Improve the apiserver startup logic

Use a post-start-hook waiting for crd to be ready, then discover
the apiserver status by actually checking the healthz endpoint,
which accounts for post-start-hook success

services/apiserver.go

@@ -62,8 +62,6 @@ import (
 const audience = "https://kubernetes.default.svc.cluster.local"
 const apiserverPort = 6443
 
-var pathRegex = regexp.MustCompile(`^/apis/([^/]+)/([^/]+)$`)
-
 var kubeApiserver = &Unit{
 	Name:         "kube-apiserver",
 	Dependencies: []*Unit{etcd, pkiCA, pkiMaster, vpn, kubeLogger},
@@ -94,33 +92,37 @@ var kubeApiserver = &Unit{
 		if err != nil {
 			return fmt.Errorf("could not initialize aggregator: %w", err)
 		}
-		// Register api services for all delegate servers
+		// Start registration services, which auto register all resources and all
+		// crd to the aggregator server
 		apiRegistrationClient, err := apiregistrationclient.NewForConfig(clients.KubeConfig)
 		if err != nil {
 			return fmt.Errorf("could not initialize aggregator client: %w", err)
 		}
 		registrationController := autoregister.NewAutoRegisterController(aggregatorServer.APIRegistrationInformers.Apiregistration().V1().APIServices(), apiRegistrationClient)
-		crdController := crdregistration.NewCRDRegistrationController(
-			extensionServer.Informers.Apiextensions().V1().CustomResourceDefinitions(),
-			registrationController,
-		)
+		crdController := crdregistration.NewCRDRegistrationController(extensionServer.Informers.Apiextensions().V1().CustomResourceDefinitions(), registrationController)
+		// Start by discovery resource paths from apiserver and registering them manually
+		// TODO We completely ignore priorities, wchih might become an issue at some point
+		pathRegex := regexp.MustCompile(`^/apis/([^/]+)/([^/]+)$`)
 		for _, resourcePath := range apiServer.GenericAPIServer.ListedPaths() {
-			match := pathRegex.FindStringSubmatch(resourcePath)
-			if match == nil {
-				continue
+			if match := pathRegex.FindStringSubmatch(resourcePath); match != nil {
+				registrationController.AddAPIServiceToSyncOnStart(&v1.APIService{
+					ObjectMeta: meta.ObjectMeta{Name: match[2] + "." + match[1]},
+					Spec: v1.APIServiceSpec{
+						Group:                match[1],
+						Version:              match[2],
+						GroupPriorityMinimum: 1,
+						VersionPriority:      1,
+					},
+				})
 			}
-			registrationController.AddAPIServiceToSyncOnStart(&v1.APIService{
-				ObjectMeta: meta.ObjectMeta{Name: match[2] + "." + match[1]},
-				Spec: v1.APIServiceSpec{
-					Group:                match[1],
-					Version:              match[2],
-					GroupPriorityMinimum: 1, // default value, TODO might be an issue
-					VersionPriority:      1,
-				},
-			})
 		}
-		go registrationController.Run(1, ctx.Done())
-		go crdController.Run(1, ctx.Done())
+		// Wait for CRD to be ready then start both controllers in a post-start-hook
+		err = aggregatorServer.GenericAPIServer.AddPostStartHook("autoregistration", func(context server.PostStartHookContext) error {
+			go crdController.Run(1, ctx.Done())
+			crdController.WaitForInitialSync()
+			go registrationController.Run(1, ctx.Done())
+			return nil
+		})
 
 		// Finally start the apiserver
 		server := aggregatorServer.GenericAPIServer.PrepareRun()
@@ -133,8 +135,9 @@ var kubeApiserver = &Unit{
 		if err != nil {
 			return false
 		}
-		_, err = clients.Client.CoreV1().Nodes().List(context.Background(), meta.ListOptions{})
-		if err != nil {
+		status := 0
+		result := clients.Client.Discovery().RESTClient().Get().AbsPath("/healthz").Do(context.TODO()).StatusCode(&status)
+		if result.Error() != nil || status != 200 {
 			return false
 		}
 		return true