Run all k8s native components

pkg/cluster/cluster.go

@@ -42,7 +42,7 @@ func New(settings *ClusterSettings, node *NodeSettings) (*Cluster, error) {
 		if err != nil {
 			return nil, err
 		}
-		masterCerts, err := NewMasterCerts("master")
+		masterCerts, err := NewMasterCerts("master", c.networking.NodeAddress.IP)
 		if err != nil {
 			return nil, err
 		}

pkg/cluster/kubeconfig.go

+package cluster
+
+import (
+	"os"
+	"text/template"
+)
+
+const kubeconfigTemplate = `
+apiVersion: v1
+clusters:
+- cluster:
+    server: {{.URL}}
+    certificate-authority: {{.CACert}}
+  name: local
+contexts:
+- context:
+    cluster: local
+    namespace: default
+    user: user
+  name: Default
+current-context: Default
+kind: Config
+preferences: {}
+users:
+- name: user
+  user:
+    client-certificate: {{.ClientCert}}
+    client-key: {{.ClientKey}}
+`
+
+type KubeConfig struct {
+	URL        string
+	CACert     string
+	ClientCert string
+	ClientKey  string
+}
+
+func (k *KubeConfig) Write(path string) error {
+	t, _ := template.New("kubeconfig").Parse(kubeconfigTemplate)
+	file, err := os.Create(path)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+	return t.Execute(file, k)
+}
+
+const kubeletConfigTemplate = `
+kind: KubeletConfiguration
+apiVersion: kubelet.config.k8s.io/v1beta1
+authentication:
+  anonymous:
+    enabled: false
+  webhook:
+    enabled: true
+  x509:
+    clientCAFile: "{{.CACert}}"
+authorization:
+  mode: Webhook
+clusterDomain: "cluster.local"
+imageMinimumGCAge: "120h"
+resolvConf: "/etc/resolv.conf"
+cgroupDriver: systemd
+runtimeRequestTimeout: "15m"
+tlsCertFile: "{{.TLSCert}}"
+tlsPrivateKeyFile: "{{.TLSKey}}"
+`
+
+type KubeletConfig struct {
+	CACert  string
+	TLSCert string
+	TLSKey  string
+}
+
+func (k *KubeletConfig) Write(path string) error {
+	t, _ := template.New("kubeletconfig").Parse(kubeletConfigTemplate)
+	file, err := os.Create(path)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+	return t.Execute(file, k)
+}

pkg/cluster/meta.go

@@ -61,23 +61,25 @@ func (s *HeptoState) String() string {
 	return res
 }
 
-func (s *HeptoState) Merge(b []byte) error {
+func (s *HeptoState) Merge(b []byte) (bool, error) {
 	remote := new(HeptoState)
 	err := json.Unmarshal(b, remote)
 	if err != nil {
-		return err
+		return false, err
 	}
 	if remote.PKI == nil {
-		return nil
+		return false, nil
 	}
-	s.PKI.Merge(remote.PKI)
+	change := false
+	change = change || s.PKI.Merge(remote.PKI)
 	for name, remoteCerts := range remote.Certificates {
 		_, ok := s.Certificates[name]
 		if ok {
-			s.Certificates[name].Merge(remoteCerts)
+			change = change || s.Certificates[name].Merge(remoteCerts)
 		} else {
 			s.Certificates[name] = remoteCerts
+			change = true
 		}
 	}
-	return nil
+	return change, nil
 }

pkg/cluster/pki.go

@@ -2,6 +2,7 @@ package cluster
 
 import (
 	"crypto/x509"
+	"net"
 	"os"
 	"path/filepath"
 
@@ -30,43 +31,59 @@ type NodeCerts struct {
 // Master certs
 type MasterCerts struct {
 	// Certificate for exposing the apiserver
-	Service *pki.Certificate `json:"service"`
+	Service *pki.Certificate
 	// Certificate for signing tokens
-	Tokens *pki.Certificate `json:"tokens"`
+	Tokens *pki.Certificate
 	// Certificate for authenticating against kubelets
-	Kubelet *pki.Certificate `json:"kubelet"`
+	Kubelet *pki.Certificate
+	// Service certificate for the controller manager
+	Controllers *pki.Certificate
+	// API client certificate for the controller manager
+	ControllersClient *pki.Certificate
+	// API client certificate for the scheduler
+	SchedulerClient *pki.Certificate
 }
 
 // Merge PKI
-func (n *ClusterPKI) Merge(other *ClusterPKI) {
-	if other.API != nil {
+func (n *ClusterPKI) Merge(other *ClusterPKI) bool {
+	change := false
+	if n.API == nil && other.API != nil {
 		n.API = other.API
+		change = true
 	}
-	if other.Services != nil {
+	if n.Services == nil && other.Services != nil {
 		n.Services = other.Services
+		change = true
 	}
-	if other.Kubelet != nil {
+	if n.Kubelet == nil && other.Kubelet != nil {
 		n.Kubelet = other.Kubelet
+		change = true
 	}
+	return change
 }
 
 // Merge a single node or master certificate
-func mergeCert(local *pki.Certificate, remote *pki.Certificate) {
+func mergeCert(local *pki.Certificate, remote *pki.Certificate) bool {
+	change := false
 	// Import CSR to master for signing
-	if remote.CSR != nil {
+	if local.CSR == nil && remote.CSR != nil {
 		local.CSR = remote.CSR
+		change = true
 	}
 	// Import and save cert back to node
-	if remote.Cert != nil {
+	if local.Cert == nil && remote.Cert != nil {
 		local.Cert = remote.Cert
 		local.Save()
+		change = true
 	}
+	return change
 }
 
 // Merge node certificates
-func (n *NodeCerts) Merge(other *NodeCerts) {
-	mergeCert(n.Service, other.Service)
-	mergeCert(n.API, other.API)
+func (n *NodeCerts) Merge(other *NodeCerts) bool {
+	change := mergeCert(n.Service, other.Service)
+	change = change || mergeCert(n.API, other.API)
+	return change
 }
 
 func NewClusterPKI(path string) (*ClusterPKI, error) {
@@ -99,7 +116,7 @@ func NewNodeCerts(path string, nodeName string) (*NodeCerts, error) {
 	if err != nil {
 		return nil, err
 	}
-	err = serviceCert.MakeCSR(pki.NewServerTemplate([]string{nodeName}))
+	err = serviceCert.MakeCSR(pki.NewServerTemplate([]string{nodeName}, []net.IP{}))
 	if err != nil {
 		return nil, err
 	}
@@ -108,7 +125,7 @@ func NewNodeCerts(path string, nodeName string) (*NodeCerts, error) {
 	if err != nil {
 		return nil, err
 	}
-	err = apiClientCert.MakeCSR(pki.NewClientTemplate(nodeName, ""))
+	err = apiClientCert.MakeCSR(pki.NewClientTemplate("system:nodes:"+nodeName, "system:nodes"))
 	if err != nil {
 		return nil, err
 	}
@@ -118,7 +135,7 @@ func NewNodeCerts(path string, nodeName string) (*NodeCerts, error) {
 	}, nil
 }
 
-func NewMasterCerts(path string) (*MasterCerts, error) {
+func NewMasterCerts(path string, ip net.IP) (*MasterCerts, error) {
 	err := os.MkdirAll(path, 0755)
 	if err != nil {
 		return nil, err
@@ -128,7 +145,7 @@ func NewMasterCerts(path string) (*MasterCerts, error) {
 	if err != nil {
 		return nil, err
 	}
-	err = serviceCert.MakeCSR(pki.NewServerTemplate([]string{"apiserver"}))
+	err = serviceCert.MakeCSR(pki.NewServerTemplate([]string{"apiserver"}, []net.IP{ip}))
 	if err != nil {
 		return nil, err
 	}
@@ -146,10 +163,40 @@ func NewMasterCerts(path string) (*MasterCerts, error) {
 	if err != nil {
 		return nil, err
 	}
+	// Controller manager certificate
+	controllersCert, err := pki.GetCertificate(filepath.Join(path, "kubelet"))
+	if err != nil {
+		return nil, err
+	}
+	err = controllersCert.MakeCSR(pki.NewServerTemplate([]string{"controllers"}, []net.IP{ip}))
+	if err != nil {
+		return nil, err
+	}
+	// Controller manager API client certificate
+	controllersClientCert, err := pki.GetCertificate(filepath.Join(path, "controllers-client"))
+	if err != nil {
+		return nil, err
+	}
+	err = controllersClientCert.MakeCSR(pki.NewClientTemplate("system:kube-controller-manager", ""))
+	if err != nil {
+		return nil, err
+	}
+	// Scheduler API client certificate
+	schedulerClientCert, err := pki.GetCertificate(filepath.Join(path, "scheduler-client"))
+	if err != nil {
+		return nil, err
+	}
+	err = schedulerClientCert.MakeCSR(pki.NewClientTemplate("system:kube-scheduler", ""))
+	if err != nil {
+		return nil, err
+	}
 	return &MasterCerts{
-		Service: serviceCert,
-		Tokens:  tokenKey,
-		Kubelet: kubeletClientCert,
+		Service:           serviceCert,
+		Tokens:            tokenKey,
+		Kubelet:           kubeletClientCert,
+		Controllers:       controllersCert,
+		ControllersClient: controllersClientCert,
+		SchedulerClient:   schedulerClientCert,
 	}, nil
 }
 
@@ -164,11 +211,14 @@ func signCert(p *pki.PKI, c *pki.Certificate, template *x509.Certificate) {
 }
 
 func (p *ClusterPKI) SignNodeCerts(name string, n *NodeCerts) {
-	signCert(p.Services, n.Service, pki.NewServerTemplate([]string{name}))
-	signCert(p.API, n.API, pki.NewClientTemplate(name, ""))
+	signCert(p.Services, n.Service, pki.NewServerTemplate([]string{name}, []net.IP{}))
+	signCert(p.API, n.API, pki.NewClientTemplate("system:node:"+name, "system:nodes"))
 }
 
 func (p *ClusterPKI) SignMasterCerts(m *MasterCerts) {
-	signCert(p.Services, m.Service, pki.NewServerTemplate(m.Service.CSR.DNSNames))
+	signCert(p.Services, m.Service, pki.NewServerTemplate(m.Service.CSR.DNSNames, m.Service.CSR.IPAddresses))
 	signCert(p.Kubelet, m.Kubelet, pki.NewClientTemplate(m.Kubelet.CSR.Subject.CommonName, ""))
+	signCert(p.Services, m.Controllers, pki.NewServerTemplate(m.Controllers.CSR.DNSNames, m.Controllers.CSR.IPAddresses))
+	signCert(p.API, m.ControllersClient, pki.NewClientTemplate(m.ControllersClient.CSR.Subject.CommonName, ""))
+	signCert(p.API, m.SchedulerClient, pki.NewClientTemplate(m.SchedulerClient.CSR.Subject.CommonName, ""))
 }

pkg/cluster/services.go

@@ -2,6 +2,8 @@ package cluster
 
 import (
 	"context"
+	"fmt"
+	"net"
 
 	"forge.tedomum.net/acides/hepto/hepto/pkg/wrappers"
 	"github.com/sirupsen/logrus"
@@ -36,8 +38,22 @@ func (s *ClusterServices) Update(c *Cluster) {
 	}
 	if c.node.Role == Master {
 		s.startEtcd()
-		s.startK8sMaster(c.pki, c.masterCerts)
+		s.startK8sMaster(c.networking, c.pki, c.masterCerts)
 		s.started = true
+	} else if c.node.Role == Node {
+		if c.pki.Services == nil || c.pki.Services.Cert == nil {
+			return
+		}
+		if c.certs.API.Cert == nil || c.certs.Service.Cert == nil {
+			return
+		}
+		logrus.Debug("looking for master node...")
+		for _, node := range c.ml.Nodes() {
+			if node.NodeMeta.Role == string(Master) {
+				s.startK8sNode(node.Addr, c.pki, c.certs)
+				s.started = true
+			}
+		}
 	}
 }
 
@@ -51,27 +67,93 @@ func (s *ClusterServices) startEtcd() {
 	go s.watch(service)
 }
 
-func (s *ClusterServices) startK8sMaster(pki *ClusterPKI, certs *MasterCerts) {
+func (s *ClusterServices) startK8sMaster(net *ClusterNetworking, pki *ClusterPKI, certs *MasterCerts) {
 	api, err := wrappers.APIServer(s.ctx, []string{
-		"--bind-address", "::",
+		"--bind-address", net.NodeAddress.IP.String(),
+		"--service-cluster-ip-range", net.ServiceNet.String(),
 		"--tls-cert-file", certs.Service.CertPath(),
 		"--tls-private-key-file", certs.Service.KeyPath(),
 		"--client-ca-file", pki.API.CertPath(),
 		"--kubelet-certificate-authority", pki.Kubelet.CertPath(),
 		"--kubelet-client-certificate", certs.Kubelet.CertPath(),
 		"--kubelet-client-key", certs.Kubelet.KeyPath(),
-		"--etcd-servers", "http://127.0.0.1:2379",
+		"--etcd-servers", "http://localhost:2379",
 		"--service-account-signing-key-file", certs.Tokens.KeyPath(),
 		"--service-account-key-file", certs.Tokens.KeyPath(),
 		"--service-account-issuer", "https://kubernetes.default.svc.cluster.local",
 		"--api-audiences", "https://kubernetes.default.svc.cluster.local",
-		// TODO: currently hardcoded
-		"--oidc-issuer-url", "https://auth.test.tedomum.net/sso/oidc/7bb37b11-2d43-40ad-a40a-4634d6257588",
-		"--oidc-client-id", "Fd1SDTs0yHGqIK6KsogQzw3W",
 		"--authorization-mode", "Node,RBAC",
 	})
 	if err != nil {
 		logrus.Fatal(err)
 	}
+	cmConfig := KubeConfig{
+		URL:        fmt.Sprintf("https://[%s]:6443", net.NodeAddress.IP.String()),
+		CACert:     pki.Services.CertPath(),
+		ClientCert: certs.ControllersClient.CertPath(),
+		ClientKey:  certs.ControllersClient.KeyPath(),
+	}
+	cmConfigPath := "/cm.yaml"
+	err = cmConfig.Write(cmConfigPath)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+	cm, err := wrappers.ControllerManager(s.ctx, []string{
+		"--kubeconfig", cmConfigPath,
+		"--tls-cert-file", certs.Controllers.CertPath(),
+		"--tls-private-key-file", certs.Controllers.KeyPath(),
+		"--service-account-private-key-file", certs.Tokens.KeyPath(),
+		"--use-service-account-credentials",
+	})
+	if err != nil {
+		logrus.Fatal(err)
+	}
+	schedulerConfig := KubeConfig{
+		URL:        fmt.Sprintf("https://[%s]:6443", net.NodeAddress.IP.String()),
+		CACert:     pki.Services.CertPath(),
+		ClientCert: certs.SchedulerClient.CertPath(),
+		ClientKey:  certs.SchedulerClient.KeyPath(),
+	}
+	schedulerConfigPath := "/scheduler.yaml"
+	err = schedulerConfig.Write(schedulerConfigPath)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+	scheduler, err := wrappers.Scheduler(s.ctx, []string{
+		"--kubeconfig", schedulerConfigPath,
+	})
+	if err != nil {
+		logrus.Fatal(err)
+	}
 	go s.watch(api)
+	go s.watch(cm)
+	go s.watch(scheduler)
+}
+
+func (s *ClusterServices) startK8sNode(master net.IP, pki *ClusterPKI, certs *NodeCerts) {
+	kubeletKubeConfig := KubeConfig{
+		URL:        fmt.Sprintf("https://[%s]:6443", master.String()),
+		CACert:     pki.Services.CertPath(),
+		ClientCert: certs.API.CertPath(),
+		ClientKey:  certs.API.KeyPath(),
+	}
+	kubeletKubeConfigPath := "/kubelet-kubeconfig.yaml"
+	kubeletKubeConfig.Write(kubeletKubeConfigPath)
+	kubeletConfig := KubeletConfig{
+		CACert:  pki.Kubelet.CertPath(),
+		TLSCert: certs.Service.CertPath(),
+		TLSKey:  certs.Service.KeyPath(),
+	}
+	kubeletConfigPath := "/kubelet.yaml"
+	kubeletConfig.Write(kubeletConfigPath)
+	kubelet, err := wrappers.Kubelet(s.ctx, []string{
+		"--kubeconfig", kubeletKubeConfigPath,
+		"--config", kubeletConfigPath,
+		"--container-runtime", "remote",
+		"--container-runtime-endpoint", "/run/containerd/containerd.sock",
+	})
+	if err != nil {
+		logrus.Fatal(err)
+	}
+	go s.watch(kubelet)
 }