diff --git a/pkg/cluster/certs.go b/pkg/cluster/certs.go index 9edb8bb2d23e365e62d96556b7025a7a0d721daa..8e18bf4358e133c6f9e070f4e82120295625c858 100644 --- a/pkg/cluster/certs.go +++ b/pkg/cluster/certs.go @@ -1,31 +1,32 @@ package cluster import ( + "forge.tedomum.net/acides/hepto/hepto/pkg/pki" "github.com/sirupsen/logrus" ) func (c *Cluster) initCerts() { // Prepare the cluster PKI if c.node.Role == Master { - pki, err := NewClusterPKI("pki") + ca, err := pki.NewClusterPKI("pki") if err != nil { logrus.Fatal("could not initialize pki: ", err) } - masterCerts, err := NewMasterCerts("master", c.networking.NodeAddress.IP) + masterCerts, err := pki.NewMasterCerts("master", c.networking.NodeAddress.IP) if err != nil { logrus.Fatal("could not initialize master certs: ", err) } - c.pki = pki + c.pki = ca c.masterCerts = masterCerts } c.ml.State.PKI = c.pki // Initialize node certificates - certs, err := NewNodeCerts("certs", c.node.Name) + certs, err := pki.NewNodeCerts("certs", c.node.Name) if err != nil { logrus.Fatal("could not initialize node certificates: ", err) } c.certs = certs - c.ml.State.Certificates = make(map[string]*NodeCerts) + c.ml.State.Certificates = make(map[string]*pki.NodeCerts) c.ml.State.Certificates[c.node.Name] = certs } diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 0e6f9f0677d9069dffb2cc1f54109f1871f5a9c7..89c3d68017a62ef04b1d217e73ce1156f26e65cb 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -4,6 +4,7 @@ package cluster import ( "forge.tedomum.net/acides/hepto/hepto/pkg/sml" + "forge.tedomum.net/acides/hepto/hepto/pkg/pki" "forge.tedomum.net/acides/hepto/hepto/pkg/wg" "github.com/sirupsen/logrus" ) @@ -14,9 +15,9 @@ type Cluster struct { vpn *wg.Wireguard networking *ClusterNetworking node *NodeSettings - certs *NodeCerts - masterCerts *MasterCerts - pki *ClusterPKI + certs *pki.NodeCerts + masterCerts *pki.MasterCerts + pki *pki.ClusterPKI services *ClusterServices } @@ -26,7 +27,7 @@ func New(settings *ClusterSettings, node *NodeSettings) *Cluster { node: node, networking: NewClusterNetworking(settings.Name, node.Name), ml: sml.New[HeptoMeta, HeptoState](node.Name, node.IP, node.Port, node.Anchors, settings.Key), - pki: &ClusterPKI{}, + pki: &pki.ClusterPKI{}, services: NewClusterServices(), } } diff --git a/pkg/cluster/meta.go b/pkg/cluster/meta.go index b4d0f70a13912c51ca8e3e5bcec93711c6a05328..13c268b9e4c60258acc3bd11fd8bdc1ba8b5b745 100644 --- a/pkg/cluster/meta.go +++ b/pkg/cluster/meta.go @@ -4,6 +4,8 @@ import ( "encoding/json" "fmt" "strings" + + "forge.tedomum.net/acides/hepto/hepto/pkg/pki" ) // Represents a node metadata @@ -17,10 +19,10 @@ type HeptoMeta struct { // Represents the cluster state type HeptoState struct { // Cluster CAs public certificates - PKI *ClusterPKI `json:"ca"` + PKI *pki.ClusterPKI `json:"ca"` // Certificate per node, this should only // be updated by the node itself - Certificates map[string]*NodeCerts `json:"nodes"` + Certificates map[string]*pki.NodeCerts `json:"nodes"` } func (m *HeptoMeta) Encode() ([]byte, error) { diff --git a/pkg/cluster/pki.go b/pkg/cluster/pki.go deleted file mode 100644 index 9590ff3668799d6b913fbe9aa4dd31b5dcac89ea..0000000000000000000000000000000000000000 --- a/pkg/cluster/pki.go +++ /dev/null @@ -1,224 +0,0 @@ -package cluster - -import ( - "crypto/x509" - "net" - "os" - "path/filepath" - - "forge.tedomum.net/acides/hepto/hepto/pkg/pekahi" - "github.com/sirupsen/logrus" -) - -// Cluster PKI is made of three different PKIs -type ClusterPKI struct { - // Signs services exposed over the cluster - Services *pekahi.PKI `json:"services"` - // Signs kubelet client certificates (master) - Kubelet *pekahi.PKI `json:"kubelet"` - // Signs apiserver client certificates (nodes and controller) - API *pekahi.PKI `json:"api"` -} - -// Node certs -type NodeCerts struct { - // Certificate for exposing the kubelet service - Service *pekahi.Certificate `json:"service"` - // Node certificate for accessing the apiserver - API *pekahi.Certificate `json:"api"` -} - -// Master certs -type MasterCerts struct { - // Certificate for exposing the apiserver - Service *pekahi.Certificate - // Certificate for signing tokens - Tokens *pekahi.Certificate - // Certificate for authenticating against kubelets - Kubelet *pekahi.Certificate - // Service certificate for the controller manager - Controllers *pekahi.Certificate - // API client certificate for the controller manager - ControllersClient *pekahi.Certificate - // API client certificate for the scheduler - SchedulerClient *pekahi.Certificate -} - -// Merge PKI -func (n *ClusterPKI) Merge(other *ClusterPKI) bool { - change := false - if n.API == nil && other.API != nil { - n.API = other.API - change = true - } - if n.Services == nil && other.Services != nil { - n.Services = other.Services - change = true - } - if n.Kubelet == nil && other.Kubelet != nil { - n.Kubelet = other.Kubelet - change = true - } - return change -} - -// Merge a single node or master certificate -func mergeCert(local *pekahi.Certificate, remote *pekahi.Certificate) bool { - change := false - // Import CSR to master for signing - if local.CSR == nil && remote.CSR != nil { - local.CSR = remote.CSR - change = true - } - // Import and save cert back to node - if local.Cert == nil && remote.Cert != nil { - local.Cert = remote.Cert - local.Save() - change = true - } - return change -} - -// Merge node certificates -func (n *NodeCerts) Merge(other *NodeCerts) bool { - change := mergeCert(n.Service, other.Service) - change = change || mergeCert(n.API, other.API) - return change -} - -func NewClusterPKI(path string) (*ClusterPKI, error) { - err := os.MkdirAll(path, 0755) - if err != nil { - return nil, err - } - servicesCA, err := pekahi.GetPKI(filepath.Join(path, "services")) - if err != nil { - return nil, err - } - kubeletCA, err := pekahi.GetPKI(filepath.Join(path, "kubelet")) - if err != nil { - return nil, err - } - apiserverCA, err := pekahi.GetPKI(filepath.Join(path, "api")) - if err != nil { - return nil, err - } - return &ClusterPKI{servicesCA, kubeletCA, apiserverCA}, nil -} - -func NewNodeCerts(path string, nodeName string) (*NodeCerts, error) { - err := os.MkdirAll(path, 0755) - if err != nil { - return nil, err - } - // Service certificate - serviceCert, err := pekahi.GetCertificate(filepath.Join(path, "service")) - if err != nil { - return nil, err - } - err = serviceCert.MakeCSR(pekahi.NewServerTemplate([]string{nodeName}, []net.IP{})) - if err != nil { - return nil, err - } - // API certificate - apiClientCert, err := pekahi.GetCertificate(filepath.Join(path, "api")) - if err != nil { - return nil, err - } - err = apiClientCert.MakeCSR(pekahi.NewClientTemplate("system:nodes:"+nodeName, "system:nodes")) - if err != nil { - return nil, err - } - return &NodeCerts{ - Service: serviceCert, - API: apiClientCert, - }, nil -} - -func NewMasterCerts(path string, ip net.IP) (*MasterCerts, error) { - err := os.MkdirAll(path, 0755) - if err != nil { - return nil, err - } - // Service certificate - serviceCert, err := pekahi.GetCertificate(filepath.Join(path, "service")) - if err != nil { - return nil, err - } - err = serviceCert.MakeCSR(pekahi.NewServerTemplate([]string{"apiserver"}, []net.IP{ip})) - if err != nil { - return nil, err - } - // Tokens key - tokenKey, err := pekahi.GetCertificate(filepath.Join(path, "tokens")) - if err != nil { - return nil, err - } - // Kubelet certificate - kubeletClientCert, err := pekahi.GetCertificate(filepath.Join(path, "kubelet")) - if err != nil { - return nil, err - } - err = kubeletClientCert.MakeCSR(pekahi.NewClientTemplate("apiserver", "")) - if err != nil { - return nil, err - } - // Controller manager certificate - controllersCert, err := pekahi.GetCertificate(filepath.Join(path, "kubelet")) - if err != nil { - return nil, err - } - err = controllersCert.MakeCSR(pekahi.NewServerTemplate([]string{"controllers"}, []net.IP{ip})) - if err != nil { - return nil, err - } - // Controller manager API client certificate - controllersClientCert, err := pekahi.GetCertificate(filepath.Join(path, "controllers-client")) - if err != nil { - return nil, err - } - err = controllersClientCert.MakeCSR(pekahi.NewClientTemplate("system:kube-controller-manager", "")) - if err != nil { - return nil, err - } - // Scheduler API client certificate - schedulerClientCert, err := pekahi.GetCertificate(filepath.Join(path, "scheduler-client")) - if err != nil { - return nil, err - } - err = schedulerClientCert.MakeCSR(pekahi.NewClientTemplate("system:kube-scheduler", "")) - if err != nil { - return nil, err - } - return &MasterCerts{ - Service: serviceCert, - Tokens: tokenKey, - Kubelet: kubeletClientCert, - Controllers: controllersCert, - ControllersClient: controllersClientCert, - SchedulerClient: schedulerClientCert, - }, nil -} - -func signCert(p *pekahi.PKI, c *pekahi.Certificate, template *x509.Certificate) { - if c.CSR != nil && c.Cert == nil { - logrus.Info("signing certificate ", c.CSR.Subject.String()) - err := p.Sign(c, template) - if err != nil { - logrus.Warnf("cannot sign API certificate for %s: %s", c.CSR.Subject.String(), err) - } - } -} - -func (p *ClusterPKI) SignNodeCerts(name string, n *NodeCerts) { - signCert(p.Services, n.Service, pekahi.NewServerTemplate([]string{name}, []net.IP{})) - signCert(p.API, n.API, pekahi.NewClientTemplate("system:node:"+name, "system:nodes")) -} - -func (p *ClusterPKI) SignMasterCerts(m *MasterCerts) { - signCert(p.Services, m.Service, pekahi.NewServerTemplate(m.Service.CSR.DNSNames, m.Service.CSR.IPAddresses)) - signCert(p.Kubelet, m.Kubelet, pekahi.NewClientTemplate(m.Kubelet.CSR.Subject.CommonName, "")) - signCert(p.Services, m.Controllers, pekahi.NewServerTemplate(m.Controllers.CSR.DNSNames, m.Controllers.CSR.IPAddresses)) - signCert(p.API, m.ControllersClient, pekahi.NewClientTemplate(m.ControllersClient.CSR.Subject.CommonName, "")) - signCert(p.API, m.SchedulerClient, pekahi.NewClientTemplate(m.SchedulerClient.CSR.Subject.CommonName, "")) -} diff --git a/pkg/cluster/services.go b/pkg/cluster/services.go index 0a903b534b69f32208f8f446418a7ddc5dfc005d..50a54dd48756958499db7a9718514123b281438e 100644 --- a/pkg/cluster/services.go +++ b/pkg/cluster/services.go @@ -5,6 +5,7 @@ import ( "fmt" "net" + "forge.tedomum.net/acides/hepto/hepto/pkg/pki" "forge.tedomum.net/acides/hepto/hepto/pkg/wrappers" "github.com/sirupsen/logrus" "go.etcd.io/etcd/server/v3/embed" @@ -67,14 +68,14 @@ func (s *ClusterServices) startEtcd() { go s.watch(service) } -func (s *ClusterServices) startK8sMaster(net *ClusterNetworking, pki *ClusterPKI, certs *MasterCerts) { +func (s *ClusterServices) startK8sMaster(net *ClusterNetworking, ca *pki.ClusterPKI, certs *pki.MasterCerts) { api, err := wrappers.APIServer(s.ctx, []string{ "--bind-address", net.NodeAddress.IP.String(), "--service-cluster-ip-range", net.ServiceNet.String(), "--tls-cert-file", certs.Service.CertPath(), "--tls-private-key-file", certs.Service.KeyPath(), - "--client-ca-file", pki.API.CertPath(), - "--kubelet-certificate-authority", pki.Kubelet.CertPath(), + "--client-ca-file", ca.API.CertPath(), + "--kubelet-certificate-authority", ca.Kubelet.CertPath(), "--kubelet-client-certificate", certs.Kubelet.CertPath(), "--kubelet-client-key", certs.Kubelet.KeyPath(), "--etcd-servers", "http://localhost:2379", @@ -89,7 +90,7 @@ func (s *ClusterServices) startK8sMaster(net *ClusterNetworking, pki *ClusterPKI } cmConfig := KubeConfig{ URL: fmt.Sprintf("https://[%s]:6443", net.NodeAddress.IP.String()), - CACert: pki.Services.CertPath(), + CACert: ca.Services.CertPath(), ClientCert: certs.ControllersClient.CertPath(), ClientKey: certs.ControllersClient.KeyPath(), } @@ -110,7 +111,7 @@ func (s *ClusterServices) startK8sMaster(net *ClusterNetworking, pki *ClusterPKI } schedulerConfig := KubeConfig{ URL: fmt.Sprintf("https://[%s]:6443", net.NodeAddress.IP.String()), - CACert: pki.Services.CertPath(), + CACert: ca.Services.CertPath(), ClientCert: certs.SchedulerClient.CertPath(), ClientKey: certs.SchedulerClient.KeyPath(), } @@ -130,17 +131,17 @@ func (s *ClusterServices) startK8sMaster(net *ClusterNetworking, pki *ClusterPKI go s.watch(scheduler) } -func (s *ClusterServices) startK8sNode(master net.IP, pki *ClusterPKI, certs *NodeCerts) { +func (s *ClusterServices) startK8sNode(master net.IP, ca *pki.ClusterPKI, certs *pki.NodeCerts) { kubeletKubeConfig := KubeConfig{ URL: fmt.Sprintf("https://[%s]:6443", master.String()), - CACert: pki.Services.CertPath(), + CACert: ca.Services.CertPath(), ClientCert: certs.API.CertPath(), ClientKey: certs.API.KeyPath(), } kubeletKubeConfigPath := "/kubelet-kubeconfig.yaml" kubeletKubeConfig.Write(kubeletKubeConfigPath) kubeletConfig := KubeletConfig{ - CACert: pki.Kubelet.CertPath(), + CACert: ca.Kubelet.CertPath(), TLSCert: certs.Service.CertPath(), TLSKey: certs.Service.KeyPath(), } diff --git a/pkg/pki/ca.go b/pkg/pki/ca.go new file mode 100644 index 0000000000000000000000000000000000000000..f79e8db75d1a0800b90f7cd4cc26557719c02740 --- /dev/null +++ b/pkg/pki/ca.go @@ -0,0 +1,56 @@ +package pki + +import ( + "os" + "path/filepath" + + "forge.tedomum.net/acides/hepto/hepto/pkg/pekahi" +) + +// Cluster PKI is made of three different PKIs +type ClusterPKI struct { + // Signs services exposed over the cluster + Services *pekahi.PKI `json:"services"` + // Signs kubelet client certificates (master) + Kubelet *pekahi.PKI `json:"kubelet"` + // Signs apiserver client certificates (nodes and controller) + API *pekahi.PKI `json:"api"` +} + +func NewClusterPKI(path string) (*ClusterPKI, error) { + err := os.MkdirAll(path, 0755) + if err != nil { + return nil, err + } + servicesCA, err := pekahi.GetPKI(filepath.Join(path, "services")) + if err != nil { + return nil, err + } + kubeletCA, err := pekahi.GetPKI(filepath.Join(path, "kubelet")) + if err != nil { + return nil, err + } + apiserverCA, err := pekahi.GetPKI(filepath.Join(path, "api")) + if err != nil { + return nil, err + } + return &ClusterPKI{servicesCA, kubeletCA, apiserverCA}, nil +} + +// Merge PKI +func (n *ClusterPKI) Merge(other *ClusterPKI) bool { + change := false + if n.API == nil && other.API != nil { + n.API = other.API + change = true + } + if n.Services == nil && other.Services != nil { + n.Services = other.Services + change = true + } + if n.Kubelet == nil && other.Kubelet != nil { + n.Kubelet = other.Kubelet + change = true + } + return change +} diff --git a/pkg/pki/master.go b/pkg/pki/master.go new file mode 100644 index 0000000000000000000000000000000000000000..abf027b601234e5625677121218298461a575500 --- /dev/null +++ b/pkg/pki/master.go @@ -0,0 +1,98 @@ +package pki + +import ( + "net" + "os" + "path/filepath" + + "forge.tedomum.net/acides/hepto/hepto/pkg/pekahi" +) + +// Master certs +type MasterCerts struct { + // Certificate for exposing the apiserver + Service *pekahi.Certificate + // Certificate for signing tokens + Tokens *pekahi.Certificate + // Certificate for authenticating against kubelets + Kubelet *pekahi.Certificate + // Service certificate for the controller manager + Controllers *pekahi.Certificate + // API client certificate for the controller manager + ControllersClient *pekahi.Certificate + // API client certificate for the scheduler + SchedulerClient *pekahi.Certificate +} + +func NewMasterCerts(path string, ip net.IP) (*MasterCerts, error) { + err := os.MkdirAll(path, 0755) + if err != nil { + return nil, err + } + // Service certificate + serviceCert, err := pekahi.GetCertificate(filepath.Join(path, "service")) + if err != nil { + return nil, err + } + err = serviceCert.MakeCSR(pekahi.NewServerTemplate([]string{"apiserver"}, []net.IP{ip})) + if err != nil { + return nil, err + } + // Tokens key + tokenKey, err := pekahi.GetCertificate(filepath.Join(path, "tokens")) + if err != nil { + return nil, err + } + // Kubelet certificate + kubeletClientCert, err := pekahi.GetCertificate(filepath.Join(path, "kubelet")) + if err != nil { + return nil, err + } + err = kubeletClientCert.MakeCSR(pekahi.NewClientTemplate("apiserver", "")) + if err != nil { + return nil, err + } + // Controller manager certificate + controllersCert, err := pekahi.GetCertificate(filepath.Join(path, "kubelet")) + if err != nil { + return nil, err + } + err = controllersCert.MakeCSR(pekahi.NewServerTemplate([]string{"controllers"}, []net.IP{ip})) + if err != nil { + return nil, err + } + // Controller manager API client certificate + controllersClientCert, err := pekahi.GetCertificate(filepath.Join(path, "controllers-client")) + if err != nil { + return nil, err + } + err = controllersClientCert.MakeCSR(pekahi.NewClientTemplate("system:kube-controller-manager", "")) + if err != nil { + return nil, err + } + // Scheduler API client certificate + schedulerClientCert, err := pekahi.GetCertificate(filepath.Join(path, "scheduler-client")) + if err != nil { + return nil, err + } + err = schedulerClientCert.MakeCSR(pekahi.NewClientTemplate("system:kube-scheduler", "")) + if err != nil { + return nil, err + } + return &MasterCerts{ + Service: serviceCert, + Tokens: tokenKey, + Kubelet: kubeletClientCert, + Controllers: controllersCert, + ControllersClient: controllersClientCert, + SchedulerClient: schedulerClientCert, + }, nil +} + +func (p *ClusterPKI) SignMasterCerts(m *MasterCerts) { + signCert(p.Services, m.Service, pekahi.NewServerTemplate(m.Service.CSR.DNSNames, m.Service.CSR.IPAddresses)) + signCert(p.Kubelet, m.Kubelet, pekahi.NewClientTemplate(m.Kubelet.CSR.Subject.CommonName, "")) + signCert(p.Services, m.Controllers, pekahi.NewServerTemplate(m.Controllers.CSR.DNSNames, m.Controllers.CSR.IPAddresses)) + signCert(p.API, m.ControllersClient, pekahi.NewClientTemplate(m.ControllersClient.CSR.Subject.CommonName, "")) + signCert(p.API, m.SchedulerClient, pekahi.NewClientTemplate(m.SchedulerClient.CSR.Subject.CommonName, "")) +} diff --git a/pkg/pki/node.go b/pkg/pki/node.go new file mode 100644 index 0000000000000000000000000000000000000000..ba2e9e053566596054362edef71b6cbe2edbd2d6 --- /dev/null +++ b/pkg/pki/node.go @@ -0,0 +1,58 @@ +package pki + +import ( + "net" + "os" + "path/filepath" + + "forge.tedomum.net/acides/hepto/hepto/pkg/pekahi" +) + +// Node certs +type NodeCerts struct { + // Certificate for exposing the kubelet service + Service *pekahi.Certificate `json:"service"` + // Node certificate for accessing the apiserver + API *pekahi.Certificate `json:"api"` +} + +func NewNodeCerts(path string, nodeName string) (*NodeCerts, error) { + err := os.MkdirAll(path, 0755) + if err != nil { + return nil, err + } + // Service certificate + serviceCert, err := pekahi.GetCertificate(filepath.Join(path, "service")) + if err != nil { + return nil, err + } + err = serviceCert.MakeCSR(pekahi.NewServerTemplate([]string{nodeName}, []net.IP{})) + if err != nil { + return nil, err + } + // API certificate + apiClientCert, err := pekahi.GetCertificate(filepath.Join(path, "api")) + if err != nil { + return nil, err + } + err = apiClientCert.MakeCSR(pekahi.NewClientTemplate("system:nodes:"+nodeName, "system:nodes")) + if err != nil { + return nil, err + } + return &NodeCerts{ + Service: serviceCert, + API: apiClientCert, + }, nil +} + +// Merge node certificates +func (n *NodeCerts) Merge(other *NodeCerts) bool { + change := mergeCert(n.Service, other.Service) + change = change || mergeCert(n.API, other.API) + return change +} + +func (p *ClusterPKI) SignNodeCerts(name string, n *NodeCerts) { + signCert(p.Services, n.Service, pekahi.NewServerTemplate([]string{name}, []net.IP{})) + signCert(p.API, n.API, pekahi.NewClientTemplate("system:node:"+name, "system:nodes")) +} diff --git a/pkg/pki/utils.go b/pkg/pki/utils.go new file mode 100644 index 0000000000000000000000000000000000000000..dcd9814073acafffd7723fef4bfd026199a6360d --- /dev/null +++ b/pkg/pki/utils.go @@ -0,0 +1,35 @@ +package pki + +import ( + "crypto/x509" + + "forge.tedomum.net/acides/hepto/hepto/pkg/pekahi" + "github.com/sirupsen/logrus" +) + +// Merge a single node or master certificate +func mergeCert(local *pekahi.Certificate, remote *pekahi.Certificate) bool { + change := false + // Import CSR to master for signing + if local.CSR == nil && remote.CSR != nil { + local.CSR = remote.CSR + change = true + } + // Import and save cert back to node + if local.Cert == nil && remote.Cert != nil { + local.Cert = remote.Cert + local.Save() + change = true + } + return change +} + +func signCert(p *pekahi.PKI, c *pekahi.Certificate, template *x509.Certificate) { + if c.CSR != nil && c.Cert == nil { + logrus.Info("signing certificate ", c.CSR.Subject.String()) + err := p.Sign(c, template) + if err != nil { + logrus.Warnf("cannot sign certificate for %s: %s", c.CSR.Subject.String(), err) + } + } +}