Skip to content
Snippets Groups Projects
Commit ee1bdaf9 authored by kaiyou's avatar kaiyou
Browse files

Refactor the PKI and move code to service units

parent cd28a2a3
No related branches found
No related tags found
No related merge requests found
......@@ -76,8 +76,8 @@ func (u *Unit[S]) DependenciesReady() bool {
if dep.Ready != nil && !dep.Ready(dep, u.Manager.State) {
return false
} else if dep.Ready == nil && !dep.Running {
return false
}
return false
}
}
return true
}
package pki
import (
"go.acides.org/pekahi"
)
// Cluster PKI is made of three different CAs
type ClusterCA struct {
// Signs services exposed over the cluster
TLS *pekahi.Certificate `json:"tls"`
// Signs kubelet client certificates (master)
Kubelet *pekahi.Certificate `json:"kubelet"`
// Signs apiserver client certificates (nodes and controller)
API *pekahi.Certificate `json:"api"`
}
// Cluster CA as it is held by the master node
func NewClusterCA(path string) (*ClusterCA, error) {
bundle, err := pekahi.NewFileBundle(path)
if err != nil {
return nil, err
}
tlsCA, err := bundle.GetCA("tls")
if err != nil {
return nil, err
}
kubeletCA, err := bundle.GetCA("kubelet")
if err != nil {
return nil, err
}
apiserverCA, err := bundle.GetCA("api")
if err != nil {
return nil, err
}
return &ClusterCA{tlsCA, kubeletCA, apiserverCA}, nil
}
// Empty CA for receiving certificates
func EmptyClusterCA(path string) (*ClusterCA, error) {
bundle, err := pekahi.NewFileBundle(path)
if err != nil {
return nil, err
}
return &ClusterCA{
TLS: bundle.GetCertificate("tls"),
Kubelet: bundle.GetCertificate("kubelet"),
API: bundle.GetCertificate("api"),
}, nil
}
// Merge the CA
func (n *ClusterCA) Merge(remote *ClusterCA) bool {
change := mergeCert(n.TLS, remote.TLS)
change = mergeCert(n.Kubelet, remote.Kubelet) || change
change = mergeCert(n.API, remote.API) || change
return change
}
package pki
import (
"net"
"go.acides.org/pekahi"
)
// Master certs
type MasterCerts struct {
// Certificate for exposing the apiserver
TLS *pekahi.Certificate
// Certificate for signing API server tokens
APITokens *pekahi.Certificate
// Certificate for signing etcd tokens
EtcdTokens *pekahi.Certificate
// Certificate for authenticating against kubelets
Kubelet *pekahi.Certificate
// Service certificate for the controller manager
ControllersTLS *pekahi.Certificate
// API client certificate for the controller manager
ControllersAPI *pekahi.Certificate
// API client certificate for the scheduler
SchedulerAPI *pekahi.Certificate
// Root access to the API server
RootClient *pekahi.Certificate
}
func NewMasterCerts(path string, apiName string, nodeIP, apiIP net.IP) (*MasterCerts, error) {
// See: https://kubernetes.io/docs/setup/best-practices/certificates/
bundle, err := pekahi.NewFileBundle(path)
if err != nil {
return nil, err
}
// TLS certificate
tlsCert, err := bundle.GetCertOrCSR("tls",
pekahi.NewServerTemplate([]string{"kube-apiserver", apiName}, []net.IP{nodeIP, apiIP}),
)
if err != nil {
return nil, err
}
// API tokens key
apiTokenKey, err := bundle.GetCertWithKey("api-tokens")
if err != nil {
return nil, err
}
// Etcd tokens key
etcdTokenKey, err := bundle.GetCertWithKey("etcd-tokens")
if err != nil {
return nil, err
}
// Kubelet certificate
kubeletCert, err := bundle.GetCertOrCSR("kubelet",
pekahi.NewClientTemplate("kube-apiserver-kubelet-client", "system:masters"),
)
if err != nil {
return nil, err
}
// Controller manager certificate
controllersTLSCert, err := bundle.GetCertOrCSR("controllers-tls",
pekahi.NewServerTemplate([]string{"controllers"}, []net.IP{nodeIP}),
)
if err != nil {
return nil, err
}
// Controller manager API client certificate
controllersAPICert, err := bundle.GetCertOrCSR("controllers-api",
pekahi.NewClientTemplate("system:kube-controller-manager", ""),
)
if err != nil {
return nil, err
}
// Scheduler API client certificate
schedulerAPICert, err := bundle.GetCertOrCSR("scheduler-api",
pekahi.NewClientTemplate("system:kube-scheduler", ""),
)
if err != nil {
return nil, err
}
// Root client certificate
rootClientCert, err := bundle.GetCertOrCSR("root",
pekahi.NewClientTemplate("root", "system:masters"),
)
return &MasterCerts{
TLS: tlsCert,
APITokens: apiTokenKey,
EtcdTokens: etcdTokenKey,
Kubelet: kubeletCert,
ControllersTLS: controllersTLSCert,
ControllersAPI: controllersAPICert,
SchedulerAPI: schedulerAPICert,
RootClient: rootClientCert,
}, nil
}
func (ca *ClusterCA) SignMasterCerts(m *MasterCerts) {
signCert(ca.TLS, m.TLS, pekahi.NewServerTemplate(m.TLS.CSR.DNSNames, m.TLS.CSR.IPAddresses))
signCert(ca.Kubelet, m.Kubelet, pekahi.NewClientTemplate(m.Kubelet.CSR.Subject.CommonName, "system:masters"))
signCert(ca.TLS, m.ControllersTLS, pekahi.NewServerTemplate(m.ControllersTLS.CSR.DNSNames, m.ControllersTLS.CSR.IPAddresses))
signCert(ca.API, m.ControllersAPI, pekahi.NewClientTemplate(m.ControllersAPI.CSR.Subject.CommonName, ""))
signCert(ca.API, m.SchedulerAPI, pekahi.NewClientTemplate(m.SchedulerAPI.CSR.Subject.CommonName, ""))
signCert(ca.API, m.RootClient, pekahi.NewClientTemplate(m.RootClient.CSR.Subject.CommonName, "system:masters"))
}
package pki
import (
"net"
"go.acides.org/pekahi"
)
// Node certs
type NodeCerts struct {
// Certificate for exposing the kubelet service
TLS *pekahi.Certificate `json:"tls"`
// Node certificate for accessing the apiserver
API *pekahi.Certificate `json:"api"`
}
func NewNodeCerts(path string, nodeName string) (*NodeCerts, error) {
// See: https://kubernetes.io/docs/setup/best-practices/certificates/
bundle, err := pekahi.NewFileBundle(path)
if err != nil {
return nil, err
}
// Service certificate
tlsCert, err := bundle.GetCertOrCSR("tls",
pekahi.NewServerTemplate([]string{nodeName}, []net.IP{}),
)
if err != nil {
return nil, err
}
// API certificate
apiCert, err := bundle.GetCertOrCSR("api",
pekahi.NewClientTemplate("system:nodes:"+nodeName, "system:nodes"),
)
if err != nil {
return nil, err
}
return &NodeCerts{
TLS: tlsCert,
API: apiCert,
}, nil
}
// Merge node certificates
func (n *NodeCerts) Merge(other *NodeCerts) bool {
change := mergeCert(n.TLS, other.TLS)
change = mergeCert(n.API, other.API) || change
return change
}
func (ca *ClusterCA) SignNodeCerts(name string, n *NodeCerts) {
signCert(ca.TLS, n.TLS, pekahi.NewServerTemplate([]string{name}, []net.IP{}))
signCert(ca.API, n.API, pekahi.NewClientTemplate("system:node:"+name, "system:nodes"))
}
package pki
import (
"crypto/x509"
"go.acides.org/pekahi"
)
// Merge a single node or master certificate
func mergeCert(local *pekahi.Certificate, remote *pekahi.Certificate) bool {
change := false
// Do not merge nothing
if remote == nil {
return change
}
// Import CSR to master for signing
if local.CSR == nil && remote.CSR != nil {
local.CSR = remote.CSR
change = true
}
// Import and save cert back to node
if local.Cert == nil && remote.Cert != nil {
local.Cert = remote.Cert
local.Save()
change = true
}
return change
}
func signCert(ca *pekahi.Certificate, c *pekahi.Certificate, template *x509.Certificate) {
if c.CSR != nil && c.Cert == nil {
ca.Sign(c, template)
}
}
......@@ -2,71 +2,237 @@ package services
import (
"context"
"net"
"path"
"go.acides.org/hepto/pkg/pki"
"go.acides.org/pekahi"
)
const CertsPath = "/certs"
var certs = &Unit{
Name: "certificates",
// Cluster PKI is made of three different CAs
type ClusterCA struct {
// Signs services exposed over the cluster
TLS *pekahi.Certificate `json:"tls"`
// Signs kubelet client certificates (master)
Kubelet *pekahi.Certificate `json:"kubelet"`
// Signs apiserver client certificates (nodes and controller)
API *pekahi.Certificate `json:"api"`
}
// Master certs
type MasterCerts struct {
// Certificate for exposing the apiserver
TLS *pekahi.Certificate
// Certificate for signing API server tokens
APITokens *pekahi.Certificate
// Certificate for signing etcd tokens
EtcdTokens *pekahi.Certificate
// Certificate for authenticating against kubelets
Kubelet *pekahi.Certificate
// Service certificate for the controller manager
ControllersTLS *pekahi.Certificate
// API client certificate for the controller manager
ControllersAPI *pekahi.Certificate
// API client certificate for the scheduler
SchedulerAPI *pekahi.Certificate
// Root access to the API server
RootClient *pekahi.Certificate
}
// Node certs
type NodeCerts struct {
// Certificate for exposing the kubelet service
TLS *pekahi.Certificate `json:"tls"`
// Node certificate for accessing the apiserver
API *pekahi.Certificate `json:"api"`
}
// PKI manager on the master node
var pkiManager = &Unit{
Name: "pki-manager",
Run: func(u *Unit, c *Cluster, ctx context.Context) error {
bundle, err := pekahi.NewFileBundle(path.Join(CertsPath, "pki"))
if err != nil {
return err
}
tlsCA, err := bundle.GetCA("tls")
if err != nil {
return err
}
kubeletCA, err := bundle.GetCA("kubelet")
if err != nil {
return err
}
apiserverCA, err := bundle.GetCA("api")
if err != nil {
return err
}
c.pki = &ClusterCA{tlsCA, kubeletCA, apiserverCA}
c.state.PKI = c.pki
return nil
},
Wake: func(u *Unit, c *Cluster) error {
if c.node.Role == "master" {
for name, certs := range c.state.Certificates {
for name, certs := range c.state.Certificates {
if certs.TLS.Cert == nil || certs.API.Cert == nil {
c.pki.TLS.Sign(certs.TLS, pekahi.NewServerTemplate([]string{name}, []net.IP{}))
c.pki.API.Sign(certs.API, pekahi.NewClientTemplate("system:node"+name, "system:nodes"))
u.Manager.Logger.Info("signing node cert", "node", name)
c.pki.SignNodeCerts(name, certs)
}
}
return nil
},
}
// Certificates for a master node
var masterCerts = &Unit{
Name: "pki-master",
// Currently only runs on master nodes and handles singing
// directly
Dependencies: []*Unit{pkiManager},
Run: func(u *Unit, c *Cluster, ctx context.Context) error {
// See: https://kubernetes.io/docs/setup/best-practices/certificates/
bundle, err := pekahi.NewFileBundle(path.Join(CertsPath, "master"))
if err != nil {
return err
}
// TLS certificate
tlsCert, err := bundle.GetCertOrCSR("tls",
pekahi.NewServerTemplate(
[]string{"kube-apiserver", "kubernetes.default"},
[]net.IP{c.networking.NodeAddress.IP, c.networking.APIAddress},
),
)
if err != nil {
return err
}
// API tokens key
apiTokenKey, err := bundle.GetCertWithKey("api-tokens")
if err != nil {
return err
}
// Etcd tokens key
etcdTokenKey, err := bundle.GetCertWithKey("etcd-tokens")
if err != nil {
return err
}
// Kubelet certificate
kubeletCert, err := bundle.GetCertOrCSR("kubelet",
pekahi.NewClientTemplate("kube-apiserver-kubelet-client", "system:masters"),
)
if err != nil {
return err
}
// Controller manager certificate
controllersTLSCert, err := bundle.GetCertOrCSR("controllers-tls",
pekahi.NewServerTemplate([]string{"controllers"}, []net.IP{c.networking.NodeAddress.IP}),
)
if err != nil {
return err
}
// Controller manager API client certificate
controllersAPICert, err := bundle.GetCertOrCSR("controllers-api",
pekahi.NewClientTemplate("system:kube-controller-manager", ""),
)
if err != nil {
return err
}
// Scheduler API client certificate
schedulerAPICert, err := bundle.GetCertOrCSR("scheduler-api",
pekahi.NewClientTemplate("system:kube-scheduler", ""),
)
if err != nil {
return err
}
// Root client certificate
rootClientCert, err := bundle.GetCertOrCSR("root",
pekahi.NewClientTemplate("root", "system:masters"),
)
m := &MasterCerts{
TLS: tlsCert,
APITokens: apiTokenKey,
EtcdTokens: etcdTokenKey,
Kubelet: kubeletCert,
ControllersTLS: controllersTLSCert,
ControllersAPI: controllersAPICert,
SchedulerAPI: schedulerAPICert,
RootClient: rootClientCert,
}
c.pki.TLS.Sign(m.TLS, pekahi.NewServerTemplate(m.TLS.CSR.DNSNames, m.TLS.CSR.IPAddresses))
c.pki.Kubelet.Sign(m.Kubelet, pekahi.NewClientTemplate(m.Kubelet.CSR.Subject.CommonName, "system:masters"))
c.pki.TLS.Sign(m.ControllersTLS, pekahi.NewServerTemplate(m.ControllersTLS.CSR.DNSNames, m.ControllersTLS.CSR.IPAddresses))
c.pki.API.Sign(m.ControllersAPI, pekahi.NewClientTemplate(m.ControllersAPI.CSR.Subject.CommonName, ""))
c.pki.API.Sign(m.SchedulerAPI, pekahi.NewClientTemplate(m.SchedulerAPI.CSR.Subject.CommonName, ""))
c.pki.API.Sign(m.RootClient, pekahi.NewClientTemplate(m.RootClient.CSR.Subject.CommonName, "system:masters"))
c.masterCerts = m
u.Manager.Logger.Info("master pki initialized")
return nil
},
}
// Unit for discovering certificate authorities from memberlist
var pkiCA = &Unit{
Name: "pki-ca",
Ready: func(u *Unit, c *Cluster) bool {
return (c.pki != nil &&
c.certs != nil &&
c.pki.TLS.Cert != nil &&
c.pki.Kubelet.Cert != nil &&
c.pki.API.Cert != nil &&
c.certs.TLS.Cert != nil &&
c.certs.API.Cert != nil)
c.pki.API.Cert != nil)
},
Run: func(u *Unit, c *Cluster, ctx context.Context) error {
// Prepare the cluster PKI
if c.node.Role == "master" {
ca, err := pki.NewClusterCA(path.Join(CertsPath, "pki"))
if err != nil {
return err
}
masterCerts, err := pki.NewMasterCerts(
path.Join(CertsPath, "master"),
"kubernetes.default",
c.networking.NodeAddress.IP,
c.networking.APIAddress,
)
if err != nil {
return err
}
c.pki = ca
c.masterCerts = masterCerts
c.pki.SignMasterCerts(c.masterCerts)
u.Manager.Logger.Info("master pki initialized")
} else {
ca, err := pki.EmptyClusterCA(path.Join(CertsPath, "pki"))
if err != nil {
return err
}
c.pki = ca
u.Manager.Logger.Info("placeholder pki initialized")
if c.pki != nil {
return nil
}
bundle, err := pekahi.NewFileBundle(path.Join(CertsPath, "pki"))
if err != nil {
return nil
}
c.pki = &ClusterCA{
TLS: bundle.GetCertificate("tls"),
Kubelet: bundle.GetCertificate("kubelet"),
API: bundle.GetCertificate("api"),
}
c.state.PKI = c.pki
// Initialize node certificates
certs, err := pki.NewNodeCerts(path.Join(CertsPath, "node"), c.node.Name)
u.Manager.Logger.Info("placeholder pki initialized")
return nil
},
}
// Unit for requesting and getting node certs
var nodeCerts = &Unit{
Name: "pki-node",
Run: func(u *Unit, c *Cluster, ctx context.Context) error {
// See: https://kubernetes.io/docs/setup/best-practices/certificates/
bundle, err := pekahi.NewFileBundle(path.Join(CertsPath, "node"))
if err != nil {
return err
}
// Service certificate
tlsCert, err := bundle.GetCertOrCSR("tls",
pekahi.NewServerTemplate([]string{c.node.Name}, []net.IP{}),
)
if err != nil {
return err
}
// API certificate
apiCert, err := bundle.GetCertOrCSR("api",
pekahi.NewClientTemplate("system:nodes:"+c.node.Name, "system:nodes"),
)
if err != nil {
return err
}
c.certs = certs
c.state.Certificates = make(map[string]*pki.NodeCerts)
c.state.Certificates[c.node.Name] = certs
c.certs = &NodeCerts{
TLS: tlsCert,
API: apiCert,
}
c.state.Certificates[c.node.Name] = c.certs
u.Manager.Logger.Info("node certificates initialized")
return nil
},
Ready: func(u *Unit, c *Cluster) bool {
return (c.certs != nil &&
c.certs.API.Cert != nil &&
c.certs.TLS.Cert != nil)
},
}
......@@ -8,7 +8,7 @@ import (
var memberlist = &Unit{
Name: "memberlist",
Dependencies: []*Unit{certs},
Dependencies: []*Unit{pkiCA},
Run: func(u *Unit, c *Cluster, ctx context.Context) error {
node := u.Manager.State.node
ml := sml.New[HeptoMeta, HeptoState](node.Name, node.IP, node.Port, node.Anchors, c.settings.Key, c.settings.Logger)
......@@ -37,7 +37,7 @@ var remote_master = &Unit{
}
return nil
},
Ready: func(u *Unit, c *Cluster) bool {
Ready: func(u *Unit, c *Cluster) bool {
return c.masterNode != nil
},
}
......@@ -11,7 +11,7 @@ var EtcdPath = "/etcd"
var etcd = &Unit{
Name: "etcd",
Dependencies: []*Unit{certs},
Dependencies: []*Unit{masterCerts},
Run: func(u *Unit, c *Cluster, ctx context.Context) error {
config := embed.NewConfig()
config.Dir = EtcdPath
......
......@@ -27,7 +27,7 @@ var configPath = "/etc"
var apiserver = &Unit{
Name: "kube-apiserver",
Dependencies: []*Unit{etcd, certs, vpn},
Dependencies: []*Unit{etcd, masterCerts, pkiCA, vpn},
Run: func(u *Unit, c *Cluster, ctx context.Context) error {
args := []string{
"--bind-address", c.networking.NodeAddress.IP.String(),
......@@ -81,7 +81,7 @@ var apiserver = &Unit{
var controller_manager = &Unit{
Name: "kube-controller-manager",
Dependencies: []*Unit{apiserver, certs},
Dependencies: []*Unit{apiserver, masterCerts, pkiCA},
Run: func(u *Unit, c *Cluster, ctx context.Context) error {
cmConfig := KubeConfig{
URL: fmt.Sprintf("https://[%s]:6443", c.networking.NodeAddress.IP.String()),
......@@ -126,7 +126,7 @@ var controller_manager = &Unit{
var kube_scheduler = &Unit{
Name: "kube-scheduler",
Dependencies: []*Unit{apiserver, certs},
Dependencies: []*Unit{apiserver, masterCerts, pkiCA},
Run: func(u *Unit, c *Cluster, ctx context.Context) error {
schedulerConfig := KubeConfig{
URL: fmt.Sprintf("https://[%s]:6443", c.networking.NodeAddress.IP.String()),
......@@ -156,7 +156,7 @@ var kube_scheduler = &Unit{
var kube_kubelet = &Unit{
Name: "kubelet",
Dependencies: []*Unit{remote_master, containerd, certs},
Dependencies: []*Unit{remote_master, containerd, nodeCerts, pkiCA},
Run: func(u *Unit, c *Cluster, ctx context.Context) error {
kubeletKubeConfig := KubeConfig{
URL: fmt.Sprintf("https://[%s]:6443", c.masterNode.String()),
......
......@@ -8,7 +8,6 @@ import (
"github.com/go-logr/logr"
"go.acides.org/hepto/pkg/daeman"
"go.acides.org/hepto/pkg/pki"
"go.acides.org/hepto/pkg/utils"
"go.acides.org/hepto/pkg/wg"
"go.uber.org/zap"
......@@ -56,9 +55,9 @@ type Cluster struct {
state *HeptoState
masterNode *HeptoMeta
certs *pki.NodeCerts
masterCerts *pki.MasterCerts
pki *pki.ClusterCA
certs *NodeCerts
masterCerts *MasterCerts
pki *ClusterCA
vpn *wg.Wireguard
}
......@@ -88,7 +87,9 @@ func NewManager(settings *ClusterSettings, node *NodeSettings, logger logr.Logge
VpnIP: networking.NodeAddress.IP,
},
nodes: []*HeptoMeta{},
state: &HeptoState{},
state: &HeptoState{
Certificates: make(map[string]*NodeCerts),
},
}
units := []*Unit{}
switch node.Role {
......
......@@ -4,7 +4,7 @@ import (
"encoding/json"
"net"
"go.acides.org/hepto/pkg/pki"
"go.acides.org/pekahi"
)
// Represents a node metadata
......@@ -45,10 +45,10 @@ func (m *HeptoMeta) Key() string {
// Represents the cluster state
type HeptoState struct {
// Cluster CAs public certificates
PKI *pki.ClusterCA `json:"ca"`
PKI *ClusterCA `json:"ca"`
// Certificate per node, this should only
// be updated by the node itself
Certificates map[string]*pki.NodeCerts `json:"nodes"`
Certificates map[string]*NodeCerts `json:"nodes"`
}
func (m *HeptoMeta) Encode() ([]byte, error) {
......@@ -85,11 +85,14 @@ func (s *HeptoState) Merge(b []byte) (bool, error) {
return false, nil
}
change := false
change = s.PKI.Merge(remote.PKI) || change
change = mergeCert(s.PKI.TLS, remote.PKI.TLS) || change
change = mergeCert(s.PKI.Kubelet, remote.PKI.Kubelet) || change
change = mergeCert(s.PKI.API, remote.PKI.API) || change
for name, remoteCerts := range remote.Certificates {
_, ok := s.Certificates[name]
certs, ok := s.Certificates[name]
if ok {
change = s.Certificates[name].Merge(remoteCerts) || change
change = mergeCert(certs.TLS, remoteCerts.TLS) || change
change = mergeCert(certs.API, remoteCerts.API) || change
} else {
s.Certificates[name] = remoteCerts
change = true
......@@ -97,3 +100,24 @@ func (s *HeptoState) Merge(b []byte) (bool, error) {
}
return change, nil
}
// Merge a single node or master certificate
func mergeCert(local *pekahi.Certificate, remote *pekahi.Certificate) bool {
change := false
// Do not merge nothing
if remote == nil {
return change
}
// Import CSR to master for signing
if local.CSR == nil && remote.CSR != nil {
local.CSR = remote.CSR
change = true
}
// Import and save cert back to node
if local.Cert == nil && remote.Cert != nil {
local.Cert = remote.Cert
local.Save()
change = true
}
return change
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment