From ff7dd6f2a9aa804392755ab1b57d9812bd5d76b7 Mon Sep 17 00:00:00 2001
From: kaiyou <dev@kaiyou.fr>
Date: Sun, 2 Apr 2023 12:35:01 +0200
Subject: [PATCH] Inject the data directory into the service modules
---
cmd/hepto/config.go | 8 +-------
cmd/hepto/service.go | 1 +
services/certs.go | 8 +++-----
services/containerd.go | 14 ++++++--------
services/etcd.go | 5 ++---
services/k8s.go | 15 ++++++++++-----
services/manager.go | 4 +++-
7 files changed, 26 insertions(+), 29 deletions(-)
diff --git a/cmd/hepto/config.go b/cmd/hepto/config.go
index b8c2e76..603ce94 100644
--- a/cmd/hepto/config.go
+++ b/cmd/hepto/config.go
@@ -51,13 +51,7 @@ func (c *Config) SetupContainer() error {
return err
}
// Setup data persistence mounts
- c.Container.Mounts[services.CertsPath] = path.Join(c.Container.Data, "/certs")
- if c.Node.Role == "master" || c.Node.Role == "full" {
- c.Container.Mounts[services.EtcdPath] = path.Join(c.Container.Data, "/etcd")
- }
- if c.Node.Role == "node" || c.Node.Role == "full" {
- c.Container.Mounts[services.ContainerdPath] = path.Join(c.Container.Data, "/containerd")
- }
+ c.Container.Mounts[c.Cluster.DataDir] = c.Container.Data
// Set the container IP mask if required, defaults to /64
if len(c.Container.IP.Mask) == 0 {
mask := c.Container.IP.IP.DefaultMask()
diff --git a/cmd/hepto/service.go b/cmd/hepto/service.go
index ba85280..6a34a10 100644
--- a/cmd/hepto/service.go
+++ b/cmd/hepto/service.go
@@ -23,6 +23,7 @@ var Hepto = &cobra.Command{
Long: `Hepto is a Kubernetes distribution designed for geo-distributed
deployments, including across links with noticeable latency.`,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+ config.Cluster.DataDir = "/data"
// Print version if requested, verflag flags are declared
// by init() functions deep in k8s code
verflag.PrintAndExitIfRequested()
diff --git a/services/certs.go b/services/certs.go
index 6499759..8f7a654 100644
--- a/services/certs.go
+++ b/services/certs.go
@@ -8,8 +8,6 @@ import (
"go.acides.org/pekahi"
)
-const CertsPath = "/certs"
-
// Cluster PKI is made of three different CAs
type ClusterCA struct {
// Signs services exposed over the cluster
@@ -52,7 +50,7 @@ type NodeCerts struct {
var pkiManager = &Unit{
Name: "pki-manager",
Start: func(u *Unit, c *Cluster, ctx context.Context) error {
- bundle, err := pekahi.NewFileBundle(path.Join(CertsPath, "pki"))
+ bundle, err := pekahi.NewFileBundle(path.Join(c.settings.DataDir, "pki/ca"))
if err != nil {
return err
}
@@ -103,7 +101,7 @@ var pkiMaster = &Unit{
Dependencies: []*Unit{pkiManager},
Start: func(u *Unit, c *Cluster, ctx context.Context) error {
// See: https://kubernetes.io/docs/setup/best-practices/certificates/
- bundle, err := pekahi.NewFileBundle(path.Join(CertsPath, "master"))
+ bundle, err := pekahi.NewFileBundle(path.Join(c.settings.DataDir, "pki/master"))
if err != nil {
return err
}
@@ -187,7 +185,7 @@ var pkiNode = &Unit{
Name: "pki-node",
Start: func(u *Unit, c *Cluster, ctx context.Context) error {
// See: https://kubernetes.io/docs/setup/best-practices/certificates/
- bundle, err := pekahi.NewFileBundle(path.Join(CertsPath, "node"))
+ bundle, err := pekahi.NewFileBundle(path.Join(c.settings.DataDir, "pki/node"))
if err != nil {
return err
}
diff --git a/services/containerd.go b/services/containerd.go
index 244f526..f6becc2 100644
--- a/services/containerd.go
+++ b/services/containerd.go
@@ -3,6 +3,7 @@ package services
import (
"context"
"os"
+ "path"
"strings"
"github.com/containerd/containerd/defaults"
@@ -41,11 +42,8 @@ import (
)
const (
- ContainerdPath = "/containerd"
- ContainerdSocket = "/run/containerd.sock"
- ContainerdTTRPC = "/run/containerd.sock.ttrpc"
- ContainerdState = "/run/containerd"
- ContainerdTempMounts = "/run/containerd/tmpmounts"
+ ContainerdSocket = "/containerd.sock"
+ ContainerdTTRPC = "/containerd.sock.ttrpc"
)
var containerd = &Unit{
@@ -60,8 +58,8 @@ var containerd = &Unit{
}
cfg := &config.Config{
Version: 2,
- Root: ContainerdPath,
- State: ContainerdState,
+ Root: path.Join(c.settings.DataDir, "containerd"),
+ State: "/containerd",
GRPC: config.GRPCConfig{
Address: ContainerdSocket,
MaxRecvMsgSize: defaults.DefaultMaxRecvMsgSize,
@@ -76,7 +74,7 @@ var containerd = &Unit{
if err != nil {
return err
}
- err = mount.SetTempMountLocation(ContainerdTempMounts)
+ err = mount.SetTempMountLocation("/containerd/mounts")
if err != nil {
return err
}
diff --git a/services/etcd.go b/services/etcd.go
index e390ede..bcbfd0d 100644
--- a/services/etcd.go
+++ b/services/etcd.go
@@ -3,18 +3,17 @@ package services
import (
"context"
"fmt"
+ "path"
"go.etcd.io/etcd/server/v3/embed"
)
-var EtcdPath = "/etcd"
-
var etcd = &Unit{
Name: "etcd",
Dependencies: []*Unit{pkiMaster},
Run: func(u *Unit, c *Cluster, ctx context.Context) error {
config := embed.NewConfig()
- config.Dir = EtcdPath
+ config.Dir = path.Join(c.settings.DataDir, "etcd")
config.AuthToken = fmt.Sprintf("jwt,priv-key=%s,sign-method=ES256", c.masterCerts.EtcdTokens.KeyPath())
config.ZapLoggerBuilder = embed.NewZapLoggerBuilder(c.settings.ZapLogger)
server, err := embed.StartEtcd(config)
diff --git a/services/k8s.go b/services/k8s.go
index d99caee..97ff677 100644
--- a/services/k8s.go
+++ b/services/k8s.go
@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"path"
+ "time"
"github.com/spf13/pflag"
"go.acides.org/hepto/utils"
@@ -30,8 +31,6 @@ import (
"k8s.io/kubernetes/pkg/util/filesystem"
)
-var configPath = "/etc"
-
var kubeLogger = &Unit{
Name: "kube-logger",
Start: func(u *Unit, c *Cluster, ctx context.Context) error {
@@ -124,7 +123,7 @@ var kubeControllerManager = &Unit{
ClientCert: c.masterCerts.ControllersAPI.CertPath(),
ClientKey: c.masterCerts.ControllersAPI.KeyPath(),
}
- cmConfigPath := path.Join(configPath, "controller-manager.yaml")
+ cmConfigPath := "/controller-manager.yaml"
err := utils.WriteConfig(cmConfig, cmConfigPath)
if err != nil {
return err
@@ -211,28 +210,32 @@ var kubeKubelet = &Unit{
Name: "kubelet",
Dependencies: []*Unit{masterDiscovery, containerdGRPC, containerdTTRPC, pkiCA, pkiNode, kubeLogger},
Run: func(u *Unit, c *Cluster, ctx context.Context) error {
+ time.Sleep(10 * time.Second)
+ dataPath := path.Join(c.settings.DataDir, "kubelet")
kubeletKubeConfig := KubeConfig{
URL: fmt.Sprintf("https://[%s]:6443", c.masterNode.VpnIP.String()),
CACert: c.pki.TLS.CertPath(),
ClientCert: c.certs.API.CertPath(),
ClientKey: c.certs.API.KeyPath(),
}
- kubeletKubeConfigPath := path.Join(configPath, "kubelet-kubeconfig.yaml")
+ kubeletKubeConfigPath := "/kubelet-kubeconfig.yaml"
err := utils.WriteConfig(kubeletKubeConfig, kubeletKubeConfigPath)
if err != nil {
return err
}
kubeletConfig := KubeletConfig{
+ Data: dataPath,
CACert: c.pki.Kubelet.CertPath(),
TLSCert: c.certs.TLS.CertPath(),
TLSKey: c.certs.TLS.KeyPath(),
}
- kubeletConfigPath := path.Join(configPath, "kubelet.yaml")
+ kubeletConfigPath := "/kubelet.yaml"
err = utils.WriteConfig(kubeletConfig, kubeletConfigPath)
if err != nil {
return err
}
args := []string{
+ "--root-dir", dataPath,
"--kubeconfig", kubeletKubeConfigPath,
"--config", kubeletConfigPath,
"--container-runtime-endpoint", "unix://" + ContainerdSocket,
@@ -351,9 +354,11 @@ fileCheckFrequency: 1m
httpCheckFrequency: 1m
nodeStatusUpdateFrequency: 20s
nodeStatusReportFrequency: 5m
+volumePluginDir: "{{.Data}}/volume-plugins"
`
type KubeletConfig struct {
+ Data string
CACert string
TLSCert string
TLSKey string
diff --git a/services/manager.go b/services/manager.go
index 2a1e04b..6954a5e 100644
--- a/services/manager.go
+++ b/services/manager.go
@@ -18,6 +18,8 @@ import (
)
type ClusterSettings struct {
+ // Data storage path
+ DataDir string
// Logger interface
Logger logr.Logger
// Concrete zap logger for etcd
@@ -74,7 +76,7 @@ type Unit = daeman.Unit[*Cluster]
func NewManager(settings *ClusterSettings, node *NodeSettings, logger logr.Logger) (*daeman.Manager[*Cluster], error) {
networking := NewClusterNetworking(settings.Name, node.Name)
// Initialize an empty pki for discovery
- bundle, err := pekahi.NewFileBundle(path.Join(CertsPath, "pki"))
+ bundle, err := pekahi.NewFileBundle(path.Join(settings.DataDir, "pki/ca"))
if err != nil {
return nil, err
}
--
GitLab