Inject the data directory into the service modules

cmd/hepto/config.go

@@ -51,13 +51,7 @@ func (c *Config) SetupContainer() error {
 		return err
 	}
 	// Setup data persistence mounts
-	c.Container.Mounts[services.CertsPath] = path.Join(c.Container.Data, "/certs")
-	if c.Node.Role == "master" || c.Node.Role == "full" {
-		c.Container.Mounts[services.EtcdPath] = path.Join(c.Container.Data, "/etcd")
-	}
-	if c.Node.Role == "node" || c.Node.Role == "full" {
-		c.Container.Mounts[services.ContainerdPath] = path.Join(c.Container.Data, "/containerd")
-	}
+	c.Container.Mounts[c.Cluster.DataDir] = c.Container.Data
 	// Set the container IP mask if required, defaults to /64
 	if len(c.Container.IP.Mask) == 0 {
 		mask := c.Container.IP.IP.DefaultMask()

cmd/hepto/service.go

@@ -23,6 +23,7 @@ var Hepto = &cobra.Command{
 	Long: `Hepto is a Kubernetes distribution designed for geo-distributed
 	deployments, including across links with noticeable latency.`,
 	PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+		config.Cluster.DataDir = "/data"
 		// Print version if requested, verflag flags are declared
 		// by init() functions deep in k8s code
 		verflag.PrintAndExitIfRequested()

services/certs.go

@@ -8,8 +8,6 @@ import (
 	"go.acides.org/pekahi"
 )
 
-const CertsPath = "/certs"
-
 // Cluster PKI is made of three different CAs
 type ClusterCA struct {
 	// Signs services exposed over the cluster
@@ -52,7 +50,7 @@ type NodeCerts struct {
 var pkiManager = &Unit{
 	Name: "pki-manager",
 	Start: func(u *Unit, c *Cluster, ctx context.Context) error {
-		bundle, err := pekahi.NewFileBundle(path.Join(CertsPath, "pki"))
+		bundle, err := pekahi.NewFileBundle(path.Join(c.settings.DataDir, "pki/ca"))
 		if err != nil {
 			return err
 		}
@@ -103,7 +101,7 @@ var pkiMaster = &Unit{
 	Dependencies: []*Unit{pkiManager},
 	Start: func(u *Unit, c *Cluster, ctx context.Context) error {
 		// See: https://kubernetes.io/docs/setup/best-practices/certificates/
-		bundle, err := pekahi.NewFileBundle(path.Join(CertsPath, "master"))
+		bundle, err := pekahi.NewFileBundle(path.Join(c.settings.DataDir, "pki/master"))
 		if err != nil {
 			return err
 		}
@@ -187,7 +185,7 @@ var pkiNode = &Unit{
 	Name: "pki-node",
 	Start: func(u *Unit, c *Cluster, ctx context.Context) error {
 		// See: https://kubernetes.io/docs/setup/best-practices/certificates/
-		bundle, err := pekahi.NewFileBundle(path.Join(CertsPath, "node"))
+		bundle, err := pekahi.NewFileBundle(path.Join(c.settings.DataDir, "pki/node"))
 		if err != nil {
 			return err
 		}

services/containerd.go

@@ -3,6 +3,7 @@ package services
 import (
 	"context"
 	"os"
+	"path"
 	"strings"
 
 	"github.com/containerd/containerd/defaults"
@@ -41,11 +42,8 @@ import (
 )
 
 const (
-	ContainerdPath       = "/containerd"
-	ContainerdSocket     = "/run/containerd.sock"
-	ContainerdTTRPC      = "/run/containerd.sock.ttrpc"
-	ContainerdState      = "/run/containerd"
-	ContainerdTempMounts = "/run/containerd/tmpmounts"
+	ContainerdSocket = "/containerd.sock"
+	ContainerdTTRPC  = "/containerd.sock.ttrpc"
 )
 
 var containerd = &Unit{
@@ -60,8 +58,8 @@ var containerd = &Unit{
 		}
 		cfg := &config.Config{
 			Version: 2,
-			Root:    ContainerdPath,
-			State:   ContainerdState,
+			Root:    path.Join(c.settings.DataDir, "containerd"),
+			State:   "/containerd",
 			GRPC: config.GRPCConfig{
 				Address:        ContainerdSocket,
 				MaxRecvMsgSize: defaults.DefaultMaxRecvMsgSize,
@@ -76,7 +74,7 @@ var containerd = &Unit{
 		if err != nil {
 			return err
 		}
-		err = mount.SetTempMountLocation(ContainerdTempMounts)
+		err = mount.SetTempMountLocation("/containerd/mounts")
 		if err != nil {
 			return err
 		}

services/etcd.go

@@ -3,18 +3,17 @@ package services
 import (
 	"context"
 	"fmt"
+	"path"
 
 	"go.etcd.io/etcd/server/v3/embed"
 )
 
-var EtcdPath = "/etcd"
-
 var etcd = &Unit{
 	Name:         "etcd",
 	Dependencies: []*Unit{pkiMaster},
 	Run: func(u *Unit, c *Cluster, ctx context.Context) error {
 		config := embed.NewConfig()
-		config.Dir = EtcdPath
+		config.Dir = path.Join(c.settings.DataDir, "etcd")
 		config.AuthToken = fmt.Sprintf("jwt,priv-key=%s,sign-method=ES256", c.masterCerts.EtcdTokens.KeyPath())
 		config.ZapLoggerBuilder = embed.NewZapLoggerBuilder(c.settings.ZapLogger)
 		server, err := embed.StartEtcd(config)

services/k8s.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"path"
+	"time"
 
 	"github.com/spf13/pflag"
 	"go.acides.org/hepto/utils"
@@ -30,8 +31,6 @@ import (
 	"k8s.io/kubernetes/pkg/util/filesystem"
 )
 
-var configPath = "/etc"
-
 var kubeLogger = &Unit{
 	Name: "kube-logger",
 	Start: func(u *Unit, c *Cluster, ctx context.Context) error {
@@ -124,7 +123,7 @@ var kubeControllerManager = &Unit{
 			ClientCert: c.masterCerts.ControllersAPI.CertPath(),
 			ClientKey:  c.masterCerts.ControllersAPI.KeyPath(),
 		}
-		cmConfigPath := path.Join(configPath, "controller-manager.yaml")
+		cmConfigPath := "/controller-manager.yaml"
 		err := utils.WriteConfig(cmConfig, cmConfigPath)
 		if err != nil {
 			return err
@@ -211,28 +210,32 @@ var kubeKubelet = &Unit{
 	Name:         "kubelet",
 	Dependencies: []*Unit{masterDiscovery, containerdGRPC, containerdTTRPC, pkiCA, pkiNode, kubeLogger},
 	Run: func(u *Unit, c *Cluster, ctx context.Context) error {
+		time.Sleep(10 * time.Second)
+		dataPath := path.Join(c.settings.DataDir, "kubelet")
 		kubeletKubeConfig := KubeConfig{
 			URL:        fmt.Sprintf("https://[%s]:6443", c.masterNode.VpnIP.String()),
 			CACert:     c.pki.TLS.CertPath(),
 			ClientCert: c.certs.API.CertPath(),
 			ClientKey:  c.certs.API.KeyPath(),
 		}
-		kubeletKubeConfigPath := path.Join(configPath, "kubelet-kubeconfig.yaml")
+		kubeletKubeConfigPath := "/kubelet-kubeconfig.yaml"
 		err := utils.WriteConfig(kubeletKubeConfig, kubeletKubeConfigPath)
 		if err != nil {
 			return err
 		}
 		kubeletConfig := KubeletConfig{
+			Data:    dataPath,
 			CACert:  c.pki.Kubelet.CertPath(),
 			TLSCert: c.certs.TLS.CertPath(),
 			TLSKey:  c.certs.TLS.KeyPath(),
 		}
-		kubeletConfigPath := path.Join(configPath, "kubelet.yaml")
+		kubeletConfigPath := "/kubelet.yaml"
 		err = utils.WriteConfig(kubeletConfig, kubeletConfigPath)
 		if err != nil {
 			return err
 		}
 		args := []string{
+			"--root-dir", dataPath,
 			"--kubeconfig", kubeletKubeConfigPath,
 			"--config", kubeletConfigPath,
 			"--container-runtime-endpoint", "unix://" + ContainerdSocket,
@@ -351,9 +354,11 @@ fileCheckFrequency: 1m
 httpCheckFrequency: 1m
 nodeStatusUpdateFrequency: 20s
 nodeStatusReportFrequency: 5m
+volumePluginDir: "{{.Data}}/volume-plugins"
 `
 
 type KubeletConfig struct {
+	Data    string
 	CACert  string
 	TLSCert string
 	TLSKey  string

services/manager.go

@@ -18,6 +18,8 @@ import (
 )
 
 type ClusterSettings struct {
+	// Data storage path
+	DataDir string
 	// Logger interface
 	Logger logr.Logger
 	// Concrete zap logger for etcd
@@ -74,7 +76,7 @@ type Unit = daeman.Unit[*Cluster]
 func NewManager(settings *ClusterSettings, node *NodeSettings, logger logr.Logger) (*daeman.Manager[*Cluster], error) {
 	networking := NewClusterNetworking(settings.Name, node.Name)
 	// Initialize an empty pki for discovery
-	bundle, err := pekahi.NewFileBundle(path.Join(CertsPath, "pki"))
+	bundle, err := pekahi.NewFileBundle(path.Join(settings.DataDir, "pki/ca"))
 	if err != nil {
 		return nil, err
 	}