diff --git a/cmd/hepto/defaults.go b/cmd/hepto/defaults.go index c3c3a312797d7c848ec1a028e435189b5ee1bb03..4d09f68de39a987b5063257faaec020017340d2e 100644 --- a/cmd/hepto/defaults.go +++ b/cmd/hepto/defaults.go @@ -2,6 +2,8 @@ package hepto import ( "net" + + "k8s.io/component-helpers/node/util/sysctl" ) // Default to Cloudflare DNS (2606:4700:4700::1111, 2606:4700:4700::1001) @@ -43,9 +45,37 @@ var additionalCapabilities = []string{ } // Required devices for kubernetes -var additionalDevices = []string{ +var requiredDevices = []string{ "/dev/kmsg", "/dev/full", "/dev/tty", "/dev/ptmx", + "/dev/null", + "/dev/zero", + "/dev/random", + "/dev/urandom", +} + +// General sysctl configs +// Copied from https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/cm/container_manager_linux.go +var systemSettings = map[string]int{ + sysctl.VMOvercommitMemory: sysctl.VMOvercommitMemoryAlways, + sysctl.VMPanicOnOOM: sysctl.VMPanicOnOOMInvokeOOMKiller, + sysctl.KernelPanic: sysctl.KernelPanicRebootTimeout, + sysctl.KernelPanicOnOops: sysctl.KernelPanicOnOopsAlways, + sysctl.RootMaxKeys: sysctl.RootMaxKeysSetting, + sysctl.RootMaxBytes: sysctl.RootMaxBytesSetting, +} + +// Desired system modules +var desiredModules = []string{ + "nf_conntrack", // Required by most CNI + "ip6_tables", // Required by kube-router, kube-proxy, Calico, and even Cilium + "ip6table_filter", + "ip6table_nat", + "ip6table_mangle", + "ip_tables", // Not used since hepto is IPv6 only, but required by most CNI + "iptable_filter", + "iptable_nat", + "iptable_mangle", } diff --git a/cmd/hepto/hepto.go b/cmd/hepto/hepto.go index dae826fb4f0e2809ade063cfcc8db100e3e77d48..9486d3559e2434ec69970058beb30f00c11e6306 100644 --- a/cmd/hepto/hepto.go +++ b/cmd/hepto/hepto.go @@ -10,6 +10,8 @@ import ( "github.com/go-logr/zapr" "github.com/spf13/cobra" + "github.com/spf13/pflag" + "github.com/spf13/viper" "go.acides.org/dolly" "go.acides.org/hepto/services" "go.acides.org/hepto/utils" @@ -17,17 +19,29 @@ import ( "k8s.io/component-helpers/node/util/sysctl" ) +// The main hepto command var Hepto = &cobra.Command{ Use: "hepto", Short: "A highly opinionated geo-distributed Kubernetes distro", Long: `Hepto is a Kubernetes distribution designed for geo-distributed deployments, including across links with noticeable latency.`, + PersistentPreRun: func(cmd *cobra.Command, args []string) { + viper.SetEnvPrefix("HEPTO") + viper.AutomaticEnv() + cmd.Flags().VisitAll(func(f *pflag.Flag) { + if !f.Changed && viper.IsSet(f.Name) { + cmd.Flags().Set(f.Name, fmt.Sprintf("%v", viper.Get(f.Name))) + } + }) + }, PreRunE: func(cmd *cobra.Command, args []string) error { - config.Cluster.DataDir = "/data" + err := config.Validate() + if err != nil { + return err + } // Print version if requested, verflag flags are declared // by init() functions deep in k8s code verflag.PrintAndExitIfRequested() - // Initialize logging, default to warn level zapLogger, logrusLogger, err := utils.NewLoggers(config.LogLevel, os.Stderr) if err != nil { @@ -41,17 +55,8 @@ var Hepto = &cobra.Command{ config.Cluster.ZapLogger = zapLogger config.Cluster.LogrusLogger = logrusLogger // Set the proper sysctl for the cluster - // Copied from https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/cm/container_manager_linux.go - desiredState := map[string]int{ - sysctl.VMOvercommitMemory: sysctl.VMOvercommitMemoryAlways, - sysctl.VMPanicOnOOM: sysctl.VMPanicOnOOMInvokeOOMKiller, - sysctl.KernelPanic: sysctl.KernelPanicRebootTimeout, - sysctl.KernelPanicOnOops: sysctl.KernelPanicOnOopsAlways, - sysctl.RootMaxKeys: sysctl.RootMaxKeysSetting, - sysctl.RootMaxBytes: sysctl.RootMaxBytesSetting, - } s := sysctl.New() - for key, value := range desiredState { + for key, value := range systemSettings { config.Logger.Info("setting sysctl", key, value) err := s.SetSysctl(key, value) if err != nil { @@ -61,17 +66,6 @@ var Hepto = &cobra.Command{ // Load useful kernel modules for later operations // This uses exec at the moment, which is undesireable in case hepto is used in a very // limited environment - desiredModules := []string{ - "nf_conntrack", // Required by most CNI - "ip6_tables", // Required by kube-router, kube-proxy, Calico, and even Cilium - "ip6table_filter", - "ip6table_nat", - "ip6table_mangle", - "ip_tables", // Not used since hepto is IPv6 only, but required by most CNI - "iptable_filter", - "iptable_nat", - "iptable_mangle", - } for _, mod := range desiredModules { config.Logger.Info("loading module", "module", mod) err := exec.Command("modprobe", "--", mod).Run() @@ -83,10 +77,7 @@ var Hepto = &cobra.Command{ }, RunE: func(cmd *cobra.Command, args []string) error { dataPath := path.Join(config.DataDir, config.Cluster.Name, config.Node.Name) - err := os.MkdirAll(dataPath, 0o755) - if err != nil { - return err - } + config.Cluster.DataDir = "/data" self, err := os.Executable() if err != nil { return err @@ -102,10 +93,7 @@ var Hepto = &cobra.Command{ c.AddAll(dolly.DefaultVolumes()) c.Add(dolly.NewBind(dataPath, config.Cluster.DataDir)) c.Add(dolly.NewBind(self, self)) - c.AddAll(dolly.NewDevicesOrPanic( - "/dev/null", "/dev/zero", "/dev/random", "/dev/urandom", - "/dev/kmsg", "/dev/full", "/dev/tty", "/dev/ptmx", - )) + c.AddAll(dolly.NewDevicesOrPanic(requiredDevices...)) for src, dst := range config.Mounts { c.Add(dolly.NewBind(src, dst)) }