From 40edcdef44f80b36629ff3b97f2d158606f272cf Mon Sep 17 00:00:00 2001
From: kaiyou <dev@kaiyou.fr>
Date: Tue, 7 Feb 2023 21:56:33 +0100
Subject: [PATCH] Switch to using cobra only, and add kubectl subcommand
---
cmd/hepto.go | 84 +++++-----------------------
cmd/hepto/config.go | 42 --------------
cmd/hepto/hooks.go | 87 +++++++++++++++++++++++++++++
cmd/hepto/root.go | 81 +++++----------------------
cmd/hepto/service.go | 130 +++++++++++++++++++++++++++++++++++++++++++
go.mod | 2 +-
go.sum | 4 +-
7 files changed, 247 insertions(+), 183 deletions(-)
create mode 100644 cmd/hepto/hooks.go
create mode 100644 cmd/hepto/service.go
diff --git a/cmd/hepto.go b/cmd/hepto.go
index 667a15a..f49e77e 100644
--- a/cmd/hepto.go
+++ b/cmd/hepto.go
@@ -1,86 +1,30 @@
package main
import (
- "bytes"
- "context"
"fmt"
- "net/http"
- "net/http/pprof"
"os"
"path/filepath"
- "strings"
- "syscall"
- containerd "github.com/containerd/containerd/cmd/containerd/command"
- ctr "github.com/containerd/containerd/cmd/ctr/app"
- "github.com/containerd/containerd/plugin"
- "github.com/containerd/containerd/runtime/v2/runc/manager"
- _ "github.com/containerd/containerd/runtime/v2/runc/task/plugin"
- shimv2 "github.com/containerd/containerd/runtime/v2/shim"
- runc "github.com/opencontainers/runc/cmd"
"go.acides.org/hepto/cmd/hepto"
- "golang.org/x/sys/unix"
- kubectl "k8s.io/kubectl/pkg/cmd"
)
func main() {
- bin := filepath.Base(os.Args[0])
- // TODO move this to a separate function
- // Enable pprof if required in the current or parent command line
- template := "/proc/%d/cmdline"
- cmdline, _ := os.ReadFile(fmt.Sprintf(template, syscall.Getpid()))
- pcmdline, _ := os.ReadFile(fmt.Sprintf(template, syscall.Getppid()))
- arg := []byte("--pprof")
- if bytes.Contains(cmdline, arg) || bytes.Contains(pcmdline, arg) {
- mux := http.NewServeMux()
- for _, handler := range []string{"allocs", "blocks", "cmdline", "goroutine", "heap", "mutex", "profile", "threadcreate", "trace"} {
- mux.Handle(fmt.Sprintf("/debug/pprof/%s", handler), pprof.Handler(handler))
- }
- mux.HandleFunc("/debug/pprof/", pprof.Index)
- go http.ListenAndServe(":0", mux)
- }
- // Hook external execs to the single binary
- var err error
- if bin == "mount" {
- // Hook the mount command for mounting configmaps
- // This is fairly naive mount implementation, kubelet only evers calls
- // mount with very simple very formatted arguments in that order:
- // mount -t tmpfs -o size=1234 /src /dst
- err = unix.Mount(os.Args[5], os.Args[6], os.Args[2], 0, os.Args[4])
- } else if bin == "umount" {
- // Same for umount
- err = unix.Unmount(os.Args[1], 0)
- } else if bin == "containerd" || (len(os.Args) > 1 && os.Args[1] == "publish") {
- // Containerd is also available under hepto name, guess based on
- // call arguments
- // This is some of an edge case, where containerd uses os.Executable
- // to get the current binary path (hence hepto single binary) then
- // passes that path as -publish-binary to its shim for callback
- err = containerd.App().Run(os.Args)
- } else if bin == "containerd-shim-runc-v2" || (len(os.Args) > 1 && os.Args[1] == "-namespace") {
- // Run the containerd shim
- // It is also available under hepto name, for similar reasons as
- // containerd, hence the different guess condition
- plugins := plugin.Graph(func(*plugin.Registration) bool { return false })
- for _, plug := range plugins {
- plug.Disable = !strings.HasPrefix(plug.URI(), "io.containerd.ttrpc")
+ // This is a multi-program binary, pass the program to
+ // cobra so that it handles subcommands properly
+ program := filepath.Base(os.Args[0])
+ if len(os.Args) > 1 {
+ switch os.Args[1] {
+ case "publish":
+ program = hepto.Containerd.Use
+ case "-namespare":
+ program = hepto.Shim.Use
}
- shimv2.RunManager(context.Background(), manager.NewShimManager("io.containerd.runc.v2"))
- } else if bin == "runc" {
- // Runc, as called by containerd shim
- runc.Run()
- } else if bin == "ctr" {
- // Run containerd cli client, for debugging purposes
- err = ctr.New().Run(os.Args)
- } else if bin == "kubectl" {
- // Run kubectl client, for debugging purposes
- err = kubectl.NewDefaultKubectlCommand().Execute()
- } else {
- // If no hook ran a different command, simply run hepto
- err = hepto.Hepto.Execute()
}
- if err != nil {
- fmt.Printf("unexpected error: %v", err)
+ // Use hepto as the default command in case no other matches
+ hepto.Hepto.Aliases = []string{program}
+ hepto.Root.SetArgs(append([]string{program}, os.Args[1:]...))
+ if err := hepto.Root.Execute(); err != nil {
+ fmt.Println(err)
os.Exit(1)
}
}
diff --git a/cmd/hepto/config.go b/cmd/hepto/config.go
index 64836a8..5c08097 100644
--- a/cmd/hepto/config.go
+++ b/cmd/hepto/config.go
@@ -7,18 +7,13 @@ import (
"github.com/go-logr/logr"
"github.com/go-logr/zapr"
- "github.com/spf13/cobra"
- "github.com/spf13/pflag"
- "github.com/spf13/viper"
"go.acides.org/hepto/pkg/cluster"
"go.acides.org/selfcontain"
- "k8s.io/component-base/version/verflag"
)
type Config struct {
DataDir string
BypassIPCheck bool
- Shell string
Pprof bool
Logger logr.Logger
LogLevel int
@@ -30,13 +25,6 @@ type Config struct {
var config Config
func (c *Config) Complete() error {
- // Print version if requested, verflag flags are declared
- // by init() functions deep in k8s code
- verflag.PrintAndExitIfRequested()
- // Mount a shell if required
- if c.Shell != "" {
- c.Container.Mounts[c.Shell] = c.Shell
- }
// Initialize logging, default to warn level
zapLogger, err := NewLogger(c.LogLevel)
if err != nil {
@@ -70,33 +58,3 @@ func (c *Config) Complete() error {
c.Container.Devices = additionalDevices
return nil
}
-
-func init() {
- // Hide unwanted flags declared inside k8s init() directly
- pflag.CommandLine.MarkHidden("azure-container-registry-config")
- cobra.OnInitialize(viper.AutomaticEnv)
-
- // General config
- Hepto.Flags().CountVarP(&config.LogLevel, "verbose", "v", "Make logs more verbose")
- Hepto.Flags().StringVar(&config.Shell, "shell", "", "Path to a debug shell instead of hepto")
- Hepto.Flags().BoolVar(&config.Pprof, "pprof", false, "Enable Golang pprof profiling")
- Hepto.Flags().StringVar(&config.DataDir, "data", "/var/lib", "Data root directory")
- Hepto.Flags().BoolVar(&config.BypassIPCheck, "bypass-ip-check", false, "Bypass initial IP check")
-
- // Cluster settings
- Hepto.Flags().StringVar(&config.Cluster.Name, "cluster", "hepto", "Hepto cluster name")
- Hepto.Flags().BytesHexVar(&config.Cluster.Key, "key", []byte{}, "Main cluster 32bytes key, hex-encoded")
-
- // Container settings
- Hepto.Flags().StringVar(&config.Container.Master, "iface", "eth0", "Master network interface")
- Hepto.Flags().IPVar(&config.Container.IP.IP, "ip", net.IP{}, "IP address for the public interface")
- Hepto.Flags().IPVar(&config.Container.GW, "gw", net.IP{}, "IP address of the network gateway")
- Hepto.Flags().IPSliceVar(&config.Container.DNS, "dns", defaultDNS, "DNS server IP addresses")
- Hepto.Flags().StringToStringVar(&config.Container.Mounts, "bind", map[string]string{}, "Additional bind mounts")
-
- // Node settings
- Hepto.Flags().IntVar(&config.Node.Port, "discovery-port", 7123, "TCP port used for discovering the cluster")
- Hepto.Flags().StringVar(&config.Node.Name, "name", "", "Hepto node name")
- Hepto.Flags().StringSliceVar(&config.Node.Anchors, "anchors", []string{}, "List of cluster anchors")
- Hepto.Flags().Var(&config.Node.Role, "role", "Node role inside the cluster")
-}
diff --git a/cmd/hepto/hooks.go b/cmd/hepto/hooks.go
new file mode 100644
index 0000000..8edcca4
--- /dev/null
+++ b/cmd/hepto/hooks.go
@@ -0,0 +1,87 @@
+package hepto
+
+import (
+ "context"
+ "os"
+ "strings"
+
+ containerd "github.com/containerd/containerd/cmd/containerd/command"
+ ctr "github.com/containerd/containerd/cmd/ctr/app"
+ "github.com/containerd/containerd/plugin"
+ "github.com/containerd/containerd/runtime/v2/runc/manager"
+ _ "github.com/containerd/containerd/runtime/v2/runc/task/plugin"
+ shimv2 "github.com/containerd/containerd/runtime/v2/shim"
+ runc "github.com/opencontainers/runc/cmd"
+ "github.com/spf13/cobra"
+ "golang.org/x/sys/unix"
+ kubectl "k8s.io/kubectl/pkg/cmd"
+)
+
+var Mount = &cobra.Command{
+ Use: "mount",
+ RunE: func(cmd *cobra.Command, args []string) error {
+ // Hook the mount command for mounting configmaps
+ // This is fairly naive mount implementation, kubelet only evers calls
+ // mount with very simple very formatted arguments in that order:
+ // mount -t tmpfs -o size=1234 /src /dst
+ return unix.Mount(os.Args[5], os.Args[6], os.Args[2], 0, os.Args[4])
+ },
+}
+
+var Umount = &cobra.Command{
+ Use: "umount",
+ RunE: func(cmd *cobra.Command, args []string) error {
+ // Hook the umount command for mounting configmaps
+ return unix.Unmount(os.Args[1], 0)
+ },
+}
+
+var Containerd = &cobra.Command{
+ Use: "containerd",
+ RunE: func(cmd *cobra.Command, args []string) error {
+ // Containerd is also available under hepto name, guess based on
+ // call arguments
+ // This is some of an edge case, where containerd uses os.Executable
+ // to get the current binary path (hence hepto single binary) then
+ // passes that path as -publish-binary to its shim for callback
+ return containerd.App().Run(os.Args)
+ },
+}
+
+var Shim = &cobra.Command{
+ Use: "shim",
+ Run: func(cmd *cobra.Command, args []string) {
+ // Run the containerd shim
+ // It is also available under hepto name, for similar reasons as
+ // containerd, hence the different guess condition
+ plugins := plugin.Graph(func(*plugin.Registration) bool { return false })
+ for _, plug := range plugins {
+ plug.Disable = !strings.HasPrefix(plug.URI(), "io.containerd.ttrpc")
+ }
+ shimv2.RunManager(context.Background(), manager.NewShimManager("io.containerd.runc.v2"))
+
+ },
+}
+
+var Runc = &cobra.Command{
+ Use: "runc",
+ Run: func(cmd *cobra.Command, args []string) {
+ runc.Run()
+ },
+}
+
+var Ctr = &cobra.Command{
+ Use: "ctr",
+ RunE: func(cmd *cobra.Command, args []string) error {
+ return ctr.New().Run(os.Args)
+
+ },
+}
+
+var Kubectl = &cobra.Command{
+ Use: "kubectl",
+ DisableFlagParsing: true,
+ RunE: func(cmd *cobra.Command, args []string) error {
+ return kubectl.NewDefaultKubectlCommand().Execute()
+ },
+}
diff --git a/cmd/hepto/root.go b/cmd/hepto/root.go
index d78467d..d428aa4 100644
--- a/cmd/hepto/root.go
+++ b/cmd/hepto/root.go
@@ -1,77 +1,22 @@
package hepto
import (
- "fmt"
- "net"
- "os"
- "os/exec"
- "time"
-
"github.com/spf13/cobra"
- "go.acides.org/hepto/pkg/cluster"
- "go.acides.org/selfcontain"
)
-var Hepto = &cobra.Command{
- Use: "hepto",
- Short: "A highly opinionated geo-distributed Kubernetes distro",
- Long: `Hepto is a Kubernetes distribution designed for geo-distributed
- deployments, including across links with noticeable latency.`,
- Run: func(cmd *cobra.Command, args []string) {
- config.Complete()
- err := selfcontain.RunFun(&config.Container, func() {
- cluster.Sysctl()
- }, func() {
- if config.Shell != "" {
- cmd := exec.Command(config.Shell)
- cmd.Stdin = os.Stdin
- cmd.Stdout = os.Stdout
- cmd.Stderr = os.Stderr
- err := cmd.Run()
- if err != nil {
- config.Logger.Error(err, "could not run shell")
- }
- return
- }
- config.Node.IP = waitForIP()
- config.Logger.Info("found current IP", "ip", config.Node.IP.String())
- c := cluster.New(&config.Cluster, &config.Node)
- c.Run()
- })
- if err != nil {
- config.Logger.Error(err, "could not initialize the wrapping container")
- }
- },
+// This is not a proper cobra command, but a placeholder instead
+// for handling the multi-program behavior
+var Root = &cobra.Command{
+ // We use error returns for actual errors, not flag errors
+ SilenceUsage: true,
}
-// Guess the current IP address
-func waitForIP() net.IP {
- // This is very useful for debugging, especially in network isolated
- // environments
- if config.BypassIPCheck {
- config.Logger.Info("bypassing IP check")
- if len(config.Container.IP.IP) > 0 {
- return config.Container.IP.IP
- }
- return net.ParseIP("::1")
- }
- // Otherwisem rely on opening a socket and probing the socket local
- // address for more reliability
- config.Logger.Info("determining current IP address")
- for {
- time.Sleep(time.Second)
- target := fmt.Sprintf("[%s]:53", config.Container.DNS[0].String())
- config.Logger.Info("connecting outbound to guess IP", "target", target)
- conn, err := net.Dial("udp", target)
- if err != nil {
- config.Logger.Info("could not connect")
- continue
- }
- localAddr := conn.LocalAddr().(*net.UDPAddr).IP
- conn.Close()
- if localAddr.IsLoopback() || localAddr.IsLinkLocalUnicast() {
- continue
- }
- return localAddr
- }
+func init() {
+ Root.AddCommand(Mount)
+ Root.AddCommand(Containerd)
+ Root.AddCommand(Shim)
+ Root.AddCommand(Runc)
+ Root.AddCommand(Kubectl)
+ Root.AddCommand(Ctr)
+ Root.AddCommand(Hepto)
}
diff --git a/cmd/hepto/service.go b/cmd/hepto/service.go
new file mode 100644
index 0000000..42c3d02
--- /dev/null
+++ b/cmd/hepto/service.go
@@ -0,0 +1,130 @@
+package hepto
+
+import (
+ "fmt"
+ "net"
+ "os"
+ "time"
+
+ "github.com/spf13/cobra"
+ "github.com/spf13/pflag"
+ "github.com/spf13/viper"
+ "go.acides.org/hepto/pkg/cluster"
+ "go.acides.org/selfcontain"
+ "k8s.io/component-base/version/verflag"
+)
+
+var Hepto = &cobra.Command{
+ Use: "hepto",
+ Short: "A highly opinionated geo-distributed Kubernetes distro",
+ Long: `Hepto is a Kubernetes distribution designed for geo-distributed
+ deployments, including across links with noticeable latency.`,
+ PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+ // Print version if requested, verflag flags are declared
+ // by init() functions deep in k8s code
+ verflag.PrintAndExitIfRequested()
+ // Complete the configuration
+ return config.Complete()
+ },
+}
+
+var Start = &cobra.Command{
+ Use: "start",
+ Short: "Start the hepto service",
+ RunE: func(cmd *cobra.Command, args []string) error {
+ cluster.Sysctl()
+ newArgs := append([]string{Run.Use}, os.Args[2:]...)
+ return selfcontain.RunWithArgs(&config.Container, newArgs)
+ },
+}
+
+var Run = &cobra.Command{
+ Use: "run",
+ Short: "Actually run hepto inside the container",
+ // Determine current IP, which should run only once inside the container
+ PreRun: func(cmd *cobra.Command, args []string) {
+ // This is very useful for debugging, especially in network isolated
+ // environments
+ if config.BypassIPCheck {
+ config.Logger.Info("bypassing IP check")
+ if len(config.Container.IP.IP) > 0 {
+ config.Node.IP = config.Container.IP.IP
+ } else {
+ config.Node.IP = net.ParseIP("::1")
+ }
+ return
+ }
+ // Otherwisem rely on opening a socket and probing the socket local
+ // address for more reliability
+ config.Logger.Info("determining current IP address")
+ for {
+ time.Sleep(time.Second)
+ target := fmt.Sprintf("[%s]:53", config.Container.DNS[0].String())
+ config.Logger.Info("connecting outbound to guess IP", "target", target)
+ conn, err := net.Dial("udp", target)
+ if err != nil {
+ config.Logger.Info("could not connect")
+ continue
+ }
+ localAddr := conn.LocalAddr().(*net.UDPAddr).IP
+ conn.Close()
+ if localAddr.IsLoopback() || localAddr.IsLinkLocalUnicast() {
+ continue
+ }
+ config.Logger.Info("found current IP", "ip", config.Node.IP.String())
+ config.Node.IP = localAddr
+ break
+ }
+ },
+ Run: func(cmd *cobra.Command, args []string) {
+ c := cluster.New(&config.Cluster, &config.Node)
+ c.Run()
+ },
+}
+
+var RunKubectl = &cobra.Command{
+ Use: "kubectl",
+ Short: "Run kubectl in the cluster",
+ RunE: func(cmd *cobra.Command, args []string) error {
+ container, err := selfcontain.Get(&config.Container)
+ if err != nil {
+ return err
+ }
+ return container.Exec(append(
+ []string{"/bin/kubectl", "--kubeconfig", "/root/.kube/config"},
+ args...,
+ ))
+ },
+}
+
+func init() {
+ // Hide unwanted flags declared inside k8s init() directly
+ pflag.CommandLine.MarkHidden("azure-container-registry-config")
+ cobra.OnInitialize(viper.AutomaticEnv)
+ Hepto.AddCommand(Start)
+ Hepto.AddCommand(Run)
+ Hepto.AddCommand(RunKubectl)
+
+ // General config
+ Hepto.PersistentFlags().CountVarP(&config.LogLevel, "verbose", "v", "Make logs more verbose")
+ Hepto.PersistentFlags().BoolVar(&config.Pprof, "pprof", false, "Enable Golang pprof profiling")
+ Hepto.PersistentFlags().StringVar(&config.DataDir, "data", "/var/lib", "Data root directory")
+ Hepto.PersistentFlags().BoolVar(&config.BypassIPCheck, "bypass-ip-check", false, "Bypass initial IP check")
+
+ // Cluster settings
+ Hepto.PersistentFlags().StringVar(&config.Cluster.Name, "cluster", "hepto", "Hepto cluster name")
+ Hepto.PersistentFlags().BytesHexVar(&config.Cluster.Key, "key", []byte{}, "Main cluster 32bytes key, hex-encoded")
+
+ // Container settings
+ Hepto.PersistentFlags().StringVar(&config.Container.Master, "iface", "eth0", "Master network interface")
+ Hepto.PersistentFlags().IPVar(&config.Container.IP.IP, "ip", net.IP{}, "IP address for the public interface")
+ Hepto.PersistentFlags().IPVar(&config.Container.GW, "gw", net.IP{}, "IP address of the network gateway")
+ Hepto.PersistentFlags().IPSliceVar(&config.Container.DNS, "dns", defaultDNS, "DNS server IP addresses")
+ Hepto.PersistentFlags().StringToStringVar(&config.Container.Mounts, "bind", map[string]string{}, "Additional bind mounts")
+
+ // Node settings
+ Hepto.PersistentFlags().IntVar(&config.Node.Port, "discovery-port", 7123, "TCP port used for discovering the cluster")
+ Hepto.PersistentFlags().StringVar(&config.Node.Name, "name", "", "Hepto node name")
+ Hepto.PersistentFlags().StringSliceVar(&config.Node.Anchors, "anchors", []string{}, "List of cluster anchors")
+ Hepto.PersistentFlags().Var(&config.Node.Role, "role", "Node role inside the cluster")
+}
diff --git a/go.mod b/go.mod
index 1347bf5..02a16d4 100644
--- a/go.mod
+++ b/go.mod
@@ -71,7 +71,7 @@ require (
github.com/spf13/viper v1.15.0
github.com/vishvananda/netlink v1.2.1-beta.2
go.acides.org/pekahi v0.1.1
- go.acides.org/selfcontain v0.1.1
+ go.acides.org/selfcontain v0.2.0
go.acides.org/sml v0.1.1
go.etcd.io/etcd/server/v3 v3.5.7
go.uber.org/zap v1.24.0
diff --git a/go.sum b/go.sum
index 8c16cad..c45e734 100644
--- a/go.sum
+++ b/go.sum
@@ -1326,8 +1326,8 @@ github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.acides.org/pekahi v0.1.1 h1:lohNKOhw9Fz5K1Q6K3tP7XFWc+d/O29D9AEXnY2EKU8=
go.acides.org/pekahi v0.1.1/go.mod h1:AxgN7Ss6dCRHoNOVWMymkmDafWYdDV7ce6jPl5bqyRc=
-go.acides.org/selfcontain v0.1.1 h1:a3TvW2TaujF4NDnVpFd7D9QPfkrFBXvIXN3VAIneZh4=
-go.acides.org/selfcontain v0.1.1/go.mod h1:cyKYsVw1scp6MTVIhquG+2OJrsyaDCwkXlsBvMO+cws=
+go.acides.org/selfcontain v0.2.0 h1:7b9rfBIGOqpPjqIaXo2h8OZUeCu8+XrFh9zzToi2JKM=
+go.acides.org/selfcontain v0.2.0/go.mod h1:cyKYsVw1scp6MTVIhquG+2OJrsyaDCwkXlsBvMO+cws=
go.acides.org/sml v0.1.1 h1:v424XQ1RhgZHfKG+rV0VZ8YL32HC1UlqbIqv4E4q5JU=
go.acides.org/sml v0.1.1/go.mod h1:lBAbmfk5FFmK5yt7pFoqwwGNtEfWL/aXAdCj0ry7Kj0=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
--
GitLab