diff --git a/cmd/hepto/config.go b/cmd/hepto/config.go
index 304cfab57e22f9b53441089a17147b60dbca8001..cadc06faec834eac000c538e96b31504137312d3 100644
--- a/cmd/hepto/config.go
+++ b/cmd/hepto/config.go
@@ -6,7 +6,7 @@ import (
"path"
"forge.tedomum.net/acides/hepto/pkg/cluster"
- "forge.tedomum.net/acides/hepto/pkg/selfcontain"
+ "forge.tedomum.net/acides/libs/selfcontain"
"github.com/go-logr/logr"
"github.com/go-logr/zapr"
"github.com/spf13/cobra"
diff --git a/cmd/hepto/root.go b/cmd/hepto/root.go
index c2d7c26fc5e180b038c43ae95bf11c70d5359c81..4f6b169e6c6763fdf036912da02d1527e1bfd660 100644
--- a/cmd/hepto/root.go
+++ b/cmd/hepto/root.go
@@ -8,7 +8,7 @@ import (
"time"
"forge.tedomum.net/acides/hepto/pkg/cluster"
- "forge.tedomum.net/acides/hepto/pkg/selfcontain"
+ "forge.tedomum.net/acides/libs/selfcontain"
"github.com/spf13/cobra"
)
diff --git a/go.mod b/go.mod
index 503fceadfc370a8680a088930d3a0bd2b50d8160..18a3454f51e421e8e1d7cc9deb5166ff6748bf03 100644
--- a/go.mod
+++ b/go.mod
@@ -61,11 +61,11 @@ replace (
require (
forge.tedomum.net/acides/libs/pekahi v0.1.0
+ forge.tedomum.net/acides/libs/selfcontain v0.1.0
+ forge.tedomum.net/acides/libs/sml v0.1.0
github.com/containerd/containerd v1.6.15
- github.com/containernetworking/plugins v1.2.0
github.com/go-logr/logr v1.2.3
github.com/go-logr/zapr v1.2.3
- github.com/hashicorp/memberlist v0.5.0
github.com/opencontainers/runc v1.1.4
github.com/pkg/errors v0.9.1
github.com/sirupsen/logrus v1.9.0
@@ -75,7 +75,6 @@ require (
github.com/vishvananda/netlink v1.2.1-beta.2
go.etcd.io/etcd/server/v3 v3.5.7
go.uber.org/zap v1.24.0
- golang.org/x/net v0.5.0
golang.org/x/sys v0.4.0
golang.zx2c4.com/wireguard/wgctrl v0.0.0-20221104135756-97bc4ad4a1cb
k8s.io/apiserver v0.26.1
@@ -135,6 +134,7 @@ require (
github.com/containerd/ttrpc v1.1.1-0.20220420014843-944ef4a40df3 // indirect
github.com/containerd/typeurl v1.0.3-0.20220422153119-7f6e6d160d67 // indirect
github.com/containernetworking/cni v1.1.2 // indirect
+ github.com/containernetworking/plugins v1.2.0 // indirect
github.com/containers/ocicrypt v1.1.7 // indirect
github.com/coreos/go-oidc v2.2.1+incompatible // indirect
github.com/coreos/go-semver v0.3.1 // indirect
@@ -191,6 +191,7 @@ require (
github.com/hashicorp/go-sockaddr v1.0.2 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
+ github.com/hashicorp/memberlist v0.5.0 // indirect
github.com/imdario/mergo v0.3.13 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/intel/goresctrl v0.3.0 // indirect
@@ -291,6 +292,7 @@ require (
go.uber.org/multierr v1.9.0 // indirect
golang.org/x/crypto v0.5.0 // indirect
golang.org/x/mod v0.7.0 // indirect
+ golang.org/x/net v0.5.0 // indirect
golang.org/x/oauth2 v0.4.0 // indirect
golang.org/x/sync v0.1.0 // indirect
golang.org/x/term v0.4.0 // indirect
diff --git a/go.sum b/go.sum
index 305538efa50bc74f3f77526aaab4cdf956857136..2af81e451f2f26fd9d4b359de8d4b0d2ce7023a7 100644
--- a/go.sum
+++ b/go.sum
@@ -373,6 +373,10 @@ cloud.google.com/go/workflows v1.9.0/go.mod h1:ZGkj1aFIOd9c8Gerkjjq7OW7I5+l6cSvT
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
forge.tedomum.net/acides/libs/pekahi v0.1.0 h1:6FUFwnHTtdcLb7wcXhb/+nwbjrgL0Wd3Mj0pQkJ/vl0=
forge.tedomum.net/acides/libs/pekahi v0.1.0/go.mod h1:EoxRk98rl6+UxbY+5lYkayo9tIHVrXKrECUso0O7mSY=
+forge.tedomum.net/acides/libs/selfcontain v0.1.0 h1:8mNW+bx6IXQxKS409v26CFDJg2cbshIA3CZIxSNJ74E=
+forge.tedomum.net/acides/libs/selfcontain v0.1.0/go.mod h1:LbP6zohwtluRxXRWSwYsfrcDvPg1G0B6jc00QMnfPXw=
+forge.tedomum.net/acides/libs/sml v0.1.0 h1:PY2vmd3k+YlvyGx0el7gHwhVljVxAXp0/oEPfODtZr0=
+forge.tedomum.net/acides/libs/sml v0.1.0/go.mod h1:6hL231StaBRHklykR7amZwGTcMsa3ojR+L1ohs71oVE=
github.com/AdaLogics/go-fuzz-headers v0.0.0-20220824214621-3c06a36a6952/go.mod h1:i9fr2JpcEcY/IHEvzCM3qXUZYOQHgR89dt4es1CgMhc=
github.com/AdaLogics/go-fuzz-headers v0.0.0-20221118232415-3345c89a7c72/go.mod h1:VzwV+t+dZ9j/H867F1M2ziD+yLHtB46oM35FxxMJ4d0=
github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1 h1:EKPd1INOIyr5hWOWhvpmQpY6tKjeG0hT1s3AMC/9fic=
diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go
index 4c0e16608bcb42a3eebfc8b5a06d34404d7b7c6a..7f9cfac86014c3a501dd4d682cb616a34dc5acd5 100644
--- a/pkg/cluster/cluster.go
+++ b/pkg/cluster/cluster.go
@@ -7,11 +7,10 @@ import (
"fmt"
"os"
- "k8s.io/component-helpers/node/util/sysctl"
-
"forge.tedomum.net/acides/hepto/pkg/pki"
- "forge.tedomum.net/acides/hepto/pkg/sml"
"forge.tedomum.net/acides/hepto/pkg/wg"
+ "forge.tedomum.net/acides/libs/sml"
+ "k8s.io/component-helpers/node/util/sysctl"
)
type Cluster struct {
diff --git a/pkg/cluster/services.go b/pkg/cluster/services.go
index a01ff9ac7911edfff7da68200ede6c465944a0b0..89f49fc3854a61973b4eb8448653aec8960e7211 100644
--- a/pkg/cluster/services.go
+++ b/pkg/cluster/services.go
@@ -15,7 +15,6 @@ const configPath = "/config"
const etcdPath = "/etcd"
const binPath = "/bin"
const containerdPath = "/containerd"
-const imagePath = "/images"
func (c *Cluster) watchService(name string, errCh <-chan error) {
c.settings.Logger.Info("service started", "name", name)
diff --git a/pkg/selfcontain/config.go b/pkg/selfcontain/config.go
deleted file mode 100644
index 8c924ab2cab04f6867cfa53615169557f3472a4c..0000000000000000000000000000000000000000
--- a/pkg/selfcontain/config.go
+++ /dev/null
@@ -1,125 +0,0 @@
-package selfcontain
-
-import (
- "net"
- "os"
- "path"
-
- "github.com/go-logr/logr"
- "github.com/opencontainers/runc/libcontainer/configs"
- "github.com/opencontainers/runc/libcontainer/devices"
- "golang.org/x/sys/unix"
-)
-
-type Config struct {
- // Logger interface
- Logger logr.Logger
- // Container name
- Name string
- // Path to container data storage
- Data string
- // Name of the master interface for IPvlan
- Master string
- // Public IP of the container, can be nulled for autoconfiguration
- IP net.IPNet
- // Default gateway for the container, can be nulled for autoconfiguration
- GW net.IP
- // List of DNS servers for the container
- DNS []net.IP
- // List of non-standard capabilities (required capabilities are always enabled)
- Capabilities []string
- // List of non-standard devices
- Devices []string
- // List of non-standard bind-mounts
- Mounts map[string]string
-}
-
-// Turns a selfcontain configuration into a runc/libcontainer one
-func (c *Config) toLibcontainer() (*configs.Config, error) {
- // Setup devices, first copy base devices, then discover and add
- // configured devices
- devicePaths := append(baseDevices, c.Devices...)
- allowedDevices := make([]*devices.Device, len(devicePaths))
- deviceRules := make([]*devices.Rule, len(devicePaths))
- for n, path := range devicePaths {
- device, err := devices.DeviceFromPath(path, "rw")
- if err != nil {
- return nil, err
- }
- device.Rule.Allow = true
- allowedDevices[n] = device
- deviceRules[n] = &device.Rule
- }
- // Pts uses explicit wildcard rule
- deviceRules = append(deviceRules, &devices.Rule{
- Type: devices.CharDevice,
- Major: 136,
- Minor: devices.Wildcard,
- Permissions: "rwm",
- Allow: true,
- })
- // Setup mounts by appending bind mounts to default mounts (in that order,
- // otherwise mounting / messes up with bind mounts)
- mounts := []*configs.Mount{}
- for dest, source := range c.Mounts {
- c.Logger.Info("setting up mount", "source", source, "dest", dest)
- // Explicitely ignore errors here, since many error cases are actually
- // fine (already exists, not a directory, etc.) and actual issues are caught
- // later
- os.MkdirAll(source, 0o700)
- mount := &configs.Mount{
- Source: source,
- Destination: dest,
- Device: "bind",
- Flags: unix.MS_BIND | unix.MS_NOSUID | unix.MS_NODEV,
- }
- mounts = append(mounts, mount)
- }
- mounts = append(baseMounts, mounts...)
- // Concatenate base cpabilieis and additional ones from config
- capabilities := append(baseCapabilities, c.Capabilities...)
- // Create the rootfs directory (will later be mounted as tmpfs)
- root := path.Join(c.Data, "root")
- err := os.MkdirAll(root, 0o755)
- if err != nil {
- return nil, err
- }
- // Build the configuration
- return &configs.Config{
- // Any path would do, since this gets overwritten by mounting tmpfs to /
- // Still we create a subdirectory to avoid exposing other state accidentally
- Rootfs: root,
- RootPropagation: unix.MS_SHARED | unix.MS_REC,
- Hostname: c.Name,
- // Make all capabilities inherited and ambiant
- Capabilities: &configs.Capabilities{
- Bounding: capabilities,
- Effective: capabilities,
- Inheritable: capabilities,
- Permitted: capabilities,
- Ambient: capabilities,
- },
- Namespaces: []configs.Namespace{
- {Type: configs.NEWNS},
- {Type: configs.NEWUTS},
- {Type: configs.NEWIPC},
- {Type: configs.NEWPID},
- {Type: configs.NEWNET},
- {Type: configs.NEWCGROUP},
- },
- Devices: allowedDevices,
- Cgroups: &configs.Cgroup{
- Name: c.Name,
- Systemd: false,
- Resources: &configs.Resources{
- MemorySwappiness: nil,
- Devices: deviceRules,
- },
- },
- MaskPaths: maskedPath,
- ReadonlyPaths: readOnlyPath,
- Mounts: mounts,
- Networks: baseNets,
- ParentDeathSignal: 15,
- }, nil
-}
diff --git a/pkg/selfcontain/container.go b/pkg/selfcontain/container.go
deleted file mode 100644
index c9d15cd74cda38e169ef81df7bba5a499ff65ab1..0000000000000000000000000000000000000000
--- a/pkg/selfcontain/container.go
+++ /dev/null
@@ -1,116 +0,0 @@
-// The selfcontain package provides a containment structure to move the current
-// process inside a restricted container.
-//
-// This is accomplished thanks to runc/libcontainer library which in turns uses
-// C bindings to namespace primitives. Containment is not much configurable and
-// is fine-tuned to hepto itself.
-package selfcontain
-
-import (
- "os"
- "path/filepath"
-
- "github.com/opencontainers/runc/libcontainer"
- "github.com/opencontainers/runc/libcontainer/configs"
- _ "github.com/opencontainers/runc/libcontainer/nsenter"
-)
-
-type Container struct {
- config *Config
- self string
- container libcontainer.Container
- process libcontainer.Process
-}
-
-// Containerize the current process by runnig the current binary inside a container
-func New(config *Config) (*Container, error) {
- // Resolve self from the host
- self, err := os.Executable()
- if err != nil {
- return nil, err
- }
- // Prepare a libcontainer factory using the init path and args
- factoryConfig := func(f *libcontainer.LinuxFactory) error {
- f.InitPath = self
- f.InitArgs = []string{os.Args[0], argInit}
- return nil
- }
- factory, err := libcontainer.New(config.Data, factoryConfig)
- if err != nil {
- return nil, err
- }
- // Create and wrap the libcontainer instance
- config.Mounts[self] = self
- containerConfig, err := config.toLibcontainer()
- if err != nil {
- return nil, err
- }
- container, err := factory.Create(config.Name, containerConfig)
- if err != nil {
- return nil, err
- }
- return &Container{
- config: config,
- self: self,
- container: container,
- }, nil
-}
-
-func (c *Container) Start(args []string) error {
- process := libcontainer.Process{
- Args: append([]string{c.self}, args...),
- Stdin: os.Stdin,
- Stdout: os.Stdout,
- Stderr: os.Stderr,
- Init: true,
- }
- // Simply start the process instead of running, at this point init will be waiting
- // and listening on libcontainer control pipe
- err := c.container.Start(&process)
- if err != nil {
- c.container.Destroy()
- return err
- }
- pid, err := process.Pid()
- if err != nil {
- c.container.Destroy()
- return err
- }
- c.config.Logger.Info("container started", "parent", os.Getpid(), "container", pid)
- c.process = process
- return nil
-}
-
-// Actually run the container and block until the process has returned
-func (c *Container) Run() error {
- defer c.Destroy()
- err := c.container.Exec()
- if err != nil {
- return err
- }
- _, err = c.process.Wait()
- return err
-}
-
-func (c *Container) Destroy() error {
- c.config.Logger.Info("destroying container", "name", c.config.Name)
- err := c.container.Destroy()
- if err != nil {
- c.config.Logger.Error(err, "could not destroy")
- return err
- }
- err = os.RemoveAll(filepath.Join(c.config.Data, c.config.Name))
- if err != nil {
- c.config.Logger.Error(err, "could not cleanup")
- return err
- }
- return nil
-}
-
-func (c *Container) GetNS(nsType configs.NamespaceType) (string, error) {
- state, err := c.container.State()
- if err != nil {
- return "", err
- }
- return state.NamespacePaths[nsType], nil
-}
diff --git a/pkg/selfcontain/defaults.go b/pkg/selfcontain/defaults.go
deleted file mode 100644
index 991555cf36f86ea3a69b4ba35948603d4badfddf..0000000000000000000000000000000000000000
--- a/pkg/selfcontain/defaults.go
+++ /dev/null
@@ -1,95 +0,0 @@
-package selfcontain
-
-import (
- "github.com/opencontainers/runc/libcontainer/configs"
- "golang.org/x/sys/unix"
-)
-
-// This argument is passed back to the forked process to notify it should behave
-// as a libcontainer init, which in turn is handled by init()
-const argInit = "selfcontain-arg-libcontainer"
-
-// Restrict access to the bare minimum for container to run properly
-// See https://pkg.go.dev/github.com/opencontainers/runc@v1.0.2/libcontainer/specconv
-// for comments about issues with default restrictions
-var baseDevices = []string{
- "/dev/null",
- "/dev/zero",
- "/dev/urandom",
- "/dev/random",
-}
-
-// These path will be mounted as a default base inside the container
-var baseMounts = []*configs.Mount{
- // Start by mounting an empty root
- {
- Source: "tmpfs",
- Destination: "/",
- Device: "tmpfs",
- Flags: unix.MS_NOSUID | unix.MS_STRICTATIME,
- },
- // Used by so many programs for reflection that they must be mounted
- {
- Source: "proc",
- Destination: "/proc",
- Device: "proc",
- Flags: unix.MS_NOEXEC | unix.MS_NOSUID | unix.MS_NODEV,
- },
- {
- Source: "sysfs",
- Destination: "/sys",
- Device: "sysfs",
- Flags: unix.MS_NOEXEC | unix.MS_NOSUID | unix.MS_NODEV,
- PropagationFlags: []int{unix.MS_SHARED | unix.MS_REC},
- },
- // Used for container management
- {
- Source: "cgroup",
- Destination: "/sys/fs/cgroup",
- Device: "cgroup",
- Flags: unix.MS_NOEXEC | unix.MS_NOSUID | unix.MS_NODEV,
- PropagationFlags: []int{unix.MS_SHARED | unix.MS_REC},
- },
- // Dedicated pts instead of device rules
- {
- Source: "devpts",
- Destination: "/dev/pts",
- Device: "devpts",
- Flags: unix.MS_NOSUID | unix.MS_NOEXEC,
- Data: "newinstance,ptmxmode=0666,mode=0620,gid=5",
- },
-}
-
-// Restrict capabilities to strictly required capabilities
-// All capabilities are inheritable and ambient, so that init execve works properly
-var baseCapabilities = []string{
- // Required for later setting up networking
- "CAP_NET_ADMIN",
-}
-
-// These networks will be setup as a default base inside the container
-var baseNets = []*configs.Network{
- {
- Type: "loopback",
- Address: "127.0.0.1/0",
- Gateway: "localhost",
- },
-}
-
-// These path should not be readable at all from the container, despite /proc being
-// mounted there
-var maskedPath = []string{
- // This might leak system memory otherwise
- "/proc/kcore",
- "/sys/firmware",
-}
-
-// These path should never be written from inside the container
-var readOnlyPath = []string{
- // Kernel configuration shall not be modified
- //"/proc/sys",
- // IRQ shall not be triggered or setup
- "/proc/sysrq-trigger", "/proc/irq",
- // System but shall not be written to
- "/proc/bus",
-}
diff --git a/pkg/selfcontain/init.go b/pkg/selfcontain/init.go
deleted file mode 100644
index 0a9146ee36938195dd502cbd73eee2c8a7c2235a..0000000000000000000000000000000000000000
--- a/pkg/selfcontain/init.go
+++ /dev/null
@@ -1,37 +0,0 @@
-package selfcontain
-
-import (
- "fmt"
- "os"
- "runtime"
-
- "github.com/opencontainers/runc/libcontainer"
-)
-
-// libcontainer uses a three-step containerization technique:
-// 1. spawn a fifo for later communication with containerized init
-// 2. unshare the current process and fork/execve /proc/self/exe with a
-// special argument to trigger later initialization
-// 3. Use the fifo to communicatie with init and initialize mounts, etc.
-//
-// This init checks for said special argument and call into libcontainer
-// initialization routines, which in turn will execve the Process provided
-// command, which in our specific case is /proc/self/exe again.
-//
-// It seems we cannot avoid such complexity without dropping all of
-// libcontainer, which would make containerization even more complex.
-func init() {
- if len(os.Args) > 1 && os.Args[1] == argInit {
- // Do not start the full featured runtime
- runtime.GOMAXPROCS(1)
- runtime.LockOSThread()
- // Run libcontainer initialization, which will fork/exec to the
- // provided process executable, a.k.a ourselves
- factory, _ := libcontainer.New("")
- err := factory.StartInitialization()
- if err != nil {
- fmt.Printf("could not initialize app: %v", err)
- os.Exit(1)
- }
- }
-}
diff --git a/pkg/selfcontain/net.go b/pkg/selfcontain/net.go
deleted file mode 100644
index 07f1b268ac4ccde43c38f4e536d9110d1886a1b9..0000000000000000000000000000000000000000
--- a/pkg/selfcontain/net.go
+++ /dev/null
@@ -1,209 +0,0 @@
-package selfcontain
-
-import (
- "fmt"
- "io/ioutil"
- "net"
- "os"
- "path"
- "strings"
-
- "github.com/containernetworking/plugins/pkg/ns"
- "github.com/containernetworking/plugins/pkg/utils/sysctl"
- "github.com/opencontainers/runc/libcontainer/configs"
- "github.com/vishvananda/netlink"
-)
-
-const ACCEPT_RA = "net.ipv6.conf.eth0.accept_ra"
-const ACCEPT_PINFO = "net.ipv6.conf.eth0.accept_ra_pinfo"
-const ACCEPT_DFTRTR = "net.ipv6.conf.eth0.accept_ra_defrtr"
-
-// Setup networking inside the container
-// This must be called from outside the container, since it requires both access to the
-// host networking stack and the namespace networking stack
-func (c *Container) SetupNetworking(etc string) error {
- ifaceName, err := c.setupIPVlan(c.config.Master, 1500)
- if err != nil {
- return fmt.Errorf("could not create interface: %w", err)
- }
- netns, err := c.findNetNS()
- if err != nil {
- return fmt.Errorf("unable to find netns: %w", err)
- }
- err = netns.Do(func(_ ns.NetNS) error {
- // Rename the interface
- iface, err := netlink.LinkByName(ifaceName)
- if err != nil {
- return fmt.Errorf("could not find interface: %w", err)
- }
- err = netlink.LinkSetName(iface, "eth0")
- if err != nil {
- return fmt.Errorf("could not rename: %w", err)
- }
- err = netlink.LinkSetUp(iface)
- if err != nil {
- return fmt.Errorf("could not set the interface up: %w", err)
- }
- // Setup addresses and routes
- err = setupRA(iface)
- if err != nil {
- return fmt.Errorf("could not enable RA: %w", err)
- }
- err = setupAddress(iface, c.config.IP)
- if err != nil {
- return fmt.Errorf("could not set the address: %w", err)
- }
- err = setupGw(iface, c.config.GW)
- if err != nil {
- return fmt.Errorf("could not set the gateway: %w", err)
- }
- // Setup DNS
- err = setupDNS(c.config.DNS, etc)
- if err != nil {
- return fmt.Errorf("could not set the DNS: %w", err)
- }
- err = setupCerts(etc)
- if err != nil {
- return fmt.Errorf("could not set certificates: %w", err)
- }
- return nil
- })
- return err
-}
-
-func (c *Container) findNetNS() (ns.NetNS, error) {
- nsPath, err := c.GetNS(configs.NEWNET)
- if err != nil {
- return nil, err
- }
- netns, err := ns.GetNS(nsPath)
- if err != nil {
- return nil, err
- }
- return netns, nil
-}
-
-func (c *Container) setupIPVlan(master string, mtu int) (string, error) {
- tmpName := "vethtmp"
- netns, err := c.findNetNS()
- if err != nil {
- return "", err
- }
- masterIface, err := netlink.LinkByName(master)
- if err != nil {
- return "", err
- }
- ipvlan := &netlink.IPVlan{
- LinkAttrs: netlink.LinkAttrs{
- MTU: mtu,
- Name: tmpName,
- ParentIndex: masterIface.Attrs().Index,
- Namespace: netlink.NsFd(int(netns.Fd())),
- },
- Mode: netlink.IPVLAN_MODE_L2,
- }
- err = netlink.LinkAdd(ipvlan)
- if err != nil {
- return "", err
- }
- return tmpName, nil
-}
-
-func setupRA(iface netlink.Link) error {
- // Accept router advertisement, even when forwarding is
- // enabled, this is further specified by setupAddress
- // and setupGw
- _, err := sysctl.Sysctl(ACCEPT_RA, "2")
- return err
-}
-
-func setupAddress(iface netlink.Link, ip net.IPNet) error {
- // Accept router advertisement for addresses if required,
- // otherwise use provided IP
- accept_pinfo := "1"
- if len(ip.IP) > 0 {
- accept_pinfo = "0"
- addr := &netlink.Addr{
- IPNet: &ip,
- }
- err := netlink.AddrAdd(iface, addr)
- if err != nil {
- return err
- }
- }
- _, err := sysctl.Sysctl(ACCEPT_PINFO, accept_pinfo)
- return err
-}
-
-func setupGw(iface netlink.Link, gw net.IP) error {
- // Accept router advertisement for default routes if required,
- // otherwise use provided gateway
- accept_defrtr := "1"
- if len(gw) > 0 {
- // First add a link-local route to the gateway, so that
- // out-of-lan default routes are handled properly
- bits := 8 * len(gw)
- err := netlink.RouteAdd(&netlink.Route{
- LinkIndex: iface.Attrs().Index,
- Scope: netlink.SCOPE_LINK,
- Dst: &net.IPNet{
- IP: gw,
- Mask: net.CIDRMask(bits, bits),
- },
- })
- if err != nil {
- return err
- }
- err = netlink.RouteAdd(&netlink.Route{
- LinkIndex: iface.Attrs().Index,
- Scope: netlink.SCOPE_UNIVERSE,
- Dst: &net.IPNet{},
- Gw: gw,
- })
- if err != nil {
- return err
- }
- }
- _, err := sysctl.Sysctl(ACCEPT_DFTRTR, accept_defrtr)
- return err
-}
-
-func setupDNS(servers []net.IP, etc string) error {
- lines := make([]string, len(servers))
- for i, server := range servers {
- lines[i] = fmt.Sprintf("nameserver %s", server.String())
- }
- resolv := []byte(strings.Join(lines, "\n"))
- err := ioutil.WriteFile(path.Join(etc, "resolv.conf"), resolv, 0644)
- return err
-}
-
-func setupCerts(etc string) error {
- // This is bluntly copied from go x509 package, because it is not
- // exported unfortunately
- var certFiles = []string{
- "/etc/ssl/certs/ca-certificates.crt", // Debian/Ubuntu/Gentoo etc.
- "/etc/pki/tls/certs/ca-bundle.crt", // Fedora/RHEL 6
- "/etc/ssl/ca-bundle.pem", // OpenSUSE
- "/etc/pki/tls/cacert.pem", // OpenELEC
- "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", // CentOS/RHEL 7
- "/etc/ssl/cert.pem", // Alpine Linux
- }
- for _, file := range certFiles {
- data, err := os.ReadFile(file)
- if err != nil {
- continue
- }
- dest := path.Join(etc, file[4:])
- err = os.MkdirAll(path.Dir(dest), 0o755)
- if err != nil {
- return err
- }
- err = os.WriteFile(dest, data, 0o644)
- if err != nil {
- return err
- }
- return nil
- }
- return fmt.Errorf("no certificate available")
-}
diff --git a/pkg/selfcontain/utils.go b/pkg/selfcontain/utils.go
deleted file mode 100644
index b0c8d4d00cee7c40578305302dfad52c78311f9d..0000000000000000000000000000000000000000
--- a/pkg/selfcontain/utils.go
+++ /dev/null
@@ -1,76 +0,0 @@
-package selfcontain
-
-import (
- "os"
- "os/signal"
- "path"
-)
-
-type runnable func()
-
-const argRunFun = "selfcontain-run-fun"
-
-func RunFun(config *Config, setup runnable, run runnable) error {
- // Run the function if we are indeed inside the container
- for _, arg := range os.Args {
- if arg == argRunFun {
- config.Logger.Info("now running inside the container")
- err := Evacuate()
- if err != nil {
- config.Logger.Error(err, "could not evacuate")
- os.Exit(1)
- }
- config.Logger.Info("starting the main routine")
- run()
- return nil
- }
- }
- // Otherwise containerize ourselves
- setup()
- config.Logger.Info("setting up a new container")
- // Prepare etc dir for later setting up networking
- etc := path.Join(config.Data, "etc")
- config.Mounts["/etc"] = etc
- // Create the container itsel
- c, err := New(config)
- if err != nil {
- return err
- }
- defer c.Destroy()
- config.Logger.Info("starting the container init process")
- err = c.Start(append(os.Args, argRunFun))
- if err != nil {
- return err
- }
- config.Logger.Info("setting up container networking")
- err = c.SetupNetworking(etc)
- if err != nil {
- return err
- }
- // Make sure we are notified and that we destroy the
- // container upon being interrupted
- s := make(chan os.Signal, 1)
- signal.Notify(s, os.Interrupt)
- go func() {
- <-s
- config.Logger.Info("interrupt signal caught, tearing down")
- c.Destroy()
- }()
- config.Logger.Info("container is ready, handing over")
- return c.Run()
-}
-
-// Evacuate cgroups, which is required for many in-container use cases
-// Remaining in the root cgroup would prevent creating any domain sub-cgroup
-func Evacuate() error {
- // Libcontainer cgroup manager is not designed for evacuation and will
- // fail in such a case, so we are using cgroupfs directly, which is
- // explicitely available due to defaults, and simple since we are
- // the only running process at the moment
- err := os.Mkdir("/sys/fs/cgroup/selfcontain", 0o755)
- if err != nil && !os.IsExist(err) {
- return err
- }
- err = os.WriteFile("/sys/fs/cgroup/selfcontain/cgroup.procs", []byte("0"), 0o755)
- return err
-}
diff --git a/pkg/sml/delegate.go b/pkg/sml/delegate.go
deleted file mode 100644
index efccd18006a466943fa45282773c2d8cbee8afdb..0000000000000000000000000000000000000000
--- a/pkg/sml/delegate.go
+++ /dev/null
@@ -1,66 +0,0 @@
-package sml
-
-import (
- "github.com/hashicorp/memberlist"
-)
-
-// Cluster implements the memberlist.Delegate interface
-func (m *Memberlist[M, S, MP, SP]) NotifyConflict(node, other *memberlist.Node) {
-}
-
-// Cluter implements the memberlist.Delegate interface
-func (m *Memberlist[M, S, MP, SP]) NodeMeta(limit int) []byte {
- n, err := m.Meta.Encode()
- if err != nil {
- m.logger.Info("could not encode node metadata")
- return []byte{}
- }
- return n
-}
-
-// Clutser implements the memberlist.Delegate interface
-func (m *Memberlist[M, S, MP, SP]) NotifyMsg([]byte) {
-}
-
-// Cluster implements the memberlist.Delegate interface
-func (m *Memberlist[M, S, MP, SP]) GetBroadcasts(overhead, limit int) [][]byte {
- return nil
-}
-
-// Cluster implements the memberlist.Delegate interface
-func (m *Memberlist[M, S, MP, SP]) LocalState(join bool) []byte {
- s, err := m.State.Encode()
- if err != nil {
- m.logger.Info("could not encode local state")
- return []byte{}
- }
- return s
-}
-
-// Clutser implements the memberlist.Delegate interface
-func (m *Memberlist[M, S, MP, SP]) MergeRemoteState(buf []byte, join bool) {
- m.logger.Info("merging remote state")
- change, err := m.State.Merge(buf)
- if err != nil {
- m.logger.Error(err, "could not merge remote state")
- }
- if change {
- m.nodeChanges <- struct{}{}
- }
-}
-
-// Cluster implements the EventDelegate interface
-func (m *Memberlist[M, S, MP, SP]) NotifyJoin(n *memberlist.Node) {
- m.logger.Info("node joined", "name", n.Name)
- m.nodeChanges <- struct{}{}
-}
-
-// Node implements the EventDelegate interface
-func (m *Memberlist[M, S, MP, SP]) NotifyLeave(n *memberlist.Node) {
- m.nodeChanges <- struct{}{}
-}
-
-// Node implements the EventDelegate interface
-func (m *Memberlist[M, S, MP, SP]) NotifyUpdate(n *memberlist.Node) {
- m.nodeChanges <- struct{}{}
-}
diff --git a/pkg/sml/instrumentation.go b/pkg/sml/instrumentation.go
deleted file mode 100644
index 249595204f4647326be8d75b752bb49e320113dc..0000000000000000000000000000000000000000
--- a/pkg/sml/instrumentation.go
+++ /dev/null
@@ -1,9 +0,0 @@
-package sml
-
-type Instrumentation interface {
- // Instrumentation data was updated
- Updates() <-chan struct{}
- // Get the current minimum observed path MTU with any other node in the
- // cluster, useful for setting cluster-wide MTU
- MinMTU() int
-}
diff --git a/pkg/sml/memberlist.go b/pkg/sml/memberlist.go
deleted file mode 100644
index 553e0570894bb78df481d1d43f2085924381c028..0000000000000000000000000000000000000000
--- a/pkg/sml/memberlist.go
+++ /dev/null
@@ -1,177 +0,0 @@
-// Simple Memberlist is a wrapper around hashicorp memberlist, that
-// leverages the gossip protocol and exposes a simpler interface.
-// Main features include: automatic rejoining a list of cluster anchors,
-// providing a channel of node changes, encoding and decoding of node
-// metadata, plus caching of decoded metadata.
-package sml
-
-import (
- "net"
- "time"
-
- "github.com/go-logr/logr"
- "github.com/hashicorp/memberlist"
-)
-
-type Memberlist[M any, S any, MP MetaPointer[M], SP StatePointer[S]] struct {
- Meta MP
- State SP
- name string
- nodeName string
- anchors []string
- key []byte
- logger logr.Logger
- config *memberlist.Config
- ml *memberlist.Memberlist
- nodeCache []Node[M, MP]
- nodeChanges chan struct{}
- chans []chan struct{}
- transport *instrumentedTransport
-}
-
-type logWriter struct {
- logr.Logger
-}
-
-func (w logWriter) Write(b []byte) (int, error) {
- w.Info(string(b))
- return len(b), nil
-}
-
-func New[M any, S any, MP MetaPointer[M], SP StatePointer[S]](nodeName string, nodeIP net.IP, port int, anchors []string, key []byte, logger logr.Logger) *Memberlist[M, S, MP, SP] {
- config := memberlist.DefaultLANConfig()
- config.LogOutput = logWriter{logger}
- config.SecretKey = key
- config.BindAddr = nodeIP.String()
- config.BindPort = port
- config.AdvertiseAddr = nodeIP.String()
- config.AdvertisePort = port
- config.Name = nodeName
- config.SecretKey = key
-
- m := &Memberlist[M, S, MP, SP]{
- nodeName: nodeName,
- nodeChanges: make(chan struct{}, 100),
- anchors: anchors,
- key: key,
- logger: logger,
- config: config,
- Meta: new(M),
- State: new(S),
- }
-
- // Memberlist is its own memberlist delegate implementation
- m.config.Delegate = m
- m.config.Conflict = m
- m.config.Events = m
-
- return m
-}
-
-// Start the memberlist cluster by listening on main sockets
-func (m *Memberlist[M, S, MP, SP]) Start() error {
- m.logger.Info("starting the cluster transport")
- tc := &memberlist.NetTransportConfig{
- BindAddrs: []string{m.config.BindAddr},
- BindPort: m.config.BindPort,
- }
- transport, err := NewTransport(tc, m.logger)
- if err != nil {
- return err
- }
- m.transport = transport
- m.config.Transport = transport
- ml, err := memberlist.Create(m.config)
- if err != nil {
- return err
- }
- m.ml = ml
- go m.join()
- return nil
-}
-
-// Run the memberlist cluster main loop, that awaits cluster changes, maintains
-// the cluster state and propagates information to channels
-func (m *Memberlist[M, S, MP, SP]) Run() error {
- ticker := time.Tick(10 * time.Second)
- for {
- select {
- case <-ticker:
- go m.join()
- case <-m.nodeChanges:
- m.updateCache()
- m.logger.Info("network topology changed", "nodes", m.Nodes())
- for _, c := range m.chans {
- c <- struct{}{}
- }
- }
- }
-}
-
-// Get a channel for notifications of network changes
-func (m *Memberlist[M, S, MP, SP]) Events() <-chan struct{} {
- // Buffer to avoid blocking later
- // TODO: be more organized about channels we deliver
- c := make(chan struct{}, 100)
- m.chans = append(m.chans, c)
- return c
-}
-
-// Get the list of current cluster nodes
-func (m *Memberlist[M, S, MP, SP]) Nodes() []Node[M, MP] {
- return m.nodeCache
-}
-
-// Get the instrumentation interface
-func (m *Memberlist[M, S, MP, SP]) Instr() Instrumentation {
- return m.transport
-}
-
-// Update the current node
-func (m *Memberlist[M, S, MP, SP]) Update() {
- m.logger.Info("updating the memberlist cluster")
- m.ml.UpdateNode(1 * time.Second)
-}
-
-// Update the node cache after a network change, goes through all the
-// nodes and decodes metadata
-func (m *Memberlist[M, S, MP, SP]) updateCache() {
- members := m.ml.Members()
- m.logger.Info("updating the node cache", "count", len(members))
- var cache []Node[M, MP]
- for _, mlNode := range members {
- meta := new(M)
- pointer := MP(meta)
- err := pointer.Decode(mlNode.Meta)
- if err == nil {
- cache = append(cache, Node[M, MP]{mlNode, pointer})
- } else {
- m.logger.Info("could not decode meta", "node", mlNode.Name)
- }
- }
- m.nodeCache = cache
-}
-
-// Try and join any anchor that is not currently a cluster member
-func (m *Memberlist[M, S, MP, SP]) join() error {
- addrs := []string{}
- members := m.ml.Members()
- for _, candidate := range m.anchors {
- found := false
- for _, node := range members {
- if node.Address() == candidate {
- found = true
- break
- }
- }
- if found {
- continue
- }
- addrs = append(addrs, candidate)
- }
- if len(addrs) > 0 {
- m.logger.Info("joining cluster nodes", "addresses", addrs)
- }
- _, err := m.ml.Join(addrs)
- return err
-}
diff --git a/pkg/sml/node.go b/pkg/sml/node.go
deleted file mode 100644
index 49ddd9d036426fdec90e6f624d54a592e3429f29..0000000000000000000000000000000000000000
--- a/pkg/sml/node.go
+++ /dev/null
@@ -1,46 +0,0 @@
-package sml
-
-import (
- "github.com/hashicorp/memberlist"
-)
-
-// Cluster data can be encoded and decoded to bytes, required
-// by Memberlist wire protocol
-type ClusterData interface {
- Encode() ([]byte, error)
- Decode([]byte) error
- String() string
-}
-
-type NodeMeta interface {
- ClusterData
-}
-
-// Pointer type to node meta, for generics trickery
-// This is required because we create meta instances while the
-// interface specification requires pointer receivers for decoding
-type MetaPointer[M any] interface {
- NodeMeta
- *M
-}
-
-// Cluster state is node data that can be merged, crdt-style
-type ClusterState interface {
- ClusterData
- Merge([]byte) (bool, error)
-}
-
-// Pointer type to cluster state, for generics trickery
-// This is required because we create state instances while the
-// interface specification requires pointer receivers for decoding and
-// merging
-type StatePointer[S any] interface {
- ClusterState
- *S
-}
-
-// Represents a full node for easy browsing
-type Node[M any, MP MetaPointer[M]] struct {
- *memberlist.Node
- NodeMeta MP
-}
diff --git a/pkg/sml/transport.go b/pkg/sml/transport.go
deleted file mode 100644
index 4cd8dc475e171915390a67097ce9c8639485478f..0000000000000000000000000000000000000000
--- a/pkg/sml/transport.go
+++ /dev/null
@@ -1,89 +0,0 @@
-package sml
-
-import (
- "net"
- "time"
-
- "github.com/go-logr/logr"
- "github.com/hashicorp/memberlist"
- "golang.org/x/net/ipv6"
-)
-
-// Network transport for memberlist, instrumented with metrology
-// and automatic path MTU discovery
-type instrumentedTransport struct {
- memberlist.NetTransport
- // Map between node address and known path MTU
- pmtu map[string]int
- // Last evaluated minimum path MTU
- minPmtu int
- // Instrumentation updates
- updates chan struct{}
- // Logger interface
- logger logr.Logger
-}
-
-// Create a new instrumented transport
-func NewTransport(config *memberlist.NetTransportConfig, logger logr.Logger) (*instrumentedTransport, error) {
- nt, err := memberlist.NewNetTransport(config)
- if err != nil {
- return nil, err
- }
- return &instrumentedTransport{
- *nt,
- make(map[string]int),
- 1500,
- make(chan struct{}, 100),
- logger,
- }, nil
-}
-
-// See Transport.
-func (t *instrumentedTransport) DialAddressTimeout(a memberlist.Address, timeout time.Duration) (net.Conn, error) {
- addr := a.Addr
-
- t.logger.Info("memberlist dialing", "address", addr)
- dialer := net.Dialer{Timeout: timeout}
- conn, err := dialer.Dial("tcp", addr)
- if err == nil {
- wrapped := ipv6.NewConn(conn)
- mtu, err := wrapped.PathMTU()
- if err == nil {
- t.logger.Info("discovered MTU", "node", addr, "mtu", mtu)
- prev, _ := t.pmtu[addr]
- t.pmtu[addr] = mtu
- if prev != mtu {
- t.updateMinMTU()
- }
- } else {
- t.logger.Error(err, "could not discover MTU")
- }
- } else {
- t.logger.Error(err, "could not connect to remote")
- }
- return conn, err
-}
-
-// Instrumented transport implements instrumentation
-func (t *instrumentedTransport) Updates() <-chan struct{} {
- return t.updates
-}
-
-// Instrumented transport implements instrumentation
-func (t *instrumentedTransport) MinMTU() int {
- return t.minPmtu
-}
-
-// Update the minimum MTU and notify upstream if required
-func (t *instrumentedTransport) updateMinMTU() {
- min := 1500
- for _, mtu := range t.pmtu {
- if mtu < min {
- min = mtu
- }
- }
- if min != t.minPmtu {
- t.updates <- struct{}{}
- }
- t.minPmtu = min
-}