diff --git a/cmd/hepto/config.go b/cmd/hepto/config.go
index f04329df3d70f9331cbe6e7d51d41b43b3bfce3c..15cef956d1e97c72daf34b99fe9bc40c7749bd27 100644
--- a/cmd/hepto/config.go
+++ b/cmd/hepto/config.go
@@ -7,32 +7,21 @@ import (
"forge.tedomum.net/acides/hepto/hepto/pkg/cluster"
"forge.tedomum.net/acides/hepto/hepto/pkg/selfcontain"
- "forge.tedomum.net/acides/hepto/hepto/pkg/types"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)
type Config struct {
DataDir string
- Contained bool
-
- Cluster cluster.ClusterSettings
- Network selfcontain.NetworkSettings
- Node cluster.NodeSettings
+ Cluster cluster.ClusterSettings
+ Container selfcontain.Config
+ Node cluster.NodeSettings
}
// Default to Cloudflare DNS (2606:4700:4700::1111, 2606:4700:4700::1001)
-var defaultDNS = types.AddressSlice{
- {IP: net.IP{0x26, 0x06, 0x47, 0, 0x47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x11, 0x11}},
- {IP: net.IP{0x26, 0x06, 0x47, 0, 0x47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x10, 0x01}},
-}
-
-func (c Config) LibcontainerPath() string {
- return c.dataPath("containers")
-}
-
-func (c Config) RootPath() string {
- return c.dataPath("root")
+var defaultDNS = []net.IP{
+ {0x26, 0x06, 0x47, 0, 0x47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x11, 0x11},
+ {0x26, 0x06, 0x47, 0, 0x47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x10, 0x01},
}
func (c Config) dataPath(subPath string) string {
@@ -41,6 +30,12 @@ func (c Config) dataPath(subPath string) string {
return dir
}
+func (c *Config) Complete() {
+ c.Container.Root = c.dataPath("root")
+ c.Container.Data = c.dataPath("containers")
+ c.Container.Name = c.Node.Name
+}
+
var config Config
func init() {
@@ -48,17 +43,16 @@ func init() {
// General config
rootCmd.Flags().StringVar(&config.DataDir, "data", "/var/lib", "Data root directory")
- rootCmd.Flags().BoolVar(&config.Contained, "contained", false, "Internal flag for containerized process")
// Cluster settings
rootCmd.Flags().StringVar(&config.Cluster.Name, "cluster", "hepto", "Hepto cluster name")
rootCmd.Flags().BytesHexVar(&config.Cluster.Key, "key", []byte{}, "Main cluster 32bytes key, hex-encoded")
- // Network settings
- rootCmd.Flags().StringVar(&config.Network.Master, "iface", "eth0", "Master network interface")
- rootCmd.Flags().Var(&config.Network.IP, "ip", "IP address for the public interface")
- rootCmd.Flags().Var(&config.Network.GW, "gw", "IP address of the network gateway")
- rootCmd.Flags().Var(&config.Network.DNS, "dns", "DNS server IP addresses")
+ // Container settings
+ rootCmd.Flags().StringVar(&config.Container.Master, "iface", "eth0", "Master network interface")
+ rootCmd.Flags().IPNetVar(&config.Container.IP, "ip", net.IPNet{}, "IP address for the public interface")
+ rootCmd.Flags().IPVar(&config.Container.GW, "gw", net.IP{}, "IP address of the network gateway")
+ rootCmd.Flags().IPSliceVar(&config.Container.DNS, "dns", defaultDNS, "DNS server IP addresses")
// Node settings
rootCmd.Flags().IntVar(&config.Node.Port, "discovery-port", 7123, "TCP port used for discovering the cluster")
diff --git a/cmd/hepto/root.go b/cmd/hepto/root.go
index dfa43abc4dfb37ef4568da343b5d721bf458b309..8ce63433a95335a97cd9b0a018b75a19dbf18d98 100644
--- a/cmd/hepto/root.go
+++ b/cmd/hepto/root.go
@@ -4,7 +4,6 @@ import (
"fmt"
"net"
"os"
- "os/signal"
"time"
"forge.tedomum.net/acides/hepto/hepto/pkg/cluster"
@@ -20,60 +19,26 @@ var rootCmd = &cobra.Command{
deployments, including across links with noticeable latency.`,
Run: func(cmd *cobra.Command, args []string) {
logrus.SetLevel(logrus.DebugLevel)
- if config.Contained {
- run()
- } else {
- containerize()
+ config.Complete()
+ err := selfcontain.RunFun(&config.Container, func() {
+ config.Node.IP = waitForIP()
+ logrus.Debug("current IP is ", config.Node.IP.String())
+ c := cluster.New(&config.Cluster, &config.Node)
+ c.Run()
+ })
+ if err != nil {
+ logrus.Fatal(err)
}
},
}
-// Actually run hepto
-func run() {
- config.Node.IP = waitForIP()
- logrus.Debug("current IP is ", config.Node.IP.String())
- c := cluster.New(&config.Cluster, &config.Node)
- c.Run()
-}
-
-// Run ourselves inside a container
-func containerize() {
- args := append(os.Args[1:], "--contained")
- c, err := selfcontain.New(config.Node.Name, config.LibcontainerPath(), config.RootPath(), args)
- if err != nil {
- logrus.Fatal(err)
- }
- // Use default DNS if required
- if len(config.Network.DNS) == 0 {
- config.Network.DNS = defaultDNS
- }
- err = c.SetupNetworking(&config.Network, config.RootPath())
- if err != nil {
- c.Destroy()
- logrus.Fatal(err)
- }
- // Make sure we are notified and that we destroy the
- // container upon being interrupted
- s := make(chan os.Signal, 1)
- signal.Notify(s, os.Interrupt)
- go func() {
- <-s
- c.Destroy()
- }()
- err = c.Run()
- if err != nil {
- c.Destroy()
- logrus.Fatal(err)
- }
-}
-
// Guess the current IP address
func waitForIP() net.IP {
logrus.Info("determining current IP address...")
for {
time.Sleep(time.Second)
- logrus.Debug("sending a probe UDP packet to ", defaultDNS[0].IP.String())
- conn, err := net.Dial("udp", fmt.Sprintf("[%s]:53", defaultDNS[0].IP.String()))
+ logrus.Debug("sending a probe UDP packet to ", defaultDNS[0].String())
+ conn, err := net.Dial("udp", fmt.Sprintf("[%s]:53", defaultDNS[0].String()))
if err != nil {
logrus.Debug(err)
continue
diff --git a/pkg/selfcontain/config.go b/pkg/selfcontain/config.go
new file mode 100644
index 0000000000000000000000000000000000000000..212b0b1ecde9b5291da8666eb5e3ce55f5bdd89b
--- /dev/null
+++ b/pkg/selfcontain/config.go
@@ -0,0 +1,22 @@
+package selfcontain
+
+import (
+ "net"
+)
+
+type Config struct {
+ // Container name
+ Name string
+ // Path to on-file container data
+ Data string
+ // Path to container root filesystem
+ Root string
+ // Name of the master interface for IPvlan
+ Master string
+ // Public IP of the container, can be nulled for autoconfiguration
+ IP net.IPNet
+ // Default gateway for the container, can be nulled for autoconfiguration
+ GW net.IP
+ // List of DNS servers for the container
+ DNS []net.IP
+}
diff --git a/pkg/selfcontain/container.go b/pkg/selfcontain/container.go
index e4bf4e38340e744277502ed8aa2994c5011bc6c4..4f5e49d1acbe99371b5e6d50e374a501bb3e8e78 100644
--- a/pkg/selfcontain/container.go
+++ b/pkg/selfcontain/container.go
@@ -17,14 +17,14 @@ import (
)
type Container struct {
- Name string
- dataPath string
+ config *Config
+ self string
container libcontainer.Container
process libcontainer.Process
}
// Containerize the current process by runnig the current binary inside a container
-func New(name string, dataPath string, newRoot string, newArgs []string) (*Container, error) {
+func New(config *Config) (*Container, error) {
// Get the current binary path to initialize InitPath and the command, otherwise
// libcontainer uses /proc/self/exe which a. requires persistent access to /proc,
// which we might drop at some point, b. requires hacking reexec so that "exe" is
@@ -47,18 +47,26 @@ func New(name string, dataPath string, newRoot string, newArgs []string) (*Conta
f.InitArgs = []string{os.Args[0], argInit}
return nil
}
- factory, err := libcontainer.New(dataPath, factoryConfig)
+ factory, err := libcontainer.New(config.Data, factoryConfig)
if err != nil {
return nil, err
}
// Use the default container configuration and create the container and process
- config := makeConfig(name, newRoot, self)
- container, err := factory.Create(name, config)
+ defaults := makeDefaults(config.Name, config.Root, self)
+ container, err := factory.Create(config.Name, defaults)
if err != nil {
return nil, err
}
+ return &Container{
+ config: config,
+ self: self,
+ container: container,
+ }, nil
+}
+
+func (c *Container) Start(args []string) error {
process := libcontainer.Process{
- Args: append([]string{filepath.Join("/", filepath.Base(self))}, newArgs...),
+ Args: append([]string{filepath.Join("/", filepath.Base(c.self))}, args...),
Stdin: os.Stdin,
Stdout: os.Stdout,
Stderr: os.Stderr,
@@ -66,24 +74,20 @@ func New(name string, dataPath string, newRoot string, newArgs []string) (*Conta
}
// Simply start the process instead of running, at this point init will be waiting
// and listening on libcontainer control pipe
- err = container.Start(&process)
+ err := c.container.Start(&process)
if err != nil {
- container.Destroy()
- return nil, err
+ c.container.Destroy()
+ return err
}
pid, err := process.Pid()
if err != nil {
- container.Destroy()
- return nil, err
+ c.container.Destroy()
+ return err
}
logrus.Debug("running with pid ", os.Getpid())
logrus.Debug("namespaced with pid ", pid)
- return &Container{
- Name: name,
- dataPath: dataPath,
- container: container,
- process: process,
- }, nil
+ c.process = process
+ return nil
}
// Actually run the container and block until the process has returned
@@ -98,13 +102,13 @@ func (c *Container) Run() error {
}
func (c *Container) Destroy() error {
- logrus.Info("destroying container ", c.Name)
+ logrus.Info("destroying container ", c.config.Name)
err := c.container.Destroy()
if err != nil {
logrus.Error(err)
return err
}
- err = os.RemoveAll(filepath.Join(c.dataPath, c.Name))
+ err = os.RemoveAll(filepath.Join(c.config.Data, c.config.Name))
if err != nil {
logrus.Error(err)
return err
diff --git a/pkg/selfcontain/defaults.go b/pkg/selfcontain/defaults.go
index c81a299312ad42020f739e96cc84063e28ff44dc..2a3933e904c9cb4f0d21c0f2c1016f117f25fff2 100644
--- a/pkg/selfcontain/defaults.go
+++ b/pkg/selfcontain/defaults.go
@@ -10,7 +10,7 @@ import (
// This argument is passed back to the forked process to notify it should behave
// as a libcontainer init, which in turn is handled by init()
-const argInit = "hepto-arg-libcontainer"
+const argInit = "selfcontain-arg-libcontainer"
// Restrict access to the bare minimum for hepto to run properly
// See https://pkg.go.dev/github.com/opencontainers/runc@v1.0.2/libcontainer/specconv
@@ -103,14 +103,6 @@ var defaultMounts = []*configs.Mount{
Flags: unix.MS_NOSUID | unix.MS_STRICTATIME,
Data: "mode=755",
},
- // TODO: needs explainations
- {
- Source: "/run/systemd",
- Destination: "/run/systemd",
- Device: "bind",
- Flags: unix.MS_NOSUID | unix.MS_STRICTATIME,
- Data: "mode=755",
- },
}
// Restrict capabilities to strictly required capabilities
@@ -152,14 +144,14 @@ var readOnlyPath = []string{
}
// Make a container configuration using defaults
-func makeConfig(name string, newRoot string, self string) *configs.Config {
+func makeDefaults(name string, root string, self string) *configs.Config {
deviceRules := []*devices.Rule{}
for _, device := range allowedDevices {
deviceRules = append(deviceRules, &device.Rule)
}
// Finally returns the configuration
return &configs.Config{
- Rootfs: newRoot,
+ Rootfs: root,
RootPropagation: unix.MS_SHARED | unix.MS_REC,
Hostname: name,
Capabilities: &configs.Capabilities{
@@ -180,7 +172,7 @@ func makeConfig(name string, newRoot string, self string) *configs.Config {
Devices: allowedDevices,
Cgroups: &configs.Cgroup{
Name: name,
- Systemd: true,
+ Systemd: false,
Resources: &configs.Resources{
MemorySwappiness: nil,
Devices: deviceRules,
diff --git a/pkg/selfcontain/net.go b/pkg/selfcontain/net.go
index 956b417d5db2a6403dd9582177f62b137f4fe957..c6c260c90cb77d22a86a6f5d08ba52e4d75608b4 100644
--- a/pkg/selfcontain/net.go
+++ b/pkg/selfcontain/net.go
@@ -8,7 +8,6 @@ import (
"path"
"strings"
- "forge.tedomum.net/acides/hepto/hepto/pkg/types"
"github.com/containernetworking/plugins/pkg/ns"
"github.com/containernetworking/plugins/pkg/utils/sysctl"
"github.com/opencontainers/runc/libcontainer/configs"
@@ -18,22 +17,11 @@ import (
const ACCEPT_PINFO = "net.ipv6.conf.eth0.accept_ra_pinfo"
const ACCEPT_DFTRTR = "net.ipv6.conf.eth0.accept_ra_defrtr"
-type NetworkSettings struct {
- // Name of the master interface for IPvlan
- Master string
- // Public IP of the container, can be nulled for autoconfiguration
- IP types.Address
- // Default gateway for the container, can be nulled for autoconfiguration
- GW types.Address
- // List of DNS servers for the container
- DNS types.AddressSlice
-}
-
// Setup networking inside the container
// This must be called from outside the container, since it requires both access to the
// host networking stack and the namespace networking stack
-func (c *Container) SetupNetworking(settings *NetworkSettings, root string) error {
- ifaceName, err := c.setupIPVlan(settings.Master, 1500)
+func (c *Container) SetupNetworking() error {
+ ifaceName, err := c.setupIPVlan(c.config.Master, 1500)
if err != nil {
return err
}
@@ -56,20 +44,20 @@ func (c *Container) SetupNetworking(settings *NetworkSettings, root string) erro
return err
}
// Setup addresses and routes
- err = setupAddress(iface, settings.IP.IPNet())
+ err = setupAddress(iface, c.config.IP)
if err != nil {
return err
}
- err = setupGw(iface, settings.GW.IP)
+ err = setupGw(iface, c.config.GW)
if err != nil {
return err
}
// Setup DNS
- err = setupDNS(settings.DNS.IPSlice(), root)
+ err = setupDNS(c.config.DNS, c.config.Root)
if err != nil {
return err
}
- err = setupHosts(root)
+ err = setupHosts(c.config.Root)
if err != nil {
return err
}
@@ -116,14 +104,14 @@ func (c *Container) setupIPVlan(master string, mtu int) (string, error) {
return tmpName, nil
}
-func setupAddress(iface netlink.Link, ip *net.IPNet) error {
+func setupAddress(iface netlink.Link, ip net.IPNet) error {
// Accept router advertisement for addresses if required,
// otherwise use provided IP
accept_ra := "1"
if len(ip.IP) > 0 {
accept_ra = "0"
addr := &netlink.Addr{
- IPNet: ip,
+ IPNet: &ip,
}
err := netlink.AddrAdd(iface, addr)
if err != nil {
diff --git a/pkg/selfcontain/utils.go b/pkg/selfcontain/utils.go
new file mode 100644
index 0000000000000000000000000000000000000000..6a6eb149a7fd9780ebfa5072747683b392f26541
--- /dev/null
+++ b/pkg/selfcontain/utils.go
@@ -0,0 +1,46 @@
+package selfcontain
+
+import (
+ "os"
+ "os/signal"
+
+ "github.com/sirupsen/logrus"
+)
+
+type runnable func()
+
+const argRunFun = "selfcontain-run-fun"
+
+func RunFun(config *Config, f runnable) error {
+ // Run the function if we are indeed inside the container
+ for _, arg := range os.Args {
+ if arg == argRunFun {
+ logrus.Debug("we are running inside the container...")
+ f()
+ return nil
+ }
+ }
+ // Otherwise containerize ourselves
+ c, err := New(config)
+ if err != nil {
+ return err
+ }
+ defer c.Destroy()
+ err = c.Start(append(os.Args, argRunFun))
+ if err != nil {
+ return err
+ }
+ err = c.SetupNetworking()
+ if err != nil {
+ return err
+ }
+ // Make sure we are notified and that we destroy the
+ // container upon being interrupted
+ s := make(chan os.Signal, 1)
+ signal.Notify(s, os.Interrupt)
+ go func() {
+ <-s
+ c.Destroy()
+ }()
+ return c.Run()
+}