Add minio for s3 datastore in tests

test/templates/minio-config.sh

+#!/bin/ash
+set -x
+wget https://dl.min.io/client/mc/release/linux-amd64/mc
+chmod +x mc
+mv mc /usr/local/bin/mc
+mc alias set local {{.ConsoleAddress}} admin test1234
+mc admin user svcacct add local admin --access-key mykey --secret-key mysecret
+mc mb local/mybucket
+echo 'This line marks WaitFor as done'

test/templates/mmr.config.yaml

@@ -27,3 +27,14 @@ redis:
       addr: "{{.RedisAddr}}"
 accessTokens:
   maxCacheTimeSeconds: 43200
+datastores:
+  - type: s3
+    id: "s3_internal"
+    forKinds: [ "thumbnails", "remote_media", "local_media", "archives" ]
+    opts:
+      tempPath: "/tmp/mediarepo_s3_upload"
+      endpoint: "{{.S3Endpoint}}"
+      bucketName: "mybucket"
+      accessKeyId: "mykey"
+      accessSecret: "mysecret"
+      ssl: false
\ No newline at end of file

test/test_internals/deps.go

@@ -18,6 +18,7 @@ type ContainerDeps struct {
 	ctx            context.Context
 	pgContainer    *postgres.PostgresContainer
 	redisContainer testcontainers.Container
+	minioDep       *MinioDep
 	depNet         *NetworkDep
 
 	Homeservers []*SynapseDep
@@ -88,6 +89,12 @@ func MakeTestDeps() (*ContainerDeps, error) {
 		return nil, err
 	}
 
+	// Start a minio (s3) container
+	minioDep, err := MakeMinio(depNet)
+	if err != nil {
+		return nil, err
+	}
+
 	// Start two MMRs for testing
 	mmrs, err := makeMmrInstances(ctx, 2, depNet, mmrTmplArgs{
 		Homeservers: []mmrHomeserverTmplArgs{
@@ -102,6 +109,7 @@ func MakeTestDeps() (*ContainerDeps, error) {
 		},
 		RedisAddr:          fmt.Sprintf("%s:%d", redisHost, 6379), // we're behind the network for redis
 		PgConnectionString: pgConnStr,
+		S3Endpoint:         minioDep.Endpoint,
 	})
 	if err != nil {
 		return nil, err
@@ -111,6 +119,7 @@ func MakeTestDeps() (*ContainerDeps, error) {
 		ctx:            ctx,
 		pgContainer:    pgContainer,
 		redisContainer: redisContainer,
+		minioDep:       minioDep,
 		Homeservers:    []*SynapseDep{syn1, syn2},
 		Machines:       mmrs,
 		depNet:         depNet,
@@ -130,6 +139,7 @@ func (c *ContainerDeps) Teardown() {
 	if err := c.pgContainer.Terminate(c.ctx); err != nil {
 		log.Fatalf("Error shutting down mmr-postgres container: %s", err.Error())
 	}
+	c.minioDep.Teardown()
 	c.depNet.Teardown()
 }
 

test/test_internals/deps_minio.go

+package test_internals
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"path"
+	"strings"
+	"text/template"
+	"time"
+
+	"github.com/docker/go-connections/nat"
+	"github.com/testcontainers/testcontainers-go"
+	"github.com/testcontainers/testcontainers-go/wait"
+)
+
+type minioTmplArgs struct {
+	ConsoleAddress string
+}
+
+type MinioDep struct {
+	ctx       context.Context
+	container testcontainers.Container
+
+	Endpoint string
+}
+
+func MakeMinio(depNet *NetworkDep) (*MinioDep, error) {
+	ctx := context.Background()
+
+	// Start the minio container
+	consolePort, _ := nat.NewPort("tcp", "9090")
+	container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
+		ContainerRequest: testcontainers.ContainerRequest{
+			Image:        "quay.io/minio/minio:RELEASE.2023-08-09T23-30-22Z",
+			ExposedPorts: []string{"9000/tcp", "9090/tcp"},
+			Env: map[string]string{
+				"MINIO_ROOT_USER":     "admin",
+				"MINIO_ROOT_PASSWORD": "test1234",
+			},
+			WaitingFor: wait.ForHTTP("/login").WithPort(consolePort),
+			Networks:   []string{depNet.NetId},
+			Cmd:        []string{"server", "/data", "--console-address", ":9090"},
+			// we don't bind any volumes because we don't care if we lose the data
+		},
+		Started: true,
+	})
+	if err != nil {
+		return nil, err
+	}
+	if _, _, err = container.Exec(ctx, []string{"mkdir", "-p", "/data"}); err != nil {
+		return nil, err
+	}
+
+	// Find the minio connection details
+	minioIp, err := container.ContainerIP(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	// Prepare the test script
+	t, err := template.New("minio-config.sh").ParseFiles(path.Join(".", "test", "templates", "minio-config.sh"))
+	if err != nil {
+		return nil, err
+	}
+	w := new(strings.Builder)
+	//goland:noinspection HttpUrlsUsage
+	err = t.Execute(w, minioTmplArgs{
+		ConsoleAddress: fmt.Sprintf("http://%s:%d", minioIp, 9000), // we're behind the network
+	})
+	if err != nil {
+		return nil, err
+	}
+	f, err := os.CreateTemp(os.TempDir(), "mmr-minio")
+	if err != nil {
+		return nil, err
+	}
+	_, err = f.Write([]byte(strings.ReplaceAll(w.String(), "\r\n", "\n"))) // dos2unix now instead of in the container
+	if err != nil {
+		return nil, err
+	}
+	err = f.Close()
+	if err != nil {
+		return nil, err
+	}
+
+	// Use an intermediary container to set up the minio instance
+	_, err = testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
+		ContainerRequest: testcontainers.ContainerRequest{
+			Image:      "docker.io/alpine:3.18.3",
+			Networks:   []string{depNet.NetId},
+			WaitingFor: wait.ForLog("This line marks WaitFor as done").WithStartupTimeout(30 * time.Second),
+			Mounts: []testcontainers.ContainerMount{
+				testcontainers.BindMount(f.Name(), "/run.sh"),
+			},
+			Cmd: []string{"ash", "-c", "chmod +x /run.sh && /run.sh"},
+		},
+		Started: true,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return &MinioDep{
+		ctx:       ctx,
+		container: container,
+		Endpoint:  fmt.Sprintf("%s:%d", minioIp, 9000), // we're behind the network
+	}, nil
+}
+
+func (c *MinioDep) Teardown() {
+	if err := c.container.Terminate(c.ctx); err != nil {
+		log.Fatalf("Error shutting down minio: %s", err.Error())
+	}
+}

test/test_internals/deps_mmr.go

@@ -26,6 +26,7 @@ type mmrTmplArgs struct {
 	Homeservers        []mmrHomeserverTmplArgs
 	RedisAddr          string
 	PgConnectionString string
+	S3Endpoint         string
 }
 
 type mmrContainer struct {

test/upload_suite_test.go

@@ -40,7 +40,7 @@ func (s *UploadTestSuite) TestUpload() {
 	contentType, img, err := test_internals.MakeTestImage(512, 512)
 	res, err := client.Upload("image"+util.ExtensionForContentType(contentType), contentType, img)
 	assert.NoError(t, err)
-	log.Println(res.MxcUri)
+	log.Println(res.MxcUri) // TODO: Test download (from other instance) matches what we uploaded
 	assert.NotEmpty(t, res.MxcUri)
 }