Add structured logging support

CHANGELOG.md

@@ -7,6 +7,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## [Unreleased]
 
+### Added
+
+* Added support for structured logging (JSON).
+
 ### Changed
 
 * Turned color-coded logs off by default. This can be changed in the config.

cmd/export_synapse_for_import/main.go

@@ -58,7 +58,7 @@ func main() {
 		realPsqlPassword = *postgresPassword
 	}
 
-	err := logging.Setup(config.Get().General.LogDirectory)
+	err := logging.Setup(config.Get().General.LogDirectory, config.Get().General.LogColors, config.Get().General.JsonLogs)
 	if err != nil {
 		panic(err)
 	}

cmd/gdpr_export/main.go

@@ -45,7 +45,7 @@ func main() {
 	assets.SetupTemplates(*templatesPath)
 
 	var err error
-	err = logging.Setup(config.Get().General.LogDirectory)
+	err = logging.Setup(config.Get().General.LogDirectory, config.Get().General.LogColors, config.Get().General.JsonLogs)
 	if err != nil {
 		panic(err)
 	}

cmd/gdpr_import/main.go

@@ -35,7 +35,7 @@ func main() {
 	assets.SetupMigrations(*migrationsPath)
 
 	var err error
-	err = logging.Setup(config.Get().General.LogDirectory)
+	err = logging.Setup(config.Get().General.LogDirectory, config.Get().General.LogColors, config.Get().General.JsonLogs)
 	if err != nil {
 		panic(err)
 	}

cmd/import_synapse/main.go

@@ -72,7 +72,7 @@ func main() {
 		realPsqlPassword = *postgresPassword
 	}
 
-	err := logging.Setup(config.Get().General.LogDirectory)
+	err := logging.Setup(config.Get().General.LogDirectory, config.Get().General.LogColors, config.Get().General.JsonLogs)
 	if err != nil {
 		panic(err)
 	}

cmd/media_repo/main.go

@@ -59,7 +59,7 @@ func main() {
 	assets.SetupTemplates(*templatesPath)
 	assets.SetupAssets(*assetsPath)
 
-	err := logging.Setup(config.Get().General.LogDirectory, config.Get().General.LogColors)
+	err := logging.Setup(config.Get().General.LogDirectory, config.Get().General.LogColors, config.Get().General.JsonLogs)
 	if err != nil {
 		panic(err)
 	}

common/config/conf_main.go

@@ -25,6 +25,7 @@ func NewDefaultMainConfig() MainRepoConfig {
 			Port:             8000,
 			LogDirectory:     "logs",
 			LogColors:        false,
+			JsonLogs:         false,
 			TrustAnyForward:  false,
 			UseForwardedHost: true,
 		},

common/config/models_main.go

@@ -5,6 +5,7 @@ type GeneralConfig struct {
 	Port             int    `yaml:"port"`
 	LogDirectory     string `yaml:"logDirectory"`
 	LogColors        bool   `yaml:"logColors"`
+	JsonLogs         bool   `yaml:"jsonLogs"`
 	TrustAnyForward  bool   `yaml:"trustAnyForwardedAddress"`
 	UseForwardedHost bool   `yaml:"useForwardedHost"`
 }

common/logging/logger.go

@@ -19,17 +19,24 @@ func (f utcFormatter) Format(entry *logrus.Entry) ([]byte, error) {
 	return f.Formatter.Format(entry)
 }
 
-func Setup(dir string, colors bool) error {
-	formatter := &utcFormatter{
-		&logrus.TextFormatter{
+func Setup(dir string, colors bool, json bool) error {
+	var lineFormatter logrus.Formatter
+	if json {
+		lineFormatter = &logrus.JSONFormatter{
+			TimestampFormat:  "2006-01-02 15:04:05.000 Z07:00",
+			DisableTimestamp: false,
+		}
+	} else {
+		lineFormatter = &logrus.TextFormatter{
 			TimestampFormat:  "2006-01-02 15:04:05.000 Z07:00",
 			FullTimestamp:    true,
 			ForceColors:      colors,
 			DisableColors:    !colors,
 			DisableTimestamp: false,
 			QuoteEmptyFields: true,
-		},
+		}
 	}
+	formatter := &utcFormatter{lineFormatter}
 	logrus.SetFormatter(formatter)
 	logrus.SetOutput(os.Stdout)
 

config.sample.yaml

@@ -15,6 +15,10 @@ repo:
   # appear in logs which render them unreadable, which is why colors are disabled by default.
   logColors: false
 
+  # Set to true to enable JSON logging for consumption by things like logstash. Note that this is
+  # incompatible with the log color option and will always render without colors.
+  jsonLogs: false
+
   # If true, the media repo will accept any X-Forwarded-For header without validation. In most cases
   # this option should be left as "false". Note that the media repo already expects an X-Forwarded-For
   # header, but validates it to ensure the IP being given makes sense.