Skip to content
Snippets Groups Projects
Commit 33aa8cad authored by Travis Ralston's avatar Travis Ralston
Browse files

Remove IPFS

parent a6de9482
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 18 additions and 2294 deletions
CHANGELOG.md
+ 3
− 1
View file @ 33aa8cad
......@@ -7,7 +7,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
## [Unreleased]
*Nothing yet.*
### Removed
* IPFS support has been removed due to maintenance burden.
## [1.2.13] - February 12, 2023
......
api/features/feature_routes.go
+ 0
− 5
View file @ 33aa8cad
......@@ -4,11 +4,6 @@ import (
"net/http"
)
const IPFSDownloadRoute = "/_matrix/media/unstable/io.t2bot.ipfs/download/{server:[a-zA-Z0-9.:\\-_]+}/{ipfsContentId:[^/]+}"
const IPFSLiveDownloadRouteR0 = "/_matrix/media/r0/download/{server:[a-zA-Z0-9.:\\-_]+}/ipfs:{ipfsContentId:[^/]+}"
const IPFSLiveDownloadRouteV1 = "/_matrix/media/v1/download/{server:[a-zA-Z0-9.:\\-_]+}/ipfs:{ipfsContentId:[^/]+}"
const IPFSLiveDownloadRouteUnstable = "/_matrix/media/v1/download/{server:[a-zA-Z0-9.:\\-_]+}/ipfs:{ipfsContentId:[^/]+}"
func IsRoute(r *http.Request, route string) bool {
uri := r.URL.Path
return uri == route
......
api/unstable/ipfs_download.go deleted 100644 → 0
+ 0
− 49
View file @ a6de9482
package unstable
import (
"github.com/getsentry/sentry-go"
"net/http"
"github.com/gorilla/mux"
"github.com/sirupsen/logrus"
"github.com/turt2live/matrix-media-repo/api"
"github.com/turt2live/matrix-media-repo/api/r0"
"github.com/turt2live/matrix-media-repo/common/rcontext"
"github.com/turt2live/matrix-media-repo/ipfs_proxy"
)
func IPFSDownload(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
params := mux.Vars(r)
server := params["server"]
ipfsContentId := params["ipfsContentId"]
targetDisposition := r.URL.Query().Get("org.matrix.msc2702.asAttachment")
if targetDisposition == "true" {
targetDisposition = "attachment"
} else if targetDisposition == "false" {
targetDisposition = "inline"
} else {
targetDisposition = "infer"
}
rctx = rctx.LogWithFields(logrus.Fields{
"ipfsContentId": ipfsContentId,
"server": server,
})
obj, err := ipfs_proxy.GetObject(ipfsContentId, rctx)
if err != nil {
rctx.Log.Error(err)
sentry.CaptureException(err)
return api.InternalServerError("unexpected error")
}
return &r0.DownloadMediaResponse{
ContentType: obj.ContentType,
Filename: obj.FileName,
SizeBytes: obj.SizeBytes,
Data: obj.Data,
TargetDisposition: targetDisposition,
}
}
api/webserver/webserver.go
+ 3
− 11
View file @ 33aa8cad
......@@ -3,8 +3,6 @@ package webserver
import (
"context"
"encoding/json"
"github.com/getsentry/sentry-go"
sentryhttp "github.com/getsentry/sentry-go/http"
"net"
"net/http"
"os"
......@@ -13,12 +11,14 @@ import (
"sync"
"time"
"github.com/getsentry/sentry-go"
sentryhttp "github.com/getsentry/sentry-go/http"
"github.com/didip/tollbooth"
"github.com/gorilla/mux"
"github.com/sirupsen/logrus"
"github.com/turt2live/matrix-media-repo/api"
"github.com/turt2live/matrix-media-repo/api/custom"
"github.com/turt2live/matrix-media-repo/api/features"
"github.com/turt2live/matrix-media-repo/api/r0"
"github.com/turt2live/matrix-media-repo/api/unstable"
"github.com/turt2live/matrix-media-repo/api/webserver/debug"
......@@ -85,7 +85,6 @@ func Init() *sync.WaitGroup {
appendToImportHandler := handler{api.RepoAdminRoute(custom.AppendToImport), "append_to_import", counter, false}
stopImportHandler := handler{api.RepoAdminRoute(custom.StopImport), "stop_import", counter, false}
versionHandler := handler{api.AccessTokenOptionalRoute(custom.GetVersion), "get_version", counter, false}
ipfsDownloadHandler := handler{api.AccessTokenOptionalRoute(unstable.IPFSDownload), "ipfs_download", counter, false}
logoutHandler := handler{api.AccessTokenRequiredRoute(r0.Logout), "logout", counter, false}
logoutAllHandler := handler{api.AccessTokenRequiredRoute(r0.LogoutAll), "logout_all", counter, false}
getMediaAttrsHandler := handler{api.AccessTokenRequiredRoute(custom.GetAttributes), "get_media_attributes", counter, false}
......@@ -160,13 +159,6 @@ func Init() *sync.WaitGroup {
}
}
if config.Get().Features.IPFS.Enabled {
routes = append(routes, definedRoute{features.IPFSDownloadRoute, route{"GET", ipfsDownloadHandler}})
routes = append(routes, definedRoute{features.IPFSLiveDownloadRouteR0, route{"GET", ipfsDownloadHandler}})
routes = append(routes, definedRoute{features.IPFSLiveDownloadRouteV1, route{"GET", ipfsDownloadHandler}})
routes = append(routes, definedRoute{features.IPFSLiveDownloadRouteUnstable, route{"GET", ipfsDownloadHandler}})
}
for _, def := range routes {
logrus.Info("Registering route: " + def.route.method + " " + def.path)
rtr.Handle(def.path, def.route.handler).Methods(def.route.method)
......
cmd/media_repo/reloads.go
+ 0
− 17
View file @ 33aa8cad
......@@ -6,7 +6,6 @@ import (
"github.com/turt2live/matrix-media-repo/common/globals"
"github.com/turt2live/matrix-media-repo/common/runtime"
"github.com/turt2live/matrix-media-repo/internal_cache"
"github.com/turt2live/matrix-media-repo/ipfs_proxy"
"github.com/turt2live/matrix-media-repo/metrics"
"github.com/turt2live/matrix-media-repo/plugins"
"github.com/turt2live/matrix-media-repo/storage"
......@@ -19,7 +18,6 @@ func setupReloads() {
reloadDatabaseOnChan(globals.DatabaseReloadChan)
reloadDatastoresOnChan(globals.DatastoresReloadChan)
reloadRecurringTasksOnChan(globals.RecurringTasksReloadChan)
reloadIpfsOnChan(globals.IPFSReloadChan)
reloadAccessTokensOnChan(globals.AccessTokenReloadChan)
reloadCacheOnChan(globals.CacheReplaceChan)
reloadPluginsOnChan(globals.PluginReloadChan)
......@@ -33,7 +31,6 @@ func stopReloads() {
globals.DatastoresReloadChan <- false
globals.AccessTokenReloadChan <- false
globals.RecurringTasksReloadChan <- false
globals.IPFSReloadChan <- false
globals.CacheReplaceChan <- false
globals.PluginReloadChan <- false
}
......@@ -111,20 +108,6 @@ func reloadRecurringTasksOnChan(reloadChan chan bool) {
}()
}
func reloadIpfsOnChan(reloadChan chan bool) {
go func() {
defer close(reloadChan)
for {
shouldReload := <-reloadChan
if shouldReload {
ipfs_proxy.Reload()
} else {
ipfs_proxy.Stop()
}
}
}()
}
func reloadAccessTokensOnChan(reloadChan chan bool) {
go func() {
defer close(reloadChan)
......
common/config/conf_min_shared.go
+ 0
− 7
View file @ 33aa8cad
......@@ -53,13 +53,6 @@ func NewDefaultMinimumRepoConfig() MinimumRepoConfig {
YComponents: 3,
Punch: 1,
},
IPFS: IPFSConfig{
Enabled: false,
Daemon: IPFSDaemonConfig{
Enabled: true,
RepoPath: "./ipfs",
},
},
},
AccessTokens: AccessTokenConfig{
MaxCacheTimeSeconds: 0,
......
common/config/models_domain.go
+ 0
− 11
View file @ 33aa8cad
......@@ -88,7 +88,6 @@ type TimeoutsConfig struct {
type FeatureConfig struct {
MSC2448Blurhash MSC2448Config `yaml:"MSC2448"`
IPFS IPFSConfig `yaml:"IPFS"`
Redis RedisConfig `yaml:"redis"`
}
......@@ -103,16 +102,6 @@ type MSC2448Config struct {
Punch int `yaml:"punch"`
}
type IPFSConfig struct {
Enabled bool `yaml:"enabled"`
Daemon IPFSDaemonConfig `yaml:"builtInDaemon"`
}
type IPFSDaemonConfig struct {
Enabled bool `yaml:"enabled"`
RepoPath string `yaml:"repoPath"`
}
type AccessTokenConfig struct {
MaxCacheTimeSeconds int `yaml:"maxCacheTimeSeconds"`
UseAppservices bool `yaml:"useLocalAppserviceConfig"`
......
common/config/watch.go
+ 2
− 11
View file @ 33aa8cad
package config
import (
"github.com/getsentry/sentry-go"
"time"
"github.com/getsentry/sentry-go"
"github.com/bep/debounce"
"github.com/fsnotify/fsnotify"
"github.com/sirupsen/logrus"
......@@ -92,13 +93,6 @@ func onFileChanged() {
logrus.Warn("Log configuration changed - restart the media repo to apply changes")
}
ipfsDaemonChange := configNew.Features.IPFS.Daemon.Enabled != configNow.Features.IPFS.Daemon.Enabled
ipfsDaemonPathChange := configNew.Features.IPFS.Daemon.RepoPath != configNow.Features.IPFS.Daemon.RepoPath
if ipfsDaemonChange || ipfsDaemonPathChange {
logrus.Warn("IPFS Daemon options changed - reloading")
globals.IPFSReloadChan <- true
}
redisEnabledChange := configNew.Features.Redis.Enabled != configNow.Features.Redis.Enabled
redisShardsChange := hasRedisShardConfigChanged(configNew, configNow)
if redisEnabledChange || redisShardsChange {
......@@ -125,9 +119,6 @@ func hasWebFeatureChanged(configNew *MainRepoConfig, configNow *MainRepoConfig)
if configNew.Features.MSC2448Blurhash.Enabled != configNow.Features.MSC2448Blurhash.Enabled {
return true
}
if configNew.Features.IPFS.Enabled != configNow.Features.IPFS.Enabled {
return true
}
return false
}
......
common/globals/reload.go
+ 0
− 1
View file @ 33aa8cad
......@@ -5,7 +5,6 @@ var MetricsReloadChan = make(chan bool)
var DatabaseReloadChan = make(chan bool)
var DatastoresReloadChan = make(chan bool)
var RecurringTasksReloadChan = make(chan bool)
var IPFSReloadChan = make(chan bool)
var AccessTokenReloadChan = make(chan bool)
var CacheReplaceChan = make(chan bool)
var PluginReloadChan = make(chan bool)
common/runtime/init.go
+ 1
− 4
View file @ 33aa8cad
......@@ -2,13 +2,13 @@ package runtime
import (
"fmt"
"github.com/getsentry/sentry-go"
"github.com/sirupsen/logrus"
"github.com/turt2live/matrix-media-repo/common/config"
"github.com/turt2live/matrix-media-repo/common/rcontext"
"github.com/turt2live/matrix-media-repo/common/version"
"github.com/turt2live/matrix-media-repo/ipfs_proxy"
"github.com/turt2live/matrix-media-repo/plugins"
"github.com/turt2live/matrix-media-repo/storage"
"github.com/turt2live/matrix-media-repo/storage/datastore"
......@@ -22,9 +22,6 @@ func RunStartupSequence() {
LoadDatabase()
LoadDatastores()
plugins.ReloadPlugins()
logrus.Info("Starting IPFS (if enabled)...")
ipfs_proxy.Reload()
}
func LoadDatabase() {
......
config.sample.yaml
+ 0
− 32
View file @ 33aa8cad
......@@ -178,20 +178,6 @@ datastores:
# See https://aws.amazon.com/s3/storage-classes/ for details; uncomment to use.
#storageClass: STANDARD
# The media repo does support an IPFS datastore, but only if the IPFS feature is enabled. If
# the feature is not enabled, this will not work. Note that IPFS support is experimental at
# the moment and not recommended for general use.
#
# NOTE: Everything you upload to IPFS will be publicly accessible, even when the media repo
# puts authentication on the download endpoints. Only use this option for cases where you
# expect your media to be publicly accessible.
- type: ipfs
enabled: false # Enable this to use IPFS support
forKinds: ["local_media"]
# The IPFS datastore currently has no options. It will use the daemon or HTTP API configured
# in the IPFS section of your main config.
opts: {}
# Options for controlling archives. Archives are exports of a particular user's content for
# the purpose of GDPR or moving media to a different server.
archiving:
......@@ -544,24 +530,6 @@ featureSupport:
# make the effect more subtle, larger values make it stronger.
punch: 1
# IPFS Support
# This is currently experimental and might not work at all.
IPFS:
# Whether or not IPFS support is enabled for use in the media repo.
enabled: false
# Options for the built in IPFS daemon
builtInDaemon:
# Enable this to spawn an in-process IPFS node to use instead of a localhost
# HTTP agent. If this is disabled, the media repo will assume you have an HTTP
# IPFS agent running and accessible. Defaults to using a daemon (true).
enabled: true
# If the Daemon is enabled, set this to the location where the IPFS files should
# be stored. If you're using Docker, this should be something like "/data/ipfs"
# so it can be mapped to a volume.
repoPath: "./ipfs"
# Support for redis as a cache mechanism
#
# Note: Enabling Redis support will mean that the existing cache mechanism will do nothing.
......
controllers/upload_controller/upload_controller.go
+ 3
− 21
View file @ 33aa8cad
package upload_controller
import (
"fmt"
"github.com/getsentry/sentry-go"
"io"
"io/ioutil"
"strconv"
"time"
"github.com/getsentry/sentry-go"
"github.com/patrickmn/go-cache"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
......@@ -142,25 +142,7 @@ func UploadMedia(contents io.ReadCloser, contentLength int64, contentType string
_ = recentMediaIds.Add(mediaId, true, cache.DefaultExpiration)
var existingFile *AlreadyUploadedFile = nil
ds, err := datastore.PickDatastore(common.KindLocalMedia, ctx)
if err != nil {
return nil, err
}
if ds.Type == "ipfs" {
// Do the upload now so we can pick the media ID to point to IPFS
info, err := ds.UploadFile(util_byte_seeker.NewByteSeeker(dataBytes), contentLength, ctx)
if err != nil {
return nil, err
}
existingFile = &AlreadyUploadedFile{
DS: ds,
ObjectInfo: info,
}
mediaId = fmt.Sprintf("ipfs:%s", info.Location[len("ipfs/"):])
}
m, err := StoreDirect(existingFile, util_byte_seeker.NewByteSeeker(dataBytes), contentLength, contentType, filename, userId, origin, mediaId, common.KindLocalMedia, ctx, true)
m, err := StoreDirect(nil, util_byte_seeker.NewByteSeeker(dataBytes), contentLength, contentType, filename, userId, origin, mediaId, common.KindLocalMedia, ctx, true)
if err != nil {
return m, err
}
......
docs/ipfs.md deleted 100644 → 0
+ 0
− 12
View file @ a6de9482
# IPFS Support
TODO: An actual instructional document.
## Bugs/TODO
* [ ] Copies nearly everything into memory instead of streaming
* [ ] Downloads don't work because the nodes can be split sometimes (multiple links, RawData off the parent doesn't work)
* [ ] Delete support (see TODO)
* [ ] Overwrite support (if possible, might need to add support for changing media locations)
* [ ] General stability testing
* [ ] Use file names from database, not IPFS (because IPFS doesn't have any)
\ No newline at end of file
go.mod
+ 6
− 195
View file @ 33aa8cad
......@@ -28,11 +28,6 @@ require (
github.com/gorilla/mux v1.8.0
github.com/hashicorp/go-hclog v1.2.2
github.com/hashicorp/go-plugin v1.4.4
github.com/ipfs/go-cid v0.2.0
github.com/ipfs/go-ipfs-api v0.3.0
github.com/ipfs/go-ipfs-files v0.1.1
github.com/ipfs/interface-go-ipfs-core v0.7.0
github.com/ipfs/kubo v0.14.0
github.com/k3a/html2text v1.0.8
github.com/kettek/apng v0.0.0-20220622131934-4e92eded13be
github.com/lestrrat/go-file-rotatelogs v0.0.0-20180223000712-d3151e2a480f
......@@ -54,227 +49,44 @@ require (
)
require (
bazil.org/fuse v0.0.0-20200524192727-fb710f7dfd05 // indirect
github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect
github.com/Jeffail/gabs v1.4.0 // indirect
github.com/Stebalien/go-bitfield v1.0.0 // indirect
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 // indirect
github.com/andybalholm/cascadia v1.3.1 // indirect
github.com/benbjohnson/clock v1.3.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/blang/semver/v4 v4.0.0 // indirect
github.com/btcsuite/btcd v0.22.1 // indirect
github.com/btcsuite/btcd/btcec/v2 v2.2.1 // indirect
github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 // indirect
github.com/cenkalti/backoff v2.2.1+incompatible // indirect
github.com/cenkalti/backoff/v4 v4.1.3 // indirect
github.com/ceramicnetwork/go-dag-jose v0.1.0 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/cheekybits/genny v1.0.0 // indirect
github.com/containerd/cgroups v1.0.4 // indirect
github.com/coreos/go-systemd/v22 v22.3.2 // indirect
github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect
github.com/cskr/pubsub v1.0.2 // indirect
github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
github.com/dgraph-io/badger v1.6.2 // indirect
github.com/dgraph-io/ristretto v0.1.0 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/dsoprea/go-utility/v2 v2.0.0-20200717064901-2fccff4aa15e // indirect
github.com/elastic/gosigar v0.14.2 // indirect
github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 // indirect
github.com/fatih/color v1.13.0 // indirect
github.com/flynn/noise v1.0.0 // indirect
github.com/francoispqt/gojay v1.2.13 // indirect
github.com/go-errors/errors v1.4.2 // indirect
github.com/go-logr/logr v1.2.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
github.com/godbus/dbus/v5 v5.1.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect
github.com/golang/glog v1.0.0 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/google/gopacket v1.1.19 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/gorilla/websocket v1.5.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.2 // indirect
github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c // indirect
github.com/hajimehoshi/go-mp3 v0.3.3 // indirect
github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/huin/goupnp v1.0.3 // indirect
github.com/icza/bitio v1.1.0 // indirect
github.com/ipfs/bbloom v0.0.4 // indirect
github.com/ipfs/go-bitfield v1.0.0 // indirect
github.com/ipfs/go-bitswap v0.9.0 // indirect
github.com/ipfs/go-block-format v0.0.3 // indirect
github.com/ipfs/go-blockservice v0.4.0 // indirect
github.com/ipfs/go-cidutil v0.1.0 // indirect
github.com/ipfs/go-datastore v0.5.1 // indirect
github.com/ipfs/go-delegated-routing v0.3.0 // indirect
github.com/ipfs/go-ds-badger v0.3.0 // indirect
github.com/ipfs/go-ds-flatfs v0.5.1 // indirect
github.com/ipfs/go-ds-leveldb v0.5.0 // indirect
github.com/ipfs/go-ds-measure v0.2.0 // indirect
github.com/ipfs/go-fetcher v1.6.1 // indirect
github.com/ipfs/go-filestore v1.2.0 // indirect
github.com/ipfs/go-fs-lock v0.0.7 // indirect
github.com/ipfs/go-graphsync v0.13.2 // indirect
github.com/ipfs/go-ipfs-blockstore v1.2.0 // indirect
github.com/ipfs/go-ipfs-chunker v0.0.5 // indirect
github.com/ipfs/go-ipfs-delay v0.0.1 // indirect
github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect
github.com/ipfs/go-ipfs-exchange-interface v0.2.0 // indirect
github.com/ipfs/go-ipfs-exchange-offline v0.3.0 // indirect
github.com/ipfs/go-ipfs-keystore v0.0.2 // indirect
github.com/ipfs/go-ipfs-pinner v0.2.1 // indirect
github.com/ipfs/go-ipfs-posinfo v0.0.1 // indirect
github.com/ipfs/go-ipfs-pq v0.0.2 // indirect
github.com/ipfs/go-ipfs-provider v0.7.1 // indirect
github.com/ipfs/go-ipfs-routing v0.2.1 // indirect
github.com/ipfs/go-ipfs-util v0.0.2 // indirect
github.com/ipfs/go-ipld-cbor v0.0.6 // indirect
github.com/ipfs/go-ipld-format v0.4.0 // indirect
github.com/ipfs/go-ipld-git v0.1.1 // indirect
github.com/ipfs/go-ipld-legacy v0.1.1 // indirect
github.com/ipfs/go-ipns v0.1.2 // indirect
github.com/ipfs/go-log v1.0.5 // indirect
github.com/ipfs/go-log/v2 v2.5.1 // indirect
github.com/ipfs/go-merkledag v0.6.0 // indirect
github.com/ipfs/go-metrics-interface v0.0.1 // indirect
github.com/ipfs/go-mfs v0.2.1 // indirect
github.com/ipfs/go-namesys v0.5.0 // indirect
github.com/ipfs/go-path v0.3.0 // indirect
github.com/ipfs/go-peertaskqueue v0.7.1 // indirect
github.com/ipfs/go-unixfs v0.4.0 // indirect
github.com/ipfs/go-unixfsnode v1.4.0 // indirect
github.com/ipfs/go-verifcid v0.0.2 // indirect
github.com/ipld/edelweiss v0.1.5 // indirect
github.com/ipld/go-codec-dagpb v1.4.2 // indirect
github.com/ipld/go-ipld-prime v0.18.0 // indirect
github.com/jackpal/go-nat-pmp v1.0.2 // indirect
github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
github.com/jbenet/goprocess v0.1.4 // indirect
github.com/jfreymuth/oggvorbis v1.0.3 // indirect
github.com/jfreymuth/vorbis v1.0.2 // indirect
github.com/jonboulle/clockwork v0.1.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.15.9 // indirect
github.com/klauspost/cpuid/v2 v2.1.0 // indirect
github.com/koron/go-ssdp v0.0.3 // indirect
github.com/libp2p/go-buffer-pool v0.1.0 // indirect
github.com/libp2p/go-cidranger v1.1.0 // indirect
github.com/libp2p/go-doh-resolver v0.4.0 // indirect
github.com/libp2p/go-eventbus v0.2.1 // indirect
github.com/libp2p/go-flow-metrics v0.1.0 // indirect
github.com/libp2p/go-libp2p v0.20.3 // indirect
github.com/libp2p/go-libp2p-asn-util v0.2.0 // indirect
github.com/libp2p/go-libp2p-core v0.16.1 // indirect
github.com/libp2p/go-libp2p-discovery v0.7.0 // indirect
github.com/libp2p/go-libp2p-kad-dht v0.16.0 // indirect
github.com/libp2p/go-libp2p-kbucket v0.4.7 // indirect
github.com/libp2p/go-libp2p-loggables v0.1.0 // indirect
github.com/libp2p/go-libp2p-peerstore v0.7.1 // indirect
github.com/libp2p/go-libp2p-pubsub v0.7.1 // indirect
github.com/libp2p/go-libp2p-pubsub-router v0.5.0 // indirect
github.com/libp2p/go-libp2p-record v0.1.3 // indirect
github.com/libp2p/go-libp2p-resource-manager v0.3.0 // indirect
github.com/libp2p/go-libp2p-routing-helpers v0.2.3 // indirect
github.com/libp2p/go-libp2p-swarm v0.11.0 // indirect
github.com/libp2p/go-libp2p-xor v0.1.0 // indirect
github.com/libp2p/go-mplex v0.7.0 // indirect
github.com/libp2p/go-msgio v0.2.0 // indirect
github.com/libp2p/go-nat v0.1.0 // indirect
github.com/libp2p/go-netroute v0.2.0 // indirect
github.com/libp2p/go-openssl v0.0.7 // indirect
github.com/libp2p/go-reuseport v0.2.0 // indirect
github.com/libp2p/go-yamux/v3 v3.1.2 // indirect
github.com/libp2p/zeroconf/v2 v2.1.1 // indirect
github.com/lucas-clemente/quic-go v0.27.1 // indirect
github.com/marten-seemann/qtls-go1-16 v0.1.5 // indirect
github.com/marten-seemann/qtls-go1-17 v0.1.2 // indirect
github.com/marten-seemann/qtls-go1-18 v0.1.2 // indirect
github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect
github.com/kr/pretty v0.3.0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.16 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/mewkiz/flac v1.0.7 // indirect
github.com/mewkiz/pkg v0.0.0-20211102230744-16a6ce8f1b77 // indirect
github.com/miekg/dns v1.1.50 // indirect
github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect
github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/mr-tron/base58 v1.2.0 // indirect
github.com/multiformats/go-base32 v0.0.4 // indirect
github.com/multiformats/go-base36 v0.1.0 // indirect
github.com/multiformats/go-multiaddr v0.6.0 // indirect
github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect
github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect
github.com/multiformats/go-multibase v0.1.1 // indirect
github.com/multiformats/go-multicodec v0.5.0 // indirect
github.com/multiformats/go-multihash v0.2.1 // indirect
github.com/multiformats/go-multistream v0.3.3 // indirect
github.com/multiformats/go-varint v0.0.6 // indirect
github.com/nxadm/tail v1.4.8 // indirect
github.com/onsi/ginkgo v1.16.5 // indirect
github.com/opencontainers/runtime-spec v1.0.2 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/openzipkin/zipkin-go v0.4.0 // indirect
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.37.0 // indirect
github.com/prometheus/procfs v0.8.0 // indirect
github.com/raulk/go-watchdog v1.3.0 // indirect
github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
github.com/tidwall/gjson v1.14.3 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.0 // indirect
github.com/wI2L/jsondiff v0.2.0 // indirect
github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect
github.com/whyrusleeping/cbor-gen v0.0.0-20220514204315-f29c37e9c44c // indirect
github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect
github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect
github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 // indirect
github.com/whyrusleeping/tar-utils v0.0.0-20201201191210-20a61371de5b // indirect
github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee // indirect
go.opencensus.io v0.23.0 // indirect
go.opentelemetry.io/otel v1.9.0 // indirect
go.opentelemetry.io/otel/exporters/jaeger v1.9.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.9.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.9.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.9.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.9.0 // indirect
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.9.0 // indirect
go.opentelemetry.io/otel/exporters/zipkin v1.9.0 // indirect
go.opentelemetry.io/otel/sdk v1.9.0 // indirect
go.opentelemetry.io/otel/trace v1.9.0 // indirect
go.opentelemetry.io/proto/otlp v0.19.0 // indirect
go.uber.org/atomic v1.10.0 // indirect
go.uber.org/dig v1.15.0 // indirect
go.uber.org/fx v1.18.1 // indirect
go.uber.org/multierr v1.8.0 // indirect
go.uber.org/zap v1.22.0 // indirect
go4.org v0.0.0-20201209231011-d4a079459e60 // indirect
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd // indirect
github.com/smartystreets/assertions v1.0.0 // indirect
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab // indirect
golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/tools v0.1.12 // indirect
golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
google.golang.org/protobuf v1.28.1 // indirect
gopkg.in/square/go-jose.v2 v2.6.0 // indirect
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
lukechampine.com/blake3 v1.1.7 // indirect
)
require (
......@@ -285,7 +97,6 @@ require (
github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect
github.com/fastly/go-utils v0.0.0-20180712184237-d95a45783239 // indirect
github.com/go-sql-driver/mysql v1.5.0 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/hashicorp/yamux v0.1.1 // indirect
github.com/jehiah/go-strftime v0.0.0-20171201141054-1d33003b3869 // indirect
github.com/lestrrat/go-envload v0.0.0-20180220120943-6ed08b54a570 // indirect
......
go.sum
+ 0
− 1594
View file @ 33aa8cad
Source diff could not be displayed: it is too large. Options to address this: view the blob.
ipfs_proxy/api.go deleted 100644 → 0
+ 0
− 69
View file @ a6de9482
package ipfs_proxy
import (
"github.com/getsentry/sentry-go"
"io"
"github.com/sirupsen/logrus"
"github.com/turt2live/matrix-media-repo/common/config"
"github.com/turt2live/matrix-media-repo/common/rcontext"
"github.com/turt2live/matrix-media-repo/ipfs_proxy/ipfs_embedded"
"github.com/turt2live/matrix-media-repo/ipfs_proxy/ipfs_local"
"github.com/turt2live/matrix-media-repo/ipfs_proxy/ipfs_models"
)
var implementation IPFSImplementation
func Reload() {
if implementation != nil {
implementation.Stop()
}
implementation = nil
if !config.Get().Features.IPFS.Enabled {
return
}
if config.Get().Features.IPFS.Daemon.Enabled {
logrus.Info("Starting up local IPFS daemon...")
impl, err := ipfs_embedded.NewEmbeddedIPFSNode()
if err != nil {
sentry.CaptureException(err)
panic(err)
}
implementation = impl
} else {
logrus.Info("Using localhost IPFS HTTP agent...")
impl, err := ipfs_local.NewLocalIPFSImplementation()
if err != nil {
sentry.CaptureException(err)
panic(err)
}
implementation = impl
}
}
func Stop() {
if implementation != nil {
implementation.Stop()
}
implementation = nil
}
func getImpl() IPFSImplementation {
if implementation == nil {
Reload()
}
if implementation == nil {
panic("missing ipfs implementation object")
}
return implementation
}
func GetObject(contentId string, ctx rcontext.RequestContext) (*ipfs_models.IPFSObject, error) {
return getImpl().GetObject(contentId, ctx)
}
func PutObject(data io.Reader, ctx rcontext.RequestContext) (string, error) {
return getImpl().PutObject(data, ctx)
}
ipfs_proxy/iface.go deleted 100644 → 0
+ 0
− 14
View file @ a6de9482
package ipfs_proxy
import (
"io"
"github.com/turt2live/matrix-media-repo/common/rcontext"
"github.com/turt2live/matrix-media-repo/ipfs_proxy/ipfs_models"
)
type IPFSImplementation interface {
GetObject(contentId string, ctx rcontext.RequestContext) (*ipfs_models.IPFSObject, error)
PutObject(data io.Reader, ctx rcontext.RequestContext) (string, error)
Stop()
}
ipfs_proxy/ipfs_embedded/impl.go deleted 100644 → 0
+ 0
− 167
View file @ a6de9482
package ipfs_embedded
import (
"bytes"
"context"
"io"
"io/ioutil"
"path"
"path/filepath"
"time"
"github.com/ipfs/go-cid"
files "github.com/ipfs/go-ipfs-files"
icore "github.com/ipfs/interface-go-ipfs-core"
icorepath "github.com/ipfs/interface-go-ipfs-core/path"
ipfsConfig "github.com/ipfs/kubo/config"
"github.com/ipfs/kubo/core"
"github.com/ipfs/kubo/core/bootstrap"
"github.com/ipfs/kubo/core/coreapi"
"github.com/ipfs/kubo/core/node/libp2p"
"github.com/ipfs/kubo/plugin/loader"
"github.com/ipfs/kubo/repo/fsrepo"
"github.com/sirupsen/logrus"
"github.com/turt2live/matrix-media-repo/common/config"
"github.com/turt2live/matrix-media-repo/common/rcontext"
"github.com/turt2live/matrix-media-repo/ipfs_proxy/ipfs_models"
"github.com/turt2live/matrix-media-repo/util"
)
type IPFSEmbedded struct {
api icore.CoreAPI
node *core.IpfsNode
ctx context.Context
cancelCtxFn context.CancelFunc
}
func NewEmbeddedIPFSNode() (IPFSEmbedded, error) {
// Startup routine modified from:
// https://github.com/ipfs/kubo/blob/083ef47ce84a5bd9a93f0ce0afaf668881dc1f35/docs/examples/go-ipfs-as-a-library/main.go
basePath := config.Get().Features.IPFS.Daemon.RepoPath
dataPath := path.Join(basePath, "data")
ctx, cancel := context.WithCancel(context.Background())
blank := IPFSEmbedded{}
// Load plugins
logrus.Info("Loading plugins for IPFS embedded node...")
plugins, err := loader.NewPluginLoader(filepath.Join(basePath, "plugins"))
if err != nil {
cancel()
return blank, err
}
err = plugins.Initialize()
if err != nil {
cancel()
return blank, err
}
err = plugins.Inject()
if err != nil {
cancel()
return blank, err
}
logrus.Info("Generating config for IPFS embedded node")
cfg, err := ipfsConfig.Init(ioutil.Discard, 2048)
if err != nil {
cancel()
return blank, err
}
logrus.Info("Initializing IPFS embedded node")
err = fsrepo.Init(dataPath, cfg)
if err != nil {
cancel()
return blank, err
}
logrus.Info("Starting fsrepo for IPFS embedded node")
repo, err := fsrepo.Open(dataPath)
if err != nil {
cancel()
return blank, err
}
// Create the node from the repo
nodeOptions := &core.BuildCfg{
Online: true,
Routing: libp2p.DHTOption,
Repo: repo,
}
logrus.Info("Building IPFS embedded node")
node, err := core.NewNode(ctx, nodeOptions)
if err != nil {
cancel()
return blank, err
}
logrus.Info("Generating API reference for IPFS embedded node")
api, err := coreapi.NewCoreAPI(node)
if err != nil {
cancel()
return blank, err
}
// Connect to peers so we can actually get started
logrus.Info("Connecting to peers for IPFS embedded node")
err = node.Bootstrap(bootstrap.DefaultBootstrapConfig)
if err != nil {
cancel()
return blank, err
}
logrus.Info("Done building IPFS embedded node")
return IPFSEmbedded{
api: api,
node: node,
ctx: ctx,
cancelCtxFn: cancel,
}, nil
}
func (i IPFSEmbedded) GetObject(contentId string, ctx rcontext.RequestContext) (*ipfs_models.IPFSObject, error) {
ctx.Log.Info("Getting object from embedded IPFS node")
ipfsCid, err := cid.Decode(contentId)
if err != nil {
return nil, err
}
ctx.Log.Info("Resolving path and node")
timeoutCtx, cancel := context.WithTimeout(ctx.Context, 10*time.Second)
defer cancel()
ipfsPath := icorepath.IpfsPath(ipfsCid)
r, err := i.api.Object().Data(timeoutCtx, ipfsPath)
if err != nil {
return nil, err
}
b, err := ioutil.ReadAll(r)
if err != nil {
return nil, err
}
ctx.Log.Info("Returning object")
return &ipfs_models.IPFSObject{
ContentType: "application/octet-stream", // TODO: Actually fetch
FileName: "ipfs.dat", // TODO: Actually fetch
SizeBytes: int64(len(b)),
Data: util.BufferToStream(bytes.NewBuffer(b)), // stream to avoid log spam
}, nil
}
func (i IPFSEmbedded) PutObject(data io.Reader, ctx rcontext.RequestContext) (string, error) {
ipfsFile := files.NewReaderFile(data)
p, err := i.api.Unixfs().Add(ctx.Context, ipfsFile)
if err != nil {
return "", err
}
return p.Cid().String(), nil
}
func (i IPFSEmbedded) Stop() {
i.cancelCtxFn()
_ = i.node.Close()
}
ipfs_proxy/ipfs_local/impl.go deleted 100644 → 0
+ 0
− 61
View file @ a6de9482
package ipfs_local
import (
"bytes"
"io"
"io/ioutil"
"github.com/ipfs/go-cid"
httpapi "github.com/ipfs/go-ipfs-api"
"github.com/turt2live/matrix-media-repo/common/rcontext"
"github.com/turt2live/matrix-media-repo/ipfs_proxy/ipfs_models"
"github.com/turt2live/matrix-media-repo/util"
)
type IPFSLocal struct {
client *httpapi.Shell
}
func NewLocalIPFSImplementation() (IPFSLocal, error) {
client := httpapi.NewLocalShell()
return IPFSLocal{
client: client,
}, nil
}
func (i IPFSLocal) GetObject(contentId string, ctx rcontext.RequestContext) (*ipfs_models.IPFSObject, error) {
ipfsCid, err := cid.Decode(contentId)
if err != nil {
return nil, err
}
c, err := i.client.Cat(ipfsCid.String())
if err != nil {
return nil, err
}
defer c.Close()
b, err := ioutil.ReadAll(c)
if err != nil {
return nil, err
}
return &ipfs_models.IPFSObject{
ContentType: "application/octet-stream", // TODO: Actually fetch
FileName: "ipfs.dat", // TODO: Actually fetch
SizeBytes: int64(len(b)),
Data: util.BufferToStream(bytes.NewBuffer(b)), // stream to avoid log spam
}, nil
}
func (i IPFSLocal) PutObject(data io.Reader, ctx rcontext.RequestContext) (string, error) {
p, err := i.client.Add(data)
if err != nil {
return "", err
}
return p, nil
}
func (i IPFSLocal) Stop() {
// Nothing to do
}
ipfs_proxy/ipfs_models/object.go deleted 100644 → 0
+ 0
− 12
View file @ a6de9482
package ipfs_models
import (
"io"
)
type IPFSObject struct {
ContentType string
FileName string
SizeBytes int64
Data io.ReadCloser
}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment