From abe72c87d2e293e48e3cfb07c51407cc729849ea Mon Sep 17 00:00:00 2001
From: Travis Ralston <travpc@gmail.com>
Date: Sun, 16 Aug 2020 16:27:47 -0600
Subject: [PATCH] Improve audio sampling speed by using the seeker interface
We make our own ReadCloser+ReadSeeker implementation because a NopCloser actually destroys the seeker implementation.
---
api/unstable/info.go | 4 +--
thumbnailing/i/flac.go | 4 +--
thumbnailing/i/mp3.go | 41 +++++++------------------------
thumbnailing/i/ogg.go | 4 +--
thumbnailing/i/wav.go | 4 +--
util/streams.go | 5 ----
util/util_audio/fastsample.go | 39 +++++++++++++++++++++++++++++
util/util_byte_seeker/seekable.go | 25 +++++++++++++++++++
8 files changed, 81 insertions(+), 45 deletions(-)
create mode 100644 util/util_audio/fastsample.go
create mode 100644 util/util_byte_seeker/seekable.go
diff --git a/api/unstable/info.go b/api/unstable/info.go
index e880aec2..5f669d86 100644
--- a/api/unstable/info.go
+++ b/api/unstable/info.go
@@ -18,8 +18,8 @@ import (
"github.com/turt2live/matrix-media-repo/storage"
"github.com/turt2live/matrix-media-repo/thumbnailing"
"github.com/turt2live/matrix-media-repo/thumbnailing/i"
- "github.com/turt2live/matrix-media-repo/util"
"github.com/turt2live/matrix-media-repo/util/cleanup"
+ "github.com/turt2live/matrix-media-repo/util/util_byte_seeker"
)
type mediaInfoHashes struct {
@@ -123,7 +123,7 @@ func MediaInfo(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo)
}
if strings.HasPrefix(response.ContentType, "audio/") {
- generator, err := thumbnailing.GetGenerator(util.ByteCloser(b), response.ContentType, false)
+ generator, err := thumbnailing.GetGenerator(util_byte_seeker.NewByteSeeker(b), response.ContentType, false)
if err == nil {
if audiogenerator, ok := generator.(i.AudioGenerator); ok {
audioInfo, err := audiogenerator.GetAudioData(b, 768, rctx)
diff --git a/thumbnailing/i/flac.go b/thumbnailing/i/flac.go
index 17e743b2..1c45f6e5 100644
--- a/thumbnailing/i/flac.go
+++ b/thumbnailing/i/flac.go
@@ -8,7 +8,7 @@ import (
"github.com/turt2live/matrix-media-repo/common/rcontext"
"github.com/turt2live/matrix-media-repo/thumbnailing/m"
"github.com/turt2live/matrix-media-repo/thumbnailing/u"
- "github.com/turt2live/matrix-media-repo/util"
+ "github.com/turt2live/matrix-media-repo/util/util_byte_seeker"
)
type flacGenerator struct {
@@ -27,7 +27,7 @@ func (d flacGenerator) matches(img []byte, contentType string) bool {
}
func (d flacGenerator) decode(b []byte) (beep.StreamSeekCloser, beep.Format, error) {
- audio, format, err := flac.Decode(util.ByteCloser(b))
+ audio, format, err := flac.Decode(util_byte_seeker.NewByteSeeker(b))
if err != nil {
return audio, format, errors.New("flac: error decoding audio: " + err.Error())
}
diff --git a/thumbnailing/i/mp3.go b/thumbnailing/i/mp3.go
index 8df89242..b3c7f8dd 100644
--- a/thumbnailing/i/mp3.go
+++ b/thumbnailing/i/mp3.go
@@ -6,7 +6,6 @@ import (
"image"
"image/color"
"image/draw"
- "io"
"io/ioutil"
"math"
"path"
@@ -20,7 +19,8 @@ import (
"github.com/turt2live/matrix-media-repo/common/rcontext"
"github.com/turt2live/matrix-media-repo/thumbnailing/m"
"github.com/turt2live/matrix-media-repo/thumbnailing/u"
- "github.com/turt2live/matrix-media-repo/util"
+ "github.com/turt2live/matrix-media-repo/util/util_audio"
+ "github.com/turt2live/matrix-media-repo/util/util_byte_seeker"
)
type mp3Generator struct {
@@ -39,7 +39,7 @@ func (d mp3Generator) matches(img []byte, contentType string) bool {
}
func (d mp3Generator) decode(b []byte) (beep.StreamSeekCloser, beep.Format, error) {
- audio, format, err := mp3.Decode(util.ByteCloser(b))
+ audio, format, err := mp3.Decode(util_byte_seeker.NewByteSeeker(b))
if err != nil {
return audio, format, errors.New("mp3: error decoding audio: " + err.Error())
}
@@ -67,39 +67,16 @@ func (d mp3Generator) GetAudioData(b []byte, nKeys int, ctx rcontext.RequestCont
}
func (d mp3Generator) GetDataFromStream(audio beep.StreamSeekCloser, format beep.Format, nKeys int) (*m.AudioInfo, error) {
- allSamples := make([][2]float64, 0)
-
- moreSamples := true
- samples := make([][2]float64, 100000) // a 3 minute mp3 has roughly 7 million samples, so reduce the number of iterations we have to do
- for moreSamples {
- n, ok := audio.Stream(*&samples)
- if n == 0 {
- moreSamples = false
- }
- if !ok && audio.Err() != nil && audio.Err() != io.EOF {
- return nil, errors.New("beep-visual: error sampling audio: " + audio.Err().Error())
- }
- for i, v := range samples {
- if i >= n {
- break
- }
- allSamples = append(allSamples, v)
- }
- }
-
- downsampled := make([][2]float64, 0)
- everyNth := int(math.Round(float64(len(allSamples)) / float64(nKeys)))
- for i, s := range allSamples {
- if i%everyNth != 0 {
- continue
- }
- downsampled = append(downsampled, s)
+ totalSamples := audio.Len()
+ downsampled, err := util_audio.FastSampleAudio(audio, nKeys)
+ if err != nil {
+ return nil, err
}
return &m.AudioInfo{
- Duration: format.SampleRate.D(len(allSamples)),
+ Duration: format.SampleRate.D(totalSamples),
Channels: format.NumChannels,
- TotalSamples: len(allSamples),
+ TotalSamples: totalSamples,
KeySamples: downsampled,
}, nil
}
diff --git a/thumbnailing/i/ogg.go b/thumbnailing/i/ogg.go
index ef2eaba2..8b73cb41 100644
--- a/thumbnailing/i/ogg.go
+++ b/thumbnailing/i/ogg.go
@@ -8,7 +8,7 @@ import (
"github.com/turt2live/matrix-media-repo/common/rcontext"
"github.com/turt2live/matrix-media-repo/thumbnailing/m"
"github.com/turt2live/matrix-media-repo/thumbnailing/u"
- "github.com/turt2live/matrix-media-repo/util"
+ "github.com/turt2live/matrix-media-repo/util/util_byte_seeker"
)
type oggGenerator struct {
@@ -27,7 +27,7 @@ func (d oggGenerator) matches(img []byte, contentType string) bool {
}
func (d oggGenerator) decode(b []byte) (beep.StreamSeekCloser, beep.Format, error) {
- audio, format, err := vorbis.Decode(util.ByteCloser(b))
+ audio, format, err := vorbis.Decode(util_byte_seeker.NewByteSeeker(b))
if err != nil {
return audio, format, errors.New("ogg: error decoding audio: " + err.Error())
}
diff --git a/thumbnailing/i/wav.go b/thumbnailing/i/wav.go
index 7d63b5ab..265fc28f 100644
--- a/thumbnailing/i/wav.go
+++ b/thumbnailing/i/wav.go
@@ -8,7 +8,7 @@ import (
"github.com/turt2live/matrix-media-repo/common/rcontext"
"github.com/turt2live/matrix-media-repo/thumbnailing/m"
"github.com/turt2live/matrix-media-repo/thumbnailing/u"
- "github.com/turt2live/matrix-media-repo/util"
+ "github.com/turt2live/matrix-media-repo/util/util_byte_seeker"
)
type wavGenerator struct {
@@ -27,7 +27,7 @@ func (d wavGenerator) matches(img []byte, contentType string) bool {
}
func (d wavGenerator) decode(b []byte) (beep.StreamSeekCloser, beep.Format, error) {
- audio, format, err := wav.Decode(util.ByteCloser(b))
+ audio, format, err := wav.Decode(util_byte_seeker.NewByteSeeker(b))
if err != nil {
return audio, format, errors.New("wav: error decoding audio: " + err.Error())
}
diff --git a/util/streams.go b/util/streams.go
index e01d01c9..a51adcdb 100644
--- a/util/streams.go
+++ b/util/streams.go
@@ -50,8 +50,3 @@ func GetSha256HashOfStream(r io.ReadCloser) (string, error) {
return hex.EncodeToString(hasher.Sum(nil)), nil
}
-
-func ByteCloser(b []byte) io.ReadCloser {
- buf := bytes.NewBuffer(b)
- return ioutil.NopCloser(buf)
-}
\ No newline at end of file
diff --git a/util/util_audio/fastsample.go b/util/util_audio/fastsample.go
new file mode 100644
index 00000000..bef57ca3
--- /dev/null
+++ b/util/util_audio/fastsample.go
@@ -0,0 +1,39 @@
+package util_audio
+
+import (
+ "errors"
+ "math"
+
+ "github.com/faiface/beep"
+)
+
+func FastSampleAudio(stream beep.StreamSeekCloser, numSamples int) ([][2]float64, error) {
+ everyNth := int(math.Round(float64(stream.Len()) / float64(numSamples)))
+ samples := make([][2]float64, numSamples)
+ totalRead := 0
+ for i := range samples {
+ pos := i * everyNth
+ if stream.Position() != pos {
+ err := stream.Seek(pos)
+ if err != nil {
+ return nil, errors.New("fast-sample: could not seek: " + err.Error())
+ }
+ }
+
+ sample := make([][2]float64, 1)
+ n, _ := stream.Stream(sample)
+ if stream.Err() != nil {
+ return nil, errors.New("fast-sample: could not stream: " + stream.Err().Error())
+ }
+ if n > 0 {
+ samples[i] = sample[0]
+ totalRead++
+ } else {
+ break
+ }
+ }
+ if totalRead != len(samples) {
+ return samples[:totalRead], nil
+ }
+ return samples, nil
+}
diff --git a/util/util_byte_seeker/seekable.go b/util/util_byte_seeker/seekable.go
new file mode 100644
index 00000000..1257ec72
--- /dev/null
+++ b/util/util_byte_seeker/seekable.go
@@ -0,0 +1,25 @@
+package util_byte_seeker
+
+import (
+ "bytes"
+)
+
+type ByteSeeker struct {
+ s *bytes.Reader
+}
+
+func NewByteSeeker(b []byte) ByteSeeker {
+ return ByteSeeker{s: bytes.NewReader(b)}
+}
+
+func (s ByteSeeker) Close() error {
+ return nil // no-op
+}
+
+func (s ByteSeeker) Read(p []byte) (n int, err error) {
+ return s.s.Read(p)
+}
+
+func (s ByteSeeker) Seek(offset int64, whence int) (int64, error) {
+ return s.s.Seek(offset, whence)
+}
--
GitLab