Improve audio sampling speed by using the seeker interface

We make our own ReadCloser+ReadSeeker implementation because a NopCloser actually destroys the seeker implementation.

api/unstable/info.go

@@ -18,8 +18,8 @@ import (
 	"github.com/turt2live/matrix-media-repo/storage"
 	"github.com/turt2live/matrix-media-repo/thumbnailing"
 	"github.com/turt2live/matrix-media-repo/thumbnailing/i"
-	"github.com/turt2live/matrix-media-repo/util"
 	"github.com/turt2live/matrix-media-repo/util/cleanup"
+	"github.com/turt2live/matrix-media-repo/util/util_byte_seeker"
 )
 
 type mediaInfoHashes struct {
@@ -123,7 +123,7 @@ func MediaInfo(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo)
 	}
 
 	if strings.HasPrefix(response.ContentType, "audio/") {
-		generator, err := thumbnailing.GetGenerator(util.ByteCloser(b), response.ContentType, false)
+		generator, err := thumbnailing.GetGenerator(util_byte_seeker.NewByteSeeker(b), response.ContentType, false)
 		if err == nil {
 			if audiogenerator, ok := generator.(i.AudioGenerator); ok {
 				audioInfo, err := audiogenerator.GetAudioData(b, 768, rctx)

thumbnailing/i/flac.go

@@ -8,7 +8,7 @@ import (
 	"github.com/turt2live/matrix-media-repo/common/rcontext"
 	"github.com/turt2live/matrix-media-repo/thumbnailing/m"
 	"github.com/turt2live/matrix-media-repo/thumbnailing/u"
-	"github.com/turt2live/matrix-media-repo/util"
+	"github.com/turt2live/matrix-media-repo/util/util_byte_seeker"
 )
 
 type flacGenerator struct {
@@ -27,7 +27,7 @@ func (d flacGenerator) matches(img []byte, contentType string) bool {
 }
 
 func (d flacGenerator) decode(b []byte) (beep.StreamSeekCloser, beep.Format, error) {
-	audio, format, err := flac.Decode(util.ByteCloser(b))
+	audio, format, err := flac.Decode(util_byte_seeker.NewByteSeeker(b))
 	if err != nil {
 		return audio, format, errors.New("flac: error decoding audio: " + err.Error())
 	}

thumbnailing/i/mp3.go

@@ -6,7 +6,6 @@ import (
 	"image"
 	"image/color"
 	"image/draw"
-	"io"
 	"io/ioutil"
 	"math"
 	"path"
@@ -20,7 +19,8 @@ import (
 	"github.com/turt2live/matrix-media-repo/common/rcontext"
 	"github.com/turt2live/matrix-media-repo/thumbnailing/m"
 	"github.com/turt2live/matrix-media-repo/thumbnailing/u"
-	"github.com/turt2live/matrix-media-repo/util"
+	"github.com/turt2live/matrix-media-repo/util/util_audio"
+	"github.com/turt2live/matrix-media-repo/util/util_byte_seeker"
 )
 
 type mp3Generator struct {
@@ -39,7 +39,7 @@ func (d mp3Generator) matches(img []byte, contentType string) bool {
 }
 
 func (d mp3Generator) decode(b []byte) (beep.StreamSeekCloser, beep.Format, error) {
-	audio, format, err := mp3.Decode(util.ByteCloser(b))
+	audio, format, err := mp3.Decode(util_byte_seeker.NewByteSeeker(b))
 	if err != nil {
 		return audio, format, errors.New("mp3: error decoding audio: " + err.Error())
 	}
@@ -67,39 +67,16 @@ func (d mp3Generator) GetAudioData(b []byte, nKeys int, ctx rcontext.RequestCont
 }
 
 func (d mp3Generator) GetDataFromStream(audio beep.StreamSeekCloser, format beep.Format, nKeys int) (*m.AudioInfo, error) {
-	allSamples := make([][2]float64, 0)
-
-	moreSamples := true
-	samples := make([][2]float64, 100000) // a 3 minute mp3 has roughly 7 million samples, so reduce the number of iterations we have to do
-	for moreSamples {
-		n, ok := audio.Stream(*&samples)
-		if n == 0 {
-			moreSamples = false
-		}
-		if !ok && audio.Err() != nil && audio.Err() != io.EOF {
-			return nil, errors.New("beep-visual: error sampling audio: " + audio.Err().Error())
-		}
-		for i, v := range samples {
-			if i >= n {
-				break
-			}
-			allSamples = append(allSamples, v)
-		}
-	}
-
-	downsampled := make([][2]float64, 0)
-	everyNth := int(math.Round(float64(len(allSamples)) / float64(nKeys)))
-	for i, s := range allSamples {
-		if i%everyNth != 0 {
-			continue
-		}
-		downsampled = append(downsampled, s)
+	totalSamples := audio.Len()
+	downsampled, err := util_audio.FastSampleAudio(audio, nKeys)
+	if err != nil {
+		return nil, err
 	}
 
 	return &m.AudioInfo{
-		Duration:     format.SampleRate.D(len(allSamples)),
+		Duration:     format.SampleRate.D(totalSamples),
 		Channels:     format.NumChannels,
-		TotalSamples: len(allSamples),
+		TotalSamples: totalSamples,
 		KeySamples:   downsampled,
 	}, nil
 }

thumbnailing/i/ogg.go

@@ -8,7 +8,7 @@ import (
 	"github.com/turt2live/matrix-media-repo/common/rcontext"
 	"github.com/turt2live/matrix-media-repo/thumbnailing/m"
 	"github.com/turt2live/matrix-media-repo/thumbnailing/u"
-	"github.com/turt2live/matrix-media-repo/util"
+	"github.com/turt2live/matrix-media-repo/util/util_byte_seeker"
 )
 
 type oggGenerator struct {
@@ -27,7 +27,7 @@ func (d oggGenerator) matches(img []byte, contentType string) bool {
 }
 
 func (d oggGenerator) decode(b []byte) (beep.StreamSeekCloser, beep.Format, error) {
-	audio, format, err := vorbis.Decode(util.ByteCloser(b))
+	audio, format, err := vorbis.Decode(util_byte_seeker.NewByteSeeker(b))
 	if err != nil {
 		return audio, format, errors.New("ogg: error decoding audio: " + err.Error())
 	}

thumbnailing/i/wav.go

@@ -8,7 +8,7 @@ import (
 	"github.com/turt2live/matrix-media-repo/common/rcontext"
 	"github.com/turt2live/matrix-media-repo/thumbnailing/m"
 	"github.com/turt2live/matrix-media-repo/thumbnailing/u"
-	"github.com/turt2live/matrix-media-repo/util"
+	"github.com/turt2live/matrix-media-repo/util/util_byte_seeker"
 )
 
 type wavGenerator struct {
@@ -27,7 +27,7 @@ func (d wavGenerator) matches(img []byte, contentType string) bool {
 }
 
 func (d wavGenerator) decode(b []byte) (beep.StreamSeekCloser, beep.Format, error) {
-	audio, format, err := wav.Decode(util.ByteCloser(b))
+	audio, format, err := wav.Decode(util_byte_seeker.NewByteSeeker(b))
 	if err != nil {
 		return audio, format, errors.New("wav: error decoding audio: " + err.Error())
 	}

util/streams.go

@@ -50,8 +50,3 @@ func GetSha256HashOfStream(r io.ReadCloser) (string, error) {
 
 	return hex.EncodeToString(hasher.Sum(nil)), nil
 }
-
-func ByteCloser(b []byte) io.ReadCloser {
-	buf := bytes.NewBuffer(b)
-	return ioutil.NopCloser(buf)
-}
\ No newline at end of file

util/util_audio/fastsample.go

+package util_audio
+
+import (
+	"errors"
+	"math"
+
+	"github.com/faiface/beep"
+)
+
+func FastSampleAudio(stream beep.StreamSeekCloser, numSamples int) ([][2]float64, error) {
+	everyNth := int(math.Round(float64(stream.Len()) / float64(numSamples)))
+	samples := make([][2]float64, numSamples)
+	totalRead := 0
+	for i := range samples {
+		pos := i * everyNth
+		if stream.Position() != pos {
+			err := stream.Seek(pos)
+			if err != nil {
+				return nil, errors.New("fast-sample: could not seek: " + err.Error())
+			}
+		}
+
+		sample := make([][2]float64, 1)
+		n, _ := stream.Stream(sample)
+		if stream.Err() != nil {
+			return nil, errors.New("fast-sample: could not stream: " + stream.Err().Error())
+		}
+		if n > 0 {
+			samples[i] = sample[0]
+			totalRead++
+		} else {
+			break
+		}
+	}
+	if totalRead != len(samples) {
+		return samples[:totalRead], nil
+	}
+	return samples, nil
+}

util/util_byte_seeker/seekable.go

+package util_byte_seeker
+
+import (
+	"bytes"
+)
+
+type ByteSeeker struct {
+	s *bytes.Reader
+}
+
+func NewByteSeeker(b []byte) ByteSeeker {
+	return ByteSeeker{s: bytes.NewReader(b)}
+}
+
+func (s ByteSeeker) Close() error {
+	return nil // no-op
+}
+
+func (s ByteSeeker) Read(p []byte) (n int, err error) {
+	return s.s.Read(p)
+}
+
+func (s ByteSeeker) Seek(offset int64, whence int) (int64, error) {
+	return s.s.Seek(offset, whence)
+}