From 8f0bc73d7d5f4c88cbc5588a0ece12b3855c8f98 Mon Sep 17 00:00:00 2001
From: Chocobozzz <me@florianbigard.com>
Date: Thu, 11 Apr 2019 15:38:53 +0200
Subject: [PATCH] Add ability to limit videos history size
---
config/default.yaml | 7 ++++
config/production.yaml.example | 6 ++++
server.ts | 2 ++
server/controllers/api/users/my-history.ts | 2 +-
server/helpers/core-utils.ts | 2 +-
server/helpers/custom-validators/videos.ts | 3 +-
server/initializers/config.ts | 11 ++++--
server/initializers/constants.ts | 9 ++---
server/lib/schedulers/abstract-scheduler.ts | 3 +-
.../remove-old-history-scheduler.ts | 32 +++++++++++++++++
server/middlewares/cache.ts | 4 +--
server/models/account/user-video-history.ts | 14 +++++++-
server/tests/api/videos/videos-history.ts | 34 +++++++++++++++++--
shared/utils/videos/videos.ts | 4 ++-
14 files changed, 116 insertions(+), 17 deletions(-)
create mode 100644 server/lib/schedulers/remove-old-history-scheduler.ts
diff --git a/config/default.yaml b/config/default.yaml
index 617159c2cc..d45d84b90c 100644
--- a/config/default.yaml
+++ b/config/default.yaml
@@ -111,6 +111,13 @@ tracker:
# Reject peers that do a lot of announces (could improve privacy of TCP/UDP peers)
reject_too_many_announces: false
+history:
+ videos:
+ # If you want to limit users videos history
+ # -1 means there is no limitations
+ # Other values could be '6 months' or '30 days' etc (PeerTube will periodically delete old entries from database)
+ max_age: -1
+
cache:
previews:
size: 500 # Max number of previews you want to cache
diff --git a/config/production.yaml.example b/config/production.yaml.example
index dd5c9769bd..b813a65e9c 100644
--- a/config/production.yaml.example
+++ b/config/production.yaml.example
@@ -112,6 +112,12 @@ tracker:
# Reject peers that do a lot of announces (could improve privacy of TCP/UDP peers)
reject_too_many_announces: false
+history:
+ videos:
+ # If you want to limit users videos history
+ # -1 means there is no limitations
+ # Other values could be '6 months' or '30 days' etc (PeerTube will periodically delete old entries from database)
+ max_age: -1
###############################################################################
#
diff --git a/server.ts b/server.ts
index 110ae1ab8e..f4f0c4d68a 100644
--- a/server.ts
+++ b/server.ts
@@ -105,6 +105,7 @@ import { RemoveOldJobsScheduler } from './server/lib/schedulers/remove-old-jobs-
import { UpdateVideosScheduler } from './server/lib/schedulers/update-videos-scheduler'
import { YoutubeDlUpdateScheduler } from './server/lib/schedulers/youtube-dl-update-scheduler'
import { VideosRedundancyScheduler } from './server/lib/schedulers/videos-redundancy-scheduler'
+import { RemoveOldHistoryScheduler } from './server/lib/schedulers/remove-old-history-scheduler'
import { isHTTPSignatureDigestValid } from './server/helpers/peertube-crypto'
import { PeerTubeSocket } from './server/lib/peertube-socket'
import { updateStreamingPlaylistsInfohashesIfNeeded } from './server/lib/hls'
@@ -240,6 +241,7 @@ async function startApplication () {
UpdateVideosScheduler.Instance.enable()
YoutubeDlUpdateScheduler.Instance.enable()
VideosRedundancyScheduler.Instance.enable()
+ RemoveOldHistoryScheduler.Instance.enable()
// Redis initialization
Redis.Instance.init()
diff --git a/server/controllers/api/users/my-history.ts b/server/controllers/api/users/my-history.ts
index b30d3aec2b..7025c0ff15 100644
--- a/server/controllers/api/users/my-history.ts
+++ b/server/controllers/api/users/my-history.ts
@@ -48,7 +48,7 @@ async function removeUserHistory (req: express.Request, res: express.Response) {
const beforeDate = req.body.beforeDate || null
await sequelizeTypescript.transaction(t => {
- return UserVideoHistoryModel.removeHistoryBefore(user, beforeDate, t)
+ return UserVideoHistoryModel.removeUserHistoryBefore(user, beforeDate, t)
})
// Do not send the delete to other instances, we delete OUR copy of this video abuse
diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts
index f6d90bfca3..305d3b71e0 100644
--- a/server/helpers/core-utils.ts
+++ b/server/helpers/core-utils.ts
@@ -40,7 +40,7 @@ const timeTable = {
month: 3600000 * 24 * 30
}
-export function parseDuration (duration: number | string): number {
+export function parseDurationToMs (duration: number | string): number {
if (typeof duration === 'number') return duration
if (typeof duration === 'string') {
diff --git a/server/helpers/custom-validators/videos.ts b/server/helpers/custom-validators/videos.ts
index eb08ae4ad9..214db17a12 100644
--- a/server/helpers/custom-validators/videos.ts
+++ b/server/helpers/custom-validators/videos.ts
@@ -5,7 +5,8 @@ import 'multer'
import * as validator from 'validator'
import { UserRight, VideoFilter, VideoPrivacy, VideoRateType } from '../../../shared'
import {
- CONSTRAINTS_FIELDS, MIMETYPES,
+ CONSTRAINTS_FIELDS,
+ MIMETYPES,
VIDEO_CATEGORIES,
VIDEO_LICENCES,
VIDEO_PRIVACIES,
diff --git a/server/initializers/config.ts b/server/initializers/config.ts
index 8dd62cba8c..1f374dea9d 100644
--- a/server/initializers/config.ts
+++ b/server/initializers/config.ts
@@ -2,7 +2,7 @@ import { IConfig } from 'config'
import { dirname, join } from 'path'
import { VideosRedundancy } from '../../shared/models'
// Do not use barrels, remain constants as independent as possible
-import { buildPath, parseBytes, parseDuration, root } from '../helpers/core-utils'
+import { buildPath, parseBytes, parseDurationToMs, root } from '../helpers/core-utils'
import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type'
import * as bytes from 'bytes'
@@ -80,7 +80,7 @@ const CONFIG = {
},
REDUNDANCY: {
VIDEOS: {
- CHECK_INTERVAL: parseDuration(config.get<string>('redundancy.videos.check_interval')),
+ CHECK_INTERVAL: parseDurationToMs(config.get<string>('redundancy.videos.check_interval')),
STRATEGIES: buildVideosRedundancy(config.get<any[]>('redundancy.videos.strategies'))
}
},
@@ -94,6 +94,11 @@ const CONFIG = {
PRIVATE: config.get<boolean>('tracker.private'),
REJECT_TOO_MANY_ANNOUNCES: config.get<boolean>('tracker.reject_too_many_announces')
},
+ HISTORY: {
+ VIDEOS: {
+ MAX_AGE: parseDurationToMs(config.get('history.videos.max_age'))
+ }
+ },
ADMIN: {
get EMAIL () { return config.get<string>('admin.email') }
},
@@ -216,7 +221,7 @@ function buildVideosRedundancy (objs: any[]): VideosRedundancy[] {
return objs.map(obj => {
return Object.assign({}, obj, {
- minLifetime: parseDuration(obj.min_lifetime),
+ minLifetime: parseDurationToMs(obj.min_lifetime),
size: bytes.parse(obj.size),
minViews: obj.min_views
})
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index a0609d7cde..f008cd291d 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -158,12 +158,12 @@ const JOB_REQUEST_TIMEOUT = 3000 // 3 seconds
const JOB_COMPLETED_LIFETIME = 60000 * 60 * 24 * 2 // 2 days
const VIDEO_IMPORT_TIMEOUT = 1000 * 3600 // 1 hour
-// 1 hour
-let SCHEDULER_INTERVALS_MS = {
+const SCHEDULER_INTERVALS_MS = {
actorFollowScores: 60000 * 60, // 1 hour
removeOldJobs: 60000 * 60, // 1 hour
updateVideos: 60000, // 1 minute
- youtubeDLUpdate: 60000 * 60 * 24 // 1 day
+ youtubeDLUpdate: 60000 * 60 * 24, // 1 day
+ removeOldHistory: 60000 * 60 * 24 // 1 day
}
// ---------------------------------------------------------------------------
@@ -591,6 +591,7 @@ if (isTestInstance() === true) {
SCHEDULER_INTERVALS_MS.actorFollowScores = 1000
SCHEDULER_INTERVALS_MS.removeOldJobs = 10000
+ SCHEDULER_INTERVALS_MS.removeOldHistory = 5000
SCHEDULER_INTERVALS_MS.updateVideos = 5000
REPEAT_JOBS[ 'videos-views' ] = { every: 5000 }
@@ -734,7 +735,7 @@ function buildVideosExtname () {
}
function loadLanguages () {
- VIDEO_LANGUAGES = buildLanguages()
+ Object.assign(VIDEO_LANGUAGES, buildLanguages())
}
function buildLanguages () {
diff --git a/server/lib/schedulers/abstract-scheduler.ts b/server/lib/schedulers/abstract-scheduler.ts
index 86ea7aa38b..0e60889117 100644
--- a/server/lib/schedulers/abstract-scheduler.ts
+++ b/server/lib/schedulers/abstract-scheduler.ts
@@ -1,4 +1,5 @@
import { logger } from '../../helpers/logger'
+import * as Bluebird from 'bluebird'
export abstract class AbstractScheduler {
@@ -30,5 +31,5 @@ export abstract class AbstractScheduler {
}
}
- protected abstract internalExecute (): Promise<any>
+ protected abstract internalExecute (): Promise<any> | Bluebird<any>
}
diff --git a/server/lib/schedulers/remove-old-history-scheduler.ts b/server/lib/schedulers/remove-old-history-scheduler.ts
new file mode 100644
index 0000000000..1b5ff83944
--- /dev/null
+++ b/server/lib/schedulers/remove-old-history-scheduler.ts
@@ -0,0 +1,32 @@
+import { logger } from '../../helpers/logger'
+import { AbstractScheduler } from './abstract-scheduler'
+import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
+import { UserVideoHistoryModel } from '../../models/account/user-video-history'
+import { CONFIG } from '../../initializers/config'
+import { isTestInstance } from '../../helpers/core-utils'
+
+export class RemoveOldHistoryScheduler extends AbstractScheduler {
+
+ private static instance: AbstractScheduler
+
+ protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.removeOldHistory
+
+ private constructor () {
+ super()
+ }
+
+ protected internalExecute () {
+ if (CONFIG.HISTORY.VIDEOS.MAX_AGE === -1) return
+
+ logger.info('Removing old videos history.')
+
+ const now = new Date()
+ const beforeDate = new Date(now.getTime() - CONFIG.HISTORY.VIDEOS.MAX_AGE).toISOString()
+
+ return UserVideoHistoryModel.removeOldHistory(beforeDate)
+ }
+
+ static get Instance () {
+ return this.instance || (this.instance = new this())
+ }
+}
diff --git a/server/middlewares/cache.ts b/server/middlewares/cache.ts
index 8ffe757009..e83d8d569b 100644
--- a/server/middlewares/cache.ts
+++ b/server/middlewares/cache.ts
@@ -1,6 +1,6 @@
import * as express from 'express'
import * as AsyncLock from 'async-lock'
-import { parseDuration } from '../helpers/core-utils'
+import { parseDurationToMs } from '../helpers/core-utils'
import { Redis } from '../lib/redis'
import { logger } from '../helpers/logger'
@@ -24,7 +24,7 @@ function cacheRoute (lifetimeArg: string | number) {
res.send = (body) => {
if (res.statusCode >= 200 && res.statusCode < 400) {
const contentType = res.get('content-type')
- const lifetime = parseDuration(lifetimeArg)
+ const lifetime = parseDurationToMs(lifetimeArg)
Redis.Instance.setCachedRoute(req, body, lifetime, contentType, res.statusCode)
.then(() => done())
diff --git a/server/models/account/user-video-history.ts b/server/models/account/user-video-history.ts
index 15cb399c95..49d2def815 100644
--- a/server/models/account/user-video-history.ts
+++ b/server/models/account/user-video-history.ts
@@ -67,7 +67,7 @@ export class UserVideoHistoryModel extends Model<UserVideoHistoryModel> {
})
}
- static removeHistoryBefore (user: UserModel, beforeDate: string, t: Transaction) {
+ static removeUserHistoryBefore (user: UserModel, beforeDate: string, t: Transaction) {
const query: DestroyOptions = {
where: {
userId: user.id
@@ -83,4 +83,16 @@ export class UserVideoHistoryModel extends Model<UserVideoHistoryModel> {
return UserVideoHistoryModel.destroy(query)
}
+
+ static removeOldHistory (beforeDate: string) {
+ const query: DestroyOptions = {
+ where: {
+ updatedAt: {
+ [Op.lt]: beforeDate
+ }
+ }
+ }
+
+ return UserVideoHistoryModel.destroy(query)
+ }
}
diff --git a/server/tests/api/videos/videos-history.ts b/server/tests/api/videos/videos-history.ts
index f654a422bd..f7d3a6aeb0 100644
--- a/server/tests/api/videos/videos-history.ts
+++ b/server/tests/api/videos/videos-history.ts
@@ -7,14 +7,15 @@ import {
flushTests,
getVideosListWithToken,
getVideoWithToken,
- killallServers,
+ killallServers, reRunServer,
runServer,
searchVideoWithToken,
ServerInfo,
setAccessTokensToServers,
updateMyUser,
uploadVideo,
- userLogin
+ userLogin,
+ wait
} from '../../../../shared/utils'
import { Video, VideoDetails } from '../../../../shared/models/videos'
import { listMyVideosHistory, removeMyVideosHistory, userWatchVideo } from '../../../../shared/utils/videos/video-history'
@@ -192,6 +193,35 @@ describe('Test videos history', function () {
expect(videos[1].name).to.equal('video 3')
})
+ it('Should not clean old history', async function () {
+ this.timeout(50000)
+
+ killallServers([ server ])
+
+ await reRunServer(server, { history: { videos: { max_age: '10 days' } } })
+
+ await wait(6000)
+
+ // Should still have history
+
+ const res = await listMyVideosHistory(server.url, server.accessToken)
+
+ expect(res.body.total).to.equal(2)
+ })
+
+ it('Should clean old history', async function () {
+ this.timeout(50000)
+
+ killallServers([ server ])
+
+ await reRunServer(server, { history: { videos: { max_age: '5 seconds' } } })
+
+ await wait(6000)
+
+ const res = await listMyVideosHistory(server.url, server.accessToken)
+ expect(res.body.total).to.equal(0)
+ })
+
after(async function () {
killallServers([ server ])
diff --git a/shared/utils/videos/videos.ts b/shared/utils/videos/videos.ts
index 54c6bccec4..b5a07b792d 100644
--- a/shared/utils/videos/videos.ts
+++ b/shared/utils/videos/videos.ts
@@ -18,9 +18,11 @@ import {
} from '../'
import * as validator from 'validator'
import { VideoDetails, VideoPrivacy } from '../../models/videos'
-import { VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES, VIDEO_PRIVACIES } from '../../../server/initializers/constants'
+import { VIDEO_CATEGORIES, VIDEO_LANGUAGES, loadLanguages, VIDEO_LICENCES, VIDEO_PRIVACIES } from '../../../server/initializers/constants'
import { dateIsValid, webtorrentAdd } from '../miscs/miscs'
+loadLanguages()
+
type VideoAttributes = {
name?: string
category?: number
--
GitLab