From d74d29ad9e35929491cf37223398d2535ab23de0 Mon Sep 17 00:00:00 2001
From: Chocobozzz <me@florianbigard.com>
Date: Tue, 19 Mar 2019 14:23:17 +0100
Subject: [PATCH] Limit user tokens cache
---
server.ts | 8 ++++----
server/controllers/static.ts | 3 +--
server/initializers/constants.ts | 13 ++++++++++---
server/initializers/installer.ts | 10 +++++-----
.../abstract-video-static-file-cache.ts | 0
.../actor-follow-score-cache.ts | 0
server/lib/{cache => files-cache}/index.ts | 0
.../{cache => files-cache}/videos-caption-cache.ts | 4 ++--
.../{cache => files-cache}/videos-preview-cache.ts | 4 ++--
.../handlers/activitypub-http-broadcast.ts | 3 +--
.../job-queue/handlers/activitypub-http-unicast.ts | 2 +-
server/lib/oauth-model.ts | 12 +++++++++---
server/lib/schedulers/actor-follow-scheduler.ts | 2 +-
13 files changed, 36 insertions(+), 25 deletions(-)
rename server/lib/{cache => files-cache}/abstract-video-static-file-cache.ts (100%)
rename server/lib/{cache => files-cache}/actor-follow-score-cache.ts (100%)
rename server/lib/{cache => files-cache}/index.ts (100%)
rename server/lib/{cache => files-cache}/videos-caption-cache.ts (91%)
rename server/lib/{cache => files-cache}/videos-preview-cache.ts (88%)
diff --git a/server.ts b/server.ts
index 9fe7411755..df56bcd822 100644
--- a/server.ts
+++ b/server.ts
@@ -28,7 +28,7 @@ import { checkMissedConfig, checkFFmpeg } from './server/initializers/checker-be
// Do not use barrels because we don't want to load all modules here (we need to initialize database first)
import { logger } from './server/helpers/logger'
-import { API_VERSION, CONFIG, CACHE } from './server/initializers/constants'
+import { API_VERSION, CONFIG, FILES_CACHE } from './server/initializers/constants'
const missed = checkMissedConfig()
if (missed.length !== 0) {
@@ -82,7 +82,7 @@ migrate()
import { installApplication } from './server/initializers'
import { Emailer } from './server/lib/emailer'
import { JobQueue } from './server/lib/job-queue'
-import { VideosPreviewCache, VideosCaptionCache } from './server/lib/cache'
+import { VideosPreviewCache, VideosCaptionCache } from './server/lib/files-cache'
import {
activityPubRouter,
apiRouter,
@@ -218,8 +218,8 @@ async function startApplication () {
])
// Caches initializations
- VideosPreviewCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE, CACHE.PREVIEWS.MAX_AGE)
- VideosCaptionCache.Instance.init(CONFIG.CACHE.VIDEO_CAPTIONS.SIZE, CACHE.VIDEO_CAPTIONS.MAX_AGE)
+ VideosPreviewCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE, FILES_CACHE.PREVIEWS.MAX_AGE)
+ VideosCaptionCache.Instance.init(CONFIG.CACHE.VIDEO_CAPTIONS.SIZE, FILES_CACHE.VIDEO_CAPTIONS.MAX_AGE)
// Enable Schedulers
ActorFollowScheduler.Instance.enable()
diff --git a/server/controllers/static.ts b/server/controllers/static.ts
index 7b14320e45..e65c7afd39 100644
--- a/server/controllers/static.ts
+++ b/server/controllers/static.ts
@@ -8,11 +8,10 @@ import {
STATIC_MAX_AGE,
STATIC_PATHS
} from '../initializers'
-import { VideosPreviewCache } from '../lib/cache'
+import { VideosCaptionCache, VideosPreviewCache } from '../lib/files-cache'
import { cacheRoute } from '../middlewares/cache'
import { asyncMiddleware, videosGetValidator } from '../middlewares'
import { VideoModel } from '../models/video/video'
-import { VideosCaptionCache } from '../lib/cache/videos-caption-cache'
import { UserModel } from '../models/account/user'
import { VideoCommentModel } from '../models/video/video-comment'
import { HttpNodeinfoDiasporaSoftwareNsSchema20 } from '../../shared/models/nodeinfo'
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 7fac8a4d6f..7a3ec38741 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -660,7 +660,7 @@ const EMBED_SIZE = {
}
// Sub folders of cache directory
-const CACHE = {
+const FILES_CACHE = {
PREVIEWS: {
DIRECTORY: join(CONFIG.STORAGE.CACHE_DIR, 'previews'),
MAX_AGE: 1000 * 3600 * 3 // 3 hours
@@ -671,6 +671,12 @@ const CACHE = {
}
}
+const CACHE = {
+ USER_TOKENS: {
+ MAX_SIZE: 10000
+ }
+}
+
const HLS_STREAMING_PLAYLIST_DIRECTORY = join(CONFIG.STORAGE.STREAMING_PLAYLISTS_DIR, 'hls')
const HLS_REDUNDANCY_DIRECTORY = join(CONFIG.STORAGE.REDUNDANCY_DIR, 'hls')
@@ -741,7 +747,7 @@ if (isTestInstance() === true) {
JOB_ATTEMPTS['email'] = 1
- CACHE.VIDEO_CAPTIONS.MAX_AGE = 3000
+ FILES_CACHE.VIDEO_CAPTIONS.MAX_AGE = 3000
MEMOIZE_TTL.OVERVIEWS_SAMPLE = 1
ROUTE_CACHE_LIFETIME.OVERVIEWS.VIDEOS = '0ms'
@@ -759,7 +765,7 @@ export {
ACCEPT_HEADERS,
BCRYPT_SALT_SIZE,
TRACKER_RATE_LIMITS,
- CACHE,
+ FILES_CACHE,
CONFIG,
CONSTRAINTS_FIELDS,
EMBED_SIZE,
@@ -799,6 +805,7 @@ export {
VIDEO_TRANSCODING_FPS,
FFMPEG_NICE,
VIDEO_ABUSE_STATES,
+ CACHE,
JOB_REQUEST_TIMEOUT,
USER_PASSWORD_RESET_LIFETIME,
MEMOIZE_TTL,
diff --git a/server/initializers/installer.ts b/server/initializers/installer.ts
index cd2c942fd1..07af96b689 100644
--- a/server/initializers/installer.ts
+++ b/server/initializers/installer.ts
@@ -6,7 +6,7 @@ import { UserModel } from '../models/account/user'
import { ApplicationModel } from '../models/application/application'
import { OAuthClientModel } from '../models/oauth/oauth-client'
import { applicationExist, clientsExist, usersExist } from './checker-after-init'
-import { CACHE, CONFIG, HLS_STREAMING_PLAYLIST_DIRECTORY, LAST_MIGRATION_VERSION } from './constants'
+import { FILES_CACHE, CONFIG, HLS_STREAMING_PLAYLIST_DIRECTORY, LAST_MIGRATION_VERSION } from './constants'
import { sequelizeTypescript } from './database'
import { remove, ensureDir } from 'fs-extra'
@@ -42,8 +42,8 @@ export {
// ---------------------------------------------------------------------------
function removeCacheAndTmpDirectories () {
- const cacheDirectories = Object.keys(CACHE)
- .map(k => CACHE[k].DIRECTORY)
+ const cacheDirectories = Object.keys(FILES_CACHE)
+ .map(k => FILES_CACHE[k].DIRECTORY)
const tasks: Promise<any>[] = []
@@ -60,8 +60,8 @@ function removeCacheAndTmpDirectories () {
function createDirectoriesIfNotExist () {
const storage = CONFIG.STORAGE
- const cacheDirectories = Object.keys(CACHE)
- .map(k => CACHE[k].DIRECTORY)
+ const cacheDirectories = Object.keys(FILES_CACHE)
+ .map(k => FILES_CACHE[k].DIRECTORY)
const tasks: Promise<void>[] = []
for (const key of Object.keys(storage)) {
diff --git a/server/lib/cache/abstract-video-static-file-cache.ts b/server/lib/files-cache/abstract-video-static-file-cache.ts
similarity index 100%
rename from server/lib/cache/abstract-video-static-file-cache.ts
rename to server/lib/files-cache/abstract-video-static-file-cache.ts
diff --git a/server/lib/cache/actor-follow-score-cache.ts b/server/lib/files-cache/actor-follow-score-cache.ts
similarity index 100%
rename from server/lib/cache/actor-follow-score-cache.ts
rename to server/lib/files-cache/actor-follow-score-cache.ts
diff --git a/server/lib/cache/index.ts b/server/lib/files-cache/index.ts
similarity index 100%
rename from server/lib/cache/index.ts
rename to server/lib/files-cache/index.ts
diff --git a/server/lib/cache/videos-caption-cache.ts b/server/lib/files-cache/videos-caption-cache.ts
similarity index 91%
rename from server/lib/cache/videos-caption-cache.ts
rename to server/lib/files-cache/videos-caption-cache.ts
index f240affbca..fe5b441afc 100644
--- a/server/lib/cache/videos-caption-cache.ts
+++ b/server/lib/files-cache/videos-caption-cache.ts
@@ -1,5 +1,5 @@
import { join } from 'path'
-import { CACHE, CONFIG } from '../../initializers'
+import { FILES_CACHE, CONFIG } from '../../initializers'
import { VideoModel } from '../../models/video/video'
import { VideoCaptionModel } from '../../models/video/video-caption'
import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache'
@@ -42,7 +42,7 @@ class VideosCaptionCache extends AbstractVideoStaticFileCache <GetPathParam> {
if (!video) return undefined
const remoteStaticPath = videoCaption.getCaptionStaticPath()
- const destPath = join(CACHE.VIDEO_CAPTIONS.DIRECTORY, videoCaption.getCaptionName())
+ const destPath = join(FILES_CACHE.VIDEO_CAPTIONS.DIRECTORY, videoCaption.getCaptionName())
return this.saveRemoteVideoFileAndReturnPath(video, remoteStaticPath, destPath)
}
diff --git a/server/lib/cache/videos-preview-cache.ts b/server/lib/files-cache/videos-preview-cache.ts
similarity index 88%
rename from server/lib/cache/videos-preview-cache.ts
rename to server/lib/files-cache/videos-preview-cache.ts
index a5d6f5b627..01cd3647ea 100644
--- a/server/lib/cache/videos-preview-cache.ts
+++ b/server/lib/files-cache/videos-preview-cache.ts
@@ -1,5 +1,5 @@
import { join } from 'path'
-import { CACHE, CONFIG, STATIC_PATHS } from '../../initializers'
+import { FILES_CACHE, CONFIG, STATIC_PATHS } from '../../initializers'
import { VideoModel } from '../../models/video/video'
import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache'
@@ -31,7 +31,7 @@ class VideosPreviewCache extends AbstractVideoStaticFileCache <string> {
if (video.isOwned()) throw new Error('Cannot load remote preview of owned video.')
const remoteStaticPath = join(STATIC_PATHS.PREVIEWS, video.getPreviewName())
- const destPath = join(CACHE.PREVIEWS.DIRECTORY, video.getPreviewName())
+ const destPath = join(FILES_CACHE.PREVIEWS.DIRECTORY, video.getPreviewName())
return this.saveRemoteVideoFileAndReturnPath(video, remoteStaticPath, destPath)
}
diff --git a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts
index 9493945ff6..2b1e21c393 100644
--- a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts
+++ b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts
@@ -2,10 +2,9 @@ import * as Bull from 'bull'
import * as Bluebird from 'bluebird'
import { logger } from '../../../helpers/logger'
import { doRequest } from '../../../helpers/requests'
-import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils'
import { BROADCAST_CONCURRENCY, JOB_REQUEST_TIMEOUT } from '../../../initializers'
-import { ActorFollowScoreCache } from '../../cache'
+import { ActorFollowScoreCache } from '../../files-cache'
export type ActivitypubHttpBroadcastPayload = {
uris: string[]
diff --git a/server/lib/job-queue/handlers/activitypub-http-unicast.ts b/server/lib/job-queue/handlers/activitypub-http-unicast.ts
index 3973dcdc8c..59de7119ad 100644
--- a/server/lib/job-queue/handlers/activitypub-http-unicast.ts
+++ b/server/lib/job-queue/handlers/activitypub-http-unicast.ts
@@ -3,7 +3,7 @@ import { logger } from '../../../helpers/logger'
import { doRequest } from '../../../helpers/requests'
import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils'
import { JOB_REQUEST_TIMEOUT } from '../../../initializers'
-import { ActorFollowScoreCache } from '../../cache'
+import { ActorFollowScoreCache } from '../../files-cache'
export type ActivitypubHttpUnicastPayload = {
uri: string
diff --git a/server/lib/oauth-model.ts b/server/lib/oauth-model.ts
index 2cd2ae97cf..5b4a2bcf9e 100644
--- a/server/lib/oauth-model.ts
+++ b/server/lib/oauth-model.ts
@@ -4,12 +4,12 @@ import { logger } from '../helpers/logger'
import { UserModel } from '../models/account/user'
import { OAuthClientModel } from '../models/oauth/oauth-client'
import { OAuthTokenModel } from '../models/oauth/oauth-token'
-import { CONFIG } from '../initializers/constants'
+import { CONFIG, CACHE } from '../initializers/constants'
import { Transaction } from 'sequelize'
type TokenInfo = { accessToken: string, refreshToken: string, accessTokenExpiresAt: Date, refreshTokenExpiresAt: Date }
-const accessTokenCache: { [ accessToken: string ]: OAuthTokenModel } = {}
-const userHavingToken: { [ userId: number ]: string } = {}
+let accessTokenCache: { [ accessToken: string ]: OAuthTokenModel } = {}
+let userHavingToken: { [ userId: number ]: string } = {}
// ---------------------------------------------------------------------------
@@ -43,6 +43,12 @@ function getAccessToken (bearerToken: string) {
return OAuthTokenModel.getByTokenAndPopulateUser(bearerToken)
.then(tokenModel => {
if (tokenModel) {
+ // Reinit our cache
+ if (Object.keys(accessTokenCache).length > CACHE.USER_TOKENS.MAX_SIZE) {
+ accessTokenCache = {}
+ userHavingToken = {}
+ }
+
accessTokenCache[ bearerToken ] = tokenModel
userHavingToken[ tokenModel.userId ] = tokenModel.accessToken
}
diff --git a/server/lib/schedulers/actor-follow-scheduler.ts b/server/lib/schedulers/actor-follow-scheduler.ts
index 3967be7f8f..05e6bd139e 100644
--- a/server/lib/schedulers/actor-follow-scheduler.ts
+++ b/server/lib/schedulers/actor-follow-scheduler.ts
@@ -3,7 +3,7 @@ import { logger } from '../../helpers/logger'
import { ActorFollowModel } from '../../models/activitypub/actor-follow'
import { AbstractScheduler } from './abstract-scheduler'
import { SCHEDULER_INTERVALS_MS } from '../../initializers'
-import { ActorFollowScoreCache } from '../cache'
+import { ActorFollowScoreCache } from '../files-cache'
export class ActorFollowScheduler extends AbstractScheduler {
--
GitLab