From 2d4037ba619ad923c00261b9cd05597c9c836569 Mon Sep 17 00:00:00 2001
From: Kroese <kroese@users.noreply.github.com>
Date: Mon, 11 Dec 2023 11:24:51 +0100
Subject: [PATCH] Extend sitemap span (#4231)
* Extend sitemap span
* Keep cargo_fmt happy
* Add FETCH_LIMIT_SITEMAP
* Use FETCH_LIMIT_SITEMAP
* Keep cargo_fmt happy
* Update utils.rs
* Use SITEMAP_DAYS
* Keep cargo_fmt happy
* Sitemap
* Keep cargo_fmt happy
* Sitemap
* Sitemap
* Increase to 31 days
---
crates/api/src/sitemap.rs | 4 ++--
crates/db_schema/src/impls/post.rs | 13 +++++++++++--
crates/db_schema/src/utils.rs | 2 ++
3 files changed, 15 insertions(+), 4 deletions(-)
diff --git a/crates/api/src/sitemap.rs b/crates/api/src/sitemap.rs
index 438a8b8e8..b1b961350 100644
--- a/crates/api/src/sitemap.rs
+++ b/crates/api/src/sitemap.rs
@@ -26,7 +26,7 @@ async fn generate_urlset(
}
pub async fn get_sitemap(context: Data<LemmyContext>) -> LemmyResult<HttpResponse> {
- info!("Generating sitemap with posts from last {} hours...", 24);
+ info!("Generating sitemap...",);
let posts = Post::list_for_sitemap(&mut context.pool()).await?;
info!("Loaded latest {} posts", posts.len());
@@ -36,7 +36,7 @@ pub async fn get_sitemap(context: Data<LemmyContext>) -> LemmyResult<HttpRespons
Ok(
HttpResponse::Ok()
.content_type("application/xml")
- .insert_header(header::CacheControl(vec![CacheDirective::MaxAge(86_400)])) // 24 h
+ .insert_header(header::CacheControl(vec![CacheDirective::MaxAge(3_600)])) // 1 h
.body(buf),
)
}
diff --git a/crates/db_schema/src/impls/post.rs b/crates/db_schema/src/impls/post.rs
index f7ff633ec..d65fd2398 100644
--- a/crates/db_schema/src/impls/post.rs
+++ b/crates/db_schema/src/impls/post.rs
@@ -29,7 +29,15 @@ use crate::{
PostUpdateForm,
},
traits::{Crud, Likeable, Saveable},
- utils::{get_conn, naive_now, DbPool, DELETED_REPLACEMENT_TEXT, FETCH_LIMIT_MAX},
+ utils::{
+ get_conn,
+ naive_now,
+ DbPool,
+ DELETED_REPLACEMENT_TEXT,
+ FETCH_LIMIT_MAX,
+ SITEMAP_DAYS,
+ SITEMAP_LIMIT,
+ },
};
use ::url::Url;
use chrono::{Duration, Utc};
@@ -109,8 +117,9 @@ impl Post {
.filter(local.eq(true))
.filter(deleted.eq(false))
.filter(removed.eq(false))
- .filter(published.ge(Utc::now().naive_utc() - Duration::days(1)))
+ .filter(published.ge(Utc::now().naive_utc() - Duration::days(SITEMAP_DAYS)))
.order(published.desc())
+ .limit(SITEMAP_LIMIT)
.load::<(DbUrl, chrono::DateTime<Utc>)>(conn)
.await
}
diff --git a/crates/db_schema/src/utils.rs b/crates/db_schema/src/utils.rs
index 9ac2d4c88..2b1179bee 100644
--- a/crates/db_schema/src/utils.rs
+++ b/crates/db_schema/src/utils.rs
@@ -49,6 +49,8 @@ use url::Url;
const FETCH_LIMIT_DEFAULT: i64 = 10;
pub const FETCH_LIMIT_MAX: i64 = 50;
+pub const SITEMAP_LIMIT: i64 = 50000;
+pub const SITEMAP_DAYS: i64 = 31;
const POOL_TIMEOUT: Option<Duration> = Some(Duration::from_secs(5));
pub const RANK_DEFAULT: f64 = 0.0001;
--
GitLab