lumither/nocturne
1
0 Fork 0
mirror of https://github.com/Lumither/nocturne.git synced 2026-10-17 10:26:21 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat(blog::refresh) add hash check to boost idx speed

Browse Source
This commit is contained in:
lumither committed 2024-07-21 01:12:15 +08:00
1 parent de27479242
commit 0864f0b950
15 files changed
+426 -199
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

backend/src/api/post/errors/mod.rs
+1
View File
@@ -0,0 +1 @@
pub mod refresh;
backend/src/api/post/errors/refresh.rs
+61
View File
@@ -0,0 +1,61 @@
use thiserror::Error;
#[derive(Error, Debug)]
pub enum PostIdxError {
#[error("Failed to parse UUID <{}>: {}", id, err_msg)]
InvalidUUID { id: String, err_msg: String },
#[error(
"Failed to parse `last update time` for <{}> when creating post index: {}",
id,
err_msg
)]
InvalidLastUpdate { id: String, err_msg: String },
#[error(
"Failed to parse `first update time` for <{}> when creating post index: {}",
id,
err_msg
)]
InvalidFirstUpdate { id: String, err_msg: String },
#[error(
"Failed to parse `tag` for <{}> when creating post index: <{}>: {}",
id,
invalid_string,
err_msg
)]
InvalidTagString {
id: String,
invalid_string: String,
err_msg: String,
},
#[error(
"Database error on writing `{}` to table `{}`: <{}>: {}",
data_desc,
db_table,
id,
err_msg
)]
DBWriteFailure {
data_desc: String,
db_table: String,
id: String,
err_msg: String,
},
#[error(
"Database error on Reading `{}` to table `{}`: <{}>: {}",
data_desc,
db_table,
id,
err_msg
)]
DBReadFailure {
data_desc: String,
db_table: String,
id: String,
err_msg: String,
},
}
backend/src/api/post/mod.rs
+2
View File
@@ -1 +1,3 @@
mod errors;
pub mod refresh;
mod utils;
backend/src/api/post/refresh.rs
+83 -144
View File
@@ -1,53 +1,60 @@
use std::env;
use std::{env, fs};
use std::path::Path;
use std::str::FromStr;
use axum::extract::State;
use axum::http::StatusCode;
use axum::Json;
use chrono::DateTime;
use futures::{stream, StreamExt};
use serde_json::{json, Map, Value};
use sha2::{Digest, Sha256};
use sqlx::{PgPool, query};
use tracing::error;
use uuid::Uuid;
use crate::constants::GLOBAL_TIME_FORMAT;
use crate::model::{post, POST_DB_MODEL, TAG_DB_MODEL};
use crate::api::post::errors::refresh::PostIdxError;
use crate::api::post::utils::{build_post_idx, post};
use crate::model::blog::{POST_BASE_MODEL, POST_HASH_MODEL, POST_META_MODEL, TAG_MODEL};
pub async fn refresh(
State(db_connection): State<PgPool>,
) -> Result<StatusCode, (StatusCode, Json<Value>)> {
// search post
let data_repo_dir = match env::var("GIT_WORK_DIR") {
Ok(value) => value,
Err(_) => {
return Err((
StatusCode::INTERNAL_SERVER_ERROR,
Json::from(json!(format!(
"{{error: {}}}",
Json::from(json!((
"error",
"error at parsing environment variable: GIT_WORK_DIR"
))),
));
}
};
let md_list = search_md(Path::new(&data_repo_dir));
// generate post idx
let md_path = search_md(Path::new(&data_repo_dir));
// dump old idx
query("DROP TABLE IF EXISTS Post;")
.execute(&db_connection)
.await
.unwrap();
query("DROP TABLE IF EXISTS Tag;")
.execute(&db_connection)
.await
.unwrap();
// generate new idx
let post_list: Vec<Map<_, _>> = md_list
let post_list_with_hash: Vec<(Map<_, _>, Vec<_>)> = md_path
.into_iter()
.filter_map(|post_path| match post::from_path(&post_path) {
Ok(post) => Some(post),
Ok(post) => match fs::read(&post_path) {
Ok(bytes) => {
let mut hasher = Sha256::new();
hasher.update(bytes);
let hash = hasher.finalize().to_vec();
Some((post, hash))
}
Err(e) => {
error!(
"Failed to read file while hashing the post <{}>, skipping: {}",
post_path,
e.to_string()
);
None
}
},
Err(e) => {
error!(
"Failed to parse post <{}>, skipping: {}",
@@ -59,138 +66,44 @@ pub async fn refresh(
})
.collect();
query(POST_DB_MODEL).execute(&db_connection).await.unwrap();
query(TAG_DB_MODEL).execute(&db_connection).await.unwrap();
// todo: concurrency
for post in post_list {
let meta = &post["meta"];
let post_id = match Uuid::from_str(post["post_id"].as_str().unwrap()) {
Ok(id) => id,
Err(e) => {
error!(
"Failed to parse UUID <{}>: {}",
post["post_id"].as_str().unwrap(),
e.to_string()
);
continue;
}
};
let title = post["title"].as_str();
let summary = post
.get("summary")
.and_then(|summary| summary.as_str())
.unwrap_or("");
let content = post["content"].as_str();
let last_update = match DateTime::parse_from_str(
post["last_update"].as_str().unwrap(),
GLOBAL_TIME_FORMAT,
) {
Ok(time) => time,
Err(e) => {
error!(
"Failed to parse last update time for <{}> on db table initialization: {}",
post_id.to_string(),
e.to_string()
);
continue;
}
};
let first_update = match DateTime::parse_from_str(
post["first_update"].as_str().unwrap(),
GLOBAL_TIME_FORMAT,
) {
Ok(time) => time,
Err(e) => {
error!(
"Failed to parse first update time for <{}> on db table initialization: {}",
post_id.to_string(),
e.to_string()
);
continue;
}
};
let sub_title = meta
.get("sub_title")
.and_then(|sub_title| sub_title.as_str())
.unwrap_or("");
let category = meta
.get("category")
.and_then(|category| category.as_str())
.unwrap_or("N/A");
let header_img = meta
.get("header_img")
.and_then(|header_img| header_img.as_str())
.unwrap_or("");
// table Post
match query(
r##"
INSERT INTO Post (post_id, title, summary, content, last_update, first_update, sub_title, category, header_img)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
"##,
)
.bind(post_id)
.bind(title)
.bind(summary)
.bind(content)
.bind(last_update)
.bind(first_update)
.bind(sub_title)
.bind(category)
.bind(header_img)
// db create
query(POST_BASE_MODEL)
.execute(&db_connection)
.await
{
Ok(_) => {}
Err(e) => {
error!("Database error on initializing post data (insert):<{}> : {}", post_id.to_string(), e.to_string());
return Err((
StatusCode::INTERNAL_SERVER_ERROR,
Json::from(json!(("error:", e.to_string()))),
));
.unwrap();
query(POST_META_MODEL)
.execute(&db_connection)
.await
.unwrap();
query(TAG_MODEL).execute(&db_connection).await.unwrap();
query(POST_HASH_MODEL)
.execute(&db_connection)
.await
.unwrap();
let stream = stream::iter(post_list_with_hash).for_each_concurrent(None, |(post, hash)| {
let db_clone = db_connection.clone();
async move {
if let Err(e) = build_post_idx(&db_clone, post, hash).await {
error!("{}", e.to_string());
}
}
});
// table Tag
if let Some(tags_string) = meta.get("tags").to_owned() {
// parse tags string
let parsed: Value = serde_json::from_str(tags_string.as_str().unwrap()).unwrap();
let string_array = parsed.as_array().unwrap();
let tags: Vec<String> = string_array
.iter()
.map(|value| value.as_str().unwrap().to_string())
.collect();
for tag in tags {
match query(
r#"
INSERT INTO Tag
VALUES ($1, $2);
"#,
)
.bind(Uuid::from_str(post["post_id"].as_str().unwrap()).unwrap())
.bind(tag)
.execute(&db_connection)
.await
{
Ok(_) => {}
Err(e) => {
return Err((
StatusCode::INTERNAL_SERVER_ERROR,
Json::from(json!(format!("{{error: {}}}", e.to_string()))),
));
}
}
}
}
}
stream.await;
Ok(StatusCode::OK)
}
/// Search a dir, return a list of markdown files
///
/// # Arguments
/// * `entry`: dir path
///
/// # Returns
/// Vec<String, Global>
/// * `String`: path of a markdown file
///
fn search_md(entry: &Path) -> Vec<String> {
if entry.ends_with(".git") {
return vec![];
@@ -212,3 +125,29 @@ fn search_md(entry: &Path) -> Vec<String> {
vec![]
}
}
async fn build_post_idx(
db_connection: &PgPool,
post: Map<String, Value>,
hash: Vec<u8>,
) -> Result<(), PostIdxError> {
let meta = &post["meta"];
// extract post id
let post_id = match Uuid::from_str(post["post_id"].as_str().unwrap()) {
Ok(id) => id,
Err(e) => {
return Err(PostIdxError::InvalidUUID {
id: post["post_id"].as_str().unwrap().to_string(),
err_msg: e.to_string(),
})
}
};
if build_post_idx::hash(db_connection, post_id, hash).await? {
build_post_idx::base(db_connection, &post, post_id, meta).await?;
build_post_idx::tag(db_connection, post_id, meta).await?;
}
Ok(())
}
backend/src/api/post/utils/build_post_idx.rs
+214
View File
@@ -0,0 +1,214 @@
use base64::Engine;
use base64::engine::general_purpose;
use chrono::DateTime;
use serde_json::{Map, Value};
use sqlx::{PgPool, query, Row};
use uuid::Uuid;
use crate::api::post::errors::refresh::PostIdxError;
use crate::constants::GLOBAL_TIME_FORMAT;
pub async fn base(
db_connection: &PgPool,
post: &Map<String, Value>,
post_id: Uuid,
meta: &Value,
) -> Result<(), PostIdxError> {
let title = post["title"].as_str();
let summary = post
.get("summary")
.and_then(|summary| summary.as_str())
.unwrap_or("");
let content = post["content"].as_str();
let last_update =
match DateTime::parse_from_str(post["last_update"].as_str().unwrap(), GLOBAL_TIME_FORMAT) {
Ok(time) => time,
Err(e) => {
return Err(PostIdxError::InvalidLastUpdate {
id: post_id.to_string(),
err_msg: e.to_string(),
});
}
};
let first_update = match DateTime::parse_from_str(
post["first_update"].as_str().unwrap(),
GLOBAL_TIME_FORMAT,
) {
Ok(time) => time,
Err(e) => {
return Err(PostIdxError::InvalidFirstUpdate {
id: post_id.to_string(),
err_msg: e.to_string(),
})
}
};
let sub_title = meta
.get("sub_title")
.and_then(|sub_title| sub_title.as_str())
.unwrap_or("");
let category = meta
.get("category")
.and_then(|category| category.as_str())
.unwrap_or("N/A");
let header_img = meta
.get("header_img")
.and_then(|header_img| header_img.as_str())
.unwrap_or("");
match query(
r##"
INSERT INTO Post (post_id, title, summary, content, last_update, first_update, sub_title, category, header_img)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
ON CONFLICT (post_id) DO
UPDATE SET
title = EXCLUDED.title,
summary = EXCLUDED.summary,
content = EXCLUDED.content,
last_update = EXCLUDED.last_update,
first_update = EXCLUDED.first_update,
sub_title = EXCLUDED.sub_title,
category = EXCLUDED.category,
header_img = EXCLUDED.header_img;
"##,
)
.bind(post_id)
.bind(title)
.bind(summary)
.bind(content)
.bind(last_update)
.bind(first_update)
.bind(sub_title)
.bind(category)
.bind(header_img)
.execute(db_connection)
.await
{
Ok(_) => {}
Err(e) => {
return Err(
PostIdxError::DBWriteFailure {
data_desc: "post base information".to_string(),
db_table: "Post".to_string(),
id: post_id.to_string(),
err_msg: e.to_string()
}
)
}
}
Ok(())
}
pub async fn tag(db_connection: &PgPool, post_id: Uuid, meta: &Value) -> Result<(), PostIdxError> {
if let Some(tags_string) = meta.get("tags").to_owned() {
// parse tags string
let parsed: Value = match serde_json::from_str(tags_string.as_str().unwrap()) {
Ok(value) => value,
Err(e) => {
return Err(PostIdxError::InvalidTagString {
id: post_id.to_string(),
invalid_string: tags_string.to_string().to_string(),
err_msg: e.to_string(),
})
}
};
let string_array = parsed.as_array().unwrap();
let tags: Vec<String> = string_array
.iter()
.map(|value| value.as_str().unwrap().to_string())
.collect();
// remove old record
if let Err(e) = query("DELETE FROM Tag WHERE post_id = $1")
.bind(post_id)
.execute(db_connection)
.await
{
return Err(PostIdxError::DBWriteFailure {
data_desc: "post tag information".to_string(),
db_table: "Tag".to_string(),
id: post_id.to_string(),
err_msg: e.to_string(),
});
}
// add new record
for tag in tags {
match query("INSERT INTO Tag VALUES ($1, $2);")
.bind(post_id)
.bind(tag)
.execute(db_connection)
.await
{
Ok(_) => {}
Err(e) => {
return Err(PostIdxError::DBWriteFailure {
data_desc: "post tag information".to_string(),
db_table: "Tag".to_string(),
id: post_id.to_string(),
err_msg: e.to_string(),
})
}
}
}
}
Ok(())
}
/// # Arguments
///
/// * `db_connection`: database connection
/// * `post_id`: UUID of the post
/// * `hash`: Hex String
///
/// # Returns
/// Result<bool, PostIdxError>
/// - `bool`: if the hash is different from database, return true, else false
pub async fn hash(
db_connection: &PgPool,
post_id: Uuid,
hash: Vec<u8>,
) -> Result<bool, PostIdxError> {
let base64_form = general_purpose::STANDARD.encode(hash);
match query("SELECT hash FROM Hash WHERE post_id = $1")
.bind(post_id)
.fetch_optional(db_connection)
.await
{
Ok(row) => {
let db_record: String = match row {
None => "".to_string(),
Some(row) => row.try_get("hash").unwrap_or("".to_string()),
};
if base64_form == db_record {
Ok(false)
} else {
if let Err(e) = query(
r#"
INSERT INTO Hash (post_id, hash) VALUES ($1, $2)
ON CONFLICT (post_id) DO UPDATE SET hash = EXCLUDED.hash"#,
)
.bind(post_id)
.bind(&base64_form)
.execute(db_connection)
.await
{
return Err(PostIdxError::DBWriteFailure {
data_desc: "post hash".to_string(),
db_table: "Hash".to_string(),
id: post_id.to_string(),
err_msg: e.to_string(),
});
}
Ok(true)
}
}
Err(e) => Err(PostIdxError::DBReadFailure {
data_desc: "post hash".to_string(),
db_table: "Hash".to_string(),
id: post_id.to_string(),
err_msg: e.to_string(),
}),
}
}
backend/src/api/post/utils/mod.rs
+2
View File
@@ -0,0 +1,2 @@
pub mod build_post_idx;
pub mod post;
backend/src/model/post.rs → backend/src/api/post/utils/post.rs
View File
File renamed without changes.
backend/src/model/blog/hash.sql
+5
View File
@@ -0,0 +1,5 @@
CREATE TABLE IF NOT EXISTS Hash
(
post_id UUID PRIMARY KEY,
hash VARCHAR(44)
);
backend/src/model/blog/meta.sql
+4
View File
@@ -0,0 +1,4 @@
CREATE TABLE IF NOT EXISTS Meta
(
post_id UUID PRIMARY KEY
);
backend/src/model/blog/mod.rs
+38
View File
@@ -0,0 +1,38 @@
pub const POST_BASE_MODEL: &str = r##"
CREATE TABLE IF NOT EXISTS Post
(
post_id UUID PRIMARY KEY,
title VARCHAR(255) NOT NULL,
category VARCHAR(255) NOT NULL,
header_img TEXT,
summary TEXT,
sub_title TEXT,
content TEXT NOT NULL,
last_update TIMESTAMPTZ NOT NULL,
first_update TIMESTAMPTZ NOT NULL
);
"##;
// base mode, will add more cols in runtime
pub const POST_META_MODEL: &str = r##"
CREATE TABLE IF NOT EXISTS Meta
(
post_id UUID PRIMARY KEY
);
"##;
pub const POST_HASH_MODEL: &str = r##"
CREATE TABLE IF NOT EXISTS Hash
(
post_id UUID PRIMARY KEY,
hash VARCHAR(44)
);
"##;
pub const TAG_MODEL: &str = r##"
CREATE TABLE IF NOT EXISTS Tag
(
post_id UUID NOT NULL,
tag VARCHAR(255) NOT NULL
);
"##;
backend/src/model/blog/post.sql
+12
View File
@@ -0,0 +1,12 @@
CREATE TABLE IF NOT EXISTS Post
(
post_id UUID PRIMARY KEY,
title VARCHAR(255) NOT NULL,
category VARCHAR(255) NOT NULL,
header_img TEXT,
summary TEXT,
sub_title TEXT,
content TEXT NOT NULL,
last_update TIMESTAMPTZ NOT NULL,
first_update TIMESTAMPTZ NOT NULL
);
backend/src/model/blog/tag.sql
+5
View File
@@ -0,0 +1,5 @@
CREATE TABLE IF NOT EXISTS Tag
(
post_id UUID NOT NULL,
tag VARCHAR(255) NOT NULL
);
backend/src/model/mod.rs
+1 -24
View File
@@ -1,24 +1 @@
pub mod post;
pub const POST_DB_MODEL: &str = r##"
CREATE TABLE IF NOT EXISTS Post
(
post_id UUID PRIMARY KEY,
title VARCHAR(255) NOT NULL,
category VARCHAR(255) NOT NULL,
header_img TEXT,
summary TEXT,
sub_title TEXT,
content TEXT NOT NULL,
last_update TIMESTAMPTZ NOT NULL,
first_update TIMESTAMPTZ NOT NULL
);
"##;
pub const TAG_DB_MODEL: &str = r##"
CREATE TABLE IF NOT EXISTS Tag
(
post_id UUID NOT NULL,
tag VARCHAR(255) NOT NULL
);
"##;
pub mod blog;