diff --git a/containers/rust-dev.Dockerfile b/containers/rust-dev.Dockerfile new file mode 100644 index 0000000..ca1ac07 --- /dev/null +++ b/containers/rust-dev.Dockerfile @@ -0,0 +1,8 @@ +FROM rust + +RUN cargo install cargo-watch && \ + mkdir -p /opt/code && \ + touch /opt/code/dev-entry.sh && \ + chmod +x /opt/code/dev-entry.sh + +CMD [ "/opt/code/dev-entry.sh" ] diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 093e0d6..ce00844 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -2,14 +2,23 @@ version: '3.7' services: filed: build: - context: filed - dockerfile: Dockerfile.dev + context: containers + dockerfile: rust-dev.Dockerfile networks: bfile: volumes: - './filed:/opt/code' - '/opt/code/target' - './volatile/files:/opt/user_uploads' + janitord: + build: + context: containers + dockerfile: rust-dev.Dockerfile + networks: + bfile: + volumes: + - './janitor:/opt/code' + - './volatile/files:/opt/user_uploads' caddy: image: caddy:alpine volumes: diff --git a/filed/Dockerfile.dev b/filed/Dockerfile.dev deleted file mode 100644 index 995c30c..0000000 --- a/filed/Dockerfile.dev +++ /dev/null @@ -1,17 +0,0 @@ -# --- build --- -FROM rust as builder - -WORKDIR /opt/code -COPY . . - -# No build is done during this step -# since the directory will be mounted anyways -# to the dev's machine. - -# Therefore installing & compiling in the dockerfile -# would be moronic, to say at least - -# However, cargo watch needs to be installed -RUN cargo install cargo-watch - -CMD [ "/opt/code/dev-entry.sh" ] diff --git a/janitor/.gitignore b/janitor/.gitignore index 5ffa2a4..70e4a54 100644 --- a/janitor/.gitignore +++ b/janitor/.gitignore @@ -1,3 +1,4 @@ .env .DS_Store target +Dockerfile \ No newline at end of file diff --git a/janitor/Cargo.lock b/janitor/Cargo.lock index 369d25c..ed06e23 100644 --- a/janitor/Cargo.lock +++ b/janitor/Cargo.lock @@ -26,6 +26,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "autocfg" version = "1.1.0" @@ -80,6 +95,21 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-targets", +] + [[package]] name = "combine" version = "4.6.6" @@ -90,6 +120,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "core-foundation-sys" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" + [[package]] name = "dotenvy" version = "0.15.7" @@ -142,6 +178,29 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" +[[package]] +name = "iana-time-zone" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "idna" version = "0.4.0" @@ -162,11 +221,14 @@ checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" name = "janitor" version = "0.1.0" dependencies = [ + "chrono", "dotenvy", "femme", "log", "parse_duration", "redis", + "serde", + "serde_json", "tokio", ] @@ -842,6 +904,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.48.0" diff --git a/janitor/Cargo.toml b/janitor/Cargo.toml index fc521cc..47151c6 100644 --- a/janitor/Cargo.toml +++ b/janitor/Cargo.toml @@ -6,9 +6,12 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +chrono = { version = "0.4.31", features = ["serde"] } dotenvy = "0.15.7" femme = "2.2.1" log = "0.4.20" parse_duration = "2.1.1" redis = { version = "0.23.3", features = ["tokio"] } +serde = { version = "1.0.188", features = ["derive"] } +serde_json = "1.0.107" tokio = { version = "1.33.0", features = ["full"] } diff --git a/janitor/Dockerfile.prod b/janitor/Dockerfile.prod new file mode 100644 index 0000000..8239767 --- /dev/null +++ b/janitor/Dockerfile.prod @@ -0,0 +1,16 @@ +# --- build --- +FROM rust:alpine as builder + +WORKDIR /opt/build +COPY . . + +RUN apk add --no-cache musl-dev upx + +RUN cargo b -r +RUN strip target/release/filed && upx --best target/release/filed + +# --- deploy --- +FROM busybox:musl + +COPY --from=builder /opt/build/target/release/filed /bin/filed +CMD [ "/bin/filed" ] diff --git a/janitor/dev-entry.sh b/janitor/dev-entry.sh new file mode 100755 index 0000000..dbb8503 --- /dev/null +++ b/janitor/dev-entry.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +cd /opt/code + +cargo check +cargo build + +cargo watch -w src -x run \ No newline at end of file diff --git a/janitor/src/clean.rs b/janitor/src/clean.rs index e657501..d84bee6 100644 --- a/janitor/src/clean.rs +++ b/janitor/src/clean.rs @@ -1,7 +1,66 @@ -use std::error::Error; -use crate::state::State; +use redis::Commands; +use tokio::task::JoinSet; +use std::{error::Error, path::Path}; + +use crate::{state::State, file::File}; + +async fn check_key(key: String, mut client: redis::Client) -> bool { + + #[cfg(debug_assertions)] + log::debug!("Checking object {}", key); + + let val: String = client.get(key.clone()).unwrap(); + let file: File = serde_json::from_str(val.as_str()).unwrap(); + + if ! Path::new(&file.path.clone()).exists() { + #[cfg(debug_assertions)] { + log::debug!("Object {key} is marked for deletion because it doesn't exist in the filesystem"); + } + client.del::(key).unwrap(); + return true; + } + + let stat = tokio::fs::metadata(file.clone().path).await.unwrap(); + if ! stat.is_file() { + client.del::(key).unwrap(); + return true; + } + + false +} pub async fn clean(state: State) -> Result<(), Box> { + + let mut redis = state.redis.clone(); + let keys: Vec = redis.keys(format!("{}*", state.env.redis.prefix))?; + let objects = keys.len(); + #[cfg(debug_assertions)] + log::debug!("Got {} objects", objects); + + let mut set: JoinSet = JoinSet::new(); + for key in keys { + set.spawn(check_key(key, redis.clone())); + } + + #[cfg(debug_assertions)] + let mut del_count: u32 = 0; + + while let Some(_deleted) = set.join_next().await { + + #[cfg(debug_assertions)] { + if _deleted.is_ok() { + if _deleted.unwrap() { + del_count += 1; + } + } + } + + } + + #[cfg(debug_assertions)] + log::debug!("Deleted {} objects", del_count); + + Ok(()) } \ No newline at end of file diff --git a/janitor/src/env.rs b/janitor/src/env.rs index 2e2401c..89d3b0d 100644 --- a/janitor/src/env.rs +++ b/janitor/src/env.rs @@ -1,5 +1,5 @@ -use std::{error::Error, env::var, time::Duration}; +use std::{error::Error, env::var, time::Duration, path::Path}; #[derive(Debug, Clone)] pub struct RedisEnv { @@ -13,7 +13,8 @@ pub struct RedisEnv { pub struct Env { pub redis: RedisEnv, pub clean_del: Duration, - pub clean_errdel: Duration + pub clean_errdel: Duration, + pub usercont_dir: String } impl Env { @@ -24,10 +25,18 @@ impl Env { pass: var("REDIS_PASS")?.to_string(), host: var("REDIS_HOST")?.to_string(), port: var("REDIS_PORT")?.parse()?, - prefix: var("REDIS_PASS")?.to_string() + prefix: var("REDIS_PREFIX")?.to_string() }, clean_del: parse_duration::parse(var("CLEAN_DEL")?.as_str())?, - clean_errdel: parse_duration::parse(var("CLEAN_ERRDEL")?.as_str())? + clean_errdel: parse_duration::parse(var("CLEAN_ERRDEL")?.as_str())?, + usercont_dir: { + let dir = var("USERCONTENT_DIR")?; + let dir = dir.as_str(); + if ! Path::new(dir).is_dir() { + return Err("Path specified in USERCONTENT_DIR is not a directory!".into()); + } + dir.to_string() + } } ) } diff --git a/janitor/src/file.rs b/janitor/src/file.rs new file mode 100644 index 0000000..f036b29 --- /dev/null +++ b/janitor/src/file.rs @@ -0,0 +1,19 @@ + +use chrono::{DateTime, Local}; +use serde::{Serialize, Deserialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct File { + pub path: String, + pub size: usize, + pub name: Option, + pub mime: String, + pub delete_at: DateTime, + sha512: String +} + +impl File { + pub fn expired(self: &Self) -> bool { + self.delete_at > chrono::Local::now() + } +} \ No newline at end of file diff --git a/janitor/src/main.rs b/janitor/src/main.rs index 2e6f75f..41f6fab 100644 --- a/janitor/src/main.rs +++ b/janitor/src/main.rs @@ -2,6 +2,7 @@ mod clean; mod state; mod env; +mod file; pub fn redis_conn(env: env::Env) -> Result { log::info!("Connecting to redis DB on {}", env.redis.host); @@ -10,17 +11,40 @@ pub fn redis_conn(env: env::Env) -> Result { #[tokio::main] async fn main() { + #[cfg(debug_assertions)] { + femme::with_level(log::LevelFilter::Debug); + } + #[cfg(not(debug_assertions))] { + femme::with_level(log::LevelFilter::Info); + } + dotenvy::dotenv().unwrap(); let env = crate::env::Env::load().unwrap(); let statee = crate::state::State { - redis: redis_conn(env).unwrap() + redis: redis_conn(env.clone()).unwrap(), + env: env.clone() }; loop { - let res = clean::clean(statee).await; + + #[cfg(debug_assertions)] + log::debug!("Initiating clean process"); + + let envy = env.clone(); + let res = clean::clean(statee.clone()).await; if res.is_err() { - log::error!("Error while cleaning") + log::error!("Error while cleaning: {}", res.unwrap_err()); + log::error!("Retrying in {}", std::env::var("CLEAN_ERRDEL").unwrap()); + tokio::time::sleep(envy.clean_errdel).await; + continue; } + + #[cfg(debug_assertions)] { + log::debug!("Cleaned successfully"); + log::debug!("Next clean is scheduled in {}", std::env::var("CLEAN_DEL").unwrap()) + } + + tokio::time::sleep(envy.clean_errdel).await; } } diff --git a/janitor/src/state.rs b/janitor/src/state.rs index 55e087f..1e02f25 100644 --- a/janitor/src/state.rs +++ b/janitor/src/state.rs @@ -1,5 +1,9 @@ use redis::Client; +use crate::env::Env; + +#[derive(Debug, Clone)] pub struct State { - pub redis: Client + pub redis: Client, + pub env: Env } \ No newline at end of file