get janitord running in docker

containers/rust-dev.Dockerfile

@@ -0,0 +1,8 @@
+FROM rust
+
+RUN cargo install cargo-watch && \
+    mkdir -p /opt/code && \
+    touch /opt/code/dev-entry.sh && \
+    chmod +x /opt/code/dev-entry.sh
+
+CMD [ "/opt/code/dev-entry.sh" ]

docker-compose.dev.yml

@@ -2,14 +2,23 @@ version: '3.7'
 services:
     filed:
         build:
-            context: filed
-            dockerfile: Dockerfile.dev
+            context: containers
+            dockerfile: rust-dev.Dockerfile
         networks:
             bfile:
         volumes:
             - './filed:/opt/code'
             - '/opt/code/target'
             - './volatile/files:/opt/user_uploads'
+    janitord:
+        build:
+            context: containers
+            dockerfile: rust-dev.Dockerfile
+        networks:
+            bfile:
+        volumes:
+            - './janitor:/opt/code'
+            - './volatile/files:/opt/user_uploads'
     caddy:
         image: caddy:alpine
         volumes:

filed/Dockerfile.dev

@@ -1,17 +0,0 @@
-# --- build ---
-FROM rust as builder
-
-WORKDIR /opt/code
-COPY . .
-
-# No build is done during this step
-# since the directory will be mounted anyways
-# to the dev's machine.
-
-# Therefore installing & compiling in the dockerfile
-# would be moronic, to say at least
-
-# However, cargo watch needs to be installed
-RUN cargo install cargo-watch
-
-CMD [ "/opt/code/dev-entry.sh" ]

janitor/.gitignore

@@ -1,3 +1,4 @@
 .env
 .DS_Store
 target
+Dockerfile

janitor/Cargo.lock

@@ -26,6 +26,21 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "android-tzdata"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0"
+
+[[package]]
+name = "android_system_properties"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "autocfg"
 version = "1.1.0"
@@ -80,6 +95,21 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
+[[package]]
+name = "chrono"
+version = "0.4.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38"
+dependencies = [
+ "android-tzdata",
+ "iana-time-zone",
+ "js-sys",
+ "num-traits",
+ "serde",
+ "wasm-bindgen",
+ "windows-targets",
+]
+
 [[package]]
 name = "combine"
 version = "4.6.6"
@@ -90,6 +120,12 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "core-foundation-sys"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa"
+
 [[package]]
 name = "dotenvy"
 version = "0.15.7"
@@ -142,6 +178,29 @@ version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7"
 
+[[package]]
+name = "iana-time-zone"
+version = "0.1.57"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613"
+dependencies = [
+ "android_system_properties",
+ "core-foundation-sys",
+ "iana-time-zone-haiku",
+ "js-sys",
+ "wasm-bindgen",
+ "windows",
+]
+
+[[package]]
+name = "iana-time-zone-haiku"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
+dependencies = [
+ "cc",
+]
+
 [[package]]
 name = "idna"
 version = "0.4.0"
@@ -162,11 +221,14 @@ checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38"
 name = "janitor"
 version = "0.1.0"
 dependencies = [
+ "chrono",
  "dotenvy",
  "femme",
  "log",
  "parse_duration",
  "redis",
+ "serde",
+ "serde_json",
  "tokio",
 ]
 
@@ -842,6 +904,15 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
+[[package]]
+name = "windows"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f"
+dependencies = [
+ "windows-targets",
+]
+
 [[package]]
 name = "windows-sys"
 version = "0.48.0"

janitor/Cargo.toml

@@ -6,9 +6,12 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+chrono = { version = "0.4.31", features = ["serde"] }
 dotenvy = "0.15.7"
 femme = "2.2.1"
 log = "0.4.20"
 parse_duration = "2.1.1"
 redis = { version = "0.23.3", features = ["tokio"] }
+serde = { version = "1.0.188", features = ["derive"] }
+serde_json = "1.0.107"
 tokio = { version = "1.33.0", features = ["full"] }

janitor/Dockerfile.prod

@@ -0,0 +1,16 @@
+# --- build ---
+FROM rust:alpine as builder
+
+WORKDIR /opt/build
+COPY . .
+
+RUN apk add --no-cache musl-dev upx
+
+RUN cargo b -r
+RUN strip target/release/filed && upx --best target/release/filed
+
+# --- deploy ---
+FROM busybox:musl
+
+COPY --from=builder /opt/build/target/release/filed /bin/filed
+CMD [ "/bin/filed" ]

janitor/dev-entry.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+cd /opt/code
+
+cargo check
+cargo build
+
+cargo watch -w src -x run

janitor/src/clean.rs

@@ -1,7 +1,66 @@
-use std::error::Error;
 
-use crate::state::State;
+use redis::Commands;
+use tokio::task::JoinSet;
+use std::{error::Error, path::Path};
+
+use crate::{state::State, file::File};
+
+async fn check_key(key: String, mut client: redis::Client) -> bool {
+
+    #[cfg(debug_assertions)]
+    log::debug!("Checking object {}", key);
+
+    let val: String = client.get(key.clone()).unwrap();
+    let file: File = serde_json::from_str(val.as_str()).unwrap();
+
+    if ! Path::new(&file.path.clone()).exists() {
+        #[cfg(debug_assertions)] {
+            log::debug!("Object {key} is marked for deletion because it doesn't exist in the filesystem");
+        }
+        client.del::<String, ()>(key).unwrap();
+        return true;
+    }
+
+    let stat = tokio::fs::metadata(file.clone().path).await.unwrap();
+    if ! stat.is_file() {
+        client.del::<String, ()>(key).unwrap();
+        return true;
+    }
+
+    false
+}
 
 pub async fn clean(state: State) -> Result<(), Box<dyn Error>> {
 
+    let mut redis = state.redis.clone();
+    let keys: Vec<String> = redis.keys(format!("{}*", state.env.redis.prefix))?;
+    let objects = keys.len();
+    
+    #[cfg(debug_assertions)]
+    log::debug!("Got {} objects", objects);
+    
+    let mut set: JoinSet<bool> = JoinSet::new();
+    for key in keys {
+        set.spawn(check_key(key, redis.clone()));
+    }
+
+    #[cfg(debug_assertions)]
+    let mut del_count: u32 = 0;
+
+    while let Some(_deleted) = set.join_next().await {
+
+        #[cfg(debug_assertions)] {
+            if _deleted.is_ok() {
+                if _deleted.unwrap() {
+                    del_count += 1;
+                }
+            }
+        }
+
+    }
+
+    #[cfg(debug_assertions)]
+    log::debug!("Deleted {} objects", del_count);
+
+    Ok(())
 }

janitor/src/env.rs

@@ -1,5 +1,5 @@
 
-use std::{error::Error, env::var, time::Duration};
+use std::{error::Error, env::var, time::Duration, path::Path};
 
 #[derive(Debug, Clone)]
 pub struct RedisEnv {
@@ -13,7 +13,8 @@ pub struct RedisEnv {
 pub struct Env {
     pub redis: RedisEnv,
     pub clean_del: Duration,
-    pub clean_errdel: Duration
+    pub clean_errdel: Duration,
+    pub usercont_dir: String
 }
 
 impl Env {
@@ -24,10 +25,18 @@ impl Env {
                     pass: var("REDIS_PASS")?.to_string(),
                     host: var("REDIS_HOST")?.to_string(),
                     port: var("REDIS_PORT")?.parse()?,
-                    prefix: var("REDIS_PASS")?.to_string()
+                    prefix: var("REDIS_PREFIX")?.to_string()
                 },
                 clean_del: parse_duration::parse(var("CLEAN_DEL")?.as_str())?,
-                clean_errdel: parse_duration::parse(var("CLEAN_ERRDEL")?.as_str())?
+                clean_errdel: parse_duration::parse(var("CLEAN_ERRDEL")?.as_str())?,
+                usercont_dir: {
+                    let dir = var("USERCONTENT_DIR")?;
+                    let dir = dir.as_str();
+                    if ! Path::new(dir).is_dir() {
+                        return Err("Path specified in USERCONTENT_DIR is not a directory!".into());
+                    }
+                    dir.to_string()
+                }
             }
         )
     }

janitor/src/file.rs

@@ -0,0 +1,19 @@
+
+use chrono::{DateTime, Local};
+use serde::{Serialize, Deserialize};
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct File {
+    pub path: String,
+    pub size: usize,
+    pub name: Option<String>,
+    pub mime: String,
+    pub delete_at: DateTime<Local>,
+    sha512: String
+}
+
+impl File {
+    pub fn expired(self: &Self) -> bool {
+        self.delete_at > chrono::Local::now()
+    }
+}

janitor/src/main.rs

@@ -2,6 +2,7 @@
 mod clean;
 mod state;
 mod env;
+mod file;
 
 pub fn redis_conn(env: env::Env) -> Result<redis::Client, redis::RedisError> {
     log::info!("Connecting to redis DB on {}", env.redis.host);
@@ -10,17 +11,40 @@ pub fn redis_conn(env: env::Env) -> Result<redis::Client, redis::RedisError> {
 
 #[tokio::main]
 async fn main() {
+    #[cfg(debug_assertions)] {
+        femme::with_level(log::LevelFilter::Debug);
+    }
+    #[cfg(not(debug_assertions))] {
+        femme::with_level(log::LevelFilter::Info);
+    }
+
     dotenvy::dotenv().unwrap();
 
     let env = crate::env::Env::load().unwrap();
     let statee = crate::state::State {
-        redis: redis_conn(env).unwrap()
+        redis: redis_conn(env.clone()).unwrap(),
+        env: env.clone()
     };
 
     loop {
-        let res = clean::clean(statee).await;
+
+        #[cfg(debug_assertions)]
+        log::debug!("Initiating clean process");
+
+        let envy = env.clone();
+        let res = clean::clean(statee.clone()).await;
         if res.is_err() {
-            log::error!("Error while cleaning")
-        }
+            log::error!("Error while cleaning: {}", res.unwrap_err());
+            log::error!("Retrying in {}", std::env::var("CLEAN_ERRDEL").unwrap());
+            tokio::time::sleep(envy.clean_errdel).await;
+            continue;
+        }
+    
+        #[cfg(debug_assertions)] {
+            log::debug!("Cleaned successfully");
+            log::debug!("Next clean is scheduled in {}", std::env::var("CLEAN_DEL").unwrap())
+        }
+
+        tokio::time::sleep(envy.clean_errdel).await;
     }
 }

janitor/src/state.rs

@@ -1,5 +1,9 @@
 use redis::Client;
 
+use crate::env::Env;
+
+#[derive(Debug, Clone)]
 pub struct State {
-    pub redis: Client
+    pub redis: Client,
+    pub env: Env
 }