get janitord running in docker
This commit is contained in:
parent
95bc1fc47c
commit
32d5666477
|
@ -0,0 +1,8 @@
|
||||||
|
FROM rust
|
||||||
|
|
||||||
|
RUN cargo install cargo-watch && \
|
||||||
|
mkdir -p /opt/code && \
|
||||||
|
touch /opt/code/dev-entry.sh && \
|
||||||
|
chmod +x /opt/code/dev-entry.sh
|
||||||
|
|
||||||
|
CMD [ "/opt/code/dev-entry.sh" ]
|
|
@ -2,14 +2,23 @@ version: '3.7'
|
||||||
services:
|
services:
|
||||||
filed:
|
filed:
|
||||||
build:
|
build:
|
||||||
context: filed
|
context: containers
|
||||||
dockerfile: Dockerfile.dev
|
dockerfile: rust-dev.Dockerfile
|
||||||
networks:
|
networks:
|
||||||
bfile:
|
bfile:
|
||||||
volumes:
|
volumes:
|
||||||
- './filed:/opt/code'
|
- './filed:/opt/code'
|
||||||
- '/opt/code/target'
|
- '/opt/code/target'
|
||||||
- './volatile/files:/opt/user_uploads'
|
- './volatile/files:/opt/user_uploads'
|
||||||
|
janitord:
|
||||||
|
build:
|
||||||
|
context: containers
|
||||||
|
dockerfile: rust-dev.Dockerfile
|
||||||
|
networks:
|
||||||
|
bfile:
|
||||||
|
volumes:
|
||||||
|
- './janitor:/opt/code'
|
||||||
|
- './volatile/files:/opt/user_uploads'
|
||||||
caddy:
|
caddy:
|
||||||
image: caddy:alpine
|
image: caddy:alpine
|
||||||
volumes:
|
volumes:
|
||||||
|
|
|
@ -1,17 +0,0 @@
|
||||||
# --- build ---
|
|
||||||
FROM rust as builder
|
|
||||||
|
|
||||||
WORKDIR /opt/code
|
|
||||||
COPY . .
|
|
||||||
|
|
||||||
# No build is done during this step
|
|
||||||
# since the directory will be mounted anyways
|
|
||||||
# to the dev's machine.
|
|
||||||
|
|
||||||
# Therefore installing & compiling in the dockerfile
|
|
||||||
# would be moronic, to say at least
|
|
||||||
|
|
||||||
# However, cargo watch needs to be installed
|
|
||||||
RUN cargo install cargo-watch
|
|
||||||
|
|
||||||
CMD [ "/opt/code/dev-entry.sh" ]
|
|
|
@ -1,3 +1,4 @@
|
||||||
.env
|
.env
|
||||||
.DS_Store
|
.DS_Store
|
||||||
target
|
target
|
||||||
|
Dockerfile
|
|
@ -26,6 +26,21 @@ dependencies = [
|
||||||
"memchr",
|
"memchr",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "android-tzdata"
|
||||||
|
version = "0.1.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "android_system_properties"
|
||||||
|
version = "0.1.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
|
||||||
|
dependencies = [
|
||||||
|
"libc",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "autocfg"
|
name = "autocfg"
|
||||||
version = "1.1.0"
|
version = "1.1.0"
|
||||||
|
@ -80,6 +95,21 @@ version = "1.0.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
|
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "chrono"
|
||||||
|
version = "0.4.31"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38"
|
||||||
|
dependencies = [
|
||||||
|
"android-tzdata",
|
||||||
|
"iana-time-zone",
|
||||||
|
"js-sys",
|
||||||
|
"num-traits",
|
||||||
|
"serde",
|
||||||
|
"wasm-bindgen",
|
||||||
|
"windows-targets",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "combine"
|
name = "combine"
|
||||||
version = "4.6.6"
|
version = "4.6.6"
|
||||||
|
@ -90,6 +120,12 @@ dependencies = [
|
||||||
"memchr",
|
"memchr",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "core-foundation-sys"
|
||||||
|
version = "0.8.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "dotenvy"
|
name = "dotenvy"
|
||||||
version = "0.15.7"
|
version = "0.15.7"
|
||||||
|
@ -142,6 +178,29 @@ version = "0.3.3"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7"
|
checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "iana-time-zone"
|
||||||
|
version = "0.1.57"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613"
|
||||||
|
dependencies = [
|
||||||
|
"android_system_properties",
|
||||||
|
"core-foundation-sys",
|
||||||
|
"iana-time-zone-haiku",
|
||||||
|
"js-sys",
|
||||||
|
"wasm-bindgen",
|
||||||
|
"windows",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "iana-time-zone-haiku"
|
||||||
|
version = "0.1.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
|
||||||
|
dependencies = [
|
||||||
|
"cc",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "idna"
|
name = "idna"
|
||||||
version = "0.4.0"
|
version = "0.4.0"
|
||||||
|
@ -162,11 +221,14 @@ checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38"
|
||||||
name = "janitor"
|
name = "janitor"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"chrono",
|
||||||
"dotenvy",
|
"dotenvy",
|
||||||
"femme",
|
"femme",
|
||||||
"log",
|
"log",
|
||||||
"parse_duration",
|
"parse_duration",
|
||||||
"redis",
|
"redis",
|
||||||
|
"serde",
|
||||||
|
"serde_json",
|
||||||
"tokio",
|
"tokio",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
@ -842,6 +904,15 @@ version = "0.4.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows"
|
||||||
|
version = "0.48.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f"
|
||||||
|
dependencies = [
|
||||||
|
"windows-targets",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "windows-sys"
|
name = "windows-sys"
|
||||||
version = "0.48.0"
|
version = "0.48.0"
|
||||||
|
|
|
@ -6,9 +6,12 @@ edition = "2021"
|
||||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
|
chrono = { version = "0.4.31", features = ["serde"] }
|
||||||
dotenvy = "0.15.7"
|
dotenvy = "0.15.7"
|
||||||
femme = "2.2.1"
|
femme = "2.2.1"
|
||||||
log = "0.4.20"
|
log = "0.4.20"
|
||||||
parse_duration = "2.1.1"
|
parse_duration = "2.1.1"
|
||||||
redis = { version = "0.23.3", features = ["tokio"] }
|
redis = { version = "0.23.3", features = ["tokio"] }
|
||||||
|
serde = { version = "1.0.188", features = ["derive"] }
|
||||||
|
serde_json = "1.0.107"
|
||||||
tokio = { version = "1.33.0", features = ["full"] }
|
tokio = { version = "1.33.0", features = ["full"] }
|
||||||
|
|
|
@ -0,0 +1,16 @@
|
||||||
|
# --- build ---
|
||||||
|
FROM rust:alpine as builder
|
||||||
|
|
||||||
|
WORKDIR /opt/build
|
||||||
|
COPY . .
|
||||||
|
|
||||||
|
RUN apk add --no-cache musl-dev upx
|
||||||
|
|
||||||
|
RUN cargo b -r
|
||||||
|
RUN strip target/release/filed && upx --best target/release/filed
|
||||||
|
|
||||||
|
# --- deploy ---
|
||||||
|
FROM busybox:musl
|
||||||
|
|
||||||
|
COPY --from=builder /opt/build/target/release/filed /bin/filed
|
||||||
|
CMD [ "/bin/filed" ]
|
|
@ -0,0 +1,8 @@
|
||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
cd /opt/code
|
||||||
|
|
||||||
|
cargo check
|
||||||
|
cargo build
|
||||||
|
|
||||||
|
cargo watch -w src -x run
|
|
@ -1,7 +1,66 @@
|
||||||
use std::error::Error;
|
|
||||||
|
|
||||||
use crate::state::State;
|
use redis::Commands;
|
||||||
|
use tokio::task::JoinSet;
|
||||||
|
use std::{error::Error, path::Path};
|
||||||
|
|
||||||
|
use crate::{state::State, file::File};
|
||||||
|
|
||||||
|
async fn check_key(key: String, mut client: redis::Client) -> bool {
|
||||||
|
|
||||||
|
#[cfg(debug_assertions)]
|
||||||
|
log::debug!("Checking object {}", key);
|
||||||
|
|
||||||
|
let val: String = client.get(key.clone()).unwrap();
|
||||||
|
let file: File = serde_json::from_str(val.as_str()).unwrap();
|
||||||
|
|
||||||
|
if ! Path::new(&file.path.clone()).exists() {
|
||||||
|
#[cfg(debug_assertions)] {
|
||||||
|
log::debug!("Object {key} is marked for deletion because it doesn't exist in the filesystem");
|
||||||
|
}
|
||||||
|
client.del::<String, ()>(key).unwrap();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
let stat = tokio::fs::metadata(file.clone().path).await.unwrap();
|
||||||
|
if ! stat.is_file() {
|
||||||
|
client.del::<String, ()>(key).unwrap();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
false
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn clean(state: State) -> Result<(), Box<dyn Error>> {
|
pub async fn clean(state: State) -> Result<(), Box<dyn Error>> {
|
||||||
|
|
||||||
|
let mut redis = state.redis.clone();
|
||||||
|
let keys: Vec<String> = redis.keys(format!("{}*", state.env.redis.prefix))?;
|
||||||
|
let objects = keys.len();
|
||||||
|
|
||||||
|
#[cfg(debug_assertions)]
|
||||||
|
log::debug!("Got {} objects", objects);
|
||||||
|
|
||||||
|
let mut set: JoinSet<bool> = JoinSet::new();
|
||||||
|
for key in keys {
|
||||||
|
set.spawn(check_key(key, redis.clone()));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(debug_assertions)]
|
||||||
|
let mut del_count: u32 = 0;
|
||||||
|
|
||||||
|
while let Some(_deleted) = set.join_next().await {
|
||||||
|
|
||||||
|
#[cfg(debug_assertions)] {
|
||||||
|
if _deleted.is_ok() {
|
||||||
|
if _deleted.unwrap() {
|
||||||
|
del_count += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(debug_assertions)]
|
||||||
|
log::debug!("Deleted {} objects", del_count);
|
||||||
|
|
||||||
|
Ok(())
|
||||||
}
|
}
|
|
@ -1,5 +1,5 @@
|
||||||
|
|
||||||
use std::{error::Error, env::var, time::Duration};
|
use std::{error::Error, env::var, time::Duration, path::Path};
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct RedisEnv {
|
pub struct RedisEnv {
|
||||||
|
@ -13,7 +13,8 @@ pub struct RedisEnv {
|
||||||
pub struct Env {
|
pub struct Env {
|
||||||
pub redis: RedisEnv,
|
pub redis: RedisEnv,
|
||||||
pub clean_del: Duration,
|
pub clean_del: Duration,
|
||||||
pub clean_errdel: Duration
|
pub clean_errdel: Duration,
|
||||||
|
pub usercont_dir: String
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Env {
|
impl Env {
|
||||||
|
@ -24,10 +25,18 @@ impl Env {
|
||||||
pass: var("REDIS_PASS")?.to_string(),
|
pass: var("REDIS_PASS")?.to_string(),
|
||||||
host: var("REDIS_HOST")?.to_string(),
|
host: var("REDIS_HOST")?.to_string(),
|
||||||
port: var("REDIS_PORT")?.parse()?,
|
port: var("REDIS_PORT")?.parse()?,
|
||||||
prefix: var("REDIS_PASS")?.to_string()
|
prefix: var("REDIS_PREFIX")?.to_string()
|
||||||
},
|
},
|
||||||
clean_del: parse_duration::parse(var("CLEAN_DEL")?.as_str())?,
|
clean_del: parse_duration::parse(var("CLEAN_DEL")?.as_str())?,
|
||||||
clean_errdel: parse_duration::parse(var("CLEAN_ERRDEL")?.as_str())?
|
clean_errdel: parse_duration::parse(var("CLEAN_ERRDEL")?.as_str())?,
|
||||||
|
usercont_dir: {
|
||||||
|
let dir = var("USERCONTENT_DIR")?;
|
||||||
|
let dir = dir.as_str();
|
||||||
|
if ! Path::new(dir).is_dir() {
|
||||||
|
return Err("Path specified in USERCONTENT_DIR is not a directory!".into());
|
||||||
|
}
|
||||||
|
dir.to_string()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,19 @@
|
||||||
|
|
||||||
|
use chrono::{DateTime, Local};
|
||||||
|
use serde::{Serialize, Deserialize};
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct File {
|
||||||
|
pub path: String,
|
||||||
|
pub size: usize,
|
||||||
|
pub name: Option<String>,
|
||||||
|
pub mime: String,
|
||||||
|
pub delete_at: DateTime<Local>,
|
||||||
|
sha512: String
|
||||||
|
}
|
||||||
|
|
||||||
|
impl File {
|
||||||
|
pub fn expired(self: &Self) -> bool {
|
||||||
|
self.delete_at > chrono::Local::now()
|
||||||
|
}
|
||||||
|
}
|
|
@ -2,6 +2,7 @@
|
||||||
mod clean;
|
mod clean;
|
||||||
mod state;
|
mod state;
|
||||||
mod env;
|
mod env;
|
||||||
|
mod file;
|
||||||
|
|
||||||
pub fn redis_conn(env: env::Env) -> Result<redis::Client, redis::RedisError> {
|
pub fn redis_conn(env: env::Env) -> Result<redis::Client, redis::RedisError> {
|
||||||
log::info!("Connecting to redis DB on {}", env.redis.host);
|
log::info!("Connecting to redis DB on {}", env.redis.host);
|
||||||
|
@ -10,17 +11,40 @@ pub fn redis_conn(env: env::Env) -> Result<redis::Client, redis::RedisError> {
|
||||||
|
|
||||||
#[tokio::main]
|
#[tokio::main]
|
||||||
async fn main() {
|
async fn main() {
|
||||||
|
#[cfg(debug_assertions)] {
|
||||||
|
femme::with_level(log::LevelFilter::Debug);
|
||||||
|
}
|
||||||
|
#[cfg(not(debug_assertions))] {
|
||||||
|
femme::with_level(log::LevelFilter::Info);
|
||||||
|
}
|
||||||
|
|
||||||
dotenvy::dotenv().unwrap();
|
dotenvy::dotenv().unwrap();
|
||||||
|
|
||||||
let env = crate::env::Env::load().unwrap();
|
let env = crate::env::Env::load().unwrap();
|
||||||
let statee = crate::state::State {
|
let statee = crate::state::State {
|
||||||
redis: redis_conn(env).unwrap()
|
redis: redis_conn(env.clone()).unwrap(),
|
||||||
|
env: env.clone()
|
||||||
};
|
};
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
let res = clean::clean(statee).await;
|
|
||||||
|
#[cfg(debug_assertions)]
|
||||||
|
log::debug!("Initiating clean process");
|
||||||
|
|
||||||
|
let envy = env.clone();
|
||||||
|
let res = clean::clean(statee.clone()).await;
|
||||||
if res.is_err() {
|
if res.is_err() {
|
||||||
log::error!("Error while cleaning")
|
log::error!("Error while cleaning: {}", res.unwrap_err());
|
||||||
|
log::error!("Retrying in {}", std::env::var("CLEAN_ERRDEL").unwrap());
|
||||||
|
tokio::time::sleep(envy.clean_errdel).await;
|
||||||
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(debug_assertions)] {
|
||||||
|
log::debug!("Cleaned successfully");
|
||||||
|
log::debug!("Next clean is scheduled in {}", std::env::var("CLEAN_DEL").unwrap())
|
||||||
|
}
|
||||||
|
|
||||||
|
tokio::time::sleep(envy.clean_errdel).await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,5 +1,9 @@
|
||||||
use redis::Client;
|
use redis::Client;
|
||||||
|
|
||||||
|
use crate::env::Env;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
pub struct State {
|
pub struct State {
|
||||||
pub redis: Client
|
pub redis: Client,
|
||||||
|
pub env: Env
|
||||||
}
|
}
|
Loading…
Reference in New Issue