blek
/
bfile
1
0
Fork 0
Code Issues 6 Pull Requests 1 Packages Projects Releases 1 Wiki Activity

get janitord running in docker

This commit is contained in:
blek 2023-10-12 20:39:28 +10:00
parent 95bc1fc47c
commit 32d5666477
Signed by: blek
GPG Key ID: 14546221E3595D0C
13 changed files with 243 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
containers/rust-dev.Dockerfile Normal file
View File

@ -0,0 +1,8 @@
FROM rust
RUN cargo install cargo-watch && \
mkdir -p /opt/code && \
touch /opt/code/dev-entry.sh && \
chmod +x /opt/code/dev-entry.sh
CMD [ "/opt/code/dev-entry.sh" ]

13
docker-compose.dev.yml
View File

@ -2,14 +2,23 @@ version: '3.7'
services:
filed:
build:
context: filed
dockerfile: Dockerfile.dev
context: containers
dockerfile: rust-dev.Dockerfile
networks:
bfile:
volumes:
- './filed:/opt/code'
- '/opt/code/target'
- './volatile/files:/opt/user_uploads'
janitord:
build:
context: containers
dockerfile: rust-dev.Dockerfile
networks:
bfile:
volumes:
- './janitor:/opt/code'
- './volatile/files:/opt/user_uploads'
caddy:
image: caddy:alpine
volumes:

17
filed/Dockerfile.dev
View File

@ -1,17 +0,0 @@
# --- build ---
FROM rust as builder
WORKDIR /opt/code
COPY . .
# No build is done during this step
# since the directory will be mounted anyways
# to the dev's machine.
# Therefore installing & compiling in the dockerfile
# would be moronic, to say at least
# However, cargo watch needs to be installed
RUN cargo install cargo-watch
CMD [ "/opt/code/dev-entry.sh" ]

1
janitor/.gitignore vendored
View File

@ -1,3 +1,4 @@
.env
.DS_Store
target
Dockerfile

71
janitor/Cargo.lock generated
View File

@ -26,6 +26,21 @@ dependencies = [
"memchr",
]
[[package]]
name = "android-tzdata"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0"
[[package]]
name = "android_system_properties"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
dependencies = [
"libc",
]
[[package]]
name = "autocfg"
version = "1.1.0"
@ -80,6 +95,21 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "chrono"
version = "0.4.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38"
dependencies = [
"android-tzdata",
"iana-time-zone",
"js-sys",
"num-traits",
"serde",
"wasm-bindgen",
"windows-targets",
]
[[package]]
name = "combine"
version = "4.6.6"
@ -90,6 +120,12 @@ dependencies = [
"memchr",
]
[[package]]
name = "core-foundation-sys"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa"
[[package]]
name = "dotenvy"
version = "0.15.7"
@ -142,6 +178,29 @@ version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7"
[[package]]
name = "iana-time-zone"
version = "0.1.57"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613"
dependencies = [
"android_system_properties",
"core-foundation-sys",
"iana-time-zone-haiku",
"js-sys",
"wasm-bindgen",
"windows",
]
[[package]]
name = "iana-time-zone-haiku"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
dependencies = [
"cc",
]
[[package]]
name = "idna"
version = "0.4.0"
@ -162,11 +221,14 @@ checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38"
name = "janitor"
version = "0.1.0"
dependencies = [
"chrono",
"dotenvy",
"femme",
"log",
"parse_duration",
"redis",
"serde",
"serde_json",
"tokio",
]
@ -842,6 +904,15 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windows"
version = "0.48.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f"
dependencies = [
"windows-targets",
]
[[package]]
name = "windows-sys"
version = "0.48.0"

3
janitor/Cargo.toml
View File

@ -6,9 +6,12 @@ edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
chrono = { version = "0.4.31", features = ["serde"] }
dotenvy = "0.15.7"
femme = "2.2.1"
log = "0.4.20"
parse_duration = "2.1.1"
redis = { version = "0.23.3", features = ["tokio"] }
serde = { version = "1.0.188", features = ["derive"] }
serde_json = "1.0.107"
tokio = { version = "1.33.0", features = ["full"] }

16
janitor/Dockerfile.prod Normal file
View File

@ -0,0 +1,16 @@
# --- build ---
FROM rust:alpine as builder
WORKDIR /opt/build
COPY . .
RUN apk add --no-cache musl-dev upx
RUN cargo b -r
RUN strip target/release/filed && upx --best target/release/filed
# --- deploy ---
FROM busybox:musl
COPY --from=builder /opt/build/target/release/filed /bin/filed
CMD [ "/bin/filed" ]

8
janitor/dev-entry.sh Executable file
View File

@ -0,0 +1,8 @@
#!/bin/sh
cd /opt/code
cargo check
cargo build
cargo watch -w src -x run

63
janitor/src/clean.rs
View File

@ -1,7 +1,66 @@
use std::error::Error;
use crate::state::State;
use redis::Commands;
use tokio::task::JoinSet;
use std::{error::Error, path::Path};
use crate::{state::State, file::File};
async fn check_key(key: String, mut client: redis::Client) -> bool {
#[cfg(debug_assertions)]
log::debug!("Checking object {}", key);
let val: String = client.get(key.clone()).unwrap();
let file: File = serde_json::from_str(val.as_str()).unwrap();
if ! Path::new(&file.path.clone()).exists() {
#[cfg(debug_assertions)] {
log::debug!("Object {key} is marked for deletion because it doesn't exist in the filesystem");
}
client.del::<String, ()>(key).unwrap();
return true;
}
let stat = tokio::fs::metadata(file.clone().path).await.unwrap();
if ! stat.is_file() {
client.del::<String, ()>(key).unwrap();
return true;
}
false
}
pub async fn clean(state: State) -> Result<(), Box<dyn Error>> {
let mut redis = state.redis.clone();
let keys: Vec<String> = redis.keys(format!("{}*", state.env.redis.prefix))?;
let objects = keys.len();
#[cfg(debug_assertions)]
log::debug!("Got {} objects", objects);
let mut set: JoinSet<bool> = JoinSet::new();
for key in keys {
set.spawn(check_key(key, redis.clone()));
}
#[cfg(debug_assertions)]
let mut del_count: u32 = 0;
while let Some(_deleted) = set.join_next().await {
#[cfg(debug_assertions)] {
if _deleted.is_ok() {
if _deleted.unwrap() {
del_count += 1;
}
}
}
}
#[cfg(debug_assertions)]
log::debug!("Deleted {} objects", del_count);
Ok(())
}

17
janitor/src/env.rs
View File

@ -1,5 +1,5 @@
use std::{error::Error, env::var, time::Duration};
use std::{error::Error, env::var, time::Duration, path::Path};
#[derive(Debug, Clone)]
pub struct RedisEnv {
@ -13,7 +13,8 @@ pub struct RedisEnv {
pub struct Env {
pub redis: RedisEnv,
pub clean_del: Duration,
pub clean_errdel: Duration
pub clean_errdel: Duration,
pub usercont_dir: String
}
impl Env {
@ -24,10 +25,18 @@ impl Env {
pass: var("REDIS_PASS")?.to_string(),
host: var("REDIS_HOST")?.to_string(),
port: var("REDIS_PORT")?.parse()?,
prefix: var("REDIS_PASS")?.to_string()
prefix: var("REDIS_PREFIX")?.to_string()
},
clean_del: parse_duration::parse(var("CLEAN_DEL")?.as_str())?,
clean_errdel: parse_duration::parse(var("CLEAN_ERRDEL")?.as_str())?
clean_errdel: parse_duration::parse(var("CLEAN_ERRDEL")?.as_str())?,
usercont_dir: {
let dir = var("USERCONTENT_DIR")?;
let dir = dir.as_str();
if ! Path::new(dir).is_dir() {
return Err("Path specified in USERCONTENT_DIR is not a directory!".into());
}
dir.to_string()
}
}
)
}

19
janitor/src/file.rs Normal file
View File

@ -0,0 +1,19 @@
use chrono::{DateTime, Local};
use serde::{Serialize, Deserialize};
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct File {
pub path: String,
pub size: usize,
pub name: Option<String>,
pub mime: String,
pub delete_at: DateTime<Local>,
sha512: String
}
impl File {
pub fn expired(self: &Self) -> bool {
self.delete_at > chrono::Local::now()
}
}

30
janitor/src/main.rs
View File

@ -2,6 +2,7 @@
mod clean;
mod state;
mod env;
mod file;
pub fn redis_conn(env: env::Env) -> Result<redis::Client, redis::RedisError> {
log::info!("Connecting to redis DB on {}", env.redis.host);
@ -10,17 +11,40 @@ pub fn redis_conn(env: env::Env) -> Result<redis::Client, redis::RedisError> {
#[tokio::main]
async fn main() {
#[cfg(debug_assertions)] {
femme::with_level(log::LevelFilter::Debug);
}
#[cfg(not(debug_assertions))] {
femme::with_level(log::LevelFilter::Info);
}
dotenvy::dotenv().unwrap();
let env = crate::env::Env::load().unwrap();
let statee = crate::state::State {
redis: redis_conn(env).unwrap()
redis: redis_conn(env.clone()).unwrap(),
env: env.clone()
};
loop {
let res = clean::clean(statee).await;
#[cfg(debug_assertions)]
log::debug!("Initiating clean process");
let envy = env.clone();
let res = clean::clean(statee.clone()).await;
if res.is_err() {
log::error!("Error while cleaning")
log::error!("Error while cleaning: {}", res.unwrap_err());
log::error!("Retrying in {}", std::env::var("CLEAN_ERRDEL").unwrap());
tokio::time::sleep(envy.clean_errdel).await;
continue;
}
#[cfg(debug_assertions)] {
log::debug!("Cleaned successfully");
log::debug!("Next clean is scheduled in {}", std::env::var("CLEAN_DEL").unwrap())
}
tokio::time::sleep(envy.clean_errdel).await;
}
}

6
janitor/src/state.rs
View File

@ -1,5 +1,9 @@
use redis::Client;
use crate::env::Env;
#[derive(Debug, Clone)]
pub struct State {
pub redis: Client
pub redis: Client,
pub env: Env
}