From c6e1f80b472a51d7dc50e88c997674a7702e0d67 Mon Sep 17 00:00:00 2001 From: blek Date: Sun, 1 Oct 2023 14:17:58 +1000 Subject: [PATCH] all the stuff for uploading the files --- filed/Cargo.lock | 161 ++++++++++++++++++++++++++++++++++++++ filed/Cargo.toml | 6 ++ filed/README.md | 3 + filed/src/env.rs | 17 +++- filed/src/files/lookup.rs | 76 ++++++++++++++++++ filed/src/files/mod.rs | 68 ++++++++++++++++ filed/src/main.rs | 1 + 7 files changed, 331 insertions(+), 1 deletion(-) create mode 100644 filed/src/files/lookup.rs create mode 100644 filed/src/files/mod.rs diff --git a/filed/Cargo.lock b/filed/Cargo.lock index dcf75ad..a94ec72 100644 --- a/filed/Cargo.lock +++ b/filed/Cargo.lock @@ -17,6 +17,21 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "askama" version = "0.12.0" @@ -130,6 +145,21 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-targets", +] + [[package]] name = "combine" version = "4.6.6" @@ -140,6 +170,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "core-foundation-sys" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" + [[package]] name = "cpufeatures" version = "0.2.9" @@ -221,11 +257,17 @@ version = "0.1.0" dependencies = [ "askama", "bytes", + "chrono", "dotenvy", "femme", "futures-util", + "hex", "log", + "num", "redis", + "serde", + "serde_json", + "sha2", "tokio", "warp", ] @@ -381,6 +423,12 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "http" version = "0.2.9" @@ -448,6 +496,29 @@ dependencies = [ "want", ] +[[package]] +name = "iana-time-zone" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "idna" version = "0.4.0" @@ -581,6 +652,76 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "num" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05180d69e3da0e530ba2a1dae5110317e49e3b7f3d41be227dc5f92e49ee7af" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "num-complex" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ba157ca0885411de85d6ca030ba7e2a83a28636056c7c699b07c8b6f7383214" +dependencies = [ + "num-traits", + "serde", +] + +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" +dependencies = [ + "autocfg", + "num-bigint", + "num-integer", + "num-traits", + "serde", +] + [[package]] name = "num-traits" version = "0.2.16" @@ -819,6 +960,17 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "slab" version = "0.4.9" @@ -1314,6 +1466,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.48.0" diff --git a/filed/Cargo.toml b/filed/Cargo.toml index 78067f3..f5bb6df 100644 --- a/filed/Cargo.toml +++ b/filed/Cargo.toml @@ -8,10 +8,16 @@ edition = "2021" [dependencies] askama = "0.12.0" bytes = "1.5.0" +chrono = { version = "0.4.31", features = ["serde"] } dotenvy = "0.15.7" femme = "2.2.1" futures-util = "0.3.28" +hex = "0.4.3" log = "0.4.20" +num = { version = "0.4.1", features = ["serde"] } redis = { version = "0.23.3", features = ["tokio"] } +serde = { version = "1.0.188", features = ["derive"] } +serde_json = "1.0.107" +sha2 = "0.10.8" tokio = { version = "1.32.0", features = ["rt", "macros", "rt-multi-thread"] } warp = "0.3.6" diff --git a/filed/README.md b/filed/README.md index 2a1e4b0..37747dd 100644 --- a/filed/README.md +++ b/filed/README.md @@ -6,3 +6,6 @@ This module is released under the GPLv3 with additions, copy of which is include To get started with this, copy either `Dockerfile.dev` or `Dockerfile.prod` to `Dockerfile`, depending on your environment. Then either build it manually or start it up using the `docker-compose.yml` file, which is provided in the top level directory. + +## Deploying notes +Files will be saved in `/opt/useruploads`. Mount that directory into a volume or host directory to easily back up the data. diff --git a/filed/src/env.rs b/filed/src/env.rs index 8df99bf..c1a7d43 100644 --- a/filed/src/env.rs +++ b/filed/src/env.rs @@ -3,7 +3,7 @@ This file provides the `loadenv` function that will do just that. */ -use std::{env::var, net::SocketAddr}; +use std::{env::var, net::SocketAddr, path::Path}; #[derive(Debug, Clone)] pub struct Redis { @@ -18,6 +18,7 @@ pub struct Env { pub logging: bool, pub listen: SocketAddr, pub redis: Redis, + pub filedir: String } fn get_var, O: From>(name: T) -> Result { @@ -39,7 +40,21 @@ pub fn loadenv() -> Result> { host: get_var("REDIS_HOST")?, port: get_var::<&str, String>("REDIS_PORT")?.parse().unwrap(), prefix: get_var("REDIS_PREFIX")? + }, + filedir: { + let spath: String = get_var("USERCONTENT_DIR")?; + let path = Path::new(&spath); + if ! path.exists() { + return Err(format!("USERCONTENT_DIR is set to \"{}\", which does not exist!", &spath).into()) + } + spath } } ) +} + +impl Env { + pub fn usercontent_dir(self: &Self) -> Box<&Path> { + Box::new(Path::new(&self.filedir)) + } } \ No newline at end of file diff --git a/filed/src/files/lookup.rs b/filed/src/files/lookup.rs new file mode 100644 index 0000000..346c759 --- /dev/null +++ b/filed/src/files/lookup.rs @@ -0,0 +1,76 @@ + +use std::error::Error; + +use redis::{Client, Commands}; + +use crate::env::Env; + +use super::File; + +pub struct FileFinder { + conn: Client, + env: Env +} + +pub enum LookupKind { + ByName, + ByHash +} + +impl FileFinder { + pub fn new(conn: Client, env: Env) -> FileFinder { + FileFinder { conn, env } + } + fn find(self: &Self, key: String) -> Result, Box> { + let mut conn = self.conn.get_connection()?; + + if ! conn.exists(&key)? { + return Ok(None) + } + + let data: String = conn.get(&key)?; + Ok(Some(serde_json::from_str(data.as_str())?)) + } + pub fn find_by_name(self: &Self, name: String) -> Result, Box> { + Ok(self.find(format!("{}-name-{}", self.env.redis.prefix, name))?) + } + pub fn find_by_hash(self: &Self, hash: String) -> Result, Box> { + Ok(self.find(format!("{}-hash-{}", self.env.redis.prefix, hash))?) + } + + fn save_int(self: &Self, file: &File, key: String) -> Result<(), Box> { + let mut conn = self.conn.get_connection()?; + conn.set(key, serde_json::to_string(&file)?)?; + Ok(()) + } + + pub fn save(self: &Self, file: &File, kind: LookupKind) -> Result<(), Box> { + let file = file.clone(); + let midfix = match kind { + LookupKind::ByName => "-name-", + LookupKind::ByHash => "-hash-" + }; + + match kind { + LookupKind::ByName => { + if (&file).name.is_none() { + return Err("Filename can't be None when LookupKind is ByName!".into()) + } + } + _ => () + } + + self.save_int( + &file, + format!( + "{}{}{}", + self.env.redis.prefix, + midfix, + match kind { + LookupKind::ByName => (&file).name.as_ref().unwrap().clone(), + LookupKind::ByHash => (&file).hash() + } + ) + ) + } +} \ No newline at end of file diff --git a/filed/src/files/mod.rs b/filed/src/files/mod.rs new file mode 100644 index 0000000..3be296b --- /dev/null +++ b/filed/src/files/mod.rs @@ -0,0 +1,68 @@ +// this whole crate is just a boilerplate +#![allow(unused)] + +use std::{sync::Arc, error::Error, ops::Add}; + +use chrono::{DateTime, Local}; +use num::BigUint; +use sha2::{Sha512, Digest, digest::FixedOutput}; +use serde::{Serialize, Deserialize}; +use tokio::fs; + +use crate::env::Env; + +pub mod lookup; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct File { + pub path: String, + pub size: BigUint, + pub name: Option, + pub mime: String, + pub delete_at: DateTime, + sha512: String +} + +impl File { + pub fn comp_hash(self: &Self, other: &Sha512) -> bool { + let mut hash = other.clone(); + hex::encode(hash.finalize_fixed()) == self.sha512 + } + pub fn hash(self: &Self) -> String { + self.sha512.clone() + } + + pub async fn create(data: Vec, mime: String, name: Option, env: Env) -> Result> { + + let mut filename = String::new(); + let mut hash = Sha512::new(); + hash.update(&data); + let hash = hex::encode(hash.finalize_fixed()); + + match name { + Some(name) => filename = name, + None => filename = hash.clone() + } + + let path = env.usercontent_dir().join(&filename); + if ! path.exists() { + fs::write(&path, &data).await; + } else { + return Err("File already uploaded".into()); + } + + let expires = Local::now(); + expires.add(chrono::Duration::minutes(30)); + + Ok( + File { + path: path.display().to_string(), + size: BigUint::from(data.len()), + name: Some(filename), + mime, + delete_at: expires, + sha512: hash + } + ) + } +} \ No newline at end of file diff --git a/filed/src/main.rs b/filed/src/main.rs index e5f4b01..46e529b 100644 --- a/filed/src/main.rs +++ b/filed/src/main.rs @@ -2,6 +2,7 @@ #![warn(clippy::suspicious)] #![warn(clippy::correctness)] +mod files; mod env; mod web; mod db;