blek
/
bfile
1
0
Fork 0
Code Issues 6 Pull Requests 1 Packages Projects Releases 1 Wiki Activity

all the stuff for uploading the files

This commit is contained in:
blek 2023-10-01 14:17:58 +10:00
parent dc6d19a478
commit c6e1f80b47
Signed by: blek
GPG Key ID: 14546221E3595D0C
7 changed files with 331 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

161
filed/Cargo.lock generated
View File

@ -17,6 +17,21 @@ version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
[[package]]
name = "android-tzdata"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0"
[[package]]
name = "android_system_properties"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
dependencies = [
"libc",
]
[[package]]
name = "askama"
version = "0.12.0"
@ -130,6 +145,21 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "chrono"
version = "0.4.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38"
dependencies = [
"android-tzdata",
"iana-time-zone",
"js-sys",
"num-traits",
"serde",
"wasm-bindgen",
"windows-targets",
]
[[package]]
name = "combine"
version = "4.6.6"
@ -140,6 +170,12 @@ dependencies = [
"memchr",
]
[[package]]
name = "core-foundation-sys"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa"
[[package]]
name = "cpufeatures"
version = "0.2.9"
@ -221,11 +257,17 @@ version = "0.1.0"
dependencies = [
"askama",
"bytes",
"chrono",
"dotenvy",
"femme",
"futures-util",
"hex",
"log",
"num",
"redis",
"serde",
"serde_json",
"sha2",
"tokio",
"warp",
]
@ -381,6 +423,12 @@ version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7"
[[package]]
name = "hex"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
[[package]]
name = "http"
version = "0.2.9"
@ -448,6 +496,29 @@ dependencies = [
"want",
]
[[package]]
name = "iana-time-zone"
version = "0.1.57"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613"
dependencies = [
"android_system_properties",
"core-foundation-sys",
"iana-time-zone-haiku",
"js-sys",
"wasm-bindgen",
"windows",
]
[[package]]
name = "iana-time-zone-haiku"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
dependencies = [
"cc",
]
[[package]]
name = "idna"
version = "0.4.0"
@ -581,6 +652,76 @@ dependencies = [
"minimal-lexical",
]
[[package]]
name = "num"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b05180d69e3da0e530ba2a1dae5110317e49e3b7f3d41be227dc5f92e49ee7af"
dependencies = [
"num-bigint",
"num-complex",
"num-integer",
"num-iter",
"num-rational",
"num-traits",
]
[[package]]
name = "num-bigint"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0"
dependencies = [
"autocfg",
"num-integer",
"num-traits",
"serde",
]
[[package]]
name = "num-complex"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ba157ca0885411de85d6ca030ba7e2a83a28636056c7c699b07c8b6f7383214"
dependencies = [
"num-traits",
"serde",
]
[[package]]
name = "num-integer"
version = "0.1.45"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9"
dependencies = [
"autocfg",
"num-traits",
]
[[package]]
name = "num-iter"
version = "0.1.43"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252"
dependencies = [
"autocfg",
"num-integer",
"num-traits",
]
[[package]]
name = "num-rational"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0"
dependencies = [
"autocfg",
"num-bigint",
"num-integer",
"num-traits",
"serde",
]
[[package]]
name = "num-traits"
version = "0.2.16"
@ -819,6 +960,17 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012"
[[package]]
name = "sha2"
version = "0.10.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8"
dependencies = [
"cfg-if",
"cpufeatures",
"digest",
]
[[package]]
name = "slab"
version = "0.4.9"
@ -1314,6 +1466,15 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windows"
version = "0.48.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f"
dependencies = [
"windows-targets",
]
[[package]]
name = "windows-sys"
version = "0.48.0"

6
filed/Cargo.toml
View File

@ -8,10 +8,16 @@ edition = "2021"
[dependencies]
askama = "0.12.0"
bytes = "1.5.0"
chrono = { version = "0.4.31", features = ["serde"] }
dotenvy = "0.15.7"
femme = "2.2.1"
futures-util = "0.3.28"
hex = "0.4.3"
log = "0.4.20"
num = { version = "0.4.1", features = ["serde"] }
redis = { version = "0.23.3", features = ["tokio"] }
serde = { version = "1.0.188", features = ["derive"] }
serde_json = "1.0.107"
sha2 = "0.10.8"
tokio = { version = "1.32.0", features = ["rt", "macros", "rt-multi-thread"] }
warp = "0.3.6"

3
filed/README.md
View File

@ -6,3 +6,6 @@ This module is released under the GPLv3 with additions, copy of which is include
To get started with this, copy either `Dockerfile.dev` or `Dockerfile.prod` to `Dockerfile`, depending on your environment.
Then either build it manually or start it up using the `docker-compose.yml` file, which is provided in the top level directory.
## Deploying notes
Files will be saved in `/opt/useruploads`. Mount that directory into a volume or host directory to easily back up the data.

17
filed/src/env.rs
View File

@ -3,7 +3,7 @@
This file provides the `loadenv` function that will do just that.
*/
use std::{env::var, net::SocketAddr};
use std::{env::var, net::SocketAddr, path::Path};
#[derive(Debug, Clone)]
pub struct Redis {
@ -18,6 +18,7 @@ pub struct Env {
pub logging: bool,
pub listen: SocketAddr,
pub redis: Redis,
pub filedir: String
}
fn get_var<T: Into<String>, O: From<String>>(name: T) -> Result<O, String> {
@ -39,7 +40,21 @@ pub fn loadenv() -> Result<Env, Box<dyn std::error::Error>> {
host: get_var("REDIS_HOST")?,
port: get_var::<&str, String>("REDIS_PORT")?.parse().unwrap(),
prefix: get_var("REDIS_PREFIX")?
},
filedir: {
let spath: String = get_var("USERCONTENT_DIR")?;
let path = Path::new(&spath);
if ! path.exists() {
return Err(format!("USERCONTENT_DIR is set to \"{}\", which does not exist!", &spath).into())
}
spath
}
}
)
}
impl Env {
pub fn usercontent_dir(self: &Self) -> Box<&Path> {
Box::new(Path::new(&self.filedir))
}
}

76
filed/src/files/lookup.rs Normal file
View File

@ -0,0 +1,76 @@
use std::error::Error;
use redis::{Client, Commands};
use crate::env::Env;
use super::File;
pub struct FileFinder {
conn: Client,
env: Env
}
pub enum LookupKind {
ByName,
ByHash
}
impl FileFinder {
pub fn new(conn: Client, env: Env) -> FileFinder {
FileFinder { conn, env }
}
fn find(self: &Self, key: String) -> Result<Option<File>, Box<dyn Error>> {
let mut conn = self.conn.get_connection()?;
if ! conn.exists(&key)? {
return Ok(None)
}
let data: String = conn.get(&key)?;
Ok(Some(serde_json::from_str(data.as_str())?))
}
pub fn find_by_name(self: &Self, name: String) -> Result<Option<File>, Box<dyn Error>> {
Ok(self.find(format!("{}-name-{}", self.env.redis.prefix, name))?)
}
pub fn find_by_hash(self: &Self, hash: String) -> Result<Option<File>, Box<dyn Error>> {
Ok(self.find(format!("{}-hash-{}", self.env.redis.prefix, hash))?)
}
fn save_int(self: &Self, file: &File, key: String) -> Result<(), Box<dyn Error>> {
let mut conn = self.conn.get_connection()?;
conn.set(key, serde_json::to_string(&file)?)?;
Ok(())
}
pub fn save(self: &Self, file: &File, kind: LookupKind) -> Result<(), Box<dyn Error>> {
let file = file.clone();
let midfix = match kind {
LookupKind::ByName => "-name-",
LookupKind::ByHash => "-hash-"
};
match kind {
LookupKind::ByName => {
if (&file).name.is_none() {
return Err("Filename can't be None when LookupKind is ByName!".into())
}
}
_ => ()
}
self.save_int(
&file,
format!(
"{}{}{}",
self.env.redis.prefix,
midfix,
match kind {
LookupKind::ByName => (&file).name.as_ref().unwrap().clone(),
LookupKind::ByHash => (&file).hash()
}
)
)
}
}

68
filed/src/files/mod.rs Normal file
View File

@ -0,0 +1,68 @@
// this whole crate is just a boilerplate
#![allow(unused)]
use std::{sync::Arc, error::Error, ops::Add};
use chrono::{DateTime, Local};
use num::BigUint;
use sha2::{Sha512, Digest, digest::FixedOutput};
use serde::{Serialize, Deserialize};
use tokio::fs;
use crate::env::Env;
pub mod lookup;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct File {
pub path: String,
pub size: BigUint,
pub name: Option<String>,
pub mime: String,
pub delete_at: DateTime<Local>,
sha512: String
}
impl File {
pub fn comp_hash(self: &Self, other: &Sha512) -> bool {
let mut hash = other.clone();
hex::encode(hash.finalize_fixed()) == self.sha512
}
pub fn hash(self: &Self) -> String {
self.sha512.clone()
}
pub async fn create(data: Vec<u8>, mime: String, name: Option<String>, env: Env) -> Result<File, Box<dyn Error>> {
let mut filename = String::new();
let mut hash = Sha512::new();
hash.update(&data);
let hash = hex::encode(hash.finalize_fixed());
match name {
Some(name) => filename = name,
None => filename = hash.clone()
}
let path = env.usercontent_dir().join(&filename);
if ! path.exists() {
fs::write(&path, &data).await;
} else {
return Err("File already uploaded".into());
}
let expires = Local::now();
expires.add(chrono::Duration::minutes(30));
Ok(
File {
path: path.display().to_string(),
size: BigUint::from(data.len()),
name: Some(filename),
mime,
delete_at: expires,
sha512: hash
}
)
}
}

1
filed/src/main.rs
View File

@ -2,6 +2,7 @@
#![warn(clippy::suspicious)]
#![warn(clippy::correctness)]
mod files;
mod env;
mod web;
mod db;