diff --git a/conf/caddy/Caddyfile b/conf/caddy/Caddyfile index e52811c..8984b7a 100644 --- a/conf/caddy/Caddyfile +++ b/conf/caddy/Caddyfile @@ -4,4 +4,8 @@ uri * strip_prefix /qr reverse_proxy http://qr } + handle /resource/* { + uri * strip_prefix /resource + reverse_proxy http://resourced + } } \ No newline at end of file diff --git a/containers/go-dev.Dockerfile b/containers/go-dev.Dockerfile new file mode 100644 index 0000000..c78435d --- /dev/null +++ b/containers/go-dev.Dockerfile @@ -0,0 +1,10 @@ +FROM golang:alpine + +RUN go install github.com/cosmtrek/air@latest + +WORKDIR /opt/code + +# The directory will be mounted +# COPY . . + +CMD [ "air", "-c", ".air.toml" ] diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 774aea5..f3d3871 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -21,6 +21,15 @@ services: volumes: - './janitor:/opt/code' - './volatile/files:/opt/user_uploads' + resourced: + build: + context: containers + dockerfile: go-dev.Dockerfile + networks: + bfile: + volumes: + - './resource:/opt/code' + - '/opt/code/tmp' caddy: image: caddy:alpine volumes: diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 4ac8eb2..fcbd1fe 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -19,6 +19,15 @@ services: volumes: - './janitor:/config:ro' - './volatile/files:/opt/user_uploads' + resourced: + build: + context: resource + dockerfile: Dockerfile.prod + networks: + bfile: + pid: host # prefork + volumes: + - './resource:/opt/cont' caddy: image: caddy:alpine volumes: diff --git a/resource/.air.toml b/resource/.air.toml new file mode 100644 index 0000000..b928116 --- /dev/null +++ b/resource/.air.toml @@ -0,0 +1,46 @@ +root = "." +testdata_dir = "testdata" +tmp_dir = "tmp" + +[build] + args_bin = [] + bin = "./tmp/main" + cmd = "go build -o ./tmp/main ." + delay = 1000 + exclude_dir = ["assets", "tmp", "vendor", "testdata"] + exclude_file = [] + exclude_regex = [ ".*\\.md" ] + exclude_unchanged = false + follow_symlink = false + full_bin = "" + include_dir = [] + include_ext = ["go", "tpl", "tmpl", "html"] + include_file = [] + kill_delay = "0s" + log = "build-errors.log" + poll = false + poll_interval = 0 + post_cmd = [] + pre_cmd = [] + rerun = false + rerun_delay = 500 + send_interrupt = false + stop_on_error = false + +[color] + app = "" + build = "yellow" + main = "magenta" + runner = "green" + watcher = "cyan" + +[log] + main_only = false + time = false + +[misc] + clean_on_exit = false + +[screen] + clear_on_rebuild = false + keep_scroll = true diff --git a/resource/.gitignore b/resource/.gitignore new file mode 100644 index 0000000..bac04d0 --- /dev/null +++ b/resource/.gitignore @@ -0,0 +1,5 @@ +resourced.toml +resourced +.DS_Store +.env +tmp diff --git a/resource/Dockerfile.prod b/resource/Dockerfile.prod new file mode 100644 index 0000000..0451ed0 --- /dev/null +++ b/resource/Dockerfile.prod @@ -0,0 +1,22 @@ +FROM golang:alpine3.17 as builder + +WORKDIR /opt/build +COPY . . + +RUN apk add --no-cache git musl-dev upx binutils + +RUN go build . && \ + strip resourced && \ + upx resourced + +FROM alpine:3.17 + +WORKDIR /opt/code +COPY --from=builder /opt/build/resourced /usr/bin/resourced + +# Note +# ----- +# Since this is running with prefork, don't +# forget to set --pid=host when running this app + +CMD [ "sh", "-c", "/usr/bin/resourced" ] diff --git a/resource/README.md b/resource/README.md new file mode 100644 index 0000000..09823a3 --- /dev/null +++ b/resource/README.md @@ -0,0 +1,15 @@ +# resourceD +A simple microservice for serving resources + +## Building +```sh +go build . +``` + +## Running +Either run it the normal way via the provided docker compose file, or `go run .`. + +Also dont forget to create `resourced.toml` + +## Configuration +The file `resourced.toml.example` serves as a both an example and reference, since it has a lot of comments. \ No newline at end of file diff --git a/resource/api_ref.swagger.yml b/resource/api_ref.swagger.yml new file mode 100644 index 0000000..ea60424 --- /dev/null +++ b/resource/api_ref.swagger.yml @@ -0,0 +1,48 @@ +openapi: 3.0.3 +info: + title: ResourceD API + description: |- + This is the ResourceD API docs. + license: + name: GPLv3 + url: https://www.gnu.org/licenses/gpl-3.0.en.html + version: "1.0" +servers: + - url: http://localhost/resource +tags: + - name: Data API + description: API for serving data + - name: System API + description: API for serving system data + +paths: + /{id}: + get: + description: Get a resource + summary: Get a resource. Send browser requests to this URL + tags: + - Data API + responses: + 200: + description: Returns a resource in its binary data + 302: + description: Resource is an external (http) link and the redirect is being forwarded to that link + 404: + description: Not found + 500: + description: Internal error + /info/is_enabled: + get: + description: Check if resourceD is enabled + summary: Check if resourceD is enabled + tags: + - System API + responses: + 200: + description: Ok + content: + application/json: + schema: + type: boolean + example: true + \ No newline at end of file diff --git a/resource/go.mod b/resource/go.mod new file mode 100644 index 0000000..4333b70 --- /dev/null +++ b/resource/go.mod @@ -0,0 +1,20 @@ +module blek/resourced + +go 1.21.3 + +require ( + github.com/BurntSushi/toml v1.3.2 // indirect + github.com/andybalholm/brotli v1.0.5 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/gofiber/fiber/v2 v2.50.0 // indirect + github.com/google/uuid v1.3.1 // indirect + github.com/klauspost/compress v1.16.7 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect + github.com/mattn/go-runewidth v0.0.15 // indirect + github.com/rivo/uniseg v0.2.0 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/valyala/fasthttp v1.50.0 // indirect + github.com/valyala/tcplisten v1.0.0 // indirect + golang.org/x/sys v0.13.0 // indirect +) diff --git a/resource/go.sum b/resource/go.sum new file mode 100644 index 0000000..c9e3a79 --- /dev/null +++ b/resource/go.sum @@ -0,0 +1,31 @@ +github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= +github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= +github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/gofiber/fiber/v2 v2.50.0 h1:ia0JaB+uw3GpNSCR5nvC5dsaxXjRU5OEu36aytx+zGw= +github.com/gofiber/fiber/v2 v2.50.0/go.mod h1:21eytvay9Is7S6z+OgPi7c7n4++tnClWmhpimVHMimw= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= +github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.50.0 h1:H7fweIlBm0rXLs2q0XbalvJ6r0CUPFWK3/bB4N13e9M= +github.com/valyala/fasthttp v1.50.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA= +github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= +github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/resource/main.go b/resource/main.go new file mode 100644 index 0000000..a63adde --- /dev/null +++ b/resource/main.go @@ -0,0 +1,152 @@ +package main + +import ( + "log" + "fmt" + "regexp" + "strings" + "net/http" + "io/ioutil" +) + +import ( + "github.com/BurntSushi/toml" + "github.com/gofiber/fiber/v2" + "github.com/dustin/go-humanize" +) + +type Resource struct { + Url string `toml:"url"` + Proxied bool `toml:"proxied",default:false` + Mime string `toml:"mime"` +} +func (self Resource) Get() ([]byte, error) { + return ioutil.ReadFile(self.Url[7:]) +} + +type ResourceDConfig struct { + Enabled bool `toml:"enabled"` + ListenURL string `toml:"listen_url"` + ProxyCacheMinSize string `toml:"proxy_cache_min_size",default:5MB` +} + +type Config struct { + ResourceD ResourceDConfig `toml:"resourceD"` + Resource map[string]Resource `toml:"resource"` +} + +func (self Config) Validate() int { + re, err := regexp.Compile(`^(file|http(s|))://`) + if err != nil { panic(err) } + + for key, res := range self.Resource { + if ! re.MatchString(res.Url) { + panic(fmt.Sprintf("Resource %s has invalid URL: %s\nOnly file://, http:// and https:// URLs are allowed", key, res.Url)) + } + } + + return 0 +} +func (self Resource) GetProxied() ([]byte, error) { + + cached, exists := ProxyResourceCache[self.Url]; + if exists { + return cached, nil + } + + resp, err := http.Get(self.Url) + if err != nil { return make([]byte, 0, 0), err } + + buf, err := ioutil.ReadAll(resp.Body) + if err != nil { return make([]byte, 0, 0), err } + + // cache only those that are less than 5 mb + if len(buf) > ProxyCacheMinSize { + ProxyResourceCache[self.Url] = buf + } + + return buf, nil +} + +var ProxyResourceCache map[string][]byte = make(map[string][]byte) +var ProxyCacheMinSize int + +func main() { + var conf Config + + data, err := ioutil.ReadFile("resourced.toml") + if err != nil { panic(err) } + + a, err := toml.Decode(string(data), &conf) + if err != nil { panic(err) } + _ = a + + cache_min, err := humanize.ParseBytes(conf.ResourceD.ProxyCacheMinSize) + if err != nil { panic(err) } + ProxyCacheMinSize = int(cache_min) + + conf.Validate() + + if ! conf.ResourceD.Enabled { + fmt.Println("\x1b[33m[warn] resourceD is disabled. No resources will be served\x1b[0m") + } + + app := fiber.New(fiber.Config { + Prefork: true, + CaseSensitive: false, + StrictRouting: true, + ServerHeader: "", + AppName: "blek! File resourceD", + }) + + app.Get("/info/is_enabled", func (c *fiber.Ctx) error { + return c.JSON(conf.ResourceD.Enabled) + }) + + app.Use(func (c *fiber.Ctx) error { + if ! conf.ResourceD.Enabled { + return c.Status(fiber.StatusNotFound).SendString("ResourceD is disabled") + } + return c.Next() + }) + + app.Get("/:id", func (c *fiber.Ctx) error { + res, exists := conf.Resource[c.Params("id")] + if ! exists { + return c.Status(fiber.StatusNotFound).SendString("Resource not found") + } + + if ! strings.HasPrefix(res.Url, "file://") { + + if res.Proxied { + data, err := res.GetProxied() + if err != nil { + log.Fatalln(err) + // we failed, send a redirect instead + // the next line would be the one with + // c.Location(res.Url) + } else { + c.Response().Header.SetContentType(res.Mime) + c.Response().Header.SetContentLength(len(data)) + return c.Send(data) + } + } + + c.Location(res.Url) + c.Status(302) + return nil + } + + data, err := res.Get() + if err != nil { + panic(err) + } + + c.Response().Header.SetContentType(res.Mime) + c.Response().Header.SetContentLength(len(data)) + + return c.Send(data) + }) + + log.Fatal(app.Listen(conf.ResourceD.ListenURL)) +} \ No newline at end of file diff --git a/resource/resourced.toml.example b/resource/resourced.toml.example new file mode 100644 index 0000000..c74bae7 --- /dev/null +++ b/resource/resourced.toml.example @@ -0,0 +1,47 @@ + +# The resourceD config +[resourceD] + +# Whether to enable the resourceD. +# If this is false, resourceD will start but respond to +# all requests with 404 +# It is false by default because resourceD is not required in a default installation. +enabled=true + +# URL to listen on +listen_url="0.0.0.0:80" + +# Minibal size of a file to be cached +# File size is parsed via this library: +# https://github.com/dustin/go-humanize +proxy_cache_min_size="5MB" + +# Resource ID must be like a java package name +# At least one X.X. is required +# +# Examples: +# org.university.logo +# dev.indie_guy.logo +# com.pany.logo +# Test your names here: https://regex101.com/r/wQdOup/2 +[resource."com.example.logo"] + +# Can also be an external link +# If an external link is specified, +# the resource will be returned as a 302 redirect to the link +url="file:///some/where" + +# File type, as according to this: +# https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types +mime="image/jpg" + +# (if url is an http(s) url)) +# Whether to proxy the resource. +# true - Send the resource like a regular file +# false - Send a 302 HTTP redirect to the URL (default) +# +# It is not recommended to set this to `true` +# unless you are referring to a resource that is +# available only in the local network +# +# proxied=true \ No newline at end of file