Add resource service #19

Merged
blek merged 13 commits from resource-service into 0.2-dev 2023-11-11 09:09:48 +01:00
13 changed files with 418 additions and 0 deletions

View File

@ -4,4 +4,8 @@
uri * strip_prefix /qr uri * strip_prefix /qr
reverse_proxy http://qr reverse_proxy http://qr
} }
handle /resource/* {
uri * strip_prefix /resource
reverse_proxy http://resourced
}
} }

View File

@ -0,0 +1,10 @@
FROM golang:alpine
RUN go install github.com/cosmtrek/air@latest
WORKDIR /opt/code
# The directory will be mounted
# COPY . .
CMD [ "air", "-c", ".air.toml" ]

View File

@ -21,6 +21,15 @@ services:
volumes: volumes:
- './janitor:/opt/code' - './janitor:/opt/code'
- './volatile/files:/opt/user_uploads' - './volatile/files:/opt/user_uploads'
resourced:
build:
context: containers
dockerfile: go-dev.Dockerfile
networks:
bfile:
volumes:
- './resource:/opt/code'
- '/opt/code/tmp'
caddy: caddy:
image: caddy:alpine image: caddy:alpine
volumes: volumes:

View File

@ -19,6 +19,15 @@ services:
volumes: volumes:
- './janitor:/config:ro' - './janitor:/config:ro'
- './volatile/files:/opt/user_uploads' - './volatile/files:/opt/user_uploads'
resourced:
build:
context: resource
dockerfile: Dockerfile.prod
networks:
bfile:
pid: host # prefork
volumes:
- './resource:/opt/cont'
caddy: caddy:
image: caddy:alpine image: caddy:alpine
volumes: volumes:

46
resource/.air.toml Normal file
View File

@ -0,0 +1,46 @@
root = "."
testdata_dir = "testdata"
tmp_dir = "tmp"
[build]
args_bin = []
bin = "./tmp/main"
cmd = "go build -o ./tmp/main ."
delay = 1000
exclude_dir = ["assets", "tmp", "vendor", "testdata"]
exclude_file = []
exclude_regex = [ ".*\\.md" ]
exclude_unchanged = false
follow_symlink = false
full_bin = ""
include_dir = []
include_ext = ["go", "tpl", "tmpl", "html"]
include_file = []
kill_delay = "0s"
log = "build-errors.log"
poll = false
poll_interval = 0
post_cmd = []
pre_cmd = []
rerun = false
rerun_delay = 500
send_interrupt = false
stop_on_error = false
[color]
app = ""
build = "yellow"
main = "magenta"
runner = "green"
watcher = "cyan"
[log]
main_only = false
time = false
[misc]
clean_on_exit = false
[screen]
clear_on_rebuild = false
keep_scroll = true

5
resource/.gitignore vendored Normal file
View File

@ -0,0 +1,5 @@
resourced.toml
resourced
.DS_Store
.env
tmp

22
resource/Dockerfile.prod Normal file
View File

@ -0,0 +1,22 @@
FROM golang:alpine3.17 as builder
WORKDIR /opt/build
COPY . .
RUN apk add --no-cache git musl-dev upx binutils
RUN go build . && \
strip resourced && \
upx resourced
FROM alpine:3.17
WORKDIR /opt/code
COPY --from=builder /opt/build/resourced /usr/bin/resourced
# Note
# -----
# Since this is running with prefork, don't
# forget to set --pid=host when running this app
CMD [ "sh", "-c", "/usr/bin/resourced" ]

15
resource/README.md Normal file
View File

@ -0,0 +1,15 @@
# resourceD
A simple microservice for serving resources
## Building
```sh
go build .
```
## Running
Either run it the normal way via the provided docker compose file, or `go run .`.
Also dont forget to create `resourced.toml`
## Configuration
The file `resourced.toml.example` serves as a both an example and reference, since it has a lot of comments.

View File

@ -0,0 +1,48 @@
openapi: 3.0.3
info:
title: ResourceD API
description: |-
This is the ResourceD API docs.
license:
name: GPLv3
url: https://www.gnu.org/licenses/gpl-3.0.en.html
version: "1.0"
servers:
- url: http://localhost/resource
tags:
- name: Data API
description: API for serving data
- name: System API
description: API for serving system data
paths:
/{id}:
get:
description: Get a resource
summary: Get a resource. Send browser requests to this URL
tags:
- Data API
responses:
200:
description: Returns a resource in its binary data
302:
description: Resource is an external (http) link and the redirect is being forwarded to that link
404:
description: Not found
500:
description: Internal error
/info/is_enabled:
get:
description: Check if resourceD is enabled
summary: Check if resourceD is enabled
tags:
- System API
responses:
200:
description: Ok
content:
application/json:
schema:
type: boolean
example: true

20
resource/go.mod Normal file
View File

@ -0,0 +1,20 @@
module blek/resourced
go 1.21.3
require (
github.com/BurntSushi/toml v1.3.2 // indirect
github.com/andybalholm/brotli v1.0.5 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/gofiber/fiber/v2 v2.50.0 // indirect
github.com/google/uuid v1.3.1 // indirect
github.com/klauspost/compress v1.16.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.19 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/rivo/uniseg v0.2.0 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/fasthttp v1.50.0 // indirect
github.com/valyala/tcplisten v1.0.0 // indirect
golang.org/x/sys v0.13.0 // indirect
)

31
resource/go.sum Normal file
View File

@ -0,0 +1,31 @@
github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=
github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/gofiber/fiber/v2 v2.50.0 h1:ia0JaB+uw3GpNSCR5nvC5dsaxXjRU5OEu36aytx+zGw=
github.com/gofiber/fiber/v2 v2.50.0/go.mod h1:21eytvay9Is7S6z+OgPi7c7n4++tnClWmhpimVHMimw=
github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.50.0 h1:H7fweIlBm0rXLs2q0XbalvJ6r0CUPFWK3/bB4N13e9M=
github.com/valyala/fasthttp v1.50.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA=
github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

152
resource/main.go Normal file
View File

@ -0,0 +1,152 @@
package main
import (
"log"
"fmt"
"regexp"
"strings"
"net/http"
"io/ioutil"
)
import (
"github.com/BurntSushi/toml"
"github.com/gofiber/fiber/v2"
"github.com/dustin/go-humanize"
)
type Resource struct {
Url string `toml:"url"`
Proxied bool `toml:"proxied",default:false`
Mime string `toml:"mime"`
}
func (self Resource) Get() ([]byte, error) {
return ioutil.ReadFile(self.Url[7:])
}
type ResourceDConfig struct {
Enabled bool `toml:"enabled"`
ListenURL string `toml:"listen_url"`
ProxyCacheMinSize string `toml:"proxy_cache_min_size",default:5MB`
}
type Config struct {
ResourceD ResourceDConfig `toml:"resourceD"`
Resource map[string]Resource `toml:"resource"`
}
func (self Config) Validate() int {
re, err := regexp.Compile(`^(file|http(s|))://`)
if err != nil { panic(err) }
for key, res := range self.Resource {
if ! re.MatchString(res.Url) {
panic(fmt.Sprintf("Resource %s has invalid URL: %s\nOnly file://, http:// and https:// URLs are allowed", key, res.Url))
}
}
return 0
}
func (self Resource) GetProxied() ([]byte, error) {
cached, exists := ProxyResourceCache[self.Url];
if exists {
return cached, nil
}
resp, err := http.Get(self.Url)
if err != nil { return make([]byte, 0, 0), err }
buf, err := ioutil.ReadAll(resp.Body)
if err != nil { return make([]byte, 0, 0), err }
// cache only those that are less than 5 mb
if len(buf) > ProxyCacheMinSize {
ProxyResourceCache[self.Url] = buf
}
return buf, nil
}
var ProxyResourceCache map[string][]byte = make(map[string][]byte)
var ProxyCacheMinSize int
func main() {
var conf Config
data, err := ioutil.ReadFile("resourced.toml")
if err != nil { panic(err) }
a, err := toml.Decode(string(data), &conf)
if err != nil { panic(err) }
_ = a
cache_min, err := humanize.ParseBytes(conf.ResourceD.ProxyCacheMinSize)
if err != nil { panic(err) }
ProxyCacheMinSize = int(cache_min)
conf.Validate()
if ! conf.ResourceD.Enabled {
fmt.Println("\x1b[33m[warn] resourceD is disabled. No resources will be served\x1b[0m")
}
app := fiber.New(fiber.Config {
Prefork: true,
CaseSensitive: false,
StrictRouting: true,
ServerHeader: "",
AppName: "blek! File resourceD",
})
app.Get("/info/is_enabled", func (c *fiber.Ctx) error {
return c.JSON(conf.ResourceD.Enabled)
})
app.Use(func (c *fiber.Ctx) error {
if ! conf.ResourceD.Enabled {
return c.Status(fiber.StatusNotFound).SendString("ResourceD is disabled")
}
return c.Next()
})
app.Get("/:id", func (c *fiber.Ctx) error {
res, exists := conf.Resource[c.Params("id")]
if ! exists {
return c.Status(fiber.StatusNotFound).SendString("Resource not found")
}
if ! strings.HasPrefix(res.Url, "file://") {
if res.Proxied {
data, err := res.GetProxied()
if err != nil {
log.Fatalln(err)
// we failed, send a redirect instead
// the next line would be the one with
// c.Location(res.Url)
} else {
c.Response().Header.SetContentType(res.Mime)
c.Response().Header.SetContentLength(len(data))
return c.Send(data)
}
}
c.Location(res.Url)
c.Status(302)
return nil
}
data, err := res.Get()
if err != nil {
panic(err)
}
c.Response().Header.SetContentType(res.Mime)
c.Response().Header.SetContentLength(len(data))
return c.Send(data)
})
log.Fatal(app.Listen(conf.ResourceD.ListenURL))
}

View File

@ -0,0 +1,47 @@
# The resourceD config
[resourceD]
# Whether to enable the resourceD.
# If this is false, resourceD will start but respond to
# all requests with 404
# It is false by default because resourceD is not required in a default installation.
enabled=true
# URL to listen on
listen_url="0.0.0.0:80"
# Minibal size of a file to be cached
# File size is parsed via this library:
# https://github.com/dustin/go-humanize
proxy_cache_min_size="5MB"
# Resource ID must be like a java package name
# At least one X.X. is required
#
# Examples:
# org.university.logo
# dev.indie_guy.logo
# com.pany.logo
# Test your names here: https://regex101.com/r/wQdOup/2
[resource."com.example.logo"]
# Can also be an external link
# If an external link is specified,
# the resource will be returned as a 302 redirect to the link
url="file:///some/where"
# File type, as according to this:
# https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types
mime="image/jpg"
# (if url is an http(s) url))
# Whether to proxy the resource.
# true - Send the resource like a regular file
# false - Send a 302 HTTP redirect to the URL (default)
#
# It is not recommended to set this to `true`
# unless you are referring to a resource that is
# available only in the local network
#
# proxied=true