From cfd5255ea438436bd7749dab42f69d4cb613b67f Mon Sep 17 00:00:00 2001 From: blek Date: Sun, 5 Nov 2023 13:13:02 +1000 Subject: [PATCH 01/13] init repo --- resource/.gitignore | 4 +++ resource/README.md | 15 +++++++++ resource/go.mod | 19 +++++++++++ resource/go.sum | 29 +++++++++++++++++ resource/main.go | 56 +++++++++++++++++++++++++++++++++ resource/resourced.toml.example | 31 ++++++++++++++++++ 6 files changed, 154 insertions(+) create mode 100644 resource/.gitignore create mode 100644 resource/README.md create mode 100644 resource/go.mod create mode 100644 resource/go.sum create mode 100644 resource/main.go create mode 100644 resource/resourced.toml.example diff --git a/resource/.gitignore b/resource/.gitignore new file mode 100644 index 0000000..4bf8d84 --- /dev/null +++ b/resource/.gitignore @@ -0,0 +1,4 @@ +resourced.toml +resourced +.DS_Store +.env diff --git a/resource/README.md b/resource/README.md new file mode 100644 index 0000000..09823a3 --- /dev/null +++ b/resource/README.md @@ -0,0 +1,15 @@ +# resourceD +A simple microservice for serving resources + +## Building +```sh +go build . +``` + +## Running +Either run it the normal way via the provided docker compose file, or `go run .`. + +Also dont forget to create `resourced.toml` + +## Configuration +The file `resourced.toml.example` serves as a both an example and reference, since it has a lot of comments. \ No newline at end of file diff --git a/resource/go.mod b/resource/go.mod new file mode 100644 index 0000000..0ee4b39 --- /dev/null +++ b/resource/go.mod @@ -0,0 +1,19 @@ +module blek/resourced + +go 1.21.3 + +require ( + github.com/BurntSushi/toml v1.3.2 // indirect + github.com/andybalholm/brotli v1.0.5 // indirect + github.com/gofiber/fiber/v2 v2.50.0 // indirect + github.com/google/uuid v1.3.1 // indirect + github.com/klauspost/compress v1.16.7 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect + github.com/mattn/go-runewidth v0.0.15 // indirect + github.com/rivo/uniseg v0.2.0 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/valyala/fasthttp v1.50.0 // indirect + github.com/valyala/tcplisten v1.0.0 // indirect + golang.org/x/sys v0.13.0 // indirect +) diff --git a/resource/go.sum b/resource/go.sum new file mode 100644 index 0000000..76c49cb --- /dev/null +++ b/resource/go.sum @@ -0,0 +1,29 @@ +github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= +github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= +github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/gofiber/fiber/v2 v2.50.0 h1:ia0JaB+uw3GpNSCR5nvC5dsaxXjRU5OEu36aytx+zGw= +github.com/gofiber/fiber/v2 v2.50.0/go.mod h1:21eytvay9Is7S6z+OgPi7c7n4++tnClWmhpimVHMimw= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= +github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.50.0 h1:H7fweIlBm0rXLs2q0XbalvJ6r0CUPFWK3/bB4N13e9M= +github.com/valyala/fasthttp v1.50.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA= +github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= +github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/resource/main.go b/resource/main.go new file mode 100644 index 0000000..5bf03de --- /dev/null +++ b/resource/main.go @@ -0,0 +1,56 @@ +package main + +import ( + "fmt" + "log" + "io/ioutil" +) + +import ( + "github.com/BurntSushi/toml" + "github.com/gofiber/fiber/v2" +) + +type Resource struct { + Url string `toml:"url"` + Mime string `toml:"mime"` +} + +type ResourceDConfig struct { + Enabled bool `toml:"enabled"` + ListenURL string `toml:"listen_url"` +} + +type Config struct { + ResourceD ResourceDConfig `toml:"resourceD"` + Resource map[string]Resource `toml:"resource"` +} + +func main() { + var conf Config + + data, err := ioutil.ReadFile("resourced.toml") + if err != nil { panic(err) } + + a, err := toml.Decode(string(data), &conf) + if err != nil { panic(err) } + _ = a + + app := fiber.New() + + app.Get("/:id", func (c *fiber.Ctx) error { + + if ! conf.ResourceD.Enabled { + return c.Status(fiber.StatusNotFound).SendString("Resource not found") + } + + res, exists := conf.Resource[c.Params("id")] + if ! exists { + return c.Status(fiber.StatusNotFound).SendString("Resource not found") + } + + return c.SendString(fmt.Sprintf("Id: %s\nUrl: %s\nType: %s", c.Params("id"), res.Url, res.Mime)) + }) + + log.Fatal(app.Listen(conf.ResourceD.ListenURL)) +} \ No newline at end of file diff --git a/resource/resourced.toml.example b/resource/resourced.toml.example new file mode 100644 index 0000000..7cf863b --- /dev/null +++ b/resource/resourced.toml.example @@ -0,0 +1,31 @@ + +# The resourceD config +[ResourceD] + +# Whether to enable the resourceD. +# If this is false, resourceD will start but respond to +# all requests with 404 +# It is false by default because resourceD is not required in a default installation. +Enabled=true + +# URL to listen on +ListenUrl="0.0.0.0:80" + +# Resource ID must be like a java package name +# At least one X.X. is required +# +# Examples: +# org.university.logo +# dev.indie_guy.logo +# com.pany.logo +# Test your names here: https://regex101.com/r/wQdOup/2 +# +[Resource."com.example.logo"] +# Can also be an external link +# If an external link is specified, +# the resource will be returned as a 302 redirect to the link +Url="file:///some/where" + +# File type, as according to this: +# https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types +mime="image/jpg" -- 2.40.1 From 264d3595b62dfb9b01d7a620f861bf420a8bfb5f Mon Sep 17 00:00:00 2001 From: blek Date: Sun, 5 Nov 2023 13:36:24 +1000 Subject: [PATCH 02/13] actually serve the resources --- resource/main.go | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/resource/main.go b/resource/main.go index 5bf03de..ac40530 100644 --- a/resource/main.go +++ b/resource/main.go @@ -1,7 +1,6 @@ package main import ( - "fmt" "log" "io/ioutil" ) @@ -15,6 +14,9 @@ type Resource struct { Url string `toml:"url"` Mime string `toml:"mime"` } +func (self Resource) Get() ([]byte, error) { + return ioutil.ReadFile(self.Url[7:]) +} type ResourceDConfig struct { Enabled bool `toml:"enabled"` @@ -36,20 +38,36 @@ func main() { if err != nil { panic(err) } _ = a - app := fiber.New() + app := fiber.New(fiber.Config { + Prefork: true, + CaseSensitive: false, + StrictRouting: true, + ServerHeader: "", + AppName: "blek! File resourceD", + }) + + app.Use(func (c *fiber.Ctx) error { + if ! conf.ResourceD.Enabled { + return c.Status(fiber.StatusNotFound).SendString("ResourceD is disabled") + } + return c.Next() + }) app.Get("/:id", func (c *fiber.Ctx) error { - if ! conf.ResourceD.Enabled { - return c.Status(fiber.StatusNotFound).SendString("Resource not found") - } - res, exists := conf.Resource[c.Params("id")] if ! exists { return c.Status(fiber.StatusNotFound).SendString("Resource not found") } - return c.SendString(fmt.Sprintf("Id: %s\nUrl: %s\nType: %s", c.Params("id"), res.Url, res.Mime)) + data, err := res.Get() + if err != nil { + panic(err) + } + + c.Response().Header.SetContentType(res.Mime) + c.Response().Header.SetContentLength(len(data)) + return c.Send(data) }) log.Fatal(app.Listen(conf.ResourceD.ListenURL)) -- 2.40.1 From a9df435ebe4281fdea9acf5815d97a75402702e1 Mon Sep 17 00:00:00 2001 From: blek Date: Sun, 5 Nov 2023 13:50:53 +1000 Subject: [PATCH 03/13] support for http(s):// urls --- resource/main.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/resource/main.go b/resource/main.go index ac40530..0c349b6 100644 --- a/resource/main.go +++ b/resource/main.go @@ -2,6 +2,9 @@ package main import ( "log" + "fmt" + "regexp" + "strings" "io/ioutil" ) @@ -28,6 +31,19 @@ type Config struct { Resource map[string]Resource `toml:"resource"` } +func (self Config) Validate() int { + re, err := regexp.Compile(`^(file|http(s|))://`) + if err != nil { panic(err) } + + for key, res := range self.Resource { + if ! re.MatchString(res.Url) { + panic(fmt.Sprintf("Resource %s has invalid URL: %s\nOnly file://, http:// and https:// URLs are allowed", key, res.Url)) + } + } + + return 0 +} + func main() { var conf Config @@ -37,6 +53,8 @@ func main() { a, err := toml.Decode(string(data), &conf) if err != nil { panic(err) } _ = a + + conf.Validate() app := fiber.New(fiber.Config { Prefork: true, @@ -60,6 +78,12 @@ func main() { return c.Status(fiber.StatusNotFound).SendString("Resource not found") } + if ! strings.HasPrefix(res.Url, "file://") { + c.Location(res.Url) + c.Status(302) + return nil + } + data, err := res.Get() if err != nil { panic(err) @@ -67,6 +91,7 @@ func main() { c.Response().Header.SetContentType(res.Mime) c.Response().Header.SetContentLength(len(data)) + return c.Send(data) }) -- 2.40.1 From 318d7788a841476b92ba31ea59bfef1b6486d6a7 Mon Sep 17 00:00:00 2001 From: blek Date: Sun, 5 Nov 2023 14:09:30 +1000 Subject: [PATCH 04/13] proxied resources --- resource/main.go | 25 ++++++++++++++++++++++--- resource/resourced.toml.example | 12 +++++++++++- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/resource/main.go b/resource/main.go index 0c349b6..c0a4002 100644 --- a/resource/main.go +++ b/resource/main.go @@ -5,6 +5,7 @@ import ( "fmt" "regexp" "strings" + "net/http" "io/ioutil" ) @@ -14,8 +15,9 @@ import ( ) type Resource struct { - Url string `toml:"url"` - Mime string `toml:"mime"` + Url string `toml:"url"` + Proxied bool `toml:"proxied",default:false` + Mime string `toml:"mime"` } func (self Resource) Get() ([]byte, error) { return ioutil.ReadFile(self.Url[7:]) @@ -72,13 +74,30 @@ func main() { }) app.Get("/:id", func (c *fiber.Ctx) error { - res, exists := conf.Resource[c.Params("id")] if ! exists { return c.Status(fiber.StatusNotFound).SendString("Resource not found") } if ! strings.HasPrefix(res.Url, "file://") { + + if res.Proxied { + resp, err := http.Get(res.Url) + if err != nil { + log.Fatalln(err) + } else { + c.Response().Header.SetContentType(res.Mime) + // c.Response().Header.SetContentLength(resp.ContentLength) + buf, err := ioutil.ReadAll(resp.Body) + if err != nil { + log.Fatalln(err) + return c.Status(500).SendString("Internal error") + } + + return c.Send(buf) + } + } + c.Location(res.Url) c.Status(302) return nil diff --git a/resource/resourced.toml.example b/resource/resourced.toml.example index 7cf863b..e8fca0e 100644 --- a/resource/resourced.toml.example +++ b/resource/resourced.toml.example @@ -19,7 +19,6 @@ ListenUrl="0.0.0.0:80" # dev.indie_guy.logo # com.pany.logo # Test your names here: https://regex101.com/r/wQdOup/2 -# [Resource."com.example.logo"] # Can also be an external link # If an external link is specified, @@ -29,3 +28,14 @@ Url="file:///some/where" # File type, as according to this: # https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types mime="image/jpg" + +# (if url is an http(s) url)) +# Whether to proxy the resource. +# true - Send the resource like a regular file +# false - Send a 302 HTTP redirect to the URL (default) +# +# It is not recommended to set this to `true` +# unless you are referring to a resource that is +# available only in the local network +# +# proxied=true \ No newline at end of file -- 2.40.1 From 3902d2aecb7a3d9fd8de67485d36e5d04f393efa Mon Sep 17 00:00:00 2001 From: blek Date: Sun, 5 Nov 2023 15:05:30 +1000 Subject: [PATCH 05/13] cached proxied resources --- resource/main.go | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/resource/main.go b/resource/main.go index c0a4002..513c349 100644 --- a/resource/main.go +++ b/resource/main.go @@ -45,6 +45,25 @@ func (self Config) Validate() int { return 0 } +func (self Resource) GetProxied() ([]byte, error) { + + cached, exists := ProxyResourceCache[self.Url]; + if exists { + return cached, nil + } + + resp, err := http.Get(self.Url) + if err != nil { return make([]byte, 0, 0), err } + + buf, err := ioutil.ReadAll(resp.Body) + if err != nil { return make([]byte, 0, 0), err } + + ProxyResourceCache[self.Url] = buf + + return buf, nil +} + +var ProxyResourceCache map[string][]byte = make(map[string][]byte) func main() { var conf Config @@ -82,19 +101,15 @@ func main() { if ! strings.HasPrefix(res.Url, "file://") { if res.Proxied { - resp, err := http.Get(res.Url) + data, err := res.GetProxied() if err != nil { log.Fatalln(err) + // we failed, send a redirect instead + // (next executed line would be 115) } else { c.Response().Header.SetContentType(res.Mime) - // c.Response().Header.SetContentLength(resp.ContentLength) - buf, err := ioutil.ReadAll(resp.Body) - if err != nil { - log.Fatalln(err) - return c.Status(500).SendString("Internal error") - } - - return c.Send(buf) + c.Response().Header.SetContentLength(len(data)) + return c.Send(data) } } -- 2.40.1 From 354163a0007a007d59b61e4dfe18097dbab254a6 Mon Sep 17 00:00:00 2001 From: blek Date: Sun, 5 Nov 2023 15:25:21 +1000 Subject: [PATCH 06/13] add option to configure min size to cache proxy --- resource/go.mod | 1 + resource/go.sum | 2 ++ resource/main.go | 16 +++++++++++++--- resource/resourced.toml.example | 5 +++++ 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/resource/go.mod b/resource/go.mod index 0ee4b39..4333b70 100644 --- a/resource/go.mod +++ b/resource/go.mod @@ -5,6 +5,7 @@ go 1.21.3 require ( github.com/BurntSushi/toml v1.3.2 // indirect github.com/andybalholm/brotli v1.0.5 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/gofiber/fiber/v2 v2.50.0 // indirect github.com/google/uuid v1.3.1 // indirect github.com/klauspost/compress v1.16.7 // indirect diff --git a/resource/go.sum b/resource/go.sum index 76c49cb..c9e3a79 100644 --- a/resource/go.sum +++ b/resource/go.sum @@ -2,6 +2,8 @@ github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8 github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/gofiber/fiber/v2 v2.50.0 h1:ia0JaB+uw3GpNSCR5nvC5dsaxXjRU5OEu36aytx+zGw= github.com/gofiber/fiber/v2 v2.50.0/go.mod h1:21eytvay9Is7S6z+OgPi7c7n4++tnClWmhpimVHMimw= github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= diff --git a/resource/main.go b/resource/main.go index 513c349..2822d34 100644 --- a/resource/main.go +++ b/resource/main.go @@ -12,6 +12,7 @@ import ( import ( "github.com/BurntSushi/toml" "github.com/gofiber/fiber/v2" + "github.com/dustin/go-humanize" ) type Resource struct { @@ -24,8 +25,9 @@ func (self Resource) Get() ([]byte, error) { } type ResourceDConfig struct { - Enabled bool `toml:"enabled"` - ListenURL string `toml:"listen_url"` + Enabled bool `toml:"enabled"` + ListenURL string `toml:"listen_url"` + ProxyCacheMinSize string `toml:"proxy_cache_min_size",default:5MB` } type Config struct { @@ -58,12 +60,16 @@ func (self Resource) GetProxied() ([]byte, error) { buf, err := ioutil.ReadAll(resp.Body) if err != nil { return make([]byte, 0, 0), err } - ProxyResourceCache[self.Url] = buf + // cache only those that are less than 5 mb + if len(buf) > ProxyCacheMinSize { + ProxyResourceCache[self.Url] = buf + } return buf, nil } var ProxyResourceCache map[string][]byte = make(map[string][]byte) +var ProxyCacheMinSize int func main() { var conf Config @@ -75,6 +81,10 @@ func main() { if err != nil { panic(err) } _ = a + cache_min, err := humanize.ParseBytes(conf.ResourceD.ProxyCacheMinSize) + if err != nil { panic(err) } + ProxyCacheMinSize = int(cache_min) + conf.Validate() app := fiber.New(fiber.Config { diff --git a/resource/resourced.toml.example b/resource/resourced.toml.example index e8fca0e..043888b 100644 --- a/resource/resourced.toml.example +++ b/resource/resourced.toml.example @@ -11,6 +11,11 @@ Enabled=true # URL to listen on ListenUrl="0.0.0.0:80" +# Minibal size of a file to be cached +# File size is parsed via this library: +# https://github.com/dustin/go-humanize +proxy_cache_min_size="5MB" + # Resource ID must be like a java package name # At least one X.X. is required # -- 2.40.1 From 47287a319b4af95956b21c1f7e743e8acbf71b42 Mon Sep 17 00:00:00 2001 From: blek Date: Sun, 5 Nov 2023 15:26:58 +1000 Subject: [PATCH 07/13] fix wrong symbols in the example config --- resource/resourced.toml.example | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/resource/resourced.toml.example b/resource/resourced.toml.example index 043888b..c74bae7 100644 --- a/resource/resourced.toml.example +++ b/resource/resourced.toml.example @@ -1,15 +1,15 @@ # The resourceD config -[ResourceD] +[resourceD] # Whether to enable the resourceD. # If this is false, resourceD will start but respond to # all requests with 404 # It is false by default because resourceD is not required in a default installation. -Enabled=true +enabled=true # URL to listen on -ListenUrl="0.0.0.0:80" +listen_url="0.0.0.0:80" # Minibal size of a file to be cached # File size is parsed via this library: @@ -24,11 +24,12 @@ proxy_cache_min_size="5MB" # dev.indie_guy.logo # com.pany.logo # Test your names here: https://regex101.com/r/wQdOup/2 -[Resource."com.example.logo"] +[resource."com.example.logo"] + # Can also be an external link # If an external link is specified, # the resource will be returned as a 302 redirect to the link -Url="file:///some/where" +url="file:///some/where" # File type, as according to this: # https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types -- 2.40.1 From 855ac150ef338346d1f95cae74d77c2a588d9c4b Mon Sep 17 00:00:00 2001 From: blek Date: Sun, 5 Nov 2023 15:33:30 +1000 Subject: [PATCH 08/13] add route to get if the resourced is enabled --- resource/main.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/resource/main.go b/resource/main.go index 2822d34..c17a903 100644 --- a/resource/main.go +++ b/resource/main.go @@ -95,6 +95,10 @@ func main() { AppName: "blek! File resourceD", }) + app.Get("/info/is_enabled", func (c *fiber.Ctx) error { + return c.JSON(conf.ResourceD.Enabled) + }) + app.Use(func (c *fiber.Ctx) error { if ! conf.ResourceD.Enabled { return c.Status(fiber.StatusNotFound).SendString("ResourceD is disabled") -- 2.40.1 From d1d7c45c5eaddfc4e912e2290866aa82a79c4965 Mon Sep 17 00:00:00 2001 From: blek Date: Sun, 5 Nov 2023 15:49:44 +1000 Subject: [PATCH 09/13] add swagger reference --- resource/api_ref.swagger.yml | 48 ++++++++++++++++++++++++++++++++++++ resource/main.go | 3 ++- 2 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 resource/api_ref.swagger.yml diff --git a/resource/api_ref.swagger.yml b/resource/api_ref.swagger.yml new file mode 100644 index 0000000..ea60424 --- /dev/null +++ b/resource/api_ref.swagger.yml @@ -0,0 +1,48 @@ +openapi: 3.0.3 +info: + title: ResourceD API + description: |- + This is the ResourceD API docs. + license: + name: GPLv3 + url: https://www.gnu.org/licenses/gpl-3.0.en.html + version: "1.0" +servers: + - url: http://localhost/resource +tags: + - name: Data API + description: API for serving data + - name: System API + description: API for serving system data + +paths: + /{id}: + get: + description: Get a resource + summary: Get a resource. Send browser requests to this URL + tags: + - Data API + responses: + 200: + description: Returns a resource in its binary data + 302: + description: Resource is an external (http) link and the redirect is being forwarded to that link + 404: + description: Not found + 500: + description: Internal error + /info/is_enabled: + get: + description: Check if resourceD is enabled + summary: Check if resourceD is enabled + tags: + - System API + responses: + 200: + description: Ok + content: + application/json: + schema: + type: boolean + example: true + \ No newline at end of file diff --git a/resource/main.go b/resource/main.go index c17a903..fc910a1 100644 --- a/resource/main.go +++ b/resource/main.go @@ -119,7 +119,8 @@ func main() { if err != nil { log.Fatalln(err) // we failed, send a redirect instead - // (next executed line would be 115) + // the next line would be the one with + // c.Location(res.Url) } else { c.Response().Header.SetContentType(res.Mime) c.Response().Header.SetContentLength(len(data)) -- 2.40.1 From 9a9f3cd58d4c1d640c62a75e769d94f5c75bda3c Mon Sep 17 00:00:00 2001 From: blek Date: Sat, 11 Nov 2023 17:21:22 +1000 Subject: [PATCH 10/13] add air to resourceD more info: https://github.com/cosmtrek/air --- resource/.air.toml | 46 +++++++++++++++++++++++++++++++++++++++++++++ resource/.gitignore | 1 + 2 files changed, 47 insertions(+) create mode 100644 resource/.air.toml diff --git a/resource/.air.toml b/resource/.air.toml new file mode 100644 index 0000000..b928116 --- /dev/null +++ b/resource/.air.toml @@ -0,0 +1,46 @@ +root = "." +testdata_dir = "testdata" +tmp_dir = "tmp" + +[build] + args_bin = [] + bin = "./tmp/main" + cmd = "go build -o ./tmp/main ." + delay = 1000 + exclude_dir = ["assets", "tmp", "vendor", "testdata"] + exclude_file = [] + exclude_regex = [ ".*\\.md" ] + exclude_unchanged = false + follow_symlink = false + full_bin = "" + include_dir = [] + include_ext = ["go", "tpl", "tmpl", "html"] + include_file = [] + kill_delay = "0s" + log = "build-errors.log" + poll = false + poll_interval = 0 + post_cmd = [] + pre_cmd = [] + rerun = false + rerun_delay = 500 + send_interrupt = false + stop_on_error = false + +[color] + app = "" + build = "yellow" + main = "magenta" + runner = "green" + watcher = "cyan" + +[log] + main_only = false + time = false + +[misc] + clean_on_exit = false + +[screen] + clear_on_rebuild = false + keep_scroll = true diff --git a/resource/.gitignore b/resource/.gitignore index 4bf8d84..bac04d0 100644 --- a/resource/.gitignore +++ b/resource/.gitignore @@ -2,3 +2,4 @@ resourced.toml resourced .DS_Store .env +tmp -- 2.40.1 From c43b8c8fb3e2e1dc59f3d4bd7c4f5e901c716c32 Mon Sep 17 00:00:00 2001 From: blek Date: Sat, 11 Nov 2023 17:34:21 +1000 Subject: [PATCH 11/13] deploy resourceD within dev compose file --- conf/caddy/Caddyfile | 4 ++++ containers/go-dev.Dockerfile | 10 ++++++++++ docker-compose.dev.yml | 9 +++++++++ 3 files changed, 23 insertions(+) create mode 100644 containers/go-dev.Dockerfile diff --git a/conf/caddy/Caddyfile b/conf/caddy/Caddyfile index e52811c..8984b7a 100644 --- a/conf/caddy/Caddyfile +++ b/conf/caddy/Caddyfile @@ -4,4 +4,8 @@ uri * strip_prefix /qr reverse_proxy http://qr } + handle /resource/* { + uri * strip_prefix /resource + reverse_proxy http://resourced + } } \ No newline at end of file diff --git a/containers/go-dev.Dockerfile b/containers/go-dev.Dockerfile new file mode 100644 index 0000000..c78435d --- /dev/null +++ b/containers/go-dev.Dockerfile @@ -0,0 +1,10 @@ +FROM golang:alpine + +RUN go install github.com/cosmtrek/air@latest + +WORKDIR /opt/code + +# The directory will be mounted +# COPY . . + +CMD [ "air", "-c", ".air.toml" ] diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 774aea5..f3d3871 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -21,6 +21,15 @@ services: volumes: - './janitor:/opt/code' - './volatile/files:/opt/user_uploads' + resourced: + build: + context: containers + dockerfile: go-dev.Dockerfile + networks: + bfile: + volumes: + - './resource:/opt/code' + - '/opt/code/tmp' caddy: image: caddy:alpine volumes: -- 2.40.1 From b834f5df9eb0afed9151e1e7bc80b7b0776327c7 Mon Sep 17 00:00:00 2001 From: blek Date: Sat, 11 Nov 2023 17:58:50 +1000 Subject: [PATCH 12/13] deploy resourced to production dockerfile Note: I did not test the compose file on my machine before submitting the commit. May or may not work. --- docker-compose.prod.yml | 9 +++++++++ resource/Dockerfile.prod | 22 ++++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 resource/Dockerfile.prod diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 4ac8eb2..fcbd1fe 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -19,6 +19,15 @@ services: volumes: - './janitor:/config:ro' - './volatile/files:/opt/user_uploads' + resourced: + build: + context: resource + dockerfile: Dockerfile.prod + networks: + bfile: + pid: host # prefork + volumes: + - './resource:/opt/cont' caddy: image: caddy:alpine volumes: diff --git a/resource/Dockerfile.prod b/resource/Dockerfile.prod new file mode 100644 index 0000000..0451ed0 --- /dev/null +++ b/resource/Dockerfile.prod @@ -0,0 +1,22 @@ +FROM golang:alpine3.17 as builder + +WORKDIR /opt/build +COPY . . + +RUN apk add --no-cache git musl-dev upx binutils + +RUN go build . && \ + strip resourced && \ + upx resourced + +FROM alpine:3.17 + +WORKDIR /opt/code +COPY --from=builder /opt/build/resourced /usr/bin/resourced + +# Note +# ----- +# Since this is running with prefork, don't +# forget to set --pid=host when running this app + +CMD [ "sh", "-c", "/usr/bin/resourced" ] -- 2.40.1 From 5a3153bbbee61f52166a401436ce1deccacaed65 Mon Sep 17 00:00:00 2001 From: blek Date: Sat, 11 Nov 2023 18:02:04 +1000 Subject: [PATCH 13/13] print a warning if resourced is disabled --- resource/main.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/resource/main.go b/resource/main.go index fc910a1..a63adde 100644 --- a/resource/main.go +++ b/resource/main.go @@ -86,6 +86,10 @@ func main() { ProxyCacheMinSize = int(cache_min) conf.Validate() + + if ! conf.ResourceD.Enabled { + fmt.Println("\x1b[33m[warn] resourceD is disabled. No resources will be served\x1b[0m") + } app := fiber.New(fiber.Config { Prefork: true, -- 2.40.1