From e531fd949319d8b4169be13c6c592f4bd1a850c1 Mon Sep 17 00:00:00 2001 From: xuu Date: Sun, 5 Nov 2023 08:30:15 -0700 Subject: [PATCH] feat: add image --- .gitea/workflows/deploy.yml | 8 +- go.mod | 1 + go.sum | 3 + v2/app.image.go | 26 ++++ v2/artifact/artifact.go | 12 -- v2/go.mod | 8 +- v2/image/image.go | 256 ++++++++++++++++++++++++++++++++++++ 7 files changed, 294 insertions(+), 20 deletions(-) create mode 100644 v2/app.image.go create mode 100644 v2/image/image.go diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index b89b3f4..820adf7 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -1,8 +1,8 @@ name: Deploy on: - # push: - # branches: [ "master" ] + push: + branches: [ "master" ] release: types: [ published ] @@ -16,10 +16,10 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: 1.21.1 + go-version: 1.21.3 - name: Install - run: go install -ldflags "-s -w" go.sour.is/paste/cmd/paste/v2@latest + run: go install -ldflags "-s -w" go.sour.is/paste/cmd/paste/v2@master - run: mv $(go env GOPATH)/bin/paste sour.is-paste diff --git a/go.mod b/go.mod index f9f657e..1367fa9 100644 --- a/go.mod +++ b/go.mod @@ -55,6 +55,7 @@ require ( github.com/vektah/gqlparser/v2 v2.5.6 // indirect github.com/yeqown/reedsolomon v1.0.0 // indirect github.com/yosssi/gmq v0.0.1 // indirect + go.opentelemetry.io/otel v1.19.0 golang.org/x/image v0.5.0 // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/text v0.13.0 // indirect diff --git a/go.sum b/go.sum index ab47b97..5e1a2f9 100644 --- a/go.sum +++ b/go.sum @@ -313,6 +313,7 @@ github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -347,6 +348,8 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs= +go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= diff --git a/v2/app.image.go b/v2/app.image.go new file mode 100644 index 0000000..72567f0 --- /dev/null +++ b/v2/app.image.go @@ -0,0 +1,26 @@ +package main + +import ( + "context" + + "go.sour.is/paste/v2/image" + "go.sour.is/pkg/env" + "go.sour.is/pkg/lg" + "go.sour.is/pkg/service" +) + +var _ = apps.Register(40, func(ctx context.Context, svc *service.Harness) error { + _, span := lg.Span(ctx) + defer span.End() + + store := env.Default("IMAGE_STORE", "data/") + + a, err := image.New(store, -1) + if err != nil { + return err + } + svc.Add(a) + span.AddEvent("register image") + + return nil +}) diff --git a/v2/artifact/artifact.go b/v2/artifact/artifact.go index 1a7cdbf..7dd8f20 100644 --- a/v2/artifact/artifact.go +++ b/v2/artifact/artifact.go @@ -23,18 +23,6 @@ import ( "github.com/gomarkdown/markdown/parser" ) -// func init() { -// a := &Artifact{} -// httpsrv.RegisterModule("artifact", a.config) - -// httpsrv.HttpRegister("artifact", httpsrv.HttpRoutes{ -// {Name: "get-path", Method: "GET", Pattern: "/a/{name}/{path:.*}", HandlerFunc: a.get}, -// {Name: "get", Method: "GET", Pattern: "/a/{name}", HandlerFunc: a.get}, -// {Name: "put", Method: "PUT", Pattern: "/a", HandlerFunc: a.put}, -// {Name: "get", Method: "GET", Pattern: "/a", HandlerFunc: a.list}, -// }) -// } - // artifact stores items to disk type artifact struct { store string diff --git a/v2/go.mod b/v2/go.mod index 7b4d3c1..ffad8d9 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -1,10 +1,13 @@ module go.sour.is/paste/v2 -go 1.21.3 +go 1.21 require ( github.com/gomarkdown/markdown v0.0.0-20200824053859-8c8b3816f167 + github.com/h2non/filetype v1.1.0 github.com/rs/cors v1.6.0 + go.opentelemetry.io/otel v1.18.0 + go.opentelemetry.io/otel/trace v1.18.0 go.sour.is/paste v0.0.0-20231022150928-96338f2f4441 go.sour.is/pkg v0.0.6 golang.org/x/sys v0.13.0 @@ -23,7 +26,6 @@ require ( github.com/golang/protobuf v1.5.3 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect - github.com/h2non/filetype v1.1.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/magiconair/properties v1.8.1 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect @@ -42,14 +44,12 @@ require ( github.com/subosito/gotenv v1.2.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 // indirect go.opentelemetry.io/contrib/instrumentation/runtime v0.42.0 // indirect - go.opentelemetry.io/otel v1.18.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.18.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.18.0 // indirect go.opentelemetry.io/otel/exporters/prometheus v0.41.0 // indirect go.opentelemetry.io/otel/metric v1.18.0 // indirect go.opentelemetry.io/otel/sdk v1.18.0 // indirect go.opentelemetry.io/otel/sdk/metric v0.41.0 // indirect - go.opentelemetry.io/otel/trace v1.18.0 // indirect go.opentelemetry.io/proto/otlp v1.0.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/net v0.17.0 // indirect diff --git a/v2/image/image.go b/v2/image/image.go new file mode 100644 index 0000000..e6ac9fd --- /dev/null +++ b/v2/image/image.go @@ -0,0 +1,256 @@ +package image + +import ( + "context" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/h2non/filetype" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + "golang.org/x/sys/unix" + + "go.sour.is/paste/src/pkg/readutil" + "go.sour.is/pkg/lg" +) + +type image struct { + store string + maxSize int64 +} + +const DefaultMaxSize = 500 * 1024 * 1024 + +func New(store string, maxSize int64) (a *image, err error) { + a = &image{ + store: store, + maxSize: DefaultMaxSize, + } + + if maxSize > 0 { + a.maxSize = maxSize + } + + if !chkStore(a.store) { + return nil, fmt.Errorf("image Store location [%s] does not exist or is not writable", a.store) + } + + return a, nil +} +func (a *image) RegisterHTTP(mux *http.ServeMux) { + mux.Handle("/i", http.StripPrefix("/i", a)) + mux.Handle("/i/", http.StripPrefix("/i/", a)) + mux.Handle("3/upload", http.StripPrefix("/3/upload", a)) + +} +func (a *image) ServeHTTP(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + ctx, span := lg.Span(ctx) + defer span.End() + + switch r.Method { + case http.MethodGet: + name := strings.TrimPrefix(r.URL.Path, "/") + + a.get(ctx, w, name) + case http.MethodPost: + length := 0 + if h := r.Header.Get("Content-Length"); h != "" { + if i, err := strconv.Atoi(h); err != nil { + length = i + } + } + id, err := a.put(ctx, w, r.Body, length) + switch { + case errors.Is(err, ErrGone): + w.WriteHeader(http.StatusGone) + case errors.Is(err, ErrNotFound): + w.WriteHeader(http.StatusNotFound) + case errors.Is(err, ErrReadingContent): + w.WriteHeader(http.StatusInternalServerError) + case errors.Is(err, ErrUnsupportedType): + w.WriteHeader(http.StatusUnsupportedMediaType) + } + + type data struct{ + Link string `json:"link"` + DeleteHash string `json:"deletehash"` + } + var resp = struct{ + Data data `json:"data"` + Success bool `json:"success"` + Status int `json:"status"` + }{ + Data: data{ + Link: fmt.Sprintf("https://%s/%s", r.Host, id), + }, + Success: true, + Status: 200, + } + + json.NewEncoder(w).Encode(resp) + + default: + http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) + } +} + +func (a *image) get(ctx context.Context, w http.ResponseWriter, name string) error { + _, span := lg.Span(ctx) + defer span.End() + + ext := filepath.Ext(name) + id := strings.TrimSuffix(name, ext) + + fname := filepath.Join(a.store, id) + + if !chkFile(fname) { + return fmt.Errorf("%w: %s", ErrNotFound, fname) + } + + if chkGone(fname) { + return fmt.Errorf("%w: %s", ErrGone, fname) + } + + f, err := os.Open(fname) + if err != nil { + return err + } + defer f.Close() + + pr := readutil.NewPreviewReader(f) + + mime, err := readutil.ReadMIME(pr, name) + if err != nil { + return err + } + w.Header().Set("Content-Type", mime) + w.Header().Set("X-Content-Type-Options", "nosniff") + + _, _ = io.Copy(w, pr.Drain()) + return nil +} + +func (a *image) put(ctx context.Context, w http.ResponseWriter, r io.ReadCloser, length int) (string, error) { + _, span := lg.Span(ctx) + defer span.End() + + defer r.Close() + + if length > 0 { + if int64(length) > a.maxSize { + return "", ErrSizeTooLarge + } + } + + rdr := io.LimitReader(r, a.maxSize) + pr := readutil.NewPreviewReader(rdr) + if !isImageOrVideo(pr) { + return "", ErrUnsupportedType + } + rdr = pr.Drain() + + s256 := sha256.New() + tmp, err := os.CreateTemp(a.store, "image-") + if err != nil { + return "", fmt.Errorf("%w: %w", ErrBadInput, err) + } + + defer os.Remove(tmp.Name()) + + m := io.MultiWriter(s256, tmp) + if _, err := io.Copy(m, rdr); err != nil { + return "", fmt.Errorf("%w: %w", ErrBadInput, err) + } + tmp.Close() + + id := base64.RawURLEncoding.EncodeToString(s256.Sum(nil)[12:]) + fname := filepath.Join(a.store, id) + + span.AddEvent("image: moving file", trace.WithAttributes(attribute.String("src", tmp.Name()), attribute.String("dst", fname))) + _ = os.Rename(tmp.Name(), fname) + + return id, nil +} + +func isImageOrVideo(in io.Reader) bool { + buf := make([]byte, 320) + _, err := in.Read(buf) + if err != nil { + return false + } + return filetype.IsImage(buf) || filetype.IsVideo(buf) +} + +func chkStore(path string) bool { + file, err := os.Stat(path) + + if err != nil && os.IsNotExist(err) { + err = os.MkdirAll(path, 0744) + if err != nil { + return false + } + + file, err = os.Stat(path) + } + + if err != nil { + return false + } + + if !file.IsDir() { + return false + } + + if unix.Access(path, unix.W_OK&unix.R_OK) != nil { + return false + } + + return true +} +func chkFile(path string) bool { + file, err := os.Stat(path) + if err != nil { + return false + } + + if file.IsDir() { + return false + } + + if unix.Access(path, unix.W_OK&unix.R_OK) != nil { + return false + } + + return true +} +func chkGone(path string) bool { + file, err := os.Stat(path) + if err != nil { + return true + } + + if file.Size() == 0 { + return true + } + + return false +} + +var ( + ErrNotFound = errors.New("not found") + ErrGone = errors.New("gone") + ErrReadingContent = errors.New("reading content") + ErrSizeTooLarge = errors.New("size too large") + ErrBadInput = errors.New("bad input") + ErrUnsupportedType = errors.New("unsupported type") +)