add image and artifact stores

This commit is contained in:
Xuu
2020-09-04 17:18:58 -06:00
parent a55162b6b4
commit 61e4b82e84
18 changed files with 2090 additions and 224 deletions

View File

@@ -254,7 +254,7 @@ func publicSwaggerJson() (*asset, error) {
return nil, err
}
info := bindataFileInfo{name: "public/swagger.json", size: 4895, mode: os.FileMode(0644), modTime: time.Unix(1598898929, 0)}
info := bindataFileInfo{name: "public/swagger.json", size: 4895, mode: os.FileMode(0644), modTime: time.Unix(1599169378, 0)}
a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x71, 0xcc, 0xe, 0x5f, 0xc1, 0xe9, 0xd6, 0xa9, 0x46, 0x93, 0xde, 0x3d, 0x71, 0x10, 0x7f, 0xa9, 0x1e, 0x98, 0x11, 0x70, 0xc6, 0xcf, 0x1c, 0xa, 0x57, 0x84, 0xfc, 0x3c, 0xf7, 0x28, 0x3, 0x67}}
return a, nil
}

193
src/routes/artifact.go Normal file
View File

@@ -0,0 +1,193 @@
package routes
import (
"archive/tar"
"crypto/sha256"
"encoding/base64"
"io"
"io/ioutil"
"net/http"
"os"
"path/filepath"
"strconv"
"strings"
"github.com/gorilla/mux"
"sour.is/x/toolbox/httpsrv"
"sour.is/x/toolbox/log"
)
func init() {
a := &Artifact{}
httpsrv.RegisterModule("artifact", a.config)
httpsrv.HttpRegister("artifact", httpsrv.HttpRoutes{
{Name: "get-path", Method: "GET", Pattern: "/a/{name}/{path:.*}", HandlerFunc: a.get},
{Name: "get", Method: "GET", Pattern: "/a/{name}", HandlerFunc: a.get},
{Name: "put", Method: "PUT", Pattern: "/a", HandlerFunc: a.put},
})
}
type Artifact struct {
store string
maxSize int64
}
func (a *Artifact) config(config map[string]string) {
a.store = "data/"
if config["store"] != "" {
a.store = config["store"]
}
if !chkStore(a.store) {
log.Criticalf("Artifact Store location [%s] does not exist or is not writable.", a.store)
}
log.Noticef("Artifact Store location set to [%s]", a.store)
a.maxSize = 50 * 1024 * 1024
if s, ok := config["max-size"]; ok {
if i, err := strconv.Atoi(s); err == nil {
a.maxSize = int64(i)
}
}
log.Noticef("Artifact: store max-size set to [%d]", a.maxSize)
}
func (a *Artifact) get(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
name := vars["name"]
path, hasPath := vars["path"]
ext := filepath.Ext(name)
id := strings.TrimSuffix(name, ext)
fname := filepath.Join(a.store, id)
if !chkFile(fname) {
httpsrv.WriteError(w, http.StatusNotFound, "Not Found")
return
}
if chkGone(fname) {
httpsrv.WriteError(w, http.StatusGone, "Gone")
return
}
f, err := os.Open(fname)
if err != nil {
log.Fatal(err)
}
defer f.Close()
if !hasPath {
pr := NewPreviewReader(f)
mime, err := ReadMIME(pr, name)
if err != nil {
httpsrv.WriteError(w, http.StatusInternalServerError, err.Error())
return
}
w.Header().Set("Content-Type", mime)
w.Header().Set("X-Content-Type-Options", "nosniff")
io.Copy(w, pr.Drain())
return
}
rdr, err := Decompress(f)
if err != nil {
httpsrv.WriteError(w, http.StatusInternalServerError, err.Error())
return
}
if cl, ok := rdr.(io.ReadCloser); ok {
defer cl.Close()
}
tr := tar.NewReader(rdr)
if path == "..." {
var paths []string
for {
hdr, err := tr.Next()
if err == io.EOF {
break // End of archive
}
if err != nil {
log.Fatal(err)
}
paths = append(paths, hdr.Name)
}
httpsrv.WriteObject(w, http.StatusOK, paths)
return
}
for {
hdr, err := tr.Next()
if err == io.EOF {
break // End of archive
}
if err != nil {
log.Fatal(err)
}
if hdr.Name == path {
pr := NewPreviewReader(tr)
mime, err := ReadMIME(pr, hdr.Name)
if err != nil {
httpsrv.WriteError(w, http.StatusInternalServerError, err.Error())
return
}
w.Header().Set("Content-Type", mime)
if _, err := io.Copy(w, pr.Drain()); err != nil {
log.Fatal(err)
}
w.WriteHeader(http.StatusOK)
return
}
}
httpsrv.WriteError(w, http.StatusNotFound, "Not Found in Archive")
}
func (a *Artifact) put(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
if h := r.Header.Get("Content-Length"); h != "" {
if i, err := strconv.Atoi(h); err == nil {
if int64(i) > a.maxSize {
http.Error(w, "ERR Size Too Large", http.StatusRequestEntityTooLarge)
return
}
}
}
rdr := io.LimitReader(r.Body, 500*1024*1024)
pr := NewPreviewReader(rdr)
rdr = pr.Drain()
s256 := sha256.New()
tmp, err := ioutil.TempFile(a.store, "arch-")
if err != nil {
httpsrv.WriteError(w, 400, err.Error())
return
}
defer os.Remove(tmp.Name())
m := io.MultiWriter(s256, tmp)
if _, err := io.Copy(m, rdr); err != nil {
httpsrv.WriteError(w, 400, err.Error())
}
tmp.Close()
id := base64.RawURLEncoding.EncodeToString(s256.Sum(nil)[12:])
fname := filepath.Join(a.store, id)
log.Debugs("Artifact: moving file", "src", tmp.Name(), "dst", fname)
os.Rename(tmp.Name(), fname)
httpsrv.WriteText(w, http.StatusCreated, id)
}

File diff suppressed because one or more lines are too long

37
src/routes/decompress.go Normal file
View File

@@ -0,0 +1,37 @@
package routes
import (
"compress/bzip2"
"compress/gzip"
"io"
"github.com/andybalholm/brotli"
xz "github.com/remyoudompheng/go-liblzma"
"sour.is/x/toolbox/log"
)
func Decompress(in io.Reader) (io.Reader, error) {
rdr := NewPreviewReader(in)
mime, err := ReadMIMEWithSize(rdr, "", 32768)
if err != nil {
return rdr.Drain(), err
}
r := rdr.Drain()
switch mime {
case "application/gzip":
r, err = gzip.NewReader(r)
case "application/x-bzip2":
r = bzip2.NewReader(r)
case "application/x-xz":
r, err = xz.NewReader(r)
case "application/brotli":
r = brotli.NewReader(r)
}
log.Debugs("Decompress:", "mime", mime)
return r, err
}

149
src/routes/image.go Normal file
View File

@@ -0,0 +1,149 @@
package routes
import (
"crypto/sha256"
"encoding/base64"
"io"
"io/ioutil"
"net/http"
"os"
"path/filepath"
"strconv"
"strings"
"github.com/gorilla/mux"
"github.com/h2non/filetype"
"sour.is/x/toolbox/httpsrv"
"sour.is/x/toolbox/log"
)
func init() {
a := &Image{}
httpsrv.RegisterModule("image", a.config)
httpsrv.HttpRegister("image", httpsrv.HttpRoutes{
{Name: "getImage", Method: "GET", Pattern: "/i/{name}", HandlerFunc: a.get},
{Name: "putImage", Method: "PUT", Pattern: "/i", HandlerFunc: a.put},
})
}
type Image struct {
store string
maxSize int64
}
func (a *Image) config(config map[string]string) {
a.store = "data/"
if config["store"] != "" {
a.store = config["store"]
}
if !chkStore(a.store) {
log.Criticalf("Image: store location [%s] does not exist or is not writable.", a.store)
}
log.Noticef("Image: store location set to [%s]", a.store)
a.maxSize = 10 * 1024 * 1024
if s, ok := config["max-size"]; ok {
if i, err := strconv.Atoi(s); err == nil {
a.maxSize = int64(i)
}
}
log.Noticef("Image: store max-size set to [%d]", a.maxSize)
}
func (a *Image) get(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
name := vars["name"]
ext := filepath.Ext(name)
id := strings.TrimSuffix(name, ext)
fname := filepath.Join(a.store, id)
if !chkFile(fname) {
httpsrv.WriteError(w, http.StatusNotFound, "Not Found")
return
}
if chkGone(fname) {
httpsrv.WriteError(w, http.StatusGone, "Gone")
return
}
f, err := os.Open(fname)
if err != nil {
log.Fatal(err)
}
defer f.Close()
pr := NewPreviewReader(f)
mime, err := ReadMIME(pr, name)
w.Header().Set("Content-Type", mime)
w.Header().Set("X-Content-Type-Options", "nosniff")
io.Copy(w, pr.Drain())
}
func (a *Image) put(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
if h := r.Header.Get("Content-Length"); h != "" {
log.Debugs("Artifact:", "content-length", h, "max-length", a.maxSize)
if i, err := strconv.Atoi(h); err == nil {
if int64(i) > a.maxSize {
log.Error("ERR Size Too Large")
http.Error(w, "ERR Size Too Large", http.StatusRequestEntityTooLarge)
return
}
}
}
rdr := io.LimitReader(r.Body, a.maxSize)
pr := NewPreviewReader(rdr)
if !isImageOrVideo(pr) {
httpsrv.WriteError(w, http.StatusUnsupportedMediaType, "ERR Not Image")
return
}
rdr = pr.Drain()
s256 := sha256.New()
tmp, err := ioutil.TempFile(a.store, "image-")
if err != nil {
httpsrv.WriteError(w, 400, err.Error())
return
}
defer os.Remove(tmp.Name())
m := io.MultiWriter(s256, tmp)
if _, err = io.Copy(m, rdr); err != nil {
httpsrv.WriteError(w, 400, err.Error())
return
}
tmp.Close()
id := base64.RawURLEncoding.EncodeToString(s256.Sum(nil)[12:])
fname := filepath.Join(a.store, id)
log.Debugs("Image: moving file", "src", tmp.Name(), "dst", fname)
err = os.Rename(tmp.Name(), fname)
if err != nil {
httpsrv.WriteError(w, 400, err.Error())
return
}
httpsrv.WriteText(w, http.StatusCreated, id)
}
func isImageOrVideo(in io.Reader) bool {
buf := make([]byte, 320)
_, err := in.Read(buf)
if err != nil {
return false
}
return filetype.IsImage(buf) || filetype.IsVideo(buf)
}

View File

@@ -5,11 +5,11 @@ import (
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/json"
"io"
"io/ioutil"
"net/http"
"os"
"path/filepath"
"strconv"
"strings"
"time"
@@ -20,55 +20,67 @@ import (
"sour.is/x/toolbox/log"
)
var store string
var randBytes int
func init() {
httpsrv.RegisterModule("paste", setConfig)
p := &Paste{}
httpsrv.RegisterModule("paste", p.setConfig)
httpsrv.HttpRegister("paste", httpsrv.HttpRoutes{
{Name: "getRandom", Method: "GET", Pattern: "/paste/rng", HandlerFunc: getRandom},
{Name: "getPaste", Method: "GET", Pattern: "/paste/{id}", HandlerFunc: getPaste},
{Name: "getPaste", Method: "GET", Pattern: "/paste/get/{id}", HandlerFunc: getPaste},
{Name: "postPaste", Method: "POST", Pattern: "/paste", HandlerFunc: postPaste},
{Name: "getRandom", Method: "GET", Pattern: "/paste/rng", HandlerFunc: p.getRandom},
{Name: "getPaste", Method: "GET", Pattern: "/paste/{id}", HandlerFunc: p.getPaste},
{Name: "getPaste", Method: "GET", Pattern: "/paste/get/{id}", HandlerFunc: p.getPaste},
{Name: "postPaste", Method: "POST", Pattern: "/paste", HandlerFunc: p.postPaste},
{Name: "getRandom", Method: "GET", Pattern: "/api/rng", HandlerFunc: getRandom},
{Name: "getPaste", Method: "GET", Pattern: "/api/{id}", HandlerFunc: getPaste},
{Name: "getPaste", Method: "GET", Pattern: "/api/get/{id}", HandlerFunc: getPaste},
{Name: "postPaste", Method: "POST", Pattern: "/api", HandlerFunc: postPaste},
{Name: "getRandom", Method: "GET", Pattern: "/api/rng", HandlerFunc: p.getRandom},
{Name: "getPaste", Method: "GET", Pattern: "/api/{id}", HandlerFunc: p.getPaste},
{Name: "getPaste", Method: "GET", Pattern: "/api/get/{id}", HandlerFunc: p.getPaste},
{Name: "postPaste", Method: "POST", Pattern: "/api", HandlerFunc: p.postPaste},
})
}
func setConfig(config map[string]string) {
type Paste struct {
store string
randBytes int
}
store = "data/"
func (p *Paste) setConfig(config map[string]string) {
p.store = "data/"
if config["store"] != "" {
store = config["store"]
p.store = config["store"]
}
if !chkStore(store) {
log.Criticalf("[routes::Paste] Store location [%s] does not exist or is not writable.", store)
if !chkStore(p.store) {
log.Criticalf("Paste: store location [%s] does not exist or is not writable.", p.store)
}
log.Noticef("[paste::getPaste] Store location set to [%s]", store)
log.Noticef("Paste: store location set to [%s]", p.store)
randBytes = 1024
p.randBytes = 1024
if config["random"] != "" {
randBytes, _ = strconv.Atoi(config["random"])
log.Noticef("[paste:getRandom] set random size to %d bytes", randBytes)
p.randBytes, _ = strconv.Atoi(config["random"])
log.Noticef("Paste: set random size to %d bytes", p.randBytes)
}
}
func chkStore(path string) bool {
file, err := os.Stat(path)
if err == nil {
return true
if err != nil && os.IsNotExist(err) {
err = os.MkdirAll(path, 0744)
if err != nil {
return false
}
file, err = os.Stat(path)
}
if os.IsNotExist(err) {
if err != nil {
return false
}
if !file.IsDir() {
return false
}
if unix.Access(path, unix.W_OK&unix.R_OK) != nil {
return false
}
@@ -78,15 +90,14 @@ func chkStore(path string) bool {
func chkFile(path string) bool {
file, err := os.Stat(path)
if err == nil {
return true
}
if os.IsNotExist(err) {
if err != nil {
return false
}
if file.IsDir() {
return false
}
if unix.Access(path, unix.W_OK&unix.R_OK) != nil {
return false
}
@@ -99,50 +110,55 @@ func chkGone(path string) bool {
if err != nil {
return true
}
if file.Size() == 0 {
return true
}
return false
}
func getRandom(w http.ResponseWriter, r *http.Request) {
s := make([]byte, randBytes)
rand.Read(s)
func (p *Paste) getRandom(w http.ResponseWriter, _ *http.Request) {
s := make([]byte, p.randBytes)
_, _ = rand.Read(s)
w.Header().Set("content-type", "application/octet-stream")
w.WriteHeader(http.StatusOK)
w.Write(s)
_, _ = w.Write(s)
}
func getPaste(w http.ResponseWriter, r *http.Request) {
func (p *Paste) getPaste(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
id := vars["id"]
if !chkFile(store + id) {
w.WriteHeader(http.StatusNotFound)
w.Write([]byte("ERR Not Found"))
fname := filepath.Join(p.store, id)
if !chkFile(fname) {
httpsrv.WriteText(w, http.StatusNotFound, "ERR Not Found")
return
}
if chkGone(store + id) {
w.WriteHeader(http.StatusGone)
w.Write([]byte("ERR Gone"))
if chkGone(fname) {
httpsrv.WriteText(w, http.StatusGone, "ERR Gone")
return
}
head, err := os.Open(store + id)
f, err := os.Open(fname)
if err != nil {
log.Fatal(err)
httpsrv.WriteText(w, http.StatusInternalServerError, "ERR Reading Content")
}
defer head.Close()
defer f.Close()
pr := NewPreviewReader(f)
keep := true
scanner := bufio.NewScanner(head)
scanner := bufio.NewScanner(pr)
for scanner.Scan() {
txt := scanner.Text()
log.Debug(txt)
// End of header found
if txt == "" {
break
}
@@ -157,9 +173,9 @@ func getPaste(w http.ResponseWriter, r *http.Request) {
log.Debugf("%d > %d", now, exp)
if now > exp {
w.WriteHeader(http.StatusGone)
w.Write([]byte("ERR Gone"))
_, _ = w.Write([]byte("ERR Gone"))
deleteFile(store + id)
deleteFile(fname)
return
}
@@ -174,47 +190,46 @@ func getPaste(w http.ResponseWriter, r *http.Request) {
}
}
file, _ := os.Open(store + id)
defer file.Close()
w.WriteHeader(http.StatusOK)
scanner = bufio.NewScanner(file)
for scanner.Scan() {
w.Write(scanner.Bytes())
w.Write([]byte("\n"))
}
io.Copy(w, pr.Drain())
if !keep {
deleteFile(store + id)
deleteFile(fname)
}
}
func postPaste(w http.ResponseWriter, r *http.Request) {
func (p *Paste) postPaste(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
body, err := ioutil.ReadAll(io.LimitReader(r.Body, 1048576))
checkErr(err, w)
rdr := io.LimitReader(r.Body, 1048576)
s256 := sha256.Sum256(body)
id := base64.RawURLEncoding.EncodeToString(s256[12:])
s256 := sha256.New()
tmp, _ := ioutil.TempFile(p.store, "arch-")
defer os.Remove(tmp.Name())
ioutil.WriteFile(store+id, body, 0644)
m := io.MultiWriter(s256, tmp)
if _, err := io.Copy(m, rdr); err != nil {
httpsrv.WriteError(w, 400, err.Error())
}
tmp.Close()
w.WriteHeader(http.StatusCreated)
w.Write([]byte("OK " + id))
id := base64.RawURLEncoding.EncodeToString(s256.Sum(nil)[12:])
fname := filepath.Join(p.store, id)
w.WriteHeader(http.StatusCreated)
os.Rename(tmp.Name(), fname)
httpsrv.WriteText(w, http.StatusCreated, "OK "+id)
}
func checkErr(err error, w http.ResponseWriter) {
func checkErr(err error, w http.ResponseWriter) bool {
if err != nil {
w.WriteHeader(http.StatusBadRequest)
json.NewEncoder(w).Encode(err)
panic(err)
httpsrv.WriteObject(w, http.StatusBadRequest, err)
return true
}
return false
}
func deleteFile(path string) {
ioutil.WriteFile(path, []byte(""), 0644)
_ = ioutil.WriteFile(path, nil, 0644)
}

View File

@@ -0,0 +1,72 @@
package routes
import (
"bytes"
"errors"
"io"
"sour.is/x/toolbox/log"
)
type PreviewReader struct {
r io.Reader
buf bytes.Buffer
}
func NewPreviewReader(r io.Reader) *PreviewReader {
return &PreviewReader{r: r}
}
var ErrReaderDrained = errors.New("io.Reader has been drained")
func (pr *PreviewReader) Read(p []byte) (n int, err error) {
if pr.r == nil {
return 0, ErrReaderDrained
}
i, err := pr.r.Read(p)
log.Debugf("PreviewReader: buffer %d bytes", i)
_, berr := pr.buf.Write(p[:i])
if berr != nil {
return i, berr
}
return i, err
}
func (pr *PreviewReader) Drain() io.Reader {
dr := &drainReader{r: pr.r, buf: &pr.buf}
pr.r = nil
return dr
}
type drainReader struct {
r io.Reader
buf *bytes.Buffer
}
func (dr *drainReader) Read(p []byte) (n int, err error) {
i := 0
// log.Debugs("drainReader:", "buf", dr.buf.Len(), "p", len(p))
if dr.buf.Len() > 0 {
i, err = dr.buf.Read(p)
if err != nil && err != io.EOF {
return i, err
}
if err != nil && err == io.EOF {
err = nil
}
if err != nil {
return i, err
}
}
ri, err := dr.r.Read(p[i:])
// log.Debugs("drainReader:", "drain", i, "read", ri, "cap", len(p), "err", err)
return ri + i, err
}

View File

@@ -1,11 +1,26 @@
package routes
import "sour.is/x/toolbox/httpsrv"
import (
"fmt"
"net/http"
"sour.is/x/toolbox/httpsrv"
)
func init() {
httpsrv.AssetRegister("paste", httpsrv.AssetRoutes{
{Name: "Assets", Path: "/", HandlerFunc: httpsrv.FsHtml5(assetFS())},
asset := assetFS()
fmt.Printf("%#v\n", asset)
httpsrv.HttpRegister("ui", httpsrv.HttpRoutes{
{
Name: "Assets",
Method: "GET",
Pattern: "/ui{path:.*}",
HandlerFunc: http.StripPrefix("/ui", http.FileServer(httpsrv.FsHtml5(asset))).ServeHTTP,
},
{Name: "Root", Method: "GET", Pattern: "/", HandlerFunc: http.RedirectHandler("/ui/", http.StatusFound).ServeHTTP},
})
}
//go:generate go run github.com/sour-is/go-assetfs/cmd/assetfs -pkg routes -prefix ../../ ../../public/ ../../public/static/css/ ../../public/static/js/
//go:generate go run github.com/sour-is/go-assetfs/cmd/assetfs -pkg routes -prefix ../../ ../../public/...

1113
src/routes/read-mime.go Normal file

File diff suppressed because it is too large Load Diff