add image and artifact stores

src/routes/artifact.go

@@ -0,0 +1,193 @@
+package routes
+
+import (
+	"archive/tar"
+	"crypto/sha256"
+	"encoding/base64"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"github.com/gorilla/mux"
+	"sour.is/x/toolbox/httpsrv"
+	"sour.is/x/toolbox/log"
+)
+
+func init() {
+	a := &Artifact{}
+	httpsrv.RegisterModule("artifact", a.config)
+
+	httpsrv.HttpRegister("artifact", httpsrv.HttpRoutes{
+		{Name: "get-path", Method: "GET", Pattern: "/a/{name}/{path:.*}", HandlerFunc: a.get},
+		{Name: "get", Method: "GET", Pattern: "/a/{name}", HandlerFunc: a.get},
+		{Name: "put", Method: "PUT", Pattern: "/a", HandlerFunc: a.put},
+	})
+}
+
+type Artifact struct {
+	store   string
+	maxSize int64
+}
+
+func (a *Artifact) config(config map[string]string) {
+	a.store = "data/"
+	if config["store"] != "" {
+		a.store = config["store"]
+	}
+
+	if !chkStore(a.store) {
+		log.Criticalf("Artifact Store location [%s] does not exist or is not writable.", a.store)
+	}
+	log.Noticef("Artifact Store location set to [%s]", a.store)
+
+	a.maxSize = 50 * 1024 * 1024
+	if s, ok := config["max-size"]; ok {
+		if i, err := strconv.Atoi(s); err == nil {
+			a.maxSize = int64(i)
+		}
+	}
+	log.Noticef("Artifact: store max-size set to [%d]", a.maxSize)
+}
+
+func (a *Artifact) get(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+
+	name := vars["name"]
+	path, hasPath := vars["path"]
+
+	ext := filepath.Ext(name)
+	id := strings.TrimSuffix(name, ext)
+
+	fname := filepath.Join(a.store, id)
+
+	if !chkFile(fname) {
+		httpsrv.WriteError(w, http.StatusNotFound, "Not Found")
+
+		return
+	}
+
+	if chkGone(fname) {
+		httpsrv.WriteError(w, http.StatusGone, "Gone")
+
+		return
+	}
+
+	f, err := os.Open(fname)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer f.Close()
+
+	if !hasPath {
+		pr := NewPreviewReader(f)
+
+		mime, err := ReadMIME(pr, name)
+		if err != nil {
+			httpsrv.WriteError(w, http.StatusInternalServerError, err.Error())
+			return
+		}
+		w.Header().Set("Content-Type", mime)
+		w.Header().Set("X-Content-Type-Options", "nosniff")
+
+		io.Copy(w, pr.Drain())
+		return
+	}
+
+	rdr, err := Decompress(f)
+	if err != nil {
+		httpsrv.WriteError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	if cl, ok := rdr.(io.ReadCloser); ok {
+		defer cl.Close()
+	}
+
+	tr := tar.NewReader(rdr)
+	if path == "..." {
+		var paths []string
+		for {
+			hdr, err := tr.Next()
+			if err == io.EOF {
+				break // End of archive
+			}
+			if err != nil {
+				log.Fatal(err)
+			}
+			paths = append(paths, hdr.Name)
+		}
+
+		httpsrv.WriteObject(w, http.StatusOK, paths)
+		return
+	}
+
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break // End of archive
+		}
+		if err != nil {
+			log.Fatal(err)
+		}
+		if hdr.Name == path {
+			pr := NewPreviewReader(tr)
+
+			mime, err := ReadMIME(pr, hdr.Name)
+			if err != nil {
+				httpsrv.WriteError(w, http.StatusInternalServerError, err.Error())
+				return
+			}
+			w.Header().Set("Content-Type", mime)
+
+			if _, err := io.Copy(w, pr.Drain()); err != nil {
+				log.Fatal(err)
+			}
+			w.WriteHeader(http.StatusOK)
+			return
+		}
+	}
+
+	httpsrv.WriteError(w, http.StatusNotFound, "Not Found in Archive")
+}
+func (a *Artifact) put(w http.ResponseWriter, r *http.Request) {
+	defer r.Body.Close()
+
+	if h := r.Header.Get("Content-Length"); h != "" {
+		if i, err := strconv.Atoi(h); err == nil {
+			if int64(i) > a.maxSize {
+				http.Error(w, "ERR Size Too Large", http.StatusRequestEntityTooLarge)
+				return
+			}
+		}
+	}
+
+	rdr := io.LimitReader(r.Body, 500*1024*1024)
+	pr := NewPreviewReader(rdr)
+	rdr = pr.Drain()
+
+	s256 := sha256.New()
+	tmp, err := ioutil.TempFile(a.store, "arch-")
+	if err != nil {
+		httpsrv.WriteError(w, 400, err.Error())
+		return
+	}
+
+	defer os.Remove(tmp.Name())
+
+	m := io.MultiWriter(s256, tmp)
+	if _, err := io.Copy(m, rdr); err != nil {
+		httpsrv.WriteError(w, 400, err.Error())
+	}
+	tmp.Close()
+
+	id := base64.RawURLEncoding.EncodeToString(s256.Sum(nil)[12:])
+	fname := filepath.Join(a.store, id)
+
+	log.Debugs("Artifact: moving file", "src", tmp.Name(), "dst", fname)
+	os.Rename(tmp.Name(), fname)
+
+	httpsrv.WriteText(w, http.StatusCreated, id)
+}

src/routes/decompress.go

@@ -0,0 +1,37 @@
+package routes
+
+import (
+	"compress/bzip2"
+	"compress/gzip"
+	"io"
+
+	"github.com/andybalholm/brotli"
+	xz "github.com/remyoudompheng/go-liblzma"
+	"sour.is/x/toolbox/log"
+)
+
+func Decompress(in io.Reader) (io.Reader, error) {
+	rdr := NewPreviewReader(in)
+	mime, err := ReadMIMEWithSize(rdr, "", 32768)
+	if err != nil {
+		return rdr.Drain(), err
+	}
+	r := rdr.Drain()
+
+	switch mime {
+	case "application/gzip":
+		r, err = gzip.NewReader(r)
+
+	case "application/x-bzip2":
+		r = bzip2.NewReader(r)
+
+	case "application/x-xz":
+		r, err = xz.NewReader(r)
+
+	case "application/brotli":
+		r = brotli.NewReader(r)
+	}
+	log.Debugs("Decompress:", "mime", mime)
+
+	return r, err
+}

src/routes/image.go

@@ -0,0 +1,149 @@
+package routes
+
+import (
+	"crypto/sha256"
+	"encoding/base64"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"github.com/gorilla/mux"
+	"github.com/h2non/filetype"
+	"sour.is/x/toolbox/httpsrv"
+	"sour.is/x/toolbox/log"
+)
+
+func init() {
+	a := &Image{}
+	httpsrv.RegisterModule("image", a.config)
+
+	httpsrv.HttpRegister("image", httpsrv.HttpRoutes{
+		{Name: "getImage", Method: "GET", Pattern: "/i/{name}", HandlerFunc: a.get},
+		{Name: "putImage", Method: "PUT", Pattern: "/i", HandlerFunc: a.put},
+	})
+}
+
+type Image struct {
+	store   string
+	maxSize int64
+}
+
+func (a *Image) config(config map[string]string) {
+	a.store = "data/"
+	if config["store"] != "" {
+		a.store = config["store"]
+	}
+
+	if !chkStore(a.store) {
+		log.Criticalf("Image: store location [%s] does not exist or is not writable.", a.store)
+	}
+	log.Noticef("Image: store location set to [%s]", a.store)
+
+	a.maxSize = 10 * 1024 * 1024
+	if s, ok := config["max-size"]; ok {
+		if i, err := strconv.Atoi(s); err == nil {
+			a.maxSize = int64(i)
+		}
+	}
+	log.Noticef("Image: store max-size set to [%d]", a.maxSize)
+}
+
+func (a *Image) get(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+
+	name := vars["name"]
+
+	ext := filepath.Ext(name)
+	id := strings.TrimSuffix(name, ext)
+
+	fname := filepath.Join(a.store, id)
+
+	if !chkFile(fname) {
+		httpsrv.WriteError(w, http.StatusNotFound, "Not Found")
+
+		return
+	}
+
+	if chkGone(fname) {
+		httpsrv.WriteError(w, http.StatusGone, "Gone")
+
+		return
+	}
+
+	f, err := os.Open(fname)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer f.Close()
+
+	pr := NewPreviewReader(f)
+
+	mime, err := ReadMIME(pr, name)
+	w.Header().Set("Content-Type", mime)
+	w.Header().Set("X-Content-Type-Options", "nosniff")
+
+	io.Copy(w, pr.Drain())
+}
+
+func (a *Image) put(w http.ResponseWriter, r *http.Request) {
+	defer r.Body.Close()
+
+	if h := r.Header.Get("Content-Length"); h != "" {
+		log.Debugs("Artifact:", "content-length", h, "max-length", a.maxSize)
+		if i, err := strconv.Atoi(h); err == nil {
+			if int64(i) > a.maxSize {
+				log.Error("ERR Size Too Large")
+				http.Error(w, "ERR Size Too Large", http.StatusRequestEntityTooLarge)
+				return
+			}
+		}
+	}
+
+	rdr := io.LimitReader(r.Body, a.maxSize)
+	pr := NewPreviewReader(rdr)
+	if !isImageOrVideo(pr) {
+		httpsrv.WriteError(w, http.StatusUnsupportedMediaType, "ERR Not Image")
+		return
+	}
+	rdr = pr.Drain()
+
+	s256 := sha256.New()
+	tmp, err := ioutil.TempFile(a.store, "image-")
+	if err != nil {
+		httpsrv.WriteError(w, 400, err.Error())
+		return
+	}
+	defer os.Remove(tmp.Name())
+
+	m := io.MultiWriter(s256, tmp)
+	if _, err = io.Copy(m, rdr); err != nil {
+		httpsrv.WriteError(w, 400, err.Error())
+		return
+	}
+	tmp.Close()
+
+	id := base64.RawURLEncoding.EncodeToString(s256.Sum(nil)[12:])
+	fname := filepath.Join(a.store, id)
+
+	log.Debugs("Image: moving file", "src", tmp.Name(), "dst", fname)
+	err = os.Rename(tmp.Name(), fname)
+	if err != nil {
+		httpsrv.WriteError(w, 400, err.Error())
+		return
+	}
+
+	httpsrv.WriteText(w, http.StatusCreated, id)
+}
+
+func isImageOrVideo(in io.Reader) bool {
+	buf := make([]byte, 320)
+	_, err := in.Read(buf)
+	if err != nil {
+		return false
+	}
+	return filetype.IsImage(buf) || filetype.IsVideo(buf)
+}

src/routes/paste.go

@@ -5,11 +5,11 @@ import (
 	"crypto/rand"
 	"crypto/sha256"
 	"encoding/base64"
-	"encoding/json"
 	"io"
 	"io/ioutil"
 	"net/http"
 	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
@@ -20,55 +20,67 @@ import (
 	"sour.is/x/toolbox/log"
 )
 
-var store string
-var randBytes int
-
 func init() {
-	httpsrv.RegisterModule("paste", setConfig)
+	p := &Paste{}
+
+	httpsrv.RegisterModule("paste", p.setConfig)
 
 	httpsrv.HttpRegister("paste", httpsrv.HttpRoutes{
-		{Name: "getRandom", Method: "GET", Pattern: "/paste/rng", HandlerFunc: getRandom},
-		{Name: "getPaste", Method: "GET", Pattern: "/paste/{id}", HandlerFunc: getPaste},
-		{Name: "getPaste", Method: "GET", Pattern: "/paste/get/{id}", HandlerFunc: getPaste},
-		{Name: "postPaste", Method: "POST", Pattern: "/paste", HandlerFunc: postPaste},
+		{Name: "getRandom", Method: "GET", Pattern: "/paste/rng", HandlerFunc: p.getRandom},
+		{Name: "getPaste", Method: "GET", Pattern: "/paste/{id}", HandlerFunc: p.getPaste},
+		{Name: "getPaste", Method: "GET", Pattern: "/paste/get/{id}", HandlerFunc: p.getPaste},
+		{Name: "postPaste", Method: "POST", Pattern: "/paste", HandlerFunc: p.postPaste},
 
-		{Name: "getRandom", Method: "GET", Pattern: "/api/rng", HandlerFunc: getRandom},
-		{Name: "getPaste", Method: "GET", Pattern: "/api/{id}", HandlerFunc: getPaste},
-		{Name: "getPaste", Method: "GET", Pattern: "/api/get/{id}", HandlerFunc: getPaste},
-		{Name: "postPaste", Method: "POST", Pattern: "/api", HandlerFunc: postPaste},
+		{Name: "getRandom", Method: "GET", Pattern: "/api/rng", HandlerFunc: p.getRandom},
+		{Name: "getPaste", Method: "GET", Pattern: "/api/{id}", HandlerFunc: p.getPaste},
+		{Name: "getPaste", Method: "GET", Pattern: "/api/get/{id}", HandlerFunc: p.getPaste},
+		{Name: "postPaste", Method: "POST", Pattern: "/api", HandlerFunc: p.postPaste},
 	})
 }
 
-func setConfig(config map[string]string) {
+type Paste struct {
+	store     string
+	randBytes int
+}
 
-	store = "data/"
+func (p *Paste) setConfig(config map[string]string) {
+	p.store = "data/"
 	if config["store"] != "" {
-		store = config["store"]
+		p.store = config["store"]
 	}
 
-	if !chkStore(store) {
-		log.Criticalf("[routes::Paste] Store location [%s] does not exist or is not writable.", store)
+	if !chkStore(p.store) {
+		log.Criticalf("Paste: store location [%s] does not exist or is not writable.", p.store)
 	}
-	log.Noticef("[paste::getPaste] Store location set to [%s]", store)
+	log.Noticef("Paste: store location set to [%s]", p.store)
 
-	randBytes = 1024
+	p.randBytes = 1024
 	if config["random"] != "" {
-		randBytes, _ = strconv.Atoi(config["random"])
-		log.Noticef("[paste:getRandom] set random size to %d bytes", randBytes)
+		p.randBytes, _ = strconv.Atoi(config["random"])
+		log.Noticef("Paste: set random size to %d bytes", p.randBytes)
 	}
 }
 
 func chkStore(path string) bool {
 	file, err := os.Stat(path)
-	if err == nil {
-		return true
+
+	if err != nil && os.IsNotExist(err) {
+		err = os.MkdirAll(path, 0744)
+		if err != nil {
+			return false
+		}
+
+		file, err = os.Stat(path)
 	}
-	if os.IsNotExist(err) {
+
+	if err != nil {
 		return false
 	}
+
 	if !file.IsDir() {
 		return false
 	}
+
 	if unix.Access(path, unix.W_OK&unix.R_OK) != nil {
 		return false
 	}
@@ -78,15 +90,14 @@ func chkStore(path string) bool {
 
 func chkFile(path string) bool {
 	file, err := os.Stat(path)
-	if err == nil {
-		return true
-	}
-	if os.IsNotExist(err) {
+	if err != nil {
 		return false
 	}
+
 	if file.IsDir() {
 		return false
 	}
+
 	if unix.Access(path, unix.W_OK&unix.R_OK) != nil {
 		return false
 	}
@@ -99,50 +110,55 @@ func chkGone(path string) bool {
 	if err != nil {
 		return true
 	}
+
 	if file.Size() == 0 {
 		return true
 	}
+
 	return false
 }
 
-func getRandom(w http.ResponseWriter, r *http.Request) {
-	s := make([]byte, randBytes)
-	rand.Read(s)
+func (p *Paste) getRandom(w http.ResponseWriter, _ *http.Request) {
+	s := make([]byte, p.randBytes)
+	_, _ = rand.Read(s)
 
 	w.Header().Set("content-type", "application/octet-stream")
 	w.WriteHeader(http.StatusOK)
-	w.Write(s)
+	_, _ = w.Write(s)
 }
 
-func getPaste(w http.ResponseWriter, r *http.Request) {
-
+func (p *Paste) getPaste(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	id := vars["id"]
 
-	if !chkFile(store + id) {
-		w.WriteHeader(http.StatusNotFound)
-		w.Write([]byte("ERR Not Found"))
+	fname := filepath.Join(p.store, id)
+
+	if !chkFile(fname) {
+		httpsrv.WriteText(w, http.StatusNotFound, "ERR Not Found")
 		return
 	}
 
-	if chkGone(store + id) {
-		w.WriteHeader(http.StatusGone)
-		w.Write([]byte("ERR Gone"))
+	if chkGone(fname) {
+		httpsrv.WriteText(w, http.StatusGone, "ERR Gone")
 		return
 	}
 
-	head, err := os.Open(store + id)
+	f, err := os.Open(fname)
 	if err != nil {
 		log.Fatal(err)
+		httpsrv.WriteText(w, http.StatusInternalServerError, "ERR Reading Content")
 	}
-	defer head.Close()
+	defer f.Close()
+
+	pr := NewPreviewReader(f)
 
 	keep := true
-	scanner := bufio.NewScanner(head)
+	scanner := bufio.NewScanner(pr)
 	for scanner.Scan() {
 		txt := scanner.Text()
 		log.Debug(txt)
 
+		// End of header found
 		if txt == "" {
 			break
 		}
@@ -157,9 +173,9 @@ func getPaste(w http.ResponseWriter, r *http.Request) {
 			log.Debugf("%d > %d", now, exp)
 			if now > exp {
 				w.WriteHeader(http.StatusGone)
-				w.Write([]byte("ERR Gone"))
+				_, _ = w.Write([]byte("ERR Gone"))
 
-				deleteFile(store + id)
+				deleteFile(fname)
 
 				return
 			}
@@ -174,47 +190,46 @@ func getPaste(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	file, _ := os.Open(store + id)
-	defer file.Close()
-
 	w.WriteHeader(http.StatusOK)
-
-	scanner = bufio.NewScanner(file)
-	for scanner.Scan() {
-		w.Write(scanner.Bytes())
-		w.Write([]byte("\n"))
-	}
+	io.Copy(w, pr.Drain())
 
 	if !keep {
-		deleteFile(store + id)
+		deleteFile(fname)
 	}
 }
 
-func postPaste(w http.ResponseWriter, r *http.Request) {
+func (p *Paste) postPaste(w http.ResponseWriter, r *http.Request) {
 	defer r.Body.Close()
 
-	body, err := ioutil.ReadAll(io.LimitReader(r.Body, 1048576))
-	checkErr(err, w)
+	rdr := io.LimitReader(r.Body, 1048576)
 
-	s256 := sha256.Sum256(body)
-	id := base64.RawURLEncoding.EncodeToString(s256[12:])
+	s256 := sha256.New()
+	tmp, _ := ioutil.TempFile(p.store, "arch-")
+	defer os.Remove(tmp.Name())
 
-	ioutil.WriteFile(store+id, body, 0644)
+	m := io.MultiWriter(s256, tmp)
+	if _, err := io.Copy(m, rdr); err != nil {
+		httpsrv.WriteError(w, 400, err.Error())
+	}
+	tmp.Close()
 
-	w.WriteHeader(http.StatusCreated)
-	w.Write([]byte("OK " + id))
+	id := base64.RawURLEncoding.EncodeToString(s256.Sum(nil)[12:])
+	fname := filepath.Join(p.store, id)
 
-	w.WriteHeader(http.StatusCreated)
+	os.Rename(tmp.Name(), fname)
+
+	httpsrv.WriteText(w, http.StatusCreated, "OK "+id)
 }
 
-func checkErr(err error, w http.ResponseWriter) {
+func checkErr(err error, w http.ResponseWriter) bool {
 	if err != nil {
-		w.WriteHeader(http.StatusBadRequest)
-		json.NewEncoder(w).Encode(err)
-		panic(err)
+		httpsrv.WriteObject(w, http.StatusBadRequest, err)
+		return true
 	}
+
+	return false
 }
 
 func deleteFile(path string) {
-	ioutil.WriteFile(path, []byte(""), 0644)
+	_ = ioutil.WriteFile(path, nil, 0644)
 }

src/routes/preview-reader.go

@@ -0,0 +1,72 @@
+package routes
+
+import (
+	"bytes"
+	"errors"
+	"io"
+
+	"sour.is/x/toolbox/log"
+)
+
+type PreviewReader struct {
+	r   io.Reader
+	buf bytes.Buffer
+}
+
+func NewPreviewReader(r io.Reader) *PreviewReader {
+	return &PreviewReader{r: r}
+}
+
+var ErrReaderDrained = errors.New("io.Reader has been drained")
+
+func (pr *PreviewReader) Read(p []byte) (n int, err error) {
+	if pr.r == nil {
+		return 0, ErrReaderDrained
+	}
+
+	i, err := pr.r.Read(p)
+
+	log.Debugf("PreviewReader: buffer %d bytes", i)
+	_, berr := pr.buf.Write(p[:i])
+	if berr != nil {
+		return i, berr
+	}
+
+	return i, err
+}
+
+func (pr *PreviewReader) Drain() io.Reader {
+	dr := &drainReader{r: pr.r, buf: &pr.buf}
+	pr.r = nil
+	return dr
+}
+
+type drainReader struct {
+	r   io.Reader
+	buf *bytes.Buffer
+}
+
+func (dr *drainReader) Read(p []byte) (n int, err error) {
+	i := 0
+	// log.Debugs("drainReader:", "buf", dr.buf.Len(), "p", len(p))
+	if dr.buf.Len() > 0 {
+		i, err = dr.buf.Read(p)
+		if err != nil && err != io.EOF {
+			return i, err
+		}
+
+		if err != nil && err == io.EOF {
+			err = nil
+		}
+
+		if err != nil {
+			return i, err
+		}
+	}
+
+	ri, err := dr.r.Read(p[i:])
+
+	// log.Debugs("drainReader:", "drain", i, "read", ri, "cap", len(p), "err", err)
+
+	return ri + i, err
+}

src/routes/public.go

@@ -1,11 +1,26 @@
 package routes
 
-import "sour.is/x/toolbox/httpsrv"
+import (
+	"fmt"
+	"net/http"
+
+	"sour.is/x/toolbox/httpsrv"
+)
 
 func init() {
-	httpsrv.AssetRegister("paste", httpsrv.AssetRoutes{
-		{Name: "Assets", Path: "/", HandlerFunc: httpsrv.FsHtml5(assetFS())},
+	asset := assetFS()
+	fmt.Printf("%#v\n", asset)
+
+	httpsrv.HttpRegister("ui", httpsrv.HttpRoutes{
+		{
+			Name:        "Assets",
+			Method:      "GET",
+			Pattern:     "/ui{path:.*}",
+			HandlerFunc: http.StripPrefix("/ui", http.FileServer(httpsrv.FsHtml5(asset))).ServeHTTP,
+		},
+
+		{Name: "Root", Method: "GET", Pattern: "/", HandlerFunc: http.RedirectHandler("/ui/", http.StatusFound).ServeHTTP},
 	})
 }
 
-//go:generate go run github.com/sour-is/go-assetfs/cmd/assetfs -pkg routes  -prefix ../../ ../../public/ ../../public/static/css/ ../../public/static/js/
+//go:generate go run github.com/sour-is/go-assetfs/cmd/assetfs -pkg routes  -prefix ../../ ../../public/...