sour-is/go-paste
1
0
Fork 0
Code Pull Requests Actions Releases 6 Activity

update shorturl to use badgerhold

Browse Source
This commit is contained in:
Xuu
2020-10-30 11:08:04 -06:00
parent db4ac49a49
commit 6c01df76f5
15 changed files with 16578 additions and 279 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
src/routes/artifact.go
View File

@@ -4,6 +4,7 @@ import (
"archive/tar"
"crypto/sha256"
"encoding/base64"
"fmt"
"io"
"io/ioutil"
"net/http"
@@ -30,9 +31,11 @@ func init() {
{Name: "get-path", Method: "GET", Pattern: "/a/{name}/{path:.*}", HandlerFunc: a.get},
{Name: "get", Method: "GET", Pattern: "/a/{name}", HandlerFunc: a.get},
{Name: "put", Method: "PUT", Pattern: "/a", HandlerFunc: a.put},
{Name: "get", Method: "GET", Pattern: "/a", HandlerFunc: a.list},
})
}
// Artifact stores items to disk
type Artifact struct {
store string
maxSize int64
@@ -90,7 +93,7 @@ func (a *Artifact) get(w http.ResponseWriter, r *http.Request) {
if !hasPath {
pr := readutil.NewPreviewReader(f)
mime, err := ReadMIME(pr, name)
mime, err := readutil.ReadMIME(pr, name)
if err != nil {
httpsrv.WriteError(w, http.StatusInternalServerError, err.Error())
return
@@ -102,8 +105,8 @@ func (a *Artifact) get(w http.ResponseWriter, r *http.Request) {
return
}
rdr, err := Decompress(f)
if err != nil {
rdr, err := readutil.Decompress(f)
if err != nil && err != readutil.ErrUnsupportedType {
httpsrv.WriteError(w, http.StatusInternalServerError, err.Error())
return
}
@@ -161,14 +164,14 @@ func (a *Artifact) get(w http.ResponseWriter, r *http.Request) {
renderer := html.NewRenderer(opts)
b := markdown.ToHTML(md, p, renderer)
w.Write(b)
_, _ = w.Write(b)
w.WriteHeader(http.StatusOK)
return
}
pr := readutil.NewPreviewReader(tr)
mime, err := ReadMIME(pr, hdr.Name)
mime, err := readutil.ReadMIME(pr, hdr.Name)
if err != nil {
httpsrv.WriteError(w, http.StatusInternalServerError, err.Error())
return
@@ -220,7 +223,24 @@ func (a *Artifact) put(w http.ResponseWriter, r *http.Request) {
fname := filepath.Join(a.store, id)
log.Debugs("Artifact: moving file", "src", tmp.Name(), "dst", fname)
os.Rename(tmp.Name(), fname)
_ = os.Rename(tmp.Name(), fname)
httpsrv.WriteText(w, http.StatusCreated, id)
}
func (a *Artifact) list(w http.ResponseWriter, r *http.Request) {
err := filepath.Walk(a.store, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() {
return nil
}
_, err = fmt.Fprintln(w, "FILE: ", info.Name())
return err
})
if err != nil {
httpsrv.WriteError(w, http.StatusInternalServerError, err.Error())
return
}
}

208
src/routes/bindata.go
View File

File diff suppressed because one or more lines are too long

38
src/routes/decompress.go
View File

@@ -1,38 +0,0 @@
package routes
import (
"compress/bzip2"
"compress/gzip"
"io"
"github.com/andybalholm/brotli"
xz "github.com/remyoudompheng/go-liblzma"
"sour.is/x/paste/src/pkg/readutil"
"sour.is/x/toolbox/log"
)
func Decompress(in io.Reader) (io.Reader, error) {
rdr := readutil.NewPreviewReader(in)
mime, err := ReadMIMEWithSize(rdr, "", 32768)
if err != nil {
return rdr.Drain(), err
}
r := rdr.Drain()
switch mime {
case "application/gzip":
r, err = gzip.NewReader(r)
case "application/x-bzip2":
r = bzip2.NewReader(r)
case "application/x-xz":
r, err = xz.NewReader(r)
case "application/brotli":
r = brotli.NewReader(r)
}
log.Debugs("Decompress:", "mime", mime)
return r, err
}

2
src/routes/identity.go
View File

@@ -388,7 +388,7 @@ func (s *identity) getStyle(ctx context.Context, email string) (*Style, error) {
hash := md5.New()
email = strings.TrimSpace(strings.ToLower(email))
hash.Write([]byte(email))
_, _ = hash.Write([]byte(email))
id := hash.Sum(nil)

16
src/routes/image.go
View File

@@ -20,7 +20,7 @@ import (
)
func init() {
a := &Image{}
a := &image{}
httpsrv.RegisterModule("image", a.config)
httpsrv.HttpRegister("image", httpsrv.HttpRoutes{
@@ -30,12 +30,12 @@ func init() {
})
}
type Image struct {
type image struct {
store string
maxSize int64
}
func (a *Image) config(config map[string]string) {
func (a *image) config(config map[string]string) {
a.store = "data/"
if config["store"] != "" {
a.store = config["store"]
@@ -55,7 +55,7 @@ func (a *Image) config(config map[string]string) {
log.Noticef("Image: store max-size set to [%d]", a.maxSize)
}
func (a *Image) get(w http.ResponseWriter, r *http.Request) {
func (a *image) get(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
name := vars["name"]
@@ -85,14 +85,14 @@ func (a *Image) get(w http.ResponseWriter, r *http.Request) {
pr := readutil.NewPreviewReader(f)
mime, err := ReadMIME(pr, name)
mime, _ := readutil.ReadMIME(pr, name)
w.Header().Set("Content-Type", mime)
w.Header().Set("X-Content-Type-Options", "nosniff")
io.Copy(w, pr.Drain())
_, _ = io.Copy(w, pr.Drain())
}
func (a *Image) put(w http.ResponseWriter, r *http.Request) {
func (a *image) put(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
if h := r.Header.Get("Content-Length"); h != "" {
@@ -151,6 +151,6 @@ func isImageOrVideo(in io.Reader) bool {
return filetype.IsImage(buf) || filetype.IsVideo(buf)
}
func (a *Image) getStyle(w http.ResponseWriter, r *http.Request) {
func (a *image) getStyle(w http.ResponseWriter, r *http.Request) {
}

18
src/routes/paste.go
View File

@@ -192,7 +192,7 @@ func (p *Paste) getPaste(w http.ResponseWriter, r *http.Request) {
}
w.WriteHeader(http.StatusOK)
io.Copy(w, pr.Drain())
_, _ = io.Copy(w, pr.Drain())
if !keep {
deleteFile(fname)
@@ -217,19 +217,19 @@ func (p *Paste) postPaste(w http.ResponseWriter, r *http.Request) {
id := base64.RawURLEncoding.EncodeToString(s256.Sum(nil)[12:])
fname := filepath.Join(p.store, id)
os.Rename(tmp.Name(), fname)
_ = os.Rename(tmp.Name(), fname)
httpsrv.WriteText(w, http.StatusCreated, "OK "+id)
}
func checkErr(err error, w http.ResponseWriter) bool {
if err != nil {
httpsrv.WriteObject(w, http.StatusBadRequest, err)
return true
}
// func checkErr(err error, w http.ResponseWriter) bool {
// if err != nil {
// httpsrv.WriteObject(w, http.StatusBadRequest, err)
// return true
// }
return false
}
// return false
// }
func deleteFile(path string) {
_ = ioutil.WriteFile(path, nil, 0644)

1114
src/routes/read-mime.go
View File

File diff suppressed because it is too large Load Diff

192
src/routes/shorturl.go
View File

@@ -1,99 +1,131 @@
package routes
import (
"bytes"
"encoding/json"
"net/http"
"net/url"
"regexp"
"github.com/coreos/bbolt"
"github.com/gorilla/mux"
"github.com/timshannon/badgerhold"
"sour.is/x/toolbox/httpsrv"
"sour.is/x/toolbox/log"
"sour.is/x/toolbox/uuid"
)
type logger struct{}
func (logger) Errorf(s string, o ...interface{}) { log.Errorf(s, o...) }
func (logger) Warningf(s string, o ...interface{}) { log.Warningf(s, o...) }
func (logger) Infof(s string, o ...interface{}) { log.Infof(s, o...) }
func (logger) Debugf(s string, o ...interface{}) { log.Debugf(s, o...) }
func init() {
s := &shortDB{}
httpsrv.RegisterModule("short", s.config)
httpsrv.HttpRegister("short", httpsrv.HttpRoutes{
{Name: "getShort", Method: "GET", Pattern: "/s/{id}", HandlerFunc: s.getShort},
{Name: "putShort", Method: "PUT", Pattern: "/s/{id}", HandlerFunc: s.putShort},
{Name: "get", Method: "GET", Pattern: "/s/{id}", HandlerFunc: s.get},
{Name: "put", Method: "PUT", Pattern: "/s/{id}", HandlerFunc: s.put},
})
}
type shortDB struct {
path string
bucket string
options badgerhold.Options
}
func (s *shortDB) config(config map[string]string) {
s.bucket = "shortURL"
if config["bucket"] != "" {
s.bucket = config["bucket"]
s.options = badgerhold.DefaultOptions
s.options.Options = s.options.WithLogger(logger{})
var ok bool
if s.options.Dir, ok = config["index"]; !ok {
s.options.Dir = "data"
}
s.path = "data/meta.db"
if config["store"] != "" {
s.path = config["store"]
if s.options.ValueDir, ok = config["value"]; !ok {
s.options.ValueDir = "data"
}
db, err := bbolt.Open(s.path, 0666, nil)
if err != nil {
log.Fatalf("ShortURL: failed to open db at [%s]", s.path)
}
defer db.Close()
db.Update(func(tx *bbolt.Tx) error {
_, err := tx.CreateBucketIfNotExists([]byte(s.bucket))
if err != nil {
log.Fatalf("ShortURL: create bucket: %s", err)
store, err := badgerhold.Open(s.options)
defer func() {
if err = store.Close(); err != nil {
log.Fatal(err)
}
return nil
})
log.Noticef("ShortURL: opened db at [%s] bucket [%s]", s.path, s.bucket)
}()
if err != nil {
log.Fatal(err)
}
}
func (s *shortDB) getShort(w http.ResponseWriter, r *http.Request) {
func (s *shortDB) get(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
id := vars["id"]
url := s.GetURL(id)
if url == nil {
httpsrv.WriteError(w, 404, "not found")
store, err := badgerhold.Open(s.options)
defer func() {
if err = store.Close(); err != nil {
log.Fatal(err)
}
}()
if err != nil {
httpsrv.WriteText(w, 500, err.Error())
return
}
var url shortURL
err = store.Get(id, &url)
if err != nil && err != badgerhold.ErrNotFound {
httpsrv.WriteText(w, 500, err.Error())
return
}
if err == badgerhold.ErrNotFound {
httpsrv.WriteText(w, 404, "not found")
return
}
w.Header().Set("Location", url.URL)
w.WriteHeader(http.StatusFound)
}
func (s *shortDB) putShort(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
func (s *shortDB) put(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
id := vars["id"]
secret := r.FormValue("secret")
u, err := url.Parse(r.FormValue("url"))
if err != nil {
httpsrv.WriteError(w, 400, "bad url")
httpsrv.WriteText(w, 400, "bad url")
return
}
short := s.GetURL(id)
store, err := badgerhold.Open(s.options)
defer func() {
if err = store.Close(); err != nil {
log.Fatal(err)
}
}()
if err != nil {
httpsrv.WriteText(w, 500, err.Error())
return
}
if short == nil {
short = newshort(id, secret, u.String())
url := &shortURL{}
err = store.Get(id, url)
if err != nil && err != badgerhold.ErrNotFound {
httpsrv.WriteText(w, 500, err.Error())
return
}
s.PutURL(short.ID, short)
httpsrv.WriteObject(w, 200, short)
if err == badgerhold.ErrNotFound {
url = newShortURL(id, secret, u.String())
err := store.Insert(url.ID, url)
if err != nil {
httpsrv.WriteText(w, 500, err.Error())
return
}
httpsrv.WriteObject(w, 200, url)
return
}
@@ -102,24 +134,27 @@ func (s *shortDB) putShort(w http.ResponseWriter, r *http.Request) {
return
}
if secret != short.Secret {
if secret != url.Secret {
httpsrv.WriteError(w, 403, "forbidden")
return
}
short.URL = u.String()
err = store.Upsert(url.ID, url)
if err != nil {
httpsrv.WriteText(w, 500, err.Error())
return
}
s.PutURL(short.ID, short)
httpsrv.WriteObject(w, 200, short)
httpsrv.WriteObject(w, 200, url)
}
type shortURL struct {
ID string
ID string `badgerhold:"unique"`
URL string
Secret string
}
func newshort(id, secret, u string) *shortURL {
func newShortURL(id, secret, u string) *shortURL {
m, err := regexp.MatchString("[a-z-]{1,64}", id)
if id == "" || !m || err != nil {
id = uuid.V4()
@@ -130,64 +165,3 @@ func newshort(id, secret, u string) *shortURL {
}
return &shortURL{ID: id, Secret: secret, URL: u}
}
func (s *shortURL) Bytes() []byte {
if s == nil {
return nil
}
var w bytes.Buffer
json.NewEncoder(&w).Encode(*s)
return w.Bytes()
}
func URLFromBytes(b []byte) *shortURL {
if len(b) == 0 {
return nil
}
var s shortURL
json.Unmarshal(b, &s)
log.Debug(s)
return &s
}
func (s *shortDB) GetURL(id string) *shortURL {
db, err := bbolt.Open(s.path, 0666, nil)
if err != nil {
log.Errorf("ShortURL: failed to open db at [%s]", s.path)
return nil
}
defer db.Close()
var url *shortURL
err = db.View(func(tx *bbolt.Tx) error {
b := tx.Bucket([]byte(s.bucket))
v := b.Get([]byte(id))
url = URLFromBytes(v)
return nil
})
if err != nil {
log.Errorf("ShortURL: failed to open db at [%s]", s.path)
}
return url
}
func (s *shortDB) PutURL(id string, url *shortURL) {
db, err := bbolt.Open(s.path, 0666, nil)
if err != nil {
log.Errorf("ShortURL: failed to open db at [%s]", s.path)
return
}
defer db.Close()
err = db.Update(func(tx *bbolt.Tx) error {
b := tx.Bucket([]byte(s.bucket))
return b.Put([]byte(id), url.Bytes())
})
if err != nil {
log.Errorf("ShortURL: failed to write db at [%s]", s.path)
}
}