From 9049ab043e40d5f0e43ba5c8706e7b9c71f8e4a3 Mon Sep 17 00:00:00 2001 From: Xuu Date: Mon, 7 Sep 2020 10:45:20 -0600 Subject: [PATCH] make shorturl persist --- .drone.yml | 2 +- Makefile | 11 ++- cmd/paste/config.go | 2 + go.mod | 3 +- go.sum | 4 ++ src/routes/preview-reader.go | 15 +++- src/routes/shorturl.go | 128 +++++++++++++++++++++++++++-------- 7 files changed, 132 insertions(+), 33 deletions(-) diff --git a/.drone.yml b/.drone.yml index 3c3a79e..899ccfb 100644 --- a/.drone.yml +++ b/.drone.yml @@ -39,10 +39,10 @@ steps: - name: Build Release commands: - cd debian + - make tag - make build - make copy - make deploy - - make tag trigger: event: - promote diff --git a/Makefile b/Makefile index 9f05fa8..b36584c 100644 --- a/Makefile +++ b/Makefile @@ -3,8 +3,12 @@ DOCS_ASSET=src/docs/bindata.go SOURCE=$(wildcard cmd/paste/*.go) $(filter-out src/routes/bindata.go, $(wildcard src/routes/*.go)) BINARY=paste +PKG=./cmd/paste -VERSION:=$(shell debian/inc_version.sh -p $(shell git describe --tags `git rev-list --tags --max-count=1`)) +VERSION=$(shell git describe --tags `git rev-list --tags --max-count=1`|cut -b2-) +VERSION_PAT=$(shell debian/inc_version.sh -p $(VERSION)) +VERSION_MIN=$(shell debian/inc_version.sh -m $(VERSION)) +VERSION_MAJ=$(shell debian/inc_version.sh -M $(VERSION)) DATE:=$(shell date -u +%FT%TZ) define DUMMY_BINDATA @@ -27,7 +31,9 @@ test: $(ROUTE_ASSET) $(DOCS_ASSET) go test ./... go vet ./... run: $(BINARY) - ./$(BINARY) -vv serve + go run \ + -ldflags "-X main.AppVersion=$(VERSION_PAT) -X main.AppBuild=$(DATE)" \ + $(PKG) -vv serve $(BINARY): $(SOURCE) $(ROUTE_ASSET) $(DOCS_ASSET) go build -v \ @@ -48,6 +54,7 @@ $(DOCS_ASSET): echo "$$DUMMY_BINDATA" > src/docs/bindata.go go generate "sour.is/x/paste/cmd/paste" go generate "sour.is/x/paste/src/docs" + deploy: $(SOURCE) $(ROUTE_ASSET) cd debian && make diff --git a/cmd/paste/config.go b/cmd/paste/config.go index 5aafaa3..e889954 100644 --- a/cmd/paste/config.go +++ b/cmd/paste/config.go @@ -54,6 +54,8 @@ store = "data/artifact" [module.image] store = "data/image" +[module.short] +store = "data/meta.db" ` var args map[string]interface{} diff --git a/go.mod b/go.mod index 5ed31c6..1fdb6ba 100644 --- a/go.mod +++ b/go.mod @@ -5,16 +5,17 @@ go 1.14 require ( github.com/99designs/gqlgen v0.12.2 github.com/andybalholm/brotli v1.0.0 + github.com/coreos/bbolt v1.3.2 github.com/docopt/docopt.go v0.0.0-20180111231733-ee0de3bc6815 github.com/go-swagger/go-swagger v0.25.0 github.com/golang/gddo v0.0.0-20200831202555-721e228c7686 // indirect github.com/gorilla/mux v1.8.0 github.com/h2non/filetype v1.1.0 - github.com/patrickmn/go-cache v2.1.0+incompatible github.com/remyoudompheng/go-liblzma v0.0.0-20190506200333-81bf2d431b96 github.com/sour-is/go-assetfs v1.0.0 github.com/spf13/viper v1.7.1 github.com/vektah/dataloaden v0.3.0 + go.etcd.io/bbolt v1.3.5 // indirect golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a sour.is/x/toolbox v0.12.17 ) diff --git a/go.sum b/go.sum index 474d9ce..b0ee0d4 100644 --- a/go.sum +++ b/go.sum @@ -58,6 +58,7 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cgilling/dbstats v0.0.0-20150427045024-c9db8cf218e6/go.mod h1:fsf3+k/VvGOE9sF2B9d6PBcZOzQIlDJhn2LhBqF/4VY= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -504,6 +505,8 @@ github.com/yosssi/gmq v0.0.1 h1:GhlDVaAQoi3Mvjul/qJXXGfL4JBeE0GQwbWp3eIsja8= github.com/yosssi/gmq v0.0.1/go.mod h1:mReykazh0U1JabvuWh1PEbzzJftqOQWsjr0Lwg5jL1Y= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= +go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.3.0/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE= @@ -610,6 +613,7 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a h1:i47hUS795cOydZI4AwJQCKXOr4BvxzvikwDoDtHhP2Y= diff --git a/src/routes/preview-reader.go b/src/routes/preview-reader.go index 700fa66..2e6a7ce 100644 --- a/src/routes/preview-reader.go +++ b/src/routes/preview-reader.go @@ -3,6 +3,7 @@ package routes import ( "bytes" "errors" + "fmt" "io" "sour.is/x/toolbox/log" @@ -46,9 +47,10 @@ type drainReader struct { buf *bytes.Buffer } +var _ io.Seeker = (*drainReader)(nil) + func (dr *drainReader) Read(p []byte) (n int, err error) { i := 0 - // log.Debugs("drainReader:", "buf", dr.buf.Len(), "p", len(p)) if dr.buf.Len() > 0 { i, err = dr.buf.Read(p) if err != nil && err != io.EOF { @@ -70,3 +72,14 @@ func (dr *drainReader) Read(p []byte) (n int, err error) { return ri + i, err } + +// Seek attempt if the underlying reader supports it. +func (dr *drainReader) Seek(offset int64, whence int) (int64, error) { + if dr.buf.Len() > 0 { + return 0, fmt.Errorf("unable to seek") + } + if r, ok := dr.r.(io.Seeker); ok { + return r.Seek(offset, whence) + } + return 0, fmt.Errorf("unable to seek") +} diff --git a/src/routes/shorturl.go b/src/routes/shorturl.go index ff4c61b..756ea25 100644 --- a/src/routes/shorturl.go +++ b/src/routes/shorturl.go @@ -1,19 +1,22 @@ package routes import ( + "bytes" + "encoding/json" "net/http" "net/url" "regexp" - "time" + "github.com/coreos/bbolt" "github.com/gorilla/mux" - "github.com/patrickmn/go-cache" "sour.is/x/toolbox/httpsrv" + "sour.is/x/toolbox/log" "sour.is/x/toolbox/uuid" ) func init() { - s := NewShortManager(365 * 24 * time.Hour) + s := &shortDB{} + httpsrv.RegisterModule("short", s.config) httpsrv.HttpRegister("short", httpsrv.HttpRoutes{ {Name: "getShort", Method: "GET", Pattern: "/s/{id}", HandlerFunc: s.getShort}, @@ -21,38 +24,40 @@ func init() { }) } -type shortManager struct { - defaultExpire time.Duration - db *cache.Cache +type shortDB struct { + path string + bucket string } -type shortURL struct { - ID string - URL string - Secret string -} - -func NewShortManager(defaultExpire time.Duration) *shortManager { - return &shortManager{ - defaultExpire: defaultExpire, - db: cache.New(defaultExpire, defaultExpire/10), +func (s *shortDB) config(config map[string]string) { + s.bucket = "shortURL" + if config["bucket"] != "" { + s.bucket = config["bucket"] } -} -func (s *shortManager) GetURL(id string) *shortURL { - if u, ok := s.db.Get(id); ok { - if url, ok := u.(*shortURL); ok { - return url + s.path = "data/meta.db" + if config["store"] != "" { + s.path = config["store"] + } + + db, err := bbolt.Open(s.path, 0666, nil) + if err != nil { + log.Fatalf("ShortURL: failed to open db at [%s]", s.path) + } + defer db.Close() + + db.Update(func(tx *bbolt.Tx) error { + _, err := tx.CreateBucketIfNotExists([]byte(s.bucket)) + if err != nil { + log.Fatalf("ShortURL: create bucket: %s", err) } - } + return nil + }) - return nil -} -func (s *shortManager) PutURL(id string, url *shortURL) { - s.db.SetDefault(id, url) + log.Noticef("ShortURL: opened db at [%s] bucket [%s]", s.path, s.bucket) } -func (s *shortManager) getShort(w http.ResponseWriter, r *http.Request) { +func (s *shortDB) getShort(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) id := vars["id"] @@ -67,7 +72,7 @@ func (s *shortManager) getShort(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusFound) } -func (s *shortManager) putShort(w http.ResponseWriter, r *http.Request) { +func (s *shortDB) putShort(w http.ResponseWriter, r *http.Request) { defer r.Body.Close() vars := mux.Vars(r) @@ -107,6 +112,12 @@ func (s *shortManager) putShort(w http.ResponseWriter, r *http.Request) { httpsrv.WriteObject(w, 200, short) } +type shortURL struct { + ID string + URL string + Secret string +} + func newshort(id, secret, u string) *shortURL { m, err := regexp.MatchString("[a-z-]{1,64}", id) if id == "" || !m || err != nil { @@ -118,3 +129,64 @@ func newshort(id, secret, u string) *shortURL { } return &shortURL{ID: id, Secret: secret, URL: u} } + +func (s *shortURL) Bytes() []byte { + if s == nil { + return nil + } + + var w bytes.Buffer + json.NewEncoder(&w).Encode(*s) + return w.Bytes() +} + +func URLFromBytes(b []byte) *shortURL { + if len(b) == 0 { + return nil + } + var s shortURL + json.Unmarshal(b, &s) + + log.Debug(s) + return &s +} + +func (s *shortDB) GetURL(id string) *shortURL { + db, err := bbolt.Open(s.path, 0666, nil) + if err != nil { + log.Errorf("ShortURL: failed to open db at [%s]", s.path) + return nil + } + defer db.Close() + + var url *shortURL + + err = db.View(func(tx *bbolt.Tx) error { + b := tx.Bucket([]byte(s.bucket)) + v := b.Get([]byte(id)) + + url = URLFromBytes(v) + + return nil + }) + if err != nil { + log.Errorf("ShortURL: failed to open db at [%s]", s.path) + } + return url +} +func (s *shortDB) PutURL(id string, url *shortURL) { + db, err := bbolt.Open(s.path, 0666, nil) + if err != nil { + log.Errorf("ShortURL: failed to open db at [%s]", s.path) + return + } + defer db.Close() + + err = db.Update(func(tx *bbolt.Tx) error { + b := tx.Bucket([]byte(s.bucket)) + return b.Put([]byte(id), url.Bytes()) + }) + if err != nil { + log.Errorf("ShortURL: failed to write db at [%s]", s.path) + } +}