make shorturl persist

.drone.yml

@@ -39,10 +39,10 @@ steps:
 - name: Build Release
   commands:
     - cd debian
+    - make tag
     - make build
     - make copy
     - make deploy
-    - make tag
 trigger:
   event:
   - promote

Makefile

@@ -3,8 +3,12 @@ DOCS_ASSET=src/docs/bindata.go
 
 SOURCE=$(wildcard cmd/paste/*.go) $(filter-out src/routes/bindata.go, $(wildcard src/routes/*.go))
 BINARY=paste
+PKG=./cmd/paste
 
-VERSION:=$(shell debian/inc_version.sh -p $(shell git describe --tags `git rev-list --tags --max-count=1`))
+VERSION=$(shell git describe --tags `git rev-list --tags --max-count=1`|cut -b2-)
+VERSION_PAT=$(shell debian/inc_version.sh -p $(VERSION))
+VERSION_MIN=$(shell debian/inc_version.sh -m $(VERSION))
+VERSION_MAJ=$(shell debian/inc_version.sh -M $(VERSION))
 DATE:=$(shell date -u +%FT%TZ)
 
 define DUMMY_BINDATA
@@ -27,7 +31,9 @@ test: $(ROUTE_ASSET) $(DOCS_ASSET)
 	go test ./...
 	go vet ./...
 run: $(BINARY)
-	./$(BINARY) -vv serve
+	go run \
+	  -ldflags "-X main.AppVersion=$(VERSION_PAT) -X main.AppBuild=$(DATE)" \
+	  $(PKG) -vv serve
 
 $(BINARY): $(SOURCE) $(ROUTE_ASSET) $(DOCS_ASSET)
 	go build -v \
@@ -48,6 +54,7 @@ $(DOCS_ASSET):
 	echo "$$DUMMY_BINDATA" > src/docs/bindata.go
 	go generate "sour.is/x/paste/cmd/paste"
 	go generate "sour.is/x/paste/src/docs"
+
 deploy: $(SOURCE) $(ROUTE_ASSET)
 	cd debian && make
 

cmd/paste/config.go

@@ -54,6 +54,8 @@ store = "data/artifact"
 [module.image]
 store = "data/image"
 
+[module.short]
+store = "data/meta.db"
 `
 
 var args map[string]interface{}

go.mod

@@ -5,16 +5,17 @@ go 1.14
 require (
 	github.com/99designs/gqlgen v0.12.2
 	github.com/andybalholm/brotli v1.0.0
+	github.com/coreos/bbolt v1.3.2
 	github.com/docopt/docopt.go v0.0.0-20180111231733-ee0de3bc6815
 	github.com/go-swagger/go-swagger v0.25.0
 	github.com/golang/gddo v0.0.0-20200831202555-721e228c7686 // indirect
 	github.com/gorilla/mux v1.8.0
 	github.com/h2non/filetype v1.1.0
-	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/remyoudompheng/go-liblzma v0.0.0-20190506200333-81bf2d431b96
 	github.com/sour-is/go-assetfs v1.0.0
 	github.com/spf13/viper v1.7.1
 	github.com/vektah/dataloaden v0.3.0
+	go.etcd.io/bbolt v1.3.5 // indirect
 	golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a
 	sour.is/x/toolbox v0.12.17
 )

go.sum

@@ -58,6 +58,7 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cgilling/dbstats v0.0.0-20150427045024-c9db8cf218e6/go.mod h1:fsf3+k/VvGOE9sF2B9d6PBcZOzQIlDJhn2LhBqF/4VY=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@@ -504,6 +505,8 @@ github.com/yosssi/gmq v0.0.1 h1:GhlDVaAQoi3Mvjul/qJXXGfL4JBeE0GQwbWp3eIsja8=
 github.com/yosssi/gmq v0.0.1/go.mod h1:mReykazh0U1JabvuWh1PEbzzJftqOQWsjr0Lwg5jL1Y=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0=
+go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
 go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
 go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
 go.mongodb.org/mongo-driver v1.3.0/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE=
@@ -610,6 +613,7 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a h1:i47hUS795cOydZI4AwJQCKXOr4BvxzvikwDoDtHhP2Y=

src/routes/preview-reader.go

@@ -3,6 +3,7 @@ package routes
 import (
 	"bytes"
 	"errors"
+	"fmt"
 	"io"
 
 	"sour.is/x/toolbox/log"
@@ -46,9 +47,10 @@ type drainReader struct {
 	buf *bytes.Buffer
 }
 
+var _ io.Seeker = (*drainReader)(nil)
+
 func (dr *drainReader) Read(p []byte) (n int, err error) {
 	i := 0
-	// log.Debugs("drainReader:", "buf", dr.buf.Len(), "p", len(p))
 	if dr.buf.Len() > 0 {
 		i, err = dr.buf.Read(p)
 		if err != nil && err != io.EOF {
@@ -70,3 +72,14 @@ func (dr *drainReader) Read(p []byte) (n int, err error) {
 
 	return ri + i, err
 }
+
+// Seek attempt if the underlying reader supports it.
+func (dr *drainReader) Seek(offset int64, whence int) (int64, error) {
+	if dr.buf.Len() > 0 {
+		return 0, fmt.Errorf("unable to seek")
+	}
+	if r, ok := dr.r.(io.Seeker); ok {
+		return r.Seek(offset, whence)
+	}
+	return 0, fmt.Errorf("unable to seek")
+}

src/routes/shorturl.go

@@ -1,19 +1,22 @@
 package routes
 
 import (
+	"bytes"
+	"encoding/json"
 	"net/http"
 	"net/url"
 	"regexp"
-	"time"
 
+	"github.com/coreos/bbolt"
 	"github.com/gorilla/mux"
-	"github.com/patrickmn/go-cache"
 	"sour.is/x/toolbox/httpsrv"
+	"sour.is/x/toolbox/log"
 	"sour.is/x/toolbox/uuid"
 )
 
 func init() {
-	s := NewShortManager(365 * 24 * time.Hour)
+	s := &shortDB{}
+	httpsrv.RegisterModule("short", s.config)
 
 	httpsrv.HttpRegister("short", httpsrv.HttpRoutes{
 		{Name: "getShort", Method: "GET", Pattern: "/s/{id}", HandlerFunc: s.getShort},
@@ -21,38 +24,40 @@ func init() {
 	})
 }
 
-type shortManager struct {
+type shortDB struct {
-	defaultExpire time.Duration
+	path   string
-	db            *cache.Cache
+	bucket string
 }
 
-type shortURL struct {
+func (s *shortDB) config(config map[string]string) {
-	ID     string
+	s.bucket = "shortURL"
-	URL    string
+	if config["bucket"] != "" {
-	Secret string
+		s.bucket = config["bucket"]
-}
-
-func NewShortManager(defaultExpire time.Duration) *shortManager {
-	return &shortManager{
-		defaultExpire: defaultExpire,
-		db:            cache.New(defaultExpire, defaultExpire/10),
 	}
-}
 
-func (s *shortManager) GetURL(id string) *shortURL {
+	s.path = "data/meta.db"
-	if u, ok := s.db.Get(id); ok {
+	if config["store"] != "" {
-		if url, ok := u.(*shortURL); ok {
+		s.path = config["store"]
-			return url
+	}
+
+	db, err := bbolt.Open(s.path, 0666, nil)
+	if err != nil {
+		log.Fatalf("ShortURL: failed to open db at [%s]", s.path)
+	}
+	defer db.Close()
+
+	db.Update(func(tx *bbolt.Tx) error {
+		_, err := tx.CreateBucketIfNotExists([]byte(s.bucket))
+		if err != nil {
+			log.Fatalf("ShortURL: create bucket: %s", err)
 		}
-	}
+		return nil
+	})
 
-	return nil
+	log.Noticef("ShortURL: opened db at [%s] bucket [%s]", s.path, s.bucket)
-}
-func (s *shortManager) PutURL(id string, url *shortURL) {
-	s.db.SetDefault(id, url)
 }
 
-func (s *shortManager) getShort(w http.ResponseWriter, r *http.Request) {
+func (s *shortDB) getShort(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 
 	id := vars["id"]
@@ -67,7 +72,7 @@ func (s *shortManager) getShort(w http.ResponseWriter, r *http.Request) {
 	w.WriteHeader(http.StatusFound)
 }
 
-func (s *shortManager) putShort(w http.ResponseWriter, r *http.Request) {
+func (s *shortDB) putShort(w http.ResponseWriter, r *http.Request) {
 	defer r.Body.Close()
 
 	vars := mux.Vars(r)
@@ -107,6 +112,12 @@ func (s *shortManager) putShort(w http.ResponseWriter, r *http.Request) {
 	httpsrv.WriteObject(w, 200, short)
 }
 
+type shortURL struct {
+	ID     string
+	URL    string
+	Secret string
+}
+
 func newshort(id, secret, u string) *shortURL {
 	m, err := regexp.MatchString("[a-z-]{1,64}", id)
 	if id == "" || !m || err != nil {
@@ -118,3 +129,64 @@ func newshort(id, secret, u string) *shortURL {
 	}
 	return &shortURL{ID: id, Secret: secret, URL: u}
 }
+
+func (s *shortURL) Bytes() []byte {
+	if s == nil {
+		return nil
+	}
+
+	var w bytes.Buffer
+	json.NewEncoder(&w).Encode(*s)
+	return w.Bytes()
+}
+
+func URLFromBytes(b []byte) *shortURL {
+	if len(b) == 0 {
+		return nil
+	}
+	var s shortURL
+	json.Unmarshal(b, &s)
+
+	log.Debug(s)
+	return &s
+}
+
+func (s *shortDB) GetURL(id string) *shortURL {
+	db, err := bbolt.Open(s.path, 0666, nil)
+	if err != nil {
+		log.Errorf("ShortURL: failed to open db at [%s]", s.path)
+		return nil
+	}
+	defer db.Close()
+
+	var url *shortURL
+
+	err = db.View(func(tx *bbolt.Tx) error {
+		b := tx.Bucket([]byte(s.bucket))
+		v := b.Get([]byte(id))
+
+		url = URLFromBytes(v)
+
+		return nil
+	})
+	if err != nil {
+		log.Errorf("ShortURL: failed to open db at [%s]", s.path)
+	}
+	return url
+}
+func (s *shortDB) PutURL(id string, url *shortURL) {
+	db, err := bbolt.Open(s.path, 0666, nil)
+	if err != nil {
+		log.Errorf("ShortURL: failed to open db at [%s]", s.path)
+		return
+	}
+	defer db.Close()
+
+	err = db.Update(func(tx *bbolt.Tx) error {
+		b := tx.Bucket([]byte(s.bucket))
+		return b.Put([]byte(id), url.Bytes())
+	})
+	if err != nil {
+		log.Errorf("ShortURL: failed to write db at [%s]", s.path)
+	}
+}