change to index handling and initial markdown support

.drone.yml

@@ -39,10 +39,10 @@ steps:
 - name: Build Release
   commands:
     - cd debian
+    - make tag
     - make build
     - make copy
     - make deploy
-    - make tag
 trigger:
   event:
   - promote

Makefile

@@ -3,8 +3,12 @@ DOCS_ASSET=src/docs/bindata.go
 
 SOURCE=$(wildcard cmd/paste/*.go) $(filter-out src/routes/bindata.go, $(wildcard src/routes/*.go))
 BINARY=paste
+PKG=./cmd/paste
 
-VERSION:=$(shell debian/inc_version.sh -p $(shell git describe --tags `git rev-list --tags --max-count=1`))
+VERSION=$(shell git describe --tags `git rev-list --tags --max-count=1`|cut -b2-)
+VERSION_PAT=$(shell debian/inc_version.sh -p $(VERSION))
+VERSION_MIN=$(shell debian/inc_version.sh -m $(VERSION))
+VERSION_MAJ=$(shell debian/inc_version.sh -M $(VERSION))
 DATE:=$(shell date -u +%FT%TZ)
 
 define DUMMY_BINDATA
@@ -27,7 +31,9 @@ test: $(ROUTE_ASSET) $(DOCS_ASSET)
 	go test ./...
 	go vet ./...
 run: $(BINARY)
-	./$(BINARY) -vv serve
+	go run \
+	  -ldflags "-X main.AppVersion=$(VERSION_PAT) -X main.AppBuild=$(DATE)" \
+	  $(PKG) -vv serve
 
 $(BINARY): $(SOURCE) $(ROUTE_ASSET) $(DOCS_ASSET)
 	go build -v \
@@ -48,6 +54,7 @@ $(DOCS_ASSET):
 	echo "$$DUMMY_BINDATA" > src/docs/bindata.go
 	go generate "sour.is/x/paste/cmd/paste"
 	go generate "sour.is/x/paste/src/docs"
+
 deploy: $(SOURCE) $(ROUTE_ASSET)
 	cd debian && make
 

cmd/paste/config.go

@@ -54,6 +54,8 @@ store = "data/artifact"
 [module.image]
 store = "data/image"
 
+[module.short]
+store = "data/meta.db"
 `
 
 var args map[string]interface{}

go.mod

@@ -5,16 +5,18 @@ go 1.14
 require (
 	github.com/99designs/gqlgen v0.12.2
 	github.com/andybalholm/brotli v1.0.0
+	github.com/coreos/bbolt v1.3.2
 	github.com/docopt/docopt.go v0.0.0-20180111231733-ee0de3bc6815
 	github.com/go-swagger/go-swagger v0.25.0
 	github.com/golang/gddo v0.0.0-20200831202555-721e228c7686 // indirect
+	github.com/gomarkdown/markdown v0.0.0-20200824053859-8c8b3816f167
 	github.com/gorilla/mux v1.8.0
 	github.com/h2non/filetype v1.1.0
-	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/remyoudompheng/go-liblzma v0.0.0-20190506200333-81bf2d431b96
 	github.com/sour-is/go-assetfs v1.0.0
 	github.com/spf13/viper v1.7.1
 	github.com/vektah/dataloaden v0.3.0
+	go.etcd.io/bbolt v1.3.5 // indirect
 	golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a
 	sour.is/x/toolbox v0.12.17
 )

go.sum

@@ -58,6 +58,7 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cgilling/dbstats v0.0.0-20150427045024-c9db8cf218e6/go.mod h1:fsf3+k/VvGOE9sF2B9d6PBcZOzQIlDJhn2LhBqF/4VY=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@@ -221,6 +222,8 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomarkdown/markdown v0.0.0-20200824053859-8c8b3816f167 h1:LP/6EfrZ/LyCc+SXvANDrIJ4sP9u2NAtqyv6QknetNQ=
+github.com/gomarkdown/markdown v0.0.0-20200824053859-8c8b3816f167/go.mod h1:aii0r/K0ZnHv7G0KF7xy1v0A7s2Ljrb5byB7MO5p6TU=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.1.1-0.20171103154506-982329095285/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
@@ -504,6 +507,8 @@ github.com/yosssi/gmq v0.0.1 h1:GhlDVaAQoi3Mvjul/qJXXGfL4JBeE0GQwbWp3eIsja8=
 github.com/yosssi/gmq v0.0.1/go.mod h1:mReykazh0U1JabvuWh1PEbzzJftqOQWsjr0Lwg5jL1Y=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0=
+go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
 go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
 go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
 go.mongodb.org/mongo-driver v1.3.0/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE=
@@ -515,6 +520,7 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+golang.org/dl v0.0.0-20190829154251-82a15e2f2ead/go.mod h1:IUMfjQLJQd4UTqG1Z90tenwKoCX93Gn3MAQJMOSBsDQ=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@@ -610,6 +616,7 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a h1:i47hUS795cOydZI4AwJQCKXOr4BvxzvikwDoDtHhP2Y=

src/routes/artifact.go

@@ -15,6 +15,10 @@ import (
 	"github.com/gorilla/mux"
 	"sour.is/x/toolbox/httpsrv"
 	"sour.is/x/toolbox/log"
+
+	"github.com/gomarkdown/markdown"
+	"github.com/gomarkdown/markdown/html"
+	"github.com/gomarkdown/markdown/parser"
 )
 
 func init() {
@@ -78,7 +82,7 @@ func (a *Artifact) get(w http.ResponseWriter, r *http.Request) {
 
 	f, err := os.Open(fname)
 	if err != nil {
-		log.Fatal(err)
+		log.Error(err)
 	}
 	defer f.Close()
 
@@ -107,7 +111,7 @@ func (a *Artifact) get(w http.ResponseWriter, r *http.Request) {
 	}
 
 	tr := tar.NewReader(rdr)
-	if path == "..." {
+	if path == "@" {
 		var paths []string
 		for {
 			hdr, err := tr.Next()
@@ -115,7 +119,7 @@ func (a *Artifact) get(w http.ResponseWriter, r *http.Request) {
 				break // End of archive
 			}
 			if err != nil {
-				log.Fatal(err)
+				log.Error(err)
 			}
 			paths = append(paths, hdr.Name)
 		}
@@ -130,9 +134,37 @@ func (a *Artifact) get(w http.ResponseWriter, r *http.Request) {
 			break // End of archive
 		}
 		if err != nil {
-			log.Fatal(err)
+			log.Error(err)
 		}
+		if path == "~" && hdr.Name == "index.html" {
+			path = hdr.Name
+		}
+		if path == "~" && hdr.Name == "index.md" {
+			path = hdr.Name
+		}
+
 		if hdr.Name == path {
+			if strings.HasSuffix(hdr.Name, ".md") {
+				md, err := ioutil.ReadAll(tr)
+				if err != nil {
+					httpsrv.WriteError(w, http.StatusInternalServerError, err.Error())
+					return
+				}
+
+				w.Header().Set("Content-Type", "text/html")
+				extensions := parser.CommonExtensions | parser.AutoHeadingIDs
+				p := parser.NewWithExtensions(extensions)
+
+				htmlFlags := html.CommonFlags | html.HrefTargetBlank
+				opts := html.RendererOptions{Flags: htmlFlags}
+				renderer := html.NewRenderer(opts)
+
+				b := markdown.ToHTML(md, p, renderer)
+				w.Write(b)
+				w.WriteHeader(http.StatusOK)
+				return
+			}
+
 			pr := NewPreviewReader(tr)
 
 			mime, err := ReadMIME(pr, hdr.Name)
@@ -143,7 +175,7 @@ func (a *Artifact) get(w http.ResponseWriter, r *http.Request) {
 			w.Header().Set("Content-Type", mime)
 
 			if _, err := io.Copy(w, pr.Drain()); err != nil {
-				log.Fatal(err)
+				log.Error(err)
 			}
 			w.WriteHeader(http.StatusOK)
 			return

src/routes/preview-reader.go

@@ -3,6 +3,7 @@ package routes
 import (
 	"bytes"
 	"errors"
+	"fmt"
 	"io"
 
 	"sour.is/x/toolbox/log"
@@ -46,9 +47,10 @@ type drainReader struct {
 	buf *bytes.Buffer
 }
 
+var _ io.Seeker = (*drainReader)(nil)
+
 func (dr *drainReader) Read(p []byte) (n int, err error) {
 	i := 0
-	// log.Debugs("drainReader:", "buf", dr.buf.Len(), "p", len(p))
 	if dr.buf.Len() > 0 {
 		i, err = dr.buf.Read(p)
 		if err != nil && err != io.EOF {
@@ -70,3 +72,14 @@ func (dr *drainReader) Read(p []byte) (n int, err error) {
 
 	return ri + i, err
 }
+
+// Seek attempt if the underlying reader supports it.
+func (dr *drainReader) Seek(offset int64, whence int) (int64, error) {
+	if dr.buf.Len() > 0 {
+		return 0, fmt.Errorf("unable to seek")
+	}
+	if r, ok := dr.r.(io.Seeker); ok {
+		return r.Seek(offset, whence)
+	}
+	return 0, fmt.Errorf("unable to seek")
+}

src/routes/shorturl.go

@@ -1,19 +1,22 @@
 package routes
 
 import (
+	"bytes"
+	"encoding/json"
 	"net/http"
 	"net/url"
 	"regexp"
-	"time"
 
+	"github.com/coreos/bbolt"
 	"github.com/gorilla/mux"
-	"github.com/patrickmn/go-cache"
 	"sour.is/x/toolbox/httpsrv"
+	"sour.is/x/toolbox/log"
 	"sour.is/x/toolbox/uuid"
 )
 
 func init() {
-	s := NewShortManager(365 * 24 * time.Hour)
+	s := &shortDB{}
+	httpsrv.RegisterModule("short", s.config)
 
 	httpsrv.HttpRegister("short", httpsrv.HttpRoutes{
 		{Name: "getShort", Method: "GET", Pattern: "/s/{id}", HandlerFunc: s.getShort},
@@ -21,38 +24,40 @@ func init() {
 	})
 }
 
-type shortManager struct {
+type shortDB struct {
-	defaultExpire time.Duration
+	path   string
-	db            *cache.Cache
+	bucket string
 }
 
-type shortURL struct {
+func (s *shortDB) config(config map[string]string) {
-	ID     string
+	s.bucket = "shortURL"
-	URL    string
+	if config["bucket"] != "" {
-	Secret string
+		s.bucket = config["bucket"]
 	}
 
-func NewShortManager(defaultExpire time.Duration) *shortManager {
+	s.path = "data/meta.db"
-	return &shortManager{
+	if config["store"] != "" {
-		defaultExpire: defaultExpire,
+		s.path = config["store"]
-		db:            cache.New(defaultExpire, defaultExpire/10),
-	}
 	}
 
-func (s *shortManager) GetURL(id string) *shortURL {
+	db, err := bbolt.Open(s.path, 0666, nil)
-	if u, ok := s.db.Get(id); ok {
+	if err != nil {
-		if url, ok := u.(*shortURL); ok {
+		log.Fatalf("ShortURL: failed to open db at [%s]", s.path)
-			return url
-		}
 	}
+	defer db.Close()
 
+	db.Update(func(tx *bbolt.Tx) error {
+		_, err := tx.CreateBucketIfNotExists([]byte(s.bucket))
+		if err != nil {
+			log.Fatalf("ShortURL: create bucket: %s", err)
+		}
 		return nil
-}
+	})
-func (s *shortManager) PutURL(id string, url *shortURL) {
+
-	s.db.SetDefault(id, url)
+	log.Noticef("ShortURL: opened db at [%s] bucket [%s]", s.path, s.bucket)
 }
 
-func (s *shortManager) getShort(w http.ResponseWriter, r *http.Request) {
+func (s *shortDB) getShort(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 
 	id := vars["id"]
@@ -67,7 +72,7 @@ func (s *shortManager) getShort(w http.ResponseWriter, r *http.Request) {
 	w.WriteHeader(http.StatusFound)
 }
 
-func (s *shortManager) putShort(w http.ResponseWriter, r *http.Request) {
+func (s *shortDB) putShort(w http.ResponseWriter, r *http.Request) {
 	defer r.Body.Close()
 
 	vars := mux.Vars(r)
@@ -107,6 +112,12 @@ func (s *shortManager) putShort(w http.ResponseWriter, r *http.Request) {
 	httpsrv.WriteObject(w, 200, short)
 }
 
+type shortURL struct {
+	ID     string
+	URL    string
+	Secret string
+}
+
 func newshort(id, secret, u string) *shortURL {
 	m, err := regexp.MatchString("[a-z-]{1,64}", id)
 	if id == "" || !m || err != nil {
@@ -118,3 +129,64 @@ func newshort(id, secret, u string) *shortURL {
 	}
 	return &shortURL{ID: id, Secret: secret, URL: u}
 }
+
+func (s *shortURL) Bytes() []byte {
+	if s == nil {
+		return nil
+	}
+
+	var w bytes.Buffer
+	json.NewEncoder(&w).Encode(*s)
+	return w.Bytes()
+}
+
+func URLFromBytes(b []byte) *shortURL {
+	if len(b) == 0 {
+		return nil
+	}
+	var s shortURL
+	json.Unmarshal(b, &s)
+
+	log.Debug(s)
+	return &s
+}
+
+func (s *shortDB) GetURL(id string) *shortURL {
+	db, err := bbolt.Open(s.path, 0666, nil)
+	if err != nil {
+		log.Errorf("ShortURL: failed to open db at [%s]", s.path)
+		return nil
+	}
+	defer db.Close()
+
+	var url *shortURL
+
+	err = db.View(func(tx *bbolt.Tx) error {
+		b := tx.Bucket([]byte(s.bucket))
+		v := b.Get([]byte(id))
+
+		url = URLFromBytes(v)
+
+		return nil
+	})
+	if err != nil {
+		log.Errorf("ShortURL: failed to open db at [%s]", s.path)
+	}
+	return url
+}
+func (s *shortDB) PutURL(id string, url *shortURL) {
+	db, err := bbolt.Open(s.path, 0666, nil)
+	if err != nil {
+		log.Errorf("ShortURL: failed to open db at [%s]", s.path)
+		return
+	}
+	defer db.Close()
+
+	err = db.Update(func(tx *bbolt.Tx) error {
+		b := tx.Bucket([]byte(s.bucket))
+		return b.Put([]byte(id), url.Bytes())
+	})
+	if err != nil {
+		log.Errorf("ShortURL: failed to write db at [%s]", s.path)
+	}
+}