chore: refactor

This commit is contained in:
xuu 2025-03-31 17:39:24 -06:00
parent cd2c9abd1b
commit ef65b115b7
Signed by: xuu
GPG Key ID: 8B3B0604F164E04F
5 changed files with 304 additions and 264 deletions

35
feed.go
View File

@ -21,13 +21,14 @@ import (
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"go.sour.is/xt/internal/otel"
"go.sour.is/xt/internal/uuid"
"go.yarn.social/lextwt"
"go.yarn.social/types"
)
type Feed struct {
FeedID uuid
ParentID uuid
FeedID uuid.UUID
ParentID uuid.UUID
HashURI string
URI string
Nick string
@ -236,7 +237,7 @@ func storeFeed(ctx context.Context, db db, f types.TwtFile) error {
loadTS := time.Now()
refreshRate := 600
feedID := urlNS.UUID5(cmp.Or(f.Twter().HashingURI, f.Twter().URI))
feedID := uuid.UrlNS.UUID5(cmp.Or(f.Twter().HashingURI, f.Twter().URI))
tx, err := db.BeginTx(ctx, nil)
if err != nil {
@ -269,13 +270,13 @@ func storeFeed(ctx context.Context, db db, f types.TwtFile) error {
for _, twt := range twts {
twtID := makeULID(twt)
mentions := make(uuids, 0, len(twt.Mentions()))
mentions := make(uuid.UUIDs, 0, len(twt.Mentions()))
for _, mention := range twt.Mentions() {
followMap[mention.Twter().URI] = mention.Twter().Nick
mentions = append(mentions, urlNS.UUID5(mention.Twter().URI))
mentions = append(mentions, uuid.UrlNS.UUID5(mention.Twter().URI))
}
tags := make(strList, 0, len(twt.Tags()))
tags := make(uuid.List, 0, len(twt.Tags()))
for _, tag := range twt.Tags() {
tags = append(tags, tag.Text())
}
@ -335,7 +336,7 @@ func storeFeed(ctx context.Context, db db, f types.TwtFile) error {
}
part = uri[:strings.LastIndex(uri, "/")+1] + part
childID := urlNS.UUID5(part)
childID := uuid.UrlNS.UUID5(part)
fmt.Println("found prev", uri, part)
args = append(args,
childID, // feed_id
@ -351,7 +352,7 @@ func storeFeed(ctx context.Context, db db, f types.TwtFile) error {
for uri, nick := range followMap {
args = append(args,
urlNS.UUID5(uri), // feed_id
uuid.UrlNS.UUID5(uri), // feed_id
nil, // parent_id
nick, // nick
uri, // uri
@ -389,7 +390,7 @@ func storeRegistry(ctx context.Context, db db, in io.Reader) error {
nick := twt.Twter().DomainNick()
uri := twt.Twter().URI
feedID := urlNS.UUID5(uri)
feedID := uuid.UrlNS.UUID5(uri)
twtID := makeULID(twt)
text := fmt.Sprintf("%+l", twt)
@ -405,13 +406,13 @@ func storeRegistry(ctx context.Context, db db, in io.Reader) error {
twters[uri] = nick
mentions := make(uuids, 0, len(twt.Mentions()))
mentions := make(uuid.UUIDs, 0, len(twt.Mentions()))
for _, mention := range twt.Mentions() {
twters[uri] = nick
mentions = append(mentions, urlNS.UUID5(mention.Twter().URI))
mentions = append(mentions, uuid.UrlNS.UUID5(mention.Twter().URI))
}
tags := make(strList, 0, len(twt.Tags()))
tags := make(uuid.List, 0, len(twt.Tags()))
for _, tag := range twt.Tags() {
tags = append(tags, tag.Text())
}
@ -466,7 +467,7 @@ func storeRegistry(ctx context.Context, db db, in io.Reader) error {
// continue
// }
feedID := urlNS.UUID5(uri)
feedID := uuid.UrlNS.UUID5(uri)
args = append(args,
feedID, // feed_id
@ -641,7 +642,7 @@ func fetchTwts(ctx context.Context, db db, uri string, limit int, offset int64)
where := `where feed_id in (select feed_id from feeds where state != 'permanantly-dead')`
if uri != "" {
feed_id := urlNS.UUID5(uri)
feed_id := uuid.UrlNS.UUID5(uri)
where = "where feed_id = ?"
args = append(args, feed_id)
}
@ -732,7 +733,7 @@ func fetchUsers(ctx context.Context, db db, uri, q string) ([]types.Twt, error)
args := make([]any, 0)
if uri != "" {
where = `where feed_id = ? or parent_id = ?`
feed_id := urlNS.UUID5(uri)
feed_id := uuid.UrlNS.UUID5(uri)
args = append(args, feed_id, feed_id)
} else if q != "" {
where = `where nick like ?`
@ -783,7 +784,7 @@ func fetchUsers(ctx context.Context, db db, uri, q string) ([]types.Twt, error)
return twts, nil
}
func fetchMentions(ctx context.Context, db db, mention uuid, limit int, offset int64) ([]types.Twt, int64, int64, error) {
func fetchMentions(ctx context.Context, db db, mention uuid.UUID, limit int, offset int64) ([]types.Twt, int64, int64, error) {
ctx, span := otel.Span(ctx)
defer span.End()
@ -871,7 +872,7 @@ func fetchMentions(ctx context.Context, db db, mention uuid, limit int, offset i
return twts, offset, end, err
}
func fetchConv(ctx context.Context, db db, hash string, limit int, offset int64) ([]types.Twt, int64, int64, error) {
func fetchConv(ctx context.Context, db db, hash string, _ int, offset int64) ([]types.Twt, int64, int64, error) {
ctx, span := otel.Span(ctx)
defer span.End()

225
http-api.go Normal file
View File

@ -0,0 +1,225 @@
package main
import (
"fmt"
"net/http"
"slices"
"sort"
"strconv"
"strings"
"time"
"go.sour.is/xt/internal/otel"
"go.sour.is/xt/internal/uuid"
"go.yarn.social/lextwt"
)
type API struct {
app *appState
db db
hostname string
}
func (a *API) plain(w http.ResponseWriter, r *http.Request) {
reg := lextwt.NewTwtRegistry(mkPreambleDocs(a.hostname), nil)
reg.WriteTo(w)
}
func (a *API) conv(w http.ResponseWriter, r *http.Request) {
ctx, span := otel.Span(r.Context())
defer span.End()
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
hash := r.PathValue("hash")
if (len(hash) < 6 || len(hash) > 8) && !notAny(hash, "abcdefghijklmnopqrstuvwxyz234567") {
w.WriteHeader(http.StatusBadRequest)
return
}
limit := 100
if v, ok := strconv.Atoi(r.URL.Query().Get("limit")); ok == nil {
limit = v
}
var offset int64 = 0
if v, ok := strconv.ParseInt(r.URL.Query().Get("offset"), 10, 64); ok == nil {
offset = v
}
twts, offset, end, err := fetchConv(ctx, a.db, hash, limit, offset)
span.RecordError(err)
if err != nil {
http.Error(w, "ERR", 500)
return
}
preamble := mkPreamble(a.hostname, "", "/api/plain/conv/"+hash, limit, int64(len(twts)), offset, end)
reg := lextwt.NewTwtRegistry(preamble, twts)
reg.WriteTo(w)
}
func (a *API) mentions(w http.ResponseWriter, r *http.Request) {
ctx, span := otel.Span(r.Context())
defer span.End()
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
uri := r.URL.Query().Get("uri")
if uri == "" {
reg := lextwt.NewTwtRegistry(mkPreambleDocs(a.hostname), nil)
reg.WriteTo(w)
return
}
mention := uuid.UrlNS.UUID5(uri)
limit := 100
if v, ok := strconv.Atoi(r.URL.Query().Get("limit")); ok == nil {
limit = v
}
var offset int64 = 0
if v, ok := strconv.ParseInt(r.URL.Query().Get("offset"), 10, 64); ok == nil {
offset = v
}
twts, offset, end, err := fetchMentions(ctx, a.db, mention, limit, offset)
span.RecordError(err)
if err != nil {
http.Error(w, "ERR", 500)
return
}
preamble := mkPreamble(a.hostname, uri, "/api/plain/mentions", limit, int64(len(twts)), offset, end)
reg := lextwt.NewTwtRegistry(preamble, twts)
reg.WriteTo(w)
}
func (a *API) twt(w http.ResponseWriter, r *http.Request) {
ctx, span := otel.Span(r.Context())
defer span.End()
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
uri := r.URL.Query().Get("uri")
limit := 100
if v, ok := strconv.Atoi(r.URL.Query().Get("limit")); ok == nil {
limit = v
}
limit = min(100, max(1, limit))
var offset int64 = 0
if v, ok := strconv.ParseInt(r.URL.Query().Get("offset"), 10, 64); ok == nil {
offset = v
}
twts, offset, end, err := fetchTwts(ctx, a.db, uri, limit, offset)
span.RecordError(err)
if err != nil {
http.Error(w, "ERR", 500)
return
}
preamble := mkPreamble(a.hostname, uri, "/api/plain/twt", limit, int64(len(twts)), offset, end)
reg := lextwt.NewTwtRegistry(preamble, twts)
reg.WriteTo(w)
}
func (a *API) users(w http.ResponseWriter, r *http.Request) {
ctx, span := otel.Span(r.Context())
defer span.End()
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
uri := r.URL.Query().Get("uri")
q := r.URL.Query().Get("q")
twts, err := fetchUsers(ctx, a.db, uri, q)
if err != nil {
http.Error(w, "ERR", 500)
return
}
reg := lextwt.NewTwtRegistry(mkPreambleDocs(a.hostname), twts)
reg.WriteTo(w)
}
func (a *API) queue(w http.ResponseWriter, r *http.Request) {
lis := slices.Collect(a.app.queue.Iter())
sort.Slice(lis, func(i, j int) bool {
return lis[i].NextScanOn.Time.Before(lis[j].LastScanOn.Time)
})
for _, feed := range lis {
fmt.Fprintln(w, feed.State, feed.NextScanOn.Time.Format(time.RFC3339), feed.Nick, feed.URI)
}
}
func notAny(s string, chars string) bool {
for _, c := range s {
if !strings.ContainsRune(chars, c) {
return false
}
}
return true
}
func mkqry(uri string, limit int, offset int64) string {
qry := make([]string, 0, 3)
if uri != "" {
qry = append(qry, "uri="+uri)
}
if limit != 100 {
qry = append(qry, fmt.Sprint("limit=", limit))
}
if offset != 0 {
qry = append(qry, fmt.Sprint("offset=", offset))
}
if len(qry) == 0 {
return ""
}
return "?" + strings.Join(qry, "&")
}
func add(preamble lextwt.Comments, text string, v ...any) lextwt.Comments {
if len(v) > 0 {
text = fmt.Sprintf(text, v...)
}
return append(preamble, lextwt.NewComment("# "+text))
}
func mkPreamble(hostname, uri, path string, limit int, length, offset, end int64) lextwt.Comments {
uri += path
preamble := add(mkPreambleDocs(hostname), "twt range = 1 %d", end)
preamble = add(preamble, "self = %s%s", hostname, mkqry(uri, limit, offset))
if next := offset + length; next < end {
preamble = add(preamble, "next = %s%s", hostname, mkqry(uri, limit, next))
}
if prev := offset - int64(limit); prev > 0 {
preamble = add(preamble, "prev = %s%s", hostname, mkqry(uri, limit, prev))
}
return preamble
}
var mkPreambleDocs = func(hostname string) lextwt.Comments {
c := add(nil, iAmTheWatcher)
c = add(c, "")
c = add(c, "Usage:")
c = add(c, " %s/api/plain/users View list of users and latest twt date.", hostname)
c = add(c, " %s/api/plain/twt View all twts.", hostname)
c = add(c, " %s/api/plain/mentions?uri=:uri View all mentions for uri.", hostname)
c = add(c, " %s/api/plain/conv/:hash View all twts for a conversation subject.", hostname)
c = add(c, "")
c = add(c, "Options:")
c = add(c, " uri Filter to show a specific users twts.")
c = add(c, " offset Start index for quey.")
c = add(c, " limit Count of items to return (going back in time).")
return add(c, "")
}

30
http-html.go Normal file
View File

@ -0,0 +1,30 @@
package main
import (
"net/http"
"go.sour.is/xt/internal/otel"
)
type HTML struct {
app *appState
db db
hostname string
}
func (a *HTML) healthcheck(w http.ResponseWriter, r *http.Request) {
_, span := otel.Span(r.Context())
defer span.End()
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
w.Write([]byte("ok"))
}
func (a *HTML) home(w http.ResponseWriter, r *http.Request) {
_, span := otel.Span(r.Context())
defer span.End()
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
w.Write([]byte("ok"))
}

230
http.go
View File

@ -5,35 +5,12 @@ import (
"errors"
"fmt"
"net/http"
"slices"
"sort"
"strconv"
"strings"
"time"
"go.yarn.social/lextwt"
"go.sour.is/xt/internal/otel"
)
const iAmTheWatcher = "I am the Watcher. I am your guide through this vast new twtiverse."
var mkPreambleDocs = func(hostname string) lextwt.Comments {
c := add(nil, iAmTheWatcher)
c = add(c, "")
c = add(c, "Usage:")
c = add(c, " %s/api/plain/users View list of users and latest twt date.", hostname)
c = add(c, " %s/api/plain/twt View all twts.", hostname)
c = add(c, " %s/api/plain/mentions?uri=:uri View all mentions for uri.", hostname)
c = add(c, " %s/api/plain/conv/:hash View all twts for a conversation subject.", hostname)
c = add(c, "")
c = add(c, "Options:")
c = add(c, " uri Filter to show a specific users twts.")
c = add(c, " offset Start index for quey.")
c = add(c, " limit Count of items to return (going back in time).")
return add(c, "")
}
func httpServer(ctx context.Context, app *appState) error {
ctx, span := otel.Span(ctx)
defer span.End()
@ -52,13 +29,14 @@ func httpServer(ctx context.Context, app *appState) error {
hostname: app.args.Hostname,
}
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
_, span := otel.Span(r.Context())
defer span.End()
html := HTML{
app: app,
db: db,
hostname: app.args.Hostname,
}
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
w.Write([]byte("ok"))
})
http.HandleFunc("/", html.home)
http.HandleFunc("/health", html.healthcheck)
http.HandleFunc("/api/plain", api.plain)
http.HandleFunc("/api/plain/conv/{hash}", api.conv)
@ -82,197 +60,3 @@ func httpServer(ctx context.Context, app *appState) error {
return nil
}
func notAny(s string, chars string) bool {
for _, c := range s {
if !strings.ContainsRune(chars, c) {
return false
}
}
return true
}
func mkqry(uri string, limit int, offset int64) string {
qry := make([]string, 0, 3)
if uri != "" {
qry = append(qry, "uri="+uri)
}
if limit != 100 {
qry = append(qry, fmt.Sprint("limit=", limit))
}
if offset != 0 {
qry = append(qry, fmt.Sprint("offset=", offset))
}
if len(qry) == 0 {
return ""
}
return "?" + strings.Join(qry, "&")
}
func add(preamble lextwt.Comments, text string, v ...any) lextwt.Comments {
if len(v) > 0 {
text = fmt.Sprintf(text, v...)
}
return append(preamble, lextwt.NewComment("# "+text))
}
func mkPreamble(hostname, uri, path string, limit int, length, offset, end int64) lextwt.Comments {
uri += path
preamble := add(mkPreambleDocs(hostname), "twt range = 1 %d", end)
preamble = add(preamble, "self = %s%s", hostname, mkqry(uri, limit, offset))
if next := offset + length; next < end {
preamble = add(preamble, "next = %s%s", hostname, mkqry(uri, limit, next))
}
if prev := offset - int64(limit); prev > 0 {
preamble = add(preamble, "prev = %s%s", hostname, mkqry(uri, limit, prev))
}
return preamble
}
type API struct {
app *appState
db db
hostname string
}
func (a *API) plain(w http.ResponseWriter, r *http.Request) {
reg := lextwt.NewTwtRegistry(mkPreambleDocs(a.hostname), nil)
reg.WriteTo(w)
}
func (a *API) conv(w http.ResponseWriter, r *http.Request) {
ctx, span := otel.Span(r.Context())
defer span.End()
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
hash := r.PathValue("hash")
if (len(hash) < 6 || len(hash) > 8) && !notAny(hash, "abcdefghijklmnopqrstuvwxyz234567") {
w.WriteHeader(http.StatusBadRequest)
return
}
limit := 100
if v, ok := strconv.Atoi(r.URL.Query().Get("limit")); ok == nil {
limit = v
}
var offset int64 = 0
if v, ok := strconv.ParseInt(r.URL.Query().Get("offset"), 10, 64); ok == nil {
offset = v
}
twts, offset, end, err := fetchConv(ctx, a.db, hash, limit, offset)
span.RecordError(err)
if err != nil {
http.Error(w, "ERR", 500)
return
}
preamble := mkPreamble(a.hostname, "", "/api/plain/conv/"+hash, limit, int64(len(twts)), offset, end)
reg := lextwt.NewTwtRegistry(preamble, twts)
reg.WriteTo(w)
}
func (a *API) mentions(w http.ResponseWriter, r *http.Request) {
ctx, span := otel.Span(r.Context())
defer span.End()
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
uri := r.URL.Query().Get("uri")
if uri == "" {
reg := lextwt.NewTwtRegistry(mkPreambleDocs(a.hostname), nil)
reg.WriteTo(w)
return
}
mention := urlNS.UUID5(uri)
limit := 100
if v, ok := strconv.Atoi(r.URL.Query().Get("limit")); ok == nil {
limit = v
}
var offset int64 = 0
if v, ok := strconv.ParseInt(r.URL.Query().Get("offset"), 10, 64); ok == nil {
offset = v
}
twts, offset, end, err := fetchMentions(ctx, a.db, mention, limit, offset)
span.RecordError(err)
if err != nil {
http.Error(w, "ERR", 500)
return
}
preamble := mkPreamble(a.hostname, uri, "/api/plain/mentions", limit, int64(len(twts)), offset, end)
reg := lextwt.NewTwtRegistry(preamble, twts)
reg.WriteTo(w)
}
func (a *API) twt(w http.ResponseWriter, r *http.Request) {
ctx, span := otel.Span(r.Context())
defer span.End()
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
uri := r.URL.Query().Get("uri")
limit := 100
if v, ok := strconv.Atoi(r.URL.Query().Get("limit")); ok == nil {
limit = v
}
limit = min(100, max(1, limit))
var offset int64 = 0
if v, ok := strconv.ParseInt(r.URL.Query().Get("offset"), 10, 64); ok == nil {
offset = v
}
twts, offset, end, err := fetchTwts(ctx, a.db, uri, limit, offset)
span.RecordError(err)
if err != nil {
http.Error(w, "ERR", 500)
return
}
preamble := mkPreamble(a.hostname, uri, "/api/plain/twt", limit, int64(len(twts)), offset, end)
reg := lextwt.NewTwtRegistry(preamble, twts)
reg.WriteTo(w)
}
func (a *API) users(w http.ResponseWriter, r *http.Request) {
ctx, span := otel.Span(r.Context())
defer span.End()
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
uri := r.URL.Query().Get("uri")
q := r.URL.Query().Get("q")
twts, err := fetchUsers(ctx, a.db, uri, q)
if err != nil {
http.Error(w, "ERR", 500)
return
}
reg := lextwt.NewTwtRegistry(mkPreambleDocs(a.hostname), twts)
reg.WriteTo(w)
}
func (a *API) queue(w http.ResponseWriter, r *http.Request) {
lis := slices.Collect(a.app.queue.Iter())
sort.Slice(lis, func(i, j int) bool {
return lis[i].NextScanOn.Time.Before(lis[j].LastScanOn.Time)
})
for _, feed := range lis {
fmt.Fprintln(w, feed.State, feed.NextScanOn.Time.Format(time.RFC3339), feed.Nick, feed.URI)
}
}

View File

@ -1,4 +1,4 @@
package main
package uuid
import (
"crypto/sha1"
@ -8,54 +8,54 @@ import (
"strings"
)
type uuid [16]byte
type UUID [16]byte
var urlNS = uuid{0x6b, 0xa7, 0xb8, 0x10, 0x9d, 0xad, 0x11, 0xd1, 0x80, 0xb4, 0x00, 0xc0, 0x4f, 0xd4, 0x30, 0xc8}
var UrlNS = UUID{0x6b, 0xa7, 0xb8, 0x10, 0x9d, 0xad, 0x11, 0xd1, 0x80, 0xb4, 0x00, 0xc0, 0x4f, 0xd4, 0x30, 0xc8}
func (u uuid) UUID5(value string) uuid {
func (u UUID) UUID5(value string) UUID {
h := sha1.New()
h.Write(u[:])
h.Write([]byte(value))
return uuid(h.Sum(nil))
return UUID(h.Sum(nil))
}
func (u *uuid) UnmarshalText(data string) error {
func (u *UUID) UnmarshalText(data string) error {
data = strings.Trim(data, "{}")
data = strings.ReplaceAll(data, "-", "")
_, err := hex.Decode(u[:], []byte(data))
return err
}
func (u uuid) MarshalText() string {
func (u UUID) MarshalText() string {
s := hex.EncodeToString(u[:])
return fmt.Sprintf("{%s-%s-%s-%s-%s}", s[:8], s[8:12], s[12:16], s[16:20], s[20:])
}
func (u uuid) Value() (driver.Value, error) {
func (u UUID) Value() (driver.Value, error) {
return u[:], nil
}
func (u *uuid) Scan(value any) error {
func (u *UUID) Scan(value any) error {
if value == nil {
return nil
}
*u = uuid(value.([]byte))
*u = UUID(value.([]byte))
return nil
}
type uuids []uuid
type UUIDs []UUID
func (lis uuids) ToStrList() strList {
arr := make(strList, len(lis))
func (lis UUIDs) ToStrList() List {
arr := make(List, len(lis))
for i, v := range lis {
arr[i] = v.MarshalText()
}
return arr
}
type strList []string
type List []string
func (l *strList) Scan(value any) error {
func (l *List) Scan(value any) error {
s := value.(string)
s = strings.Trim(s, "[]")
for _, v := range strings.Split(s, ",") {
@ -67,7 +67,7 @@ func (l *strList) Scan(value any) error {
return nil
}
func (l strList) Value() (driver.Value, error) {
func (l List) Value() (driver.Value, error) {
arr := make([]string, len(l))
for i, v := range l {
arr[i] = "\"" + v + "\""