chore: fix twt hash

cmd/load/main.go

@@ -1,92 +0,0 @@
-package main
-
-import (
-	"context"
-	"database/sql"
-	"fmt"
-	"hash/fnv"
-	"iter"
-	"os"
-
-	"github.com/oklog/ulid/v2"
-	"github.com/uptrace/opentelemetry-go-extra/otelsql"
-	"go.yarn.social/lextwt"
-	"go.yarn.social/types"
-)
-
-func main() {
-	in := os.Stdin
-	if len(os.Args) != 2 {
-		fmt.Fprint(os.Stderr, "usage: ", os.Args[0], "[db file]")
-	}
-
-	db, err := DB(context.Background(), os.Args[1])
-	if err != nil {
-		panic(err)
-	}
-	_ = db
-
-	for line := range lextwt.IterRegistry(in) {
-		_ = line
-	}
-}
-
-const MaxVariableNumber = 32766
-
-func DB(ctx context.Context, cnx string) (*sql.DB, error) {
-	// return sql.Open(app.args.dbtype, app.args.dbfile)
-
-	db, err := otelsql.Open("sqlite", cnx)
-	if err != nil {
-		return db, err
-	}
-
-	return db, err
-}
-
-func makeULID(twt types.Twt) ulid.ULID {
-	h64 := fnv.New64a()
-	h16 := fnv.New32a()
-	text := []byte(fmt.Sprintf("%+l", twt))
-	b := make([]byte, 10)
-	copy(b, h16.Sum(text)[:2])
-	copy(b[2:], h64.Sum(text))
-	u := ulid.ULID{}
-	u.SetTime(ulid.Timestamp(twt.Created()))
-	u.SetEntropy(b)
-
-	return u
-}
-
-func chunk(args []any, qry func(int) (string, int), maxArgs int) iter.Seq2[string, []any] {
-	_, size := qry(1)
-	itemsPerIter := maxArgs / size
-
-	if len(args) < size {
-		return func(yield func(string, []any) bool) {}
-	}
-
-	if len(args) < maxArgs {
-		return func(yield func(string, []any) bool) {
-			query, _ := qry(len(args) / size)
-			yield(query, args)
-		}
-	}
-
-	return func(yield func(string, []any) bool) {
-		for len(args) > 0 {
-			if len(args) > maxArgs {
-				query, size := qry(itemsPerIter)
-				if !yield(query, args[:size]) {
-					return
-				}
-				args = args[size:]
-				continue
-			}
-
-			query, _ := qry(len(args) / size)
-			yield(query, args)
-			return
-		}
-	}
-}

feed.go

@@ -3,10 +3,10 @@ package main
 import (
 	"cmp"
 	"context"
+	"crypto/sha3"
 	"database/sql"
 	"database/sql/driver"
 	"fmt"
-	"hash/fnv"
 	"io"
 	"iter"
 	"net/http"
@@ -378,6 +378,7 @@ func storeRegistry(ctx context.Context, db db, in io.Reader) error {
 	twters := make(map[string]string)
 	args := make([]any, 0, 1024*16)
 
+	i := 0
 	for line := range lextwt.IterRegistry(in) {
 		twt, ok := line.(*lextwt.Twt)
 		if !ok {
@@ -431,6 +432,8 @@ func storeRegistry(ctx context.Context, db db, in io.Reader) error {
 		)
 
 		if len(args) >= 16*1022 {
+			i+=len(args)
+			fmt.Println("store", i/7, i)
 			tx, err := db.BeginTx(ctx, nil)
 			if err != nil {
 				return err
@@ -560,15 +563,10 @@ func (n TwtTime) Value() (driver.Value, error) {
 }
 
 func makeULID(twt types.Twt) ulid.ULID {
-	h64 := fnv.New64a()
+	text := fmt.Appendf(nil, "%s\t%+l", cmp.Or(twt.Twter().HashingURI, twt.Twter().URI), twt)
-	h16 := fnv.New32a()
-	text := []byte(fmt.Sprintf("%+l", twt))
-	b := make([]byte, 10)
-	copy(b, h16.Sum(text)[:2])
-	copy(b[2:], h64.Sum(text))
 	u := ulid.ULID{}
 	u.SetTime(ulid.Timestamp(twt.Created()))
-	u.SetEntropy(b)
+	u.SetEntropy(sha3.SumSHAKE128(text, 10))
 
 	return u
 }

go.mod

@@ -1,6 +1,6 @@
 module go.sour.is/xt
 
-go 1.23.2
+go 1.24.1
 
 require (
 	github.com/mattn/go-sqlite3 v1.14.24

http.go

@@ -145,7 +145,7 @@ func httpServer(ctx context.Context, app *appState) error {
 	
 			return
 		}
-		mention := urlNS.UUID5(uri).MarshalText()
+		mention := urlNS.UUID5(uri)
 
 		args := make([]any, 0, 3)
 		args = append(args, mention)
@@ -163,9 +163,9 @@ func httpServer(ctx context.Context, app *appState) error {
 
 		var end int64
 		err = db.QueryRowContext(ctx, `
-			select count(*) n from twts, json_each(mentions) where value = ?`, args...).Scan(&end)
+			select count(*) n from twt_mentions where feed_id = ?`, args...).Scan(&end)
 		span.RecordError(err)
-		fmt.Println(mention, end, err)
+		fmt.Println(mention.MarshalText(), end, err)
 
 		if offset < 1 {
 			offset += end
@@ -191,7 +191,7 @@ func httpServer(ctx context.Context, app *appState) error {
 			) using (feed_id)
 			where rowid in (
 				select rowid from twts
-				where ulid in (select ulid from twts, json_each(mentions) where value = ?)
+				where ulid in (select ulid from twt_mentions where feed_id = ?)
 				order by ulid asc
 				limit ?
 				offset ?
@@ -231,6 +231,11 @@ func httpServer(ctx context.Context, app *appState) error {
 			twt, _ := lextwt.ParseLine(o.Text, &twter)
 			twts = append(twts, twt)
 		}
+		if err := rows.Err(); err != nil {
+			fmt.Println(err)
+			return
+		}
+
 		preamble := add(PREAMBLE_DOCS, "twt range = 1  %d", end)
 		preamble = add(preamble, "self = %s/mentions%s", hostname, mkqry(uri, limit, offset))
 		if next := offset + int64(len(twts)); next < end {