chore: fix twt hash
This commit is contained in:
parent
fb957ed25d
commit
fc762f3bf3
GPG Key ID:
8B3B0604F164E04F
@ -1,92 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"iter"
|
||||
"os"
|
||||
|
||||
"github.com/oklog/ulid/v2"
|
||||
"github.com/uptrace/opentelemetry-go-extra/otelsql"
|
||||
"go.yarn.social/lextwt"
|
||||
"go.yarn.social/types"
|
||||
)
|
||||
|
||||
func main() {
|
||||
in := os.Stdin
|
||||
if len(os.Args) != 2 {
|
||||
fmt.Fprint(os.Stderr, "usage: ", os.Args[0], "[db file]")
|
||||
}
|
||||
|
||||
db, err := DB(context.Background(), os.Args[1])
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
_ = db
|
||||
|
||||
for line := range lextwt.IterRegistry(in) {
|
||||
_ = line
|
||||
}
|
||||
}
|
||||
|
||||
const MaxVariableNumber = 32766
|
||||
|
||||
func DB(ctx context.Context, cnx string) (*sql.DB, error) {
|
||||
// return sql.Open(app.args.dbtype, app.args.dbfile)
|
||||
|
||||
db, err := otelsql.Open("sqlite", cnx)
|
||||
if err != nil {
|
||||
return db, err
|
||||
}
|
||||
|
||||
return db, err
|
||||
}
|
||||
|
||||
func makeULID(twt types.Twt) ulid.ULID {
|
||||
h64 := fnv.New64a()
|
||||
h16 := fnv.New32a()
|
||||
text := []byte(fmt.Sprintf("%+l", twt))
|
||||
b := make([]byte, 10)
|
||||
copy(b, h16.Sum(text)[:2])
|
||||
copy(b[2:], h64.Sum(text))
|
||||
u := ulid.ULID{}
|
||||
u.SetTime(ulid.Timestamp(twt.Created()))
|
||||
u.SetEntropy(b)
|
||||
|
||||
return u
|
||||
}
|
||||
|
||||
func chunk(args []any, qry func(int) (string, int), maxArgs int) iter.Seq2[string, []any] {
|
||||
_, size := qry(1)
|
||||
itemsPerIter := maxArgs / size
|
||||
|
||||
if len(args) < size {
|
||||
return func(yield func(string, []any) bool) {}
|
||||
}
|
||||
|
||||
if len(args) < maxArgs {
|
||||
return func(yield func(string, []any) bool) {
|
||||
query, _ := qry(len(args) / size)
|
||||
yield(query, args)
|
||||
}
|
||||
}
|
||||
|
||||
return func(yield func(string, []any) bool) {
|
||||
for len(args) > 0 {
|
||||
if len(args) > maxArgs {
|
||||
query, size := qry(itemsPerIter)
|
||||
if !yield(query, args[:size]) {
|
||||
return
|
||||
}
|
||||
args = args[size:]
|
||||
continue
|
||||
}
|
||||
|
||||
query, _ := qry(len(args) / size)
|
||||
yield(query, args)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
14
feed.go
14
feed.go
@ -3,10 +3,10 @@ package main
|
||||
import (
|
||||
"cmp"
|
||||
"context"
|
||||
"crypto/sha3"
|
||||
"database/sql"
|
||||
"database/sql/driver"
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"io"
|
||||
"iter"
|
||||
"net/http"
|
||||
@ -378,6 +378,7 @@ func storeRegistry(ctx context.Context, db db, in io.Reader) error {
|
||||
twters := make(map[string]string)
|
||||
args := make([]any, 0, 1024*16)
|
||||
|
||||
i := 0
|
||||
for line := range lextwt.IterRegistry(in) {
|
||||
twt, ok := line.(*lextwt.Twt)
|
||||
if !ok {
|
||||
@ -431,6 +432,8 @@ func storeRegistry(ctx context.Context, db db, in io.Reader) error {
|
||||
)
|
||||
|
||||
if len(args) >= 16*1022 {
|
||||
i+=len(args)
|
||||
fmt.Println("store", i/7, i)
|
||||
tx, err := db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
@ -560,15 +563,10 @@ func (n TwtTime) Value() (driver.Value, error) {
|
||||
}
|
||||
|
||||
func makeULID(twt types.Twt) ulid.ULID {
|
||||
h64 := fnv.New64a()
|
||||
h16 := fnv.New32a()
|
||||
text := []byte(fmt.Sprintf("%+l", twt))
|
||||
b := make([]byte, 10)
|
||||
copy(b, h16.Sum(text)[:2])
|
||||
copy(b[2:], h64.Sum(text))
|
||||
text := fmt.Appendf(nil, "%s\t%+l", cmp.Or(twt.Twter().HashingURI, twt.Twter().URI), twt)
|
||||
u := ulid.ULID{}
|
||||
u.SetTime(ulid.Timestamp(twt.Created()))
|
||||
u.SetEntropy(b)
|
||||
u.SetEntropy(sha3.SumSHAKE128(text, 10))
|
||||
|
||||
return u
|
||||
}
|
||||
|
||||
2
go.mod
2
go.mod
@ -1,6 +1,6 @@
|
||||
module go.sour.is/xt
|
||||
|
||||
go 1.23.2
|
||||
go 1.24.1
|
||||
|
||||
require (
|
||||
github.com/mattn/go-sqlite3 v1.14.24
|
||||
|
||||
13
http.go
13
http.go
@ -145,7 +145,7 @@ func httpServer(ctx context.Context, app *appState) error {
|
||||
|
||||
return
|
||||
}
|
||||
mention := urlNS.UUID5(uri).MarshalText()
|
||||
mention := urlNS.UUID5(uri)
|
||||
|
||||
args := make([]any, 0, 3)
|
||||
args = append(args, mention)
|
||||
@ -163,9 +163,9 @@ func httpServer(ctx context.Context, app *appState) error {
|
||||
|
||||
var end int64
|
||||
err = db.QueryRowContext(ctx, `
|
||||
select count(*) n from twts, json_each(mentions) where value = ?`, args...).Scan(&end)
|
||||
select count(*) n from twt_mentions where feed_id = ?`, args...).Scan(&end)
|
||||
span.RecordError(err)
|
||||
fmt.Println(mention, end, err)
|
||||
fmt.Println(mention.MarshalText(), end, err)
|
||||
|
||||
if offset < 1 {
|
||||
offset += end
|
||||
@ -191,7 +191,7 @@ func httpServer(ctx context.Context, app *appState) error {
|
||||
) using (feed_id)
|
||||
where rowid in (
|
||||
select rowid from twts
|
||||
where ulid in (select ulid from twts, json_each(mentions) where value = ?)
|
||||
where ulid in (select ulid from twt_mentions where feed_id = ?)
|
||||
order by ulid asc
|
||||
limit ?
|
||||
offset ?
|
||||
@ -231,6 +231,11 @@ func httpServer(ctx context.Context, app *appState) error {
|
||||
twt, _ := lextwt.ParseLine(o.Text, &twter)
|
||||
twts = append(twts, twt)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
fmt.Println(err)
|
||||
return
|
||||
}
|
||||
|
||||
preamble := add(PREAMBLE_DOCS, "twt range = 1 %d", end)
|
||||
preamble = add(preamble, "self = %s/mentions%s", hostname, mkqry(uri, limit, offset))
|
||||
if next := offset + int64(len(twts)); next < end {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user