chore: changes for otel and fixes to loop

cmd/load/main.go

@@ -0,0 +1,92 @@
+package main
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"hash/fnv"
+	"iter"
+	"os"
+
+	"github.com/oklog/ulid/v2"
+	"github.com/uptrace/opentelemetry-go-extra/otelsql"
+	"go.yarn.social/lextwt"
+	"go.yarn.social/types"
+)
+
+func main() {
+	in := os.Stdin
+	if len(os.Args) != 2 {
+		fmt.Fprint(os.Stderr, "usage: ", os.Args[0], "[db file]")
+	}
+
+	db, err := DB(context.Background(), os.Args[1])
+	if err != nil {
+		panic(err)
+	}
+	_ = db
+
+	for line := range lextwt.IterRegistry(in) {
+		_ = line
+	}
+}
+
+const MaxVariableNumber = 32766
+
+func DB(ctx context.Context, cnx string) (*sql.DB, error) {
+	// return sql.Open(app.args.dbtype, app.args.dbfile)
+
+	db, err := otelsql.Open("sqlite", cnx)
+	if err != nil {
+		return db, err
+	}
+
+	return db, err
+}
+
+func makeULID(twt types.Twt) ulid.ULID {
+	h64 := fnv.New64a()
+	h16 := fnv.New32a()
+	text := []byte(fmt.Sprintf("%+l", twt))
+	b := make([]byte, 10)
+	copy(b, h16.Sum(text)[:2])
+	copy(b[2:], h64.Sum(text))
+	u := ulid.ULID{}
+	u.SetTime(ulid.Timestamp(twt.Created()))
+	u.SetEntropy(b)
+
+	return u
+}
+
+func chunk(args []any, qry func(int) (string, int), maxArgs int) iter.Seq2[string, []any] {
+	_, size := qry(1)
+	itemsPerIter := maxArgs / size
+
+	if len(args) < size {
+		return func(yield func(string, []any) bool) {}
+	}
+
+	if len(args) < maxArgs {
+		return func(yield func(string, []any) bool) {
+			query, _ := qry(len(args) / size)
+			yield(query, args)
+		}
+	}
+
+	return func(yield func(string, []any) bool) {
+		for len(args) > 0 {
+			if len(args) > maxArgs {
+				query, size := qry(itemsPerIter)
+				if !yield(query, args[:size]) {
+					return
+				}
+				args = args[size:]
+				continue
+			}
+
+			query, _ := qry(len(args) / size)
+			yield(query, args)
+			return
+		}
+	}
+}