sour-is/xt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: many fixes to code

Browse Source
This commit is contained in:
xuu 2025-03-13 22:36:24 -06:00
parent ed5b43300b
commit 42fe9176b7
Signed by: xuu
GPG Key ID: 8B3B0604F164E04F
15 changed files with 794 additions and 285 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
about.me
View File

@ -1,84 +0,0 @@
.ce
Preamble
.sp
We, the people of the United States, in order
to form a more perfect Union, establish justice, insure
domestic tranquility, provide for the common defense, promote
the general welfare,
and secure the blessing of liberty to ourselves and our
posterity do ordain and establish this Constitution for the
United States of America.
.sp
.nr aR 0 1
.af aR I
.de AR
.ce
.ps 16
.ft B
Article \\n+(aR
.nr sE 0 1
.af sE 1
.ps 12
.ft P
..
.de SE
.sp
.ft B
\\s-2SECTION \\n+(sE:\\s+2
.ft P
.nr pP 0 1
.af pP 1
..
.de PP
.sp
.ft I
\\s-3Paragraph \\n+(pP:\\s+3
.ft P
..
.AR
.SE
Legislative powers; in whom vested:
.PP
All legislative powers herein granted shall be vested in a
Congress of the United States, which shall consist of a Senate
and a House of Representatives.
.SE
House of Representatives, how and by whom chosen, Qualifications
of a Representative. Representatives and direct taxes, how
apportioned. Enumeration. Vacancies to be filled. Power of
choosing officers and of impeachment.
.PP
The House of Representatives shall be composed of members chosen
every second year by the people of the several states, and the
electors in each State shall have the qualifications requisite
for electors of the most numerous branch of the State Legislature.
.PP
No person shall be a Representative who shall not have attained
to the age of twenty-five years, and been seven years a citizen
of the United States, and who shall not, when elected, be an
inhabitant of that State in which he shall be chosen.
.PP
Representatives and direct taxes shall be apportioned among the
several States which maybe included within this Union, according
to their respective numbers, which shall be determined by adding
to the whole number of free persons, including those bound for
service for a term of years, and excluding Indians not taxed,
three-fifths of all other persons. The actual enumeration shall
be made within three years after the first meeting of the
Congress of the United States, and within every subsequent term
of ten years, in such manner as they shall by law direct. The
number of Representatives shall not exceed one for every thirty
thousand, but each State shall have at least one Representative;
and until such enumeration shall be made, the State of New
Hampshire shall be entitled to choose three, Massachusetts eight,
Rhode Island and Providence Plantations one, Connecticut
five, New York six, New Jersey four, Pennsylvania eight,
Delaware one, Maryland six, Virginia ten, North Carolina five,
South Carolina five, and Georgia three.
.PP
When vacancies happen in the representation from any State, the
Executive Authority thereof shall issue writs of election to fill
such vacancies.
.PP
The House of Representatives shall choose their Speaker and other
officers; and shall have the sole power of impeachment.

76
service.go → app.go
View File

@ -6,6 +6,8 @@ import (
"fmt" "fmt"
"iter" "iter"
"os" "os"
"runtime/debug"
"strconv"
"strings" "strings"
"sync" "sync"
@ -13,7 +15,9 @@ import (
_ "github.com/mattn/go-sqlite3" _ "github.com/mattn/go-sqlite3"
"github.com/uptrace/opentelemetry-go-extra/otelsql" "github.com/uptrace/opentelemetry-go-extra/otelsql"
"go.opentelemetry.io/otel/attribute"
semconv "go.opentelemetry.io/otel/semconv/v1.4.0" semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
"go.opentelemetry.io/otel/trace"
"go.sour.is/xt/internal/otel" "go.sour.is/xt/internal/otel"
"go.yarn.social/lextwt" "go.yarn.social/lextwt"
"go.yarn.social/types" "go.yarn.social/types"
@ -24,12 +28,15 @@ func run(c *console) error {
ctx, span := otel.Span(c.Context) ctx, span := otel.Span(c.Context)
defer span.End() defer span.End()
bi, _ := debug.ReadBuildInfo()
span.AddEvent(name, trace.WithAttributes(attribute.String("version", bi.Main.Version)))
a := c.Args() a := c.Args()
app := &appState{ app := &appState{
args: a, args: a,
feeds: sync.Map{}, feeds: sync.Map{},
queue: FibHeap(func(a, b *Feed) bool { queue: FibHeap(func(a, b *Feed) bool {
return a.LastScanOn.Time.Before(b.LastScanOn.Time) return a.NextScanOn.Time.Before(b.NextScanOn.Time)
}), }),
} }
@ -38,7 +45,7 @@ func run(c *console) error {
ctx, span := otel.Span(ctx) ctx, span := otel.Span(ctx)
defer span.End() defer span.End()
db, err := app.DB() db, err := app.DB(ctx)
if err != nil { if err != nil {
return err return err
} }
@ -76,7 +83,7 @@ func run(c *console) error {
return fmt.Errorf("%w: %w", ErrParseFailed, err) return fmt.Errorf("%w: %w", ErrParseFailed, err)
} }
db, err := app.DB() db, err := app.DB(ctx)
if err != nil { if err != nil {
return err return err
} }
@ -91,10 +98,16 @@ func run(c *console) error {
wg, ctx := errgroup.WithContext(ctx) wg, ctx := errgroup.WithContext(ctx)
c.Context = ctx c.Context = ctx
wg.Go(func() error { return refreshLoop(c, app) }) wg.Go(func() error {
return refreshLoop(c, app)
})
go httpServer(c, app) go httpServer(c, app)
wg.Wait() err = wg.Wait()
if err != nil {
return err
}
return c.Context.Err() return c.Context.Err()
} }
@ -104,12 +117,61 @@ type appState struct {
queue *fibHeap[Feed] queue *fibHeap[Feed]
} }
func (app *appState) DB() (*sql.DB, error) { type db struct {
*sql.DB
Version string
Params map[string]string
MaxLength int
MaxVariableNumber int
}
func (app *appState) DB(ctx context.Context) (db, error) {
// return sql.Open(app.args.dbtype, app.args.dbfile) // return sql.Open(app.args.dbtype, app.args.dbfile)
return otelsql.Open(app.args.dbtype, app.args.dbfile, var err error
db := db{Params: make(map[string]string)}
db.DB, err = otelsql.Open(app.args.dbtype, app.args.dbfile,
otelsql.WithAttributes(semconv.DBSystemSqlite), otelsql.WithAttributes(semconv.DBSystemSqlite),
otelsql.WithDBName("mydb")) otelsql.WithDBName("mydb"))
if err != nil {
return db, err
}
rows, err := db.DB.QueryContext(ctx, `select sqlite_version()`)
if err != nil {
return db, err
}
if rows.Next() {
rows.Scan(&db.Version)
}
if err = rows.Err(); err != nil {
return db, err
}
rows, err = db.DB.QueryContext(ctx, `pragma compile_options`)
if err != nil {
return db, err
}
for rows.Next() {
var key, value string
rows.Scan(&key)
key, value, _ = strings.Cut(key, "=")
db.Params[key] = value
}
if err = rows.Err(); err != nil {
return db, err
}
if m, ok := db.Params["MAX_VARIABLE_NUMBER"]; ok {
db.MaxVariableNumber, _ = strconv.Atoi(m)
}
if m, ok := db.Params["MAX_LENGTH"]; ok {
db.MaxLength, _ = strconv.Atoi(m)
}
return db, err
} }
func (app *appState) Feed(feedID string) *Feed { func (app *appState) Feed(feedID string) *Feed {

0
service_test.go → app_test.go
View File

333
feed.go
View File

@ -4,7 +4,9 @@ import (
"cmp" "cmp"
"context" "context"
"database/sql" "database/sql"
"database/sql/driver"
"fmt" "fmt"
"hash/fnv"
"iter" "iter"
"net/http" "net/http"
"net/url" "net/url"
@ -14,6 +16,7 @@ import (
_ "embed" _ "embed"
"github.com/oklog/ulid/v2"
"go.sour.is/xt/internal/otel" "go.sour.is/xt/internal/otel"
"go.yarn.social/lextwt" "go.yarn.social/lextwt"
"go.yarn.social/types" "go.yarn.social/types"
@ -21,22 +24,22 @@ import (
type Feed struct { type Feed struct {
FeedID uuid FeedID uuid
FetchURI string ParentID uuid
HashURI string
URI string URI string
Nick string Nick string
LastScanOn sql.NullTime State State
LastScanOn TwtTime
RefreshRate int RefreshRate int
NextScanOn TwtTime
LastModified sql.NullTime LastModified TwtTime
LastError sql.NullString LastError sql.NullString
ETag sql.NullString ETag sql.NullString
Version string Version string
DiscloseFeedURL string DiscloseFeedURL string
DiscloseNick string DiscloseNick string
FirstFetch bool
State State
} }
type State string type State string
@ -47,43 +50,34 @@ const (
Cold State = "cold" Cold State = "cold"
Warm State = "warm" Warm State = "warm"
Hot State = "hot" Hot State = "hot"
Once State = "once"
) )
var ( var (
//go:embed init.sql //go:embed init.sql
initSQL string initSQL string
insertFeed = ` insertFeed = func(r int) (string, int) {
insert into feeds repeat := ""
(feed_id, uri, nick, last_scan_on, refresh_rate) if r > 1 {
values (?, ?, ?, ?, ?) repeat = strings.Repeat(", (?, ?, ?, ?, ?, ?, ?)", r-1)
ON CONFLICT (feed_id) DO NOTHING }
` return `
insert into feeds (
insertTwt = ` feed_id,
insert into twts parent_id,
(feed_id, hash, conv, dt, text, mentions, tags) nick,
values (?, ?, ?, ?, ?, ?, ?) uri,
ON CONFLICT (feed_id, hash) DO NOTHING state,
` last_scan_on,
refresh_rate
fetchFeeds = ` )
select values (?, ?, ?, ?, ?, ?, ?)` + repeat + `
feed_id, ON CONFLICT (feed_id) DO NOTHING`, r * 7
uri, }
nick,
last_scan_on,
refresh_rate,
last_modified_on,
last_etag
from feeds
where datetime(
coalesce(last_scan_on, '1901-01-01'),
'+'||refresh_rate||' seconds'
) < datetime(current_timestamp, '+10 minutes')
`
updateFeed = ` updateFeed = `
update feeds set update feeds set
state = ?,
last_scan_on = ?, last_scan_on = ?,
refresh_rate = ?, refresh_rate = ?,
last_modified_on = ?, last_modified_on = ?,
@ -91,21 +85,83 @@ var (
last_error = ? last_error = ?
where feed_id = ? where feed_id = ?
` `
insertTwt = func(r int) (string, int) {
repeat := ""
if r > 1 {
repeat = strings.Repeat(", (?, ?, ?, ?, ?, ?, ?)", r-1)
}
return `
insert into twts
(feed_id, ulid, text, hash, conv, mentions, tags)
values (?, ?, ?, ?, ?, ?, ?)` + repeat + `
ON CONFLICT (feed_id, ulid) DO NOTHING`, r * 7
}
fetchFeeds = `
select
feed_id,
parent_id,
coalesce(hashing_uri, uri) hash_uri,
uri,
nick,
state,
last_scan_on,
strftime(
'%Y-%m-%dT%H:%M:%fZ',
coalesce(last_scan_on, '1901-01-01'),
'+'||refresh_rate||' seconds'
) next_scan_on,
refresh_rate,
last_modified_on,
last_etag
from feeds
left join (
select
feed_id parent_id,
uri hashing_uri
from feeds
where parent_id is null
) using (parent_id)
where datetime(
coalesce(last_scan_on, '1901-01-01'),
'+'||refresh_rate||' seconds'
) < datetime(current_timestamp, '+10 minutes')
`
) )
func (f *Feed) Save(ctx context.Context, db *sql.DB) error { func (f *Feed) Create(ctx context.Context, db db) error {
ctx, span := otel.Span(ctx)
defer span.End()
query, _ := insertFeed(1)
_, err := db.ExecContext(
ctx,
query,
f.FeedID, // feed_id
f.ParentID, // parent_id
f.Nick, // nick
f.URI, // uri
f.State, // state
f.LastScanOn, // last_scan_on
f.RefreshRate, // refresh_rate
)
return err
}
func (f *Feed) Save(ctx context.Context, db db) error {
ctx, span := otel.Span(ctx) ctx, span := otel.Span(ctx)
defer span.End() defer span.End()
_, err := db.ExecContext( _, err := db.ExecContext(
ctx, ctx,
updateFeed, updateFeed,
f.LastScanOn, f.State, // state
f.RefreshRate, f.LastScanOn, // last_scan_on
f.LastModified, f.RefreshRate, // refresh_rate
f.ETag, f.LastModified, // last_modified_on
f.LastError, f.ETag, // last_etag
f.FeedID, f.LastError, // last_error
f.FeedID, // feed_id
) )
return err return err
} }
@ -117,9 +173,13 @@ func (f *Feed) Scan(res interface{ Scan(...any) error }) error {
f.Version = "0.0.1" f.Version = "0.0.1"
err = res.Scan( err = res.Scan(
&f.FeedID, &f.FeedID,
&f.ParentID,
&f.HashURI,
&f.URI, &f.URI,
&f.Nick, &f.Nick,
&f.State,
&f.LastScanOn, &f.LastScanOn,
&f.NextScanOn,
&f.RefreshRate, &f.RefreshRate,
&f.LastModified, &f.LastModified,
&f.ETag, &f.ETag,
@ -128,19 +188,10 @@ func (f *Feed) Scan(res interface{ Scan(...any) error }) error {
return err return err
} }
if !f.LastScanOn.Valid {
f.FirstFetch = true
f.LastScanOn.Time = time.Now()
f.LastScanOn.Valid = true
} else {
f.LastScanOn.Time = f.LastScanOn.Time.Add(time.Duration(f.RefreshRate) * time.Second)
}
f.FetchURI = f.URI
return err return err
} }
func loadFeeds(ctx context.Context, db *sql.DB) (iter.Seq[Feed], error) { func loadFeeds(ctx context.Context, db db) (iter.Seq[Feed], error) {
ctx, span := otel.Span(ctx) ctx, span := otel.Span(ctx)
var err error var err error
@ -159,6 +210,7 @@ func loadFeeds(ctx context.Context, db *sql.DB) (iter.Seq[Feed], error) {
var f Feed var f Feed
err = f.Scan(res) err = f.Scan(res)
if err != nil { if err != nil {
span.RecordError(err)
return return
} }
if !yield(f) { if !yield(f) {
@ -168,7 +220,7 @@ func loadFeeds(ctx context.Context, db *sql.DB) (iter.Seq[Feed], error) {
}, err }, err
} }
func storeFeed(ctx context.Context, db *sql.DB, f types.TwtFile) error { func storeFeed(ctx context.Context, db db, f types.TwtFile) error {
ctx, span := otel.Span(ctx) ctx, span := otel.Span(ctx)
defer span.End() defer span.End()
@ -201,20 +253,11 @@ func storeFeed(ctx context.Context, db *sql.DB, f types.TwtFile) error {
defer tx.Rollback() defer tx.Rollback()
_, err = tx.ExecContext( twts := f.Twts()
ctx, _, size := insertTwt(len(twts))
insertFeed, args := make([]any, 0, size)
feedID,
f.Twter().HashingURI,
f.Twter().DomainNick(),
loadTS,
refreshRate,
)
if err != nil {
return err
}
for _, twt := range f.Twts() { for _, twt := range twts {
mentions := make(uuids, 0, len(twt.Mentions())) mentions := make(uuids, 0, len(twt.Mentions()))
for _, mention := range twt.Mentions() { for _, mention := range twt.Mentions() {
followMap[mention.Twter().Nick] = mention.Twter().URI followMap[mention.Twter().Nick] = mention.Twter().URI
@ -233,32 +276,76 @@ func storeFeed(ctx context.Context, db *sql.DB, f types.TwtFile) error {
subjectTag = tag.Text() subjectTag = tag.Text()
} }
} }
args = append(
args,
feedID, // feed_id
makeULID(twt), // ulid
fmt.Sprintf("%+l", twt), // text
subjectTag, // conv
twt.Hash(), // hash
mentions.ToStrList(), // mentions
tags, // tags
)
}
for query, args := range chunk(args, insertTwt, db.MaxVariableNumber) {
fmt.Println("store", f.Twter().URI, len(args))
_, err = tx.ExecContext( _, err = tx.ExecContext(
ctx, ctx,
insertTwt, query,
feedID, args...,
twt.Hash(),
subjectTag,
twt.Created(),
fmt.Sprint(twt),
mentions.ToStrList(),
tags,
) )
if err != nil { if err != nil {
return err return err
} }
} }
args = args[:0]
args = append(args,
feedID, // feed_id
nil, // parent_id
f.Twter().DomainNick(), // nick
f.Twter().URI, // uri
"warm", // state
TwtTime{Time: loadTS, Valid: true}, // last_scan_on
refreshRate, // refresh_rate
)
if prev, ok := f.Info().GetN("prev", 0); ok {
_, part, ok := strings.Cut(prev.Value(), " ")
if ok {
uri:= f.Twter().URI
part = uri[:strings.LastIndex(uri, "/")+1] + part
childID := urlNS.UUID5(part)
args = append(args,
childID, // feed_id
feedID, // parent_id
f.Twter().DomainNick(), // nick
part, // uri
"once", // state
nil, // last_scan_on
refreshRate, // refresh_rate
)
}
}
for nick, uri := range followMap { for nick, uri := range followMap {
args = append(args,
urlNS.UUID5(uri), // feed_id
nil, // parent_id
nick, // nick
uri, // uri
"warm", // state
nil, // last_scan_on
refreshRate, // refresh_rate
)
}
for query, args := range chunk(args, insertFeed, db.MaxVariableNumber) {
_, err = tx.ExecContext( _, err = tx.ExecContext(
ctx, ctx,
insertFeed, query,
urlNS.UUID5(uri), args...,
uri,
nick,
nil,
refreshRate,
) )
if err != nil { if err != nil {
return err return err
@ -270,11 +357,11 @@ func storeFeed(ctx context.Context, db *sql.DB, f types.TwtFile) error {
func (feed *Feed) MakeHTTPRequest(ctx context.Context) (*http.Request, error) { func (feed *Feed) MakeHTTPRequest(ctx context.Context) (*http.Request, error) {
feed.State = "fetch" feed.State = "fetch"
if strings.Contains(feed.FetchURI, "lublin.se") { if strings.Contains(feed.URI, "lublin.se") {
return nil, fmt.Errorf("%w: permaban: %s", ErrPermanentlyDead, feed.URI) return nil, fmt.Errorf("%w: permaban: %s", ErrPermanentlyDead, feed.URI)
} }
req, err := http.NewRequestWithContext(ctx, "GET", feed.FetchURI, nil) req, err := http.NewRequestWithContext(ctx, "GET", feed.URI, nil)
if err != nil { if err != nil {
return nil, fmt.Errorf("creating HTTP request failed: %w", err) return nil, fmt.Errorf("creating HTTP request failed: %w", err)
} }
@ -298,3 +385,83 @@ func (feed *Feed) MakeHTTPRequest(ctx context.Context) (*http.Request, error) {
return req, nil return req, nil
} }
type TwtTime struct {
Time time.Time
Valid bool // Valid is true if Time is not NULL
}
// Scan implements the [Scanner] interface.
func (n *TwtTime) Scan(value any) error {
var err error
switch value := value.(type) {
case nil:
n.Time, n.Valid = time.Time{}, false
return nil
case string:
n.Valid = true
n.Time, err = time.Parse(time.RFC3339, value)
case time.Time:
n.Valid = true
n.Time = value
}
return err
}
// Value implements the [driver.Valuer] interface.
func (n TwtTime) Value() (driver.Value, error) {
if !n.Valid {
return nil, nil
}
return n.Time.Format(time.RFC3339), nil
}
func makeULID(twt types.Twt) ulid.ULID {
h64 := fnv.New64a()
h16 := fnv.New32a()
text := []byte(fmt.Sprintf("%+l", twt))
b := make([]byte, 10)
copy(b, h16.Sum(text)[:2])
copy(b[2:], h64.Sum(text))
u := ulid.ULID{}
u.SetTime(ulid.Timestamp(twt.Created()))
u.SetEntropy(b)
return u
}
func chunk(args []any, qry func(int) (string, int), maxArgs int) iter.Seq2[string, []any] {
_, size := qry(1)
itemsPerIter := maxArgs / size
if len(args) < size {
return func(yield func(string, []any) bool) {}
}
if len(args) < maxArgs {
return func(yield func(string, []any) bool) {
query, _ := qry(len(args) / size)
yield(query, args)
}
}
return func(yield func(string, []any) bool) {
for len(args) > 0 {
if len(args) > maxArgs {
query, size := qry(itemsPerIter)
if !yield(query, args[:size]) {
return
}
args = args[size:]
continue
}
query, _ := qry(len(args) / size)
yield(query, args)
return
}
}
}

48
fetcher.go
View File

@ -8,6 +8,8 @@ import (
"net/http" "net/http"
"sync" "sync"
"time" "time"
"go.sour.is/xt/internal/otel"
) )
var ( var (
@ -62,7 +64,7 @@ func NewHTTPFetcher() *httpFetcher {
ForceAttemptHTTP2: false, ForceAttemptHTTP2: false,
MaxIdleConns: 100, MaxIdleConns: 100,
IdleConnTimeout: 10 * time.Second, IdleConnTimeout: 10 * time.Second,
TLSHandshakeTimeout: 10 * time.Second, TLSHandshakeTimeout: 5 * time.Second,
ExpectContinueTimeout: 1 * time.Second, ExpectContinueTimeout: 1 * time.Second,
}, },
}, },
@ -70,6 +72,10 @@ func NewHTTPFetcher() *httpFetcher {
} }
func (f *httpFetcher) Fetch(ctx context.Context, request *Feed) *Response { func (f *httpFetcher) Fetch(ctx context.Context, request *Feed) *Response {
ctx, span := otel.Span(ctx)
defer span.End()
defer fmt.Println("fetch done", request.URI)
response := &Response{ response := &Response{
Request: request, Request: request,
} }
@ -79,8 +85,9 @@ func (f *httpFetcher) Fetch(ctx context.Context, request *Feed) *Response {
response.err = err response.err = err
return response return response
} }
span.AddEvent("start request")
res, err := f.client.Do(req) res, err := f.client.Do(req)
span.AddEvent("got response")
if err != nil { if err != nil {
if errors.Is(err, &net.DNSError{}) { if errors.Is(err, &net.DNSError{}) {
response.err = fmt.Errorf("%w: %s", ErrTemporarilyDead, err) response.err = fmt.Errorf("%w: %s", ErrTemporarilyDead, err)
@ -97,7 +104,7 @@ func (f *httpFetcher) Fetch(ctx context.Context, request *Feed) *Response {
case 304: case 304:
response.err = fmt.Errorf("%w: %s", ErrUnmodified, res.Status) response.err = fmt.Errorf("%w: %s", ErrUnmodified, res.Status)
case 400, 406, 502, 503: case 400, 406, 429, 502, 503:
response.err = fmt.Errorf("%w: %s", ErrTemporarilyDead, res.Status) response.err = fmt.Errorf("%w: %s", ErrTemporarilyDead, res.Status)
case 403, 404, 410: case 403, 404, 410:
@ -121,29 +128,54 @@ func NewFuncPool[IN, OUT any](
size int, size int,
fetch func(ctx context.Context, request IN) OUT, fetch func(ctx context.Context, request IN) OUT,
) (*pool[IN, OUT], func()) { ) (*pool[IN, OUT], func()) {
ctx, span := otel.Span(ctx)
defer span.End()
var wg sync.WaitGroup var wg sync.WaitGroup
in := make(chan IN, size) in := make(chan IN, size)
out := make(chan OUT, size) out := make(chan OUT)
wg.Add(size) wg.Add(size)
for range size { for range size {
go func() { go func() {
ctx, span := otel.Span(ctx)
defer span.End()
defer wg.Done() defer wg.Done()
for request := range in { for request := range in {
ctx, cancel := context.WithTimeoutCause(ctx, 15*time.Second, fmt.Errorf("GOT STUCK"))
defer cancel()
ctx, span := otel.Span(ctx)
defer span.End()
span.AddEvent("start fetch")
r := fetch(ctx, request)
span.AddEvent("got fetch")
select { select {
case <-ctx.Done(): case <-ctx.Done():
return return
case out <- fetch(ctx, request): case out <- r:
span.AddEvent("sent queue")
case <-time.After(20 * time.Second):
fmt.Println("GOT STUCK", request)
span.AddEvent("GOT STUCK")
cancel()
} }
} }
}() }()
} }
return &pool[IN, OUT]{ return &pool[IN, OUT]{
in: in, in: in,
out: out, out: out,
}, func() { close(in); wg.Wait(); close(out) } }, func() {
close(in)
wg.Wait()
close(out)
}
} }
func (f *pool[IN, OUT]) Fetch(request IN) { func (f *pool[IN, OUT]) Fetch(request IN) {

3
go.mod
View File

@ -41,6 +41,7 @@ require (
github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/matryer/is v1.4.1 github.com/matryer/is v1.4.1
github.com/oklog/ulid/v2 v2.1.0
github.com/prometheus/client_golang v1.20.5 github.com/prometheus/client_golang v1.20.5
github.com/sirupsen/logrus v1.9.3 // indirect github.com/sirupsen/logrus v1.9.3 // indirect
github.com/uptrace/opentelemetry-go-extra/otelsql v0.3.2 github.com/uptrace/opentelemetry-go-extra/otelsql v0.3.2
@ -53,7 +54,7 @@ require (
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.34.0 go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.34.0
go.opentelemetry.io/otel/sdk v1.34.0 go.opentelemetry.io/otel/sdk v1.34.0
go.opentelemetry.io/otel/sdk/log v0.10.0 go.opentelemetry.io/otel/sdk/log v0.10.0
go.opentelemetry.io/otel/sdk/metric v1.34.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.34.0
go.yarn.social/types v0.0.0-20250108134258-ed75fa653ede go.yarn.social/types v0.0.0-20250108134258-ed75fa653ede
golang.org/x/crypto v0.33.0 // indirect golang.org/x/crypto v0.33.0 // indirect
golang.org/x/sync v0.11.0 golang.org/x/sync v0.11.0

5
go.sum
View File

@ -35,6 +35,11 @@ github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBW
github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/oklog/ulid/v2 v2.1.0 h1:+9lhoxAP56we25tyYETBBY1YLA2SaoLvUFgrP2miPJU=
github.com/oklog/ulid/v2 v2.1.0/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ=
github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y=

6
go.work Normal file
View File

@ -0,0 +1,6 @@
go 1.24.0
use (
.
../go-lextwt
)

191
go.work.sum Normal file
View File

@ -0,0 +1,191 @@
cel.dev/expr v0.19.0 h1:lXuo+nDhpyJSpWxpPVi5cPUwzKb+dsdOiw6IreM5yt0=
cel.dev/expr v0.19.0/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw=
cloud.google.com/go v0.26.0 h1:e0WKqKTd5BnrG8aKH3J3h+QvEIQtSUcf2n5UZ5ZgLtQ=
cloud.google.com/go/compute/metadata v0.5.2 h1:UxK4uu/Tn+I3p2dYWTfiX4wva7aYlKixAHn3fyqngqo=
cloud.google.com/go/compute/metadata v0.5.2/go.mod h1:C66sj2AluDcIqakBq/M8lw8/ybHgOZqin2obFxa/E5k=
dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s=
dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0 h1:3c8yed4lgqTt+oTQ+JNMDo+F4xprBf+O/il4ZC0nRLw=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0/go.mod h1:obipzmGjfSjam60XLwGfqUkJsfiheAl+TUjG+4yzyPM=
github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ=
github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE=
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78=
github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0=
github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0=
github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
github.com/alecthomas/kingpin/v2 v2.4.0 h1:f48lwail6p8zpO1bC4TxtqACaGqHYA22qkHjHpqDjYY=
github.com/alecthomas/kingpin/v2 v2.4.0/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE=
github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc=
github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc=
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4=
github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWpi6yML8=
github.com/aymanbagabas/go-udiff v0.2.0/go.mod h1:RE4Ex0qsGkTAJoQdQQCA0uG+nAzJO/pI/QwceO5fgrA=
github.com/badgerodon/ioutil v0.0.0-20150716134133-06e58e34b867 h1:nsDNoesoGwPzPkcrR1w1uzPUtiqwCXoNnkWC7nUuRHI=
github.com/badgerodon/ioutil v0.0.0-20150716134133-06e58e34b867/go.mod h1:Ctq1YQi0dOq7QgBLZZ7p1Fr3IbAAqL/yMqDIHoe9WtE=
github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g=
github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw=
github.com/charmbracelet/harmonica v0.2.0 h1:8NxJWRWg/bzKqqEaaeFNipOu77YR5t8aSwG4pgaUBiQ=
github.com/charmbracelet/harmonica v0.2.0/go.mod h1:KSri/1RMQOZLbw7AHqgcBycp8pgJnQMYYT8QZRqZ1Ao=
github.com/charmbracelet/x/exp/golden v0.0.0-20250213125511-a0c32e22e4fc h1:Tp8vprGbBhTAeyCNgrWPYIPyVEo9qmFRAcRhSdeind4=
github.com/charmbracelet/x/exp/golden v0.0.0-20250213125511-a0c32e22e4fc/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U=
github.com/charmbracelet/x/exp/teatest v0.0.0-20250213125511-a0c32e22e4fc h1:p4x5lOqNZEELobbdrm00kW5xrQwXZtIrUW8yTPK16aU=
github.com/charmbracelet/x/exp/teatest v0.0.0-20250213125511-a0c32e22e4fc/go.mod h1:ag+SpTUkiN/UuUGYPX3Ci4fR1oF3XX97PpGhiXK7i6U=
github.com/cilium/ebpf v0.11.0 h1:V8gS/bTCCjX9uUnkUFUpPsksM8n1lXBAvHcpiFk1X2Y=
github.com/cilium/ebpf v0.11.0/go.mod h1:WE7CZAnqOL2RouJ4f1uyNhqr2P4CCvXFIqdRDUgWsVs=
github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI=
github.com/cloudflare/circl v1.5.0 h1:hxIWksrX6XN5a1L2TI/h53AGPhNHoUBo+TD1ms9+pys=
github.com/cloudflare/circl v1.5.0/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f h1:WBZRG4aNOuI15bLRrCgN8fCq8E5Xuty6jGbmSNEvSsU=
github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 h1:QVw89YDxXxEe+l8gU8ETbOasdwEV+avkR75ZzsVV9WI=
github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8=
github.com/cosiner/argv v0.1.0 h1:BVDiEL32lwHukgJKP87btEPenzrrHUjajs/8yzaqcXg=
github.com/cosiner/argv v0.1.0/go.mod h1:EusR6TucWKX+zFgtdUsKT2Cvg45K5rtpCcWz4hK06d8=
github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/creack/pty v1.1.20 h1:VIPb/a2s17qNeQgDnkfZC35RScx+blkKF8GV68n80J4=
github.com/creack/pty v1.1.20/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
github.com/cyphar/filepath-securejoin v0.3.4 h1:VBWugsJh2ZxJmLFSM06/0qzQyiQX2Qs0ViKrUAcqdZ8=
github.com/cyphar/filepath-securejoin v0.3.4/go.mod h1:8s/MCNJREmFK0H02MF6Ihv1nakJe4L/w3WZLHNkvlYM=
github.com/derekparker/trie v0.0.0-20230829180723-39f4de51ef7d h1:hUWoLdw5kvo2xCsqlsIBMvWUc1QCSsCYD2J2+Fg6YoU=
github.com/derekparker/trie v0.0.0-20230829180723-39f4de51ef7d/go.mod h1:C7Es+DLenIpPc9J6IYw4jrK0h7S9bKj4DNl8+KxGEXU=
github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
github.com/envoyproxy/go-control-plane v0.13.1 h1:vPfJZCkob6yTMEgS+0TwfTUfbHjfy/6vOJ8hUWX/uXE=
github.com/envoyproxy/go-control-plane v0.13.1/go.mod h1:X45hY0mufo6Fd0KW3rqsGvQMw58jvjymeCzBU3mWyHw=
github.com/envoyproxy/protoc-gen-validate v1.1.0 h1:tntQDh69XqOCOZsDz0lVJQez/2L6Uu2PdjCQwWCJ3bM=
github.com/envoyproxy/protoc-gen-validate v1.1.0/go.mod h1:sXRDRVmzEbkM7CVcM06s9shE/m23dg3wzjl0UWqJ2q4=
github.com/felixge/fgprof v0.9.5 h1:8+vR6yu2vvSKn08urWyEuxx75NWPEvybbkBirEpsbVY=
github.com/felixge/fgprof v0.9.5/go.mod h1:yKl+ERSa++RYOs32d8K6WEXCB4uXdLls4ZaZPpayhMM=
github.com/go-delve/liner v1.2.3-0.20231231155935-4726ab1d7f62 h1:IGtvsNyIuRjl04XAOFGACozgUD7A82UffYxZt4DWbvA=
github.com/go-delve/liner v1.2.3-0.20231231155935-4726ab1d7f62/go.mod h1:biJCRbqp51wS+I92HMqn5H8/A0PAhxn2vyOT+JqhiGI=
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic=
github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU=
github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow=
github.com/go-git/go-git/v5 v5.12.0 h1:7Md+ndsjrzZxbddRDZjF14qK+NN56sy6wkqaVrjZtys=
github.com/go-git/go-git/v5 v5.12.0/go.mod h1:FTM9VKtnI2m65hNI/TenDDDnUf2Q9FHnXYjuz9i5OEY=
github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU=
github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA=
github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
github.com/golang/glog v1.2.3 h1:oDTdz9f5VGVVNGu/Q7UXKWYsD0873HXLHdJUNBsSEKM=
github.com/golang/glog v1.2.3/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1 h1:G5FRp8JnTd7RQH5kemVNlMeyXQAztQ3mOWV95KxsXH8=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/google/go-dap v0.12.0 h1:rVcjv3SyMIrpaOoTAdFDyHs99CwVOItIJGKLQFQhNeM=
github.com/google/go-dap v0.12.0/go.mod h1:tNjCASCm5cqePi/RVXXWEVqtnNLV1KTWtYOqu6rZNzc=
github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 h1:y3N7Bm7Y9/CtpiVkw/ZWj6lSlDF3F74SfKwfTCer72Q=
github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c=
github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU=
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4=
github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/pkg/profile v1.7.0 h1:hnbDkaNWPCLMO9wGLdBFTIZvzDrDfBM2072E1S9gJkA=
github.com/pkg/profile v1.7.0/go.mod h1:8Uer0jas47ZQMJ7VD+OHknK4YDY07LPUC6dEvqDjvNo=
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo=
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
github.com/rogpeppe/fastuuid v1.2.0 h1:Ppwyp6VYCF1nvBTXL3trRso7mXMlRrw9ooo375wvi2s=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sahilm/fuzzy v0.1.1 h1:ceu5RHF8DGgoi+/dR5PsECjCDH1BE3Fnmpo7aVXOdRA=
github.com/sahilm/fuzzy v0.1.1/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8ZK9Y=
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8=
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
github.com/skeema/knownhosts v1.3.0 h1:AM+y0rI04VksttfwjkSTNQorvGqmwATnvnAHpSgc0LY=
github.com/skeema/knownhosts v1.3.0/go.mod h1:sPINvnADmT/qYH1kfv+ePMmOBTH6Tbl7b5LvTDjFK7M=
github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc=
github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU=
github.com/yuin/goldmark v1.4.13 h1:fVcFKWvrslecOb/tg+Cc05dkeYx540o0FuFt3nUVDoE=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.opentelemetry.io/contrib/detectors/gcp v1.32.0 h1:P78qWqkLSShicHmAzfECaTgvslqHxblNE9j62Ws1NK8=
go.opentelemetry.io/contrib/detectors/gcp v1.32.0/go.mod h1:TVqo0Sda4Cv8gCIixd7LuLwW4EylumVWfhjZJjDD4DU=
go.starlark.net v0.0.0-20231101134539-556fd59b42f6 h1:+eC0F/k4aBLC4szgOcjd7bDTEnpxADJyWJE0yowgM3E=
go.starlark.net v0.0.0-20231101134539-556fd59b42f6/go.mod h1:LcLNIzVOMp4oV+uusnpk+VU+SzXaJakUuBjoCSWH5dM=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 h1:XQyxROzUlZH+WIQwySDgnISgOivlhjIEwaQaJEJrrN0=
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE=
golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/telemetry v0.0.0-20240521205824-bda55230c457/go.mod h1:pRgIJT+bRLFKnoM1ldnzKoxTIn14Yxz928LQRYYgIN0=
golang.org/x/telemetry v0.0.0-20241106142447-58a1122356f5 h1:TCDqnvbBsFapViksHcHySl/sW4+rTGNIAoJJesHRuMM=
golang.org/x/telemetry v0.0.0-20241106142447-58a1122356f5/go.mod h1:8nZWdGp9pq73ZI//QJyckMQab3yq7hoWi7SI0UIusVI=
golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s=
golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y=
golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g=
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc h1:/hemPrYIhOhy8zYrNj+069zDB68us2sMGsfkFJO0iZs=
rsc.io/pdf v0.1.1 h1:k1MczvYDUvJBe93bYd7wrZLLUEcLZAuF824/I4e5Xr4=
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=

117
http.go
View File

@ -9,57 +9,148 @@ import (
"strings" "strings"
"time" "time"
"go.yarn.social/lextwt"
"go.yarn.social/types"
"go.sour.is/xt/internal/otel" "go.sour.is/xt/internal/otel"
) )
func httpServer(c *console, app *appState) error { func httpServer(c *console, app *appState) error {
otel.Info("start http server") ctx, span := otel.Span(c)
defer span.End()
db, err := app.DB() span.AddEvent("start http server")
db, err := app.DB(ctx)
if err != nil { if err != nil {
otel.Info("missing db", err) span.RecordError(fmt.Errorf("%w: missing db", err))
return err return err
} }
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
_, span := otel.Span(r.Context())
defer span.End()
w.Header().Set("Content-Type", "text/html; charset=utf-8") w.Header().Set("Content-Type", "text/html; charset=utf-8")
w.Write([]byte("ok")) w.Write([]byte("ok"))
}) })
http.HandleFunc("/conv/{hash}", func(w http.ResponseWriter, r *http.Request) { http.HandleFunc("/conv/{hash}", func(w http.ResponseWriter, r *http.Request) {
ctx, span := otel.Span(r.Context())
defer span.End()
hash := r.PathValue("hash") hash := r.PathValue("hash")
if (len(hash) < 6 || len(hash) > 8) && !notAny(hash, "abcdefghijklmnopqrstuvwxyz234567") { if (len(hash) < 6 || len(hash) > 8) && !notAny(hash, "abcdefghijklmnopqrstuvwxyz234567") {
w.WriteHeader(http.StatusBadRequest) w.WriteHeader(http.StatusBadRequest)
return return
} }
w.Header().Set("Content-Type", "text/html; charset=utf-8") w.Header().Set("Content-Type", "text/plain; charset=utf-8")
w.Write([]byte(hash))
rows, err := db.QueryContext(r.Context(), "SELECT feed_id, hash, conv, dt, text FROM twt WHERE hash = $1 or conv = $1", hash) rows, err := db.QueryContext(
ctx,
`SELECT
feed_id,
hash,
conv,
nick,
uri,
text
FROM twts
JOIN (
SELECT
feed_id,
nick,
uri
FROM feeds
) using (feed_id)
WHERE
hash = $1 or
conv = $1
order by ulid asc`,
hash,
)
if err != nil { if err != nil {
otel.Info("error", err) span.RecordError(err)
return return
} }
defer rows.Close() defer rows.Close()
var twts []types.Twt
for rows.Next() { for rows.Next() {
var twt struct { var o struct {
FeedID string FeedID string
Hash string Hash string
Conv string Conv string
Dt time.Time Dt string
Nick string
URI string
Text string Text string
} }
err = rows.Scan(&twt.FeedID, &twt.Hash, &twt.Conv, &twt.Dt, &twt.Text) err = rows.Scan(&o.FeedID, &o.Hash, &o.Conv, &o.Nick, &o.URI, &o.Text)
if err != nil { if err != nil {
otel.Error(err) span.RecordError(err)
return return
} }
twter := types.NewTwter(o.Nick, o.URI)
o.Text = strings.ReplaceAll(o.Text, "\n", "\u2028")
twt, _ := lextwt.ParseLine(o.Text, &twter)
twts = append(twts, twt)
} }
var preamble lextwt.Comments
preamble = append(preamble, lextwt.NewComment("# self = /conv/"+hash))
reg := lextwt.NewTwtRegistry(preamble, twts)
reg.WriteTo(w)
}) })
http.HandleFunc("/feeds", func(w http.ResponseWriter, r *http.Request) { http.HandleFunc("/users", func(w http.ResponseWriter, r *http.Request) {
ctx, span := otel.Span(r.Context())
defer span.End()
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
rows, err := db.QueryContext(
ctx,
`SELECT
feed_id,
uri,
nick,
last_scan_on
FROM feeds
where parent_id is null
order by nick, uri`,
)
if err != nil {
span.RecordError(err)
return
}
defer rows.Close()
var twts []types.Twt
for rows.Next() {
var o struct {
FeedID string
URI string
Nick string
Dt TwtTime
}
err = rows.Scan(&o.FeedID, &o.URI, &o.Nick, &o.Dt)
if err != nil {
span.RecordError(err)
return
}
twts = append(twts, lextwt.NewTwt(
types.NewTwter(o.Nick, o.URI),
lextwt.NewDateTime(o.Dt.Time, o.Dt.Time.Format(time.RFC3339)),
nil,
))
}
reg := lextwt.NewTwtRegistry(nil, twts)
reg.WriteTo(w)
})
http.HandleFunc("/queue", func(w http.ResponseWriter, r *http.Request) {
lis := slices.Collect(app.queue.Iter()) lis := slices.Collect(app.queue.Iter())
sort.Slice(lis, func(i, j int) bool { sort.Slice(lis, func(i, j int) bool {
return lis[i].LastScanOn.Time.Before(lis[j].LastScanOn.Time) return lis[i].LastScanOn.Time.Before(lis[j].LastScanOn.Time)
@ -77,7 +168,7 @@ func httpServer(c *console, app *appState) error {
c.AddCancel(srv.Shutdown) c.AddCancel(srv.Shutdown)
err = srv.ListenAndServe() err = srv.ListenAndServe()
if !errors.Is(err, http.ErrServerClosed) { if !errors.Is(err, http.ErrServerClosed) {
otel.Error(err) span.RecordError(err)
return err return err
} }

12
init.sql
View File

@ -2,8 +2,10 @@ PRAGMA journal_mode=WAL;
create table if not exists feeds ( create table if not exists feeds (
feed_id blob primary key, feed_id blob primary key,
uri text, parent_id blob,
nick text, nick text,
uri text,
state string,
last_scan_on timestamp, last_scan_on timestamp,
refresh_rate int default 600, refresh_rate int default 600,
last_modified_on timestamp, last_modified_on timestamp,
@ -13,12 +15,12 @@ create table if not exists feeds (
create table if not exists twts ( create table if not exists twts (
feed_id blob, feed_id blob,
hash text, ulid blob,
conv text,
dt text, -- timestamp with timezone
text text, text text,
conv text,
hash text,
mentions text, -- json mentions text, -- json
tags text, -- json tags text, -- json
primary key (feed_id, hash) primary key (feed_id, ulid)
); );

61
internal/otel/otel.go
View File

@ -17,13 +17,11 @@ import (
"go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp" "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
"go.opentelemetry.io/otel/exporters/stdout/stdoutlog" "go.opentelemetry.io/otel/exporters/stdout/stdoutlog"
"go.opentelemetry.io/otel/exporters/stdout/stdoutmetric"
"go.opentelemetry.io/otel/exporters/stdout/stdouttrace" "go.opentelemetry.io/otel/exporters/stdout/stdouttrace"
"go.opentelemetry.io/otel/log/global" "go.opentelemetry.io/otel/log/global"
"go.opentelemetry.io/otel/metric" "go.opentelemetry.io/otel/metric"
"go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/propagation"
"go.opentelemetry.io/otel/sdk/log" "go.opentelemetry.io/otel/sdk/log"
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
"go.opentelemetry.io/otel/sdk/resource" "go.opentelemetry.io/otel/sdk/resource"
sdktrace "go.opentelemetry.io/otel/sdk/trace" sdktrace "go.opentelemetry.io/otel/sdk/trace"
semconv "go.opentelemetry.io/otel/semconv/v1.26.0" semconv "go.opentelemetry.io/otel/semconv/v1.26.0"
@ -45,19 +43,50 @@ func Init(ctx context.Context, name string) (shutdown func(context.Context) erro
} }
func Meter() metric.Meter { return meter } func Meter() metric.Meter { return meter }
func Error(err error, v ...any) { // func Error(err error, v ...any) {
// if err == nil {
// return
// }
// fmt.Println("ERR:", append([]any{err}, v...))
// logger.Error(err.Error(), v...)
// }
// func Info(msg string, v ...any) { fmt.Println(append([]any{msg}, v...)); logger.Info(msg, v...) }
type spanny struct{
trace.Span
}
func (s *spanny) RecordError(err error, options ...trace.EventOption) {
if err == nil { if err == nil {
return return
} }
logger.Error(err.Error(), v...) ec := trace.NewEventConfig(options...)
attrs := make([]any, len(ec.Attributes()))
for i, v := range ec.Attributes() {
attrs[i] = v
}
fmt.Println(append([]any{"ERR:", err}, attrs...)...)
logger.Error(err.Error(), attrs...)
s.Span.RecordError(err, options...)
} }
func Info(msg string, v ...any) { logger.Info(msg, v...) } func (s *spanny) AddEvent(name string, options ...trace.EventOption) {
ec := trace.NewEventConfig(options...)
attrs := make([]any, len(ec.Attributes()))
for i, v := range ec.Attributes() {
attrs[i] = v
}
fmt.Println(append([]any{name}, attrs...)...)
logger.Info(name, attrs...)
}
func Span(ctx context.Context, opts ...trace.SpanStartOption) (context.Context, trace.Span) { func Span(ctx context.Context, opts ...trace.SpanStartOption) (context.Context, trace.Span) {
name, attrs := Attrs() name, attrs := Attrs()
ctx, span := tracer.Start(ctx, name, opts...) ctx, span := tracer.Start(ctx, name, opts...)
span.SetAttributes(attrs...) span.SetAttributes(attrs...)
return ctx, span return ctx, &spanny{span}
} }
func Attrs() (string, []attribute.KeyValue) { func Attrs() (string, []attribute.KeyValue) {
var attrs []attribute.KeyValue var attrs []attribute.KeyValue
@ -192,17 +221,17 @@ func newTraceProvider(ctx context.Context, name string) (func(context.Context) e
} }
func newMeterProvider(ctx context.Context, name string) (func(context.Context) error, error) { func newMeterProvider(ctx context.Context, name string) (func(context.Context) error, error) {
metricExporter, err := stdoutmetric.New() // metricExporter, err := stdoutmetric.New()
if err != nil { // if err != nil {
return nil, err // return nil, err
} // }
meterProvider := sdkmetric.NewMeterProvider( // meterProvider := sdkmetric.NewMeterProvider(
sdkmetric.WithReader(sdkmetric.NewPeriodicReader(metricExporter, // sdkmetric.WithReader(sdkmetric.NewPeriodicReader(metricExporter,
// Default is 1m. Set to 3s for demonstrative purposes. // // Default is 1m. Set to 3s for demonstrative purposes.
sdkmetric.WithInterval(3*time.Second))), // sdkmetric.WithInterval(3*time.Second))),
) // )
otel.SetMeterProvider(meterProvider) // otel.SetMeterProvider(meterProvider)
http.Handle("/metrics", promhttp.Handler()) http.Handle("/metrics", promhttp.Handler())
return func(ctx context.Context) error { return nil }, nil return func(ctx context.Context) error { return nil }, nil

6
main.go
View File

@ -8,7 +8,6 @@ import (
"io" "io"
"os" "os"
"os/signal" "os/signal"
"runtime/debug"
"strings" "strings"
"go.opentelemetry.io/otel/metric" "go.opentelemetry.io/otel/metric"
@ -27,7 +26,7 @@ func main() {
dbfile: env("XT_DBFILE", "file:twt.db"), dbfile: env("XT_DBFILE", "file:twt.db"),
baseFeed: env("XT_BASE_FEED", "feed"), baseFeed: env("XT_BASE_FEED", "feed"),
Nick: env("XT_NICK", "xuu"), Nick: env("XT_NICK", "xuu"),
URI: env("XT_URI", "https://txt.sour.is/users/xuu/twtxt.txt"), URI: env("XT_URI", "https://txt.sour.is/user/xuu/twtxt.txt"),
Listen: env("XT_LISTEN", ":8080"), Listen: env("XT_LISTEN", ":8080"),
}) })
@ -41,9 +40,6 @@ func main() {
m_up.Record(ctx, 1) m_up.Record(ctx, 1)
defer m_up.Record(context.Background(), 0) defer m_up.Record(context.Background(), 0)
bi, _ := debug.ReadBuildInfo()
otel.Info(name, "version", bi.Main.Version)
err = run(console) err = run(console)
if !errors.Is(err, context.Canceled) { if !errors.Is(err, context.Canceled) {
console.IfFatal(err) console.IfFatal(err)

2
otel.go
View File

@ -1,2 +0,0 @@
package main

135
refresh-loop.go
View File

@ -1,14 +1,16 @@
package main package main
import ( import (
"context"
"errors" "errors"
"fmt" "fmt"
"io" "io"
"os" "os"
"path/filepath" "path/filepath"
"strings"
"time" "time"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"go.sour.is/xt/internal/otel" "go.sour.is/xt/internal/otel"
"go.yarn.social/lextwt" "go.yarn.social/lextwt"
"go.yarn.social/types" "go.yarn.social/types"
@ -27,61 +29,91 @@ func refreshLoop(c *console, app *appState) error {
defer span.End() defer span.End()
f := NewHTTPFetcher() f := NewHTTPFetcher()
fetch, close := NewFuncPool(c.Context, 25, f.Fetch) fetch, close := NewFuncPool(ctx, 25, f.Fetch)
defer close() defer close()
db, err := app.DB() db, err := app.DB(c)
if err != nil { if err != nil {
otel.Error(err, "missing db") span.RecordError(err)
return err return err
} }
go processorLoop(ctx, db, fetch)
queue := app.queue queue := app.queue
otel.Info("start refresh loop") span.AddEvent("start refresh loop")
for c.Context.Err() == nil {
if queue.IsEmpty() {
otel.Info("load feeds")
it, err := loadFeeds(c.Context, db) for ctx.Err() == nil {
if queue.IsEmpty() {
span.AddEvent("load feeds")
it, err := loadFeeds(ctx, db)
span.RecordError(err)
for f := range it { for f := range it {
queue.Insert(&f) queue.Insert(&f)
} }
if err != nil {
otel.Error(err)
return err
}
} }
span.AddEvent("queue size", trace.WithAttributes(attribute.Int("size", int(queue.count))))
f := queue.ExtractMin() f := queue.ExtractMin()
if f == nil { if f == nil {
otel.Info("sleeping for ", TenMinutes*time.Second) span.AddEvent("sleeping for ", trace.WithAttributes(attribute.Int("seconds", int(TenMinutes))))
select { select {
case <-time.After(TenMinutes * time.Second): case <-time.After(TenMinutes * time.Second):
case <-c.Done(): case <-c.Done():
return nil return nil
} }
continue continue
} }
otel.Info("queue size", queue.count, "next", f.URI, "next scan on", f.LastScanOn.Time.Format(time.RFC3339)) span.AddEvent("next", trace.WithAttributes(
attribute.Int("size", int(queue.count)),
attribute.String("uri", f.URI),
attribute.String("scan on", f.LastScanOn.Time.Format(time.RFC3339)),
))
if time.Until(f.LastScanOn.Time) > 2*time.Hour { if time.Until(f.NextScanOn.Time) > 2*time.Hour {
otel.Info("too soon", f.URI) span.AddEvent("too soon", trace.WithAttributes(attribute.String("uri", f.URI)))
continue continue
} }
select { select {
case <-c.Done(): case <-ctx.Done():
return nil return nil
case t := <-time.After(time.Until(f.LastScanOn.Time)): case t := <-time.After(time.Until(f.NextScanOn.Time)):
otel.Info("fetch", t.Format(time.RFC3339), f.Nick, f.URI) span.AddEvent("fetch", trace.WithAttributes(
attribute.Int("size", int(queue.count)),
attribute.String("uri", f.URI),
attribute.String("timeout", t.Format(time.RFC3339)),
attribute.String("scan on", f.NextScanOn.Time.Format(time.RFC3339)),
))
fetch.Fetch(f) fetch.Fetch(f)
}
}
span.RecordError(ctx.Err())
return ctx.Err()
}
func processorLoop(ctx context.Context, db db, fetch *pool[*Feed, *Response]) {
ctx, span := otel.Span(ctx)
defer span.End()
for ctx.Err() == nil {
select {
case <-ctx.Done():
return
case res := <-fetch.Out(): case res := <-fetch.Out():
otel.Info("got response:", res.Request.URI)
f := res.Request f := res.Request
span.AddEvent("got response", trace.WithAttributes(
attribute.String("uri", f.URI),
attribute.String("scan on", f.NextScanOn.Time.Format(time.RFC3339)),
))
f.LastScanOn.Time = time.Now() f.LastScanOn.Time = time.Now()
f.LastScanOn.Valid = true
err := res.err err := res.err
if res.err != nil { if res.err != nil {
if errors.Is(err, ErrPermanentlyDead) { if errors.Is(err, ErrPermanentlyDead) {
@ -94,12 +126,10 @@ func refreshLoop(c *console, app *appState) error {
f.RefreshRate = OneDay f.RefreshRate = OneDay
} }
otel.Error(err) span.RecordError(err)
f.LastError.String, f.LastError.Valid = err.Error(), true f.LastError.String, f.LastError.Valid = err.Error(), true
err = f.Save(c.Context, db) err = f.Save(ctx, db)
if err != nil { span.RecordError(err)
otel.Error(err)
}
continue continue
} }
@ -107,15 +137,16 @@ func refreshLoop(c *console, app *appState) error {
f.ETag.String, f.ETag.Valid = res.ETag(), true f.ETag.String, f.ETag.Valid = res.ETag(), true
f.LastModified.Time, f.LastModified.Valid = res.LastModified(), true f.LastModified.Time, f.LastModified.Valid = res.LastModified(), true
span.AddEvent("read feed")
cpy, err := os.OpenFile(filepath.Join("feeds", urlNS.UUID5(f.URI).MarshalText()), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) cpy, err := os.OpenFile(filepath.Join("feeds", urlNS.UUID5(f.URI).MarshalText()), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil { if err != nil {
otel.Error(fmt.Errorf("%w: %w", ErrParseFailed, err)) span.RecordError(fmt.Errorf("%w: %w", ErrParseFailed, err))
f.LastError.String, f.LastError.Valid = err.Error(), true f.LastError.String, f.LastError.Valid = err.Error(), true
f.RefreshRate = OneDay f.RefreshRate = OneDay
err = f.Save(c.Context, db) err = f.Save(ctx, db)
otel.Error(err) span.RecordError(err)
continue continue
} }
@ -123,55 +154,37 @@ func refreshLoop(c *console, app *appState) error {
rdr = lextwt.TwtFixer(rdr) rdr = lextwt.TwtFixer(rdr)
twtfile, err := lextwt.ParseFile(rdr, &types.Twter{Nick: f.Nick, URI: f.URI}) twtfile, err := lextwt.ParseFile(rdr, &types.Twter{Nick: f.Nick, URI: f.URI})
if err != nil { if err != nil {
otel.Error(fmt.Errorf("%w: %w", ErrParseFailed, err)) span.RecordError(fmt.Errorf("%w: %w", ErrParseFailed, err))
f.LastError.String, f.LastError.Valid = err.Error(), true f.LastError.String, f.LastError.Valid = err.Error(), true
f.RefreshRate = OneDay f.RefreshRate = OneDay
err = f.Save(c.Context, db) err = f.Save(ctx, db)
otel.Error(err) span.RecordError(err)
continue continue
} }
cpy.Close()
if prev, ok := twtfile.Info().GetN("prev", 0); f.FirstFetch && ok { span.AddEvent("parse complete", trace.WithAttributes(attribute.Int("count", twtfile.Twts().Len())))
_, part, ok := strings.Cut(prev.Value(), " ")
if ok {
part = f.URI[:strings.LastIndex(f.URI, "/")+1] + part
queue.Insert(&Feed{
FetchURI: part,
URI: f.URI,
Nick: f.Nick,
LastScanOn: f.LastScanOn,
RefreshRate: f.RefreshRate,
})
}
}
err = storeFeed(ctx, db, twtfile) err = storeFeed(ctx, db, twtfile)
if err != nil { if err != nil {
otel.Error(err) span.RecordError(err)
f.LastError.String, f.LastError.Valid = err.Error(), true f.LastError.String, f.LastError.Valid = err.Error(), true
err = f.Save(c.Context, db) err = f.Save(ctx, db)
otel.Error(err) span.RecordError(err)
return err continue
} }
cpy.Close()
f.LastScanOn.Time = time.Now()
f.RefreshRate = TenMinutes f.RefreshRate = TenMinutes
f.LastError.String = "" f.LastError.String = ""
err = f.Save(c.Context, db) err = f.Save(ctx, db)
if err != nil { span.RecordError(err)
otel.Error(err)
return err
}
} }
} }
return c.Context.Err() span.RecordError(ctx.Err())
} }