chore: many fixes to code
This commit is contained in:
parent
ed5b43300b
commit
42fe9176b7
GPG Key ID:
8B3B0604F164E04F
84
about.me
84
about.me
@ -1,84 +0,0 @@
|
||||
.ce
|
||||
Preamble
|
||||
.sp
|
||||
We, the people of the United States, in order
|
||||
to form a more perfect Union, establish justice, insure
|
||||
domestic tranquility, provide for the common defense, promote
|
||||
the general welfare,
|
||||
and secure the blessing of liberty to ourselves and our
|
||||
posterity do ordain and establish this Constitution for the
|
||||
United States of America.
|
||||
.sp
|
||||
.nr aR 0 1
|
||||
.af aR I
|
||||
.de AR
|
||||
.ce
|
||||
.ps 16
|
||||
.ft B
|
||||
Article \\n+(aR
|
||||
.nr sE 0 1
|
||||
.af sE 1
|
||||
.ps 12
|
||||
.ft P
|
||||
..
|
||||
.de SE
|
||||
.sp
|
||||
.ft B
|
||||
\\s-2SECTION \\n+(sE:\\s+2
|
||||
.ft P
|
||||
.nr pP 0 1
|
||||
.af pP 1
|
||||
..
|
||||
.de PP
|
||||
.sp
|
||||
.ft I
|
||||
\\s-3Paragraph \\n+(pP:\\s+3
|
||||
.ft P
|
||||
..
|
||||
.AR
|
||||
.SE
|
||||
Legislative powers; in whom vested:
|
||||
.PP
|
||||
All legislative powers herein granted shall be vested in a
|
||||
Congress of the United States, which shall consist of a Senate
|
||||
and a House of Representatives.
|
||||
.SE
|
||||
House of Representatives, how and by whom chosen, Qualifications
|
||||
of a Representative. Representatives and direct taxes, how
|
||||
apportioned. Enumeration. Vacancies to be filled. Power of
|
||||
choosing officers and of impeachment.
|
||||
.PP
|
||||
The House of Representatives shall be composed of members chosen
|
||||
every second year by the people of the several states, and the
|
||||
electors in each State shall have the qualifications requisite
|
||||
for electors of the most numerous branch of the State Legislature.
|
||||
.PP
|
||||
No person shall be a Representative who shall not have attained
|
||||
to the age of twenty-five years, and been seven years a citizen
|
||||
of the United States, and who shall not, when elected, be an
|
||||
inhabitant of that State in which he shall be chosen.
|
||||
.PP
|
||||
Representatives and direct taxes shall be apportioned among the
|
||||
several States which maybe included within this Union, according
|
||||
to their respective numbers, which shall be determined by adding
|
||||
to the whole number of free persons, including those bound for
|
||||
service for a term of years, and excluding Indians not taxed,
|
||||
three-fifths of all other persons. The actual enumeration shall
|
||||
be made within three years after the first meeting of the
|
||||
Congress of the United States, and within every subsequent term
|
||||
of ten years, in such manner as they shall by law direct. The
|
||||
number of Representatives shall not exceed one for every thirty
|
||||
thousand, but each State shall have at least one Representative;
|
||||
and until such enumeration shall be made, the State of New
|
||||
Hampshire shall be entitled to choose three, Massachusetts eight,
|
||||
Rhode Island and Providence Plantations one, Connecticut
|
||||
five, New York six, New Jersey four, Pennsylvania eight,
|
||||
Delaware one, Maryland six, Virginia ten, North Carolina five,
|
||||
South Carolina five, and Georgia three.
|
||||
.PP
|
||||
When vacancies happen in the representation from any State, the
|
||||
Executive Authority thereof shall issue writs of election to fill
|
||||
such vacancies.
|
||||
.PP
|
||||
The House of Representatives shall choose their Speaker and other
|
||||
officers; and shall have the sole power of impeachment.
|
||||
@ -6,6 +6,8 @@ import (
|
||||
"fmt"
|
||||
"iter"
|
||||
"os"
|
||||
"runtime/debug"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
@ -13,7 +15,9 @@ import (
|
||||
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
"github.com/uptrace/opentelemetry-go-extra/otelsql"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
"go.sour.is/xt/internal/otel"
|
||||
"go.yarn.social/lextwt"
|
||||
"go.yarn.social/types"
|
||||
@ -24,12 +28,15 @@ func run(c *console) error {
|
||||
ctx, span := otel.Span(c.Context)
|
||||
defer span.End()
|
||||
|
||||
bi, _ := debug.ReadBuildInfo()
|
||||
span.AddEvent(name, trace.WithAttributes(attribute.String("version", bi.Main.Version)))
|
||||
|
||||
a := c.Args()
|
||||
app := &appState{
|
||||
args: a,
|
||||
feeds: sync.Map{},
|
||||
queue: FibHeap(func(a, b *Feed) bool {
|
||||
return a.LastScanOn.Time.Before(b.LastScanOn.Time)
|
||||
return a.NextScanOn.Time.Before(b.NextScanOn.Time)
|
||||
}),
|
||||
}
|
||||
|
||||
@ -38,7 +45,7 @@ func run(c *console) error {
|
||||
ctx, span := otel.Span(ctx)
|
||||
defer span.End()
|
||||
|
||||
db, err := app.DB()
|
||||
db, err := app.DB(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -76,7 +83,7 @@ func run(c *console) error {
|
||||
return fmt.Errorf("%w: %w", ErrParseFailed, err)
|
||||
}
|
||||
|
||||
db, err := app.DB()
|
||||
db, err := app.DB(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -91,10 +98,16 @@ func run(c *console) error {
|
||||
wg, ctx := errgroup.WithContext(ctx)
|
||||
c.Context = ctx
|
||||
|
||||
wg.Go(func() error { return refreshLoop(c, app) })
|
||||
wg.Go(func() error {
|
||||
return refreshLoop(c, app)
|
||||
})
|
||||
go httpServer(c, app)
|
||||
|
||||
wg.Wait()
|
||||
err = wg.Wait()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return c.Context.Err()
|
||||
}
|
||||
|
||||
@ -104,12 +117,61 @@ type appState struct {
|
||||
queue *fibHeap[Feed]
|
||||
}
|
||||
|
||||
func (app *appState) DB() (*sql.DB, error) {
|
||||
type db struct {
|
||||
*sql.DB
|
||||
Version string
|
||||
Params map[string]string
|
||||
MaxLength int
|
||||
MaxVariableNumber int
|
||||
}
|
||||
|
||||
func (app *appState) DB(ctx context.Context) (db, error) {
|
||||
// return sql.Open(app.args.dbtype, app.args.dbfile)
|
||||
|
||||
return otelsql.Open(app.args.dbtype, app.args.dbfile,
|
||||
var err error
|
||||
db := db{Params: make(map[string]string)}
|
||||
db.DB, err = otelsql.Open(app.args.dbtype, app.args.dbfile,
|
||||
otelsql.WithAttributes(semconv.DBSystemSqlite),
|
||||
otelsql.WithDBName("mydb"))
|
||||
if err != nil {
|
||||
return db, err
|
||||
}
|
||||
|
||||
rows, err := db.DB.QueryContext(ctx, `select sqlite_version()`)
|
||||
if err != nil {
|
||||
return db, err
|
||||
}
|
||||
|
||||
if rows.Next() {
|
||||
rows.Scan(&db.Version)
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
return db, err
|
||||
}
|
||||
|
||||
rows, err = db.DB.QueryContext(ctx, `pragma compile_options`)
|
||||
if err != nil {
|
||||
return db, err
|
||||
}
|
||||
|
||||
for rows.Next() {
|
||||
var key, value string
|
||||
rows.Scan(&key)
|
||||
key, value, _ = strings.Cut(key, "=")
|
||||
db.Params[key] = value
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
return db, err
|
||||
}
|
||||
|
||||
if m, ok := db.Params["MAX_VARIABLE_NUMBER"]; ok {
|
||||
db.MaxVariableNumber, _ = strconv.Atoi(m)
|
||||
}
|
||||
if m, ok := db.Params["MAX_LENGTH"]; ok {
|
||||
db.MaxLength, _ = strconv.Atoi(m)
|
||||
}
|
||||
|
||||
return db, err
|
||||
}
|
||||
|
||||
func (app *appState) Feed(feedID string) *Feed {
|
||||
333
feed.go
333
feed.go
@ -4,7 +4,9 @@ import (
|
||||
"cmp"
|
||||
"context"
|
||||
"database/sql"
|
||||
"database/sql/driver"
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"iter"
|
||||
"net/http"
|
||||
"net/url"
|
||||
@ -14,6 +16,7 @@ import (
|
||||
|
||||
_ "embed"
|
||||
|
||||
"github.com/oklog/ulid/v2"
|
||||
"go.sour.is/xt/internal/otel"
|
||||
"go.yarn.social/lextwt"
|
||||
"go.yarn.social/types"
|
||||
@ -21,22 +24,22 @@ import (
|
||||
|
||||
type Feed struct {
|
||||
FeedID uuid
|
||||
FetchURI string
|
||||
ParentID uuid
|
||||
HashURI string
|
||||
URI string
|
||||
Nick string
|
||||
LastScanOn sql.NullTime
|
||||
State State
|
||||
LastScanOn TwtTime
|
||||
RefreshRate int
|
||||
NextScanOn TwtTime
|
||||
|
||||
LastModified sql.NullTime
|
||||
LastModified TwtTime
|
||||
LastError sql.NullString
|
||||
ETag sql.NullString
|
||||
|
||||
Version string
|
||||
DiscloseFeedURL string
|
||||
DiscloseNick string
|
||||
FirstFetch bool
|
||||
|
||||
State State
|
||||
}
|
||||
|
||||
type State string
|
||||
@ -47,43 +50,34 @@ const (
|
||||
Cold State = "cold"
|
||||
Warm State = "warm"
|
||||
Hot State = "hot"
|
||||
Once State = "once"
|
||||
)
|
||||
|
||||
var (
|
||||
//go:embed init.sql
|
||||
initSQL string
|
||||
|
||||
insertFeed = `
|
||||
insert into feeds
|
||||
(feed_id, uri, nick, last_scan_on, refresh_rate)
|
||||
values (?, ?, ?, ?, ?)
|
||||
ON CONFLICT (feed_id) DO NOTHING
|
||||
`
|
||||
|
||||
insertTwt = `
|
||||
insert into twts
|
||||
(feed_id, hash, conv, dt, text, mentions, tags)
|
||||
values (?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT (feed_id, hash) DO NOTHING
|
||||
`
|
||||
|
||||
fetchFeeds = `
|
||||
select
|
||||
feed_id,
|
||||
uri,
|
||||
nick,
|
||||
last_scan_on,
|
||||
refresh_rate,
|
||||
last_modified_on,
|
||||
last_etag
|
||||
from feeds
|
||||
where datetime(
|
||||
coalesce(last_scan_on, '1901-01-01'),
|
||||
'+'||refresh_rate||' seconds'
|
||||
) < datetime(current_timestamp, '+10 minutes')
|
||||
`
|
||||
insertFeed = func(r int) (string, int) {
|
||||
repeat := ""
|
||||
if r > 1 {
|
||||
repeat = strings.Repeat(", (?, ?, ?, ?, ?, ?, ?)", r-1)
|
||||
}
|
||||
return `
|
||||
insert into feeds (
|
||||
feed_id,
|
||||
parent_id,
|
||||
nick,
|
||||
uri,
|
||||
state,
|
||||
last_scan_on,
|
||||
refresh_rate
|
||||
)
|
||||
values (?, ?, ?, ?, ?, ?, ?)` + repeat + `
|
||||
ON CONFLICT (feed_id) DO NOTHING`, r * 7
|
||||
}
|
||||
updateFeed = `
|
||||
update feeds set
|
||||
update feeds set
|
||||
state = ?,
|
||||
last_scan_on = ?,
|
||||
refresh_rate = ?,
|
||||
last_modified_on = ?,
|
||||
@ -91,21 +85,83 @@ var (
|
||||
last_error = ?
|
||||
where feed_id = ?
|
||||
`
|
||||
|
||||
insertTwt = func(r int) (string, int) {
|
||||
repeat := ""
|
||||
if r > 1 {
|
||||
repeat = strings.Repeat(", (?, ?, ?, ?, ?, ?, ?)", r-1)
|
||||
}
|
||||
return `
|
||||
insert into twts
|
||||
(feed_id, ulid, text, hash, conv, mentions, tags)
|
||||
values (?, ?, ?, ?, ?, ?, ?)` + repeat + `
|
||||
ON CONFLICT (feed_id, ulid) DO NOTHING`, r * 7
|
||||
}
|
||||
|
||||
fetchFeeds = `
|
||||
select
|
||||
feed_id,
|
||||
parent_id,
|
||||
coalesce(hashing_uri, uri) hash_uri,
|
||||
uri,
|
||||
nick,
|
||||
state,
|
||||
last_scan_on,
|
||||
strftime(
|
||||
'%Y-%m-%dT%H:%M:%fZ',
|
||||
coalesce(last_scan_on, '1901-01-01'),
|
||||
'+'||refresh_rate||' seconds'
|
||||
) next_scan_on,
|
||||
refresh_rate,
|
||||
last_modified_on,
|
||||
last_etag
|
||||
from feeds
|
||||
left join (
|
||||
select
|
||||
feed_id parent_id,
|
||||
uri hashing_uri
|
||||
from feeds
|
||||
where parent_id is null
|
||||
) using (parent_id)
|
||||
where datetime(
|
||||
coalesce(last_scan_on, '1901-01-01'),
|
||||
'+'||refresh_rate||' seconds'
|
||||
) < datetime(current_timestamp, '+10 minutes')
|
||||
`
|
||||
)
|
||||
|
||||
func (f *Feed) Save(ctx context.Context, db *sql.DB) error {
|
||||
func (f *Feed) Create(ctx context.Context, db db) error {
|
||||
ctx, span := otel.Span(ctx)
|
||||
defer span.End()
|
||||
query, _ := insertFeed(1)
|
||||
_, err := db.ExecContext(
|
||||
ctx,
|
||||
query,
|
||||
f.FeedID, // feed_id
|
||||
f.ParentID, // parent_id
|
||||
f.Nick, // nick
|
||||
f.URI, // uri
|
||||
f.State, // state
|
||||
f.LastScanOn, // last_scan_on
|
||||
f.RefreshRate, // refresh_rate
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
func (f *Feed) Save(ctx context.Context, db db) error {
|
||||
ctx, span := otel.Span(ctx)
|
||||
defer span.End()
|
||||
|
||||
_, err := db.ExecContext(
|
||||
ctx,
|
||||
updateFeed,
|
||||
f.LastScanOn,
|
||||
f.RefreshRate,
|
||||
f.LastModified,
|
||||
f.ETag,
|
||||
f.LastError,
|
||||
f.FeedID,
|
||||
f.State, // state
|
||||
f.LastScanOn, // last_scan_on
|
||||
f.RefreshRate, // refresh_rate
|
||||
f.LastModified, // last_modified_on
|
||||
f.ETag, // last_etag
|
||||
f.LastError, // last_error
|
||||
f.FeedID, // feed_id
|
||||
)
|
||||
return err
|
||||
}
|
||||
@ -117,9 +173,13 @@ func (f *Feed) Scan(res interface{ Scan(...any) error }) error {
|
||||
f.Version = "0.0.1"
|
||||
err = res.Scan(
|
||||
&f.FeedID,
|
||||
&f.ParentID,
|
||||
&f.HashURI,
|
||||
&f.URI,
|
||||
&f.Nick,
|
||||
&f.State,
|
||||
&f.LastScanOn,
|
||||
&f.NextScanOn,
|
||||
&f.RefreshRate,
|
||||
&f.LastModified,
|
||||
&f.ETag,
|
||||
@ -128,19 +188,10 @@ func (f *Feed) Scan(res interface{ Scan(...any) error }) error {
|
||||
return err
|
||||
}
|
||||
|
||||
if !f.LastScanOn.Valid {
|
||||
f.FirstFetch = true
|
||||
f.LastScanOn.Time = time.Now()
|
||||
f.LastScanOn.Valid = true
|
||||
} else {
|
||||
f.LastScanOn.Time = f.LastScanOn.Time.Add(time.Duration(f.RefreshRate) * time.Second)
|
||||
}
|
||||
|
||||
f.FetchURI = f.URI
|
||||
return err
|
||||
}
|
||||
|
||||
func loadFeeds(ctx context.Context, db *sql.DB) (iter.Seq[Feed], error) {
|
||||
func loadFeeds(ctx context.Context, db db) (iter.Seq[Feed], error) {
|
||||
ctx, span := otel.Span(ctx)
|
||||
|
||||
var err error
|
||||
@ -159,6 +210,7 @@ func loadFeeds(ctx context.Context, db *sql.DB) (iter.Seq[Feed], error) {
|
||||
var f Feed
|
||||
err = f.Scan(res)
|
||||
if err != nil {
|
||||
span.RecordError(err)
|
||||
return
|
||||
}
|
||||
if !yield(f) {
|
||||
@ -168,7 +220,7 @@ func loadFeeds(ctx context.Context, db *sql.DB) (iter.Seq[Feed], error) {
|
||||
}, err
|
||||
}
|
||||
|
||||
func storeFeed(ctx context.Context, db *sql.DB, f types.TwtFile) error {
|
||||
func storeFeed(ctx context.Context, db db, f types.TwtFile) error {
|
||||
ctx, span := otel.Span(ctx)
|
||||
defer span.End()
|
||||
|
||||
@ -201,20 +253,11 @@ func storeFeed(ctx context.Context, db *sql.DB, f types.TwtFile) error {
|
||||
|
||||
defer tx.Rollback()
|
||||
|
||||
_, err = tx.ExecContext(
|
||||
ctx,
|
||||
insertFeed,
|
||||
feedID,
|
||||
f.Twter().HashingURI,
|
||||
f.Twter().DomainNick(),
|
||||
loadTS,
|
||||
refreshRate,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
twts := f.Twts()
|
||||
_, size := insertTwt(len(twts))
|
||||
args := make([]any, 0, size)
|
||||
|
||||
for _, twt := range f.Twts() {
|
||||
for _, twt := range twts {
|
||||
mentions := make(uuids, 0, len(twt.Mentions()))
|
||||
for _, mention := range twt.Mentions() {
|
||||
followMap[mention.Twter().Nick] = mention.Twter().URI
|
||||
@ -233,32 +276,76 @@ func storeFeed(ctx context.Context, db *sql.DB, f types.TwtFile) error {
|
||||
subjectTag = tag.Text()
|
||||
}
|
||||
}
|
||||
args = append(
|
||||
args,
|
||||
feedID, // feed_id
|
||||
makeULID(twt), // ulid
|
||||
fmt.Sprintf("%+l", twt), // text
|
||||
subjectTag, // conv
|
||||
twt.Hash(), // hash
|
||||
mentions.ToStrList(), // mentions
|
||||
tags, // tags
|
||||
)
|
||||
}
|
||||
for query, args := range chunk(args, insertTwt, db.MaxVariableNumber) {
|
||||
fmt.Println("store", f.Twter().URI, len(args))
|
||||
|
||||
_, err = tx.ExecContext(
|
||||
ctx,
|
||||
insertTwt,
|
||||
feedID,
|
||||
twt.Hash(),
|
||||
subjectTag,
|
||||
twt.Created(),
|
||||
fmt.Sprint(twt),
|
||||
mentions.ToStrList(),
|
||||
tags,
|
||||
query,
|
||||
args...,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
args = args[:0]
|
||||
args = append(args,
|
||||
feedID, // feed_id
|
||||
nil, // parent_id
|
||||
f.Twter().DomainNick(), // nick
|
||||
f.Twter().URI, // uri
|
||||
"warm", // state
|
||||
TwtTime{Time: loadTS, Valid: true}, // last_scan_on
|
||||
refreshRate, // refresh_rate
|
||||
)
|
||||
|
||||
if prev, ok := f.Info().GetN("prev", 0); ok {
|
||||
_, part, ok := strings.Cut(prev.Value(), " ")
|
||||
if ok {
|
||||
uri:= f.Twter().URI
|
||||
part = uri[:strings.LastIndex(uri, "/")+1] + part
|
||||
childID := urlNS.UUID5(part)
|
||||
|
||||
args = append(args,
|
||||
childID, // feed_id
|
||||
feedID, // parent_id
|
||||
f.Twter().DomainNick(), // nick
|
||||
part, // uri
|
||||
"once", // state
|
||||
nil, // last_scan_on
|
||||
refreshRate, // refresh_rate
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
for nick, uri := range followMap {
|
||||
args = append(args,
|
||||
urlNS.UUID5(uri), // feed_id
|
||||
nil, // parent_id
|
||||
nick, // nick
|
||||
uri, // uri
|
||||
"warm", // state
|
||||
nil, // last_scan_on
|
||||
refreshRate, // refresh_rate
|
||||
)
|
||||
}
|
||||
for query, args := range chunk(args, insertFeed, db.MaxVariableNumber) {
|
||||
_, err = tx.ExecContext(
|
||||
ctx,
|
||||
insertFeed,
|
||||
urlNS.UUID5(uri),
|
||||
uri,
|
||||
nick,
|
||||
nil,
|
||||
refreshRate,
|
||||
query,
|
||||
args...,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
@ -270,11 +357,11 @@ func storeFeed(ctx context.Context, db *sql.DB, f types.TwtFile) error {
|
||||
|
||||
func (feed *Feed) MakeHTTPRequest(ctx context.Context) (*http.Request, error) {
|
||||
feed.State = "fetch"
|
||||
if strings.Contains(feed.FetchURI, "lublin.se") {
|
||||
if strings.Contains(feed.URI, "lublin.se") {
|
||||
return nil, fmt.Errorf("%w: permaban: %s", ErrPermanentlyDead, feed.URI)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", feed.FetchURI, nil)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", feed.URI, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating HTTP request failed: %w", err)
|
||||
}
|
||||
@ -298,3 +385,83 @@ func (feed *Feed) MakeHTTPRequest(ctx context.Context) (*http.Request, error) {
|
||||
|
||||
return req, nil
|
||||
}
|
||||
|
||||
type TwtTime struct {
|
||||
Time time.Time
|
||||
Valid bool // Valid is true if Time is not NULL
|
||||
}
|
||||
|
||||
// Scan implements the [Scanner] interface.
|
||||
func (n *TwtTime) Scan(value any) error {
|
||||
var err error
|
||||
|
||||
switch value := value.(type) {
|
||||
case nil:
|
||||
n.Time, n.Valid = time.Time{}, false
|
||||
return nil
|
||||
case string:
|
||||
n.Valid = true
|
||||
n.Time, err = time.Parse(time.RFC3339, value)
|
||||
case time.Time:
|
||||
n.Valid = true
|
||||
n.Time = value
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Value implements the [driver.Valuer] interface.
|
||||
func (n TwtTime) Value() (driver.Value, error) {
|
||||
if !n.Valid {
|
||||
return nil, nil
|
||||
}
|
||||
return n.Time.Format(time.RFC3339), nil
|
||||
}
|
||||
|
||||
func makeULID(twt types.Twt) ulid.ULID {
|
||||
h64 := fnv.New64a()
|
||||
h16 := fnv.New32a()
|
||||
text := []byte(fmt.Sprintf("%+l", twt))
|
||||
b := make([]byte, 10)
|
||||
copy(b, h16.Sum(text)[:2])
|
||||
copy(b[2:], h64.Sum(text))
|
||||
u := ulid.ULID{}
|
||||
u.SetTime(ulid.Timestamp(twt.Created()))
|
||||
u.SetEntropy(b)
|
||||
|
||||
return u
|
||||
}
|
||||
|
||||
func chunk(args []any, qry func(int) (string, int), maxArgs int) iter.Seq2[string, []any] {
|
||||
_, size := qry(1)
|
||||
itemsPerIter := maxArgs / size
|
||||
|
||||
if len(args) < size {
|
||||
return func(yield func(string, []any) bool) {}
|
||||
}
|
||||
|
||||
if len(args) < maxArgs {
|
||||
return func(yield func(string, []any) bool) {
|
||||
query, _ := qry(len(args) / size)
|
||||
yield(query, args)
|
||||
}
|
||||
}
|
||||
|
||||
return func(yield func(string, []any) bool) {
|
||||
for len(args) > 0 {
|
||||
if len(args) > maxArgs {
|
||||
query, size := qry(itemsPerIter)
|
||||
if !yield(query, args[:size]) {
|
||||
return
|
||||
}
|
||||
args = args[size:]
|
||||
continue
|
||||
}
|
||||
|
||||
query, _ := qry(len(args) / size)
|
||||
yield(query, args)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
48
fetcher.go
48
fetcher.go
@ -8,6 +8,8 @@ import (
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go.sour.is/xt/internal/otel"
|
||||
)
|
||||
|
||||
var (
|
||||
@ -62,7 +64,7 @@ func NewHTTPFetcher() *httpFetcher {
|
||||
ForceAttemptHTTP2: false,
|
||||
MaxIdleConns: 100,
|
||||
IdleConnTimeout: 10 * time.Second,
|
||||
TLSHandshakeTimeout: 10 * time.Second,
|
||||
TLSHandshakeTimeout: 5 * time.Second,
|
||||
ExpectContinueTimeout: 1 * time.Second,
|
||||
},
|
||||
},
|
||||
@ -70,6 +72,10 @@ func NewHTTPFetcher() *httpFetcher {
|
||||
}
|
||||
|
||||
func (f *httpFetcher) Fetch(ctx context.Context, request *Feed) *Response {
|
||||
ctx, span := otel.Span(ctx)
|
||||
defer span.End()
|
||||
|
||||
defer fmt.Println("fetch done", request.URI)
|
||||
response := &Response{
|
||||
Request: request,
|
||||
}
|
||||
@ -79,8 +85,9 @@ func (f *httpFetcher) Fetch(ctx context.Context, request *Feed) *Response {
|
||||
response.err = err
|
||||
return response
|
||||
}
|
||||
|
||||
span.AddEvent("start request")
|
||||
res, err := f.client.Do(req)
|
||||
span.AddEvent("got response")
|
||||
if err != nil {
|
||||
if errors.Is(err, &net.DNSError{}) {
|
||||
response.err = fmt.Errorf("%w: %s", ErrTemporarilyDead, err)
|
||||
@ -97,7 +104,7 @@ func (f *httpFetcher) Fetch(ctx context.Context, request *Feed) *Response {
|
||||
case 304:
|
||||
response.err = fmt.Errorf("%w: %s", ErrUnmodified, res.Status)
|
||||
|
||||
case 400, 406, 502, 503:
|
||||
case 400, 406, 429, 502, 503:
|
||||
response.err = fmt.Errorf("%w: %s", ErrTemporarilyDead, res.Status)
|
||||
|
||||
case 403, 404, 410:
|
||||
@ -121,29 +128,54 @@ func NewFuncPool[IN, OUT any](
|
||||
size int,
|
||||
fetch func(ctx context.Context, request IN) OUT,
|
||||
) (*pool[IN, OUT], func()) {
|
||||
ctx, span := otel.Span(ctx)
|
||||
defer span.End()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
|
||||
in := make(chan IN, size)
|
||||
out := make(chan OUT, size)
|
||||
out := make(chan OUT)
|
||||
|
||||
wg.Add(size)
|
||||
for range size {
|
||||
go func() {
|
||||
ctx, span := otel.Span(ctx)
|
||||
defer span.End()
|
||||
|
||||
defer wg.Done()
|
||||
for request := range in {
|
||||
ctx, cancel := context.WithTimeoutCause(ctx, 15*time.Second, fmt.Errorf("GOT STUCK"))
|
||||
defer cancel()
|
||||
|
||||
ctx, span := otel.Span(ctx)
|
||||
defer span.End()
|
||||
|
||||
span.AddEvent("start fetch")
|
||||
r := fetch(ctx, request)
|
||||
span.AddEvent("got fetch")
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case out <- fetch(ctx, request):
|
||||
case out <- r:
|
||||
span.AddEvent("sent queue")
|
||||
case <-time.After(20 * time.Second):
|
||||
fmt.Println("GOT STUCK", request)
|
||||
span.AddEvent("GOT STUCK")
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
return &pool[IN, OUT]{
|
||||
in: in,
|
||||
out: out,
|
||||
}, func() { close(in); wg.Wait(); close(out) }
|
||||
in: in,
|
||||
out: out,
|
||||
}, func() {
|
||||
close(in)
|
||||
wg.Wait()
|
||||
close(out)
|
||||
}
|
||||
}
|
||||
|
||||
func (f *pool[IN, OUT]) Fetch(request IN) {
|
||||
|
||||
3
go.mod
3
go.mod
@ -41,6 +41,7 @@ require (
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/matryer/is v1.4.1
|
||||
github.com/oklog/ulid/v2 v2.1.0
|
||||
github.com/prometheus/client_golang v1.20.5
|
||||
github.com/sirupsen/logrus v1.9.3 // indirect
|
||||
github.com/uptrace/opentelemetry-go-extra/otelsql v0.3.2
|
||||
@ -53,7 +54,7 @@ require (
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.34.0
|
||||
go.opentelemetry.io/otel/sdk v1.34.0
|
||||
go.opentelemetry.io/otel/sdk/log v0.10.0
|
||||
go.opentelemetry.io/otel/sdk/metric v1.34.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/metric v1.34.0
|
||||
go.yarn.social/types v0.0.0-20250108134258-ed75fa653ede
|
||||
golang.org/x/crypto v0.33.0 // indirect
|
||||
golang.org/x/sync v0.11.0
|
||||
|
||||
5
go.sum
5
go.sum
@ -35,6 +35,11 @@ github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBW
|
||||
github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||
github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
|
||||
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
|
||||
github.com/oklog/ulid/v2 v2.1.0 h1:+9lhoxAP56we25tyYETBBY1YLA2SaoLvUFgrP2miPJU=
|
||||
github.com/oklog/ulid/v2 v2.1.0/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ=
|
||||
github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y=
|
||||
|
||||
191
go.work.sum
Normal file
191
go.work.sum
Normal file
@ -0,0 +1,191 @@
|
||||
cel.dev/expr v0.19.0 h1:lXuo+nDhpyJSpWxpPVi5cPUwzKb+dsdOiw6IreM5yt0=
|
||||
cel.dev/expr v0.19.0/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw=
|
||||
cloud.google.com/go v0.26.0 h1:e0WKqKTd5BnrG8aKH3J3h+QvEIQtSUcf2n5UZ5ZgLtQ=
|
||||
cloud.google.com/go/compute/metadata v0.5.2 h1:UxK4uu/Tn+I3p2dYWTfiX4wva7aYlKixAHn3fyqngqo=
|
||||
cloud.google.com/go/compute/metadata v0.5.2/go.mod h1:C66sj2AluDcIqakBq/M8lw8/ybHgOZqin2obFxa/E5k=
|
||||
dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s=
|
||||
dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
|
||||
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0 h1:3c8yed4lgqTt+oTQ+JNMDo+F4xprBf+O/il4ZC0nRLw=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0/go.mod h1:obipzmGjfSjam60XLwGfqUkJsfiheAl+TUjG+4yzyPM=
|
||||
github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ=
|
||||
github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE=
|
||||
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
|
||||
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
|
||||
github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78=
|
||||
github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0=
|
||||
github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0=
|
||||
github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
|
||||
github.com/alecthomas/kingpin/v2 v2.4.0 h1:f48lwail6p8zpO1bC4TxtqACaGqHYA22qkHjHpqDjYY=
|
||||
github.com/alecthomas/kingpin/v2 v2.4.0/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE=
|
||||
github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc=
|
||||
github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
|
||||
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc=
|
||||
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
|
||||
github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg=
|
||||
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
|
||||
github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4=
|
||||
github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
|
||||
github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWpi6yML8=
|
||||
github.com/aymanbagabas/go-udiff v0.2.0/go.mod h1:RE4Ex0qsGkTAJoQdQQCA0uG+nAzJO/pI/QwceO5fgrA=
|
||||
github.com/badgerodon/ioutil v0.0.0-20150716134133-06e58e34b867 h1:nsDNoesoGwPzPkcrR1w1uzPUtiqwCXoNnkWC7nUuRHI=
|
||||
github.com/badgerodon/ioutil v0.0.0-20150716134133-06e58e34b867/go.mod h1:Ctq1YQi0dOq7QgBLZZ7p1Fr3IbAAqL/yMqDIHoe9WtE=
|
||||
github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g=
|
||||
github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw=
|
||||
github.com/charmbracelet/harmonica v0.2.0 h1:8NxJWRWg/bzKqqEaaeFNipOu77YR5t8aSwG4pgaUBiQ=
|
||||
github.com/charmbracelet/harmonica v0.2.0/go.mod h1:KSri/1RMQOZLbw7AHqgcBycp8pgJnQMYYT8QZRqZ1Ao=
|
||||
github.com/charmbracelet/x/exp/golden v0.0.0-20250213125511-a0c32e22e4fc h1:Tp8vprGbBhTAeyCNgrWPYIPyVEo9qmFRAcRhSdeind4=
|
||||
github.com/charmbracelet/x/exp/golden v0.0.0-20250213125511-a0c32e22e4fc/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U=
|
||||
github.com/charmbracelet/x/exp/teatest v0.0.0-20250213125511-a0c32e22e4fc h1:p4x5lOqNZEELobbdrm00kW5xrQwXZtIrUW8yTPK16aU=
|
||||
github.com/charmbracelet/x/exp/teatest v0.0.0-20250213125511-a0c32e22e4fc/go.mod h1:ag+SpTUkiN/UuUGYPX3Ci4fR1oF3XX97PpGhiXK7i6U=
|
||||
github.com/cilium/ebpf v0.11.0 h1:V8gS/bTCCjX9uUnkUFUpPsksM8n1lXBAvHcpiFk1X2Y=
|
||||
github.com/cilium/ebpf v0.11.0/go.mod h1:WE7CZAnqOL2RouJ4f1uyNhqr2P4CCvXFIqdRDUgWsVs=
|
||||
github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI=
|
||||
github.com/cloudflare/circl v1.5.0 h1:hxIWksrX6XN5a1L2TI/h53AGPhNHoUBo+TD1ms9+pys=
|
||||
github.com/cloudflare/circl v1.5.0/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs=
|
||||
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f h1:WBZRG4aNOuI15bLRrCgN8fCq8E5Xuty6jGbmSNEvSsU=
|
||||
github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 h1:QVw89YDxXxEe+l8gU8ETbOasdwEV+avkR75ZzsVV9WI=
|
||||
github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8=
|
||||
github.com/cosiner/argv v0.1.0 h1:BVDiEL32lwHukgJKP87btEPenzrrHUjajs/8yzaqcXg=
|
||||
github.com/cosiner/argv v0.1.0/go.mod h1:EusR6TucWKX+zFgtdUsKT2Cvg45K5rtpCcWz4hK06d8=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||
github.com/creack/pty v1.1.20 h1:VIPb/a2s17qNeQgDnkfZC35RScx+blkKF8GV68n80J4=
|
||||
github.com/creack/pty v1.1.20/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
|
||||
github.com/cyphar/filepath-securejoin v0.3.4 h1:VBWugsJh2ZxJmLFSM06/0qzQyiQX2Qs0ViKrUAcqdZ8=
|
||||
github.com/cyphar/filepath-securejoin v0.3.4/go.mod h1:8s/MCNJREmFK0H02MF6Ihv1nakJe4L/w3WZLHNkvlYM=
|
||||
github.com/derekparker/trie v0.0.0-20230829180723-39f4de51ef7d h1:hUWoLdw5kvo2xCsqlsIBMvWUc1QCSsCYD2J2+Fg6YoU=
|
||||
github.com/derekparker/trie v0.0.0-20230829180723-39f4de51ef7d/go.mod h1:C7Es+DLenIpPc9J6IYw4jrK0h7S9bKj4DNl8+KxGEXU=
|
||||
github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
|
||||
github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
|
||||
github.com/envoyproxy/go-control-plane v0.13.1 h1:vPfJZCkob6yTMEgS+0TwfTUfbHjfy/6vOJ8hUWX/uXE=
|
||||
github.com/envoyproxy/go-control-plane v0.13.1/go.mod h1:X45hY0mufo6Fd0KW3rqsGvQMw58jvjymeCzBU3mWyHw=
|
||||
github.com/envoyproxy/protoc-gen-validate v1.1.0 h1:tntQDh69XqOCOZsDz0lVJQez/2L6Uu2PdjCQwWCJ3bM=
|
||||
github.com/envoyproxy/protoc-gen-validate v1.1.0/go.mod h1:sXRDRVmzEbkM7CVcM06s9shE/m23dg3wzjl0UWqJ2q4=
|
||||
github.com/felixge/fgprof v0.9.5 h1:8+vR6yu2vvSKn08urWyEuxx75NWPEvybbkBirEpsbVY=
|
||||
github.com/felixge/fgprof v0.9.5/go.mod h1:yKl+ERSa++RYOs32d8K6WEXCB4uXdLls4ZaZPpayhMM=
|
||||
github.com/go-delve/liner v1.2.3-0.20231231155935-4726ab1d7f62 h1:IGtvsNyIuRjl04XAOFGACozgUD7A82UffYxZt4DWbvA=
|
||||
github.com/go-delve/liner v1.2.3-0.20231231155935-4726ab1d7f62/go.mod h1:biJCRbqp51wS+I92HMqn5H8/A0PAhxn2vyOT+JqhiGI=
|
||||
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
|
||||
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic=
|
||||
github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU=
|
||||
github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow=
|
||||
github.com/go-git/go-git/v5 v5.12.0 h1:7Md+ndsjrzZxbddRDZjF14qK+NN56sy6wkqaVrjZtys=
|
||||
github.com/go-git/go-git/v5 v5.12.0/go.mod h1:FTM9VKtnI2m65hNI/TenDDDnUf2Q9FHnXYjuz9i5OEY=
|
||||
github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU=
|
||||
github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
|
||||
github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA=
|
||||
github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
|
||||
github.com/golang/glog v1.2.3 h1:oDTdz9f5VGVVNGu/Q7UXKWYsD0873HXLHdJUNBsSEKM=
|
||||
github.com/golang/glog v1.2.3/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w=
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/mock v1.1.1 h1:G5FRp8JnTd7RQH5kemVNlMeyXQAztQ3mOWV95KxsXH8=
|
||||
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
||||
github.com/google/go-dap v0.12.0 h1:rVcjv3SyMIrpaOoTAdFDyHs99CwVOItIJGKLQFQhNeM=
|
||||
github.com/google/go-dap v0.12.0/go.mod h1:tNjCASCm5cqePi/RVXXWEVqtnNLV1KTWtYOqu6rZNzc=
|
||||
github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
|
||||
github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 h1:y3N7Bm7Y9/CtpiVkw/ZWj6lSlDF3F74SfKwfTCer72Q=
|
||||
github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
|
||||
github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c=
|
||||
github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
|
||||
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
|
||||
github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
|
||||
github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
|
||||
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
|
||||
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
|
||||
github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
|
||||
github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
|
||||
github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
|
||||
github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
|
||||
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU=
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
|
||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
|
||||
github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4=
|
||||
github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI=
|
||||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
|
||||
github.com/pkg/profile v1.7.0 h1:hnbDkaNWPCLMO9wGLdBFTIZvzDrDfBM2072E1S9gJkA=
|
||||
github.com/pkg/profile v1.7.0/go.mod h1:8Uer0jas47ZQMJ7VD+OHknK4YDY07LPUC6dEvqDjvNo=
|
||||
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo=
|
||||
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
|
||||
github.com/rogpeppe/fastuuid v1.2.0 h1:Ppwyp6VYCF1nvBTXL3trRso7mXMlRrw9ooo375wvi2s=
|
||||
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
|
||||
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
|
||||
github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
|
||||
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
||||
github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
|
||||
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/sahilm/fuzzy v0.1.1 h1:ceu5RHF8DGgoi+/dR5PsECjCDH1BE3Fnmpo7aVXOdRA=
|
||||
github.com/sahilm/fuzzy v0.1.1/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8ZK9Y=
|
||||
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8=
|
||||
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
|
||||
github.com/skeema/knownhosts v1.3.0 h1:AM+y0rI04VksttfwjkSTNQorvGqmwATnvnAHpSgc0LY=
|
||||
github.com/skeema/knownhosts v1.3.0/go.mod h1:sPINvnADmT/qYH1kfv+ePMmOBTH6Tbl7b5LvTDjFK7M=
|
||||
github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
|
||||
github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
|
||||
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
|
||||
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
|
||||
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
|
||||
github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
|
||||
github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
|
||||
github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc=
|
||||
github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU=
|
||||
github.com/yuin/goldmark v1.4.13 h1:fVcFKWvrslecOb/tg+Cc05dkeYx540o0FuFt3nUVDoE=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.32.0 h1:P78qWqkLSShicHmAzfECaTgvslqHxblNE9j62Ws1NK8=
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.32.0/go.mod h1:TVqo0Sda4Cv8gCIixd7LuLwW4EylumVWfhjZJjDD4DU=
|
||||
go.starlark.net v0.0.0-20231101134539-556fd59b42f6 h1:+eC0F/k4aBLC4szgOcjd7bDTEnpxADJyWJE0yowgM3E=
|
||||
go.starlark.net v0.0.0-20231101134539-556fd59b42f6/go.mod h1:LcLNIzVOMp4oV+uusnpk+VU+SzXaJakUuBjoCSWH5dM=
|
||||
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 h1:XQyxROzUlZH+WIQwySDgnISgOivlhjIEwaQaJEJrrN0=
|
||||
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||
golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE=
|
||||
golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
|
||||
golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/telemetry v0.0.0-20240521205824-bda55230c457/go.mod h1:pRgIJT+bRLFKnoM1ldnzKoxTIn14Yxz928LQRYYgIN0=
|
||||
golang.org/x/telemetry v0.0.0-20241106142447-58a1122356f5 h1:TCDqnvbBsFapViksHcHySl/sW4+rTGNIAoJJesHRuMM=
|
||||
golang.org/x/telemetry v0.0.0-20241106142447-58a1122356f5/go.mod h1:8nZWdGp9pq73ZI//QJyckMQab3yq7hoWi7SI0UIusVI=
|
||||
golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s=
|
||||
golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y=
|
||||
golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g=
|
||||
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
||||
golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
|
||||
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE=
|
||||
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
|
||||
google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||
gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
|
||||
gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
|
||||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc h1:/hemPrYIhOhy8zYrNj+069zDB68us2sMGsfkFJO0iZs=
|
||||
rsc.io/pdf v0.1.1 h1:k1MczvYDUvJBe93bYd7wrZLLUEcLZAuF824/I4e5Xr4=
|
||||
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
|
||||
117
http.go
117
http.go
@ -9,57 +9,148 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go.yarn.social/lextwt"
|
||||
"go.yarn.social/types"
|
||||
|
||||
"go.sour.is/xt/internal/otel"
|
||||
)
|
||||
|
||||
func httpServer(c *console, app *appState) error {
|
||||
otel.Info("start http server")
|
||||
ctx, span := otel.Span(c)
|
||||
defer span.End()
|
||||
|
||||
db, err := app.DB()
|
||||
span.AddEvent("start http server")
|
||||
|
||||
db, err := app.DB(ctx)
|
||||
if err != nil {
|
||||
otel.Info("missing db", err)
|
||||
span.RecordError(fmt.Errorf("%w: missing db", err))
|
||||
return err
|
||||
}
|
||||
|
||||
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
||||
_, span := otel.Span(r.Context())
|
||||
defer span.End()
|
||||
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
w.Write([]byte("ok"))
|
||||
})
|
||||
|
||||
http.HandleFunc("/conv/{hash}", func(w http.ResponseWriter, r *http.Request) {
|
||||
ctx, span := otel.Span(r.Context())
|
||||
defer span.End()
|
||||
|
||||
hash := r.PathValue("hash")
|
||||
if (len(hash) < 6 || len(hash) > 8) && !notAny(hash, "abcdefghijklmnopqrstuvwxyz234567") {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
w.Write([]byte(hash))
|
||||
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||
|
||||
rows, err := db.QueryContext(r.Context(), "SELECT feed_id, hash, conv, dt, text FROM twt WHERE hash = $1 or conv = $1", hash)
|
||||
rows, err := db.QueryContext(
|
||||
ctx,
|
||||
`SELECT
|
||||
feed_id,
|
||||
hash,
|
||||
conv,
|
||||
nick,
|
||||
uri,
|
||||
text
|
||||
FROM twts
|
||||
JOIN (
|
||||
SELECT
|
||||
feed_id,
|
||||
nick,
|
||||
uri
|
||||
FROM feeds
|
||||
) using (feed_id)
|
||||
WHERE
|
||||
hash = $1 or
|
||||
conv = $1
|
||||
order by ulid asc`,
|
||||
hash,
|
||||
)
|
||||
if err != nil {
|
||||
otel.Info("error", err)
|
||||
span.RecordError(err)
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var twts []types.Twt
|
||||
for rows.Next() {
|
||||
var twt struct {
|
||||
var o struct {
|
||||
FeedID string
|
||||
Hash string
|
||||
Conv string
|
||||
Dt time.Time
|
||||
Dt string
|
||||
Nick string
|
||||
URI string
|
||||
Text string
|
||||
}
|
||||
err = rows.Scan(&twt.FeedID, &twt.Hash, &twt.Conv, &twt.Dt, &twt.Text)
|
||||
err = rows.Scan(&o.FeedID, &o.Hash, &o.Conv, &o.Nick, &o.URI, &o.Text)
|
||||
if err != nil {
|
||||
otel.Error(err)
|
||||
span.RecordError(err)
|
||||
return
|
||||
}
|
||||
twter := types.NewTwter(o.Nick, o.URI)
|
||||
o.Text = strings.ReplaceAll(o.Text, "\n", "\u2028")
|
||||
twt, _ := lextwt.ParseLine(o.Text, &twter)
|
||||
twts = append(twts, twt)
|
||||
}
|
||||
var preamble lextwt.Comments
|
||||
preamble = append(preamble, lextwt.NewComment("# self = /conv/"+hash))
|
||||
|
||||
reg := lextwt.NewTwtRegistry(preamble, twts)
|
||||
reg.WriteTo(w)
|
||||
})
|
||||
|
||||
http.HandleFunc("/feeds", func(w http.ResponseWriter, r *http.Request) {
|
||||
http.HandleFunc("/users", func(w http.ResponseWriter, r *http.Request) {
|
||||
ctx, span := otel.Span(r.Context())
|
||||
defer span.End()
|
||||
|
||||
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||
|
||||
rows, err := db.QueryContext(
|
||||
ctx,
|
||||
`SELECT
|
||||
feed_id,
|
||||
uri,
|
||||
nick,
|
||||
last_scan_on
|
||||
FROM feeds
|
||||
where parent_id is null
|
||||
order by nick, uri`,
|
||||
)
|
||||
if err != nil {
|
||||
span.RecordError(err)
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var twts []types.Twt
|
||||
for rows.Next() {
|
||||
var o struct {
|
||||
FeedID string
|
||||
URI string
|
||||
Nick string
|
||||
Dt TwtTime
|
||||
}
|
||||
err = rows.Scan(&o.FeedID, &o.URI, &o.Nick, &o.Dt)
|
||||
if err != nil {
|
||||
span.RecordError(err)
|
||||
return
|
||||
}
|
||||
twts = append(twts, lextwt.NewTwt(
|
||||
types.NewTwter(o.Nick, o.URI),
|
||||
lextwt.NewDateTime(o.Dt.Time, o.Dt.Time.Format(time.RFC3339)),
|
||||
nil,
|
||||
))
|
||||
}
|
||||
reg := lextwt.NewTwtRegistry(nil, twts)
|
||||
reg.WriteTo(w)
|
||||
})
|
||||
|
||||
http.HandleFunc("/queue", func(w http.ResponseWriter, r *http.Request) {
|
||||
lis := slices.Collect(app.queue.Iter())
|
||||
sort.Slice(lis, func(i, j int) bool {
|
||||
return lis[i].LastScanOn.Time.Before(lis[j].LastScanOn.Time)
|
||||
@ -77,7 +168,7 @@ func httpServer(c *console, app *appState) error {
|
||||
c.AddCancel(srv.Shutdown)
|
||||
err = srv.ListenAndServe()
|
||||
if !errors.Is(err, http.ErrServerClosed) {
|
||||
otel.Error(err)
|
||||
span.RecordError(err)
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
12
init.sql
12
init.sql
@ -2,8 +2,10 @@ PRAGMA journal_mode=WAL;
|
||||
|
||||
create table if not exists feeds (
|
||||
feed_id blob primary key,
|
||||
uri text,
|
||||
parent_id blob,
|
||||
nick text,
|
||||
uri text,
|
||||
state string,
|
||||
last_scan_on timestamp,
|
||||
refresh_rate int default 600,
|
||||
last_modified_on timestamp,
|
||||
@ -13,12 +15,12 @@ create table if not exists feeds (
|
||||
|
||||
create table if not exists twts (
|
||||
feed_id blob,
|
||||
hash text,
|
||||
conv text,
|
||||
dt text, -- timestamp with timezone
|
||||
ulid blob,
|
||||
text text,
|
||||
conv text,
|
||||
hash text,
|
||||
mentions text, -- json
|
||||
tags text, -- json
|
||||
primary key (feed_id, hash)
|
||||
primary key (feed_id, ulid)
|
||||
);
|
||||
|
||||
|
||||
@ -17,13 +17,11 @@ import (
|
||||
"go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp"
|
||||
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
|
||||
"go.opentelemetry.io/otel/exporters/stdout/stdoutlog"
|
||||
"go.opentelemetry.io/otel/exporters/stdout/stdoutmetric"
|
||||
"go.opentelemetry.io/otel/exporters/stdout/stdouttrace"
|
||||
"go.opentelemetry.io/otel/log/global"
|
||||
"go.opentelemetry.io/otel/metric"
|
||||
"go.opentelemetry.io/otel/propagation"
|
||||
"go.opentelemetry.io/otel/sdk/log"
|
||||
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
|
||||
"go.opentelemetry.io/otel/sdk/resource"
|
||||
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
||||
semconv "go.opentelemetry.io/otel/semconv/v1.26.0"
|
||||
@ -45,19 +43,50 @@ func Init(ctx context.Context, name string) (shutdown func(context.Context) erro
|
||||
}
|
||||
|
||||
func Meter() metric.Meter { return meter }
|
||||
func Error(err error, v ...any) {
|
||||
// func Error(err error, v ...any) {
|
||||
// if err == nil {
|
||||
// return
|
||||
// }
|
||||
// fmt.Println("ERR:", append([]any{err}, v...))
|
||||
// logger.Error(err.Error(), v...)
|
||||
// }
|
||||
// func Info(msg string, v ...any) { fmt.Println(append([]any{msg}, v...)); logger.Info(msg, v...) }
|
||||
|
||||
type spanny struct{
|
||||
trace.Span
|
||||
}
|
||||
func (s *spanny) RecordError(err error, options ...trace.EventOption) {
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
logger.Error(err.Error(), v...)
|
||||
ec := trace.NewEventConfig(options...)
|
||||
|
||||
attrs := make([]any, len(ec.Attributes()))
|
||||
for i, v := range ec.Attributes() {
|
||||
attrs[i] = v
|
||||
}
|
||||
|
||||
fmt.Println(append([]any{"ERR:", err}, attrs...)...)
|
||||
logger.Error(err.Error(), attrs...)
|
||||
s.Span.RecordError(err, options...)
|
||||
}
|
||||
func Info(msg string, v ...any) { logger.Info(msg, v...) }
|
||||
func (s *spanny) AddEvent(name string, options ...trace.EventOption) {
|
||||
ec := trace.NewEventConfig(options...)
|
||||
|
||||
attrs := make([]any, len(ec.Attributes()))
|
||||
for i, v := range ec.Attributes() {
|
||||
attrs[i] = v
|
||||
}
|
||||
fmt.Println(append([]any{name}, attrs...)...)
|
||||
logger.Info(name, attrs...)
|
||||
}
|
||||
|
||||
func Span(ctx context.Context, opts ...trace.SpanStartOption) (context.Context, trace.Span) {
|
||||
name, attrs := Attrs()
|
||||
ctx, span := tracer.Start(ctx, name, opts...)
|
||||
span.SetAttributes(attrs...)
|
||||
|
||||
return ctx, span
|
||||
return ctx, &spanny{span}
|
||||
}
|
||||
func Attrs() (string, []attribute.KeyValue) {
|
||||
var attrs []attribute.KeyValue
|
||||
@ -192,17 +221,17 @@ func newTraceProvider(ctx context.Context, name string) (func(context.Context) e
|
||||
}
|
||||
|
||||
func newMeterProvider(ctx context.Context, name string) (func(context.Context) error, error) {
|
||||
metricExporter, err := stdoutmetric.New()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// metricExporter, err := stdoutmetric.New()
|
||||
// if err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
|
||||
meterProvider := sdkmetric.NewMeterProvider(
|
||||
sdkmetric.WithReader(sdkmetric.NewPeriodicReader(metricExporter,
|
||||
// Default is 1m. Set to 3s for demonstrative purposes.
|
||||
sdkmetric.WithInterval(3*time.Second))),
|
||||
)
|
||||
otel.SetMeterProvider(meterProvider)
|
||||
// meterProvider := sdkmetric.NewMeterProvider(
|
||||
// sdkmetric.WithReader(sdkmetric.NewPeriodicReader(metricExporter,
|
||||
// // Default is 1m. Set to 3s for demonstrative purposes.
|
||||
// sdkmetric.WithInterval(3*time.Second))),
|
||||
// )
|
||||
// otel.SetMeterProvider(meterProvider)
|
||||
|
||||
http.Handle("/metrics", promhttp.Handler())
|
||||
return func(ctx context.Context) error { return nil }, nil
|
||||
|
||||
6
main.go
6
main.go
@ -8,7 +8,6 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
"os/signal"
|
||||
"runtime/debug"
|
||||
"strings"
|
||||
|
||||
"go.opentelemetry.io/otel/metric"
|
||||
@ -27,7 +26,7 @@ func main() {
|
||||
dbfile: env("XT_DBFILE", "file:twt.db"),
|
||||
baseFeed: env("XT_BASE_FEED", "feed"),
|
||||
Nick: env("XT_NICK", "xuu"),
|
||||
URI: env("XT_URI", "https://txt.sour.is/users/xuu/twtxt.txt"),
|
||||
URI: env("XT_URI", "https://txt.sour.is/user/xuu/twtxt.txt"),
|
||||
Listen: env("XT_LISTEN", ":8080"),
|
||||
})
|
||||
|
||||
@ -41,9 +40,6 @@ func main() {
|
||||
m_up.Record(ctx, 1)
|
||||
defer m_up.Record(context.Background(), 0)
|
||||
|
||||
bi, _ := debug.ReadBuildInfo()
|
||||
otel.Info(name, "version", bi.Main.Version)
|
||||
|
||||
err = run(console)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
console.IfFatal(err)
|
||||
|
||||
135
refresh-loop.go
135
refresh-loop.go
@ -1,14 +1,16 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
"go.sour.is/xt/internal/otel"
|
||||
"go.yarn.social/lextwt"
|
||||
"go.yarn.social/types"
|
||||
@ -27,61 +29,91 @@ func refreshLoop(c *console, app *appState) error {
|
||||
defer span.End()
|
||||
|
||||
f := NewHTTPFetcher()
|
||||
fetch, close := NewFuncPool(c.Context, 25, f.Fetch)
|
||||
fetch, close := NewFuncPool(ctx, 25, f.Fetch)
|
||||
defer close()
|
||||
|
||||
db, err := app.DB()
|
||||
db, err := app.DB(c)
|
||||
if err != nil {
|
||||
otel.Error(err, "missing db")
|
||||
span.RecordError(err)
|
||||
return err
|
||||
}
|
||||
|
||||
go processorLoop(ctx, db, fetch)
|
||||
|
||||
queue := app.queue
|
||||
|
||||
otel.Info("start refresh loop")
|
||||
for c.Context.Err() == nil {
|
||||
if queue.IsEmpty() {
|
||||
otel.Info("load feeds")
|
||||
span.AddEvent("start refresh loop")
|
||||
|
||||
it, err := loadFeeds(c.Context, db)
|
||||
for ctx.Err() == nil {
|
||||
if queue.IsEmpty() {
|
||||
span.AddEvent("load feeds")
|
||||
|
||||
it, err := loadFeeds(ctx, db)
|
||||
span.RecordError(err)
|
||||
for f := range it {
|
||||
queue.Insert(&f)
|
||||
}
|
||||
if err != nil {
|
||||
otel.Error(err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
span.AddEvent("queue size", trace.WithAttributes(attribute.Int("size", int(queue.count))))
|
||||
|
||||
f := queue.ExtractMin()
|
||||
if f == nil {
|
||||
otel.Info("sleeping for ", TenMinutes*time.Second)
|
||||
span.AddEvent("sleeping for ", trace.WithAttributes(attribute.Int("seconds", int(TenMinutes))))
|
||||
select {
|
||||
case <-time.After(TenMinutes * time.Second):
|
||||
|
||||
case <-c.Done():
|
||||
return nil
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
otel.Info("queue size", queue.count, "next", f.URI, "next scan on", f.LastScanOn.Time.Format(time.RFC3339))
|
||||
span.AddEvent("next", trace.WithAttributes(
|
||||
attribute.Int("size", int(queue.count)),
|
||||
attribute.String("uri", f.URI),
|
||||
attribute.String("scan on", f.LastScanOn.Time.Format(time.RFC3339)),
|
||||
))
|
||||
|
||||
if time.Until(f.LastScanOn.Time) > 2*time.Hour {
|
||||
otel.Info("too soon", f.URI)
|
||||
if time.Until(f.NextScanOn.Time) > 2*time.Hour {
|
||||
span.AddEvent("too soon", trace.WithAttributes(attribute.String("uri", f.URI)))
|
||||
continue
|
||||
}
|
||||
|
||||
select {
|
||||
case <-c.Done():
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case t := <-time.After(time.Until(f.LastScanOn.Time)):
|
||||
otel.Info("fetch", t.Format(time.RFC3339), f.Nick, f.URI)
|
||||
case t := <-time.After(time.Until(f.NextScanOn.Time)):
|
||||
span.AddEvent("fetch", trace.WithAttributes(
|
||||
attribute.Int("size", int(queue.count)),
|
||||
attribute.String("uri", f.URI),
|
||||
attribute.String("timeout", t.Format(time.RFC3339)),
|
||||
attribute.String("scan on", f.NextScanOn.Time.Format(time.RFC3339)),
|
||||
))
|
||||
|
||||
fetch.Fetch(f)
|
||||
}
|
||||
|
||||
}
|
||||
span.RecordError(ctx.Err())
|
||||
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
func processorLoop(ctx context.Context, db db, fetch *pool[*Feed, *Response]) {
|
||||
ctx, span := otel.Span(ctx)
|
||||
defer span.End()
|
||||
|
||||
for ctx.Err() == nil {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case res := <-fetch.Out():
|
||||
otel.Info("got response:", res.Request.URI)
|
||||
f := res.Request
|
||||
span.AddEvent("got response", trace.WithAttributes(
|
||||
attribute.String("uri", f.URI),
|
||||
attribute.String("scan on", f.NextScanOn.Time.Format(time.RFC3339)),
|
||||
))
|
||||
|
||||
f.LastScanOn.Time = time.Now()
|
||||
f.LastScanOn.Valid = true
|
||||
err := res.err
|
||||
if res.err != nil {
|
||||
if errors.Is(err, ErrPermanentlyDead) {
|
||||
@ -94,12 +126,10 @@ func refreshLoop(c *console, app *appState) error {
|
||||
f.RefreshRate = OneDay
|
||||
}
|
||||
|
||||
otel.Error(err)
|
||||
span.RecordError(err)
|
||||
f.LastError.String, f.LastError.Valid = err.Error(), true
|
||||
err = f.Save(c.Context, db)
|
||||
if err != nil {
|
||||
otel.Error(err)
|
||||
}
|
||||
err = f.Save(ctx, db)
|
||||
span.RecordError(err)
|
||||
|
||||
continue
|
||||
}
|
||||
@ -107,15 +137,16 @@ func refreshLoop(c *console, app *appState) error {
|
||||
f.ETag.String, f.ETag.Valid = res.ETag(), true
|
||||
f.LastModified.Time, f.LastModified.Valid = res.LastModified(), true
|
||||
|
||||
span.AddEvent("read feed")
|
||||
cpy, err := os.OpenFile(filepath.Join("feeds", urlNS.UUID5(f.URI).MarshalText()), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
|
||||
if err != nil {
|
||||
otel.Error(fmt.Errorf("%w: %w", ErrParseFailed, err))
|
||||
span.RecordError(fmt.Errorf("%w: %w", ErrParseFailed, err))
|
||||
|
||||
f.LastError.String, f.LastError.Valid = err.Error(), true
|
||||
f.RefreshRate = OneDay
|
||||
|
||||
err = f.Save(c.Context, db)
|
||||
otel.Error(err)
|
||||
err = f.Save(ctx, db)
|
||||
span.RecordError(err)
|
||||
|
||||
continue
|
||||
}
|
||||
@ -123,55 +154,37 @@ func refreshLoop(c *console, app *appState) error {
|
||||
rdr = lextwt.TwtFixer(rdr)
|
||||
twtfile, err := lextwt.ParseFile(rdr, &types.Twter{Nick: f.Nick, URI: f.URI})
|
||||
if err != nil {
|
||||
otel.Error(fmt.Errorf("%w: %w", ErrParseFailed, err))
|
||||
span.RecordError(fmt.Errorf("%w: %w", ErrParseFailed, err))
|
||||
|
||||
f.LastError.String, f.LastError.Valid = err.Error(), true
|
||||
f.RefreshRate = OneDay
|
||||
|
||||
err = f.Save(c.Context, db)
|
||||
otel.Error(err)
|
||||
err = f.Save(ctx, db)
|
||||
span.RecordError(err)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
if prev, ok := twtfile.Info().GetN("prev", 0); f.FirstFetch && ok {
|
||||
_, part, ok := strings.Cut(prev.Value(), " ")
|
||||
if ok {
|
||||
part = f.URI[:strings.LastIndex(f.URI, "/")+1] + part
|
||||
queue.Insert(&Feed{
|
||||
FetchURI: part,
|
||||
URI: f.URI,
|
||||
Nick: f.Nick,
|
||||
LastScanOn: f.LastScanOn,
|
||||
RefreshRate: f.RefreshRate,
|
||||
})
|
||||
}
|
||||
}
|
||||
cpy.Close()
|
||||
span.AddEvent("parse complete", trace.WithAttributes(attribute.Int("count", twtfile.Twts().Len())))
|
||||
|
||||
err = storeFeed(ctx, db, twtfile)
|
||||
if err != nil {
|
||||
otel.Error(err)
|
||||
span.RecordError(err)
|
||||
|
||||
f.LastError.String, f.LastError.Valid = err.Error(), true
|
||||
err = f.Save(c.Context, db)
|
||||
err = f.Save(ctx, db)
|
||||
|
||||
otel.Error(err)
|
||||
return err
|
||||
span.RecordError(err)
|
||||
continue
|
||||
}
|
||||
|
||||
cpy.Close()
|
||||
|
||||
f.LastScanOn.Time = time.Now()
|
||||
f.RefreshRate = TenMinutes
|
||||
f.LastError.String = ""
|
||||
|
||||
err = f.Save(c.Context, db)
|
||||
if err != nil {
|
||||
otel.Error(err)
|
||||
return err
|
||||
}
|
||||
err = f.Save(ctx, db)
|
||||
span.RecordError(err)
|
||||
}
|
||||
}
|
||||
|
||||
return c.Context.Err()
|
||||
span.RecordError(ctx.Err())
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user