From ed5b43300be055584abe2df71082ab96cc29f329 Mon Sep 17 00:00:00 2001 From: xuu Date: Mon, 24 Feb 2025 17:28:09 -0700 Subject: [PATCH 01/13] feat: add otel --- .gitignore | 2 + feed.go | 30 +++-- go.mod | 46 ++++++- go.sum | 96 +++++++++++++-- http.go | 33 +++-- internal/otel/otel.go | 272 ++++++++++++++++++++++++++++++++++++++++++ main.go | 121 ++++++++++++++----- otel.go | 2 + refresh-loop.go | 81 ++++++++----- service.go | 40 +++++-- 10 files changed, 612 insertions(+), 111 deletions(-) create mode 100644 internal/otel/otel.go create mode 100644 otel.go diff --git a/.gitignore b/.gitignore index 24a5a9f..1949f0b 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ feed __debug* feeds/ +/xt +.env diff --git a/feed.go b/feed.go index 51a0d8a..27b04bd 100644 --- a/feed.go +++ b/feed.go @@ -14,6 +14,7 @@ import ( _ "embed" + "go.sour.is/xt/internal/otel" "go.yarn.social/lextwt" "go.yarn.social/types" ) @@ -76,7 +77,10 @@ var ( last_modified_on, last_etag from feeds - where datetime(last_scan_on, '+'||refresh_rate||' seconds') < datetime(current_timestamp, '+10 minutes') + where datetime( + coalesce(last_scan_on, '1901-01-01'), + '+'||refresh_rate||' seconds' + ) < datetime(current_timestamp, '+10 minutes') ` updateFeed = ` update feeds set @@ -90,7 +94,9 @@ var ( ) func (f *Feed) Save(ctx context.Context, db *sql.DB) error { - fmt.Println(f.FetchURI, " ", f.LastModified, " ", f.LastError) + ctx, span := otel.Span(ctx) + defer span.End() + _, err := db.ExecContext( ctx, updateFeed, @@ -135,6 +141,8 @@ func (f *Feed) Scan(res interface{ Scan(...any) error }) error { } func loadFeeds(ctx context.Context, db *sql.DB) (iter.Seq[Feed], error) { + ctx, span := otel.Span(ctx) + var err error var res *sql.Rows @@ -145,6 +153,8 @@ func loadFeeds(ctx context.Context, db *sql.DB) (iter.Seq[Feed], error) { } return func(yield func(Feed) bool) { + defer span.End() + for res.Next() { var f Feed err = f.Scan(res) @@ -158,13 +168,16 @@ func loadFeeds(ctx context.Context, db *sql.DB) (iter.Seq[Feed], error) { }, err } -func storeFeed(db *sql.DB, f types.TwtFile) error { +func storeFeed(ctx context.Context, db *sql.DB, f types.TwtFile) error { + ctx, span := otel.Span(ctx) + defer span.End() + loadTS := time.Now() refreshRate := 600 feedID := urlNS.UUID5(cmp.Or(f.Twter().HashingURI, f.Twter().URI)) - tx, err := db.Begin() + tx, err := db.BeginTx(ctx, nil) if err != nil { return err } @@ -188,7 +201,8 @@ func storeFeed(db *sql.DB, f types.TwtFile) error { defer tx.Rollback() - _, err = tx.Exec( + _, err = tx.ExecContext( + ctx, insertFeed, feedID, f.Twter().HashingURI, @@ -220,7 +234,8 @@ func storeFeed(db *sql.DB, f types.TwtFile) error { } } - _, err = tx.Exec( + _, err = tx.ExecContext( + ctx, insertTwt, feedID, twt.Hash(), @@ -236,7 +251,8 @@ func storeFeed(db *sql.DB, f types.TwtFile) error { } for nick, uri := range followMap { - _, err = tx.Exec( + _, err = tx.ExecContext( + ctx, insertFeed, urlNS.UUID5(uri), uri, diff --git a/go.mod b/go.mod index 358000c..f453882 100644 --- a/go.mod +++ b/go.mod @@ -4,25 +4,59 @@ go 1.23.2 require ( github.com/mattn/go-sqlite3 v1.14.24 - go.yarn.social/lextwt v0.0.0-20240908172157-7b9ae633db51 + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.6.0 + go.yarn.social/lextwt v0.0.0-20250213063805-7adc6ca07564 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.25.1 // indirect + github.com/klauspost/compress v1.17.9 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 // indirect + go.opentelemetry.io/proto/otlp v1.5.0 // indirect + golang.org/x/net v0.34.0 // indirect + golang.org/x/text v0.22.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250115164207-1a7da9e5054f // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect + google.golang.org/protobuf v1.36.3 // indirect ) require ( github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/uuid v1.6.0 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect - go.opentelemetry.io/otel/metric v1.34.0 // indirect - go.opentelemetry.io/otel/trace v1.34.0 // indirect + go.opentelemetry.io/otel/log v0.10.0 + go.opentelemetry.io/otel/metric v1.34.0 + go.opentelemetry.io/otel/trace v1.34.0 ) require ( github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/matryer/is v1.4.1 + github.com/prometheus/client_golang v1.20.5 github.com/sirupsen/logrus v1.9.3 // indirect + github.com/uptrace/opentelemetry-go-extra/otelsql v0.3.2 github.com/writeas/go-strip-markdown/v2 v2.1.1 // indirect + go.opentelemetry.io/contrib/bridges/otelslog v0.9.0 go.opentelemetry.io/otel v1.34.0 - go.yarn.social/types v0.0.0-20230305013457-e4d91e351ac8 - golang.org/x/crypto v0.27.0 // indirect - golang.org/x/sys v0.25.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 + go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.10.0 + go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.34.0 + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.34.0 + go.opentelemetry.io/otel/sdk v1.34.0 + go.opentelemetry.io/otel/sdk/log v0.10.0 + go.opentelemetry.io/otel/sdk/metric v1.34.0 // indirect + go.yarn.social/types v0.0.0-20250108134258-ed75fa653ede + golang.org/x/crypto v0.33.0 // indirect + golang.org/x/sync v0.11.0 + golang.org/x/sys v0.30.0 // indirect + google.golang.org/grpc v1.70.0 // indirect ) diff --git a/go.sum b/go.sum index 0f9aef5..2ca56a6 100644 --- a/go.sum +++ b/go.sum @@ -1,46 +1,118 @@ -github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= -github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.25.1 h1:VNqngBF40hVlDloBruUehVYC3ArSgIyScOAyMRqBxRg= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.25.1/go.mod h1:RBRO7fro65R6tjKzYgLAFo0t1QEXY1Dp+i/bvpRiqiQ= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/matryer/is v1.4.1 h1:55ehd8zaGABKLXQUe2awZ99BD/PTc2ls+KV/dXphgEQ= github.com/matryer/is v1.4.1/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/uptrace/opentelemetry-go-extra/otelsql v0.3.2 h1:ZjUj9BLYf9PEqBn8W/OapxhPjVRdC6CsXTdULHsyk5c= +github.com/uptrace/opentelemetry-go-extra/otelsql v0.3.2/go.mod h1:O8bHQfyinKwTXKkiKNGmLQS7vRsqRxIQTFZpYpHK3IQ= github.com/writeas/go-strip-markdown/v2 v2.1.1 h1:hAxUM21Uhznf/FnbVGiJciqzska6iLei22Ijc3q2e28= github.com/writeas/go-strip-markdown/v2 v2.1.1/go.mod h1:UvvgPJgn1vvN8nWuE5e7v/+qmDu3BSVnKAB6Gl7hFzA= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/bridges/otelslog v0.9.0 h1:N+78eXSlu09kii5nkiM+01YbtWe01oZLPPLhNlEKhus= +go.opentelemetry.io/contrib/bridges/otelslog v0.9.0/go.mod h1:/2KhfLAhtQpgnhIk1f+dftA3fuuMcZjiz//Dc9yfaEs= go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY= go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.6.0 h1:QSKmLBzbFULSyHzOdO9JsN9lpE4zkrz1byYGmJecdVE= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.6.0/go.mod h1:sTQ/NH8Yrirf0sJ5rWqVu+oT82i4zL9FaF6rWcqnptM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 h1:OeNbIYk/2C15ckl7glBlOBp5+WlYsOElzTNmiPW/x60= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0/go.mod h1:7Bept48yIeqxP2OZ9/AqIpYS94h2or0aB4FypJTc8ZM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 h1:tgJ0uaNS4c98WRNUEx5U3aDlrDOI5Rs+1Vifcw4DJ8U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0/go.mod h1:U7HYyW0zt/a9x5J1Kjs+r1f/d4ZHnYFclhYY2+YbeoE= +go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.10.0 h1:GKCEAZLEpEf78cUvudQdTg0aET2ObOZRB2HtXA0qPAI= +go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.10.0/go.mod h1:9/zqSWLCmHT/9Jo6fYeUDRRogOLL60ABLsHWS99lF8s= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.34.0 h1:czJDQwFrMbOr9Kk+BPo1y8WZIIFIK58SA1kykuVeiOU= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.34.0/go.mod h1:lT7bmsxOe58Tq+JIOkTQMCGXdu47oA+VJKLZHbaBKbs= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.34.0 h1:jBpDk4HAUsrnVO1FsfCfCOTEc/MkInJmvfCHYLFiT80= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.34.0/go.mod h1:H9LUIM1daaeZaz91vZcfeM0fejXPmgCYE8ZhzqfJuiU= +go.opentelemetry.io/otel/log v0.10.0 h1:1CXmspaRITvFcjA4kyVszuG4HjA61fPDxMb7q3BuyF0= +go.opentelemetry.io/otel/log v0.10.0/go.mod h1:PbVdm9bXKku/gL0oFfUF4wwsQsOPlpo4VEqjvxih+FM= go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ= go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE= +go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= +go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= +go.opentelemetry.io/otel/sdk/log v0.10.0 h1:lR4teQGWfeDVGoute6l0Ou+RpFqQ9vaPdrNJlST0bvw= +go.opentelemetry.io/otel/sdk/log v0.10.0/go.mod h1:A+V1UTWREhWAittaQEG4bYm4gAZa6xnvVu+xKrIRkzo= +go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= +go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= -go.uber.org/ratelimit v0.3.1 h1:K4qVE+byfv/B3tC+4nYWP7v/6SimcO7HzHekoMNBma0= -go.uber.org/ratelimit v0.3.1/go.mod h1:6euWsTB6U/Nb3X++xEUXA8ciPJvr19Q/0h1+oDcJhRk= -go.yarn.social/lextwt v0.0.0-20240908172157-7b9ae633db51 h1:XEjx0jSNv1h22gwGfQBfMypWv/YZXWGTRbqh3B8xfIs= -go.yarn.social/lextwt v0.0.0-20240908172157-7b9ae633db51/go.mod h1:CWAZuBHZfGaqa0FreSeLG+pzK3rHP2TNAG7Zh6QlRiM= -go.yarn.social/types v0.0.0-20230305013457-e4d91e351ac8 h1:zfnniiSO/WO65mSpdQzGYJ9pM0rYg/BKgrSm8h2mTyA= -go.yarn.social/types v0.0.0-20230305013457-e4d91e351ac8/go.mod h1:+xnDkQ0T0S8emxWIsvxlCAoyF8gBaj0q81hr/VrKc0c= -golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= -golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= +go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= +go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yarn.social/lextwt v0.0.0-20250213063805-7adc6ca07564 h1:z+IAMtxNKWcLNm9nLzJwHw6OPkV5JoQYmmFohaUvcKI= +go.yarn.social/lextwt v0.0.0-20250213063805-7adc6ca07564/go.mod h1:JOPCOh+3bHv+BMaFZpKzw6soiXbIlZD5b2f7YKDDjqk= +go.yarn.social/types v0.0.0-20250108134258-ed75fa653ede h1:XV9tuDQ605xxH4qIQPRHM1bOa7k0rJZ2RqA5kz2Nun4= +go.yarn.social/types v0.0.0-20250108134258-ed75fa653ede/go.mod h1:+xnDkQ0T0S8emxWIsvxlCAoyF8gBaj0q81hr/VrKc0c= +golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus= +golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= +golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= +golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= +golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= +golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= -golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= +google.golang.org/genproto/googleapis/api v0.0.0-20250115164207-1a7da9e5054f h1:gap6+3Gk41EItBuyi4XX/bp4oqJ3UwuIMl25yGinuAA= +google.golang.org/genproto/googleapis/api v0.0.0-20250115164207-1a7da9e5054f/go.mod h1:Ic02D47M+zbarjYYUlK57y316f2MoN0gjAwI3f2S95o= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f h1:OxYkA3wjPsZyBylwymxSHa7ViiW1Sml4ToBrncvFehI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f/go.mod h1:+2Yz8+CLJbIfL9z73EW45avw8Lmge3xVElCP9zEKi50= +google.golang.org/grpc v1.70.0 h1:pWFv03aZoHzlRKHWicjsZytKAiYCtNS0dHbXnIdq7jQ= +google.golang.org/grpc v1.70.0/go.mod h1:ofIJqVKDXx/JiXrwr2IG4/zwdH9txy3IlF40RmcJSQw= +google.golang.org/protobuf v1.36.3 h1:82DV7MYdb8anAVi3qge1wSnMDrnKK7ebr+I0hHRN1BU= +google.golang.org/protobuf v1.36.3/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/http.go b/http.go index 93514a9..0069cee 100644 --- a/http.go +++ b/http.go @@ -1,23 +1,24 @@ package main import ( - "context" + "errors" "fmt" "net/http" "slices" "sort" "strings" "time" + + "go.sour.is/xt/internal/otel" ) -func httpServer(c console, app *appState) { - c.Log("start http server") +func httpServer(c *console, app *appState) error { + otel.Info("start http server") db, err := app.DB() if err != nil { - c.Log("missing db", err) - c.abort() - return + otel.Info("missing db", err) + return err } http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { @@ -37,7 +38,7 @@ func httpServer(c console, app *appState) { rows, err := db.QueryContext(r.Context(), "SELECT feed_id, hash, conv, dt, text FROM twt WHERE hash = $1 or conv = $1", hash) if err != nil { - c.Log(err) + otel.Info("error", err) return } defer rows.Close() @@ -52,7 +53,7 @@ func httpServer(c console, app *appState) { } err = rows.Scan(&twt.FeedID, &twt.Hash, &twt.Conv, &twt.Dt, &twt.Text) if err != nil { - c.Log(err) + otel.Error(err) return } } @@ -73,18 +74,14 @@ func httpServer(c console, app *appState) { Handler: http.DefaultServeMux, } - go func() { - <-c.Done() - c.Log("stop http server") - srv.Shutdown(context.Background()) - }() - + c.AddCancel(srv.Shutdown) err = srv.ListenAndServe() - if err != nil { - c.Log(err) - c.abort() - return + if !errors.Is(err, http.ErrServerClosed) { + otel.Error(err) + return err } + + return nil } func notAny(s string, chars string) bool { diff --git a/internal/otel/otel.go b/internal/otel/otel.go new file mode 100644 index 0000000..a35e945 --- /dev/null +++ b/internal/otel/otel.go @@ -0,0 +1,272 @@ +package otel + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net/http" + "os" + "runtime" + "time" + + "github.com/prometheus/client_golang/prometheus/promhttp" + "go.opentelemetry.io/contrib/bridges/otelslog" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + "go.opentelemetry.io/otel/exporters/stdout/stdoutlog" + "go.opentelemetry.io/otel/exporters/stdout/stdoutmetric" + "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" + "go.opentelemetry.io/otel/log/global" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/log" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.26.0" + "go.opentelemetry.io/otel/trace" +) + +var ( + tracer trace.Tracer + meter metric.Meter + logger *slog.Logger +) + +func Init(ctx context.Context, name string) (shutdown func(context.Context) error, err error) { + tracer = otel.Tracer(name) + meter = otel.Meter(name) + logger = otelslog.NewLogger(name) + + return setupOTelSDK(ctx, name) +} + +func Meter() metric.Meter { return meter } +func Error(err error, v ...any) { + if err == nil { + return + } + logger.Error(err.Error(), v...) +} +func Info(msg string, v ...any) { logger.Info(msg, v...) } +func Span(ctx context.Context, opts ...trace.SpanStartOption) (context.Context, trace.Span) { + name, attrs := Attrs() + ctx, span := tracer.Start(ctx, name, opts...) + span.SetAttributes(attrs...) + + return ctx, span +} +func Attrs() (string, []attribute.KeyValue) { + var attrs []attribute.KeyValue + var name string + if pc, file, line, ok := runtime.Caller(2); ok { + if fn := runtime.FuncForPC(pc); fn != nil { + name = fn.Name() + } + attrs = append(attrs, + attribute.String("pc", fmt.Sprintf("%v", pc)), + attribute.String("file", file), + attribute.Int("line", line), + ) + } + return name, attrs +} + +// setupOTelSDK bootstraps the OpenTelemetry pipeline. +// If it does not return an error, make sure to call shutdown for proper cleanup. +func setupOTelSDK(ctx context.Context, name string) (shutdown func(context.Context) error, err error) { + var shutdownFuncs []func(context.Context) error + + // shutdown calls cleanup functions registered via shutdownFuncs. + // The errors from the calls are joined. + // Each registered cleanup will be invoked once. + shutdown = func(ctx context.Context) error { + fmt.Println("shutdown") + var err error + for _, fn := range shutdownFuncs { + err = errors.Join(err, fn(ctx)) + } + shutdownFuncs = nil + return err + } + + // playShutdown := otelplay.ConfigureOpentelemetry(ctx) + // shutdownFuncs = append(shutdownFuncs, func(ctx context.Context) error { playShutdown(); return nil }) + + // handleErr calls shutdown for cleanup and makes sure that all errors are returned. + handleErr := func(inErr error) { + err = errors.Join(inErr, shutdown(ctx)) + } + + // Set up propagator. + prop := newPropagator() + otel.SetTextMapPropagator(prop) + + // Set up trace provider. + tracerShutdown, err := newTraceProvider(ctx, name) + if err != nil { + handleErr(err) + return + } + shutdownFuncs = append(shutdownFuncs, tracerShutdown) + + // Set up meter provider. + meterShutdown, err := newMeterProvider(ctx, name) + if err != nil { + handleErr(err) + return + } + shutdownFuncs = append(shutdownFuncs, meterShutdown) + + // Set up logger provider. + loggerShutdown, err := newLoggerProvider(ctx, name) + if err != nil { + handleErr(err) + return + } + shutdownFuncs = append(shutdownFuncs, loggerShutdown) + + return +} + +func newPropagator() propagation.TextMapPropagator { + return propagation.NewCompositeTextMapPropagator( + propagation.TraceContext{}, + propagation.Baggage{}, + ) +} + +func newTraceProvider(ctx context.Context, name string) (func(context.Context) error, error) { + r, err := resource.Merge( + resource.Default(), + resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceName(name), + ), + ) + if err != nil { + return nil, err + } + + if v := env("XT_TRACER", ""); v != "" { + fmt.Println("use tracer", v) + exp, err := otlptracegrpc.New( + ctx, + otlptracegrpc.WithEndpoint(v), + otlptracegrpc.WithInsecure(), + ) + if err != nil { + return nil, err + } + + tracerProvider := sdktrace.NewTracerProvider( + sdktrace.WithBatcher(exp), + sdktrace.WithResource(r), + ) + otel.SetTracerProvider(tracerProvider) + return func(ctx context.Context) error { + return tracerProvider.Shutdown(ctx) + }, nil + } + + traceExporter, err := stdouttrace.New( + stdouttrace.WithWriter(os.Stderr), + stdouttrace.WithPrettyPrint(), + ) + if err != nil { + return nil, err + } + + tracerProvider := sdktrace.NewTracerProvider( + sdktrace.WithResource(r), + sdktrace.WithBatcher(traceExporter, + // Default is 5s. Set to 1s for demonstrative purposes. + sdktrace.WithBatchTimeout(time.Second)), + ) + otel.SetTracerProvider(tracerProvider) + + return tracerProvider.Shutdown, nil +} + +func newMeterProvider(ctx context.Context, name string) (func(context.Context) error, error) { + metricExporter, err := stdoutmetric.New() + if err != nil { + return nil, err + } + + meterProvider := sdkmetric.NewMeterProvider( + sdkmetric.WithReader(sdkmetric.NewPeriodicReader(metricExporter, + // Default is 1m. Set to 3s for demonstrative purposes. + sdkmetric.WithInterval(3*time.Second))), + ) + otel.SetMeterProvider(meterProvider) + + http.Handle("/metrics", promhttp.Handler()) + return func(ctx context.Context) error { return nil }, nil +} + +func newLoggerProvider(ctx context.Context, name string) (func(context.Context) error, error) { + r, err := resource.Merge( + resource.Default(), + resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceName(name), + ), + ) + if err != nil { + return nil, err + } + + if v := env("XT_LOGGER", ""); v != "" { + fmt.Println("use logger", v) + + exp, err := otlploghttp.New( + ctx, + otlploghttp.WithInsecure(), + otlploghttp.WithEndpointURL(v), + ) + if err != nil { + return nil, err + } + + processor := log.NewBatchProcessor(exp) + provider := log.NewLoggerProvider( + log.WithProcessor(processor), + log.WithResource(r), + ) + global.SetLoggerProvider(provider) + + return processor.Shutdown, nil + + } + + // return func(ctx context.Context) error { return nil }, nil + + logExporter, err := stdoutlog.New( + stdoutlog.WithPrettyPrint(), + stdoutlog.WithWriter(os.Stderr), + ) + if err != nil { + return nil, err + } + + loggerProvider := log.NewLoggerProvider( + log.WithProcessor( + log.NewBatchProcessor(logExporter), + ), + log.WithResource(r), + ) + global.SetLoggerProvider(loggerProvider) + + return loggerProvider.Shutdown, nil +} + +func env(key, def string) string { + if v, ok := os.LookupEnv(key); ok { + return v + } + return def +} diff --git a/main.go b/main.go index 26e95ef..03c0f6d 100644 --- a/main.go +++ b/main.go @@ -1,45 +1,112 @@ package main import ( + "bufio" "context" "errors" "fmt" "io" "os" "os/signal" + "runtime/debug" + "strings" - "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/metric" + "go.sour.is/xt/internal/otel" ) const name = "go.sour.is/xt" -var ( - tracer = otel.Tracer(name) - meter = otel.Meter(name) -) +var m_up metric.Int64Gauge -type contextKey struct{ name string } +func main() { + dotEnv() // load .env + + ctx, console := newConsole(args{ + dbtype: env("XT_DBTYPE", "sqlite3"), + dbfile: env("XT_DBFILE", "file:twt.db"), + baseFeed: env("XT_BASE_FEED", "feed"), + Nick: env("XT_NICK", "xuu"), + URI: env("XT_URI", "https://txt.sour.is/users/xuu/twtxt.txt"), + Listen: env("XT_LISTEN", ":8080"), + }) + + finish, err := otel.Init(ctx, name) + console.IfFatal(err) + console.AddCancel(finish) + + m_up, err = otel.Meter().Int64Gauge("up") + console.IfFatal(err) + + m_up.Record(ctx, 1) + defer m_up.Record(context.Background(), 0) + + bi, _ := debug.ReadBuildInfo() + otel.Info(name, "version", bi.Main.Version) + + err = run(console) + if !errors.Is(err, context.Canceled) { + console.IfFatal(err) + } +} type console struct { io.Reader io.Writer err io.Writer context.Context - abort func() + abort func() + cancelfns []func(context.Context) error } -func (c console) Log(v ...any) { fmt.Fprintln(c.err, v...) } -func (c console) Args() args { +func newConsole(args args) (context.Context, *console) { + ctx := context.Background() + ctx, abort := context.WithCancel(ctx) + ctx, stop := signal.NotifyContext(ctx, os.Interrupt) + go func() { <-ctx.Done(); stop() }() // restore interrupt function + + console := &console{Reader: os.Stdin, Writer: os.Stdout, err: os.Stderr, Context: ctx, abort: abort} + console.Set("console", console) + console.Set("args", args) + return ctx, console +} + +func (c *console) Args() args { v, ok := c.Get("args").(args) if !ok { return args{} } return v } +func (c *console) Shutdown() error { + fmt.Fprintln(c.err, "shutting down ", len(c.cancelfns), " cancel functions...") + defer fmt.Fprintln(c.err, "done") + + c.abort() + var err error + for _, fn := range c.cancelfns { + err = errors.Join(err, fn(c.Context)) + } + return err +} +func (c *console) AddCancel(fn func(context.Context) error) { c.cancelfns = append(c.cancelfns, fn) } + +func (c *console) IfFatal(err error) { + if err == nil { + return + } + fmt.Fprintln(c.err, err) + c.abort() + os.Exit(1) +} + +type contextKey struct{ name string } + func (c *console) Set(name string, value any) { c.Context = context.WithValue(c.Context, contextKey{name}, value) } -func (c console) Get(name string) any { + +func (c *console) Get(name string) any { return c.Context.Value(contextKey{name}) } @@ -59,25 +126,25 @@ func env(key, def string) string { return def } -func main() { - ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) - console := console{os.Stdin, os.Stdout, os.Stderr, ctx, stop} - - go func() { <-ctx.Done(); console.Log("shutdown"); stop() }() - - args := args{ - dbtype: env("XT_DBTYPE", "sqlite3"), - dbfile: env("XT_DBFILE", "file:twt.db"), - baseFeed: env("XT_BASE_FEED", "feed"), - Nick: env("XT_NICK", "xuu"), - URI: env("XT_URI", "https://txt.sour.is/users/xuu/twtxt.txt"), - Listen: env("XT_LISTEN", ":8040"), +func dotEnv() { + fd, err := os.Open(".env") + if err != nil { + return } - console.Set("args", args) + scan := bufio.NewScanner(fd) - if err := run(console); err != nil && !errors.Is(err, context.Canceled) { - fmt.Println(err) - os.Exit(1) + for scan.Scan() { + line := scan.Text() + + if strings.HasPrefix(line, "#") { + continue + } + key, val, ok := strings.Cut(line, "=") + if !ok { + continue + } + + os.Setenv(strings.TrimSpace(key), strings.TrimSpace(val)) } } diff --git a/otel.go b/otel.go new file mode 100644 index 0000000..c9ecbf5 --- /dev/null +++ b/otel.go @@ -0,0 +1,2 @@ +package main + diff --git a/refresh-loop.go b/refresh-loop.go index 1fe5e30..9905502 100644 --- a/refresh-loop.go +++ b/refresh-loop.go @@ -9,6 +9,7 @@ import ( "strings" "time" + "go.sour.is/xt/internal/otel" "go.yarn.social/lextwt" "go.yarn.social/types" ) @@ -21,8 +22,9 @@ const ( TwoMinutes = 60 ) -func refreshLoop(c console, app *appState) { - defer c.abort() +func refreshLoop(c *console, app *appState) error { + ctx, span := otel.Span(c.Context) + defer span.End() f := NewHTTPFetcher() fetch, close := NewFuncPool(c.Context, 25, f.Fetch) @@ -30,60 +32,58 @@ func refreshLoop(c console, app *appState) { db, err := app.DB() if err != nil { - c.Log("missing db") - c.abort() - return + otel.Error(err, "missing db") + return err } queue := app.queue - c.Log("start refresh loop") - for c.Err() == nil { + otel.Info("start refresh loop") + for c.Context.Err() == nil { if queue.IsEmpty() { - c.Log("load feeds") + otel.Info("load feeds") it, err := loadFeeds(c.Context, db) for f := range it { queue.Insert(&f) } if err != nil { - c.Log(err) - return + otel.Error(err) + return err } } f := queue.ExtractMin() if f == nil { - c.Log("sleeping for ", TenMinutes*time.Second) + otel.Info("sleeping for ", TenMinutes*time.Second) select { case <-time.After(TenMinutes * time.Second): case <-c.Done(): - return + return nil } continue } - c.Log("queue size", queue.count, "next", f.URI, "next scan on", f.LastScanOn.Time.Format(time.RFC3339)) + otel.Info("queue size", queue.count, "next", f.URI, "next scan on", f.LastScanOn.Time.Format(time.RFC3339)) if time.Until(f.LastScanOn.Time) > 2*time.Hour { - c.Log("too soon", f.URI) + otel.Info("too soon", f.URI) continue } select { case <-c.Done(): - return + return nil case t := <-time.After(time.Until(f.LastScanOn.Time)): - c.Log("fetch", t.Format(time.RFC3339), f.Nick, f.URI) + otel.Info("fetch", t.Format(time.RFC3339), f.Nick, f.URI) fetch.Fetch(f) case res := <-fetch.Out(): - c.Log("got response:", res.Request.URI) + otel.Info("got response:", res.Request.URI) f := res.Request f.LastScanOn.Time = time.Now() err := res.err if res.err != nil { - f.LastError.String, f.LastError.Valid = err.Error(), true if errors.Is(err, ErrPermanentlyDead) { f.RefreshRate = TenYear } @@ -94,11 +94,11 @@ func refreshLoop(c console, app *appState) { f.RefreshRate = OneDay } - c.Log(err) + otel.Error(err) + f.LastError.String, f.LastError.Valid = err.Error(), true err = f.Save(c.Context, db) if err != nil { - c.Log(err) - return + otel.Error(err) } continue @@ -108,13 +108,30 @@ func refreshLoop(c console, app *appState) { f.LastModified.Time, f.LastModified.Valid = res.LastModified(), true cpy, err := os.OpenFile(filepath.Join("feeds", urlNS.UUID5(f.URI).MarshalText()), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) - rdr := io.TeeReader(res.Body, cpy) + if err != nil { + otel.Error(fmt.Errorf("%w: %w", ErrParseFailed, err)) + f.LastError.String, f.LastError.Valid = err.Error(), true + f.RefreshRate = OneDay + + err = f.Save(c.Context, db) + otel.Error(err) + + continue + } + rdr := io.TeeReader(res.Body, cpy) + rdr = lextwt.TwtFixer(rdr) twtfile, err := lextwt.ParseFile(rdr, &types.Twter{Nick: f.Nick, URI: f.URI}) if err != nil { - c.Log(fmt.Errorf("%w: %w", ErrParseFailed, err)) + otel.Error(fmt.Errorf("%w: %w", ErrParseFailed, err)) + + f.LastError.String, f.LastError.Valid = err.Error(), true f.RefreshRate = OneDay - return + + err = f.Save(c.Context, db) + otel.Error(err) + + continue } if prev, ok := twtfile.Info().GetN("prev", 0); f.FirstFetch && ok { @@ -131,13 +148,15 @@ func refreshLoop(c console, app *appState) { } } - err = storeFeed(db, twtfile) + err = storeFeed(ctx, db, twtfile) if err != nil { - c.Log(err) + otel.Error(err) + f.LastError.String, f.LastError.Valid = err.Error(), true err = f.Save(c.Context, db) - c.Log(err) - return + + otel.Error(err) + return err } cpy.Close() @@ -148,9 +167,11 @@ func refreshLoop(c console, app *appState) { err = f.Save(c.Context, db) if err != nil { - c.Log(err) - return + otel.Error(err) + return err } } } + + return c.Context.Err() } diff --git a/service.go b/service.go index a63978e..6344876 100644 --- a/service.go +++ b/service.go @@ -12,11 +12,18 @@ import ( _ "embed" _ "github.com/mattn/go-sqlite3" + "github.com/uptrace/opentelemetry-go-extra/otelsql" + semconv "go.opentelemetry.io/otel/semconv/v1.4.0" + "go.sour.is/xt/internal/otel" "go.yarn.social/lextwt" "go.yarn.social/types" + "golang.org/x/sync/errgroup" ) -func run(c console) error { +func run(c *console) error { + ctx, span := otel.Span(c.Context) + defer span.End() + a := c.Args() app := &appState{ args: a, @@ -28,6 +35,9 @@ func run(c console) error { // Setup DB err := func(ctx context.Context) error { + ctx, span := otel.Span(ctx) + defer span.End() + db, err := app.DB() if err != nil { return err @@ -42,13 +52,16 @@ func run(c console) error { } return nil - }(c.Context) + }(ctx) if err != nil { return err } // Seed File - err = func() error { + err = func(ctx context.Context) error { + ctx, span := otel.Span(ctx) + defer span.End() + f, err := os.Open(a.baseFeed) if err != nil { return err @@ -69,29 +82,34 @@ func run(c console) error { } defer db.Close() - return storeFeed(db, twtfile) - }() + return storeFeed(ctx, db, twtfile) + }(ctx) if err != nil { return err } - go refreshLoop(c, app) + wg, ctx := errgroup.WithContext(ctx) + c.Context = ctx + + wg.Go(func() error { return refreshLoop(c, app) }) go httpServer(c, app) - <-c.Done() - return c.Err() + wg.Wait() + return c.Context.Err() } type appState struct { args args feeds sync.Map queue *fibHeap[Feed] - - } func (app *appState) DB() (*sql.DB, error) { - return sql.Open(app.args.dbtype, app.args.dbfile) + // return sql.Open(app.args.dbtype, app.args.dbfile) + + return otelsql.Open(app.args.dbtype, app.args.dbfile, + otelsql.WithAttributes(semconv.DBSystemSqlite), + otelsql.WithDBName("mydb")) } func (app *appState) Feed(feedID string) *Feed { From 42fe9176b752fb0cdd32e70c019406565e25c6d4 Mon Sep 17 00:00:00 2001 From: xuu Date: Thu, 13 Mar 2025 22:36:24 -0600 Subject: [PATCH 02/13] chore: many fixes to code --- about.me | 84 --------- service.go => app.go | 76 +++++++- service_test.go => app_test.go | 0 feed.go | 333 +++++++++++++++++++++++++-------- fetcher.go | 48 ++++- go.mod | 3 +- go.sum | 5 + go.work | 6 + go.work.sum | 191 +++++++++++++++++++ http.go | 117 ++++++++++-- init.sql | 12 +- internal/otel/otel.go | 61 ++++-- main.go | 6 +- otel.go | 2 - refresh-loop.go | 135 +++++++------ 15 files changed, 794 insertions(+), 285 deletions(-) delete mode 100644 about.me rename service.go => app.go (58%) rename service_test.go => app_test.go (100%) create mode 100644 go.work create mode 100644 go.work.sum delete mode 100644 otel.go diff --git a/about.me b/about.me deleted file mode 100644 index dcf15fa..0000000 --- a/about.me +++ /dev/null @@ -1,84 +0,0 @@ -.ce -Preamble -.sp -We, the people of the United States, in order -to form a more perfect Union, establish justice, insure -domestic tranquility, provide for the common defense, promote -the general welfare, -and secure the blessing of liberty to ourselves and our -posterity do ordain and establish this Constitution for the -United States of America. -.sp -.nr aR 0 1 -.af aR I -.de AR -.ce -.ps 16 -.ft B -Article \\n+(aR -.nr sE 0 1 -.af sE 1 -.ps 12 -.ft P -.. -.de SE -.sp -.ft B -\\s-2SECTION \\n+(sE:\\s+2 -.ft P -.nr pP 0 1 -.af pP 1 -.. -.de PP -.sp -.ft I -\\s-3Paragraph \\n+(pP:\\s+3 -.ft P -.. -.AR -.SE -Legislative powers; in whom vested: -.PP -All legislative powers herein granted shall be vested in a -Congress of the United States, which shall consist of a Senate -and a House of Representatives. -.SE -House of Representatives, how and by whom chosen, Qualifications -of a Representative. Representatives and direct taxes, how -apportioned. Enumeration. Vacancies to be filled. Power of -choosing officers and of impeachment. -.PP -The House of Representatives shall be composed of members chosen -every second year by the people of the several states, and the -electors in each State shall have the qualifications requisite -for electors of the most numerous branch of the State Legislature. -.PP -No person shall be a Representative who shall not have attained -to the age of twenty-five years, and been seven years a citizen -of the United States, and who shall not, when elected, be an -inhabitant of that State in which he shall be chosen. -.PP -Representatives and direct taxes shall be apportioned among the -several States which maybe included within this Union, according -to their respective numbers, which shall be determined by adding -to the whole number of free persons, including those bound for -service for a term of years, and excluding Indians not taxed, -three-fifths of all other persons. The actual enumeration shall -be made within three years after the first meeting of the -Congress of the United States, and within every subsequent term -of ten years, in such manner as they shall by law direct. The -number of Representatives shall not exceed one for every thirty -thousand, but each State shall have at least one Representative; -and until such enumeration shall be made, the State of New -Hampshire shall be entitled to choose three, Massachusetts eight, -Rhode Island and Providence Plantations one, Connecticut -five, New York six, New Jersey four, Pennsylvania eight, -Delaware one, Maryland six, Virginia ten, North Carolina five, -South Carolina five, and Georgia three. -.PP -When vacancies happen in the representation from any State, the -Executive Authority thereof shall issue writs of election to fill -such vacancies. -.PP -The House of Representatives shall choose their Speaker and other -officers; and shall have the sole power of impeachment. \ No newline at end of file diff --git a/service.go b/app.go similarity index 58% rename from service.go rename to app.go index 6344876..1688f82 100644 --- a/service.go +++ b/app.go @@ -6,6 +6,8 @@ import ( "fmt" "iter" "os" + "runtime/debug" + "strconv" "strings" "sync" @@ -13,7 +15,9 @@ import ( _ "github.com/mattn/go-sqlite3" "github.com/uptrace/opentelemetry-go-extra/otelsql" + "go.opentelemetry.io/otel/attribute" semconv "go.opentelemetry.io/otel/semconv/v1.4.0" + "go.opentelemetry.io/otel/trace" "go.sour.is/xt/internal/otel" "go.yarn.social/lextwt" "go.yarn.social/types" @@ -24,12 +28,15 @@ func run(c *console) error { ctx, span := otel.Span(c.Context) defer span.End() + bi, _ := debug.ReadBuildInfo() + span.AddEvent(name, trace.WithAttributes(attribute.String("version", bi.Main.Version))) + a := c.Args() app := &appState{ args: a, feeds: sync.Map{}, queue: FibHeap(func(a, b *Feed) bool { - return a.LastScanOn.Time.Before(b.LastScanOn.Time) + return a.NextScanOn.Time.Before(b.NextScanOn.Time) }), } @@ -38,7 +45,7 @@ func run(c *console) error { ctx, span := otel.Span(ctx) defer span.End() - db, err := app.DB() + db, err := app.DB(ctx) if err != nil { return err } @@ -76,7 +83,7 @@ func run(c *console) error { return fmt.Errorf("%w: %w", ErrParseFailed, err) } - db, err := app.DB() + db, err := app.DB(ctx) if err != nil { return err } @@ -91,10 +98,16 @@ func run(c *console) error { wg, ctx := errgroup.WithContext(ctx) c.Context = ctx - wg.Go(func() error { return refreshLoop(c, app) }) + wg.Go(func() error { + return refreshLoop(c, app) + }) go httpServer(c, app) - wg.Wait() + err = wg.Wait() + if err != nil { + return err + } + return c.Context.Err() } @@ -104,12 +117,61 @@ type appState struct { queue *fibHeap[Feed] } -func (app *appState) DB() (*sql.DB, error) { +type db struct { + *sql.DB + Version string + Params map[string]string + MaxLength int + MaxVariableNumber int +} + +func (app *appState) DB(ctx context.Context) (db, error) { // return sql.Open(app.args.dbtype, app.args.dbfile) - return otelsql.Open(app.args.dbtype, app.args.dbfile, + var err error + db := db{Params: make(map[string]string)} + db.DB, err = otelsql.Open(app.args.dbtype, app.args.dbfile, otelsql.WithAttributes(semconv.DBSystemSqlite), otelsql.WithDBName("mydb")) + if err != nil { + return db, err + } + + rows, err := db.DB.QueryContext(ctx, `select sqlite_version()`) + if err != nil { + return db, err + } + + if rows.Next() { + rows.Scan(&db.Version) + } + if err = rows.Err(); err != nil { + return db, err + } + + rows, err = db.DB.QueryContext(ctx, `pragma compile_options`) + if err != nil { + return db, err + } + + for rows.Next() { + var key, value string + rows.Scan(&key) + key, value, _ = strings.Cut(key, "=") + db.Params[key] = value + } + if err = rows.Err(); err != nil { + return db, err + } + + if m, ok := db.Params["MAX_VARIABLE_NUMBER"]; ok { + db.MaxVariableNumber, _ = strconv.Atoi(m) + } + if m, ok := db.Params["MAX_LENGTH"]; ok { + db.MaxLength, _ = strconv.Atoi(m) + } + + return db, err } func (app *appState) Feed(feedID string) *Feed { diff --git a/service_test.go b/app_test.go similarity index 100% rename from service_test.go rename to app_test.go diff --git a/feed.go b/feed.go index 27b04bd..17d980b 100644 --- a/feed.go +++ b/feed.go @@ -4,7 +4,9 @@ import ( "cmp" "context" "database/sql" + "database/sql/driver" "fmt" + "hash/fnv" "iter" "net/http" "net/url" @@ -14,6 +16,7 @@ import ( _ "embed" + "github.com/oklog/ulid/v2" "go.sour.is/xt/internal/otel" "go.yarn.social/lextwt" "go.yarn.social/types" @@ -21,22 +24,22 @@ import ( type Feed struct { FeedID uuid - FetchURI string + ParentID uuid + HashURI string URI string Nick string - LastScanOn sql.NullTime + State State + LastScanOn TwtTime RefreshRate int + NextScanOn TwtTime - LastModified sql.NullTime + LastModified TwtTime LastError sql.NullString ETag sql.NullString Version string DiscloseFeedURL string DiscloseNick string - FirstFetch bool - - State State } type State string @@ -47,43 +50,34 @@ const ( Cold State = "cold" Warm State = "warm" Hot State = "hot" + Once State = "once" ) var ( //go:embed init.sql initSQL string - insertFeed = ` - insert into feeds - (feed_id, uri, nick, last_scan_on, refresh_rate) - values (?, ?, ?, ?, ?) - ON CONFLICT (feed_id) DO NOTHING - ` - - insertTwt = ` - insert into twts - (feed_id, hash, conv, dt, text, mentions, tags) - values (?, ?, ?, ?, ?, ?, ?) - ON CONFLICT (feed_id, hash) DO NOTHING - ` - - fetchFeeds = ` - select - feed_id, - uri, - nick, - last_scan_on, - refresh_rate, - last_modified_on, - last_etag - from feeds - where datetime( - coalesce(last_scan_on, '1901-01-01'), - '+'||refresh_rate||' seconds' - ) < datetime(current_timestamp, '+10 minutes') - ` + insertFeed = func(r int) (string, int) { + repeat := "" + if r > 1 { + repeat = strings.Repeat(", (?, ?, ?, ?, ?, ?, ?)", r-1) + } + return ` + insert into feeds ( + feed_id, + parent_id, + nick, + uri, + state, + last_scan_on, + refresh_rate + ) + values (?, ?, ?, ?, ?, ?, ?)` + repeat + ` + ON CONFLICT (feed_id) DO NOTHING`, r * 7 + } updateFeed = ` - update feeds set + update feeds set + state = ?, last_scan_on = ?, refresh_rate = ?, last_modified_on = ?, @@ -91,21 +85,83 @@ var ( last_error = ? where feed_id = ? ` + + insertTwt = func(r int) (string, int) { + repeat := "" + if r > 1 { + repeat = strings.Repeat(", (?, ?, ?, ?, ?, ?, ?)", r-1) + } + return ` + insert into twts + (feed_id, ulid, text, hash, conv, mentions, tags) + values (?, ?, ?, ?, ?, ?, ?)` + repeat + ` + ON CONFLICT (feed_id, ulid) DO NOTHING`, r * 7 + } + + fetchFeeds = ` + select + feed_id, + parent_id, + coalesce(hashing_uri, uri) hash_uri, + uri, + nick, + state, + last_scan_on, + strftime( + '%Y-%m-%dT%H:%M:%fZ', + coalesce(last_scan_on, '1901-01-01'), + '+'||refresh_rate||' seconds' + ) next_scan_on, + refresh_rate, + last_modified_on, + last_etag + from feeds + left join ( + select + feed_id parent_id, + uri hashing_uri + from feeds + where parent_id is null + ) using (parent_id) + where datetime( + coalesce(last_scan_on, '1901-01-01'), + '+'||refresh_rate||' seconds' + ) < datetime(current_timestamp, '+10 minutes') + ` ) -func (f *Feed) Save(ctx context.Context, db *sql.DB) error { +func (f *Feed) Create(ctx context.Context, db db) error { + ctx, span := otel.Span(ctx) + defer span.End() + query, _ := insertFeed(1) + _, err := db.ExecContext( + ctx, + query, + f.FeedID, // feed_id + f.ParentID, // parent_id + f.Nick, // nick + f.URI, // uri + f.State, // state + f.LastScanOn, // last_scan_on + f.RefreshRate, // refresh_rate + ) + return err +} + +func (f *Feed) Save(ctx context.Context, db db) error { ctx, span := otel.Span(ctx) defer span.End() _, err := db.ExecContext( ctx, updateFeed, - f.LastScanOn, - f.RefreshRate, - f.LastModified, - f.ETag, - f.LastError, - f.FeedID, + f.State, // state + f.LastScanOn, // last_scan_on + f.RefreshRate, // refresh_rate + f.LastModified, // last_modified_on + f.ETag, // last_etag + f.LastError, // last_error + f.FeedID, // feed_id ) return err } @@ -117,9 +173,13 @@ func (f *Feed) Scan(res interface{ Scan(...any) error }) error { f.Version = "0.0.1" err = res.Scan( &f.FeedID, + &f.ParentID, + &f.HashURI, &f.URI, &f.Nick, + &f.State, &f.LastScanOn, + &f.NextScanOn, &f.RefreshRate, &f.LastModified, &f.ETag, @@ -128,19 +188,10 @@ func (f *Feed) Scan(res interface{ Scan(...any) error }) error { return err } - if !f.LastScanOn.Valid { - f.FirstFetch = true - f.LastScanOn.Time = time.Now() - f.LastScanOn.Valid = true - } else { - f.LastScanOn.Time = f.LastScanOn.Time.Add(time.Duration(f.RefreshRate) * time.Second) - } - - f.FetchURI = f.URI return err } -func loadFeeds(ctx context.Context, db *sql.DB) (iter.Seq[Feed], error) { +func loadFeeds(ctx context.Context, db db) (iter.Seq[Feed], error) { ctx, span := otel.Span(ctx) var err error @@ -159,6 +210,7 @@ func loadFeeds(ctx context.Context, db *sql.DB) (iter.Seq[Feed], error) { var f Feed err = f.Scan(res) if err != nil { + span.RecordError(err) return } if !yield(f) { @@ -168,7 +220,7 @@ func loadFeeds(ctx context.Context, db *sql.DB) (iter.Seq[Feed], error) { }, err } -func storeFeed(ctx context.Context, db *sql.DB, f types.TwtFile) error { +func storeFeed(ctx context.Context, db db, f types.TwtFile) error { ctx, span := otel.Span(ctx) defer span.End() @@ -201,20 +253,11 @@ func storeFeed(ctx context.Context, db *sql.DB, f types.TwtFile) error { defer tx.Rollback() - _, err = tx.ExecContext( - ctx, - insertFeed, - feedID, - f.Twter().HashingURI, - f.Twter().DomainNick(), - loadTS, - refreshRate, - ) - if err != nil { - return err - } + twts := f.Twts() + _, size := insertTwt(len(twts)) + args := make([]any, 0, size) - for _, twt := range f.Twts() { + for _, twt := range twts { mentions := make(uuids, 0, len(twt.Mentions())) for _, mention := range twt.Mentions() { followMap[mention.Twter().Nick] = mention.Twter().URI @@ -233,32 +276,76 @@ func storeFeed(ctx context.Context, db *sql.DB, f types.TwtFile) error { subjectTag = tag.Text() } } + args = append( + args, + feedID, // feed_id + makeULID(twt), // ulid + fmt.Sprintf("%+l", twt), // text + subjectTag, // conv + twt.Hash(), // hash + mentions.ToStrList(), // mentions + tags, // tags + ) + } + for query, args := range chunk(args, insertTwt, db.MaxVariableNumber) { + fmt.Println("store", f.Twter().URI, len(args)) _, err = tx.ExecContext( ctx, - insertTwt, - feedID, - twt.Hash(), - subjectTag, - twt.Created(), - fmt.Sprint(twt), - mentions.ToStrList(), - tags, + query, + args..., ) if err != nil { return err } } + args = args[:0] + args = append(args, + feedID, // feed_id + nil, // parent_id + f.Twter().DomainNick(), // nick + f.Twter().URI, // uri + "warm", // state + TwtTime{Time: loadTS, Valid: true}, // last_scan_on + refreshRate, // refresh_rate + ) + + if prev, ok := f.Info().GetN("prev", 0); ok { + _, part, ok := strings.Cut(prev.Value(), " ") + if ok { + uri:= f.Twter().URI + part = uri[:strings.LastIndex(uri, "/")+1] + part + childID := urlNS.UUID5(part) + + args = append(args, + childID, // feed_id + feedID, // parent_id + f.Twter().DomainNick(), // nick + part, // uri + "once", // state + nil, // last_scan_on + refreshRate, // refresh_rate + ) + } + } + for nick, uri := range followMap { + args = append(args, + urlNS.UUID5(uri), // feed_id + nil, // parent_id + nick, // nick + uri, // uri + "warm", // state + nil, // last_scan_on + refreshRate, // refresh_rate + ) + } + for query, args := range chunk(args, insertFeed, db.MaxVariableNumber) { _, err = tx.ExecContext( ctx, - insertFeed, - urlNS.UUID5(uri), - uri, - nick, - nil, - refreshRate, + query, + args..., ) if err != nil { return err @@ -270,11 +357,11 @@ func storeFeed(ctx context.Context, db *sql.DB, f types.TwtFile) error { func (feed *Feed) MakeHTTPRequest(ctx context.Context) (*http.Request, error) { feed.State = "fetch" - if strings.Contains(feed.FetchURI, "lublin.se") { + if strings.Contains(feed.URI, "lublin.se") { return nil, fmt.Errorf("%w: permaban: %s", ErrPermanentlyDead, feed.URI) } - req, err := http.NewRequestWithContext(ctx, "GET", feed.FetchURI, nil) + req, err := http.NewRequestWithContext(ctx, "GET", feed.URI, nil) if err != nil { return nil, fmt.Errorf("creating HTTP request failed: %w", err) } @@ -298,3 +385,83 @@ func (feed *Feed) MakeHTTPRequest(ctx context.Context) (*http.Request, error) { return req, nil } + +type TwtTime struct { + Time time.Time + Valid bool // Valid is true if Time is not NULL +} + +// Scan implements the [Scanner] interface. +func (n *TwtTime) Scan(value any) error { + var err error + + switch value := value.(type) { + case nil: + n.Time, n.Valid = time.Time{}, false + return nil + case string: + n.Valid = true + n.Time, err = time.Parse(time.RFC3339, value) + case time.Time: + n.Valid = true + n.Time = value + } + return err +} + +// Value implements the [driver.Valuer] interface. +func (n TwtTime) Value() (driver.Value, error) { + if !n.Valid { + return nil, nil + } + return n.Time.Format(time.RFC3339), nil +} + +func makeULID(twt types.Twt) ulid.ULID { + h64 := fnv.New64a() + h16 := fnv.New32a() + text := []byte(fmt.Sprintf("%+l", twt)) + b := make([]byte, 10) + copy(b, h16.Sum(text)[:2]) + copy(b[2:], h64.Sum(text)) + u := ulid.ULID{} + u.SetTime(ulid.Timestamp(twt.Created())) + u.SetEntropy(b) + + return u +} + +func chunk(args []any, qry func(int) (string, int), maxArgs int) iter.Seq2[string, []any] { + _, size := qry(1) + itemsPerIter := maxArgs / size + + if len(args) < size { + return func(yield func(string, []any) bool) {} + } + + if len(args) < maxArgs { + return func(yield func(string, []any) bool) { + query, _ := qry(len(args) / size) + yield(query, args) + } + } + + return func(yield func(string, []any) bool) { + for len(args) > 0 { + if len(args) > maxArgs { + query, size := qry(itemsPerIter) + if !yield(query, args[:size]) { + return + } + args = args[size:] + continue + } + + query, _ := qry(len(args) / size) + yield(query, args) + return + } + } +} + + diff --git a/fetcher.go b/fetcher.go index f4118ff..58e0cf1 100644 --- a/fetcher.go +++ b/fetcher.go @@ -8,6 +8,8 @@ import ( "net/http" "sync" "time" + + "go.sour.is/xt/internal/otel" ) var ( @@ -62,7 +64,7 @@ func NewHTTPFetcher() *httpFetcher { ForceAttemptHTTP2: false, MaxIdleConns: 100, IdleConnTimeout: 10 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, + TLSHandshakeTimeout: 5 * time.Second, ExpectContinueTimeout: 1 * time.Second, }, }, @@ -70,6 +72,10 @@ func NewHTTPFetcher() *httpFetcher { } func (f *httpFetcher) Fetch(ctx context.Context, request *Feed) *Response { + ctx, span := otel.Span(ctx) + defer span.End() + + defer fmt.Println("fetch done", request.URI) response := &Response{ Request: request, } @@ -79,8 +85,9 @@ func (f *httpFetcher) Fetch(ctx context.Context, request *Feed) *Response { response.err = err return response } - + span.AddEvent("start request") res, err := f.client.Do(req) + span.AddEvent("got response") if err != nil { if errors.Is(err, &net.DNSError{}) { response.err = fmt.Errorf("%w: %s", ErrTemporarilyDead, err) @@ -97,7 +104,7 @@ func (f *httpFetcher) Fetch(ctx context.Context, request *Feed) *Response { case 304: response.err = fmt.Errorf("%w: %s", ErrUnmodified, res.Status) - case 400, 406, 502, 503: + case 400, 406, 429, 502, 503: response.err = fmt.Errorf("%w: %s", ErrTemporarilyDead, res.Status) case 403, 404, 410: @@ -121,29 +128,54 @@ func NewFuncPool[IN, OUT any]( size int, fetch func(ctx context.Context, request IN) OUT, ) (*pool[IN, OUT], func()) { + ctx, span := otel.Span(ctx) + defer span.End() + var wg sync.WaitGroup in := make(chan IN, size) - out := make(chan OUT, size) + out := make(chan OUT) wg.Add(size) for range size { go func() { + ctx, span := otel.Span(ctx) + defer span.End() + defer wg.Done() for request := range in { + ctx, cancel := context.WithTimeoutCause(ctx, 15*time.Second, fmt.Errorf("GOT STUCK")) + defer cancel() + + ctx, span := otel.Span(ctx) + defer span.End() + + span.AddEvent("start fetch") + r := fetch(ctx, request) + span.AddEvent("got fetch") + select { case <-ctx.Done(): return - case out <- fetch(ctx, request): + case out <- r: + span.AddEvent("sent queue") + case <-time.After(20 * time.Second): + fmt.Println("GOT STUCK", request) + span.AddEvent("GOT STUCK") + cancel() } } }() } return &pool[IN, OUT]{ - in: in, - out: out, - }, func() { close(in); wg.Wait(); close(out) } + in: in, + out: out, + }, func() { + close(in) + wg.Wait() + close(out) + } } func (f *pool[IN, OUT]) Fetch(request IN) { diff --git a/go.mod b/go.mod index f453882..05e88af 100644 --- a/go.mod +++ b/go.mod @@ -41,6 +41,7 @@ require ( github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/matryer/is v1.4.1 + github.com/oklog/ulid/v2 v2.1.0 github.com/prometheus/client_golang v1.20.5 github.com/sirupsen/logrus v1.9.3 // indirect github.com/uptrace/opentelemetry-go-extra/otelsql v0.3.2 @@ -53,7 +54,7 @@ require ( go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.34.0 go.opentelemetry.io/otel/sdk v1.34.0 go.opentelemetry.io/otel/sdk/log v0.10.0 - go.opentelemetry.io/otel/sdk/metric v1.34.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.34.0 go.yarn.social/types v0.0.0-20250108134258-ed75fa653ede golang.org/x/crypto v0.33.0 // indirect golang.org/x/sync v0.11.0 diff --git a/go.sum b/go.sum index 2ca56a6..247ae07 100644 --- a/go.sum +++ b/go.sum @@ -35,6 +35,11 @@ github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBW github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/oklog/ulid/v2 v2.1.0 h1:+9lhoxAP56we25tyYETBBY1YLA2SaoLvUFgrP2miPJU= +github.com/oklog/ulid/v2 v2.1.0/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ= +github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= diff --git a/go.work b/go.work new file mode 100644 index 0000000..774fc94 --- /dev/null +++ b/go.work @@ -0,0 +1,6 @@ +go 1.24.0 + +use ( + . + ../go-lextwt +) diff --git a/go.work.sum b/go.work.sum new file mode 100644 index 0000000..cc7321d --- /dev/null +++ b/go.work.sum @@ -0,0 +1,191 @@ +cel.dev/expr v0.19.0 h1:lXuo+nDhpyJSpWxpPVi5cPUwzKb+dsdOiw6IreM5yt0= +cel.dev/expr v0.19.0/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw= +cloud.google.com/go v0.26.0 h1:e0WKqKTd5BnrG8aKH3J3h+QvEIQtSUcf2n5UZ5ZgLtQ= +cloud.google.com/go/compute/metadata v0.5.2 h1:UxK4uu/Tn+I3p2dYWTfiX4wva7aYlKixAHn3fyqngqo= +cloud.google.com/go/compute/metadata v0.5.2/go.mod h1:C66sj2AluDcIqakBq/M8lw8/ybHgOZqin2obFxa/E5k= +dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= +dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0 h1:3c8yed4lgqTt+oTQ+JNMDo+F4xprBf+O/il4ZC0nRLw= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0/go.mod h1:obipzmGjfSjam60XLwGfqUkJsfiheAl+TUjG+4yzyPM= +github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= +github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78= +github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= +github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= +github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= +github.com/alecthomas/kingpin/v2 v2.4.0 h1:f48lwail6p8zpO1bC4TxtqACaGqHYA22qkHjHpqDjYY= +github.com/alecthomas/kingpin/v2 v2.4.0/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= +github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc= +github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= +github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= +github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWpi6yML8= +github.com/aymanbagabas/go-udiff v0.2.0/go.mod h1:RE4Ex0qsGkTAJoQdQQCA0uG+nAzJO/pI/QwceO5fgrA= +github.com/badgerodon/ioutil v0.0.0-20150716134133-06e58e34b867 h1:nsDNoesoGwPzPkcrR1w1uzPUtiqwCXoNnkWC7nUuRHI= +github.com/badgerodon/ioutil v0.0.0-20150716134133-06e58e34b867/go.mod h1:Ctq1YQi0dOq7QgBLZZ7p1Fr3IbAAqL/yMqDIHoe9WtE= +github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g= +github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= +github.com/charmbracelet/harmonica v0.2.0 h1:8NxJWRWg/bzKqqEaaeFNipOu77YR5t8aSwG4pgaUBiQ= +github.com/charmbracelet/harmonica v0.2.0/go.mod h1:KSri/1RMQOZLbw7AHqgcBycp8pgJnQMYYT8QZRqZ1Ao= +github.com/charmbracelet/x/exp/golden v0.0.0-20250213125511-a0c32e22e4fc h1:Tp8vprGbBhTAeyCNgrWPYIPyVEo9qmFRAcRhSdeind4= +github.com/charmbracelet/x/exp/golden v0.0.0-20250213125511-a0c32e22e4fc/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U= +github.com/charmbracelet/x/exp/teatest v0.0.0-20250213125511-a0c32e22e4fc h1:p4x5lOqNZEELobbdrm00kW5xrQwXZtIrUW8yTPK16aU= +github.com/charmbracelet/x/exp/teatest v0.0.0-20250213125511-a0c32e22e4fc/go.mod h1:ag+SpTUkiN/UuUGYPX3Ci4fR1oF3XX97PpGhiXK7i6U= +github.com/cilium/ebpf v0.11.0 h1:V8gS/bTCCjX9uUnkUFUpPsksM8n1lXBAvHcpiFk1X2Y= +github.com/cilium/ebpf v0.11.0/go.mod h1:WE7CZAnqOL2RouJ4f1uyNhqr2P4CCvXFIqdRDUgWsVs= +github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= +github.com/cloudflare/circl v1.5.0 h1:hxIWksrX6XN5a1L2TI/h53AGPhNHoUBo+TD1ms9+pys= +github.com/cloudflare/circl v1.5.0/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f h1:WBZRG4aNOuI15bLRrCgN8fCq8E5Xuty6jGbmSNEvSsU= +github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 h1:QVw89YDxXxEe+l8gU8ETbOasdwEV+avkR75ZzsVV9WI= +github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= +github.com/cosiner/argv v0.1.0 h1:BVDiEL32lwHukgJKP87btEPenzrrHUjajs/8yzaqcXg= +github.com/cosiner/argv v0.1.0/go.mod h1:EusR6TucWKX+zFgtdUsKT2Cvg45K5rtpCcWz4hK06d8= +github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.20 h1:VIPb/a2s17qNeQgDnkfZC35RScx+blkKF8GV68n80J4= +github.com/creack/pty v1.1.20/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/cyphar/filepath-securejoin v0.3.4 h1:VBWugsJh2ZxJmLFSM06/0qzQyiQX2Qs0ViKrUAcqdZ8= +github.com/cyphar/filepath-securejoin v0.3.4/go.mod h1:8s/MCNJREmFK0H02MF6Ihv1nakJe4L/w3WZLHNkvlYM= +github.com/derekparker/trie v0.0.0-20230829180723-39f4de51ef7d h1:hUWoLdw5kvo2xCsqlsIBMvWUc1QCSsCYD2J2+Fg6YoU= +github.com/derekparker/trie v0.0.0-20230829180723-39f4de51ef7d/go.mod h1:C7Es+DLenIpPc9J6IYw4jrK0h7S9bKj4DNl8+KxGEXU= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= +github.com/envoyproxy/go-control-plane v0.13.1 h1:vPfJZCkob6yTMEgS+0TwfTUfbHjfy/6vOJ8hUWX/uXE= +github.com/envoyproxy/go-control-plane v0.13.1/go.mod h1:X45hY0mufo6Fd0KW3rqsGvQMw58jvjymeCzBU3mWyHw= +github.com/envoyproxy/protoc-gen-validate v1.1.0 h1:tntQDh69XqOCOZsDz0lVJQez/2L6Uu2PdjCQwWCJ3bM= +github.com/envoyproxy/protoc-gen-validate v1.1.0/go.mod h1:sXRDRVmzEbkM7CVcM06s9shE/m23dg3wzjl0UWqJ2q4= +github.com/felixge/fgprof v0.9.5 h1:8+vR6yu2vvSKn08urWyEuxx75NWPEvybbkBirEpsbVY= +github.com/felixge/fgprof v0.9.5/go.mod h1:yKl+ERSa++RYOs32d8K6WEXCB4uXdLls4ZaZPpayhMM= +github.com/go-delve/liner v1.2.3-0.20231231155935-4726ab1d7f62 h1:IGtvsNyIuRjl04XAOFGACozgUD7A82UffYxZt4DWbvA= +github.com/go-delve/liner v1.2.3-0.20231231155935-4726ab1d7f62/go.mod h1:biJCRbqp51wS+I92HMqn5H8/A0PAhxn2vyOT+JqhiGI= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= +github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU= +github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow= +github.com/go-git/go-git/v5 v5.12.0 h1:7Md+ndsjrzZxbddRDZjF14qK+NN56sy6wkqaVrjZtys= +github.com/go-git/go-git/v5 v5.12.0/go.mod h1:FTM9VKtnI2m65hNI/TenDDDnUf2Q9FHnXYjuz9i5OEY= +github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= +github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= +github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/golang/glog v1.2.3 h1:oDTdz9f5VGVVNGu/Q7UXKWYsD0873HXLHdJUNBsSEKM= +github.com/golang/glog v1.2.3/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1 h1:G5FRp8JnTd7RQH5kemVNlMeyXQAztQ3mOWV95KxsXH8= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/google/go-dap v0.12.0 h1:rVcjv3SyMIrpaOoTAdFDyHs99CwVOItIJGKLQFQhNeM= +github.com/google/go-dap v0.12.0/go.mod h1:tNjCASCm5cqePi/RVXXWEVqtnNLV1KTWtYOqu6rZNzc= +github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= +github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 h1:y3N7Bm7Y9/CtpiVkw/ZWj6lSlDF3F74SfKwfTCer72Q= +github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= +github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= +github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= +github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= +github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= +github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/profile v1.7.0 h1:hnbDkaNWPCLMO9wGLdBFTIZvzDrDfBM2072E1S9gJkA= +github.com/pkg/profile v1.7.0/go.mod h1:8Uer0jas47ZQMJ7VD+OHknK4YDY07LPUC6dEvqDjvNo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= +github.com/rogpeppe/fastuuid v1.2.0 h1:Ppwyp6VYCF1nvBTXL3trRso7mXMlRrw9ooo375wvi2s= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sahilm/fuzzy v0.1.1 h1:ceu5RHF8DGgoi+/dR5PsECjCDH1BE3Fnmpo7aVXOdRA= +github.com/sahilm/fuzzy v0.1.1/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8ZK9Y= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= +github.com/skeema/knownhosts v1.3.0 h1:AM+y0rI04VksttfwjkSTNQorvGqmwATnvnAHpSgc0LY= +github.com/skeema/knownhosts v1.3.0/go.mod h1:sPINvnADmT/qYH1kfv+ePMmOBTH6Tbl7b5LvTDjFK7M= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= +github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= +github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc= +github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= +github.com/yuin/goldmark v1.4.13 h1:fVcFKWvrslecOb/tg+Cc05dkeYx540o0FuFt3nUVDoE= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.opentelemetry.io/contrib/detectors/gcp v1.32.0 h1:P78qWqkLSShicHmAzfECaTgvslqHxblNE9j62Ws1NK8= +go.opentelemetry.io/contrib/detectors/gcp v1.32.0/go.mod h1:TVqo0Sda4Cv8gCIixd7LuLwW4EylumVWfhjZJjDD4DU= +go.starlark.net v0.0.0-20231101134539-556fd59b42f6 h1:+eC0F/k4aBLC4szgOcjd7bDTEnpxADJyWJE0yowgM3E= +go.starlark.net v0.0.0-20231101134539-556fd59b42f6/go.mod h1:LcLNIzVOMp4oV+uusnpk+VU+SzXaJakUuBjoCSWH5dM= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 h1:XQyxROzUlZH+WIQwySDgnISgOivlhjIEwaQaJEJrrN0= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE= +golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/telemetry v0.0.0-20240521205824-bda55230c457/go.mod h1:pRgIJT+bRLFKnoM1ldnzKoxTIn14Yxz928LQRYYgIN0= +golang.org/x/telemetry v0.0.0-20241106142447-58a1122356f5 h1:TCDqnvbBsFapViksHcHySl/sW4+rTGNIAoJJesHRuMM= +golang.org/x/telemetry v0.0.0-20241106142447-58a1122356f5/go.mod h1:8nZWdGp9pq73ZI//QJyckMQab3yq7hoWi7SI0UIusVI= +golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= +golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= +golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc h1:/hemPrYIhOhy8zYrNj+069zDB68us2sMGsfkFJO0iZs= +rsc.io/pdf v0.1.1 h1:k1MczvYDUvJBe93bYd7wrZLLUEcLZAuF824/I4e5Xr4= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/http.go b/http.go index 0069cee..b121bd8 100644 --- a/http.go +++ b/http.go @@ -9,57 +9,148 @@ import ( "strings" "time" + "go.yarn.social/lextwt" + "go.yarn.social/types" + "go.sour.is/xt/internal/otel" ) func httpServer(c *console, app *appState) error { - otel.Info("start http server") + ctx, span := otel.Span(c) + defer span.End() - db, err := app.DB() + span.AddEvent("start http server") + + db, err := app.DB(ctx) if err != nil { - otel.Info("missing db", err) + span.RecordError(fmt.Errorf("%w: missing db", err)) return err } http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + _, span := otel.Span(r.Context()) + defer span.End() + w.Header().Set("Content-Type", "text/html; charset=utf-8") w.Write([]byte("ok")) }) http.HandleFunc("/conv/{hash}", func(w http.ResponseWriter, r *http.Request) { + ctx, span := otel.Span(r.Context()) + defer span.End() + hash := r.PathValue("hash") if (len(hash) < 6 || len(hash) > 8) && !notAny(hash, "abcdefghijklmnopqrstuvwxyz234567") { w.WriteHeader(http.StatusBadRequest) return } - w.Header().Set("Content-Type", "text/html; charset=utf-8") - w.Write([]byte(hash)) + w.Header().Set("Content-Type", "text/plain; charset=utf-8") - rows, err := db.QueryContext(r.Context(), "SELECT feed_id, hash, conv, dt, text FROM twt WHERE hash = $1 or conv = $1", hash) + rows, err := db.QueryContext( + ctx, + `SELECT + feed_id, + hash, + conv, + nick, + uri, + text + FROM twts + JOIN ( + SELECT + feed_id, + nick, + uri + FROM feeds + ) using (feed_id) + WHERE + hash = $1 or + conv = $1 + order by ulid asc`, + hash, + ) if err != nil { - otel.Info("error", err) + span.RecordError(err) return } defer rows.Close() + var twts []types.Twt for rows.Next() { - var twt struct { + var o struct { FeedID string Hash string Conv string - Dt time.Time + Dt string + Nick string + URI string Text string } - err = rows.Scan(&twt.FeedID, &twt.Hash, &twt.Conv, &twt.Dt, &twt.Text) + err = rows.Scan(&o.FeedID, &o.Hash, &o.Conv, &o.Nick, &o.URI, &o.Text) if err != nil { - otel.Error(err) + span.RecordError(err) return } + twter := types.NewTwter(o.Nick, o.URI) + o.Text = strings.ReplaceAll(o.Text, "\n", "\u2028") + twt, _ := lextwt.ParseLine(o.Text, &twter) + twts = append(twts, twt) } + var preamble lextwt.Comments + preamble = append(preamble, lextwt.NewComment("# self = /conv/"+hash)) + + reg := lextwt.NewTwtRegistry(preamble, twts) + reg.WriteTo(w) }) - http.HandleFunc("/feeds", func(w http.ResponseWriter, r *http.Request) { + http.HandleFunc("/users", func(w http.ResponseWriter, r *http.Request) { + ctx, span := otel.Span(r.Context()) + defer span.End() + + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + + rows, err := db.QueryContext( + ctx, + `SELECT + feed_id, + uri, + nick, + last_scan_on + FROM feeds + where parent_id is null + order by nick, uri`, + ) + if err != nil { + span.RecordError(err) + return + } + defer rows.Close() + + var twts []types.Twt + for rows.Next() { + var o struct { + FeedID string + URI string + Nick string + Dt TwtTime + } + err = rows.Scan(&o.FeedID, &o.URI, &o.Nick, &o.Dt) + if err != nil { + span.RecordError(err) + return + } + twts = append(twts, lextwt.NewTwt( + types.NewTwter(o.Nick, o.URI), + lextwt.NewDateTime(o.Dt.Time, o.Dt.Time.Format(time.RFC3339)), + nil, + )) + } + reg := lextwt.NewTwtRegistry(nil, twts) + reg.WriteTo(w) + }) + + http.HandleFunc("/queue", func(w http.ResponseWriter, r *http.Request) { lis := slices.Collect(app.queue.Iter()) sort.Slice(lis, func(i, j int) bool { return lis[i].LastScanOn.Time.Before(lis[j].LastScanOn.Time) @@ -77,7 +168,7 @@ func httpServer(c *console, app *appState) error { c.AddCancel(srv.Shutdown) err = srv.ListenAndServe() if !errors.Is(err, http.ErrServerClosed) { - otel.Error(err) + span.RecordError(err) return err } diff --git a/init.sql b/init.sql index 10dfc70..30e2cc5 100644 --- a/init.sql +++ b/init.sql @@ -2,8 +2,10 @@ PRAGMA journal_mode=WAL; create table if not exists feeds ( feed_id blob primary key, - uri text, + parent_id blob, nick text, + uri text, + state string, last_scan_on timestamp, refresh_rate int default 600, last_modified_on timestamp, @@ -13,12 +15,12 @@ create table if not exists feeds ( create table if not exists twts ( feed_id blob, - hash text, - conv text, - dt text, -- timestamp with timezone + ulid blob, text text, + conv text, + hash text, mentions text, -- json tags text, -- json - primary key (feed_id, hash) + primary key (feed_id, ulid) ); diff --git a/internal/otel/otel.go b/internal/otel/otel.go index a35e945..99fde23 100644 --- a/internal/otel/otel.go +++ b/internal/otel/otel.go @@ -17,13 +17,11 @@ import ( "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" "go.opentelemetry.io/otel/exporters/stdout/stdoutlog" - "go.opentelemetry.io/otel/exporters/stdout/stdoutmetric" "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" "go.opentelemetry.io/otel/log/global" "go.opentelemetry.io/otel/metric" "go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/sdk/log" - sdkmetric "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/resource" sdktrace "go.opentelemetry.io/otel/sdk/trace" semconv "go.opentelemetry.io/otel/semconv/v1.26.0" @@ -45,19 +43,50 @@ func Init(ctx context.Context, name string) (shutdown func(context.Context) erro } func Meter() metric.Meter { return meter } -func Error(err error, v ...any) { +// func Error(err error, v ...any) { +// if err == nil { +// return +// } +// fmt.Println("ERR:", append([]any{err}, v...)) +// logger.Error(err.Error(), v...) +// } +// func Info(msg string, v ...any) { fmt.Println(append([]any{msg}, v...)); logger.Info(msg, v...) } + +type spanny struct{ + trace.Span +} +func (s *spanny) RecordError(err error, options ...trace.EventOption) { if err == nil { return } - logger.Error(err.Error(), v...) + ec := trace.NewEventConfig(options...) + + attrs := make([]any, len(ec.Attributes())) + for i, v := range ec.Attributes() { + attrs[i] = v + } + + fmt.Println(append([]any{"ERR:", err}, attrs...)...) + logger.Error(err.Error(), attrs...) + s.Span.RecordError(err, options...) } -func Info(msg string, v ...any) { logger.Info(msg, v...) } +func (s *spanny) AddEvent(name string, options ...trace.EventOption) { + ec := trace.NewEventConfig(options...) + + attrs := make([]any, len(ec.Attributes())) + for i, v := range ec.Attributes() { + attrs[i] = v + } + fmt.Println(append([]any{name}, attrs...)...) + logger.Info(name, attrs...) +} + func Span(ctx context.Context, opts ...trace.SpanStartOption) (context.Context, trace.Span) { name, attrs := Attrs() ctx, span := tracer.Start(ctx, name, opts...) span.SetAttributes(attrs...) - return ctx, span + return ctx, &spanny{span} } func Attrs() (string, []attribute.KeyValue) { var attrs []attribute.KeyValue @@ -192,17 +221,17 @@ func newTraceProvider(ctx context.Context, name string) (func(context.Context) e } func newMeterProvider(ctx context.Context, name string) (func(context.Context) error, error) { - metricExporter, err := stdoutmetric.New() - if err != nil { - return nil, err - } + // metricExporter, err := stdoutmetric.New() + // if err != nil { + // return nil, err + // } - meterProvider := sdkmetric.NewMeterProvider( - sdkmetric.WithReader(sdkmetric.NewPeriodicReader(metricExporter, - // Default is 1m. Set to 3s for demonstrative purposes. - sdkmetric.WithInterval(3*time.Second))), - ) - otel.SetMeterProvider(meterProvider) + // meterProvider := sdkmetric.NewMeterProvider( + // sdkmetric.WithReader(sdkmetric.NewPeriodicReader(metricExporter, + // // Default is 1m. Set to 3s for demonstrative purposes. + // sdkmetric.WithInterval(3*time.Second))), + // ) + // otel.SetMeterProvider(meterProvider) http.Handle("/metrics", promhttp.Handler()) return func(ctx context.Context) error { return nil }, nil diff --git a/main.go b/main.go index 03c0f6d..679e345 100644 --- a/main.go +++ b/main.go @@ -8,7 +8,6 @@ import ( "io" "os" "os/signal" - "runtime/debug" "strings" "go.opentelemetry.io/otel/metric" @@ -27,7 +26,7 @@ func main() { dbfile: env("XT_DBFILE", "file:twt.db"), baseFeed: env("XT_BASE_FEED", "feed"), Nick: env("XT_NICK", "xuu"), - URI: env("XT_URI", "https://txt.sour.is/users/xuu/twtxt.txt"), + URI: env("XT_URI", "https://txt.sour.is/user/xuu/twtxt.txt"), Listen: env("XT_LISTEN", ":8080"), }) @@ -41,9 +40,6 @@ func main() { m_up.Record(ctx, 1) defer m_up.Record(context.Background(), 0) - bi, _ := debug.ReadBuildInfo() - otel.Info(name, "version", bi.Main.Version) - err = run(console) if !errors.Is(err, context.Canceled) { console.IfFatal(err) diff --git a/otel.go b/otel.go deleted file mode 100644 index c9ecbf5..0000000 --- a/otel.go +++ /dev/null @@ -1,2 +0,0 @@ -package main - diff --git a/refresh-loop.go b/refresh-loop.go index 9905502..2625c53 100644 --- a/refresh-loop.go +++ b/refresh-loop.go @@ -1,14 +1,16 @@ package main import ( + "context" "errors" "fmt" "io" "os" "path/filepath" - "strings" "time" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.sour.is/xt/internal/otel" "go.yarn.social/lextwt" "go.yarn.social/types" @@ -27,61 +29,91 @@ func refreshLoop(c *console, app *appState) error { defer span.End() f := NewHTTPFetcher() - fetch, close := NewFuncPool(c.Context, 25, f.Fetch) + fetch, close := NewFuncPool(ctx, 25, f.Fetch) defer close() - db, err := app.DB() + db, err := app.DB(c) if err != nil { - otel.Error(err, "missing db") + span.RecordError(err) return err } + go processorLoop(ctx, db, fetch) + queue := app.queue - otel.Info("start refresh loop") - for c.Context.Err() == nil { - if queue.IsEmpty() { - otel.Info("load feeds") + span.AddEvent("start refresh loop") - it, err := loadFeeds(c.Context, db) + for ctx.Err() == nil { + if queue.IsEmpty() { + span.AddEvent("load feeds") + + it, err := loadFeeds(ctx, db) + span.RecordError(err) for f := range it { queue.Insert(&f) } - if err != nil { - otel.Error(err) - return err - } } + span.AddEvent("queue size", trace.WithAttributes(attribute.Int("size", int(queue.count)))) f := queue.ExtractMin() if f == nil { - otel.Info("sleeping for ", TenMinutes*time.Second) + span.AddEvent("sleeping for ", trace.WithAttributes(attribute.Int("seconds", int(TenMinutes)))) select { case <-time.After(TenMinutes * time.Second): - case <-c.Done(): return nil } continue } - otel.Info("queue size", queue.count, "next", f.URI, "next scan on", f.LastScanOn.Time.Format(time.RFC3339)) + span.AddEvent("next", trace.WithAttributes( + attribute.Int("size", int(queue.count)), + attribute.String("uri", f.URI), + attribute.String("scan on", f.LastScanOn.Time.Format(time.RFC3339)), + )) - if time.Until(f.LastScanOn.Time) > 2*time.Hour { - otel.Info("too soon", f.URI) + if time.Until(f.NextScanOn.Time) > 2*time.Hour { + span.AddEvent("too soon", trace.WithAttributes(attribute.String("uri", f.URI))) continue } - select { - case <-c.Done(): + case <-ctx.Done(): return nil - case t := <-time.After(time.Until(f.LastScanOn.Time)): - otel.Info("fetch", t.Format(time.RFC3339), f.Nick, f.URI) + case t := <-time.After(time.Until(f.NextScanOn.Time)): + span.AddEvent("fetch", trace.WithAttributes( + attribute.Int("size", int(queue.count)), + attribute.String("uri", f.URI), + attribute.String("timeout", t.Format(time.RFC3339)), + attribute.String("scan on", f.NextScanOn.Time.Format(time.RFC3339)), + )) + fetch.Fetch(f) + } + + } + span.RecordError(ctx.Err()) + + return ctx.Err() +} + +func processorLoop(ctx context.Context, db db, fetch *pool[*Feed, *Response]) { + ctx, span := otel.Span(ctx) + defer span.End() + + for ctx.Err() == nil { + select { + case <-ctx.Done(): + return case res := <-fetch.Out(): - otel.Info("got response:", res.Request.URI) f := res.Request + span.AddEvent("got response", trace.WithAttributes( + attribute.String("uri", f.URI), + attribute.String("scan on", f.NextScanOn.Time.Format(time.RFC3339)), + )) + f.LastScanOn.Time = time.Now() + f.LastScanOn.Valid = true err := res.err if res.err != nil { if errors.Is(err, ErrPermanentlyDead) { @@ -94,12 +126,10 @@ func refreshLoop(c *console, app *appState) error { f.RefreshRate = OneDay } - otel.Error(err) + span.RecordError(err) f.LastError.String, f.LastError.Valid = err.Error(), true - err = f.Save(c.Context, db) - if err != nil { - otel.Error(err) - } + err = f.Save(ctx, db) + span.RecordError(err) continue } @@ -107,15 +137,16 @@ func refreshLoop(c *console, app *appState) error { f.ETag.String, f.ETag.Valid = res.ETag(), true f.LastModified.Time, f.LastModified.Valid = res.LastModified(), true + span.AddEvent("read feed") cpy, err := os.OpenFile(filepath.Join("feeds", urlNS.UUID5(f.URI).MarshalText()), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) if err != nil { - otel.Error(fmt.Errorf("%w: %w", ErrParseFailed, err)) + span.RecordError(fmt.Errorf("%w: %w", ErrParseFailed, err)) f.LastError.String, f.LastError.Valid = err.Error(), true f.RefreshRate = OneDay - err = f.Save(c.Context, db) - otel.Error(err) + err = f.Save(ctx, db) + span.RecordError(err) continue } @@ -123,55 +154,37 @@ func refreshLoop(c *console, app *appState) error { rdr = lextwt.TwtFixer(rdr) twtfile, err := lextwt.ParseFile(rdr, &types.Twter{Nick: f.Nick, URI: f.URI}) if err != nil { - otel.Error(fmt.Errorf("%w: %w", ErrParseFailed, err)) + span.RecordError(fmt.Errorf("%w: %w", ErrParseFailed, err)) f.LastError.String, f.LastError.Valid = err.Error(), true f.RefreshRate = OneDay - err = f.Save(c.Context, db) - otel.Error(err) + err = f.Save(ctx, db) + span.RecordError(err) continue } - - if prev, ok := twtfile.Info().GetN("prev", 0); f.FirstFetch && ok { - _, part, ok := strings.Cut(prev.Value(), " ") - if ok { - part = f.URI[:strings.LastIndex(f.URI, "/")+1] + part - queue.Insert(&Feed{ - FetchURI: part, - URI: f.URI, - Nick: f.Nick, - LastScanOn: f.LastScanOn, - RefreshRate: f.RefreshRate, - }) - } - } + cpy.Close() + span.AddEvent("parse complete", trace.WithAttributes(attribute.Int("count", twtfile.Twts().Len()))) err = storeFeed(ctx, db, twtfile) if err != nil { - otel.Error(err) + span.RecordError(err) f.LastError.String, f.LastError.Valid = err.Error(), true - err = f.Save(c.Context, db) + err = f.Save(ctx, db) - otel.Error(err) - return err + span.RecordError(err) + continue } - cpy.Close() - - f.LastScanOn.Time = time.Now() f.RefreshRate = TenMinutes f.LastError.String = "" - err = f.Save(c.Context, db) - if err != nil { - otel.Error(err) - return err - } + err = f.Save(ctx, db) + span.RecordError(err) } } - return c.Context.Err() + span.RecordError(ctx.Err()) } From 42a9b26b2263d86bdc13d17da1a3b5a11323c308 Mon Sep 17 00:00:00 2001 From: xuu Date: Mon, 24 Mar 2025 14:03:39 -0600 Subject: [PATCH 03/13] chore: changes for otel and fixes to loop --- .gitignore | 2 + app.go | 4 +- cmd/load/main.go | 92 +++++++++++++++++++++++++++++++++++++++++++ fetcher.go | 29 +++++++++++--- go.mod | 15 +++---- go.sum | 16 ++++++++ go.work.sum | 1 + internal/otel/otel.go | 58 +++++++++++++++++++++------ refresh-loop.go | 73 ++++++++++++++++++++++++++++------ 9 files changed, 252 insertions(+), 38 deletions(-) create mode 100644 cmd/load/main.go diff --git a/.gitignore b/.gitignore index 1949f0b..df69e5e 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ __debug* feeds/ /xt .env +*.txt +*.txt.xz \ No newline at end of file diff --git a/app.go b/app.go index 1688f82..791c01b 100644 --- a/app.go +++ b/app.go @@ -99,7 +99,7 @@ func run(c *console) error { c.Context = ctx wg.Go(func() error { - return refreshLoop(c, app) + return feedRefreshProcessor(c, app) }) go httpServer(c, app) @@ -132,7 +132,7 @@ func (app *appState) DB(ctx context.Context) (db, error) { db := db{Params: make(map[string]string)} db.DB, err = otelsql.Open(app.args.dbtype, app.args.dbfile, otelsql.WithAttributes(semconv.DBSystemSqlite), - otelsql.WithDBName("mydb")) + otelsql.WithDBName("xt")) if err != nil { return db, err } diff --git a/cmd/load/main.go b/cmd/load/main.go new file mode 100644 index 0000000..4ba59ef --- /dev/null +++ b/cmd/load/main.go @@ -0,0 +1,92 @@ +package main + +import ( + "context" + "database/sql" + "fmt" + "hash/fnv" + "iter" + "os" + + "github.com/oklog/ulid/v2" + "github.com/uptrace/opentelemetry-go-extra/otelsql" + "go.yarn.social/lextwt" + "go.yarn.social/types" +) + +func main() { + in := os.Stdin + if len(os.Args) != 2 { + fmt.Fprint(os.Stderr, "usage: ", os.Args[0], "[db file]") + } + + db, err := DB(context.Background(), os.Args[1]) + if err != nil { + panic(err) + } + _ = db + + for line := range lextwt.IterRegistry(in) { + _ = line + } +} + +const MaxVariableNumber = 32766 + +func DB(ctx context.Context, cnx string) (*sql.DB, error) { + // return sql.Open(app.args.dbtype, app.args.dbfile) + + db, err := otelsql.Open("sqlite", cnx) + if err != nil { + return db, err + } + + return db, err +} + +func makeULID(twt types.Twt) ulid.ULID { + h64 := fnv.New64a() + h16 := fnv.New32a() + text := []byte(fmt.Sprintf("%+l", twt)) + b := make([]byte, 10) + copy(b, h16.Sum(text)[:2]) + copy(b[2:], h64.Sum(text)) + u := ulid.ULID{} + u.SetTime(ulid.Timestamp(twt.Created())) + u.SetEntropy(b) + + return u +} + +func chunk(args []any, qry func(int) (string, int), maxArgs int) iter.Seq2[string, []any] { + _, size := qry(1) + itemsPerIter := maxArgs / size + + if len(args) < size { + return func(yield func(string, []any) bool) {} + } + + if len(args) < maxArgs { + return func(yield func(string, []any) bool) { + query, _ := qry(len(args) / size) + yield(query, args) + } + } + + return func(yield func(string, []any) bool) { + for len(args) > 0 { + if len(args) > maxArgs { + query, size := qry(itemsPerIter) + if !yield(query, args[:size]) { + return + } + args = args[size:] + continue + } + + query, _ := qry(len(args) / size) + yield(query, args) + return + } + } +} diff --git a/fetcher.go b/fetcher.go index 58e0cf1..1ec41a0 100644 --- a/fetcher.go +++ b/fetcher.go @@ -9,6 +9,8 @@ import ( "sync" "time" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" "go.sour.is/xt/internal/otel" ) @@ -50,10 +52,18 @@ func (r *Response) LastModified() time.Time { type httpFetcher struct { client *http.Client + + m_fetch_status metric.Int64Counter + m_fetch_second metric.Float64Histogram } func NewHTTPFetcher() *httpFetcher { + fetch_total, _ := otel.Meter().Int64Counter("xt_fetch_status_total") + fetch_second, _ := otel.Meter().Float64Histogram("xt_fetch_seconds") + return &httpFetcher{ + m_fetch_status: fetch_total, + m_fetch_second: fetch_second, client: &http.Client{ Transport: &http.Transport{ Proxy: http.ProxyFromEnvironment, @@ -75,6 +85,12 @@ func (f *httpFetcher) Fetch(ctx context.Context, request *Feed) *Response { ctx, span := otel.Span(ctx) defer span.End() + start := time.Now() + defer func() { + since := time.Since(start) + f.m_fetch_second.Record(ctx, since.Seconds()) + }() + defer fmt.Println("fetch done", request.URI) response := &Response{ Request: request, @@ -100,17 +116,24 @@ func (f *httpFetcher) Fetch(ctx context.Context, request *Feed) *Response { response.Response = res switch res.StatusCode { case 200: + f.m_fetch_status.Add(ctx, 1, metric.WithAttributes(attribute.String("status", "ok"))) case 304: + f.m_fetch_status.Add(ctx, 1, metric.WithAttributes(attribute.String("status", "not_modified"))) response.err = fmt.Errorf("%w: %s", ErrUnmodified, res.Status) case 400, 406, 429, 502, 503: + f.m_fetch_status.Add(ctx, 1, metric.WithAttributes(attribute.String("status", "temp_fail"))) response.err = fmt.Errorf("%w: %s", ErrTemporarilyDead, res.Status) case 403, 404, 410: + f.m_fetch_status.Add(ctx, 1, metric.WithAttributes(attribute.String("status", "perm_fail"))) + response.err = fmt.Errorf("%w: %s", ErrPermanentlyDead, res.Status) default: + f.m_fetch_status.Add(ctx, 1, metric.WithAttributes(attribute.Int("status", res.StatusCode))) + response.err = errors.New(res.Status) } @@ -134,7 +157,7 @@ func NewFuncPool[IN, OUT any]( var wg sync.WaitGroup in := make(chan IN, size) - out := make(chan OUT) + out := make(chan OUT, size) wg.Add(size) for range size { @@ -159,10 +182,6 @@ func NewFuncPool[IN, OUT any]( return case out <- r: span.AddEvent("sent queue") - case <-time.After(20 * time.Second): - fmt.Println("GOT STUCK", request) - span.AddEvent("GOT STUCK") - cancel() } } }() diff --git a/go.mod b/go.mod index 05e88af..32d1163 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/klauspost/compress v1.17.9 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/common v0.62.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 // indirect go.opentelemetry.io/proto/otlp v1.5.0 // indirect @@ -24,7 +24,7 @@ require ( golang.org/x/text v0.22.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250115164207-1a7da9e5054f // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect - google.golang.org/protobuf v1.36.3 // indirect + google.golang.org/protobuf v1.36.5 // indirect ) require ( @@ -33,8 +33,8 @@ require ( github.com/google/uuid v1.6.0 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/otel/log v0.10.0 - go.opentelemetry.io/otel/metric v1.34.0 - go.opentelemetry.io/otel/trace v1.34.0 + go.opentelemetry.io/otel/metric v1.35.0 + go.opentelemetry.io/otel/trace v1.35.0 ) require ( @@ -47,14 +47,15 @@ require ( github.com/uptrace/opentelemetry-go-extra/otelsql v0.3.2 github.com/writeas/go-strip-markdown/v2 v2.1.1 // indirect go.opentelemetry.io/contrib/bridges/otelslog v0.9.0 - go.opentelemetry.io/otel v1.34.0 + go.opentelemetry.io/otel v1.35.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 + go.opentelemetry.io/otel/exporters/prometheus v0.57.0 go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.10.0 go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.34.0 go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.34.0 - go.opentelemetry.io/otel/sdk v1.34.0 + go.opentelemetry.io/otel/sdk v1.35.0 go.opentelemetry.io/otel/sdk/log v0.10.0 - go.opentelemetry.io/otel/sdk/metric v1.34.0 + go.opentelemetry.io/otel/sdk/metric v1.35.0 go.yarn.social/types v0.0.0-20250108134258-ed75fa653ede golang.org/x/crypto v0.33.0 // indirect golang.org/x/sync v0.11.0 diff --git a/go.sum b/go.sum index 247ae07..6199cb5 100644 --- a/go.sum +++ b/go.sum @@ -48,6 +48,8 @@ github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= +github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= @@ -66,12 +68,16 @@ go.opentelemetry.io/contrib/bridges/otelslog v0.9.0 h1:N+78eXSlu09kii5nkiM+01Ybt go.opentelemetry.io/contrib/bridges/otelslog v0.9.0/go.mod h1:/2KhfLAhtQpgnhIk1f+dftA3fuuMcZjiz//Dc9yfaEs= go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY= go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI= +go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= +go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.6.0 h1:QSKmLBzbFULSyHzOdO9JsN9lpE4zkrz1byYGmJecdVE= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.6.0/go.mod h1:sTQ/NH8Yrirf0sJ5rWqVu+oT82i4zL9FaF6rWcqnptM= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 h1:OeNbIYk/2C15ckl7glBlOBp5+WlYsOElzTNmiPW/x60= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0/go.mod h1:7Bept48yIeqxP2OZ9/AqIpYS94h2or0aB4FypJTc8ZM= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 h1:tgJ0uaNS4c98WRNUEx5U3aDlrDOI5Rs+1Vifcw4DJ8U= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0/go.mod h1:U7HYyW0zt/a9x5J1Kjs+r1f/d4ZHnYFclhYY2+YbeoE= +go.opentelemetry.io/otel/exporters/prometheus v0.57.0 h1:AHh/lAP1BHrY5gBwk8ncc25FXWm/gmmY3BX258z5nuk= +go.opentelemetry.io/otel/exporters/prometheus v0.57.0/go.mod h1:QpFWz1QxqevfjwzYdbMb4Y1NnlJvqSGwyuU0B4iuc9c= go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.10.0 h1:GKCEAZLEpEf78cUvudQdTg0aET2ObOZRB2HtXA0qPAI= go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.10.0/go.mod h1:9/zqSWLCmHT/9Jo6fYeUDRRogOLL60ABLsHWS99lF8s= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.34.0 h1:czJDQwFrMbOr9Kk+BPo1y8WZIIFIK58SA1kykuVeiOU= @@ -82,14 +88,22 @@ go.opentelemetry.io/otel/log v0.10.0 h1:1CXmspaRITvFcjA4kyVszuG4HjA61fPDxMb7q3Bu go.opentelemetry.io/otel/log v0.10.0/go.mod h1:PbVdm9bXKku/gL0oFfUF4wwsQsOPlpo4VEqjvxih+FM= go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ= go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE= +go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= +go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= +go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= +go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= go.opentelemetry.io/otel/sdk/log v0.10.0 h1:lR4teQGWfeDVGoute6l0Ou+RpFqQ9vaPdrNJlST0bvw= go.opentelemetry.io/otel/sdk/log v0.10.0/go.mod h1:A+V1UTWREhWAittaQEG4bYm4gAZa6xnvVu+xKrIRkzo= go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= +go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= +go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= +go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= +go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -117,6 +131,8 @@ google.golang.org/grpc v1.70.0 h1:pWFv03aZoHzlRKHWicjsZytKAiYCtNS0dHbXnIdq7jQ= google.golang.org/grpc v1.70.0/go.mod h1:ofIJqVKDXx/JiXrwr2IG4/zwdH9txy3IlF40RmcJSQw= google.golang.org/protobuf v1.36.3 h1:82DV7MYdb8anAVi3qge1wSnMDrnKK7ebr+I0hHRN1BU= google.golang.org/protobuf v1.36.3/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/go.work.sum b/go.work.sum index cc7321d..063b17d 100644 --- a/go.work.sum +++ b/go.work.sum @@ -82,6 +82,7 @@ github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1 h1:G5FRp8JnTd7RQH5kemVNlMeyXQAztQ3mOWV95KxsXH8= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-dap v0.12.0 h1:rVcjv3SyMIrpaOoTAdFDyHs99CwVOItIJGKLQFQhNeM= github.com/google/go-dap v0.12.0/go.mod h1:tNjCASCm5cqePi/RVXXWEVqtnNLV1KTWtYOqu6rZNzc= github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= diff --git a/internal/otel/otel.go b/internal/otel/otel.go index 99fde23..6162517 100644 --- a/internal/otel/otel.go +++ b/internal/otel/otel.go @@ -10,21 +10,25 @@ import ( "runtime" "time" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "go.opentelemetry.io/contrib/bridges/otelslog" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + otelprom "go.opentelemetry.io/otel/exporters/prometheus" "go.opentelemetry.io/otel/exporters/stdout/stdoutlog" "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" - "go.opentelemetry.io/otel/log/global" + "go.opentelemetry.io/otel/log/global" "go.opentelemetry.io/otel/metric" "go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/sdk/log" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/resource" sdktrace "go.opentelemetry.io/otel/sdk/trace" semconv "go.opentelemetry.io/otel/semconv/v1.26.0" + "go.opentelemetry.io/otel/trace" ) @@ -43,6 +47,7 @@ func Init(ctx context.Context, name string) (shutdown func(context.Context) erro } func Meter() metric.Meter { return meter } + // func Error(err error, v ...any) { // if err == nil { // return @@ -52,33 +57,37 @@ func Meter() metric.Meter { return meter } // } // func Info(msg string, v ...any) { fmt.Println(append([]any{msg}, v...)); logger.Info(msg, v...) } -type spanny struct{ +type spanny struct { trace.Span } -func (s *spanny) RecordError(err error, options ...trace.EventOption) { + +func (s *spanny) RecordError(err error, options ...trace.EventOption) { if err == nil { return } ec := trace.NewEventConfig(options...) - + attrs := make([]any, len(ec.Attributes())) for i, v := range ec.Attributes() { attrs[i] = v } - fmt.Println(append([]any{"ERR:", err}, attrs...)...) - logger.Error(err.Error(), attrs...) + fmt.Println(append([]any{"ERR:", err}, attrs...)...) + logger.Error(err.Error(), attrs...) s.Span.RecordError(err, options...) } -func (s *spanny) AddEvent(name string, options ...trace.EventOption) { +func (s *spanny) AddEvent(name string, options ...trace.EventOption) { ec := trace.NewEventConfig(options...) - - attrs := make([]any, len(ec.Attributes())) + + attrs := make([]any, 2*len(ec.Attributes())) for i, v := range ec.Attributes() { - attrs[i] = v + attrs[2*i] = v.Key + attrs[2*i+1] = v.Value.Emit() } - fmt.Println(append([]any{name}, attrs...)...) - logger.Info(name, attrs...) + fmt.Println(append([]any{name}, attrs...)...) + logger.Info(name, attrs...) + + s.Span.AddEvent(name, options...) } func Span(ctx context.Context, opts ...trace.SpanStartOption) (context.Context, trace.Span) { @@ -233,8 +242,31 @@ func newMeterProvider(ctx context.Context, name string) (func(context.Context) e // ) // otel.SetMeterProvider(meterProvider) + metricExporter, err := otelprom.New( + otelprom.WithRegisterer(prometheus.DefaultRegisterer), + + // OTEL default buckets assume you're using milliseconds. Substitute defaults + // appropriate for units of seconds. + otelprom.WithAggregationSelector(func(ik sdkmetric.InstrumentKind) sdkmetric.Aggregation { + switch ik { + case sdkmetric.InstrumentKindHistogram: + return sdkmetric.AggregationExplicitBucketHistogram{ + Boundaries: prometheus.DefBuckets, + NoMinMax: false, + } + default: + return sdkmetric.DefaultAggregationSelector(ik) + } + }), + ) + + p := sdkmetric.NewMeterProvider( + sdkmetric.WithReader(metricExporter), + ) + + otel.SetMeterProvider(p) http.Handle("/metrics", promhttp.Handler()) - return func(ctx context.Context) error { return nil }, nil + return func(ctx context.Context) error { return nil }, err } func newLoggerProvider(ctx context.Context, name string) (func(context.Context) error, error) { diff --git a/refresh-loop.go b/refresh-loop.go index 2625c53..c9d4b25 100644 --- a/refresh-loop.go +++ b/refresh-loop.go @@ -7,6 +7,7 @@ import ( "io" "os" "path/filepath" + "sort" "time" "go.opentelemetry.io/otel/attribute" @@ -24,12 +25,16 @@ const ( TwoMinutes = 60 ) -func refreshLoop(c *console, app *appState) error { +func feedRefreshProcessor(c *console, app *appState) error { ctx, span := otel.Span(c.Context) defer span.End() + sleeping_time, _ := otel.Meter().Int64Counter("xt_feed_sleep") + + queue_size, _ := otel.Meter().Int64Gauge("xt_feed_queue_size") + f := NewHTTPFetcher() - fetch, close := NewFuncPool(ctx, 25, f.Fetch) + fetch, close := NewFuncPool(ctx, 40, f.Fetch) defer close() db, err := app.DB(c) @@ -54,29 +59,40 @@ func refreshLoop(c *console, app *appState) error { queue.Insert(&f) } } - span.AddEvent("queue size", trace.WithAttributes(attribute.Int("size", int(queue.count)))) + span.AddEvent("queue", trace.WithAttributes(attribute.Int("size", int(queue.count)))) + queue_size.Record(ctx, int64(queue.count)) f := queue.ExtractMin() if f == nil { + sleeping_time.Add(ctx, int64(TenMinutes)) span.AddEvent("sleeping for ", trace.WithAttributes(attribute.Int("seconds", int(TenMinutes)))) select { case <-time.After(TenMinutes * time.Second): case <-c.Done(): return nil } + span.End() + continue } span.AddEvent("next", trace.WithAttributes( attribute.Int("size", int(queue.count)), attribute.String("uri", f.URI), - attribute.String("scan on", f.LastScanOn.Time.Format(time.RFC3339)), + attribute.String("last scan on", f.LastScanOn.Time.Format(time.RFC3339)), + attribute.String("next scan on", f.NextScanOn.Time.Format(time.RFC3339)), )) - if time.Until(f.NextScanOn.Time) > 2*time.Hour { + until := time.Until(f.NextScanOn.Time) + + if until > 2*time.Hour { span.AddEvent("too soon", trace.WithAttributes(attribute.String("uri", f.URI))) + span.End() + continue } + + sleeping_time.Add(ctx, until.Milliseconds()) select { case <-ctx.Done(): return nil @@ -85,12 +101,11 @@ func refreshLoop(c *console, app *appState) error { attribute.Int("size", int(queue.count)), attribute.String("uri", f.URI), attribute.String("timeout", t.Format(time.RFC3339)), - attribute.String("scan on", f.NextScanOn.Time.Format(time.RFC3339)), + attribute.String("next scan on", f.NextScanOn.Time.Format(time.RFC3339)), )) - - fetch.Fetch(f) } + fetch.Fetch(f) } span.RecordError(ctx.Err()) @@ -101,6 +116,10 @@ func processorLoop(ctx context.Context, db db, fetch *pool[*Feed, *Response]) { ctx, span := otel.Span(ctx) defer span.End() + process_in_total, _ := otel.Meter().Int64Counter("xt_processed_in_total") + process_out_total, _ := otel.Meter().Int64Counter("xt_processed_out_total") + twts_histogram, _ := otel.Meter().Float64Histogram("xt_twt1k_bucket") + for ctx.Err() == nil { select { case <-ctx.Done(): @@ -112,6 +131,7 @@ func processorLoop(ctx context.Context, db db, fetch *pool[*Feed, *Response]) { attribute.String("scan on", f.NextScanOn.Time.Format(time.RFC3339)), )) + process_in_total.Add(ctx, 1) f.LastScanOn.Time = time.Now() f.LastScanOn.Valid = true err := res.err @@ -153,6 +173,8 @@ func processorLoop(ctx context.Context, db db, fetch *pool[*Feed, *Response]) { rdr := io.TeeReader(res.Body, cpy) rdr = lextwt.TwtFixer(rdr) twtfile, err := lextwt.ParseFile(rdr, &types.Twter{Nick: f.Nick, URI: f.URI}) + cpy.Close() + if err != nil { span.RecordError(fmt.Errorf("%w: %w", ErrParseFailed, err)) @@ -164,8 +186,10 @@ func processorLoop(ctx context.Context, db db, fetch *pool[*Feed, *Response]) { continue } - cpy.Close() - span.AddEvent("parse complete", trace.WithAttributes(attribute.Int("count", twtfile.Twts().Len()))) + + count := twtfile.Twts().Len() + span.AddEvent("parse complete", trace.WithAttributes(attribute.Int("count", count))) + twts_histogram.Record(ctx, float64(count)/1000) err = storeFeed(ctx, db, twtfile) if err != nil { @@ -178,13 +202,40 @@ func processorLoop(ctx context.Context, db db, fetch *pool[*Feed, *Response]) { continue } - f.RefreshRate = TenMinutes + f.RefreshRate = checkTemp(twtfile.Twts()) f.LastError.String = "" err = f.Save(ctx, db) span.RecordError(err) + process_out_total.Add(ctx, 1) } } span.RecordError(ctx.Err()) } + +func checkTemp(twts types.Twts) int { + if len(twts) < 5 { + return OneDay + } + sort.Sort(sort.Reverse(twts)) + + now := time.Now() + + since_first := now.Sub(twts[0].Created()) + since_fifth := now.Sub(twts[4].Created()) + + if since_first < 2 * time.Hour || since_fifth < 8 * time.Hour { + return TwoMinutes + } + + if since_first < 4 * time.Hour || since_fifth < 16 * time.Hour{ + return TenMinutes + } + + if since_first < 8 * time.Hour || since_fifth < 32 * time.Hour{ + return TenMinutes + } + + return OneDay +} From 8ae61d4a270aab81696fbf8fe31fdc5551bfd4e3 Mon Sep 17 00:00:00 2001 From: xuu Date: Mon, 24 Mar 2025 17:01:50 -0600 Subject: [PATCH 04/13] chore: fix queue sort, add feed temp --- http.go | 4 ++-- refresh-loop.go | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/http.go b/http.go index b121bd8..c588eda 100644 --- a/http.go +++ b/http.go @@ -153,10 +153,10 @@ func httpServer(c *console, app *appState) error { http.HandleFunc("/queue", func(w http.ResponseWriter, r *http.Request) { lis := slices.Collect(app.queue.Iter()) sort.Slice(lis, func(i, j int) bool { - return lis[i].LastScanOn.Time.Before(lis[j].LastScanOn.Time) + return lis[i].NextScanOn.Time.Before(lis[j].LastScanOn.Time) }) for _, feed := range lis { - fmt.Fprintln(w, feed.State, feed.LastScanOn.Time.Format(time.RFC3339), feed.Nick, feed.URI) + fmt.Fprintln(w, feed.State, feed.NextScanOn.Time.Format(time.RFC3339), feed.Nick, feed.URI) } }) diff --git a/refresh-loop.go b/refresh-loop.go index c9d4b25..e1f0b99 100644 --- a/refresh-loop.go +++ b/refresh-loop.go @@ -91,12 +91,14 @@ func feedRefreshProcessor(c *console, app *appState) error { continue } - + span.AddEvent( + "till next", + trace.WithAttributes(attribute.String("time", until.String()))) sleeping_time.Add(ctx, until.Milliseconds()) select { case <-ctx.Done(): return nil - case t := <-time.After(time.Until(f.NextScanOn.Time)): + case t := <-time.After(until): span.AddEvent("fetch", trace.WithAttributes( attribute.Int("size", int(queue.count)), attribute.String("uri", f.URI), From 7ab409403f559802c1d2c09ddd7250489c8bcaec Mon Sep 17 00:00:00 2001 From: xuu Date: Mon, 24 Mar 2025 17:18:46 -0600 Subject: [PATCH 05/13] chore: add 500 status --- fetcher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fetcher.go b/fetcher.go index 1ec41a0..37d018a 100644 --- a/fetcher.go +++ b/fetcher.go @@ -122,7 +122,7 @@ func (f *httpFetcher) Fetch(ctx context.Context, request *Feed) *Response { f.m_fetch_status.Add(ctx, 1, metric.WithAttributes(attribute.String("status", "not_modified"))) response.err = fmt.Errorf("%w: %s", ErrUnmodified, res.Status) - case 400, 406, 429, 502, 503: + case 400, 406, 429, 500, 502, 503: f.m_fetch_status.Add(ctx, 1, metric.WithAttributes(attribute.String("status", "temp_fail"))) response.err = fmt.Errorf("%w: %s", ErrTemporarilyDead, res.Status) From dae57540e31f707d4f375ea96486f05353cdd116 Mon Sep 17 00:00:00 2001 From: xuu Date: Mon, 24 Mar 2025 18:28:57 -0600 Subject: [PATCH 06/13] chore: refine checkTemp --- refresh-loop.go | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/refresh-loop.go b/refresh-loop.go index e1f0b99..3ecef61 100644 --- a/refresh-loop.go +++ b/refresh-loop.go @@ -19,6 +19,7 @@ import ( const ( TenYear = 3153600000 + OneMonth = OneDay * 30 OneDay = 86400 OneHour = 3600 TenMinutes = 600 @@ -218,14 +219,12 @@ func processorLoop(ctx context.Context, db db, fetch *pool[*Feed, *Response]) { func checkTemp(twts types.Twts) int { if len(twts) < 5 { - return OneDay + return 7*OneDay } - sort.Sort(sort.Reverse(twts)) + sort.Sort(twts) - now := time.Now() - - since_first := now.Sub(twts[0].Created()) - since_fifth := now.Sub(twts[4].Created()) + since_first := -time.Until(twts[0].Created()) + since_fifth := -time.Until(twts[4].Created()) if since_first < 2 * time.Hour || since_fifth < 8 * time.Hour { return TwoMinutes @@ -236,8 +235,24 @@ func checkTemp(twts types.Twts) int { } if since_first < 8 * time.Hour || since_fifth < 32 * time.Hour{ - return TenMinutes + return 2*TenMinutes } - return OneDay + if since_first < 16 * time.Hour || since_fifth < 64 * time.Hour{ + return 4*TenMinutes + } + + if since_first < 24 * time.Hour || since_fifth < 128 * time.Hour{ + return OneDay + } + + if since_first < 48 * time.Hour || since_fifth < 256 * time.Hour{ + return 2*OneDay + } + + if since_first < 96 * time.Hour || since_fifth < 512 * time.Hour{ + return 7*OneDay + } + + return OneMonth } From 2de06ec4d936eb7d75b4469f679634fa0837f2b6 Mon Sep 17 00:00:00 2001 From: xuu Date: Mon, 24 Mar 2025 22:29:18 -0600 Subject: [PATCH 07/13] chore: adjust timing --- feed.go | 28 +++++++++++++--------------- refresh-loop.go | 2 +- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/feed.go b/feed.go index 17d980b..a907498 100644 --- a/feed.go +++ b/feed.go @@ -71,7 +71,7 @@ var ( state, last_scan_on, refresh_rate - ) + ) values (?, ?, ?, ?, ?, ?, ?)` + repeat + ` ON CONFLICT (feed_id) DO NOTHING`, r * 7 } @@ -92,26 +92,26 @@ var ( repeat = strings.Repeat(", (?, ?, ?, ?, ?, ?, ?)", r-1) } return ` - insert into twts - (feed_id, ulid, text, hash, conv, mentions, tags) + insert into twts + (feed_id, ulid, text, hash, conv, mentions, tags) values (?, ?, ?, ?, ?, ?, ?)` + repeat + ` ON CONFLICT (feed_id, ulid) DO NOTHING`, r * 7 } fetchFeeds = ` - select + select feed_id, - parent_id, - coalesce(hashing_uri, uri) hash_uri, + parent_id, + coalesce(hashing_uri, uri) hash_uri, uri, nick, - state, + state, last_scan_on, strftime( '%Y-%m-%dT%H:%M:%fZ', - coalesce(last_scan_on, '1901-01-01'), + coalesce(last_scan_on, '1901-01-01'), '+'||refresh_rate||' seconds' - ) next_scan_on, + ) next_scan_on, refresh_rate, last_modified_on, last_etag @@ -124,9 +124,9 @@ var ( where parent_id is null ) using (parent_id) where datetime( - coalesce(last_scan_on, '1901-01-01'), + coalesce(last_scan_on, '1901-01-01'), '+'||refresh_rate||' seconds' - ) < datetime(current_timestamp, '+10 minutes') + ) < datetime(current_timestamp, '+2 minutes') ` ) @@ -310,7 +310,7 @@ func storeFeed(ctx context.Context, db db, f types.TwtFile) error { TwtTime{Time: loadTS, Valid: true}, // last_scan_on refreshRate, // refresh_rate ) - + if prev, ok := f.Info().GetN("prev", 0); ok { _, part, ok := strings.Cut(prev.Value(), " ") if ok { @@ -319,7 +319,7 @@ func storeFeed(ctx context.Context, db db, f types.TwtFile) error { childID := urlNS.UUID5(part) args = append(args, - childID, // feed_id + childID, // feed_id feedID, // parent_id f.Twter().DomainNick(), // nick part, // uri @@ -463,5 +463,3 @@ func chunk(args []any, qry func(int) (string, int), maxArgs int) iter.Seq2[strin } } } - - diff --git a/refresh-loop.go b/refresh-loop.go index 3ecef61..c0789f6 100644 --- a/refresh-loop.go +++ b/refresh-loop.go @@ -65,7 +65,7 @@ func feedRefreshProcessor(c *console, app *appState) error { f := queue.ExtractMin() if f == nil { - sleeping_time.Add(ctx, int64(TenMinutes)) + sleeping_time.Add(ctx, int64(TwoMinutes)) span.AddEvent("sleeping for ", trace.WithAttributes(attribute.Int("seconds", int(TenMinutes)))) select { case <-time.After(TenMinutes * time.Second): From 2fdc43b7de9a5f83a712b1dba51557388be03b11 Mon Sep 17 00:00:00 2001 From: xuu Date: Tue, 25 Mar 2025 17:05:21 -0600 Subject: [PATCH 08/13] chore: add last twt on support --- feed.go | 20 +++++++++----- http.go | 31 +++++++++++++--------- refresh-loop.go | 69 +++++++++++++++++++++++++++++++++++++------------ 3 files changed, 84 insertions(+), 36 deletions(-) diff --git a/feed.go b/feed.go index a907498..409146a 100644 --- a/feed.go +++ b/feed.go @@ -32,6 +32,7 @@ type Feed struct { LastScanOn TwtTime RefreshRate int NextScanOn TwtTime + LastTwtOn TwtTime LastModified TwtTime LastError sql.NullString @@ -102,20 +103,25 @@ var ( select feed_id, parent_id, - coalesce(hashing_uri, uri) hash_uri, + coalesce(hashing_uri, uri) hash_uri, uri, nick, state, last_scan_on, strftime( - '%Y-%m-%dT%H:%M:%fZ', + '%Y-%m-%dT%H:%M:%fZ', coalesce(last_scan_on, '1901-01-01'), - '+'||refresh_rate||' seconds' - ) next_scan_on, + '+'||abs(refresh_rate + cast(random() % 30 as int))||' seconds' + ) next_scan_on, + coalesce(last_twt_on, '1901-01-01T00:00:00Z') last_twt_on, refresh_rate, last_modified_on, last_etag from feeds + left join ( + select feed_id, max(strftime('%Y-%m-%dT%H:%M:%fZ', (substring(text, 1, instr(text, ' ')-1)))) last_twt_on + from twts group by feed_id + ) using (feed_id) left join ( select feed_id parent_id, @@ -125,8 +131,8 @@ var ( ) using (parent_id) where datetime( coalesce(last_scan_on, '1901-01-01'), - '+'||refresh_rate||' seconds' - ) < datetime(current_timestamp, '+2 minutes') + '+'||abs(refresh_rate+cast(random()%30 as int))||' seconds' + ) < datetime(current_timestamp, '+3 minutes') ` ) @@ -180,6 +186,7 @@ func (f *Feed) Scan(res interface{ Scan(...any) error }) error { &f.State, &f.LastScanOn, &f.NextScanOn, + &f.LastTwtOn, &f.RefreshRate, &f.LastModified, &f.ETag, @@ -356,7 +363,6 @@ func storeFeed(ctx context.Context, db db, f types.TwtFile) error { } func (feed *Feed) MakeHTTPRequest(ctx context.Context) (*http.Request, error) { - feed.State = "fetch" if strings.Contains(feed.URI, "lublin.se") { return nil, fmt.Errorf("%w: permaban: %s", ErrPermanentlyDead, feed.URI) } diff --git a/http.go b/http.go index c588eda..0453b41 100644 --- a/http.go +++ b/http.go @@ -31,11 +31,11 @@ func httpServer(c *console, app *appState) error { _, span := otel.Span(r.Context()) defer span.End() - w.Header().Set("Content-Type", "text/html; charset=utf-8") + w.Header().Set("Content-Type", "text/plain; charset=utf-8") w.Write([]byte("ok")) }) - http.HandleFunc("/conv/{hash}", func(w http.ResponseWriter, r *http.Request) { + http.HandleFunc("/api/plain/conv/{hash}", func(w http.ResponseWriter, r *http.Request) { ctx, span := otel.Span(r.Context()) defer span.End() @@ -104,7 +104,7 @@ func httpServer(c *console, app *appState) error { reg.WriteTo(w) }) - http.HandleFunc("/users", func(w http.ResponseWriter, r *http.Request) { + http.HandleFunc("/api/plain/users", func(w http.ResponseWriter, r *http.Request) { ctx, span := otel.Span(r.Context()) defer span.End() @@ -112,14 +112,20 @@ func httpServer(c *console, app *appState) error { rows, err := db.QueryContext( ctx, - `SELECT - feed_id, - uri, + `SELECT + feed_id, + uri, nick, - last_scan_on + last_scan_on, + last_twt_on FROM feeds - where parent_id is null - order by nick, uri`, + left join ( + select feed_id, max(strftime('%Y-%m-%dT%H:%M:%fZ', (substring(text, 1, instr(text, ' ')-1)))) last_twt_on + from twts group by feed_id + ) using (feed_id) + where parent_id is null and state not in ('permanantly-dead', 'frozen') and last_twt_on is not null + order by nick, uri + `, ) if err != nil { span.RecordError(err) @@ -134,15 +140,16 @@ func httpServer(c *console, app *appState) error { URI string Nick string Dt TwtTime + LastTwtOn TwtTime } - err = rows.Scan(&o.FeedID, &o.URI, &o.Nick, &o.Dt) + err = rows.Scan(&o.FeedID, &o.URI, &o.Nick, &o.Dt, &o.LastTwtOn) if err != nil { span.RecordError(err) return } twts = append(twts, lextwt.NewTwt( types.NewTwter(o.Nick, o.URI), - lextwt.NewDateTime(o.Dt.Time, o.Dt.Time.Format(time.RFC3339)), + lextwt.NewDateTime(o.Dt.Time, o.LastTwtOn.Time.Format(time.RFC3339)), nil, )) } @@ -150,7 +157,7 @@ func httpServer(c *console, app *appState) error { reg.WriteTo(w) }) - http.HandleFunc("/queue", func(w http.ResponseWriter, r *http.Request) { + http.HandleFunc("/api/plain/queue", func(w http.ResponseWriter, r *http.Request) { lis := slices.Collect(app.queue.Iter()) sort.Slice(lis, func(i, j int) bool { return lis[i].NextScanOn.Time.Before(lis[j].LastScanOn.Time) diff --git a/refresh-loop.go b/refresh-loop.go index c0789f6..0edf3ab 100644 --- a/refresh-loop.go +++ b/refresh-loop.go @@ -23,7 +23,7 @@ const ( OneDay = 86400 OneHour = 3600 TenMinutes = 600 - TwoMinutes = 60 + TwoMinutes = 120 ) func feedRefreshProcessor(c *console, app *appState) error { @@ -66,9 +66,9 @@ func feedRefreshProcessor(c *console, app *appState) error { f := queue.ExtractMin() if f == nil { sleeping_time.Add(ctx, int64(TwoMinutes)) - span.AddEvent("sleeping for ", trace.WithAttributes(attribute.Int("seconds", int(TenMinutes)))) + span.AddEvent("sleeping for ", trace.WithAttributes(attribute.Int("seconds", int(TwoMinutes)))) select { - case <-time.After(TenMinutes * time.Second): + case <-time.After(TwoMinutes * time.Second): case <-c.Done(): return nil } @@ -93,7 +93,7 @@ func feedRefreshProcessor(c *console, app *appState) error { continue } span.AddEvent( - "till next", + "till next", trace.WithAttributes(attribute.String("time", until.String()))) sleeping_time.Add(ctx, until.Milliseconds()) select { @@ -140,13 +140,14 @@ func processorLoop(ctx context.Context, db db, fetch *pool[*Feed, *Response]) { err := res.err if res.err != nil { if errors.Is(err, ErrPermanentlyDead) { + f.State = "permanantly-dead" f.RefreshRate = TenYear } if errors.Is(err, ErrTemporarilyDead) { - f.RefreshRate = OneDay + f.RefreshRate, f.State = tsTemp(f.LastTwtOn.Time) } if errors.Is(err, ErrUnmodified) { - f.RefreshRate = OneDay + f.RefreshRate, f.State = tsTemp(f.LastTwtOn.Time) } span.RecordError(err) @@ -205,7 +206,7 @@ func processorLoop(ctx context.Context, db db, fetch *pool[*Feed, *Response]) { continue } - f.RefreshRate = checkTemp(twtfile.Twts()) + f.RefreshRate, f.State = checkTemp(twtfile.Twts()) f.LastError.String = "" err = f.Save(ctx, db) @@ -217,9 +218,9 @@ func processorLoop(ctx context.Context, db db, fetch *pool[*Feed, *Response]) { span.RecordError(ctx.Err()) } -func checkTemp(twts types.Twts) int { +func checkTemp(twts types.Twts) (int, State) { if len(twts) < 5 { - return 7*OneDay + return 7*OneDay, "cold" } sort.Sort(twts) @@ -227,32 +228,66 @@ func checkTemp(twts types.Twts) int { since_fifth := -time.Until(twts[4].Created()) if since_first < 2 * time.Hour || since_fifth < 8 * time.Hour { - return TwoMinutes + return TwoMinutes, "hot" } if since_first < 4 * time.Hour || since_fifth < 16 * time.Hour{ - return TenMinutes + return TenMinutes, "hot" } if since_first < 8 * time.Hour || since_fifth < 32 * time.Hour{ - return 2*TenMinutes + return 2*TenMinutes, "warm" } if since_first < 16 * time.Hour || since_fifth < 64 * time.Hour{ - return 4*TenMinutes + return 4*TenMinutes, "warm" } if since_first < 24 * time.Hour || since_fifth < 128 * time.Hour{ - return OneDay + return OneDay, "cold" } if since_first < 48 * time.Hour || since_fifth < 256 * time.Hour{ - return 2*OneDay + return 2*OneDay, "cold" } if since_first < 96 * time.Hour || since_fifth < 512 * time.Hour{ - return 7*OneDay + return 7*OneDay, "frozen" } - return OneMonth + return OneMonth, "frozen" +} + +func tsTemp(ts time.Time) (int, State) { + since_first := -time.Until(ts) + + if since_first < 2 * time.Hour { + return TwoMinutes, "hot" + } + + if since_first < 4 * time.Hour { + return TenMinutes, "hot" + } + + if since_first < 8 * time.Hour { + return 2*TenMinutes, "warm" + } + + if since_first < 16 * time.Hour { + return 4*TenMinutes, "warm" + } + + if since_first < 24 * time.Hour { + return OneDay, "cold" + } + + if since_first < 48 * time.Hour { + return 2*OneDay, "cold" + } + + if since_first < 96 * time.Hour { + return 7*OneDay, "frozen" + } + + return OneMonth, "frozen" } From 5c97bfb182db62a9ce9c64da339cd655b7dc98ba Mon Sep 17 00:00:00 2001 From: xuu Date: Tue, 25 Mar 2025 18:52:30 -0600 Subject: [PATCH 09/13] chore: add twt endpoints --- http.go | 89 ++++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 85 insertions(+), 4 deletions(-) diff --git a/http.go b/http.go index 0453b41..2adb74a 100644 --- a/http.go +++ b/http.go @@ -6,6 +6,7 @@ import ( "net/http" "slices" "sort" + "strconv" "strings" "time" @@ -104,6 +105,86 @@ func httpServer(c *console, app *appState) error { reg.WriteTo(w) }) + http.HandleFunc("/api/plain/twt", func(w http.ResponseWriter, r *http.Request) { + ctx, span := otel.Span(r.Context()) + defer span.End() + + limit := 100 + if v, ok := strconv.Atoi(r.URL.Query().Get("limit")); ok == nil { + limit = v + } + offset := 0 + if v, ok := strconv.Atoi(r.URL.Query().Get("offset")); ok == nil { + offset = v + } + + args := []any{limit, offset} + + uriqry := "" + if u := r.URL.Query().Get("uri"); u != "" { + uriqry = "and uri = ?" + args = append([]any{u}, args...) + } + + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + rows, err := db.QueryContext( + ctx, + `SELECT + feed_id, + hash, + conv, + nick, + uri, + text + FROM twts + JOIN ( + SELECT + feed_id, + nick, + uri + FROM feeds + where state not in ('frozen', 'permanantly-dead') + `+uriqry+` + ) using (feed_id) + order by ulid desc + limit ? + offset ? + `, args..., + ) + if err != nil { + span.RecordError(err) + return + } + defer rows.Close() + + var twts []types.Twt + for rows.Next() { + var o struct { + FeedID string + Hash string + Conv string + Dt string + Nick string + URI string + Text string + } + err = rows.Scan(&o.FeedID, &o.Hash, &o.Conv, &o.Nick, &o.URI, &o.Text) + if err != nil { + span.RecordError(err) + return + } + twter := types.NewTwter(o.Nick, o.URI) + o.Text = strings.ReplaceAll(o.Text, "\n", "\u2028") + twt, _ := lextwt.ParseLine(o.Text, &twter) + twts = append(twts, twt) + } + var preamble lextwt.Comments + preamble = append(preamble, lextwt.NewComment("# self = /api/plain/twts")) + + reg := lextwt.NewTwtRegistry(preamble, twts) + reg.WriteTo(w) + }) + http.HandleFunc("/api/plain/users", func(w http.ResponseWriter, r *http.Request) { ctx, span := otel.Span(r.Context()) defer span.End() @@ -136,10 +217,10 @@ func httpServer(c *console, app *appState) error { var twts []types.Twt for rows.Next() { var o struct { - FeedID string - URI string - Nick string - Dt TwtTime + FeedID string + URI string + Nick string + Dt TwtTime LastTwtOn TwtTime } err = rows.Scan(&o.FeedID, &o.URI, &o.Nick, &o.Dt, &o.LastTwtOn) From b34c9bc99fdd0216724455f0c6dfef4fad5c88ca Mon Sep 17 00:00:00 2001 From: xuu Date: Wed, 26 Mar 2025 18:54:44 -0600 Subject: [PATCH 10/13] chore: changes to feed --- feed.go | 11 ++++++++-- http.go | 62 +++++++++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 56 insertions(+), 17 deletions(-) diff --git a/feed.go b/feed.go index 409146a..0510a7d 100644 --- a/feed.go +++ b/feed.go @@ -322,9 +322,16 @@ func storeFeed(ctx context.Context, db db, f types.TwtFile) error { _, part, ok := strings.Cut(prev.Value(), " ") if ok { uri:= f.Twter().URI + if u, ok := f.Info().GetN("url", 0); ok { + uri = u.Value() + } + if u, ok := f.Info().GetN("uri", 0); ok { + uri = u.Value() + } + part = uri[:strings.LastIndex(uri, "/")+1] + part childID := urlNS.UUID5(part) - + fmt.Println("found prev", uri, part) args = append(args, childID, // feed_id feedID, // parent_id @@ -332,7 +339,7 @@ func storeFeed(ctx context.Context, db db, f types.TwtFile) error { part, // uri "once", // state nil, // last_scan_on - refreshRate, // refresh_rate + 0, // refresh_rate ) } } diff --git a/http.go b/http.go index 2adb74a..9b49b5a 100644 --- a/http.go +++ b/http.go @@ -109,34 +109,36 @@ func httpServer(c *console, app *appState) error { ctx, span := otel.Span(r.Context()) defer span.End() + args := make([]any, 0, 3) + uriarg := "" + uri := r.URL.Query().Get("uri") + if uri != "" { + uriarg = "and uri = ?" + args = append(args, uri) + } + limit := 100 if v, ok := strconv.Atoi(r.URL.Query().Get("limit")); ok == nil { limit = v } + offset := 0 if v, ok := strconv.Atoi(r.URL.Query().Get("offset")); ok == nil { offset = v } - - args := []any{limit, offset} - - uriqry := "" - if u := r.URL.Query().Get("uri"); u != "" { - uriqry = "and uri = ?" - args = append([]any{u}, args...) - } + args = append(args, limit, offset) w.Header().Set("Content-Type", "text/plain; charset=utf-8") rows, err := db.QueryContext( ctx, - `SELECT - feed_id, - hash, - conv, + `SELECT + feed_id, + hash, + conv, nick, uri, text - FROM twts + FROM twts JOIN ( SELECT feed_id, @@ -144,7 +146,7 @@ func httpServer(c *console, app *appState) error { uri FROM feeds where state not in ('frozen', 'permanantly-dead') - `+uriqry+` + `+uriarg+` ) using (feed_id) order by ulid desc limit ? @@ -179,7 +181,12 @@ func httpServer(c *console, app *appState) error { twts = append(twts, twt) } var preamble lextwt.Comments - preamble = append(preamble, lextwt.NewComment("# self = /api/plain/twts")) + preamble = append(preamble, lextwt.NewComment("# I am the Watcher. I am your guide through this vast new twtiverse.")) + preamble = append(preamble, lextwt.NewComment("# self = /api/plain/twts"+mkqry(uri, limit, offset))) + preamble = append(preamble, lextwt.NewComment("# next = /api/plain/twts"+mkqry(uri, limit, offset+len(twts)))) + if offset > 0 { + preamble = append(preamble, lextwt.NewComment("# prev = /api/plain/twts"+mkqry(uri, limit, offset-limit))) + } reg := lextwt.NewTwtRegistry(preamble, twts) reg.WriteTo(w) @@ -271,3 +278,28 @@ func notAny(s string, chars string) bool { } return true } + + +func mkqry(uri string, limit, offset int) string { + qry := make([]string, 0, 3) + + if uri != "" { + qry = append(qry, "uri=" + uri) + } + + limit = min(100, max(1, limit)) + if limit != 100 { + qry = append(qry, fmt.Sprint("limit=", limit)) + } + + offset = max(0, offset) + if offset != 0 { + qry = append(qry, fmt.Sprint("offset=", offset)) + } + + if len(qry) == 0 { + return "" + } + + return "?" + strings.Join(qry, "&") +} From fe28b7c2ad4686982b2b16cb7a393e6fa6d29849 Mon Sep 17 00:00:00 2001 From: xuu Date: Wed, 26 Mar 2025 15:38:25 -0600 Subject: [PATCH 11/13] chore: go fmt --- feed.go | 19 +++++++++-------- http.go | 2 +- internal/otel/otel.go | 2 +- refresh-loop.go | 48 +++++++++++++++++++++---------------------- uuid.go | 6 +++--- 5 files changed, 40 insertions(+), 37 deletions(-) diff --git a/feed.go b/feed.go index 0510a7d..120475e 100644 --- a/feed.go +++ b/feed.go @@ -32,7 +32,7 @@ type Feed struct { LastScanOn TwtTime RefreshRate int NextScanOn TwtTime - LastTwtOn TwtTime + LastTwtOn TwtTime LastModified TwtTime LastError sql.NullString @@ -321,7 +321,7 @@ func storeFeed(ctx context.Context, db db, f types.TwtFile) error { if prev, ok := f.Info().GetN("prev", 0); ok { _, part, ok := strings.Cut(prev.Value(), " ") if ok { - uri:= f.Twter().URI + uri := f.Twter().URI if u, ok := f.Info().GetN("url", 0); ok { uri = u.Value() } @@ -333,13 +333,13 @@ func storeFeed(ctx context.Context, db db, f types.TwtFile) error { childID := urlNS.UUID5(part) fmt.Println("found prev", uri, part) args = append(args, - childID, // feed_id - feedID, // parent_id + childID, // feed_id + feedID, // parent_id f.Twter().DomainNick(), // nick - part, // uri - "once", // state - nil, // last_scan_on - 0, // refresh_rate + part, // uri + "once", // state + nil, // last_scan_on + 0, // refresh_rate ) } } @@ -373,6 +373,9 @@ func (feed *Feed) MakeHTTPRequest(ctx context.Context) (*http.Request, error) { if strings.Contains(feed.URI, "lublin.se") { return nil, fmt.Errorf("%w: permaban: %s", ErrPermanentlyDead, feed.URI) } + if strings.Contains(feed.URI, "enotty.dk") { + return nil, fmt.Errorf("%w: permaban: %s", ErrPermanentlyDead, feed.URI) + } req, err := http.NewRequestWithContext(ctx, "GET", feed.URI, nil) if err != nil { diff --git a/http.go b/http.go index 9b49b5a..47d443d 100644 --- a/http.go +++ b/http.go @@ -106,7 +106,7 @@ func httpServer(c *console, app *appState) error { }) http.HandleFunc("/api/plain/twt", func(w http.ResponseWriter, r *http.Request) { - ctx, span := otel.Span(r.Context()) + ctx, span := otel.Span(r.Context()) defer span.End() args := make([]any, 0, 3) diff --git a/internal/otel/otel.go b/internal/otel/otel.go index 6162517..95bcca0 100644 --- a/internal/otel/otel.go +++ b/internal/otel/otel.go @@ -20,7 +20,7 @@ import ( otelprom "go.opentelemetry.io/otel/exporters/prometheus" "go.opentelemetry.io/otel/exporters/stdout/stdoutlog" "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" - "go.opentelemetry.io/otel/log/global" + "go.opentelemetry.io/otel/log/global" "go.opentelemetry.io/otel/metric" "go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/sdk/log" diff --git a/refresh-loop.go b/refresh-loop.go index 0edf3ab..186e643 100644 --- a/refresh-loop.go +++ b/refresh-loop.go @@ -140,7 +140,7 @@ func processorLoop(ctx context.Context, db db, fetch *pool[*Feed, *Response]) { err := res.err if res.err != nil { if errors.Is(err, ErrPermanentlyDead) { - f.State = "permanantly-dead" + f.State = "permanantly-dead" f.RefreshRate = TenYear } if errors.Is(err, ErrTemporarilyDead) { @@ -220,39 +220,39 @@ func processorLoop(ctx context.Context, db db, fetch *pool[*Feed, *Response]) { func checkTemp(twts types.Twts) (int, State) { if len(twts) < 5 { - return 7*OneDay, "cold" + return 7 * OneDay, "cold" } sort.Sort(twts) since_first := -time.Until(twts[0].Created()) since_fifth := -time.Until(twts[4].Created()) - if since_first < 2 * time.Hour || since_fifth < 8 * time.Hour { + if since_first < 2*time.Hour || since_fifth < 8*time.Hour { return TwoMinutes, "hot" } - if since_first < 4 * time.Hour || since_fifth < 16 * time.Hour{ + if since_first < 4*time.Hour || since_fifth < 16*time.Hour { return TenMinutes, "hot" } - if since_first < 8 * time.Hour || since_fifth < 32 * time.Hour{ - return 2*TenMinutes, "warm" + if since_first < 8*time.Hour || since_fifth < 32*time.Hour { + return 2 * TenMinutes, "warm" } - if since_first < 16 * time.Hour || since_fifth < 64 * time.Hour{ - return 4*TenMinutes, "warm" + if since_first < 16*time.Hour || since_fifth < 64*time.Hour { + return 4 * TenMinutes, "warm" } - if since_first < 24 * time.Hour || since_fifth < 128 * time.Hour{ + if since_first < 24*time.Hour || since_fifth < 128*time.Hour { return OneDay, "cold" } - if since_first < 48 * time.Hour || since_fifth < 256 * time.Hour{ - return 2*OneDay, "cold" + if since_first < 48*time.Hour || since_fifth < 256*time.Hour { + return 2 * OneDay, "cold" } - if since_first < 96 * time.Hour || since_fifth < 512 * time.Hour{ - return 7*OneDay, "frozen" + if since_first < 96*time.Hour || since_fifth < 512*time.Hour { + return 7 * OneDay, "frozen" } return OneMonth, "frozen" @@ -261,32 +261,32 @@ func checkTemp(twts types.Twts) (int, State) { func tsTemp(ts time.Time) (int, State) { since_first := -time.Until(ts) - if since_first < 2 * time.Hour { + if since_first < 2*time.Hour { return TwoMinutes, "hot" } - if since_first < 4 * time.Hour { + if since_first < 4*time.Hour { return TenMinutes, "hot" } - if since_first < 8 * time.Hour { - return 2*TenMinutes, "warm" + if since_first < 8*time.Hour { + return 2 * TenMinutes, "warm" } - if since_first < 16 * time.Hour { - return 4*TenMinutes, "warm" + if since_first < 16*time.Hour { + return 4 * TenMinutes, "warm" } - if since_first < 24 * time.Hour { + if since_first < 24*time.Hour { return OneDay, "cold" } - if since_first < 48 * time.Hour { - return 2*OneDay, "cold" + if since_first < 48*time.Hour { + return 2 * OneDay, "cold" } - if since_first < 96 * time.Hour { - return 7*OneDay, "frozen" + if since_first < 96*time.Hour { + return 7 * OneDay, "frozen" } return OneMonth, "frozen" diff --git a/uuid.go b/uuid.go index bd3dc87..fd0b181 100644 --- a/uuid.go +++ b/uuid.go @@ -70,8 +70,8 @@ func (l *strList) Scan(value any) error { func (l strList) Value() (driver.Value, error) { arr := make([]string, len(l)) for i, v := range l { - arr[i] = "\""+v+"\"" + arr[i] = "\"" + v + "\"" } - return "["+strings.Join(arr, ",") +"]", nil -} \ No newline at end of file + return "[" + strings.Join(arr, ",") + "]", nil +} From a0614435ff35b52007178d759751af0c9582bcb8 Mon Sep 17 00:00:00 2001 From: xuu Date: Wed, 26 Mar 2025 18:52:45 -0600 Subject: [PATCH 12/13] chore: deps --- go.mod | 10 +-- go.sum | 10 +++ go.work | 6 -- go.work.sum | 192 ---------------------------------------------------- http.go | 40 ++++++----- 5 files changed, 39 insertions(+), 219 deletions(-) delete mode 100644 go.work delete mode 100644 go.work.sum diff --git a/go.mod b/go.mod index 32d1163..34a7a1b 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.23.2 require ( github.com/mattn/go-sqlite3 v1.14.24 go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.6.0 - go.yarn.social/lextwt v0.0.0-20250213063805-7adc6ca07564 + go.yarn.social/lextwt v0.1.5-0.20250327005027-02d9b44de4dd ) require ( @@ -21,7 +21,7 @@ require ( go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 // indirect go.opentelemetry.io/proto/otlp v1.5.0 // indirect golang.org/x/net v0.34.0 // indirect - golang.org/x/text v0.22.0 // indirect + golang.org/x/text v0.23.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250115164207-1a7da9e5054f // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect google.golang.org/protobuf v1.36.5 // indirect @@ -57,8 +57,8 @@ require ( go.opentelemetry.io/otel/sdk/log v0.10.0 go.opentelemetry.io/otel/sdk/metric v1.35.0 go.yarn.social/types v0.0.0-20250108134258-ed75fa653ede - golang.org/x/crypto v0.33.0 // indirect - golang.org/x/sync v0.11.0 - golang.org/x/sys v0.30.0 // indirect + golang.org/x/crypto v0.36.0 // indirect + golang.org/x/sync v0.12.0 + golang.org/x/sys v0.31.0 // indirect google.golang.org/grpc v1.70.0 // indirect ) diff --git a/go.sum b/go.sum index 6199cb5..08cd712 100644 --- a/go.sum +++ b/go.sum @@ -110,19 +110,29 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yarn.social/lextwt v0.0.0-20250213063805-7adc6ca07564 h1:z+IAMtxNKWcLNm9nLzJwHw6OPkV5JoQYmmFohaUvcKI= go.yarn.social/lextwt v0.0.0-20250213063805-7adc6ca07564/go.mod h1:JOPCOh+3bHv+BMaFZpKzw6soiXbIlZD5b2f7YKDDjqk= +go.yarn.social/lextwt v0.1.5-0.20250327005027-02d9b44de4dd h1:Np3zWtQ0GB9WhRFCPblaItLVtdy8Y35QKL+PUvRR/t8= +go.yarn.social/lextwt v0.1.5-0.20250327005027-02d9b44de4dd/go.mod h1:P36NPegLbhbFa1A0JOLsDyIQcdM0zdmx8kPKACXry4A= go.yarn.social/types v0.0.0-20250108134258-ed75fa653ede h1:XV9tuDQ605xxH4qIQPRHM1bOa7k0rJZ2RqA5kz2Nun4= go.yarn.social/types v0.0.0-20250108134258-ed75fa653ede/go.mod h1:+xnDkQ0T0S8emxWIsvxlCAoyF8gBaj0q81hr/VrKc0c= golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus= golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= +golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= +golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= google.golang.org/genproto/googleapis/api v0.0.0-20250115164207-1a7da9e5054f h1:gap6+3Gk41EItBuyi4XX/bp4oqJ3UwuIMl25yGinuAA= google.golang.org/genproto/googleapis/api v0.0.0-20250115164207-1a7da9e5054f/go.mod h1:Ic02D47M+zbarjYYUlK57y316f2MoN0gjAwI3f2S95o= google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f h1:OxYkA3wjPsZyBylwymxSHa7ViiW1Sml4ToBrncvFehI= diff --git a/go.work b/go.work deleted file mode 100644 index 774fc94..0000000 --- a/go.work +++ /dev/null @@ -1,6 +0,0 @@ -go 1.24.0 - -use ( - . - ../go-lextwt -) diff --git a/go.work.sum b/go.work.sum deleted file mode 100644 index 063b17d..0000000 --- a/go.work.sum +++ /dev/null @@ -1,192 +0,0 @@ -cel.dev/expr v0.19.0 h1:lXuo+nDhpyJSpWxpPVi5cPUwzKb+dsdOiw6IreM5yt0= -cel.dev/expr v0.19.0/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw= -cloud.google.com/go v0.26.0 h1:e0WKqKTd5BnrG8aKH3J3h+QvEIQtSUcf2n5UZ5ZgLtQ= -cloud.google.com/go/compute/metadata v0.5.2 h1:UxK4uu/Tn+I3p2dYWTfiX4wva7aYlKixAHn3fyqngqo= -cloud.google.com/go/compute/metadata v0.5.2/go.mod h1:C66sj2AluDcIqakBq/M8lw8/ybHgOZqin2obFxa/E5k= -dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= -dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0 h1:3c8yed4lgqTt+oTQ+JNMDo+F4xprBf+O/il4ZC0nRLw= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0/go.mod h1:obipzmGjfSjam60XLwGfqUkJsfiheAl+TUjG+4yzyPM= -github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= -github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= -github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= -github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= -github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78= -github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= -github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= -github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= -github.com/alecthomas/kingpin/v2 v2.4.0 h1:f48lwail6p8zpO1bC4TxtqACaGqHYA22qkHjHpqDjYY= -github.com/alecthomas/kingpin/v2 v2.4.0/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= -github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc= -github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= -github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= -github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= -github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= -github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= -github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWpi6yML8= -github.com/aymanbagabas/go-udiff v0.2.0/go.mod h1:RE4Ex0qsGkTAJoQdQQCA0uG+nAzJO/pI/QwceO5fgrA= -github.com/badgerodon/ioutil v0.0.0-20150716134133-06e58e34b867 h1:nsDNoesoGwPzPkcrR1w1uzPUtiqwCXoNnkWC7nUuRHI= -github.com/badgerodon/ioutil v0.0.0-20150716134133-06e58e34b867/go.mod h1:Ctq1YQi0dOq7QgBLZZ7p1Fr3IbAAqL/yMqDIHoe9WtE= -github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g= -github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= -github.com/charmbracelet/harmonica v0.2.0 h1:8NxJWRWg/bzKqqEaaeFNipOu77YR5t8aSwG4pgaUBiQ= -github.com/charmbracelet/harmonica v0.2.0/go.mod h1:KSri/1RMQOZLbw7AHqgcBycp8pgJnQMYYT8QZRqZ1Ao= -github.com/charmbracelet/x/exp/golden v0.0.0-20250213125511-a0c32e22e4fc h1:Tp8vprGbBhTAeyCNgrWPYIPyVEo9qmFRAcRhSdeind4= -github.com/charmbracelet/x/exp/golden v0.0.0-20250213125511-a0c32e22e4fc/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U= -github.com/charmbracelet/x/exp/teatest v0.0.0-20250213125511-a0c32e22e4fc h1:p4x5lOqNZEELobbdrm00kW5xrQwXZtIrUW8yTPK16aU= -github.com/charmbracelet/x/exp/teatest v0.0.0-20250213125511-a0c32e22e4fc/go.mod h1:ag+SpTUkiN/UuUGYPX3Ci4fR1oF3XX97PpGhiXK7i6U= -github.com/cilium/ebpf v0.11.0 h1:V8gS/bTCCjX9uUnkUFUpPsksM8n1lXBAvHcpiFk1X2Y= -github.com/cilium/ebpf v0.11.0/go.mod h1:WE7CZAnqOL2RouJ4f1uyNhqr2P4CCvXFIqdRDUgWsVs= -github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= -github.com/cloudflare/circl v1.5.0 h1:hxIWksrX6XN5a1L2TI/h53AGPhNHoUBo+TD1ms9+pys= -github.com/cloudflare/circl v1.5.0/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f h1:WBZRG4aNOuI15bLRrCgN8fCq8E5Xuty6jGbmSNEvSsU= -github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 h1:QVw89YDxXxEe+l8gU8ETbOasdwEV+avkR75ZzsVV9WI= -github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= -github.com/cosiner/argv v0.1.0 h1:BVDiEL32lwHukgJKP87btEPenzrrHUjajs/8yzaqcXg= -github.com/cosiner/argv v0.1.0/go.mod h1:EusR6TucWKX+zFgtdUsKT2Cvg45K5rtpCcWz4hK06d8= -github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/creack/pty v1.1.20 h1:VIPb/a2s17qNeQgDnkfZC35RScx+blkKF8GV68n80J4= -github.com/creack/pty v1.1.20/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= -github.com/cyphar/filepath-securejoin v0.3.4 h1:VBWugsJh2ZxJmLFSM06/0qzQyiQX2Qs0ViKrUAcqdZ8= -github.com/cyphar/filepath-securejoin v0.3.4/go.mod h1:8s/MCNJREmFK0H02MF6Ihv1nakJe4L/w3WZLHNkvlYM= -github.com/derekparker/trie v0.0.0-20230829180723-39f4de51ef7d h1:hUWoLdw5kvo2xCsqlsIBMvWUc1QCSsCYD2J2+Fg6YoU= -github.com/derekparker/trie v0.0.0-20230829180723-39f4de51ef7d/go.mod h1:C7Es+DLenIpPc9J6IYw4jrK0h7S9bKj4DNl8+KxGEXU= -github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= -github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= -github.com/envoyproxy/go-control-plane v0.13.1 h1:vPfJZCkob6yTMEgS+0TwfTUfbHjfy/6vOJ8hUWX/uXE= -github.com/envoyproxy/go-control-plane v0.13.1/go.mod h1:X45hY0mufo6Fd0KW3rqsGvQMw58jvjymeCzBU3mWyHw= -github.com/envoyproxy/protoc-gen-validate v1.1.0 h1:tntQDh69XqOCOZsDz0lVJQez/2L6Uu2PdjCQwWCJ3bM= -github.com/envoyproxy/protoc-gen-validate v1.1.0/go.mod h1:sXRDRVmzEbkM7CVcM06s9shE/m23dg3wzjl0UWqJ2q4= -github.com/felixge/fgprof v0.9.5 h1:8+vR6yu2vvSKn08urWyEuxx75NWPEvybbkBirEpsbVY= -github.com/felixge/fgprof v0.9.5/go.mod h1:yKl+ERSa++RYOs32d8K6WEXCB4uXdLls4ZaZPpayhMM= -github.com/go-delve/liner v1.2.3-0.20231231155935-4726ab1d7f62 h1:IGtvsNyIuRjl04XAOFGACozgUD7A82UffYxZt4DWbvA= -github.com/go-delve/liner v1.2.3-0.20231231155935-4726ab1d7f62/go.mod h1:biJCRbqp51wS+I92HMqn5H8/A0PAhxn2vyOT+JqhiGI= -github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= -github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= -github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU= -github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow= -github.com/go-git/go-git/v5 v5.12.0 h1:7Md+ndsjrzZxbddRDZjF14qK+NN56sy6wkqaVrjZtys= -github.com/go-git/go-git/v5 v5.12.0/go.mod h1:FTM9VKtnI2m65hNI/TenDDDnUf2Q9FHnXYjuz9i5OEY= -github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= -github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= -github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= -github.com/golang/glog v1.2.3 h1:oDTdz9f5VGVVNGu/Q7UXKWYsD0873HXLHdJUNBsSEKM= -github.com/golang/glog v1.2.3/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1 h1:G5FRp8JnTd7RQH5kemVNlMeyXQAztQ3mOWV95KxsXH8= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/go-dap v0.12.0 h1:rVcjv3SyMIrpaOoTAdFDyHs99CwVOItIJGKLQFQhNeM= -github.com/google/go-dap v0.12.0/go.mod h1:tNjCASCm5cqePi/RVXXWEVqtnNLV1KTWtYOqu6rZNzc= -github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= -github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 h1:y3N7Bm7Y9/CtpiVkw/ZWj6lSlDF3F74SfKwfTCer72Q= -github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= -github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= -github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= -github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= -github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= -github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= -github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= -github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= -github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= -github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= -github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= -github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= -github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= -github.com/pkg/profile v1.7.0 h1:hnbDkaNWPCLMO9wGLdBFTIZvzDrDfBM2072E1S9gJkA= -github.com/pkg/profile v1.7.0/go.mod h1:8Uer0jas47ZQMJ7VD+OHknK4YDY07LPUC6dEvqDjvNo= -github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= -github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= -github.com/rogpeppe/fastuuid v1.2.0 h1:Ppwyp6VYCF1nvBTXL3trRso7mXMlRrw9ooo375wvi2s= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= -github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sahilm/fuzzy v0.1.1 h1:ceu5RHF8DGgoi+/dR5PsECjCDH1BE3Fnmpo7aVXOdRA= -github.com/sahilm/fuzzy v0.1.1/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8ZK9Y= -github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= -github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/skeema/knownhosts v1.3.0 h1:AM+y0rI04VksttfwjkSTNQorvGqmwATnvnAHpSgc0LY= -github.com/skeema/knownhosts v1.3.0/go.mod h1:sPINvnADmT/qYH1kfv+ePMmOBTH6Tbl7b5LvTDjFK7M= -github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= -github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= -github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= -github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= -github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc= -github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= -github.com/yuin/goldmark v1.4.13 h1:fVcFKWvrslecOb/tg+Cc05dkeYx540o0FuFt3nUVDoE= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.opentelemetry.io/contrib/detectors/gcp v1.32.0 h1:P78qWqkLSShicHmAzfECaTgvslqHxblNE9j62Ws1NK8= -go.opentelemetry.io/contrib/detectors/gcp v1.32.0/go.mod h1:TVqo0Sda4Cv8gCIixd7LuLwW4EylumVWfhjZJjDD4DU= -go.starlark.net v0.0.0-20231101134539-556fd59b42f6 h1:+eC0F/k4aBLC4szgOcjd7bDTEnpxADJyWJE0yowgM3E= -go.starlark.net v0.0.0-20231101134539-556fd59b42f6/go.mod h1:LcLNIzVOMp4oV+uusnpk+VU+SzXaJakUuBjoCSWH5dM= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 h1:XQyxROzUlZH+WIQwySDgnISgOivlhjIEwaQaJEJrrN0= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE= -golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= -golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/telemetry v0.0.0-20240521205824-bda55230c457/go.mod h1:pRgIJT+bRLFKnoM1ldnzKoxTIn14Yxz928LQRYYgIN0= -golang.org/x/telemetry v0.0.0-20241106142447-58a1122356f5 h1:TCDqnvbBsFapViksHcHySl/sW4+rTGNIAoJJesHRuMM= -golang.org/x/telemetry v0.0.0-20241106142447-58a1122356f5/go.mod h1:8nZWdGp9pq73ZI//QJyckMQab3yq7hoWi7SI0UIusVI= -golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= -golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= -golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= -golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= -gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc h1:/hemPrYIhOhy8zYrNj+069zDB68us2sMGsfkFJO0iZs= -rsc.io/pdf v0.1.1 h1:k1MczvYDUvJBe93bYd7wrZLLUEcLZAuF824/I4e5Xr4= -rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/http.go b/http.go index 47d443d..19938b6 100644 --- a/http.go +++ b/http.go @@ -113,8 +113,9 @@ func httpServer(c *console, app *appState) error { uriarg := "" uri := r.URL.Query().Get("uri") if uri != "" { - uriarg = "and uri = ?" - args = append(args, uri) + feed_id := urlNS.UUID5(uri) + uriarg = "and feed_id = ?" + args = append(args, feed_id) } limit := 100 @@ -126,7 +127,6 @@ func httpServer(c *console, app *appState) error { if v, ok := strconv.Atoi(r.URL.Query().Get("offset")); ok == nil { offset = v } - args = append(args, limit, offset) w.Header().Set("Content-Type", "text/plain; charset=utf-8") rows, err := db.QueryContext( @@ -198,22 +198,30 @@ func httpServer(c *console, app *appState) error { w.Header().Set("Content-Type", "text/plain; charset=utf-8") + where := `where parent_id is null and state not in ('permanantly-dead', 'frozen') and last_twt_on is not null` + args := make([]any, 0) + if uri := r.URL.Query().Get("uri"); uri != "" { + where = `where feed_id = ? or parent_id = ?` + feed_id := urlNS.UUID5(uri) + args = append(args, feed_id, feed_id) + } + rows, err := db.QueryContext( ctx, `SELECT - feed_id, - uri, - nick, - last_scan_on, - last_twt_on - FROM feeds - left join ( - select feed_id, max(strftime('%Y-%m-%dT%H:%M:%fZ', (substring(text, 1, instr(text, ' ')-1)))) last_twt_on - from twts group by feed_id - ) using (feed_id) - where parent_id is null and state not in ('permanantly-dead', 'frozen') and last_twt_on is not null - order by nick, uri - `, + feed_id, + uri, + nick, + last_scan_on, + last_twt_on + FROM feeds + left join ( + select feed_id, max(strftime('%Y-%m-%dT%H:%M:%fZ', (substring(text, 1, instr(text, ' ')-1)))) last_twt_on + from twts group by feed_id + ) using (feed_id) + `+where+` + order by nick, uri + `,args..., ) if err != nil { span.RecordError(err) From 22d77d6aef64e60e8a5c557f82932504f6951ba9 Mon Sep 17 00:00:00 2001 From: xuu Date: Wed, 26 Mar 2025 19:03:20 -0600 Subject: [PATCH 13/13] chore: fix --- http.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http.go b/http.go index 19938b6..1a2a236 100644 --- a/http.go +++ b/http.go @@ -127,7 +127,7 @@ func httpServer(c *console, app *appState) error { if v, ok := strconv.Atoi(r.URL.Query().Get("offset")); ok == nil { offset = v } - + args = append(args, limit, offset) w.Header().Set("Content-Type", "text/plain; charset=utf-8") rows, err := db.QueryContext( ctx,