package main import ( "context" "errors" "fmt" "net/http" "slices" "sort" "strconv" "strings" "time" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "go.yarn.social/lextwt" "go.yarn.social/types" "go.sour.is/xt/internal/otel" ) const iAmTheWatcher = "I am the Watcher. I am your guide through this vast new twtiverse." const hostname = "https://watcher.sour.is" var PREAMBLE_DOCS = func() lextwt.Comments { c := add(nil, iAmTheWatcher) c = add(c, "") c = add(c,"Usage:") c = add(c," %s/api/plain/users View list of users and latest twt date.", hostname) c = add(c," %s/api/plain/twt View all twts in decending order.", hostname) c = add(c," %s/api/plain/conv/:hash View all twts for a conversation subject.", hostname) c = add(c,"") c = add(c,"Options:") c = add(c," uri Filter to show a specific users twts.") c = add(c," offset Start index for quey.") c = add(c," limit Count of items to return (going back in time).") return add(c,"") }() func httpServer(ctx context.Context, app *appState) error { ctx, span := otel.Span(ctx) defer span.End() span.AddEvent("start http server") db, err := app.DB(ctx) if err != nil { span.RecordError(fmt.Errorf("%w: missing db", err)) return err } http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { _, span := otel.Span(r.Context()) defer span.End() w.Header().Set("Content-Type", "text/plain; charset=utf-8") w.Write([]byte("ok")) }) http.HandleFunc("/api/plain", func(w http.ResponseWriter, r *http.Request) { reg := lextwt.NewTwtRegistry(PREAMBLE_DOCS, nil) reg.WriteTo(w) }) http.HandleFunc("/api/plain/conv/{hash}", func(w http.ResponseWriter, r *http.Request) { ctx, span := otel.Span(r.Context()) defer span.End() hash := r.PathValue("hash") if (len(hash) < 6 || len(hash) > 8) && !notAny(hash, "abcdefghijklmnopqrstuvwxyz234567") { w.WriteHeader(http.StatusBadRequest) return } w.Header().Set("Content-Type", "text/plain; charset=utf-8") rows, err := db.QueryContext( ctx, ` SELECT feed_id, hash, conv, nick, uri, text FROM twts JOIN ( SELECT feed_id, nick, uri FROM feeds ) using (feed_id) WHERE hash = $1 or conv = $1 order by ulid asc`, hash, ) if err != nil { span.RecordError(err) return } defer rows.Close() var twts []types.Twt for rows.Next() { var o struct { FeedID string Hash string Conv string Dt string Nick string URI string Text string } err = rows.Scan(&o.FeedID, &o.Hash, &o.Conv, &o.Nick, &o.URI, &o.Text) if err != nil { span.RecordError(err) return } twter := types.NewTwter(o.Nick, o.URI) o.Text = strings.ReplaceAll(o.Text, "\n", "\u2028") twt, _ := lextwt.ParseLine(o.Text, &twter) twts = append(twts, twt) } preamble := add(PREAMBLE_DOCS, "self = /conv/%s", hash) reg := lextwt.NewTwtRegistry(preamble, twts) reg.WriteTo(w) }) http.HandleFunc("/api/plain/twt", func(w http.ResponseWriter, r *http.Request) { ctx, span := otel.Span(r.Context()) defer span.End() args := make([]any, 0, 3) where := `where feed_id in (select feed_id from feeds where state != 'permanantly-dead')` uri := r.URL.Query().Get("uri") if uri != "" { feed_id := urlNS.UUID5(uri) where = "where feed_id = ?" args = append(args, feed_id) } limit := 100 if v, ok := strconv.Atoi(r.URL.Query().Get("limit")); ok == nil { limit = v } var offset int64 = 0 if v, ok := strconv.ParseInt(r.URL.Query().Get("offset"), 10, 64); ok == nil { offset = v } var end int64 err = db.QueryRowContext(ctx, ` select count(*) n from twts `+where+``, args...).Scan(&end) span.RecordError(err) if offset < 1 { offset += end } limit = min(100, max(1, limit)) offset = max(1, offset) args = append(args, limit, offset-int64(limit)) span.AddEvent("twts", trace.WithAttributes( attribute.Int("limit", limit), attribute.Int64("offset-end", offset), attribute.Int64("offset-start", offset-int64(limit)), attribute.Int64("max", end), )) qry := ` SELECT feed_id, hash, conv, coalesce(nick, 'nobody') nick, coalesce(uri, 'https://empty.txt') uri, text FROM twts join ( select feed_id, nick, uri from feeds ) using (feed_id) where rowid in ( select rowid from twts ` + where +` order by ulid asc limit ? offset ? ) order by ulid asc` fmt.Println(qry, args) w.Header().Set("Content-Type", "text/plain; charset=utf-8") rows, err := db.QueryContext( ctx, qry, args..., ) if err != nil { span.RecordError(err) return } defer rows.Close() var twts []types.Twt for rows.Next() { var o struct { FeedID string Hash string Conv string Dt string Nick string URI string Text string } err = rows.Scan(&o.FeedID, &o.Hash, &o.Conv, &o.Nick, &o.URI, &o.Text) if err != nil { span.RecordError(err) return } twter := types.NewTwter(o.Nick, o.URI) twt, _ := lextwt.ParseLine(o.Text, &twter) twts = append(twts, twt) } preamble := add(PREAMBLE_DOCS, "twt range = 1 %d", end) preamble = add(preamble, "self = %s/api/plain/twt%s", hostname, mkqry(uri, limit, offset)) if next := offset + int64(len(twts)); next < end { preamble = add(preamble, "next = %s/api/plain/twt%s", hostname, mkqry(uri, limit, next)) } if prev := offset - int64(limit); prev > 0 { preamble = add(preamble, "prev = %s/api/plain/twt%s", hostname, mkqry(uri, limit, prev)) } reg := lextwt.NewTwtRegistry(preamble, twts) reg.WriteTo(w) }) http.HandleFunc("/api/plain/users", func(w http.ResponseWriter, r *http.Request) { ctx, span := otel.Span(r.Context()) defer span.End() w.Header().Set("Content-Type", "text/plain; charset=utf-8") where := `where parent_id is null and state not in ('permanantly-dead', 'frozen') and last_twt_on is not null` args := make([]any, 0) if uri := r.URL.Query().Get("uri"); uri != "" { where = `where feed_id = ? or parent_id = ?` feed_id := urlNS.UUID5(uri) args = append(args, feed_id, feed_id) } qry := ` SELECT feed_id, uri, nick, last_scan_on, coalesce(last_twt_on, last_scan_on) last_twt_on FROM feeds left join last_twt_on using (feed_id) ` + where + ` order by nick, uri ` fmt.Println(qry, args) rows, err := db.QueryContext(ctx, qry, args...) if err != nil { span.RecordError(err) return } defer rows.Close() var twts []types.Twt for rows.Next() { var o struct { FeedID string URI string Nick string Dt TwtTime LastTwtOn TwtTime } err = rows.Scan(&o.FeedID, &o.URI, &o.Nick, &o.Dt, &o.LastTwtOn) if err != nil { span.RecordError(err) return } twts = append(twts, lextwt.NewTwt( types.NewTwter(o.Nick, o.URI), lextwt.NewDateTime(o.Dt.Time, o.LastTwtOn.Time.Format(time.RFC3339)), nil, )) } reg := lextwt.NewTwtRegistry(PREAMBLE_DOCS, twts) reg.WriteTo(w) }) http.HandleFunc("/api/plain/queue", func(w http.ResponseWriter, r *http.Request) { lis := slices.Collect(app.queue.Iter()) sort.Slice(lis, func(i, j int) bool { return lis[i].NextScanOn.Time.Before(lis[j].LastScanOn.Time) }) for _, feed := range lis { fmt.Fprintln(w, feed.State, feed.NextScanOn.Time.Format(time.RFC3339), feed.Nick, feed.URI) } }) srv := &http.Server{ Addr: app.args.Listen, Handler: http.DefaultServeMux, } app.AddCancel(srv.Shutdown) err = srv.ListenAndServe() if !errors.Is(err, http.ErrServerClosed) { span.RecordError(err) return err } return nil } func notAny(s string, chars string) bool { for _, c := range s { if !strings.ContainsRune(chars, c) { return false } } return true } func mkqry(uri string, limit int, offset int64) string { qry := make([]string, 0, 3) if uri != "" { qry = append(qry, "uri="+uri) } if limit != 100 { qry = append(qry, fmt.Sprint("limit=", limit)) } if offset != 0 { qry = append(qry, fmt.Sprint("offset=", offset)) } if len(qry) == 0 { return "" } return "?" + strings.Join(qry, "&") } func add(preamble lextwt.Comments, text string, v ...any) lextwt.Comments { if len(v) > 0 { text = fmt.Sprintf(text, v...) } return append(preamble, lextwt.NewComment("# "+text)) }