chore: refactor app harness

feed.go

@@ -1,8 +1,21 @@
 package main
 
 import (
+	"cmp"
 	"context"
 	"database/sql"
+	"fmt"
+	"iter"
+	"net/http"
+	"net/url"
+	"slices"
+	"strings"
+	"time"
+
+	_ "embed"
+
+	"go.yarn.social/lextwt"
+	"go.yarn.social/types"
 )
 
 type Feed struct {
@@ -17,13 +30,67 @@ type Feed struct {
 	LastError    sql.NullString
 	ETag         sql.NullString
 
+	Version         string
 	DiscloseFeedURL string
 	DiscloseNick    string
+	FirstFetch      bool
 
-	Version string
+	State State
 }
 
+type State string
+
+const (
+	PermanentlyDead State = "permanantly-dead"
+	Frozen          State = "frozen"
+	Cold            State = "cold"
+	Warm            State = "warm"
+	Hot             State = "hot"
+)
+
+var (
+	//go:embed init.sql
+	initSQL string
+
+	insertFeed = `
+		insert into feeds 
+			(feed_id, uri, nick, last_scan_on, refresh_rate) 
+		values (?, ?, ?, ?, ?) 
+		ON CONFLICT (feed_id) DO NOTHING
+	`
+
+	insertTwt = `
+		insert into twts 
+			(feed_id, hash, conv, dt, text, mentions, tags) 
+		values (?, ?, ?, ?, ?, ?, ?)
+		ON CONFLICT (feed_id, hash) DO NOTHING
+	`
+
+	fetchFeeds = `
+		select 
+			feed_id, 
+			uri, 
+			nick, 
+			last_scan_on, 
+			refresh_rate,
+			last_modified_on,
+			last_etag
+		from feeds
+		where datetime(last_scan_on, '+'||refresh_rate||' seconds') < datetime(current_timestamp, '+10 minutes')
+	`
+	updateFeed = `
+		update feeds set 
+			last_scan_on = ?,
+			refresh_rate = ?,
+			last_modified_on = ?,
+			last_etag = ?,
+			last_error = ?
+		where feed_id = ?
+	`
+)
+
 func (f *Feed) Save(ctx context.Context, db *sql.DB) error {
+	fmt.Println(f.FetchURI, " ", f.LastModified, " ", f.LastError)
 	_, err := db.ExecContext(
 		ctx,
 		updateFeed,
@@ -36,3 +103,182 @@ func (f *Feed) Save(ctx context.Context, db *sql.DB) error {
 	)
 	return err
 }
+
+func (f *Feed) Scan(res interface{ Scan(...any) error }) error {
+	f.State = "load"
+	var err error
+
+	f.Version = "0.0.1"
+	err = res.Scan(
+		&f.FeedID,
+		&f.URI,
+		&f.Nick,
+		&f.LastScanOn,
+		&f.RefreshRate,
+		&f.LastModified,
+		&f.ETag,
+	)
+	if err != nil {
+		return err
+	}
+
+	if !f.LastScanOn.Valid {
+		f.FirstFetch = true
+		f.LastScanOn.Time = time.Now()
+		f.LastScanOn.Valid = true
+	} else {
+		f.LastScanOn.Time = f.LastScanOn.Time.Add(time.Duration(f.RefreshRate) * time.Second)
+	}
+
+	f.FetchURI = f.URI
+	return err
+}
+
+func loadFeeds(ctx context.Context, db *sql.DB) (iter.Seq[Feed], error) {
+	var err error
+	var res *sql.Rows
+
+	res, err = db.QueryContext(ctx, fetchFeeds)
+
+	if err != nil {
+		return slices.Values([]Feed{}), err
+	}
+
+	return func(yield func(Feed) bool) {
+		for res.Next() {
+			var f Feed
+			err = f.Scan(res)
+			if err != nil {
+				return
+			}
+			if !yield(f) {
+				return
+			}
+		}
+	}, err
+}
+
+func storeFeed(db *sql.DB, f types.TwtFile) error {
+	loadTS := time.Now()
+	refreshRate := 600
+
+	feedID := urlNS.UUID5(cmp.Or(f.Twter().HashingURI, f.Twter().URI))
+
+	tx, err := db.Begin()
+	if err != nil {
+		return err
+	}
+
+	followers := f.Info().GetAll("follow")
+	followMap := make(map[string]string, len(followers))
+	for _, f := range f.Info().GetAll("follow") {
+		nick, uri, ok := strings.Cut(f.Value(), "http")
+		if !ok {
+			continue
+		}
+		nick = strings.TrimSpace(nick)
+		uri = "http" + strings.TrimSpace(uri)
+
+		if _, err := url.Parse(uri); err != nil {
+			continue
+		}
+
+		followMap[nick] = uri
+	}
+
+	defer tx.Rollback()
+
+	_, err = tx.Exec(
+		insertFeed,
+		feedID,
+		f.Twter().HashingURI,
+		f.Twter().DomainNick(),
+		loadTS,
+		refreshRate,
+	)
+	if err != nil {
+		return err
+	}
+
+	for _, twt := range f.Twts() {
+		mentions := make(uuids, 0, len(twt.Mentions()))
+		for _, mention := range twt.Mentions() {
+			followMap[mention.Twter().Nick] = mention.Twter().URI
+			mentions = append(mentions, urlNS.UUID5(mention.Twter().URI))
+		}
+
+		tags := make(strList, 0, len(twt.Tags()))
+		for _, tag := range twt.Tags() {
+			tags = append(tags, tag.Text())
+		}
+
+		subject := twt.Subject()
+		subjectTag := ""
+		if subject != nil {
+			if tag, ok := subject.Tag().(*lextwt.Tag); ok && tag != nil {
+				subjectTag = tag.Text()
+			}
+		}
+
+		_, err = tx.Exec(
+			insertTwt,
+			feedID,
+			twt.Hash(),
+			subjectTag,
+			twt.Created(),
+			fmt.Sprint(twt),
+			mentions.ToStrList(),
+			tags,
+		)
+		if err != nil {
+			return err
+		}
+	}
+
+	for nick, uri := range followMap {
+		_, err = tx.Exec(
+			insertFeed,
+			urlNS.UUID5(uri),
+			uri,
+			nick,
+			nil,
+			refreshRate,
+		)
+		if err != nil {
+			return err
+		}
+	}
+
+	return tx.Commit()
+}
+
+func (feed *Feed) MakeHTTPRequest(ctx context.Context) (*http.Request, error) {
+	feed.State = "fetch"
+	if strings.Contains(feed.FetchURI, "lublin.se") {
+		return nil, fmt.Errorf("%w: permaban: %s", ErrPermanentlyDead, feed.URI)
+	}
+
+	req, err := http.NewRequestWithContext(ctx, "GET", feed.FetchURI, nil)
+	if err != nil {
+		return nil, fmt.Errorf("creating HTTP request failed: %w", err)
+	}
+
+	req.Header.Add("Accept", "text/plain")
+
+	if !feed.LastModified.Valid {
+		req.Header.Add("If-Modified-Since", feed.LastModified.Time.Format(http.TimeFormat))
+	}
+
+	if feed.ETag.Valid {
+		req.Header.Add("If-None-Match", feed.ETag.String)
+	}
+
+	if feed.DiscloseFeedURL != "" && feed.DiscloseNick != "" {
+		req.Header.Set("User-Agent", fmt.Sprintf("xt/%s (+%s; @%s)",
+			feed.Version, feed.DiscloseFeedURL, feed.DiscloseNick))
+	} else {
+		req.Header.Set("User-Agent", fmt.Sprintf("xt/%s", feed.Version))
+	}
+
+	return req, nil
+}