xt/feed.go

package main

import (
	"cmp"
	"context"
	"database/sql"
	"fmt"
	"iter"
	"net/http"
	"net/url"
	"slices"
	"strings"
	"time"

	_ "embed"

	"go.yarn.social/lextwt"
	"go.yarn.social/types"
)

type Feed struct {
	FeedID      uuid
	FetchURI    string
	URI         string
	Nick        string
	LastScanOn  sql.NullTime
	RefreshRate int

	LastModified sql.NullTime
	LastError    sql.NullString
	ETag         sql.NullString

	Version         string
	DiscloseFeedURL string
	DiscloseNick    string
	FirstFetch      bool

	State State
}

type State string

const (
	PermanentlyDead State = "permanantly-dead"
	Frozen          State = "frozen"
	Cold            State = "cold"
	Warm            State = "warm"
	Hot             State = "hot"
)

var (
	//go:embed init.sql
	initSQL string

	insertFeed = `
		insert into feeds 
			(feed_id, uri, nick, last_scan_on, refresh_rate) 
		values (?, ?, ?, ?, ?) 
		ON CONFLICT (feed_id) DO NOTHING
	`

	insertTwt = `
		insert into twts 
			(feed_id, hash, conv, dt, text, mentions, tags) 
		values (?, ?, ?, ?, ?, ?, ?)
		ON CONFLICT (feed_id, hash) DO NOTHING
	`

	fetchFeeds = `
		select 
			feed_id, 
			uri, 
			nick, 
			last_scan_on, 
			refresh_rate,
			last_modified_on,
			last_etag
		from feeds
		where datetime(last_scan_on, '+'||refresh_rate||' seconds') < datetime(current_timestamp, '+10 minutes')
	`
	updateFeed = `
		update feeds set 
			last_scan_on = ?,
			refresh_rate = ?,
			last_modified_on = ?,
			last_etag = ?,
			last_error = ?
		where feed_id = ?
	`
)

func (f *Feed) Save(ctx context.Context, db *sql.DB) error {
	fmt.Println(f.FetchURI, " ", f.LastModified, " ", f.LastError)
	_, err := db.ExecContext(
		ctx,
		updateFeed,
		f.LastScanOn,
		f.RefreshRate,
		f.LastModified,
		f.ETag,
		f.LastError,
		f.FeedID,
	)
	return err
}

func (f *Feed) Scan(res interface{ Scan(...any) error }) error {
	f.State = "load"
	var err error

	f.Version = "0.0.1"
	err = res.Scan(
		&f.FeedID,
		&f.URI,
		&f.Nick,
		&f.LastScanOn,
		&f.RefreshRate,
		&f.LastModified,
		&f.ETag,
	)
	if err != nil {
		return err
	}

	if !f.LastScanOn.Valid {
		f.FirstFetch = true
		f.LastScanOn.Time = time.Now()
		f.LastScanOn.Valid = true
	} else {
		f.LastScanOn.Time = f.LastScanOn.Time.Add(time.Duration(f.RefreshRate) * time.Second)
	}

	f.FetchURI = f.URI
	return err
}

func loadFeeds(ctx context.Context, db *sql.DB) (iter.Seq[Feed], error) {
	var err error
	var res *sql.Rows

	res, err = db.QueryContext(ctx, fetchFeeds)

	if err != nil {
		return slices.Values([]Feed{}), err
	}

	return func(yield func(Feed) bool) {
		for res.Next() {
			var f Feed
			err = f.Scan(res)
			if err != nil {
				return
			}
			if !yield(f) {
				return
			}
		}
	}, err
}

func storeFeed(db *sql.DB, f types.TwtFile) error {
	loadTS := time.Now()
	refreshRate := 600

	feedID := urlNS.UUID5(cmp.Or(f.Twter().HashingURI, f.Twter().URI))

	tx, err := db.Begin()
	if err != nil {
		return err
	}

	followers := f.Info().GetAll("follow")
	followMap := make(map[string]string, len(followers))
	for _, f := range f.Info().GetAll("follow") {
		nick, uri, ok := strings.Cut(f.Value(), "http")
		if !ok {
			continue
		}
		nick = strings.TrimSpace(nick)
		uri = "http" + strings.TrimSpace(uri)

		if _, err := url.Parse(uri); err != nil {
			continue
		}

		followMap[nick] = uri
	}

	defer tx.Rollback()

	_, err = tx.Exec(
		insertFeed,
		feedID,
		f.Twter().HashingURI,
		f.Twter().DomainNick(),
		loadTS,
		refreshRate,
	)
	if err != nil {
		return err
	}

	for _, twt := range f.Twts() {
		mentions := make(uuids, 0, len(twt.Mentions()))
		for _, mention := range twt.Mentions() {
			followMap[mention.Twter().Nick] = mention.Twter().URI
			mentions = append(mentions, urlNS.UUID5(mention.Twter().URI))
		}

		tags := make(strList, 0, len(twt.Tags()))
		for _, tag := range twt.Tags() {
			tags = append(tags, tag.Text())
		}

		subject := twt.Subject()
		subjectTag := ""
		if subject != nil {
			if tag, ok := subject.Tag().(*lextwt.Tag); ok && tag != nil {
				subjectTag = tag.Text()
			}
		}

		_, err = tx.Exec(
			insertTwt,
			feedID,
			twt.Hash(),
			subjectTag,
			twt.Created(),
			fmt.Sprint(twt),
			mentions.ToStrList(),
			tags,
		)
		if err != nil {
			return err
		}
	}

	for nick, uri := range followMap {
		_, err = tx.Exec(
			insertFeed,
			urlNS.UUID5(uri),
			uri,
			nick,
			nil,
			refreshRate,
		)
		if err != nil {
			return err
		}
	}

	return tx.Commit()
}

func (feed *Feed) MakeHTTPRequest(ctx context.Context) (*http.Request, error) {
	feed.State = "fetch"
	if strings.Contains(feed.FetchURI, "lublin.se") {
		return nil, fmt.Errorf("%w: permaban: %s", ErrPermanentlyDead, feed.URI)
	}

	req, err := http.NewRequestWithContext(ctx, "GET", feed.FetchURI, nil)
	if err != nil {
		return nil, fmt.Errorf("creating HTTP request failed: %w", err)
	}

	req.Header.Add("Accept", "text/plain")

	if !feed.LastModified.Valid {
		req.Header.Add("If-Modified-Since", feed.LastModified.Time.Format(http.TimeFormat))
	}

	if feed.ETag.Valid {
		req.Header.Add("If-None-Match", feed.ETag.String)
	}

	if feed.DiscloseFeedURL != "" && feed.DiscloseNick != "" {
		req.Header.Set("User-Agent", fmt.Sprintf("xt/%s (+%s; @%s)",
			feed.Version, feed.DiscloseFeedURL, feed.DiscloseNick))
	} else {
		req.Header.Set("User-Agent", fmt.Sprintf("xt/%s", feed.Version))
	}

	return req, nil
}