xt/refresh-loop.go

package main

import (
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"
	"time"

	"go.yarn.social/lextwt"
	"go.yarn.social/types"
)

const (
	TenYear    = 3153600000
	OneDay     = 86400
	OneHour    = 3600
	TenMinutes = 600
	TwoMinutes = 60
)

func refreshLoop(c console, app *appState) {
	defer c.abort()

	f := NewHTTPFetcher()
	fetch, close := NewFuncPool(c.Context, 25, f.Fetch)
	defer close()

	db, err := app.DB()
	if err != nil {
		c.Log("missing db")
		c.abort()
		return
	}

	queue := app.queue

	c.Log("start refresh loop")
	for c.Err() == nil {
		if queue.IsEmpty() {
			c.Log("load feeds")

			it, err := loadFeeds(c.Context, db)
			for f := range it {
				queue.Insert(&f)
			}
			if err != nil {
				c.Log(err)
				return
			}
		}

		f := queue.ExtractMin()
		if f == nil {
			c.Log("sleeping for ", TenMinutes*time.Second)
			select {
			case <-time.After(TenMinutes * time.Second):

			case <-c.Done():
				return
			}
			continue
		}

		c.Log("queue size", queue.count, "next", f.URI, "next scan on", f.LastScanOn.Time.Format(time.RFC3339))

		if time.Until(f.LastScanOn.Time) > 2*time.Hour {
			c.Log("too soon", f.URI)
			continue
		}

		select {
		case <-c.Done():
			return
		case t := <-time.After(time.Until(f.LastScanOn.Time)):
			c.Log("fetch", t.Format(time.RFC3339), f.Nick, f.URI)
			fetch.Fetch(f)
		case res := <-fetch.Out():
			c.Log("got response:", res.Request.URI)
			f := res.Request
			f.LastScanOn.Time = time.Now()
			err := res.err
			if res.err != nil {
				f.LastError.String, f.LastError.Valid = err.Error(), true
				if errors.Is(err, ErrPermanentlyDead) {
					f.RefreshRate = TenYear
				}
				if errors.Is(err, ErrTemporarilyDead) {
					f.RefreshRate = OneDay
				}
				if errors.Is(err, ErrUnmodified) {
					f.RefreshRate = OneDay
				}

				c.Log(err)
				err = f.Save(c.Context, db)
				if err != nil {
					c.Log(err)
					return
				}

				continue
			}

			f.ETag.String, f.ETag.Valid = res.ETag(), true
			f.LastModified.Time, f.LastModified.Valid = res.LastModified(), true

			cpy, err := os.OpenFile(filepath.Join("feeds", urlNS.UUID5(f.URI).MarshalText()), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
			rdr := io.TeeReader(res.Body, cpy)

			twtfile, err := lextwt.ParseFile(rdr, &types.Twter{Nick: f.Nick, URI: f.URI})
			if err != nil {
				c.Log(fmt.Errorf("%w: %w", ErrParseFailed, err))
				f.RefreshRate = OneDay
				return
			}

			if prev, ok := twtfile.Info().GetN("prev", 0); f.FirstFetch && ok {
				_, part, ok := strings.Cut(prev.Value(), " ")
				if ok {
					part = f.URI[:strings.LastIndex(f.URI, "/")+1] + part
					queue.Insert(&Feed{
						FetchURI:    part,
						URI:         f.URI,
						Nick:        f.Nick,
						LastScanOn:  f.LastScanOn,
						RefreshRate: f.RefreshRate,
					})
				}
			}

			err = storeFeed(db, twtfile)
			if err != nil {
				c.Log(err)

				err = f.Save(c.Context, db)
				c.Log(err)
				return
			}

			cpy.Close()

			f.LastScanOn.Time = time.Now()
			f.RefreshRate = TenMinutes
			f.LastError.String = ""

			err = f.Save(c.Context, db)
			if err != nil {
				c.Log(err)
				return
			}
		}
	}
}