package main import ( "errors" "fmt" "io" "os" "path/filepath" "strings" "time" "go.yarn.social/lextwt" "go.yarn.social/types" ) const ( TenYear = 3153600000 OneDay = 86400 OneHour = 3600 TenMinutes = 600 TwoMinutes = 60 ) func refreshLoop(c console, app *appState) { defer c.abort() f := NewHTTPFetcher() fetch, close := NewFuncPool(c.Context, 25, f.Fetch) defer close() db, err := app.DB() if err != nil { c.Log("missing db") c.abort() return } queue := app.queue c.Log("start refresh loop") for c.Err() == nil { if queue.IsEmpty() { c.Log("load feeds") it, err := loadFeeds(c.Context, db) for f := range it { queue.Insert(&f) } if err != nil { c.Log(err) return } } f := queue.ExtractMin() if f == nil { c.Log("sleeping for ", TenMinutes*time.Second) select { case <-time.After(TenMinutes * time.Second): case <-c.Done(): return } continue } c.Log("queue size", queue.count, "next", f.URI, "next scan on", f.LastScanOn.Time.Format(time.RFC3339)) if time.Until(f.LastScanOn.Time) > 2*time.Hour { c.Log("too soon", f.URI) continue } select { case <-c.Done(): return case t := <-time.After(time.Until(f.LastScanOn.Time)): c.Log("fetch", t.Format(time.RFC3339), f.Nick, f.URI) fetch.Fetch(f) case res := <-fetch.Out(): c.Log("got response:", res.Request.URI) f := res.Request f.LastScanOn.Time = time.Now() err := res.err if res.err != nil { f.LastError.String, f.LastError.Valid = err.Error(), true if errors.Is(err, ErrPermanentlyDead) { f.RefreshRate = TenYear } if errors.Is(err, ErrTemporarilyDead) { f.RefreshRate = OneDay } if errors.Is(err, ErrUnmodified) { f.RefreshRate = OneDay } c.Log(err) err = f.Save(c.Context, db) if err != nil { c.Log(err) return } continue } f.ETag.String, f.ETag.Valid = res.ETag(), true f.LastModified.Time, f.LastModified.Valid = res.LastModified(), true cpy, err := os.OpenFile(filepath.Join("feeds", urlNS.UUID5(f.URI).MarshalText()), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) rdr := io.TeeReader(res.Body, cpy) twtfile, err := lextwt.ParseFile(rdr, &types.Twter{Nick: f.Nick, URI: f.URI}) if err != nil { c.Log(fmt.Errorf("%w: %w", ErrParseFailed, err)) f.RefreshRate = OneDay return } if prev, ok := twtfile.Info().GetN("prev", 0); f.FirstFetch && ok { _, part, ok := strings.Cut(prev.Value(), " ") if ok { part = f.URI[:strings.LastIndex(f.URI, "/")+1] + part queue.Insert(&Feed{ FetchURI: part, URI: f.URI, Nick: f.Nick, LastScanOn: f.LastScanOn, RefreshRate: f.RefreshRate, }) } } err = storeFeed(db, twtfile) if err != nil { c.Log(err) err = f.Save(c.Context, db) c.Log(err) return } cpy.Close() f.LastScanOn.Time = time.Now() f.RefreshRate = TenMinutes f.LastError.String = "" err = f.Save(c.Context, db) if err != nil { c.Log(err) return } } } }