xt/fetcher.go

package main

import (
	"context"
	"errors"
	"fmt"
	"net"
	"net/http"
	"strings"
	"time"
)

var (
	ErrUnmodified      = errors.New("unmodified")
	ErrPermanentlyDead = errors.New("permanently dead")
	ErrTemporarilyDead = errors.New("temporarily dead")
	ErrParseFailed     = errors.New("parse failed")
)

type Response struct {
	*http.Response
}

func (r *Response) ETag() string {
	return r.Header.Get("ETag")
}
func (r *Response) Read(b []byte) (int, error) {
	return r.Body.Read(b)
}

// Close closes the Response.Body, which is necessary to free up resources
func (r *Response) Close() {
	r.Body.Close()
}
func (r *Response) ContentType() string {
	return r.Header.Get("Content-Type")
}
func (r *Response) LastModified() time.Time {
	lastModified := time.Now()
	if lm, err := time.Parse(http.TimeFormat, r.Header.Get("Last-Modified")); err == nil {
		lastModified = lm
	}

	return lastModified
}

type httpFetcher struct {
	client *http.Client
}

func NewHTTPFetcher() *httpFetcher {
	return &httpFetcher{
		client: &http.Client{
			Transport: &http.Transport{
				Proxy: http.ProxyFromEnvironment,
				DialContext: (&net.Dialer{
					Timeout:   5 * time.Second,
					KeepAlive: 5 * time.Second,
				}).DialContext,
				ForceAttemptHTTP2:     false,
				MaxIdleConns:          100,
				IdleConnTimeout:       10 * time.Second,
				TLSHandshakeTimeout:   10 * time.Second,
				ExpectContinueTimeout: 1 * time.Second,
			},
		},
	}
}

func (f *httpFetcher) Fetch(ctx context.Context, request *Feed) (*Response, error) {
	if strings.Contains(request.URI, "lublin.se") {
		return nil, fmt.Errorf("%w: permaban: %s", ErrPermanentlyDead, request.URI)
	}

	req, err := http.NewRequestWithContext(ctx, "GET", request.URI, nil)
	if err != nil {
		return nil, fmt.Errorf("creating HTTP request failed: %w", err)
	}

	req.Header.Add("Accept", "text/plain")

	if !request.LastModified.Valid {
		req.Header.Add("If-Modified-Since", request.LastModified.Time.Format(http.TimeFormat))
	}

	if request.ETag.Valid {
		req.Header.Add("If-None-Match", request.ETag.String)
	}

	if request.DiscloseFeedURL != "" && request.DiscloseNick != "" {
		req.Header.Set("User-Agent", fmt.Sprintf("xt/%s (+%s; @%s)",
			request.Version, request.DiscloseFeedURL, request.DiscloseNick))
	} else {
		req.Header.Set("User-Agent", fmt.Sprintf("xt/%s", request.Version))
	}

	res, err := f.client.Do(req)
	if err != nil {
		if errors.Is(err, &net.DNSError{}) {
			return nil, fmt.Errorf("%w: %s", ErrTemporarilyDead, err)
		}
		return nil, fmt.Errorf("%w: %w", ErrPermanentlyDead, err)
	}

	response := &Response{
		Response: res,
	}

	switch res.StatusCode {
	case 200:
		return response, nil

	case 304:
		return response, fmt.Errorf("%w: %s", ErrUnmodified, res.Status)

	case 400, 406, 502, 503:
		return response, fmt.Errorf("%w: %s", ErrTemporarilyDead, res.Status)

	case 403, 404, 410:
		return response, fmt.Errorf("%w: %s", ErrPermanentlyDead, res.Status)

	default:
		return response, errors.New(res.Status)
	}
}