package main import ( "context" "errors" "fmt" "net" "net/http" "strings" "time" ) var ( ErrUnmodified = errors.New("unmodified") ErrPermanentlyDead = errors.New("permanently dead") ErrTemporarilyDead = errors.New("temporarily dead") ErrParseFailed = errors.New("parse failed") ) type Response struct { *http.Response } func (r *Response) ETag() string { return r.Header.Get("ETag") } func (r *Response) Read(b []byte) (int, error) { return r.Body.Read(b) } // Close closes the Response.Body, which is necessary to free up resources func (r *Response) Close() { r.Body.Close() } func (r *Response) ContentType() string { return r.Header.Get("Content-Type") } func (r *Response) LastModified() time.Time { lastModified := time.Now() if lm, err := time.Parse(http.TimeFormat, r.Header.Get("Last-Modified")); err == nil { lastModified = lm } return lastModified } type httpFetcher struct { client *http.Client } func NewHTTPFetcher() *httpFetcher { return &httpFetcher{ client: &http.Client{ Transport: &http.Transport{ Proxy: http.ProxyFromEnvironment, DialContext: (&net.Dialer{ Timeout: 5 * time.Second, KeepAlive: 5 * time.Second, }).DialContext, ForceAttemptHTTP2: false, MaxIdleConns: 100, IdleConnTimeout: 10 * time.Second, TLSHandshakeTimeout: 10 * time.Second, ExpectContinueTimeout: 1 * time.Second, }, }, } } func (f *httpFetcher) Fetch(ctx context.Context, request *Feed) (*Response, error) { if strings.Contains(request.FetchURI, "lublin.se") { return nil, fmt.Errorf("%w: permaban: %s", ErrPermanentlyDead, request.URI) } req, err := http.NewRequestWithContext(ctx, "GET", request.FetchURI, nil) if err != nil { return nil, fmt.Errorf("creating HTTP request failed: %w", err) } req.Header.Add("Accept", "text/plain") if !request.LastModified.Valid { req.Header.Add("If-Modified-Since", request.LastModified.Time.Format(http.TimeFormat)) } if request.ETag.Valid { req.Header.Add("If-None-Match", request.ETag.String) } if request.DiscloseFeedURL != "" && request.DiscloseNick != "" { req.Header.Set("User-Agent", fmt.Sprintf("xt/%s (+%s; @%s)", request.Version, request.DiscloseFeedURL, request.DiscloseNick)) } else { req.Header.Set("User-Agent", fmt.Sprintf("xt/%s", request.Version)) } res, err := f.client.Do(req) if err != nil { if errors.Is(err, &net.DNSError{}) { return nil, fmt.Errorf("%w: %s", ErrTemporarilyDead, err) } return nil, fmt.Errorf("%w: %w", ErrTemporarilyDead, err) } response := &Response{ Response: res, } switch res.StatusCode { case 200: return response, nil case 304: return response, fmt.Errorf("%w: %s", ErrUnmodified, res.Status) case 400, 406, 502, 503: return response, fmt.Errorf("%w: %s", ErrTemporarilyDead, res.Status) case 403, 404, 410: return response, fmt.Errorf("%w: %s", ErrPermanentlyDead, res.Status) default: return response, errors.New(res.Status) } }