126 lines
2.9 KiB
Go
126 lines
2.9 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"net"
|
|
"net/http"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
var (
|
|
ErrUnmodified = errors.New("unmodified")
|
|
ErrPermanentlyDead = errors.New("permanently dead")
|
|
ErrTemporarilyDead = errors.New("temporarily dead")
|
|
ErrParseFailed = errors.New("parse failed")
|
|
)
|
|
|
|
type Response struct {
|
|
*http.Response
|
|
}
|
|
|
|
func (r *Response) ETag() string {
|
|
return r.Header.Get("ETag")
|
|
}
|
|
func (r *Response) Read(b []byte) (int, error) {
|
|
return r.Body.Read(b)
|
|
}
|
|
|
|
// Close closes the Response.Body, which is necessary to free up resources
|
|
func (r *Response) Close() {
|
|
r.Body.Close()
|
|
}
|
|
func (r *Response) ContentType() string {
|
|
return r.Header.Get("Content-Type")
|
|
}
|
|
func (r *Response) LastModified() time.Time {
|
|
lastModified := time.Now()
|
|
if lm, err := time.Parse(http.TimeFormat, r.Header.Get("Last-Modified")); err == nil {
|
|
lastModified = lm
|
|
}
|
|
|
|
return lastModified
|
|
}
|
|
|
|
type httpFetcher struct {
|
|
client *http.Client
|
|
}
|
|
|
|
func NewHTTPFetcher() *httpFetcher {
|
|
return &httpFetcher{
|
|
client: &http.Client{
|
|
Transport: &http.Transport{
|
|
Proxy: http.ProxyFromEnvironment,
|
|
DialContext: (&net.Dialer{
|
|
Timeout: 5 * time.Second,
|
|
KeepAlive: 5 * time.Second,
|
|
}).DialContext,
|
|
ForceAttemptHTTP2: false,
|
|
MaxIdleConns: 100,
|
|
IdleConnTimeout: 10 * time.Second,
|
|
TLSHandshakeTimeout: 10 * time.Second,
|
|
ExpectContinueTimeout: 1 * time.Second,
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
func (f *httpFetcher) Fetch(ctx context.Context, request *Feed) (*Response, error) {
|
|
if strings.Contains(request.URI, "lublin.se") {
|
|
return nil, fmt.Errorf("%w: permaban: %s", ErrPermanentlyDead, request.URI)
|
|
}
|
|
|
|
req, err := http.NewRequestWithContext(ctx, "GET", request.URI, nil)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("creating HTTP request failed: %w", err)
|
|
}
|
|
|
|
req.Header.Add("Accept", "text/plain")
|
|
|
|
if !request.LastModified.Valid {
|
|
req.Header.Add("If-Modified-Since", request.LastModified.Time.Format(http.TimeFormat))
|
|
}
|
|
|
|
if request.ETag.Valid {
|
|
req.Header.Add("If-None-Match", request.ETag.String)
|
|
}
|
|
|
|
if request.DiscloseFeedURL != "" && request.DiscloseNick != "" {
|
|
req.Header.Set("User-Agent", fmt.Sprintf("xt/%s (+%s; @%s)",
|
|
request.Version, request.DiscloseFeedURL, request.DiscloseNick))
|
|
} else {
|
|
req.Header.Set("User-Agent", fmt.Sprintf("xt/%s", request.Version))
|
|
}
|
|
|
|
res, err := f.client.Do(req)
|
|
if err != nil {
|
|
if errors.Is(err, &net.DNSError{}) {
|
|
return nil, fmt.Errorf("%w: %s", ErrTemporarilyDead, err)
|
|
}
|
|
return nil, fmt.Errorf("%w: %w", ErrPermanentlyDead, err)
|
|
}
|
|
|
|
response := &Response{
|
|
Response: res,
|
|
}
|
|
|
|
switch res.StatusCode {
|
|
case 200:
|
|
return response, nil
|
|
|
|
case 304:
|
|
return response, fmt.Errorf("%w: %s", ErrUnmodified, res.Status)
|
|
|
|
case 400, 406, 502, 503:
|
|
return response, fmt.Errorf("%w: %s", ErrTemporarilyDead, res.Status)
|
|
|
|
case 403, 404, 410:
|
|
return response, fmt.Errorf("%w: %s", ErrPermanentlyDead, res.Status)
|
|
|
|
default:
|
|
return response, errors.New(res.Status)
|
|
}
|
|
}
|