indexer/scraper/transport.go

package scraper

import (
	"crypto/tls"
	"net"
	"net/http"
	"strings"
	"sync"
	"time"

	"github.com/quic-go/quic-go"
	"github.com/quic-go/quic-go/http3"
)

// scraperTransport is the shared HTTP stack used by every client in this
// package (robots.txt checks, type detection, and the forge scrapers). It
// composes:
//
//   - a single stdlib *http.Transport that negotiates HTTP/1.1 vs HTTP/2 over
//     TLS via ALPN automatically (no logic here);
//   - an HTTP/3 transport from quic-go, tried first on each host and falling
//     back to the stdlib transport if the host does not support h3.
//
// Protocol selection is "newest available first with automatic fallback":
// the first request to a host attempts HTTP/3, and falls back to HTTP/2 or
// HTTP/1.1 (negotiated via ALPN) if the host has no h3. A per-host negative
// cache remembers hosts that failed h3 so we don't keep re-triggering a
// doomed QUIC handshake on every request.
//
// Sharing one transport means idle keep-alive connections are reused across
// the whole scrape of a host, and each origin's negotiation (h3 or h1/h2) is
// done once for a while.
type scraperTransport struct {
	// h12 is the plain stdlib transport: HTTP/1.1 and HTTP/2.
	h12 *http.Transport
	// h3 is the HTTP/3 transport (lazily created, quic-go).
	h3 *http3.Transport

	mu     sync.Mutex
	cached map[string]h3Capability
	forced map[string]struct{} // hosts that try h3 even during a negative cool-down
}

// h3Capability is the cached knowledge about a single host's HTTP/3 support.
type h3Capability struct {
	capable  bool
	expireAt time.Time
}

const (
	// h3NegativeTTL is how long a failed h3 attempt is remembered, so a host
	// without h3 isn't re-triggered on every request. After it expires, h3 is
	// tried again (the host may have started serving it).
	h3NegativeTTL = time.Hour
	// h3HandshakeTimeout aborts a QUIC handshake (e.g. to a host with UDP
	// closed) instead of hanging the scrape. This bounds the per-host h3
	// attempt before we fall back to HTTP/2/1.1.
	h3HandshakeTimeout = 5 * time.Second
)

// sharedTransport is the single transport used by the whole process.
var sharedTransport = newScraperTransport()

// newScraperTransport builds the shared transport.
func newScraperTransport() *scraperTransport {
	t12 := &http.Transport{
		MaxIdleConns:          100,
		MaxIdleConnsPerHost:   10,
		IdleConnTimeout:       90 * time.Second,
		ExpectContinueTimeout: 1 * time.Second,
		DialContext: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 30 * time.Second,
		}).DialContext,
		// Our TLS config disables Go's implicit HTTP/2 enablement, so opt in
		// explicitly: stdlib then negotiates HTTP/2 via ALPN when the server
		// supports it, and falls back to HTTP/1.1 otherwise.
		ForceAttemptHTTP2: true,
		TLSClientConfig: &tls.Config{
			MinVersion: tls.VersionTLS12,
		},
	}
	return &scraperTransport{
		h12:    t12,
		cached: map[string]h3Capability{},
		forced: map[string]struct{}{},
	}
}

// httpClientFor returns an *http.Client that shares sharedTransport with a
// per-client timeout. Timeouts differ per use (robots probes are short, the
// SDK scrape is a bit more generous) but the underlying connection pool is
// the same.
func httpClientFor(timeout time.Duration) *http.Client {
	return &http.Client{
		Transport: sharedTransport,
		Timeout:   timeout,
	}
}

// EnableHTTP3 forces a host to keep trying HTTP/3 even while a previous
// failure is being remembered (during the negative cool-down). It is an
// override for operators who know a host serves h3. By default, h3 is already
// tried automatically on every host, so this is only needed to bypass the
// negative cache.
func EnableHTTP3(host string) {
	if host == "" {
		return
	}
	sharedTransport.forceH3(host)
}

// forceH3 marks a host as always trying h3 (overrides the negative cache).
func (t *scraperTransport) forceH3(host string) {
	t.mu.Lock()
	defer t.mu.Unlock()
	t.forced[host] = struct{}{}
}

// h3Transport lazily builds the quic-go HTTP/3 transport.
func (t *scraperTransport) h3Transport() *http3.Transport {
	if t.h3 != nil {
		return t.h3
	}
	t.mu.Lock()
	defer t.mu.Unlock()
	if t.h3 == nil {
		t.h3 = &http3.Transport{
			TLSClientConfig: &tls.Config{
				MinVersion: tls.VersionTLS12,
			},
			QUICConfig: &quic.Config{
				// Fail fast against hosts with UDP closed instead of hanging.
				HandshakeIdleTimeout: h3HandshakeTimeout,
				MaxIdleTimeout:       90 * time.Second,
			},
		}
	}
	return t.h3
}

// RoundTrip implements http.RoundTripper.
func (t *scraperTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	host := req.URL.Hostname()

	if t.shouldTryH3(host) {
		resp, err := t.h3Transport().RoundTrip(req)
		if err != nil {
			// h3 failed: cache the negative result and fall back to h1/h2 for
			// this very request so the caller sees a successful response.
			t.recordH3(host, false)
			return t.h12.RoundTrip(req)
		}
		t.observeAltSvc(resp, host)
		return resp, nil
	}

	resp, err := t.h12.RoundTrip(req)
	if err != nil {
		return nil, err
	}
	// If the h1/h2 response advertises h3, learn the capability so subsequent
	// requests to this host try h3. The current request stays on h1/h2.
	t.observeAltSvc(resp, host)
	return resp, nil
}

// shouldTryH3 reports whether a host should be attempted over h3 first.
// The default is yes (newest-available-first); it is only gated by an
// unexpired negative capability (the host recently failed h3) unless the
// host is forced.
func (t *scraperTransport) shouldTryH3(host string) bool {
	t.mu.Lock()
	defer t.mu.Unlock()
	if _, ok := t.forced[host]; ok {
		return true
	}
	c, ok := t.cached[host]
	if !ok {
		return true // unknown host: try h3 first
	}
	// A negative result rejects h3; a positive result is fine (and the
	// default is already to try). Either way, once expired, try again.
	if !c.capable {
		return time.Now().After(c.expireAt)
	}
	return true
}

// recordH3 stores a learned h3 capability for a host. A positive capability
// (h3 succeeded) prevents a negative cool-down; a negative one gates h3 off
// for h3NegativeTTL so we don't re-attempt a doomed QUIC handshake. Only the
// negative case matters for gating (positive just confirms the default of
// trying h3); both use the same TTL for simplicity.
func (t *scraperTransport) recordH3(host string, capable bool) {
	t.mu.Lock()
	defer t.mu.Unlock()
	t.cached[host] = h3Capability{capable: capable, expireAt: time.Now().Add(h3NegativeTTL)}
}

// observeAltSvc records a positive h3 capability if a response advertises h3
// via Alt-Svc.
func (t *scraperTransport) observeAltSvc(resp *http.Response, host string) {
	if resp == nil || !advertisesH3(resp.Header) {
		return
	}
	t.recordH3(host, true)
}

// advertisesH3 reports whether an Alt-Svc header advertises the h3 protocol.
func advertisesH3(hdr http.Header) bool {
	if hdr == nil {
		return false
	}
	for _, v := range hdr.Values("Alt-Svc") {
		if strings.Contains(strings.ToLower(v), "h3") {
			return true
		}
	}
	return false
}