🔥

scraper/detect.go

@@ -0,0 +1,131 @@
+// Forge type detection.
+//
+// The forge type is not configured; each scrape run re-detects it, because the
+// hoster may have changed the backend (e.g. moved from Gitea to Forgejo).
+//
+// Forgejo and Gitea are hard to tell apart: Forgejo is a fork of Gitea and
+// both serve /api/v1/version. The discriminator is the Forgejo-native
+// /api/forgejo/v1/version endpoint, which only Forgejo serves. An instance is
+// therefore classified Gitea only when that check fails and /api/v1/version
+// answers.
+package scraper
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+)
+
+// Type is the kind of forge software an instance runs, e.g. "forgejo".
+type Type string
+
+// Known forge types. Add a detector per type as they are implemented; until
+// then the default is TypeUnknown.
+const (
+	TypeForgejo Type = "forgejo"
+	TypeGitea   Type = "gitea"
+	TypeUnknown Type = "unknown"
+)
+
+// Endpoints probed for detection. /api/forgejo/v1/version is Forgejo-native;
+// /api/v1/version is shared by Forgejo, Gitea and Gogs.
+const (
+	forgejoVersionPath = "/api/forgejo/v1/version"
+	apiVersionPath     = "/api/v1/version"
+)
+
+// versionResponse is the JSON shape of the version endpoints used for
+// detection. Only the version string matters.
+type versionResponse struct {
+	Version string `json:"version"`
+}
+
+// detectHTTPClient is used for probing version endpoints. It follows
+// redirects and has a short timeout, sharing the transport used by the rest
+// of the scraper so HTTP/2 connections are reused.
+var detectHTTPClient = httpClientFor(10 * time.Second)
+
+// DetectType probes an instance at fqdn to determine its forge type and
+// software version. scheme is the connection scheme ("https" by default, or
+// "http" if forced by the config); fqdn is the bare hostname.
+//
+// Robots.txt is honored: if the instance disallows /api/ the probe is not
+// made and TypeUnknown is returned.
+func DetectType(scheme, fqdn string) (Type, string, error) {
+	return detectAt(scheme + "://" + fqdn)
+}
+
+// detectAt is DetectType for an already-schemed base URL. It is separate so
+// tests can exercise detection against an HTTP server.
+func detectAt(baseURL string) (Type, string, error) {
+	// Determine Forgejo status once: both branches need it, because a Gitea
+	// classification requires ruling Forgejo out first (Forgejo also serves
+	// /api/v1/version).
+	fjVersion, isFJ, err := probeVersion(baseURL, forgejoVersionPath)
+	if err != nil {
+		return TypeUnknown, "", fmt.Errorf("probe forgejo: %w", err)
+	}
+	if isFJ {
+		return TypeForgejo, fjVersion, nil
+	}
+
+	// Not Forgejo: the shared /api/v1/version endpoint implies Gitea. Gogs
+	// also serves it, so it would be misclassified here until a Gogs
+	// detector (and its own discriminator) is added.
+	version, ok, err := probeVersion(baseURL, apiVersionPath)
+	if err != nil {
+		return TypeUnknown, "", fmt.Errorf("probe gitea: %w", err)
+	}
+	if ok {
+		return TypeGitea, version, nil
+	}
+
+	return TypeUnknown, "", nil
+}
+
+// probeVersion fetches versionPath under baseURL and reports the advertised
+// version. It returns ok=false if the endpoint does not exist (404) or does
+// not look like our version JSON. Robots.txt is honored before probing.
+func probeVersion(baseURL, versionPath string) (version string, ok bool, err error) {
+	// Honor robots: probing under /api/ when it is disallowed would violate
+	// the instance's crawling rules.
+	allowed, err := Allowed(baseURL)
+	if err != nil {
+		return "", false, fmt.Errorf("check robots.txt: %w", err)
+	}
+	if !allowed {
+		return "", false, nil
+	}
+
+	u := strings.TrimSuffix(baseURL, "/") + versionPath
+	req, err := http.NewRequest(http.MethodGet, u, nil)
+	if err != nil {
+		return "", false, fmt.Errorf("build version request: %w", err)
+	}
+	req.Header.Set("User-Agent", UserAgent)
+
+	resp, err := detectHTTPClient.Do(req)
+	if err != nil {
+		return "", false, fmt.Errorf("fetch version endpoint: %w", err)
+	}
+	defer func() { _ = resp.Body.Close() }()
+
+	if resp.StatusCode == http.StatusNotFound {
+		return "", false, nil
+	}
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return "", false, fmt.Errorf("version endpoint returned %s", resp.Status)
+	}
+
+	var v versionResponse
+	if err := json.NewDecoder(resp.Body).Decode(&v); err != nil {
+		// A 2xx that isn't our JSON shape isn't a forge we know.
+		return "", false, nil
+	}
+	if v.Version == "" {
+		return "", false, nil
+	}
+	return v.Version, true, nil
+}

scraper/detect_test.go

@@ -0,0 +1,102 @@
+package scraper
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+// newDetectServer serves a robots.txt that allows /api/ plus the given
+// version paths. Returns the server's fqdn (host:port).
+func newDetectServer(t *testing.T, versionPaths ...string) string {
+	t.Helper()
+	mux := http.NewServeMux()
+	mux.HandleFunc("/robots.txt", func(w http.ResponseWriter, r *http.Request) {
+		_, _ = fmt.Fprintln(w, "User-agent: *")
+		_, _ = fmt.Fprintln(w, "Allow: /")
+	})
+	for _, p := range versionPaths {
+		p := p
+		mux.HandleFunc(p, func(w http.ResponseWriter, r *http.Request) {
+			w.Header().Set("Content-Type", "application/json")
+			_, _ = fmt.Fprintln(w, `{"version":"16.0.5"}`)
+		})
+	}
+	srv := httptest.NewServer(mux)
+	t.Cleanup(srv.Close)
+	return srv.Listener.Addr().String()
+}
+
+func TestDetectForgejo(t *testing.T) {
+	// Forgejo serves BOTH endpoints; the Forgejo-native one must win.
+	fqdn := newDetectServer(t, "/api/forgejo/v1/version", "/api/v1/version")
+
+	typ, version, err := detectAt("http://" + fqdn)
+	if err != nil {
+		t.Fatalf("detect: %v", err)
+	}
+	if typ != TypeForgejo {
+		t.Errorf("expected TypeForgejo, got %q", typ)
+	}
+	if version != "16.0.5" {
+		t.Errorf("expected version 16.0.5, got %q", version)
+	}
+}
+
+func TestDetectGitea(t *testing.T) {
+	// Gitea serves only the shared /api/v1/version endpoint.
+	fqdn := newDetectServer(t, "/api/v1/version")
+
+	typ, version, err := detectAt("http://" + fqdn)
+	if err != nil {
+		t.Fatalf("detect: %v", err)
+	}
+	if typ != TypeGitea {
+		t.Errorf("expected TypeGitea, got %q", typ)
+	}
+	if version != "16.0.5" {
+		t.Errorf("expected version 16.0.5, got %q", version)
+	}
+}
+
+func TestDetectUnknown(t *testing.T) {
+	// A server that serves robots but no version endpoint.
+	fqdn := newDetectServer(t)
+
+	typ, version, err := detectAt("http://" + fqdn)
+	if err != nil {
+		t.Fatalf("detect: %v", err)
+	}
+	if typ != TypeUnknown {
+		t.Errorf("expected TypeUnknown, got %q", typ)
+	}
+	if version != "" {
+		t.Errorf("expected empty version, got %q", version)
+	}
+}
+
+func TestDetectHonorsRobotsDisallow(t *testing.T) {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/robots.txt", func(w http.ResponseWriter, r *http.Request) {
+		_, _ = fmt.Fprintln(w, "User-agent: *")
+		_, _ = fmt.Fprintln(w, "Disallow: /api/")
+	})
+	mux.HandleFunc("/api/forgejo/v1/version", func(w http.ResponseWriter, r *http.Request) {
+		_, _ = fmt.Fprintln(w, `{"version":"16.0.5"}`)
+	})
+	srv := httptest.NewServer(mux)
+	t.Cleanup(srv.Close)
+	fqdn := srv.Listener.Addr().String()
+
+	typ, version, err := detectAt("http://" + fqdn)
+	if err != nil {
+		t.Fatalf("detect: %v", err)
+	}
+	if typ != TypeUnknown {
+		t.Errorf("expected TypeUnknown when /api/ is disallowed, got %q", typ)
+	}
+	if version != "" {
+		t.Errorf("expected empty version when disallowed, got %q", version)
+	}
+}

scraper/forgejo.go

@@ -17,12 +17,19 @@ type ForgejoScraper struct {
 	Client   *forgejo.Client
 	BaseURL  string
 	PageSize int
+	// Identity of the forge this scraper talks to, attached to every Project.
+	ForgeIdentity
 }
 
-// NewForgejoScraper creates a scraper for a Forgejo instance.
-// BaseURL should be the instance URL, e.g. https://codeberg.org.
-func NewForgejoScraper(baseURL string) (*ForgejoScraper, error) {
-	client, err := forgejo.NewClient(baseURL)
+// NewForgejoScraper creates a scraper for a Forgejo instance. scheme is the
+// connection scheme ("https" by default, or "http" if forced); fqdn is the
+// bare hostname (e.g. "git.example.org") and is the forge identity attached to
+// each Project. The connection URL is built as scheme://fqdn.
+func NewForgejoScraper(scheme, fqdn, forgeType string) (*ForgejoScraper, error) {
+	host := strings.TrimSuffix(fqdn, "/")
+	baseURL := scheme + "://" + host
+
+	client, err := forgejo.NewClient(baseURL, forgejo.SetHTTPClient(httpClientFor(30*time.Second)))
 	if err != nil {
 		return nil, fmt.Errorf("create Forgejo client: %w", err)
 	}
@@ -30,8 +37,12 @@ func NewForgejoScraper(baseURL string) (*ForgejoScraper, error) {
 
 	return &ForgejoScraper{
 		Client:   client,
-		BaseURL:  strings.TrimSuffix(baseURL, "/"),
+		BaseURL:  baseURL,
 		PageSize: 50,
+		ForgeIdentity: ForgeIdentity{
+			ForgeURL:  host,
+			ForgeType: forgeType,
+		},
 	}, nil
 }
 
@@ -142,6 +153,7 @@ func (s *ForgejoScraper) GetProjects() ([]Project, error) {
 			readme := s.readmeFor(owner, name, repo.DefaultBranch, root)
 
 			projects = append(projects, Project{
+				ForgeIdentity: s.ForgeIdentity,
 				Name:          repo.FullName,
 				Url:           repo.HTMLURL,
 				Description:   repo.Description,

scraper/robots.go

@@ -13,17 +13,23 @@ import (
 // instances can set robots rules for us specifically.
 const UserAgent = "ffd-indexer/0.1"
 
-// robotsHTTPClient is used for fetching robots.txt. It does not follow the
-// SDK client's settings; it is a plain, short-timeout client.
-var robotsHTTPClient = &http.Client{Timeout: 10 * time.Second}
+// robotsHTTPClient fetches robots.txt using the shared transport.
+var robotsHTTPClient = httpClientFor(10 * time.Second)
 
 // Allowed reports whether an instance at baseURL permits crawling its API
 // according to its robots.txt. It only consults the rules for the
 // User-Agent "*"; a block of /api/ (the API root) disallows scraping.
 //
+// baseURL may be a bare hostname (e.g. "git.example.org"); the https scheme is
+// added internally when missing. It may also be a full URL or a host:port
+// (used in tests).
+//
 // A failure to fetch or parse robots.txt is treated as an error; callers
 // decide whether that is fatal.
 func Allowed(baseURL string) (bool, error) {
+	if !strings.Contains(baseURL, "://") {
+		baseURL = "https://" + baseURL
+	}
 	u, err := url.Parse(baseURL)
 	if err != nil {
 		return false, fmt.Errorf("parse base URL: %w", err)
@@ -73,7 +79,7 @@ func allowedFor(body, userAgent string) bool {
 	groups := parseRobotGroups(body)
 
 	// A group that explicitly names our user agent takes precedence over a
-	// "*" group, regardless of order. Otherwise fall back to "*".
+	// "*" group, regardless of order. Otherwise, fall back to "*".
 	var star *robotGroup
 	for _, g := range groups {
 		for _, a := range g.agents {
@@ -100,7 +106,7 @@ func parseRobotGroups(body string) []*robotGroup {
 	var groups []*robotGroup
 	var cur *robotGroup
 
-	for _, raw := range strings.Split(body, "\n") {
+	for raw := range strings.SplitSeq(body, "\n") {
 		line := strings.TrimSpace(raw)
 		// Strip an inline comment (not part of a rule's result).
 		if i := strings.IndexByte(line, '#'); i >= 0 {

scraper/robots_test.go

@@ -10,12 +10,12 @@ func TestAllowedFor(t *testing.T) {
 		t.Error("expected /api/ allowed when only archive is disallowed")
 	}
 
-	// codeberg: /api/ disallowed under "*".
-	codeberg := `User-agent: *
+	// a forge that disallows /api/ under "*".
+	fixture := `User-agent: *
 Disallow: /api/
 Disallow: /avatars/
 `
-	if allowedFor(codeberg, ua) {
+	if allowedFor(fixture, ua) {
 		t.Error("expected /api/ blocked when /api/ is disallowed")
 	}
 

scraper/scraper.go

@@ -4,6 +4,8 @@ package scraper
 
 import (
 	"fmt"
+	"net/url"
+	"strings"
 	"time"
 )
 
@@ -14,18 +16,56 @@ type Scraper interface {
 	GetProjects() ([]Project, error)
 }
 
-// NewScraper returns a Scraper for the given forge type, e.g. "forgejo".
-func NewScraper(typ, baseURL string) (Scraper, error) {
+// Scheme splits an input forge address into the connection scheme ("https"
+// by default) and the bare hostname (the forge's identity). The input may be
+// a bare FQDN ("git.example.org") or prefixed with a protocol to force http
+// or https ("http://git.example.org"). The returned host is always
+// protocol-free, so the database and metrics labels never carry a scheme.
+func Scheme(addr string) (scheme, fqdn string) {
+	scheme = "https"
+	if i := strings.Index(addr, "://"); i >= 0 {
+		scheme = addr[:i]
+		addr = addr[i+3:]
+	}
+	if u, err := url.Parse("//" + addr); err == nil {
+		if h := u.Hostname(); h != "" {
+			fqdn = h
+		} else {
+			fqdn = addr
+		}
+	} else {
+		fqdn = addr
+	}
+	fqdn = strings.TrimRight(fqdn, "/")
+	return scheme, fqdn
+}
+
+// NewScraper returns a Scraper for the given forge type, e.g. TypeForgejo.
+// scheme is the connection scheme ("https" unless the user forced http); fqdn
+// is the forge's bare hostname, used as the identity attached to every
+// returned Project. The connection URL is built as scheme://fqdn.
+func NewScraper(typ Type, scheme, fqdn string) (Scraper, error) {
 	switch typ {
-	case "forgejo":
-		return NewForgejoScraper(baseURL)
+	// Forgejo and Gitea expose the same /api/v1 surface (Forgejo is a fork
+	// of Gitea), so both are scraped with the Forgejo scraper class.
+	case TypeForgejo, TypeGitea:
+		return NewForgejoScraper(scheme, fqdn, string(typ))
 	default:
 		return nil, fmt.Errorf("unknown forge type %q", typ)
 	}
 }
 
+// ForgeIdentity identifies which forge a Project was scraped from. ForgeURL
+// is the bare hostname of the instance (e.g. "git.example.org"); the HTTP
+// scheme is added internally when a connection URL is needed.
+type ForgeIdentity struct {
+	ForgeURL  string
+	ForgeType string
+}
+
 // Project is a generic, forge-agnostic representation of an open source project.
 type Project struct {
+	ForgeIdentity
 	Name          string    `json:"name"`
 	Url           string    `json:"url"`
 	Description   string    `json:"description"`

scraper/scraper_test.go

@@ -0,0 +1,24 @@
+package scraper
+
+import "testing"
+
+func TestScheme(t *testing.T) {
+	tests := []struct {
+		in     string
+		scheme string
+		fqdn   string
+	}{
+		{"git.example.org", "https", "git.example.org"},
+		{"https://git.example.org", "https", "git.example.org"},
+		{"http://git.example.org", "http", "git.example.org"},
+		{"https://git.example.org/", "https", "git.example.org"},
+		{"http://git.example.net", "http", "git.example.net"},
+	}
+	for _, tt := range tests {
+		scheme, fqdn := Scheme(tt.in)
+		if scheme != tt.scheme || fqdn != tt.fqdn {
+			t.Errorf("Scheme(%q) = (%q, %q), want (%q, %q)",
+				tt.in, scheme, fqdn, tt.scheme, tt.fqdn)
+		}
+	}
+}

scraper/transport.go

@@ -0,0 +1,221 @@
+package scraper
+
+import (
+	"crypto/tls"
+	"net"
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/quic-go/quic-go"
+	"github.com/quic-go/quic-go/http3"
+)
+
+// scraperTransport is the shared HTTP stack used by every client in this
+// package (robots.txt checks, type detection, and the forge scrapers). It
+// composes:
+//
+//   - a single stdlib *http.Transport that negotiates HTTP/1.1 vs HTTP/2 over
+//     TLS via ALPN automatically (no logic here);
+//   - an HTTP/3 transport from quic-go, tried first on each host and falling
+//     back to the stdlib transport if the host does not support h3.
+//
+// Protocol selection is "newest available first with automatic fallback":
+// the first request to a host attempts HTTP/3, and falls back to HTTP/2 or
+// HTTP/1.1 (negotiated via ALPN) if the host has no h3. A per-host negative
+// cache remembers hosts that failed h3 so we don't keep re-triggering a
+// doomed QUIC handshake on every request.
+//
+// Sharing one transport means idle keep-alive connections are reused across
+// the whole scrape of a host, and each origin's negotiation (h3 or h1/h2) is
+// done once for a while.
+type scraperTransport struct {
+	// h12 is the plain stdlib transport: HTTP/1.1 and HTTP/2.
+	h12 *http.Transport
+	// h3 is the HTTP/3 transport (lazily created, quic-go).
+	h3 *http3.Transport
+
+	mu     sync.Mutex
+	cached map[string]h3Capability
+	forced map[string]struct{} // hosts that try h3 even during a negative cool-down
+}
+
+// h3Capability is the cached knowledge about a single host's HTTP/3 support.
+type h3Capability struct {
+	capable  bool
+	expireAt time.Time
+}
+
+const (
+	// h3NegativeTTL is how long a failed h3 attempt is remembered, so a host
+	// without h3 isn't re-triggered on every request. After it expires, h3 is
+	// tried again (the host may have started serving it).
+	h3NegativeTTL = time.Hour
+	// h3HandshakeTimeout aborts a QUIC handshake (e.g. to a host with UDP
+	// closed) instead of hanging the scrape. This bounds the per-host h3
+	// attempt before we fall back to HTTP/2/1.1.
+	h3HandshakeTimeout = 5 * time.Second
+)
+
+// sharedTransport is the single transport used by the whole process.
+var sharedTransport = newScraperTransport()
+
+// newScraperTransport builds the shared transport.
+func newScraperTransport() *scraperTransport {
+	t12 := &http.Transport{
+		MaxIdleConns:          100,
+		MaxIdleConnsPerHost:   10,
+		IdleConnTimeout:       90 * time.Second,
+		ExpectContinueTimeout: 1 * time.Second,
+		DialContext: (&net.Dialer{
+			Timeout:   10 * time.Second,
+			KeepAlive: 30 * time.Second,
+		}).DialContext,
+		// Our TLS config disables Go's implicit HTTP/2 enablement, so opt in
+		// explicitly: stdlib then negotiates HTTP/2 via ALPN when the server
+		// supports it, and falls back to HTTP/1.1 otherwise.
+		ForceAttemptHTTP2: true,
+		TLSClientConfig: &tls.Config{
+			MinVersion: tls.VersionTLS12,
+		},
+	}
+	return &scraperTransport{
+		h12:    t12,
+		cached: map[string]h3Capability{},
+		forced: map[string]struct{}{},
+	}
+}
+
+// httpClientFor returns an *http.Client that shares sharedTransport with a
+// per-client timeout. Timeouts differ per use (robots probes are short, the
+// SDK scrape is a bit more generous) but the underlying connection pool is
+// the same.
+func httpClientFor(timeout time.Duration) *http.Client {
+	return &http.Client{
+		Transport: sharedTransport,
+		Timeout:   timeout,
+	}
+}
+
+// EnableHTTP3 forces a host to keep trying HTTP/3 even while a previous
+// failure is being remembered (during the negative cool-down). It is an
+// override for operators who know a host serves h3. By default, h3 is already
+// tried automatically on every host, so this is only needed to bypass the
+// negative cache.
+func EnableHTTP3(host string) {
+	if host == "" {
+		return
+	}
+	sharedTransport.forceH3(host)
+}
+
+// forceH3 marks a host as always trying h3 (overrides the negative cache).
+func (t *scraperTransport) forceH3(host string) {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	t.forced[host] = struct{}{}
+}
+
+// h3Transport lazily builds the quic-go HTTP/3 transport.
+func (t *scraperTransport) h3Transport() *http3.Transport {
+	if t.h3 != nil {
+		return t.h3
+	}
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	if t.h3 == nil {
+		t.h3 = &http3.Transport{
+			TLSClientConfig: &tls.Config{
+				MinVersion: tls.VersionTLS12,
+			},
+			QUICConfig: &quic.Config{
+				// Fail fast against hosts with UDP closed instead of hanging.
+				HandshakeIdleTimeout: h3HandshakeTimeout,
+				MaxIdleTimeout:       90 * time.Second,
+			},
+		}
+	}
+	return t.h3
+}
+
+// RoundTrip implements http.RoundTripper.
+func (t *scraperTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	host := req.URL.Hostname()
+
+	if t.shouldTryH3(host) {
+		resp, err := t.h3Transport().RoundTrip(req)
+		if err != nil {
+			// h3 failed: cache the negative result and fall back to h1/h2 for
+			// this very request so the caller sees a successful response.
+			t.recordH3(host, false)
+			return t.h12.RoundTrip(req)
+		}
+		t.observeAltSvc(resp, host)
+		return resp, nil
+	}
+
+	resp, err := t.h12.RoundTrip(req)
+	if err != nil {
+		return nil, err
+	}
+	// If the h1/h2 response advertises h3, learn the capability so subsequent
+	// requests to this host try h3. The current request stays on h1/h2.
+	t.observeAltSvc(resp, host)
+	return resp, nil
+}
+
+// shouldTryH3 reports whether a host should be attempted over h3 first.
+// The default is yes (newest-available-first); it is only gated by an
+// unexpired negative capability (the host recently failed h3) unless the
+// host is forced.
+func (t *scraperTransport) shouldTryH3(host string) bool {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	if _, ok := t.forced[host]; ok {
+		return true
+	}
+	c, ok := t.cached[host]
+	if !ok {
+		return true // unknown host: try h3 first
+	}
+	// A negative result rejects h3; a positive result is fine (and the
+	// default is already to try). Either way, once expired, try again.
+	if !c.capable {
+		return time.Now().After(c.expireAt)
+	}
+	return true
+}
+
+// recordH3 stores a learned h3 capability for a host. A positive capability
+// (h3 succeeded) prevents a negative cool-down; a negative one gates h3 off
+// for h3NegativeTTL so we don't re-attempt a doomed QUIC handshake. Only the
+// negative case matters for gating (positive just confirms the default of
+// trying h3); both use the same TTL for simplicity.
+func (t *scraperTransport) recordH3(host string, capable bool) {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	t.cached[host] = h3Capability{capable: capable, expireAt: time.Now().Add(h3NegativeTTL)}
+}
+
+// observeAltSvc records a positive h3 capability if a response advertises h3
+// via Alt-Svc.
+func (t *scraperTransport) observeAltSvc(resp *http.Response, host string) {
+	if resp == nil || !advertisesH3(resp.Header) {
+		return
+	}
+	t.recordH3(host, true)
+}
+
+// advertisesH3 reports whether an Alt-Svc header advertises the h3 protocol.
+func advertisesH3(hdr http.Header) bool {
+	if hdr == nil {
+		return false
+	}
+	for _, v := range hdr.Values("Alt-Svc") {
+		if strings.Contains(strings.ToLower(v), "h3") {
+			return true
+		}
+	}
+	return false
+}

scraper/transport_test.go

@@ -0,0 +1,143 @@
+package scraper
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+)
+
+func TestAdvertisesH3(t *testing.T) {
+	cases := []struct {
+		hdr  http.Header
+		want bool
+	}{
+		{http.Header{"Alt-Svc": []string{`h3=":443";ma=2592000`}}, true},
+		{http.Header{"Alt-Svc": []string{`h2=":443";ma=2592000`}}, false},
+		{http.Header{}, false},
+		{nil, false},
+	}
+	for i, c := range cases {
+		if got := advertisesH3(c.hdr); got != c.want {
+			t.Errorf("case %d: advertisesH3(%v) = %v, want %v", i, c.hdr, got, c.want)
+		}
+	}
+}
+
+func TestH3CapabilityCache(t *testing.T) {
+	tr := newScraperTransport()
+	host := "example.test"
+
+	// Unknown host: try h3 first (newest-available-first).
+	if !tr.shouldTryH3(host) {
+		t.Errorf("unknown host should attempt h3 first")
+	}
+
+	// Positive capability keeps h3 enabled.
+	tr.recordH3(host, true)
+	if !tr.shouldTryH3(host) {
+		t.Errorf("capable host should attempt h3")
+	}
+
+	// Negative capability gates h3 off.
+	tr.recordH3(host, false)
+	if tr.shouldTryH3(host) {
+		t.Errorf("negative capability should disable h3")
+	}
+
+	// A forced host tries h3 even during a negative cool-down.
+	tr.recordH3(host, false)
+	tr.forceH3(host)
+	if !tr.shouldTryH3(host) {
+		t.Errorf("forced host should attempt h3")
+	}
+}
+
+func TestH3NegativeExpiry(t *testing.T) {
+	tr := newScraperTransport()
+	host := "expiry.test"
+
+	// A negative capability older than its TTL must re-enable h3.
+	tr.mu.Lock()
+	tr.cached[host] = h3Capability{capable: false, expireAt: time.Now().Add(-time.Second)}
+	tr.mu.Unlock()
+
+	if !tr.shouldTryH3(host) {
+		t.Errorf("expired negative capability should attempt h3 again")
+	}
+}
+
+func TestObserveAltSvcLearnsCapability(t *testing.T) {
+	tr := newScraperTransport()
+	host := "advertises.test"
+
+	// Alt-Svc advertisement records a positive capability.
+	tr.observeAltSvc(&http.Response{Header: http.Header{"Alt-Svc": []string{`h3=":443"`}}}, host)
+	tr.mu.Lock()
+	c := tr.cached[host]
+	tr.mu.Unlock()
+	if !c.capable {
+		t.Errorf("alt-svc advertisement should record a positive h3 capability")
+	}
+
+	// A response without alt-svc must not clear an existing capability.
+	tr.observeAltSvc(&http.Response{Header: http.Header{}}, host)
+	tr.mu.Lock()
+	c = tr.cached[host]
+	tr.mu.Unlock()
+	if !c.capable {
+		t.Errorf("no alt-svc must not clear an existing capability")
+	}
+}
+
+// TestRoundTripFallsBackToH12 verifies that when HTTP/3 is not available (the
+// httptest server speaks only HTTP/1.1 over TLS — no QUIC), the transport
+// falls back to the stdlib (h1/h2) transport and still returns a response.
+func TestRoundTripFallsBackToH12(t *testing.T) {
+	srv := newTLSServer(t)
+	cfg := srv.TLS
+	cfg.NextProtos = []string{"http/1.1"} // keep it HTTP/1.1 so result is deterministic
+
+	tr := newScraperTransport()
+	// Make the h3 handshake fail fast instead of waiting the 5s const.
+	quicCfg := tr.h3Transport().QUICConfig
+	quicCfg.HandshakeIdleTimeout = 200 * time.Millisecond
+
+	// Trust the httptest server's self-signed cert in both sub-transports.
+	pool := x509.NewCertPool()
+	pool.AddCert(srv.Certificate())
+	cerr := &tls.Config{MinVersion: tls.VersionTLS12, RootCAs: pool}
+	tr.h12.TLSClientConfig = cerr
+	tr.h3.TLSClientConfig = cerr
+
+	addr := srv.Listener.Addr().String()
+	req, err := http.NewRequest("GET", "https://"+addr+"/", nil)
+	if err != nil {
+		t.Fatalf("new request: %v", err)
+	}
+	resp, err := tr.RoundTrip(req)
+	if err != nil {
+		t.Fatalf("RoundTrip: %v", err)
+	}
+	defer func() { _ = resp.Body.Close() }()
+	if resp.StatusCode != http.StatusOK {
+		t.Errorf("status = %d, want 200", resp.StatusCode)
+	}
+	// It must have worked over h12: HTTP/1.x (we pinned ALPN) and definitely
+	// not HTTP/3.
+	if resp.ProtoMajor >= 3 {
+		t.Errorf("unexpected proto %s (should have fallen back to h1/h2)", resp.Proto)
+	}
+}
+
+func newTLSServer(t *testing.T) *httptest.Server {
+	t.Helper()
+	s := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write([]byte("ok"))
+	}))
+	s.StartTLS()
+	t.Cleanup(s.Close)
+	return s
+}