indexer/scraper/detect.go

// Forge type detection.
//
// The forge type is not configured; each scrape run re-detects it, because the
// hoster may have changed the backend (e.g. moved from Gitea to Forgejo).
//
// Forgejo and Gitea are hard to tell apart: Forgejo is a fork of Gitea and
// both serve /api/v1/version. The discriminator is the Forgejo-native
// /api/forgejo/v1/version endpoint, which only Forgejo serves. An instance is
// therefore classified Gitea only when that check fails and /api/v1/version
// answers.
package scraper

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strings"
	"time"
)

// Type is the kind of forge software an instance runs, e.g. "forgejo".
type Type string

// Known forge types. Add a detector per type as they are implemented; until
// then the default is TypeUnknown.
const (
	TypeForgejo Type = "forgejo"
	TypeGitea   Type = "gitea"
	TypeUnknown Type = "unknown"
)

// Endpoints probed for detection. /api/forgejo/v1/version is Forgejo-native;
// /api/v1/version is shared by Forgejo, Gitea and Gogs.
const (
	forgejoVersionPath = "/api/forgejo/v1/version"
	apiVersionPath     = "/api/v1/version"
)

// versionResponse is the JSON shape of the version endpoints used for
// detection. Only the version string matters.
type versionResponse struct {
	Version string `json:"version"`
}

// detectHTTPClient is used for probing version endpoints. It follows
// redirects and has a short timeout, sharing the transport used by the rest
// of the scraper so HTTP/2 connections are reused.
var detectHTTPClient = httpClientFor(10 * time.Second)

// DetectType probes an instance at fqdn to determine its forge type and
// software version. scheme is the connection scheme ("https" by default, or
// "http" if forced by the config); fqdn is the bare hostname.
//
// Robots.txt is honored: if the instance disallows /api/ the probe is not
// made and TypeUnknown is returned.
func DetectType(scheme, fqdn string) (Type, string, error) {
	return detectAt(scheme + "://" + fqdn)
}

// detectAt is DetectType for an already-schemed base URL. It is separate so
// tests can exercise detection against an HTTP server.
func detectAt(baseURL string) (Type, string, error) {
	// Determine Forgejo status once: both branches need it, because a Gitea
	// classification requires ruling Forgejo out first (Forgejo also serves
	// /api/v1/version).
	fjVersion, isFJ, err := probeVersion(baseURL, forgejoVersionPath)
	if err != nil {
		return TypeUnknown, "", fmt.Errorf("probe forgejo: %w", err)
	}
	if isFJ {
		return TypeForgejo, fjVersion, nil
	}

	// Not Forgejo: the shared /api/v1/version endpoint implies Gitea. Gogs
	// also serves it, so it would be misclassified here until a Gogs
	// detector (and its own discriminator) is added.
	version, ok, err := probeVersion(baseURL, apiVersionPath)
	if err != nil {
		return TypeUnknown, "", fmt.Errorf("probe gitea: %w", err)
	}
	if ok {
		return TypeGitea, version, nil
	}

	return TypeUnknown, "", nil
}

// probeVersion fetches versionPath under baseURL and reports the advertised
// version. It returns ok=false if the endpoint does not exist (404) or does
// not look like our version JSON. Robots.txt is honored before probing.
func probeVersion(baseURL, versionPath string) (version string, ok bool, err error) {
	// Honor robots: probing under /api/ when it is disallowed would violate
	// the instance's crawling rules.
	allowed, err := Allowed(baseURL)
	if err != nil {
		return "", false, fmt.Errorf("check robots.txt: %w", err)
	}
	if !allowed {
		return "", false, nil
	}

	u := strings.TrimSuffix(baseURL, "/") + versionPath
	req, err := http.NewRequest(http.MethodGet, u, nil)
	if err != nil {
		return "", false, fmt.Errorf("build version request: %w", err)
	}
	req.Header.Set("User-Agent", UserAgent)

	resp, err := detectHTTPClient.Do(req)
	if err != nil {
		return "", false, fmt.Errorf("fetch version endpoint: %w", err)
	}
	defer func() { _ = resp.Body.Close() }()

	if resp.StatusCode == http.StatusNotFound {
		return "", false, nil
	}
	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		return "", false, fmt.Errorf("version endpoint returned %s", resp.Status)
	}

	var v versionResponse
	if err := json.NewDecoder(resp.Body).Decode(&v); err != nil {
		// A 2xx that isn't our JSON shape isn't a forge we know.
		return "", false, nil
	}
	if v.Version == "" {
		return "", false, nil
	}
	return v.Version, true, nil
}