// Forge type detection. // // The forge type is not configured; each scrape run re-detects it, because the // hoster may have changed the backend (e.g. moved from Gitea to Forgejo). // // Forgejo and Gitea are hard to tell apart: Forgejo is a fork of Gitea and // both serve /api/v1/version. The discriminator is the Forgejo-native // /api/forgejo/v1/version endpoint, which only Forgejo serves. An instance is // therefore classified Gitea only when that check fails and /api/v1/version // answers. package scraper import ( "encoding/json" "fmt" "net/http" "strings" "time" ) // Type is the kind of forge software an instance runs, e.g. "forgejo". type Type string // Known forge types. Add a detector per type as they are implemented; until // then the default is TypeUnknown. const ( TypeForgejo Type = "forgejo" TypeGitea Type = "gitea" TypeUnknown Type = "unknown" ) // Endpoints probed for detection. /api/forgejo/v1/version is Forgejo-native; // /api/v1/version is shared by Forgejo, Gitea and Gogs. const ( forgejoVersionPath = "/api/forgejo/v1/version" apiVersionPath = "/api/v1/version" ) // versionResponse is the JSON shape of the version endpoints used for // detection. Only the version string matters. type versionResponse struct { Version string `json:"version"` } // detectHTTPClient is used for probing version endpoints. It follows // redirects and has a short timeout, sharing the transport used by the rest // of the scraper so HTTP/2 connections are reused. var detectHTTPClient = httpClientFor(10 * time.Second) // DetectType probes an instance at fqdn to determine its forge type and // software version. scheme is the connection scheme ("https" by default, or // "http" if forced by the config); fqdn is the bare hostname. // // Robots.txt is honored: if the instance disallows /api/ the probe is not // made and TypeUnknown is returned. func DetectType(scheme, fqdn string) (Type, string, error) { return detectAt(scheme + "://" + fqdn) } // detectAt is DetectType for an already-schemed base URL. It is separate so // tests can exercise detection against an HTTP server. func detectAt(baseURL string) (Type, string, error) { // Determine Forgejo status once: both branches need it, because a Gitea // classification requires ruling Forgejo out first (Forgejo also serves // /api/v1/version). fjVersion, isFJ, err := probeVersion(baseURL, forgejoVersionPath) if err != nil { return TypeUnknown, "", fmt.Errorf("probe forgejo: %w", err) } if isFJ { return TypeForgejo, fjVersion, nil } // Not Forgejo: the shared /api/v1/version endpoint implies Gitea. Gogs // also serves it, so it would be misclassified here until a Gogs // detector (and its own discriminator) is added. version, ok, err := probeVersion(baseURL, apiVersionPath) if err != nil { return TypeUnknown, "", fmt.Errorf("probe gitea: %w", err) } if ok { return TypeGitea, version, nil } return TypeUnknown, "", nil } // probeVersion fetches versionPath under baseURL and reports the advertised // version. It returns ok=false if the endpoint does not exist (404) or does // not look like our version JSON. Robots.txt is honored before probing. func probeVersion(baseURL, versionPath string) (version string, ok bool, err error) { // Honor robots: probing under /api/ when it is disallowed would violate // the instance's crawling rules. allowed, err := Allowed(baseURL) if err != nil { return "", false, fmt.Errorf("check robots.txt: %w", err) } if !allowed { return "", false, nil } u := strings.TrimSuffix(baseURL, "/") + versionPath req, err := http.NewRequest(http.MethodGet, u, nil) if err != nil { return "", false, fmt.Errorf("build version request: %w", err) } req.Header.Set("User-Agent", UserAgent) resp, err := detectHTTPClient.Do(req) if err != nil { return "", false, fmt.Errorf("fetch version endpoint: %w", err) } defer func() { _ = resp.Body.Close() }() if resp.StatusCode == http.StatusNotFound { return "", false, nil } if resp.StatusCode < 200 || resp.StatusCode >= 300 { return "", false, fmt.Errorf("version endpoint returned %s", resp.Status) } var v versionResponse if err := json.NewDecoder(resp.Body).Decode(&v); err != nil { // A 2xx that isn't our JSON shape isn't a forge we know. return "", false, nil } if v.Version == "" { return "", false, nil } return v.Version, true, nil }