63c6f4b761
Some checks failed
CI pipeline for ffd-indexer / checks (go test + golangci) (push) Failing after 26s
CI pipeline for ffd-indexer / build binaries + containers (push) Failing after 3s
CI pipeline for ffd-indexer / format (gofmt + nixfmt) (push) Failing after 3s
CI pipeline for ffd-indexer / integration (live scrape + Typesense) (push) Failing after 20s
87 lines
2.9 KiB
Go
87 lines
2.9 KiB
Go
// Package scraper defines the interfaces and shared types used to discover
|
|
// projects from forge instances (Forgejo, Gogs, ...).
|
|
package scraper
|
|
|
|
import (
|
|
"fmt"
|
|
"net/url"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// Scraper is implemented by concrete, per-forge scrapers.
|
|
type Scraper interface {
|
|
// GetProjects retrieves every project discoverable through this forge's
|
|
// public API and maps it to the generic Project type.
|
|
GetProjects() ([]Project, error)
|
|
}
|
|
|
|
// Scheme splits an input forge address into the connection scheme ("https"
|
|
// by default) and the bare hostname (the forge's identity). The input may be
|
|
// a bare FQDN ("git.example.org") or prefixed with a protocol to force http
|
|
// or https ("http://git.example.org"). The returned host is always
|
|
// protocol-free, so the database and metrics labels never carry a scheme.
|
|
func Scheme(addr string) (scheme, fqdn string) {
|
|
scheme = "https"
|
|
if i := strings.Index(addr, "://"); i >= 0 {
|
|
scheme = addr[:i]
|
|
addr = addr[i+3:]
|
|
}
|
|
if u, err := url.Parse("//" + addr); err == nil {
|
|
if h := u.Hostname(); h != "" {
|
|
fqdn = h
|
|
} else {
|
|
fqdn = addr
|
|
}
|
|
} else {
|
|
fqdn = addr
|
|
}
|
|
fqdn = strings.TrimRight(fqdn, "/")
|
|
return scheme, fqdn
|
|
}
|
|
|
|
// NewScraper returns a Scraper for the given forge type, e.g. TypeForgejo.
|
|
// scheme is the connection scheme ("https" unless the user forced http); fqdn
|
|
// is the forge's bare hostname, used as the identity attached to every
|
|
// returned Project. The connection URL is built as scheme://fqdn.
|
|
func NewScraper(typ Type, scheme, fqdn string) (Scraper, error) {
|
|
switch typ {
|
|
// Forgejo and Gitea expose the same /api/v1 surface (Forgejo is a fork
|
|
// of Gitea), so both are scraped with the Forgejo scraper class.
|
|
case TypeForgejo, TypeGitea:
|
|
return NewForgejoScraper(scheme, fqdn, string(typ))
|
|
default:
|
|
return nil, fmt.Errorf("unknown forge type %q", typ)
|
|
}
|
|
}
|
|
|
|
// ForgeIdentity identifies which forge a Project was scraped from. ForgeURL
|
|
// is the bare hostname of the instance (e.g. "git.example.org"); the HTTP
|
|
// scheme is added internally when a connection URL is needed.
|
|
type ForgeIdentity struct {
|
|
ForgeURL string
|
|
ForgeType string
|
|
}
|
|
|
|
// Project is a generic, forge-agnostic representation of an open source project.
|
|
type Project struct {
|
|
ForgeIdentity
|
|
Name string `json:"name"`
|
|
Url string `json:"url"`
|
|
Description string `json:"description"`
|
|
Topics []string `json:"topics"`
|
|
OpenIssues int `json:"open_issues"`
|
|
OpenPRs int `json:"open_prs"`
|
|
LatestCommit time.Time `json:"latest_commit"`
|
|
LatestRelease Release `json:"latest_release"`
|
|
Languages []string `json:"languages"`
|
|
Readme string `json:"readme"`
|
|
Licenses []string `json:"license"`
|
|
UsesAI bool `json:"uses_ai"`
|
|
}
|
|
|
|
// Release is a named release with the date it was published.
|
|
type Release struct {
|
|
Name string `json:"name"`
|
|
Date time.Time `json:"date"`
|
|
}
|