// Package scraper defines the interfaces and shared types used to discover // projects from forge instances (Forgejo, Gogs, ...). package scraper import ( "fmt" "net/url" "strings" "time" ) // Scraper is implemented by concrete, per-forge scrapers. type Scraper interface { // GetProjects retrieves every project discoverable through this forge's // public API and maps it to the generic Project type. GetProjects() ([]Project, error) } // Scheme splits an input forge address into the connection scheme ("https" // by default) and the bare hostname (the forge's identity). The input may be // a bare FQDN ("git.example.org") or prefixed with a protocol to force http // or https ("http://git.example.org"). The returned host is always // protocol-free, so the database and metrics labels never carry a scheme. func Scheme(addr string) (scheme, fqdn string) { scheme = "https" if i := strings.Index(addr, "://"); i >= 0 { scheme = addr[:i] addr = addr[i+3:] } if u, err := url.Parse("//" + addr); err == nil { if h := u.Hostname(); h != "" { fqdn = h } else { fqdn = addr } } else { fqdn = addr } fqdn = strings.TrimRight(fqdn, "/") return scheme, fqdn } // NewScraper returns a Scraper for the given forge type, e.g. TypeForgejo. // scheme is the connection scheme ("https" unless the user forced http); fqdn // is the forge's bare hostname, used as the identity attached to every // returned Project. The connection URL is built as scheme://fqdn. func NewScraper(typ Type, scheme, fqdn string) (Scraper, error) { switch typ { // Forgejo and Gitea expose the same /api/v1 surface (Forgejo is a fork // of Gitea), so both are scraped with the Forgejo scraper class. case TypeForgejo, TypeGitea: return NewForgejoScraper(scheme, fqdn, string(typ)) default: return nil, fmt.Errorf("unknown forge type %q", typ) } } // ForgeIdentity identifies which forge a Project was scraped from. ForgeURL // is the bare hostname of the instance (e.g. "git.example.org"); the HTTP // scheme is added internally when a connection URL is needed. type ForgeIdentity struct { ForgeURL string ForgeType string } // Project is a generic, forge-agnostic representation of an open source project. type Project struct { ForgeIdentity Name string `json:"name"` Url string `json:"url"` Description string `json:"description"` Topics []string `json:"topics"` OpenIssues int `json:"open_issues"` OpenPRs int `json:"open_prs"` LatestCommit time.Time `json:"latest_commit"` LatestRelease Release `json:"latest_release"` Languages []string `json:"languages"` Readme string `json:"readme"` Licenses []string `json:"license"` UsesAI bool `json:"uses_ai"` } // Release is a named release with the date it was published. type Release struct { Name string `json:"name"` Date time.Time `json:"date"` }