From bde13e9b685366457aac9a6859c6daf5d95d88d4 Mon Sep 17 00:00:00 2001 From: "Ricardo (XenGi) Band" Date: Mon, 28 Sep 2026 15:36:42 +0200 Subject: [PATCH] scraper done; scrapes to local sqlite db --- .gitignore | 5 +- README.md | 43 +++++- config.go | 14 ++ flake.lock | 85 ++++++++++++ flake.nix | 21 +-- forgejo.go | 85 ------------ go.mod | 53 ++++++- go.sum | 236 +++++++++++++++++++++++++++++-- gomod2nix.toml | 270 ++++++++++++++++++++++++++++++++++++ main.go | 68 ++++----- scraper.go | 23 --- scraper/forgejo.go | 230 ++++++++++++++++++++++++++++++ scraper/forgejo_licenses.go | 96 +++++++++++++ scraper/forgejo_readme.go | 70 ++++++++++ scraper/robots.go | 157 +++++++++++++++++++++ scraper/robots_test.go | 60 ++++++++ scraper/scraper.go | 47 +++++++ store/store.go | 190 +++++++++++++++++++++++++ 18 files changed, 1587 insertions(+), 166 deletions(-) create mode 100644 config.go create mode 100644 flake.lock delete mode 100644 forgejo.go create mode 100644 gomod2nix.toml delete mode 100644 scraper.go create mode 100644 scraper/forgejo.go create mode 100644 scraper/forgejo_licenses.go create mode 100644 scraper/forgejo_readme.go create mode 100644 scraper/robots.go create mode 100644 scraper/robots_test.go create mode 100644 scraper/scraper.go create mode 100644 store/store.go diff --git a/.gitignore b/.gitignore index cead4e1..c935c59 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ config.yaml models.json Containerfile +result +index.db # Created by https://www.toptal.com/developers/gitignore/api/windows,linux,macos,direnv,vim,go,goland+all # Edit at https://www.toptal.com/developers/gitignore?templates=windows,linux,macos,direnv,vim,go,goland+all @@ -142,7 +144,8 @@ fabric.properties .LSOverride # Icon must end with two \r -Icon +Icon + # Thumbnails ._* diff --git a/README.md b/README.md index 070201e..13156ae 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,44 @@ # indexer -Federated Forge Discovery Indexer \ No newline at end of file +Federated Forge Discovery Indexer + +Builds an index of public open source software projects from configured forges. + +## Scrape behavior + +For each configured forge, the indexer: + +1. **Checks robots.txt** (`GET /robots.txt`) before any API call. If the + instance disallows `/api/`, the instance is skipped. +2. **Lists public source repositories** via the repository search API + (`GET /api/v1/repos/search`), paginated. Forks and mirrors are excluded. +3. For each repository, enriches it via the Forgejo API: reads the root + directory once, then fetches topics, languages, latest commit, latest + release, the README, and detects licenses from the repo's files. + +Errors are non-fatal: an instance or a single repo that fails is skipped and +the run continues. + +### Request volume + +**Per instance:** + +| Request | Count | +|---|---| +| `GET /robots.txt` | 1 | +| `GET /api/v1/repos/search` (page) | `ceil(N/50)` where N = total repos | + +**Per repository:** + +| Request | Endpoint | Purpose | +|---|---|---| +| list root contents | `GET /api/v1/repos/{o}/{r}/contents/` | shared: README selection + license scan (1 call) | +| languages | `GET /api/v1/repos/{o}/{r}/languages` | `Project.Languages` | +| topics | `GET /api/v1/repos/{o}/{r}/topics` | `Project.Topics` | +| latest commit | `GET /api/v1/repos/{o}/{r}/commits?limit=1` | `Project.LatestCommit` | +| latest release | `GET /api/v1/repos/{o}/{r}/releases/latest` | `Project.LatestRelease` | +| README | `GET /api/v1/repos/{o}/{r}/raw/{branch}/{file}` | `Project.Readme` | +| license files | 1+ × `GET /api/v1/repos/{o}/{r}/raw/{branch}/{file}` | SPDX detection | + +So roughly **6–8 API requests per repository** plus the paginated search +requests and a single robots.txt check per instance. diff --git a/config.go b/config.go new file mode 100644 index 0000000..87b50b1 --- /dev/null +++ b/config.go @@ -0,0 +1,14 @@ +package main + +// Forge is a single forge instance to scrape. +type Forge struct { + Name string `yaml:"name"` + Url string `yaml:"url"` + Type string `yaml:"type"` +} + +// Config is the top-level configuration file. +type Config struct { + DbPath string `yaml:"db_path"` + Forges []Forge `yaml:"forges"` +} diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..bf253d3 --- /dev/null +++ b/flake.lock @@ -0,0 +1,85 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "gomod2nix": { + "inputs": { + "flake-utils": [ + "flake-utils" + ], + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1770585520, + "narHash": "sha256-yBz9Ozd5Wb56i3e3cHZ8WcbzCQ9RlVaiW18qDYA/AzA=", + "owner": "tweag", + "repo": "gomod2nix", + "rev": "1201ddd1279c35497754f016ef33d5e060f3da8d", + "type": "github" + }, + "original": { + "owner": "tweag", + "repo": "gomod2nix", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1790573947, + "narHash": "sha256-zfer1SLJvACKM0J7c9AusGvIp4q4B6l1Z1KgiVInG8Y=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "e6025a0259c7e80f81b11effc15f65d95b708cba", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable-small", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "gomod2nix": "gomod2nix", + "nixpkgs": "nixpkgs" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix index 50b30fd..7e77d0e 100644 --- a/flake.nix +++ b/flake.nix @@ -15,20 +15,22 @@ inherit system; overlays = [ gomod2nix.overlays.default ]; }; + go = pkgs.go_1_27; ffd-indexer = pkgs.buildGoApplication { + inherit go; pname = "ffd-indexer"; version = "0.0.1"; src = ./.; modules = ./gomod2nix.toml; }; - go-test = pkgs.stdenvNoCC.mkDerivation { + go-test = pkgs.stdenv.mkDerivation { name = "go-test"; dontBuild = true; src = ./.; doCheck = true; - nativeBuildInputs = with pkgs; [ + nativeBuildInputs = [ go - writableTmpDirAsHomeHook + pkgs.writableTmpDirAsHomeHook ]; checkPhase = '' go test -v ./... @@ -37,15 +39,15 @@ mkdir "$out" ''; }; - go-lint = pkgs.stdenvNoCC.mkDerivation { + go-lint = pkgs.stdenv.mkDerivation { name = "go-lint"; dontBuild = true; src = ./.; doCheck = true; - nativeBuildInputs = with pkgs; [ - golangci-lint + nativeBuildInputs = [ + pkgs.golangci-lint go - writableTmpDirAsHomeHook + pkgs.writableTmpDirAsHomeHook ]; checkPhase = '' golangci-lint run @@ -59,13 +61,14 @@ checks = { inherit go-test go-lint; }; formatter = pkgs.nixfmt-tree; devShells.default = pkgs.mkShell { - buildInputs = with pkgs; [ + buildInputs = [ go - go-tools # staticcheck + pkgs.go-tools # staticcheck gomod2nix.packages.${system}.default ]; packages = with pkgs; [ mkcert + sqlite # debugging the index db ]; }; packages = { diff --git a/forgejo.go b/forgejo.go deleted file mode 100644 index 0aef149..0000000 --- a/forgejo.go +++ /dev/null @@ -1,85 +0,0 @@ -// forgejo.go -package main - -import ( - "fmt" - - forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3" -) - -// ForgejoScraper retrieves public repositories from a Forgejo instance. -type ForgejoScraper struct { - Client *forgejo.Client - PageSize int -} - -// NewForgejoScraper creates a scraper for a Forgejo instance. -// BaseURL should be the instance URL, e.g. https://codeberg.org. -func NewForgejoScraper(baseURL string) (*ForgejoScraper, error) { - client, err := forgejo.NewClient(baseURL) - if err != nil { - return nil, fmt.Errorf("create Forgejo client: %w", err) - } - - return &ForgejoScraper{ - Client: client, - PageSize: 50, - }, nil -} - -// GetProjects retrieves every public repository visible through the -// Forgejo repository search API and maps it to the generic Project type. -func (s *ForgejoScraper) GetProjects() ([]Project, error) { - pageSize := s.PageSize - if pageSize <= 0 { - pageSize = 50 - } - - isPrivate := false - var projects []Project - - for page := 1; ; page++ { - repos, _, err := s.Client.SearchRepos(forgejo.SearchRepoOptions{ - ListOptions: forgejo.ListOptions{ - Page: page, - PageSize: pageSize, - }, - IsPrivate: &isPrivate, - Sort: "alpha", - Order: "asc", - }) - if err != nil { - return nil, fmt.Errorf( - "list public Forgejo repositories (page %d): %w", - page, - err, - ) - } - - if len(repos) == 0 { - break - } - - for _, repo := range repos { - if repo == nil { - continue - } - - projects = append(projects, Project{ - Name: repo.FullName, - Url: repo.HTMLURL, - Description: repo.Description, - Topics: repo.Topics, - OpenIssues: repo.OpenIssues, - OpenPRs: repo.OpenPulls, - }) - } - - // A short page indicates that there are no further results. - if len(repos) < pageSize { - break - } - } - - return projects, nil -} diff --git a/go.mod b/go.mod index a899122..774a74c 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,27 @@ module git.xengi.de/ffd/indexer go 1.27.1 require ( - codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3 v3.0.0 // indirect + codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3 v3.0.0 + github.com/go-enry/go-license-detector/v4 v4.3.1 + go.yaml.in/yaml/v4 v4.0.0-rc.6 + modernc.org/sqlite v1.59.0 +) + +require ( github.com/42wim/httpsig v1.2.3 // indirect + github.com/Microsoft/go-winio v0.6.0 // indirect + github.com/ProtonMail/go-crypto v0.0.0-20220930113650-c6815a8c17ad // indirect + github.com/acomagu/bufpipe v1.0.3 // indirect + github.com/cloudflare/circl v1.2.0 // indirect github.com/davidmz/go-pageant v1.0.2 // indirect + github.com/dgryski/go-minhash v0.0.0-20190315135803-ad340ca03076 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/ekzhu/minhash-lsh v0.0.0-20190924033628-faac2c6342f8 // indirect + github.com/emirpasic/gods v1.18.1 // indirect github.com/go-fed/httpsig v1.1.0 // indirect + github.com/go-git/gcfg v1.5.0 // indirect + github.com/go-git/go-billy/v5 v5.3.1 // indirect + github.com/go-git/go-git/v5 v5.4.2 // indirect github.com/go-openapi/errors v0.22.6 // indirect github.com/go-openapi/strfmt v0.25.0 // indirect github.com/go-openapi/swag v0.25.4 // indirect @@ -24,12 +41,36 @@ require ( github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/go-version v1.7.0 // indirect + github.com/hhatto/gorst v0.0.0-20181029133204-ca9f730cac5b // indirect + github.com/imdario/mergo v0.3.13 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/jdkato/prose v1.2.1 // indirect + github.com/kevinburke/ssh_config v1.2.0 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/montanaflynn/stats v0.7.1 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect github.com/oklog/ulid v1.3.1 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/sergi/go-diff v1.2.0 // indirect + github.com/shogo82148/go-shuffle v1.0.1 // indirect + github.com/xanzy/ssh-agent v0.3.2 // indirect go.mongodb.org/mongo-driver v1.17.6 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect - golang.org/x/crypto v0.45.0 // indirect - golang.org/x/net v0.47.0 // indirect - golang.org/x/sys v0.38.0 // indirect - golang.org/x/text v0.31.0 // indirect + golang.org/x/crypto v0.54.0 // indirect + golang.org/x/exp v0.0.0-20221006183845-316c7553db56 // indirect + golang.org/x/mod v0.38.0 // indirect + golang.org/x/net v0.57.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.40.0 // indirect + golang.org/x/tools v0.48.0 // indirect + gonum.org/v1/gonum v0.8.2 // indirect + gopkg.in/neurosnap/sentences.v1 v1.0.7 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect + modernc.org/libc v1.75.7 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.12.1 // indirect ) diff --git a/go.sum b/go.sum index 69937f7..1f92ed9 100644 --- a/go.sum +++ b/go.sum @@ -2,10 +2,62 @@ codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3 v3.0.0 h1:s2fK+FBwvcYsmKDjNhmoe7B8 codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3 v3.0.0/go.mod h1:Is2jTpS1dizeXm4skQv/ES3QVqnzcNhn2GzZXpiw9f8= github.com/42wim/httpsig v1.2.3 h1:xb0YyWhkYj57SPtfSttIobJUPJZB9as1nsfo7KWVcEs= github.com/42wim/httpsig v1.2.3/go.mod h1:nZq9OlYKDrUBhptd77IHx4/sZZD+IxTBADvAPI9G/EM= +github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= +github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= +github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= +github.com/Microsoft/go-winio v0.6.0 h1:slsWYD/zyx7lCXoZVlvQrj0hPTM1HI4+v1sIda2yDvg= +github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE= +github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo= +github.com/ProtonMail/go-crypto v0.0.0-20220930113650-c6815a8c17ad h1:QeeqI2zxxgZVe11UrYFXXx6gVxPVF40ygekjBzEg4XY= +github.com/ProtonMail/go-crypto v0.0.0-20220930113650-c6815a8c17ad/go.mod h1:UBYPn8k0D56RtnR8RFQMjmh4KrZzWJ5o7Z9SYjossQ8= +github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk= +github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= +github.com/bwesterb/go-ristretto v1.2.1/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= +github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= +github.com/cloudflare/circl v1.2.0 h1:NheeISPSUcYftKlfrLuOo4T62FkmD4t4jviLfFFYaec= +github.com/cloudflare/circl v1.2.0/go.mod h1:Ch2UgYr6ti2KTtlejELlROl0YIYj7SLjAC8M+INXlMk= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davidmz/go-pageant v1.0.2 h1:bPblRCh5jGU+Uptpz6LgMZGD5hJoOt7otgT454WvHn0= github.com/davidmz/go-pageant v1.0.2/go.mod h1:P2EDDnMqIwG5Rrp05dTRITj9z2zpGcD9efWSkTNKLIE= +github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc h1:8WFBn63wegobsYAX0YjD+8suexZDga5CctH4CCTx2+8= +github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc/go.mod h1:c9O8+fpSOX1DM8cPNSkX/qsBWdkD4yd2dpciOWQjpBw= +github.com/dgryski/go-minhash v0.0.0-20190315135803-ad340ca03076 h1:EB7M2v8Svo3kvIDy+P1YDE22XskDQP+TEYGzeDwPAN4= +github.com/dgryski/go-minhash v0.0.0-20190315135803-ad340ca03076/go.mod h1:VBi0XHpFy0xiMySf6YpVbRqrupW4RprJ5QTyN+XvGSM= +github.com/dgryski/go-spooky v0.0.0-20170606183049-ed3d087f40e2 h1:lx1ZQgST/imDhmLpYDma1O3Cx9L+4Ie4E8S2RjFPQ30= +github.com/dgryski/go-spooky v0.0.0-20170606183049-ed3d087f40e2/go.mod h1:hgHYKsoIw7S/hlWtP7wD1wZ7SX1jPTtKko5X9jrOgPQ= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/ekzhu/minhash-lsh v0.0.0-20190924033628-faac2c6342f8 h1:+Tje+xk1lmGKSJjYNtgCFsU1HtQzz0kCm1DFbKlvFBo= +github.com/ekzhu/minhash-lsh v0.0.0-20190924033628-faac2c6342f8/go.mod h1:yEtCVi+QamvzjEH4U/m6ZGkALIkF2xfQnFp0BcKmIOk= +github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0= +github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/go-enry/go-license-detector/v4 v4.3.1 h1:BajEVdTffFcs8RACmblySVhfEIuT58TmXx27RgVfUdc= +github.com/go-enry/go-license-detector/v4 v4.3.1/go.mod h1:YVJKPE01WQNjN/bdM6V0I/9KxvwEAAv0Ef9pi92K6w0= github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI= github.com/go-fed/httpsig v1.1.0/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM= +github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4= +github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E= +github.com/go-git/go-billy/v5 v5.2.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= +github.com/go-git/go-billy/v5 v5.3.1 h1:CPiOUAzKtMRvolEKw+bG1PLRpT7D3LIs3/3ey4Aiu34= +github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= +github.com/go-git/go-git-fixtures/v4 v4.2.1 h1:n9gGL1Ct/yIw+nfsfr8s4+sbhT+Ncu2SubfXjIWgci8= +github.com/go-git/go-git-fixtures/v4 v4.2.1/go.mod h1:K8zd3kDUAykwTdDCr+I0per6Y6vMiRR/nnVTBtavnB0= +github.com/go-git/go-git/v5 v5.4.2 h1:BXyZu9t0VkbiHtqrsvdq39UDhGJTl1h55VW6CSC4aY4= +github.com/go-git/go-git/v5 v5.4.2/go.mod h1:gQ1kArt6d+n+BGd+/B/I74HwRTLhth2+zti4ihgckDc= github.com/go-openapi/errors v0.22.6 h1:eDxcf89O8odEnohIXwEjY1IB4ph5vmbUsBMsFNwXWPo= github.com/go-openapi/errors v0.22.6/go.mod h1:z9S8ASTUqx7+CP1Q8dD8ewGH/1JWFFLX/2PmAYNQLgk= github.com/go-openapi/strfmt v0.25.0 h1:7R0RX7mbKLa9EYCTHRcCuIPcaqlyQiWNPTXwClK0saQ= @@ -22,6 +74,8 @@ github.com/go-openapi/swag/jsonname v0.25.4 h1:bZH0+MsS03MbnwBXYhuTttMOqk+5KcQ98 github.com/go-openapi/swag/jsonname v0.25.4/go.mod h1:GPVEk9CWVhNvWhZgrnvRA6utbAltopbKwDu8mXNUMag= github.com/go-openapi/swag/jsonutils v0.25.4 h1:VSchfbGhD4UTf4vCdR2F4TLBdLwHyUDTd1/q4i+jGZA= github.com/go-openapi/swag/jsonutils v0.25.4/go.mod h1:7OYGXpvVFPn4PpaSdPHJBtF0iGnbEaTk8AvBkoWnaAY= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.4 h1:IACsSvBhiNJwlDix7wq39SS2Fh7lUOCJRmx/4SN4sVo= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.4/go.mod h1:Mt0Ost9l3cUzVv4OEZG+WSeoHwjWLnarzMePNDAOBiM= github.com/go-openapi/swag/loading v0.25.4 h1:jN4MvLj0X6yhCDduRsxDDw1aHe+ZWoLjW+9ZQWIKn2s= github.com/go-openapi/swag/loading v0.25.4/go.mod h1:rpUM1ZiyEP9+mNLIQUdMiD7dCETXvkkC30z53i+ftTE= github.com/go-openapi/swag/mangling v0.25.4 h1:2b9kBJk9JvPgxr36V23FxJLdwBrpijI26Bx5JH4Hp48= @@ -34,38 +88,204 @@ github.com/go-openapi/swag/typeutils v0.25.4 h1:1/fbZOUN472NTc39zpa+YGHn3jzHWhv4 github.com/go-openapi/swag/typeutils v0.25.4/go.mod h1:Ou7g//Wx8tTLS9vG0UmzfCsjZjKhpjxayRKTHXf2pTE= github.com/go-openapi/swag/yamlutils v0.25.4 h1:6jdaeSItEUb7ioS9lFoCZ65Cne1/RZtPBZ9A56h92Sw= github.com/go-openapi/swag/yamlutils v0.25.4/go.mod h1:MNzq1ulQu+yd8Kl7wPOut/YHAAU/H6hL91fF+E2RFwc= +github.com/go-openapi/testify/enable/yaml/v2 v2.0.2 h1:0+Y41Pz1NkbTHz8NngxTuAXxEodtNSI1WG1c/m5Akw4= +github.com/go-openapi/testify/enable/yaml/v2 v2.0.2/go.mod h1:kme83333GCtJQHXQ8UKX3IBZu6z8T5Dvy5+CW3NLUUg= +github.com/go-openapi/testify/v2 v2.0.2 h1:X999g3jeLcoY8qctY/c/Z8iBHTbwLz7R2WXd6Ub6wls= +github.com/go-openapi/testify/v2 v2.0.2/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54= github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hhatto/gorst v0.0.0-20181029133204-ca9f730cac5b h1:Jdu2tbAxkRouSILp2EbposIb8h4gO+2QuZEn3d9sKAc= +github.com/hhatto/gorst v0.0.0-20181029133204-ca9f730cac5b/go.mod h1:HmaZGXHdSwQh1jnUlBGN2BeEYOHACLVGzYOXCbsLvxY= +github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= +github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/jdkato/prose v1.2.1 h1:Fp3UnJmLVISmlc57BgKUzdjr0lOtjqTZicL3PaYy6cU= +github.com/jdkato/prose v1.2.1/go.mod h1:AiRHgVagnEx2JbQRQowVBKjG0bcs/vtkGCH1dYAL1rA= +github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= +github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= +github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/montanaflynn/stats v0.6.3/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= +github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/neurosnap/sentences v1.0.6 h1:iBVUivNtlwGkYsJblWV8GGVFmXzZzak907Ci8aA0VTE= +github.com/neurosnap/sentences v1.0.6/go.mod h1:pg1IapvYpWCJJm/Etxeh0+gtMf1rI1STY9S7eUCPbDc= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= +github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/shogo82148/go-shuffle v0.0.0-20180218125048-27e6095f230d/go.mod h1:2htx6lmL0NGLHlO8ZCf+lQBGBHIbEujyywxJArf+2Yc= +github.com/shogo82148/go-shuffle v1.0.1 h1:4swIpHXLMAz14DE4YTgakgadpRN0n1wE1dieGnOTVFU= +github.com/shogo82148/go-shuffle v1.0.1/go.mod h1:HQPjVgUUZ9TNgm4/K/iXRuAdhPsQrXnAGgtk/9kqbBY= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0= +github.com/xanzy/ssh-agent v0.3.2 h1:eKj4SX2Fe7mui28ZgnFW5fmTz1EIr7ugo5s6wDxdHBM= +github.com/xanzy/ssh-agent v0.3.2/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= go.mongodb.org/mongo-driver v1.17.6 h1:87JUG1wZfWsr6rIz3ZmpH90rL5tea7O3IHuSwHUpsss= go.mongodb.org/mongo-driver v1.17.6/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4= go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= +golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= -golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= -golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= +golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= +golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20221006183845-316c7553db56 h1:BrYbdKcCNjLyrN6aKqXy4hPw9qGI8IATkj4EWv9Q+kQ= +golang.org/x/exp v0.0.0-20221006183845-316c7553db56/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= +golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= +golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= -golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= +golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220315194320-039c03cc5b86/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= -golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= +golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.8.2 h1:CCXrcPKiGGotvnN6jfUsKk4rRqm7q09/YbKb5xCEvtM= +gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0 h1:OE9mWmgKkjJyEmDAAtGMPjXu+YNeGvK9VTSHY6+Qihc= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/neurosnap/sentences.v1 v1.0.6/go.mod h1:YlK+SN+fLQZj+kY3r8DkGDhDr91+S3JmTb5LSxFRQo0= +gopkg.in/neurosnap/sentences.v1 v1.0.7 h1:gpTUYnqthem4+o8kyTLiYIB05W+IvdQFYR29erfe8uU= +gopkg.in/neurosnap/sentences.v1 v1.0.7/go.mod h1:YlK+SN+fLQZj+kY3r8DkGDhDr91+S3JmTb5LSxFRQo0= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/cc/v4 v4.29.2 h1:h6+9ciCnPKutf4I03CvheAvDLX7+IHlqR6Iy6J+cgd8= +modernc.org/cc/v4 v4.29.2/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.35.0 h1:F+TUsmw09QxLzmi3aeYYGxjAXarmZaKgj3mKQHNaA8w= +modernc.org/ccgo/v4 v4.35.0/go.mod h1:qrVGs9S3Sr2Ztcg9ve+kTAYMp5a3YvWjo+SoN06kJ5I= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc= +modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.75.7 h1:o3DTP9/0p9pKmY2WCKQaySW6wIiZhNM7wc2lUoyhfew= +modernc.org/libc v1.75.7/go.mod h1:bO5o2ztHxBb2rjz0PgdHN0sSMw57CgxGFLZ3Qd/QpVQ= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g= +modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.59.0 h1:X1es1GpqBlS/5T+vbM4HLUdaa8OtQx468DF2vrx+38A= +modernc.org/sqlite v1.59.0/go.mod h1:+paeT2A3iPRHkQDwG7oA6Tk0zQd5woMEI8q7orfry8k= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/gomod2nix.toml b/gomod2nix.toml new file mode 100644 index 0000000..c152ad5 --- /dev/null +++ b/gomod2nix.toml @@ -0,0 +1,270 @@ +schema = 3 + +[mod] + [mod.'codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3'] + version = 'v3.0.0' + hash = 'sha256-/fb4qz0lZt7vuxOELzthAiUy1oDC5x/PYCch3FUNWig=' + + [mod.'github.com/42wim/httpsig'] + version = 'v1.2.3' + hash = 'sha256-Yrf21S5q6mupTtyk8izVKQciJC/aRDe7YRKL0cBkRpQ=' + + [mod.'github.com/Microsoft/go-winio'] + version = 'v0.6.0' + hash = 'sha256-TQ0AvXZfuT3PDnb4p/h5zmyT8m9VPcwxlXMJuVD6ltE=' + + [mod.'github.com/ProtonMail/go-crypto'] + version = 'v0.0.0-20220930113650-c6815a8c17ad' + hash = 'sha256-mem+jc60bsyLIJblnHbnG5e3+KFWf2TfZo9In9viYwU=' + + [mod.'github.com/acomagu/bufpipe'] + version = 'v1.0.3' + hash = 'sha256-hIQf2lqZZv4dN8JwRyhNoBjStL9Mgih6f7Hy8RDU5Kg=' + + [mod.'github.com/cloudflare/circl'] + version = 'v1.2.0' + hash = 'sha256-9L1/4M8Za4qcrtti/1r1E7TgpdfUKHM+9IyuX3fuJyQ=' + + [mod.'github.com/davidmz/go-pageant'] + version = 'v1.0.2' + hash = 'sha256-gfwm6i0ZGQhX2c0lDRe7qVXyHkhoHeemYpaj+O0aYyw=' + + [mod.'github.com/dgryski/go-minhash'] + version = 'v0.0.0-20190315135803-ad340ca03076' + hash = 'sha256-CYfWI07pOkDyBgoHRUSY9QzOR/cWNLIa08M+0khbKX4=' + + [mod.'github.com/dustin/go-humanize'] + version = 'v1.0.1' + hash = 'sha256-yuvxYYngpfVkUg9yAmG99IUVmADTQA0tMbBXe0Fq0Mc=' + + [mod.'github.com/ekzhu/minhash-lsh'] + version = 'v0.0.0-20190924033628-faac2c6342f8' + hash = 'sha256-vZH3kLGjMEuDNiWHpTVXv1z0ZMYi40zai8DlOk7ivZM=' + + [mod.'github.com/emirpasic/gods'] + version = 'v1.18.1' + hash = 'sha256-hGDKddjLj+5dn2woHtXKUdd49/3xdsqnhx7VEdCu1m4=' + + [mod.'github.com/go-enry/go-license-detector/v4'] + version = 'v4.3.1' + hash = 'sha256-S9LKXjn5dL5FETOOAk+bs7bIVdu2x7MIhfjpZuXzuLo=' + + [mod.'github.com/go-fed/httpsig'] + version = 'v1.1.0' + hash = 'sha256-ok0PEEvSR010nh6vB3nDCyKSSVvoB50/UjVvBKOYXsA=' + + [mod.'github.com/go-git/gcfg'] + version = 'v1.5.0' + hash = 'sha256-A62eSLI/0y4sfwCsZhe/uoSC9Z3TryyW+PyWIcknYdE=' + + [mod.'github.com/go-git/go-billy/v5'] + version = 'v5.3.1' + hash = 'sha256-UdShPsStI2OBJig8Zv4Htyv5NiwKnYI41pwe4hIxv3Q=' + + [mod.'github.com/go-git/go-git/v5'] + version = 'v5.4.2' + hash = 'sha256-+RvDli5tPRLEs1pA9W/ZqCCm896y97ObTqLp3hLydpo=' + + [mod.'github.com/go-openapi/errors'] + version = 'v0.22.6' + hash = 'sha256-JcWNO3VO/OxaMptpMuFs6pYD7xFDtC5H8vn1EiF9w20=' + + [mod.'github.com/go-openapi/strfmt'] + version = 'v0.25.0' + hash = 'sha256-xz4aMKsHsDhUPVc27hBa4FrywKQABhLhEBNg3qClL9Y=' + + [mod.'github.com/go-openapi/swag'] + version = 'v0.25.4' + hash = 'sha256-aFQeLf95JC5gWLwIJXxhZHK9RxiGhh9IYV0MEYldDr4=' + + [mod.'github.com/go-openapi/swag/cmdutils'] + version = 'v0.25.4' + hash = 'sha256-nhlq5w/ULjwy7nptKcRryjZPlv3QG0FmgcBSOclqRlA=' + + [mod.'github.com/go-openapi/swag/conv'] + version = 'v0.25.4' + hash = 'sha256-uHgTdZC76LMSsq+x+RyeclnOgBsS0ID2cLgfjzFk0KQ=' + + [mod.'github.com/go-openapi/swag/fileutils'] + version = 'v0.25.4' + hash = 'sha256-NWacswHRvNZ81WzazA8yUmeDymtne2l8zLdArBmeYAU=' + + [mod.'github.com/go-openapi/swag/jsonname'] + version = 'v0.25.4' + hash = 'sha256-Z6uETeudh8W+/SGxFBnOB/VlJeRPRkFgCyJmntpz7bc=' + + [mod.'github.com/go-openapi/swag/jsonutils'] + version = 'v0.25.4' + hash = 'sha256-TQGeMImUuL5BWDoBHKAjZ1BvvxPXkFrV3TMgb82IrLk=' + + [mod.'github.com/go-openapi/swag/loading'] + version = 'v0.25.4' + hash = 'sha256-xr8OnFqB/kwBj3yf9HiLJwDcpL7I3/qJYlKb6VWbVFA=' + + [mod.'github.com/go-openapi/swag/mangling'] + version = 'v0.25.4' + hash = 'sha256-NrmwNgRJIUlOaAGt+XutHWSHy9e1COMnVE8B0UY5+sQ=' + + [mod.'github.com/go-openapi/swag/netutils'] + version = 'v0.25.4' + hash = 'sha256-wRMCQX/wIOvWm1g2UURobhzI47OldVqzlYYzwotLNTU=' + + [mod.'github.com/go-openapi/swag/stringutils'] + version = 'v0.25.4' + hash = 'sha256-rVo5NBH+oLLX9kaemUYWILd+TKw/TDTB8UdSz+mn3m8=' + + [mod.'github.com/go-openapi/swag/typeutils'] + version = 'v0.25.4' + hash = 'sha256-59RHnK6ugsAUc+A8DZCj7gZ3bJnt01Al7T/1kzM9PpA=' + + [mod.'github.com/go-openapi/swag/yamlutils'] + version = 'v0.25.4' + hash = 'sha256-zba7QX7Ds05oZq1opP/vwRSBEDaJ1pKm+9DgrsOuR9w=' + + [mod.'github.com/go-viper/mapstructure/v2'] + version = 'v2.4.0' + hash = 'sha256-lLfcV9z4n94hDhgyXJlde4bFB0hfzlbh+polqcJCwGE=' + + [mod.'github.com/google/uuid'] + version = 'v1.6.0' + hash = 'sha256-VWl9sqUzdOuhW0KzQlv0gwwUQClYkmZwSydHG2sALYw=' + + [mod.'github.com/hashicorp/go-version'] + version = 'v1.7.0' + hash = 'sha256-Umc/Q2mxRrPF4aEcDuDJq4lFBT+PSsnyANfyFwUIaxI=' + + [mod.'github.com/hhatto/gorst'] + version = 'v0.0.0-20181029133204-ca9f730cac5b' + hash = 'sha256-u1rB/Z20x3UYMKaLjlDP3bHnD7nMrVs5AHR48km/H1k=' + + [mod.'github.com/imdario/mergo'] + version = 'v0.3.13' + hash = 'sha256-03LKAZXgR5YUMeK5IRh2ds2GyV6uSbdq5QCPbAzlTus=' + + [mod.'github.com/jbenet/go-context'] + version = 'v0.0.0-20150711004518-d14ea06fba99' + hash = 'sha256-VANNCWNNpARH/ILQV9sCQsBWgyL2iFT+4AHZREpxIWE=' + + [mod.'github.com/jdkato/prose'] + version = 'v1.2.1' + hash = 'sha256-xjGGiunKj2XUGuUMxdKrObOUAoV1QUmwkPVhyjLc3S8=' + + [mod.'github.com/kevinburke/ssh_config'] + version = 'v1.2.0' + hash = 'sha256-Ta7ZOmyX8gG5tzWbY2oES70EJPfI90U7CIJS9EAce0s=' + + [mod.'github.com/mattn/go-isatty'] + version = 'v0.0.24' + hash = 'sha256-pmq8KKIb2+qMF3E1cR2B7NAj5joF2tZduyXjIfIlkTk=' + + [mod.'github.com/mitchellh/go-homedir'] + version = 'v1.1.0' + hash = 'sha256-oduBKXHAQG8X6aqLEpqZHs5DOKe84u6WkBwi4W6cv3k=' + + [mod.'github.com/montanaflynn/stats'] + version = 'v0.7.1' + hash = 'sha256-0QXUA/syOtDSBooh3isAffrVAjpSoBmSMMuZfO5maHg=' + + [mod.'github.com/ncruces/go-strftime'] + version = 'v1.0.0' + hash = 'sha256-GYIwYDONuv/yTE0AEugCHQbtV3oiBaco93xUNYFcVBQ=' + + [mod.'github.com/oklog/ulid'] + version = 'v1.3.1' + hash = 'sha256-LNn883rYNiaoY9sGEPIzlMRx5UwGThdYTjXqfzeGc9k=' + + [mod.'github.com/pkg/errors'] + version = 'v0.9.1' + hash = 'sha256-mNfQtcrQmu3sNg/7IwiieKWOgFQOVVe2yXgKBpe/wZw=' + + [mod.'github.com/remyoudompheng/bigfft'] + version = 'v0.0.0-20230129092748-24d4a6f8daec' + hash = 'sha256-vYmpyCE37eBYP/navhaLV4oX4/nu0Z/StAocLIFqrmM=' + + [mod.'github.com/russross/blackfriday/v2'] + version = 'v2.1.0' + hash = 'sha256-R+84l1si8az5yDqd5CYcFrTyNZ1eSYlpXKq6nFt4OTQ=' + + [mod.'github.com/sergi/go-diff'] + version = 'v1.2.0' + hash = 'sha256-d2higuBRee4ylRuCpPQV5+g0XK5yNQgDpudNor0qD2o=' + + [mod.'github.com/shogo82148/go-shuffle'] + version = 'v1.0.1' + hash = 'sha256-E5Xt69A2vXlVPzyHPyWK0FyQ+IaMZzqQCnUltedeZhg=' + + [mod.'github.com/xanzy/ssh-agent'] + version = 'v0.3.2' + hash = 'sha256-QavRCX75546hnWCOtBwiUdmyk6AoLjv3POoJXng5kMI=' + + [mod.'go.mongodb.org/mongo-driver'] + version = 'v1.17.6' + hash = 'sha256-B1ePaR77HKDksfG6PLwOmOVPYac9g13i8Ld/xWOCIiE=' + + [mod.'go.yaml.in/yaml/v3'] + version = 'v3.0.4' + hash = 'sha256-NkGFiDPoCxbr3LFsI6OCygjjkY0rdmg5ggvVVwpyDQ4=' + + [mod.'go.yaml.in/yaml/v4'] + version = 'v4.0.0-rc.6' + hash = 'sha256-9aOIQpQ53KLv/Hb14104qYKLVrZUcCGLwzYp6UGLP90=' + + [mod.'golang.org/x/crypto'] + version = 'v0.54.0' + hash = 'sha256-m8jsqlPuoPxGDjClzcAVbXf1aiU07L+9j7vYcqJu/0Q=' + + [mod.'golang.org/x/exp'] + version = 'v0.0.0-20221006183845-316c7553db56' + hash = 'sha256-2aATMtazbT3i+X3m0Ju54GKOggcyeX9dAJFxO+H5IxQ=' + + [mod.'golang.org/x/mod'] + version = 'v0.38.0' + hash = 'sha256-BpKfvsmb7ecEDvTBKx9ZHauxoWRkOAZCqg4BrHM2/S0=' + + [mod.'golang.org/x/net'] + version = 'v0.57.0' + hash = 'sha256-gKo8UMw4hfETBHm8N5GOfuNadse9TWEQXJo2YWq5bY4=' + + [mod.'golang.org/x/sync'] + version = 'v0.22.0' + hash = 'sha256-VZjl0fAM0p/nI81zh+pdBjzxFupx0UcKYXBBpL2ZS7k=' + + [mod.'golang.org/x/sys'] + version = 'v0.47.0' + hash = 'sha256-TpbRyWWqHjddP6QzUgAbaLd2EE0S+GYNRUIDJd18r98=' + + [mod.'golang.org/x/text'] + version = 'v0.40.0' + hash = 'sha256-LJfnki46XEreGbSgjl+DeqgcTsINTOu2owyXNvijMcA=' + + [mod.'golang.org/x/tools'] + version = 'v0.48.0' + hash = 'sha256-9cRNUaup6fexA5S1zc+Ic3aaoxAgKWntyMJ2xaOdiP8=' + + [mod.'gonum.org/v1/gonum'] + version = 'v0.8.2' + hash = 'sha256-eFUTetKRBa9QzDUEtd6aU8KsTJQgBF6zLAveY4Ybi58=' + + [mod.'gopkg.in/neurosnap/sentences.v1'] + version = 'v1.0.7' + hash = 'sha256-AaBiqQ5Mtrzk4dE0lZidyUtDzYVy7aonEeh5oXrq/VI=' + + [mod.'gopkg.in/warnings.v0'] + version = 'v0.1.2' + hash = 'sha256-ATVL9yEmgYbkJ1DkltDGRn/auGAjqGOfjQyBYyUo8s8=' + + [mod.'modernc.org/libc'] + version = 'v1.75.7' + hash = 'sha256-Iq3Pcr3RrbhC82RWDM1I4gyKh3V/HI1gVQzoGR+0YYo=' + + [mod.'modernc.org/mathutil'] + version = 'v1.7.1' + hash = 'sha256-COZ5rF2GhQVR1r6a0DanJ8qwQ94JSKdQxTMWrDzE0Cc=' + + [mod.'modernc.org/memory'] + version = 'v1.12.1' + hash = 'sha256-Y/kW9/A5uF+Y66kDbxrh4muFgE3CEee6YzSl2FMPcRg=' + + [mod.'modernc.org/sqlite'] + version = 'v1.59.0' + hash = 'sha256-CaNi9UvS9y6aiiSy3GUlRHX84og+i4MWOZmxbFONeb4=' diff --git a/main.go b/main.go index d8e7db7..99aed99 100644 --- a/main.go +++ b/main.go @@ -5,18 +5,11 @@ import ( "os" "go.yaml.in/yaml/v4" + + "git.xengi.de/ffd/indexer/scraper" + "git.xengi.de/ffd/indexer/store" ) -type Forge struct { - Name string `yaml:"name"` - Url string `yaml:"url"` - Type string `yaml:"type"` -} - -type Config struct { - Forges []Forge `yaml:"forges"` -} - func main() { data, err := os.ReadFile("config.yaml") if err != nil { @@ -27,33 +20,42 @@ func main() { if err := yaml.Unmarshal(data, &config); err != nil { panic(err) } - var projects []Project - var allProjects []Project + + db, err := store.Open(config.DbPath) + if err != nil { + panic(err) + } + defer func() { _ = db.Close() }() + for _, forge := range config.Forges { - fmt.Printf("%+v: %+v\n", forge.Name, forge.Url) + fmt.Printf("Scraping %s (%s, %s)\n", forge.Name, forge.Type, forge.Url) - switch forge.Type { - case "forgejo": - scraper, err := NewForgejoScraper(forge.Url) - if err != nil { - panic(err) - } - - projects, err := scraper.GetProjects() - if err != nil { - panic(err) - } - default: - projects = []Project{} - fmt.Errorf("unknown forge: %s", forge.Type) + s, err := scraper.NewScraper(forge.Type, forge.Url) + if err != nil { + fmt.Fprintf(os.Stderr, " skip %s: %v\n", forge.Name, err) + continue } - fmt.Printf("Found %d repositories\n", len(projects)) - - allProjects = append(allProjects, projects...) - - for repo := range allProjects { - fmt.Printf("%+v\n", repo) + projects, err := s.GetProjects() + if err != nil { + fmt.Fprintf(os.Stderr, " skip %s: %v\n", forge.Name, err) + continue } + + fmt.Printf(" found %d projects\n", len(projects)) + for _, p := range projects { + if err := db.Upsert(p); err != nil { + fmt.Fprintf(os.Stderr, " store %s: %v\n", p.Name, err) + continue + } + fmt.Printf(" %s\n", p.Name) + } + + all, err := db.All() + if err != nil { + fmt.Fprintf(os.Stderr, " count: %v\n", err) + continue + } + fmt.Printf(" total in db: %d\n", len(all)) } } diff --git a/scraper.go b/scraper.go deleted file mode 100644 index 473f58a..0000000 --- a/scraper.go +++ /dev/null @@ -1,23 +0,0 @@ -package main - -import ( - "time" -) - -type Project struct { - Name string `json:"name"` - Url string `json:"url"` - Description string `json:"description"` - Topics []string `json:"topics"` - OpenIssues int `json:"open_issues"` - ClosedIssues int `json:"closed_issues"` - OpenPRs int `json:"open_prs"` - ClosedPRs int `json:"closed_prs"` - LatestCommit time.Time `json:"latest_commit"` - LatestTag time.Time `json:"latest_tag"` - LatestRelease time.Time `json:"latest_release"` - Contributors int `json:"contributors"` - Languages []string `json:"languages"` - Readme string `json:"readme"` - Licence string `json:"licence"` -} diff --git a/scraper/forgejo.go b/scraper/forgejo.go new file mode 100644 index 0000000..f2023ee --- /dev/null +++ b/scraper/forgejo.go @@ -0,0 +1,230 @@ +// Package scraper forgejo.go - Forgejo scraper. +package scraper + +import ( + "fmt" + "sort" + "strings" + "time" + + forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3" + + "github.com/go-enry/go-license-detector/v4/licensedb/filer" +) + +// ForgejoScraper retrieves public repositories from a Forgejo instance. +type ForgejoScraper struct { + Client *forgejo.Client + BaseURL string + PageSize int +} + +// NewForgejoScraper creates a scraper for a Forgejo instance. +// BaseURL should be the instance URL, e.g. https://codeberg.org. +func NewForgejoScraper(baseURL string) (*ForgejoScraper, error) { + client, err := forgejo.NewClient(baseURL) + if err != nil { + return nil, fmt.Errorf("create Forgejo client: %w", err) + } + client.SetUserAgent(UserAgent) + + return &ForgejoScraper{ + Client: client, + BaseURL: strings.TrimSuffix(baseURL, "/"), + PageSize: 50, + }, nil +} + +// GetProjects retrieves every public repository visible through the +// Forgejo repository search API and maps it to the generic Project type. +func (s *ForgejoScraper) GetProjects() ([]Project, error) { + // Honor the instance's robots.txt before making any API call: + // if /api/ is disallowed, do not scrape this instance. + ok, err := Allowed(s.BaseURL) + if err != nil { + return nil, fmt.Errorf("check robots.txt: %w", err) + } + if !ok { + return nil, fmt.Errorf("instance robots.txt disallows /api/") + } + + pageSize := s.PageSize + if pageSize <= 0 { + pageSize = 50 + } + + isPrivate := false + var projects []Project + + for page := 1; ; page++ { + repos, _, err := s.Client.SearchRepos(forgejo.SearchRepoOptions{ + Page: page, + PageSize: pageSize, + IsPrivate: &isPrivate, + Type: forgejo.RepoTypeSource, + Sort: "alpha", + Order: "asc", + }) + if err != nil { + return nil, fmt.Errorf( + "list public Forgejo repositories (page %d): %w", + page, + err, + ) + } + + if len(repos) == 0 { + break + } + + for _, repo := range repos { + if repo == nil { + continue + } + + owner, name, ok := strings.Cut(repo.FullName, "/") + if !ok { + continue + } + + // List the repository root once and reuse it for readme selection + // and license detection (they previously each listed the root). + entries, _, err := s.Client.ListContents(owner, name, repo.DefaultBranch, "") + if err != nil { + // best-effort: fall back to empty root so the other fields still work + entries = nil + } + root := make([]filer.File, 0, len(entries)) + for _, e := range entries { + if e == nil { + continue + } + root = append(root, filer.File{Name: e.Name, IsDir: e.Type == "dir"}) + } + + // topics currently requires one extra HTTP call per repository. + // Future optimization: the /repos/search response already includes + // the topics field, so they could be gathered by parsing the + // SearchRepos payload itself (e.g. via ListRepos) instead. + // On failure the repo is kept in the results with empty topics. + topics, err := s.topics(owner, name) + if err != nil { + topics = nil + } + + // languages requires one extra HTTP call per repository + // (GET /repos/{owner}/{repo}/languages, via GetRepoLanguages). + // It returns a map[language]bytes; we order it most-dominant-first + // below. On failure the repo is kept with no languages. + langs, _, err := s.Client.GetRepoLanguages(owner, name) + if err != nil { + langs = nil + } + + // Latest commit: one HTTP call per repository + // (GET /repos/{owner}/{repo}/commits?limit=1, via ListRepoCommits), returns the + // newest commit first. SHA + date are mapped into LatestCommit. + latestCommit, _ := s.latestCommit(owner, name) + + // Latest release: one HTTP call per repository + // (GET /repos/{owner}/{repo}/releases/latest, via GetLatestRelease). + // Returns a single Release (server already excludes drafts and prereleases). + latestRelease, _ := s.latestRelease(owner, name) + + // Licenses: scan repo files on its default branch with + // go-license-detector, reusing the root listing. May require reads + // of candidate license files. On failure the repo is kept with no + // licenses. + licenses := s.licensesFor(owner, name, repo.DefaultBranch, root) + + // Readme: pick the default README (md/rst/txt/no extension) from the + // shared root listing and fetch it. Non-fatal when absent. + readme := s.readmeFor(owner, name, repo.DefaultBranch, root) + + projects = append(projects, Project{ + Name: repo.FullName, + Url: repo.HTMLURL, + Description: repo.Description, + OpenIssues: repo.OpenIssues, + OpenPRs: repo.OpenPulls, + Topics: topics, + Languages: orderedLanguages(langs), + LatestCommit: latestCommit, + LatestRelease: latestRelease, + Licenses: licenses, + Readme: readme, + }) + } + + // A short page indicates that there are no further results. + if len(repos) < pageSize { + break + } + } + + return projects, nil +} + +// topics returns the repository's topics. +func (s *ForgejoScraper) topics(owner, name string) ([]string, error) { + topics, _, err := s.Client.ListRepoTopics(owner, name, forgejo.ListRepoTopicsOptions{}) + if err != nil { + return nil, fmt.Errorf("list topics for %s/%s: %w", owner, name, err) + } + return topics, nil +} + +// latestCommit returns the most recent commit's time. On failure it returns +// the zero time and the error, leaving the Project's LatestCommit unset. +func (s *ForgejoScraper) latestCommit(owner, name string) (time.Time, error) { + commits, _, err := s.Client.ListRepoCommits(owner, name, forgejo.ListCommitOptions{ + PageSize: 1, + }) + if err != nil { + return time.Time{}, err + } + if len(commits) == 0 || commits[0] == nil { + return time.Time{}, nil + } + return commits[0].Created, nil +} + +// latestRelease returns the most recent non-draft, non-prerelease release. +// On failure (e.g. no releases) it returns a zero Release and the error, +// leaving the Project's LatestRelease unset. +func (s *ForgejoScraper) latestRelease(owner, name string) (Release, error) { + r, _, err := s.Client.GetLatestRelease(owner, name) + if err != nil { + return Release{}, err + } + if r == nil { + return Release{}, nil + } + return Release{ + Name: r.Title, + Date: r.PublishedAt, + }, nil +} + +// orderedLanguages returns the languages of a repo ordered by byte share, +// most dominant first. Ties are broken alphabetically for stable output. +func orderedLanguages(langs map[string]int64) []string { + if len(langs) == 0 { + return nil + } + + names := make([]string, 0, len(langs)) + for name := range langs { + names = append(names, name) + } + + // Descending byte count; alphabetically as a stable tie-breaker. + sort.Slice(names, func(i, j int) bool { + if langs[names[i]] != langs[names[j]] { + return langs[names[i]] > langs[names[j]] + } + return names[i] < names[j] + }) + + return names +} diff --git a/scraper/forgejo_licenses.go b/scraper/forgejo_licenses.go new file mode 100644 index 0000000..8094912 --- /dev/null +++ b/scraper/forgejo_licenses.go @@ -0,0 +1,96 @@ +package scraper + +import ( + "fmt" + + forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3" + + "github.com/go-enry/go-license-detector/v4/licensedb" + "github.com/go-enry/go-license-detector/v4/licensedb/filer" +) + +// forgejoFiler adapts the Forgejo content API to the filer.Filer interface so +// that go-license-detector can inspect a repository's files without a local +// checkout. ReadDir serves a cached root listing when available (to avoid an +// extra ListContents call) and falls back to the API otherwise. ReadFile +// downloads a file's bytes at the given ref (default branch). +type forgejoFiler struct { + client *forgejo.Client + owner string + repo string + ref string + // cachedRoot, when non-nil, is served for ReadDir("") to avoid a second + // root listing that was already fetched by the caller. + cachedRoot []filer.File +} + +var _ filer.Filer = (*forgejoFiler)(nil) + +func (f *forgejoFiler) ReadDir(dirPath string) ([]filer.File, error) { + if dirPath == "" && f.cachedRoot != nil { + return f.cachedRoot, nil + } + entries, _, err := f.client.ListContents(f.owner, f.repo, f.ref, dirPath) + if err != nil { + return nil, fmt.Errorf("list contents %q: %w", dirPath, err) + } + files := make([]filer.File, 0, len(entries)) + for _, e := range entries { + if e == nil { + continue + } + files = append(files, filer.File{ + Name: e.Name, + IsDir: e.Type == "dir", + }) + } + return files, nil +} + +func (f *forgejoFiler) ReadFile(path string) ([]byte, error) { + // GetFile returns raw file bytes (binary-safe, base64-decoded). + data, _, err := f.client.GetFile(f.owner, f.repo, f.ref, path) + if err != nil { + return nil, fmt.Errorf("read file %q: %w", path, err) + } + return data, nil +} + +func (f *forgejoFiler) Close() {} + +func (f *forgejoFiler) PathsAreAlwaysSlash() bool { return true } + +// licensesFor returns the SPDX identifiers detected for the repository. It +// uses go-license-detector over the repository's files on its default branch. +// The given root listing is served to the detector to avoid a duplicate, +// already-available API call. An empty slice is returned when no license can +// be determined (the repo is still kept in results -- license failures are +// non-fatal). +func (s *ForgejoScraper) licensesFor(owner, repo, ref string, root []filer.File) []string { + detector := &forgejoFiler{ + client: s.Client, + owner: owner, + repo: repo, + ref: ref, + cachedRoot: root, + } + + matches, err := licensedb.Detect(detector) + if err != nil { + return nil + } + + // Keep only the highest-confidence match, sorted deterministically. + ids := make([]string, 0, len(matches)) + bestID, bestConf := "", float32(-1) + for id, m := range matches { + if m.Confidence > bestConf { + bestConf = m.Confidence + bestID = id + } + } + if bestID != "" { + ids = append(ids, bestID) + } + return ids +} diff --git a/scraper/forgejo_readme.go b/scraper/forgejo_readme.go new file mode 100644 index 0000000..fe7122a --- /dev/null +++ b/scraper/forgejo_readme.go @@ -0,0 +1,70 @@ +package scraper + +import ( + "sort" + "strings" + + "github.com/go-enry/go-license-detector/v4/licensedb/filer" +) + +// readmeCandidates are the filenames looked for, in priority order, when +// retrieving a repository's README. +var readmeCandidates = []string{"README.md", "README.rst", "README.txt", "README"} + +// readmeFor returns the content of the repository's README on the given ref +// (default branch), picking from root (the pre-fetched root file listing) the +// first candidate found (case-insensitively). An empty string is returned when +// no README can be found; the repo is still kept in results either way. +func (s *ForgejoScraper) readmeFor(owner, name, ref string, root []filer.File) string { + // Collect root file names, lowercased for case-insensitive matching, while + // keeping the real name for the GetFile call. + type rootFile struct { + real string + low string + } + files := make([]rootFile, 0, len(root)) + for _, e := range root { + if e.IsDir { + continue + } + files = append(files, rootFile{real: e.Name, low: strings.ToLower(e.Name)}) + } + + // Candidate priority wins; ties broken by alphabetical real name for + // deterministic picks when a repo has e.g. both README and README.md. + sort.SliceStable(files, func(i, j int) bool { + pi, pj := readmePriority(files[i].low), readmePriority(files[j].low) + if pi != pj { + return pi < pj // lower = higher priority + } + return files[i].real < files[j].real + }) + + var chosen string + for i := range files { + if readmePriority(files[i].low) >= 0 { + chosen = files[i].real + break + } + } + if chosen == "" { + return "" + } + + data, _, err := s.Client.GetFile(owner, name, ref, chosen) + if err != nil { + return "" + } + return string(data) +} + +// readmePriority returns the index of a lowercased filename in +// readmeCandidates, or -1 if it is not a readme candidate. +func readmePriority(low string) int { + for i, c := range readmeCandidates { + if low == strings.ToLower(c) { + return i + } + } + return -1 +} diff --git a/scraper/robots.go b/scraper/robots.go new file mode 100644 index 0000000..bbbbd92 --- /dev/null +++ b/scraper/robots.go @@ -0,0 +1,157 @@ +package scraper + +import ( + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" +) + +// UserAgent identifies this crawler to servers. Keep it stable so that +// instances can set robots rules for us specifically. +const UserAgent = "ffd-indexer/0.1" + +// robotsHTTPClient is used for fetching robots.txt. It does not follow the +// SDK client's settings; it is a plain, short-timeout client. +var robotsHTTPClient = &http.Client{Timeout: 10 * time.Second} + +// Allowed reports whether an instance at baseURL permits crawling its API +// according to its robots.txt. It only consults the rules for the +// User-Agent "*"; a block of /api/ (the API root) disallows scraping. +// +// A failure to fetch or parse robots.txt is treated as an error; callers +// decide whether that is fatal. +func Allowed(baseURL string) (bool, error) { + u, err := url.Parse(baseURL) + if err != nil { + return false, fmt.Errorf("parse base URL: %w", err) + } + u.Path = "/robots.txt" + + req, err := http.NewRequest(http.MethodGet, u.String(), nil) + if err != nil { + return false, fmt.Errorf("build robots request: %w", err) + } + req.Header.Set("User-Agent", UserAgent) + + resp, err := robotsHTTPClient.Do(req) + if err != nil { + return false, fmt.Errorf("fetch robots.txt: %w", err) + } + defer func() { + _ = resp.Body.Close() + }() + + // A 404 or similar means "no robots.txt": assume allowed. + if resp.StatusCode == http.StatusNotFound { + return true, nil + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return false, fmt.Errorf("robots.txt returned %s", resp.Status) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return false, fmt.Errorf("read robots.txt: %w", err) + } + + return allowedFor(string(body), UserAgent), nil +} + +// allowedFor parses a robots.txt body and reports whether a crawler with the +// given user agent may crawl the path "/api/". It implements a reasonable +// subset of the robots.txt standard: +// +// - Only the "*" group and an explicit group matching userAgent are +// considered; later groups override earlier ones for the same agent. +// - Disallow entries are matched as path prefixes (a trailing "*" is +// stripped). No other wildcard/glob support. +// - An empty group (no rules) is "allow everything". +func allowedFor(body, userAgent string) bool { + groups := parseRobotGroups(body) + + // A group that explicitly names our user agent takes precedence over a + // "*" group, regardless of order. Otherwise fall back to "*". + var star *robotGroup + for _, g := range groups { + for _, a := range g.agents { + if strings.EqualFold(a, "*") { + star = g + } else if strings.EqualFold(a, userAgent) { + return !matchDisallow(g, "/api/") + } + } + } + if star != nil { + return !matchDisallow(star, "/api/") + } + return true +} + +type robotGroup struct { + agents []string + disallow []string +} + +// parseRobotGroups splits a robots.txt body into agent groups. +func parseRobotGroups(body string) []*robotGroup { + var groups []*robotGroup + var cur *robotGroup + + for _, raw := range strings.Split(body, "\n") { + line := strings.TrimSpace(raw) + // Strip an inline comment (not part of a rule's result). + if i := strings.IndexByte(line, '#'); i >= 0 { + line = strings.TrimSpace(line[:i]) + } + if line == "" { + continue + } + + key, val, ok := strings.Cut(line, ":") + if !ok { + continue + } + key = strings.ToLower(strings.TrimSpace(key)) + val = strings.TrimSpace(val) + + switch key { + case "user-agent": + // A User-agent line starts a new group, unless the current + // group is still collecting agents (has no Disallow yet). + if cur == nil || len(cur.disallow) > 0 { + cur = &robotGroup{} + groups = append(groups, cur) + } + cur.agents = append(cur.agents, val) + case "disallow": + if cur != nil { + cur.disallow = append(cur.disallow, val) + } + } + } + + return groups +} + +// matchDisallow reports whether a path is disallowed by a group's rules. +// A trailing "*" is stripped. Rules are applied in order, later rules +// overriding earlier ones: an empty Disallow rule revokes all preceding +// disallows (allow everything). +func matchDisallow(g *robotGroup, path string) bool { + disallowed := false + for _, rule := range g.disallow { + rule = strings.TrimSpace(rule) + rule = strings.TrimSuffix(rule, "*") + if rule == "" { + disallowed = false // empty Disallow: allow everything + continue + } + if strings.HasPrefix(path, rule) { + disallowed = true + } + } + return disallowed +} diff --git a/scraper/robots_test.go b/scraper/robots_test.go new file mode 100644 index 0000000..15b569c --- /dev/null +++ b/scraper/robots_test.go @@ -0,0 +1,60 @@ +package scraper + +import "testing" + +func TestAllowedFor(t *testing.T) { + const ua = "ffd-indexer/0.1" + + // xengi / cccb: only /archive/ blocked -> /api/ allowed. + if !allowedFor("User-agent: *\nDisallow: /*/*/archive/\n", ua) { + t.Error("expected /api/ allowed when only archive is disallowed") + } + + // codeberg: /api/ disallowed under "*". + codeberg := `User-agent: * +Disallow: /api/ +Disallow: /avatars/ +` + if allowedFor(codeberg, ua) { + t.Error("expected /api/ blocked when /api/ is disallowed") + } + + // Explicit agent group takes precedence over "*" (agent name matches + // UserAgent exactly, including the version). + // Here the explicit group is for "ffd-indexer" (no version), so it does + // not match "ffd-indexer/0.1"; the "*" group (blocks /api/) applies. + if allowedFor("User-agent: *\nDisallow: /api/\n\nUser-agent: ffd-indexer\nDisallow: /whatever/\n", ua) { + t.Error("expected unmatched explicit agent to fall back to * which blocks /api/") + } + + // A matching explicit agent group takes precedence over "*". + if !allowedFor("User-agent: *\nDisallow: /api/\n\nUser-agent: ffd-indexer/0.1\nDisallow: /whatever/\n", ua) { + t.Error("expected explicit agent rule to permit /api/") + } + + // No rules at all -> allow. + if !allowedFor("", ua) { + t.Error("expected empty robots to allow") + } + + // Only an unrelated agent listed -> no star, no match -> allow. + if !allowedFor("User-agent: SomeBot\nDisallow: /api/\n", ua) { + t.Error("expected unrelated agent not to block us") + } + + // Disallow "/" blocks everything including /api/. + if allowedFor("User-agent: *\nDisallow: /\n", ua) { + t.Error("expected Disallow / to block /api/") + } + + // Empty Disallow (allowed) overrides a previous block in the same group. + if !allowedFor("User-agent: *\nDisallow: /api/\nDisallow:\n", ua) { + t.Error("expected empty Disallow to allow /api/") + } + + // Wildcard on a non-prefix rule should not over-block (substring-only). + // "/*.pdf" should not match /api/. + if !allowedFor("User-agent: *\nDisallow: /*.pdf$\n", ua) { + t.Error("expected unrelated wildcard to not block /api/") + } +} diff --git a/scraper/scraper.go b/scraper/scraper.go new file mode 100644 index 0000000..a4f97ee --- /dev/null +++ b/scraper/scraper.go @@ -0,0 +1,47 @@ +// Package scraper defines the interfaces and shared types used to discover +// projects from forge instances (Forgejo, Gogs, ...). +package scraper + +import ( + "fmt" + "time" +) + +// Scraper is implemented by concrete, per-forge scrapers. +type Scraper interface { + // GetProjects retrieves every project discoverable through this forge's + // public API and maps it to the generic Project type. + GetProjects() ([]Project, error) +} + +// NewScraper returns a Scraper for the given forge type, e.g. "forgejo". +func NewScraper(typ, baseURL string) (Scraper, error) { + switch typ { + case "forgejo": + return NewForgejoScraper(baseURL) + default: + return nil, fmt.Errorf("unknown forge type %q", typ) + } +} + +// Project is a generic, forge-agnostic representation of an open source project. +type Project struct { + Name string `json:"name"` + Url string `json:"url"` + Description string `json:"description"` + Topics []string `json:"topics"` + OpenIssues int `json:"open_issues"` + OpenPRs int `json:"open_prs"` + LatestCommit time.Time `json:"latest_commit"` + LatestRelease Release `json:"latest_release"` + Languages []string `json:"languages"` + Readme string `json:"readme"` + Licenses []string `json:"license"` + UsesAI bool `json:"uses_ai"` +} + +// Release is a named release with the date it was published. +type Release struct { + Name string `json:"name"` + Date time.Time `json:"date"` +} diff --git a/store/store.go b/store/store.go new file mode 100644 index 0000000..c892be8 --- /dev/null +++ b/store/store.go @@ -0,0 +1,190 @@ +// Package store provides durable persistence of scraped projects in SQLite. +// +// The database is the system of record: scrapers write discovered projects +// here, and a search index (e.g. Typesense) can be built or rebuilt from it +// without re-scraping the forges. Projects are stored as one row each, with +// the URL as the natural key so re-running the scrape upserts in place. +package store + +import ( + "database/sql" + "encoding/json" + "fmt" + "time" + + _ "modernc.org/sqlite" + + "git.xengi.de/ffd/indexer/scraper" +) + +const schema = ` +CREATE TABLE IF NOT EXISTS projects ( + url TEXT PRIMARY KEY, + name TEXT NOT NULL, + description TEXT NOT NULL DEFAULT '', + topics TEXT NOT NULL DEFAULT '[]', -- JSON array + languages TEXT NOT NULL DEFAULT '[]', -- JSON array + licenses TEXT NOT NULL DEFAULT '[]', -- JSON array + open_issues INTEGER NOT NULL DEFAULT 0, + open_prs INTEGER NOT NULL DEFAULT 0, + latest_commit INTEGER NOT NULL DEFAULT 0, -- unix epoch seconds (UTC) + latest_release_name TEXT NOT NULL DEFAULT '', + latest_release_date INTEGER NOT NULL DEFAULT 0, -- unix epoch seconds (UTC) + readme TEXT NOT NULL DEFAULT '', + uses_ai INTEGER NOT NULL DEFAULT 0, + scraped_at INTEGER NOT NULL DEFAULT 0 -- unix epoch seconds (UTC) +); +` + +// Store wraps a SQLite database holding scraped projects. +type Store struct { + db *sql.DB +} + +// Open opens (creating if needed) the SQLite database at path. +func Open(path string) (*Store, error) { + db, err := sql.Open("sqlite", path) + if err != nil { + return nil, fmt.Errorf("open sqlite db %q: %w", path, err) + } + if _, err := db.Exec(schema); err != nil { + _ = db.Close() + return nil, fmt.Errorf("create schema: %w", err) + } + return &Store{db: db}, nil +} + +// Close closes the underlying database. +func (s *Store) Close() error { + return s.db.Close() +} + +const upsertSQL = ` +INSERT INTO projects ( + url, name, description, topics, languages, licenses, + open_issues, open_prs, latest_commit, + latest_release_name, latest_release_date, + readme, uses_ai, scraped_at +) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +ON CONFLICT(url) DO UPDATE SET + name = excluded.name, + description = excluded.description, + topics = excluded.topics, + languages = excluded.languages, + licenses = excluded.licenses, + open_issues = excluded.open_issues, + open_prs = excluded.open_prs, + latest_commit = excluded.latest_commit, + latest_release_name = excluded.latest_release_name, + latest_release_date = excluded.latest_release_date, + readme = excluded.readme, + uses_ai = excluded.uses_ai, + scraped_at = excluded.scraped_at; +` + +// Upsert inserts or updates a project, keyed on its URL. +func (s *Store) Upsert(p scraper.Project) error { + topics, err := json.Marshal(p.Topics) + if err != nil { + return fmt.Errorf("marshal topics: %w", err) + } + langs, err := json.Marshal(p.Languages) + if err != nil { + return fmt.Errorf("marshal languages: %w", err) + } + licenses, err := json.Marshal(p.Licenses) + if err != nil { + return fmt.Errorf("marshal licenses: %w", err) + } + + releaseDate := timeToUnix(p.LatestRelease.Date) + latestCommit := timeToUnix(p.LatestCommit) + + _, err = s.db.Exec(upsertSQL, + p.Url, + p.Name, + p.Description, + string(topics), + string(langs), + string(licenses), + p.OpenIssues, + p.OpenPRs, + latestCommit, + p.LatestRelease.Name, + releaseDate, + p.Readme, + boolToInt(p.UsesAI), + time.Now().UTC().Unix(), + ) + return err +} + +// All returns every stored project. +func (s *Store) All() ([]scraper.Project, error) { + rows, err := s.db.Query(`SELECT + url, name, description, topics, languages, licenses, + open_issues, open_prs, latest_commit, + latest_release_name, latest_release_date, + readme, uses_ai + FROM projects`) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + + var projects []scraper.Project + for rows.Next() { + var ( + p scraper.Project + topicsJSON string + langsJSON string + licensesJSON string + latestCommit int64 + relName string + relDate int64 + usesAI int + ) + if err := rows.Scan( + &p.Url, &p.Name, &p.Description, + &topicsJSON, &langsJSON, &licensesJSON, + &p.OpenIssues, &p.OpenPRs, &latestCommit, + &relName, &relDate, + &p.Readme, &usesAI, + ); err != nil { + return nil, err + } + + p.UsesAI = usesAI != 0 + if err := json.Unmarshal([]byte(topicsJSON), &p.Topics); err != nil { + return nil, err + } + if err := json.Unmarshal([]byte(langsJSON), &p.Languages); err != nil { + return nil, err + } + if err := json.Unmarshal([]byte(licensesJSON), &p.Licenses); err != nil { + return nil, err + } + p.LatestCommit = time.Unix(latestCommit, 0).UTC() + p.LatestRelease.Name = relName + p.LatestRelease.Date = time.Unix(relDate, 0).UTC() + + projects = append(projects, p) + } + return projects, rows.Err() +} + +// timeToUnix returns the Unix epoch seconds for a time, or 0 for the zero +// time (which is stored as a 0/absent value). +func timeToUnix(t time.Time) int64 { + if t.IsZero() { + return 0 + } + return t.Unix() +} + +func boolToInt(b bool) int { + if b { + return 1 + } + return 0 +}