diff --git a/.editorconfig b/.editorconfig deleted file mode 100644 index d5da92e..0000000 --- a/.editorconfig +++ /dev/null @@ -1,11 +0,0 @@ -# EditorConfig is awesome: https://editorconfig.org -root = true - -[*] -end_of_line = lf -insert_final_newline = true - -[*.{go,nix,yaml}] -charset = utf-8 -indent_style = space -indent_size = 2 diff --git a/.gitignore b/.gitignore index c935c59..921d1fe 100644 --- a/.gitignore +++ b/.gitignore @@ -1,17 +1,4 @@ -config.yaml -models.json -Containerfile -result -index.db - -# Created by https://www.toptal.com/developers/gitignore/api/windows,linux,macos,direnv,vim,go,goland+all -# Edit at https://www.toptal.com/developers/gitignore?templates=windows,linux,macos,direnv,vim,go,goland+all - -### direnv ### -.direnv -.envrc - -### Go ### +# ---> Go # If you prefer the allow list template instead of the deny list, see community template: # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore # @@ -25,195 +12,23 @@ index.db # Test binary, built with `go test -c` *.test -# Output of the go coverage tool, specifically when used with LiteIDE +# Code coverage profiles and other test artifacts *.out +coverage.* +*.coverprofile +profile.cov # Dependency directories (remove the comment below to include it) # vendor/ # Go workspace file go.work +go.work.sum -### GoLand+all ### -# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider -# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 +# env file +.env -# User-specific stuff -.idea/**/workspace.xml -.idea/**/tasks.xml -.idea/**/usage.statistics.xml -.idea/**/dictionaries -.idea/**/shelf +# Editor/IDE +# .idea/ +# .vscode/ -# AWS User-specific -.idea/**/aws.xml - -# Generated files -.idea/**/contentModel.xml - -# Sensitive or high-churn files -.idea/**/dataSources/ -.idea/**/dataSources.ids -.idea/**/dataSources.local.xml -.idea/**/sqlDataSources.xml -.idea/**/dynamic.xml -.idea/**/uiDesigner.xml -.idea/**/dbnavigator.xml - -# Gradle -.idea/**/gradle.xml -.idea/**/libraries - -# Gradle and Maven with auto-import -# When using Gradle or Maven with auto-import, you should exclude module files, -# since they will be recreated, and may cause churn. Uncomment if using -# auto-import. -# .idea/artifacts -# .idea/compiler.xml -# .idea/jarRepositories.xml -# .idea/modules.xml -# .idea/*.iml -# .idea/modules -# *.iml -# *.ipr - -# CMake -cmake-build-*/ - -# Mongo Explorer plugin -.idea/**/mongoSettings.xml - -# File-based project format -*.iws - -# IntelliJ -out/ - -# mpeltonen/sbt-idea plugin -.idea_modules/ - -# JIRA plugin -atlassian-ide-plugin.xml - -# Cursive Clojure plugin -.idea/replstate.xml - -# SonarLint plugin -.idea/sonarlint/ - -# Crashlytics plugin (for Android Studio and IntelliJ) -com_crashlytics_export_strings.xml -crashlytics.properties -crashlytics-build.properties -fabric.properties - -# Editor-based Rest Client -.idea/httpRequests - -# Android studio 3.1+ serialized cache file -.idea/caches/build_file_checksums.ser - -### GoLand+all Patch ### -# Ignore everything but code style settings and run configurations -# that are supposed to be shared within teams. - -.idea/* - -!.idea/codeStyles -!.idea/runConfigurations - -### Linux ### -*~ - -# temporary files which can be created if a process still has a handle open of a deleted file -.fuse_hidden* - -# KDE directory preferences -.directory - -# Linux trash folder which might appear on any partition or disk -.Trash-* - -# .nfs files are created when an open file is removed but is still being accessed -.nfs* - -### macOS ### -# General -.DS_Store -.AppleDouble -.LSOverride - -# Icon must end with two \r -Icon - - -# Thumbnails -._* - -# Files that might appear in the root of a volume -.DocumentRevisions-V100 -.fseventsd -.Spotlight-V100 -.TemporaryItems -.Trashes -.VolumeIcon.icns -.com.apple.timemachine.donotpresent - -# Directories potentially created on remote AFP share -.AppleDB -.AppleDesktop -Network Trash Folder -Temporary Items -.apdisk - -### macOS Patch ### -# iCloud generated files -*.icloud - -### Vim ### -# Swap -[._]*.s[a-v][a-z] -!*.svg # comment out if you don't need vector files -[._]*.sw[a-p] -[._]s[a-rt-v][a-z] -[._]ss[a-gi-z] -[._]sw[a-p] - -# Session -Session.vim -Sessionx.vim - -# Temporary -.netrwhist -# Auto-generated tag files -tags -# Persistent undo -[._]*.un~ - -### Windows ### -# Windows thumbnail cache files -Thumbs.db -Thumbs.db:encryptable -ehthumbs.db -ehthumbs_vista.db - -# Dump file -*.stackdump - -# Folder config file -[Dd]esktop.ini - -# Recycle Bin used on file shares -$RECYCLE.BIN/ - -# Windows Installer files -*.cab -*.msi -*.msix -*.msm -*.msp - -# Windows shortcuts -*.lnk - -# End of https://www.toptal.com/developers/gitignore/api/windows,linux,macos,direnv,vim,go,goland+all diff --git a/README.md b/README.md index 13156ae..070201e 100644 --- a/README.md +++ b/README.md @@ -1,44 +1,3 @@ # indexer -Federated Forge Discovery Indexer - -Builds an index of public open source software projects from configured forges. - -## Scrape behavior - -For each configured forge, the indexer: - -1. **Checks robots.txt** (`GET /robots.txt`) before any API call. If the - instance disallows `/api/`, the instance is skipped. -2. **Lists public source repositories** via the repository search API - (`GET /api/v1/repos/search`), paginated. Forks and mirrors are excluded. -3. For each repository, enriches it via the Forgejo API: reads the root - directory once, then fetches topics, languages, latest commit, latest - release, the README, and detects licenses from the repo's files. - -Errors are non-fatal: an instance or a single repo that fails is skipped and -the run continues. - -### Request volume - -**Per instance:** - -| Request | Count | -|---|---| -| `GET /robots.txt` | 1 | -| `GET /api/v1/repos/search` (page) | `ceil(N/50)` where N = total repos | - -**Per repository:** - -| Request | Endpoint | Purpose | -|---|---|---| -| list root contents | `GET /api/v1/repos/{o}/{r}/contents/` | shared: README selection + license scan (1 call) | -| languages | `GET /api/v1/repos/{o}/{r}/languages` | `Project.Languages` | -| topics | `GET /api/v1/repos/{o}/{r}/topics` | `Project.Topics` | -| latest commit | `GET /api/v1/repos/{o}/{r}/commits?limit=1` | `Project.LatestCommit` | -| latest release | `GET /api/v1/repos/{o}/{r}/releases/latest` | `Project.LatestRelease` | -| README | `GET /api/v1/repos/{o}/{r}/raw/{branch}/{file}` | `Project.Readme` | -| license files | 1+ × `GET /api/v1/repos/{o}/{r}/raw/{branch}/{file}` | SPDX detection | - -So roughly **6–8 API requests per repository** plus the paginated search -requests and a single robots.txt check per instance. +Federated Forge Discovery Indexer \ No newline at end of file diff --git a/config.go b/config.go deleted file mode 100644 index 87b50b1..0000000 --- a/config.go +++ /dev/null @@ -1,14 +0,0 @@ -package main - -// Forge is a single forge instance to scrape. -type Forge struct { - Name string `yaml:"name"` - Url string `yaml:"url"` - Type string `yaml:"type"` -} - -// Config is the top-level configuration file. -type Config struct { - DbPath string `yaml:"db_path"` - Forges []Forge `yaml:"forges"` -} diff --git a/flake.lock b/flake.lock deleted file mode 100644 index bf253d3..0000000 --- a/flake.lock +++ /dev/null @@ -1,85 +0,0 @@ -{ - "nodes": { - "flake-utils": { - "inputs": { - "systems": "systems" - }, - "locked": { - "lastModified": 1731533236, - "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, - "gomod2nix": { - "inputs": { - "flake-utils": [ - "flake-utils" - ], - "nixpkgs": [ - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1770585520, - "narHash": "sha256-yBz9Ozd5Wb56i3e3cHZ8WcbzCQ9RlVaiW18qDYA/AzA=", - "owner": "tweag", - "repo": "gomod2nix", - "rev": "1201ddd1279c35497754f016ef33d5e060f3da8d", - "type": "github" - }, - "original": { - "owner": "tweag", - "repo": "gomod2nix", - "type": "github" - } - }, - "nixpkgs": { - "locked": { - "lastModified": 1790573947, - "narHash": "sha256-zfer1SLJvACKM0J7c9AusGvIp4q4B6l1Z1KgiVInG8Y=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "e6025a0259c7e80f81b11effc15f65d95b708cba", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixos-unstable-small", - "repo": "nixpkgs", - "type": "github" - } - }, - "root": { - "inputs": { - "flake-utils": "flake-utils", - "gomod2nix": "gomod2nix", - "nixpkgs": "nixpkgs" - } - }, - "systems": { - "locked": { - "lastModified": 1681028828, - "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", - "owner": "nix-systems", - "repo": "default", - "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", - "type": "github" - }, - "original": { - "owner": "nix-systems", - "repo": "default", - "type": "github" - } - } - }, - "root": "root", - "version": 7 -} diff --git a/flake.nix b/flake.nix deleted file mode 100644 index 7e77d0e..0000000 --- a/flake.nix +++ /dev/null @@ -1,86 +0,0 @@ -{ - description = "Federated Forge Discovery Indexer"; - inputs = { - nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable-small"; - flake-utils.url = "github:numtide/flake-utils"; - gomod2nix = { - url = "github:tweag/gomod2nix"; - inputs.nixpkgs.follows = "nixpkgs"; - inputs.flake-utils.follows = "flake-utils"; - }; - }; - outputs = { self, nixpkgs, flake-utils, gomod2nix }: flake-utils.lib.eachDefaultSystem (system: - let - pkgs = import nixpkgs { - inherit system; - overlays = [ gomod2nix.overlays.default ]; - }; - go = pkgs.go_1_27; - ffd-indexer = pkgs.buildGoApplication { - inherit go; - pname = "ffd-indexer"; - version = "0.0.1"; - src = ./.; - modules = ./gomod2nix.toml; - }; - go-test = pkgs.stdenv.mkDerivation { - name = "go-test"; - dontBuild = true; - src = ./.; - doCheck = true; - nativeBuildInputs = [ - go - pkgs.writableTmpDirAsHomeHook - ]; - checkPhase = '' - go test -v ./... - ''; - installPhase = '' - mkdir "$out" - ''; - }; - go-lint = pkgs.stdenv.mkDerivation { - name = "go-lint"; - dontBuild = true; - src = ./.; - doCheck = true; - nativeBuildInputs = [ - pkgs.golangci-lint - go - pkgs.writableTmpDirAsHomeHook - ]; - checkPhase = '' - golangci-lint run - ''; - installPhase = '' - mkdir "$out" - ''; - }; - in - { - checks = { inherit go-test go-lint; }; - formatter = pkgs.nixfmt-tree; - devShells.default = pkgs.mkShell { - buildInputs = [ - go - pkgs.go-tools # staticcheck - gomod2nix.packages.${system}.default - ]; - packages = with pkgs; [ - mkcert - sqlite # debugging the index db - ]; - }; - packages = { - default = ffd-indexer; - container = pkgs.dockerTools.buildImage { - name = "ffd-indexer"; - config = { - Cmd = [ "${ffd-indexer}/bin/ffd-indexer" ]; - }; - }; - }; - nixosModules.default = import ./option.nix; - } - ); -} diff --git a/go.mod b/go.mod deleted file mode 100644 index 774a74c..0000000 --- a/go.mod +++ /dev/null @@ -1,76 +0,0 @@ -module git.xengi.de/ffd/indexer - -go 1.27.1 - -require ( - codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3 v3.0.0 - github.com/go-enry/go-license-detector/v4 v4.3.1 - go.yaml.in/yaml/v4 v4.0.0-rc.6 - modernc.org/sqlite v1.59.0 -) - -require ( - github.com/42wim/httpsig v1.2.3 // indirect - github.com/Microsoft/go-winio v0.6.0 // indirect - github.com/ProtonMail/go-crypto v0.0.0-20220930113650-c6815a8c17ad // indirect - github.com/acomagu/bufpipe v1.0.3 // indirect - github.com/cloudflare/circl v1.2.0 // indirect - github.com/davidmz/go-pageant v1.0.2 // indirect - github.com/dgryski/go-minhash v0.0.0-20190315135803-ad340ca03076 // indirect - github.com/dustin/go-humanize v1.0.1 // indirect - github.com/ekzhu/minhash-lsh v0.0.0-20190924033628-faac2c6342f8 // indirect - github.com/emirpasic/gods v1.18.1 // indirect - github.com/go-fed/httpsig v1.1.0 // indirect - github.com/go-git/gcfg v1.5.0 // indirect - github.com/go-git/go-billy/v5 v5.3.1 // indirect - github.com/go-git/go-git/v5 v5.4.2 // indirect - github.com/go-openapi/errors v0.22.6 // indirect - github.com/go-openapi/strfmt v0.25.0 // indirect - github.com/go-openapi/swag v0.25.4 // indirect - github.com/go-openapi/swag/cmdutils v0.25.4 // indirect - github.com/go-openapi/swag/conv v0.25.4 // indirect - github.com/go-openapi/swag/fileutils v0.25.4 // indirect - github.com/go-openapi/swag/jsonname v0.25.4 // indirect - github.com/go-openapi/swag/jsonutils v0.25.4 // indirect - github.com/go-openapi/swag/loading v0.25.4 // indirect - github.com/go-openapi/swag/mangling v0.25.4 // indirect - github.com/go-openapi/swag/netutils v0.25.4 // indirect - github.com/go-openapi/swag/stringutils v0.25.4 // indirect - github.com/go-openapi/swag/typeutils v0.25.4 // indirect - github.com/go-openapi/swag/yamlutils v0.25.4 // indirect - github.com/go-viper/mapstructure/v2 v2.4.0 // indirect - github.com/google/uuid v1.6.0 // indirect - github.com/hashicorp/go-version v1.7.0 // indirect - github.com/hhatto/gorst v0.0.0-20181029133204-ca9f730cac5b // indirect - github.com/imdario/mergo v0.3.13 // indirect - github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect - github.com/jdkato/prose v1.2.1 // indirect - github.com/kevinburke/ssh_config v1.2.0 // indirect - github.com/mattn/go-isatty v0.0.24 // indirect - github.com/mitchellh/go-homedir v1.1.0 // indirect - github.com/montanaflynn/stats v0.7.1 // indirect - github.com/ncruces/go-strftime v1.0.0 // indirect - github.com/oklog/ulid v1.3.1 // indirect - github.com/pkg/errors v0.9.1 // indirect - github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/sergi/go-diff v1.2.0 // indirect - github.com/shogo82148/go-shuffle v1.0.1 // indirect - github.com/xanzy/ssh-agent v0.3.2 // indirect - go.mongodb.org/mongo-driver v1.17.6 // indirect - go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/crypto v0.54.0 // indirect - golang.org/x/exp v0.0.0-20221006183845-316c7553db56 // indirect - golang.org/x/mod v0.38.0 // indirect - golang.org/x/net v0.57.0 // indirect - golang.org/x/sync v0.22.0 // indirect - golang.org/x/sys v0.47.0 // indirect - golang.org/x/text v0.40.0 // indirect - golang.org/x/tools v0.48.0 // indirect - gonum.org/v1/gonum v0.8.2 // indirect - gopkg.in/neurosnap/sentences.v1 v1.0.7 // indirect - gopkg.in/warnings.v0 v0.1.2 // indirect - modernc.org/libc v1.75.7 // indirect - modernc.org/mathutil v1.7.1 // indirect - modernc.org/memory v1.12.1 // indirect -) diff --git a/go.sum b/go.sum deleted file mode 100644 index 1f92ed9..0000000 --- a/go.sum +++ /dev/null @@ -1,291 +0,0 @@ -codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3 v3.0.0 h1:s2fK+FBwvcYsmKDjNhmoe7B8q9zsgs0UrSlYe9r4XjM= -codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3 v3.0.0/go.mod h1:Is2jTpS1dizeXm4skQv/ES3QVqnzcNhn2GzZXpiw9f8= -github.com/42wim/httpsig v1.2.3 h1:xb0YyWhkYj57SPtfSttIobJUPJZB9as1nsfo7KWVcEs= -github.com/42wim/httpsig v1.2.3/go.mod h1:nZq9OlYKDrUBhptd77IHx4/sZZD+IxTBADvAPI9G/EM= -github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= -github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= -github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= -github.com/Microsoft/go-winio v0.6.0 h1:slsWYD/zyx7lCXoZVlvQrj0hPTM1HI4+v1sIda2yDvg= -github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE= -github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo= -github.com/ProtonMail/go-crypto v0.0.0-20220930113650-c6815a8c17ad h1:QeeqI2zxxgZVe11UrYFXXx6gVxPVF40ygekjBzEg4XY= -github.com/ProtonMail/go-crypto v0.0.0-20220930113650-c6815a8c17ad/go.mod h1:UBYPn8k0D56RtnR8RFQMjmh4KrZzWJ5o7Z9SYjossQ8= -github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk= -github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= -github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= -github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= -github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= -github.com/bwesterb/go-ristretto v1.2.1/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= -github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= -github.com/cloudflare/circl v1.2.0 h1:NheeISPSUcYftKlfrLuOo4T62FkmD4t4jviLfFFYaec= -github.com/cloudflare/circl v1.2.0/go.mod h1:Ch2UgYr6ti2KTtlejELlROl0YIYj7SLjAC8M+INXlMk= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davidmz/go-pageant v1.0.2 h1:bPblRCh5jGU+Uptpz6LgMZGD5hJoOt7otgT454WvHn0= -github.com/davidmz/go-pageant v1.0.2/go.mod h1:P2EDDnMqIwG5Rrp05dTRITj9z2zpGcD9efWSkTNKLIE= -github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc h1:8WFBn63wegobsYAX0YjD+8suexZDga5CctH4CCTx2+8= -github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc/go.mod h1:c9O8+fpSOX1DM8cPNSkX/qsBWdkD4yd2dpciOWQjpBw= -github.com/dgryski/go-minhash v0.0.0-20190315135803-ad340ca03076 h1:EB7M2v8Svo3kvIDy+P1YDE22XskDQP+TEYGzeDwPAN4= -github.com/dgryski/go-minhash v0.0.0-20190315135803-ad340ca03076/go.mod h1:VBi0XHpFy0xiMySf6YpVbRqrupW4RprJ5QTyN+XvGSM= -github.com/dgryski/go-spooky v0.0.0-20170606183049-ed3d087f40e2 h1:lx1ZQgST/imDhmLpYDma1O3Cx9L+4Ie4E8S2RjFPQ30= -github.com/dgryski/go-spooky v0.0.0-20170606183049-ed3d087f40e2/go.mod h1:hgHYKsoIw7S/hlWtP7wD1wZ7SX1jPTtKko5X9jrOgPQ= -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/ekzhu/minhash-lsh v0.0.0-20190924033628-faac2c6342f8 h1:+Tje+xk1lmGKSJjYNtgCFsU1HtQzz0kCm1DFbKlvFBo= -github.com/ekzhu/minhash-lsh v0.0.0-20190924033628-faac2c6342f8/go.mod h1:yEtCVi+QamvzjEH4U/m6ZGkALIkF2xfQnFp0BcKmIOk= -github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= -github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= -github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= -github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= -github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= -github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0= -github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= -github.com/go-enry/go-license-detector/v4 v4.3.1 h1:BajEVdTffFcs8RACmblySVhfEIuT58TmXx27RgVfUdc= -github.com/go-enry/go-license-detector/v4 v4.3.1/go.mod h1:YVJKPE01WQNjN/bdM6V0I/9KxvwEAAv0Ef9pi92K6w0= -github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI= -github.com/go-fed/httpsig v1.1.0/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM= -github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4= -github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E= -github.com/go-git/go-billy/v5 v5.2.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= -github.com/go-git/go-billy/v5 v5.3.1 h1:CPiOUAzKtMRvolEKw+bG1PLRpT7D3LIs3/3ey4Aiu34= -github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= -github.com/go-git/go-git-fixtures/v4 v4.2.1 h1:n9gGL1Ct/yIw+nfsfr8s4+sbhT+Ncu2SubfXjIWgci8= -github.com/go-git/go-git-fixtures/v4 v4.2.1/go.mod h1:K8zd3kDUAykwTdDCr+I0per6Y6vMiRR/nnVTBtavnB0= -github.com/go-git/go-git/v5 v5.4.2 h1:BXyZu9t0VkbiHtqrsvdq39UDhGJTl1h55VW6CSC4aY4= -github.com/go-git/go-git/v5 v5.4.2/go.mod h1:gQ1kArt6d+n+BGd+/B/I74HwRTLhth2+zti4ihgckDc= -github.com/go-openapi/errors v0.22.6 h1:eDxcf89O8odEnohIXwEjY1IB4ph5vmbUsBMsFNwXWPo= -github.com/go-openapi/errors v0.22.6/go.mod h1:z9S8ASTUqx7+CP1Q8dD8ewGH/1JWFFLX/2PmAYNQLgk= -github.com/go-openapi/strfmt v0.25.0 h1:7R0RX7mbKLa9EYCTHRcCuIPcaqlyQiWNPTXwClK0saQ= -github.com/go-openapi/strfmt v0.25.0/go.mod h1:nNXct7OzbwrMY9+5tLX4I21pzcmE6ccMGXl3jFdPfn8= -github.com/go-openapi/swag v0.25.4 h1:OyUPUFYDPDBMkqyxOTkqDYFnrhuhi9NR6QVUvIochMU= -github.com/go-openapi/swag v0.25.4/go.mod h1:zNfJ9WZABGHCFg2RnY0S4IOkAcVTzJ6z2Bi+Q4i6qFQ= -github.com/go-openapi/swag/cmdutils v0.25.4 h1:8rYhB5n6WawR192/BfUu2iVlxqVR9aRgGJP6WaBoW+4= -github.com/go-openapi/swag/cmdutils v0.25.4/go.mod h1:pdae/AFo6WxLl5L0rq87eRzVPm/XRHM3MoYgRMvG4A0= -github.com/go-openapi/swag/conv v0.25.4 h1:/Dd7p0LZXczgUcC/Ikm1+YqVzkEeCc9LnOWjfkpkfe4= -github.com/go-openapi/swag/conv v0.25.4/go.mod h1:3LXfie/lwoAv0NHoEuY1hjoFAYkvlqI/Bn5EQDD3PPU= -github.com/go-openapi/swag/fileutils v0.25.4 h1:2oI0XNW5y6UWZTC7vAxC8hmsK/tOkWXHJQH4lKjqw+Y= -github.com/go-openapi/swag/fileutils v0.25.4/go.mod h1:cdOT/PKbwcysVQ9Tpr0q20lQKH7MGhOEb6EwmHOirUk= -github.com/go-openapi/swag/jsonname v0.25.4 h1:bZH0+MsS03MbnwBXYhuTttMOqk+5KcQ9869Vye1bNHI= -github.com/go-openapi/swag/jsonname v0.25.4/go.mod h1:GPVEk9CWVhNvWhZgrnvRA6utbAltopbKwDu8mXNUMag= -github.com/go-openapi/swag/jsonutils v0.25.4 h1:VSchfbGhD4UTf4vCdR2F4TLBdLwHyUDTd1/q4i+jGZA= -github.com/go-openapi/swag/jsonutils v0.25.4/go.mod h1:7OYGXpvVFPn4PpaSdPHJBtF0iGnbEaTk8AvBkoWnaAY= -github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.4 h1:IACsSvBhiNJwlDix7wq39SS2Fh7lUOCJRmx/4SN4sVo= -github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.4/go.mod h1:Mt0Ost9l3cUzVv4OEZG+WSeoHwjWLnarzMePNDAOBiM= -github.com/go-openapi/swag/loading v0.25.4 h1:jN4MvLj0X6yhCDduRsxDDw1aHe+ZWoLjW+9ZQWIKn2s= -github.com/go-openapi/swag/loading v0.25.4/go.mod h1:rpUM1ZiyEP9+mNLIQUdMiD7dCETXvkkC30z53i+ftTE= -github.com/go-openapi/swag/mangling v0.25.4 h1:2b9kBJk9JvPgxr36V23FxJLdwBrpijI26Bx5JH4Hp48= -github.com/go-openapi/swag/mangling v0.25.4/go.mod h1:6dxwu6QyORHpIIApsdZgb6wBk/DPU15MdyYj/ikn0Hg= -github.com/go-openapi/swag/netutils v0.25.4 h1:Gqe6K71bGRb3ZQLusdI8p/y1KLgV4M/k+/HzVSqT8H0= -github.com/go-openapi/swag/netutils v0.25.4/go.mod h1:m2W8dtdaoX7oj9rEttLyTeEFFEBvnAx9qHd5nJEBzYg= -github.com/go-openapi/swag/stringutils v0.25.4 h1:O6dU1Rd8bej4HPA3/CLPciNBBDwZj9HiEpdVsb8B5A8= -github.com/go-openapi/swag/stringutils v0.25.4/go.mod h1:GTsRvhJW5xM5gkgiFe0fV3PUlFm0dr8vki6/VSRaZK0= -github.com/go-openapi/swag/typeutils v0.25.4 h1:1/fbZOUN472NTc39zpa+YGHn3jzHWhv42wAJSN91wRw= -github.com/go-openapi/swag/typeutils v0.25.4/go.mod h1:Ou7g//Wx8tTLS9vG0UmzfCsjZjKhpjxayRKTHXf2pTE= -github.com/go-openapi/swag/yamlutils v0.25.4 h1:6jdaeSItEUb7ioS9lFoCZ65Cne1/RZtPBZ9A56h92Sw= -github.com/go-openapi/swag/yamlutils v0.25.4/go.mod h1:MNzq1ulQu+yd8Kl7wPOut/YHAAU/H6hL91fF+E2RFwc= -github.com/go-openapi/testify/enable/yaml/v2 v2.0.2 h1:0+Y41Pz1NkbTHz8NngxTuAXxEodtNSI1WG1c/m5Akw4= -github.com/go-openapi/testify/enable/yaml/v2 v2.0.2/go.mod h1:kme83333GCtJQHXQ8UKX3IBZu6z8T5Dvy5+CW3NLUUg= -github.com/go-openapi/testify/v2 v2.0.2 h1:X999g3jeLcoY8qctY/c/Z8iBHTbwLz7R2WXd6Ub6wls= -github.com/go-openapi/testify/v2 v2.0.2/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54= -github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= -github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= -github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo= -github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= -github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= -github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= -github.com/hhatto/gorst v0.0.0-20181029133204-ca9f730cac5b h1:Jdu2tbAxkRouSILp2EbposIb8h4gO+2QuZEn3d9sKAc= -github.com/hhatto/gorst v0.0.0-20181029133204-ca9f730cac5b/go.mod h1:HmaZGXHdSwQh1jnUlBGN2BeEYOHACLVGzYOXCbsLvxY= -github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= -github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= -github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= -github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= -github.com/jdkato/prose v1.2.1 h1:Fp3UnJmLVISmlc57BgKUzdjr0lOtjqTZicL3PaYy6cU= -github.com/jdkato/prose v1.2.1/go.mod h1:AiRHgVagnEx2JbQRQowVBKjG0bcs/vtkGCH1dYAL1rA= -github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= -github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= -github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= -github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= -github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= -github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= -github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= -github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/montanaflynn/stats v0.6.3/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= -github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= -github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= -github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= -github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= -github.com/neurosnap/sentences v1.0.6 h1:iBVUivNtlwGkYsJblWV8GGVFmXzZzak907Ci8aA0VTE= -github.com/neurosnap/sentences v1.0.6/go.mod h1:pg1IapvYpWCJJm/Etxeh0+gtMf1rI1STY9S7eUCPbDc= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= -github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shogo82148/go-shuffle v0.0.0-20180218125048-27e6095f230d/go.mod h1:2htx6lmL0NGLHlO8ZCf+lQBGBHIbEujyywxJArf+2Yc= -github.com/shogo82148/go-shuffle v1.0.1 h1:4swIpHXLMAz14DE4YTgakgadpRN0n1wE1dieGnOTVFU= -github.com/shogo82148/go-shuffle v1.0.1/go.mod h1:HQPjVgUUZ9TNgm4/K/iXRuAdhPsQrXnAGgtk/9kqbBY= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= -github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0= -github.com/xanzy/ssh-agent v0.3.2 h1:eKj4SX2Fe7mui28ZgnFW5fmTz1EIr7ugo5s6wDxdHBM= -github.com/xanzy/ssh-agent v0.3.2/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= -go.mongodb.org/mongo-driver v1.17.6 h1:87JUG1wZfWsr6rIz3ZmpH90rL5tea7O3IHuSwHUpsss= -go.mongodb.org/mongo-driver v1.17.6/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= -go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= -go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4= -go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= -golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= -golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= -golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20221006183845-316c7553db56 h1:BrYbdKcCNjLyrN6aKqXy4hPw9qGI8IATkj4EWv9Q+kQ= -golang.org/x/exp v0.0.0-20221006183845-316c7553db56/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= -golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= -golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= -golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= -golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= -golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= -golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220315194320-039c03cc5b86/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= -golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= -golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= -golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= -golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= -golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= -gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= -gonum.org/v1/gonum v0.8.2 h1:CCXrcPKiGGotvnN6jfUsKk4rRqm7q09/YbKb5xCEvtM= -gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= -gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0 h1:OE9mWmgKkjJyEmDAAtGMPjXu+YNeGvK9VTSHY6+Qihc= -gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= -gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/neurosnap/sentences.v1 v1.0.6/go.mod h1:YlK+SN+fLQZj+kY3r8DkGDhDr91+S3JmTb5LSxFRQo0= -gopkg.in/neurosnap/sentences.v1 v1.0.7 h1:gpTUYnqthem4+o8kyTLiYIB05W+IvdQFYR29erfe8uU= -gopkg.in/neurosnap/sentences.v1 v1.0.7/go.mod h1:YlK+SN+fLQZj+kY3r8DkGDhDr91+S3JmTb5LSxFRQo0= -gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= -gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -modernc.org/cc/v4 v4.29.2 h1:h6+9ciCnPKutf4I03CvheAvDLX7+IHlqR6Iy6J+cgd8= -modernc.org/cc/v4 v4.29.2/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= -modernc.org/ccgo/v4 v4.35.0 h1:F+TUsmw09QxLzmi3aeYYGxjAXarmZaKgj3mKQHNaA8w= -modernc.org/ccgo/v4 v4.35.0/go.mod h1:qrVGs9S3Sr2Ztcg9ve+kTAYMp5a3YvWjo+SoN06kJ5I= -modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= -modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= -modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= -modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= -modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc= -modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= -modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= -modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= -modernc.org/libc v1.75.7 h1:o3DTP9/0p9pKmY2WCKQaySW6wIiZhNM7wc2lUoyhfew= -modernc.org/libc v1.75.7/go.mod h1:bO5o2ztHxBb2rjz0PgdHN0sSMw57CgxGFLZ3Qd/QpVQ= -modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= -modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= -modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g= -modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= -modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= -modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= -modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= -modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= -modernc.org/sqlite v1.59.0 h1:X1es1GpqBlS/5T+vbM4HLUdaa8OtQx468DF2vrx+38A= -modernc.org/sqlite v1.59.0/go.mod h1:+paeT2A3iPRHkQDwG7oA6Tk0zQd5woMEI8q7orfry8k= -modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= -modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= -modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= -modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= -rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/gomod2nix.toml b/gomod2nix.toml deleted file mode 100644 index c152ad5..0000000 --- a/gomod2nix.toml +++ /dev/null @@ -1,270 +0,0 @@ -schema = 3 - -[mod] - [mod.'codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3'] - version = 'v3.0.0' - hash = 'sha256-/fb4qz0lZt7vuxOELzthAiUy1oDC5x/PYCch3FUNWig=' - - [mod.'github.com/42wim/httpsig'] - version = 'v1.2.3' - hash = 'sha256-Yrf21S5q6mupTtyk8izVKQciJC/aRDe7YRKL0cBkRpQ=' - - [mod.'github.com/Microsoft/go-winio'] - version = 'v0.6.0' - hash = 'sha256-TQ0AvXZfuT3PDnb4p/h5zmyT8m9VPcwxlXMJuVD6ltE=' - - [mod.'github.com/ProtonMail/go-crypto'] - version = 'v0.0.0-20220930113650-c6815a8c17ad' - hash = 'sha256-mem+jc60bsyLIJblnHbnG5e3+KFWf2TfZo9In9viYwU=' - - [mod.'github.com/acomagu/bufpipe'] - version = 'v1.0.3' - hash = 'sha256-hIQf2lqZZv4dN8JwRyhNoBjStL9Mgih6f7Hy8RDU5Kg=' - - [mod.'github.com/cloudflare/circl'] - version = 'v1.2.0' - hash = 'sha256-9L1/4M8Za4qcrtti/1r1E7TgpdfUKHM+9IyuX3fuJyQ=' - - [mod.'github.com/davidmz/go-pageant'] - version = 'v1.0.2' - hash = 'sha256-gfwm6i0ZGQhX2c0lDRe7qVXyHkhoHeemYpaj+O0aYyw=' - - [mod.'github.com/dgryski/go-minhash'] - version = 'v0.0.0-20190315135803-ad340ca03076' - hash = 'sha256-CYfWI07pOkDyBgoHRUSY9QzOR/cWNLIa08M+0khbKX4=' - - [mod.'github.com/dustin/go-humanize'] - version = 'v1.0.1' - hash = 'sha256-yuvxYYngpfVkUg9yAmG99IUVmADTQA0tMbBXe0Fq0Mc=' - - [mod.'github.com/ekzhu/minhash-lsh'] - version = 'v0.0.0-20190924033628-faac2c6342f8' - hash = 'sha256-vZH3kLGjMEuDNiWHpTVXv1z0ZMYi40zai8DlOk7ivZM=' - - [mod.'github.com/emirpasic/gods'] - version = 'v1.18.1' - hash = 'sha256-hGDKddjLj+5dn2woHtXKUdd49/3xdsqnhx7VEdCu1m4=' - - [mod.'github.com/go-enry/go-license-detector/v4'] - version = 'v4.3.1' - hash = 'sha256-S9LKXjn5dL5FETOOAk+bs7bIVdu2x7MIhfjpZuXzuLo=' - - [mod.'github.com/go-fed/httpsig'] - version = 'v1.1.0' - hash = 'sha256-ok0PEEvSR010nh6vB3nDCyKSSVvoB50/UjVvBKOYXsA=' - - [mod.'github.com/go-git/gcfg'] - version = 'v1.5.0' - hash = 'sha256-A62eSLI/0y4sfwCsZhe/uoSC9Z3TryyW+PyWIcknYdE=' - - [mod.'github.com/go-git/go-billy/v5'] - version = 'v5.3.1' - hash = 'sha256-UdShPsStI2OBJig8Zv4Htyv5NiwKnYI41pwe4hIxv3Q=' - - [mod.'github.com/go-git/go-git/v5'] - version = 'v5.4.2' - hash = 'sha256-+RvDli5tPRLEs1pA9W/ZqCCm896y97ObTqLp3hLydpo=' - - [mod.'github.com/go-openapi/errors'] - version = 'v0.22.6' - hash = 'sha256-JcWNO3VO/OxaMptpMuFs6pYD7xFDtC5H8vn1EiF9w20=' - - [mod.'github.com/go-openapi/strfmt'] - version = 'v0.25.0' - hash = 'sha256-xz4aMKsHsDhUPVc27hBa4FrywKQABhLhEBNg3qClL9Y=' - - [mod.'github.com/go-openapi/swag'] - version = 'v0.25.4' - hash = 'sha256-aFQeLf95JC5gWLwIJXxhZHK9RxiGhh9IYV0MEYldDr4=' - - [mod.'github.com/go-openapi/swag/cmdutils'] - version = 'v0.25.4' - hash = 'sha256-nhlq5w/ULjwy7nptKcRryjZPlv3QG0FmgcBSOclqRlA=' - - [mod.'github.com/go-openapi/swag/conv'] - version = 'v0.25.4' - hash = 'sha256-uHgTdZC76LMSsq+x+RyeclnOgBsS0ID2cLgfjzFk0KQ=' - - [mod.'github.com/go-openapi/swag/fileutils'] - version = 'v0.25.4' - hash = 'sha256-NWacswHRvNZ81WzazA8yUmeDymtne2l8zLdArBmeYAU=' - - [mod.'github.com/go-openapi/swag/jsonname'] - version = 'v0.25.4' - hash = 'sha256-Z6uETeudh8W+/SGxFBnOB/VlJeRPRkFgCyJmntpz7bc=' - - [mod.'github.com/go-openapi/swag/jsonutils'] - version = 'v0.25.4' - hash = 'sha256-TQGeMImUuL5BWDoBHKAjZ1BvvxPXkFrV3TMgb82IrLk=' - - [mod.'github.com/go-openapi/swag/loading'] - version = 'v0.25.4' - hash = 'sha256-xr8OnFqB/kwBj3yf9HiLJwDcpL7I3/qJYlKb6VWbVFA=' - - [mod.'github.com/go-openapi/swag/mangling'] - version = 'v0.25.4' - hash = 'sha256-NrmwNgRJIUlOaAGt+XutHWSHy9e1COMnVE8B0UY5+sQ=' - - [mod.'github.com/go-openapi/swag/netutils'] - version = 'v0.25.4' - hash = 'sha256-wRMCQX/wIOvWm1g2UURobhzI47OldVqzlYYzwotLNTU=' - - [mod.'github.com/go-openapi/swag/stringutils'] - version = 'v0.25.4' - hash = 'sha256-rVo5NBH+oLLX9kaemUYWILd+TKw/TDTB8UdSz+mn3m8=' - - [mod.'github.com/go-openapi/swag/typeutils'] - version = 'v0.25.4' - hash = 'sha256-59RHnK6ugsAUc+A8DZCj7gZ3bJnt01Al7T/1kzM9PpA=' - - [mod.'github.com/go-openapi/swag/yamlutils'] - version = 'v0.25.4' - hash = 'sha256-zba7QX7Ds05oZq1opP/vwRSBEDaJ1pKm+9DgrsOuR9w=' - - [mod.'github.com/go-viper/mapstructure/v2'] - version = 'v2.4.0' - hash = 'sha256-lLfcV9z4n94hDhgyXJlde4bFB0hfzlbh+polqcJCwGE=' - - [mod.'github.com/google/uuid'] - version = 'v1.6.0' - hash = 'sha256-VWl9sqUzdOuhW0KzQlv0gwwUQClYkmZwSydHG2sALYw=' - - [mod.'github.com/hashicorp/go-version'] - version = 'v1.7.0' - hash = 'sha256-Umc/Q2mxRrPF4aEcDuDJq4lFBT+PSsnyANfyFwUIaxI=' - - [mod.'github.com/hhatto/gorst'] - version = 'v0.0.0-20181029133204-ca9f730cac5b' - hash = 'sha256-u1rB/Z20x3UYMKaLjlDP3bHnD7nMrVs5AHR48km/H1k=' - - [mod.'github.com/imdario/mergo'] - version = 'v0.3.13' - hash = 'sha256-03LKAZXgR5YUMeK5IRh2ds2GyV6uSbdq5QCPbAzlTus=' - - [mod.'github.com/jbenet/go-context'] - version = 'v0.0.0-20150711004518-d14ea06fba99' - hash = 'sha256-VANNCWNNpARH/ILQV9sCQsBWgyL2iFT+4AHZREpxIWE=' - - [mod.'github.com/jdkato/prose'] - version = 'v1.2.1' - hash = 'sha256-xjGGiunKj2XUGuUMxdKrObOUAoV1QUmwkPVhyjLc3S8=' - - [mod.'github.com/kevinburke/ssh_config'] - version = 'v1.2.0' - hash = 'sha256-Ta7ZOmyX8gG5tzWbY2oES70EJPfI90U7CIJS9EAce0s=' - - [mod.'github.com/mattn/go-isatty'] - version = 'v0.0.24' - hash = 'sha256-pmq8KKIb2+qMF3E1cR2B7NAj5joF2tZduyXjIfIlkTk=' - - [mod.'github.com/mitchellh/go-homedir'] - version = 'v1.1.0' - hash = 'sha256-oduBKXHAQG8X6aqLEpqZHs5DOKe84u6WkBwi4W6cv3k=' - - [mod.'github.com/montanaflynn/stats'] - version = 'v0.7.1' - hash = 'sha256-0QXUA/syOtDSBooh3isAffrVAjpSoBmSMMuZfO5maHg=' - - [mod.'github.com/ncruces/go-strftime'] - version = 'v1.0.0' - hash = 'sha256-GYIwYDONuv/yTE0AEugCHQbtV3oiBaco93xUNYFcVBQ=' - - [mod.'github.com/oklog/ulid'] - version = 'v1.3.1' - hash = 'sha256-LNn883rYNiaoY9sGEPIzlMRx5UwGThdYTjXqfzeGc9k=' - - [mod.'github.com/pkg/errors'] - version = 'v0.9.1' - hash = 'sha256-mNfQtcrQmu3sNg/7IwiieKWOgFQOVVe2yXgKBpe/wZw=' - - [mod.'github.com/remyoudompheng/bigfft'] - version = 'v0.0.0-20230129092748-24d4a6f8daec' - hash = 'sha256-vYmpyCE37eBYP/navhaLV4oX4/nu0Z/StAocLIFqrmM=' - - [mod.'github.com/russross/blackfriday/v2'] - version = 'v2.1.0' - hash = 'sha256-R+84l1si8az5yDqd5CYcFrTyNZ1eSYlpXKq6nFt4OTQ=' - - [mod.'github.com/sergi/go-diff'] - version = 'v1.2.0' - hash = 'sha256-d2higuBRee4ylRuCpPQV5+g0XK5yNQgDpudNor0qD2o=' - - [mod.'github.com/shogo82148/go-shuffle'] - version = 'v1.0.1' - hash = 'sha256-E5Xt69A2vXlVPzyHPyWK0FyQ+IaMZzqQCnUltedeZhg=' - - [mod.'github.com/xanzy/ssh-agent'] - version = 'v0.3.2' - hash = 'sha256-QavRCX75546hnWCOtBwiUdmyk6AoLjv3POoJXng5kMI=' - - [mod.'go.mongodb.org/mongo-driver'] - version = 'v1.17.6' - hash = 'sha256-B1ePaR77HKDksfG6PLwOmOVPYac9g13i8Ld/xWOCIiE=' - - [mod.'go.yaml.in/yaml/v3'] - version = 'v3.0.4' - hash = 'sha256-NkGFiDPoCxbr3LFsI6OCygjjkY0rdmg5ggvVVwpyDQ4=' - - [mod.'go.yaml.in/yaml/v4'] - version = 'v4.0.0-rc.6' - hash = 'sha256-9aOIQpQ53KLv/Hb14104qYKLVrZUcCGLwzYp6UGLP90=' - - [mod.'golang.org/x/crypto'] - version = 'v0.54.0' - hash = 'sha256-m8jsqlPuoPxGDjClzcAVbXf1aiU07L+9j7vYcqJu/0Q=' - - [mod.'golang.org/x/exp'] - version = 'v0.0.0-20221006183845-316c7553db56' - hash = 'sha256-2aATMtazbT3i+X3m0Ju54GKOggcyeX9dAJFxO+H5IxQ=' - - [mod.'golang.org/x/mod'] - version = 'v0.38.0' - hash = 'sha256-BpKfvsmb7ecEDvTBKx9ZHauxoWRkOAZCqg4BrHM2/S0=' - - [mod.'golang.org/x/net'] - version = 'v0.57.0' - hash = 'sha256-gKo8UMw4hfETBHm8N5GOfuNadse9TWEQXJo2YWq5bY4=' - - [mod.'golang.org/x/sync'] - version = 'v0.22.0' - hash = 'sha256-VZjl0fAM0p/nI81zh+pdBjzxFupx0UcKYXBBpL2ZS7k=' - - [mod.'golang.org/x/sys'] - version = 'v0.47.0' - hash = 'sha256-TpbRyWWqHjddP6QzUgAbaLd2EE0S+GYNRUIDJd18r98=' - - [mod.'golang.org/x/text'] - version = 'v0.40.0' - hash = 'sha256-LJfnki46XEreGbSgjl+DeqgcTsINTOu2owyXNvijMcA=' - - [mod.'golang.org/x/tools'] - version = 'v0.48.0' - hash = 'sha256-9cRNUaup6fexA5S1zc+Ic3aaoxAgKWntyMJ2xaOdiP8=' - - [mod.'gonum.org/v1/gonum'] - version = 'v0.8.2' - hash = 'sha256-eFUTetKRBa9QzDUEtd6aU8KsTJQgBF6zLAveY4Ybi58=' - - [mod.'gopkg.in/neurosnap/sentences.v1'] - version = 'v1.0.7' - hash = 'sha256-AaBiqQ5Mtrzk4dE0lZidyUtDzYVy7aonEeh5oXrq/VI=' - - [mod.'gopkg.in/warnings.v0'] - version = 'v0.1.2' - hash = 'sha256-ATVL9yEmgYbkJ1DkltDGRn/auGAjqGOfjQyBYyUo8s8=' - - [mod.'modernc.org/libc'] - version = 'v1.75.7' - hash = 'sha256-Iq3Pcr3RrbhC82RWDM1I4gyKh3V/HI1gVQzoGR+0YYo=' - - [mod.'modernc.org/mathutil'] - version = 'v1.7.1' - hash = 'sha256-COZ5rF2GhQVR1r6a0DanJ8qwQ94JSKdQxTMWrDzE0Cc=' - - [mod.'modernc.org/memory'] - version = 'v1.12.1' - hash = 'sha256-Y/kW9/A5uF+Y66kDbxrh4muFgE3CEee6YzSl2FMPcRg=' - - [mod.'modernc.org/sqlite'] - version = 'v1.59.0' - hash = 'sha256-CaNi9UvS9y6aiiSy3GUlRHX84og+i4MWOZmxbFONeb4=' diff --git a/main.go b/main.go deleted file mode 100644 index 99aed99..0000000 --- a/main.go +++ /dev/null @@ -1,61 +0,0 @@ -package main - -import ( - "fmt" - "os" - - "go.yaml.in/yaml/v4" - - "git.xengi.de/ffd/indexer/scraper" - "git.xengi.de/ffd/indexer/store" -) - -func main() { - data, err := os.ReadFile("config.yaml") - if err != nil { - panic(err) - } - var config Config - - if err := yaml.Unmarshal(data, &config); err != nil { - panic(err) - } - - db, err := store.Open(config.DbPath) - if err != nil { - panic(err) - } - defer func() { _ = db.Close() }() - - for _, forge := range config.Forges { - fmt.Printf("Scraping %s (%s, %s)\n", forge.Name, forge.Type, forge.Url) - - s, err := scraper.NewScraper(forge.Type, forge.Url) - if err != nil { - fmt.Fprintf(os.Stderr, " skip %s: %v\n", forge.Name, err) - continue - } - - projects, err := s.GetProjects() - if err != nil { - fmt.Fprintf(os.Stderr, " skip %s: %v\n", forge.Name, err) - continue - } - - fmt.Printf(" found %d projects\n", len(projects)) - for _, p := range projects { - if err := db.Upsert(p); err != nil { - fmt.Fprintf(os.Stderr, " store %s: %v\n", p.Name, err) - continue - } - fmt.Printf(" %s\n", p.Name) - } - - all, err := db.All() - if err != nil { - fmt.Fprintf(os.Stderr, " count: %v\n", err) - continue - } - fmt.Printf(" total in db: %d\n", len(all)) - } -} diff --git a/option.nix b/option.nix deleted file mode 100644 index d003b41..0000000 --- a/option.nix +++ /dev/null @@ -1,57 +0,0 @@ -{ - config, - lib, - pkgs, - options, - ... -}: - -let - cfg = config.services.federated-forge-discovery.indexer; - configFile = pkgs.writeText "config.ini" (pkgs.lib.generators.toINI { } cfg); - execCommand = "${cfg.package}/bin/ffd-indexer -c '${configFile}'"; -in -{ - options.services.federated-forge-discovery.indexer = { - enable = lib.mkEnableOption "Enables the Federated Forge Discovery Indexer systemd service."; - openFirewall = lib.mkOption { - - }; - package = lib.mkOption { - description = "Package to use."; - type = lib.types.package; - default = lib.literalExpression '' - pkgs.ffd-indexer - ''; - }; - port = lib.mkOption { - description = "Port to listen for search engines"; - type = lib.types.port; - default = 47810; - }; - }; - - config = lib.mkIf cfg.enable { - systemd.services."ffd-indexer" = { - description = "Federated Forge Discovery Indexer"; - wants = [ "network-online.target" ]; - after = [ "network-online.target" ]; - serviceConfig = { - Restart = "always"; - RestartSec = 30; - ExecStart = execCommand; - User = "ffd-indexer"; - Group = "ffd-indexer"; - AmbientCapabilities = lib.mkIf (cfg.port < 1000) [ "CAP_NET_BIND_SERVICE" ]; - CapabilityBoundingSet = lib.mkIf (cfg.port < 1000) [ "CAP_NET_BIND_SERVICE" ]; - NoNewPrivileges = true; - }; - wantedBy = [ "multi-user.target" ]; - }; - }; - - #meta = { - # maintainers = with lib.maintainers; [ xengi ]; - # doc = ./default.xml; - #}; -} diff --git a/scraper/forgejo.go b/scraper/forgejo.go deleted file mode 100644 index f2023ee..0000000 --- a/scraper/forgejo.go +++ /dev/null @@ -1,230 +0,0 @@ -// Package scraper forgejo.go - Forgejo scraper. -package scraper - -import ( - "fmt" - "sort" - "strings" - "time" - - forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3" - - "github.com/go-enry/go-license-detector/v4/licensedb/filer" -) - -// ForgejoScraper retrieves public repositories from a Forgejo instance. -type ForgejoScraper struct { - Client *forgejo.Client - BaseURL string - PageSize int -} - -// NewForgejoScraper creates a scraper for a Forgejo instance. -// BaseURL should be the instance URL, e.g. https://codeberg.org. -func NewForgejoScraper(baseURL string) (*ForgejoScraper, error) { - client, err := forgejo.NewClient(baseURL) - if err != nil { - return nil, fmt.Errorf("create Forgejo client: %w", err) - } - client.SetUserAgent(UserAgent) - - return &ForgejoScraper{ - Client: client, - BaseURL: strings.TrimSuffix(baseURL, "/"), - PageSize: 50, - }, nil -} - -// GetProjects retrieves every public repository visible through the -// Forgejo repository search API and maps it to the generic Project type. -func (s *ForgejoScraper) GetProjects() ([]Project, error) { - // Honor the instance's robots.txt before making any API call: - // if /api/ is disallowed, do not scrape this instance. - ok, err := Allowed(s.BaseURL) - if err != nil { - return nil, fmt.Errorf("check robots.txt: %w", err) - } - if !ok { - return nil, fmt.Errorf("instance robots.txt disallows /api/") - } - - pageSize := s.PageSize - if pageSize <= 0 { - pageSize = 50 - } - - isPrivate := false - var projects []Project - - for page := 1; ; page++ { - repos, _, err := s.Client.SearchRepos(forgejo.SearchRepoOptions{ - Page: page, - PageSize: pageSize, - IsPrivate: &isPrivate, - Type: forgejo.RepoTypeSource, - Sort: "alpha", - Order: "asc", - }) - if err != nil { - return nil, fmt.Errorf( - "list public Forgejo repositories (page %d): %w", - page, - err, - ) - } - - if len(repos) == 0 { - break - } - - for _, repo := range repos { - if repo == nil { - continue - } - - owner, name, ok := strings.Cut(repo.FullName, "/") - if !ok { - continue - } - - // List the repository root once and reuse it for readme selection - // and license detection (they previously each listed the root). - entries, _, err := s.Client.ListContents(owner, name, repo.DefaultBranch, "") - if err != nil { - // best-effort: fall back to empty root so the other fields still work - entries = nil - } - root := make([]filer.File, 0, len(entries)) - for _, e := range entries { - if e == nil { - continue - } - root = append(root, filer.File{Name: e.Name, IsDir: e.Type == "dir"}) - } - - // topics currently requires one extra HTTP call per repository. - // Future optimization: the /repos/search response already includes - // the topics field, so they could be gathered by parsing the - // SearchRepos payload itself (e.g. via ListRepos) instead. - // On failure the repo is kept in the results with empty topics. - topics, err := s.topics(owner, name) - if err != nil { - topics = nil - } - - // languages requires one extra HTTP call per repository - // (GET /repos/{owner}/{repo}/languages, via GetRepoLanguages). - // It returns a map[language]bytes; we order it most-dominant-first - // below. On failure the repo is kept with no languages. - langs, _, err := s.Client.GetRepoLanguages(owner, name) - if err != nil { - langs = nil - } - - // Latest commit: one HTTP call per repository - // (GET /repos/{owner}/{repo}/commits?limit=1, via ListRepoCommits), returns the - // newest commit first. SHA + date are mapped into LatestCommit. - latestCommit, _ := s.latestCommit(owner, name) - - // Latest release: one HTTP call per repository - // (GET /repos/{owner}/{repo}/releases/latest, via GetLatestRelease). - // Returns a single Release (server already excludes drafts and prereleases). - latestRelease, _ := s.latestRelease(owner, name) - - // Licenses: scan repo files on its default branch with - // go-license-detector, reusing the root listing. May require reads - // of candidate license files. On failure the repo is kept with no - // licenses. - licenses := s.licensesFor(owner, name, repo.DefaultBranch, root) - - // Readme: pick the default README (md/rst/txt/no extension) from the - // shared root listing and fetch it. Non-fatal when absent. - readme := s.readmeFor(owner, name, repo.DefaultBranch, root) - - projects = append(projects, Project{ - Name: repo.FullName, - Url: repo.HTMLURL, - Description: repo.Description, - OpenIssues: repo.OpenIssues, - OpenPRs: repo.OpenPulls, - Topics: topics, - Languages: orderedLanguages(langs), - LatestCommit: latestCommit, - LatestRelease: latestRelease, - Licenses: licenses, - Readme: readme, - }) - } - - // A short page indicates that there are no further results. - if len(repos) < pageSize { - break - } - } - - return projects, nil -} - -// topics returns the repository's topics. -func (s *ForgejoScraper) topics(owner, name string) ([]string, error) { - topics, _, err := s.Client.ListRepoTopics(owner, name, forgejo.ListRepoTopicsOptions{}) - if err != nil { - return nil, fmt.Errorf("list topics for %s/%s: %w", owner, name, err) - } - return topics, nil -} - -// latestCommit returns the most recent commit's time. On failure it returns -// the zero time and the error, leaving the Project's LatestCommit unset. -func (s *ForgejoScraper) latestCommit(owner, name string) (time.Time, error) { - commits, _, err := s.Client.ListRepoCommits(owner, name, forgejo.ListCommitOptions{ - PageSize: 1, - }) - if err != nil { - return time.Time{}, err - } - if len(commits) == 0 || commits[0] == nil { - return time.Time{}, nil - } - return commits[0].Created, nil -} - -// latestRelease returns the most recent non-draft, non-prerelease release. -// On failure (e.g. no releases) it returns a zero Release and the error, -// leaving the Project's LatestRelease unset. -func (s *ForgejoScraper) latestRelease(owner, name string) (Release, error) { - r, _, err := s.Client.GetLatestRelease(owner, name) - if err != nil { - return Release{}, err - } - if r == nil { - return Release{}, nil - } - return Release{ - Name: r.Title, - Date: r.PublishedAt, - }, nil -} - -// orderedLanguages returns the languages of a repo ordered by byte share, -// most dominant first. Ties are broken alphabetically for stable output. -func orderedLanguages(langs map[string]int64) []string { - if len(langs) == 0 { - return nil - } - - names := make([]string, 0, len(langs)) - for name := range langs { - names = append(names, name) - } - - // Descending byte count; alphabetically as a stable tie-breaker. - sort.Slice(names, func(i, j int) bool { - if langs[names[i]] != langs[names[j]] { - return langs[names[i]] > langs[names[j]] - } - return names[i] < names[j] - }) - - return names -} diff --git a/scraper/forgejo_licenses.go b/scraper/forgejo_licenses.go deleted file mode 100644 index 8094912..0000000 --- a/scraper/forgejo_licenses.go +++ /dev/null @@ -1,96 +0,0 @@ -package scraper - -import ( - "fmt" - - forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3" - - "github.com/go-enry/go-license-detector/v4/licensedb" - "github.com/go-enry/go-license-detector/v4/licensedb/filer" -) - -// forgejoFiler adapts the Forgejo content API to the filer.Filer interface so -// that go-license-detector can inspect a repository's files without a local -// checkout. ReadDir serves a cached root listing when available (to avoid an -// extra ListContents call) and falls back to the API otherwise. ReadFile -// downloads a file's bytes at the given ref (default branch). -type forgejoFiler struct { - client *forgejo.Client - owner string - repo string - ref string - // cachedRoot, when non-nil, is served for ReadDir("") to avoid a second - // root listing that was already fetched by the caller. - cachedRoot []filer.File -} - -var _ filer.Filer = (*forgejoFiler)(nil) - -func (f *forgejoFiler) ReadDir(dirPath string) ([]filer.File, error) { - if dirPath == "" && f.cachedRoot != nil { - return f.cachedRoot, nil - } - entries, _, err := f.client.ListContents(f.owner, f.repo, f.ref, dirPath) - if err != nil { - return nil, fmt.Errorf("list contents %q: %w", dirPath, err) - } - files := make([]filer.File, 0, len(entries)) - for _, e := range entries { - if e == nil { - continue - } - files = append(files, filer.File{ - Name: e.Name, - IsDir: e.Type == "dir", - }) - } - return files, nil -} - -func (f *forgejoFiler) ReadFile(path string) ([]byte, error) { - // GetFile returns raw file bytes (binary-safe, base64-decoded). - data, _, err := f.client.GetFile(f.owner, f.repo, f.ref, path) - if err != nil { - return nil, fmt.Errorf("read file %q: %w", path, err) - } - return data, nil -} - -func (f *forgejoFiler) Close() {} - -func (f *forgejoFiler) PathsAreAlwaysSlash() bool { return true } - -// licensesFor returns the SPDX identifiers detected for the repository. It -// uses go-license-detector over the repository's files on its default branch. -// The given root listing is served to the detector to avoid a duplicate, -// already-available API call. An empty slice is returned when no license can -// be determined (the repo is still kept in results -- license failures are -// non-fatal). -func (s *ForgejoScraper) licensesFor(owner, repo, ref string, root []filer.File) []string { - detector := &forgejoFiler{ - client: s.Client, - owner: owner, - repo: repo, - ref: ref, - cachedRoot: root, - } - - matches, err := licensedb.Detect(detector) - if err != nil { - return nil - } - - // Keep only the highest-confidence match, sorted deterministically. - ids := make([]string, 0, len(matches)) - bestID, bestConf := "", float32(-1) - for id, m := range matches { - if m.Confidence > bestConf { - bestConf = m.Confidence - bestID = id - } - } - if bestID != "" { - ids = append(ids, bestID) - } - return ids -} diff --git a/scraper/forgejo_readme.go b/scraper/forgejo_readme.go deleted file mode 100644 index fe7122a..0000000 --- a/scraper/forgejo_readme.go +++ /dev/null @@ -1,70 +0,0 @@ -package scraper - -import ( - "sort" - "strings" - - "github.com/go-enry/go-license-detector/v4/licensedb/filer" -) - -// readmeCandidates are the filenames looked for, in priority order, when -// retrieving a repository's README. -var readmeCandidates = []string{"README.md", "README.rst", "README.txt", "README"} - -// readmeFor returns the content of the repository's README on the given ref -// (default branch), picking from root (the pre-fetched root file listing) the -// first candidate found (case-insensitively). An empty string is returned when -// no README can be found; the repo is still kept in results either way. -func (s *ForgejoScraper) readmeFor(owner, name, ref string, root []filer.File) string { - // Collect root file names, lowercased for case-insensitive matching, while - // keeping the real name for the GetFile call. - type rootFile struct { - real string - low string - } - files := make([]rootFile, 0, len(root)) - for _, e := range root { - if e.IsDir { - continue - } - files = append(files, rootFile{real: e.Name, low: strings.ToLower(e.Name)}) - } - - // Candidate priority wins; ties broken by alphabetical real name for - // deterministic picks when a repo has e.g. both README and README.md. - sort.SliceStable(files, func(i, j int) bool { - pi, pj := readmePriority(files[i].low), readmePriority(files[j].low) - if pi != pj { - return pi < pj // lower = higher priority - } - return files[i].real < files[j].real - }) - - var chosen string - for i := range files { - if readmePriority(files[i].low) >= 0 { - chosen = files[i].real - break - } - } - if chosen == "" { - return "" - } - - data, _, err := s.Client.GetFile(owner, name, ref, chosen) - if err != nil { - return "" - } - return string(data) -} - -// readmePriority returns the index of a lowercased filename in -// readmeCandidates, or -1 if it is not a readme candidate. -func readmePriority(low string) int { - for i, c := range readmeCandidates { - if low == strings.ToLower(c) { - return i - } - } - return -1 -} diff --git a/scraper/robots.go b/scraper/robots.go deleted file mode 100644 index bbbbd92..0000000 --- a/scraper/robots.go +++ /dev/null @@ -1,157 +0,0 @@ -package scraper - -import ( - "fmt" - "io" - "net/http" - "net/url" - "strings" - "time" -) - -// UserAgent identifies this crawler to servers. Keep it stable so that -// instances can set robots rules for us specifically. -const UserAgent = "ffd-indexer/0.1" - -// robotsHTTPClient is used for fetching robots.txt. It does not follow the -// SDK client's settings; it is a plain, short-timeout client. -var robotsHTTPClient = &http.Client{Timeout: 10 * time.Second} - -// Allowed reports whether an instance at baseURL permits crawling its API -// according to its robots.txt. It only consults the rules for the -// User-Agent "*"; a block of /api/ (the API root) disallows scraping. -// -// A failure to fetch or parse robots.txt is treated as an error; callers -// decide whether that is fatal. -func Allowed(baseURL string) (bool, error) { - u, err := url.Parse(baseURL) - if err != nil { - return false, fmt.Errorf("parse base URL: %w", err) - } - u.Path = "/robots.txt" - - req, err := http.NewRequest(http.MethodGet, u.String(), nil) - if err != nil { - return false, fmt.Errorf("build robots request: %w", err) - } - req.Header.Set("User-Agent", UserAgent) - - resp, err := robotsHTTPClient.Do(req) - if err != nil { - return false, fmt.Errorf("fetch robots.txt: %w", err) - } - defer func() { - _ = resp.Body.Close() - }() - - // A 404 or similar means "no robots.txt": assume allowed. - if resp.StatusCode == http.StatusNotFound { - return true, nil - } - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - return false, fmt.Errorf("robots.txt returned %s", resp.Status) - } - - body, err := io.ReadAll(resp.Body) - if err != nil { - return false, fmt.Errorf("read robots.txt: %w", err) - } - - return allowedFor(string(body), UserAgent), nil -} - -// allowedFor parses a robots.txt body and reports whether a crawler with the -// given user agent may crawl the path "/api/". It implements a reasonable -// subset of the robots.txt standard: -// -// - Only the "*" group and an explicit group matching userAgent are -// considered; later groups override earlier ones for the same agent. -// - Disallow entries are matched as path prefixes (a trailing "*" is -// stripped). No other wildcard/glob support. -// - An empty group (no rules) is "allow everything". -func allowedFor(body, userAgent string) bool { - groups := parseRobotGroups(body) - - // A group that explicitly names our user agent takes precedence over a - // "*" group, regardless of order. Otherwise fall back to "*". - var star *robotGroup - for _, g := range groups { - for _, a := range g.agents { - if strings.EqualFold(a, "*") { - star = g - } else if strings.EqualFold(a, userAgent) { - return !matchDisallow(g, "/api/") - } - } - } - if star != nil { - return !matchDisallow(star, "/api/") - } - return true -} - -type robotGroup struct { - agents []string - disallow []string -} - -// parseRobotGroups splits a robots.txt body into agent groups. -func parseRobotGroups(body string) []*robotGroup { - var groups []*robotGroup - var cur *robotGroup - - for _, raw := range strings.Split(body, "\n") { - line := strings.TrimSpace(raw) - // Strip an inline comment (not part of a rule's result). - if i := strings.IndexByte(line, '#'); i >= 0 { - line = strings.TrimSpace(line[:i]) - } - if line == "" { - continue - } - - key, val, ok := strings.Cut(line, ":") - if !ok { - continue - } - key = strings.ToLower(strings.TrimSpace(key)) - val = strings.TrimSpace(val) - - switch key { - case "user-agent": - // A User-agent line starts a new group, unless the current - // group is still collecting agents (has no Disallow yet). - if cur == nil || len(cur.disallow) > 0 { - cur = &robotGroup{} - groups = append(groups, cur) - } - cur.agents = append(cur.agents, val) - case "disallow": - if cur != nil { - cur.disallow = append(cur.disallow, val) - } - } - } - - return groups -} - -// matchDisallow reports whether a path is disallowed by a group's rules. -// A trailing "*" is stripped. Rules are applied in order, later rules -// overriding earlier ones: an empty Disallow rule revokes all preceding -// disallows (allow everything). -func matchDisallow(g *robotGroup, path string) bool { - disallowed := false - for _, rule := range g.disallow { - rule = strings.TrimSpace(rule) - rule = strings.TrimSuffix(rule, "*") - if rule == "" { - disallowed = false // empty Disallow: allow everything - continue - } - if strings.HasPrefix(path, rule) { - disallowed = true - } - } - return disallowed -} diff --git a/scraper/robots_test.go b/scraper/robots_test.go deleted file mode 100644 index 15b569c..0000000 --- a/scraper/robots_test.go +++ /dev/null @@ -1,60 +0,0 @@ -package scraper - -import "testing" - -func TestAllowedFor(t *testing.T) { - const ua = "ffd-indexer/0.1" - - // xengi / cccb: only /archive/ blocked -> /api/ allowed. - if !allowedFor("User-agent: *\nDisallow: /*/*/archive/\n", ua) { - t.Error("expected /api/ allowed when only archive is disallowed") - } - - // codeberg: /api/ disallowed under "*". - codeberg := `User-agent: * -Disallow: /api/ -Disallow: /avatars/ -` - if allowedFor(codeberg, ua) { - t.Error("expected /api/ blocked when /api/ is disallowed") - } - - // Explicit agent group takes precedence over "*" (agent name matches - // UserAgent exactly, including the version). - // Here the explicit group is for "ffd-indexer" (no version), so it does - // not match "ffd-indexer/0.1"; the "*" group (blocks /api/) applies. - if allowedFor("User-agent: *\nDisallow: /api/\n\nUser-agent: ffd-indexer\nDisallow: /whatever/\n", ua) { - t.Error("expected unmatched explicit agent to fall back to * which blocks /api/") - } - - // A matching explicit agent group takes precedence over "*". - if !allowedFor("User-agent: *\nDisallow: /api/\n\nUser-agent: ffd-indexer/0.1\nDisallow: /whatever/\n", ua) { - t.Error("expected explicit agent rule to permit /api/") - } - - // No rules at all -> allow. - if !allowedFor("", ua) { - t.Error("expected empty robots to allow") - } - - // Only an unrelated agent listed -> no star, no match -> allow. - if !allowedFor("User-agent: SomeBot\nDisallow: /api/\n", ua) { - t.Error("expected unrelated agent not to block us") - } - - // Disallow "/" blocks everything including /api/. - if allowedFor("User-agent: *\nDisallow: /\n", ua) { - t.Error("expected Disallow / to block /api/") - } - - // Empty Disallow (allowed) overrides a previous block in the same group. - if !allowedFor("User-agent: *\nDisallow: /api/\nDisallow:\n", ua) { - t.Error("expected empty Disallow to allow /api/") - } - - // Wildcard on a non-prefix rule should not over-block (substring-only). - // "/*.pdf" should not match /api/. - if !allowedFor("User-agent: *\nDisallow: /*.pdf$\n", ua) { - t.Error("expected unrelated wildcard to not block /api/") - } -} diff --git a/scraper/scraper.go b/scraper/scraper.go deleted file mode 100644 index a4f97ee..0000000 --- a/scraper/scraper.go +++ /dev/null @@ -1,47 +0,0 @@ -// Package scraper defines the interfaces and shared types used to discover -// projects from forge instances (Forgejo, Gogs, ...). -package scraper - -import ( - "fmt" - "time" -) - -// Scraper is implemented by concrete, per-forge scrapers. -type Scraper interface { - // GetProjects retrieves every project discoverable through this forge's - // public API and maps it to the generic Project type. - GetProjects() ([]Project, error) -} - -// NewScraper returns a Scraper for the given forge type, e.g. "forgejo". -func NewScraper(typ, baseURL string) (Scraper, error) { - switch typ { - case "forgejo": - return NewForgejoScraper(baseURL) - default: - return nil, fmt.Errorf("unknown forge type %q", typ) - } -} - -// Project is a generic, forge-agnostic representation of an open source project. -type Project struct { - Name string `json:"name"` - Url string `json:"url"` - Description string `json:"description"` - Topics []string `json:"topics"` - OpenIssues int `json:"open_issues"` - OpenPRs int `json:"open_prs"` - LatestCommit time.Time `json:"latest_commit"` - LatestRelease Release `json:"latest_release"` - Languages []string `json:"languages"` - Readme string `json:"readme"` - Licenses []string `json:"license"` - UsesAI bool `json:"uses_ai"` -} - -// Release is a named release with the date it was published. -type Release struct { - Name string `json:"name"` - Date time.Time `json:"date"` -} diff --git a/store/store.go b/store/store.go deleted file mode 100644 index c892be8..0000000 --- a/store/store.go +++ /dev/null @@ -1,190 +0,0 @@ -// Package store provides durable persistence of scraped projects in SQLite. -// -// The database is the system of record: scrapers write discovered projects -// here, and a search index (e.g. Typesense) can be built or rebuilt from it -// without re-scraping the forges. Projects are stored as one row each, with -// the URL as the natural key so re-running the scrape upserts in place. -package store - -import ( - "database/sql" - "encoding/json" - "fmt" - "time" - - _ "modernc.org/sqlite" - - "git.xengi.de/ffd/indexer/scraper" -) - -const schema = ` -CREATE TABLE IF NOT EXISTS projects ( - url TEXT PRIMARY KEY, - name TEXT NOT NULL, - description TEXT NOT NULL DEFAULT '', - topics TEXT NOT NULL DEFAULT '[]', -- JSON array - languages TEXT NOT NULL DEFAULT '[]', -- JSON array - licenses TEXT NOT NULL DEFAULT '[]', -- JSON array - open_issues INTEGER NOT NULL DEFAULT 0, - open_prs INTEGER NOT NULL DEFAULT 0, - latest_commit INTEGER NOT NULL DEFAULT 0, -- unix epoch seconds (UTC) - latest_release_name TEXT NOT NULL DEFAULT '', - latest_release_date INTEGER NOT NULL DEFAULT 0, -- unix epoch seconds (UTC) - readme TEXT NOT NULL DEFAULT '', - uses_ai INTEGER NOT NULL DEFAULT 0, - scraped_at INTEGER NOT NULL DEFAULT 0 -- unix epoch seconds (UTC) -); -` - -// Store wraps a SQLite database holding scraped projects. -type Store struct { - db *sql.DB -} - -// Open opens (creating if needed) the SQLite database at path. -func Open(path string) (*Store, error) { - db, err := sql.Open("sqlite", path) - if err != nil { - return nil, fmt.Errorf("open sqlite db %q: %w", path, err) - } - if _, err := db.Exec(schema); err != nil { - _ = db.Close() - return nil, fmt.Errorf("create schema: %w", err) - } - return &Store{db: db}, nil -} - -// Close closes the underlying database. -func (s *Store) Close() error { - return s.db.Close() -} - -const upsertSQL = ` -INSERT INTO projects ( - url, name, description, topics, languages, licenses, - open_issues, open_prs, latest_commit, - latest_release_name, latest_release_date, - readme, uses_ai, scraped_at -) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) -ON CONFLICT(url) DO UPDATE SET - name = excluded.name, - description = excluded.description, - topics = excluded.topics, - languages = excluded.languages, - licenses = excluded.licenses, - open_issues = excluded.open_issues, - open_prs = excluded.open_prs, - latest_commit = excluded.latest_commit, - latest_release_name = excluded.latest_release_name, - latest_release_date = excluded.latest_release_date, - readme = excluded.readme, - uses_ai = excluded.uses_ai, - scraped_at = excluded.scraped_at; -` - -// Upsert inserts or updates a project, keyed on its URL. -func (s *Store) Upsert(p scraper.Project) error { - topics, err := json.Marshal(p.Topics) - if err != nil { - return fmt.Errorf("marshal topics: %w", err) - } - langs, err := json.Marshal(p.Languages) - if err != nil { - return fmt.Errorf("marshal languages: %w", err) - } - licenses, err := json.Marshal(p.Licenses) - if err != nil { - return fmt.Errorf("marshal licenses: %w", err) - } - - releaseDate := timeToUnix(p.LatestRelease.Date) - latestCommit := timeToUnix(p.LatestCommit) - - _, err = s.db.Exec(upsertSQL, - p.Url, - p.Name, - p.Description, - string(topics), - string(langs), - string(licenses), - p.OpenIssues, - p.OpenPRs, - latestCommit, - p.LatestRelease.Name, - releaseDate, - p.Readme, - boolToInt(p.UsesAI), - time.Now().UTC().Unix(), - ) - return err -} - -// All returns every stored project. -func (s *Store) All() ([]scraper.Project, error) { - rows, err := s.db.Query(`SELECT - url, name, description, topics, languages, licenses, - open_issues, open_prs, latest_commit, - latest_release_name, latest_release_date, - readme, uses_ai - FROM projects`) - if err != nil { - return nil, err - } - defer func() { _ = rows.Close() }() - - var projects []scraper.Project - for rows.Next() { - var ( - p scraper.Project - topicsJSON string - langsJSON string - licensesJSON string - latestCommit int64 - relName string - relDate int64 - usesAI int - ) - if err := rows.Scan( - &p.Url, &p.Name, &p.Description, - &topicsJSON, &langsJSON, &licensesJSON, - &p.OpenIssues, &p.OpenPRs, &latestCommit, - &relName, &relDate, - &p.Readme, &usesAI, - ); err != nil { - return nil, err - } - - p.UsesAI = usesAI != 0 - if err := json.Unmarshal([]byte(topicsJSON), &p.Topics); err != nil { - return nil, err - } - if err := json.Unmarshal([]byte(langsJSON), &p.Languages); err != nil { - return nil, err - } - if err := json.Unmarshal([]byte(licensesJSON), &p.Licenses); err != nil { - return nil, err - } - p.LatestCommit = time.Unix(latestCommit, 0).UTC() - p.LatestRelease.Name = relName - p.LatestRelease.Date = time.Unix(relDate, 0).UTC() - - projects = append(projects, p) - } - return projects, rows.Err() -} - -// timeToUnix returns the Unix epoch seconds for a time, or 0 for the zero -// time (which is stored as a 0/absent value). -func timeToUnix(t time.Time) int64 { - if t.IsZero() { - return 0 - } - return t.Unix() -} - -func boolToInt(b bool) int { - if b { - return 1 - } - return 0 -}