scraper done; scrapes to local sqlite db

initial dump
2026-09-28 15:36:42 +02:00 · 2026-09-28 12:21:17 +02:00
18 changed files with 2039 additions and 12 deletions
--- a/.editorconfig
+++ b/.editorconfig
@ -0,0 +1,11 @@
+# EditorConfig is awesome: https://editorconfig.org
+root = true
+
+[*]
+end_of_line = lf
+insert_final_newline = true
+
+[*.{go,nix,yaml}]
+charset = utf-8
+indent_style = space
+indent_size = 2
--- a/.gitignore
+++ b/.gitignore
@ -1,4 +1,17 @@
-# ---> Go
+config.yaml
+models.json
+Containerfile
+result
+index.db
+
+# Created by https://www.toptal.com/developers/gitignore/api/windows,linux,macos,direnv,vim,go,goland+all
+# Edit at https://www.toptal.com/developers/gitignore?templates=windows,linux,macos,direnv,vim,go,goland+all
+
+### direnv ###
+.direnv
+.envrc
+
+### Go ###
 # If you prefer the allow list template instead of the deny list, see community template:
 # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
 #
@ -12,23 +25,195 @@
 # Test binary, built with `go test -c`
 *.test

-# Code coverage profiles and other test artifacts
+# Output of the go coverage tool, specifically when used with LiteIDE
 *.out
-coverage.*
-*.coverprofile
-profile.cov

 # Dependency directories (remove the comment below to include it)
 # vendor/

 # Go workspace file
 go.work
-go.work.sum

-# env file
-.env
+### GoLand+all ###
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839

-# Editor/IDE
-# .idea/
-# .vscode/
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf

+# AWS User-specific
+.idea/**/aws.xml
+
+# Generated files
+.idea/**/contentModel.xml
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+
+# Gradle
+.idea/**/gradle.xml
+.idea/**/libraries
+
+# Gradle and Maven with auto-import
+# When using Gradle or Maven with auto-import, you should exclude module files,
+# since they will be recreated, and may cause churn.  Uncomment if using
+# auto-import.
+# .idea/artifacts
+# .idea/compiler.xml
+# .idea/jarRepositories.xml
+# .idea/modules.xml
+# .idea/*.iml
+# .idea/modules
+# *.iml
+# *.ipr
+
+# CMake
+cmake-build-*/
+
+# Mongo Explorer plugin
+.idea/**/mongoSettings.xml
+
+# File-based project format
+*.iws
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# SonarLint plugin
+.idea/sonarlint/
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+# Editor-based Rest Client
+.idea/httpRequests
+
+# Android studio 3.1+ serialized cache file
+.idea/caches/build_file_checksums.ser
+
+### GoLand+all Patch ###
+# Ignore everything but code style settings and run configurations
+# that are supposed to be shared within teams.
+
+.idea/*
+
+!.idea/codeStyles
+!.idea/runConfigurations
+
+### Linux ###
+*~
+
+# temporary files which can be created if a process still has a handle open of a deleted file
+.fuse_hidden*
+
+# KDE directory preferences
+.directory
+
+# Linux trash folder which might appear on any partition or disk
+.Trash-*
+
+# .nfs files are created when an open file is removed but is still being accessed
+.nfs*
+
+### macOS ###
+# General
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+### macOS Patch ###
+# iCloud generated files
+*.icloud
+
+### Vim ###
+# Swap
+[._]*.s[a-v][a-z]
+!*.svg  # comment out if you don't need vector files
+[._]*.sw[a-p]
+[._]s[a-rt-v][a-z]
+[._]ss[a-gi-z]
+[._]sw[a-p]
+
+# Session
+Session.vim
+Sessionx.vim
+
+# Temporary
+.netrwhist
+# Auto-generated tag files
+tags
+# Persistent undo
+[._]*.un~
+
+### Windows ###
+# Windows thumbnail cache files
+Thumbs.db
+Thumbs.db:encryptable
+ehthumbs.db
+ehthumbs_vista.db
+
+# Dump file
+*.stackdump
+
+# Folder config file
+[Dd]esktop.ini
+
+# Recycle Bin used on file shares
+$RECYCLE.BIN/
+
+# Windows Installer files
+*.cab
+*.msi
+*.msix
+*.msm
+*.msp
+
+# Windows shortcuts
+*.lnk
+
+# End of https://www.toptal.com/developers/gitignore/api/windows,linux,macos,direnv,vim,go,goland+all
--- a/README.md
+++ b/README.md
@ -1,3 +1,44 @@
 # indexer

 Federated Forge Discovery Indexer
+
+Builds an index of public open source software projects from configured forges.
+
+## Scrape behavior
+
+For each configured forge, the indexer:
+
+1. **Checks robots.txt** (`GET /robots.txt`) before any API call. If the
+   instance disallows `/api/`, the instance is skipped.
+2. **Lists public source repositories** via the repository search API
+   (`GET /api/v1/repos/search`), paginated. Forks and mirrors are excluded.
+3. For each repository, enriches it via the Forgejo API: reads the root
+   directory once, then fetches topics, languages, latest commit, latest
+   release, the README, and detects licenses from the repo's files.
+
+Errors are non-fatal: an instance or a single repo that fails is skipped and
+the run continues.
+
+### Request volume
+
+**Per instance:**
+
+| Request | Count |
+|---|---|
+| `GET /robots.txt` | 1 |
+| `GET /api/v1/repos/search` (page) | `ceil(N/50)` where N = total repos |
+
+**Per repository:**
+
+| Request | Endpoint | Purpose |
+|---|---|---|
+| list root contents | `GET /api/v1/repos/{o}/{r}/contents/` | shared: README selection + license scan (1 call) |
+| languages | `GET /api/v1/repos/{o}/{r}/languages` | `Project.Languages` |
+| topics | `GET /api/v1/repos/{o}/{r}/topics` | `Project.Topics` |
+| latest commit | `GET /api/v1/repos/{o}/{r}/commits?limit=1` | `Project.LatestCommit` |
+| latest release | `GET /api/v1/repos/{o}/{r}/releases/latest` | `Project.LatestRelease` |
+| README | `GET /api/v1/repos/{o}/{r}/raw/{branch}/{file}` | `Project.Readme` |
+| license files | 1+ × `GET /api/v1/repos/{o}/{r}/raw/{branch}/{file}` | SPDX detection |
+
+So roughly **6–8 API requests per repository** plus the paginated search
+requests and a single robots.txt check per instance.
--- a/config.go
+++ b/config.go
@ -0,0 +1,14 @@
+package main
+
+// Forge is a single forge instance to scrape.
+type Forge struct {
+	Name string `yaml:"name"`
+	Url  string `yaml:"url"`
+	Type string `yaml:"type"`
+}
+
+// Config is the top-level configuration file.
+type Config struct {
+	DbPath string  `yaml:"db_path"`
+	Forges []Forge `yaml:"forges"`
+}
--- a/flake.lock
+++ b/flake.lock
@ -0,0 +1,85 @@
+{
+  "nodes": {
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1731533236,
+        "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "gomod2nix": {
+      "inputs": {
+        "flake-utils": [
+          "flake-utils"
+        ],
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1770585520,
+        "narHash": "sha256-yBz9Ozd5Wb56i3e3cHZ8WcbzCQ9RlVaiW18qDYA/AzA=",
+        "owner": "tweag",
+        "repo": "gomod2nix",
+        "rev": "1201ddd1279c35497754f016ef33d5e060f3da8d",
+        "type": "github"
+      },
+      "original": {
+        "owner": "tweag",
+        "repo": "gomod2nix",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1790573947,
+        "narHash": "sha256-zfer1SLJvACKM0J7c9AusGvIp4q4B6l1Z1KgiVInG8Y=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "e6025a0259c7e80f81b11effc15f65d95b708cba",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-unstable-small",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "flake-utils": "flake-utils",
+        "gomod2nix": "gomod2nix",
+        "nixpkgs": "nixpkgs"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}
--- a/flake.nix
+++ b/flake.nix
@ -0,0 +1,86 @@
+{
+  description = "Federated Forge Discovery Indexer";
+  inputs = {
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable-small";
+    flake-utils.url = "github:numtide/flake-utils";
+    gomod2nix = {
+      url = "github:tweag/gomod2nix";
+      inputs.nixpkgs.follows = "nixpkgs";
+      inputs.flake-utils.follows = "flake-utils";
+    };
+  };
+  outputs = { self, nixpkgs, flake-utils, gomod2nix }: flake-utils.lib.eachDefaultSystem (system:
+    let
+      pkgs = import nixpkgs {
+        inherit system;
+        overlays = [ gomod2nix.overlays.default ];
+      };
+      go = pkgs.go_1_27;
+      ffd-indexer = pkgs.buildGoApplication {
+        inherit go;
+        pname = "ffd-indexer";
+        version = "0.0.1";
+        src = ./.;
+        modules = ./gomod2nix.toml;
+      };
+      go-test = pkgs.stdenv.mkDerivation {
+        name = "go-test";
+        dontBuild = true;
+        src = ./.;
+        doCheck = true;
+        nativeBuildInputs = [
+          go
+          pkgs.writableTmpDirAsHomeHook
+        ];
+        checkPhase = ''
+          go test -v ./...
+        '';
+        installPhase = ''
+          mkdir "$out"
+        '';
+      };
+      go-lint = pkgs.stdenv.mkDerivation {
+        name = "go-lint";
+        dontBuild = true;
+        src = ./.;
+        doCheck = true;
+        nativeBuildInputs = [
+          pkgs.golangci-lint
+          go
+          pkgs.writableTmpDirAsHomeHook
+        ];
+        checkPhase = ''
+          golangci-lint run
+        '';
+        installPhase = ''
+          mkdir "$out"
+        '';
+      };
+    in
+    {
+      checks = { inherit go-test go-lint; };
+      formatter = pkgs.nixfmt-tree;
+      devShells.default = pkgs.mkShell {
+        buildInputs = [
+          go
+          pkgs.go-tools # staticcheck
+          gomod2nix.packages.${system}.default
+        ];
+        packages = with pkgs; [
+          mkcert
+          sqlite # debugging the index db
+        ];
+      };
+      packages = {
+        default = ffd-indexer;
+        container = pkgs.dockerTools.buildImage {
+          name = "ffd-indexer";
+          config = {
+            Cmd = [ "${ffd-indexer}/bin/ffd-indexer" ];
+          };
+        };
+      };
+      nixosModules.default = import ./option.nix;
+    }
+  );
+}
--- a/go.mod
+++ b/go.mod
@ -0,0 +1,76 @@
+module git.xengi.de/ffd/indexer
+
+go 1.27.1
+
+require (
+	codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3 v3.0.0
+	github.com/go-enry/go-license-detector/v4 v4.3.1
+	go.yaml.in/yaml/v4 v4.0.0-rc.6
+	modernc.org/sqlite v1.59.0
+)
+
+require (
+	github.com/42wim/httpsig v1.2.3 // indirect
+	github.com/Microsoft/go-winio v0.6.0 // indirect
+	github.com/ProtonMail/go-crypto v0.0.0-20220930113650-c6815a8c17ad // indirect
+	github.com/acomagu/bufpipe v1.0.3 // indirect
+	github.com/cloudflare/circl v1.2.0 // indirect
+	github.com/davidmz/go-pageant v1.0.2 // indirect
+	github.com/dgryski/go-minhash v0.0.0-20190315135803-ad340ca03076 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/ekzhu/minhash-lsh v0.0.0-20190924033628-faac2c6342f8 // indirect
+	github.com/emirpasic/gods v1.18.1 // indirect
+	github.com/go-fed/httpsig v1.1.0 // indirect
+	github.com/go-git/gcfg v1.5.0 // indirect
+	github.com/go-git/go-billy/v5 v5.3.1 // indirect
+	github.com/go-git/go-git/v5 v5.4.2 // indirect
+	github.com/go-openapi/errors v0.22.6 // indirect
+	github.com/go-openapi/strfmt v0.25.0 // indirect
+	github.com/go-openapi/swag v0.25.4 // indirect
+	github.com/go-openapi/swag/cmdutils v0.25.4 // indirect
+	github.com/go-openapi/swag/conv v0.25.4 // indirect
+	github.com/go-openapi/swag/fileutils v0.25.4 // indirect
+	github.com/go-openapi/swag/jsonname v0.25.4 // indirect
+	github.com/go-openapi/swag/jsonutils v0.25.4 // indirect
+	github.com/go-openapi/swag/loading v0.25.4 // indirect
+	github.com/go-openapi/swag/mangling v0.25.4 // indirect
+	github.com/go-openapi/swag/netutils v0.25.4 // indirect
+	github.com/go-openapi/swag/stringutils v0.25.4 // indirect
+	github.com/go-openapi/swag/typeutils v0.25.4 // indirect
+	github.com/go-openapi/swag/yamlutils v0.25.4 // indirect
+	github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/hashicorp/go-version v1.7.0 // indirect
+	github.com/hhatto/gorst v0.0.0-20181029133204-ca9f730cac5b // indirect
+	github.com/imdario/mergo v0.3.13 // indirect
+	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
+	github.com/jdkato/prose v1.2.1 // indirect
+	github.com/kevinburke/ssh_config v1.2.0 // indirect
+	github.com/mattn/go-isatty v0.0.24 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/montanaflynn/stats v0.7.1 // indirect
+	github.com/ncruces/go-strftime v1.0.0 // indirect
+	github.com/oklog/ulid v1.3.1 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/sergi/go-diff v1.2.0 // indirect
+	github.com/shogo82148/go-shuffle v1.0.1 // indirect
+	github.com/xanzy/ssh-agent v0.3.2 // indirect
+	go.mongodb.org/mongo-driver v1.17.6 // indirect
+	go.yaml.in/yaml/v3 v3.0.4 // indirect
+	golang.org/x/crypto v0.54.0 // indirect
+	golang.org/x/exp v0.0.0-20221006183845-316c7553db56 // indirect
+	golang.org/x/mod v0.38.0 // indirect
+	golang.org/x/net v0.57.0 // indirect
+	golang.org/x/sync v0.22.0 // indirect
+	golang.org/x/sys v0.47.0 // indirect
+	golang.org/x/text v0.40.0 // indirect
+	golang.org/x/tools v0.48.0 // indirect
+	gonum.org/v1/gonum v0.8.2 // indirect
+	gopkg.in/neurosnap/sentences.v1 v1.0.7 // indirect
+	gopkg.in/warnings.v0 v0.1.2 // indirect
+	modernc.org/libc v1.75.7 // indirect
+	modernc.org/mathutil v1.7.1 // indirect
+	modernc.org/memory v1.12.1 // indirect
+)
--- a/go.sum
+++ b/go.sum
@ -0,0 +1,291 @@
+codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3 v3.0.0 h1:s2fK+FBwvcYsmKDjNhmoe7B8q9zsgs0UrSlYe9r4XjM=
+codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3 v3.0.0/go.mod h1:Is2jTpS1dizeXm4skQv/ES3QVqnzcNhn2GzZXpiw9f8=
+github.com/42wim/httpsig v1.2.3 h1:xb0YyWhkYj57SPtfSttIobJUPJZB9as1nsfo7KWVcEs=
+github.com/42wim/httpsig v1.2.3/go.mod h1:nZq9OlYKDrUBhptd77IHx4/sZZD+IxTBADvAPI9G/EM=
+github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
+github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0=
+github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
+github.com/Microsoft/go-winio v0.6.0 h1:slsWYD/zyx7lCXoZVlvQrj0hPTM1HI4+v1sIda2yDvg=
+github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE=
+github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo=
+github.com/ProtonMail/go-crypto v0.0.0-20220930113650-c6815a8c17ad h1:QeeqI2zxxgZVe11UrYFXXx6gVxPVF40ygekjBzEg4XY=
+github.com/ProtonMail/go-crypto v0.0.0-20220930113650-c6815a8c17ad/go.mod h1:UBYPn8k0D56RtnR8RFQMjmh4KrZzWJ5o7Z9SYjossQ8=
+github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk=
+github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
+github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
+github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
+github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
+github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
+github.com/bwesterb/go-ristretto v1.2.1/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
+github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I=
+github.com/cloudflare/circl v1.2.0 h1:NheeISPSUcYftKlfrLuOo4T62FkmD4t4jviLfFFYaec=
+github.com/cloudflare/circl v1.2.0/go.mod h1:Ch2UgYr6ti2KTtlejELlROl0YIYj7SLjAC8M+INXlMk=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davidmz/go-pageant v1.0.2 h1:bPblRCh5jGU+Uptpz6LgMZGD5hJoOt7otgT454WvHn0=
+github.com/davidmz/go-pageant v1.0.2/go.mod h1:P2EDDnMqIwG5Rrp05dTRITj9z2zpGcD9efWSkTNKLIE=
+github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc h1:8WFBn63wegobsYAX0YjD+8suexZDga5CctH4CCTx2+8=
+github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc/go.mod h1:c9O8+fpSOX1DM8cPNSkX/qsBWdkD4yd2dpciOWQjpBw=
+github.com/dgryski/go-minhash v0.0.0-20190315135803-ad340ca03076 h1:EB7M2v8Svo3kvIDy+P1YDE22XskDQP+TEYGzeDwPAN4=
+github.com/dgryski/go-minhash v0.0.0-20190315135803-ad340ca03076/go.mod h1:VBi0XHpFy0xiMySf6YpVbRqrupW4RprJ5QTyN+XvGSM=
+github.com/dgryski/go-spooky v0.0.0-20170606183049-ed3d087f40e2 h1:lx1ZQgST/imDhmLpYDma1O3Cx9L+4Ie4E8S2RjFPQ30=
+github.com/dgryski/go-spooky v0.0.0-20170606183049-ed3d087f40e2/go.mod h1:hgHYKsoIw7S/hlWtP7wD1wZ7SX1jPTtKko5X9jrOgPQ=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/ekzhu/minhash-lsh v0.0.0-20190924033628-faac2c6342f8 h1:+Tje+xk1lmGKSJjYNtgCFsU1HtQzz0kCm1DFbKlvFBo=
+github.com/ekzhu/minhash-lsh v0.0.0-20190924033628-faac2c6342f8/go.mod h1:yEtCVi+QamvzjEH4U/m6ZGkALIkF2xfQnFp0BcKmIOk=
+github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
+github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
+github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
+github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
+github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
+github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0=
+github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
+github.com/go-enry/go-license-detector/v4 v4.3.1 h1:BajEVdTffFcs8RACmblySVhfEIuT58TmXx27RgVfUdc=
+github.com/go-enry/go-license-detector/v4 v4.3.1/go.mod h1:YVJKPE01WQNjN/bdM6V0I/9KxvwEAAv0Ef9pi92K6w0=
+github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI=
+github.com/go-fed/httpsig v1.1.0/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM=
+github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4=
+github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E=
+github.com/go-git/go-billy/v5 v5.2.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0=
+github.com/go-git/go-billy/v5 v5.3.1 h1:CPiOUAzKtMRvolEKw+bG1PLRpT7D3LIs3/3ey4Aiu34=
+github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0=
+github.com/go-git/go-git-fixtures/v4 v4.2.1 h1:n9gGL1Ct/yIw+nfsfr8s4+sbhT+Ncu2SubfXjIWgci8=
+github.com/go-git/go-git-fixtures/v4 v4.2.1/go.mod h1:K8zd3kDUAykwTdDCr+I0per6Y6vMiRR/nnVTBtavnB0=
+github.com/go-git/go-git/v5 v5.4.2 h1:BXyZu9t0VkbiHtqrsvdq39UDhGJTl1h55VW6CSC4aY4=
+github.com/go-git/go-git/v5 v5.4.2/go.mod h1:gQ1kArt6d+n+BGd+/B/I74HwRTLhth2+zti4ihgckDc=
+github.com/go-openapi/errors v0.22.6 h1:eDxcf89O8odEnohIXwEjY1IB4ph5vmbUsBMsFNwXWPo=
+github.com/go-openapi/errors v0.22.6/go.mod h1:z9S8ASTUqx7+CP1Q8dD8ewGH/1JWFFLX/2PmAYNQLgk=
+github.com/go-openapi/strfmt v0.25.0 h1:7R0RX7mbKLa9EYCTHRcCuIPcaqlyQiWNPTXwClK0saQ=
+github.com/go-openapi/strfmt v0.25.0/go.mod h1:nNXct7OzbwrMY9+5tLX4I21pzcmE6ccMGXl3jFdPfn8=
+github.com/go-openapi/swag v0.25.4 h1:OyUPUFYDPDBMkqyxOTkqDYFnrhuhi9NR6QVUvIochMU=
+github.com/go-openapi/swag v0.25.4/go.mod h1:zNfJ9WZABGHCFg2RnY0S4IOkAcVTzJ6z2Bi+Q4i6qFQ=
+github.com/go-openapi/swag/cmdutils v0.25.4 h1:8rYhB5n6WawR192/BfUu2iVlxqVR9aRgGJP6WaBoW+4=
+github.com/go-openapi/swag/cmdutils v0.25.4/go.mod h1:pdae/AFo6WxLl5L0rq87eRzVPm/XRHM3MoYgRMvG4A0=
+github.com/go-openapi/swag/conv v0.25.4 h1:/Dd7p0LZXczgUcC/Ikm1+YqVzkEeCc9LnOWjfkpkfe4=
+github.com/go-openapi/swag/conv v0.25.4/go.mod h1:3LXfie/lwoAv0NHoEuY1hjoFAYkvlqI/Bn5EQDD3PPU=
+github.com/go-openapi/swag/fileutils v0.25.4 h1:2oI0XNW5y6UWZTC7vAxC8hmsK/tOkWXHJQH4lKjqw+Y=
+github.com/go-openapi/swag/fileutils v0.25.4/go.mod h1:cdOT/PKbwcysVQ9Tpr0q20lQKH7MGhOEb6EwmHOirUk=
+github.com/go-openapi/swag/jsonname v0.25.4 h1:bZH0+MsS03MbnwBXYhuTttMOqk+5KcQ9869Vye1bNHI=
+github.com/go-openapi/swag/jsonname v0.25.4/go.mod h1:GPVEk9CWVhNvWhZgrnvRA6utbAltopbKwDu8mXNUMag=
+github.com/go-openapi/swag/jsonutils v0.25.4 h1:VSchfbGhD4UTf4vCdR2F4TLBdLwHyUDTd1/q4i+jGZA=
+github.com/go-openapi/swag/jsonutils v0.25.4/go.mod h1:7OYGXpvVFPn4PpaSdPHJBtF0iGnbEaTk8AvBkoWnaAY=
+github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.4 h1:IACsSvBhiNJwlDix7wq39SS2Fh7lUOCJRmx/4SN4sVo=
+github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.4/go.mod h1:Mt0Ost9l3cUzVv4OEZG+WSeoHwjWLnarzMePNDAOBiM=
+github.com/go-openapi/swag/loading v0.25.4 h1:jN4MvLj0X6yhCDduRsxDDw1aHe+ZWoLjW+9ZQWIKn2s=
+github.com/go-openapi/swag/loading v0.25.4/go.mod h1:rpUM1ZiyEP9+mNLIQUdMiD7dCETXvkkC30z53i+ftTE=
+github.com/go-openapi/swag/mangling v0.25.4 h1:2b9kBJk9JvPgxr36V23FxJLdwBrpijI26Bx5JH4Hp48=
+github.com/go-openapi/swag/mangling v0.25.4/go.mod h1:6dxwu6QyORHpIIApsdZgb6wBk/DPU15MdyYj/ikn0Hg=
+github.com/go-openapi/swag/netutils v0.25.4 h1:Gqe6K71bGRb3ZQLusdI8p/y1KLgV4M/k+/HzVSqT8H0=
+github.com/go-openapi/swag/netutils v0.25.4/go.mod h1:m2W8dtdaoX7oj9rEttLyTeEFFEBvnAx9qHd5nJEBzYg=
+github.com/go-openapi/swag/stringutils v0.25.4 h1:O6dU1Rd8bej4HPA3/CLPciNBBDwZj9HiEpdVsb8B5A8=
+github.com/go-openapi/swag/stringutils v0.25.4/go.mod h1:GTsRvhJW5xM5gkgiFe0fV3PUlFm0dr8vki6/VSRaZK0=
+github.com/go-openapi/swag/typeutils v0.25.4 h1:1/fbZOUN472NTc39zpa+YGHn3jzHWhv42wAJSN91wRw=
+github.com/go-openapi/swag/typeutils v0.25.4/go.mod h1:Ou7g//Wx8tTLS9vG0UmzfCsjZjKhpjxayRKTHXf2pTE=
+github.com/go-openapi/swag/yamlutils v0.25.4 h1:6jdaeSItEUb7ioS9lFoCZ65Cne1/RZtPBZ9A56h92Sw=
+github.com/go-openapi/swag/yamlutils v0.25.4/go.mod h1:MNzq1ulQu+yd8Kl7wPOut/YHAAU/H6hL91fF+E2RFwc=
+github.com/go-openapi/testify/enable/yaml/v2 v2.0.2 h1:0+Y41Pz1NkbTHz8NngxTuAXxEodtNSI1WG1c/m5Akw4=
+github.com/go-openapi/testify/enable/yaml/v2 v2.0.2/go.mod h1:kme83333GCtJQHXQ8UKX3IBZu6z8T5Dvy5+CW3NLUUg=
+github.com/go-openapi/testify/v2 v2.0.2 h1:X999g3jeLcoY8qctY/c/Z8iBHTbwLz7R2WXd6Ub6wls=
+github.com/go-openapi/testify/v2 v2.0.2/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54=
+github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
+github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
+github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo=
+github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY=
+github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
+github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
+github.com/hhatto/gorst v0.0.0-20181029133204-ca9f730cac5b h1:Jdu2tbAxkRouSILp2EbposIb8h4gO+2QuZEn3d9sKAc=
+github.com/hhatto/gorst v0.0.0-20181029133204-ca9f730cac5b/go.mod h1:HmaZGXHdSwQh1jnUlBGN2BeEYOHACLVGzYOXCbsLvxY=
+github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
+github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk=
+github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg=
+github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
+github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
+github.com/jdkato/prose v1.2.1 h1:Fp3UnJmLVISmlc57BgKUzdjr0lOtjqTZicL3PaYy6cU=
+github.com/jdkato/prose v1.2.1/go.mod h1:AiRHgVagnEx2JbQRQowVBKjG0bcs/vtkGCH1dYAL1rA=
+github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
+github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
+github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
+github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
+github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A=
+github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA=
+github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
+github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/montanaflynn/stats v0.6.3/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE=
+github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
+github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
+github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
+github.com/neurosnap/sentences v1.0.6 h1:iBVUivNtlwGkYsJblWV8GGVFmXzZzak907Ci8aA0VTE=
+github.com/neurosnap/sentences v1.0.6/go.mod h1:pg1IapvYpWCJJm/Etxeh0+gtMf1rI1STY9S7eUCPbDc=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
+github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
+github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
+github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
+github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
+github.com/shogo82148/go-shuffle v0.0.0-20180218125048-27e6095f230d/go.mod h1:2htx6lmL0NGLHlO8ZCf+lQBGBHIbEujyywxJArf+2Yc=
+github.com/shogo82148/go-shuffle v1.0.1 h1:4swIpHXLMAz14DE4YTgakgadpRN0n1wE1dieGnOTVFU=
+github.com/shogo82148/go-shuffle v1.0.1/go.mod h1:HQPjVgUUZ9TNgm4/K/iXRuAdhPsQrXnAGgtk/9kqbBY=
+github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
+github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0=
+github.com/xanzy/ssh-agent v0.3.2 h1:eKj4SX2Fe7mui28ZgnFW5fmTz1EIr7ugo5s6wDxdHBM=
+github.com/xanzy/ssh-agent v0.3.2/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
+go.mongodb.org/mongo-driver v1.17.6 h1:87JUG1wZfWsr6rIz3ZmpH90rL5tea7O3IHuSwHUpsss=
+go.mongodb.org/mongo-driver v1.17.6/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ=
+go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
+go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
+go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
+golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
+golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
+golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
+golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
+golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20221006183845-316c7553db56 h1:BrYbdKcCNjLyrN6aKqXy4hPw9qGI8IATkj4EWv9Q+kQ=
+golang.org/x/exp v0.0.0-20221006183845-316c7553db56/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
+golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
+golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk=
+golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
+golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
+golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
+golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220315194320-039c03cc5b86/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
+golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
+golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
+golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
+golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE=
+golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk=
+gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
+gonum.org/v1/gonum v0.8.2 h1:CCXrcPKiGGotvnN6jfUsKk4rRqm7q09/YbKb5xCEvtM=
+gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0=
+gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0 h1:OE9mWmgKkjJyEmDAAtGMPjXu+YNeGvK9VTSHY6+Qihc=
+gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
+gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/neurosnap/sentences.v1 v1.0.6/go.mod h1:YlK+SN+fLQZj+kY3r8DkGDhDr91+S3JmTb5LSxFRQo0=
+gopkg.in/neurosnap/sentences.v1 v1.0.7 h1:gpTUYnqthem4+o8kyTLiYIB05W+IvdQFYR29erfe8uU=
+gopkg.in/neurosnap/sentences.v1 v1.0.7/go.mod h1:YlK+SN+fLQZj+kY3r8DkGDhDr91+S3JmTb5LSxFRQo0=
+gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
+gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+modernc.org/cc/v4 v4.29.2 h1:h6+9ciCnPKutf4I03CvheAvDLX7+IHlqR6Iy6J+cgd8=
+modernc.org/cc/v4 v4.29.2/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
+modernc.org/ccgo/v4 v4.35.0 h1:F+TUsmw09QxLzmi3aeYYGxjAXarmZaKgj3mKQHNaA8w=
+modernc.org/ccgo/v4 v4.35.0/go.mod h1:qrVGs9S3Sr2Ztcg9ve+kTAYMp5a3YvWjo+SoN06kJ5I=
+modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=
+modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU=
+modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
+modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
+modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc=
+modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
+modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
+modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
+modernc.org/libc v1.75.7 h1:o3DTP9/0p9pKmY2WCKQaySW6wIiZhNM7wc2lUoyhfew=
+modernc.org/libc v1.75.7/go.mod h1:bO5o2ztHxBb2rjz0PgdHN0sSMw57CgxGFLZ3Qd/QpVQ=
+modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
+modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
+modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g=
+modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
+modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg=
+modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
+modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
+modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
+modernc.org/sqlite v1.59.0 h1:X1es1GpqBlS/5T+vbM4HLUdaa8OtQx468DF2vrx+38A=
+modernc.org/sqlite v1.59.0/go.mod h1:+paeT2A3iPRHkQDwG7oA6Tk0zQd5woMEI8q7orfry8k=
+modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
+modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
+modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
+modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
+rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
--- a/gomod2nix.toml
+++ b/gomod2nix.toml
@ -0,0 +1,270 @@
+schema = 3
+
+[mod]
+  [mod.'codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3']
+    version = 'v3.0.0'
+    hash = 'sha256-/fb4qz0lZt7vuxOELzthAiUy1oDC5x/PYCch3FUNWig='
+
+  [mod.'github.com/42wim/httpsig']
+    version = 'v1.2.3'
+    hash = 'sha256-Yrf21S5q6mupTtyk8izVKQciJC/aRDe7YRKL0cBkRpQ='
+
+  [mod.'github.com/Microsoft/go-winio']
+    version = 'v0.6.0'
+    hash = 'sha256-TQ0AvXZfuT3PDnb4p/h5zmyT8m9VPcwxlXMJuVD6ltE='
+
+  [mod.'github.com/ProtonMail/go-crypto']
+    version = 'v0.0.0-20220930113650-c6815a8c17ad'
+    hash = 'sha256-mem+jc60bsyLIJblnHbnG5e3+KFWf2TfZo9In9viYwU='
+
+  [mod.'github.com/acomagu/bufpipe']
+    version = 'v1.0.3'
+    hash = 'sha256-hIQf2lqZZv4dN8JwRyhNoBjStL9Mgih6f7Hy8RDU5Kg='
+
+  [mod.'github.com/cloudflare/circl']
+    version = 'v1.2.0'
+    hash = 'sha256-9L1/4M8Za4qcrtti/1r1E7TgpdfUKHM+9IyuX3fuJyQ='
+
+  [mod.'github.com/davidmz/go-pageant']
+    version = 'v1.0.2'
+    hash = 'sha256-gfwm6i0ZGQhX2c0lDRe7qVXyHkhoHeemYpaj+O0aYyw='
+
+  [mod.'github.com/dgryski/go-minhash']
+    version = 'v0.0.0-20190315135803-ad340ca03076'
+    hash = 'sha256-CYfWI07pOkDyBgoHRUSY9QzOR/cWNLIa08M+0khbKX4='
+
+  [mod.'github.com/dustin/go-humanize']
+    version = 'v1.0.1'
+    hash = 'sha256-yuvxYYngpfVkUg9yAmG99IUVmADTQA0tMbBXe0Fq0Mc='
+
+  [mod.'github.com/ekzhu/minhash-lsh']
+    version = 'v0.0.0-20190924033628-faac2c6342f8'
+    hash = 'sha256-vZH3kLGjMEuDNiWHpTVXv1z0ZMYi40zai8DlOk7ivZM='
+
+  [mod.'github.com/emirpasic/gods']
+    version = 'v1.18.1'
+    hash = 'sha256-hGDKddjLj+5dn2woHtXKUdd49/3xdsqnhx7VEdCu1m4='
+
+  [mod.'github.com/go-enry/go-license-detector/v4']
+    version = 'v4.3.1'
+    hash = 'sha256-S9LKXjn5dL5FETOOAk+bs7bIVdu2x7MIhfjpZuXzuLo='
+
+  [mod.'github.com/go-fed/httpsig']
+    version = 'v1.1.0'
+    hash = 'sha256-ok0PEEvSR010nh6vB3nDCyKSSVvoB50/UjVvBKOYXsA='
+
+  [mod.'github.com/go-git/gcfg']
+    version = 'v1.5.0'
+    hash = 'sha256-A62eSLI/0y4sfwCsZhe/uoSC9Z3TryyW+PyWIcknYdE='
+
+  [mod.'github.com/go-git/go-billy/v5']
+    version = 'v5.3.1'
+    hash = 'sha256-UdShPsStI2OBJig8Zv4Htyv5NiwKnYI41pwe4hIxv3Q='
+
+  [mod.'github.com/go-git/go-git/v5']
+    version = 'v5.4.2'
+    hash = 'sha256-+RvDli5tPRLEs1pA9W/ZqCCm896y97ObTqLp3hLydpo='
+
+  [mod.'github.com/go-openapi/errors']
+    version = 'v0.22.6'
+    hash = 'sha256-JcWNO3VO/OxaMptpMuFs6pYD7xFDtC5H8vn1EiF9w20='
+
+  [mod.'github.com/go-openapi/strfmt']
+    version = 'v0.25.0'
+    hash = 'sha256-xz4aMKsHsDhUPVc27hBa4FrywKQABhLhEBNg3qClL9Y='
+
+  [mod.'github.com/go-openapi/swag']
+    version = 'v0.25.4'
+    hash = 'sha256-aFQeLf95JC5gWLwIJXxhZHK9RxiGhh9IYV0MEYldDr4='
+
+  [mod.'github.com/go-openapi/swag/cmdutils']
+    version = 'v0.25.4'
+    hash = 'sha256-nhlq5w/ULjwy7nptKcRryjZPlv3QG0FmgcBSOclqRlA='
+
+  [mod.'github.com/go-openapi/swag/conv']
+    version = 'v0.25.4'
+    hash = 'sha256-uHgTdZC76LMSsq+x+RyeclnOgBsS0ID2cLgfjzFk0KQ='
+
+  [mod.'github.com/go-openapi/swag/fileutils']
+    version = 'v0.25.4'
+    hash = 'sha256-NWacswHRvNZ81WzazA8yUmeDymtne2l8zLdArBmeYAU='
+
+  [mod.'github.com/go-openapi/swag/jsonname']
+    version = 'v0.25.4'
+    hash = 'sha256-Z6uETeudh8W+/SGxFBnOB/VlJeRPRkFgCyJmntpz7bc='
+
+  [mod.'github.com/go-openapi/swag/jsonutils']
+    version = 'v0.25.4'
+    hash = 'sha256-TQGeMImUuL5BWDoBHKAjZ1BvvxPXkFrV3TMgb82IrLk='
+
+  [mod.'github.com/go-openapi/swag/loading']
+    version = 'v0.25.4'
+    hash = 'sha256-xr8OnFqB/kwBj3yf9HiLJwDcpL7I3/qJYlKb6VWbVFA='
+
+  [mod.'github.com/go-openapi/swag/mangling']
+    version = 'v0.25.4'
+    hash = 'sha256-NrmwNgRJIUlOaAGt+XutHWSHy9e1COMnVE8B0UY5+sQ='
+
+  [mod.'github.com/go-openapi/swag/netutils']
+    version = 'v0.25.4'
+    hash = 'sha256-wRMCQX/wIOvWm1g2UURobhzI47OldVqzlYYzwotLNTU='
+
+  [mod.'github.com/go-openapi/swag/stringutils']
+    version = 'v0.25.4'
+    hash = 'sha256-rVo5NBH+oLLX9kaemUYWILd+TKw/TDTB8UdSz+mn3m8='
+
+  [mod.'github.com/go-openapi/swag/typeutils']
+    version = 'v0.25.4'
+    hash = 'sha256-59RHnK6ugsAUc+A8DZCj7gZ3bJnt01Al7T/1kzM9PpA='
+
+  [mod.'github.com/go-openapi/swag/yamlutils']
+    version = 'v0.25.4'
+    hash = 'sha256-zba7QX7Ds05oZq1opP/vwRSBEDaJ1pKm+9DgrsOuR9w='
+
+  [mod.'github.com/go-viper/mapstructure/v2']
+    version = 'v2.4.0'
+    hash = 'sha256-lLfcV9z4n94hDhgyXJlde4bFB0hfzlbh+polqcJCwGE='
+
+  [mod.'github.com/google/uuid']
+    version = 'v1.6.0'
+    hash = 'sha256-VWl9sqUzdOuhW0KzQlv0gwwUQClYkmZwSydHG2sALYw='
+
+  [mod.'github.com/hashicorp/go-version']
+    version = 'v1.7.0'
+    hash = 'sha256-Umc/Q2mxRrPF4aEcDuDJq4lFBT+PSsnyANfyFwUIaxI='
+
+  [mod.'github.com/hhatto/gorst']
+    version = 'v0.0.0-20181029133204-ca9f730cac5b'
+    hash = 'sha256-u1rB/Z20x3UYMKaLjlDP3bHnD7nMrVs5AHR48km/H1k='
+
+  [mod.'github.com/imdario/mergo']
+    version = 'v0.3.13'
+    hash = 'sha256-03LKAZXgR5YUMeK5IRh2ds2GyV6uSbdq5QCPbAzlTus='
+
+  [mod.'github.com/jbenet/go-context']
+    version = 'v0.0.0-20150711004518-d14ea06fba99'
+    hash = 'sha256-VANNCWNNpARH/ILQV9sCQsBWgyL2iFT+4AHZREpxIWE='
+
+  [mod.'github.com/jdkato/prose']
+    version = 'v1.2.1'
+    hash = 'sha256-xjGGiunKj2XUGuUMxdKrObOUAoV1QUmwkPVhyjLc3S8='
+
+  [mod.'github.com/kevinburke/ssh_config']
+    version = 'v1.2.0'
+    hash = 'sha256-Ta7ZOmyX8gG5tzWbY2oES70EJPfI90U7CIJS9EAce0s='
+
+  [mod.'github.com/mattn/go-isatty']
+    version = 'v0.0.24'
+    hash = 'sha256-pmq8KKIb2+qMF3E1cR2B7NAj5joF2tZduyXjIfIlkTk='
+
+  [mod.'github.com/mitchellh/go-homedir']
+    version = 'v1.1.0'
+    hash = 'sha256-oduBKXHAQG8X6aqLEpqZHs5DOKe84u6WkBwi4W6cv3k='
+
+  [mod.'github.com/montanaflynn/stats']
+    version = 'v0.7.1'
+    hash = 'sha256-0QXUA/syOtDSBooh3isAffrVAjpSoBmSMMuZfO5maHg='
+
+  [mod.'github.com/ncruces/go-strftime']
+    version = 'v1.0.0'
+    hash = 'sha256-GYIwYDONuv/yTE0AEugCHQbtV3oiBaco93xUNYFcVBQ='
+
+  [mod.'github.com/oklog/ulid']
+    version = 'v1.3.1'
+    hash = 'sha256-LNn883rYNiaoY9sGEPIzlMRx5UwGThdYTjXqfzeGc9k='
+
+  [mod.'github.com/pkg/errors']
+    version = 'v0.9.1'
+    hash = 'sha256-mNfQtcrQmu3sNg/7IwiieKWOgFQOVVe2yXgKBpe/wZw='
+
+  [mod.'github.com/remyoudompheng/bigfft']
+    version = 'v0.0.0-20230129092748-24d4a6f8daec'
+    hash = 'sha256-vYmpyCE37eBYP/navhaLV4oX4/nu0Z/StAocLIFqrmM='
+
+  [mod.'github.com/russross/blackfriday/v2']
+    version = 'v2.1.0'
+    hash = 'sha256-R+84l1si8az5yDqd5CYcFrTyNZ1eSYlpXKq6nFt4OTQ='
+
+  [mod.'github.com/sergi/go-diff']
+    version = 'v1.2.0'
+    hash = 'sha256-d2higuBRee4ylRuCpPQV5+g0XK5yNQgDpudNor0qD2o='
+
+  [mod.'github.com/shogo82148/go-shuffle']
+    version = 'v1.0.1'
+    hash = 'sha256-E5Xt69A2vXlVPzyHPyWK0FyQ+IaMZzqQCnUltedeZhg='
+
+  [mod.'github.com/xanzy/ssh-agent']
+    version = 'v0.3.2'
+    hash = 'sha256-QavRCX75546hnWCOtBwiUdmyk6AoLjv3POoJXng5kMI='
+
+  [mod.'go.mongodb.org/mongo-driver']
+    version = 'v1.17.6'
+    hash = 'sha256-B1ePaR77HKDksfG6PLwOmOVPYac9g13i8Ld/xWOCIiE='
+
+  [mod.'go.yaml.in/yaml/v3']
+    version = 'v3.0.4'
+    hash = 'sha256-NkGFiDPoCxbr3LFsI6OCygjjkY0rdmg5ggvVVwpyDQ4='
+
+  [mod.'go.yaml.in/yaml/v4']
+    version = 'v4.0.0-rc.6'
+    hash = 'sha256-9aOIQpQ53KLv/Hb14104qYKLVrZUcCGLwzYp6UGLP90='
+
+  [mod.'golang.org/x/crypto']
+    version = 'v0.54.0'
+    hash = 'sha256-m8jsqlPuoPxGDjClzcAVbXf1aiU07L+9j7vYcqJu/0Q='
+
+  [mod.'golang.org/x/exp']
+    version = 'v0.0.0-20221006183845-316c7553db56'
+    hash = 'sha256-2aATMtazbT3i+X3m0Ju54GKOggcyeX9dAJFxO+H5IxQ='
+
+  [mod.'golang.org/x/mod']
+    version = 'v0.38.0'
+    hash = 'sha256-BpKfvsmb7ecEDvTBKx9ZHauxoWRkOAZCqg4BrHM2/S0='
+
+  [mod.'golang.org/x/net']
+    version = 'v0.57.0'
+    hash = 'sha256-gKo8UMw4hfETBHm8N5GOfuNadse9TWEQXJo2YWq5bY4='
+
+  [mod.'golang.org/x/sync']
+    version = 'v0.22.0'
+    hash = 'sha256-VZjl0fAM0p/nI81zh+pdBjzxFupx0UcKYXBBpL2ZS7k='
+
+  [mod.'golang.org/x/sys']
+    version = 'v0.47.0'
+    hash = 'sha256-TpbRyWWqHjddP6QzUgAbaLd2EE0S+GYNRUIDJd18r98='
+
+  [mod.'golang.org/x/text']
+    version = 'v0.40.0'
+    hash = 'sha256-LJfnki46XEreGbSgjl+DeqgcTsINTOu2owyXNvijMcA='
+
+  [mod.'golang.org/x/tools']
+    version = 'v0.48.0'
+    hash = 'sha256-9cRNUaup6fexA5S1zc+Ic3aaoxAgKWntyMJ2xaOdiP8='
+
+  [mod.'gonum.org/v1/gonum']
+    version = 'v0.8.2'
+    hash = 'sha256-eFUTetKRBa9QzDUEtd6aU8KsTJQgBF6zLAveY4Ybi58='
+
+  [mod.'gopkg.in/neurosnap/sentences.v1']
+    version = 'v1.0.7'
+    hash = 'sha256-AaBiqQ5Mtrzk4dE0lZidyUtDzYVy7aonEeh5oXrq/VI='
+
+  [mod.'gopkg.in/warnings.v0']
+    version = 'v0.1.2'
+    hash = 'sha256-ATVL9yEmgYbkJ1DkltDGRn/auGAjqGOfjQyBYyUo8s8='
+
+  [mod.'modernc.org/libc']
+    version = 'v1.75.7'
+    hash = 'sha256-Iq3Pcr3RrbhC82RWDM1I4gyKh3V/HI1gVQzoGR+0YYo='
+
+  [mod.'modernc.org/mathutil']
+    version = 'v1.7.1'
+    hash = 'sha256-COZ5rF2GhQVR1r6a0DanJ8qwQ94JSKdQxTMWrDzE0Cc='
+
+  [mod.'modernc.org/memory']
+    version = 'v1.12.1'
+    hash = 'sha256-Y/kW9/A5uF+Y66kDbxrh4muFgE3CEee6YzSl2FMPcRg='
+
+  [mod.'modernc.org/sqlite']
+    version = 'v1.59.0'
+    hash = 'sha256-CaNi9UvS9y6aiiSy3GUlRHX84og+i4MWOZmxbFONeb4='
--- a/main.go
+++ b/main.go
@ -0,0 +1,61 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"go.yaml.in/yaml/v4"
+
+	"git.xengi.de/ffd/indexer/scraper"
+	"git.xengi.de/ffd/indexer/store"
+)
+
+func main() {
+	data, err := os.ReadFile("config.yaml")
+	if err != nil {
+		panic(err)
+	}
+	var config Config
+
+	if err := yaml.Unmarshal(data, &config); err != nil {
+		panic(err)
+	}
+
+	db, err := store.Open(config.DbPath)
+	if err != nil {
+		panic(err)
+	}
+	defer func() { _ = db.Close() }()
+
+	for _, forge := range config.Forges {
+		fmt.Printf("Scraping %s (%s, %s)\n", forge.Name, forge.Type, forge.Url)
+
+		s, err := scraper.NewScraper(forge.Type, forge.Url)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "  skip %s: %v\n", forge.Name, err)
+			continue
+		}
+
+		projects, err := s.GetProjects()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "  skip %s: %v\n", forge.Name, err)
+			continue
+		}
+
+		fmt.Printf("  found %d projects\n", len(projects))
+		for _, p := range projects {
+			if err := db.Upsert(p); err != nil {
+				fmt.Fprintf(os.Stderr, "    store %s: %v\n", p.Name, err)
+				continue
+			}
+			fmt.Printf("    %s\n", p.Name)
+		}
+
+		all, err := db.All()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "  count: %v\n", err)
+			continue
+		}
+		fmt.Printf("  total in db: %d\n", len(all))
+	}
+}
--- a/option.nix
+++ b/option.nix
@ -0,0 +1,57 @@
+{
+  config,
+  lib,
+  pkgs,
+  options,
+  ...
+}:
+
+let
+  cfg = config.services.federated-forge-discovery.indexer;
+  configFile = pkgs.writeText "config.ini" (pkgs.lib.generators.toINI { } cfg);
+  execCommand = "${cfg.package}/bin/ffd-indexer -c '${configFile}'";
+in
+{
+  options.services.federated-forge-discovery.indexer = {
+    enable = lib.mkEnableOption "Enables the Federated Forge Discovery Indexer systemd service.";
+    openFirewall = lib.mkOption {
+
+    };
+    package = lib.mkOption {
+      description = "Package to use.";
+      type = lib.types.package;
+      default = lib.literalExpression ''
+        pkgs.ffd-indexer
+      '';
+    };
+    port = lib.mkOption {
+      description = "Port to listen for search engines";
+      type = lib.types.port;
+      default = 47810;
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    systemd.services."ffd-indexer" = {
+      description = "Federated Forge Discovery Indexer";
+      wants = [ "network-online.target" ];
+      after = [ "network-online.target" ];
+      serviceConfig = {
+        Restart = "always";
+        RestartSec = 30;
+        ExecStart = execCommand;
+        User = "ffd-indexer";
+        Group = "ffd-indexer";
+        AmbientCapabilities = lib.mkIf (cfg.port < 1000) [ "CAP_NET_BIND_SERVICE" ];
+        CapabilityBoundingSet = lib.mkIf (cfg.port < 1000) [ "CAP_NET_BIND_SERVICE" ];
+        NoNewPrivileges = true;
+      };
+      wantedBy = [ "multi-user.target" ];
+    };
+  };
+
+  #meta = {
+  #  maintainers = with lib.maintainers; [ xengi ];
+  #  doc = ./default.xml;
+  #};
+}
--- a/scraper/forgejo.go
+++ b/scraper/forgejo.go
@ -0,0 +1,230 @@
+// Package scraper forgejo.go - Forgejo scraper.
+package scraper
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+
+	forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3"
+
+	"github.com/go-enry/go-license-detector/v4/licensedb/filer"
+)
+
+// ForgejoScraper retrieves public repositories from a Forgejo instance.
+type ForgejoScraper struct {
+	Client   *forgejo.Client
+	BaseURL  string
+	PageSize int
+}
+
+// NewForgejoScraper creates a scraper for a Forgejo instance.
+// BaseURL should be the instance URL, e.g. https://codeberg.org.
+func NewForgejoScraper(baseURL string) (*ForgejoScraper, error) {
+	client, err := forgejo.NewClient(baseURL)
+	if err != nil {
+		return nil, fmt.Errorf("create Forgejo client: %w", err)
+	}
+	client.SetUserAgent(UserAgent)
+
+	return &ForgejoScraper{
+		Client:   client,
+		BaseURL:  strings.TrimSuffix(baseURL, "/"),
+		PageSize: 50,
+	}, nil
+}
+
+// GetProjects retrieves every public repository visible through the
+// Forgejo repository search API and maps it to the generic Project type.
+func (s *ForgejoScraper) GetProjects() ([]Project, error) {
+	// Honor the instance's robots.txt before making any API call:
+	// if /api/ is disallowed, do not scrape this instance.
+	ok, err := Allowed(s.BaseURL)
+	if err != nil {
+		return nil, fmt.Errorf("check robots.txt: %w", err)
+	}
+	if !ok {
+		return nil, fmt.Errorf("instance robots.txt disallows /api/")
+	}
+
+	pageSize := s.PageSize
+	if pageSize <= 0 {
+		pageSize = 50
+	}
+
+	isPrivate := false
+	var projects []Project
+
+	for page := 1; ; page++ {
+		repos, _, err := s.Client.SearchRepos(forgejo.SearchRepoOptions{
+			Page:      page,
+			PageSize:  pageSize,
+			IsPrivate: &isPrivate,
+			Type:      forgejo.RepoTypeSource,
+			Sort:      "alpha",
+			Order:     "asc",
+		})
+		if err != nil {
+			return nil, fmt.Errorf(
+				"list public Forgejo repositories (page %d): %w",
+				page,
+				err,
+			)
+		}
+
+		if len(repos) == 0 {
+			break
+		}
+
+		for _, repo := range repos {
+			if repo == nil {
+				continue
+			}
+
+			owner, name, ok := strings.Cut(repo.FullName, "/")
+			if !ok {
+				continue
+			}
+
+			// List the repository root once and reuse it for readme selection
+			// and license detection (they previously each listed the root).
+			entries, _, err := s.Client.ListContents(owner, name, repo.DefaultBranch, "")
+			if err != nil {
+				// best-effort: fall back to empty root so the other fields still work
+				entries = nil
+			}
+			root := make([]filer.File, 0, len(entries))
+			for _, e := range entries {
+				if e == nil {
+					continue
+				}
+				root = append(root, filer.File{Name: e.Name, IsDir: e.Type == "dir"})
+			}
+
+			// topics currently requires one extra HTTP call per repository.
+			// Future optimization: the /repos/search response already includes
+			// the topics field, so they could be gathered by parsing the
+			// SearchRepos payload itself (e.g. via ListRepos) instead.
+			// On failure the repo is kept in the results with empty topics.
+			topics, err := s.topics(owner, name)
+			if err != nil {
+				topics = nil
+			}
+
+			// languages requires one extra HTTP call per repository
+			// (GET /repos/{owner}/{repo}/languages, via GetRepoLanguages).
+			// It returns a map[language]bytes; we order it most-dominant-first
+			// below. On failure the repo is kept with no languages.
+			langs, _, err := s.Client.GetRepoLanguages(owner, name)
+			if err != nil {
+				langs = nil
+			}
+
+			// Latest commit: one HTTP call per repository
+			// (GET /repos/{owner}/{repo}/commits?limit=1, via ListRepoCommits), returns the
+			// newest commit first. SHA + date are mapped into LatestCommit.
+			latestCommit, _ := s.latestCommit(owner, name)
+
+			// Latest release: one HTTP call per repository
+			// (GET /repos/{owner}/{repo}/releases/latest, via GetLatestRelease).
+			// Returns a single Release (server already excludes drafts and prereleases).
+			latestRelease, _ := s.latestRelease(owner, name)
+
+			// Licenses: scan repo files on its default branch with
+			// go-license-detector, reusing the root listing. May require reads
+			// of candidate license files. On failure the repo is kept with no
+			// licenses.
+			licenses := s.licensesFor(owner, name, repo.DefaultBranch, root)
+
+			// Readme: pick the default README (md/rst/txt/no extension) from the
+			// shared root listing and fetch it. Non-fatal when absent.
+			readme := s.readmeFor(owner, name, repo.DefaultBranch, root)
+
+			projects = append(projects, Project{
+				Name:          repo.FullName,
+				Url:           repo.HTMLURL,
+				Description:   repo.Description,
+				OpenIssues:    repo.OpenIssues,
+				OpenPRs:       repo.OpenPulls,
+				Topics:        topics,
+				Languages:     orderedLanguages(langs),
+				LatestCommit:  latestCommit,
+				LatestRelease: latestRelease,
+				Licenses:      licenses,
+				Readme:        readme,
+			})
+		}
+
+		// A short page indicates that there are no further results.
+		if len(repos) < pageSize {
+			break
+		}
+	}
+
+	return projects, nil
+}
+
+// topics returns the repository's topics.
+func (s *ForgejoScraper) topics(owner, name string) ([]string, error) {
+	topics, _, err := s.Client.ListRepoTopics(owner, name, forgejo.ListRepoTopicsOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("list topics for %s/%s: %w", owner, name, err)
+	}
+	return topics, nil
+}
+
+// latestCommit returns the most recent commit's time. On failure it returns
+// the zero time and the error, leaving the Project's LatestCommit unset.
+func (s *ForgejoScraper) latestCommit(owner, name string) (time.Time, error) {
+	commits, _, err := s.Client.ListRepoCommits(owner, name, forgejo.ListCommitOptions{
+		PageSize: 1,
+	})
+	if err != nil {
+		return time.Time{}, err
+	}
+	if len(commits) == 0 || commits[0] == nil {
+		return time.Time{}, nil
+	}
+	return commits[0].Created, nil
+}
+
+// latestRelease returns the most recent non-draft, non-prerelease release.
+// On failure (e.g. no releases) it returns a zero Release and the error,
+// leaving the Project's LatestRelease unset.
+func (s *ForgejoScraper) latestRelease(owner, name string) (Release, error) {
+	r, _, err := s.Client.GetLatestRelease(owner, name)
+	if err != nil {
+		return Release{}, err
+	}
+	if r == nil {
+		return Release{}, nil
+	}
+	return Release{
+		Name: r.Title,
+		Date: r.PublishedAt,
+	}, nil
+}
+
+// orderedLanguages returns the languages of a repo ordered by byte share,
+// most dominant first. Ties are broken alphabetically for stable output.
+func orderedLanguages(langs map[string]int64) []string {
+	if len(langs) == 0 {
+		return nil
+	}
+
+	names := make([]string, 0, len(langs))
+	for name := range langs {
+		names = append(names, name)
+	}
+
+	// Descending byte count; alphabetically as a stable tie-breaker.
+	sort.Slice(names, func(i, j int) bool {
+		if langs[names[i]] != langs[names[j]] {
+			return langs[names[i]] > langs[names[j]]
+		}
+		return names[i] < names[j]
+	})
+
+	return names
+}
--- a/scraper/forgejo_licenses.go
+++ b/scraper/forgejo_licenses.go
@ -0,0 +1,96 @@
+package scraper
+
+import (
+	"fmt"
+
+	forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3"
+
+	"github.com/go-enry/go-license-detector/v4/licensedb"
+	"github.com/go-enry/go-license-detector/v4/licensedb/filer"
+)
+
+// forgejoFiler adapts the Forgejo content API to the filer.Filer interface so
+// that go-license-detector can inspect a repository's files without a local
+// checkout. ReadDir serves a cached root listing when available (to avoid an
+// extra ListContents call) and falls back to the API otherwise. ReadFile
+// downloads a file's bytes at the given ref (default branch).
+type forgejoFiler struct {
+	client *forgejo.Client
+	owner  string
+	repo   string
+	ref    string
+	// cachedRoot, when non-nil, is served for ReadDir("") to avoid a second
+	// root listing that was already fetched by the caller.
+	cachedRoot []filer.File
+}
+
+var _ filer.Filer = (*forgejoFiler)(nil)
+
+func (f *forgejoFiler) ReadDir(dirPath string) ([]filer.File, error) {
+	if dirPath == "" && f.cachedRoot != nil {
+		return f.cachedRoot, nil
+	}
+	entries, _, err := f.client.ListContents(f.owner, f.repo, f.ref, dirPath)
+	if err != nil {
+		return nil, fmt.Errorf("list contents %q: %w", dirPath, err)
+	}
+	files := make([]filer.File, 0, len(entries))
+	for _, e := range entries {
+		if e == nil {
+			continue
+		}
+		files = append(files, filer.File{
+			Name:  e.Name,
+			IsDir: e.Type == "dir",
+		})
+	}
+	return files, nil
+}
+
+func (f *forgejoFiler) ReadFile(path string) ([]byte, error) {
+	// GetFile returns raw file bytes (binary-safe, base64-decoded).
+	data, _, err := f.client.GetFile(f.owner, f.repo, f.ref, path)
+	if err != nil {
+		return nil, fmt.Errorf("read file %q: %w", path, err)
+	}
+	return data, nil
+}
+
+func (f *forgejoFiler) Close() {}
+
+func (f *forgejoFiler) PathsAreAlwaysSlash() bool { return true }
+
+// licensesFor returns the SPDX identifiers detected for the repository. It
+// uses go-license-detector over the repository's files on its default branch.
+// The given root listing is served to the detector to avoid a duplicate,
+// already-available API call. An empty slice is returned when no license can
+// be determined (the repo is still kept in results -- license failures are
+// non-fatal).
+func (s *ForgejoScraper) licensesFor(owner, repo, ref string, root []filer.File) []string {
+	detector := &forgejoFiler{
+		client:     s.Client,
+		owner:      owner,
+		repo:       repo,
+		ref:        ref,
+		cachedRoot: root,
+	}
+
+	matches, err := licensedb.Detect(detector)
+	if err != nil {
+		return nil
+	}
+
+	// Keep only the highest-confidence match, sorted deterministically.
+	ids := make([]string, 0, len(matches))
+	bestID, bestConf := "", float32(-1)
+	for id, m := range matches {
+		if m.Confidence > bestConf {
+			bestConf = m.Confidence
+			bestID = id
+		}
+	}
+	if bestID != "" {
+		ids = append(ids, bestID)
+	}
+	return ids
+}
--- a/scraper/forgejo_readme.go
+++ b/scraper/forgejo_readme.go
@ -0,0 +1,70 @@
+package scraper
+
+import (
+	"sort"
+	"strings"
+
+	"github.com/go-enry/go-license-detector/v4/licensedb/filer"
+)
+
+// readmeCandidates are the filenames looked for, in priority order, when
+// retrieving a repository's README.
+var readmeCandidates = []string{"README.md", "README.rst", "README.txt", "README"}
+
+// readmeFor returns the content of the repository's README on the given ref
+// (default branch), picking from root (the pre-fetched root file listing) the
+// first candidate found (case-insensitively). An empty string is returned when
+// no README can be found; the repo is still kept in results either way.
+func (s *ForgejoScraper) readmeFor(owner, name, ref string, root []filer.File) string {
+	// Collect root file names, lowercased for case-insensitive matching, while
+	// keeping the real name for the GetFile call.
+	type rootFile struct {
+		real string
+		low  string
+	}
+	files := make([]rootFile, 0, len(root))
+	for _, e := range root {
+		if e.IsDir {
+			continue
+		}
+		files = append(files, rootFile{real: e.Name, low: strings.ToLower(e.Name)})
+	}
+
+	// Candidate priority wins; ties broken by alphabetical real name for
+	// deterministic picks when a repo has e.g. both README and README.md.
+	sort.SliceStable(files, func(i, j int) bool {
+		pi, pj := readmePriority(files[i].low), readmePriority(files[j].low)
+		if pi != pj {
+			return pi < pj // lower = higher priority
+		}
+		return files[i].real < files[j].real
+	})
+
+	var chosen string
+	for i := range files {
+		if readmePriority(files[i].low) >= 0 {
+			chosen = files[i].real
+			break
+		}
+	}
+	if chosen == "" {
+		return ""
+	}
+
+	data, _, err := s.Client.GetFile(owner, name, ref, chosen)
+	if err != nil {
+		return ""
+	}
+	return string(data)
+}
+
+// readmePriority returns the index of a lowercased filename in
+// readmeCandidates, or -1 if it is not a readme candidate.
+func readmePriority(low string) int {
+	for i, c := range readmeCandidates {
+		if low == strings.ToLower(c) {
+			return i
+		}
+	}
+	return -1
+}
--- a/scraper/robots.go
+++ b/scraper/robots.go
@ -0,0 +1,157 @@
+package scraper
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+)
+
+// UserAgent identifies this crawler to servers. Keep it stable so that
+// instances can set robots rules for us specifically.
+const UserAgent = "ffd-indexer/0.1"
+
+// robotsHTTPClient is used for fetching robots.txt. It does not follow the
+// SDK client's settings; it is a plain, short-timeout client.
+var robotsHTTPClient = &http.Client{Timeout: 10 * time.Second}
+
+// Allowed reports whether an instance at baseURL permits crawling its API
+// according to its robots.txt. It only consults the rules for the
+// User-Agent "*"; a block of /api/ (the API root) disallows scraping.
+//
+// A failure to fetch or parse robots.txt is treated as an error; callers
+// decide whether that is fatal.
+func Allowed(baseURL string) (bool, error) {
+	u, err := url.Parse(baseURL)
+	if err != nil {
+		return false, fmt.Errorf("parse base URL: %w", err)
+	}
+	u.Path = "/robots.txt"
+
+	req, err := http.NewRequest(http.MethodGet, u.String(), nil)
+	if err != nil {
+		return false, fmt.Errorf("build robots request: %w", err)
+	}
+	req.Header.Set("User-Agent", UserAgent)
+
+	resp, err := robotsHTTPClient.Do(req)
+	if err != nil {
+		return false, fmt.Errorf("fetch robots.txt: %w", err)
+	}
+	defer func() {
+		_ = resp.Body.Close()
+	}()
+
+	// A 404 or similar means "no robots.txt": assume allowed.
+	if resp.StatusCode == http.StatusNotFound {
+		return true, nil
+	}
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return false, fmt.Errorf("robots.txt returned %s", resp.Status)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return false, fmt.Errorf("read robots.txt: %w", err)
+	}
+
+	return allowedFor(string(body), UserAgent), nil
+}
+
+// allowedFor parses a robots.txt body and reports whether a crawler with the
+// given user agent may crawl the path "/api/". It implements a reasonable
+// subset of the robots.txt standard:
+//
+//   - Only the "*" group and an explicit group matching userAgent are
+//     considered; later groups override earlier ones for the same agent.
+//   - Disallow entries are matched as path prefixes (a trailing "*" is
+//     stripped). No other wildcard/glob support.
+//   - An empty group (no rules) is "allow everything".
+func allowedFor(body, userAgent string) bool {
+	groups := parseRobotGroups(body)
+
+	// A group that explicitly names our user agent takes precedence over a
+	// "*" group, regardless of order. Otherwise fall back to "*".
+	var star *robotGroup
+	for _, g := range groups {
+		for _, a := range g.agents {
+			if strings.EqualFold(a, "*") {
+				star = g
+			} else if strings.EqualFold(a, userAgent) {
+				return !matchDisallow(g, "/api/")
+			}
+		}
+	}
+	if star != nil {
+		return !matchDisallow(star, "/api/")
+	}
+	return true
+}
+
+type robotGroup struct {
+	agents   []string
+	disallow []string
+}
+
+// parseRobotGroups splits a robots.txt body into agent groups.
+func parseRobotGroups(body string) []*robotGroup {
+	var groups []*robotGroup
+	var cur *robotGroup
+
+	for _, raw := range strings.Split(body, "\n") {
+		line := strings.TrimSpace(raw)
+		// Strip an inline comment (not part of a rule's result).
+		if i := strings.IndexByte(line, '#'); i >= 0 {
+			line = strings.TrimSpace(line[:i])
+		}
+		if line == "" {
+			continue
+		}
+
+		key, val, ok := strings.Cut(line, ":")
+		if !ok {
+			continue
+		}
+		key = strings.ToLower(strings.TrimSpace(key))
+		val = strings.TrimSpace(val)
+
+		switch key {
+		case "user-agent":
+			// A User-agent line starts a new group, unless the current
+			// group is still collecting agents (has no Disallow yet).
+			if cur == nil || len(cur.disallow) > 0 {
+				cur = &robotGroup{}
+				groups = append(groups, cur)
+			}
+			cur.agents = append(cur.agents, val)
+		case "disallow":
+			if cur != nil {
+				cur.disallow = append(cur.disallow, val)
+			}
+		}
+	}
+
+	return groups
+}
+
+// matchDisallow reports whether a path is disallowed by a group's rules.
+// A trailing "*" is stripped. Rules are applied in order, later rules
+// overriding earlier ones: an empty Disallow rule revokes all preceding
+// disallows (allow everything).
+func matchDisallow(g *robotGroup, path string) bool {
+	disallowed := false
+	for _, rule := range g.disallow {
+		rule = strings.TrimSpace(rule)
+		rule = strings.TrimSuffix(rule, "*")
+		if rule == "" {
+			disallowed = false // empty Disallow: allow everything
+			continue
+		}
+		if strings.HasPrefix(path, rule) {
+			disallowed = true
+		}
+	}
+	return disallowed
+}
--- a/scraper/robots_test.go
+++ b/scraper/robots_test.go
@ -0,0 +1,60 @@
+package scraper
+
+import "testing"
+
+func TestAllowedFor(t *testing.T) {
+	const ua = "ffd-indexer/0.1"
+
+	// xengi / cccb: only /archive/ blocked -> /api/ allowed.
+	if !allowedFor("User-agent: *\nDisallow: /*/*/archive/\n", ua) {
+		t.Error("expected /api/ allowed when only archive is disallowed")
+	}
+
+	// codeberg: /api/ disallowed under "*".
+	codeberg := `User-agent: *
+Disallow: /api/
+Disallow: /avatars/
+`
+	if allowedFor(codeberg, ua) {
+		t.Error("expected /api/ blocked when /api/ is disallowed")
+	}
+
+	// Explicit agent group takes precedence over "*" (agent name matches
+	// UserAgent exactly, including the version).
+	// Here the explicit group is for "ffd-indexer" (no version), so it does
+	// not match "ffd-indexer/0.1"; the "*" group (blocks /api/) applies.
+	if allowedFor("User-agent: *\nDisallow: /api/\n\nUser-agent: ffd-indexer\nDisallow: /whatever/\n", ua) {
+		t.Error("expected unmatched explicit agent to fall back to * which blocks /api/")
+	}
+
+	// A matching explicit agent group takes precedence over "*".
+	if !allowedFor("User-agent: *\nDisallow: /api/\n\nUser-agent: ffd-indexer/0.1\nDisallow: /whatever/\n", ua) {
+		t.Error("expected explicit agent rule to permit /api/")
+	}
+
+	// No rules at all -> allow.
+	if !allowedFor("", ua) {
+		t.Error("expected empty robots to allow")
+	}
+
+	// Only an unrelated agent listed -> no star, no match -> allow.
+	if !allowedFor("User-agent: SomeBot\nDisallow: /api/\n", ua) {
+		t.Error("expected unrelated agent not to block us")
+	}
+
+	// Disallow "/" blocks everything including /api/.
+	if allowedFor("User-agent: *\nDisallow: /\n", ua) {
+		t.Error("expected Disallow / to block /api/")
+	}
+
+	// Empty Disallow (allowed) overrides a previous block in the same group.
+	if !allowedFor("User-agent: *\nDisallow: /api/\nDisallow:\n", ua) {
+		t.Error("expected empty Disallow to allow /api/")
+	}
+
+	// Wildcard on a non-prefix rule should not over-block (substring-only).
+	// "/*.pdf" should not match /api/.
+	if !allowedFor("User-agent: *\nDisallow: /*.pdf$\n", ua) {
+		t.Error("expected unrelated wildcard to not block /api/")
+	}
+}
--- a/scraper/scraper.go
+++ b/scraper/scraper.go
@ -0,0 +1,47 @@
+// Package scraper defines the interfaces and shared types used to discover
+// projects from forge instances (Forgejo, Gogs, ...).
+package scraper
+
+import (
+	"fmt"
+	"time"
+)
+
+// Scraper is implemented by concrete, per-forge scrapers.
+type Scraper interface {
+	// GetProjects retrieves every project discoverable through this forge's
+	// public API and maps it to the generic Project type.
+	GetProjects() ([]Project, error)
+}
+
+// NewScraper returns a Scraper for the given forge type, e.g. "forgejo".
+func NewScraper(typ, baseURL string) (Scraper, error) {
+	switch typ {
+	case "forgejo":
+		return NewForgejoScraper(baseURL)
+	default:
+		return nil, fmt.Errorf("unknown forge type %q", typ)
+	}
+}
+
+// Project is a generic, forge-agnostic representation of an open source project.
+type Project struct {
+	Name          string    `json:"name"`
+	Url           string    `json:"url"`
+	Description   string    `json:"description"`
+	Topics        []string  `json:"topics"`
+	OpenIssues    int       `json:"open_issues"`
+	OpenPRs       int       `json:"open_prs"`
+	LatestCommit  time.Time `json:"latest_commit"`
+	LatestRelease Release   `json:"latest_release"`
+	Languages     []string  `json:"languages"`
+	Readme        string    `json:"readme"`
+	Licenses      []string  `json:"license"`
+	UsesAI        bool      `json:"uses_ai"`
+}
+
+// Release is a named release with the date it was published.
+type Release struct {
+	Name string    `json:"name"`
+	Date time.Time `json:"date"`
+}
--- a/store/store.go
+++ b/store/store.go
@ -0,0 +1,190 @@
+// Package store provides durable persistence of scraped projects in SQLite.
+//
+// The database is the system of record: scrapers write discovered projects
+// here, and a search index (e.g. Typesense) can be built or rebuilt from it
+// without re-scraping the forges. Projects are stored as one row each, with
+// the URL as the natural key so re-running the scrape upserts in place.
+package store
+
+import (
+	"database/sql"
+	"encoding/json"
+	"fmt"
+	"time"
+
+	_ "modernc.org/sqlite"
+
+	"git.xengi.de/ffd/indexer/scraper"
+)
+
+const schema = `
+CREATE TABLE IF NOT EXISTS projects (
+	url                TEXT PRIMARY KEY,
+	name               TEXT NOT NULL,
+	description        TEXT NOT NULL DEFAULT '',
+	topics             TEXT NOT NULL DEFAULT '[]',   -- JSON array
+	languages          TEXT NOT NULL DEFAULT '[]',   -- JSON array
+	licenses           TEXT NOT NULL DEFAULT '[]',   -- JSON array
+	open_issues         INTEGER NOT NULL DEFAULT 0,
+	open_prs            INTEGER NOT NULL DEFAULT 0,
+	latest_commit       INTEGER NOT NULL DEFAULT 0,   -- unix epoch seconds (UTC)
+	latest_release_name  TEXT   NOT NULL DEFAULT '',
+	latest_release_date INTEGER NOT NULL DEFAULT 0,   -- unix epoch seconds (UTC)
+	readme              TEXT   NOT NULL DEFAULT '',
+	uses_ai             INTEGER NOT NULL DEFAULT 0,
+	scraped_at          INTEGER NOT NULL DEFAULT 0    -- unix epoch seconds (UTC)
+);
+`
+
+// Store wraps a SQLite database holding scraped projects.
+type Store struct {
+	db *sql.DB
+}
+
+// Open opens (creating if needed) the SQLite database at path.
+func Open(path string) (*Store, error) {
+	db, err := sql.Open("sqlite", path)
+	if err != nil {
+		return nil, fmt.Errorf("open sqlite db %q: %w", path, err)
+	}
+	if _, err := db.Exec(schema); err != nil {
+		_ = db.Close()
+		return nil, fmt.Errorf("create schema: %w", err)
+	}
+	return &Store{db: db}, nil
+}
+
+// Close closes the underlying database.
+func (s *Store) Close() error {
+	return s.db.Close()
+}
+
+const upsertSQL = `
+INSERT INTO projects (
+	url, name, description, topics, languages, licenses,
+	open_issues, open_prs, latest_commit,
+	latest_release_name, latest_release_date,
+	readme, uses_ai, scraped_at
+) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ON CONFLICT(url) DO UPDATE SET
+	name = excluded.name,
+	description = excluded.description,
+	topics = excluded.topics,
+	languages = excluded.languages,
+	licenses = excluded.licenses,
+	open_issues = excluded.open_issues,
+	open_prs = excluded.open_prs,
+	latest_commit = excluded.latest_commit,
+	latest_release_name = excluded.latest_release_name,
+	latest_release_date = excluded.latest_release_date,
+	readme = excluded.readme,
+	uses_ai = excluded.uses_ai,
+	scraped_at = excluded.scraped_at;
+`
+
+// Upsert inserts or updates a project, keyed on its URL.
+func (s *Store) Upsert(p scraper.Project) error {
+	topics, err := json.Marshal(p.Topics)
+	if err != nil {
+		return fmt.Errorf("marshal topics: %w", err)
+	}
+	langs, err := json.Marshal(p.Languages)
+	if err != nil {
+		return fmt.Errorf("marshal languages: %w", err)
+	}
+	licenses, err := json.Marshal(p.Licenses)
+	if err != nil {
+		return fmt.Errorf("marshal licenses: %w", err)
+	}
+
+	releaseDate := timeToUnix(p.LatestRelease.Date)
+	latestCommit := timeToUnix(p.LatestCommit)
+
+	_, err = s.db.Exec(upsertSQL,
+		p.Url,
+		p.Name,
+		p.Description,
+		string(topics),
+		string(langs),
+		string(licenses),
+		p.OpenIssues,
+		p.OpenPRs,
+		latestCommit,
+		p.LatestRelease.Name,
+		releaseDate,
+		p.Readme,
+		boolToInt(p.UsesAI),
+		time.Now().UTC().Unix(),
+	)
+	return err
+}
+
+// All returns every stored project.
+func (s *Store) All() ([]scraper.Project, error) {
+	rows, err := s.db.Query(`SELECT
+		url, name, description, topics, languages, licenses,
+		open_issues, open_prs, latest_commit,
+		latest_release_name, latest_release_date,
+		readme, uses_ai
+		FROM projects`)
+	if err != nil {
+		return nil, err
+	}
+	defer func() { _ = rows.Close() }()
+
+	var projects []scraper.Project
+	for rows.Next() {
+		var (
+			p            scraper.Project
+			topicsJSON   string
+			langsJSON    string
+			licensesJSON string
+			latestCommit int64
+			relName      string
+			relDate      int64
+			usesAI       int
+		)
+		if err := rows.Scan(
+			&p.Url, &p.Name, &p.Description,
+			&topicsJSON, &langsJSON, &licensesJSON,
+			&p.OpenIssues, &p.OpenPRs, &latestCommit,
+			&relName, &relDate,
+			&p.Readme, &usesAI,
+		); err != nil {
+			return nil, err
+		}
+
+		p.UsesAI = usesAI != 0
+		if err := json.Unmarshal([]byte(topicsJSON), &p.Topics); err != nil {
+			return nil, err
+		}
+		if err := json.Unmarshal([]byte(langsJSON), &p.Languages); err != nil {
+			return nil, err
+		}
+		if err := json.Unmarshal([]byte(licensesJSON), &p.Licenses); err != nil {
+			return nil, err
+		}
+		p.LatestCommit = time.Unix(latestCommit, 0).UTC()
+		p.LatestRelease.Name = relName
+		p.LatestRelease.Date = time.Unix(relDate, 0).UTC()
+
+		projects = append(projects, p)
+	}
+	return projects, rows.Err()
+}
+
+// timeToUnix returns the Unix epoch seconds for a time, or 0 for the zero
+// time (which is stored as a 0/absent value).
+func timeToUnix(t time.Time) int64 {
+	if t.IsZero() {
+		return 0
+	}
+	return t.Unix()
+}
+
+func boolToInt(b bool) int {
+	if b {
+		return 1
+	}
+	return 0
+}
Author	SHA1	Message	Date
Ricardo (XenGi) Band	bde13e9b68	scraper done; scrapes to local sqlite db	2026-09-28 15:36:42 +02:00
Ricardo (XenGi) Band	448552016b	initial dump	2026-09-28 12:21:17 +02:00