From 448552016bc4f2d0e4659e0dd358d1f9efa5ae81 Mon Sep 17 00:00:00 2001 From: "Ricardo (XenGi) Band" Date: Mon, 28 Sep 2026 12:21:17 +0200 Subject: [PATCH 1/2] initial dump --- .editorconfig | 11 +++ .gitignore | 204 +++++++++++++++++++++++++++++++++++++++++++++++--- flake.nix | 83 ++++++++++++++++++++ forgejo.go | 85 +++++++++++++++++++++ go.mod | 35 +++++++++ go.sum | 71 ++++++++++++++++++ main.go | 59 +++++++++++++++ option.nix | 57 ++++++++++++++ scraper.go | 23 ++++++ 9 files changed, 617 insertions(+), 11 deletions(-) create mode 100644 .editorconfig create mode 100644 flake.nix create mode 100644 forgejo.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go create mode 100644 option.nix create mode 100644 scraper.go diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..d5da92e --- /dev/null +++ b/.editorconfig @@ -0,0 +1,11 @@ +# EditorConfig is awesome: https://editorconfig.org +root = true + +[*] +end_of_line = lf +insert_final_newline = true + +[*.{go,nix,yaml}] +charset = utf-8 +indent_style = space +indent_size = 2 diff --git a/.gitignore b/.gitignore index 921d1fe..cead4e1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,15 @@ -# ---> Go +config.yaml +models.json +Containerfile + +# Created by https://www.toptal.com/developers/gitignore/api/windows,linux,macos,direnv,vim,go,goland+all +# Edit at https://www.toptal.com/developers/gitignore?templates=windows,linux,macos,direnv,vim,go,goland+all + +### direnv ### +.direnv +.envrc + +### Go ### # If you prefer the allow list template instead of the deny list, see community template: # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore # @@ -12,23 +23,194 @@ # Test binary, built with `go test -c` *.test -# Code coverage profiles and other test artifacts +# Output of the go coverage tool, specifically when used with LiteIDE *.out -coverage.* -*.coverprofile -profile.cov # Dependency directories (remove the comment below to include it) # vendor/ # Go workspace file go.work -go.work.sum -# env file -.env +### GoLand+all ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 -# Editor/IDE -# .idea/ -# .vscode/ +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf +# AWS User-specific +.idea/**/aws.xml + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# SonarLint plugin +.idea/sonarlint/ + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +### GoLand+all Patch ### +# Ignore everything but code style settings and run configurations +# that are supposed to be shared within teams. + +.idea/* + +!.idea/codeStyles +!.idea/runConfigurations + +### Linux ### +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +### macOS ### +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +### macOS Patch ### +# iCloud generated files +*.icloud + +### Vim ### +# Swap +[._]*.s[a-v][a-z] +!*.svg # comment out if you don't need vector files +[._]*.sw[a-p] +[._]s[a-rt-v][a-z] +[._]ss[a-gi-z] +[._]sw[a-p] + +# Session +Session.vim +Sessionx.vim + +# Temporary +.netrwhist +# Auto-generated tag files +tags +# Persistent undo +[._]*.un~ + +### Windows ### +# Windows thumbnail cache files +Thumbs.db +Thumbs.db:encryptable +ehthumbs.db +ehthumbs_vista.db + +# Dump file +*.stackdump + +# Folder config file +[Dd]esktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msix +*.msm +*.msp + +# Windows shortcuts +*.lnk + +# End of https://www.toptal.com/developers/gitignore/api/windows,linux,macos,direnv,vim,go,goland+all diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..50b30fd --- /dev/null +++ b/flake.nix @@ -0,0 +1,83 @@ +{ + description = "Federated Forge Discovery Indexer"; + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable-small"; + flake-utils.url = "github:numtide/flake-utils"; + gomod2nix = { + url = "github:tweag/gomod2nix"; + inputs.nixpkgs.follows = "nixpkgs"; + inputs.flake-utils.follows = "flake-utils"; + }; + }; + outputs = { self, nixpkgs, flake-utils, gomod2nix }: flake-utils.lib.eachDefaultSystem (system: + let + pkgs = import nixpkgs { + inherit system; + overlays = [ gomod2nix.overlays.default ]; + }; + ffd-indexer = pkgs.buildGoApplication { + pname = "ffd-indexer"; + version = "0.0.1"; + src = ./.; + modules = ./gomod2nix.toml; + }; + go-test = pkgs.stdenvNoCC.mkDerivation { + name = "go-test"; + dontBuild = true; + src = ./.; + doCheck = true; + nativeBuildInputs = with pkgs; [ + go + writableTmpDirAsHomeHook + ]; + checkPhase = '' + go test -v ./... + ''; + installPhase = '' + mkdir "$out" + ''; + }; + go-lint = pkgs.stdenvNoCC.mkDerivation { + name = "go-lint"; + dontBuild = true; + src = ./.; + doCheck = true; + nativeBuildInputs = with pkgs; [ + golangci-lint + go + writableTmpDirAsHomeHook + ]; + checkPhase = '' + golangci-lint run + ''; + installPhase = '' + mkdir "$out" + ''; + }; + in + { + checks = { inherit go-test go-lint; }; + formatter = pkgs.nixfmt-tree; + devShells.default = pkgs.mkShell { + buildInputs = with pkgs; [ + go + go-tools # staticcheck + gomod2nix.packages.${system}.default + ]; + packages = with pkgs; [ + mkcert + ]; + }; + packages = { + default = ffd-indexer; + container = pkgs.dockerTools.buildImage { + name = "ffd-indexer"; + config = { + Cmd = [ "${ffd-indexer}/bin/ffd-indexer" ]; + }; + }; + }; + nixosModules.default = import ./option.nix; + } + ); +} diff --git a/forgejo.go b/forgejo.go new file mode 100644 index 0000000..0aef149 --- /dev/null +++ b/forgejo.go @@ -0,0 +1,85 @@ +// forgejo.go +package main + +import ( + "fmt" + + forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3" +) + +// ForgejoScraper retrieves public repositories from a Forgejo instance. +type ForgejoScraper struct { + Client *forgejo.Client + PageSize int +} + +// NewForgejoScraper creates a scraper for a Forgejo instance. +// BaseURL should be the instance URL, e.g. https://codeberg.org. +func NewForgejoScraper(baseURL string) (*ForgejoScraper, error) { + client, err := forgejo.NewClient(baseURL) + if err != nil { + return nil, fmt.Errorf("create Forgejo client: %w", err) + } + + return &ForgejoScraper{ + Client: client, + PageSize: 50, + }, nil +} + +// GetProjects retrieves every public repository visible through the +// Forgejo repository search API and maps it to the generic Project type. +func (s *ForgejoScraper) GetProjects() ([]Project, error) { + pageSize := s.PageSize + if pageSize <= 0 { + pageSize = 50 + } + + isPrivate := false + var projects []Project + + for page := 1; ; page++ { + repos, _, err := s.Client.SearchRepos(forgejo.SearchRepoOptions{ + ListOptions: forgejo.ListOptions{ + Page: page, + PageSize: pageSize, + }, + IsPrivate: &isPrivate, + Sort: "alpha", + Order: "asc", + }) + if err != nil { + return nil, fmt.Errorf( + "list public Forgejo repositories (page %d): %w", + page, + err, + ) + } + + if len(repos) == 0 { + break + } + + for _, repo := range repos { + if repo == nil { + continue + } + + projects = append(projects, Project{ + Name: repo.FullName, + Url: repo.HTMLURL, + Description: repo.Description, + Topics: repo.Topics, + OpenIssues: repo.OpenIssues, + OpenPRs: repo.OpenPulls, + }) + } + + // A short page indicates that there are no further results. + if len(repos) < pageSize { + break + } + } + + return projects, nil +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..a899122 --- /dev/null +++ b/go.mod @@ -0,0 +1,35 @@ +module git.xengi.de/ffd/indexer + +go 1.27.1 + +require ( + codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3 v3.0.0 // indirect + github.com/42wim/httpsig v1.2.3 // indirect + github.com/davidmz/go-pageant v1.0.2 // indirect + github.com/go-fed/httpsig v1.1.0 // indirect + github.com/go-openapi/errors v0.22.6 // indirect + github.com/go-openapi/strfmt v0.25.0 // indirect + github.com/go-openapi/swag v0.25.4 // indirect + github.com/go-openapi/swag/cmdutils v0.25.4 // indirect + github.com/go-openapi/swag/conv v0.25.4 // indirect + github.com/go-openapi/swag/fileutils v0.25.4 // indirect + github.com/go-openapi/swag/jsonname v0.25.4 // indirect + github.com/go-openapi/swag/jsonutils v0.25.4 // indirect + github.com/go-openapi/swag/loading v0.25.4 // indirect + github.com/go-openapi/swag/mangling v0.25.4 // indirect + github.com/go-openapi/swag/netutils v0.25.4 // indirect + github.com/go-openapi/swag/stringutils v0.25.4 // indirect + github.com/go-openapi/swag/typeutils v0.25.4 // indirect + github.com/go-openapi/swag/yamlutils v0.25.4 // indirect + github.com/go-viper/mapstructure/v2 v2.4.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/hashicorp/go-version v1.7.0 // indirect + github.com/oklog/ulid v1.3.1 // indirect + go.mongodb.org/mongo-driver v1.17.6 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect + golang.org/x/crypto v0.45.0 // indirect + golang.org/x/net v0.47.0 // indirect + golang.org/x/sys v0.38.0 // indirect + golang.org/x/text v0.31.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..69937f7 --- /dev/null +++ b/go.sum @@ -0,0 +1,71 @@ +codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3 v3.0.0 h1:s2fK+FBwvcYsmKDjNhmoe7B8q9zsgs0UrSlYe9r4XjM= +codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3 v3.0.0/go.mod h1:Is2jTpS1dizeXm4skQv/ES3QVqnzcNhn2GzZXpiw9f8= +github.com/42wim/httpsig v1.2.3 h1:xb0YyWhkYj57SPtfSttIobJUPJZB9as1nsfo7KWVcEs= +github.com/42wim/httpsig v1.2.3/go.mod h1:nZq9OlYKDrUBhptd77IHx4/sZZD+IxTBADvAPI9G/EM= +github.com/davidmz/go-pageant v1.0.2 h1:bPblRCh5jGU+Uptpz6LgMZGD5hJoOt7otgT454WvHn0= +github.com/davidmz/go-pageant v1.0.2/go.mod h1:P2EDDnMqIwG5Rrp05dTRITj9z2zpGcD9efWSkTNKLIE= +github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI= +github.com/go-fed/httpsig v1.1.0/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM= +github.com/go-openapi/errors v0.22.6 h1:eDxcf89O8odEnohIXwEjY1IB4ph5vmbUsBMsFNwXWPo= +github.com/go-openapi/errors v0.22.6/go.mod h1:z9S8ASTUqx7+CP1Q8dD8ewGH/1JWFFLX/2PmAYNQLgk= +github.com/go-openapi/strfmt v0.25.0 h1:7R0RX7mbKLa9EYCTHRcCuIPcaqlyQiWNPTXwClK0saQ= +github.com/go-openapi/strfmt v0.25.0/go.mod h1:nNXct7OzbwrMY9+5tLX4I21pzcmE6ccMGXl3jFdPfn8= +github.com/go-openapi/swag v0.25.4 h1:OyUPUFYDPDBMkqyxOTkqDYFnrhuhi9NR6QVUvIochMU= +github.com/go-openapi/swag v0.25.4/go.mod h1:zNfJ9WZABGHCFg2RnY0S4IOkAcVTzJ6z2Bi+Q4i6qFQ= +github.com/go-openapi/swag/cmdutils v0.25.4 h1:8rYhB5n6WawR192/BfUu2iVlxqVR9aRgGJP6WaBoW+4= +github.com/go-openapi/swag/cmdutils v0.25.4/go.mod h1:pdae/AFo6WxLl5L0rq87eRzVPm/XRHM3MoYgRMvG4A0= +github.com/go-openapi/swag/conv v0.25.4 h1:/Dd7p0LZXczgUcC/Ikm1+YqVzkEeCc9LnOWjfkpkfe4= +github.com/go-openapi/swag/conv v0.25.4/go.mod h1:3LXfie/lwoAv0NHoEuY1hjoFAYkvlqI/Bn5EQDD3PPU= +github.com/go-openapi/swag/fileutils v0.25.4 h1:2oI0XNW5y6UWZTC7vAxC8hmsK/tOkWXHJQH4lKjqw+Y= +github.com/go-openapi/swag/fileutils v0.25.4/go.mod h1:cdOT/PKbwcysVQ9Tpr0q20lQKH7MGhOEb6EwmHOirUk= +github.com/go-openapi/swag/jsonname v0.25.4 h1:bZH0+MsS03MbnwBXYhuTttMOqk+5KcQ9869Vye1bNHI= +github.com/go-openapi/swag/jsonname v0.25.4/go.mod h1:GPVEk9CWVhNvWhZgrnvRA6utbAltopbKwDu8mXNUMag= +github.com/go-openapi/swag/jsonutils v0.25.4 h1:VSchfbGhD4UTf4vCdR2F4TLBdLwHyUDTd1/q4i+jGZA= +github.com/go-openapi/swag/jsonutils v0.25.4/go.mod h1:7OYGXpvVFPn4PpaSdPHJBtF0iGnbEaTk8AvBkoWnaAY= +github.com/go-openapi/swag/loading v0.25.4 h1:jN4MvLj0X6yhCDduRsxDDw1aHe+ZWoLjW+9ZQWIKn2s= +github.com/go-openapi/swag/loading v0.25.4/go.mod h1:rpUM1ZiyEP9+mNLIQUdMiD7dCETXvkkC30z53i+ftTE= +github.com/go-openapi/swag/mangling v0.25.4 h1:2b9kBJk9JvPgxr36V23FxJLdwBrpijI26Bx5JH4Hp48= +github.com/go-openapi/swag/mangling v0.25.4/go.mod h1:6dxwu6QyORHpIIApsdZgb6wBk/DPU15MdyYj/ikn0Hg= +github.com/go-openapi/swag/netutils v0.25.4 h1:Gqe6K71bGRb3ZQLusdI8p/y1KLgV4M/k+/HzVSqT8H0= +github.com/go-openapi/swag/netutils v0.25.4/go.mod h1:m2W8dtdaoX7oj9rEttLyTeEFFEBvnAx9qHd5nJEBzYg= +github.com/go-openapi/swag/stringutils v0.25.4 h1:O6dU1Rd8bej4HPA3/CLPciNBBDwZj9HiEpdVsb8B5A8= +github.com/go-openapi/swag/stringutils v0.25.4/go.mod h1:GTsRvhJW5xM5gkgiFe0fV3PUlFm0dr8vki6/VSRaZK0= +github.com/go-openapi/swag/typeutils v0.25.4 h1:1/fbZOUN472NTc39zpa+YGHn3jzHWhv42wAJSN91wRw= +github.com/go-openapi/swag/typeutils v0.25.4/go.mod h1:Ou7g//Wx8tTLS9vG0UmzfCsjZjKhpjxayRKTHXf2pTE= +github.com/go-openapi/swag/yamlutils v0.25.4 h1:6jdaeSItEUb7ioS9lFoCZ65Cne1/RZtPBZ9A56h92Sw= +github.com/go-openapi/swag/yamlutils v0.25.4/go.mod h1:MNzq1ulQu+yd8Kl7wPOut/YHAAU/H6hL91fF+E2RFwc= +github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= +github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= +github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +go.mongodb.org/mongo-driver v1.17.6 h1:87JUG1wZfWsr6rIz3ZmpH90rL5tea7O3IHuSwHUpsss= +go.mongodb.org/mongo-driver v1.17.6/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4= +go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= +golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= +golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/main.go b/main.go new file mode 100644 index 0000000..d8e7db7 --- /dev/null +++ b/main.go @@ -0,0 +1,59 @@ +package main + +import ( + "fmt" + "os" + + "go.yaml.in/yaml/v4" +) + +type Forge struct { + Name string `yaml:"name"` + Url string `yaml:"url"` + Type string `yaml:"type"` +} + +type Config struct { + Forges []Forge `yaml:"forges"` +} + +func main() { + data, err := os.ReadFile("config.yaml") + if err != nil { + panic(err) + } + var config Config + + if err := yaml.Unmarshal(data, &config); err != nil { + panic(err) + } + var projects []Project + var allProjects []Project + for _, forge := range config.Forges { + fmt.Printf("%+v: %+v\n", forge.Name, forge.Url) + + switch forge.Type { + case "forgejo": + scraper, err := NewForgejoScraper(forge.Url) + if err != nil { + panic(err) + } + + projects, err := scraper.GetProjects() + if err != nil { + panic(err) + } + default: + projects = []Project{} + fmt.Errorf("unknown forge: %s", forge.Type) + } + + fmt.Printf("Found %d repositories\n", len(projects)) + + allProjects = append(allProjects, projects...) + + for repo := range allProjects { + fmt.Printf("%+v\n", repo) + } + } +} diff --git a/option.nix b/option.nix new file mode 100644 index 0000000..d003b41 --- /dev/null +++ b/option.nix @@ -0,0 +1,57 @@ +{ + config, + lib, + pkgs, + options, + ... +}: + +let + cfg = config.services.federated-forge-discovery.indexer; + configFile = pkgs.writeText "config.ini" (pkgs.lib.generators.toINI { } cfg); + execCommand = "${cfg.package}/bin/ffd-indexer -c '${configFile}'"; +in +{ + options.services.federated-forge-discovery.indexer = { + enable = lib.mkEnableOption "Enables the Federated Forge Discovery Indexer systemd service."; + openFirewall = lib.mkOption { + + }; + package = lib.mkOption { + description = "Package to use."; + type = lib.types.package; + default = lib.literalExpression '' + pkgs.ffd-indexer + ''; + }; + port = lib.mkOption { + description = "Port to listen for search engines"; + type = lib.types.port; + default = 47810; + }; + }; + + config = lib.mkIf cfg.enable { + systemd.services."ffd-indexer" = { + description = "Federated Forge Discovery Indexer"; + wants = [ "network-online.target" ]; + after = [ "network-online.target" ]; + serviceConfig = { + Restart = "always"; + RestartSec = 30; + ExecStart = execCommand; + User = "ffd-indexer"; + Group = "ffd-indexer"; + AmbientCapabilities = lib.mkIf (cfg.port < 1000) [ "CAP_NET_BIND_SERVICE" ]; + CapabilityBoundingSet = lib.mkIf (cfg.port < 1000) [ "CAP_NET_BIND_SERVICE" ]; + NoNewPrivileges = true; + }; + wantedBy = [ "multi-user.target" ]; + }; + }; + + #meta = { + # maintainers = with lib.maintainers; [ xengi ]; + # doc = ./default.xml; + #}; +} diff --git a/scraper.go b/scraper.go new file mode 100644 index 0000000..473f58a --- /dev/null +++ b/scraper.go @@ -0,0 +1,23 @@ +package main + +import ( + "time" +) + +type Project struct { + Name string `json:"name"` + Url string `json:"url"` + Description string `json:"description"` + Topics []string `json:"topics"` + OpenIssues int `json:"open_issues"` + ClosedIssues int `json:"closed_issues"` + OpenPRs int `json:"open_prs"` + ClosedPRs int `json:"closed_prs"` + LatestCommit time.Time `json:"latest_commit"` + LatestTag time.Time `json:"latest_tag"` + LatestRelease time.Time `json:"latest_release"` + Contributors int `json:"contributors"` + Languages []string `json:"languages"` + Readme string `json:"readme"` + Licence string `json:"licence"` +} From bde13e9b685366457aac9a6859c6daf5d95d88d4 Mon Sep 17 00:00:00 2001 From: "Ricardo (XenGi) Band" Date: Mon, 28 Sep 2026 15:36:42 +0200 Subject: [PATCH 2/2] scraper done; scrapes to local sqlite db --- .gitignore | 5 +- README.md | 43 +++++- config.go | 14 ++ flake.lock | 85 ++++++++++++ flake.nix | 21 +-- forgejo.go | 85 ------------ go.mod | 53 ++++++- go.sum | 236 +++++++++++++++++++++++++++++-- gomod2nix.toml | 270 ++++++++++++++++++++++++++++++++++++ main.go | 68 ++++----- scraper.go | 23 --- scraper/forgejo.go | 230 ++++++++++++++++++++++++++++++ scraper/forgejo_licenses.go | 96 +++++++++++++ scraper/forgejo_readme.go | 70 ++++++++++ scraper/robots.go | 157 +++++++++++++++++++++ scraper/robots_test.go | 60 ++++++++ scraper/scraper.go | 47 +++++++ store/store.go | 190 +++++++++++++++++++++++++ 18 files changed, 1587 insertions(+), 166 deletions(-) create mode 100644 config.go create mode 100644 flake.lock delete mode 100644 forgejo.go create mode 100644 gomod2nix.toml delete mode 100644 scraper.go create mode 100644 scraper/forgejo.go create mode 100644 scraper/forgejo_licenses.go create mode 100644 scraper/forgejo_readme.go create mode 100644 scraper/robots.go create mode 100644 scraper/robots_test.go create mode 100644 scraper/scraper.go create mode 100644 store/store.go diff --git a/.gitignore b/.gitignore index cead4e1..c935c59 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ config.yaml models.json Containerfile +result +index.db # Created by https://www.toptal.com/developers/gitignore/api/windows,linux,macos,direnv,vim,go,goland+all # Edit at https://www.toptal.com/developers/gitignore?templates=windows,linux,macos,direnv,vim,go,goland+all @@ -142,7 +144,8 @@ fabric.properties .LSOverride # Icon must end with two \r -Icon +Icon + # Thumbnails ._* diff --git a/README.md b/README.md index 070201e..13156ae 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,44 @@ # indexer -Federated Forge Discovery Indexer \ No newline at end of file +Federated Forge Discovery Indexer + +Builds an index of public open source software projects from configured forges. + +## Scrape behavior + +For each configured forge, the indexer: + +1. **Checks robots.txt** (`GET /robots.txt`) before any API call. If the + instance disallows `/api/`, the instance is skipped. +2. **Lists public source repositories** via the repository search API + (`GET /api/v1/repos/search`), paginated. Forks and mirrors are excluded. +3. For each repository, enriches it via the Forgejo API: reads the root + directory once, then fetches topics, languages, latest commit, latest + release, the README, and detects licenses from the repo's files. + +Errors are non-fatal: an instance or a single repo that fails is skipped and +the run continues. + +### Request volume + +**Per instance:** + +| Request | Count | +|---|---| +| `GET /robots.txt` | 1 | +| `GET /api/v1/repos/search` (page) | `ceil(N/50)` where N = total repos | + +**Per repository:** + +| Request | Endpoint | Purpose | +|---|---|---| +| list root contents | `GET /api/v1/repos/{o}/{r}/contents/` | shared: README selection + license scan (1 call) | +| languages | `GET /api/v1/repos/{o}/{r}/languages` | `Project.Languages` | +| topics | `GET /api/v1/repos/{o}/{r}/topics` | `Project.Topics` | +| latest commit | `GET /api/v1/repos/{o}/{r}/commits?limit=1` | `Project.LatestCommit` | +| latest release | `GET /api/v1/repos/{o}/{r}/releases/latest` | `Project.LatestRelease` | +| README | `GET /api/v1/repos/{o}/{r}/raw/{branch}/{file}` | `Project.Readme` | +| license files | 1+ × `GET /api/v1/repos/{o}/{r}/raw/{branch}/{file}` | SPDX detection | + +So roughly **6–8 API requests per repository** plus the paginated search +requests and a single robots.txt check per instance. diff --git a/config.go b/config.go new file mode 100644 index 0000000..87b50b1 --- /dev/null +++ b/config.go @@ -0,0 +1,14 @@ +package main + +// Forge is a single forge instance to scrape. +type Forge struct { + Name string `yaml:"name"` + Url string `yaml:"url"` + Type string `yaml:"type"` +} + +// Config is the top-level configuration file. +type Config struct { + DbPath string `yaml:"db_path"` + Forges []Forge `yaml:"forges"` +} diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..bf253d3 --- /dev/null +++ b/flake.lock @@ -0,0 +1,85 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "gomod2nix": { + "inputs": { + "flake-utils": [ + "flake-utils" + ], + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1770585520, + "narHash": "sha256-yBz9Ozd5Wb56i3e3cHZ8WcbzCQ9RlVaiW18qDYA/AzA=", + "owner": "tweag", + "repo": "gomod2nix", + "rev": "1201ddd1279c35497754f016ef33d5e060f3da8d", + "type": "github" + }, + "original": { + "owner": "tweag", + "repo": "gomod2nix", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1790573947, + "narHash": "sha256-zfer1SLJvACKM0J7c9AusGvIp4q4B6l1Z1KgiVInG8Y=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "e6025a0259c7e80f81b11effc15f65d95b708cba", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable-small", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "gomod2nix": "gomod2nix", + "nixpkgs": "nixpkgs" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix index 50b30fd..7e77d0e 100644 --- a/flake.nix +++ b/flake.nix @@ -15,20 +15,22 @@ inherit system; overlays = [ gomod2nix.overlays.default ]; }; + go = pkgs.go_1_27; ffd-indexer = pkgs.buildGoApplication { + inherit go; pname = "ffd-indexer"; version = "0.0.1"; src = ./.; modules = ./gomod2nix.toml; }; - go-test = pkgs.stdenvNoCC.mkDerivation { + go-test = pkgs.stdenv.mkDerivation { name = "go-test"; dontBuild = true; src = ./.; doCheck = true; - nativeBuildInputs = with pkgs; [ + nativeBuildInputs = [ go - writableTmpDirAsHomeHook + pkgs.writableTmpDirAsHomeHook ]; checkPhase = '' go test -v ./... @@ -37,15 +39,15 @@ mkdir "$out" ''; }; - go-lint = pkgs.stdenvNoCC.mkDerivation { + go-lint = pkgs.stdenv.mkDerivation { name = "go-lint"; dontBuild = true; src = ./.; doCheck = true; - nativeBuildInputs = with pkgs; [ - golangci-lint + nativeBuildInputs = [ + pkgs.golangci-lint go - writableTmpDirAsHomeHook + pkgs.writableTmpDirAsHomeHook ]; checkPhase = '' golangci-lint run @@ -59,13 +61,14 @@ checks = { inherit go-test go-lint; }; formatter = pkgs.nixfmt-tree; devShells.default = pkgs.mkShell { - buildInputs = with pkgs; [ + buildInputs = [ go - go-tools # staticcheck + pkgs.go-tools # staticcheck gomod2nix.packages.${system}.default ]; packages = with pkgs; [ mkcert + sqlite # debugging the index db ]; }; packages = { diff --git a/forgejo.go b/forgejo.go deleted file mode 100644 index 0aef149..0000000 --- a/forgejo.go +++ /dev/null @@ -1,85 +0,0 @@ -// forgejo.go -package main - -import ( - "fmt" - - forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3" -) - -// ForgejoScraper retrieves public repositories from a Forgejo instance. -type ForgejoScraper struct { - Client *forgejo.Client - PageSize int -} - -// NewForgejoScraper creates a scraper for a Forgejo instance. -// BaseURL should be the instance URL, e.g. https://codeberg.org. -func NewForgejoScraper(baseURL string) (*ForgejoScraper, error) { - client, err := forgejo.NewClient(baseURL) - if err != nil { - return nil, fmt.Errorf("create Forgejo client: %w", err) - } - - return &ForgejoScraper{ - Client: client, - PageSize: 50, - }, nil -} - -// GetProjects retrieves every public repository visible through the -// Forgejo repository search API and maps it to the generic Project type. -func (s *ForgejoScraper) GetProjects() ([]Project, error) { - pageSize := s.PageSize - if pageSize <= 0 { - pageSize = 50 - } - - isPrivate := false - var projects []Project - - for page := 1; ; page++ { - repos, _, err := s.Client.SearchRepos(forgejo.SearchRepoOptions{ - ListOptions: forgejo.ListOptions{ - Page: page, - PageSize: pageSize, - }, - IsPrivate: &isPrivate, - Sort: "alpha", - Order: "asc", - }) - if err != nil { - return nil, fmt.Errorf( - "list public Forgejo repositories (page %d): %w", - page, - err, - ) - } - - if len(repos) == 0 { - break - } - - for _, repo := range repos { - if repo == nil { - continue - } - - projects = append(projects, Project{ - Name: repo.FullName, - Url: repo.HTMLURL, - Description: repo.Description, - Topics: repo.Topics, - OpenIssues: repo.OpenIssues, - OpenPRs: repo.OpenPulls, - }) - } - - // A short page indicates that there are no further results. - if len(repos) < pageSize { - break - } - } - - return projects, nil -} diff --git a/go.mod b/go.mod index a899122..774a74c 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,27 @@ module git.xengi.de/ffd/indexer go 1.27.1 require ( - codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3 v3.0.0 // indirect + codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3 v3.0.0 + github.com/go-enry/go-license-detector/v4 v4.3.1 + go.yaml.in/yaml/v4 v4.0.0-rc.6 + modernc.org/sqlite v1.59.0 +) + +require ( github.com/42wim/httpsig v1.2.3 // indirect + github.com/Microsoft/go-winio v0.6.0 // indirect + github.com/ProtonMail/go-crypto v0.0.0-20220930113650-c6815a8c17ad // indirect + github.com/acomagu/bufpipe v1.0.3 // indirect + github.com/cloudflare/circl v1.2.0 // indirect github.com/davidmz/go-pageant v1.0.2 // indirect + github.com/dgryski/go-minhash v0.0.0-20190315135803-ad340ca03076 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/ekzhu/minhash-lsh v0.0.0-20190924033628-faac2c6342f8 // indirect + github.com/emirpasic/gods v1.18.1 // indirect github.com/go-fed/httpsig v1.1.0 // indirect + github.com/go-git/gcfg v1.5.0 // indirect + github.com/go-git/go-billy/v5 v5.3.1 // indirect + github.com/go-git/go-git/v5 v5.4.2 // indirect github.com/go-openapi/errors v0.22.6 // indirect github.com/go-openapi/strfmt v0.25.0 // indirect github.com/go-openapi/swag v0.25.4 // indirect @@ -24,12 +41,36 @@ require ( github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/go-version v1.7.0 // indirect + github.com/hhatto/gorst v0.0.0-20181029133204-ca9f730cac5b // indirect + github.com/imdario/mergo v0.3.13 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/jdkato/prose v1.2.1 // indirect + github.com/kevinburke/ssh_config v1.2.0 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/montanaflynn/stats v0.7.1 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect github.com/oklog/ulid v1.3.1 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/sergi/go-diff v1.2.0 // indirect + github.com/shogo82148/go-shuffle v1.0.1 // indirect + github.com/xanzy/ssh-agent v0.3.2 // indirect go.mongodb.org/mongo-driver v1.17.6 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect - golang.org/x/crypto v0.45.0 // indirect - golang.org/x/net v0.47.0 // indirect - golang.org/x/sys v0.38.0 // indirect - golang.org/x/text v0.31.0 // indirect + golang.org/x/crypto v0.54.0 // indirect + golang.org/x/exp v0.0.0-20221006183845-316c7553db56 // indirect + golang.org/x/mod v0.38.0 // indirect + golang.org/x/net v0.57.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.40.0 // indirect + golang.org/x/tools v0.48.0 // indirect + gonum.org/v1/gonum v0.8.2 // indirect + gopkg.in/neurosnap/sentences.v1 v1.0.7 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect + modernc.org/libc v1.75.7 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.12.1 // indirect ) diff --git a/go.sum b/go.sum index 69937f7..1f92ed9 100644 --- a/go.sum +++ b/go.sum @@ -2,10 +2,62 @@ codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3 v3.0.0 h1:s2fK+FBwvcYsmKDjNhmoe7B8 codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3 v3.0.0/go.mod h1:Is2jTpS1dizeXm4skQv/ES3QVqnzcNhn2GzZXpiw9f8= github.com/42wim/httpsig v1.2.3 h1:xb0YyWhkYj57SPtfSttIobJUPJZB9as1nsfo7KWVcEs= github.com/42wim/httpsig v1.2.3/go.mod h1:nZq9OlYKDrUBhptd77IHx4/sZZD+IxTBADvAPI9G/EM= +github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= +github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= +github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= +github.com/Microsoft/go-winio v0.6.0 h1:slsWYD/zyx7lCXoZVlvQrj0hPTM1HI4+v1sIda2yDvg= +github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE= +github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo= +github.com/ProtonMail/go-crypto v0.0.0-20220930113650-c6815a8c17ad h1:QeeqI2zxxgZVe11UrYFXXx6gVxPVF40ygekjBzEg4XY= +github.com/ProtonMail/go-crypto v0.0.0-20220930113650-c6815a8c17ad/go.mod h1:UBYPn8k0D56RtnR8RFQMjmh4KrZzWJ5o7Z9SYjossQ8= +github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk= +github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= +github.com/bwesterb/go-ristretto v1.2.1/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= +github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= +github.com/cloudflare/circl v1.2.0 h1:NheeISPSUcYftKlfrLuOo4T62FkmD4t4jviLfFFYaec= +github.com/cloudflare/circl v1.2.0/go.mod h1:Ch2UgYr6ti2KTtlejELlROl0YIYj7SLjAC8M+INXlMk= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davidmz/go-pageant v1.0.2 h1:bPblRCh5jGU+Uptpz6LgMZGD5hJoOt7otgT454WvHn0= github.com/davidmz/go-pageant v1.0.2/go.mod h1:P2EDDnMqIwG5Rrp05dTRITj9z2zpGcD9efWSkTNKLIE= +github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc h1:8WFBn63wegobsYAX0YjD+8suexZDga5CctH4CCTx2+8= +github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc/go.mod h1:c9O8+fpSOX1DM8cPNSkX/qsBWdkD4yd2dpciOWQjpBw= +github.com/dgryski/go-minhash v0.0.0-20190315135803-ad340ca03076 h1:EB7M2v8Svo3kvIDy+P1YDE22XskDQP+TEYGzeDwPAN4= +github.com/dgryski/go-minhash v0.0.0-20190315135803-ad340ca03076/go.mod h1:VBi0XHpFy0xiMySf6YpVbRqrupW4RprJ5QTyN+XvGSM= +github.com/dgryski/go-spooky v0.0.0-20170606183049-ed3d087f40e2 h1:lx1ZQgST/imDhmLpYDma1O3Cx9L+4Ie4E8S2RjFPQ30= +github.com/dgryski/go-spooky v0.0.0-20170606183049-ed3d087f40e2/go.mod h1:hgHYKsoIw7S/hlWtP7wD1wZ7SX1jPTtKko5X9jrOgPQ= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/ekzhu/minhash-lsh v0.0.0-20190924033628-faac2c6342f8 h1:+Tje+xk1lmGKSJjYNtgCFsU1HtQzz0kCm1DFbKlvFBo= +github.com/ekzhu/minhash-lsh v0.0.0-20190924033628-faac2c6342f8/go.mod h1:yEtCVi+QamvzjEH4U/m6ZGkALIkF2xfQnFp0BcKmIOk= +github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0= +github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/go-enry/go-license-detector/v4 v4.3.1 h1:BajEVdTffFcs8RACmblySVhfEIuT58TmXx27RgVfUdc= +github.com/go-enry/go-license-detector/v4 v4.3.1/go.mod h1:YVJKPE01WQNjN/bdM6V0I/9KxvwEAAv0Ef9pi92K6w0= github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI= github.com/go-fed/httpsig v1.1.0/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM= +github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4= +github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E= +github.com/go-git/go-billy/v5 v5.2.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= +github.com/go-git/go-billy/v5 v5.3.1 h1:CPiOUAzKtMRvolEKw+bG1PLRpT7D3LIs3/3ey4Aiu34= +github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= +github.com/go-git/go-git-fixtures/v4 v4.2.1 h1:n9gGL1Ct/yIw+nfsfr8s4+sbhT+Ncu2SubfXjIWgci8= +github.com/go-git/go-git-fixtures/v4 v4.2.1/go.mod h1:K8zd3kDUAykwTdDCr+I0per6Y6vMiRR/nnVTBtavnB0= +github.com/go-git/go-git/v5 v5.4.2 h1:BXyZu9t0VkbiHtqrsvdq39UDhGJTl1h55VW6CSC4aY4= +github.com/go-git/go-git/v5 v5.4.2/go.mod h1:gQ1kArt6d+n+BGd+/B/I74HwRTLhth2+zti4ihgckDc= github.com/go-openapi/errors v0.22.6 h1:eDxcf89O8odEnohIXwEjY1IB4ph5vmbUsBMsFNwXWPo= github.com/go-openapi/errors v0.22.6/go.mod h1:z9S8ASTUqx7+CP1Q8dD8ewGH/1JWFFLX/2PmAYNQLgk= github.com/go-openapi/strfmt v0.25.0 h1:7R0RX7mbKLa9EYCTHRcCuIPcaqlyQiWNPTXwClK0saQ= @@ -22,6 +74,8 @@ github.com/go-openapi/swag/jsonname v0.25.4 h1:bZH0+MsS03MbnwBXYhuTttMOqk+5KcQ98 github.com/go-openapi/swag/jsonname v0.25.4/go.mod h1:GPVEk9CWVhNvWhZgrnvRA6utbAltopbKwDu8mXNUMag= github.com/go-openapi/swag/jsonutils v0.25.4 h1:VSchfbGhD4UTf4vCdR2F4TLBdLwHyUDTd1/q4i+jGZA= github.com/go-openapi/swag/jsonutils v0.25.4/go.mod h1:7OYGXpvVFPn4PpaSdPHJBtF0iGnbEaTk8AvBkoWnaAY= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.4 h1:IACsSvBhiNJwlDix7wq39SS2Fh7lUOCJRmx/4SN4sVo= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.4/go.mod h1:Mt0Ost9l3cUzVv4OEZG+WSeoHwjWLnarzMePNDAOBiM= github.com/go-openapi/swag/loading v0.25.4 h1:jN4MvLj0X6yhCDduRsxDDw1aHe+ZWoLjW+9ZQWIKn2s= github.com/go-openapi/swag/loading v0.25.4/go.mod h1:rpUM1ZiyEP9+mNLIQUdMiD7dCETXvkkC30z53i+ftTE= github.com/go-openapi/swag/mangling v0.25.4 h1:2b9kBJk9JvPgxr36V23FxJLdwBrpijI26Bx5JH4Hp48= @@ -34,38 +88,204 @@ github.com/go-openapi/swag/typeutils v0.25.4 h1:1/fbZOUN472NTc39zpa+YGHn3jzHWhv4 github.com/go-openapi/swag/typeutils v0.25.4/go.mod h1:Ou7g//Wx8tTLS9vG0UmzfCsjZjKhpjxayRKTHXf2pTE= github.com/go-openapi/swag/yamlutils v0.25.4 h1:6jdaeSItEUb7ioS9lFoCZ65Cne1/RZtPBZ9A56h92Sw= github.com/go-openapi/swag/yamlutils v0.25.4/go.mod h1:MNzq1ulQu+yd8Kl7wPOut/YHAAU/H6hL91fF+E2RFwc= +github.com/go-openapi/testify/enable/yaml/v2 v2.0.2 h1:0+Y41Pz1NkbTHz8NngxTuAXxEodtNSI1WG1c/m5Akw4= +github.com/go-openapi/testify/enable/yaml/v2 v2.0.2/go.mod h1:kme83333GCtJQHXQ8UKX3IBZu6z8T5Dvy5+CW3NLUUg= +github.com/go-openapi/testify/v2 v2.0.2 h1:X999g3jeLcoY8qctY/c/Z8iBHTbwLz7R2WXd6Ub6wls= +github.com/go-openapi/testify/v2 v2.0.2/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54= github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hhatto/gorst v0.0.0-20181029133204-ca9f730cac5b h1:Jdu2tbAxkRouSILp2EbposIb8h4gO+2QuZEn3d9sKAc= +github.com/hhatto/gorst v0.0.0-20181029133204-ca9f730cac5b/go.mod h1:HmaZGXHdSwQh1jnUlBGN2BeEYOHACLVGzYOXCbsLvxY= +github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= +github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/jdkato/prose v1.2.1 h1:Fp3UnJmLVISmlc57BgKUzdjr0lOtjqTZicL3PaYy6cU= +github.com/jdkato/prose v1.2.1/go.mod h1:AiRHgVagnEx2JbQRQowVBKjG0bcs/vtkGCH1dYAL1rA= +github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= +github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= +github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/montanaflynn/stats v0.6.3/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= +github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/neurosnap/sentences v1.0.6 h1:iBVUivNtlwGkYsJblWV8GGVFmXzZzak907Ci8aA0VTE= +github.com/neurosnap/sentences v1.0.6/go.mod h1:pg1IapvYpWCJJm/Etxeh0+gtMf1rI1STY9S7eUCPbDc= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= +github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/shogo82148/go-shuffle v0.0.0-20180218125048-27e6095f230d/go.mod h1:2htx6lmL0NGLHlO8ZCf+lQBGBHIbEujyywxJArf+2Yc= +github.com/shogo82148/go-shuffle v1.0.1 h1:4swIpHXLMAz14DE4YTgakgadpRN0n1wE1dieGnOTVFU= +github.com/shogo82148/go-shuffle v1.0.1/go.mod h1:HQPjVgUUZ9TNgm4/K/iXRuAdhPsQrXnAGgtk/9kqbBY= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0= +github.com/xanzy/ssh-agent v0.3.2 h1:eKj4SX2Fe7mui28ZgnFW5fmTz1EIr7ugo5s6wDxdHBM= +github.com/xanzy/ssh-agent v0.3.2/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= go.mongodb.org/mongo-driver v1.17.6 h1:87JUG1wZfWsr6rIz3ZmpH90rL5tea7O3IHuSwHUpsss= go.mongodb.org/mongo-driver v1.17.6/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4= go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= +golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= -golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= -golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= +golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= +golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20221006183845-316c7553db56 h1:BrYbdKcCNjLyrN6aKqXy4hPw9qGI8IATkj4EWv9Q+kQ= +golang.org/x/exp v0.0.0-20221006183845-316c7553db56/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= +golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= +golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= -golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= +golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220315194320-039c03cc5b86/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= -golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= +golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.8.2 h1:CCXrcPKiGGotvnN6jfUsKk4rRqm7q09/YbKb5xCEvtM= +gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0 h1:OE9mWmgKkjJyEmDAAtGMPjXu+YNeGvK9VTSHY6+Qihc= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/neurosnap/sentences.v1 v1.0.6/go.mod h1:YlK+SN+fLQZj+kY3r8DkGDhDr91+S3JmTb5LSxFRQo0= +gopkg.in/neurosnap/sentences.v1 v1.0.7 h1:gpTUYnqthem4+o8kyTLiYIB05W+IvdQFYR29erfe8uU= +gopkg.in/neurosnap/sentences.v1 v1.0.7/go.mod h1:YlK+SN+fLQZj+kY3r8DkGDhDr91+S3JmTb5LSxFRQo0= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/cc/v4 v4.29.2 h1:h6+9ciCnPKutf4I03CvheAvDLX7+IHlqR6Iy6J+cgd8= +modernc.org/cc/v4 v4.29.2/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.35.0 h1:F+TUsmw09QxLzmi3aeYYGxjAXarmZaKgj3mKQHNaA8w= +modernc.org/ccgo/v4 v4.35.0/go.mod h1:qrVGs9S3Sr2Ztcg9ve+kTAYMp5a3YvWjo+SoN06kJ5I= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc= +modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.75.7 h1:o3DTP9/0p9pKmY2WCKQaySW6wIiZhNM7wc2lUoyhfew= +modernc.org/libc v1.75.7/go.mod h1:bO5o2ztHxBb2rjz0PgdHN0sSMw57CgxGFLZ3Qd/QpVQ= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g= +modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.59.0 h1:X1es1GpqBlS/5T+vbM4HLUdaa8OtQx468DF2vrx+38A= +modernc.org/sqlite v1.59.0/go.mod h1:+paeT2A3iPRHkQDwG7oA6Tk0zQd5woMEI8q7orfry8k= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/gomod2nix.toml b/gomod2nix.toml new file mode 100644 index 0000000..c152ad5 --- /dev/null +++ b/gomod2nix.toml @@ -0,0 +1,270 @@ +schema = 3 + +[mod] + [mod.'codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3'] + version = 'v3.0.0' + hash = 'sha256-/fb4qz0lZt7vuxOELzthAiUy1oDC5x/PYCch3FUNWig=' + + [mod.'github.com/42wim/httpsig'] + version = 'v1.2.3' + hash = 'sha256-Yrf21S5q6mupTtyk8izVKQciJC/aRDe7YRKL0cBkRpQ=' + + [mod.'github.com/Microsoft/go-winio'] + version = 'v0.6.0' + hash = 'sha256-TQ0AvXZfuT3PDnb4p/h5zmyT8m9VPcwxlXMJuVD6ltE=' + + [mod.'github.com/ProtonMail/go-crypto'] + version = 'v0.0.0-20220930113650-c6815a8c17ad' + hash = 'sha256-mem+jc60bsyLIJblnHbnG5e3+KFWf2TfZo9In9viYwU=' + + [mod.'github.com/acomagu/bufpipe'] + version = 'v1.0.3' + hash = 'sha256-hIQf2lqZZv4dN8JwRyhNoBjStL9Mgih6f7Hy8RDU5Kg=' + + [mod.'github.com/cloudflare/circl'] + version = 'v1.2.0' + hash = 'sha256-9L1/4M8Za4qcrtti/1r1E7TgpdfUKHM+9IyuX3fuJyQ=' + + [mod.'github.com/davidmz/go-pageant'] + version = 'v1.0.2' + hash = 'sha256-gfwm6i0ZGQhX2c0lDRe7qVXyHkhoHeemYpaj+O0aYyw=' + + [mod.'github.com/dgryski/go-minhash'] + version = 'v0.0.0-20190315135803-ad340ca03076' + hash = 'sha256-CYfWI07pOkDyBgoHRUSY9QzOR/cWNLIa08M+0khbKX4=' + + [mod.'github.com/dustin/go-humanize'] + version = 'v1.0.1' + hash = 'sha256-yuvxYYngpfVkUg9yAmG99IUVmADTQA0tMbBXe0Fq0Mc=' + + [mod.'github.com/ekzhu/minhash-lsh'] + version = 'v0.0.0-20190924033628-faac2c6342f8' + hash = 'sha256-vZH3kLGjMEuDNiWHpTVXv1z0ZMYi40zai8DlOk7ivZM=' + + [mod.'github.com/emirpasic/gods'] + version = 'v1.18.1' + hash = 'sha256-hGDKddjLj+5dn2woHtXKUdd49/3xdsqnhx7VEdCu1m4=' + + [mod.'github.com/go-enry/go-license-detector/v4'] + version = 'v4.3.1' + hash = 'sha256-S9LKXjn5dL5FETOOAk+bs7bIVdu2x7MIhfjpZuXzuLo=' + + [mod.'github.com/go-fed/httpsig'] + version = 'v1.1.0' + hash = 'sha256-ok0PEEvSR010nh6vB3nDCyKSSVvoB50/UjVvBKOYXsA=' + + [mod.'github.com/go-git/gcfg'] + version = 'v1.5.0' + hash = 'sha256-A62eSLI/0y4sfwCsZhe/uoSC9Z3TryyW+PyWIcknYdE=' + + [mod.'github.com/go-git/go-billy/v5'] + version = 'v5.3.1' + hash = 'sha256-UdShPsStI2OBJig8Zv4Htyv5NiwKnYI41pwe4hIxv3Q=' + + [mod.'github.com/go-git/go-git/v5'] + version = 'v5.4.2' + hash = 'sha256-+RvDli5tPRLEs1pA9W/ZqCCm896y97ObTqLp3hLydpo=' + + [mod.'github.com/go-openapi/errors'] + version = 'v0.22.6' + hash = 'sha256-JcWNO3VO/OxaMptpMuFs6pYD7xFDtC5H8vn1EiF9w20=' + + [mod.'github.com/go-openapi/strfmt'] + version = 'v0.25.0' + hash = 'sha256-xz4aMKsHsDhUPVc27hBa4FrywKQABhLhEBNg3qClL9Y=' + + [mod.'github.com/go-openapi/swag'] + version = 'v0.25.4' + hash = 'sha256-aFQeLf95JC5gWLwIJXxhZHK9RxiGhh9IYV0MEYldDr4=' + + [mod.'github.com/go-openapi/swag/cmdutils'] + version = 'v0.25.4' + hash = 'sha256-nhlq5w/ULjwy7nptKcRryjZPlv3QG0FmgcBSOclqRlA=' + + [mod.'github.com/go-openapi/swag/conv'] + version = 'v0.25.4' + hash = 'sha256-uHgTdZC76LMSsq+x+RyeclnOgBsS0ID2cLgfjzFk0KQ=' + + [mod.'github.com/go-openapi/swag/fileutils'] + version = 'v0.25.4' + hash = 'sha256-NWacswHRvNZ81WzazA8yUmeDymtne2l8zLdArBmeYAU=' + + [mod.'github.com/go-openapi/swag/jsonname'] + version = 'v0.25.4' + hash = 'sha256-Z6uETeudh8W+/SGxFBnOB/VlJeRPRkFgCyJmntpz7bc=' + + [mod.'github.com/go-openapi/swag/jsonutils'] + version = 'v0.25.4' + hash = 'sha256-TQGeMImUuL5BWDoBHKAjZ1BvvxPXkFrV3TMgb82IrLk=' + + [mod.'github.com/go-openapi/swag/loading'] + version = 'v0.25.4' + hash = 'sha256-xr8OnFqB/kwBj3yf9HiLJwDcpL7I3/qJYlKb6VWbVFA=' + + [mod.'github.com/go-openapi/swag/mangling'] + version = 'v0.25.4' + hash = 'sha256-NrmwNgRJIUlOaAGt+XutHWSHy9e1COMnVE8B0UY5+sQ=' + + [mod.'github.com/go-openapi/swag/netutils'] + version = 'v0.25.4' + hash = 'sha256-wRMCQX/wIOvWm1g2UURobhzI47OldVqzlYYzwotLNTU=' + + [mod.'github.com/go-openapi/swag/stringutils'] + version = 'v0.25.4' + hash = 'sha256-rVo5NBH+oLLX9kaemUYWILd+TKw/TDTB8UdSz+mn3m8=' + + [mod.'github.com/go-openapi/swag/typeutils'] + version = 'v0.25.4' + hash = 'sha256-59RHnK6ugsAUc+A8DZCj7gZ3bJnt01Al7T/1kzM9PpA=' + + [mod.'github.com/go-openapi/swag/yamlutils'] + version = 'v0.25.4' + hash = 'sha256-zba7QX7Ds05oZq1opP/vwRSBEDaJ1pKm+9DgrsOuR9w=' + + [mod.'github.com/go-viper/mapstructure/v2'] + version = 'v2.4.0' + hash = 'sha256-lLfcV9z4n94hDhgyXJlde4bFB0hfzlbh+polqcJCwGE=' + + [mod.'github.com/google/uuid'] + version = 'v1.6.0' + hash = 'sha256-VWl9sqUzdOuhW0KzQlv0gwwUQClYkmZwSydHG2sALYw=' + + [mod.'github.com/hashicorp/go-version'] + version = 'v1.7.0' + hash = 'sha256-Umc/Q2mxRrPF4aEcDuDJq4lFBT+PSsnyANfyFwUIaxI=' + + [mod.'github.com/hhatto/gorst'] + version = 'v0.0.0-20181029133204-ca9f730cac5b' + hash = 'sha256-u1rB/Z20x3UYMKaLjlDP3bHnD7nMrVs5AHR48km/H1k=' + + [mod.'github.com/imdario/mergo'] + version = 'v0.3.13' + hash = 'sha256-03LKAZXgR5YUMeK5IRh2ds2GyV6uSbdq5QCPbAzlTus=' + + [mod.'github.com/jbenet/go-context'] + version = 'v0.0.0-20150711004518-d14ea06fba99' + hash = 'sha256-VANNCWNNpARH/ILQV9sCQsBWgyL2iFT+4AHZREpxIWE=' + + [mod.'github.com/jdkato/prose'] + version = 'v1.2.1' + hash = 'sha256-xjGGiunKj2XUGuUMxdKrObOUAoV1QUmwkPVhyjLc3S8=' + + [mod.'github.com/kevinburke/ssh_config'] + version = 'v1.2.0' + hash = 'sha256-Ta7ZOmyX8gG5tzWbY2oES70EJPfI90U7CIJS9EAce0s=' + + [mod.'github.com/mattn/go-isatty'] + version = 'v0.0.24' + hash = 'sha256-pmq8KKIb2+qMF3E1cR2B7NAj5joF2tZduyXjIfIlkTk=' + + [mod.'github.com/mitchellh/go-homedir'] + version = 'v1.1.0' + hash = 'sha256-oduBKXHAQG8X6aqLEpqZHs5DOKe84u6WkBwi4W6cv3k=' + + [mod.'github.com/montanaflynn/stats'] + version = 'v0.7.1' + hash = 'sha256-0QXUA/syOtDSBooh3isAffrVAjpSoBmSMMuZfO5maHg=' + + [mod.'github.com/ncruces/go-strftime'] + version = 'v1.0.0' + hash = 'sha256-GYIwYDONuv/yTE0AEugCHQbtV3oiBaco93xUNYFcVBQ=' + + [mod.'github.com/oklog/ulid'] + version = 'v1.3.1' + hash = 'sha256-LNn883rYNiaoY9sGEPIzlMRx5UwGThdYTjXqfzeGc9k=' + + [mod.'github.com/pkg/errors'] + version = 'v0.9.1' + hash = 'sha256-mNfQtcrQmu3sNg/7IwiieKWOgFQOVVe2yXgKBpe/wZw=' + + [mod.'github.com/remyoudompheng/bigfft'] + version = 'v0.0.0-20230129092748-24d4a6f8daec' + hash = 'sha256-vYmpyCE37eBYP/navhaLV4oX4/nu0Z/StAocLIFqrmM=' + + [mod.'github.com/russross/blackfriday/v2'] + version = 'v2.1.0' + hash = 'sha256-R+84l1si8az5yDqd5CYcFrTyNZ1eSYlpXKq6nFt4OTQ=' + + [mod.'github.com/sergi/go-diff'] + version = 'v1.2.0' + hash = 'sha256-d2higuBRee4ylRuCpPQV5+g0XK5yNQgDpudNor0qD2o=' + + [mod.'github.com/shogo82148/go-shuffle'] + version = 'v1.0.1' + hash = 'sha256-E5Xt69A2vXlVPzyHPyWK0FyQ+IaMZzqQCnUltedeZhg=' + + [mod.'github.com/xanzy/ssh-agent'] + version = 'v0.3.2' + hash = 'sha256-QavRCX75546hnWCOtBwiUdmyk6AoLjv3POoJXng5kMI=' + + [mod.'go.mongodb.org/mongo-driver'] + version = 'v1.17.6' + hash = 'sha256-B1ePaR77HKDksfG6PLwOmOVPYac9g13i8Ld/xWOCIiE=' + + [mod.'go.yaml.in/yaml/v3'] + version = 'v3.0.4' + hash = 'sha256-NkGFiDPoCxbr3LFsI6OCygjjkY0rdmg5ggvVVwpyDQ4=' + + [mod.'go.yaml.in/yaml/v4'] + version = 'v4.0.0-rc.6' + hash = 'sha256-9aOIQpQ53KLv/Hb14104qYKLVrZUcCGLwzYp6UGLP90=' + + [mod.'golang.org/x/crypto'] + version = 'v0.54.0' + hash = 'sha256-m8jsqlPuoPxGDjClzcAVbXf1aiU07L+9j7vYcqJu/0Q=' + + [mod.'golang.org/x/exp'] + version = 'v0.0.0-20221006183845-316c7553db56' + hash = 'sha256-2aATMtazbT3i+X3m0Ju54GKOggcyeX9dAJFxO+H5IxQ=' + + [mod.'golang.org/x/mod'] + version = 'v0.38.0' + hash = 'sha256-BpKfvsmb7ecEDvTBKx9ZHauxoWRkOAZCqg4BrHM2/S0=' + + [mod.'golang.org/x/net'] + version = 'v0.57.0' + hash = 'sha256-gKo8UMw4hfETBHm8N5GOfuNadse9TWEQXJo2YWq5bY4=' + + [mod.'golang.org/x/sync'] + version = 'v0.22.0' + hash = 'sha256-VZjl0fAM0p/nI81zh+pdBjzxFupx0UcKYXBBpL2ZS7k=' + + [mod.'golang.org/x/sys'] + version = 'v0.47.0' + hash = 'sha256-TpbRyWWqHjddP6QzUgAbaLd2EE0S+GYNRUIDJd18r98=' + + [mod.'golang.org/x/text'] + version = 'v0.40.0' + hash = 'sha256-LJfnki46XEreGbSgjl+DeqgcTsINTOu2owyXNvijMcA=' + + [mod.'golang.org/x/tools'] + version = 'v0.48.0' + hash = 'sha256-9cRNUaup6fexA5S1zc+Ic3aaoxAgKWntyMJ2xaOdiP8=' + + [mod.'gonum.org/v1/gonum'] + version = 'v0.8.2' + hash = 'sha256-eFUTetKRBa9QzDUEtd6aU8KsTJQgBF6zLAveY4Ybi58=' + + [mod.'gopkg.in/neurosnap/sentences.v1'] + version = 'v1.0.7' + hash = 'sha256-AaBiqQ5Mtrzk4dE0lZidyUtDzYVy7aonEeh5oXrq/VI=' + + [mod.'gopkg.in/warnings.v0'] + version = 'v0.1.2' + hash = 'sha256-ATVL9yEmgYbkJ1DkltDGRn/auGAjqGOfjQyBYyUo8s8=' + + [mod.'modernc.org/libc'] + version = 'v1.75.7' + hash = 'sha256-Iq3Pcr3RrbhC82RWDM1I4gyKh3V/HI1gVQzoGR+0YYo=' + + [mod.'modernc.org/mathutil'] + version = 'v1.7.1' + hash = 'sha256-COZ5rF2GhQVR1r6a0DanJ8qwQ94JSKdQxTMWrDzE0Cc=' + + [mod.'modernc.org/memory'] + version = 'v1.12.1' + hash = 'sha256-Y/kW9/A5uF+Y66kDbxrh4muFgE3CEee6YzSl2FMPcRg=' + + [mod.'modernc.org/sqlite'] + version = 'v1.59.0' + hash = 'sha256-CaNi9UvS9y6aiiSy3GUlRHX84og+i4MWOZmxbFONeb4=' diff --git a/main.go b/main.go index d8e7db7..99aed99 100644 --- a/main.go +++ b/main.go @@ -5,18 +5,11 @@ import ( "os" "go.yaml.in/yaml/v4" + + "git.xengi.de/ffd/indexer/scraper" + "git.xengi.de/ffd/indexer/store" ) -type Forge struct { - Name string `yaml:"name"` - Url string `yaml:"url"` - Type string `yaml:"type"` -} - -type Config struct { - Forges []Forge `yaml:"forges"` -} - func main() { data, err := os.ReadFile("config.yaml") if err != nil { @@ -27,33 +20,42 @@ func main() { if err := yaml.Unmarshal(data, &config); err != nil { panic(err) } - var projects []Project - var allProjects []Project + + db, err := store.Open(config.DbPath) + if err != nil { + panic(err) + } + defer func() { _ = db.Close() }() + for _, forge := range config.Forges { - fmt.Printf("%+v: %+v\n", forge.Name, forge.Url) + fmt.Printf("Scraping %s (%s, %s)\n", forge.Name, forge.Type, forge.Url) - switch forge.Type { - case "forgejo": - scraper, err := NewForgejoScraper(forge.Url) - if err != nil { - panic(err) - } - - projects, err := scraper.GetProjects() - if err != nil { - panic(err) - } - default: - projects = []Project{} - fmt.Errorf("unknown forge: %s", forge.Type) + s, err := scraper.NewScraper(forge.Type, forge.Url) + if err != nil { + fmt.Fprintf(os.Stderr, " skip %s: %v\n", forge.Name, err) + continue } - fmt.Printf("Found %d repositories\n", len(projects)) - - allProjects = append(allProjects, projects...) - - for repo := range allProjects { - fmt.Printf("%+v\n", repo) + projects, err := s.GetProjects() + if err != nil { + fmt.Fprintf(os.Stderr, " skip %s: %v\n", forge.Name, err) + continue } + + fmt.Printf(" found %d projects\n", len(projects)) + for _, p := range projects { + if err := db.Upsert(p); err != nil { + fmt.Fprintf(os.Stderr, " store %s: %v\n", p.Name, err) + continue + } + fmt.Printf(" %s\n", p.Name) + } + + all, err := db.All() + if err != nil { + fmt.Fprintf(os.Stderr, " count: %v\n", err) + continue + } + fmt.Printf(" total in db: %d\n", len(all)) } } diff --git a/scraper.go b/scraper.go deleted file mode 100644 index 473f58a..0000000 --- a/scraper.go +++ /dev/null @@ -1,23 +0,0 @@ -package main - -import ( - "time" -) - -type Project struct { - Name string `json:"name"` - Url string `json:"url"` - Description string `json:"description"` - Topics []string `json:"topics"` - OpenIssues int `json:"open_issues"` - ClosedIssues int `json:"closed_issues"` - OpenPRs int `json:"open_prs"` - ClosedPRs int `json:"closed_prs"` - LatestCommit time.Time `json:"latest_commit"` - LatestTag time.Time `json:"latest_tag"` - LatestRelease time.Time `json:"latest_release"` - Contributors int `json:"contributors"` - Languages []string `json:"languages"` - Readme string `json:"readme"` - Licence string `json:"licence"` -} diff --git a/scraper/forgejo.go b/scraper/forgejo.go new file mode 100644 index 0000000..f2023ee --- /dev/null +++ b/scraper/forgejo.go @@ -0,0 +1,230 @@ +// Package scraper forgejo.go - Forgejo scraper. +package scraper + +import ( + "fmt" + "sort" + "strings" + "time" + + forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3" + + "github.com/go-enry/go-license-detector/v4/licensedb/filer" +) + +// ForgejoScraper retrieves public repositories from a Forgejo instance. +type ForgejoScraper struct { + Client *forgejo.Client + BaseURL string + PageSize int +} + +// NewForgejoScraper creates a scraper for a Forgejo instance. +// BaseURL should be the instance URL, e.g. https://codeberg.org. +func NewForgejoScraper(baseURL string) (*ForgejoScraper, error) { + client, err := forgejo.NewClient(baseURL) + if err != nil { + return nil, fmt.Errorf("create Forgejo client: %w", err) + } + client.SetUserAgent(UserAgent) + + return &ForgejoScraper{ + Client: client, + BaseURL: strings.TrimSuffix(baseURL, "/"), + PageSize: 50, + }, nil +} + +// GetProjects retrieves every public repository visible through the +// Forgejo repository search API and maps it to the generic Project type. +func (s *ForgejoScraper) GetProjects() ([]Project, error) { + // Honor the instance's robots.txt before making any API call: + // if /api/ is disallowed, do not scrape this instance. + ok, err := Allowed(s.BaseURL) + if err != nil { + return nil, fmt.Errorf("check robots.txt: %w", err) + } + if !ok { + return nil, fmt.Errorf("instance robots.txt disallows /api/") + } + + pageSize := s.PageSize + if pageSize <= 0 { + pageSize = 50 + } + + isPrivate := false + var projects []Project + + for page := 1; ; page++ { + repos, _, err := s.Client.SearchRepos(forgejo.SearchRepoOptions{ + Page: page, + PageSize: pageSize, + IsPrivate: &isPrivate, + Type: forgejo.RepoTypeSource, + Sort: "alpha", + Order: "asc", + }) + if err != nil { + return nil, fmt.Errorf( + "list public Forgejo repositories (page %d): %w", + page, + err, + ) + } + + if len(repos) == 0 { + break + } + + for _, repo := range repos { + if repo == nil { + continue + } + + owner, name, ok := strings.Cut(repo.FullName, "/") + if !ok { + continue + } + + // List the repository root once and reuse it for readme selection + // and license detection (they previously each listed the root). + entries, _, err := s.Client.ListContents(owner, name, repo.DefaultBranch, "") + if err != nil { + // best-effort: fall back to empty root so the other fields still work + entries = nil + } + root := make([]filer.File, 0, len(entries)) + for _, e := range entries { + if e == nil { + continue + } + root = append(root, filer.File{Name: e.Name, IsDir: e.Type == "dir"}) + } + + // topics currently requires one extra HTTP call per repository. + // Future optimization: the /repos/search response already includes + // the topics field, so they could be gathered by parsing the + // SearchRepos payload itself (e.g. via ListRepos) instead. + // On failure the repo is kept in the results with empty topics. + topics, err := s.topics(owner, name) + if err != nil { + topics = nil + } + + // languages requires one extra HTTP call per repository + // (GET /repos/{owner}/{repo}/languages, via GetRepoLanguages). + // It returns a map[language]bytes; we order it most-dominant-first + // below. On failure the repo is kept with no languages. + langs, _, err := s.Client.GetRepoLanguages(owner, name) + if err != nil { + langs = nil + } + + // Latest commit: one HTTP call per repository + // (GET /repos/{owner}/{repo}/commits?limit=1, via ListRepoCommits), returns the + // newest commit first. SHA + date are mapped into LatestCommit. + latestCommit, _ := s.latestCommit(owner, name) + + // Latest release: one HTTP call per repository + // (GET /repos/{owner}/{repo}/releases/latest, via GetLatestRelease). + // Returns a single Release (server already excludes drafts and prereleases). + latestRelease, _ := s.latestRelease(owner, name) + + // Licenses: scan repo files on its default branch with + // go-license-detector, reusing the root listing. May require reads + // of candidate license files. On failure the repo is kept with no + // licenses. + licenses := s.licensesFor(owner, name, repo.DefaultBranch, root) + + // Readme: pick the default README (md/rst/txt/no extension) from the + // shared root listing and fetch it. Non-fatal when absent. + readme := s.readmeFor(owner, name, repo.DefaultBranch, root) + + projects = append(projects, Project{ + Name: repo.FullName, + Url: repo.HTMLURL, + Description: repo.Description, + OpenIssues: repo.OpenIssues, + OpenPRs: repo.OpenPulls, + Topics: topics, + Languages: orderedLanguages(langs), + LatestCommit: latestCommit, + LatestRelease: latestRelease, + Licenses: licenses, + Readme: readme, + }) + } + + // A short page indicates that there are no further results. + if len(repos) < pageSize { + break + } + } + + return projects, nil +} + +// topics returns the repository's topics. +func (s *ForgejoScraper) topics(owner, name string) ([]string, error) { + topics, _, err := s.Client.ListRepoTopics(owner, name, forgejo.ListRepoTopicsOptions{}) + if err != nil { + return nil, fmt.Errorf("list topics for %s/%s: %w", owner, name, err) + } + return topics, nil +} + +// latestCommit returns the most recent commit's time. On failure it returns +// the zero time and the error, leaving the Project's LatestCommit unset. +func (s *ForgejoScraper) latestCommit(owner, name string) (time.Time, error) { + commits, _, err := s.Client.ListRepoCommits(owner, name, forgejo.ListCommitOptions{ + PageSize: 1, + }) + if err != nil { + return time.Time{}, err + } + if len(commits) == 0 || commits[0] == nil { + return time.Time{}, nil + } + return commits[0].Created, nil +} + +// latestRelease returns the most recent non-draft, non-prerelease release. +// On failure (e.g. no releases) it returns a zero Release and the error, +// leaving the Project's LatestRelease unset. +func (s *ForgejoScraper) latestRelease(owner, name string) (Release, error) { + r, _, err := s.Client.GetLatestRelease(owner, name) + if err != nil { + return Release{}, err + } + if r == nil { + return Release{}, nil + } + return Release{ + Name: r.Title, + Date: r.PublishedAt, + }, nil +} + +// orderedLanguages returns the languages of a repo ordered by byte share, +// most dominant first. Ties are broken alphabetically for stable output. +func orderedLanguages(langs map[string]int64) []string { + if len(langs) == 0 { + return nil + } + + names := make([]string, 0, len(langs)) + for name := range langs { + names = append(names, name) + } + + // Descending byte count; alphabetically as a stable tie-breaker. + sort.Slice(names, func(i, j int) bool { + if langs[names[i]] != langs[names[j]] { + return langs[names[i]] > langs[names[j]] + } + return names[i] < names[j] + }) + + return names +} diff --git a/scraper/forgejo_licenses.go b/scraper/forgejo_licenses.go new file mode 100644 index 0000000..8094912 --- /dev/null +++ b/scraper/forgejo_licenses.go @@ -0,0 +1,96 @@ +package scraper + +import ( + "fmt" + + forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3" + + "github.com/go-enry/go-license-detector/v4/licensedb" + "github.com/go-enry/go-license-detector/v4/licensedb/filer" +) + +// forgejoFiler adapts the Forgejo content API to the filer.Filer interface so +// that go-license-detector can inspect a repository's files without a local +// checkout. ReadDir serves a cached root listing when available (to avoid an +// extra ListContents call) and falls back to the API otherwise. ReadFile +// downloads a file's bytes at the given ref (default branch). +type forgejoFiler struct { + client *forgejo.Client + owner string + repo string + ref string + // cachedRoot, when non-nil, is served for ReadDir("") to avoid a second + // root listing that was already fetched by the caller. + cachedRoot []filer.File +} + +var _ filer.Filer = (*forgejoFiler)(nil) + +func (f *forgejoFiler) ReadDir(dirPath string) ([]filer.File, error) { + if dirPath == "" && f.cachedRoot != nil { + return f.cachedRoot, nil + } + entries, _, err := f.client.ListContents(f.owner, f.repo, f.ref, dirPath) + if err != nil { + return nil, fmt.Errorf("list contents %q: %w", dirPath, err) + } + files := make([]filer.File, 0, len(entries)) + for _, e := range entries { + if e == nil { + continue + } + files = append(files, filer.File{ + Name: e.Name, + IsDir: e.Type == "dir", + }) + } + return files, nil +} + +func (f *forgejoFiler) ReadFile(path string) ([]byte, error) { + // GetFile returns raw file bytes (binary-safe, base64-decoded). + data, _, err := f.client.GetFile(f.owner, f.repo, f.ref, path) + if err != nil { + return nil, fmt.Errorf("read file %q: %w", path, err) + } + return data, nil +} + +func (f *forgejoFiler) Close() {} + +func (f *forgejoFiler) PathsAreAlwaysSlash() bool { return true } + +// licensesFor returns the SPDX identifiers detected for the repository. It +// uses go-license-detector over the repository's files on its default branch. +// The given root listing is served to the detector to avoid a duplicate, +// already-available API call. An empty slice is returned when no license can +// be determined (the repo is still kept in results -- license failures are +// non-fatal). +func (s *ForgejoScraper) licensesFor(owner, repo, ref string, root []filer.File) []string { + detector := &forgejoFiler{ + client: s.Client, + owner: owner, + repo: repo, + ref: ref, + cachedRoot: root, + } + + matches, err := licensedb.Detect(detector) + if err != nil { + return nil + } + + // Keep only the highest-confidence match, sorted deterministically. + ids := make([]string, 0, len(matches)) + bestID, bestConf := "", float32(-1) + for id, m := range matches { + if m.Confidence > bestConf { + bestConf = m.Confidence + bestID = id + } + } + if bestID != "" { + ids = append(ids, bestID) + } + return ids +} diff --git a/scraper/forgejo_readme.go b/scraper/forgejo_readme.go new file mode 100644 index 0000000..fe7122a --- /dev/null +++ b/scraper/forgejo_readme.go @@ -0,0 +1,70 @@ +package scraper + +import ( + "sort" + "strings" + + "github.com/go-enry/go-license-detector/v4/licensedb/filer" +) + +// readmeCandidates are the filenames looked for, in priority order, when +// retrieving a repository's README. +var readmeCandidates = []string{"README.md", "README.rst", "README.txt", "README"} + +// readmeFor returns the content of the repository's README on the given ref +// (default branch), picking from root (the pre-fetched root file listing) the +// first candidate found (case-insensitively). An empty string is returned when +// no README can be found; the repo is still kept in results either way. +func (s *ForgejoScraper) readmeFor(owner, name, ref string, root []filer.File) string { + // Collect root file names, lowercased for case-insensitive matching, while + // keeping the real name for the GetFile call. + type rootFile struct { + real string + low string + } + files := make([]rootFile, 0, len(root)) + for _, e := range root { + if e.IsDir { + continue + } + files = append(files, rootFile{real: e.Name, low: strings.ToLower(e.Name)}) + } + + // Candidate priority wins; ties broken by alphabetical real name for + // deterministic picks when a repo has e.g. both README and README.md. + sort.SliceStable(files, func(i, j int) bool { + pi, pj := readmePriority(files[i].low), readmePriority(files[j].low) + if pi != pj { + return pi < pj // lower = higher priority + } + return files[i].real < files[j].real + }) + + var chosen string + for i := range files { + if readmePriority(files[i].low) >= 0 { + chosen = files[i].real + break + } + } + if chosen == "" { + return "" + } + + data, _, err := s.Client.GetFile(owner, name, ref, chosen) + if err != nil { + return "" + } + return string(data) +} + +// readmePriority returns the index of a lowercased filename in +// readmeCandidates, or -1 if it is not a readme candidate. +func readmePriority(low string) int { + for i, c := range readmeCandidates { + if low == strings.ToLower(c) { + return i + } + } + return -1 +} diff --git a/scraper/robots.go b/scraper/robots.go new file mode 100644 index 0000000..bbbbd92 --- /dev/null +++ b/scraper/robots.go @@ -0,0 +1,157 @@ +package scraper + +import ( + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" +) + +// UserAgent identifies this crawler to servers. Keep it stable so that +// instances can set robots rules for us specifically. +const UserAgent = "ffd-indexer/0.1" + +// robotsHTTPClient is used for fetching robots.txt. It does not follow the +// SDK client's settings; it is a plain, short-timeout client. +var robotsHTTPClient = &http.Client{Timeout: 10 * time.Second} + +// Allowed reports whether an instance at baseURL permits crawling its API +// according to its robots.txt. It only consults the rules for the +// User-Agent "*"; a block of /api/ (the API root) disallows scraping. +// +// A failure to fetch or parse robots.txt is treated as an error; callers +// decide whether that is fatal. +func Allowed(baseURL string) (bool, error) { + u, err := url.Parse(baseURL) + if err != nil { + return false, fmt.Errorf("parse base URL: %w", err) + } + u.Path = "/robots.txt" + + req, err := http.NewRequest(http.MethodGet, u.String(), nil) + if err != nil { + return false, fmt.Errorf("build robots request: %w", err) + } + req.Header.Set("User-Agent", UserAgent) + + resp, err := robotsHTTPClient.Do(req) + if err != nil { + return false, fmt.Errorf("fetch robots.txt: %w", err) + } + defer func() { + _ = resp.Body.Close() + }() + + // A 404 or similar means "no robots.txt": assume allowed. + if resp.StatusCode == http.StatusNotFound { + return true, nil + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return false, fmt.Errorf("robots.txt returned %s", resp.Status) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return false, fmt.Errorf("read robots.txt: %w", err) + } + + return allowedFor(string(body), UserAgent), nil +} + +// allowedFor parses a robots.txt body and reports whether a crawler with the +// given user agent may crawl the path "/api/". It implements a reasonable +// subset of the robots.txt standard: +// +// - Only the "*" group and an explicit group matching userAgent are +// considered; later groups override earlier ones for the same agent. +// - Disallow entries are matched as path prefixes (a trailing "*" is +// stripped). No other wildcard/glob support. +// - An empty group (no rules) is "allow everything". +func allowedFor(body, userAgent string) bool { + groups := parseRobotGroups(body) + + // A group that explicitly names our user agent takes precedence over a + // "*" group, regardless of order. Otherwise fall back to "*". + var star *robotGroup + for _, g := range groups { + for _, a := range g.agents { + if strings.EqualFold(a, "*") { + star = g + } else if strings.EqualFold(a, userAgent) { + return !matchDisallow(g, "/api/") + } + } + } + if star != nil { + return !matchDisallow(star, "/api/") + } + return true +} + +type robotGroup struct { + agents []string + disallow []string +} + +// parseRobotGroups splits a robots.txt body into agent groups. +func parseRobotGroups(body string) []*robotGroup { + var groups []*robotGroup + var cur *robotGroup + + for _, raw := range strings.Split(body, "\n") { + line := strings.TrimSpace(raw) + // Strip an inline comment (not part of a rule's result). + if i := strings.IndexByte(line, '#'); i >= 0 { + line = strings.TrimSpace(line[:i]) + } + if line == "" { + continue + } + + key, val, ok := strings.Cut(line, ":") + if !ok { + continue + } + key = strings.ToLower(strings.TrimSpace(key)) + val = strings.TrimSpace(val) + + switch key { + case "user-agent": + // A User-agent line starts a new group, unless the current + // group is still collecting agents (has no Disallow yet). + if cur == nil || len(cur.disallow) > 0 { + cur = &robotGroup{} + groups = append(groups, cur) + } + cur.agents = append(cur.agents, val) + case "disallow": + if cur != nil { + cur.disallow = append(cur.disallow, val) + } + } + } + + return groups +} + +// matchDisallow reports whether a path is disallowed by a group's rules. +// A trailing "*" is stripped. Rules are applied in order, later rules +// overriding earlier ones: an empty Disallow rule revokes all preceding +// disallows (allow everything). +func matchDisallow(g *robotGroup, path string) bool { + disallowed := false + for _, rule := range g.disallow { + rule = strings.TrimSpace(rule) + rule = strings.TrimSuffix(rule, "*") + if rule == "" { + disallowed = false // empty Disallow: allow everything + continue + } + if strings.HasPrefix(path, rule) { + disallowed = true + } + } + return disallowed +} diff --git a/scraper/robots_test.go b/scraper/robots_test.go new file mode 100644 index 0000000..15b569c --- /dev/null +++ b/scraper/robots_test.go @@ -0,0 +1,60 @@ +package scraper + +import "testing" + +func TestAllowedFor(t *testing.T) { + const ua = "ffd-indexer/0.1" + + // xengi / cccb: only /archive/ blocked -> /api/ allowed. + if !allowedFor("User-agent: *\nDisallow: /*/*/archive/\n", ua) { + t.Error("expected /api/ allowed when only archive is disallowed") + } + + // codeberg: /api/ disallowed under "*". + codeberg := `User-agent: * +Disallow: /api/ +Disallow: /avatars/ +` + if allowedFor(codeberg, ua) { + t.Error("expected /api/ blocked when /api/ is disallowed") + } + + // Explicit agent group takes precedence over "*" (agent name matches + // UserAgent exactly, including the version). + // Here the explicit group is for "ffd-indexer" (no version), so it does + // not match "ffd-indexer/0.1"; the "*" group (blocks /api/) applies. + if allowedFor("User-agent: *\nDisallow: /api/\n\nUser-agent: ffd-indexer\nDisallow: /whatever/\n", ua) { + t.Error("expected unmatched explicit agent to fall back to * which blocks /api/") + } + + // A matching explicit agent group takes precedence over "*". + if !allowedFor("User-agent: *\nDisallow: /api/\n\nUser-agent: ffd-indexer/0.1\nDisallow: /whatever/\n", ua) { + t.Error("expected explicit agent rule to permit /api/") + } + + // No rules at all -> allow. + if !allowedFor("", ua) { + t.Error("expected empty robots to allow") + } + + // Only an unrelated agent listed -> no star, no match -> allow. + if !allowedFor("User-agent: SomeBot\nDisallow: /api/\n", ua) { + t.Error("expected unrelated agent not to block us") + } + + // Disallow "/" blocks everything including /api/. + if allowedFor("User-agent: *\nDisallow: /\n", ua) { + t.Error("expected Disallow / to block /api/") + } + + // Empty Disallow (allowed) overrides a previous block in the same group. + if !allowedFor("User-agent: *\nDisallow: /api/\nDisallow:\n", ua) { + t.Error("expected empty Disallow to allow /api/") + } + + // Wildcard on a non-prefix rule should not over-block (substring-only). + // "/*.pdf" should not match /api/. + if !allowedFor("User-agent: *\nDisallow: /*.pdf$\n", ua) { + t.Error("expected unrelated wildcard to not block /api/") + } +} diff --git a/scraper/scraper.go b/scraper/scraper.go new file mode 100644 index 0000000..a4f97ee --- /dev/null +++ b/scraper/scraper.go @@ -0,0 +1,47 @@ +// Package scraper defines the interfaces and shared types used to discover +// projects from forge instances (Forgejo, Gogs, ...). +package scraper + +import ( + "fmt" + "time" +) + +// Scraper is implemented by concrete, per-forge scrapers. +type Scraper interface { + // GetProjects retrieves every project discoverable through this forge's + // public API and maps it to the generic Project type. + GetProjects() ([]Project, error) +} + +// NewScraper returns a Scraper for the given forge type, e.g. "forgejo". +func NewScraper(typ, baseURL string) (Scraper, error) { + switch typ { + case "forgejo": + return NewForgejoScraper(baseURL) + default: + return nil, fmt.Errorf("unknown forge type %q", typ) + } +} + +// Project is a generic, forge-agnostic representation of an open source project. +type Project struct { + Name string `json:"name"` + Url string `json:"url"` + Description string `json:"description"` + Topics []string `json:"topics"` + OpenIssues int `json:"open_issues"` + OpenPRs int `json:"open_prs"` + LatestCommit time.Time `json:"latest_commit"` + LatestRelease Release `json:"latest_release"` + Languages []string `json:"languages"` + Readme string `json:"readme"` + Licenses []string `json:"license"` + UsesAI bool `json:"uses_ai"` +} + +// Release is a named release with the date it was published. +type Release struct { + Name string `json:"name"` + Date time.Time `json:"date"` +} diff --git a/store/store.go b/store/store.go new file mode 100644 index 0000000..c892be8 --- /dev/null +++ b/store/store.go @@ -0,0 +1,190 @@ +// Package store provides durable persistence of scraped projects in SQLite. +// +// The database is the system of record: scrapers write discovered projects +// here, and a search index (e.g. Typesense) can be built or rebuilt from it +// without re-scraping the forges. Projects are stored as one row each, with +// the URL as the natural key so re-running the scrape upserts in place. +package store + +import ( + "database/sql" + "encoding/json" + "fmt" + "time" + + _ "modernc.org/sqlite" + + "git.xengi.de/ffd/indexer/scraper" +) + +const schema = ` +CREATE TABLE IF NOT EXISTS projects ( + url TEXT PRIMARY KEY, + name TEXT NOT NULL, + description TEXT NOT NULL DEFAULT '', + topics TEXT NOT NULL DEFAULT '[]', -- JSON array + languages TEXT NOT NULL DEFAULT '[]', -- JSON array + licenses TEXT NOT NULL DEFAULT '[]', -- JSON array + open_issues INTEGER NOT NULL DEFAULT 0, + open_prs INTEGER NOT NULL DEFAULT 0, + latest_commit INTEGER NOT NULL DEFAULT 0, -- unix epoch seconds (UTC) + latest_release_name TEXT NOT NULL DEFAULT '', + latest_release_date INTEGER NOT NULL DEFAULT 0, -- unix epoch seconds (UTC) + readme TEXT NOT NULL DEFAULT '', + uses_ai INTEGER NOT NULL DEFAULT 0, + scraped_at INTEGER NOT NULL DEFAULT 0 -- unix epoch seconds (UTC) +); +` + +// Store wraps a SQLite database holding scraped projects. +type Store struct { + db *sql.DB +} + +// Open opens (creating if needed) the SQLite database at path. +func Open(path string) (*Store, error) { + db, err := sql.Open("sqlite", path) + if err != nil { + return nil, fmt.Errorf("open sqlite db %q: %w", path, err) + } + if _, err := db.Exec(schema); err != nil { + _ = db.Close() + return nil, fmt.Errorf("create schema: %w", err) + } + return &Store{db: db}, nil +} + +// Close closes the underlying database. +func (s *Store) Close() error { + return s.db.Close() +} + +const upsertSQL = ` +INSERT INTO projects ( + url, name, description, topics, languages, licenses, + open_issues, open_prs, latest_commit, + latest_release_name, latest_release_date, + readme, uses_ai, scraped_at +) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +ON CONFLICT(url) DO UPDATE SET + name = excluded.name, + description = excluded.description, + topics = excluded.topics, + languages = excluded.languages, + licenses = excluded.licenses, + open_issues = excluded.open_issues, + open_prs = excluded.open_prs, + latest_commit = excluded.latest_commit, + latest_release_name = excluded.latest_release_name, + latest_release_date = excluded.latest_release_date, + readme = excluded.readme, + uses_ai = excluded.uses_ai, + scraped_at = excluded.scraped_at; +` + +// Upsert inserts or updates a project, keyed on its URL. +func (s *Store) Upsert(p scraper.Project) error { + topics, err := json.Marshal(p.Topics) + if err != nil { + return fmt.Errorf("marshal topics: %w", err) + } + langs, err := json.Marshal(p.Languages) + if err != nil { + return fmt.Errorf("marshal languages: %w", err) + } + licenses, err := json.Marshal(p.Licenses) + if err != nil { + return fmt.Errorf("marshal licenses: %w", err) + } + + releaseDate := timeToUnix(p.LatestRelease.Date) + latestCommit := timeToUnix(p.LatestCommit) + + _, err = s.db.Exec(upsertSQL, + p.Url, + p.Name, + p.Description, + string(topics), + string(langs), + string(licenses), + p.OpenIssues, + p.OpenPRs, + latestCommit, + p.LatestRelease.Name, + releaseDate, + p.Readme, + boolToInt(p.UsesAI), + time.Now().UTC().Unix(), + ) + return err +} + +// All returns every stored project. +func (s *Store) All() ([]scraper.Project, error) { + rows, err := s.db.Query(`SELECT + url, name, description, topics, languages, licenses, + open_issues, open_prs, latest_commit, + latest_release_name, latest_release_date, + readme, uses_ai + FROM projects`) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + + var projects []scraper.Project + for rows.Next() { + var ( + p scraper.Project + topicsJSON string + langsJSON string + licensesJSON string + latestCommit int64 + relName string + relDate int64 + usesAI int + ) + if err := rows.Scan( + &p.Url, &p.Name, &p.Description, + &topicsJSON, &langsJSON, &licensesJSON, + &p.OpenIssues, &p.OpenPRs, &latestCommit, + &relName, &relDate, + &p.Readme, &usesAI, + ); err != nil { + return nil, err + } + + p.UsesAI = usesAI != 0 + if err := json.Unmarshal([]byte(topicsJSON), &p.Topics); err != nil { + return nil, err + } + if err := json.Unmarshal([]byte(langsJSON), &p.Languages); err != nil { + return nil, err + } + if err := json.Unmarshal([]byte(licensesJSON), &p.Licenses); err != nil { + return nil, err + } + p.LatestCommit = time.Unix(latestCommit, 0).UTC() + p.LatestRelease.Name = relName + p.LatestRelease.Date = time.Unix(relDate, 0).UTC() + + projects = append(projects, p) + } + return projects, rows.Err() +} + +// timeToUnix returns the Unix epoch seconds for a time, or 0 for the zero +// time (which is stored as a 0/absent value). +func timeToUnix(t time.Time) int64 { + if t.IsZero() { + return 0 + } + return t.Unix() +} + +func boolToInt(b bool) int { + if b { + return 1 + } + return 0 +}