From 448552016bc4f2d0e4659e0dd358d1f9efa5ae81 Mon Sep 17 00:00:00 2001 From: "Ricardo (XenGi) Band" Date: Mon, 28 Sep 2026 12:21:17 +0200 Subject: [PATCH] initial dump --- .editorconfig | 11 +++ .gitignore | 204 +++++++++++++++++++++++++++++++++++++++++++++++--- flake.nix | 83 ++++++++++++++++++++ forgejo.go | 85 +++++++++++++++++++++ go.mod | 35 +++++++++ go.sum | 71 ++++++++++++++++++ main.go | 59 +++++++++++++++ option.nix | 57 ++++++++++++++ scraper.go | 23 ++++++ 9 files changed, 617 insertions(+), 11 deletions(-) create mode 100644 .editorconfig create mode 100644 flake.nix create mode 100644 forgejo.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go create mode 100644 option.nix create mode 100644 scraper.go diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..d5da92e --- /dev/null +++ b/.editorconfig @@ -0,0 +1,11 @@ +# EditorConfig is awesome: https://editorconfig.org +root = true + +[*] +end_of_line = lf +insert_final_newline = true + +[*.{go,nix,yaml}] +charset = utf-8 +indent_style = space +indent_size = 2 diff --git a/.gitignore b/.gitignore index 921d1fe..cead4e1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,15 @@ -# ---> Go +config.yaml +models.json +Containerfile + +# Created by https://www.toptal.com/developers/gitignore/api/windows,linux,macos,direnv,vim,go,goland+all +# Edit at https://www.toptal.com/developers/gitignore?templates=windows,linux,macos,direnv,vim,go,goland+all + +### direnv ### +.direnv +.envrc + +### Go ### # If you prefer the allow list template instead of the deny list, see community template: # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore # @@ -12,23 +23,194 @@ # Test binary, built with `go test -c` *.test -# Code coverage profiles and other test artifacts +# Output of the go coverage tool, specifically when used with LiteIDE *.out -coverage.* -*.coverprofile -profile.cov # Dependency directories (remove the comment below to include it) # vendor/ # Go workspace file go.work -go.work.sum -# env file -.env +### GoLand+all ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 -# Editor/IDE -# .idea/ -# .vscode/ +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf +# AWS User-specific +.idea/**/aws.xml + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# SonarLint plugin +.idea/sonarlint/ + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +### GoLand+all Patch ### +# Ignore everything but code style settings and run configurations +# that are supposed to be shared within teams. + +.idea/* + +!.idea/codeStyles +!.idea/runConfigurations + +### Linux ### +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +### macOS ### +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +### macOS Patch ### +# iCloud generated files +*.icloud + +### Vim ### +# Swap +[._]*.s[a-v][a-z] +!*.svg # comment out if you don't need vector files +[._]*.sw[a-p] +[._]s[a-rt-v][a-z] +[._]ss[a-gi-z] +[._]sw[a-p] + +# Session +Session.vim +Sessionx.vim + +# Temporary +.netrwhist +# Auto-generated tag files +tags +# Persistent undo +[._]*.un~ + +### Windows ### +# Windows thumbnail cache files +Thumbs.db +Thumbs.db:encryptable +ehthumbs.db +ehthumbs_vista.db + +# Dump file +*.stackdump + +# Folder config file +[Dd]esktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msix +*.msm +*.msp + +# Windows shortcuts +*.lnk + +# End of https://www.toptal.com/developers/gitignore/api/windows,linux,macos,direnv,vim,go,goland+all diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..50b30fd --- /dev/null +++ b/flake.nix @@ -0,0 +1,83 @@ +{ + description = "Federated Forge Discovery Indexer"; + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable-small"; + flake-utils.url = "github:numtide/flake-utils"; + gomod2nix = { + url = "github:tweag/gomod2nix"; + inputs.nixpkgs.follows = "nixpkgs"; + inputs.flake-utils.follows = "flake-utils"; + }; + }; + outputs = { self, nixpkgs, flake-utils, gomod2nix }: flake-utils.lib.eachDefaultSystem (system: + let + pkgs = import nixpkgs { + inherit system; + overlays = [ gomod2nix.overlays.default ]; + }; + ffd-indexer = pkgs.buildGoApplication { + pname = "ffd-indexer"; + version = "0.0.1"; + src = ./.; + modules = ./gomod2nix.toml; + }; + go-test = pkgs.stdenvNoCC.mkDerivation { + name = "go-test"; + dontBuild = true; + src = ./.; + doCheck = true; + nativeBuildInputs = with pkgs; [ + go + writableTmpDirAsHomeHook + ]; + checkPhase = '' + go test -v ./... + ''; + installPhase = '' + mkdir "$out" + ''; + }; + go-lint = pkgs.stdenvNoCC.mkDerivation { + name = "go-lint"; + dontBuild = true; + src = ./.; + doCheck = true; + nativeBuildInputs = with pkgs; [ + golangci-lint + go + writableTmpDirAsHomeHook + ]; + checkPhase = '' + golangci-lint run + ''; + installPhase = '' + mkdir "$out" + ''; + }; + in + { + checks = { inherit go-test go-lint; }; + formatter = pkgs.nixfmt-tree; + devShells.default = pkgs.mkShell { + buildInputs = with pkgs; [ + go + go-tools # staticcheck + gomod2nix.packages.${system}.default + ]; + packages = with pkgs; [ + mkcert + ]; + }; + packages = { + default = ffd-indexer; + container = pkgs.dockerTools.buildImage { + name = "ffd-indexer"; + config = { + Cmd = [ "${ffd-indexer}/bin/ffd-indexer" ]; + }; + }; + }; + nixosModules.default = import ./option.nix; + } + ); +} diff --git a/forgejo.go b/forgejo.go new file mode 100644 index 0000000..0aef149 --- /dev/null +++ b/forgejo.go @@ -0,0 +1,85 @@ +// forgejo.go +package main + +import ( + "fmt" + + forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3" +) + +// ForgejoScraper retrieves public repositories from a Forgejo instance. +type ForgejoScraper struct { + Client *forgejo.Client + PageSize int +} + +// NewForgejoScraper creates a scraper for a Forgejo instance. +// BaseURL should be the instance URL, e.g. https://codeberg.org. +func NewForgejoScraper(baseURL string) (*ForgejoScraper, error) { + client, err := forgejo.NewClient(baseURL) + if err != nil { + return nil, fmt.Errorf("create Forgejo client: %w", err) + } + + return &ForgejoScraper{ + Client: client, + PageSize: 50, + }, nil +} + +// GetProjects retrieves every public repository visible through the +// Forgejo repository search API and maps it to the generic Project type. +func (s *ForgejoScraper) GetProjects() ([]Project, error) { + pageSize := s.PageSize + if pageSize <= 0 { + pageSize = 50 + } + + isPrivate := false + var projects []Project + + for page := 1; ; page++ { + repos, _, err := s.Client.SearchRepos(forgejo.SearchRepoOptions{ + ListOptions: forgejo.ListOptions{ + Page: page, + PageSize: pageSize, + }, + IsPrivate: &isPrivate, + Sort: "alpha", + Order: "asc", + }) + if err != nil { + return nil, fmt.Errorf( + "list public Forgejo repositories (page %d): %w", + page, + err, + ) + } + + if len(repos) == 0 { + break + } + + for _, repo := range repos { + if repo == nil { + continue + } + + projects = append(projects, Project{ + Name: repo.FullName, + Url: repo.HTMLURL, + Description: repo.Description, + Topics: repo.Topics, + OpenIssues: repo.OpenIssues, + OpenPRs: repo.OpenPulls, + }) + } + + // A short page indicates that there are no further results. + if len(repos) < pageSize { + break + } + } + + return projects, nil +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..a899122 --- /dev/null +++ b/go.mod @@ -0,0 +1,35 @@ +module git.xengi.de/ffd/indexer + +go 1.27.1 + +require ( + codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3 v3.0.0 // indirect + github.com/42wim/httpsig v1.2.3 // indirect + github.com/davidmz/go-pageant v1.0.2 // indirect + github.com/go-fed/httpsig v1.1.0 // indirect + github.com/go-openapi/errors v0.22.6 // indirect + github.com/go-openapi/strfmt v0.25.0 // indirect + github.com/go-openapi/swag v0.25.4 // indirect + github.com/go-openapi/swag/cmdutils v0.25.4 // indirect + github.com/go-openapi/swag/conv v0.25.4 // indirect + github.com/go-openapi/swag/fileutils v0.25.4 // indirect + github.com/go-openapi/swag/jsonname v0.25.4 // indirect + github.com/go-openapi/swag/jsonutils v0.25.4 // indirect + github.com/go-openapi/swag/loading v0.25.4 // indirect + github.com/go-openapi/swag/mangling v0.25.4 // indirect + github.com/go-openapi/swag/netutils v0.25.4 // indirect + github.com/go-openapi/swag/stringutils v0.25.4 // indirect + github.com/go-openapi/swag/typeutils v0.25.4 // indirect + github.com/go-openapi/swag/yamlutils v0.25.4 // indirect + github.com/go-viper/mapstructure/v2 v2.4.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/hashicorp/go-version v1.7.0 // indirect + github.com/oklog/ulid v1.3.1 // indirect + go.mongodb.org/mongo-driver v1.17.6 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect + golang.org/x/crypto v0.45.0 // indirect + golang.org/x/net v0.47.0 // indirect + golang.org/x/sys v0.38.0 // indirect + golang.org/x/text v0.31.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..69937f7 --- /dev/null +++ b/go.sum @@ -0,0 +1,71 @@ +codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3 v3.0.0 h1:s2fK+FBwvcYsmKDjNhmoe7B8q9zsgs0UrSlYe9r4XjM= +codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3 v3.0.0/go.mod h1:Is2jTpS1dizeXm4skQv/ES3QVqnzcNhn2GzZXpiw9f8= +github.com/42wim/httpsig v1.2.3 h1:xb0YyWhkYj57SPtfSttIobJUPJZB9as1nsfo7KWVcEs= +github.com/42wim/httpsig v1.2.3/go.mod h1:nZq9OlYKDrUBhptd77IHx4/sZZD+IxTBADvAPI9G/EM= +github.com/davidmz/go-pageant v1.0.2 h1:bPblRCh5jGU+Uptpz6LgMZGD5hJoOt7otgT454WvHn0= +github.com/davidmz/go-pageant v1.0.2/go.mod h1:P2EDDnMqIwG5Rrp05dTRITj9z2zpGcD9efWSkTNKLIE= +github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI= +github.com/go-fed/httpsig v1.1.0/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM= +github.com/go-openapi/errors v0.22.6 h1:eDxcf89O8odEnohIXwEjY1IB4ph5vmbUsBMsFNwXWPo= +github.com/go-openapi/errors v0.22.6/go.mod h1:z9S8ASTUqx7+CP1Q8dD8ewGH/1JWFFLX/2PmAYNQLgk= +github.com/go-openapi/strfmt v0.25.0 h1:7R0RX7mbKLa9EYCTHRcCuIPcaqlyQiWNPTXwClK0saQ= +github.com/go-openapi/strfmt v0.25.0/go.mod h1:nNXct7OzbwrMY9+5tLX4I21pzcmE6ccMGXl3jFdPfn8= +github.com/go-openapi/swag v0.25.4 h1:OyUPUFYDPDBMkqyxOTkqDYFnrhuhi9NR6QVUvIochMU= +github.com/go-openapi/swag v0.25.4/go.mod h1:zNfJ9WZABGHCFg2RnY0S4IOkAcVTzJ6z2Bi+Q4i6qFQ= +github.com/go-openapi/swag/cmdutils v0.25.4 h1:8rYhB5n6WawR192/BfUu2iVlxqVR9aRgGJP6WaBoW+4= +github.com/go-openapi/swag/cmdutils v0.25.4/go.mod h1:pdae/AFo6WxLl5L0rq87eRzVPm/XRHM3MoYgRMvG4A0= +github.com/go-openapi/swag/conv v0.25.4 h1:/Dd7p0LZXczgUcC/Ikm1+YqVzkEeCc9LnOWjfkpkfe4= +github.com/go-openapi/swag/conv v0.25.4/go.mod h1:3LXfie/lwoAv0NHoEuY1hjoFAYkvlqI/Bn5EQDD3PPU= +github.com/go-openapi/swag/fileutils v0.25.4 h1:2oI0XNW5y6UWZTC7vAxC8hmsK/tOkWXHJQH4lKjqw+Y= +github.com/go-openapi/swag/fileutils v0.25.4/go.mod h1:cdOT/PKbwcysVQ9Tpr0q20lQKH7MGhOEb6EwmHOirUk= +github.com/go-openapi/swag/jsonname v0.25.4 h1:bZH0+MsS03MbnwBXYhuTttMOqk+5KcQ9869Vye1bNHI= +github.com/go-openapi/swag/jsonname v0.25.4/go.mod h1:GPVEk9CWVhNvWhZgrnvRA6utbAltopbKwDu8mXNUMag= +github.com/go-openapi/swag/jsonutils v0.25.4 h1:VSchfbGhD4UTf4vCdR2F4TLBdLwHyUDTd1/q4i+jGZA= +github.com/go-openapi/swag/jsonutils v0.25.4/go.mod h1:7OYGXpvVFPn4PpaSdPHJBtF0iGnbEaTk8AvBkoWnaAY= +github.com/go-openapi/swag/loading v0.25.4 h1:jN4MvLj0X6yhCDduRsxDDw1aHe+ZWoLjW+9ZQWIKn2s= +github.com/go-openapi/swag/loading v0.25.4/go.mod h1:rpUM1ZiyEP9+mNLIQUdMiD7dCETXvkkC30z53i+ftTE= +github.com/go-openapi/swag/mangling v0.25.4 h1:2b9kBJk9JvPgxr36V23FxJLdwBrpijI26Bx5JH4Hp48= +github.com/go-openapi/swag/mangling v0.25.4/go.mod h1:6dxwu6QyORHpIIApsdZgb6wBk/DPU15MdyYj/ikn0Hg= +github.com/go-openapi/swag/netutils v0.25.4 h1:Gqe6K71bGRb3ZQLusdI8p/y1KLgV4M/k+/HzVSqT8H0= +github.com/go-openapi/swag/netutils v0.25.4/go.mod h1:m2W8dtdaoX7oj9rEttLyTeEFFEBvnAx9qHd5nJEBzYg= +github.com/go-openapi/swag/stringutils v0.25.4 h1:O6dU1Rd8bej4HPA3/CLPciNBBDwZj9HiEpdVsb8B5A8= +github.com/go-openapi/swag/stringutils v0.25.4/go.mod h1:GTsRvhJW5xM5gkgiFe0fV3PUlFm0dr8vki6/VSRaZK0= +github.com/go-openapi/swag/typeutils v0.25.4 h1:1/fbZOUN472NTc39zpa+YGHn3jzHWhv42wAJSN91wRw= +github.com/go-openapi/swag/typeutils v0.25.4/go.mod h1:Ou7g//Wx8tTLS9vG0UmzfCsjZjKhpjxayRKTHXf2pTE= +github.com/go-openapi/swag/yamlutils v0.25.4 h1:6jdaeSItEUb7ioS9lFoCZ65Cne1/RZtPBZ9A56h92Sw= +github.com/go-openapi/swag/yamlutils v0.25.4/go.mod h1:MNzq1ulQu+yd8Kl7wPOut/YHAAU/H6hL91fF+E2RFwc= +github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= +github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= +github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +go.mongodb.org/mongo-driver v1.17.6 h1:87JUG1wZfWsr6rIz3ZmpH90rL5tea7O3IHuSwHUpsss= +go.mongodb.org/mongo-driver v1.17.6/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4= +go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= +golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= +golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/main.go b/main.go new file mode 100644 index 0000000..d8e7db7 --- /dev/null +++ b/main.go @@ -0,0 +1,59 @@ +package main + +import ( + "fmt" + "os" + + "go.yaml.in/yaml/v4" +) + +type Forge struct { + Name string `yaml:"name"` + Url string `yaml:"url"` + Type string `yaml:"type"` +} + +type Config struct { + Forges []Forge `yaml:"forges"` +} + +func main() { + data, err := os.ReadFile("config.yaml") + if err != nil { + panic(err) + } + var config Config + + if err := yaml.Unmarshal(data, &config); err != nil { + panic(err) + } + var projects []Project + var allProjects []Project + for _, forge := range config.Forges { + fmt.Printf("%+v: %+v\n", forge.Name, forge.Url) + + switch forge.Type { + case "forgejo": + scraper, err := NewForgejoScraper(forge.Url) + if err != nil { + panic(err) + } + + projects, err := scraper.GetProjects() + if err != nil { + panic(err) + } + default: + projects = []Project{} + fmt.Errorf("unknown forge: %s", forge.Type) + } + + fmt.Printf("Found %d repositories\n", len(projects)) + + allProjects = append(allProjects, projects...) + + for repo := range allProjects { + fmt.Printf("%+v\n", repo) + } + } +} diff --git a/option.nix b/option.nix new file mode 100644 index 0000000..d003b41 --- /dev/null +++ b/option.nix @@ -0,0 +1,57 @@ +{ + config, + lib, + pkgs, + options, + ... +}: + +let + cfg = config.services.federated-forge-discovery.indexer; + configFile = pkgs.writeText "config.ini" (pkgs.lib.generators.toINI { } cfg); + execCommand = "${cfg.package}/bin/ffd-indexer -c '${configFile}'"; +in +{ + options.services.federated-forge-discovery.indexer = { + enable = lib.mkEnableOption "Enables the Federated Forge Discovery Indexer systemd service."; + openFirewall = lib.mkOption { + + }; + package = lib.mkOption { + description = "Package to use."; + type = lib.types.package; + default = lib.literalExpression '' + pkgs.ffd-indexer + ''; + }; + port = lib.mkOption { + description = "Port to listen for search engines"; + type = lib.types.port; + default = 47810; + }; + }; + + config = lib.mkIf cfg.enable { + systemd.services."ffd-indexer" = { + description = "Federated Forge Discovery Indexer"; + wants = [ "network-online.target" ]; + after = [ "network-online.target" ]; + serviceConfig = { + Restart = "always"; + RestartSec = 30; + ExecStart = execCommand; + User = "ffd-indexer"; + Group = "ffd-indexer"; + AmbientCapabilities = lib.mkIf (cfg.port < 1000) [ "CAP_NET_BIND_SERVICE" ]; + CapabilityBoundingSet = lib.mkIf (cfg.port < 1000) [ "CAP_NET_BIND_SERVICE" ]; + NoNewPrivileges = true; + }; + wantedBy = [ "multi-user.target" ]; + }; + }; + + #meta = { + # maintainers = with lib.maintainers; [ xengi ]; + # doc = ./default.xml; + #}; +} diff --git a/scraper.go b/scraper.go new file mode 100644 index 0000000..473f58a --- /dev/null +++ b/scraper.go @@ -0,0 +1,23 @@ +package main + +import ( + "time" +) + +type Project struct { + Name string `json:"name"` + Url string `json:"url"` + Description string `json:"description"` + Topics []string `json:"topics"` + OpenIssues int `json:"open_issues"` + ClosedIssues int `json:"closed_issues"` + OpenPRs int `json:"open_prs"` + ClosedPRs int `json:"closed_prs"` + LatestCommit time.Time `json:"latest_commit"` + LatestTag time.Time `json:"latest_tag"` + LatestRelease time.Time `json:"latest_release"` + Contributors int `json:"contributors"` + Languages []string `json:"languages"` + Readme string `json:"readme"` + Licence string `json:"licence"` +}