diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..d5da92e --- /dev/null +++ b/.editorconfig @@ -0,0 +1,11 @@ +# EditorConfig is awesome: https://editorconfig.org +root = true + +[*] +end_of_line = lf +insert_final_newline = true + +[*.{go,nix,yaml}] +charset = utf-8 +indent_style = space +indent_size = 2 diff --git a/.gitignore b/.gitignore index 921d1fe..cead4e1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,15 @@ -# ---> Go +config.yaml +models.json +Containerfile + +# Created by https://www.toptal.com/developers/gitignore/api/windows,linux,macos,direnv,vim,go,goland+all +# Edit at https://www.toptal.com/developers/gitignore?templates=windows,linux,macos,direnv,vim,go,goland+all + +### direnv ### +.direnv +.envrc + +### Go ### # If you prefer the allow list template instead of the deny list, see community template: # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore # @@ -12,23 +23,194 @@ # Test binary, built with `go test -c` *.test -# Code coverage profiles and other test artifacts +# Output of the go coverage tool, specifically when used with LiteIDE *.out -coverage.* -*.coverprofile -profile.cov # Dependency directories (remove the comment below to include it) # vendor/ # Go workspace file go.work -go.work.sum -# env file -.env +### GoLand+all ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 -# Editor/IDE -# .idea/ -# .vscode/ +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf +# AWS User-specific +.idea/**/aws.xml + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# SonarLint plugin +.idea/sonarlint/ + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +### GoLand+all Patch ### +# Ignore everything but code style settings and run configurations +# that are supposed to be shared within teams. + +.idea/* + +!.idea/codeStyles +!.idea/runConfigurations + +### Linux ### +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +### macOS ### +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +### macOS Patch ### +# iCloud generated files +*.icloud + +### Vim ### +# Swap +[._]*.s[a-v][a-z] +!*.svg # comment out if you don't need vector files +[._]*.sw[a-p] +[._]s[a-rt-v][a-z] +[._]ss[a-gi-z] +[._]sw[a-p] + +# Session +Session.vim +Sessionx.vim + +# Temporary +.netrwhist +# Auto-generated tag files +tags +# Persistent undo +[._]*.un~ + +### Windows ### +# Windows thumbnail cache files +Thumbs.db +Thumbs.db:encryptable +ehthumbs.db +ehthumbs_vista.db + +# Dump file +*.stackdump + +# Folder config file +[Dd]esktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msix +*.msm +*.msp + +# Windows shortcuts +*.lnk + +# End of https://www.toptal.com/developers/gitignore/api/windows,linux,macos,direnv,vim,go,goland+all diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..50b30fd --- /dev/null +++ b/flake.nix @@ -0,0 +1,83 @@ +{ + description = "Federated Forge Discovery Indexer"; + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable-small"; + flake-utils.url = "github:numtide/flake-utils"; + gomod2nix = { + url = "github:tweag/gomod2nix"; + inputs.nixpkgs.follows = "nixpkgs"; + inputs.flake-utils.follows = "flake-utils"; + }; + }; + outputs = { self, nixpkgs, flake-utils, gomod2nix }: flake-utils.lib.eachDefaultSystem (system: + let + pkgs = import nixpkgs { + inherit system; + overlays = [ gomod2nix.overlays.default ]; + }; + ffd-indexer = pkgs.buildGoApplication { + pname = "ffd-indexer"; + version = "0.0.1"; + src = ./.; + modules = ./gomod2nix.toml; + }; + go-test = pkgs.stdenvNoCC.mkDerivation { + name = "go-test"; + dontBuild = true; + src = ./.; + doCheck = true; + nativeBuildInputs = with pkgs; [ + go + writableTmpDirAsHomeHook + ]; + checkPhase = '' + go test -v ./... + ''; + installPhase = '' + mkdir "$out" + ''; + }; + go-lint = pkgs.stdenvNoCC.mkDerivation { + name = "go-lint"; + dontBuild = true; + src = ./.; + doCheck = true; + nativeBuildInputs = with pkgs; [ + golangci-lint + go + writableTmpDirAsHomeHook + ]; + checkPhase = '' + golangci-lint run + ''; + installPhase = '' + mkdir "$out" + ''; + }; + in + { + checks = { inherit go-test go-lint; }; + formatter = pkgs.nixfmt-tree; + devShells.default = pkgs.mkShell { + buildInputs = with pkgs; [ + go + go-tools # staticcheck + gomod2nix.packages.${system}.default + ]; + packages = with pkgs; [ + mkcert + ]; + }; + packages = { + default = ffd-indexer; + container = pkgs.dockerTools.buildImage { + name = "ffd-indexer"; + config = { + Cmd = [ "${ffd-indexer}/bin/ffd-indexer" ]; + }; + }; + }; + nixosModules.default = import ./option.nix; + } + ); +} diff --git a/forgejo.go b/forgejo.go new file mode 100644 index 0000000..0aef149 --- /dev/null +++ b/forgejo.go @@ -0,0 +1,85 @@ +// forgejo.go +package main + +import ( + "fmt" + + forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3" +) + +// ForgejoScraper retrieves public repositories from a Forgejo instance. +type ForgejoScraper struct { + Client *forgejo.Client + PageSize int +} + +// NewForgejoScraper creates a scraper for a Forgejo instance. +// BaseURL should be the instance URL, e.g. https://codeberg.org. +func NewForgejoScraper(baseURL string) (*ForgejoScraper, error) { + client, err := forgejo.NewClient(baseURL) + if err != nil { + return nil, fmt.Errorf("create Forgejo client: %w", err) + } + + return &ForgejoScraper{ + Client: client, + PageSize: 50, + }, nil +} + +// GetProjects retrieves every public repository visible through the +// Forgejo repository search API and maps it to the generic Project type. +func (s *ForgejoScraper) GetProjects() ([]Project, error) { + pageSize := s.PageSize + if pageSize <= 0 { + pageSize = 50 + } + + isPrivate := false + var projects []Project + + for page := 1; ; page++ { + repos, _, err := s.Client.SearchRepos(forgejo.SearchRepoOptions{ + ListOptions: forgejo.ListOptions{ + Page: page, + PageSize: pageSize, + }, + IsPrivate: &isPrivate, + Sort: "alpha", + Order: "asc", + }) + if err != nil { + return nil, fmt.Errorf( + "list public Forgejo repositories (page %d): %w", + page, + err, + ) + } + + if len(repos) == 0 { + break + } + + for _, repo := range repos { + if repo == nil { + continue + } + + projects = append(projects, Project{ + Name: repo.FullName, + Url: repo.HTMLURL, + Description: repo.Description, + Topics: repo.Topics, + OpenIssues: repo.OpenIssues, + OpenPRs: repo.OpenPulls, + }) + } + + // A short page indicates that there are no further results. + if len(repos) < pageSize { + break + } + } + + return projects, nil +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..a899122 --- /dev/null +++ b/go.mod @@ -0,0 +1,35 @@ +module git.xengi.de/ffd/indexer + +go 1.27.1 + +require ( + codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3 v3.0.0 // indirect + github.com/42wim/httpsig v1.2.3 // indirect + github.com/davidmz/go-pageant v1.0.2 // indirect + github.com/go-fed/httpsig v1.1.0 // indirect + github.com/go-openapi/errors v0.22.6 // indirect + github.com/go-openapi/strfmt v0.25.0 // indirect + github.com/go-openapi/swag v0.25.4 // indirect + github.com/go-openapi/swag/cmdutils v0.25.4 // indirect + github.com/go-openapi/swag/conv v0.25.4 // indirect + github.com/go-openapi/swag/fileutils v0.25.4 // indirect + github.com/go-openapi/swag/jsonname v0.25.4 // indirect + github.com/go-openapi/swag/jsonutils v0.25.4 // indirect + github.com/go-openapi/swag/loading v0.25.4 // indirect + github.com/go-openapi/swag/mangling v0.25.4 // indirect + github.com/go-openapi/swag/netutils v0.25.4 // indirect + github.com/go-openapi/swag/stringutils v0.25.4 // indirect + github.com/go-openapi/swag/typeutils v0.25.4 // indirect + github.com/go-openapi/swag/yamlutils v0.25.4 // indirect + github.com/go-viper/mapstructure/v2 v2.4.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/hashicorp/go-version v1.7.0 // indirect + github.com/oklog/ulid v1.3.1 // indirect + go.mongodb.org/mongo-driver v1.17.6 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect + golang.org/x/crypto v0.45.0 // indirect + golang.org/x/net v0.47.0 // indirect + golang.org/x/sys v0.38.0 // indirect + golang.org/x/text v0.31.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..69937f7 --- /dev/null +++ b/go.sum @@ -0,0 +1,71 @@ +codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3 v3.0.0 h1:s2fK+FBwvcYsmKDjNhmoe7B8q9zsgs0UrSlYe9r4XjM= +codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3 v3.0.0/go.mod h1:Is2jTpS1dizeXm4skQv/ES3QVqnzcNhn2GzZXpiw9f8= +github.com/42wim/httpsig v1.2.3 h1:xb0YyWhkYj57SPtfSttIobJUPJZB9as1nsfo7KWVcEs= +github.com/42wim/httpsig v1.2.3/go.mod h1:nZq9OlYKDrUBhptd77IHx4/sZZD+IxTBADvAPI9G/EM= +github.com/davidmz/go-pageant v1.0.2 h1:bPblRCh5jGU+Uptpz6LgMZGD5hJoOt7otgT454WvHn0= +github.com/davidmz/go-pageant v1.0.2/go.mod h1:P2EDDnMqIwG5Rrp05dTRITj9z2zpGcD9efWSkTNKLIE= +github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI= +github.com/go-fed/httpsig v1.1.0/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM= +github.com/go-openapi/errors v0.22.6 h1:eDxcf89O8odEnohIXwEjY1IB4ph5vmbUsBMsFNwXWPo= +github.com/go-openapi/errors v0.22.6/go.mod h1:z9S8ASTUqx7+CP1Q8dD8ewGH/1JWFFLX/2PmAYNQLgk= +github.com/go-openapi/strfmt v0.25.0 h1:7R0RX7mbKLa9EYCTHRcCuIPcaqlyQiWNPTXwClK0saQ= +github.com/go-openapi/strfmt v0.25.0/go.mod h1:nNXct7OzbwrMY9+5tLX4I21pzcmE6ccMGXl3jFdPfn8= +github.com/go-openapi/swag v0.25.4 h1:OyUPUFYDPDBMkqyxOTkqDYFnrhuhi9NR6QVUvIochMU= +github.com/go-openapi/swag v0.25.4/go.mod h1:zNfJ9WZABGHCFg2RnY0S4IOkAcVTzJ6z2Bi+Q4i6qFQ= +github.com/go-openapi/swag/cmdutils v0.25.4 h1:8rYhB5n6WawR192/BfUu2iVlxqVR9aRgGJP6WaBoW+4= +github.com/go-openapi/swag/cmdutils v0.25.4/go.mod h1:pdae/AFo6WxLl5L0rq87eRzVPm/XRHM3MoYgRMvG4A0= +github.com/go-openapi/swag/conv v0.25.4 h1:/Dd7p0LZXczgUcC/Ikm1+YqVzkEeCc9LnOWjfkpkfe4= +github.com/go-openapi/swag/conv v0.25.4/go.mod h1:3LXfie/lwoAv0NHoEuY1hjoFAYkvlqI/Bn5EQDD3PPU= +github.com/go-openapi/swag/fileutils v0.25.4 h1:2oI0XNW5y6UWZTC7vAxC8hmsK/tOkWXHJQH4lKjqw+Y= +github.com/go-openapi/swag/fileutils v0.25.4/go.mod h1:cdOT/PKbwcysVQ9Tpr0q20lQKH7MGhOEb6EwmHOirUk= +github.com/go-openapi/swag/jsonname v0.25.4 h1:bZH0+MsS03MbnwBXYhuTttMOqk+5KcQ9869Vye1bNHI= +github.com/go-openapi/swag/jsonname v0.25.4/go.mod h1:GPVEk9CWVhNvWhZgrnvRA6utbAltopbKwDu8mXNUMag= +github.com/go-openapi/swag/jsonutils v0.25.4 h1:VSchfbGhD4UTf4vCdR2F4TLBdLwHyUDTd1/q4i+jGZA= +github.com/go-openapi/swag/jsonutils v0.25.4/go.mod h1:7OYGXpvVFPn4PpaSdPHJBtF0iGnbEaTk8AvBkoWnaAY= +github.com/go-openapi/swag/loading v0.25.4 h1:jN4MvLj0X6yhCDduRsxDDw1aHe+ZWoLjW+9ZQWIKn2s= +github.com/go-openapi/swag/loading v0.25.4/go.mod h1:rpUM1ZiyEP9+mNLIQUdMiD7dCETXvkkC30z53i+ftTE= +github.com/go-openapi/swag/mangling v0.25.4 h1:2b9kBJk9JvPgxr36V23FxJLdwBrpijI26Bx5JH4Hp48= +github.com/go-openapi/swag/mangling v0.25.4/go.mod h1:6dxwu6QyORHpIIApsdZgb6wBk/DPU15MdyYj/ikn0Hg= +github.com/go-openapi/swag/netutils v0.25.4 h1:Gqe6K71bGRb3ZQLusdI8p/y1KLgV4M/k+/HzVSqT8H0= +github.com/go-openapi/swag/netutils v0.25.4/go.mod h1:m2W8dtdaoX7oj9rEttLyTeEFFEBvnAx9qHd5nJEBzYg= +github.com/go-openapi/swag/stringutils v0.25.4 h1:O6dU1Rd8bej4HPA3/CLPciNBBDwZj9HiEpdVsb8B5A8= +github.com/go-openapi/swag/stringutils v0.25.4/go.mod h1:GTsRvhJW5xM5gkgiFe0fV3PUlFm0dr8vki6/VSRaZK0= +github.com/go-openapi/swag/typeutils v0.25.4 h1:1/fbZOUN472NTc39zpa+YGHn3jzHWhv42wAJSN91wRw= +github.com/go-openapi/swag/typeutils v0.25.4/go.mod h1:Ou7g//Wx8tTLS9vG0UmzfCsjZjKhpjxayRKTHXf2pTE= +github.com/go-openapi/swag/yamlutils v0.25.4 h1:6jdaeSItEUb7ioS9lFoCZ65Cne1/RZtPBZ9A56h92Sw= +github.com/go-openapi/swag/yamlutils v0.25.4/go.mod h1:MNzq1ulQu+yd8Kl7wPOut/YHAAU/H6hL91fF+E2RFwc= +github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= +github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= +github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +go.mongodb.org/mongo-driver v1.17.6 h1:87JUG1wZfWsr6rIz3ZmpH90rL5tea7O3IHuSwHUpsss= +go.mongodb.org/mongo-driver v1.17.6/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4= +go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= +golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= +golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/main.go b/main.go new file mode 100644 index 0000000..d8e7db7 --- /dev/null +++ b/main.go @@ -0,0 +1,59 @@ +package main + +import ( + "fmt" + "os" + + "go.yaml.in/yaml/v4" +) + +type Forge struct { + Name string `yaml:"name"` + Url string `yaml:"url"` + Type string `yaml:"type"` +} + +type Config struct { + Forges []Forge `yaml:"forges"` +} + +func main() { + data, err := os.ReadFile("config.yaml") + if err != nil { + panic(err) + } + var config Config + + if err := yaml.Unmarshal(data, &config); err != nil { + panic(err) + } + var projects []Project + var allProjects []Project + for _, forge := range config.Forges { + fmt.Printf("%+v: %+v\n", forge.Name, forge.Url) + + switch forge.Type { + case "forgejo": + scraper, err := NewForgejoScraper(forge.Url) + if err != nil { + panic(err) + } + + projects, err := scraper.GetProjects() + if err != nil { + panic(err) + } + default: + projects = []Project{} + fmt.Errorf("unknown forge: %s", forge.Type) + } + + fmt.Printf("Found %d repositories\n", len(projects)) + + allProjects = append(allProjects, projects...) + + for repo := range allProjects { + fmt.Printf("%+v\n", repo) + } + } +} diff --git a/option.nix b/option.nix new file mode 100644 index 0000000..d003b41 --- /dev/null +++ b/option.nix @@ -0,0 +1,57 @@ +{ + config, + lib, + pkgs, + options, + ... +}: + +let + cfg = config.services.federated-forge-discovery.indexer; + configFile = pkgs.writeText "config.ini" (pkgs.lib.generators.toINI { } cfg); + execCommand = "${cfg.package}/bin/ffd-indexer -c '${configFile}'"; +in +{ + options.services.federated-forge-discovery.indexer = { + enable = lib.mkEnableOption "Enables the Federated Forge Discovery Indexer systemd service."; + openFirewall = lib.mkOption { + + }; + package = lib.mkOption { + description = "Package to use."; + type = lib.types.package; + default = lib.literalExpression '' + pkgs.ffd-indexer + ''; + }; + port = lib.mkOption { + description = "Port to listen for search engines"; + type = lib.types.port; + default = 47810; + }; + }; + + config = lib.mkIf cfg.enable { + systemd.services."ffd-indexer" = { + description = "Federated Forge Discovery Indexer"; + wants = [ "network-online.target" ]; + after = [ "network-online.target" ]; + serviceConfig = { + Restart = "always"; + RestartSec = 30; + ExecStart = execCommand; + User = "ffd-indexer"; + Group = "ffd-indexer"; + AmbientCapabilities = lib.mkIf (cfg.port < 1000) [ "CAP_NET_BIND_SERVICE" ]; + CapabilityBoundingSet = lib.mkIf (cfg.port < 1000) [ "CAP_NET_BIND_SERVICE" ]; + NoNewPrivileges = true; + }; + wantedBy = [ "multi-user.target" ]; + }; + }; + + #meta = { + # maintainers = with lib.maintainers; [ xengi ]; + # doc = ./default.xml; + #}; +} diff --git a/scraper.go b/scraper.go new file mode 100644 index 0000000..473f58a --- /dev/null +++ b/scraper.go @@ -0,0 +1,23 @@ +package main + +import ( + "time" +) + +type Project struct { + Name string `json:"name"` + Url string `json:"url"` + Description string `json:"description"` + Topics []string `json:"topics"` + OpenIssues int `json:"open_issues"` + ClosedIssues int `json:"closed_issues"` + OpenPRs int `json:"open_prs"` + ClosedPRs int `json:"closed_prs"` + LatestCommit time.Time `json:"latest_commit"` + LatestTag time.Time `json:"latest_tag"` + LatestRelease time.Time `json:"latest_release"` + Contributors int `json:"contributors"` + Languages []string `json:"languages"` + Readme string `json:"readme"` + Licence string `json:"licence"` +}