initial dump
This commit is contained in:
Ricardo (XenGi) Band
parent
b5339485be
commit
448552016b
9 changed files with 617 additions and 11 deletions
11
.editorconfig
Normal file
11
.editorconfig
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
# EditorConfig is awesome: https://editorconfig.org
|
||||
root = true
|
||||
|
||||
[*]
|
||||
end_of_line = lf
|
||||
insert_final_newline = true
|
||||
|
||||
[*.{go,nix,yaml}]
|
||||
charset = utf-8
|
||||
indent_style = space
|
||||
indent_size = 2
|
||||
204
.gitignore
vendored
204
.gitignore
vendored
|
|
@ -1,4 +1,15 @@
|
|||
# ---> Go
|
||||
config.yaml
|
||||
models.json
|
||||
Containerfile
|
||||
|
||||
# Created by https://www.toptal.com/developers/gitignore/api/windows,linux,macos,direnv,vim,go,goland+all
|
||||
# Edit at https://www.toptal.com/developers/gitignore?templates=windows,linux,macos,direnv,vim,go,goland+all
|
||||
|
||||
### direnv ###
|
||||
.direnv
|
||||
.envrc
|
||||
|
||||
### Go ###
|
||||
# If you prefer the allow list template instead of the deny list, see community template:
|
||||
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
|
||||
#
|
||||
|
|
@ -12,23 +23,194 @@
|
|||
# Test binary, built with `go test -c`
|
||||
*.test
|
||||
|
||||
# Code coverage profiles and other test artifacts
|
||||
# Output of the go coverage tool, specifically when used with LiteIDE
|
||||
*.out
|
||||
coverage.*
|
||||
*.coverprofile
|
||||
profile.cov
|
||||
|
||||
# Dependency directories (remove the comment below to include it)
|
||||
# vendor/
|
||||
|
||||
# Go workspace file
|
||||
go.work
|
||||
go.work.sum
|
||||
|
||||
# env file
|
||||
.env
|
||||
### GoLand+all ###
|
||||
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
|
||||
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
|
||||
|
||||
# Editor/IDE
|
||||
# .idea/
|
||||
# .vscode/
|
||||
# User-specific stuff
|
||||
.idea/**/workspace.xml
|
||||
.idea/**/tasks.xml
|
||||
.idea/**/usage.statistics.xml
|
||||
.idea/**/dictionaries
|
||||
.idea/**/shelf
|
||||
|
||||
# AWS User-specific
|
||||
.idea/**/aws.xml
|
||||
|
||||
# Generated files
|
||||
.idea/**/contentModel.xml
|
||||
|
||||
# Sensitive or high-churn files
|
||||
.idea/**/dataSources/
|
||||
.idea/**/dataSources.ids
|
||||
.idea/**/dataSources.local.xml
|
||||
.idea/**/sqlDataSources.xml
|
||||
.idea/**/dynamic.xml
|
||||
.idea/**/uiDesigner.xml
|
||||
.idea/**/dbnavigator.xml
|
||||
|
||||
# Gradle
|
||||
.idea/**/gradle.xml
|
||||
.idea/**/libraries
|
||||
|
||||
# Gradle and Maven with auto-import
|
||||
# When using Gradle or Maven with auto-import, you should exclude module files,
|
||||
# since they will be recreated, and may cause churn. Uncomment if using
|
||||
# auto-import.
|
||||
# .idea/artifacts
|
||||
# .idea/compiler.xml
|
||||
# .idea/jarRepositories.xml
|
||||
# .idea/modules.xml
|
||||
# .idea/*.iml
|
||||
# .idea/modules
|
||||
# *.iml
|
||||
# *.ipr
|
||||
|
||||
# CMake
|
||||
cmake-build-*/
|
||||
|
||||
# Mongo Explorer plugin
|
||||
.idea/**/mongoSettings.xml
|
||||
|
||||
# File-based project format
|
||||
*.iws
|
||||
|
||||
# IntelliJ
|
||||
out/
|
||||
|
||||
# mpeltonen/sbt-idea plugin
|
||||
.idea_modules/
|
||||
|
||||
# JIRA plugin
|
||||
atlassian-ide-plugin.xml
|
||||
|
||||
# Cursive Clojure plugin
|
||||
.idea/replstate.xml
|
||||
|
||||
# SonarLint plugin
|
||||
.idea/sonarlint/
|
||||
|
||||
# Crashlytics plugin (for Android Studio and IntelliJ)
|
||||
com_crashlytics_export_strings.xml
|
||||
crashlytics.properties
|
||||
crashlytics-build.properties
|
||||
fabric.properties
|
||||
|
||||
# Editor-based Rest Client
|
||||
.idea/httpRequests
|
||||
|
||||
# Android studio 3.1+ serialized cache file
|
||||
.idea/caches/build_file_checksums.ser
|
||||
|
||||
### GoLand+all Patch ###
|
||||
# Ignore everything but code style settings and run configurations
|
||||
# that are supposed to be shared within teams.
|
||||
|
||||
.idea/*
|
||||
|
||||
!.idea/codeStyles
|
||||
!.idea/runConfigurations
|
||||
|
||||
### Linux ###
|
||||
*~
|
||||
|
||||
# temporary files which can be created if a process still has a handle open of a deleted file
|
||||
.fuse_hidden*
|
||||
|
||||
# KDE directory preferences
|
||||
.directory
|
||||
|
||||
# Linux trash folder which might appear on any partition or disk
|
||||
.Trash-*
|
||||
|
||||
# .nfs files are created when an open file is removed but is still being accessed
|
||||
.nfs*
|
||||
|
||||
### macOS ###
|
||||
# General
|
||||
.DS_Store
|
||||
.AppleDouble
|
||||
.LSOverride
|
||||
|
||||
# Icon must end with two \r
|
||||
Icon
|
||||
|
||||
# Thumbnails
|
||||
._*
|
||||
|
||||
# Files that might appear in the root of a volume
|
||||
.DocumentRevisions-V100
|
||||
.fseventsd
|
||||
.Spotlight-V100
|
||||
.TemporaryItems
|
||||
.Trashes
|
||||
.VolumeIcon.icns
|
||||
.com.apple.timemachine.donotpresent
|
||||
|
||||
# Directories potentially created on remote AFP share
|
||||
.AppleDB
|
||||
.AppleDesktop
|
||||
Network Trash Folder
|
||||
Temporary Items
|
||||
.apdisk
|
||||
|
||||
### macOS Patch ###
|
||||
# iCloud generated files
|
||||
*.icloud
|
||||
|
||||
### Vim ###
|
||||
# Swap
|
||||
[._]*.s[a-v][a-z]
|
||||
!*.svg # comment out if you don't need vector files
|
||||
[._]*.sw[a-p]
|
||||
[._]s[a-rt-v][a-z]
|
||||
[._]ss[a-gi-z]
|
||||
[._]sw[a-p]
|
||||
|
||||
# Session
|
||||
Session.vim
|
||||
Sessionx.vim
|
||||
|
||||
# Temporary
|
||||
.netrwhist
|
||||
# Auto-generated tag files
|
||||
tags
|
||||
# Persistent undo
|
||||
[._]*.un~
|
||||
|
||||
### Windows ###
|
||||
# Windows thumbnail cache files
|
||||
Thumbs.db
|
||||
Thumbs.db:encryptable
|
||||
ehthumbs.db
|
||||
ehthumbs_vista.db
|
||||
|
||||
# Dump file
|
||||
*.stackdump
|
||||
|
||||
# Folder config file
|
||||
[Dd]esktop.ini
|
||||
|
||||
# Recycle Bin used on file shares
|
||||
$RECYCLE.BIN/
|
||||
|
||||
# Windows Installer files
|
||||
*.cab
|
||||
*.msi
|
||||
*.msix
|
||||
*.msm
|
||||
*.msp
|
||||
|
||||
# Windows shortcuts
|
||||
*.lnk
|
||||
|
||||
# End of https://www.toptal.com/developers/gitignore/api/windows,linux,macos,direnv,vim,go,goland+all
|
||||
|
|
|
|||
83
flake.nix
Normal file
83
flake.nix
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
{
|
||||
description = "Federated Forge Discovery Indexer";
|
||||
inputs = {
|
||||
nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable-small";
|
||||
flake-utils.url = "github:numtide/flake-utils";
|
||||
gomod2nix = {
|
||||
url = "github:tweag/gomod2nix";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
inputs.flake-utils.follows = "flake-utils";
|
||||
};
|
||||
};
|
||||
outputs = { self, nixpkgs, flake-utils, gomod2nix }: flake-utils.lib.eachDefaultSystem (system:
|
||||
let
|
||||
pkgs = import nixpkgs {
|
||||
inherit system;
|
||||
overlays = [ gomod2nix.overlays.default ];
|
||||
};
|
||||
ffd-indexer = pkgs.buildGoApplication {
|
||||
pname = "ffd-indexer";
|
||||
version = "0.0.1";
|
||||
src = ./.;
|
||||
modules = ./gomod2nix.toml;
|
||||
};
|
||||
go-test = pkgs.stdenvNoCC.mkDerivation {
|
||||
name = "go-test";
|
||||
dontBuild = true;
|
||||
src = ./.;
|
||||
doCheck = true;
|
||||
nativeBuildInputs = with pkgs; [
|
||||
go
|
||||
writableTmpDirAsHomeHook
|
||||
];
|
||||
checkPhase = ''
|
||||
go test -v ./...
|
||||
'';
|
||||
installPhase = ''
|
||||
mkdir "$out"
|
||||
'';
|
||||
};
|
||||
go-lint = pkgs.stdenvNoCC.mkDerivation {
|
||||
name = "go-lint";
|
||||
dontBuild = true;
|
||||
src = ./.;
|
||||
doCheck = true;
|
||||
nativeBuildInputs = with pkgs; [
|
||||
golangci-lint
|
||||
go
|
||||
writableTmpDirAsHomeHook
|
||||
];
|
||||
checkPhase = ''
|
||||
golangci-lint run
|
||||
'';
|
||||
installPhase = ''
|
||||
mkdir "$out"
|
||||
'';
|
||||
};
|
||||
in
|
||||
{
|
||||
checks = { inherit go-test go-lint; };
|
||||
formatter = pkgs.nixfmt-tree;
|
||||
devShells.default = pkgs.mkShell {
|
||||
buildInputs = with pkgs; [
|
||||
go
|
||||
go-tools # staticcheck
|
||||
gomod2nix.packages.${system}.default
|
||||
];
|
||||
packages = with pkgs; [
|
||||
mkcert
|
||||
];
|
||||
};
|
||||
packages = {
|
||||
default = ffd-indexer;
|
||||
container = pkgs.dockerTools.buildImage {
|
||||
name = "ffd-indexer";
|
||||
config = {
|
||||
Cmd = [ "${ffd-indexer}/bin/ffd-indexer" ];
|
||||
};
|
||||
};
|
||||
};
|
||||
nixosModules.default = import ./option.nix;
|
||||
}
|
||||
);
|
||||
}
|
||||
85
forgejo.go
Normal file
85
forgejo.go
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
// forgejo.go
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3"
|
||||
)
|
||||
|
||||
// ForgejoScraper retrieves public repositories from a Forgejo instance.
|
||||
type ForgejoScraper struct {
|
||||
Client *forgejo.Client
|
||||
PageSize int
|
||||
}
|
||||
|
||||
// NewForgejoScraper creates a scraper for a Forgejo instance.
|
||||
// BaseURL should be the instance URL, e.g. https://codeberg.org.
|
||||
func NewForgejoScraper(baseURL string) (*ForgejoScraper, error) {
|
||||
client, err := forgejo.NewClient(baseURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create Forgejo client: %w", err)
|
||||
}
|
||||
|
||||
return &ForgejoScraper{
|
||||
Client: client,
|
||||
PageSize: 50,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// GetProjects retrieves every public repository visible through the
|
||||
// Forgejo repository search API and maps it to the generic Project type.
|
||||
func (s *ForgejoScraper) GetProjects() ([]Project, error) {
|
||||
pageSize := s.PageSize
|
||||
if pageSize <= 0 {
|
||||
pageSize = 50
|
||||
}
|
||||
|
||||
isPrivate := false
|
||||
var projects []Project
|
||||
|
||||
for page := 1; ; page++ {
|
||||
repos, _, err := s.Client.SearchRepos(forgejo.SearchRepoOptions{
|
||||
ListOptions: forgejo.ListOptions{
|
||||
Page: page,
|
||||
PageSize: pageSize,
|
||||
},
|
||||
IsPrivate: &isPrivate,
|
||||
Sort: "alpha",
|
||||
Order: "asc",
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf(
|
||||
"list public Forgejo repositories (page %d): %w",
|
||||
page,
|
||||
err,
|
||||
)
|
||||
}
|
||||
|
||||
if len(repos) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
for _, repo := range repos {
|
||||
if repo == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
projects = append(projects, Project{
|
||||
Name: repo.FullName,
|
||||
Url: repo.HTMLURL,
|
||||
Description: repo.Description,
|
||||
Topics: repo.Topics,
|
||||
OpenIssues: repo.OpenIssues,
|
||||
OpenPRs: repo.OpenPulls,
|
||||
})
|
||||
}
|
||||
|
||||
// A short page indicates that there are no further results.
|
||||
if len(repos) < pageSize {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return projects, nil
|
||||
}
|
||||
35
go.mod
Normal file
35
go.mod
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
module git.xengi.de/ffd/indexer
|
||||
|
||||
go 1.27.1
|
||||
|
||||
require (
|
||||
codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3 v3.0.0 // indirect
|
||||
github.com/42wim/httpsig v1.2.3 // indirect
|
||||
github.com/davidmz/go-pageant v1.0.2 // indirect
|
||||
github.com/go-fed/httpsig v1.1.0 // indirect
|
||||
github.com/go-openapi/errors v0.22.6 // indirect
|
||||
github.com/go-openapi/strfmt v0.25.0 // indirect
|
||||
github.com/go-openapi/swag v0.25.4 // indirect
|
||||
github.com/go-openapi/swag/cmdutils v0.25.4 // indirect
|
||||
github.com/go-openapi/swag/conv v0.25.4 // indirect
|
||||
github.com/go-openapi/swag/fileutils v0.25.4 // indirect
|
||||
github.com/go-openapi/swag/jsonname v0.25.4 // indirect
|
||||
github.com/go-openapi/swag/jsonutils v0.25.4 // indirect
|
||||
github.com/go-openapi/swag/loading v0.25.4 // indirect
|
||||
github.com/go-openapi/swag/mangling v0.25.4 // indirect
|
||||
github.com/go-openapi/swag/netutils v0.25.4 // indirect
|
||||
github.com/go-openapi/swag/stringutils v0.25.4 // indirect
|
||||
github.com/go-openapi/swag/typeutils v0.25.4 // indirect
|
||||
github.com/go-openapi/swag/yamlutils v0.25.4 // indirect
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/hashicorp/go-version v1.7.0 // indirect
|
||||
github.com/oklog/ulid v1.3.1 // indirect
|
||||
go.mongodb.org/mongo-driver v1.17.6 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
|
||||
golang.org/x/crypto v0.45.0 // indirect
|
||||
golang.org/x/net v0.47.0 // indirect
|
||||
golang.org/x/sys v0.38.0 // indirect
|
||||
golang.org/x/text v0.31.0 // indirect
|
||||
)
|
||||
71
go.sum
Normal file
71
go.sum
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3 v3.0.0 h1:s2fK+FBwvcYsmKDjNhmoe7B8q9zsgs0UrSlYe9r4XjM=
|
||||
codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3 v3.0.0/go.mod h1:Is2jTpS1dizeXm4skQv/ES3QVqnzcNhn2GzZXpiw9f8=
|
||||
github.com/42wim/httpsig v1.2.3 h1:xb0YyWhkYj57SPtfSttIobJUPJZB9as1nsfo7KWVcEs=
|
||||
github.com/42wim/httpsig v1.2.3/go.mod h1:nZq9OlYKDrUBhptd77IHx4/sZZD+IxTBADvAPI9G/EM=
|
||||
github.com/davidmz/go-pageant v1.0.2 h1:bPblRCh5jGU+Uptpz6LgMZGD5hJoOt7otgT454WvHn0=
|
||||
github.com/davidmz/go-pageant v1.0.2/go.mod h1:P2EDDnMqIwG5Rrp05dTRITj9z2zpGcD9efWSkTNKLIE=
|
||||
github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI=
|
||||
github.com/go-fed/httpsig v1.1.0/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM=
|
||||
github.com/go-openapi/errors v0.22.6 h1:eDxcf89O8odEnohIXwEjY1IB4ph5vmbUsBMsFNwXWPo=
|
||||
github.com/go-openapi/errors v0.22.6/go.mod h1:z9S8ASTUqx7+CP1Q8dD8ewGH/1JWFFLX/2PmAYNQLgk=
|
||||
github.com/go-openapi/strfmt v0.25.0 h1:7R0RX7mbKLa9EYCTHRcCuIPcaqlyQiWNPTXwClK0saQ=
|
||||
github.com/go-openapi/strfmt v0.25.0/go.mod h1:nNXct7OzbwrMY9+5tLX4I21pzcmE6ccMGXl3jFdPfn8=
|
||||
github.com/go-openapi/swag v0.25.4 h1:OyUPUFYDPDBMkqyxOTkqDYFnrhuhi9NR6QVUvIochMU=
|
||||
github.com/go-openapi/swag v0.25.4/go.mod h1:zNfJ9WZABGHCFg2RnY0S4IOkAcVTzJ6z2Bi+Q4i6qFQ=
|
||||
github.com/go-openapi/swag/cmdutils v0.25.4 h1:8rYhB5n6WawR192/BfUu2iVlxqVR9aRgGJP6WaBoW+4=
|
||||
github.com/go-openapi/swag/cmdutils v0.25.4/go.mod h1:pdae/AFo6WxLl5L0rq87eRzVPm/XRHM3MoYgRMvG4A0=
|
||||
github.com/go-openapi/swag/conv v0.25.4 h1:/Dd7p0LZXczgUcC/Ikm1+YqVzkEeCc9LnOWjfkpkfe4=
|
||||
github.com/go-openapi/swag/conv v0.25.4/go.mod h1:3LXfie/lwoAv0NHoEuY1hjoFAYkvlqI/Bn5EQDD3PPU=
|
||||
github.com/go-openapi/swag/fileutils v0.25.4 h1:2oI0XNW5y6UWZTC7vAxC8hmsK/tOkWXHJQH4lKjqw+Y=
|
||||
github.com/go-openapi/swag/fileutils v0.25.4/go.mod h1:cdOT/PKbwcysVQ9Tpr0q20lQKH7MGhOEb6EwmHOirUk=
|
||||
github.com/go-openapi/swag/jsonname v0.25.4 h1:bZH0+MsS03MbnwBXYhuTttMOqk+5KcQ9869Vye1bNHI=
|
||||
github.com/go-openapi/swag/jsonname v0.25.4/go.mod h1:GPVEk9CWVhNvWhZgrnvRA6utbAltopbKwDu8mXNUMag=
|
||||
github.com/go-openapi/swag/jsonutils v0.25.4 h1:VSchfbGhD4UTf4vCdR2F4TLBdLwHyUDTd1/q4i+jGZA=
|
||||
github.com/go-openapi/swag/jsonutils v0.25.4/go.mod h1:7OYGXpvVFPn4PpaSdPHJBtF0iGnbEaTk8AvBkoWnaAY=
|
||||
github.com/go-openapi/swag/loading v0.25.4 h1:jN4MvLj0X6yhCDduRsxDDw1aHe+ZWoLjW+9ZQWIKn2s=
|
||||
github.com/go-openapi/swag/loading v0.25.4/go.mod h1:rpUM1ZiyEP9+mNLIQUdMiD7dCETXvkkC30z53i+ftTE=
|
||||
github.com/go-openapi/swag/mangling v0.25.4 h1:2b9kBJk9JvPgxr36V23FxJLdwBrpijI26Bx5JH4Hp48=
|
||||
github.com/go-openapi/swag/mangling v0.25.4/go.mod h1:6dxwu6QyORHpIIApsdZgb6wBk/DPU15MdyYj/ikn0Hg=
|
||||
github.com/go-openapi/swag/netutils v0.25.4 h1:Gqe6K71bGRb3ZQLusdI8p/y1KLgV4M/k+/HzVSqT8H0=
|
||||
github.com/go-openapi/swag/netutils v0.25.4/go.mod h1:m2W8dtdaoX7oj9rEttLyTeEFFEBvnAx9qHd5nJEBzYg=
|
||||
github.com/go-openapi/swag/stringutils v0.25.4 h1:O6dU1Rd8bej4HPA3/CLPciNBBDwZj9HiEpdVsb8B5A8=
|
||||
github.com/go-openapi/swag/stringutils v0.25.4/go.mod h1:GTsRvhJW5xM5gkgiFe0fV3PUlFm0dr8vki6/VSRaZK0=
|
||||
github.com/go-openapi/swag/typeutils v0.25.4 h1:1/fbZOUN472NTc39zpa+YGHn3jzHWhv42wAJSN91wRw=
|
||||
github.com/go-openapi/swag/typeutils v0.25.4/go.mod h1:Ou7g//Wx8tTLS9vG0UmzfCsjZjKhpjxayRKTHXf2pTE=
|
||||
github.com/go-openapi/swag/yamlutils v0.25.4 h1:6jdaeSItEUb7ioS9lFoCZ65Cne1/RZtPBZ9A56h92Sw=
|
||||
github.com/go-openapi/swag/yamlutils v0.25.4/go.mod h1:MNzq1ulQu+yd8Kl7wPOut/YHAAU/H6hL91fF+E2RFwc=
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY=
|
||||
github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
|
||||
github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
|
||||
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
|
||||
go.mongodb.org/mongo-driver v1.17.6 h1:87JUG1wZfWsr6rIz3ZmpH90rL5tea7O3IHuSwHUpsss=
|
||||
go.mongodb.org/mongo-driver v1.17.6/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ=
|
||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
|
||||
golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
|
||||
golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
|
||||
golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
|
||||
golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
|
||||
golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
59
main.go
Normal file
59
main.go
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"go.yaml.in/yaml/v4"
|
||||
)
|
||||
|
||||
type Forge struct {
|
||||
Name string `yaml:"name"`
|
||||
Url string `yaml:"url"`
|
||||
Type string `yaml:"type"`
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
Forges []Forge `yaml:"forges"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
data, err := os.ReadFile("config.yaml")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
var config Config
|
||||
|
||||
if err := yaml.Unmarshal(data, &config); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
var projects []Project
|
||||
var allProjects []Project
|
||||
for _, forge := range config.Forges {
|
||||
fmt.Printf("%+v: %+v\n", forge.Name, forge.Url)
|
||||
|
||||
switch forge.Type {
|
||||
case "forgejo":
|
||||
scraper, err := NewForgejoScraper(forge.Url)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
projects, err := scraper.GetProjects()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
default:
|
||||
projects = []Project{}
|
||||
fmt.Errorf("unknown forge: %s", forge.Type)
|
||||
}
|
||||
|
||||
fmt.Printf("Found %d repositories\n", len(projects))
|
||||
|
||||
allProjects = append(allProjects, projects...)
|
||||
|
||||
for repo := range allProjects {
|
||||
fmt.Printf("%+v\n", repo)
|
||||
}
|
||||
}
|
||||
}
|
||||
57
option.nix
Normal file
57
option.nix
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
{
|
||||
config,
|
||||
lib,
|
||||
pkgs,
|
||||
options,
|
||||
...
|
||||
}:
|
||||
|
||||
let
|
||||
cfg = config.services.federated-forge-discovery.indexer;
|
||||
configFile = pkgs.writeText "config.ini" (pkgs.lib.generators.toINI { } cfg);
|
||||
execCommand = "${cfg.package}/bin/ffd-indexer -c '${configFile}'";
|
||||
in
|
||||
{
|
||||
options.services.federated-forge-discovery.indexer = {
|
||||
enable = lib.mkEnableOption "Enables the Federated Forge Discovery Indexer systemd service.";
|
||||
openFirewall = lib.mkOption {
|
||||
|
||||
};
|
||||
package = lib.mkOption {
|
||||
description = "Package to use.";
|
||||
type = lib.types.package;
|
||||
default = lib.literalExpression ''
|
||||
pkgs.ffd-indexer
|
||||
'';
|
||||
};
|
||||
port = lib.mkOption {
|
||||
description = "Port to listen for search engines";
|
||||
type = lib.types.port;
|
||||
default = 47810;
|
||||
};
|
||||
};
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
systemd.services."ffd-indexer" = {
|
||||
description = "Federated Forge Discovery Indexer";
|
||||
wants = [ "network-online.target" ];
|
||||
after = [ "network-online.target" ];
|
||||
serviceConfig = {
|
||||
Restart = "always";
|
||||
RestartSec = 30;
|
||||
ExecStart = execCommand;
|
||||
User = "ffd-indexer";
|
||||
Group = "ffd-indexer";
|
||||
AmbientCapabilities = lib.mkIf (cfg.port < 1000) [ "CAP_NET_BIND_SERVICE" ];
|
||||
CapabilityBoundingSet = lib.mkIf (cfg.port < 1000) [ "CAP_NET_BIND_SERVICE" ];
|
||||
NoNewPrivileges = true;
|
||||
};
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
};
|
||||
};
|
||||
|
||||
#meta = {
|
||||
# maintainers = with lib.maintainers; [ xengi ];
|
||||
# doc = ./default.xml;
|
||||
#};
|
||||
}
|
||||
23
scraper.go
Normal file
23
scraper.go
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
type Project struct {
|
||||
Name string `json:"name"`
|
||||
Url string `json:"url"`
|
||||
Description string `json:"description"`
|
||||
Topics []string `json:"topics"`
|
||||
OpenIssues int `json:"open_issues"`
|
||||
ClosedIssues int `json:"closed_issues"`
|
||||
OpenPRs int `json:"open_prs"`
|
||||
ClosedPRs int `json:"closed_prs"`
|
||||
LatestCommit time.Time `json:"latest_commit"`
|
||||
LatestTag time.Time `json:"latest_tag"`
|
||||
LatestRelease time.Time `json:"latest_release"`
|
||||
Contributors int `json:"contributors"`
|
||||
Languages []string `json:"languages"`
|
||||
Readme string `json:"readme"`
|
||||
Licence string `json:"licence"`
|
||||
}
|
||||
Loading…
Reference in a new issue