63c6f4b761
Some checks failed
CI pipeline for ffd-indexer / checks (go test + golangci) (push) Failing after 26s
CI pipeline for ffd-indexer / build binaries + containers (push) Failing after 3s
CI pipeline for ffd-indexer / format (gofmt + nixfmt) (push) Failing after 3s
CI pipeline for ffd-indexer / integration (live scrape + Typesense) (push) Failing after 20s
102 lines
2.7 KiB
Go
102 lines
2.7 KiB
Go
package scraper
|
|
|
|
import (
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"testing"
|
|
)
|
|
|
|
// newDetectServer serves a robots.txt that allows /api/ plus the given
|
|
// version paths. Returns the server's fqdn (host:port).
|
|
func newDetectServer(t *testing.T, versionPaths ...string) string {
|
|
t.Helper()
|
|
mux := http.NewServeMux()
|
|
mux.HandleFunc("/robots.txt", func(w http.ResponseWriter, r *http.Request) {
|
|
_, _ = fmt.Fprintln(w, "User-agent: *")
|
|
_, _ = fmt.Fprintln(w, "Allow: /")
|
|
})
|
|
for _, p := range versionPaths {
|
|
p := p
|
|
mux.HandleFunc(p, func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_, _ = fmt.Fprintln(w, `{"version":"16.0.5"}`)
|
|
})
|
|
}
|
|
srv := httptest.NewServer(mux)
|
|
t.Cleanup(srv.Close)
|
|
return srv.Listener.Addr().String()
|
|
}
|
|
|
|
func TestDetectForgejo(t *testing.T) {
|
|
// Forgejo serves BOTH endpoints; the Forgejo-native one must win.
|
|
fqdn := newDetectServer(t, "/api/forgejo/v1/version", "/api/v1/version")
|
|
|
|
typ, version, err := detectAt("http://" + fqdn)
|
|
if err != nil {
|
|
t.Fatalf("detect: %v", err)
|
|
}
|
|
if typ != TypeForgejo {
|
|
t.Errorf("expected TypeForgejo, got %q", typ)
|
|
}
|
|
if version != "16.0.5" {
|
|
t.Errorf("expected version 16.0.5, got %q", version)
|
|
}
|
|
}
|
|
|
|
func TestDetectGitea(t *testing.T) {
|
|
// Gitea serves only the shared /api/v1/version endpoint.
|
|
fqdn := newDetectServer(t, "/api/v1/version")
|
|
|
|
typ, version, err := detectAt("http://" + fqdn)
|
|
if err != nil {
|
|
t.Fatalf("detect: %v", err)
|
|
}
|
|
if typ != TypeGitea {
|
|
t.Errorf("expected TypeGitea, got %q", typ)
|
|
}
|
|
if version != "16.0.5" {
|
|
t.Errorf("expected version 16.0.5, got %q", version)
|
|
}
|
|
}
|
|
|
|
func TestDetectUnknown(t *testing.T) {
|
|
// A server that serves robots but no version endpoint.
|
|
fqdn := newDetectServer(t)
|
|
|
|
typ, version, err := detectAt("http://" + fqdn)
|
|
if err != nil {
|
|
t.Fatalf("detect: %v", err)
|
|
}
|
|
if typ != TypeUnknown {
|
|
t.Errorf("expected TypeUnknown, got %q", typ)
|
|
}
|
|
if version != "" {
|
|
t.Errorf("expected empty version, got %q", version)
|
|
}
|
|
}
|
|
|
|
func TestDetectHonorsRobotsDisallow(t *testing.T) {
|
|
mux := http.NewServeMux()
|
|
mux.HandleFunc("/robots.txt", func(w http.ResponseWriter, r *http.Request) {
|
|
_, _ = fmt.Fprintln(w, "User-agent: *")
|
|
_, _ = fmt.Fprintln(w, "Disallow: /api/")
|
|
})
|
|
mux.HandleFunc("/api/forgejo/v1/version", func(w http.ResponseWriter, r *http.Request) {
|
|
_, _ = fmt.Fprintln(w, `{"version":"16.0.5"}`)
|
|
})
|
|
srv := httptest.NewServer(mux)
|
|
t.Cleanup(srv.Close)
|
|
fqdn := srv.Listener.Addr().String()
|
|
|
|
typ, version, err := detectAt("http://" + fqdn)
|
|
if err != nil {
|
|
t.Fatalf("detect: %v", err)
|
|
}
|
|
if typ != TypeUnknown {
|
|
t.Errorf("expected TypeUnknown when /api/ is disallowed, got %q", typ)
|
|
}
|
|
if version != "" {
|
|
t.Errorf("expected empty version when disallowed, got %q", version)
|
|
}
|
|
}
|