package main import ( "context" "flag" "fmt" "os" "time" "github.com/typesense/typesense-go/v2/typesense" "go.yaml.in/yaml/v4" "git.xengi.de/ffd/indexer/config" "git.xengi.de/ffd/indexer/internal/version" "git.xengi.de/ffd/indexer/scraper" "git.xengi.de/ffd/indexer/store" "git.xengi.de/ffd/indexer/sync" ) func main() { syncOnly := flag.Bool("sync-only", false, "rebuild the Typesense index from the existing DB without scraping") configPath := flag.String("config", "config.yaml", "path to the configuration file") showVersion := flag.Bool("version", false, "print version and commit, then exit") flag.Parse() if *showVersion { version.PrintFull() return } data, err := os.ReadFile(*configPath) if err != nil { panic(err) } var config config.Config if err := yaml.Unmarshal(data, &config); err != nil { panic(err) } db, err := store.Open(config.DbPath) if err != nil { panic(err) } defer func() { _ = db.Close() }() // Resolve the Typesense API key once: api_key_file (a sops/agenix secret) // takes precedence over the inline api_key. typesenseAPIKey, err := config.Typesense.ResolveAPIKey() if err != nil { panic(err) } if *syncOnly { if err := syncToTypesense(config.Typesense.Host, typesenseAPIKey, db); err != nil { fmt.Fprintf(os.Stderr, "typesense sync failed: %v\n", err) os.Exit(1) } fmt.Println("sync-only complete") return } for _, forge := range config.Forges { if err := forge.Validate(); err != nil { fmt.Fprintf(os.Stderr, " skip: %v\n", err) continue } // Split the configured address into a connection scheme ("https" unless // the user forced http) and the bare FQDN. The FQDN is the consistent // identity across forge_status, project identities and the metrics // labels; the scheme only affects how we connect. scheme, fqdn := scraper.Scheme(forge.URL) lastStatus, err := db.ForgeStatus(fqdn) if err != nil { fmt.Fprintf(os.Stderr, " status lookup: %v\n", err) continue } // Determine the forge type. An explicit type in the config (forgejo or // gitea) is used as-is and skips detection. "auto" probes the // Forgejo-native version endpoint first (Forgejo also serves the // shared one), then the shared /api/v1/version which implies Gitea. If // it stays unknown we cannot scrape it (no scraper for that type yet) // and record it as offline, keeping any previously recorded // type/version. var forgeType scraper.Type var version string if forge.Type != "auto" { forgeType = scraper.Type(forge.Type) } else { forgeType, version, err = scraper.DetectType(scheme, fqdn) if err != nil { fmt.Fprintf(os.Stderr, " skip %s: detect type: %v\n", fqdn, err) recordStatus(db, store.ForgeStatus{ URL: fqdn, ForgeType: lastStatus.ForgeType, Version: lastStatus.Version, Online: false, LastScrapedAt: time.Now().UTC(), LastError: err.Error(), }) continue } if forgeType == scraper.TypeUnknown { fmt.Fprintf(os.Stderr, " skip %s: unknown forge type\n", fqdn) recordStatus(db, store.ForgeStatus{ URL: fqdn, ForgeType: lastStatus.ForgeType, Version: lastStatus.Version, Online: false, LastScrapedAt: time.Now().UTC(), LastError: "unknown forge type", }) continue } } fmt.Printf("Scraping %s (%s, %s)\n", fqdn, forgeType, version) record := func(online bool, errMsg string) { recordStatus(db, store.ForgeStatus{ URL: fqdn, ForgeType: string(forgeType), Version: version, Online: online, LastScrapedAt: time.Now().UTC(), LastError: errMsg, }) } s, err := scraper.NewScraper(forgeType, scheme, fqdn) if err != nil { fmt.Fprintf(os.Stderr, " skip %s: %v\n", fqdn, err) record(false, err.Error()) continue } projects, err := s.GetProjects() if err != nil { fmt.Fprintf(os.Stderr, " skip %s: %v\n", fqdn, err) record(false, err.Error()) continue } fmt.Printf(" found %d projects\n", len(projects)) for _, p := range projects { if err := db.Upsert(p); err != nil { fmt.Fprintf(os.Stderr, " store %s: %v\n", p.Name, err) continue } fmt.Printf(" %s\n", p.Name) } all, err := db.All() if err != nil { fmt.Fprintf(os.Stderr, " count: %v\n", err) record(true, "") continue } fmt.Printf(" total in db: %d\n", len(all)) record(true, "") } // Index the stored projects into Typesense for the search UI. // A failure here is fatal: the scrape itself survived per-forge / per-repo // errors, but if we cannot put the data into Typesense the run is not // complete and should fail loudly (e.g. for systemd to catch). if err := syncToTypesense(config.Typesense.Host, typesenseAPIKey, db); err != nil { fmt.Fprintf(os.Stderr, "typesense sync failed: %v\n", err) os.Exit(1) } } // syncToTypesense pushes the store's projects into the Typesense index. func syncToTypesense(host, apiKey string, db *store.Store) error { if host == "" || apiKey == "" { return fmt.Errorf("typesense host or API key not configured") } client := typesense.NewClient( typesense.WithServer(host), typesense.WithAPIKey(apiKey), ) svc := sync.NewService(client) if err := svc.Sync(context.Background(), db); err != nil { return err } fmt.Printf("synced projects to Typesense (%s)\n", host) return nil } // recordStatus writes a forge status row, logging (but not failing on) // storage errors. func recordStatus(db *store.Store, st store.ForgeStatus) { if err := db.RecordForgeStatus(st); err != nil { fmt.Fprintf(os.Stderr, " record status: %v\n", err) } }