indexer/main.go

package main

import (
	"context"
	"flag"
	"fmt"
	"os"
	"time"

	"github.com/typesense/typesense-go/v2/typesense"
	"go.yaml.in/yaml/v4"

	"git.xengi.de/ffd/indexer/config"
	"git.xengi.de/ffd/indexer/internal/version"
	"git.xengi.de/ffd/indexer/scraper"
	"git.xengi.de/ffd/indexer/store"
	"git.xengi.de/ffd/indexer/sync"
)

func main() {
	syncOnly := flag.Bool("sync-only", false, "rebuild the Typesense index from the existing DB without scraping")
	configPath := flag.String("config", "config.yaml", "path to the configuration file")
	showVersion := flag.Bool("version", false, "print version and commit, then exit")
	flag.Parse()

	if *showVersion {
		version.PrintFull()
		return
	}

	data, err := os.ReadFile(*configPath)
	if err != nil {
		panic(err)
	}
	var config config.Config

	if err := yaml.Unmarshal(data, &config); err != nil {
		panic(err)
	}

	db, err := store.Open(config.DbPath)
	if err != nil {
		panic(err)
	}
	defer func() { _ = db.Close() }()

	// Resolve the Typesense API key once: api_key_file (a sops/agenix secret)
	// takes precedence over the inline api_key.
	typesenseAPIKey, err := config.Typesense.ResolveAPIKey()
	if err != nil {
		panic(err)
	}

	if *syncOnly {
		if err := syncToTypesense(config.Typesense.Host, typesenseAPIKey, db); err != nil {
			fmt.Fprintf(os.Stderr, "typesense sync failed: %v\n", err)
			os.Exit(1)
		}
		fmt.Println("sync-only complete")
		return
	}

	for _, forge := range config.Forges {
		if err := forge.Validate(); err != nil {
			fmt.Fprintf(os.Stderr, "  skip: %v\n", err)
			continue
		}
		// Split the configured address into a connection scheme ("https" unless
		// the user forced http) and the bare FQDN. The FQDN is the consistent
		// identity across forge_status, project identities and the metrics
		// labels; the scheme only affects how we connect.
		scheme, fqdn := scraper.Scheme(forge.URL)

		lastStatus, err := db.ForgeStatus(fqdn)
		if err != nil {
			fmt.Fprintf(os.Stderr, "  status lookup: %v\n", err)
			continue
		}

		// Determine the forge type. An explicit type in the config (forgejo or
		// gitea) is used as-is and skips detection. "auto" probes the
		// Forgejo-native version endpoint first (Forgejo also serves the
		// shared one), then the shared /api/v1/version which implies Gitea. If
		// it stays unknown we cannot scrape it (no scraper for that type yet)
		// and record it as offline, keeping any previously recorded
		// type/version.
		var forgeType scraper.Type
		var version string
		if forge.Type != "auto" {
			forgeType = scraper.Type(forge.Type)
		} else {
			forgeType, version, err = scraper.DetectType(scheme, fqdn)
			if err != nil {
				fmt.Fprintf(os.Stderr, "  skip %s: detect type: %v\n", fqdn, err)
				recordStatus(db, store.ForgeStatus{
					URL: fqdn, ForgeType: lastStatus.ForgeType, Version: lastStatus.Version,
					Online: false, LastScrapedAt: time.Now().UTC(), LastError: err.Error(),
				})
				continue
			}
			if forgeType == scraper.TypeUnknown {
				fmt.Fprintf(os.Stderr, "  skip %s: unknown forge type\n", fqdn)
				recordStatus(db, store.ForgeStatus{
					URL: fqdn, ForgeType: lastStatus.ForgeType, Version: lastStatus.Version,
					Online: false, LastScrapedAt: time.Now().UTC(), LastError: "unknown forge type",
				})
				continue
			}
		}

		fmt.Printf("Scraping %s (%s, %s)\n", fqdn, forgeType, version)
		record := func(online bool, errMsg string) {
			recordStatus(db, store.ForgeStatus{
				URL:           fqdn,
				ForgeType:     string(forgeType),
				Version:       version,
				Online:        online,
				LastScrapedAt: time.Now().UTC(),
				LastError:     errMsg,
			})
		}

		s, err := scraper.NewScraper(forgeType, scheme, fqdn)
		if err != nil {
			fmt.Fprintf(os.Stderr, "  skip %s: %v\n", fqdn, err)
			record(false, err.Error())
			continue
		}

		projects, err := s.GetProjects()
		if err != nil {
			fmt.Fprintf(os.Stderr, "  skip %s: %v\n", fqdn, err)
			record(false, err.Error())
			continue
		}

		fmt.Printf("  found %d projects\n", len(projects))
		for _, p := range projects {
			if err := db.Upsert(p); err != nil {
				fmt.Fprintf(os.Stderr, "    store %s: %v\n", p.Name, err)
				continue
			}
			fmt.Printf("    %s\n", p.Name)
		}

		all, err := db.All()
		if err != nil {
			fmt.Fprintf(os.Stderr, "  count: %v\n", err)
			record(true, "")
			continue
		}
		fmt.Printf("  total in db: %d\n", len(all))

		record(true, "")
	}

	// Index the stored projects into Typesense for the search UI.
	// A failure here is fatal: the scrape itself survived per-forge / per-repo
	// errors, but if we cannot put the data into Typesense the run is not
	// complete and should fail loudly (e.g. for systemd to catch).
	if err := syncToTypesense(config.Typesense.Host, typesenseAPIKey, db); err != nil {
		fmt.Fprintf(os.Stderr, "typesense sync failed: %v\n", err)
		os.Exit(1)
	}
}

// syncToTypesense pushes the store's projects into the Typesense index.
func syncToTypesense(host, apiKey string, db *store.Store) error {
	if host == "" || apiKey == "" {
		return fmt.Errorf("typesense host or API key not configured")
	}

	client := typesense.NewClient(
		typesense.WithServer(host),
		typesense.WithAPIKey(apiKey),
	)
	svc := sync.NewService(client)

	if err := svc.Sync(context.Background(), db); err != nil {
		return err
	}
	fmt.Printf("synced projects to Typesense (%s)\n", host)
	return nil
}

// recordStatus writes a forge status row, logging (but not failing on)
// storage errors.
func recordStatus(db *store.Store, st store.ForgeStatus) {
	if err := db.RecordForgeStatus(st); err != nil {
		fmt.Fprintf(os.Stderr, "  record status: %v\n", err)
	}
}