🔥

2026-10-01 00:37:44 +02:00 · 2026-10-01 00:37:44 +02:00 · 63c6f4b761
commit 63c6f4b761
parent bde13e9b68
32 changed files with 3319 additions and 184 deletions
--- a/.editorconfig
+++ b/.editorconfig
@ -4,6 +4,7 @@ root = true
 [*]
 end_of_line = lf
 insert_final_newline = true
+max_line_length = 116

 [*.{go,nix,yaml}]
 charset = utf-8
--- a/.forgejo/workflows/ci.yaml
+++ b/.forgejo/workflows/ci.yaml
@ -0,0 +1,159 @@
+name: CI pipeline for ffd-indexer
+
+on:
+  push:
+    branches: ["**"]
+  pull_request:
+    branches: ["**"]
+  workflow_dispatch:
+
+jobs:
+  checks:
+    name: checks (go test + golangci)
+    runs-on: debian-stable
+    container:
+      image: nixos/nix:latest
+    steps:
+      - name: Checkout
+        run: |
+          set -euo pipefail
+          git config --global --add safe.directory "$PWD"
+          ref="${FORGEJO_HEAD_REF:-$FORGEJO_REF_NAME}"
+          git clone --depth 1 --branch "$ref" "${FORGEJO_SERVER_URL}/${FORGEJO_REPOSITORY}.git" .
+      # `nix flake check` builds and runs the flake's `checks`:
+      #   go-test  -> go test -v ./...
+      #   go-lint  -> golangci-lint run
+      - name: nix flake check
+        run: nix flake check --accept-flake-config
+
+  build:
+    name: build binaries + containers
+    runs-on: debian-stable
+    container:
+      image: nixos/nix:latest
+    steps:
+      - name: Checkout
+        run: |
+          set -euo pipefail
+          git config --global --add safe.directory "$PWD"
+          ref="${FORGEJO_HEAD_REF:-$FORGEJO_REF_NAME}"
+          git clone --depth 1 --branch "$ref" "${FORGEJO_SERVER_URL}/${FORGEJO_REPOSITORY}.git" .
+      # Build every deliverable: the indexer binary, the metrics daemon and
+      # both container images. typesense-server is a local dev helper and is
+      # deliberately not built here.
+      - name: nix build
+        run: nix build --accept-flake-config .#default .#ffd-metrics .#container .#container-metrics
+
+  format:
+    name: format (gofmt + nixfmt)
+    runs-on: debian-stable
+    container:
+      image: nixos/nix:latest
+    steps:
+      - name: Checkout
+        run: |
+          set -euo pipefail
+          git config --global --add safe.directory "$PWD"
+          ref="${FORGEJO_HEAD_REF:-$FORGEJO_REF_NAME}"
+          git clone --depth 1 --branch "$ref" "${FORGEJO_SERVER_URL}/${FORGEJO_REPOSITORY}.git" .
+      - name: gofmt
+        run: |
+          nix develop --command bash -c '
+            unformatted="$(gofmt -l .)"
+            if [ -n "$unformatted" ]; then
+              echo "gofmt needed for:"
+              printf "%s\n" "$unformatted"
+              exit 1
+            fi
+            echo "gofmt: clean"
+          '
+      - name: nixfmt
+        run: |
+          git config --global --add safe.directory "$PWD"
+          nix fmt --accept-flake-config >/dev/null
+          if ! git diff --quiet -- '*.nix'; then
+            echo "nixfmt would reformat these files:"
+            git diff --name-only -- '*.nix'
+            exit 1
+          fi
+          echo "nixfmt: clean"
+
+  integration:
+    name: integration (live scrape + Typesense)
+    runs-on: debian-stable
+    container:
+      image: nixos/nix:latest
+    services:
+      typesense:
+        image: typesense/typesense:30.2
+        env:
+          TYPESENSE_API_KEY: dev-key
+          TYPESENSE_DATA_DIR: /data
+          TYPESENSE_API_PORT: "8108"
+        ports:
+          - "8108:8108"
+        options: >-
+          --tmpfs /data:rw,noexec,nosuid,size=256m
+    steps:
+      - name: Checkout
+        run: |
+          set -euo pipefail
+          git config --global --add safe.directory "$PWD"
+          ref="${FORGEJO_HEAD_REF:-$FORGEJO_REF_NAME}"
+          git clone --depth 1 --branch "$ref" "${FORGEJO_SERVER_URL}/${FORGEJO_REPOSITORY}.git" .
+      # Scrape git.xengi.de live, sync into the Typesense service, then assert
+      # end to end. Runs inside the flake's dev shell so sqlite3 and the Go
+      # toolchain are available; the typesense service is reached at the
+      # service hostname `typesense` on port 8108.
+      - name: Integration (scrape + sync + assert)
+        run: |
+          set -euo pipefail
+
+          # Config: point Typesense at the service container.
+          cat > ci-config.yaml <<'EOF'
+          db_path: "./ci-index.db"
+          typesense:
+            host: "http://typesense:8108"
+            api_key: "dev-key"
+          forges:
+            - url: git.xengi.de
+              type: auto
+          EOF
+
+          # Build the binary from the flake.
+          BIN="$(nix build .#default --accept-flake-config --no-link --print-out-paths)/bin/ffd-indexer"
+
+          # Run the whole assertion as one heredoc inside the dev shell.
+          nix develop --command bash <<'EOF'
+          set -euo pipefail
+
+          # Wait for Typesense to be healthy
+          for _ in $(seq 1 30); do
+            [ "$(curl -fsS -o /dev/null -w '%{http_code}' http://typesense:8108/health 2>/dev/null || echo 000)" = "200" ] && break
+            sleep 2
+          done
+          code="$(curl -fsS -o /dev/null -w '%{http_code}' http://typesense:8108/health 2>/dev/null || echo 000)"
+          [ "$code" = "200" ] || { echo "typesense not healthy (http $code)"; exit 1; }
+
+          # Scrape + sync (fails loudly on error).
+          "$BIN" -config ci-config.yaml
+
+          # sqlite: forge online + at least one project.
+          online="$(sqlite3 ci-index.db "SELECT 1 FROM forge_status WHERE url='git.xengi.de' AND online=1 LIMIT 1;")"
+          [ "$online" = "1" ] || { echo "forge git.xengi.de not online"; exit 1; }
+          projects="$(sqlite3 ci-index.db "SELECT COUNT(*) FROM projects;")"
+          echo "projects in sqlite: $projects"
+          [ "${projects:-0}" -ge 1 ] || { echo "no projects stored"; exit 1; }
+
+          # Typesense: collection exists and doc count == sqlite count.
+          curl -fsS -H "X-TYPESENSE-API-KEY: dev-key" http://typesense:8108/collections/projects >/dev/null
+          # python3 is not in the job image; extract `found` from the search JSON
+          # with grep (Typesense reports the total in a top-level "found" field).
+          found="$(curl -fsS -H "X-TYPESENSE-API-KEY: dev-key" \
+            "http://typesense:8108/collections/projects/documents/search?q=*&per_page=1" \
+            | grep -o '"found": *[0-9]*' | grep -o '[0-9][0-9]*' | head -1)"
+          echo "documents in Typesense: $found (sqlite: $projects)"
+          [ "$found" = "$projects" ] || { echo "Typesense count mismatch"; exit 1; }
+
+          echo "integration OK: $projects projects scraped from git.xengi.de and indexed"
+          EOF
--- a/.forgejo/workflows/container.yaml
+++ b/.forgejo/workflows/container.yaml
@ -0,0 +1,70 @@
+name: Build and publish containers on tags
+
+on:
+  push:
+    tags: ["**"]
+  workflow_dispatch:
+
+jobs:
+  container:
+    name: build + publish ffd-indexer / ffd-metrics
+    runs-on: debian-stable
+    env:
+      # Forgejo does not auto-expose secrets as env vars; map it explicitly
+      # so the whole step (including the inner heredoc) can read it.
+      REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
+    container:
+      image: nixos/nix:latest
+    steps:
+      - name: Checkout
+        run: |
+          set -euo pipefail
+          git config --global --add safe.directory "$PWD"
+          ref="${FORGEJO_HEAD_REF:-$FORGEJO_REF_NAME}"
+          git clone --depth 1 --branch "$ref" "${FORGEJO_SERVER_URL}/${FORGEJO_REPOSITORY}.git" .
+
+      # Build the two OCI images from the flake:
+      #   .#container          -> ffd-indexer  (the scraper)
+      #   .#container-metrics  -> ffd-metrics  (the metrics daemon)
+      # No container daemon/buildkit is needed: `nix develop` provides
+      # `skopeo`, which copies the resulting image tarballs straight to the
+      # Forgejo container registry.
+      #
+      # Both containers are pushed with the SAME tag; their names differ:
+      #   git.xengi.de/ffd/indexer:<tag>           ffd-indexer
+      #   git.xengi.de/ffd/indexer-metrics:<tag>   ffd-metrics
+      # Each also gets a `:latest` on its own name.
+      - name: Build and publish containers
+        run: |
+          set -euo pipefail
+
+          # Image tag: the git tag that triggered this run, without a leading
+          # "v" (e.g. v1.2.3 -> 1.2.3).
+          ref="${FORGEJO_REF_NAME:?}"
+          tag="${ref##*/}"   # drop any refs/tags/ prefix
+          tag="${tag#v}"     # drop a leading v
+          [ -n "$tag" ] || { echo "empty tag" >&2; exit 1; }
+
+          registry="${FORGEJO_SERVER_URL#https://}" # git.xengi.de
+          owner="${FORGEJO_REPOSITORY_OWNER:?}"     # ffd
+          repo="${FORGEJO_REPOSITORY##*/}"          # indexer
+          token="${REGISTRY_TOKEN:?}"
+          echo "publishing tag=$tag to $registry/$owner"
+
+          export registry owner repo tag token
+          nix develop --command bash <<'INNER'
+          set -euo pipefail
+
+          idx="$(nix build .#container --accept-flake-config --no-link --print-out-paths)"
+          met="$(nix build .#container-metrics --accept-flake-config --no-link --print-out-paths)"
+
+          # Same tag for both images, distinct names:
+          skopeo copy --dest-creds "$owner:$token" \
+            "docker-archive:$idx" "docker://$registry/$owner/$repo:$tag"
+          skopeo copy --dest-creds "$owner:$token" \
+            "docker-archive:$idx" "docker://$registry/$owner/$repo:latest"
+          skopeo copy --dest-creds "$owner:$token" \
+            "docker-archive:$met" "docker://$registry/$owner/${repo}-metrics:$tag"
+          skopeo copy --dest-creds "$owner:$token" \
+            "docker-archive:$met" "docker://$registry/$owner/${repo}-metrics:latest"
+          INNER
--- a/.gitignore
+++ b/.gitignore
@ -3,6 +3,7 @@ models.json
 Containerfile
 result
 index.db
+.typesense/

 # Created by https://www.toptal.com/developers/gitignore/api/windows,linux,macos,direnv,vim,go,goland+all
 # Edit at https://www.toptal.com/developers/gitignore?templates=windows,linux,macos,direnv,vim,go,goland+all
--- a/DOCS.md
+++ b/DOCS.md
@ -0,0 +1,423 @@
+# ffd-indexer — documentation and reference
+
+This file holds the reference material for the **Federated Forge Discovery
+Indexer**: how scraping works, the exact request volume, the full configuration
+reference, the Prometheus metric reference and PromQL queries, the supported
+forge types and type detection, the NixOS module, and development setup.
+
+See [README.md](README.md) for the project's purpose and quick start.
+
+---
+
+## Table of contents
+
+- [Architecture](#architecture)
+- [Scrape behavior](#scrape-behavior)
+  - [Robots.txt](#robotstxt)
+  - [Listing repositories](#listing-repositories)
+  - [Enrichment](#enrichment)
+  - [Request volume](#request-volume)
+  - [Error handling](#error-handling)
+- [HTTP transport](#http-transport)
+  - [HTTP/2](#http2)
+  - [HTTP/3](#http3)
+  - [Forcing a protocol](#forcing-a-protocol)
+- [Configuration reference](#configuration-reference)
+- [Prometheus metrics](#prometheus-metrics)
+  - [Metric reference](#metric-reference)
+  - [PromQL queries](#promql-queries)
+  - [Metrics daemon](#metrics-daemon)
+- [Supported forge types](#supported-forge-types)
+  - [Type detection](#type-detection)
+  - [Detection detail](#detection-detail)
+- [NixOS module](#nixos-module)
+- [Development](#development)
+
+---
+
+## Architecture
+
+Two binaries, a SQLite store, and an external search UI:
+
+```mermaid
+flowchart TD
+    subgraph Forges["Configured forges"]
+        F1["forge A"]
+        F2["forge B"]
+    end
+
+    subgraph Indexer["ffd-indexer · one-shot scrape"]
+        IV["robots.txt check \n repo listing \n per-repo enrichment"]
+    end
+
+    subgraph Metrics["ffd-metrics · long-running daemon"]
+        MV["render store as Prometheus metrics"]
+    end
+
+    DB[("SQLite \n projects · forge_status")]
+    TS["Typesense \n search index"]
+    PROM["Prometheus"]
+    UI["Search UI"]
+
+    F1 --> IV
+    F2 --> IV
+    IV -->|writes| DB
+    IV -->|syncs index| TS
+    DB -->|reads on each scrape| MV
+    MV -->|/metrics| PROM
+    UI -->|queries| TS
+```
+
+*(GitHub renders Mermaid diagrams natively in Markdown; no image to maintain.)*
+
+- **`ffd-indexer`** — the scraper. Runs once per schedule, honors robots.txt,
+  scrapes every configured forge, records per-forge status and upserts each
+  project into SQLite, then syncs the index to Typesense.
+- **`ffd-metrics`** — a long-running daemon. Reads the SQLite store on every
+  Prometheus scrape and renders it at `/metrics`. It **never** touches the
+  forges itself.
+- **SQLite** (`db_path`) — the source of truth. Holds the `projects` table
+  (with per-project forge identity and languages) and the `forge_status` table
+  (per-forge online flag, detected type, software version, last error).
+- **Typesense** (`typesense`) — optional full-text search index over the
+  stored projects, searched by the UI.
+- **Search UI** — a frontend that queries the Typesense index (not shipped in
+  this repository); the indexer's sync keeps the collection searchable by it.
+
+### Binaries
+
+| Output | Package path | What it is |
+|---|---|---|
+| `ffd-indexer` | `git.xengi.de/ffd/indexer` | one-shot scraper + sync |
+| `ffd-metrics` | `git.xengi.de/ffd/indexer/cmd/ffd-metrics` | long-running metrics daemon |
+
+Both are built from the flake (`nix build .#default .#ffd-metrics`); the
+default package installs both binaries.
+
+---
+
+## Scrape behavior
+
+The indexer connects to each forge over HTTPS, trying to negotiate HTTP/2 via
+ALPN for lower latency and multiplexing, with HTTP/1.1 as the fallback. A
+single shared transport is used for robots checks, type detection and the
+scrape itself, so keep-alive connections are reused across a host's requests.
+(Full transport detail: [HTTP transport](#http-transport).)
+
+### Robots.txt
+
+Before any API call, the indexer fetches `GET /robots.txt` and consults the
+rules for the `User-Agent: *` group. If the instance disallows `/api/` (the
+API root), the instance is **skipped** — robots rules are honored for both
+scraping and type detection. A missing robots.txt (404) is treated as
+"allowed".
+
+### Listing repositories
+
+Public source repositories are listed via the repository search API
+(`GET /api/v1/repos/search`), paginated at 50 per page. Forks and mirrors are
+excluded.
+
+### Enrichment
+
+For each repository, the indexer:
+
+- reads the repository root directory **once** (`GET /api/v1/repos/{o}/{r}/contents/`);
+  this single listing is shared by README selection and license detection;
+- fetches topics, languages, latest commit, and latest release;
+- fetches the README (raw); and
+- detects SPDX licenses from the license files present in the repo.
+
+### Request volume
+
+**Per instance:**
+
+| Request | Count |
+|---|---|
+| `GET /robots.txt` | 1 |
+| `GET /api/v1/repos/search` (page) | `ceil(N/50)` where N = total repos |
+
+**Per repository:**
+
+| Request | Endpoint | Purpose |
+|---|---|---|
+| list root contents | `GET /api/v1/repos/{o}/{r}/contents/` | shared: README selection + license scan (1 call) |
+| languages | `GET /api/v1/repos/{o}/{r}/languages` | `Project.Languages` |
+| topics | `GET /api/v1/repos/{o}/{r}/topics` | `Project.Topics` |
+| latest commit | `GET /api/v1/repos/{o}/{r}/commits?limit=1` | `Project.LatestCommit` |
+| latest release | `GET /api/v1/repos/{o}/{r}/releases/latest` | `Project.LatestRelease` |
+| README | `GET /api/v1/repos/{o}/{r}/raw/{branch}/{file}` | `Project.Readme` |
+| license files | 1+ × `GET /api/v1/repos/{o}/{r}/raw/{branch}/{file}` | SPDX detection |
+
+So roughly **6–8 API requests per repository**, plus the paginated search
+requests and a single robots.txt check per instance.
+
+### Error handling
+
+Errors are non-fatal: an instance or a single repo that fails is skipped and
+the run continues. A failed forge is still recorded in `forge_status` with
+`online=0` and its last error, so the metrics daemon always has a row per
+configured forge. A failure to sync Typesense at the end of a run is fatal
+(loud failure), so the scheduler can retry the run.
+
+---
+
+## HTTP transport
+
+The shared transport always tries to use the **newest HTTP version available**
+and automatically falls back when it isn't available:
+
+- keep-alive connections are reused across a host's requests (pagination and
+  per-repo enrichment don't re-handshake);
+- HTTP/1.1 vs HTTP/2 is negotiated automatically by Go via ALPN
+  (`ForceAttemptHTTP2` is set because the TLS config is custom);
+- HTTP/3 is tried first (per host) and falls back to HTTP/2 or HTTP/1.1 when
+  the host doesn't have it.
+
+### HTTP/2
+
+HTTP/2 is negotiated over TLS via ALPN whenever the server supports it
+(verified against live forges), falling back to HTTP/1.1 otherwise. This gives
+multiplexing and lower latency without any per-host configuration.
+
+### HTTP/3
+
+HTTP/3 is supported via `github.com/quic-go/quic-go/http3` and is tried
+**first** on each host automatically — no manual opt-in is needed:
+
+- The first request to a host attempts HTTP/3 (newest available); if the host
+  answers over h3 it is used from then on.
+- If the h3 attempt fails (e.g. the host has no UDP/QUIC), the request
+  transparently falls back to HTTP/2 or HTTP/1.1, so an h3 failure is never
+  visible to the caller.
+- A **per-host negative cache** (TTL ~1h) remembers hosts that failed h3, so a
+  host without h3 isn't re-triggered with a QUIC handshake on every request.
+  Once the TTL expires, h3 is tried again in case the host started serving it.
+- An `Alt-Svc: h3=` response header records a positive capability (confirms
+  h3 rather than gating it — the default is already to try).
+- `scraper.EnableHTTP3(host)` is only an override that keeps trying h3 even
+  during a negative cool-down; normal operation never needs it.
+
+Note: Go 1.27's stdlib has no HTTP/3 client and `x/net/http3` is only a
+registration shim; this is why the quic-go transport provides it.
+
+### Forcing a protocol
+
+If you set an explicit `url: http://...` in the config, the indexer connects
+with plain HTTP (HTTP/1.1) instead of HTTPS. The stored identity and metrics
+labels always use the bare hostname regardless of the connection scheme.
+
+---
+
+## Configuration reference
+
+Both `ffd-indexer` and `ffd-metrics` read the same YAML file
+(`-config <path>`, default `config.yaml`).
+
+```yaml
+db_path: "./index.db"
+typesense:
+  host: "http://localhost:8108"
+  api_key: "dev-key"      # inline key, or
+  # api_key_file: /run/secrets/typesense   # a secret file (takes precedence)
+metrics:
+  listen: "127.0.0.1:9090"
+forges:
+  - url: git.example.org
+    type: forgejo
+  - url: git.example.net
+    type: auto          # default when omitted
+```
+
+| Key | Type | Description |
+|---|---|---|
+| `db_path` | string | Path to the SQLite database. |
+| `typesense.host` | string | Typesense server address. |
+| `typesense.api_key` | string | Typesense API key (inline). |
+| `typesense.api_key_file` | string | Path to a file containing the API key (e.g. sops/agenix secret). Takes precedence over `api_key`; contents are trimmed of surrounding whitespace. |
+| `metrics.listen` | string (optional) | Address `ffd-metrics` binds for `/metrics`. The whole `metrics:` block may be omitted when metrics are not used. |
+| `forges[].url` | string | Forge bare hostname (FQDN); a leading scheme is accepted to force the connection protocol. Identity is always the bare hostname. |
+| `forges[].type` | string | `forgejo` \| `gitea` (pinned, recommended) or `auto` (default, experimental). |
+
+A `type` other than `forgejo`, `gitea` or `auto` is rejected at startup. See
+[Supported forge types](#supported-forge-types) for detection behavior.
+
+### Typesense API key
+
+The API key can be given inline (`typesense.api_key`) or read from a file
+(`typesense.api_key_file`) — the file takes precedence when both are set. The
+file is read at startup and trimmed of surrounding whitespace/newlines, so a
+secret deployed via **sops** or **agenix** (e.g. at `/run/secrets/...`) can be
+referenced directly without the key ever appearing in the config or the Nix
+store.
+
+### Container
+
+- `nix build .#container` — an image running `ffd-indexer` once; expects a
+  config mounted at `/etc/ffd-indexer/config.yaml` (see README quick start).
+  `db_path` is resolved relative to the container working directory, so mount a
+  volume. The image bundles CA certificates so HTTPS to forges works.
+- `nix build .#container-metrics` — an image running the `ffd-metrics` daemon,
+  exposing `:9090`.
+- `nix run .#default -- -config config.yaml` — run the indexer from the flake.
+- `nix run .#ffd-metrics -- -config /etc/ffd-indexer/config.yaml` — run the
+  metrics daemon from the flake.
+
+---
+
+## Prometheus metrics
+
+A separate long-running binary, `ffd-metrics`, serves the state of the index
+as Prometheus metrics at `/metrics`. It never scrapes the forges itself: the
+indexer's scheduled runs record per-forge status into the SQLite database, and
+`ffd-metrics` renders that state on each scrape.
+
+Both binaries share the same config file. Configure the listen address under
+`metrics.listen` (e.g. `127.0.0.1:9090`, default).
+
+### Metric reference
+
+| Metric | Type | Labels | Description |
+|---|---|---|---|
+| `ffd_forge_info` | gauge | `forge_url`, `forge_type`, `forge_version` | Constant 1 per forge with its detected type and software version |
+| `ffd_forge_online` | gauge | `forge_url`, `forge_type` | 1 if the forge was reachable and scraped successfully on its last run, else 0 (so offline forges still appear) |
+| `ffd_repository_info` | gauge | `repo_url`, `forge_url`, `forge_type` | Constant 1 per stored repository |
+| `ffd_repository_languages` | gauge | `repo_url`, `language`, `forge_url`, `forge_type` | Constant 1 per (repository, language) pair |
+
+### PromQL queries
+
+Everything beyond the primitive metrics is derived with PromQL:
+
+- repositories per forge: `count by (forge_url) (ffd_repository_info)`
+- repositories overall: `count(ffd_repository_info)`
+- forges per type: `count by (forge_type) (ffd_forge_online)`
+- language usage: `sum by (language) (ffd_repository_languages)`
+- forge versions: `ffd_forge_info` (constant gauge, read `forge_version` label)
+- offline forges: `ffd_forge_online == 0`
+
+### Metrics daemon
+
+Run it directly:
+
+```sh
+ffd-metrics -config config.yaml
+# or from the flake
+nix run .#ffd-metrics -- -config /etc/ffd-indexer/config.yaml
+```
+
+The flake also ships `container-metrics` (`nix build .#container-metrics`), an
+image that runs the daemon on `:9090`, and the NixOS module enables a
+`ffd-metrics` systemd service alongside the one-shot indexer timer.
+
+---
+
+## Supported forge types
+
+The forge software is either pinned per forge in the config (`type: forgejo` /
+`type: gitea`) or detected on every run (`type: auto`).
+
+### Type detection
+
+We recommend pinning each forge's `type` by hand; this selects the scraper
+deterministically and requires no probing. The default is `auto`, which
+re-detects the type on every run because the hoster may have changed the
+backend (e.g. Gitea→Forgejo). `auto` is **experimental**:
+
+- robots.txt is honored — a forge that disallows `/api/` will not be probed,
+  so `auto` reports `unknown` for it;
+- detection may misclassify until more detectors land;
+- the previously detected type naturally comes first because detection starts
+  with the most specific signal.
+
+Forge types without a detector are reported as `unknown`
+(`forge_type="unknown"`) and skipped; the metrics daemon still shows their
+status row.
+
+### Detection detail
+
+| Type | Detection |
+|---|---|
+| Forgejo | Serves the Forgejo-native `GET /api/forgejo/v1/version`, which returns 200 with a version JSON. Forgejo is a Gitea fork and also serves the shared `/api/v1/version`, so this endpoint is the discriminator: a hit is a strong Forgejo signal. The version is recorded and exposed as `ffd_forge_info{forge_version=...}`. |
+| Gitea | Does not serve `/api/forgejo/v1/version` (404) but does serve the shared `GET /api/v1/version`, so an instance is classified Gitea if and only if the Forgejo probe fails and the shared endpoint answers. If other forges later serve `/api/v1/version` (e.g. Gogs), that classification will need a further discriminator. |
+
+Forgejo and Gitea share the same `/api/v1` surface (Forgejo is a fork of
+Gitea), so both use the same scraper class.
+
+---
+
+## NixOS module
+
+The flake exposes `nixosModules.default` (option namespace
+`services.federated-forge-discovery.indexer`). It enables:
+
+- a **one-shot `ffd-indexer` systemd service** and a **timer** driving it
+  (`schedule`: `daily` [default], `hourly`, or `weekly` — or any systemd
+  `OnCalendar` expression);
+- a **`ffd-metrics` systemd daemon** (`Type=simple`, restart on failure);
+- the rendered config at `/etc/ffd-indexer/config.yaml`, owned by the service
+  user `ffd-indexer`;
+- a `container`/`container-metrics` integration.
+
+Options:
+
+| Option | Default | Description |
+|---|---|---|
+| `enable` | — | Enables the service and timer. |
+| `package` | — | The package to run (`ffd-indexer` from the flake). |
+| `schedule` | `"daily"` | `daily` \| `hourly` \| `weekly`, or an `OnCalendar` string. |
+| `dbPath` | — | Path to the SQLite DB. |
+| `typesense.host` | — | Typesense address. |
+| `typesense.apiKey` | `""` | Typesense API key (inline). |
+| `typesense.apiKeyFile` | `null` | Path to a secret file (sops/agenix) containing the key; takes precedence over `apiKey`. |
+| `metrics.enable` | `false` | Enable the optional `ffd-metrics` daemon (and render the `metrics:` block). Disabled by default; set `true` to run metrics. |
+| `metrics.listen` | `"127.0.0.1:9090"` | Address for the metrics daemon. |
+| `forges` | `[]` | List of `{ url, type }`; `type` is `auto` (default), `forgejo` or `gitea`. |
+
+---
+
+## Development
+
+The dev shell (`nix develop`) provides Go, `gomod2nix`, `staticcheck`, and a
+local Typesense server for development.
+
+```sh
+nix develop          # enter the dev shell (Go toolchain, gomod2nix, etc.)
+go test ./...        # run tests (store, scraper, config)
+go vet ./...
+nix flake check      # lint (golangci-lint) + all tests inside Nix
+nix build .#default .#ffd-metrics .#container .#container-metrics
+```
+
+### Continuous integration
+
+[`.forgejo/workflows/ci.yml`](.forgejo/workflows/ci.yml) runs the pipeline on
+Forgejo Actions with **no upstream actions**: the checkout is a plain
+`git clone`, and every step calls the flake (`nix flake check`, `nix build`,
+`nix develop`) inside the `nixos/nix` container image. Four isolated jobs:
+
+| Job | Enforces | Fails on |
+|---|---|---|
+| **checks** | `nix flake check` | unit tests (`go test ./...`) or golangci-lint |
+| **build** | `nix build .#default .#ffd-metrics .#container .#container-metrics` | any package failing to build |
+| **format** | `gofmt -l` and `nix fmt` | unformatted `.go` / `.nix` files |
+| **integration** | live scrape of `git.xengi.de` sent to a **Typesense `services:` container** | a scrape or sync assertion failing |
+
+The **integration** job is a real end-to-end test run inside the workflow: a
+Typesense container (the `services:` feature) is started alongside the job,
+the indexer scrapes the repository's own forge (`git.xengi.de`, `type: auto`
+→ real detection) and syncs into that service. It then asserts the forge is
+online in sqlite and that the Typesense document count equals the sqlite
+project count. The Typesense service image (`typesense/typesense:30.2`)
+matches the flake's `typesense-server` version, so CI mirrors local dev
+(where you would use `nix run .#typesense-server`). It depends on the forge
+being up — it is a live test by design.
+
+Run all checks locally with `nix flake check` + `gofmt -l .` + `nix fmt`
+(followed by a `git diff --quiet`). The runner needs the `debian-stable` label and
+outbound access to the forge, github.com (flake inputs) and cache.nixos.org.
+
+### Dependencies
+
+Go modules are vendored into Nix via `gomod2nix.toml` (generated with
+`gomod2nix generate`). After adding or upgrading a Go dependency, regenerate
+`gomod2nix.toml` before `nix flake check`/`nix build` will reflect it.
--- a/README.md
+++ b/README.md
@ -1,44 +1,169 @@
-# indexer
+[![CI Build Status](https://git.xengi.de/ffd/indexer/badges/workflows/ci.yaml/badge.svg?style=flat-square)](https://git.xengi.de/ffd/indexer/actions?workflow=ci.yaml)
+[![Container Build Status](https://git.xengi.de/ffd/indexer/badges/workflows/container.yaml/badge.svg?style=flat-square)](https://git.xengi.de/ffd/indexer/actions?workflow=container.yaml)

-Federated Forge Discovery Indexer
+# ffd-indexer

-Builds an index of public open source software projects from configured forges.
+Federated Forge Discovery Indexer.

-## Scrape behavior
+Builds and serves an index of public open-source software projects, discovered
+from self-hosted forges you configure. The indexer scrapes each forge's public
+API, enriches every repository with languages, topics, licenses and READMEs,
+stores the result in SQLite, and exposes the state as Prometheus metrics and a
+Typesense search index.

-For each configured forge, the indexer:
+> **Why this exists** — public code is scattered across many small forges, and
+> there is no federated view of it. This project fills that gap: point it at a
+> list of forges, and it continuously aggregates and indexes what is publicly
+> hosted there, so you can search and measure across all of them at once.

-1. **Checks robots.txt** (`GET /robots.txt`) before any API call. If the
-   instance disallows `/api/`, the instance is skipped.
-2. **Lists public source repositories** via the repository search API
-   (`GET /api/v1/repos/search`), paginated. Forks and mirrors are excluded.
-3. For each repository, enriches it via the Forgejo API: reads the root
-   directory once, then fetches topics, languages, latest commit, latest
-   release, the README, and detects licenses from the repo's files.
+**Status:** working. Both binaries build from a Nix flake, ship as systemd
+services and containers, and are verified end-to-end against live forges.
+License: **AGPL-3.0** (see below).

-Errors are non-fatal: an instance or a single repo that fails is skipped and
-the run continues.
+Two binaries ship together:

-### Request volume
+- **`ffd-indexer`** — the scraper. Runs once per schedule, scrapes every
+  configured forge, stores the index, and syncs it to Typesense.
+- **`ffd-metrics`** — a long-running daemon. Serves the stored index state as
+  Prometheus metrics at `/metrics`; it never scrapes forges itself.

-**Per instance:**
+---

-| Request | Count |
-|---|---|
-| `GET /robots.txt` | 1 |
-| `GET /api/v1/repos/search` (page) | `ceil(N/50)` where N = total repos |
+## Quick start

-**Per repository:**
+The fastest way to try it is the flake's container image (it runs once and
+exits, bundling CA certificates so HTTPS to forges work out of the box):

-| Request | Endpoint | Purpose |
-|---|---|---|
-| list root contents | `GET /api/v1/repos/{o}/{r}/contents/` | shared: README selection + license scan (1 call) |
-| languages | `GET /api/v1/repos/{o}/{r}/languages` | `Project.Languages` |
-| topics | `GET /api/v1/repos/{o}/{r}/topics` | `Project.Topics` |
-| latest commit | `GET /api/v1/repos/{o}/{r}/commits?limit=1` | `Project.LatestCommit` |
-| latest release | `GET /api/v1/repos/{o}/{r}/releases/latest` | `Project.LatestRelease` |
-| README | `GET /api/v1/repos/{o}/{r}/raw/{branch}/{file}` | `Project.Readme` |
-| license files | 1+ × `GET /api/v1/repos/{o}/{r}/raw/{branch}/{file}` | SPDX detection |
+```sh
+nix build .#container
+docker load < result
+docker run --rm \
+  -v "$PWD/config.yaml:/etc/ffd-indexer/config.yaml:ro" \
+  ffd-indexer
+```

-So roughly **6–8 API requests per repository** plus the paginated search
-requests and a single robots.txt check per instance.
+A minimal `config.yaml`:
+
+```yaml
+db_path: /data/index.db
+forges:
+  - url: git.example.org
+    type: forgejo
+```
+
+Mount a volume for the DB so the index persists between runs:
+
+```sh
+docker run --rm \
+  -v "$PWD/config.yaml:/etc/ffd-indexer/config.yaml:ro" \
+  -v "$PWD/data:/data" \
+  ffd-indexer
+```
+
+Or run from the flake directly (local dev):
+
+```sh
+nix run .#default -- -config config.yaml
+```
+
+**The next step** — point it at a second forge and check the output. See
+[Configuration](#configuration) for the `type` field, and [DOCS.md](DOCS.md)
+for metrics, architecture, and the NixOS module.
+
+---
+
+## Configuration
+
+The indexer reads a single YAML file (default `config.yaml`, override with
+`-config <path>`). Both binaries share the same file.
+
+```yaml
+db_path: "./index.db"              # SQLite database (index + per-forge status)
+typesense:                         # optional search index
+  host: "http://localhost:8108"
+  api_key: "dev-key"               # inline key...
+  # api_key_file: /run/secrets/ts  # ...or a secret file (takes precedence)
+metrics:
+  listen: "127.0.0.1:9090"         # ffd-metrics binds here and serves /metrics
+forges:
+  - url: git.example.org
+    type: forgejo
+  - url: git.example.net
+```
+
+- **`url`** — a bare hostname (FQDN). The indexer connects over `https://` by
+  default and adds it internally. You may prefix `http://` or `https://` to
+  force the connection protocol. The stored identity and all metrics labels
+  always use the bare hostname.
+- **`type`** — the forge's software. Set it by hand (`forgejo` or `gitea`) for
+  the recommended, deterministic setup. The default `auto` re-detects the type
+  on every run; it is **experimental** (see [DOCS.md](DOCS.md)). A forge that
+  disallows `/api/` in its robots.txt is skipped either way.
+- **`typesense.api_key` / `typesense.api_key_file`** — the Typesense API key,
+  either inlined or read from a secret file (e.g. a [sops](https://github.com/getsops/sops)/
+  [agenix](https://github.com/ryantm/agenix) secret). If both are set, the
+  **file takes precedence**, so the key never needs to appear in the config.
+
+---
+
+## What it does
+
+For each configured forge the indexer:
+
+1. **Respects robots.txt** — skips the instance if `/api/` or a sub-API is
+   disallowed.
+2. **Lists public source repositories** — paginated search; forks and mirrors
+   are excluded.
+3. **Enriches each repository** — fetches topics, languages, latest commit,
+   latest release, the README, and derives SPDX licenses from the repo's files.
+
+The indexer runs once and exits (systemd timer / container); it never stays
+resident. On each run it records per-forge status (online, type, version,
+last error) into SQLite, upserts every project, and pushes the index to
+Typesense.
+
+Everything else — request volume, the HTTP transport (HTTP/2, optional
+HTTP/3), error handling, and the full detail — is in [DOCS.md](DOCS.md).
+
+---
+
+## Verify it's working
+
+The companion daemon `ffd-metrics` serves the index's state as Prometheus
+metrics — it never scrapes forges itself; it renders the SQLite state on each
+scrape:
+
+```sh
+ffd-metrics -config config.yaml
+curl -s http://127.0.0.1:9090/metrics
+```
+
+You should see one series per configured forge: `ffd_forge_info` (type +
+software version) and `ffd_forge_online` (0/1). The full metric reference and
+ready-made PromQL queries are in [DOCS.md](DOCS.md).
+
+For a production setup, the NixOS module wires the one-shot indexer timer, the
+`ffd-metrics` daemon, and the config file — see [DOCS.md](DOCS.md).
+
+---
+
+## Contributing
+
+Contributions are welcome. Please:
+
+1. Run `go test ./...` and `go vet ./...` from the dev shell
+   (`nix develop`), and run `nix flake check` (lint + tests) before opening a
+   PR.
+2. Keep the diff small and focused, and update the README/DOCS when behavior
+   changes.
+3. Open a pull request; issues and PRs are triaged.
+
+---
+
+## License
+
+This project is licensed under the **GNU Affero General Public License v3.0**
+(AGPL-3.0) — see [LICENSE](LICENSE). It is a strong-copyleft license selected
+for network server software: if you modify it and provide the resulting
+service to others over a network, those modifications must be made available
+under the same license.
--- a/TODO.md
+++ b/TODO.md
@ -0,0 +1,4 @@
+# TODO
+
+- [ ] test suite with coverage report
+- [ ] README badges
--- a/cmd/ffd-metrics/main.go
+++ b/cmd/ffd-metrics/main.go
@ -0,0 +1,93 @@
+// Command ffd-metrics is the long-running metrics daemon for the federated
+// forge indexer.
+//
+// It reads the same SQLite store the indexer writes to and exposes the state
+// of the index to Prometheus at /metrics. It is intended to be run as a
+// separate systemd service; it performs no scraping of its own.
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"net/http"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"go.yaml.in/yaml/v4"
+
+	"git.xengi.de/ffd/indexer/config"
+	"git.xengi.de/ffd/indexer/internal/version"
+	"git.xengi.de/ffd/indexer/metrics"
+	"git.xengi.de/ffd/indexer/store"
+)
+
+func main() {
+	configPath := flag.String("config", "config.yaml", "path to the configuration file")
+	showVersion := flag.Bool("version", false, "print version and commit, then exit")
+	flag.Parse()
+
+	if *showVersion {
+		version.PrintFull()
+		return
+	}
+
+	data, err := os.ReadFile(*configPath)
+	if err != nil {
+		panic(err)
+	}
+	var cfg config.Config
+	if err := yaml.Unmarshal(data, &cfg); err != nil {
+		panic(err)
+	}
+
+	// The daemon shares the indexer's database; both processes may be open on
+	// it at once (the SQLite driver handles that with WAL-safe read queries).
+	db, err := store.Open(cfg.DbPath)
+	if err != nil {
+		panic(err)
+	}
+	defer func() { _ = db.Close() }()
+
+	registry := prometheus.NewRegistry()
+	registry.MustRegister(metrics.NewCollector(db))
+
+	mux := http.NewServeMux()
+	mux.Handle("/metrics", promhttp.HandlerFor(registry, promhttp.HandlerOpts{}))
+
+	srv := &http.Server{
+		Addr:              cfg.Metrics.Listen,
+		Handler:           mux,
+		ReadHeaderTimeout: 5 * time.Second,
+	}
+
+	// Shut down cleanly on SIGTERM/SIGINT.
+	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, os.Interrupt)
+	defer stop()
+
+	errCh := make(chan error, 1)
+	go func() {
+		fmt.Printf("ffd-metrics listening on %s\n", srv.Addr)
+		errCh <- srv.ListenAndServe()
+	}()
+
+	select {
+	case err := <-errCh:
+		if err != nil && !errors.Is(err, http.ErrServerClosed) {
+			fmt.Fprintf(os.Stderr, "metrics server: %v\n", err)
+			os.Exit(1)
+		}
+	case <-ctx.Done():
+		fmt.Println("shutting down")
+		shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		if err := srv.Shutdown(shutdownCtx); err != nil {
+			fmt.Fprintf(os.Stderr, "shutdown: %v\n", err)
+		}
+	}
+}
--- a/config.go
+++ b/config.go
@ -1,14 +0,0 @@
-package main
-
-// Forge is a single forge instance to scrape.
-type Forge struct {
-	Name string `yaml:"name"`
-	Url  string `yaml:"url"`
-	Type string `yaml:"type"`
-}
-
-// Config is the top-level configuration file.
-type Config struct {
-	DbPath string  `yaml:"db_path"`
-	Forges []Forge `yaml:"forges"`
-}
--- a/config/config.go
+++ b/config/config.go
@ -0,0 +1,102 @@
+// Package config holds the indexer's YAML configuration.
+//
+// It lives in its own package (rather than package main) so that both the
+// indexer CLI and the ffd-metrics daemon can parse the same configuration
+// file shape.
+package config
+
+import (
+	"fmt"
+	"os"
+	"strings"
+)
+
+// Forge is a single instance to scrape: its bare hostname (FQDN) and its
+// forge type. The URL is used as the forge's unique identity (the HTTP scheme
+// is added internally when a connection is needed; a scheme prefix in the URL
+// is accepted to force the connection protocol, but the stored identity is
+// always the bare hostname). Type selects the scraper: a pinned "forgejo" or
+// "gitea" is recommended and skips probing; "auto" (the default, still
+// experimental) runs the detector at scrape time so a backend change (e.g. a
+// migration from Gitea to Forgejo) is picked up automatically. Only supported
+// types are accepted.
+type Forge struct {
+	URL  string `yaml:"url"`
+	Type string `yaml:"type"`
+}
+
+// Forges is the list of forge instances to scrape.
+type Forges []Forge
+
+// TypesenseConfig holds the connection settings to the search engine.
+// ApiKey may be inlined or supplied via ApiKeyFile (a path to a file
+// containing the key, e.g. a sops/agenix secret). If both are set, the file
+// takes precedence so secrets need not live in the config file.
+type TypesenseConfig struct {
+	Host       string `yaml:"host"`
+	ApiKey     string `yaml:"api_key"`
+	ApiKeyFile string `yaml:"api_key_file"`
+}
+
+// MetricsConfig configures the Prometheus metrics daemon.
+type MetricsConfig struct {
+	// Listen is the address the ffd-metrics daemon binds and serves
+	// /metrics on, e.g. "127.0.0.1:9090".
+	Listen string `yaml:"listen"`
+}
+
+// Config is the top-level configuration file.
+type Config struct {
+	DbPath    string          `yaml:"db_path"`
+	Typesense TypesenseConfig `yaml:"typesense"`
+	Metrics   MetricsConfig   `yaml:"metrics"`
+	Forges    Forges          `yaml:"forges"`
+}
+
+// Supported forge type values accepted in the config. "auto" (the default)
+// triggers detection at scrape time; the rest pin a specific scraper.
+const (
+	ForgeTypeAuto    = "auto"
+	ForgeTypeForgejo = "forgejo"
+	ForgeTypeGitea   = "gitea"
+)
+
+// SupportedForgeTypes are the accepted values for Forge.Type, including the
+// sentinel "auto" that triggers detection at scrape time.
+var SupportedForgeTypes = map[string]bool{
+	"":               true, // default to "auto"
+	ForgeTypeAuto:    true,
+	ForgeTypeForgejo: true,
+	ForgeTypeGitea:   true,
+}
+
+// Validate normalizes a Forge's Type (empty means "auto") and rejects
+// unsupported values.
+func (f *Forge) Validate() error {
+	if f.URL == "" {
+		return fmt.Errorf("forge entry missing url")
+	}
+	if f.Type == "" {
+		f.Type = ForgeTypeAuto
+	}
+	if !SupportedForgeTypes[f.Type] {
+		return fmt.Errorf("unsupported forge type %q for %s (want %s, %s or %s)", f.Type, f.URL, ForgeTypeAuto, ForgeTypeForgejo, ForgeTypeGitea)
+	}
+	return nil
+}
+
+// ResolveAPIKey returns the effective Typesense API key. If ApiKeyFile is
+// set it is read (taking precedence over the inline ApiKey), and its contents
+// are trimmed of surrounding whitespace/newline so a plain `cat` secret file
+// works. An empty key (from either source) is returned as "" for the caller
+// to treat as "not configured".
+func (tc TypesenseConfig) ResolveAPIKey() (string, error) {
+	if tc.ApiKeyFile == "" {
+		return tc.ApiKey, nil
+	}
+	b, err := os.ReadFile(tc.ApiKeyFile)
+	if err != nil {
+		return "", fmt.Errorf("read typesense api_key_file %q: %w", tc.ApiKeyFile, err)
+	}
+	return strings.TrimSpace(string(b)), nil
+}
--- a/config/config_test.go
+++ b/config/config_test.go
@ -0,0 +1,134 @@
+package config
+
+import (
+	"os"
+	"strings"
+	"testing"
+
+	"go.yaml.in/yaml/v4"
+)
+
+func TestForgeDefaultsToAuto(t *testing.T) {
+	var f Forge
+	if err := f.Validate(); err == nil {
+		t.Fatal("expected error for missing url")
+	}
+	f = Forge{URL: "git.example.org"}
+	if err := f.Validate(); err != nil {
+		t.Fatalf("Validate: %v", err)
+	}
+	if f.Type != "auto" {
+		t.Errorf("expected default type auto, got %q", f.Type)
+	}
+}
+
+func TestForgeRejectsUnsupportedType(t *testing.T) {
+	f := Forge{URL: "example.test", Type: "github"}
+	err := f.Validate()
+	if err == nil {
+		t.Fatal("expected error for unsupported type")
+	}
+	if !strings.Contains(err.Error(), "unsupported") {
+		t.Errorf("expected unsupported-type error, got %v", err)
+	}
+}
+
+func TestForgeSupportsKnownTypes(t *testing.T) {
+	for _, typ := range []string{ForgeTypeAuto, ForgeTypeForgejo, ForgeTypeGitea} {
+		f := Forge{URL: "example.test", Type: typ}
+		if err := f.Validate(); err != nil {
+			t.Errorf("type %q should be accepted: %v", typ, err)
+		}
+	}
+}
+
+func TestParseForgesFromYAML(t *testing.T) {
+	cfg := `
+db_path: "/tmp/db"
+forges:
+  - url: git.example.org
+  - url: git.example.net
+    type: forgejo
+`
+	var c Config
+	if err := yaml.Unmarshal([]byte(cfg), &c); err != nil {
+		t.Fatalf("unmarshal: %v", err)
+	}
+	if len(c.Forges) != 2 {
+		t.Fatalf("want 2 forges, got %d", len(c.Forges))
+	}
+	if c.Forges[0].URL != "git.example.org" || c.Forges[0].Type != "" {
+		t.Errorf("forge 0: %+v", c.Forges[0])
+	}
+	if c.Forges[1].URL != "git.example.net" || c.Forges[1].Type != "forgejo" {
+		t.Errorf("forge 1: %+v", c.Forges[1])
+	}
+	if err := c.Forges[0].Validate(); err != nil {
+		t.Fatalf("validate forge 0: %v", err)
+	}
+	if c.Forges[0].Type != "auto" {
+		t.Errorf("forge 0 type should normalize to auto, got %q", c.Forges[0].Type)
+	}
+}
+
+func TestResolveAPIKeyFromInline(t *testing.T) {
+	tc := TypesenseConfig{ApiKey: "inline-key"}
+	got, err := tc.ResolveAPIKey()
+	if err != nil {
+		t.Fatalf("ResolveAPIKey: %v", err)
+	}
+	if got != "inline-key" {
+		t.Errorf("got %q, want inline-key", got)
+	}
+}
+
+func TestResolveAPIKeyFromFileTakesPrecedence(t *testing.T) {
+	p := t.TempDir() + "/key"
+	if err := os.WriteFile(p, []byte("file-key-with-newline\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	// Both set: the file wins, and surrounding whitespace is trimmed.
+	tc := TypesenseConfig{ApiKey: "inline-key", ApiKeyFile: p}
+	got, err := tc.ResolveAPIKey()
+	if err != nil {
+		t.Fatalf("ResolveAPIKey: %v", err)
+	}
+	if got != "file-key-with-newline" {
+		t.Errorf("got %q, want file key with newline trimmed", got)
+	}
+}
+
+func TestResolveAPIKeyNone(t *testing.T) {
+	tc := TypesenseConfig{}
+	got, err := tc.ResolveAPIKey()
+	if err != nil {
+		t.Fatalf("ResolveAPIKey: %v", err)
+	}
+	if got != "" {
+		t.Errorf("got %q, want empty", got)
+	}
+}
+
+func TestResolveAPIKeyMissingFile(t *testing.T) {
+	tc := TypesenseConfig{ApiKeyFile: "/nonexistent/does-not-exist"}
+	if _, err := tc.ResolveAPIKey(); err == nil {
+		t.Error("expected error for missing api_key_file")
+	}
+}
+
+func TestParseAPIKeyFileFromYAML(t *testing.T) {
+	cfg := `
+db_path: "/tmp/db"
+typesense:
+  host: "http://localhost:8108"
+  api_key: "plain"
+  api_key_file: "/run/secrets/typesense"
+`
+	var c Config
+	if err := yaml.Unmarshal([]byte(cfg), &c); err != nil {
+		t.Fatalf("unmarshal: %v", err)
+	}
+	if c.Typesense.ApiKey != "plain" || c.Typesense.ApiKeyFile != "/run/secrets/typesense" {
+		t.Errorf("typesense: %+v", c.Typesense)
+	}
+}
--- a/flake.nix
+++ b/flake.nix
@ -9,78 +9,213 @@
      inputs.flake-utils.follows = "flake-utils";
    };
  };
-  outputs = { self, nixpkgs, flake-utils, gomod2nix }: flake-utils.lib.eachDefaultSystem (system:
-    let
-      pkgs = import nixpkgs {
-        inherit system;
-        overlays = [ gomod2nix.overlays.default ];
-      };
-      go = pkgs.go_1_27;
-      ffd-indexer = pkgs.buildGoApplication {
-        inherit go;
-        pname = "ffd-indexer";
-        version = "0.0.1";
-        src = ./.;
-        modules = ./gomod2nix.toml;
-      };
-      go-test = pkgs.stdenv.mkDerivation {
-        name = "go-test";
-        dontBuild = true;
-        src = ./.;
-        doCheck = true;
-        nativeBuildInputs = [
-          go
-          pkgs.writableTmpDirAsHomeHook
-        ];
-        checkPhase = ''
-          go test -v ./...
-        '';
-        installPhase = ''
-          mkdir "$out"
-        '';
-      };
-      go-lint = pkgs.stdenv.mkDerivation {
-        name = "go-lint";
-        dontBuild = true;
-        src = ./.;
-        doCheck = true;
-        nativeBuildInputs = [
-          pkgs.golangci-lint
-          go
-          pkgs.writableTmpDirAsHomeHook
-        ];
-        checkPhase = ''
-          golangci-lint run
-        '';
-        installPhase = ''
-          mkdir "$out"
-        '';
-      };
-    in
+  outputs =
    {
-      checks = { inherit go-test go-lint; };
-      formatter = pkgs.nixfmt-tree;
-      devShells.default = pkgs.mkShell {
-        buildInputs = [
-          go
-          pkgs.go-tools # staticcheck
-          gomod2nix.packages.${system}.default
+      self,
+      nixpkgs,
+      flake-utils,
+      gomod2nix,
+    }:
+    flake-utils.lib.eachDefaultSystem (
+      system:
+      let
+        pkgs = import nixpkgs {
+          inherit system;
+          overlays = [ gomod2nix.overlays.default ];
+        };
+        go = pkgs.go_1_27;
+        version = "0.0.1";
+        # Git commit (full + short) used for container labels. Inside a real
+        # flake build (including CI's shallow clone) `self.rev` is populated;
+        # on a dirty working tree it is null and we fall back to empty so the
+        # label is simply omitted.
+        rev = self.rev or "";
+        shortRev =
+          if rev != "" then
+            builtins.substring 0 7 rev
+          else
+            (self.dirtyShortRev or "");
+        # Inject version + commit into the binaries via the Go linker, and
+        # reuse the same values for the OCI image labels.
+        goLdflags = [
+          "-X git.xengi.de/ffd/indexer/internal/version.Version=${version}"
+          "-X git.xengi.de/ffd/indexer/internal/version.Commit=${shortRev}"
        ];
-        packages = with pkgs; [
-          mkcert
-          sqlite # debugging the index db
-        ];
-      };
-      packages = {
-        default = ffd-indexer;
-        container = pkgs.dockerTools.buildImage {
-          name = "ffd-indexer";
-          config = {
-            Cmd = [ "${ffd-indexer}/bin/ffd-indexer" ];
+        ffd-indexer = pkgs.buildGoApplication {
+          inherit go;
+          pname = "ffd-indexer";
+          inherit version;
+          ldflags = goLdflags;
+          src = ./.;
+          modules = ./gomod2nix.toml;
+          # buildGoApplication runs `go install ./...`, which names each binary
+          # after the last element of its package path:
+          #   git.xengi.de/ffd/indexer          -> "indexer"
+          #   git.xengi.de/ffd/indexer/cmd/ffd-metrics -> "ffd-metrics"
+          # Rename "indexer" to the service/container name "ffd-indexer".
+          postInstall = ''
+            mv "$out/bin/indexer" "$out/bin/ffd-indexer"
+          '';
+        };
+
+        # Long-running Prometheus metrics daemon. It is a separate binary but is
+        # built in the same derivation; keep it as its own output alias for the
+        # NixOS module and containers.
+        ffd-metrics = pkgs.runCommand "ffd-metrics" { } ''
+          mkdir -p "$out/bin"
+          ln -s "${ffd-indexer}/bin/ffd-metrics" "$out/bin/ffd-metrics"
+        '';
+        go-test = pkgs.stdenv.mkDerivation {
+          name = "go-test";
+          dontBuild = true;
+          src = ./.;
+          doCheck = true;
+          nativeBuildInputs = [
+            go
+            pkgs.writableTmpDirAsHomeHook
+          ];
+          checkPhase = ''
+            go test -v ./...
+          '';
+          installPhase = ''
+            mkdir "$out"
+          '';
+        };
+        go-lint = pkgs.stdenv.mkDerivation {
+          name = "go-lint";
+          dontBuild = true;
+          src = ./.;
+          doCheck = true;
+          nativeBuildInputs = [
+            pkgs.golangci-lint
+            go
+            pkgs.writableTmpDirAsHomeHook
+          ];
+          checkPhase = ''
+            golangci-lint run
+          '';
+          installPhase = ''
+            mkdir "$out"
+          '';
+        };
+      in
+      {
+        checks = { inherit go-test go-lint; };
+        formatter = pkgs.nixfmt-tree;
+        devShells.default = pkgs.mkShell {
+          buildInputs = [
+            go
+            pkgs.go-tools # staticcheck
+            gomod2nix.packages.${system}.default
+          ];
+          packages = with pkgs; [
+            mkcert
+            sqlite # debugging the index db
+            typesense
+            skopeo # copy OCI/container images to a registry (CI publishing)
+          ];
+          shellHook = ''
+            # Local dev Typesense defaults (see packages.typesense-server).
+            export TYPESENSE_API_KEY="''${TYPESENSE_API_KEY:-dev-key}"
+            export TYPESENSE_URL="''${TYPESENSE_URL:-http://localhost:8108}"
+            echo "Typesense dev: run 'typesense-server' (port 8108, api key $TYPESENSE_API_KEY)"
+          '';
+        };
+        packages = {
+          default = ffd-indexer;
+          inherit ffd-metrics;
+          # Runs a local Typesense instance for development.
+          typesense-server = pkgs.writeScriptBin "typesense-server" ''
+            #!${pkgs.bash}/bin/bash
+            set -euo pipefail
+            ROOT="$(git rev-parse --show-toplevel 2>/dev/null || echo "$PWD")"
+            DATA_DIR="$ROOT/.typesense/data"
+            API_KEY="''${TYPESENSE_API_KEY:-dev-key}"
+            PORT="''${TYPESENSE_PORT:-8108}"
+            # Peering port follows the API port so multiple dev instances
+            # (different TYPESENSE_PORT) do not collide on the default 8107.
+            PEERING_PORT=$((PORT - 1))
+            mkdir -p "$DATA_DIR"
+            echo "Starting Typesense on 127.0.0.1:$PORT (api key: $API_KEY, data: $DATA_DIR)"
+            exec ${pkgs.typesense}/bin/typesense-server \
+              --data-dir="$DATA_DIR" \
+              --api-key="$API_KEY" \
+              --api-port="$PORT" \
+              --peering-port="$PEERING_PORT" \
+              --api-address=127.0.0.1 \
+              --enable-cors
+          '';
+          container = pkgs.dockerTools.buildImage {
+            name = "ffd-indexer";
+            contents = [
+              ffd-indexer
+              pkgs.cacert # TLS CA for HTTPS requests to the forges
+            ];
+            config = {
+              # The user must provide their own config file and mount it at
+              # /etc/ffd-indexer/config.yaml, e.g.:
+              #   docker run -v $PWD/config.yaml:/etc/ffd-indexer/config.yaml ffd-indexer
+              Labels = {
+                "org.opencontainers.image.title" = "ffd-indexer";
+                "org.opencontainers.image.description" = "Federated Forge Discovery Indexer (scraper)";
+                "org.opencontainers.image.licenses" = "AGPL-3.0-or-later";
+                "org.opencontainers.image.source" = "https://git.xengi.de/ffd/indexer";
+                "org.opencontainers.image.vendor" = "ffd";
+                "org.opencontainers.image.version" = version;
+                "org.opencontainers.image.revision" = shortRev;
+              };
+              Volumes = {
+                "/etc/ffd-indexer" = { };
+              };
+              Cmd = [
+                "${ffd-indexer}/bin/ffd-indexer"
+                "-config"
+                "/etc/ffd-indexer/config.yaml"
+              ];
+            };
+          };
+
+          # Long-running metrics image. It shares the same config file shape and
+          # reads the same DB volume as the indexer.
+          container-metrics = pkgs.dockerTools.buildImage {
+            name = "ffd-metrics";
+            contents = [
+              ffd-indexer
+              pkgs.cacert
+            ];
+            config = {
+              Labels = {
+                "org.opencontainers.image.title" = "ffd-metrics";
+                "org.opencontainers.image.description" = "Federated Forge Discovery Indexer (Prometheus metrics daemon)";
+                "org.opencontainers.image.licenses" = "AGPL-3.0-or-later";
+                "org.opencontainers.image.source" = "https://git.xengi.de/ffd/indexer";
+                "org.opencontainers.image.vendor" = "ffd";
+                "org.opencontainers.image.version" = version;
+                "org.opencontainers.image.revision" = shortRev;
+              };
+              Volumes = {
+                "/etc/ffd-indexer" = { };
+              };
+              ExposedPorts = {
+                "9090/tcp" = { };
+              };
+              Cmd = [
+                "${ffd-indexer}/bin/ffd-metrics"
+                "-config"
+                "/etc/ffd-indexer/config.yaml"
+              ];
+            };
          };
        };
+        nixosModules.default = import ./option.nix;
+      }
+    )
+    // {
+      # Overlay so the NixOS module's default package (`pkgs.ffd-indexer`)
+      # resolves to this flake's package.
+      overlays.default = final: prev: {
+        ffd-indexer = self.packages.${final.system}.default;
+        ffd-metrics = self.packages.${final.system}.ffd-metrics;
      };
-      nixosModules.default = import ./option.nix;
-    }
-  );
+    };
 }
--- a/go.mod
+++ b/go.mod
@ -5,6 +5,9 @@ go 1.27.1
 require (
 	codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v3 v3.0.0
 	github.com/go-enry/go-license-detector/v4 v4.3.1
+	github.com/prometheus/client_golang v1.24.1
+	github.com/quic-go/quic-go v0.63.0
+	github.com/typesense/typesense-go/v2 v2.0.0
 	go.yaml.in/yaml/v4 v4.0.0-rc.6
 	modernc.org/sqlite v1.59.0
 )
@ -14,6 +17,9 @@ require (
 	github.com/Microsoft/go-winio v0.6.0 // indirect
 	github.com/ProtonMail/go-crypto v0.0.0-20220930113650-c6815a8c17ad // indirect
 	github.com/acomagu/bufpipe v1.0.3 // indirect
+	github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/cloudflare/circl v1.2.0 // indirect
 	github.com/davidmz/go-pageant v1.0.2 // indirect
 	github.com/dgryski/go-minhash v0.0.0-20190315135803-ad340ca03076 // indirect
@ -49,16 +55,23 @@ require (
 	github.com/mattn/go-isatty v0.0.24 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/montanaflynn/stats v0.7.1 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/ncruces/go-strftime v1.0.0 // indirect
+	github.com/oapi-codegen/runtime v1.1.1 // indirect
 	github.com/oklog/ulid v1.3.1 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	github.com/prometheus/client_model v0.6.2 // indirect
+	github.com/prometheus/common v0.70.1 // indirect
+	github.com/prometheus/procfs v0.21.1 // indirect
+	github.com/quic-go/qpack v0.6.0 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/sergi/go-diff v1.2.0 // indirect
 	github.com/shogo82148/go-shuffle v1.0.1 // indirect
+	github.com/sony/gobreaker v0.5.0 // indirect
 	github.com/xanzy/ssh-agent v0.3.2 // indirect
 	go.mongodb.org/mongo-driver v1.17.6 // indirect
-	go.yaml.in/yaml/v3 v3.0.4 // indirect
+	go.yaml.in/yaml/v3 v3.0.5 // indirect
 	golang.org/x/crypto v0.54.0 // indirect
 	golang.org/x/exp v0.0.0-20221006183845-316c7553db56 // indirect
 	golang.org/x/mod v0.38.0 // indirect
@ -68,6 +81,7 @@ require (
 	golang.org/x/text v0.40.0 // indirect
 	golang.org/x/tools v0.48.0 // indirect
 	gonum.org/v1/gonum v0.8.2 // indirect
+	google.golang.org/protobuf v1.36.11 // indirect
 	gopkg.in/neurosnap/sentences.v1 v1.0.7 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	modernc.org/libc v1.75.7 // indirect
--- a/go.sum
+++ b/go.sum
@ -10,15 +10,23 @@ github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2B
 github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo=
 github.com/ProtonMail/go-crypto v0.0.0-20220930113650-c6815a8c17ad h1:QeeqI2zxxgZVe11UrYFXXx6gVxPVF40ygekjBzEg4XY=
 github.com/ProtonMail/go-crypto v0.0.0-20220930113650-c6815a8c17ad/go.mod h1:UBYPn8k0D56RtnR8RFQMjmh4KrZzWJ5o7Z9SYjossQ8=
+github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk=
 github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk=
 github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
 github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
+github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ=
+github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w=
 github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
 github.com/bwesterb/go-ristretto v1.2.1/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I=
 github.com/cloudflare/circl v1.2.0 h1:NheeISPSUcYftKlfrLuOo4T62FkmD4t4jviLfFFYaec=
 github.com/cloudflare/circl v1.2.0/go.mod h1:Ch2UgYr6ti2KTtlejELlROl0YIYj7SLjAC8M+INXlMk=
@ -96,8 +104,8 @@ github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9L
 github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
 github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
-github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo=
 github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
@ -116,10 +124,15 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i
 github.com/jdkato/prose v1.2.1 h1:Fp3UnJmLVISmlc57BgKUzdjr0lOtjqTZicL3PaYy6cU=
 github.com/jdkato/prose v1.2.1/go.mod h1:AiRHgVagnEx2JbQRQowVBKjG0bcs/vtkGCH1dYAL1rA=
 github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
+github.com/jinzhu/copier v0.3.4 h1:mfU6jI9PtCeUjkjQ322dlff9ELjGDu975C2p/nrubVI=
+github.com/jinzhu/copier v0.3.4/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
+github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE=
 github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
 github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
 github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
 github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
+github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk=
+github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
@ -128,6 +141,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A=
 github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA=
 github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
@ -137,18 +152,35 @@ github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrk
 github.com/montanaflynn/stats v0.6.3/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE=
 github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
 github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
 github.com/neurosnap/sentences v1.0.6 h1:iBVUivNtlwGkYsJblWV8GGVFmXzZzak907Ci8aA0VTE=
 github.com/neurosnap/sentences v1.0.6/go.mod h1:pg1IapvYpWCJJm/Etxeh0+gtMf1rI1STY9S7eUCPbDc=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/oapi-codegen/runtime v1.1.1 h1:EXLHh0DXIJnWhdRPN2w4MXAzFyE4CskzhNLUmtpMYro=
+github.com/oapi-codegen/runtime v1.1.1/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg=
 github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU=
+github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE=
+github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
+github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
+github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY=
+github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc=
+github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI=
+github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY=
+github.com/quic-go/go-ossfuzz-seeds v0.1.0 h1:APacT+iIaNF6fd8AGEiN3bT/Jtkd2jz4v4TzM7MFjy0=
+github.com/quic-go/go-ossfuzz-seeds v0.1.0/go.mod h1:3IOHRbJIc+L6YKMwfDtJAM9Vj9k0YY4muhuyUYk5tbk=
+github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8=
+github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII=
+github.com/quic-go/quic-go v0.63.0 h1:LIFGHI4PFUhhw2dDD1ARHdCff143ffMHwZtbnbuJ78A=
+github.com/quic-go/quic-go v0.63.0/go.mod h1:RAro2j2yN9a9EiPACLHT9IB2NXCvGQmmo/alT0yYI0w=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
@ -161,21 +193,33 @@ github.com/shogo82148/go-shuffle v1.0.1 h1:4swIpHXLMAz14DE4YTgakgadpRN0n1wE1dieG
 github.com/shogo82148/go-shuffle v1.0.1/go.mod h1:HQPjVgUUZ9TNgm4/K/iXRuAdhPsQrXnAGgtk/9kqbBY=
 github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/sony/gobreaker v0.5.0 h1:dRCvqm0P490vZPmy7ppEk2qCnCieBooFJ+YoXGYB+yg=
+github.com/sony/gobreaker v0.5.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY=
+github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
-github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
+github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
+github.com/typesense/typesense-go/v2 v2.0.0 h1:+MksOnrVioDqsGpz8RXkOUqhVN+yFxZwJlGDQHr/64I=
+github.com/typesense/typesense-go/v2 v2.0.0/go.mod h1:7V1ZBSfmdciL6yb2bPtWha+W53gV5WZhyOSpVgDJfao=
 github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0=
 github.com/xanzy/ssh-agent v0.3.2 h1:eKj4SX2Fe7mui28ZgnFW5fmTz1EIr7ugo5s6wDxdHBM=
 github.com/xanzy/ssh-agent v0.3.2/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
 go.mongodb.org/mongo-driver v1.17.6 h1:87JUG1wZfWsr6rIz3ZmpH90rL5tea7O3IHuSwHUpsss=
 go.mongodb.org/mongo-driver v1.17.6/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ=
-go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
-go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
+go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko=
+go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o=
+go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
+go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
+go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
+go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
 go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
 go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
 golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
@ -243,6 +287,8 @@ gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0=
 gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0 h1:OE9mWmgKkjJyEmDAAtGMPjXu+YNeGvK9VTSHY6+Qihc=
 gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
 gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc=
+google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
+google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
--- a/gomod2nix.toml
+++ b/gomod2nix.toml
@ -21,6 +21,18 @@ schema = 3
    version = 'v1.0.3'
    hash = 'sha256-hIQf2lqZZv4dN8JwRyhNoBjStL9Mgih6f7Hy8RDU5Kg='

+  [mod.'github.com/apapsch/go-jsonmerge/v2']
+    version = 'v2.0.0'
+    hash = 'sha256-xp/1B6XUN2EbddBfoUkTV3oTk+34m4kOZP+66HhfLg4='
+
+  [mod.'github.com/beorn7/perks']
+    version = 'v1.0.1'
+    hash = 'sha256-h75GUqfwJKngCJQVE5Ao5wnO3cfKD9lSIteoLp/3xJ4='
+
+  [mod.'github.com/cespare/xxhash/v2']
+    version = 'v2.3.0'
+    hash = 'sha256-7hRlwSR+fos1kx4VZmJ/7snR7zHh8ZFKX+qqqqGcQpY='
+
  [mod.'github.com/cloudflare/circl']
    version = 'v1.2.0'
    hash = 'sha256-9L1/4M8Za4qcrtti/1r1E7TgpdfUKHM+9IyuX3fuJyQ='
@ -165,10 +177,18 @@ schema = 3
    version = 'v0.7.1'
    hash = 'sha256-0QXUA/syOtDSBooh3isAffrVAjpSoBmSMMuZfO5maHg='

+  [mod.'github.com/munnerz/goautoneg']
+    version = 'v0.0.0-20191010083416-a7dc8b61c822'
+    hash = 'sha256-79URDDFenmGc9JZu+5AXHToMrtTREHb3BC84b/gym9Q='
+
  [mod.'github.com/ncruces/go-strftime']
    version = 'v1.0.0'
    hash = 'sha256-GYIwYDONuv/yTE0AEugCHQbtV3oiBaco93xUNYFcVBQ='

+  [mod.'github.com/oapi-codegen/runtime']
+    version = 'v1.1.1'
+    hash = 'sha256-GRvzpQngQPAy59KvvcwhjYqjx4gttIhOtQKjqfQcNvI='
+
  [mod.'github.com/oklog/ulid']
    version = 'v1.3.1'
    hash = 'sha256-LNn883rYNiaoY9sGEPIzlMRx5UwGThdYTjXqfzeGc9k='
@ -177,6 +197,30 @@ schema = 3
    version = 'v0.9.1'
    hash = 'sha256-mNfQtcrQmu3sNg/7IwiieKWOgFQOVVe2yXgKBpe/wZw='

+  [mod.'github.com/prometheus/client_golang']
+    version = 'v1.24.1'
+    hash = 'sha256-HAOFVYyPiU7hVS1XXMMjkbGgTT7/UN0zVHXYOnjr7is='
+
+  [mod.'github.com/prometheus/client_model']
+    version = 'v0.6.2'
+    hash = 'sha256-q6Fh6v8iNJN9ypD47LjWmx66YITa3FyRjZMRsuRTFeQ='
+
+  [mod.'github.com/prometheus/common']
+    version = 'v0.70.1'
+    hash = 'sha256-xlhVEswCWaBnAXn53KOUzquoDEnZVd/NcTbj52EJ6rE='
+
+  [mod.'github.com/prometheus/procfs']
+    version = 'v0.21.1'
+    hash = 'sha256-5SpWprdX29oVntHJyadiICOk9UAP+hu+xJM51/A8Ig4='
+
+  [mod.'github.com/quic-go/qpack']
+    version = 'v0.6.0'
+    hash = 'sha256-xaxHnTKIZt1cHK5ZqTuSTOt5RNSjQB37GlrIgEGBskM='
+
+  [mod.'github.com/quic-go/quic-go']
+    version = 'v0.63.0'
+    hash = 'sha256-ctsVUQiUjpXvs/yG75vd5lewqCphwLfaFo426WL5arE='
+
  [mod.'github.com/remyoudompheng/bigfft']
    version = 'v0.0.0-20230129092748-24d4a6f8daec'
    hash = 'sha256-vYmpyCE37eBYP/navhaLV4oX4/nu0Z/StAocLIFqrmM='
@ -193,6 +237,14 @@ schema = 3
    version = 'v1.0.1'
    hash = 'sha256-E5Xt69A2vXlVPzyHPyWK0FyQ+IaMZzqQCnUltedeZhg='

+  [mod.'github.com/sony/gobreaker']
+    version = 'v0.5.0'
+    hash = 'sha256-apNsKZeM75TYjTxqq5paHVAqIzF+Wfcw54Iro268R7Y='
+
+  [mod.'github.com/typesense/typesense-go/v2']
+    version = 'v2.0.0'
+    hash = 'sha256-MVJGuyzlXXwzYiNGAwxLwjcBpxtXRB9hj0gzciEw/xM='
+
  [mod.'github.com/xanzy/ssh-agent']
    version = 'v0.3.2'
    hash = 'sha256-QavRCX75546hnWCOtBwiUdmyk6AoLjv3POoJXng5kMI='
@ -202,8 +254,8 @@ schema = 3
    hash = 'sha256-B1ePaR77HKDksfG6PLwOmOVPYac9g13i8Ld/xWOCIiE='

  [mod.'go.yaml.in/yaml/v3']
-    version = 'v3.0.4'
-    hash = 'sha256-NkGFiDPoCxbr3LFsI6OCygjjkY0rdmg5ggvVVwpyDQ4='
+    version = 'v3.0.5'
+    hash = 'sha256-ygho+GU5kE7vPMx+dZYyNfCaeMjNxj66XmrcVf3afFE='

  [mod.'go.yaml.in/yaml/v4']
    version = 'v4.0.0-rc.6'
@ -245,6 +297,10 @@ schema = 3
    version = 'v0.8.2'
    hash = 'sha256-eFUTetKRBa9QzDUEtd6aU8KsTJQgBF6zLAveY4Ybi58='

+  [mod.'google.golang.org/protobuf']
+    version = 'v1.36.11'
+    hash = 'sha256-7W+6jntfI/awWL3JP6yQedxqP5S9o3XvPgJ2XxxsIeE='
+
  [mod.'gopkg.in/neurosnap/sentences.v1']
    version = 'v1.0.7'
    hash = 'sha256-AaBiqQ5Mtrzk4dE0lZidyUtDzYVy7aonEeh5oXrq/VI='
--- a/gomod2nix.toml.dump
+++ b/gomod2nix.toml.dump
--- a/internal/version/version.go
+++ b/internal/version/version.go
@ -0,0 +1,45 @@
+// Package version exposes build-time version and commit information for the
+// ffd-indexer binaries.
+//
+// The values are injected at build time via the Go linker:
+//
+//	go build -ldflags "-X git.xengi.de/ffd/indexer/internal/version.Version=1.2.3 \
+//	                 -X   git.xengi.de/ffd/indexer/internal/version.Commit=a1b2c3d"
+//
+// (the flake sets these from `version` and `shortRev`). When built with a
+// plain `go build`/`go run` the variables stay empty and the binary reports
+// itself as the development build.
+package version
+
+import (
+	"fmt"
+	"runtime"
+)
+
+// Version is the application version, overridden at build time.
+var Version = ""
+
+// Commit is the git commit the binary was built from, overridden at build time.
+var Commit = ""
+
+// PrintFull writes the canonical multi-line version block to stdout. This is
+// the human-readable format used by kubectl, docker, kind and friends:
+//
+//	Version:      1.2.3
+//	Commit:       a1b2c3d
+//	Go version:   go1.27.1
+//
+// Empty values are printed as "dev" so the output is always self-explanatory.
+func PrintFull() {
+	v := Version
+	if v == "" {
+		v = "dev"
+	}
+	c := Commit
+	if c == "" {
+		c = "dev"
+	}
+	fmt.Printf("Version:      %s\n", v)
+	fmt.Printf("Commit:       %s\n", c)
+	fmt.Printf("Go version:   %s\n", runtime.Version())
+}
--- a/main.go
+++ b/main.go
@ -1,21 +1,38 @@
 package main

 import (
+	"context"
+	"flag"
 	"fmt"
 	"os"
+	"time"

+	"github.com/typesense/typesense-go/v2/typesense"
 	"go.yaml.in/yaml/v4"

+	"git.xengi.de/ffd/indexer/config"
+	"git.xengi.de/ffd/indexer/internal/version"
 	"git.xengi.de/ffd/indexer/scraper"
 	"git.xengi.de/ffd/indexer/store"
+	"git.xengi.de/ffd/indexer/sync"
 )

 func main() {
-	data, err := os.ReadFile("config.yaml")
+	syncOnly := flag.Bool("sync-only", false, "rebuild the Typesense index from the existing DB without scraping")
+	configPath := flag.String("config", "config.yaml", "path to the configuration file")
+	showVersion := flag.Bool("version", false, "print version and commit, then exit")
+	flag.Parse()
+
+	if *showVersion {
+		version.PrintFull()
+		return
+	}
+
+	data, err := os.ReadFile(*configPath)
 	if err != nil {
 		panic(err)
 	}
-	var config Config
+	var config config.Config

 	if err := yaml.Unmarshal(data, &config); err != nil {
 		panic(err)
@ -27,18 +44,93 @@ func main() {
 	}
 	defer func() { _ = db.Close() }()

-	for _, forge := range config.Forges {
-		fmt.Printf("Scraping %s (%s, %s)\n", forge.Name, forge.Type, forge.Url)
+	// Resolve the Typesense API key once: api_key_file (a sops/agenix secret)
+	// takes precedence over the inline api_key.
+	typesenseAPIKey, err := config.Typesense.ResolveAPIKey()
+	if err != nil {
+		panic(err)
+	}

-		s, err := scraper.NewScraper(forge.Type, forge.Url)
+	if *syncOnly {
+		if err := syncToTypesense(config.Typesense.Host, typesenseAPIKey, db); err != nil {
+			fmt.Fprintf(os.Stderr, "typesense sync failed: %v\n", err)
+			os.Exit(1)
+		}
+		fmt.Println("sync-only complete")
+		return
+	}
+
+	for _, forge := range config.Forges {
+		if err := forge.Validate(); err != nil {
+			fmt.Fprintf(os.Stderr, "  skip: %v\n", err)
+			continue
+		}
+		// Split the configured address into a connection scheme ("https" unless
+		// the user forced http) and the bare FQDN. The FQDN is the consistent
+		// identity across forge_status, project identities and the metrics
+		// labels; the scheme only affects how we connect.
+		scheme, fqdn := scraper.Scheme(forge.URL)
+
+		lastStatus, err := db.ForgeStatus(fqdn)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "  skip %s: %v\n", forge.Name, err)
+			fmt.Fprintf(os.Stderr, "  status lookup: %v\n", err)
+			continue
+		}
+
+		// Determine the forge type. An explicit type in the config (forgejo or
+		// gitea) is used as-is and skips detection. "auto" probes the
+		// Forgejo-native version endpoint first (Forgejo also serves the
+		// shared one), then the shared /api/v1/version which implies Gitea. If
+		// it stays unknown we cannot scrape it (no scraper for that type yet)
+		// and record it as offline, keeping any previously recorded
+		// type/version.
+		var forgeType scraper.Type
+		var version string
+		if forge.Type != "auto" {
+			forgeType = scraper.Type(forge.Type)
+		} else {
+			forgeType, version, err = scraper.DetectType(scheme, fqdn)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "  skip %s: detect type: %v\n", fqdn, err)
+				recordStatus(db, store.ForgeStatus{
+					URL: fqdn, ForgeType: lastStatus.ForgeType, Version: lastStatus.Version,
+					Online: false, LastScrapedAt: time.Now().UTC(), LastError: err.Error(),
+				})
+				continue
+			}
+			if forgeType == scraper.TypeUnknown {
+				fmt.Fprintf(os.Stderr, "  skip %s: unknown forge type\n", fqdn)
+				recordStatus(db, store.ForgeStatus{
+					URL: fqdn, ForgeType: lastStatus.ForgeType, Version: lastStatus.Version,
+					Online: false, LastScrapedAt: time.Now().UTC(), LastError: "unknown forge type",
+				})
+				continue
+			}
+		}
+
+		fmt.Printf("Scraping %s (%s, %s)\n", fqdn, forgeType, version)
+		record := func(online bool, errMsg string) {
+			recordStatus(db, store.ForgeStatus{
+				URL:           fqdn,
+				ForgeType:     string(forgeType),
+				Version:       version,
+				Online:        online,
+				LastScrapedAt: time.Now().UTC(),
+				LastError:     errMsg,
+			})
+		}
+
+		s, err := scraper.NewScraper(forgeType, scheme, fqdn)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "  skip %s: %v\n", fqdn, err)
+			record(false, err.Error())
 			continue
 		}

 		projects, err := s.GetProjects()
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "  skip %s: %v\n", forge.Name, err)
+			fmt.Fprintf(os.Stderr, "  skip %s: %v\n", fqdn, err)
+			record(false, err.Error())
 			continue
 		}

@ -54,8 +146,47 @@ func main() {
 		all, err := db.All()
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "  count: %v\n", err)
+			record(true, "")
 			continue
 		}
 		fmt.Printf("  total in db: %d\n", len(all))
+
+		record(true, "")
+	}
+
+	// Index the stored projects into Typesense for the search UI.
+	// A failure here is fatal: the scrape itself survived per-forge / per-repo
+	// errors, but if we cannot put the data into Typesense the run is not
+	// complete and should fail loudly (e.g. for systemd to catch).
+	if err := syncToTypesense(config.Typesense.Host, typesenseAPIKey, db); err != nil {
+		fmt.Fprintf(os.Stderr, "typesense sync failed: %v\n", err)
+		os.Exit(1)
+	}
+}
+
+// syncToTypesense pushes the store's projects into the Typesense index.
+func syncToTypesense(host, apiKey string, db *store.Store) error {
+	if host == "" || apiKey == "" {
+		return fmt.Errorf("typesense host or API key not configured")
+	}
+
+	client := typesense.NewClient(
+		typesense.WithServer(host),
+		typesense.WithAPIKey(apiKey),
+	)
+	svc := sync.NewService(client)
+
+	if err := svc.Sync(context.Background(), db); err != nil {
+		return err
+	}
+	fmt.Printf("synced projects to Typesense (%s)\n", host)
+	return nil
+}
+
+// recordStatus writes a forge status row, logging (but not failing on)
+// storage errors.
+func recordStatus(db *store.Store, st store.ForgeStatus) {
+	if err := db.RecordForgeStatus(st); err != nil {
+		fmt.Fprintf(os.Stderr, "  record status: %v\n", err)
 	}
 }
--- a/metrics/metrics.go
+++ b/metrics/metrics.go
@ -0,0 +1,127 @@
+// Package metrics exposes the state of the federated forge index to
+// Prometheus.
+//
+// The ffd-metrics daemon is a long-running process that reads the SQLite
+// store on each scrape and renders it as Prometheus text metrics at
+// /metrics. It never touches the forges itself: the indexer's daily run
+// populates the store, and this daemon merely reflects the current DB state.
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+
+	"git.xengi.de/ffd/indexer/store"
+)
+
+// Collector implements prometheus.Collector by querying the store on every
+// Prometheus scrape. Gather is therefore always fresh but also does one
+// SQLite read per scrape.
+type Collector struct {
+	store *store.Store
+
+	forgeInfo       *prometheus.Desc
+	forgeOnline     *prometheus.Desc
+	repositoryInfo  *prometheus.Desc
+	repositoryLangs *prometheus.Desc
+}
+
+// NewCollector creates a Collector backed by st.
+func NewCollector(st *store.Store) *Collector {
+	return &Collector{
+		store: st,
+		forgeInfo: prometheus.NewDesc(
+			"ffd_forge_info",
+			"Constant gauge with the forge's type and detected software version",
+			[]string{"forge_url", "forge_type", "forge_version"},
+			nil,
+		),
+		forgeOnline: prometheus.NewDesc(
+			"ffd_forge_online",
+			"Whether a forge was reachable and scraped successfully on its last run (1) or not (0).",
+			[]string{"forge_url", "forge_type"},
+			nil,
+		),
+		repositoryInfo: prometheus.NewDesc(
+			"ffd_repository_info",
+			"Constant gauge, one series per stored repository. Count by forge_url to get repositories per forge.",
+			[]string{"repo_url", "forge_url", "forge_type"},
+			nil,
+		),
+		repositoryLangs: prometheus.NewDesc(
+			"ffd_repository_languages",
+			"Constant gauge, one series per (repository, language) pair. Sum by language to get language usage.",
+			[]string{"repo_url", "language", "forge_url", "forge_type"},
+			nil,
+		),
+	}
+}
+
+// Describe implements prometheus.Collector.
+func (c *Collector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- c.forgeInfo
+	ch <- c.forgeOnline
+	ch <- c.repositoryInfo
+	ch <- c.repositoryLangs
+}
+
+// Collect implements prometheus.Collector. Query errors are surfaced as
+// invalid metrics for each family so the scrape still produces a value (with
+// a 500 if the handler is configured to propagate errors) instead of silently
+// dropping data.
+func (c *Collector) Collect(ch chan<- prometheus.Metric) {
+	statuses, err := c.store.ForgeStatuses()
+	if err != nil {
+		c.emitInvalid(ch, err)
+		return
+	}
+
+	for _, st := range statuses {
+		online := 0.0
+		if st.Online {
+			online = 1
+		}
+		ch <- prometheus.MustNewConstMetric(
+			c.forgeInfo,
+			prometheus.GaugeValue,
+			1,
+			st.URL, st.ForgeType, st.Version,
+		)
+		ch <- prometheus.MustNewConstMetric(
+			c.forgeOnline,
+			prometheus.GaugeValue,
+			online,
+			st.URL, st.ForgeType,
+		)
+	}
+
+	repos, err := c.store.RepoIdentities()
+	if err != nil {
+		c.emitInvalid(ch, err)
+		return
+	}
+
+	for _, r := range repos {
+		forge := []string{r.ForgeURL, r.ForgeType}
+		ch <- prometheus.MustNewConstMetric(
+			c.repositoryInfo,
+			prometheus.GaugeValue,
+			1,
+			append([]string{r.URL}, forge...)...,
+		)
+		for _, lang := range r.Languages {
+			ch <- prometheus.MustNewConstMetric(
+				c.repositoryLangs,
+				prometheus.GaugeValue,
+				1,
+				append([]string{r.URL, lang}, forge...)...,
+			)
+		}
+	}
+}
+
+// emitInvalid emits one invalid metric per metric family for scrape errors.
+func (c *Collector) emitInvalid(ch chan<- prometheus.Metric, err error) {
+	for _, d := range []*prometheus.Desc{c.forgeInfo, c.forgeOnline, c.repositoryInfo, c.repositoryLangs} {
+		ch <- prometheus.NewInvalidMetric(d, err)
+	}
+}
--- a/option.nix
+++ b/option.nix
@ -2,56 +2,233 @@
  config,
  lib,
  pkgs,
-  options,
  ...
 }:

 let
  cfg = config.services.federated-forge-discovery.indexer;
-  configFile = pkgs.writeText "config.ini" (pkgs.lib.generators.toINI { } cfg);
-  execCommand = "${cfg.package}/bin/ffd-indexer -c '${configFile}'";
+  yaml = pkgs.lib.generators.toYAML { };
+
+  # NixOS package for the indexer, injected via the flake.
+  pkg = cfg.package;
+
+  # Location where the rendered config is written.
+  configDir = "/etc/ffd-indexer";
+  configFile = "${configDir}/config.yaml";
+
+  # The rendered YAML config, mirroring the config.go structure.
+  nixConfig = {
+    db_path = cfg.dbPath;
+    typesense = {
+      host = cfg.typesense.host;
+      api_key = cfg.typesense.apiKey;
+    }
+    // lib.optionalAttrs (cfg.typesense.apiKeyFile != null) {
+      # A secret file (sops/agenix) overrides the inline api_key in Go's
+      # config resolution, so it also wins here.
+      api_key_file = cfg.typesense.apiKeyFile;
+    };
+    # The metrics daemon is an optional component: only render the block
+    # when it is enabled, so an operator running just ffd-indexer can omit
+    # the metrics section entirely.
+    }
+    // lib.optionalAttrs cfg.metrics.enable {
+      metrics = {
+        listen = cfg.metrics.listen;
+      };
+    };
+    forges = cfg.forges;
+  };
+
+  configYAML = pkgs.writeText "config.yaml" (yaml nixConfig);
 in
 {
  options.services.federated-forge-discovery.indexer = {
-    enable = lib.mkEnableOption "Enables the Federated Forge Discovery Indexer systemd service.";
-    openFirewall = lib.mkOption {
+    enable = lib.mkEnableOption "Enables the Federated Forge Discovery Indexer systemd service and timer.";

-    };
    package = lib.mkOption {
-      description = "Package to use.";
+      description = "Indexer package to use.";
      type = lib.types.package;
-      default = lib.literalExpression ''
-        pkgs.ffd-indexer
-      '';
+      # Provided by the flake overlay (see flake.nix overlays.default), so
+      # `pkgs.ffd-indexer` resolves to this project's package.
+      default = pkgs.ffd-indexer;
    };
-    port = lib.mkOption {
-      description = "Port to listen for search engines";
-      type = lib.types.port;
-      default = 47810;
+
+    schedule = lib.mkOption {
+      description = ''
+        How often the indexer runs. Either one of "daily", "hourly", "weekly",
+        "monthly", or any systemd OnCalendar expression.
+      '';
+      type = lib.types.str;
+      default = "daily";
+      example = "daily";
+    };
+
+    dbPath = lib.mkOption {
+      description = "Path to the SQLite database holding the index.";
+      type = lib.types.path;
+      default = "/var/lib/ffd-indexer/index.db";
+      example = "/var/lib/ffd-indexer/index.db";
+    };
+
+    typesense = {
+      host = lib.mkOption {
+        description = "Typesense server URL.";
+        type = lib.types.str;
+        default = "http://localhost:8108";
+        example = "http://localhost:8108";
+      };
+      apiKey = lib.mkOption {
+        description = ''
+          Typesense API key. May be inlined here, or supplied via a secret file with apiKeyFile (which takes precedence)
+        '';
+        type = lib.types.str;
+        default = "";
+      };
+      apiKeyFile = lib.mkOption {
+        description = ''
+          Path to a file containing the Typesense API key. Takes precedence over apiKey, so the key need not appear in
+          the rendered config or the Nix store.
+        '';
+        type = lib.types.nullOr lib.types.path;
+        default = null;
+        example = "/run/secrets/typesense-api-key";
+      };
+    };
+
+    metrics = {
+      enable = lib.mkEnableOption "the ffd-metrics Prometheus daemon (also controls whether the metrics block is rendered into the config)";
+      listen = lib.mkOption {
+        description = "Listen address the ffd-metrics daemon binds for /metrics.";
+        type = lib.types.str;
+        default = "127.0.0.1:9090";
+        example = "127.0.0.1:9090";
+      };
+    };
+
+    forges = lib.mkOption {
+      description = ''
+        Forge instances to scrape, as a list of "url" -> "type" entries,
+        e.g. [{ url = "git.example.org"; type = "forgejo"; }]. The url is a
+        bare hostname (FQDN); a scheme prefix (http:// or https://) is
+        allowed to force the connection protocol, but the stored identity and
+        metrics labels always use the bare hostname.
+
+        The recommended type is a pinned "forgejo" or "gitea", which selects
+        the scraper deterministically and skips probing. The default "auto"
+        re-detects the type on every run and is EXPERIMENTAL: detection is
+        probed via the version API, honors robots.txt (a forge that disallows
+        /api/ is reported "unknown" unless hard-coded), and may misclassify
+        until more detectors land. Prefer setting the type by hand.
+      '';
+      type = lib.types.listOf (
+        lib.types.submodule {
+          options = {
+            url = lib.mkOption {
+              description = "Forge hostname (FQDN) to scrape, e.g. \"git.example.org\".";
+              type = lib.types.str;
+            };
+            type = lib.mkOption {
+              description = ''
+                Forge type. Recommended: "forgejo" or "gitea" (pin the scraper,
+                skip detection). Default "auto" re-detects on every scan
+                (EXPERIMENTAL).
+              '';
+              type = lib.types.enum [
+                "auto"
+                "forgejo"
+                "gitea"
+              ];
+              default = "auto";
+            };
+          };
+        }
+      );
+      default = [ ];
+      example = [
+        {
+          url = "git.example.org";
+          type = "forgejo";
+        }
+        {
+          url = "git.example.net";
+          type = "forgejo";
+        }
+      ];
    };
  };

  config = lib.mkIf cfg.enable {
-    systemd.services."ffd-indexer" = {
+    # Write the rendered config and make it readable by the service user.
+    environment.etc."${lib.removePrefix "/etc/" configFile}" = {
+      source = configYAML;
+      user = "ffd-indexer";
+      group = "ffd-indexer";
+      mode = "0400";
+    };
+
+    users.users.ffd-indexer = {
      description = "Federated Forge Discovery Indexer";
+      isSystemUser = true;
+      group = "ffd-indexer";
+      home = "/var/lib/ffd-indexer";
+      createHome = true;
+    };
+    users.groups.ffd-indexer = { };
+
+    systemd.services.ffd-indexer = {
+      description = "Federated Forge Discovery Indexer";
+      wantedBy = [ "multi-user.target" ];
      wants = [ "network-online.target" ];
      after = [ "network-online.target" ];
      serviceConfig = {
-        Restart = "always";
-        RestartSec = 30;
-        ExecStart = execCommand;
+        Type = "oneshot";
+        ExecStart = "${pkg}/bin/ffd-indexer -config ${configFile}";
        User = "ffd-indexer";
        Group = "ffd-indexer";
-        AmbientCapabilities = lib.mkIf (cfg.port < 1000) [ "CAP_NET_BIND_SERVICE" ];
-        CapabilityBoundingSet = lib.mkIf (cfg.port < 1000) [ "CAP_NET_BIND_SERVICE" ];
        NoNewPrivileges = true;
+        # The DB and Typesense data live under /var/lib/ffd-indexer.
+        StateDirectory = "ffd-indexer";
+        Restart = "on-failure";
+        RestartSec = 30;
      };
+    };
+
+    # The timer drives the schedule; the service is oneshot.
+    systemd.timers.ffd-indexer = {
+      description = "Runs the Federated Forge Discovery Indexer on schedule";
+      wantedBy = [ "timers.target" ];
+      timerConfig = {
+        OnCalendar = cfg.schedule;
+        Persistent = true;
+        Unit = "ffd-indexer.service";
+      };
+    };
+
+    # Long-running metrics daemon: reads the same DB the indexer writes to and
+    # serves Prometheus metrics at the configured address. It is a separate
+    # service so the indexer can stay a oneshot scheduled job, and it is
+    # optional (disable metrics.enable to run only the indexer).
+    systemd.services.ffd-metrics = lib.mkIf cfg.metrics.enable {
+      description = "Federated Forge Discovery Indexer metrics daemon";
      wantedBy = [ "multi-user.target" ];
+      wants = [ "network-online.target" ];
+      after = [
+        "network-online.target"
+        "ffd-indexer.service"
+      ];
+      serviceConfig = {
+        Type = "simple";
+        ExecStart = "${pkg}/bin/ffd-metrics -config ${configFile}";
+        User = "ffd-indexer";
+        Group = "ffd-indexer";
+        NoNewPrivileges = true;
+        StateDirectory = "ffd-indexer";
+        Restart = "on-failure";
+        RestartSec = 5;
+      };
    };
  };

-  #meta = {
-  #  maintainers = with lib.maintainers; [ xengi ];
-  #  doc = ./default.xml;
-  #};
+  meta.maintainers = with lib.maintainers; [ ];
 }
--- a/Show more
+++ b/Show more