🔥

2026-10-01 00:37:44 +02:00 · 2026-10-01 00:37:44 +02:00 · 63c6f4b761
commit 63c6f4b761
parent bde13e9b68
2 changed files with 3319 additions and 184 deletions
--- a/.forgejo/workflows/ci.yaml
+++ b/.forgejo/workflows/ci.yaml
@ -0,0 +1,159 @@
+name: CI pipeline for ffd-indexer
+
+on:
+  push:
+    branches: ["**"]
+  pull_request:
+    branches: ["**"]
+  workflow_dispatch:
+
+jobs:
+  checks:
+    name: checks (go test + golangci)
+    runs-on: debian-stable
+    container:
+      image: nixos/nix:latest
+    steps:
+      - name: Checkout
+        run: |
+          set -euo pipefail
+          git config --global --add safe.directory "$PWD"
+          ref="${FORGEJO_HEAD_REF:-$FORGEJO_REF_NAME}"
+          git clone --depth 1 --branch "$ref" "${FORGEJO_SERVER_URL}/${FORGEJO_REPOSITORY}.git" .
+      # `nix flake check` builds and runs the flake's `checks`:
+      #   go-test  -> go test -v ./...
+      #   go-lint  -> golangci-lint run
+      - name: nix flake check
+        run: nix flake check --accept-flake-config
+
+  build:
+    name: build binaries + containers
+    runs-on: debian-stable
+    container:
+      image: nixos/nix:latest
+    steps:
+      - name: Checkout
+        run: |
+          set -euo pipefail
+          git config --global --add safe.directory "$PWD"
+          ref="${FORGEJO_HEAD_REF:-$FORGEJO_REF_NAME}"
+          git clone --depth 1 --branch "$ref" "${FORGEJO_SERVER_URL}/${FORGEJO_REPOSITORY}.git" .
+      # Build every deliverable: the indexer binary, the metrics daemon and
+      # both container images. typesense-server is a local dev helper and is
+      # deliberately not built here.
+      - name: nix build
+        run: nix build --accept-flake-config .#default .#ffd-metrics .#container .#container-metrics
+
+  format:
+    name: format (gofmt + nixfmt)
+    runs-on: debian-stable
+    container:
+      image: nixos/nix:latest
+    steps:
+      - name: Checkout
+        run: |
+          set -euo pipefail
+          git config --global --add safe.directory "$PWD"
+          ref="${FORGEJO_HEAD_REF:-$FORGEJO_REF_NAME}"
+          git clone --depth 1 --branch "$ref" "${FORGEJO_SERVER_URL}/${FORGEJO_REPOSITORY}.git" .
+      - name: gofmt
+        run: |
+          nix develop --command bash -c '
+            unformatted="$(gofmt -l .)"
+            if [ -n "$unformatted" ]; then
+              echo "gofmt needed for:"
+              printf "%s\n" "$unformatted"
+              exit 1
+            fi
+            echo "gofmt: clean"
+          '
+      - name: nixfmt
+        run: |
+          git config --global --add safe.directory "$PWD"
+          nix fmt --accept-flake-config >/dev/null
+          if ! git diff --quiet -- '*.nix'; then
+            echo "nixfmt would reformat these files:"
+            git diff --name-only -- '*.nix'
+            exit 1
+          fi
+          echo "nixfmt: clean"
+
+  integration:
+    name: integration (live scrape + Typesense)
+    runs-on: debian-stable
+    container:
+      image: nixos/nix:latest
+    services:
+      typesense:
+        image: typesense/typesense:30.2
+        env:
+          TYPESENSE_API_KEY: dev-key
+          TYPESENSE_DATA_DIR: /data
+          TYPESENSE_API_PORT: "8108"
+        ports:
+          - "8108:8108"
+        options: >-
+          --tmpfs /data:rw,noexec,nosuid,size=256m
+    steps:
+      - name: Checkout
+        run: |
+          set -euo pipefail
+          git config --global --add safe.directory "$PWD"
+          ref="${FORGEJO_HEAD_REF:-$FORGEJO_REF_NAME}"
+          git clone --depth 1 --branch "$ref" "${FORGEJO_SERVER_URL}/${FORGEJO_REPOSITORY}.git" .
+      # Scrape git.xengi.de live, sync into the Typesense service, then assert
+      # end to end. Runs inside the flake's dev shell so sqlite3 and the Go
+      # toolchain are available; the typesense service is reached at the
+      # service hostname `typesense` on port 8108.
+      - name: Integration (scrape + sync + assert)
+        run: |
+          set -euo pipefail
+
+          # Config: point Typesense at the service container.
+          cat > ci-config.yaml <<'EOF'
+          db_path: "./ci-index.db"
+          typesense:
+            host: "http://typesense:8108"
+            api_key: "dev-key"
+          forges:
+            - url: git.xengi.de
+              type: auto
+          EOF
+
+          # Build the binary from the flake.
+          BIN="$(nix build .#default --accept-flake-config --no-link --print-out-paths)/bin/ffd-indexer"
+
+          # Run the whole assertion as one heredoc inside the dev shell.
+          nix develop --command bash <<'EOF'
+          set -euo pipefail
+
+          # Wait for Typesense to be healthy
+          for _ in $(seq 1 30); do
+            [ "$(curl -fsS -o /dev/null -w '%{http_code}' http://typesense:8108/health 2>/dev/null || echo 000)" = "200" ] && break
+            sleep 2
+          done
+          code="$(curl -fsS -o /dev/null -w '%{http_code}' http://typesense:8108/health 2>/dev/null || echo 000)"
+          [ "$code" = "200" ] || { echo "typesense not healthy (http $code)"; exit 1; }
+
+          # Scrape + sync (fails loudly on error).
+          "$BIN" -config ci-config.yaml
+
+          # sqlite: forge online + at least one project.
+          online="$(sqlite3 ci-index.db "SELECT 1 FROM forge_status WHERE url='git.xengi.de' AND online=1 LIMIT 1;")"
+          [ "$online" = "1" ] || { echo "forge git.xengi.de not online"; exit 1; }
+          projects="$(sqlite3 ci-index.db "SELECT COUNT(*) FROM projects;")"
+          echo "projects in sqlite: $projects"
+          [ "${projects:-0}" -ge 1 ] || { echo "no projects stored"; exit 1; }
+
+          # Typesense: collection exists and doc count == sqlite count.
+          curl -fsS -H "X-TYPESENSE-API-KEY: dev-key" http://typesense:8108/collections/projects >/dev/null
+          # python3 is not in the job image; extract `found` from the search JSON
+          # with grep (Typesense reports the total in a top-level "found" field).
+          found="$(curl -fsS -H "X-TYPESENSE-API-KEY: dev-key" \
+            "http://typesense:8108/collections/projects/documents/search?q=*&per_page=1" \
+            | grep -o '"found": *[0-9]*' | grep -o '[0-9][0-9]*' | head -1)"
+          echo "documents in Typesense: $found (sqlite: $projects)"
+          [ "$found" = "$projects" ] || { echo "Typesense count mismatch"; exit 1; }
+
+          echo "integration OK: $projects projects scraped from git.xengi.de and indexed"
+          EOF
--- a/.forgejo/workflows/container.yaml
+++ b/.forgejo/workflows/container.yaml
@ -0,0 +1,70 @@
+name: Build and publish containers on tags
+
+on:
+  push:
+    tags: ["**"]
+  workflow_dispatch:
+
+jobs:
+  container:
+    name: build + publish ffd-indexer / ffd-metrics
+    runs-on: debian-stable
+    env:
+      # Forgejo does not auto-expose secrets as env vars; map it explicitly
+      # so the whole step (including the inner heredoc) can read it.
+      REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
+    container:
+      image: nixos/nix:latest
+    steps:
+      - name: Checkout
+        run: |
+          set -euo pipefail
+          git config --global --add safe.directory "$PWD"
+          ref="${FORGEJO_HEAD_REF:-$FORGEJO_REF_NAME}"
+          git clone --depth 1 --branch "$ref" "${FORGEJO_SERVER_URL}/${FORGEJO_REPOSITORY}.git" .
+
+      # Build the two OCI images from the flake:
+      #   .#container          -> ffd-indexer  (the scraper)
+      #   .#container-metrics  -> ffd-metrics  (the metrics daemon)
+      # No container daemon/buildkit is needed: `nix develop` provides
+      # `skopeo`, which copies the resulting image tarballs straight to the
+      # Forgejo container registry.
+      #
+      # Both containers are pushed with the SAME tag; their names differ:
+      #   git.xengi.de/ffd/indexer:<tag>           ffd-indexer
+      #   git.xengi.de/ffd/indexer-metrics:<tag>   ffd-metrics
+      # Each also gets a `:latest` on its own name.
+      - name: Build and publish containers
+        run: |
+          set -euo pipefail
+
+          # Image tag: the git tag that triggered this run, without a leading
+          # "v" (e.g. v1.2.3 -> 1.2.3).
+          ref="${FORGEJO_REF_NAME:?}"
+          tag="${ref##*/}"   # drop any refs/tags/ prefix
+          tag="${tag#v}"     # drop a leading v
+          [ -n "$tag" ] || { echo "empty tag" >&2; exit 1; }
+
+          registry="${FORGEJO_SERVER_URL#https://}" # git.xengi.de
+          owner="${FORGEJO_REPOSITORY_OWNER:?}"     # ffd
+          repo="${FORGEJO_REPOSITORY##*/}"          # indexer
+          token="${REGISTRY_TOKEN:?}"
+          echo "publishing tag=$tag to $registry/$owner"
+
+          export registry owner repo tag token
+          nix develop --command bash <<'INNER'
+          set -euo pipefail
+
+          idx="$(nix build .#container --accept-flake-config --no-link --print-out-paths)"
+          met="$(nix build .#container-metrics --accept-flake-config --no-link --print-out-paths)"
+
+          # Same tag for both images, distinct names:
+          skopeo copy --dest-creds "$owner:$token" \
+            "docker-archive:$idx" "docker://$registry/$owner/$repo:$tag"
+          skopeo copy --dest-creds "$owner:$token" \
+            "docker-archive:$idx" "docker://$registry/$owner/$repo:latest"
+          skopeo copy --dest-creds "$owner:$token" \
+            "docker-archive:$met" "docker://$registry/$owner/${repo}-metrics:$tag"
+          skopeo copy --dest-creds "$owner:$token" \
+            "docker-archive:$met" "docker://$registry/$owner/${repo}-metrics:latest"
+          INNER