indexer/sync/sync.go

// Package sync indexes the projects stored in SQLite into a Typesense
// collection, making them searchable by the UI.
//
// The Typesense collection is the search projection of the SQLite data:
// it can be rebuilt at any time from the store without re-scraping the forges.
package sync

import (
	"context"
	"fmt"
	"time"

	"github.com/typesense/typesense-go/v2/typesense"
	"github.com/typesense/typesense-go/v2/typesense/api"
	"github.com/typesense/typesense-go/v2/typesense/api/pointer"

	"git.xengi.de/ffd/indexer/scraper"
	"git.xengi.de/ffd/indexer/store"
)

// CollectionName is the name of the Typesense collection holding projects.
const CollectionName = "projects"

// Service copies projects from the store into Typesense.
type Service struct {
	client *typesense.Client
}

// NewService creates a sync service for the given Typesense server.
func NewService(client *typesense.Client) *Service {
	return &Service{client: client}
}

// document is the JSON shape sent to Typesense for each project.
type document struct {
	ID                string   `json:"id"`
	Name              string   `json:"name"`
	URL               string   `json:"url"`
	Description       string   `json:"description"`
	Topics            []string `json:"topics"`
	Languages         []string `json:"languages"`
	Licenses          []string `json:"licenses"`
	OpenIssues        int64    `json:"open_issues"`
	OpenPRs           int64    `json:"open_prs"`
	LatestCommit      int64    `json:"latest_commit"`
	LatestReleaseName string   `json:"latest_release_name"`
	LatestReleaseDate int64    `json:"latest_release_date"`
	UsesAI            bool     `json:"uses_ai"`
	Readme            string   `json:"readme"`
}

// ensureCollection creates the projects collection if it does not exist.
func (s *Service) ensureCollection(ctx context.Context) error {
	_, err := s.client.Collection(CollectionName).Retrieve(ctx)
	if err == nil {
		return nil
	}

	schema := &api.CollectionSchema{
		Name: CollectionName,
		Fields: []api.Field{
			{Name: "name", Type: "string"},
			{Name: "url", Type: "string"},
			{Name: "description", Type: "string"},
			{Name: "readme", Type: "string"},
			{Name: "topics", Type: "string[]", Facet: pointer.True()},
			{Name: "languages", Type: "string[]", Facet: pointer.True()},
			{Name: "licenses", Type: "string[]", Facet: pointer.True()},
			{Name: "open_issues", Type: "int32"},
			{Name: "open_prs", Type: "int32"},
			{Name: "latest_commit", Type: "int64"},
			{Name: "latest_release_name", Type: "string"},
			{Name: "latest_release_date", Type: "int64"},
			{Name: "uses_ai", Type: "bool"},
		},
		DefaultSortingField: new("latest_commit"),
	}

	_, err = s.client.Collections().Create(ctx, schema)
	return err
}

// projectToDocument maps a stored project to the Typesense document shape.
func projectToDocument(p scraper.Project) document {
	return document{
		ID:                p.Url,
		Name:              p.Name,
		URL:               p.Url,
		Description:       p.Description,
		Topics:            p.Topics,
		Languages:         p.Languages,
		Licenses:          p.Licenses,
		OpenIssues:        int64(p.OpenIssues),
		OpenPRs:           int64(p.OpenPRs),
		LatestCommit:      unixTime(p.LatestCommit),
		LatestReleaseName: p.LatestRelease.Name,
		LatestReleaseDate: unixTime(p.LatestRelease.Date),
		UsesAI:            p.UsesAI,
		Readme:            p.Readme,
	}
}

// unixTime returns the Unix epoch seconds for a time, or 0 for the zero time.
func unixTime(t time.Time) int64 {
	if t.IsZero() {
		return 0
	}
	return t.Unix()
}

// Sync reads every project from the store and upserts it into Typesense.
func (s *Service) Sync(ctx context.Context, st *store.Store) error {
	if err := s.ensureCollection(ctx); err != nil {
		return fmt.Errorf("ensure collection: %w", err)
	}

	projects, err := st.All()
	if err != nil {
		return fmt.Errorf("read projects from store: %w", err)
	}

	docs := make([]any, 0, len(projects))
	for _, p := range projects {
		docs = append(docs, projectToDocument(p))
	}
	if len(docs) == 0 {
		return nil
	}

	params := &api.ImportDocumentsParams{
		// upsert so re-indexing updates in place; missing docs stay as-is.
		Action: new("upsert"),
	}
	results, err := s.client.Collection(CollectionName).Documents().Import(ctx, docs, params)
	if err != nil {
		return fmt.Errorf("bulk import into Typesense: %w", err)
	}

	var failed int
	for _, r := range results {
		if r == nil || !r.Success {
			failed++
		}
	}
	if failed > 0 {
		return fmt.Errorf("bulk import: %d of %d documents failed", failed, len(results))
	}
	return nil
}