🔥

sync/sync.go

@@ -0,0 +1,149 @@
+// Package sync indexes the projects stored in SQLite into a Typesense
+// collection, making them searchable by the UI.
+//
+// The Typesense collection is the search projection of the SQLite data:
+// it can be rebuilt at any time from the store without re-scraping the forges.
+package sync
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/typesense/typesense-go/v2/typesense"
+	"github.com/typesense/typesense-go/v2/typesense/api"
+	"github.com/typesense/typesense-go/v2/typesense/api/pointer"
+
+	"git.xengi.de/ffd/indexer/scraper"
+	"git.xengi.de/ffd/indexer/store"
+)
+
+// CollectionName is the name of the Typesense collection holding projects.
+const CollectionName = "projects"
+
+// Service copies projects from the store into Typesense.
+type Service struct {
+	client *typesense.Client
+}
+
+// NewService creates a sync service for the given Typesense server.
+func NewService(client *typesense.Client) *Service {
+	return &Service{client: client}
+}
+
+// document is the JSON shape sent to Typesense for each project.
+type document struct {
+	ID                string   `json:"id"`
+	Name              string   `json:"name"`
+	URL               string   `json:"url"`
+	Description       string   `json:"description"`
+	Topics            []string `json:"topics"`
+	Languages         []string `json:"languages"`
+	Licenses          []string `json:"licenses"`
+	OpenIssues        int64    `json:"open_issues"`
+	OpenPRs           int64    `json:"open_prs"`
+	LatestCommit      int64    `json:"latest_commit"`
+	LatestReleaseName string   `json:"latest_release_name"`
+	LatestReleaseDate int64    `json:"latest_release_date"`
+	UsesAI            bool     `json:"uses_ai"`
+	Readme            string   `json:"readme"`
+}
+
+// ensureCollection creates the projects collection if it does not exist.
+func (s *Service) ensureCollection(ctx context.Context) error {
+	_, err := s.client.Collection(CollectionName).Retrieve(ctx)
+	if err == nil {
+		return nil
+	}
+
+	schema := &api.CollectionSchema{
+		Name: CollectionName,
+		Fields: []api.Field{
+			{Name: "name", Type: "string"},
+			{Name: "url", Type: "string"},
+			{Name: "description", Type: "string"},
+			{Name: "readme", Type: "string"},
+			{Name: "topics", Type: "string[]", Facet: pointer.True()},
+			{Name: "languages", Type: "string[]", Facet: pointer.True()},
+			{Name: "licenses", Type: "string[]", Facet: pointer.True()},
+			{Name: "open_issues", Type: "int32"},
+			{Name: "open_prs", Type: "int32"},
+			{Name: "latest_commit", Type: "int64"},
+			{Name: "latest_release_name", Type: "string"},
+			{Name: "latest_release_date", Type: "int64"},
+			{Name: "uses_ai", Type: "bool"},
+		},
+		DefaultSortingField: new("latest_commit"),
+	}
+
+	_, err = s.client.Collections().Create(ctx, schema)
+	return err
+}
+
+// projectToDocument maps a stored project to the Typesense document shape.
+func projectToDocument(p scraper.Project) document {
+	return document{
+		ID:                p.Url,
+		Name:              p.Name,
+		URL:               p.Url,
+		Description:       p.Description,
+		Topics:            p.Topics,
+		Languages:         p.Languages,
+		Licenses:          p.Licenses,
+		OpenIssues:        int64(p.OpenIssues),
+		OpenPRs:           int64(p.OpenPRs),
+		LatestCommit:      unixTime(p.LatestCommit),
+		LatestReleaseName: p.LatestRelease.Name,
+		LatestReleaseDate: unixTime(p.LatestRelease.Date),
+		UsesAI:            p.UsesAI,
+		Readme:            p.Readme,
+	}
+}
+
+// unixTime returns the Unix epoch seconds for a time, or 0 for the zero time.
+func unixTime(t time.Time) int64 {
+	if t.IsZero() {
+		return 0
+	}
+	return t.Unix()
+}
+
+// Sync reads every project from the store and upserts it into Typesense.
+func (s *Service) Sync(ctx context.Context, st *store.Store) error {
+	if err := s.ensureCollection(ctx); err != nil {
+		return fmt.Errorf("ensure collection: %w", err)
+	}
+
+	projects, err := st.All()
+	if err != nil {
+		return fmt.Errorf("read projects from store: %w", err)
+	}
+
+	docs := make([]any, 0, len(projects))
+	for _, p := range projects {
+		docs = append(docs, projectToDocument(p))
+	}
+	if len(docs) == 0 {
+		return nil
+	}
+
+	params := &api.ImportDocumentsParams{
+		// upsert so re-indexing updates in place; missing docs stay as-is.
+		Action: new("upsert"),
+	}
+	results, err := s.client.Collection(CollectionName).Documents().Import(ctx, docs, params)
+	if err != nil {
+		return fmt.Errorf("bulk import into Typesense: %w", err)
+	}
+
+	var failed int
+	for _, r := range results {
+		if r == nil || !r.Success {
+			failed++
+		}
+	}
+	if failed > 0 {
+		return fmt.Errorf("bulk import: %d of %d documents failed", failed, len(results))
+	}
+	return nil
+}