// Package sync indexes the projects stored in SQLite into a Typesense // collection, making them searchable by the UI. // // The Typesense collection is the search projection of the SQLite data: // it can be rebuilt at any time from the store without re-scraping the forges. package sync import ( "context" "fmt" "time" "github.com/typesense/typesense-go/v2/typesense" "github.com/typesense/typesense-go/v2/typesense/api" "github.com/typesense/typesense-go/v2/typesense/api/pointer" "git.xengi.de/ffd/indexer/scraper" "git.xengi.de/ffd/indexer/store" ) // CollectionName is the name of the Typesense collection holding projects. const CollectionName = "projects" // Service copies projects from the store into Typesense. type Service struct { client *typesense.Client } // NewService creates a sync service for the given Typesense server. func NewService(client *typesense.Client) *Service { return &Service{client: client} } // document is the JSON shape sent to Typesense for each project. type document struct { ID string `json:"id"` Name string `json:"name"` URL string `json:"url"` Description string `json:"description"` Topics []string `json:"topics"` Languages []string `json:"languages"` Licenses []string `json:"licenses"` OpenIssues int64 `json:"open_issues"` OpenPRs int64 `json:"open_prs"` LatestCommit int64 `json:"latest_commit"` LatestReleaseName string `json:"latest_release_name"` LatestReleaseDate int64 `json:"latest_release_date"` UsesAI bool `json:"uses_ai"` Readme string `json:"readme"` } // ensureCollection creates the projects collection if it does not exist. func (s *Service) ensureCollection(ctx context.Context) error { _, err := s.client.Collection(CollectionName).Retrieve(ctx) if err == nil { return nil } schema := &api.CollectionSchema{ Name: CollectionName, Fields: []api.Field{ {Name: "name", Type: "string"}, {Name: "url", Type: "string"}, {Name: "description", Type: "string"}, {Name: "readme", Type: "string"}, {Name: "topics", Type: "string[]", Facet: pointer.True()}, {Name: "languages", Type: "string[]", Facet: pointer.True()}, {Name: "licenses", Type: "string[]", Facet: pointer.True()}, {Name: "open_issues", Type: "int32"}, {Name: "open_prs", Type: "int32"}, {Name: "latest_commit", Type: "int64"}, {Name: "latest_release_name", Type: "string"}, {Name: "latest_release_date", Type: "int64"}, {Name: "uses_ai", Type: "bool"}, }, DefaultSortingField: new("latest_commit"), } _, err = s.client.Collections().Create(ctx, schema) return err } // projectToDocument maps a stored project to the Typesense document shape. func projectToDocument(p scraper.Project) document { return document{ ID: p.Url, Name: p.Name, URL: p.Url, Description: p.Description, Topics: p.Topics, Languages: p.Languages, Licenses: p.Licenses, OpenIssues: int64(p.OpenIssues), OpenPRs: int64(p.OpenPRs), LatestCommit: unixTime(p.LatestCommit), LatestReleaseName: p.LatestRelease.Name, LatestReleaseDate: unixTime(p.LatestRelease.Date), UsesAI: p.UsesAI, Readme: p.Readme, } } // unixTime returns the Unix epoch seconds for a time, or 0 for the zero time. func unixTime(t time.Time) int64 { if t.IsZero() { return 0 } return t.Unix() } // Sync reads every project from the store and upserts it into Typesense. func (s *Service) Sync(ctx context.Context, st *store.Store) error { if err := s.ensureCollection(ctx); err != nil { return fmt.Errorf("ensure collection: %w", err) } projects, err := st.All() if err != nil { return fmt.Errorf("read projects from store: %w", err) } docs := make([]any, 0, len(projects)) for _, p := range projects { docs = append(docs, projectToDocument(p)) } if len(docs) == 0 { return nil } params := &api.ImportDocumentsParams{ // upsert so re-indexing updates in place; missing docs stay as-is. Action: new("upsert"), } results, err := s.client.Collection(CollectionName).Documents().Import(ctx, docs, params) if err != nil { return fmt.Errorf("bulk import into Typesense: %w", err) } var failed int for _, r := range results { if r == nil || !r.Success { failed++ } } if failed > 0 { return fmt.Errorf("bulk import: %d of %d documents failed", failed, len(results)) } return nil }