61 lines
1.2 KiB
Go
61 lines
1.2 KiB
Go
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
|
|
"go.yaml.in/yaml/v4"
|
|
|
|
"git.xengi.de/ffd/indexer/scraper"
|
|
"git.xengi.de/ffd/indexer/store"
|
|
)
|
|
|
|
func main() {
|
|
data, err := os.ReadFile("config.yaml")
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
var config Config
|
|
|
|
if err := yaml.Unmarshal(data, &config); err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
db, err := store.Open(config.DbPath)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
defer func() { _ = db.Close() }()
|
|
|
|
for _, forge := range config.Forges {
|
|
fmt.Printf("Scraping %s (%s, %s)\n", forge.Name, forge.Type, forge.Url)
|
|
|
|
s, err := scraper.NewScraper(forge.Type, forge.Url)
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, " skip %s: %v\n", forge.Name, err)
|
|
continue
|
|
}
|
|
|
|
projects, err := s.GetProjects()
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, " skip %s: %v\n", forge.Name, err)
|
|
continue
|
|
}
|
|
|
|
fmt.Printf(" found %d projects\n", len(projects))
|
|
for _, p := range projects {
|
|
if err := db.Upsert(p); err != nil {
|
|
fmt.Fprintf(os.Stderr, " store %s: %v\n", p.Name, err)
|
|
continue
|
|
}
|
|
fmt.Printf(" %s\n", p.Name)
|
|
}
|
|
|
|
all, err := db.All()
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, " count: %v\n", err)
|
|
continue
|
|
}
|
|
fmt.Printf(" total in db: %d\n", len(all))
|
|
}
|
|
}
|