60 lines
2 KiB
Go
60 lines
2 KiB
Go
package scraper
|
|
|
|
import "testing"
|
|
|
|
func TestAllowedFor(t *testing.T) {
|
|
const ua = "ffd-indexer/0.1"
|
|
|
|
// xengi / cccb: only /archive/ blocked -> /api/ allowed.
|
|
if !allowedFor("User-agent: *\nDisallow: /*/*/archive/\n", ua) {
|
|
t.Error("expected /api/ allowed when only archive is disallowed")
|
|
}
|
|
|
|
// codeberg: /api/ disallowed under "*".
|
|
codeberg := `User-agent: *
|
|
Disallow: /api/
|
|
Disallow: /avatars/
|
|
`
|
|
if allowedFor(codeberg, ua) {
|
|
t.Error("expected /api/ blocked when /api/ is disallowed")
|
|
}
|
|
|
|
// Explicit agent group takes precedence over "*" (agent name matches
|
|
// UserAgent exactly, including the version).
|
|
// Here the explicit group is for "ffd-indexer" (no version), so it does
|
|
// not match "ffd-indexer/0.1"; the "*" group (blocks /api/) applies.
|
|
if allowedFor("User-agent: *\nDisallow: /api/\n\nUser-agent: ffd-indexer\nDisallow: /whatever/\n", ua) {
|
|
t.Error("expected unmatched explicit agent to fall back to * which blocks /api/")
|
|
}
|
|
|
|
// A matching explicit agent group takes precedence over "*".
|
|
if !allowedFor("User-agent: *\nDisallow: /api/\n\nUser-agent: ffd-indexer/0.1\nDisallow: /whatever/\n", ua) {
|
|
t.Error("expected explicit agent rule to permit /api/")
|
|
}
|
|
|
|
// No rules at all -> allow.
|
|
if !allowedFor("", ua) {
|
|
t.Error("expected empty robots to allow")
|
|
}
|
|
|
|
// Only an unrelated agent listed -> no star, no match -> allow.
|
|
if !allowedFor("User-agent: SomeBot\nDisallow: /api/\n", ua) {
|
|
t.Error("expected unrelated agent not to block us")
|
|
}
|
|
|
|
// Disallow "/" blocks everything including /api/.
|
|
if allowedFor("User-agent: *\nDisallow: /\n", ua) {
|
|
t.Error("expected Disallow / to block /api/")
|
|
}
|
|
|
|
// Empty Disallow (allowed) overrides a previous block in the same group.
|
|
if !allowedFor("User-agent: *\nDisallow: /api/\nDisallow:\n", ua) {
|
|
t.Error("expected empty Disallow to allow /api/")
|
|
}
|
|
|
|
// Wildcard on a non-prefix rule should not over-block (substring-only).
|
|
// "/*.pdf" should not match /api/.
|
|
if !allowedFor("User-agent: *\nDisallow: /*.pdf$\n", ua) {
|
|
t.Error("expected unrelated wildcard to not block /api/")
|
|
}
|
|
}
|