From 9916ab0732d4355d1b67a225db3cd99b4c9163d2 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Fri, 15 May 2026 18:56:05 +0800 Subject: [PATCH] Add Botvrij.eu domain blocklist source Downloads the Botvrij.eu public IOC domain list (no API key required), caches it in-process with a 6h TTL, and flags any registered domain that appears directly or as a parent of a feed entry. --- README.md | 1 + checker/botvrij.go | 191 ++++++++++++++++++++++++++++++++++++++++ checker/botvrij_test.go | 94 ++++++++++++++++++++ 3 files changed, 286 insertions(+) create mode 100644 checker/botvrij.go create mode 100644 checker/botvrij_test.go diff --git a/README.md b/README.md index 7b4e8e8..0bf3f1a 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ widely-used reputation systems. | abuse.ch URLhaus | HTTPS lookup | free Auth-Key (admin) | user (default on) | | abuse.ch ThreatFox | HTTPS lookup | free Auth-Key (admin) | user (default on) | | abuse.ch MalwareBazaar| HTTPS lookup | free Auth-Key (admin) | user (default on) | +| Botvrij.eu | downloaded list | no | user (default on) | | VirusTotal v3 | HTTPS lookup | yes (admin) | admin | ### Obtaining API keys diff --git a/checker/botvrij.go b/checker/botvrij.go new file mode 100644 index 0000000..dbc4348 --- /dev/null +++ b/checker/botvrij.go @@ -0,0 +1,191 @@ +package checker + +import ( + "bufio" + "context" + "fmt" + "io" + "net/http" + "strings" + "sync" + "time" + + sdk "git.happydns.org/checker-sdk-go/checker" +) + +const ( + botvrijFeedURL = "https://www.botvrij.eu/data/ioclist.domain.simple" + botvrijDefaultTTL = 6 * time.Hour + botvrijFailBackoff = 1 * time.Minute +) + +func init() { + Register(&botvrijSource{ + cache: newBotvrijCache(botvrijFeedURL), + }) +} + +type botvrijSource struct { + cache *botvrijCache +} + +func (*botvrijSource) ID() string { return "botvrij" } +func (*botvrijSource) Name() string { return "Botvrij.eu domain blocklist" } + +func (*botvrijSource) Options() SourceOptions { + return SourceOptions{ + User: []sdk.CheckerOptionField{ + { + Id: "enable_botvrij", + Type: "bool", + Label: "Use Botvrij.eu domain blocklist", + Description: "Download the Botvrij.eu public domain blocklist (refreshed every 6h) and check the domain against it.", + Default: true, + }, + }, + } +} + +func (s *botvrijSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult { + if !sdk.GetBoolOption(opts, "enable_botvrij", true) || registered == "" { + return disabledResult(s.ID(), s.Name()) + } + + matched, size, fetched, err := s.cache.lookup(ctx, registered) + res := SourceResult{ + SourceID: s.ID(), SourceName: s.Name(), Enabled: true, + Reference: "https://botvrij.eu/", + Details: mustJSON(map[string]any{"feed_size": size, "fetched_at": fetched}), + } + if err != nil { + res.Error = err.Error() + } + if len(matched) > 0 { + res.Reasons = []string{"Malicious domain"} + for _, d := range matched { + res.Evidence = append(res.Evidence, Evidence{Label: "Domain", Value: d}) + } + } + return []SourceResult{res} +} + +func (*botvrijSource) Evaluate(r SourceResult) (bool, string) { + return evidenceEval(r, SeverityCrit) +} + +func (*botvrijSource) Diagnose(res SourceResult) Diagnosis { + domains := make([]string, 0, len(res.Evidence)) + for _, e := range res.Evidence { + domains = append(domains, e.Value) + } + previewN := min(len(domains), 5) + return Diagnosis{ + Severity: SeverityCrit, + Title: "Listed in the Botvrij.eu domain blocklist", + Detail: fmt.Sprintf( + "%d domain(s) matching this registered domain appear in the Botvrij.eu IOC list; examples: %s. Botvrij.eu tracks domains associated with malware C2, exploit kits, and other malicious infrastructure. Investigate recent DNS changes and hosted content.", + len(domains), joinNonEmpty(domains[:previewN], ", "), + ), + Fix: "https://botvrij.eu/", + FixIsURL: true, + } +} + +// ---------- cache ---------- + +type botvrijCache struct { + mu sync.Mutex + domains []string // all domains in feed + byDomain map[string]struct{} // set for O(1) exact-match test + fetchedAt time.Time + lastAttemptAt time.Time + refreshing bool + ttl time.Duration + failBackoff time.Duration + feedURL string +} + +func newBotvrijCache(feedURL string) *botvrijCache { + return &botvrijCache{ + ttl: botvrijDefaultTTL, + failBackoff: botvrijFailBackoff, + feedURL: feedURL, + } +} + +func (c *botvrijCache) lookup(ctx context.Context, registered string) (matched []string, size int, fetchedAt time.Time, err error) { + registered = strings.ToLower(strings.TrimSuffix(registered, ".")) + + c.mu.Lock() + stale := c.byDomain == nil || time.Since(c.fetchedAt) > c.ttl + doRefresh := stale && !c.refreshing && time.Since(c.lastAttemptAt) > c.failBackoff + if doRefresh { + c.refreshing = true + } + c.mu.Unlock() + + if doRefresh { + newDomains, newByDomain, ferr := c.fetch(ctx) + c.mu.Lock() + c.refreshing = false + c.lastAttemptAt = time.Now() + if ferr == nil { + c.domains = newDomains + c.byDomain = newByDomain + c.fetchedAt = c.lastAttemptAt + } else { + err = ferr + } + c.mu.Unlock() + } + + c.mu.Lock() + suffix := "." + registered + for d := range c.byDomain { + if d == registered || strings.HasSuffix(d, suffix) { + matched = append(matched, d) + } + } + size = len(c.domains) + fetchedAt = c.fetchedAt + c.mu.Unlock() + return matched, size, fetchedAt, err +} + +func (c *botvrijCache) fetch(ctx context.Context) ([]string, map[string]struct{}, error) { + reqCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, c.feedURL, nil) + if err != nil { + return nil, nil, err + } + req.Header.Set("User-Agent", "happydomain-checker-blacklist/1.0") + + resp, err := sharedHTTPClient.Do(req) + if err != nil { + return nil, nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, nil, fmt.Errorf("botvrij HTTP %d", resp.StatusCode) + } + + domains := make([]string, 0, 4096) + byDomain := make(map[string]struct{}, 4096) + scanner := bufio.NewScanner(io.LimitReader(resp.Body, 16<<20)) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + d := strings.ToLower(line) + domains = append(domains, d) + byDomain[d] = struct{}{} + } + if err := scanner.Err(); err != nil { + return nil, nil, err + } + return domains, byDomain, nil +} diff --git a/checker/botvrij_test.go b/checker/botvrij_test.go new file mode 100644 index 0000000..74366b4 --- /dev/null +++ b/checker/botvrij_test.go @@ -0,0 +1,94 @@ +package checker + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + sdk "git.happydns.org/checker-sdk-go/checker" +) + +const botvrijFakeFeed = `# Botvrij.eu IOC list - domains +# comment line +evil.com +malware.example.org +c2.badactor.net +` + +func TestBotvrijSource_Listed_ExactMatch(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(botvrijFakeFeed)) + })) + defer srv.Close() + + s := &botvrijSource{cache: newBotvrijCache(srv.URL)} + r := s.Query(context.Background(), "evil.com", "evil.com", sdk.CheckerOptions{"enable_botvrij": true})[0] + + if !r.Enabled || r.Error != "" { + t.Fatalf("expected enabled and no error, got %+v", r) + } + if len(r.Evidence) != 1 || r.Evidence[0].Value != "evil.com" { + t.Errorf("expected evidence [evil.com], got %+v", r.Evidence) + } + if listed, sev := s.Evaluate(r); !listed || sev != SeverityCrit { + t.Errorf("expected (true, crit), got (%v, %q)", listed, sev) + } +} + +func TestBotvrijSource_Listed_SubdomainInFeed(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(botvrijFakeFeed)) + })) + defer srv.Close() + + // Feed has "malware.example.org"; querying registered "example.org" should match. + s := &botvrijSource{cache: newBotvrijCache(srv.URL)} + r := s.Query(context.Background(), "sub.example.org", "example.org", sdk.CheckerOptions{"enable_botvrij": true})[0] + + if len(r.Evidence) != 1 || r.Evidence[0].Value != "malware.example.org" { + t.Errorf("expected subdomain match, got %+v", r.Evidence) + } + if listed, _ := s.Evaluate(r); !listed { + t.Error("expected listed=true") + } +} + +func TestBotvrijSource_NotListed(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(botvrijFakeFeed)) + })) + defer srv.Close() + + s := &botvrijSource{cache: newBotvrijCache(srv.URL)} + r := s.Query(context.Background(), "safe.com", "safe.com", sdk.CheckerOptions{"enable_botvrij": true})[0] + + if !r.Enabled || r.Error != "" || len(r.Evidence) != 0 { + t.Fatalf("expected clean result, got %+v", r) + } + if listed, _ := s.Evaluate(r); listed { + t.Error("expected listed=false") + } +} + +func TestBotvrijSource_Disabled(t *testing.T) { + s := &botvrijSource{cache: newBotvrijCache("http://nope")} + r := s.Query(context.Background(), "evil.com", "evil.com", sdk.CheckerOptions{"enable_botvrij": false})[0] + if r.Enabled { + t.Errorf("expected disabled, got %+v", r) + } +} + +func TestBotvrijSource_HTTPError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer srv.Close() + + s := &botvrijSource{cache: newBotvrijCache(srv.URL)} + r := s.Query(context.Background(), "evil.com", "evil.com", sdk.CheckerOptions{"enable_botvrij": true})[0] + + if r.Error == "" || r.Error != "botvrij HTTP 500" { + t.Errorf("expected HTTP 500 error, got %q", r.Error) + } +}