Add Botvrij.eu domain blocklist source
Downloads the Botvrij.eu public IOC domain list (no API key required), caches it in-process with a 6h TTL, and flags any registered domain that appears directly or as a parent of a feed entry.
This commit is contained in:
parent
6b1d2e2540
commit
9916ab0732
3 changed files with 286 additions and 0 deletions
191
checker/botvrij.go
Normal file
191
checker/botvrij.go
Normal file
|
|
@ -0,0 +1,191 @@
|
|||
package checker
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
sdk "git.happydns.org/checker-sdk-go/checker"
|
||||
)
|
||||
|
||||
const (
|
||||
botvrijFeedURL = "https://www.botvrij.eu/data/ioclist.domain.simple"
|
||||
botvrijDefaultTTL = 6 * time.Hour
|
||||
botvrijFailBackoff = 1 * time.Minute
|
||||
)
|
||||
|
||||
func init() {
|
||||
Register(&botvrijSource{
|
||||
cache: newBotvrijCache(botvrijFeedURL),
|
||||
})
|
||||
}
|
||||
|
||||
type botvrijSource struct {
|
||||
cache *botvrijCache
|
||||
}
|
||||
|
||||
func (*botvrijSource) ID() string { return "botvrij" }
|
||||
func (*botvrijSource) Name() string { return "Botvrij.eu domain blocklist" }
|
||||
|
||||
func (*botvrijSource) Options() SourceOptions {
|
||||
return SourceOptions{
|
||||
User: []sdk.CheckerOptionField{
|
||||
{
|
||||
Id: "enable_botvrij",
|
||||
Type: "bool",
|
||||
Label: "Use Botvrij.eu domain blocklist",
|
||||
Description: "Download the Botvrij.eu public domain blocklist (refreshed every 6h) and check the domain against it.",
|
||||
Default: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (s *botvrijSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult {
|
||||
if !sdk.GetBoolOption(opts, "enable_botvrij", true) || registered == "" {
|
||||
return disabledResult(s.ID(), s.Name())
|
||||
}
|
||||
|
||||
matched, size, fetched, err := s.cache.lookup(ctx, registered)
|
||||
res := SourceResult{
|
||||
SourceID: s.ID(), SourceName: s.Name(), Enabled: true,
|
||||
Reference: "https://botvrij.eu/",
|
||||
Details: mustJSON(map[string]any{"feed_size": size, "fetched_at": fetched}),
|
||||
}
|
||||
if err != nil {
|
||||
res.Error = err.Error()
|
||||
}
|
||||
if len(matched) > 0 {
|
||||
res.Reasons = []string{"Malicious domain"}
|
||||
for _, d := range matched {
|
||||
res.Evidence = append(res.Evidence, Evidence{Label: "Domain", Value: d})
|
||||
}
|
||||
}
|
||||
return []SourceResult{res}
|
||||
}
|
||||
|
||||
func (*botvrijSource) Evaluate(r SourceResult) (bool, string) {
|
||||
return evidenceEval(r, SeverityCrit)
|
||||
}
|
||||
|
||||
func (*botvrijSource) Diagnose(res SourceResult) Diagnosis {
|
||||
domains := make([]string, 0, len(res.Evidence))
|
||||
for _, e := range res.Evidence {
|
||||
domains = append(domains, e.Value)
|
||||
}
|
||||
previewN := min(len(domains), 5)
|
||||
return Diagnosis{
|
||||
Severity: SeverityCrit,
|
||||
Title: "Listed in the Botvrij.eu domain blocklist",
|
||||
Detail: fmt.Sprintf(
|
||||
"%d domain(s) matching this registered domain appear in the Botvrij.eu IOC list; examples: %s. Botvrij.eu tracks domains associated with malware C2, exploit kits, and other malicious infrastructure. Investigate recent DNS changes and hosted content.",
|
||||
len(domains), joinNonEmpty(domains[:previewN], ", "),
|
||||
),
|
||||
Fix: "https://botvrij.eu/",
|
||||
FixIsURL: true,
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- cache ----------
|
||||
|
||||
type botvrijCache struct {
|
||||
mu sync.Mutex
|
||||
domains []string // all domains in feed
|
||||
byDomain map[string]struct{} // set for O(1) exact-match test
|
||||
fetchedAt time.Time
|
||||
lastAttemptAt time.Time
|
||||
refreshing bool
|
||||
ttl time.Duration
|
||||
failBackoff time.Duration
|
||||
feedURL string
|
||||
}
|
||||
|
||||
func newBotvrijCache(feedURL string) *botvrijCache {
|
||||
return &botvrijCache{
|
||||
ttl: botvrijDefaultTTL,
|
||||
failBackoff: botvrijFailBackoff,
|
||||
feedURL: feedURL,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *botvrijCache) lookup(ctx context.Context, registered string) (matched []string, size int, fetchedAt time.Time, err error) {
|
||||
registered = strings.ToLower(strings.TrimSuffix(registered, "."))
|
||||
|
||||
c.mu.Lock()
|
||||
stale := c.byDomain == nil || time.Since(c.fetchedAt) > c.ttl
|
||||
doRefresh := stale && !c.refreshing && time.Since(c.lastAttemptAt) > c.failBackoff
|
||||
if doRefresh {
|
||||
c.refreshing = true
|
||||
}
|
||||
c.mu.Unlock()
|
||||
|
||||
if doRefresh {
|
||||
newDomains, newByDomain, ferr := c.fetch(ctx)
|
||||
c.mu.Lock()
|
||||
c.refreshing = false
|
||||
c.lastAttemptAt = time.Now()
|
||||
if ferr == nil {
|
||||
c.domains = newDomains
|
||||
c.byDomain = newByDomain
|
||||
c.fetchedAt = c.lastAttemptAt
|
||||
} else {
|
||||
err = ferr
|
||||
}
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
c.mu.Lock()
|
||||
suffix := "." + registered
|
||||
for d := range c.byDomain {
|
||||
if d == registered || strings.HasSuffix(d, suffix) {
|
||||
matched = append(matched, d)
|
||||
}
|
||||
}
|
||||
size = len(c.domains)
|
||||
fetchedAt = c.fetchedAt
|
||||
c.mu.Unlock()
|
||||
return matched, size, fetchedAt, err
|
||||
}
|
||||
|
||||
func (c *botvrijCache) fetch(ctx context.Context) ([]string, map[string]struct{}, error) {
|
||||
reqCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, c.feedURL, nil)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
req.Header.Set("User-Agent", "happydomain-checker-blacklist/1.0")
|
||||
|
||||
resp, err := sharedHTTPClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, nil, fmt.Errorf("botvrij HTTP %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
domains := make([]string, 0, 4096)
|
||||
byDomain := make(map[string]struct{}, 4096)
|
||||
scanner := bufio.NewScanner(io.LimitReader(resp.Body, 16<<20))
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
d := strings.ToLower(line)
|
||||
domains = append(domains, d)
|
||||
byDomain[d] = struct{}{}
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return domains, byDomain, nil
|
||||
}
|
||||
94
checker/botvrij_test.go
Normal file
94
checker/botvrij_test.go
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
package checker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
sdk "git.happydns.org/checker-sdk-go/checker"
|
||||
)
|
||||
|
||||
const botvrijFakeFeed = `# Botvrij.eu IOC list - domains
|
||||
# comment line
|
||||
evil.com
|
||||
malware.example.org
|
||||
c2.badactor.net
|
||||
`
|
||||
|
||||
func TestBotvrijSource_Listed_ExactMatch(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write([]byte(botvrijFakeFeed))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
s := &botvrijSource{cache: newBotvrijCache(srv.URL)}
|
||||
r := s.Query(context.Background(), "evil.com", "evil.com", sdk.CheckerOptions{"enable_botvrij": true})[0]
|
||||
|
||||
if !r.Enabled || r.Error != "" {
|
||||
t.Fatalf("expected enabled and no error, got %+v", r)
|
||||
}
|
||||
if len(r.Evidence) != 1 || r.Evidence[0].Value != "evil.com" {
|
||||
t.Errorf("expected evidence [evil.com], got %+v", r.Evidence)
|
||||
}
|
||||
if listed, sev := s.Evaluate(r); !listed || sev != SeverityCrit {
|
||||
t.Errorf("expected (true, crit), got (%v, %q)", listed, sev)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBotvrijSource_Listed_SubdomainInFeed(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write([]byte(botvrijFakeFeed))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
// Feed has "malware.example.org"; querying registered "example.org" should match.
|
||||
s := &botvrijSource{cache: newBotvrijCache(srv.URL)}
|
||||
r := s.Query(context.Background(), "sub.example.org", "example.org", sdk.CheckerOptions{"enable_botvrij": true})[0]
|
||||
|
||||
if len(r.Evidence) != 1 || r.Evidence[0].Value != "malware.example.org" {
|
||||
t.Errorf("expected subdomain match, got %+v", r.Evidence)
|
||||
}
|
||||
if listed, _ := s.Evaluate(r); !listed {
|
||||
t.Error("expected listed=true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBotvrijSource_NotListed(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write([]byte(botvrijFakeFeed))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
s := &botvrijSource{cache: newBotvrijCache(srv.URL)}
|
||||
r := s.Query(context.Background(), "safe.com", "safe.com", sdk.CheckerOptions{"enable_botvrij": true})[0]
|
||||
|
||||
if !r.Enabled || r.Error != "" || len(r.Evidence) != 0 {
|
||||
t.Fatalf("expected clean result, got %+v", r)
|
||||
}
|
||||
if listed, _ := s.Evaluate(r); listed {
|
||||
t.Error("expected listed=false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBotvrijSource_Disabled(t *testing.T) {
|
||||
s := &botvrijSource{cache: newBotvrijCache("http://nope")}
|
||||
r := s.Query(context.Background(), "evil.com", "evil.com", sdk.CheckerOptions{"enable_botvrij": false})[0]
|
||||
if r.Enabled {
|
||||
t.Errorf("expected disabled, got %+v", r)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBotvrijSource_HTTPError(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
s := &botvrijSource{cache: newBotvrijCache(srv.URL)}
|
||||
r := s.Query(context.Background(), "evil.com", "evil.com", sdk.CheckerOptions{"enable_botvrij": true})[0]
|
||||
|
||||
if r.Error == "" || r.Error != "botvrij HTTP 500" {
|
||||
t.Errorf("expected HTTP 500 error, got %q", r.Error)
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue