Add OISD domain blocklist source
Implements the OISD domainswild feed (big and small variants) as a new blacklist source. DNS0.eu was considered but shut down in October 2025.
This commit is contained in:
parent
c2cc88e1df
commit
1242a381ab
3 changed files with 308 additions and 0 deletions
|
|
@ -19,6 +19,7 @@ widely-used reputation systems.
|
|||
| abuse.ch MalwareBazaar| HTTPS lookup | free Auth-Key (admin) | user (default on) |
|
||||
| Botvrij.eu | downloaded list | no | user (default on) |
|
||||
| Disconnect.me | downloaded list | no | user (default on) |
|
||||
| OISD | downloaded list | no | user (default on) |
|
||||
| VirusTotal v3 | HTTPS lookup | yes (admin) | admin |
|
||||
|
||||
### Obtaining API keys
|
||||
|
|
|
|||
212
checker/oisd.go
Normal file
212
checker/oisd.go
Normal file
|
|
@ -0,0 +1,212 @@
|
|||
package checker
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
sdk "git.happydns.org/checker-sdk-go/checker"
|
||||
)
|
||||
|
||||
const (
|
||||
oisdBigFeedURL = "https://big.oisd.nl/domainswild"
|
||||
oisdSmallFeedURL = "https://small.oisd.nl/domainswild"
|
||||
oisdDefaultTTL = 24 * time.Hour
|
||||
oisdFailBackoff = 1 * time.Minute
|
||||
)
|
||||
|
||||
func init() {
|
||||
Register(&oisdSource{
|
||||
bigCache: newOisdCache(oisdBigFeedURL),
|
||||
smallCache: newOisdCache(oisdSmallFeedURL),
|
||||
})
|
||||
}
|
||||
|
||||
type oisdSource struct {
|
||||
bigCache *oisdCache
|
||||
smallCache *oisdCache
|
||||
}
|
||||
|
||||
func (*oisdSource) ID() string { return "oisd" }
|
||||
func (*oisdSource) Name() string { return "OISD domain blocklist" }
|
||||
|
||||
func (*oisdSource) Options() SourceOptions {
|
||||
return SourceOptions{
|
||||
User: []sdk.CheckerOptionField{
|
||||
{
|
||||
Id: "enable_oisd",
|
||||
Type: "bool",
|
||||
Label: "Use the OISD domain blocklist",
|
||||
Description: "Download the OISD domain blocklist (refreshed every 24h) and check the domain against it.",
|
||||
Default: true,
|
||||
},
|
||||
},
|
||||
Admin: []sdk.CheckerOptionField{
|
||||
{
|
||||
Id: "oisd_variant",
|
||||
Type: "string",
|
||||
Label: "OISD blocklist variant",
|
||||
Description: `Which OISD list to use: "big" (~250 k entries, recommended) or "small" (~50 k entries).`,
|
||||
Default: "big",
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (s *oisdSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult {
|
||||
if !sdk.GetBoolOption(opts, "enable_oisd", true) || registered == "" {
|
||||
return disabledResult(s.ID(), s.Name())
|
||||
}
|
||||
|
||||
cache := s.bigCache
|
||||
if stringOptDefault(opts, "oisd_variant", "big") == "small" {
|
||||
cache = s.smallCache
|
||||
}
|
||||
|
||||
matched, size, fetched, err := cache.lookup(ctx, registered)
|
||||
res := SourceResult{
|
||||
SourceID: s.ID(), SourceName: s.Name(), Enabled: true,
|
||||
Reference: "https://oisd.nl/",
|
||||
Details: mustJSON(map[string]any{"feed_size": size, "fetched_at": fetched}),
|
||||
}
|
||||
if err != nil {
|
||||
res.Error = err.Error()
|
||||
}
|
||||
if len(matched) > 0 {
|
||||
res.Reasons = []string{"Listed in OISD"}
|
||||
for _, d := range matched {
|
||||
res.Evidence = append(res.Evidence, Evidence{Label: "Domain", Value: d})
|
||||
}
|
||||
}
|
||||
return []SourceResult{res}
|
||||
}
|
||||
|
||||
func (*oisdSource) Evaluate(r SourceResult) (bool, string) {
|
||||
return evidenceEval(r, SeverityCrit)
|
||||
}
|
||||
|
||||
func (*oisdSource) Diagnose(res SourceResult) Diagnosis {
|
||||
domains := make([]string, 0, len(res.Evidence))
|
||||
for _, e := range res.Evidence {
|
||||
domains = append(domains, e.Value)
|
||||
}
|
||||
previewN := min(len(domains), 5)
|
||||
return Diagnosis{
|
||||
Severity: SeverityCrit,
|
||||
Title: "Listed in the OISD domain blocklist",
|
||||
Detail: fmt.Sprintf(
|
||||
"%d domain(s) matching this registered domain appear in the OISD blocklist; examples: %s. OISD is a large curated blocklist used by DNS resolvers, ad blockers, and firewalls worldwide. Domains listed here are blocked for millions of users. Investigate whether the domain hosts ads, trackers, or malware, or whether it has been compromised.",
|
||||
len(domains), joinNonEmpty(domains[:previewN], ", "),
|
||||
),
|
||||
Fix: "https://oisd.nl/",
|
||||
FixIsURL: true,
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- cache ----------
|
||||
|
||||
type oisdCache struct {
|
||||
mu sync.Mutex
|
||||
domains []string
|
||||
byDomain map[string]struct{}
|
||||
fetchedAt time.Time
|
||||
lastAttemptAt time.Time
|
||||
refreshing bool
|
||||
ttl time.Duration
|
||||
failBackoff time.Duration
|
||||
feedURL string
|
||||
}
|
||||
|
||||
func newOisdCache(feedURL string) *oisdCache {
|
||||
return &oisdCache{
|
||||
ttl: oisdDefaultTTL,
|
||||
failBackoff: oisdFailBackoff,
|
||||
feedURL: feedURL,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *oisdCache) lookup(ctx context.Context, registered string) (matched []string, size int, fetchedAt time.Time, err error) {
|
||||
registered = strings.ToLower(strings.TrimSuffix(registered, "."))
|
||||
|
||||
c.mu.Lock()
|
||||
stale := c.byDomain == nil || time.Since(c.fetchedAt) > c.ttl
|
||||
doRefresh := stale && !c.refreshing && time.Since(c.lastAttemptAt) > c.failBackoff
|
||||
if doRefresh {
|
||||
c.refreshing = true
|
||||
}
|
||||
feedURL := c.feedURL
|
||||
c.mu.Unlock()
|
||||
|
||||
if doRefresh {
|
||||
newDomains, newByDomain, ferr := c.fetch(ctx, feedURL)
|
||||
c.mu.Lock()
|
||||
c.refreshing = false
|
||||
c.lastAttemptAt = time.Now()
|
||||
if ferr == nil {
|
||||
c.domains = newDomains
|
||||
c.byDomain = newByDomain
|
||||
c.fetchedAt = c.lastAttemptAt
|
||||
} else {
|
||||
err = ferr
|
||||
}
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
c.mu.Lock()
|
||||
suffix := "." + registered
|
||||
for d := range c.byDomain {
|
||||
if d == registered || strings.HasSuffix(d, suffix) {
|
||||
matched = append(matched, d)
|
||||
}
|
||||
}
|
||||
size = len(c.domains)
|
||||
fetchedAt = c.fetchedAt
|
||||
c.mu.Unlock()
|
||||
return matched, size, fetchedAt, err
|
||||
}
|
||||
|
||||
func (c *oisdCache) fetch(ctx context.Context, feedURL string) ([]string, map[string]struct{}, error) {
|
||||
reqCtx, cancel := context.WithTimeout(ctx, 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, feedURL, nil)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
req.Header.Set("User-Agent", "happydomain-checker-blacklist/1.0")
|
||||
|
||||
resp, err := sharedHTTPClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, nil, fmt.Errorf("oisd HTTP %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
domains := make([]string, 0, 262144)
|
||||
byDomain := make(map[string]struct{}, 262144)
|
||||
scanner := bufio.NewScanner(io.LimitReader(resp.Body, 64<<20))
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line == "" || strings.HasPrefix(line, "!") || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
d := strings.TrimPrefix(strings.ToLower(line), "*.")
|
||||
if d == "" {
|
||||
continue
|
||||
}
|
||||
domains = append(domains, d)
|
||||
byDomain[d] = struct{}{}
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return domains, byDomain, nil
|
||||
}
|
||||
95
checker/oisd_test.go
Normal file
95
checker/oisd_test.go
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
package checker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
sdk "git.happydns.org/checker-sdk-go/checker"
|
||||
)
|
||||
|
||||
const oisdFakeFeed = `! OISD big domainswild
|
||||
# comment line
|
||||
*.evil.com
|
||||
*.malware.example.org
|
||||
*.c2.badactor.net
|
||||
`
|
||||
|
||||
func TestOisdSource_Listed_ExactMatch(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write([]byte(oisdFakeFeed))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
s := &oisdSource{bigCache: newOisdCache(srv.URL)}
|
||||
r := s.Query(context.Background(), "evil.com", "evil.com", sdk.CheckerOptions{"enable_oisd": true})[0]
|
||||
|
||||
if !r.Enabled || r.Error != "" {
|
||||
t.Fatalf("expected enabled and no error, got %+v", r)
|
||||
}
|
||||
if len(r.Evidence) != 1 || r.Evidence[0].Value != "evil.com" {
|
||||
t.Errorf("expected evidence [evil.com], got %+v", r.Evidence)
|
||||
}
|
||||
if listed, sev := s.Evaluate(r); !listed || sev != SeverityCrit {
|
||||
t.Errorf("expected (true, crit), got (%v, %q)", listed, sev)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOisdSource_Listed_SubdomainInFeed(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write([]byte(oisdFakeFeed))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
// Feed has "*.malware.example.org" → stored as "malware.example.org"; querying
|
||||
// registered "example.org" should match via suffix check.
|
||||
s := &oisdSource{bigCache: newOisdCache(srv.URL)}
|
||||
r := s.Query(context.Background(), "sub.example.org", "example.org", sdk.CheckerOptions{"enable_oisd": true})[0]
|
||||
|
||||
if len(r.Evidence) != 1 || r.Evidence[0].Value != "malware.example.org" {
|
||||
t.Errorf("expected subdomain match, got %+v", r.Evidence)
|
||||
}
|
||||
if listed, _ := s.Evaluate(r); !listed {
|
||||
t.Error("expected listed=true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOisdSource_NotListed(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write([]byte(oisdFakeFeed))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
s := &oisdSource{bigCache: newOisdCache(srv.URL)}
|
||||
r := s.Query(context.Background(), "safe.com", "safe.com", sdk.CheckerOptions{"enable_oisd": true})[0]
|
||||
|
||||
if !r.Enabled || r.Error != "" || len(r.Evidence) != 0 {
|
||||
t.Fatalf("expected clean result, got %+v", r)
|
||||
}
|
||||
if listed, _ := s.Evaluate(r); listed {
|
||||
t.Error("expected listed=false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOisdSource_Disabled(t *testing.T) {
|
||||
s := &oisdSource{bigCache: newOisdCache("http://nope")}
|
||||
r := s.Query(context.Background(), "evil.com", "evil.com", sdk.CheckerOptions{"enable_oisd": false})[0]
|
||||
if r.Enabled {
|
||||
t.Errorf("expected disabled, got %+v", r)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOisdSource_HTTPError(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
s := &oisdSource{bigCache: newOisdCache(srv.URL)}
|
||||
r := s.Query(context.Background(), "evil.com", "evil.com", sdk.CheckerOptions{"enable_oisd": true})[0]
|
||||
|
||||
if r.Error == "" || r.Error != "oisd HTTP 500" {
|
||||
t.Errorf("expected HTTP 500 error, got %q", r.Error)
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue