checker-blacklist/checker/oisd.go

package checker

import (
	"bufio"
	"context"
	"fmt"
	"io"
	"net/http"
	"strings"
	"sync"
	"time"

	sdk "git.happydns.org/checker-sdk-go/checker"
)

const (
	oisdBigFeedURL   = "https://big.oisd.nl/domainswild"
	oisdSmallFeedURL = "https://small.oisd.nl/domainswild"
	oisdDefaultTTL   = 24 * time.Hour
	oisdFailBackoff  = 1 * time.Minute
)

func init() {
	Register(&oisdSource{
		bigCache:   newOisdCache(oisdBigFeedURL),
		smallCache: newOisdCache(oisdSmallFeedURL),
	})
}

type oisdSource struct {
	bigCache   *oisdCache
	smallCache *oisdCache
}

func (*oisdSource) ID() string   { return "oisd" }
func (*oisdSource) Name() string { return "OISD domain blocklist" }

func (*oisdSource) Options() SourceOptions {
	return SourceOptions{
		User: []sdk.CheckerOptionField{
			{
				Id:          "enable_oisd",
				Type:        "bool",
				Label:       "Use the OISD domain blocklist",
				Description: "Download the OISD domain blocklist (refreshed every 24h) and check the domain against it.",
				Default:     true,
			},
		},
		Admin: []sdk.CheckerOptionField{
			{
				Id:          "oisd_variant",
				Type:        "string",
				Label:       "OISD blocklist variant",
				Description: `Which OISD list to use: "big" (~250 k entries, recommended) or "small" (~50 k entries).`,
				Default:     "big",
			},
		},
	}
}

func (s *oisdSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult {
	if !sdk.GetBoolOption(opts, "enable_oisd", true) || registered == "" {
		return disabledResult(s.ID(), s.Name())
	}

	cache := s.bigCache
	if stringOptDefault(opts, "oisd_variant", "big") == "small" {
		cache = s.smallCache
	}

	matched, size, fetched, err := cache.lookup(ctx, registered)
	res := SourceResult{
		SourceID: s.ID(), SourceName: s.Name(), Enabled: true,
		Reference: "https://oisd.nl/",
		Details:   mustJSON(map[string]any{"feed_size": size, "fetched_at": fetched}),
	}
	if err != nil {
		res.Error = err.Error()
	}
	if len(matched) > 0 {
		res.Reasons = []string{"Listed in OISD"}
		for _, d := range matched {
			res.Evidence = append(res.Evidence, Evidence{Label: "Domain", Value: d})
		}
	}
	return []SourceResult{res}
}

func (*oisdSource) Evaluate(r SourceResult) (bool, string) {
	return evidenceEval(r, SeverityCrit)
}

func (*oisdSource) Diagnose(res SourceResult) Diagnosis {
	domains := make([]string, 0, len(res.Evidence))
	for _, e := range res.Evidence {
		domains = append(domains, e.Value)
	}
	previewN := min(len(domains), 5)
	return Diagnosis{
		Severity: SeverityCrit,
		Title:    "Listed in the OISD domain blocklist",
		Detail: fmt.Sprintf(
			"%d domain(s) matching this registered domain appear in the OISD blocklist; examples: %s. OISD is a large curated blocklist used by DNS resolvers, ad blockers, and firewalls worldwide. Domains listed here are blocked for millions of users. Investigate whether the domain hosts ads, trackers, or malware, or whether it has been compromised.",
			len(domains), joinNonEmpty(domains[:previewN], ", "),
		),
		Fix:      "https://oisd.nl/",
		FixIsURL: true,
	}
}

// ---------- cache ----------

type oisdCache struct {
	mu            sync.Mutex
	domains       []string
	byDomain      map[string]struct{}
	fetchedAt     time.Time
	lastAttemptAt time.Time
	refreshing    bool
	ttl           time.Duration
	failBackoff   time.Duration
	feedURL       string
}

func newOisdCache(feedURL string) *oisdCache {
	return &oisdCache{
		ttl:         oisdDefaultTTL,
		failBackoff: oisdFailBackoff,
		feedURL:     feedURL,
	}
}

func (c *oisdCache) lookup(ctx context.Context, registered string) (matched []string, size int, fetchedAt time.Time, err error) {
	registered = strings.ToLower(strings.TrimSuffix(registered, "."))

	c.mu.Lock()
	stale := c.byDomain == nil || time.Since(c.fetchedAt) > c.ttl
	doRefresh := stale && !c.refreshing && time.Since(c.lastAttemptAt) > c.failBackoff
	if doRefresh {
		c.refreshing = true
	}
	feedURL := c.feedURL
	c.mu.Unlock()

	if doRefresh {
		newDomains, newByDomain, ferr := c.fetch(ctx, feedURL)
		c.mu.Lock()
		c.refreshing = false
		c.lastAttemptAt = time.Now()
		if ferr == nil {
			c.domains = newDomains
			c.byDomain = newByDomain
			c.fetchedAt = c.lastAttemptAt
		} else {
			err = ferr
		}
		c.mu.Unlock()
	}

	c.mu.Lock()
	suffix := "." + registered
	for d := range c.byDomain {
		if d == registered || strings.HasSuffix(d, suffix) {
			matched = append(matched, d)
		}
	}
	size = len(c.domains)
	fetchedAt = c.fetchedAt
	c.mu.Unlock()
	return matched, size, fetchedAt, err
}

func (c *oisdCache) fetch(ctx context.Context, feedURL string) ([]string, map[string]struct{}, error) {
	reqCtx, cancel := context.WithTimeout(ctx, 60*time.Second)
	defer cancel()

	req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, feedURL, nil)
	if err != nil {
		return nil, nil, err
	}
	req.Header.Set("User-Agent", "happydomain-checker-blacklist/1.0")

	resp, err := sharedHTTPClient.Do(req)
	if err != nil {
		return nil, nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return nil, nil, fmt.Errorf("oisd HTTP %d", resp.StatusCode)
	}

	domains := make([]string, 0, 262144)
	byDomain := make(map[string]struct{}, 262144)
	scanner := bufio.NewScanner(io.LimitReader(resp.Body, 64<<20))
	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())
		if line == "" || strings.HasPrefix(line, "!") || strings.HasPrefix(line, "#") {
			continue
		}
		d := strings.TrimPrefix(strings.ToLower(line), "*.")
		if d == "" {
			continue
		}
		domains = append(domains, d)
		byDomain[d] = struct{}{}
	}
	if err := scanner.Err(); err != nil {
		return nil, nil, err
	}
	return domains, byDomain, nil
}