package checker import ( "bufio" "context" "fmt" "io" "net/http" "strings" "sync" "time" sdk "git.happydns.org/checker-sdk-go/checker" ) const ( oisdBigFeedURL = "https://big.oisd.nl/domainswild" oisdSmallFeedURL = "https://small.oisd.nl/domainswild" oisdDefaultTTL = 24 * time.Hour oisdFailBackoff = 1 * time.Minute ) func init() { Register(&oisdSource{ bigCache: newOisdCache(oisdBigFeedURL), smallCache: newOisdCache(oisdSmallFeedURL), }) } type oisdSource struct { bigCache *oisdCache smallCache *oisdCache } func (*oisdSource) ID() string { return "oisd" } func (*oisdSource) Name() string { return "OISD domain blocklist" } func (*oisdSource) Options() SourceOptions { return SourceOptions{ User: []sdk.CheckerOptionField{ { Id: "enable_oisd", Type: "bool", Label: "Use the OISD domain blocklist", Description: "Download the OISD domain blocklist (refreshed every 24h) and check the domain against it.", Default: true, }, }, Admin: []sdk.CheckerOptionField{ { Id: "oisd_variant", Type: "string", Label: "OISD blocklist variant", Description: `Which OISD list to use: "big" (~250 k entries, recommended) or "small" (~50 k entries).`, Default: "big", }, }, } } func (s *oisdSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult { if !sdk.GetBoolOption(opts, "enable_oisd", true) || registered == "" { return disabledResult(s.ID(), s.Name()) } cache := s.bigCache if stringOptDefault(opts, "oisd_variant", "big") == "small" { cache = s.smallCache } matched, size, fetched, err := cache.lookup(ctx, registered) res := SourceResult{ SourceID: s.ID(), SourceName: s.Name(), Enabled: true, Reference: "https://oisd.nl/", Details: mustJSON(map[string]any{"feed_size": size, "fetched_at": fetched}), } if err != nil { res.Error = err.Error() } if len(matched) > 0 { res.Reasons = []string{"Listed in OISD"} for _, d := range matched { res.Evidence = append(res.Evidence, Evidence{Label: "Domain", Value: d}) } } return []SourceResult{res} } func (*oisdSource) Evaluate(r SourceResult) (bool, string) { return evidenceEval(r, SeverityCrit) } func (*oisdSource) Diagnose(res SourceResult) Diagnosis { domains := make([]string, 0, len(res.Evidence)) for _, e := range res.Evidence { domains = append(domains, e.Value) } previewN := min(len(domains), 5) return Diagnosis{ Severity: SeverityCrit, Title: "Listed in the OISD domain blocklist", Detail: fmt.Sprintf( "%d domain(s) matching this registered domain appear in the OISD blocklist; examples: %s. OISD is a large curated blocklist used by DNS resolvers, ad blockers, and firewalls worldwide. Domains listed here are blocked for millions of users. Investigate whether the domain hosts ads, trackers, or malware, or whether it has been compromised.", len(domains), joinNonEmpty(domains[:previewN], ", "), ), Fix: "https://oisd.nl/", FixIsURL: true, } } // ---------- cache ---------- type oisdCache struct { mu sync.Mutex domains []string byDomain map[string]struct{} fetchedAt time.Time lastAttemptAt time.Time refreshing bool ttl time.Duration failBackoff time.Duration feedURL string } func newOisdCache(feedURL string) *oisdCache { return &oisdCache{ ttl: oisdDefaultTTL, failBackoff: oisdFailBackoff, feedURL: feedURL, } } func (c *oisdCache) lookup(ctx context.Context, registered string) (matched []string, size int, fetchedAt time.Time, err error) { registered = strings.ToLower(strings.TrimSuffix(registered, ".")) c.mu.Lock() stale := c.byDomain == nil || time.Since(c.fetchedAt) > c.ttl doRefresh := stale && !c.refreshing && time.Since(c.lastAttemptAt) > c.failBackoff if doRefresh { c.refreshing = true } feedURL := c.feedURL c.mu.Unlock() if doRefresh { newDomains, newByDomain, ferr := c.fetch(ctx, feedURL) c.mu.Lock() c.refreshing = false c.lastAttemptAt = time.Now() if ferr == nil { c.domains = newDomains c.byDomain = newByDomain c.fetchedAt = c.lastAttemptAt } else { err = ferr } c.mu.Unlock() } c.mu.Lock() suffix := "." + registered for d := range c.byDomain { if d == registered || strings.HasSuffix(d, suffix) { matched = append(matched, d) } } size = len(c.domains) fetchedAt = c.fetchedAt c.mu.Unlock() return matched, size, fetchedAt, err } func (c *oisdCache) fetch(ctx context.Context, feedURL string) ([]string, map[string]struct{}, error) { reqCtx, cancel := context.WithTimeout(ctx, 60*time.Second) defer cancel() req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, feedURL, nil) if err != nil { return nil, nil, err } req.Header.Set("User-Agent", "happydomain-checker-blacklist/1.0") resp, err := sharedHTTPClient.Do(req) if err != nil { return nil, nil, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return nil, nil, fmt.Errorf("oisd HTTP %d", resp.StatusCode) } domains := make([]string, 0, 262144) byDomain := make(map[string]struct{}, 262144) scanner := bufio.NewScanner(io.LimitReader(resp.Body, 64<<20)) for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) if line == "" || strings.HasPrefix(line, "!") || strings.HasPrefix(line, "#") { continue } d := strings.TrimPrefix(strings.ToLower(line), "*.") if d == "" { continue } domains = append(domains, d) byDomain[d] = struct{}{} } if err := scanner.Err(); err != nil { return nil, nil, err } return domains, byDomain, nil }