From 01909debad2524337c1b3bc182d6bc40d0d277b4 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sun, 10 May 2026 18:59:15 +0800 Subject: [PATCH 01/15] Add CI/CD pipeline --- .drone-manifest.yml | 22 ++++++ .drone.yml | 187 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 209 insertions(+) create mode 100644 .drone-manifest.yml create mode 100644 .drone.yml diff --git a/.drone-manifest.yml b/.drone-manifest.yml new file mode 100644 index 0000000..9b05159 --- /dev/null +++ b/.drone-manifest.yml @@ -0,0 +1,22 @@ +image: happydomain/checker-blacklist:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}latest{{/if}} +{{#if build.tags}} +tags: +{{#each build.tags}} + - {{this}} +{{/each}} +{{/if}} +manifests: + - image: happydomain/checker-blacklist:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-amd64 + platform: + architecture: amd64 + os: linux + - image: happydomain/checker-blacklist:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm64 + platform: + architecture: arm64 + os: linux + variant: v8 + - image: happydomain/checker-blacklist:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm + platform: + architecture: arm + os: linux + variant: v7 diff --git a/.drone.yml b/.drone.yml new file mode 100644 index 0000000..41271dc --- /dev/null +++ b/.drone.yml @@ -0,0 +1,187 @@ +--- +kind: pipeline +type: docker +name: build-amd64 + +platform: + os: linux + arch: amd64 + +steps: + - name: checker build + image: golang:1-alpine + commands: + - apk add --no-cache git make + - make + environment: + CHECKER_VERSION: "${DRONE_BRANCH}-${DRONE_COMMIT}" + CGO_ENABLED: 0 + when: + event: + exclude: + - tag + + - name: checker build tag + image: golang:1-alpine + commands: + - apk add --no-cache git make + - make + environment: + CHECKER_VERSION: "${DRONE_SEMVER}" + CGO_ENABLED: 0 + when: + event: + - tag + + - name: publish on Docker Hub + image: plugins/docker + settings: + repo: happydomain/checker-blacklist + auto_tag: true + auto_tag_suffix: ${DRONE_STAGE_OS}-${DRONE_STAGE_ARCH} + dockerfile: Dockerfile + build_args: + - CHECKER_VERSION=${DRONE_BRANCH}-${DRONE_COMMIT} + username: + from_secret: docker_username + password: + from_secret: docker_password + when: + event: + exclude: + - tag + + - name: publish on Docker Hub (tag) + image: plugins/docker + settings: + repo: happydomain/checker-blacklist + auto_tag: true + auto_tag_suffix: ${DRONE_STAGE_OS}-${DRONE_STAGE_ARCH} + dockerfile: Dockerfile + build_args: + - CHECKER_VERSION=${DRONE_SEMVER} + username: + from_secret: docker_username + password: + from_secret: docker_password + when: + event: + - tag + +trigger: + branch: + exclude: + - renovate/* + event: + - cron + - push + - tag + +--- +kind: pipeline +type: docker +name: build-arm64 + +platform: + os: linux + arch: arm64 + +steps: + - name: checker build + image: golang:1-alpine + commands: + - apk add --no-cache git make + - make + environment: + CHECKER_VERSION: "${DRONE_BRANCH}-${DRONE_COMMIT}" + CGO_ENABLED: 0 + when: + event: + exclude: + - tag + + - name: checker build tag + image: golang:1-alpine + commands: + - apk add --no-cache git make + - make + environment: + CHECKER_VERSION: "${DRONE_SEMVER}" + CGO_ENABLED: 0 + when: + event: + - tag + + - name: publish on Docker Hub + image: plugins/docker + settings: + repo: happydomain/checker-blacklist + auto_tag: true + auto_tag_suffix: ${DRONE_STAGE_OS}-${DRONE_STAGE_ARCH} + dockerfile: Dockerfile + build_args: + - CHECKER_VERSION=${DRONE_BRANCH}-${DRONE_COMMIT} + username: + from_secret: docker_username + password: + from_secret: docker_password + when: + event: + exclude: + - tag + + - name: publish on Docker Hub (tag) + image: plugins/docker + settings: + repo: happydomain/checker-blacklist + auto_tag: true + auto_tag_suffix: ${DRONE_STAGE_OS}-${DRONE_STAGE_ARCH} + dockerfile: Dockerfile + build_args: + - CHECKER_VERSION=${DRONE_SEMVER} + username: + from_secret: docker_username + password: + from_secret: docker_password + when: + event: + - tag + +trigger: + event: + - cron + - push + - tag + +--- +kind: pipeline +name: docker-manifest + +platform: + os: linux + arch: arm64 + +steps: + - name: publish on Docker Hub + image: plugins/manifest + settings: + auto_tag: true + ignore_missing: true + spec: .drone-manifest.yml + username: + from_secret: docker_username + password: + from_secret: docker_password + +trigger: + branch: + exclude: + - renovate/* + event: + - cron + - push + - tag + +depends_on: + - build-amd64 + - build-arm64 From c437339bda9f17aa20a479c1f2f5e29be4da27f1 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Fri, 15 May 2026 18:04:10 +0800 Subject: [PATCH 02/15] Separate observation from evaluation in blacklist sources Each source's Query() method previously set r.Listed and r.Severity, embedding verdict logic inside the prober. Evaluation now lives in a dedicated Evaluate(SourceResult) (bool, string) method per source, keeping Query() as pure observation. A package-level EvaluateResult() helper looks up the source by ID and delegates to its Evaluate method; rules.go, report.go, types.go, and provider.go all call this instead of reading pre-set r.Listed/r.Severity values. An unknownSource sentinel handles results whose source is no longer registered. --- checker/dnsbl.go | 9 +++++++-- checker/openphish.go | 9 +++++++-- checker/provider.go | 2 +- checker/report.go | 16 +++++++++------- checker/report_test.go | 9 +++++---- checker/rule.go | 29 +++++++++++++++++++++++++---- checker/safebrowsing.go | 9 +++++++-- checker/source.go | 22 ++++++++++++++++++++++ checker/types.go | 4 ++-- checker/urlhaus.go | 9 +++++++-- checker/urlhaus_test.go | 12 ++++++++---- checker/virustotal.go | 28 +++++++++++++++++----------- checker/virustotal_test.go | 9 ++++++--- 13 files changed, 123 insertions(+), 44 deletions(-) diff --git a/checker/dnsbl.go b/checker/dnsbl.go index e0d7dff..34fa68a 100644 --- a/checker/dnsbl.go +++ b/checker/dnsbl.go @@ -160,8 +160,6 @@ func (s *dnsblSource) queryOne(ctx context.Context, registered string, z DNSBLZo res.BlockedQuery = true return res } - res.Listed = true - res.Severity = SeverityCrit if len(res.Reasons) == 0 { res.Reasons = append(res.Reasons, "Listed (no detail decoded)") } @@ -176,6 +174,13 @@ func (s *dnsblSource) queryOne(ctx context.Context, registered string, z DNSBLZo return res } +func (*dnsblSource) Evaluate(r SourceResult) (bool, string) { + if r.Enabled && !r.BlockedQuery && r.Error == "" && len(r.Evidence) > 0 { + return true, SeverityCrit + } + return false, "" +} + func (*dnsblSource) Diagnose(res SourceResult) Diagnosis { return Diagnosis{ Severity: SeverityCrit, diff --git a/checker/openphish.go b/checker/openphish.go index 065652a..12ab8dc 100644 --- a/checker/openphish.go +++ b/checker/openphish.go @@ -63,8 +63,6 @@ func (s *openPhishSource) Query(ctx context.Context, domain, registered string, // Fall through with whatever the cache could provide. } if len(urls) > 0 { - res.Listed = true - res.Severity = SeverityCrit res.Reasons = []string{"Phishing"} for _, u := range urls { res.Evidence = append(res.Evidence, Evidence{Label: "URL", Value: u}) @@ -73,6 +71,13 @@ func (s *openPhishSource) Query(ctx context.Context, domain, registered string, return []SourceResult{res} } +func (*openPhishSource) Evaluate(r SourceResult) (bool, string) { + if r.Enabled && r.Error == "" && len(r.Evidence) > 0 { + return true, SeverityCrit + } + return false, "" +} + func (*openPhishSource) Diagnose(res SourceResult) Diagnosis { urls := make([]string, 0, len(res.Evidence)) for _, e := range res.Evidence { diff --git a/checker/provider.go b/checker/provider.go index fd4e0ea..de33c02 100644 --- a/checker/provider.go +++ b/checker/provider.go @@ -37,7 +37,7 @@ func (p *blacklistProvider) ExtractMetrics(ctx sdk.ReportContext, collectedAt ti continue } v := 0.0 - if r.Listed { + if listed, _ := EvaluateResult(r); listed { v = 1 } metrics = append(metrics, sdk.CheckMetric{ diff --git a/checker/report.go b/checker/report.go index 5649812..c823cf7 100644 --- a/checker/report.go +++ b/checker/report.go @@ -101,7 +101,7 @@ func diagnose(d *BlacklistData) []Diagnosis { var out []Diagnosis for _, r := range d.Results { - if !r.Listed { + if listed, _ := EvaluateResult(r); !listed { continue } if s, ok := byID[r.SourceID]; ok { @@ -189,7 +189,7 @@ func buildSections(d *BlacklistData) []sourceSection { // subject sources have at most one). Plain sources skip this. if dr, ok := byID[id].(DetailRenderer); ok { for _, r := range results { - if !r.Listed && len(r.Details) == 0 { + if listed, _ := EvaluateResult(r); !listed && len(r.Details) == 0 { continue } html, err := dr.RenderDetail(r) @@ -210,7 +210,7 @@ func sectionStatus(results []SourceResult) (string, string) { if r.Enabled { enabled++ } - if r.Listed { + if l, _ := EvaluateResult(r); l { listed++ } else if r.Error != "" { errs++ @@ -238,11 +238,12 @@ func subjectStatusLabel(r SourceResult) string { switch { case !r.Enabled: return "Disabled" - case r.Listed: - return "LISTED" case r.Error != "": return "Error" } + if listed, _ := EvaluateResult(r); listed { + return "LISTED" + } return "Clean" } @@ -250,11 +251,12 @@ func subjectStatusClass(r SourceResult) string { switch { case !r.Enabled: return "muted" - case r.Listed: - return r.Severity case r.Error != "": return "warn" } + if listed, severity := EvaluateResult(r); listed { + return severity + } return "ok" } diff --git a/checker/report_test.go b/checker/report_test.go index 1ff5fad..3215745 100644 --- a/checker/report_test.go +++ b/checker/report_test.go @@ -15,8 +15,9 @@ func TestDiagnoseAndReportRender(t *testing.T) { { SourceID: "dnsbl", SourceName: "Spamhaus DBL", Subject: "dbl.spamhaus.org", - Enabled: true, Listed: true, Severity: SeverityCrit, - Reasons: []string{"Phishing domain"}, + Enabled: true, + Reasons: []string{"Phishing domain"}, + Evidence: []Evidence{{Label: "Return code", Value: "127.0.1.4"}}, LookupURL: "https://check.spamhaus.org/results/?query=example.com", RemovalURL: "https://www.spamhaus.org/dbl/removal/", }, @@ -27,7 +28,7 @@ func TestDiagnoseAndReportRender(t *testing.T) { }, { SourceID: "openphish", SourceName: "OpenPhish feed", - Enabled: true, Listed: true, Severity: SeverityCrit, + Enabled: true, Evidence: []Evidence{{Label: "URL", Value: "http://example.com/login"}}, }, }, @@ -66,7 +67,7 @@ func TestHeadline(t *testing.T) { } func TestSectionStatus(t *testing.T) { - if l, c := sectionStatus([]SourceResult{{Enabled: true, Listed: true, Severity: SeverityCrit}}); c != "crit" || !strings.HasPrefix(l, "LISTED") { + if l, c := sectionStatus([]SourceResult{{SourceID: "openphish", Enabled: true, Evidence: []Evidence{{Label: "URL", Value: "http://evil.com/"}}}}); c != "crit" || !strings.HasPrefix(l, "LISTED") { t.Errorf("sectionStatus listed = %q/%q", l, c) } if l, c := sectionStatus([]SourceResult{{Enabled: true}}); c != "ok" || l != "Clean" { diff --git a/checker/rule.go b/checker/rule.go index d389026..72d5c38 100644 --- a/checker/rule.go +++ b/checker/rule.go @@ -40,18 +40,28 @@ func (*sourceRule) Evaluate(ctx context.Context, obs sdk.ObservationGetter, opts }} } + byID := make(map[string]Source, len(Sources())) + for _, s := range Sources() { + byID[s.ID()] = s + } + out := make([]sdk.CheckState, 0, len(data.Results)) for _, r := range data.Results { - out = append(out, evaluateOne(r)) + src, ok := byID[r.SourceID] + if !ok { + src = unknownSource{} + } + out = append(out, evaluateOne(r, src)) } return out } -func evaluateOne(r SourceResult) sdk.CheckState { +func evaluateOne(r SourceResult, src Source) sdk.CheckState { subj := r.SourceName if r.Subject != "" && r.Subject != r.SourceName { subj = r.SourceName + " / " + r.Subject } + listed, severity := src.Evaluate(r) switch { case !r.Enabled: return sdk.CheckState{ @@ -72,9 +82,9 @@ func evaluateOne(r SourceResult) sdk.CheckState { Message: subj + ": query failed: " + r.Error, Code: "source_error", } - case r.Listed: + case listed: return sdk.CheckState{ - Status: severityToStatus(r.Severity), + Status: severityToStatus(severity), Subject: subj, Message: fmt.Sprintf("Listed in %s: %s", subj, joinNonEmpty(r.Reasons, "; ")), Code: "source_listed", @@ -95,6 +105,17 @@ func evaluateOne(r SourceResult) sdk.CheckState { } } +// unknownSource is a sentinel used when a SourceResult references a source ID +// that is no longer in the registry. Evaluate always returns (false, ""). +type unknownSource struct{} + +func (unknownSource) ID() string { return "" } +func (unknownSource) Name() string { return "unknown" } +func (unknownSource) Options() SourceOptions { return SourceOptions{} } +func (unknownSource) Query(_ context.Context, _, _ string, _ sdk.CheckerOptions) []SourceResult { return nil } +func (unknownSource) Diagnose(_ SourceResult) Diagnosis { return Diagnosis{} } +func (unknownSource) Evaluate(_ SourceResult) (bool, string) { return false, "" } + func severityToStatus(sev string) sdk.Status { switch sev { case SeverityCrit: diff --git a/checker/safebrowsing.go b/checker/safebrowsing.go index a9a9c03..be489a7 100644 --- a/checker/safebrowsing.go +++ b/checker/safebrowsing.go @@ -124,8 +124,6 @@ func (s *safeBrowsingSource) Query(ctx context.Context, domain, registered strin if len(parsed.Matches) == 0 { return []SourceResult{res} } - res.Listed = true - res.Severity = SeverityCrit res.Reference = "https://transparencyreport.google.com/safe-browsing/search?url=" + registered seenType := map[string]bool{} for _, m := range parsed.Matches { @@ -143,6 +141,13 @@ func (s *safeBrowsingSource) Query(ctx context.Context, domain, registered strin return []SourceResult{res} } +func (*safeBrowsingSource) Evaluate(r SourceResult) (bool, string) { + if r.Enabled && r.Error == "" && len(r.Evidence) > 0 { + return true, SeverityCrit + } + return false, "" +} + func (*safeBrowsingSource) Diagnose(res SourceResult) Diagnosis { return Diagnosis{ Severity: SeverityCrit, diff --git a/checker/source.go b/checker/source.go index 793d919..cc8b15e 100644 --- a/checker/source.go +++ b/checker/source.go @@ -48,6 +48,13 @@ type Source interface { // generic report wraps it with the title bar and severity styling. // Called only when SourceResult.Listed is true. Diagnose(res SourceResult) Diagnosis + + // Evaluate inspects an already-collected SourceResult and returns + // whether the domain is considered listed and at what severity. + // Implementations must read observation fields only (Evidence, + // Reasons, Error, Enabled, BlockedQuery, Details) and must never + // consult r.Listed or r.Severity. + Evaluate(r SourceResult) (listed bool, severity string) } // DetailRenderer is an optional interface a Source can implement when @@ -144,3 +151,18 @@ func Sources() []Source { copy(out, registry) return out } + +// EvaluateResult looks up the source that produced r from the registry +// and delegates to its Evaluate method. Returns (false, "") when the +// source is not found — a safe default that never promotes a stale +// Listed=true value. +func EvaluateResult(r SourceResult) (bool, string) { + registryMu.RLock() + defer registryMu.RUnlock() + for _, s := range registry { + if s.ID() == r.SourceID { + return s.Evaluate(r) + } + } + return false, "" +} diff --git a/checker/types.go b/checker/types.go index 4022687..48ba06c 100644 --- a/checker/types.go +++ b/checker/types.go @@ -31,7 +31,7 @@ type BlacklistData struct { func (d *BlacklistData) TotalHits() int { n := 0 for _, r := range d.Results { - if r.Listed { + if listed, _ := EvaluateResult(r); listed { n++ } } @@ -43,7 +43,7 @@ func (d *BlacklistData) TotalHits() int { func (d *BlacklistData) FilterListed() []SourceResult { out := make([]SourceResult, 0, len(d.Results)) for _, r := range d.Results { - if r.Listed { + if listed, _ := EvaluateResult(r); listed { out = append(out, r) } } diff --git a/checker/urlhaus.go b/checker/urlhaus.go index 4636ac8..9d9341e 100644 --- a/checker/urlhaus.go +++ b/checker/urlhaus.go @@ -120,8 +120,6 @@ func (s *urlhausSource) Query(ctx context.Context, domain, registered string, op if len(parsed.URLs) == 0 { return []SourceResult{res} } - res.Listed = true - res.Severity = SeverityCrit threats := map[string]bool{} details := urlhausDetails{} for _, u := range parsed.URLs { @@ -148,6 +146,13 @@ func (s *urlhausSource) Query(ctx context.Context, domain, registered string, op return []SourceResult{res} } +func (*urlhausSource) Evaluate(r SourceResult) (bool, string) { + if r.Enabled && r.Error == "" && len(r.Evidence) > 0 { + return true, SeverityCrit + } + return false, "" +} + func (*urlhausSource) Diagnose(res SourceResult) Diagnosis { online := 0 for _, e := range res.Evidence { diff --git a/checker/urlhaus_test.go b/checker/urlhaus_test.go index 0a10921..9128efd 100644 --- a/checker/urlhaus_test.go +++ b/checker/urlhaus_test.go @@ -24,8 +24,9 @@ func TestURLhausSource_NoResults(t *testing.T) { t.Fatalf("expected 1 result, got %d", len(results)) } r := results[0] - if !r.Enabled || r.Listed || r.Error != "" { - t.Fatalf("expected enabled+clean, got %+v", r) + listed, _ := s.Evaluate(r) + if !r.Enabled || listed || r.Error != "" { + t.Fatalf("expected enabled+clean, got %+v, Evaluate listed=%v", r, listed) } } @@ -48,8 +49,11 @@ func TestURLhausSource_Listed(t *testing.T) { s := &urlhausSource{endpoint: srv.URL} r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_urlhaus": true, "urlhaus_auth_key": "k"})[0] - if !r.Listed || len(r.Evidence) != 1 { - t.Fatalf("expected 1 listed evidence, got %+v", r) + if len(r.Evidence) != 1 { + t.Fatalf("expected 1 evidence item, got %+v", r) + } + if listed, severity := s.Evaluate(r); !listed || severity != SeverityCrit { + t.Errorf("expected Evaluate()=(true, crit), got (%v, %q)", listed, severity) } if r.Evidence[0].Status != "online" { t.Errorf("evidence status = %q", r.Evidence[0].Status) diff --git a/checker/virustotal.go b/checker/virustotal.go index 5213df9..bbd946e 100644 --- a/checker/virustotal.go +++ b/checker/virustotal.go @@ -146,17 +146,6 @@ func (s *virusTotalSource) Query(ctx context.Context, domain, registered string, return d.Vendors[i].Engine < d.Vendors[j].Engine }) res.Details = mustJSON(d) - - if d.Malicious == 0 && d.Suspicious == 0 { - // Clean. - return []SourceResult{res} - } - res.Listed = true - if d.Malicious > 0 { - res.Severity = SeverityCrit - } else { - res.Severity = SeverityWarn - } for _, v := range d.Vendors { res.Reasons = append(res.Reasons, v.Engine) res.Evidence = append(res.Evidence, Evidence{ @@ -167,6 +156,23 @@ func (s *virusTotalSource) Query(ctx context.Context, domain, registered string, return []SourceResult{res} } +func (*virusTotalSource) Evaluate(r SourceResult) (bool, string) { + var d vtDetails + if len(r.Details) == 0 { + return false, "" + } + if err := json.Unmarshal(r.Details, &d); err != nil { + return false, "" + } + if d.Malicious == 0 && d.Suspicious == 0 { + return false, "" + } + if d.Malicious > 0 { + return true, SeverityCrit + } + return true, SeverityWarn +} + func (*virusTotalSource) Diagnose(res SourceResult) Diagnosis { var d vtDetails _ = json.Unmarshal(res.Details, &d) diff --git a/checker/virustotal_test.go b/checker/virustotal_test.go index aba955c..e53d2cc 100644 --- a/checker/virustotal_test.go +++ b/checker/virustotal_test.go @@ -46,8 +46,8 @@ func TestVTSource_Listed(t *testing.T) { s := &virusTotalSource{endpoint: endpoint} r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"virustotal_api_key": "k"})[0] - if !r.Listed || r.Severity != SeverityCrit { - t.Errorf("expected listed+crit, got %+v", r) + if listed, severity := s.Evaluate(r); !listed || severity != SeverityCrit { + t.Errorf("expected Evaluate()=(true, crit), got (%v, %q)", listed, severity) } var d vtDetails if err := json.Unmarshal(r.Details, &d); err != nil { @@ -72,9 +72,12 @@ func TestVTSource_NotFound(t *testing.T) { s := &virusTotalSource{endpoint: endpoint} r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"virustotal_api_key": "k"})[0] - if r.Listed || r.Error != "" { + if r.Error != "" { t.Errorf("404 should be quiet not-listed: %+v", r) } + if listed, severity := s.Evaluate(r); listed || severity != "" { + t.Errorf("Evaluate() on clean result = (%v, %q), want (false, \"\")", listed, severity) + } if !strings.Contains(r.Reference, "example.com") { t.Errorf("reference URL missing: %+v", r) } From 829863e5a04e2647b1087211dc6523aa08fe6497 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Fri, 15 May 2026 21:34:52 +0800 Subject: [PATCH 03/15] Add a section on how to obtain API keys --- README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/README.md b/README.md index 6c6c138..7df0389 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,26 @@ widely-used reputation systems. | abuse.ch URLhaus | HTTPS lookup | optional Auth-Key (admin) | user (default on) | | VirusTotal v3 | HTTPS lookup | yes (admin) | admin | +### Obtaining API keys + +**Google Safe Browsing** (option: `google_safe_browsing_api_key`) +1. Go to the [Google Cloud Console](https://console.cloud.google.com/) and create or select a project. +2. Enable the *Safe Browsing API* under *APIs & Services → Library*. +3. Create an API key under *APIs & Services → Credentials*. +4. The free tier allows up to 10 000 queries/day with no billing required. + +**abuse.ch** (option: `urlhaus_auth_key` / `threatfox_auth_key` / `malwarebazaar_auth_key`) +1. Register a free account at [abuse.ch](https://abuse.ch/). +2. After login, retrieve your Auth-Key from your account profile page. +3. The same account and key works for URLhaus, ThreatFox, and MalwareBazaar — set it in each source option independently. +4. Free, no rate-limit tiers documented; the APIs are community-funded. + +**VirusTotal** (option: `virustotal_api_key`) +1. Create a free account at [virustotal.com](https://www.virustotal.com/). +2. Go to your profile and copy the API key. +3. Free tier: 4 requests/minute, 500 requests/day. No billing required. +4. The public API key is sufficient; premium keys unlock higher quotas. + DNS-based blocklists are queried in parallel. The OpenPhish feed is downloaded once per hour by the provider and cached in memory. From 6b08676ec5f641ddb198fc5d625e4a5c95f3cea6 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Fri, 15 May 2026 18:05:56 +0800 Subject: [PATCH 04/15] Add PhishTank as a new blacklist source --- README.md | 3 +- checker/phishtank.go | 241 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 243 insertions(+), 1 deletion(-) create mode 100644 checker/phishtank.go diff --git a/README.md b/README.md index 7df0389..0b541df 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,8 @@ widely-used reputation systems. | Extra DNSBL zones | DNS-based DBL | no | admin | | Google Safe Browsing | HTTPS lookup | yes (admin) | admin | | OpenPhish public feed | downloaded list | no | user (default on) | -| abuse.ch URLhaus | HTTPS lookup | optional Auth-Key (admin) | user (default on) | +| PhishTank | downloaded list | no | user (default on) | +| abuse.ch URLhaus | HTTPS lookup | free Auth-Key (admin) | user (default on) | | VirusTotal v3 | HTTPS lookup | yes (admin) | admin | ### Obtaining API keys diff --git a/checker/phishtank.go b/checker/phishtank.go new file mode 100644 index 0000000..003fb7b --- /dev/null +++ b/checker/phishtank.go @@ -0,0 +1,241 @@ +package checker + +import ( + "bufio" + "compress/gzip" + "context" + "encoding/csv" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "sync" + "time" + + sdk "git.happydns.org/checker-sdk-go/checker" +) + +const ( + phishTankFeedURL = "https://data.phishtank.com/data/online-valid.csv.gz" + phishTankDefaultTTL = 12 * time.Hour +) + +var phishTankGlobalCache = newPhishTankCache() + +func init() { Register(&phishTankSource{}) } + +type phishTankSource struct{} + +func (*phishTankSource) ID() string { return "phishtank" } +func (*phishTankSource) Name() string { return "PhishTank" } + +func (*phishTankSource) Options() SourceOptions { + return SourceOptions{ + Admin: []sdk.CheckerOptionField{ + { + Id: "phishtank_refresh_hours", + Type: "string", + Label: "PhishTank feed refresh interval (hours)", + Description: "How often to re-download the PhishTank online-valid feed. Minimum: 1. Default: 12.", + Default: "12", + }, + }, + User: []sdk.CheckerOptionField{ + { + Id: "enable_phishtank", + Type: "bool", + Label: "Use the PhishTank feed", + Description: "Download the PhishTank verified phishing list and check the domain against it.", + Default: true, + }, + }, + } +} + +func (s *phishTankSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult { + if !sdk.GetBoolOption(opts, "enable_phishtank", true) || registered == "" { + return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: false}} + } + + if ttlRaw, ok := sdk.GetOption[string](opts, "phishtank_refresh_hours"); ok && ttlRaw != "" { + if hours, err := strconv.Atoi(ttlRaw); err == nil && hours >= 1 { + phishTankGlobalCache.setTTL(time.Duration(hours) * time.Hour) + } + } + + urls, size, fetched, err := phishTankGlobalCache.lookup(ctx, registered) + res := SourceResult{ + SourceID: s.ID(), SourceName: s.Name(), Enabled: true, + Reference: "https://www.phishtank.com/", + Details: mustJSON(map[string]any{"feed_size": size, "fetched_at": fetched}), + } + if err != nil { + res.Error = err.Error() + } + if len(urls) > 0 { + res.Reasons = []string{"Phishing"} + for _, u := range urls { + res.Evidence = append(res.Evidence, Evidence{Label: "URL", Value: u}) + } + } + return []SourceResult{res} +} + +func (*phishTankSource) Evaluate(r SourceResult) (bool, string) { + if r.Enabled && r.Error == "" && len(r.Evidence) > 0 { + return true, SeverityCrit + } + return false, "" +} + +func (*phishTankSource) Diagnose(res SourceResult) Diagnosis { + urls := make([]string, 0, len(res.Evidence)) + for _, e := range res.Evidence { + urls = append(urls, e.Value) + } + previewN := min(len(urls), 5) + return Diagnosis{ + Severity: SeverityCrit, + Title: "Listed in the PhishTank phishing database", + Detail: fmt.Sprintf( + "%d URL(s) hosted on this domain are tracked as verified phishing by PhishTank. Examples: %s", + len(urls), joinNonEmpty(urls[:previewN], ", "), + ), + Fix: "https://www.phishtank.com/developer_info.php", + FixIsURL: true, + } +} + +// ---------- feed cache ---------- + +type phishTankCache struct { + mu sync.Mutex + urls []string + byHost map[string][]string + fetchedAt time.Time + lastAttemptAt time.Time + refreshing bool + ttl time.Duration + failBackoff time.Duration +} + +func newPhishTankCache() *phishTankCache { + return &phishTankCache{ + ttl: phishTankDefaultTTL, + failBackoff: 1 * time.Minute, + } +} + +func (c *phishTankCache) setTTL(d time.Duration) { + c.mu.Lock() + c.ttl = d + c.mu.Unlock() +} + +func (c *phishTankCache) lookup(ctx context.Context, domain string) (urls []string, size int, fetchedAt time.Time, err error) { + domain = strings.ToLower(strings.TrimSuffix(domain, ".")) + + c.mu.Lock() + stale := c.byHost == nil || time.Since(c.fetchedAt) > c.ttl + doRefresh := stale && !c.refreshing && time.Since(c.lastAttemptAt) > c.failBackoff + if doRefresh { + c.refreshing = true + } + c.mu.Unlock() + + if doRefresh { + newURLs, newByHost, ferr := c.fetch(ctx) + c.mu.Lock() + c.refreshing = false + c.lastAttemptAt = time.Now() + if ferr == nil { + c.urls = newURLs + c.byHost = newByHost + c.fetchedAt = c.lastAttemptAt + } else { + err = ferr + } + c.mu.Unlock() + } + + c.mu.Lock() + for host, hostURLs := range c.byHost { + if host == domain || strings.HasSuffix(host, "."+domain) { + urls = append(urls, hostURLs...) + } + } + size = len(c.urls) + fetchedAt = c.fetchedAt + c.mu.Unlock() + return urls, size, fetchedAt, err +} + +func (c *phishTankCache) fetch(ctx context.Context) ([]string, map[string][]string, error) { + reqCtx, cancel := context.WithTimeout(ctx, 120*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, phishTankFeedURL, nil) + if err != nil { + return nil, nil, err + } + req.Header.Set("User-Agent", "happydomain-checker-blacklist/1.0") + + resp, err := sharedHTTPClient.Do(req) + if err != nil { + return nil, nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, nil, fmt.Errorf("phishtank HTTP %d", resp.StatusCode) + } + + gz, err := gzip.NewReader(io.LimitReader(resp.Body, 128<<20)) + if err != nil { + return nil, nil, fmt.Errorf("phishtank gzip: %w", err) + } + defer gz.Close() + + r := csv.NewReader(bufio.NewReader(gz)) + r.ReuseRecord = true + + header, err := r.Read() + if err != nil { + return nil, nil, fmt.Errorf("phishtank csv header: %w", err) + } + urlIdx := -1 + for i, col := range header { + if col == "url" { + urlIdx = i + break + } + } + if urlIdx < 0 { + return nil, nil, fmt.Errorf("phishtank csv: no 'url' column in header") + } + + urls := make([]string, 0, 32768) + byHost := make(map[string][]string, 32768) + for { + record, err := r.Read() + if err == io.EOF { + break + } + if err != nil { + return nil, nil, fmt.Errorf("phishtank csv: %w", err) + } + if urlIdx >= len(record) { + continue + } + u := strings.TrimSpace(record[urlIdx]) + if u == "" { + continue + } + urls = append(urls, u) + if h := hostOfURL(u); h != "" { + byHost[h] = append(byHost[h], u) + } + } + return urls, byHost, nil +} From 229e7a8f0227f37ce95b9f67f9cce6993452a2e2 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Fri, 15 May 2026 18:30:46 +0800 Subject: [PATCH 05/15] Add abuse.ch ThreatFox and MalwareBazaar blacklist sources ThreatFox queries the IOC database for domain indicators (C2 servers, malware distribution, phishing); MalwareBazaar searches for malware samples tagged with the domain. Both require a free abuse.ch Auth-Key. --- README.md | 2 + checker/malwarebazaar.go | 149 ++++++++++++++++++++++++++++++++ checker/malwarebazaar_test.go | 94 +++++++++++++++++++++ checker/threatfox.go | 155 ++++++++++++++++++++++++++++++++++ checker/threatfox_test.go | 96 +++++++++++++++++++++ 5 files changed, 496 insertions(+) create mode 100644 checker/malwarebazaar.go create mode 100644 checker/malwarebazaar_test.go create mode 100644 checker/threatfox.go create mode 100644 checker/threatfox_test.go diff --git a/README.md b/README.md index 0b541df..7b4e8e8 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,8 @@ widely-used reputation systems. | OpenPhish public feed | downloaded list | no | user (default on) | | PhishTank | downloaded list | no | user (default on) | | abuse.ch URLhaus | HTTPS lookup | free Auth-Key (admin) | user (default on) | +| abuse.ch ThreatFox | HTTPS lookup | free Auth-Key (admin) | user (default on) | +| abuse.ch MalwareBazaar| HTTPS lookup | free Auth-Key (admin) | user (default on) | | VirusTotal v3 | HTTPS lookup | yes (admin) | admin | ### Obtaining API keys diff --git a/checker/malwarebazaar.go b/checker/malwarebazaar.go new file mode 100644 index 0000000..1627e6a --- /dev/null +++ b/checker/malwarebazaar.go @@ -0,0 +1,149 @@ +package checker + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "time" + + sdk "git.happydns.org/checker-sdk-go/checker" +) + +const malwareBazaarEndpoint = "https://mb-api.abuse.ch/api/v1/" + +func init() { Register(&malwareBazaarSource{endpoint: malwareBazaarEndpoint}) } + +type malwareBazaarSource struct { + endpoint string +} + +func (*malwareBazaarSource) ID() string { return "malwarebazaar" } +func (*malwareBazaarSource) Name() string { return "abuse.ch MalwareBazaar" } + +func (*malwareBazaarSource) Options() SourceOptions { + return SourceOptions{ + Admin: []sdk.CheckerOptionField{ + { + Id: "malwarebazaar_auth_key", + Type: "string", + Label: "MalwareBazaar Auth-Key", + Description: "abuse.ch MalwareBazaar Auth-Key (free, requires an abuse.ch account). Without this key the source is disabled.", + Secret: true, + }, + }, + User: []sdk.CheckerOptionField{ + { + Id: "enable_malwarebazaar", + Type: "bool", + Label: "Use abuse.ch MalwareBazaar", + Description: "Search MalwareBazaar for malware samples tagged with the domain (typically C2 infrastructure or delivery hosts).", + Default: true, + }, + }, + } +} + +func (s *malwareBazaarSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult { + authKey := stringOpt(opts, "malwarebazaar_auth_key") + if !sdk.GetBoolOption(opts, "enable_malwarebazaar", true) || registered == "" || authKey == "" { + return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: false}} + } + + res := SourceResult{ + SourceID: s.ID(), SourceName: s.Name(), Enabled: true, + Reference: "https://bazaar.abuse.ch/browse/", + } + + buf, err := json.Marshal(map[string]any{ + "query": "search_tag", + "tag": registered, + }) + if err != nil { + res.Error = err.Error() + return []SourceResult{res} + } + + reqCtx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, s.endpoint, bytes.NewReader(buf)) + if err != nil { + res.Error = err.Error() + return []SourceResult{res} + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Auth-Key", authKey) + + body, status, err := httpDo(req, 4<<20) + if err != nil { + res.Error = err.Error() + return []SourceResult{res} + } + if status != http.StatusOK { + res.Error = fmt.Sprintf("HTTP %d: %s", status, truncate(string(body), 200)) + return []SourceResult{res} + } + + var parsed struct { + QueryStatus string `json:"query_status"` + Data []struct { + SHA256Hash string `json:"sha256_hash"` + FileName string `json:"file_name"` + FileType string `json:"file_type_mime"` + Signature string `json:"signature"` + FirstSeen string `json:"first_seen"` + } `json:"data"` + } + if err := json.Unmarshal(body, &parsed); err != nil { + res.Error = "decode: " + err.Error() + return []SourceResult{res} + } + + switch parsed.QueryStatus { + case "ok": + signatures := map[string]bool{} + for _, sample := range parsed.Data { + if sample.Signature != "" && !signatures[sample.Signature] { + signatures[sample.Signature] = true + res.Reasons = append(res.Reasons, sample.Signature) + } + res.Evidence = append(res.Evidence, Evidence{ + Label: "Sample", + Value: sample.SHA256Hash, + Status: sample.FileType, + Extra: map[string]string{ + "filename": sample.FileName, + "signature": sample.Signature, + "first_seen": sample.FirstSeen, + }, + }) + } + case "no_results", "illegal_search_term": + // Clean. + default: + res.Error = "query_status=" + parsed.QueryStatus + } + return []SourceResult{res} +} + +func (*malwareBazaarSource) Evaluate(r SourceResult) (bool, string) { + if r.Enabled && r.Error == "" && len(r.Evidence) > 0 { + return true, SeverityWarn + } + return false, "" +} + +func (*malwareBazaarSource) Diagnose(res SourceResult) Diagnosis { + return Diagnosis{ + Severity: SeverityWarn, + Title: "Associated with malware samples in abuse.ch MalwareBazaar", + Detail: fmt.Sprintf( + "%d malware sample(s) are tagged with this domain; malware family/signature(s): %s. MalwareBazaar samples are tagged with their C2 domain or delivery host when known. Investigate the domain's hosting and DNS records to identify and remove malicious infrastructure.", + len(res.Evidence), joinNonEmpty(res.Reasons, ", "), + ), + Fix: res.Reference, + FixIsURL: res.Reference != "", + } +} diff --git a/checker/malwarebazaar_test.go b/checker/malwarebazaar_test.go new file mode 100644 index 0000000..62bc24a --- /dev/null +++ b/checker/malwarebazaar_test.go @@ -0,0 +1,94 @@ +package checker + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + sdk "git.happydns.org/checker-sdk-go/checker" +) + +func TestMalwareBazaarSource_NoResults(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"query_status":"no_results"}`)) + })) + defer srv.Close() + + s := &malwareBazaarSource{endpoint: srv.URL} + results := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_malwarebazaar": true, "malwarebazaar_auth_key": "k"}) + if len(results) != 1 { + t.Fatalf("expected 1 result, got %d", len(results)) + } + r := results[0] + listed, _ := s.Evaluate(r) + if !r.Enabled || listed || r.Error != "" { + t.Fatalf("expected enabled+clean, got %+v, Evaluate listed=%v", r, listed) + } +} + +func TestMalwareBazaarSource_Listed(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Auth-Key") == "" { + t.Errorf("missing Auth-Key header") + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "query_status": "ok", + "data": [{ + "sha256_hash": "aaaa1111bbbb2222cccc3333dddd4444eeee5555ffff6666aaaa1111bbbb2222", + "file_name": "evil.exe", + "file_type_mime": "application/x-dosexec", + "signature": "Emotet", + "first_seen": "2024-01-01 00:00:00" + }] + }`)) + })) + defer srv.Close() + + s := &malwareBazaarSource{endpoint: srv.URL} + r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_malwarebazaar": true, "malwarebazaar_auth_key": "k"})[0] + if len(r.Evidence) != 1 { + t.Fatalf("expected 1 evidence item, got %+v", r) + } + if listed, severity := s.Evaluate(r); !listed || severity != SeverityWarn { + t.Errorf("expected Evaluate()=(true, warn), got (%v, %q)", listed, severity) + } + if r.Evidence[0].Status != "application/x-dosexec" { + t.Errorf("evidence status = %q", r.Evidence[0].Status) + } + if len(r.Reasons) != 1 || r.Reasons[0] != "Emotet" { + t.Errorf("reasons = %v", r.Reasons) + } +} + +func TestMalwareBazaarSource_HTTPError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusUnauthorized) + _, _ = w.Write([]byte("unauthorized")) + })) + defer srv.Close() + + s := &malwareBazaarSource{endpoint: srv.URL} + r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_malwarebazaar": true, "malwarebazaar_auth_key": "k"})[0] + if r.Error == "" { + t.Errorf("expected error, got %+v", r) + } +} + +func TestMalwareBazaarSource_Disabled(t *testing.T) { + s := &malwareBazaarSource{endpoint: "http://nope"} + r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_malwarebazaar": false})[0] + if r.Enabled { + t.Errorf("expected disabled, got %+v", r) + } +} + +func TestMalwareBazaarSource_NoAuthKey(t *testing.T) { + s := &malwareBazaarSource{endpoint: "http://nope"} + r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_malwarebazaar": true})[0] + if r.Enabled { + t.Errorf("expected disabled when no auth key, got %+v", r) + } +} diff --git a/checker/threatfox.go b/checker/threatfox.go new file mode 100644 index 0000000..67fae49 --- /dev/null +++ b/checker/threatfox.go @@ -0,0 +1,155 @@ +package checker + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "time" + + sdk "git.happydns.org/checker-sdk-go/checker" +) + +const threatFoxEndpoint = "https://threatfox-api.abuse.ch/api/v1/" + +func init() { Register(&threatFoxSource{endpoint: threatFoxEndpoint}) } + +type threatFoxSource struct { + endpoint string +} + +func (*threatFoxSource) ID() string { return "threatfox" } +func (*threatFoxSource) Name() string { return "abuse.ch ThreatFox" } + +func (*threatFoxSource) Options() SourceOptions { + return SourceOptions{ + Admin: []sdk.CheckerOptionField{ + { + Id: "threatfox_auth_key", + Type: "string", + Label: "ThreatFox Auth-Key", + Description: "abuse.ch ThreatFox Auth-Key (free, requires an abuse.ch account). Without this key the source is disabled.", + Secret: true, + }, + }, + User: []sdk.CheckerOptionField{ + { + Id: "enable_threatfox", + Type: "bool", + Label: "Use abuse.ch ThreatFox", + Description: "Query ThreatFox for indicators of compromise (C2 servers, malware, phishing) associated with the domain.", + Default: true, + }, + }, + } +} + +func (s *threatFoxSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult { + authKey := stringOpt(opts, "threatfox_auth_key") + if !sdk.GetBoolOption(opts, "enable_threatfox", true) || registered == "" || authKey == "" { + return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: false}} + } + + res := SourceResult{ + SourceID: s.ID(), SourceName: s.Name(), Enabled: true, + Reference: "https://threatfox.abuse.ch/browse/", + } + + buf, err := json.Marshal(map[string]any{ + "query": "search_ioc", + "search_term": registered, + }) + if err != nil { + res.Error = err.Error() + return []SourceResult{res} + } + + reqCtx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, s.endpoint, bytes.NewReader(buf)) + if err != nil { + res.Error = err.Error() + return []SourceResult{res} + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Auth-Key", authKey) + + body, status, err := httpDo(req, 4<<20) + if err != nil { + res.Error = err.Error() + return []SourceResult{res} + } + if status != http.StatusOK { + res.Error = fmt.Sprintf("HTTP %d: %s", status, truncate(string(body), 200)) + return []SourceResult{res} + } + + var parsed struct { + QueryStatus string `json:"query_status"` + Data []struct { + IOCValue string `json:"ioc_value"` + IOCType string `json:"ioc_type"` + ThreatType string `json:"threat_type"` + MalwarePrintable string `json:"malware_printable"` + ConfidenceLevel int `json:"confidence_level"` + FirstSeen string `json:"first_seen"` + LastSeen string `json:"last_seen"` + } `json:"data"` + } + if err := json.Unmarshal(body, &parsed); err != nil { + res.Error = "decode: " + err.Error() + return []SourceResult{res} + } + + switch parsed.QueryStatus { + case "ok": + threats := map[string]bool{} + for _, ioc := range parsed.Data { + label := ioc.MalwarePrintable + if label == "" { + label = ioc.ThreatType + } + if label != "" && !threats[label] { + threats[label] = true + res.Reasons = append(res.Reasons, label) + } + res.Evidence = append(res.Evidence, Evidence{ + Label: "IOC", + Value: ioc.IOCValue, + Status: ioc.ThreatType, + Extra: map[string]string{ + "malware": ioc.MalwarePrintable, + "confidence": fmt.Sprintf("%d%%", ioc.ConfidenceLevel), + "first_seen": ioc.FirstSeen, + }, + }) + } + case "no_result": + // Clean. + default: + res.Error = "query_status=" + parsed.QueryStatus + } + return []SourceResult{res} +} + +func (*threatFoxSource) Evaluate(r SourceResult) (bool, string) { + if r.Enabled && r.Error == "" && len(r.Evidence) > 0 { + return true, SeverityCrit + } + return false, "" +} + +func (*threatFoxSource) Diagnose(res SourceResult) Diagnosis { + return Diagnosis{ + Severity: SeverityCrit, + Title: "Listed in abuse.ch ThreatFox as a threat IOC", + Detail: fmt.Sprintf( + "%d IOC(s) match this domain; threat(s): %s. ThreatFox tracks indicators of compromise including C2 servers, malware distribution hosts, and phishing infrastructure. Treat the host as compromised or abused: investigate recent DNS changes, audit hosted content, then request removal through the ThreatFox reference page.", + len(res.Evidence), joinNonEmpty(res.Reasons, ", "), + ), + Fix: res.Reference, + FixIsURL: res.Reference != "", + } +} diff --git a/checker/threatfox_test.go b/checker/threatfox_test.go new file mode 100644 index 0000000..5e5f12f --- /dev/null +++ b/checker/threatfox_test.go @@ -0,0 +1,96 @@ +package checker + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + sdk "git.happydns.org/checker-sdk-go/checker" +) + +func TestThreatFoxSource_NoResult(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"query_status":"no_result","data":[]}`)) + })) + defer srv.Close() + + s := &threatFoxSource{endpoint: srv.URL} + results := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_threatfox": true, "threatfox_auth_key": "k"}) + if len(results) != 1 { + t.Fatalf("expected 1 result, got %d", len(results)) + } + r := results[0] + listed, _ := s.Evaluate(r) + if !r.Enabled || listed || r.Error != "" { + t.Fatalf("expected enabled+clean, got %+v, Evaluate listed=%v", r, listed) + } +} + +func TestThreatFoxSource_Listed(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Auth-Key") == "" { + t.Errorf("missing Auth-Key header") + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "query_status": "ok", + "data": [{ + "ioc_value": "example.com:443", + "ioc_type": "domain", + "threat_type": "botnet_cc", + "malware_printable": "Emotet", + "confidence_level": 75, + "first_seen": "2024-01-01 00:00:00 UTC", + "last_seen": "2024-06-01 00:00:00 UTC" + }] + }`)) + })) + defer srv.Close() + + s := &threatFoxSource{endpoint: srv.URL} + r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_threatfox": true, "threatfox_auth_key": "k"})[0] + if len(r.Evidence) != 1 { + t.Fatalf("expected 1 evidence item, got %+v", r) + } + if listed, severity := s.Evaluate(r); !listed || severity != SeverityCrit { + t.Errorf("expected Evaluate()=(true, crit), got (%v, %q)", listed, severity) + } + if r.Evidence[0].Value != "example.com:443" { + t.Errorf("evidence value = %q", r.Evidence[0].Value) + } + if len(r.Reasons) != 1 || r.Reasons[0] != "Emotet" { + t.Errorf("reasons = %v", r.Reasons) + } +} + +func TestThreatFoxSource_HTTPError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusUnauthorized) + _, _ = w.Write([]byte("unauthorized")) + })) + defer srv.Close() + + s := &threatFoxSource{endpoint: srv.URL} + r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_threatfox": true, "threatfox_auth_key": "k"})[0] + if r.Error == "" { + t.Errorf("expected error, got %+v", r) + } +} + +func TestThreatFoxSource_Disabled(t *testing.T) { + s := &threatFoxSource{endpoint: "http://nope"} + r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_threatfox": false})[0] + if r.Enabled { + t.Errorf("expected disabled, got %+v", r) + } +} + +func TestThreatFoxSource_NoAuthKey(t *testing.T) { + s := &threatFoxSource{endpoint: "http://nope"} + r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_threatfox": true})[0] + if r.Enabled { + t.Errorf("expected disabled when no auth key, got %+v", r) + } +} From 061b5361ca881838da888c45dd86cc54d2bc5885 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Fri, 15 May 2026 18:38:25 +0800 Subject: [PATCH 06/15] Merge duplicate phishCache/phishTankCache into shared feedCache --- checker/feedcache.go | 89 +++++++++++++++++++++++++++ checker/openphish.go | 142 +++++++++++-------------------------------- checker/phishtank.go | 70 +-------------------- 3 files changed, 128 insertions(+), 173 deletions(-) create mode 100644 checker/feedcache.go diff --git a/checker/feedcache.go b/checker/feedcache.go new file mode 100644 index 0000000..aeaf154 --- /dev/null +++ b/checker/feedcache.go @@ -0,0 +1,89 @@ +package checker + +import ( + "context" + "net/url" + "strings" + "sync" + "time" +) + +// feedCache is a generic URL-feed cache shared between phishing-feed +// sources (OpenPhish, PhishTank). It holds a hostname-indexed snapshot +// of the feed, refreshes on TTL expiry, and ensures only one refresh is +// in flight at a time so concurrent lookups still serve stale data +// during a refresh. +type feedCache struct { + mu sync.Mutex + urls []string + byHost map[string][]string + fetchedAt time.Time + lastAttemptAt time.Time + refreshing bool + ttl time.Duration + failBackoff time.Duration + fetchFn func(ctx context.Context) (urls []string, byHost map[string][]string, err error) +} + +func newFeedCache(ttl time.Duration, fetch func(context.Context) ([]string, map[string][]string, error)) *feedCache { + if ttl <= 0 { + ttl = time.Hour + } + return &feedCache{ + ttl: ttl, + failBackoff: time.Minute, + fetchFn: fetch, + } +} + +func (c *feedCache) setTTL(d time.Duration) { + c.mu.Lock() + c.ttl = d + c.mu.Unlock() +} + +func (c *feedCache) lookup(ctx context.Context, domain string) (urls []string, size int, fetchedAt time.Time, err error) { + domain = strings.ToLower(strings.TrimSuffix(domain, ".")) + + c.mu.Lock() + stale := c.byHost == nil || time.Since(c.fetchedAt) > c.ttl + doRefresh := stale && !c.refreshing && time.Since(c.lastAttemptAt) > c.failBackoff + if doRefresh { + c.refreshing = true + } + c.mu.Unlock() + + if doRefresh { + newURLs, newByHost, ferr := c.fetchFn(ctx) + c.mu.Lock() + c.refreshing = false + c.lastAttemptAt = time.Now() + if ferr == nil { + c.urls = newURLs + c.byHost = newByHost + c.fetchedAt = c.lastAttemptAt + } else { + err = ferr + } + c.mu.Unlock() + } + + c.mu.Lock() + for host, hostURLs := range c.byHost { + if host == domain || strings.HasSuffix(host, "."+domain) { + urls = append(urls, hostURLs...) + } + } + size = len(c.urls) + fetchedAt = c.fetchedAt + c.mu.Unlock() + return urls, size, fetchedAt, err +} + +func hostOfURL(s string) string { + u, err := url.Parse(s) + if err != nil { + return "" + } + return strings.ToLower(u.Hostname()) +} diff --git a/checker/openphish.go b/checker/openphish.go index 12ab8dc..66dc08e 100644 --- a/checker/openphish.go +++ b/checker/openphish.go @@ -6,9 +6,7 @@ import ( "fmt" "io" "net/http" - "net/url" "strings" - "sync" "time" sdk "git.happydns.org/checker-sdk-go/checker" @@ -18,7 +16,7 @@ const openPhishFeedURL = "https://openphish.com/feed.txt" func init() { Register(&openPhishSource{ - cache: newPhishCache(openPhishFeedURL, 1*time.Hour), + cache: newFeedCache(1*time.Hour, openPhishFetch(openPhishFeedURL)), }) } @@ -27,7 +25,7 @@ func init() { // every URL in the feed. The cache is per-source-instance so it lives // for as long as the process. type openPhishSource struct { - cache *phishCache + cache *feedCache } func (*openPhishSource) ID() string { return "openphish" } @@ -96,114 +94,46 @@ func (*openPhishSource) Diagnose(res SourceResult) Diagnosis { } } -// ---------- feed cache ---------- +// openPhishFetch returns a fetchFn that downloads and parses the +// OpenPhish plain-text feed at feedURL. +func openPhishFetch(feedURL string) func(context.Context) ([]string, map[string][]string, error) { + return func(ctx context.Context) ([]string, map[string][]string, error) { + reqCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() -type phishCache struct { - mu sync.Mutex - urls []string - byHost map[string][]string - fetchedAt time.Time - lastAttemptAt time.Time - refreshing bool - ttl time.Duration - failBackoff time.Duration - feedURL string -} - -func newPhishCache(feedURL string, ttl time.Duration) *phishCache { - if feedURL == "" { - feedURL = openPhishFeedURL - } - if ttl <= 0 { - ttl = 1 * time.Hour - } - return &phishCache{ttl: ttl, feedURL: feedURL, failBackoff: 1 * time.Minute} -} - -func (c *phishCache) lookup(ctx context.Context, domain string) (urls []string, size int, fetchedAt time.Time, err error) { - domain = strings.ToLower(strings.TrimSuffix(domain, ".")) - - c.mu.Lock() - stale := c.byHost == nil || time.Since(c.fetchedAt) > c.ttl - doRefresh := stale && !c.refreshing && time.Since(c.lastAttemptAt) > c.failBackoff - if doRefresh { - c.refreshing = true - } - c.mu.Unlock() - - if doRefresh { - // Fetch without holding the cache lock so concurrent lookups - // can still serve stale data. Only one refresh runs at a time. - newURLs, newByHost, ferr := c.fetch(ctx) - c.mu.Lock() - c.refreshing = false - c.lastAttemptAt = time.Now() - if ferr == nil { - c.urls = newURLs - c.byHost = newByHost - c.fetchedAt = c.lastAttemptAt - } else { - err = ferr + req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, feedURL, nil) + if err != nil { + return nil, nil, err } - c.mu.Unlock() - } + req.Header.Set("User-Agent", "happydomain-checker-blacklist/1.0") - c.mu.Lock() - for host, hostURLs := range c.byHost { - if host == domain || strings.HasSuffix(host, "."+domain) { - urls = append(urls, hostURLs...) + resp, err := sharedHTTPClient.Do(req) + if err != nil { + return nil, nil, err } - } - size = len(c.urls) - fetchedAt = c.fetchedAt - c.mu.Unlock() - return urls, size, fetchedAt, err -} + defer resp.Body.Close() -func (c *phishCache) fetch(ctx context.Context) ([]string, map[string][]string, error) { - reqCtx, cancel := context.WithTimeout(ctx, 30*time.Second) - defer cancel() - - req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, c.feedURL, nil) - if err != nil { - return nil, nil, err - } - req.Header.Set("User-Agent", "happydomain-checker-blacklist/1.0") - - resp, err := sharedHTTPClient.Do(req) - if err != nil { - return nil, nil, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, nil, fmt.Errorf("openphish HTTP %d", resp.StatusCode) - } - - urls := make([]string, 0, 8192) - byHost := make(map[string][]string, 8192) - scanner := bufio.NewScanner(io.LimitReader(resp.Body, 64<<20)) - scanner.Buffer(make([]byte, 0, 64*1024), 1<<20) - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) - if line == "" || strings.HasPrefix(line, "#") { - continue + if resp.StatusCode != http.StatusOK { + return nil, nil, fmt.Errorf("openphish HTTP %d", resp.StatusCode) } - urls = append(urls, line) - if h := hostOfURL(line); h != "" { - byHost[h] = append(byHost[h], line) - } - } - if err := scanner.Err(); err != nil { - return nil, nil, err - } - return urls, byHost, nil -} -func hostOfURL(s string) string { - u, err := url.Parse(s) - if err != nil { - return "" + urls := make([]string, 0, 8192) + byHost := make(map[string][]string, 8192) + scanner := bufio.NewScanner(io.LimitReader(resp.Body, 64<<20)) + scanner.Buffer(make([]byte, 0, 64*1024), 1<<20) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + urls = append(urls, line) + if h := hostOfURL(line); h != "" { + byHost[h] = append(byHost[h], line) + } + } + if err := scanner.Err(); err != nil { + return nil, nil, err + } + return urls, byHost, nil } - return strings.ToLower(u.Hostname()) } diff --git a/checker/phishtank.go b/checker/phishtank.go index 003fb7b..6208360 100644 --- a/checker/phishtank.go +++ b/checker/phishtank.go @@ -10,7 +10,6 @@ import ( "net/http" "strconv" "strings" - "sync" "time" sdk "git.happydns.org/checker-sdk-go/checker" @@ -21,7 +20,7 @@ const ( phishTankDefaultTTL = 12 * time.Hour ) -var phishTankGlobalCache = newPhishTankCache() +var phishTankGlobalCache = newFeedCache(phishTankDefaultTTL, phishTankFetch) func init() { Register(&phishTankSource{}) } @@ -107,71 +106,8 @@ func (*phishTankSource) Diagnose(res SourceResult) Diagnosis { } } -// ---------- feed cache ---------- - -type phishTankCache struct { - mu sync.Mutex - urls []string - byHost map[string][]string - fetchedAt time.Time - lastAttemptAt time.Time - refreshing bool - ttl time.Duration - failBackoff time.Duration -} - -func newPhishTankCache() *phishTankCache { - return &phishTankCache{ - ttl: phishTankDefaultTTL, - failBackoff: 1 * time.Minute, - } -} - -func (c *phishTankCache) setTTL(d time.Duration) { - c.mu.Lock() - c.ttl = d - c.mu.Unlock() -} - -func (c *phishTankCache) lookup(ctx context.Context, domain string) (urls []string, size int, fetchedAt time.Time, err error) { - domain = strings.ToLower(strings.TrimSuffix(domain, ".")) - - c.mu.Lock() - stale := c.byHost == nil || time.Since(c.fetchedAt) > c.ttl - doRefresh := stale && !c.refreshing && time.Since(c.lastAttemptAt) > c.failBackoff - if doRefresh { - c.refreshing = true - } - c.mu.Unlock() - - if doRefresh { - newURLs, newByHost, ferr := c.fetch(ctx) - c.mu.Lock() - c.refreshing = false - c.lastAttemptAt = time.Now() - if ferr == nil { - c.urls = newURLs - c.byHost = newByHost - c.fetchedAt = c.lastAttemptAt - } else { - err = ferr - } - c.mu.Unlock() - } - - c.mu.Lock() - for host, hostURLs := range c.byHost { - if host == domain || strings.HasSuffix(host, "."+domain) { - urls = append(urls, hostURLs...) - } - } - size = len(c.urls) - fetchedAt = c.fetchedAt - c.mu.Unlock() - return urls, size, fetchedAt, err -} - -func (c *phishTankCache) fetch(ctx context.Context) ([]string, map[string][]string, error) { +// phishTankFetch downloads and parses the PhishTank gzip-compressed CSV feed. +func phishTankFetch(ctx context.Context) ([]string, map[string][]string, error) { reqCtx, cancel := context.WithTimeout(ctx, 120*time.Second) defer cancel() From 6b1d2e2540dfdec08436444cde3d3d05239eb38e Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Fri, 15 May 2026 18:41:45 +0800 Subject: [PATCH 07/15] Extract disabledResult and evidenceEval helpers to reduce boilerplate Add two shared helpers to source.go and apply them across all sources: - disabledResult(id, name) replaces the repeated inline SourceResult literal - evidenceEval(r, severity) replaces the identical Evaluate body in 6 sources --- checker/dnsbl.go | 4 +--- checker/malwarebazaar.go | 7 ++----- checker/openphish.go | 7 ++----- checker/phishtank.go | 7 ++----- checker/safebrowsing.go | 7 ++----- checker/source.go | 14 ++++++++++++++ checker/threatfox.go | 7 ++----- checker/urlhaus.go | 7 ++----- checker/virustotal.go | 2 +- 9 files changed, 28 insertions(+), 34 deletions(-) diff --git a/checker/dnsbl.go b/checker/dnsbl.go index 34fa68a..f34b7e6 100644 --- a/checker/dnsbl.go +++ b/checker/dnsbl.go @@ -93,9 +93,7 @@ var DefaultDNSBLZones = []DNSBLZone{ func (s *dnsblSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult { zones := zonesFromOptions(opts) if registered == "" || len(zones) == 0 { - return []SourceResult{{ - SourceID: s.ID(), SourceName: s.Name(), Enabled: false, - }} + return disabledResult(s.ID(), s.Name()) } out := make([]SourceResult, len(zones)) diff --git a/checker/malwarebazaar.go b/checker/malwarebazaar.go index 1627e6a..9e1af6b 100644 --- a/checker/malwarebazaar.go +++ b/checker/malwarebazaar.go @@ -48,7 +48,7 @@ func (*malwareBazaarSource) Options() SourceOptions { func (s *malwareBazaarSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult { authKey := stringOpt(opts, "malwarebazaar_auth_key") if !sdk.GetBoolOption(opts, "enable_malwarebazaar", true) || registered == "" || authKey == "" { - return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: false}} + return disabledResult(s.ID(), s.Name()) } res := SourceResult{ @@ -129,10 +129,7 @@ func (s *malwareBazaarSource) Query(ctx context.Context, domain, registered stri } func (*malwareBazaarSource) Evaluate(r SourceResult) (bool, string) { - if r.Enabled && r.Error == "" && len(r.Evidence) > 0 { - return true, SeverityWarn - } - return false, "" + return evidenceEval(r, SeverityWarn) } func (*malwareBazaarSource) Diagnose(res SourceResult) Diagnosis { diff --git a/checker/openphish.go b/checker/openphish.go index 66dc08e..e175f73 100644 --- a/checker/openphish.go +++ b/checker/openphish.go @@ -47,7 +47,7 @@ func (*openPhishSource) Options() SourceOptions { func (s *openPhishSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult { if !sdk.GetBoolOption(opts, "enable_openphish", true) || registered == "" { - return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: false}} + return disabledResult(s.ID(), s.Name()) } urls, size, fetched, err := s.cache.lookup(ctx, registered) @@ -70,10 +70,7 @@ func (s *openPhishSource) Query(ctx context.Context, domain, registered string, } func (*openPhishSource) Evaluate(r SourceResult) (bool, string) { - if r.Enabled && r.Error == "" && len(r.Evidence) > 0 { - return true, SeverityCrit - } - return false, "" + return evidenceEval(r, SeverityCrit) } func (*openPhishSource) Diagnose(res SourceResult) Diagnosis { diff --git a/checker/phishtank.go b/checker/phishtank.go index 6208360..d3505b4 100644 --- a/checker/phishtank.go +++ b/checker/phishtank.go @@ -54,7 +54,7 @@ func (*phishTankSource) Options() SourceOptions { func (s *phishTankSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult { if !sdk.GetBoolOption(opts, "enable_phishtank", true) || registered == "" { - return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: false}} + return disabledResult(s.ID(), s.Name()) } if ttlRaw, ok := sdk.GetOption[string](opts, "phishtank_refresh_hours"); ok && ttlRaw != "" { @@ -82,10 +82,7 @@ func (s *phishTankSource) Query(ctx context.Context, domain, registered string, } func (*phishTankSource) Evaluate(r SourceResult) (bool, string) { - if r.Enabled && r.Error == "" && len(r.Evidence) > 0 { - return true, SeverityCrit - } - return false, "" + return evidenceEval(r, SeverityCrit) } func (*phishTankSource) Diagnose(res SourceResult) Diagnosis { diff --git a/checker/safebrowsing.go b/checker/safebrowsing.go index be489a7..f04ec70 100644 --- a/checker/safebrowsing.go +++ b/checker/safebrowsing.go @@ -54,7 +54,7 @@ func (*safeBrowsingSource) Options() SourceOptions { func (s *safeBrowsingSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult { apiKey := stringOpt(opts, "google_safe_browsing_api_key") if apiKey == "" { - return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: false}} + return disabledResult(s.ID(), s.Name()) } if registered == "" { return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: true}} @@ -142,10 +142,7 @@ func (s *safeBrowsingSource) Query(ctx context.Context, domain, registered strin } func (*safeBrowsingSource) Evaluate(r SourceResult) (bool, string) { - if r.Enabled && r.Error == "" && len(r.Evidence) > 0 { - return true, SeverityCrit - } - return false, "" + return evidenceEval(r, SeverityCrit) } func (*safeBrowsingSource) Diagnose(res SourceResult) Diagnosis { diff --git a/checker/source.go b/checker/source.go index cc8b15e..03f47fc 100644 --- a/checker/source.go +++ b/checker/source.go @@ -152,6 +152,20 @@ func Sources() []Source { return out } +// disabledResult returns the standard "source is disabled" sentinel slice. +func disabledResult(id, name string) []SourceResult { + return []SourceResult{{SourceID: id, SourceName: name, Enabled: false}} +} + +// evidenceEval is the common Evaluate body: listed when there is at least +// one Evidence entry and no error. +func evidenceEval(r SourceResult, severity string) (bool, string) { + if r.Enabled && r.Error == "" && len(r.Evidence) > 0 { + return true, severity + } + return false, "" +} + // EvaluateResult looks up the source that produced r from the registry // and delegates to its Evaluate method. Returns (false, "") when the // source is not found — a safe default that never promotes a stale diff --git a/checker/threatfox.go b/checker/threatfox.go index 67fae49..4df96fa 100644 --- a/checker/threatfox.go +++ b/checker/threatfox.go @@ -48,7 +48,7 @@ func (*threatFoxSource) Options() SourceOptions { func (s *threatFoxSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult { authKey := stringOpt(opts, "threatfox_auth_key") if !sdk.GetBoolOption(opts, "enable_threatfox", true) || registered == "" || authKey == "" { - return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: false}} + return disabledResult(s.ID(), s.Name()) } res := SourceResult{ @@ -135,10 +135,7 @@ func (s *threatFoxSource) Query(ctx context.Context, domain, registered string, } func (*threatFoxSource) Evaluate(r SourceResult) (bool, string) { - if r.Enabled && r.Error == "" && len(r.Evidence) > 0 { - return true, SeverityCrit - } - return false, "" + return evidenceEval(r, SeverityCrit) } func (*threatFoxSource) Diagnose(res SourceResult) Diagnosis { diff --git a/checker/urlhaus.go b/checker/urlhaus.go index 9d9341e..1bac676 100644 --- a/checker/urlhaus.go +++ b/checker/urlhaus.go @@ -67,7 +67,7 @@ type urlhausURL struct { func (s *urlhausSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult { authKey := stringOpt(opts, "urlhaus_auth_key") if !sdk.GetBoolOption(opts, "enable_urlhaus", true) || registered == "" || authKey == "" { - return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: false}} + return disabledResult(s.ID(), s.Name()) } res := SourceResult{SourceID: s.ID(), SourceName: s.Name(), Enabled: true} @@ -147,10 +147,7 @@ func (s *urlhausSource) Query(ctx context.Context, domain, registered string, op } func (*urlhausSource) Evaluate(r SourceResult) (bool, string) { - if r.Enabled && r.Error == "" && len(r.Evidence) > 0 { - return true, SeverityCrit - } - return false, "" + return evidenceEval(r, SeverityCrit) } func (*urlhausSource) Diagnose(res SourceResult) Diagnosis { diff --git a/checker/virustotal.go b/checker/virustotal.go index bbd946e..14d1132 100644 --- a/checker/virustotal.go +++ b/checker/virustotal.go @@ -60,7 +60,7 @@ type vtVendorVerdict struct { func (s *virusTotalSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult { apiKey := stringOpt(opts, "virustotal_api_key") if apiKey == "" { - return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: false}} + return disabledResult(s.ID(), s.Name()) } if registered == "" { return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: true}} From 9916ab0732d4355d1b67a225db3cd99b4c9163d2 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Fri, 15 May 2026 18:56:05 +0800 Subject: [PATCH 08/15] Add Botvrij.eu domain blocklist source Downloads the Botvrij.eu public IOC domain list (no API key required), caches it in-process with a 6h TTL, and flags any registered domain that appears directly or as a parent of a feed entry. --- README.md | 1 + checker/botvrij.go | 191 ++++++++++++++++++++++++++++++++++++++++ checker/botvrij_test.go | 94 ++++++++++++++++++++ 3 files changed, 286 insertions(+) create mode 100644 checker/botvrij.go create mode 100644 checker/botvrij_test.go diff --git a/README.md b/README.md index 7b4e8e8..0bf3f1a 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ widely-used reputation systems. | abuse.ch URLhaus | HTTPS lookup | free Auth-Key (admin) | user (default on) | | abuse.ch ThreatFox | HTTPS lookup | free Auth-Key (admin) | user (default on) | | abuse.ch MalwareBazaar| HTTPS lookup | free Auth-Key (admin) | user (default on) | +| Botvrij.eu | downloaded list | no | user (default on) | | VirusTotal v3 | HTTPS lookup | yes (admin) | admin | ### Obtaining API keys diff --git a/checker/botvrij.go b/checker/botvrij.go new file mode 100644 index 0000000..dbc4348 --- /dev/null +++ b/checker/botvrij.go @@ -0,0 +1,191 @@ +package checker + +import ( + "bufio" + "context" + "fmt" + "io" + "net/http" + "strings" + "sync" + "time" + + sdk "git.happydns.org/checker-sdk-go/checker" +) + +const ( + botvrijFeedURL = "https://www.botvrij.eu/data/ioclist.domain.simple" + botvrijDefaultTTL = 6 * time.Hour + botvrijFailBackoff = 1 * time.Minute +) + +func init() { + Register(&botvrijSource{ + cache: newBotvrijCache(botvrijFeedURL), + }) +} + +type botvrijSource struct { + cache *botvrijCache +} + +func (*botvrijSource) ID() string { return "botvrij" } +func (*botvrijSource) Name() string { return "Botvrij.eu domain blocklist" } + +func (*botvrijSource) Options() SourceOptions { + return SourceOptions{ + User: []sdk.CheckerOptionField{ + { + Id: "enable_botvrij", + Type: "bool", + Label: "Use Botvrij.eu domain blocklist", + Description: "Download the Botvrij.eu public domain blocklist (refreshed every 6h) and check the domain against it.", + Default: true, + }, + }, + } +} + +func (s *botvrijSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult { + if !sdk.GetBoolOption(opts, "enable_botvrij", true) || registered == "" { + return disabledResult(s.ID(), s.Name()) + } + + matched, size, fetched, err := s.cache.lookup(ctx, registered) + res := SourceResult{ + SourceID: s.ID(), SourceName: s.Name(), Enabled: true, + Reference: "https://botvrij.eu/", + Details: mustJSON(map[string]any{"feed_size": size, "fetched_at": fetched}), + } + if err != nil { + res.Error = err.Error() + } + if len(matched) > 0 { + res.Reasons = []string{"Malicious domain"} + for _, d := range matched { + res.Evidence = append(res.Evidence, Evidence{Label: "Domain", Value: d}) + } + } + return []SourceResult{res} +} + +func (*botvrijSource) Evaluate(r SourceResult) (bool, string) { + return evidenceEval(r, SeverityCrit) +} + +func (*botvrijSource) Diagnose(res SourceResult) Diagnosis { + domains := make([]string, 0, len(res.Evidence)) + for _, e := range res.Evidence { + domains = append(domains, e.Value) + } + previewN := min(len(domains), 5) + return Diagnosis{ + Severity: SeverityCrit, + Title: "Listed in the Botvrij.eu domain blocklist", + Detail: fmt.Sprintf( + "%d domain(s) matching this registered domain appear in the Botvrij.eu IOC list; examples: %s. Botvrij.eu tracks domains associated with malware C2, exploit kits, and other malicious infrastructure. Investigate recent DNS changes and hosted content.", + len(domains), joinNonEmpty(domains[:previewN], ", "), + ), + Fix: "https://botvrij.eu/", + FixIsURL: true, + } +} + +// ---------- cache ---------- + +type botvrijCache struct { + mu sync.Mutex + domains []string // all domains in feed + byDomain map[string]struct{} // set for O(1) exact-match test + fetchedAt time.Time + lastAttemptAt time.Time + refreshing bool + ttl time.Duration + failBackoff time.Duration + feedURL string +} + +func newBotvrijCache(feedURL string) *botvrijCache { + return &botvrijCache{ + ttl: botvrijDefaultTTL, + failBackoff: botvrijFailBackoff, + feedURL: feedURL, + } +} + +func (c *botvrijCache) lookup(ctx context.Context, registered string) (matched []string, size int, fetchedAt time.Time, err error) { + registered = strings.ToLower(strings.TrimSuffix(registered, ".")) + + c.mu.Lock() + stale := c.byDomain == nil || time.Since(c.fetchedAt) > c.ttl + doRefresh := stale && !c.refreshing && time.Since(c.lastAttemptAt) > c.failBackoff + if doRefresh { + c.refreshing = true + } + c.mu.Unlock() + + if doRefresh { + newDomains, newByDomain, ferr := c.fetch(ctx) + c.mu.Lock() + c.refreshing = false + c.lastAttemptAt = time.Now() + if ferr == nil { + c.domains = newDomains + c.byDomain = newByDomain + c.fetchedAt = c.lastAttemptAt + } else { + err = ferr + } + c.mu.Unlock() + } + + c.mu.Lock() + suffix := "." + registered + for d := range c.byDomain { + if d == registered || strings.HasSuffix(d, suffix) { + matched = append(matched, d) + } + } + size = len(c.domains) + fetchedAt = c.fetchedAt + c.mu.Unlock() + return matched, size, fetchedAt, err +} + +func (c *botvrijCache) fetch(ctx context.Context) ([]string, map[string]struct{}, error) { + reqCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, c.feedURL, nil) + if err != nil { + return nil, nil, err + } + req.Header.Set("User-Agent", "happydomain-checker-blacklist/1.0") + + resp, err := sharedHTTPClient.Do(req) + if err != nil { + return nil, nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, nil, fmt.Errorf("botvrij HTTP %d", resp.StatusCode) + } + + domains := make([]string, 0, 4096) + byDomain := make(map[string]struct{}, 4096) + scanner := bufio.NewScanner(io.LimitReader(resp.Body, 16<<20)) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + d := strings.ToLower(line) + domains = append(domains, d) + byDomain[d] = struct{}{} + } + if err := scanner.Err(); err != nil { + return nil, nil, err + } + return domains, byDomain, nil +} diff --git a/checker/botvrij_test.go b/checker/botvrij_test.go new file mode 100644 index 0000000..74366b4 --- /dev/null +++ b/checker/botvrij_test.go @@ -0,0 +1,94 @@ +package checker + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + sdk "git.happydns.org/checker-sdk-go/checker" +) + +const botvrijFakeFeed = `# Botvrij.eu IOC list - domains +# comment line +evil.com +malware.example.org +c2.badactor.net +` + +func TestBotvrijSource_Listed_ExactMatch(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(botvrijFakeFeed)) + })) + defer srv.Close() + + s := &botvrijSource{cache: newBotvrijCache(srv.URL)} + r := s.Query(context.Background(), "evil.com", "evil.com", sdk.CheckerOptions{"enable_botvrij": true})[0] + + if !r.Enabled || r.Error != "" { + t.Fatalf("expected enabled and no error, got %+v", r) + } + if len(r.Evidence) != 1 || r.Evidence[0].Value != "evil.com" { + t.Errorf("expected evidence [evil.com], got %+v", r.Evidence) + } + if listed, sev := s.Evaluate(r); !listed || sev != SeverityCrit { + t.Errorf("expected (true, crit), got (%v, %q)", listed, sev) + } +} + +func TestBotvrijSource_Listed_SubdomainInFeed(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(botvrijFakeFeed)) + })) + defer srv.Close() + + // Feed has "malware.example.org"; querying registered "example.org" should match. + s := &botvrijSource{cache: newBotvrijCache(srv.URL)} + r := s.Query(context.Background(), "sub.example.org", "example.org", sdk.CheckerOptions{"enable_botvrij": true})[0] + + if len(r.Evidence) != 1 || r.Evidence[0].Value != "malware.example.org" { + t.Errorf("expected subdomain match, got %+v", r.Evidence) + } + if listed, _ := s.Evaluate(r); !listed { + t.Error("expected listed=true") + } +} + +func TestBotvrijSource_NotListed(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(botvrijFakeFeed)) + })) + defer srv.Close() + + s := &botvrijSource{cache: newBotvrijCache(srv.URL)} + r := s.Query(context.Background(), "safe.com", "safe.com", sdk.CheckerOptions{"enable_botvrij": true})[0] + + if !r.Enabled || r.Error != "" || len(r.Evidence) != 0 { + t.Fatalf("expected clean result, got %+v", r) + } + if listed, _ := s.Evaluate(r); listed { + t.Error("expected listed=false") + } +} + +func TestBotvrijSource_Disabled(t *testing.T) { + s := &botvrijSource{cache: newBotvrijCache("http://nope")} + r := s.Query(context.Background(), "evil.com", "evil.com", sdk.CheckerOptions{"enable_botvrij": false})[0] + if r.Enabled { + t.Errorf("expected disabled, got %+v", r) + } +} + +func TestBotvrijSource_HTTPError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer srv.Close() + + s := &botvrijSource{cache: newBotvrijCache(srv.URL)} + r := s.Query(context.Background(), "evil.com", "evil.com", sdk.CheckerOptions{"enable_botvrij": true})[0] + + if r.Error == "" || r.Error != "botvrij HTTP 500" { + t.Errorf("expected HTTP 500 error, got %q", r.Error) + } +} From c2cc88e1df99a8c1eb511e8361f923a1228a3b83 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Fri, 15 May 2026 21:15:10 +0800 Subject: [PATCH 09/15] Add Disconnect.me tracking-protection blocklist source Downloads and caches the Disconnect.me services.json feed (24h TTL), matching domains against the Advertising, Analytics, Social, Content, and Disconnect categories. Severity is warn (privacy classification, not malware). Reuses the shared feedCache infrastructure. --- README.md | 1 + checker/disconnect.go | 162 +++++++++++++++++++++++++++++++++++++ checker/disconnect_test.go | 127 +++++++++++++++++++++++++++++ 3 files changed, 290 insertions(+) create mode 100644 checker/disconnect.go create mode 100644 checker/disconnect_test.go diff --git a/README.md b/README.md index 0bf3f1a..f25ba23 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ widely-used reputation systems. | abuse.ch ThreatFox | HTTPS lookup | free Auth-Key (admin) | user (default on) | | abuse.ch MalwareBazaar| HTTPS lookup | free Auth-Key (admin) | user (default on) | | Botvrij.eu | downloaded list | no | user (default on) | +| Disconnect.me | downloaded list | no | user (default on) | | VirusTotal v3 | HTTPS lookup | yes (admin) | admin | ### Obtaining API keys diff --git a/checker/disconnect.go b/checker/disconnect.go new file mode 100644 index 0000000..ae93adf --- /dev/null +++ b/checker/disconnect.go @@ -0,0 +1,162 @@ +package checker + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + sdk "git.happydns.org/checker-sdk-go/checker" +) + +const disconnectFeedURL = "https://s3.amazonaws.com/lists.disconnect.me/services.json" + +func init() { + Register(&disconnectSource{ + cache: newFeedCache(24*time.Hour, disconnectFetch(disconnectFeedURL)), + }) +} + +type disconnectSource struct{ cache *feedCache } + +func (*disconnectSource) ID() string { return "disconnect" } +func (*disconnectSource) Name() string { return "Disconnect.me" } + +func (*disconnectSource) Options() SourceOptions { + return SourceOptions{ + User: []sdk.CheckerOptionField{ + { + Id: "enable_disconnect", + Type: "bool", + Label: "Use the Disconnect.me tracking-protection list", + Description: "Check the domain against the Disconnect.me blocklist used by Firefox Enhanced Tracking Protection, Brave, and uBlock Origin.", + Default: true, + }, + }, + } +} + +func (s *disconnectSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult { + if !sdk.GetBoolOption(opts, "enable_disconnect", true) { + return disabledResult(s.ID(), s.Name()) + } + if registered == "" { + return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: true}} + } + + matches, size, fetched, err := s.cache.lookup(ctx, registered) + res := SourceResult{ + SourceID: s.ID(), SourceName: s.Name(), Enabled: true, + Reference: "https://disconnect.me/trackerprotection", + Details: mustJSON(map[string]any{"feed_size": size, "fetched_at": fetched}), + } + if err != nil { + res.Error = err.Error() + } + + seenCategory := map[string]bool{} + for _, m := range matches { + parts := strings.SplitN(m, "|", 2) + category, company := parts[0], "" + if len(parts) == 2 { + company = parts[1] + } + if !seenCategory[category] { + seenCategory[category] = true + res.Reasons = append(res.Reasons, category) + } + extra := map[string]string{} + if company != "" { + extra["company"] = company + } + res.Evidence = append(res.Evidence, Evidence{ + Label: "Category", + Value: category, + Status: strings.ToLower(category), + Extra: extra, + }) + } + return []SourceResult{res} +} + +func (*disconnectSource) Evaluate(r SourceResult) (bool, string) { + return evidenceEval(r, SeverityWarn) +} + +func (*disconnectSource) Diagnose(res SourceResult) Diagnosis { + return Diagnosis{ + Severity: SeverityWarn, + Title: "Listed in Disconnect.me tracking-protection blocklist", + Detail: fmt.Sprintf( + "Category: %s. This domain appears in the Disconnect.me list used by Firefox Enhanced Tracking Protection, Brave, and uBlock Origin. Browsers and privacy tools will block third-party requests to this domain, which may affect analytics, ad delivery, or embedded widgets. The list is maintained by Disconnect.me; contact them if you believe the classification is incorrect.", + joinNonEmpty(res.Reasons, ", "), + ), + Fix: "https://disconnect.me/contact", + FixIsURL: true, + } +} + +// disconnectJSON mirrors the structure of services.json. +type disconnectJSON struct { + Categories map[string][]map[string]map[string][]string `json:"categories"` +} + +func disconnectFetch(feedURL string) func(context.Context) ([]string, map[string][]string, error) { + return func(ctx context.Context) ([]string, map[string][]string, error) { + reqCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, feedURL, nil) + if err != nil { + return nil, nil, err + } + req.Header.Set("User-Agent", "happydomain-checker-blacklist/1.0") + + resp, err := sharedHTTPClient.Do(req) + if err != nil { + return nil, nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, nil, fmt.Errorf("disconnect HTTP %d", resp.StatusCode) + } + + raw, err := io.ReadAll(io.LimitReader(resp.Body, 32<<20)) + if err != nil { + return nil, nil, fmt.Errorf("disconnect read: %w", err) + } + + var data disconnectJSON + if err := json.Unmarshal(raw, &data); err != nil { + return nil, nil, fmt.Errorf("disconnect parse: %w", err) + } + + byHost := make(map[string][]string, 4096) + for category, entities := range data.Categories { + for _, entity := range entities { + for company, sites := range entity { + for _, domains := range sites { + for _, d := range domains { + d = strings.ToLower(strings.TrimSuffix(d, ".")) + if d == "" { + continue + } + entry := category + "|" + company + byHost[d] = append(byHost[d], entry) + } + } + } + } + } + + urls := make([]string, 0, len(byHost)) + for d := range byHost { + urls = append(urls, d) + } + return urls, byHost, nil + } +} diff --git a/checker/disconnect_test.go b/checker/disconnect_test.go new file mode 100644 index 0000000..7a5e325 --- /dev/null +++ b/checker/disconnect_test.go @@ -0,0 +1,127 @@ +package checker + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" + + sdk "git.happydns.org/checker-sdk-go/checker" +) + +const disconnectFakeFeed = `{ + "categories": { + "Advertising": [ + { + "Evil Corp": { + "https://evilcorp.com": ["tracker.com", "sub.example.org"] + } + } + ], + "Analytics": [ + { + "Metrics Inc": { + "https://metrics.io": ["analytics.net"] + } + } + ] + } +}` + +func newDisconnectTestSource(srv *httptest.Server) *disconnectSource { + return &disconnectSource{ + cache: newFeedCache(time.Hour, disconnectFetch(srv.URL)), + } +} + +func TestDisconnectSource_Listed(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(disconnectFakeFeed)) + })) + defer srv.Close() + + s := newDisconnectTestSource(srv) + r := s.Query(context.Background(), "tracker.com", "tracker.com", sdk.CheckerOptions{"enable_disconnect": true})[0] + + if !r.Enabled || r.Error != "" { + t.Fatalf("expected enabled with no error, got %+v", r) + } + if len(r.Evidence) == 0 { + t.Fatalf("expected evidence, got none") + } + found := false + for _, e := range r.Evidence { + if e.Value == "Advertising" { + found = true + if e.Extra["company"] != "Evil Corp" { + t.Errorf("expected company 'Evil Corp', got %q", e.Extra["company"]) + } + } + } + if !found { + t.Errorf("expected Advertising evidence, got %+v", r.Evidence) + } + if listed, sev := s.Evaluate(r); !listed || sev != SeverityWarn { + t.Errorf("expected (true, warn), got (%v, %q)", listed, sev) + } +} + +func TestDisconnectSource_SubdomainInFeed(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(disconnectFakeFeed)) + })) + defer srv.Close() + + // Feed has "sub.example.org"; querying registered domain "example.org" should match. + s := newDisconnectTestSource(srv) + r := s.Query(context.Background(), "example.org", "example.org", sdk.CheckerOptions{"enable_disconnect": true})[0] + + if !r.Enabled || r.Error != "" { + t.Fatalf("expected enabled with no error, got %+v", r) + } + if len(r.Evidence) == 0 { + t.Errorf("expected subdomain match evidence, got none") + } +} + +func TestDisconnectSource_NotListed(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(disconnectFakeFeed)) + })) + defer srv.Close() + + s := newDisconnectTestSource(srv) + r := s.Query(context.Background(), "clean.example.com", "clean.example.com", sdk.CheckerOptions{"enable_disconnect": true})[0] + + if !r.Enabled || r.Error != "" { + t.Fatalf("expected enabled with no error, got %+v", r) + } + if len(r.Evidence) != 0 { + t.Errorf("expected no evidence for clean domain, got %+v", r.Evidence) + } + if listed, _ := s.Evaluate(r); listed { + t.Errorf("expected not listed for clean domain") + } +} + +func TestDisconnectSource_Disabled(t *testing.T) { + s := &disconnectSource{cache: newFeedCache(time.Hour, disconnectFetch("http://nope"))} + r := s.Query(context.Background(), "tracker.com", "tracker.com", sdk.CheckerOptions{"enable_disconnect": false})[0] + if r.Enabled { + t.Errorf("expected disabled result, got %+v", r) + } +} + +func TestDisconnectSource_HTTPError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer srv.Close() + + s := newDisconnectTestSource(srv) + r := s.Query(context.Background(), "tracker.com", "tracker.com", sdk.CheckerOptions{"enable_disconnect": true})[0] + if r.Error == "" { + t.Errorf("expected error on HTTP 500, got empty error") + } +} From 1242a381ab6241117188ed0eb33af8ab5a1017b5 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Fri, 15 May 2026 21:26:45 +0800 Subject: [PATCH 10/15] Add OISD domain blocklist source Implements the OISD domainswild feed (big and small variants) as a new blacklist source. DNS0.eu was considered but shut down in October 2025. --- README.md | 1 + checker/oisd.go | 212 +++++++++++++++++++++++++++++++++++++++++++ checker/oisd_test.go | 95 +++++++++++++++++++ 3 files changed, 308 insertions(+) create mode 100644 checker/oisd.go create mode 100644 checker/oisd_test.go diff --git a/README.md b/README.md index f25ba23..e630ad1 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ widely-used reputation systems. | abuse.ch MalwareBazaar| HTTPS lookup | free Auth-Key (admin) | user (default on) | | Botvrij.eu | downloaded list | no | user (default on) | | Disconnect.me | downloaded list | no | user (default on) | +| OISD | downloaded list | no | user (default on) | | VirusTotal v3 | HTTPS lookup | yes (admin) | admin | ### Obtaining API keys diff --git a/checker/oisd.go b/checker/oisd.go new file mode 100644 index 0000000..39773a1 --- /dev/null +++ b/checker/oisd.go @@ -0,0 +1,212 @@ +package checker + +import ( + "bufio" + "context" + "fmt" + "io" + "net/http" + "strings" + "sync" + "time" + + sdk "git.happydns.org/checker-sdk-go/checker" +) + +const ( + oisdBigFeedURL = "https://big.oisd.nl/domainswild" + oisdSmallFeedURL = "https://small.oisd.nl/domainswild" + oisdDefaultTTL = 24 * time.Hour + oisdFailBackoff = 1 * time.Minute +) + +func init() { + Register(&oisdSource{ + bigCache: newOisdCache(oisdBigFeedURL), + smallCache: newOisdCache(oisdSmallFeedURL), + }) +} + +type oisdSource struct { + bigCache *oisdCache + smallCache *oisdCache +} + +func (*oisdSource) ID() string { return "oisd" } +func (*oisdSource) Name() string { return "OISD domain blocklist" } + +func (*oisdSource) Options() SourceOptions { + return SourceOptions{ + User: []sdk.CheckerOptionField{ + { + Id: "enable_oisd", + Type: "bool", + Label: "Use the OISD domain blocklist", + Description: "Download the OISD domain blocklist (refreshed every 24h) and check the domain against it.", + Default: true, + }, + }, + Admin: []sdk.CheckerOptionField{ + { + Id: "oisd_variant", + Type: "string", + Label: "OISD blocklist variant", + Description: `Which OISD list to use: "big" (~250 k entries, recommended) or "small" (~50 k entries).`, + Default: "big", + }, + }, + } +} + +func (s *oisdSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult { + if !sdk.GetBoolOption(opts, "enable_oisd", true) || registered == "" { + return disabledResult(s.ID(), s.Name()) + } + + cache := s.bigCache + if stringOptDefault(opts, "oisd_variant", "big") == "small" { + cache = s.smallCache + } + + matched, size, fetched, err := cache.lookup(ctx, registered) + res := SourceResult{ + SourceID: s.ID(), SourceName: s.Name(), Enabled: true, + Reference: "https://oisd.nl/", + Details: mustJSON(map[string]any{"feed_size": size, "fetched_at": fetched}), + } + if err != nil { + res.Error = err.Error() + } + if len(matched) > 0 { + res.Reasons = []string{"Listed in OISD"} + for _, d := range matched { + res.Evidence = append(res.Evidence, Evidence{Label: "Domain", Value: d}) + } + } + return []SourceResult{res} +} + +func (*oisdSource) Evaluate(r SourceResult) (bool, string) { + return evidenceEval(r, SeverityCrit) +} + +func (*oisdSource) Diagnose(res SourceResult) Diagnosis { + domains := make([]string, 0, len(res.Evidence)) + for _, e := range res.Evidence { + domains = append(domains, e.Value) + } + previewN := min(len(domains), 5) + return Diagnosis{ + Severity: SeverityCrit, + Title: "Listed in the OISD domain blocklist", + Detail: fmt.Sprintf( + "%d domain(s) matching this registered domain appear in the OISD blocklist; examples: %s. OISD is a large curated blocklist used by DNS resolvers, ad blockers, and firewalls worldwide. Domains listed here are blocked for millions of users. Investigate whether the domain hosts ads, trackers, or malware, or whether it has been compromised.", + len(domains), joinNonEmpty(domains[:previewN], ", "), + ), + Fix: "https://oisd.nl/", + FixIsURL: true, + } +} + +// ---------- cache ---------- + +type oisdCache struct { + mu sync.Mutex + domains []string + byDomain map[string]struct{} + fetchedAt time.Time + lastAttemptAt time.Time + refreshing bool + ttl time.Duration + failBackoff time.Duration + feedURL string +} + +func newOisdCache(feedURL string) *oisdCache { + return &oisdCache{ + ttl: oisdDefaultTTL, + failBackoff: oisdFailBackoff, + feedURL: feedURL, + } +} + +func (c *oisdCache) lookup(ctx context.Context, registered string) (matched []string, size int, fetchedAt time.Time, err error) { + registered = strings.ToLower(strings.TrimSuffix(registered, ".")) + + c.mu.Lock() + stale := c.byDomain == nil || time.Since(c.fetchedAt) > c.ttl + doRefresh := stale && !c.refreshing && time.Since(c.lastAttemptAt) > c.failBackoff + if doRefresh { + c.refreshing = true + } + feedURL := c.feedURL + c.mu.Unlock() + + if doRefresh { + newDomains, newByDomain, ferr := c.fetch(ctx, feedURL) + c.mu.Lock() + c.refreshing = false + c.lastAttemptAt = time.Now() + if ferr == nil { + c.domains = newDomains + c.byDomain = newByDomain + c.fetchedAt = c.lastAttemptAt + } else { + err = ferr + } + c.mu.Unlock() + } + + c.mu.Lock() + suffix := "." + registered + for d := range c.byDomain { + if d == registered || strings.HasSuffix(d, suffix) { + matched = append(matched, d) + } + } + size = len(c.domains) + fetchedAt = c.fetchedAt + c.mu.Unlock() + return matched, size, fetchedAt, err +} + +func (c *oisdCache) fetch(ctx context.Context, feedURL string) ([]string, map[string]struct{}, error) { + reqCtx, cancel := context.WithTimeout(ctx, 60*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, feedURL, nil) + if err != nil { + return nil, nil, err + } + req.Header.Set("User-Agent", "happydomain-checker-blacklist/1.0") + + resp, err := sharedHTTPClient.Do(req) + if err != nil { + return nil, nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, nil, fmt.Errorf("oisd HTTP %d", resp.StatusCode) + } + + domains := make([]string, 0, 262144) + byDomain := make(map[string]struct{}, 262144) + scanner := bufio.NewScanner(io.LimitReader(resp.Body, 64<<20)) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "!") || strings.HasPrefix(line, "#") { + continue + } + d := strings.TrimPrefix(strings.ToLower(line), "*.") + if d == "" { + continue + } + domains = append(domains, d) + byDomain[d] = struct{}{} + } + if err := scanner.Err(); err != nil { + return nil, nil, err + } + return domains, byDomain, nil +} diff --git a/checker/oisd_test.go b/checker/oisd_test.go new file mode 100644 index 0000000..93655b2 --- /dev/null +++ b/checker/oisd_test.go @@ -0,0 +1,95 @@ +package checker + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + sdk "git.happydns.org/checker-sdk-go/checker" +) + +const oisdFakeFeed = `! OISD big domainswild +# comment line +*.evil.com +*.malware.example.org +*.c2.badactor.net +` + +func TestOisdSource_Listed_ExactMatch(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(oisdFakeFeed)) + })) + defer srv.Close() + + s := &oisdSource{bigCache: newOisdCache(srv.URL)} + r := s.Query(context.Background(), "evil.com", "evil.com", sdk.CheckerOptions{"enable_oisd": true})[0] + + if !r.Enabled || r.Error != "" { + t.Fatalf("expected enabled and no error, got %+v", r) + } + if len(r.Evidence) != 1 || r.Evidence[0].Value != "evil.com" { + t.Errorf("expected evidence [evil.com], got %+v", r.Evidence) + } + if listed, sev := s.Evaluate(r); !listed || sev != SeverityCrit { + t.Errorf("expected (true, crit), got (%v, %q)", listed, sev) + } +} + +func TestOisdSource_Listed_SubdomainInFeed(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(oisdFakeFeed)) + })) + defer srv.Close() + + // Feed has "*.malware.example.org" → stored as "malware.example.org"; querying + // registered "example.org" should match via suffix check. + s := &oisdSource{bigCache: newOisdCache(srv.URL)} + r := s.Query(context.Background(), "sub.example.org", "example.org", sdk.CheckerOptions{"enable_oisd": true})[0] + + if len(r.Evidence) != 1 || r.Evidence[0].Value != "malware.example.org" { + t.Errorf("expected subdomain match, got %+v", r.Evidence) + } + if listed, _ := s.Evaluate(r); !listed { + t.Error("expected listed=true") + } +} + +func TestOisdSource_NotListed(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(oisdFakeFeed)) + })) + defer srv.Close() + + s := &oisdSource{bigCache: newOisdCache(srv.URL)} + r := s.Query(context.Background(), "safe.com", "safe.com", sdk.CheckerOptions{"enable_oisd": true})[0] + + if !r.Enabled || r.Error != "" || len(r.Evidence) != 0 { + t.Fatalf("expected clean result, got %+v", r) + } + if listed, _ := s.Evaluate(r); listed { + t.Error("expected listed=false") + } +} + +func TestOisdSource_Disabled(t *testing.T) { + s := &oisdSource{bigCache: newOisdCache("http://nope")} + r := s.Query(context.Background(), "evil.com", "evil.com", sdk.CheckerOptions{"enable_oisd": false})[0] + if r.Enabled { + t.Errorf("expected disabled, got %+v", r) + } +} + +func TestOisdSource_HTTPError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer srv.Close() + + s := &oisdSource{bigCache: newOisdCache(srv.URL)} + r := s.Query(context.Background(), "evil.com", "evil.com", sdk.CheckerOptions{"enable_oisd": true})[0] + + if r.Error == "" || r.Error != "oisd HTTP 500" { + t.Errorf("expected HTTP 500 error, got %q", r.Error) + } +} From faae2f80c52f7cf5330e70b223aa9bbe7c33454b Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Fri, 15 May 2026 21:32:00 +0800 Subject: [PATCH 11/15] Add AlienVault OTX domain threat intelligence source --- README.md | 6 ++ checker/otx.go | 208 ++++++++++++++++++++++++++++++++++++++++++++ checker/otx_test.go | 144 ++++++++++++++++++++++++++++++ 3 files changed, 358 insertions(+) create mode 100644 checker/otx.go create mode 100644 checker/otx_test.go diff --git a/README.md b/README.md index e630ad1..9cf3eb3 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ widely-used reputation systems. | Disconnect.me | downloaded list | no | user (default on) | | OISD | downloaded list | no | user (default on) | | VirusTotal v3 | HTTPS lookup | yes (admin) | admin | +| AlienVault OTX | HTTPS lookup | free (admin) | admin | ### Obtaining API keys @@ -42,6 +43,11 @@ widely-used reputation systems. 3. Free tier: 4 requests/minute, 500 requests/day. No billing required. 4. The public API key is sufficient; premium keys unlock higher quotas. +**AlienVault OTX** (option: `otx_api_key`) +1. Register a free account at [otx.alienvault.com](https://otx.alienvault.com/). +2. Go to *Settings → API Integration* to find your personal OTX key. +3. Free, no documented rate limits for the indicator lookup API. + DNS-based blocklists are queried in parallel. The OpenPhish feed is downloaded once per hour by the provider and cached in memory. diff --git a/checker/otx.go b/checker/otx.go new file mode 100644 index 0000000..d4e39cf --- /dev/null +++ b/checker/otx.go @@ -0,0 +1,208 @@ +package checker + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "html/template" + "net/http" + "time" + + sdk "git.happydns.org/checker-sdk-go/checker" +) + +const otxEndpoint = "https://otx.alienvault.com/api/v1/indicators/domain/" + +func init() { Register(&otxSource{endpoint: otxEndpoint}) } + +type otxSource struct{ endpoint string } + +func (*otxSource) ID() string { return "otx" } +func (*otxSource) Name() string { return "AlienVault OTX" } + +func (*otxSource) Options() SourceOptions { + return SourceOptions{ + Admin: []sdk.CheckerOptionField{ + { + Id: "otx_api_key", + Type: "string", + Label: "AlienVault OTX API key", + Description: "Free OTX API key from otx.alienvault.com. Leave empty to skip OTX lookups.", + Secret: true, + }, + }, + } +} + +type otxDetails struct { + PulseCount int `json:"pulse_count"` + Reputation int `json:"reputation"` + Pulses []otxPulse `json:"pulses"` +} + +type otxPulse struct { + Name string `json:"name"` + Tags []string `json:"tags,omitempty"` + MalwareFamilies []string `json:"malware_families,omitempty"` + Adversary string `json:"adversary,omitempty"` + Created string `json:"created,omitempty"` +} + +func (s *otxSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult { + apiKey := stringOpt(opts, "otx_api_key") + if apiKey == "" { + return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: false}} + } + if registered == "" { + return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: true}} + } + + res := SourceResult{ + SourceID: s.ID(), + SourceName: s.Name(), + Enabled: true, + Reference: "https://otx.alienvault.com/indicator/domain/" + registered, + } + + reqCtx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, s.endpoint+registered+"/general", nil) + if err != nil { + res.Error = err.Error() + return []SourceResult{res} + } + req.Header.Set("X-OTX-API-KEY", apiKey) + req.Header.Set("Accept", "application/json") + + body, status, err := httpDo(req, 4<<20) + if err != nil { + res.Error = err.Error() + return []SourceResult{res} + } + if status == http.StatusNotFound { + return []SourceResult{res} + } + if status != http.StatusOK { + res.Error = fmt.Sprintf("HTTP %d: %s", status, truncate(string(body), 200)) + return []SourceResult{res} + } + + var parsed struct { + Reputation int `json:"reputation"` + PulseInfo struct { + Count int `json:"count"` + Pulses []struct { + Name string `json:"name"` + Tags []string `json:"tags"` + MalwareFamilies []struct { + DisplayName string `json:"display_name"` + } `json:"malware_families"` + Adversary string `json:"adversary"` + Created string `json:"created"` + } `json:"pulses"` + } `json:"pulse_info"` + } + if err := json.Unmarshal(body, &parsed); err != nil { + res.Error = "decode: " + err.Error() + return []SourceResult{res} + } + + d := otxDetails{ + PulseCount: parsed.PulseInfo.Count, + Reputation: parsed.Reputation, + } + + seenReason := map[string]bool{} + for _, p := range parsed.PulseInfo.Pulses { + pulse := otxPulse{ + Name: p.Name, + Tags: p.Tags, + Adversary: p.Adversary, + Created: p.Created, + } + for _, mf := range p.MalwareFamilies { + pulse.MalwareFamilies = append(pulse.MalwareFamilies, mf.DisplayName) + if !seenReason[mf.DisplayName] { + seenReason[mf.DisplayName] = true + res.Reasons = append(res.Reasons, mf.DisplayName) + } + } + if p.Adversary != "" && !seenReason[p.Adversary] { + seenReason[p.Adversary] = true + res.Reasons = append(res.Reasons, p.Adversary) + } + d.Pulses = append(d.Pulses, pulse) + res.Evidence = append(res.Evidence, Evidence{ + Label: "Pulse", + Value: p.Name, + Status: "threat", + }) + } + res.Details = mustJSON(d) + return []SourceResult{res} +} + +func (*otxSource) Evaluate(r SourceResult) (bool, string) { + if !r.Enabled || r.Error != "" || len(r.Evidence) == 0 { + return false, "" + } + var d otxDetails + _ = json.Unmarshal(r.Details, &d) + if d.Reputation < -1 { + return true, SeverityCrit + } + return true, SeverityWarn +} + +func (*otxSource) Diagnose(res SourceResult) Diagnosis { + var d otxDetails + _ = json.Unmarshal(res.Details, &d) + detail := fmt.Sprintf( + "%d threat pulse(s) reference this domain (OTX reputation score: %d). Indicators: %s. "+ + "Review the pulse details on AlienVault OTX to understand the threat context and take corrective action.", + d.PulseCount, d.Reputation, joinNonEmpty(res.Reasons, ", "), + ) + sev := SeverityWarn + if d.Reputation < -1 { + sev = SeverityCrit + } + return Diagnosis{ + Severity: sev, + Title: "Listed in AlienVault OTX threat pulses", + Detail: detail, + Fix: res.Reference, + FixIsURL: res.Reference != "", + } +} + +func (*otxSource) RenderDetail(res SourceResult) (template.HTML, error) { + var d otxDetails + if len(res.Details) > 0 { + if err := json.Unmarshal(res.Details, &d); err != nil { + return "", fmt.Errorf("otx: decode details: %w", err) + } + } + if len(d.Pulses) == 0 { + return "", nil + } + var b bytes.Buffer + if err := otxDetailTpl.Execute(&b, d); err != nil { + return "", err + } + return template.HTML(b.String()), nil +} + +var otxDetailTpl = template.Must(template.New("otx_detail").Parse(` +

OTX reputation score: {{.Reputation}}. Pulse count: {{.PulseCount}}.

+ + +{{range .Pulses}} + + + + + +{{end}} +
PulseMalware familiesAdversaryTagsCreated
{{.Name}}{{range .MalwareFamilies}}{{.}} {{end}}{{.Adversary}}{{range .Tags}}{{.}} {{end}}{{.Created}}
`)) diff --git a/checker/otx_test.go b/checker/otx_test.go new file mode 100644 index 0000000..2da5846 --- /dev/null +++ b/checker/otx_test.go @@ -0,0 +1,144 @@ +package checker + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + sdk "git.happydns.org/checker-sdk-go/checker" +) + +func newOTXServer(t *testing.T, status int, body string) (string, func()) { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("X-OTX-API-KEY") == "" { + t.Errorf("missing X-OTX-API-KEY header") + } + w.WriteHeader(status) + _, _ = w.Write([]byte(body)) + })) + return srv.URL + "/", srv.Close +} + +func TestOTXSource_NoKey(t *testing.T) { + s := &otxSource{endpoint: otxEndpoint} + r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{})[0] + if r.Enabled { + t.Errorf("expected disabled without API key, got %+v", r) + } +} + +func TestOTXSource_Listed(t *testing.T) { + body := `{ + "reputation": 0, + "pulse_info": { + "count": 1, + "pulses": [{ + "name": "Test Pulse", + "tags": ["phishing"], + "malware_families": [{"display_name": "Emotet"}], + "adversary": "", + "created": "2024-01-01T00:00:00.000Z" + }] + } + }` + endpoint, stop := newOTXServer(t, http.StatusOK, body) + defer stop() + + s := &otxSource{endpoint: endpoint} + r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"otx_api_key": "k"})[0] + + if len(r.Evidence) != 1 { + t.Fatalf("expected 1 evidence entry, got %d", len(r.Evidence)) + } + if listed, severity := s.Evaluate(r); !listed || severity != SeverityWarn { + t.Errorf("expected Evaluate()=(true, warn), got (%v, %q)", listed, severity) + } + + var d otxDetails + if err := json.Unmarshal(r.Details, &d); err != nil { + t.Fatalf("details decode: %v", err) + } + if d.PulseCount != 1 || d.Reputation != 0 { + t.Errorf("details wrong: %+v", d) + } + if len(d.Pulses) != 1 || len(d.Pulses[0].MalwareFamilies) != 1 || d.Pulses[0].MalwareFamilies[0] != "Emotet" { + t.Errorf("pulse details wrong: %+v", d.Pulses) + } + + html, err := s.RenderDetail(r) + if err != nil || !strings.Contains(string(html), "Emotet") { + t.Errorf("RenderDetail html=%q err=%v", html, err) + } +} + +func TestOTXSource_ListedCrit(t *testing.T) { + body := `{ + "reputation": -2, + "pulse_info": { + "count": 5, + "pulses": [{"name": "APT Pulse", "tags": [], "malware_families": [], "adversary": "APT28", "created": ""}] + } + }` + endpoint, stop := newOTXServer(t, http.StatusOK, body) + defer stop() + + s := &otxSource{endpoint: endpoint} + r := s.Query(context.Background(), "evil.com", "evil.com", sdk.CheckerOptions{"otx_api_key": "k"})[0] + + if listed, severity := s.Evaluate(r); !listed || severity != SeverityCrit { + t.Errorf("expected Evaluate()=(true, crit) for reputation -2, got (%v, %q)", listed, severity) + } +} + +func TestOTXSource_NotFound(t *testing.T) { + endpoint, stop := newOTXServer(t, http.StatusNotFound, `{"detail":"Not found"}`) + defer stop() + + s := &otxSource{endpoint: endpoint} + r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"otx_api_key": "k"})[0] + + if r.Error != "" { + t.Errorf("404 should be quiet not-listed, got Error=%q", r.Error) + } + if len(r.Evidence) != 0 { + t.Errorf("expected no evidence for 404, got %+v", r.Evidence) + } + if listed, _ := s.Evaluate(r); listed { + t.Errorf("Evaluate() on clean result should return false") + } + if !strings.Contains(r.Reference, "example.com") { + t.Errorf("reference URL missing domain: %+v", r) + } +} + +func TestOTXSource_HTTPError(t *testing.T) { + endpoint, stop := newOTXServer(t, http.StatusInternalServerError, `{"error":"internal"}`) + defer stop() + + s := &otxSource{endpoint: endpoint} + r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"otx_api_key": "k"})[0] + + if r.Error == "" { + t.Errorf("expected non-empty Error for HTTP 500, got %+v", r) + } +} + +func TestOTXSource_NoResults(t *testing.T) { + body := `{"reputation": 0, "pulse_info": {"count": 0, "pulses": []}}` + endpoint, stop := newOTXServer(t, http.StatusOK, body) + defer stop() + + s := &otxSource{endpoint: endpoint} + r := s.Query(context.Background(), "clean.com", "clean.com", sdk.CheckerOptions{"otx_api_key": "k"})[0] + + if len(r.Evidence) != 0 { + t.Errorf("expected no evidence for clean domain, got %+v", r.Evidence) + } + if listed, severity := s.Evaluate(r); listed || severity != "" { + t.Errorf("Evaluate() on clean domain = (%v, %q), want (false, \"\")", listed, severity) + } +} From c8bcac5a72909c035f7ec2fb5daeee4779946e8d Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Fri, 15 May 2026 21:39:43 +0800 Subject: [PATCH 12/15] Add Pulsedive domain threat intelligence source --- README.md | 6 ++ checker/pulsedive.go | 163 ++++++++++++++++++++++++++++++++++++++ checker/pulsedive_test.go | 103 ++++++++++++++++++++++++ checker/source_test.go | 2 +- 4 files changed, 273 insertions(+), 1 deletion(-) create mode 100644 checker/pulsedive.go create mode 100644 checker/pulsedive_test.go diff --git a/README.md b/README.md index 9cf3eb3..3c3b41f 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ widely-used reputation systems. | OISD | downloaded list | no | user (default on) | | VirusTotal v3 | HTTPS lookup | yes (admin) | admin | | AlienVault OTX | HTTPS lookup | free (admin) | admin | +| Pulsedive | HTTPS lookup | free (admin) | admin | ### Obtaining API keys @@ -48,6 +49,11 @@ widely-used reputation systems. 2. Go to *Settings → API Integration* to find your personal OTX key. 3. Free, no documented rate limits for the indicator lookup API. +**Pulsedive** (option: `pulsedive_api_key`) +1. Register a free account at [pulsedive.com](https://pulsedive.com/). +2. Go to your profile and copy the API key shown under *API*. +3. Free tier available; higher quotas with a paid plan. + DNS-based blocklists are queried in parallel. The OpenPhish feed is downloaded once per hour by the provider and cached in memory. diff --git a/checker/pulsedive.go b/checker/pulsedive.go new file mode 100644 index 0000000..edfeb1d --- /dev/null +++ b/checker/pulsedive.go @@ -0,0 +1,163 @@ +package checker + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "time" + + sdk "git.happydns.org/checker-sdk-go/checker" +) + +const pulsediveEndpoint = "https://pulsedive.com/api/info.php" + +func init() { Register(&pulsediveSource{endpoint: pulsediveEndpoint}) } + +type pulsediveSource struct{ endpoint string } + +func (*pulsediveSource) ID() string { return "pulsedive" } +func (*pulsediveSource) Name() string { return "Pulsedive" } + +func (*pulsediveSource) Options() SourceOptions { + return SourceOptions{ + Admin: []sdk.CheckerOptionField{ + { + Id: "pulsedive_api_key", + Type: "string", + Label: "Pulsedive API key", + Description: "Pulsedive API key (free account at pulsedive.com). Leave empty to skip Pulsedive lookups.", + Secret: true, + }, + }, + } +} + +type pulsediveDetails struct { + Risk string `json:"risk"` + Threats []pulsediveThreat `json:"threats"` +} + +type pulsediveThreat struct { + Name string `json:"name"` + Category string `json:"category"` + Risk string `json:"risk,omitempty"` +} + +func (s *pulsediveSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult { + apiKey := stringOpt(opts, "pulsedive_api_key") + if apiKey == "" || registered == "" { + return disabledResult(s.ID(), s.Name()) + } + + res := SourceResult{ + SourceID: s.ID(), + SourceName: s.Name(), + Enabled: true, + Reference: "https://pulsedive.com/indicator/" + registered, + } + + params := url.Values{ + "indicator": {registered}, + "key": {apiKey}, + } + reqURL := s.endpoint + "?" + params.Encode() + + reqCtx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, reqURL, nil) + if err != nil { + res.Error = redactSecret(err.Error(), apiKey) + return []SourceResult{res} + } + req.Header.Set("User-Agent", "happydomain-checker-blacklist/1.0") + + body, status, err := httpDo(req, 1<<20) + if err != nil { + res.Error = redactSecret(err.Error(), apiKey) + return []SourceResult{res} + } + if status != http.StatusOK { + res.Error = fmt.Sprintf("HTTP %d: %s", status, truncate(string(body), 200)) + return []SourceResult{res} + } + + // Check for "not found" before full parse — Pulsedive returns 200 + // with {"error": "Indicator not found."} for unknown indicators. + var errEnvelope struct { + Error string `json:"error"` + } + if json.Unmarshal(body, &errEnvelope) == nil && errEnvelope.Error != "" { + return []SourceResult{res} + } + + var parsed struct { + Risk string `json:"risk"` + Threats []struct { + Name string `json:"name"` + Category string `json:"category"` + Risk string `json:"risk"` + } `json:"threats"` + } + if err := json.Unmarshal(body, &parsed); err != nil { + res.Error = "decode: " + err.Error() + return []SourceResult{res} + } + + d := pulsediveDetails{Risk: parsed.Risk} + seen := map[string]bool{} + for _, t := range parsed.Threats { + d.Threats = append(d.Threats, pulsediveThreat{ + Name: t.Name, + Category: t.Category, + Risk: t.Risk, + }) + if !seen[t.Name] { + seen[t.Name] = true + res.Reasons = append(res.Reasons, t.Name) + } + res.Evidence = append(res.Evidence, Evidence{ + Label: "Threat", + Value: t.Name, + Status: t.Category, + }) + } + res.Details = mustJSON(d) + return []SourceResult{res} +} + +func (*pulsediveSource) Evaluate(r SourceResult) (bool, string) { + if !r.Enabled || r.Error != "" || len(r.Evidence) == 0 { + return false, "" + } + var d pulsediveDetails + _ = json.Unmarshal(r.Details, &d) + switch d.Risk { + case "critical", "high": + return true, SeverityCrit + default: + return true, SeverityWarn + } +} + +func (*pulsediveSource) Diagnose(res SourceResult) Diagnosis { + var d pulsediveDetails + _ = json.Unmarshal(res.Details, &d) + previewN := min(len(d.Threats), 5) + names := make([]string, 0, previewN) + for _, t := range d.Threats[:previewN] { + names = append(names, t.Name) + } + return Diagnosis{ + Severity: SeverityCrit, + Title: fmt.Sprintf("Pulsedive risk: %s — %d threat(s) associated", d.Risk, len(d.Threats)), + Detail: fmt.Sprintf( + "Pulsedive assigned a risk of %q to this domain. Associated threat(s): %s. Review the indicator page for feed context, related IPs, and historical activity, then follow up with the relevant threat's removal or remediation procedure.", + d.Risk, joinNonEmpty(names, ", "), + ), + Fix: res.Reference, + FixIsURL: res.Reference != "", + } +} diff --git a/checker/pulsedive_test.go b/checker/pulsedive_test.go new file mode 100644 index 0000000..9018b49 --- /dev/null +++ b/checker/pulsedive_test.go @@ -0,0 +1,103 @@ +package checker + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + sdk "git.happydns.org/checker-sdk-go/checker" +) + +func newPulsediveServer(t *testing.T, status int, body string) (string, func()) { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("key") == "" { + t.Errorf("missing key query parameter") + } + w.WriteHeader(status) + _, _ = w.Write([]byte(body)) + })) + return srv.URL + "/info.php", srv.Close +} + +func TestPulsediveSource_NoKey(t *testing.T) { + s := &pulsediveSource{endpoint: pulsediveEndpoint} + r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{})[0] + if r.Enabled { + t.Errorf("expected disabled without API key, got %+v", r) + } +} + +func TestPulsediveSource_Listed_High(t *testing.T) { + body := `{"risk":"high","threats":[{"name":"Emotet","category":"malware","risk":"high"}]}` + endpoint, stop := newPulsediveServer(t, http.StatusOK, body) + defer stop() + + s := &pulsediveSource{endpoint: endpoint} + r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"pulsedive_api_key": "k"})[0] + if r.Error != "" { + t.Fatalf("unexpected error: %s", r.Error) + } + if len(r.Evidence) != 1 { + t.Fatalf("expected 1 evidence, got %d", len(r.Evidence)) + } + if listed, severity := s.Evaluate(r); !listed || severity != SeverityCrit { + t.Errorf("expected (true, crit), got (%v, %q)", listed, severity) + } +} + +func TestPulsediveSource_Listed_Medium(t *testing.T) { + body := `{"risk":"medium","threats":[{"name":"SomeSpam","category":"spam","risk":"medium"}]}` + endpoint, stop := newPulsediveServer(t, http.StatusOK, body) + defer stop() + + s := &pulsediveSource{endpoint: endpoint} + r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"pulsedive_api_key": "k"})[0] + if listed, severity := s.Evaluate(r); !listed || severity != SeverityWarn { + t.Errorf("expected (true, warn), got (%v, %q)", listed, severity) + } +} + +func TestPulsediveSource_NotFound(t *testing.T) { + endpoint, stop := newPulsediveServer(t, http.StatusOK, `{"error":"Indicator not found."}`) + defer stop() + + s := &pulsediveSource{endpoint: endpoint} + r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"pulsedive_api_key": "k"})[0] + if r.Error != "" { + t.Errorf("not-found should be quiet, got error: %s", r.Error) + } + if len(r.Evidence) != 0 { + t.Errorf("expected no evidence, got %d", len(r.Evidence)) + } + if listed, _ := s.Evaluate(r); listed { + t.Errorf("expected not listed for not-found domain") + } +} + +func TestPulsediveSource_Clean(t *testing.T) { + body := `{"risk":"none","threats":[]}` + endpoint, stop := newPulsediveServer(t, http.StatusOK, body) + defer stop() + + s := &pulsediveSource{endpoint: endpoint} + r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"pulsedive_api_key": "k"})[0] + if len(r.Evidence) != 0 { + t.Errorf("expected no evidence for clean domain, got %d", len(r.Evidence)) + } + if listed, _ := s.Evaluate(r); listed { + t.Errorf("expected not listed for clean domain") + } +} + +func TestPulsediveSource_HTTPError(t *testing.T) { + endpoint, stop := newPulsediveServer(t, http.StatusInternalServerError, `internal error`) + defer stop() + + s := &pulsediveSource{endpoint: endpoint} + r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"pulsedive_api_key": "k"})[0] + if r.Error == "" { + t.Errorf("expected error on HTTP 500, got clean result") + } +} diff --git a/checker/source_test.go b/checker/source_test.go index 2bd1cd6..8f432b8 100644 --- a/checker/source_test.go +++ b/checker/source_test.go @@ -24,7 +24,7 @@ func TestRegisteredSourcesAreSane(t *testing.T) { } } // At least the built-in sources are present. - for _, want := range []string{"dnsbl", "google_safe_browsing", "openphish", "urlhaus", "virustotal"} { + for _, want := range []string{"dnsbl", "google_safe_browsing", "openphish", "pulsedive", "urlhaus", "virustotal"} { if !seen[want] { t.Errorf("missing built-in source %q", want) } From a5559ad98ffdfbebd2cfacff91702d51a7a6ad39 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Fri, 15 May 2026 21:56:25 +0800 Subject: [PATCH 13/15] Add Criminal IP domain reputation source Implements the Criminal IP API (api.criminalip.io/v1/domain/report) as a new blacklist source. Returns crit for High/Critical inbound or outbound risk scores, warn for Moderate; Safe and Low scores are not flagged. --- README.md | 6 ++ checker/criminalip.go | 189 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 195 insertions(+) create mode 100644 checker/criminalip.go diff --git a/README.md b/README.md index 3c3b41f..fd6c7dc 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ widely-used reputation systems. | VirusTotal v3 | HTTPS lookup | yes (admin) | admin | | AlienVault OTX | HTTPS lookup | free (admin) | admin | | Pulsedive | HTTPS lookup | free (admin) | admin | +| Criminal IP | HTTPS lookup | yes (admin) | admin | ### Obtaining API keys @@ -54,6 +55,11 @@ widely-used reputation systems. 2. Go to your profile and copy the API key shown under *API*. 3. Free tier available; higher quotas with a paid plan. +**Criminal IP** (option: `criminal_ip_api_key`) +1. Register a free account at [criminalip.io](https://www.criminalip.io/). +2. Go to *My Information → API Key* to find your key. +3. Free tier: 100 requests/day. Paid plans unlock higher quotas. + DNS-based blocklists are queried in parallel. The OpenPhish feed is downloaded once per hour by the provider and cached in memory. diff --git a/checker/criminalip.go b/checker/criminalip.go new file mode 100644 index 0000000..d98f2b5 --- /dev/null +++ b/checker/criminalip.go @@ -0,0 +1,189 @@ +package checker + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "time" + + sdk "git.happydns.org/checker-sdk-go/checker" +) + +const criminalIPEndpoint = "https://api.criminalip.io/v1/domain/report?query=%s" + +func init() { Register(&criminalIPSource{endpoint: criminalIPEndpoint}) } + +type criminalIPSource struct{ endpoint string } + +func (*criminalIPSource) ID() string { return "criminal_ip" } +func (*criminalIPSource) Name() string { return "Criminal IP" } + +func (*criminalIPSource) Options() SourceOptions { + return SourceOptions{ + Admin: []sdk.CheckerOptionField{ + { + Id: "criminal_ip_api_key", + Type: "string", + Label: "Criminal IP API key", + Description: "API key for api.criminalip.io. Free tier: 100 req/day. Leave empty to skip Criminal IP lookups.", + Secret: true, + }, + }, + } +} + +func (s *criminalIPSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult { + apiKey := stringOpt(opts, "criminal_ip_api_key") + if apiKey == "" { + return disabledResult(s.ID(), s.Name()) + } + if registered == "" { + return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: true}} + } + + res := SourceResult{ + SourceID: s.ID(), + SourceName: s.Name(), + Enabled: true, + Reference: "https://www.criminalip.io/domain/report/" + registered, + } + + reqCtx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + + url := fmt.Sprintf(s.endpoint, registered) + req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, url, nil) + if err != nil { + res.Error = err.Error() + return []SourceResult{res} + } + req.Header.Set("x-api-key", apiKey) + + body, status, err := httpDo(req, 1<<20) + if err != nil { + res.Error = redactSecret(err.Error(), apiKey) + return []SourceResult{res} + } + if status != http.StatusOK { + res.Error = fmt.Sprintf("HTTP %d: %s", status, redactSecret(truncate(string(body), 200), apiKey)) + return []SourceResult{res} + } + + var parsed struct { + Status int `json:"status"` + Data struct { + Score struct { + Inbound string `json:"inbound"` + Outbound string `json:"outbound"` + } `json:"score"` + SummaryInfo struct { + MaliciousInfo []string `json:"malicious_info"` + } `json:"summary_info"` + IsMalicious bool `json:"is_malicious"` + IsPhishing bool `json:"is_phishing"` + IsSpam bool `json:"is_spam"` + } `json:"data"` + } + if err := json.Unmarshal(body, &parsed); err != nil { + res.Error = "decode: " + err.Error() + return []SourceResult{res} + } + + inbound := parsed.Data.Score.Inbound + outbound := parsed.Data.Score.Outbound + + // Only populate evidence when there is an elevated score. + if criminalIPScoreLevel(inbound) > 0 || criminalIPScoreLevel(outbound) > 0 { + res.Evidence = append(res.Evidence, + Evidence{Label: "Inbound score", Value: inbound}, + Evidence{Label: "Outbound score", Value: outbound}, + ) + seen := map[string]bool{} + for _, info := range parsed.Data.SummaryInfo.MaliciousInfo { + if info != "" && !seen[info] { + seen[info] = true + res.Reasons = append(res.Reasons, info) + } + } + if parsed.Data.IsMalicious && !seen["Malicious"] { + res.Reasons = append(res.Reasons, "Malicious") + } + if parsed.Data.IsPhishing && !seen["Phishing"] { + res.Reasons = append(res.Reasons, "Phishing") + } + if parsed.Data.IsSpam && !seen["Spam"] { + res.Reasons = append(res.Reasons, "Spam") + } + } + + res.Details = mustJSON(map[string]any{ + "inbound": inbound, + "outbound": outbound, + }) + return []SourceResult{res} +} + +// criminalIPScoreLevel maps a Criminal IP score string to a numeric level +// for comparison: 0=Safe, 1=Low, 2=Moderate, 3=High, 4=Critical. +func criminalIPScoreLevel(score string) int { + switch score { + case "Low": + return 1 + case "Moderate": + return 2 + case "High": + return 3 + case "Critical": + return 4 + } + return 0 // "Safe" or unknown +} + +func (*criminalIPSource) Evaluate(r SourceResult) (bool, string) { + if !r.Enabled || r.Error != "" || len(r.Evidence) == 0 { + return false, "" + } + var d struct { + Inbound string `json:"inbound"` + Outbound string `json:"outbound"` + } + if len(r.Details) > 0 { + _ = json.Unmarshal(r.Details, &d) + } + maxLevel := criminalIPScoreLevel(d.Inbound) + if l := criminalIPScoreLevel(d.Outbound); l > maxLevel { + maxLevel = l + } + switch { + case maxLevel >= 3: + return true, SeverityCrit + case maxLevel == 2: + return true, SeverityWarn + } + return false, "" +} + +func (*criminalIPSource) Diagnose(res SourceResult) Diagnosis { + var d struct { + Inbound string `json:"inbound"` + Outbound string `json:"outbound"` + } + if len(res.Details) > 0 { + _ = json.Unmarshal(res.Details, &d) + } + sev := SeverityWarn + if criminalIPScoreLevel(d.Inbound) >= 3 || criminalIPScoreLevel(d.Outbound) >= 3 { + sev = SeverityCrit + } + return Diagnosis{ + Severity: sev, + Title: fmt.Sprintf("Criminal IP: inbound %s / outbound %s risk", d.Inbound, d.Outbound), + Detail: fmt.Sprintf( + "Criminal IP rated this domain with an inbound score of %q and an outbound score of %q. Threat category/reason(s): %s. Review the full report for connected IPs and associated URLs, then investigate any compromised infrastructure.", + d.Inbound, d.Outbound, joinNonEmpty(res.Reasons, ", "), + ), + Fix: res.Reference, + FixIsURL: res.Reference != "", + } +} From 661e67d9c2e2c04df28eb1ebc86cc59a89612598 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Fri, 15 May 2026 22:04:26 +0800 Subject: [PATCH 14/15] Add NordSpam, SpamEatingMonkey Fresh, Tiopan DBL, and SORBS RHSBL to default DNSBL zones --- README.md | 6 ++++- checker/dnsbl.go | 59 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index fd6c7dc..ca3844d 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,10 @@ widely-used reputation systems. | Spamhaus DBL | DNS-based DBL | no | admin (default on) | | SURBL multi | DNS-based DBL | no | admin (default on) | | URIBL multi | DNS-based DBL | no | admin (default on) | +| NordSpam DBL | DNS-based DBL | no | admin (default on) | +| SpamEatingMonkey Fresh| DNS-based DBL | no | admin (default on) | +| Tiopan DBL | DNS-based DBL | no | admin (default on) | +| SORBS RHSBL | DNS-based DBL | no | admin (default on) | | Extra DNSBL zones | DNS-based DBL | no | admin | | Google Safe Browsing | HTTPS lookup | yes (admin) | admin | | OpenPhish public feed | downloaded list | no | user (default on) | @@ -68,7 +72,7 @@ downloaded once per hour by the provider and cached in memory. The report opens with a diagnosis-first "Action required" section that lists the most common, high-impact problems with a one-shot remediation: -1. **Listed on Spamhaus DBL / SURBL / URIBL**: direct lookup link and +1. **Listed on Spamhaus DBL / SURBL / URIBL / NordSpam / SpamEatingMonkey / Tiopan / SORBS**: direct lookup link and removal procedure URL per operator. 2. **Flagged by Google Safe Browsing**: link to Google Search Console's security-issues review request. diff --git a/checker/dnsbl.go b/checker/dnsbl.go index f34b7e6..02d7912 100644 --- a/checker/dnsbl.go +++ b/checker/dnsbl.go @@ -88,6 +88,34 @@ var DefaultDNSBLZones = []DNSBLZone{ return v4 != nil && v4[3] == 1 }, }, + { + Zone: "dbl.nordspam.com", + Label: "NordSpam DBL", + LookupURL: "https://www.nordspam.com/", + RemovalURL: "https://www.nordspam.com/delist/", + Decode: decodeNordSpamDBL, + }, + { + Zone: "fresh.spameatingmonkey.net", + Label: "SpamEatingMonkey Fresh", + LookupURL: "https://spameatingmonkey.com/lookup", + RemovalURL: "https://spameatingmonkey.com/lookup", + Decode: decodeSEMFresh, + }, + { + Zone: "dbl.tiopan.com", + Label: "Tiopan DBL", + LookupURL: "http://www.tiopan.com/blacklist.php", + RemovalURL: "http://www.tiopan.com/blacklist.php", + Decode: decodeTiopanDBL, + }, + { + Zone: "rhsbl.sorbs.net", + Label: "SORBS RHSBL", + LookupURL: "http://www.sorbs.net/lookup.shtml", + RemovalURL: "http://www.sorbs.net/delisting/overview.shtml", + Decode: decodeSORBSRHSBL, + }, } func (s *dnsblSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult { @@ -315,6 +343,37 @@ func decodeSURBLMulti(ip net.IP) []string { return out } +func decodeNordSpamDBL(ip net.IP) []string { + if ip.String() == "127.0.0.2" { + return []string{"Listed (spam/phishing/scam/ransomware)"} + } + return []string{"Listed (code " + ip.String() + ")"} +} + +func decodeSEMFresh(ip net.IP) []string { + if ip.String() == "127.0.0.2" { + return []string{"Newly registered domain (within last 5 days)"} + } + return []string{"Listed (code " + ip.String() + ")"} +} + +func decodeTiopanDBL(ip net.IP) []string { + if ip.String() == "127.0.0.2" { + return []string{"Listed (spam/abuse)"} + } + return []string{"Listed (code " + ip.String() + ")"} +} + +func decodeSORBSRHSBL(ip net.IP) []string { + switch ip.String() { + case "127.0.0.11": + return []string{"BADCONF: domain has bad A/MX DNS records"} + case "127.0.0.12": + return []string{"NOMAIL: domain has no valid mail server"} + } + return []string{"Listed (code " + ip.String() + ")"} +} + func decodeURIBLMulti(ip net.IP) []string { v4 := ip.To4() if v4 == nil || v4[0] != 127 { From 219d9353c3cc2c31282e01a821eca8fd4f3bfbe5 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sat, 16 May 2026 09:33:19 +0800 Subject: [PATCH 15/15] Add Quad9 secure DNS blocklist source Detects domains blocked by Quad9's threat intelligence by comparing the secure resolver (9.9.9.9) against the unsecured peer (9.9.9.10). No API key required; enabled by default via the enable_quad9 user option. --- README.md | 1 + checker/quad9.go | 127 +++++++++++++++++++++++++++++++++++++++++ checker/source_test.go | 2 +- 3 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 checker/quad9.go diff --git a/README.md b/README.md index ca3844d..0058fd3 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ widely-used reputation systems. | AlienVault OTX | HTTPS lookup | free (admin) | admin | | Pulsedive | HTTPS lookup | free (admin) | admin | | Criminal IP | HTTPS lookup | yes (admin) | admin | +| Quad9 secure DNS | DNS comparison | no | user (default on) | ### Obtaining API keys diff --git a/checker/quad9.go b/checker/quad9.go new file mode 100644 index 0000000..7bf6094 --- /dev/null +++ b/checker/quad9.go @@ -0,0 +1,127 @@ +package checker + +import ( + "context" + "fmt" + "net" + "sync" + "time" + + sdk "git.happydns.org/checker-sdk-go/checker" +) + +func init() { + Register(&quad9Source{ + secure: newDNSResolver("9.9.9.9:53"), + unsecured: newDNSResolver("9.9.9.10:53"), + }) +} + +type quad9Source struct { + secure *net.Resolver + unsecured *net.Resolver +} + +func newDNSResolver(addr string) *net.Resolver { + return &net.Resolver{ + PreferGo: true, + Dial: func(ctx context.Context, network, _ string) (net.Conn, error) { + d := &net.Dialer{Timeout: 5 * time.Second} + return d.DialContext(ctx, "udp", addr) + }, + } +} + +func (*quad9Source) ID() string { return "quad9" } +func (*quad9Source) Name() string { return "Quad9 secure DNS" } + +func (*quad9Source) Options() SourceOptions { + return SourceOptions{ + User: []sdk.CheckerOptionField{ + { + Id: "enable_quad9", + Type: "bool", + Label: "Use Quad9 secure DNS check", + Description: "Compare Quad9's secure resolver (9.9.9.9) against its unsecured peer (9.9.9.10). A domain that resolves on the unsecured but returns NXDOMAIN on the secure resolver is blocked by Quad9's threat intelligence.", + Default: true, + }, + }, + } +} + +func (s *quad9Source) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult { + if !sdk.GetBoolOption(opts, "enable_quad9", true) || registered == "" { + return disabledResult(s.ID(), s.Name()) + } + + res := SourceResult{ + SourceID: s.ID(), + SourceName: s.Name(), + Enabled: true, + LookupURL: "https://www.quad9.net/result/?domain=" + registered, + } + + var ( + secureAddrs, unsecuredAddrs []net.IPAddr + secureErr, unsecuredErr error + wg sync.WaitGroup + ) + queryCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + wg.Add(2) + go func() { + defer wg.Done() + secureAddrs, secureErr = s.secure.LookupIPAddr(queryCtx, registered) + }() + go func() { + defer wg.Done() + unsecuredAddrs, unsecuredErr = s.unsecured.LookupIPAddr(queryCtx, registered) + }() + wg.Wait() + + // Suppress the unused variable warning — secureAddrs is only used as an + // existence check; the actual IPs are not surfaced. + _ = secureAddrs + + // If the unsecured resolver can't resolve the domain either, it simply + // doesn't exist — not a Quad9 block. + if unsecuredErr != nil || len(unsecuredAddrs) == 0 { + if unsecuredErr != nil { + if dnsErr, ok := unsecuredErr.(*net.DNSError); !ok || !dnsErr.IsNotFound { + res.Error = "unsecured resolver: " + unsecuredErr.Error() + } + } + return []SourceResult{res} + } + + // Domain resolves on unsecured. Check whether the secure resolver blocked it. + if dnsErr, ok := secureErr.(*net.DNSError); ok && dnsErr.IsNotFound { + res.Reasons = []string{"Blocked by Quad9 threat intelligence"} + res.Evidence = append(res.Evidence, Evidence{ + Label: "Secure resolver (9.9.9.9)", + Value: "NXDOMAIN", + }) + } else if secureErr != nil { + res.Error = "secure resolver: " + secureErr.Error() + } + + return []SourceResult{res} +} + +func (*quad9Source) Evaluate(r SourceResult) (bool, string) { + return evidenceEval(r, SeverityCrit) +} + +func (*quad9Source) Diagnose(res SourceResult) Diagnosis { + return Diagnosis{ + Severity: SeverityCrit, + Title: "Blocked by Quad9 secure DNS resolver", + Detail: fmt.Sprintf( + "This domain is on Quad9's threat intelligence blocklist: the secure resolver (9.9.9.9) returns NXDOMAIN while the unsecured peer (9.9.9.10) resolves normally. Quad9 aggregates feeds from 18+ threat intel partners (abuse.ch, Bambenek, CINS, DShield, etc.). Visitors whose ISP or device uses Quad9 cannot reach this domain. Submit a false-positive report at the Quad9 contact page if you believe the listing is incorrect.", + ), + Fix: "https://www.quad9.net/support/contact/", + FixIsURL: true, + LookupURL: res.LookupURL, + } +} diff --git a/checker/source_test.go b/checker/source_test.go index 8f432b8..c98c8b8 100644 --- a/checker/source_test.go +++ b/checker/source_test.go @@ -24,7 +24,7 @@ func TestRegisteredSourcesAreSane(t *testing.T) { } } // At least the built-in sources are present. - for _, want := range []string{"dnsbl", "google_safe_browsing", "openphish", "pulsedive", "urlhaus", "virustotal"} { + for _, want := range []string{"dnsbl", "google_safe_browsing", "openphish", "pulsedive", "quad9", "urlhaus", "virustotal"} { if !seen[want] { t.Errorf("missing built-in source %q", want) }