Compare commits

...

15 commits

Author SHA1 Message Date
219d9353c3 Add Quad9 secure DNS blocklist source
All checks were successful
continuous-integration/drone/push Build is passing
continuous-integration/drone/tag Build is passing
Detects domains blocked by Quad9's threat intelligence by comparing
the secure resolver (9.9.9.9) against the unsecured peer (9.9.9.10).
No API key required; enabled by default via the enable_quad9 user option.
2026-05-16 11:00:50 +08:00
661e67d9c2 Add NordSpam, SpamEatingMonkey Fresh, Tiopan DBL, and SORBS RHSBL to default DNSBL zones 2026-05-15 22:04:30 +08:00
a5559ad98f Add Criminal IP domain reputation source
Implements the Criminal IP API (api.criminalip.io/v1/domain/report) as a
new blacklist source. Returns crit for High/Critical inbound or outbound
risk scores, warn for Moderate; Safe and Low scores are not flagged.
2026-05-15 21:56:32 +08:00
c8bcac5a72 Add Pulsedive domain threat intelligence source 2026-05-15 21:41:38 +08:00
faae2f80c5 Add AlienVault OTX domain threat intelligence source 2026-05-15 21:41:38 +08:00
1242a381ab Add OISD domain blocklist source
Implements the OISD domainswild feed (big and small variants) as a new
blacklist source. DNS0.eu was considered but shut down in October 2025.
2026-05-15 21:41:35 +08:00
c2cc88e1df Add Disconnect.me tracking-protection blocklist source
Downloads and caches the Disconnect.me services.json feed (24h TTL),
matching domains against the Advertising, Analytics, Social, Content,
and Disconnect categories. Severity is warn (privacy classification,
not malware). Reuses the shared feedCache infrastructure.
2026-05-15 21:36:24 +08:00
9916ab0732 Add Botvrij.eu domain blocklist source
Downloads the Botvrij.eu public IOC domain list (no API key required),
caches it in-process with a 6h TTL, and flags any registered domain
that appears directly or as a parent of a feed entry.
2026-05-15 21:36:24 +08:00
6b1d2e2540 Extract disabledResult and evidenceEval helpers to reduce boilerplate
Add two shared helpers to source.go and apply them across all sources:
- disabledResult(id, name) replaces the repeated inline SourceResult literal
- evidenceEval(r, severity) replaces the identical Evaluate body in 6 sources
2026-05-15 21:36:24 +08:00
061b5361ca Merge duplicate phishCache/phishTankCache into shared feedCache 2026-05-15 21:36:24 +08:00
229e7a8f02 Add abuse.ch ThreatFox and MalwareBazaar blacklist sources
ThreatFox queries the IOC database for domain indicators (C2 servers,
malware distribution, phishing); MalwareBazaar searches for malware
samples tagged with the domain. Both require a free abuse.ch Auth-Key.
2026-05-15 21:36:24 +08:00
6b08676ec5 Add PhishTank as a new blacklist source 2026-05-15 21:36:24 +08:00
829863e5a0 Add a section on how to obtain API keys 2026-05-15 21:36:24 +08:00
c437339bda Separate observation from evaluation in blacklist sources
Each source's Query() method previously set r.Listed and r.Severity,
embedding verdict logic inside the prober. Evaluation now lives in a
dedicated Evaluate(SourceResult) (bool, string) method per source,
keeping Query() as pure observation.

A package-level EvaluateResult() helper looks up the source by ID and
delegates to its Evaluate method; rules.go, report.go, types.go, and
provider.go all call this instead of reading pre-set r.Listed/r.Severity
values. An unknownSource sentinel handles results whose source is no
longer registered.
2026-05-15 18:04:17 +08:00
01909debad Add CI/CD pipeline
All checks were successful
continuous-integration/drone/push Build is passing
2026-05-10 19:01:21 +08:00
35 changed files with 3054 additions and 159 deletions

22
.drone-manifest.yml Normal file
View file

@ -0,0 +1,22 @@
image: happydomain/checker-blacklist:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}latest{{/if}}
{{#if build.tags}}
tags:
{{#each build.tags}}
- {{this}}
{{/each}}
{{/if}}
manifests:
- image: happydomain/checker-blacklist:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-amd64
platform:
architecture: amd64
os: linux
- image: happydomain/checker-blacklist:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm64
platform:
architecture: arm64
os: linux
variant: v8
- image: happydomain/checker-blacklist:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm
platform:
architecture: arm
os: linux
variant: v7

187
.drone.yml Normal file
View file

@ -0,0 +1,187 @@
---
kind: pipeline
type: docker
name: build-amd64
platform:
os: linux
arch: amd64
steps:
- name: checker build
image: golang:1-alpine
commands:
- apk add --no-cache git make
- make
environment:
CHECKER_VERSION: "${DRONE_BRANCH}-${DRONE_COMMIT}"
CGO_ENABLED: 0
when:
event:
exclude:
- tag
- name: checker build tag
image: golang:1-alpine
commands:
- apk add --no-cache git make
- make
environment:
CHECKER_VERSION: "${DRONE_SEMVER}"
CGO_ENABLED: 0
when:
event:
- tag
- name: publish on Docker Hub
image: plugins/docker
settings:
repo: happydomain/checker-blacklist
auto_tag: true
auto_tag_suffix: ${DRONE_STAGE_OS}-${DRONE_STAGE_ARCH}
dockerfile: Dockerfile
build_args:
- CHECKER_VERSION=${DRONE_BRANCH}-${DRONE_COMMIT}
username:
from_secret: docker_username
password:
from_secret: docker_password
when:
event:
exclude:
- tag
- name: publish on Docker Hub (tag)
image: plugins/docker
settings:
repo: happydomain/checker-blacklist
auto_tag: true
auto_tag_suffix: ${DRONE_STAGE_OS}-${DRONE_STAGE_ARCH}
dockerfile: Dockerfile
build_args:
- CHECKER_VERSION=${DRONE_SEMVER}
username:
from_secret: docker_username
password:
from_secret: docker_password
when:
event:
- tag
trigger:
branch:
exclude:
- renovate/*
event:
- cron
- push
- tag
---
kind: pipeline
type: docker
name: build-arm64
platform:
os: linux
arch: arm64
steps:
- name: checker build
image: golang:1-alpine
commands:
- apk add --no-cache git make
- make
environment:
CHECKER_VERSION: "${DRONE_BRANCH}-${DRONE_COMMIT}"
CGO_ENABLED: 0
when:
event:
exclude:
- tag
- name: checker build tag
image: golang:1-alpine
commands:
- apk add --no-cache git make
- make
environment:
CHECKER_VERSION: "${DRONE_SEMVER}"
CGO_ENABLED: 0
when:
event:
- tag
- name: publish on Docker Hub
image: plugins/docker
settings:
repo: happydomain/checker-blacklist
auto_tag: true
auto_tag_suffix: ${DRONE_STAGE_OS}-${DRONE_STAGE_ARCH}
dockerfile: Dockerfile
build_args:
- CHECKER_VERSION=${DRONE_BRANCH}-${DRONE_COMMIT}
username:
from_secret: docker_username
password:
from_secret: docker_password
when:
event:
exclude:
- tag
- name: publish on Docker Hub (tag)
image: plugins/docker
settings:
repo: happydomain/checker-blacklist
auto_tag: true
auto_tag_suffix: ${DRONE_STAGE_OS}-${DRONE_STAGE_ARCH}
dockerfile: Dockerfile
build_args:
- CHECKER_VERSION=${DRONE_SEMVER}
username:
from_secret: docker_username
password:
from_secret: docker_password
when:
event:
- tag
trigger:
event:
- cron
- push
- tag
---
kind: pipeline
name: docker-manifest
platform:
os: linux
arch: arm64
steps:
- name: publish on Docker Hub
image: plugins/manifest
settings:
auto_tag: true
ignore_missing: true
spec: .drone-manifest.yml
username:
from_secret: docker_username
password:
from_secret: docker_password
trigger:
branch:
exclude:
- renovate/*
event:
- cron
- push
- tag
depends_on:
- build-amd64
- build-arm64

View file

@ -10,11 +10,60 @@ widely-used reputation systems.
| Spamhaus DBL | DNS-based DBL | no | admin (default on) |
| SURBL multi | DNS-based DBL | no | admin (default on) |
| URIBL multi | DNS-based DBL | no | admin (default on) |
| NordSpam DBL | DNS-based DBL | no | admin (default on) |
| SpamEatingMonkey Fresh| DNS-based DBL | no | admin (default on) |
| Tiopan DBL | DNS-based DBL | no | admin (default on) |
| SORBS RHSBL | DNS-based DBL | no | admin (default on) |
| Extra DNSBL zones | DNS-based DBL | no | admin |
| Google Safe Browsing | HTTPS lookup | yes (admin) | admin |
| OpenPhish public feed | downloaded list | no | user (default on) |
| abuse.ch URLhaus | HTTPS lookup | optional Auth-Key (admin) | user (default on) |
| PhishTank | downloaded list | no | user (default on) |
| abuse.ch URLhaus | HTTPS lookup | free Auth-Key (admin) | user (default on) |
| abuse.ch ThreatFox | HTTPS lookup | free Auth-Key (admin) | user (default on) |
| abuse.ch MalwareBazaar| HTTPS lookup | free Auth-Key (admin) | user (default on) |
| Botvrij.eu | downloaded list | no | user (default on) |
| Disconnect.me | downloaded list | no | user (default on) |
| OISD | downloaded list | no | user (default on) |
| VirusTotal v3 | HTTPS lookup | yes (admin) | admin |
| AlienVault OTX | HTTPS lookup | free (admin) | admin |
| Pulsedive | HTTPS lookup | free (admin) | admin |
| Criminal IP | HTTPS lookup | yes (admin) | admin |
| Quad9 secure DNS | DNS comparison | no | user (default on) |
### Obtaining API keys
**Google Safe Browsing** (option: `google_safe_browsing_api_key`)
1. Go to the [Google Cloud Console](https://console.cloud.google.com/) and create or select a project.
2. Enable the *Safe Browsing API* under *APIs & Services → Library*.
3. Create an API key under *APIs & Services → Credentials*.
4. The free tier allows up to 10 000 queries/day with no billing required.
**abuse.ch** (option: `urlhaus_auth_key` / `threatfox_auth_key` / `malwarebazaar_auth_key`)
1. Register a free account at [abuse.ch](https://abuse.ch/).
2. After login, retrieve your Auth-Key from your account profile page.
3. The same account and key works for URLhaus, ThreatFox, and MalwareBazaar — set it in each source option independently.
4. Free, no rate-limit tiers documented; the APIs are community-funded.
**VirusTotal** (option: `virustotal_api_key`)
1. Create a free account at [virustotal.com](https://www.virustotal.com/).
2. Go to your profile and copy the API key.
3. Free tier: 4 requests/minute, 500 requests/day. No billing required.
4. The public API key is sufficient; premium keys unlock higher quotas.
**AlienVault OTX** (option: `otx_api_key`)
1. Register a free account at [otx.alienvault.com](https://otx.alienvault.com/).
2. Go to *Settings → API Integration* to find your personal OTX key.
3. Free, no documented rate limits for the indicator lookup API.
**Pulsedive** (option: `pulsedive_api_key`)
1. Register a free account at [pulsedive.com](https://pulsedive.com/).
2. Go to your profile and copy the API key shown under *API*.
3. Free tier available; higher quotas with a paid plan.
**Criminal IP** (option: `criminal_ip_api_key`)
1. Register a free account at [criminalip.io](https://www.criminalip.io/).
2. Go to *My Information → API Key* to find your key.
3. Free tier: 100 requests/day. Paid plans unlock higher quotas.
DNS-based blocklists are queried in parallel. The OpenPhish feed is
downloaded once per hour by the provider and cached in memory.
@ -24,7 +73,7 @@ downloaded once per hour by the provider and cached in memory.
The report opens with a diagnosis-first "Action required" section that
lists the most common, high-impact problems with a one-shot remediation:
1. **Listed on Spamhaus DBL / SURBL / URIBL**: direct lookup link and
1. **Listed on Spamhaus DBL / SURBL / URIBL / NordSpam / SpamEatingMonkey / Tiopan / SORBS**: direct lookup link and
removal procedure URL per operator.
2. **Flagged by Google Safe Browsing**: link to Google Search Console's
security-issues review request.

191
checker/botvrij.go Normal file
View file

@ -0,0 +1,191 @@
package checker
import (
"bufio"
"context"
"fmt"
"io"
"net/http"
"strings"
"sync"
"time"
sdk "git.happydns.org/checker-sdk-go/checker"
)
const (
botvrijFeedURL = "https://www.botvrij.eu/data/ioclist.domain.simple"
botvrijDefaultTTL = 6 * time.Hour
botvrijFailBackoff = 1 * time.Minute
)
func init() {
Register(&botvrijSource{
cache: newBotvrijCache(botvrijFeedURL),
})
}
type botvrijSource struct {
cache *botvrijCache
}
func (*botvrijSource) ID() string { return "botvrij" }
func (*botvrijSource) Name() string { return "Botvrij.eu domain blocklist" }
func (*botvrijSource) Options() SourceOptions {
return SourceOptions{
User: []sdk.CheckerOptionField{
{
Id: "enable_botvrij",
Type: "bool",
Label: "Use Botvrij.eu domain blocklist",
Description: "Download the Botvrij.eu public domain blocklist (refreshed every 6h) and check the domain against it.",
Default: true,
},
},
}
}
func (s *botvrijSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult {
if !sdk.GetBoolOption(opts, "enable_botvrij", true) || registered == "" {
return disabledResult(s.ID(), s.Name())
}
matched, size, fetched, err := s.cache.lookup(ctx, registered)
res := SourceResult{
SourceID: s.ID(), SourceName: s.Name(), Enabled: true,
Reference: "https://botvrij.eu/",
Details: mustJSON(map[string]any{"feed_size": size, "fetched_at": fetched}),
}
if err != nil {
res.Error = err.Error()
}
if len(matched) > 0 {
res.Reasons = []string{"Malicious domain"}
for _, d := range matched {
res.Evidence = append(res.Evidence, Evidence{Label: "Domain", Value: d})
}
}
return []SourceResult{res}
}
func (*botvrijSource) Evaluate(r SourceResult) (bool, string) {
return evidenceEval(r, SeverityCrit)
}
func (*botvrijSource) Diagnose(res SourceResult) Diagnosis {
domains := make([]string, 0, len(res.Evidence))
for _, e := range res.Evidence {
domains = append(domains, e.Value)
}
previewN := min(len(domains), 5)
return Diagnosis{
Severity: SeverityCrit,
Title: "Listed in the Botvrij.eu domain blocklist",
Detail: fmt.Sprintf(
"%d domain(s) matching this registered domain appear in the Botvrij.eu IOC list; examples: %s. Botvrij.eu tracks domains associated with malware C2, exploit kits, and other malicious infrastructure. Investigate recent DNS changes and hosted content.",
len(domains), joinNonEmpty(domains[:previewN], ", "),
),
Fix: "https://botvrij.eu/",
FixIsURL: true,
}
}
// ---------- cache ----------
type botvrijCache struct {
mu sync.Mutex
domains []string // all domains in feed
byDomain map[string]struct{} // set for O(1) exact-match test
fetchedAt time.Time
lastAttemptAt time.Time
refreshing bool
ttl time.Duration
failBackoff time.Duration
feedURL string
}
func newBotvrijCache(feedURL string) *botvrijCache {
return &botvrijCache{
ttl: botvrijDefaultTTL,
failBackoff: botvrijFailBackoff,
feedURL: feedURL,
}
}
func (c *botvrijCache) lookup(ctx context.Context, registered string) (matched []string, size int, fetchedAt time.Time, err error) {
registered = strings.ToLower(strings.TrimSuffix(registered, "."))
c.mu.Lock()
stale := c.byDomain == nil || time.Since(c.fetchedAt) > c.ttl
doRefresh := stale && !c.refreshing && time.Since(c.lastAttemptAt) > c.failBackoff
if doRefresh {
c.refreshing = true
}
c.mu.Unlock()
if doRefresh {
newDomains, newByDomain, ferr := c.fetch(ctx)
c.mu.Lock()
c.refreshing = false
c.lastAttemptAt = time.Now()
if ferr == nil {
c.domains = newDomains
c.byDomain = newByDomain
c.fetchedAt = c.lastAttemptAt
} else {
err = ferr
}
c.mu.Unlock()
}
c.mu.Lock()
suffix := "." + registered
for d := range c.byDomain {
if d == registered || strings.HasSuffix(d, suffix) {
matched = append(matched, d)
}
}
size = len(c.domains)
fetchedAt = c.fetchedAt
c.mu.Unlock()
return matched, size, fetchedAt, err
}
func (c *botvrijCache) fetch(ctx context.Context) ([]string, map[string]struct{}, error) {
reqCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, c.feedURL, nil)
if err != nil {
return nil, nil, err
}
req.Header.Set("User-Agent", "happydomain-checker-blacklist/1.0")
resp, err := sharedHTTPClient.Do(req)
if err != nil {
return nil, nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, nil, fmt.Errorf("botvrij HTTP %d", resp.StatusCode)
}
domains := make([]string, 0, 4096)
byDomain := make(map[string]struct{}, 4096)
scanner := bufio.NewScanner(io.LimitReader(resp.Body, 16<<20))
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" || strings.HasPrefix(line, "#") {
continue
}
d := strings.ToLower(line)
domains = append(domains, d)
byDomain[d] = struct{}{}
}
if err := scanner.Err(); err != nil {
return nil, nil, err
}
return domains, byDomain, nil
}

94
checker/botvrij_test.go Normal file
View file

@ -0,0 +1,94 @@
package checker
import (
"context"
"net/http"
"net/http/httptest"
"testing"
sdk "git.happydns.org/checker-sdk-go/checker"
)
const botvrijFakeFeed = `# Botvrij.eu IOC list - domains
# comment line
evil.com
malware.example.org
c2.badactor.net
`
func TestBotvrijSource_Listed_ExactMatch(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte(botvrijFakeFeed))
}))
defer srv.Close()
s := &botvrijSource{cache: newBotvrijCache(srv.URL)}
r := s.Query(context.Background(), "evil.com", "evil.com", sdk.CheckerOptions{"enable_botvrij": true})[0]
if !r.Enabled || r.Error != "" {
t.Fatalf("expected enabled and no error, got %+v", r)
}
if len(r.Evidence) != 1 || r.Evidence[0].Value != "evil.com" {
t.Errorf("expected evidence [evil.com], got %+v", r.Evidence)
}
if listed, sev := s.Evaluate(r); !listed || sev != SeverityCrit {
t.Errorf("expected (true, crit), got (%v, %q)", listed, sev)
}
}
func TestBotvrijSource_Listed_SubdomainInFeed(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte(botvrijFakeFeed))
}))
defer srv.Close()
// Feed has "malware.example.org"; querying registered "example.org" should match.
s := &botvrijSource{cache: newBotvrijCache(srv.URL)}
r := s.Query(context.Background(), "sub.example.org", "example.org", sdk.CheckerOptions{"enable_botvrij": true})[0]
if len(r.Evidence) != 1 || r.Evidence[0].Value != "malware.example.org" {
t.Errorf("expected subdomain match, got %+v", r.Evidence)
}
if listed, _ := s.Evaluate(r); !listed {
t.Error("expected listed=true")
}
}
func TestBotvrijSource_NotListed(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte(botvrijFakeFeed))
}))
defer srv.Close()
s := &botvrijSource{cache: newBotvrijCache(srv.URL)}
r := s.Query(context.Background(), "safe.com", "safe.com", sdk.CheckerOptions{"enable_botvrij": true})[0]
if !r.Enabled || r.Error != "" || len(r.Evidence) != 0 {
t.Fatalf("expected clean result, got %+v", r)
}
if listed, _ := s.Evaluate(r); listed {
t.Error("expected listed=false")
}
}
func TestBotvrijSource_Disabled(t *testing.T) {
s := &botvrijSource{cache: newBotvrijCache("http://nope")}
r := s.Query(context.Background(), "evil.com", "evil.com", sdk.CheckerOptions{"enable_botvrij": false})[0]
if r.Enabled {
t.Errorf("expected disabled, got %+v", r)
}
}
func TestBotvrijSource_HTTPError(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
}))
defer srv.Close()
s := &botvrijSource{cache: newBotvrijCache(srv.URL)}
r := s.Query(context.Background(), "evil.com", "evil.com", sdk.CheckerOptions{"enable_botvrij": true})[0]
if r.Error == "" || r.Error != "botvrij HTTP 500" {
t.Errorf("expected HTTP 500 error, got %q", r.Error)
}
}

189
checker/criminalip.go Normal file
View file

@ -0,0 +1,189 @@
package checker
import (
"context"
"encoding/json"
"fmt"
"net/http"
"time"
sdk "git.happydns.org/checker-sdk-go/checker"
)
const criminalIPEndpoint = "https://api.criminalip.io/v1/domain/report?query=%s"
func init() { Register(&criminalIPSource{endpoint: criminalIPEndpoint}) }
type criminalIPSource struct{ endpoint string }
func (*criminalIPSource) ID() string { return "criminal_ip" }
func (*criminalIPSource) Name() string { return "Criminal IP" }
func (*criminalIPSource) Options() SourceOptions {
return SourceOptions{
Admin: []sdk.CheckerOptionField{
{
Id: "criminal_ip_api_key",
Type: "string",
Label: "Criminal IP API key",
Description: "API key for api.criminalip.io. Free tier: 100 req/day. Leave empty to skip Criminal IP lookups.",
Secret: true,
},
},
}
}
func (s *criminalIPSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult {
apiKey := stringOpt(opts, "criminal_ip_api_key")
if apiKey == "" {
return disabledResult(s.ID(), s.Name())
}
if registered == "" {
return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: true}}
}
res := SourceResult{
SourceID: s.ID(),
SourceName: s.Name(),
Enabled: true,
Reference: "https://www.criminalip.io/domain/report/" + registered,
}
reqCtx, cancel := context.WithTimeout(ctx, 15*time.Second)
defer cancel()
url := fmt.Sprintf(s.endpoint, registered)
req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, url, nil)
if err != nil {
res.Error = err.Error()
return []SourceResult{res}
}
req.Header.Set("x-api-key", apiKey)
body, status, err := httpDo(req, 1<<20)
if err != nil {
res.Error = redactSecret(err.Error(), apiKey)
return []SourceResult{res}
}
if status != http.StatusOK {
res.Error = fmt.Sprintf("HTTP %d: %s", status, redactSecret(truncate(string(body), 200), apiKey))
return []SourceResult{res}
}
var parsed struct {
Status int `json:"status"`
Data struct {
Score struct {
Inbound string `json:"inbound"`
Outbound string `json:"outbound"`
} `json:"score"`
SummaryInfo struct {
MaliciousInfo []string `json:"malicious_info"`
} `json:"summary_info"`
IsMalicious bool `json:"is_malicious"`
IsPhishing bool `json:"is_phishing"`
IsSpam bool `json:"is_spam"`
} `json:"data"`
}
if err := json.Unmarshal(body, &parsed); err != nil {
res.Error = "decode: " + err.Error()
return []SourceResult{res}
}
inbound := parsed.Data.Score.Inbound
outbound := parsed.Data.Score.Outbound
// Only populate evidence when there is an elevated score.
if criminalIPScoreLevel(inbound) > 0 || criminalIPScoreLevel(outbound) > 0 {
res.Evidence = append(res.Evidence,
Evidence{Label: "Inbound score", Value: inbound},
Evidence{Label: "Outbound score", Value: outbound},
)
seen := map[string]bool{}
for _, info := range parsed.Data.SummaryInfo.MaliciousInfo {
if info != "" && !seen[info] {
seen[info] = true
res.Reasons = append(res.Reasons, info)
}
}
if parsed.Data.IsMalicious && !seen["Malicious"] {
res.Reasons = append(res.Reasons, "Malicious")
}
if parsed.Data.IsPhishing && !seen["Phishing"] {
res.Reasons = append(res.Reasons, "Phishing")
}
if parsed.Data.IsSpam && !seen["Spam"] {
res.Reasons = append(res.Reasons, "Spam")
}
}
res.Details = mustJSON(map[string]any{
"inbound": inbound,
"outbound": outbound,
})
return []SourceResult{res}
}
// criminalIPScoreLevel maps a Criminal IP score string to a numeric level
// for comparison: 0=Safe, 1=Low, 2=Moderate, 3=High, 4=Critical.
func criminalIPScoreLevel(score string) int {
switch score {
case "Low":
return 1
case "Moderate":
return 2
case "High":
return 3
case "Critical":
return 4
}
return 0 // "Safe" or unknown
}
func (*criminalIPSource) Evaluate(r SourceResult) (bool, string) {
if !r.Enabled || r.Error != "" || len(r.Evidence) == 0 {
return false, ""
}
var d struct {
Inbound string `json:"inbound"`
Outbound string `json:"outbound"`
}
if len(r.Details) > 0 {
_ = json.Unmarshal(r.Details, &d)
}
maxLevel := criminalIPScoreLevel(d.Inbound)
if l := criminalIPScoreLevel(d.Outbound); l > maxLevel {
maxLevel = l
}
switch {
case maxLevel >= 3:
return true, SeverityCrit
case maxLevel == 2:
return true, SeverityWarn
}
return false, ""
}
func (*criminalIPSource) Diagnose(res SourceResult) Diagnosis {
var d struct {
Inbound string `json:"inbound"`
Outbound string `json:"outbound"`
}
if len(res.Details) > 0 {
_ = json.Unmarshal(res.Details, &d)
}
sev := SeverityWarn
if criminalIPScoreLevel(d.Inbound) >= 3 || criminalIPScoreLevel(d.Outbound) >= 3 {
sev = SeverityCrit
}
return Diagnosis{
Severity: sev,
Title: fmt.Sprintf("Criminal IP: inbound %s / outbound %s risk", d.Inbound, d.Outbound),
Detail: fmt.Sprintf(
"Criminal IP rated this domain with an inbound score of %q and an outbound score of %q. Threat category/reason(s): %s. Review the full report for connected IPs and associated URLs, then investigate any compromised infrastructure.",
d.Inbound, d.Outbound, joinNonEmpty(res.Reasons, ", "),
),
Fix: res.Reference,
FixIsURL: res.Reference != "",
}
}

162
checker/disconnect.go Normal file
View file

@ -0,0 +1,162 @@
package checker
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"time"
sdk "git.happydns.org/checker-sdk-go/checker"
)
const disconnectFeedURL = "https://s3.amazonaws.com/lists.disconnect.me/services.json"
func init() {
Register(&disconnectSource{
cache: newFeedCache(24*time.Hour, disconnectFetch(disconnectFeedURL)),
})
}
type disconnectSource struct{ cache *feedCache }
func (*disconnectSource) ID() string { return "disconnect" }
func (*disconnectSource) Name() string { return "Disconnect.me" }
func (*disconnectSource) Options() SourceOptions {
return SourceOptions{
User: []sdk.CheckerOptionField{
{
Id: "enable_disconnect",
Type: "bool",
Label: "Use the Disconnect.me tracking-protection list",
Description: "Check the domain against the Disconnect.me blocklist used by Firefox Enhanced Tracking Protection, Brave, and uBlock Origin.",
Default: true,
},
},
}
}
func (s *disconnectSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult {
if !sdk.GetBoolOption(opts, "enable_disconnect", true) {
return disabledResult(s.ID(), s.Name())
}
if registered == "" {
return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: true}}
}
matches, size, fetched, err := s.cache.lookup(ctx, registered)
res := SourceResult{
SourceID: s.ID(), SourceName: s.Name(), Enabled: true,
Reference: "https://disconnect.me/trackerprotection",
Details: mustJSON(map[string]any{"feed_size": size, "fetched_at": fetched}),
}
if err != nil {
res.Error = err.Error()
}
seenCategory := map[string]bool{}
for _, m := range matches {
parts := strings.SplitN(m, "|", 2)
category, company := parts[0], ""
if len(parts) == 2 {
company = parts[1]
}
if !seenCategory[category] {
seenCategory[category] = true
res.Reasons = append(res.Reasons, category)
}
extra := map[string]string{}
if company != "" {
extra["company"] = company
}
res.Evidence = append(res.Evidence, Evidence{
Label: "Category",
Value: category,
Status: strings.ToLower(category),
Extra: extra,
})
}
return []SourceResult{res}
}
func (*disconnectSource) Evaluate(r SourceResult) (bool, string) {
return evidenceEval(r, SeverityWarn)
}
func (*disconnectSource) Diagnose(res SourceResult) Diagnosis {
return Diagnosis{
Severity: SeverityWarn,
Title: "Listed in Disconnect.me tracking-protection blocklist",
Detail: fmt.Sprintf(
"Category: %s. This domain appears in the Disconnect.me list used by Firefox Enhanced Tracking Protection, Brave, and uBlock Origin. Browsers and privacy tools will block third-party requests to this domain, which may affect analytics, ad delivery, or embedded widgets. The list is maintained by Disconnect.me; contact them if you believe the classification is incorrect.",
joinNonEmpty(res.Reasons, ", "),
),
Fix: "https://disconnect.me/contact",
FixIsURL: true,
}
}
// disconnectJSON mirrors the structure of services.json.
type disconnectJSON struct {
Categories map[string][]map[string]map[string][]string `json:"categories"`
}
func disconnectFetch(feedURL string) func(context.Context) ([]string, map[string][]string, error) {
return func(ctx context.Context) ([]string, map[string][]string, error) {
reqCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, feedURL, nil)
if err != nil {
return nil, nil, err
}
req.Header.Set("User-Agent", "happydomain-checker-blacklist/1.0")
resp, err := sharedHTTPClient.Do(req)
if err != nil {
return nil, nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, nil, fmt.Errorf("disconnect HTTP %d", resp.StatusCode)
}
raw, err := io.ReadAll(io.LimitReader(resp.Body, 32<<20))
if err != nil {
return nil, nil, fmt.Errorf("disconnect read: %w", err)
}
var data disconnectJSON
if err := json.Unmarshal(raw, &data); err != nil {
return nil, nil, fmt.Errorf("disconnect parse: %w", err)
}
byHost := make(map[string][]string, 4096)
for category, entities := range data.Categories {
for _, entity := range entities {
for company, sites := range entity {
for _, domains := range sites {
for _, d := range domains {
d = strings.ToLower(strings.TrimSuffix(d, "."))
if d == "" {
continue
}
entry := category + "|" + company
byHost[d] = append(byHost[d], entry)
}
}
}
}
}
urls := make([]string, 0, len(byHost))
for d := range byHost {
urls = append(urls, d)
}
return urls, byHost, nil
}
}

127
checker/disconnect_test.go Normal file
View file

@ -0,0 +1,127 @@
package checker
import (
"context"
"net/http"
"net/http/httptest"
"testing"
"time"
sdk "git.happydns.org/checker-sdk-go/checker"
)
const disconnectFakeFeed = `{
"categories": {
"Advertising": [
{
"Evil Corp": {
"https://evilcorp.com": ["tracker.com", "sub.example.org"]
}
}
],
"Analytics": [
{
"Metrics Inc": {
"https://metrics.io": ["analytics.net"]
}
}
]
}
}`
func newDisconnectTestSource(srv *httptest.Server) *disconnectSource {
return &disconnectSource{
cache: newFeedCache(time.Hour, disconnectFetch(srv.URL)),
}
}
func TestDisconnectSource_Listed(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte(disconnectFakeFeed))
}))
defer srv.Close()
s := newDisconnectTestSource(srv)
r := s.Query(context.Background(), "tracker.com", "tracker.com", sdk.CheckerOptions{"enable_disconnect": true})[0]
if !r.Enabled || r.Error != "" {
t.Fatalf("expected enabled with no error, got %+v", r)
}
if len(r.Evidence) == 0 {
t.Fatalf("expected evidence, got none")
}
found := false
for _, e := range r.Evidence {
if e.Value == "Advertising" {
found = true
if e.Extra["company"] != "Evil Corp" {
t.Errorf("expected company 'Evil Corp', got %q", e.Extra["company"])
}
}
}
if !found {
t.Errorf("expected Advertising evidence, got %+v", r.Evidence)
}
if listed, sev := s.Evaluate(r); !listed || sev != SeverityWarn {
t.Errorf("expected (true, warn), got (%v, %q)", listed, sev)
}
}
func TestDisconnectSource_SubdomainInFeed(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte(disconnectFakeFeed))
}))
defer srv.Close()
// Feed has "sub.example.org"; querying registered domain "example.org" should match.
s := newDisconnectTestSource(srv)
r := s.Query(context.Background(), "example.org", "example.org", sdk.CheckerOptions{"enable_disconnect": true})[0]
if !r.Enabled || r.Error != "" {
t.Fatalf("expected enabled with no error, got %+v", r)
}
if len(r.Evidence) == 0 {
t.Errorf("expected subdomain match evidence, got none")
}
}
func TestDisconnectSource_NotListed(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte(disconnectFakeFeed))
}))
defer srv.Close()
s := newDisconnectTestSource(srv)
r := s.Query(context.Background(), "clean.example.com", "clean.example.com", sdk.CheckerOptions{"enable_disconnect": true})[0]
if !r.Enabled || r.Error != "" {
t.Fatalf("expected enabled with no error, got %+v", r)
}
if len(r.Evidence) != 0 {
t.Errorf("expected no evidence for clean domain, got %+v", r.Evidence)
}
if listed, _ := s.Evaluate(r); listed {
t.Errorf("expected not listed for clean domain")
}
}
func TestDisconnectSource_Disabled(t *testing.T) {
s := &disconnectSource{cache: newFeedCache(time.Hour, disconnectFetch("http://nope"))}
r := s.Query(context.Background(), "tracker.com", "tracker.com", sdk.CheckerOptions{"enable_disconnect": false})[0]
if r.Enabled {
t.Errorf("expected disabled result, got %+v", r)
}
}
func TestDisconnectSource_HTTPError(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
}))
defer srv.Close()
s := newDisconnectTestSource(srv)
r := s.Query(context.Background(), "tracker.com", "tracker.com", sdk.CheckerOptions{"enable_disconnect": true})[0]
if r.Error == "" {
t.Errorf("expected error on HTTP 500, got empty error")
}
}

View file

@ -88,14 +88,40 @@ var DefaultDNSBLZones = []DNSBLZone{
return v4 != nil && v4[3] == 1
},
},
{
Zone: "dbl.nordspam.com",
Label: "NordSpam DBL",
LookupURL: "https://www.nordspam.com/",
RemovalURL: "https://www.nordspam.com/delist/",
Decode: decodeNordSpamDBL,
},
{
Zone: "fresh.spameatingmonkey.net",
Label: "SpamEatingMonkey Fresh",
LookupURL: "https://spameatingmonkey.com/lookup",
RemovalURL: "https://spameatingmonkey.com/lookup",
Decode: decodeSEMFresh,
},
{
Zone: "dbl.tiopan.com",
Label: "Tiopan DBL",
LookupURL: "http://www.tiopan.com/blacklist.php",
RemovalURL: "http://www.tiopan.com/blacklist.php",
Decode: decodeTiopanDBL,
},
{
Zone: "rhsbl.sorbs.net",
Label: "SORBS RHSBL",
LookupURL: "http://www.sorbs.net/lookup.shtml",
RemovalURL: "http://www.sorbs.net/delisting/overview.shtml",
Decode: decodeSORBSRHSBL,
},
}
func (s *dnsblSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult {
zones := zonesFromOptions(opts)
if registered == "" || len(zones) == 0 {
return []SourceResult{{
SourceID: s.ID(), SourceName: s.Name(), Enabled: false,
}}
return disabledResult(s.ID(), s.Name())
}
out := make([]SourceResult, len(zones))
@ -160,8 +186,6 @@ func (s *dnsblSource) queryOne(ctx context.Context, registered string, z DNSBLZo
res.BlockedQuery = true
return res
}
res.Listed = true
res.Severity = SeverityCrit
if len(res.Reasons) == 0 {
res.Reasons = append(res.Reasons, "Listed (no detail decoded)")
}
@ -176,6 +200,13 @@ func (s *dnsblSource) queryOne(ctx context.Context, registered string, z DNSBLZo
return res
}
func (*dnsblSource) Evaluate(r SourceResult) (bool, string) {
if r.Enabled && !r.BlockedQuery && r.Error == "" && len(r.Evidence) > 0 {
return true, SeverityCrit
}
return false, ""
}
func (*dnsblSource) Diagnose(res SourceResult) Diagnosis {
return Diagnosis{
Severity: SeverityCrit,
@ -312,6 +343,37 @@ func decodeSURBLMulti(ip net.IP) []string {
return out
}
func decodeNordSpamDBL(ip net.IP) []string {
if ip.String() == "127.0.0.2" {
return []string{"Listed (spam/phishing/scam/ransomware)"}
}
return []string{"Listed (code " + ip.String() + ")"}
}
func decodeSEMFresh(ip net.IP) []string {
if ip.String() == "127.0.0.2" {
return []string{"Newly registered domain (within last 5 days)"}
}
return []string{"Listed (code " + ip.String() + ")"}
}
func decodeTiopanDBL(ip net.IP) []string {
if ip.String() == "127.0.0.2" {
return []string{"Listed (spam/abuse)"}
}
return []string{"Listed (code " + ip.String() + ")"}
}
func decodeSORBSRHSBL(ip net.IP) []string {
switch ip.String() {
case "127.0.0.11":
return []string{"BADCONF: domain has bad A/MX DNS records"}
case "127.0.0.12":
return []string{"NOMAIL: domain has no valid mail server"}
}
return []string{"Listed (code " + ip.String() + ")"}
}
func decodeURIBLMulti(ip net.IP) []string {
v4 := ip.To4()
if v4 == nil || v4[0] != 127 {

89
checker/feedcache.go Normal file
View file

@ -0,0 +1,89 @@
package checker
import (
"context"
"net/url"
"strings"
"sync"
"time"
)
// feedCache is a generic URL-feed cache shared between phishing-feed
// sources (OpenPhish, PhishTank). It holds a hostname-indexed snapshot
// of the feed, refreshes on TTL expiry, and ensures only one refresh is
// in flight at a time so concurrent lookups still serve stale data
// during a refresh.
type feedCache struct {
mu sync.Mutex
urls []string
byHost map[string][]string
fetchedAt time.Time
lastAttemptAt time.Time
refreshing bool
ttl time.Duration
failBackoff time.Duration
fetchFn func(ctx context.Context) (urls []string, byHost map[string][]string, err error)
}
func newFeedCache(ttl time.Duration, fetch func(context.Context) ([]string, map[string][]string, error)) *feedCache {
if ttl <= 0 {
ttl = time.Hour
}
return &feedCache{
ttl: ttl,
failBackoff: time.Minute,
fetchFn: fetch,
}
}
func (c *feedCache) setTTL(d time.Duration) {
c.mu.Lock()
c.ttl = d
c.mu.Unlock()
}
func (c *feedCache) lookup(ctx context.Context, domain string) (urls []string, size int, fetchedAt time.Time, err error) {
domain = strings.ToLower(strings.TrimSuffix(domain, "."))
c.mu.Lock()
stale := c.byHost == nil || time.Since(c.fetchedAt) > c.ttl
doRefresh := stale && !c.refreshing && time.Since(c.lastAttemptAt) > c.failBackoff
if doRefresh {
c.refreshing = true
}
c.mu.Unlock()
if doRefresh {
newURLs, newByHost, ferr := c.fetchFn(ctx)
c.mu.Lock()
c.refreshing = false
c.lastAttemptAt = time.Now()
if ferr == nil {
c.urls = newURLs
c.byHost = newByHost
c.fetchedAt = c.lastAttemptAt
} else {
err = ferr
}
c.mu.Unlock()
}
c.mu.Lock()
for host, hostURLs := range c.byHost {
if host == domain || strings.HasSuffix(host, "."+domain) {
urls = append(urls, hostURLs...)
}
}
size = len(c.urls)
fetchedAt = c.fetchedAt
c.mu.Unlock()
return urls, size, fetchedAt, err
}
func hostOfURL(s string) string {
u, err := url.Parse(s)
if err != nil {
return ""
}
return strings.ToLower(u.Hostname())
}

146
checker/malwarebazaar.go Normal file
View file

@ -0,0 +1,146 @@
package checker
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"
"time"
sdk "git.happydns.org/checker-sdk-go/checker"
)
const malwareBazaarEndpoint = "https://mb-api.abuse.ch/api/v1/"
func init() { Register(&malwareBazaarSource{endpoint: malwareBazaarEndpoint}) }
type malwareBazaarSource struct {
endpoint string
}
func (*malwareBazaarSource) ID() string { return "malwarebazaar" }
func (*malwareBazaarSource) Name() string { return "abuse.ch MalwareBazaar" }
func (*malwareBazaarSource) Options() SourceOptions {
return SourceOptions{
Admin: []sdk.CheckerOptionField{
{
Id: "malwarebazaar_auth_key",
Type: "string",
Label: "MalwareBazaar Auth-Key",
Description: "abuse.ch MalwareBazaar Auth-Key (free, requires an abuse.ch account). Without this key the source is disabled.",
Secret: true,
},
},
User: []sdk.CheckerOptionField{
{
Id: "enable_malwarebazaar",
Type: "bool",
Label: "Use abuse.ch MalwareBazaar",
Description: "Search MalwareBazaar for malware samples tagged with the domain (typically C2 infrastructure or delivery hosts).",
Default: true,
},
},
}
}
func (s *malwareBazaarSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult {
authKey := stringOpt(opts, "malwarebazaar_auth_key")
if !sdk.GetBoolOption(opts, "enable_malwarebazaar", true) || registered == "" || authKey == "" {
return disabledResult(s.ID(), s.Name())
}
res := SourceResult{
SourceID: s.ID(), SourceName: s.Name(), Enabled: true,
Reference: "https://bazaar.abuse.ch/browse/",
}
buf, err := json.Marshal(map[string]any{
"query": "search_tag",
"tag": registered,
})
if err != nil {
res.Error = err.Error()
return []SourceResult{res}
}
reqCtx, cancel := context.WithTimeout(ctx, 15*time.Second)
defer cancel()
req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, s.endpoint, bytes.NewReader(buf))
if err != nil {
res.Error = err.Error()
return []SourceResult{res}
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Auth-Key", authKey)
body, status, err := httpDo(req, 4<<20)
if err != nil {
res.Error = err.Error()
return []SourceResult{res}
}
if status != http.StatusOK {
res.Error = fmt.Sprintf("HTTP %d: %s", status, truncate(string(body), 200))
return []SourceResult{res}
}
var parsed struct {
QueryStatus string `json:"query_status"`
Data []struct {
SHA256Hash string `json:"sha256_hash"`
FileName string `json:"file_name"`
FileType string `json:"file_type_mime"`
Signature string `json:"signature"`
FirstSeen string `json:"first_seen"`
} `json:"data"`
}
if err := json.Unmarshal(body, &parsed); err != nil {
res.Error = "decode: " + err.Error()
return []SourceResult{res}
}
switch parsed.QueryStatus {
case "ok":
signatures := map[string]bool{}
for _, sample := range parsed.Data {
if sample.Signature != "" && !signatures[sample.Signature] {
signatures[sample.Signature] = true
res.Reasons = append(res.Reasons, sample.Signature)
}
res.Evidence = append(res.Evidence, Evidence{
Label: "Sample",
Value: sample.SHA256Hash,
Status: sample.FileType,
Extra: map[string]string{
"filename": sample.FileName,
"signature": sample.Signature,
"first_seen": sample.FirstSeen,
},
})
}
case "no_results", "illegal_search_term":
// Clean.
default:
res.Error = "query_status=" + parsed.QueryStatus
}
return []SourceResult{res}
}
func (*malwareBazaarSource) Evaluate(r SourceResult) (bool, string) {
return evidenceEval(r, SeverityWarn)
}
func (*malwareBazaarSource) Diagnose(res SourceResult) Diagnosis {
return Diagnosis{
Severity: SeverityWarn,
Title: "Associated with malware samples in abuse.ch MalwareBazaar",
Detail: fmt.Sprintf(
"%d malware sample(s) are tagged with this domain; malware family/signature(s): %s. MalwareBazaar samples are tagged with their C2 domain or delivery host when known. Investigate the domain's hosting and DNS records to identify and remove malicious infrastructure.",
len(res.Evidence), joinNonEmpty(res.Reasons, ", "),
),
Fix: res.Reference,
FixIsURL: res.Reference != "",
}
}

View file

@ -0,0 +1,94 @@
package checker
import (
"context"
"net/http"
"net/http/httptest"
"testing"
sdk "git.happydns.org/checker-sdk-go/checker"
)
func TestMalwareBazaarSource_NoResults(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"query_status":"no_results"}`))
}))
defer srv.Close()
s := &malwareBazaarSource{endpoint: srv.URL}
results := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_malwarebazaar": true, "malwarebazaar_auth_key": "k"})
if len(results) != 1 {
t.Fatalf("expected 1 result, got %d", len(results))
}
r := results[0]
listed, _ := s.Evaluate(r)
if !r.Enabled || listed || r.Error != "" {
t.Fatalf("expected enabled+clean, got %+v, Evaluate listed=%v", r, listed)
}
}
func TestMalwareBazaarSource_Listed(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Auth-Key") == "" {
t.Errorf("missing Auth-Key header")
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{
"query_status": "ok",
"data": [{
"sha256_hash": "aaaa1111bbbb2222cccc3333dddd4444eeee5555ffff6666aaaa1111bbbb2222",
"file_name": "evil.exe",
"file_type_mime": "application/x-dosexec",
"signature": "Emotet",
"first_seen": "2024-01-01 00:00:00"
}]
}`))
}))
defer srv.Close()
s := &malwareBazaarSource{endpoint: srv.URL}
r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_malwarebazaar": true, "malwarebazaar_auth_key": "k"})[0]
if len(r.Evidence) != 1 {
t.Fatalf("expected 1 evidence item, got %+v", r)
}
if listed, severity := s.Evaluate(r); !listed || severity != SeverityWarn {
t.Errorf("expected Evaluate()=(true, warn), got (%v, %q)", listed, severity)
}
if r.Evidence[0].Status != "application/x-dosexec" {
t.Errorf("evidence status = %q", r.Evidence[0].Status)
}
if len(r.Reasons) != 1 || r.Reasons[0] != "Emotet" {
t.Errorf("reasons = %v", r.Reasons)
}
}
func TestMalwareBazaarSource_HTTPError(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusUnauthorized)
_, _ = w.Write([]byte("unauthorized"))
}))
defer srv.Close()
s := &malwareBazaarSource{endpoint: srv.URL}
r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_malwarebazaar": true, "malwarebazaar_auth_key": "k"})[0]
if r.Error == "" {
t.Errorf("expected error, got %+v", r)
}
}
func TestMalwareBazaarSource_Disabled(t *testing.T) {
s := &malwareBazaarSource{endpoint: "http://nope"}
r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_malwarebazaar": false})[0]
if r.Enabled {
t.Errorf("expected disabled, got %+v", r)
}
}
func TestMalwareBazaarSource_NoAuthKey(t *testing.T) {
s := &malwareBazaarSource{endpoint: "http://nope"}
r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_malwarebazaar": true})[0]
if r.Enabled {
t.Errorf("expected disabled when no auth key, got %+v", r)
}
}

212
checker/oisd.go Normal file
View file

@ -0,0 +1,212 @@
package checker
import (
"bufio"
"context"
"fmt"
"io"
"net/http"
"strings"
"sync"
"time"
sdk "git.happydns.org/checker-sdk-go/checker"
)
const (
oisdBigFeedURL = "https://big.oisd.nl/domainswild"
oisdSmallFeedURL = "https://small.oisd.nl/domainswild"
oisdDefaultTTL = 24 * time.Hour
oisdFailBackoff = 1 * time.Minute
)
func init() {
Register(&oisdSource{
bigCache: newOisdCache(oisdBigFeedURL),
smallCache: newOisdCache(oisdSmallFeedURL),
})
}
type oisdSource struct {
bigCache *oisdCache
smallCache *oisdCache
}
func (*oisdSource) ID() string { return "oisd" }
func (*oisdSource) Name() string { return "OISD domain blocklist" }
func (*oisdSource) Options() SourceOptions {
return SourceOptions{
User: []sdk.CheckerOptionField{
{
Id: "enable_oisd",
Type: "bool",
Label: "Use the OISD domain blocklist",
Description: "Download the OISD domain blocklist (refreshed every 24h) and check the domain against it.",
Default: true,
},
},
Admin: []sdk.CheckerOptionField{
{
Id: "oisd_variant",
Type: "string",
Label: "OISD blocklist variant",
Description: `Which OISD list to use: "big" (~250 k entries, recommended) or "small" (~50 k entries).`,
Default: "big",
},
},
}
}
func (s *oisdSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult {
if !sdk.GetBoolOption(opts, "enable_oisd", true) || registered == "" {
return disabledResult(s.ID(), s.Name())
}
cache := s.bigCache
if stringOptDefault(opts, "oisd_variant", "big") == "small" {
cache = s.smallCache
}
matched, size, fetched, err := cache.lookup(ctx, registered)
res := SourceResult{
SourceID: s.ID(), SourceName: s.Name(), Enabled: true,
Reference: "https://oisd.nl/",
Details: mustJSON(map[string]any{"feed_size": size, "fetched_at": fetched}),
}
if err != nil {
res.Error = err.Error()
}
if len(matched) > 0 {
res.Reasons = []string{"Listed in OISD"}
for _, d := range matched {
res.Evidence = append(res.Evidence, Evidence{Label: "Domain", Value: d})
}
}
return []SourceResult{res}
}
func (*oisdSource) Evaluate(r SourceResult) (bool, string) {
return evidenceEval(r, SeverityCrit)
}
func (*oisdSource) Diagnose(res SourceResult) Diagnosis {
domains := make([]string, 0, len(res.Evidence))
for _, e := range res.Evidence {
domains = append(domains, e.Value)
}
previewN := min(len(domains), 5)
return Diagnosis{
Severity: SeverityCrit,
Title: "Listed in the OISD domain blocklist",
Detail: fmt.Sprintf(
"%d domain(s) matching this registered domain appear in the OISD blocklist; examples: %s. OISD is a large curated blocklist used by DNS resolvers, ad blockers, and firewalls worldwide. Domains listed here are blocked for millions of users. Investigate whether the domain hosts ads, trackers, or malware, or whether it has been compromised.",
len(domains), joinNonEmpty(domains[:previewN], ", "),
),
Fix: "https://oisd.nl/",
FixIsURL: true,
}
}
// ---------- cache ----------
type oisdCache struct {
mu sync.Mutex
domains []string
byDomain map[string]struct{}
fetchedAt time.Time
lastAttemptAt time.Time
refreshing bool
ttl time.Duration
failBackoff time.Duration
feedURL string
}
func newOisdCache(feedURL string) *oisdCache {
return &oisdCache{
ttl: oisdDefaultTTL,
failBackoff: oisdFailBackoff,
feedURL: feedURL,
}
}
func (c *oisdCache) lookup(ctx context.Context, registered string) (matched []string, size int, fetchedAt time.Time, err error) {
registered = strings.ToLower(strings.TrimSuffix(registered, "."))
c.mu.Lock()
stale := c.byDomain == nil || time.Since(c.fetchedAt) > c.ttl
doRefresh := stale && !c.refreshing && time.Since(c.lastAttemptAt) > c.failBackoff
if doRefresh {
c.refreshing = true
}
feedURL := c.feedURL
c.mu.Unlock()
if doRefresh {
newDomains, newByDomain, ferr := c.fetch(ctx, feedURL)
c.mu.Lock()
c.refreshing = false
c.lastAttemptAt = time.Now()
if ferr == nil {
c.domains = newDomains
c.byDomain = newByDomain
c.fetchedAt = c.lastAttemptAt
} else {
err = ferr
}
c.mu.Unlock()
}
c.mu.Lock()
suffix := "." + registered
for d := range c.byDomain {
if d == registered || strings.HasSuffix(d, suffix) {
matched = append(matched, d)
}
}
size = len(c.domains)
fetchedAt = c.fetchedAt
c.mu.Unlock()
return matched, size, fetchedAt, err
}
func (c *oisdCache) fetch(ctx context.Context, feedURL string) ([]string, map[string]struct{}, error) {
reqCtx, cancel := context.WithTimeout(ctx, 60*time.Second)
defer cancel()
req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, feedURL, nil)
if err != nil {
return nil, nil, err
}
req.Header.Set("User-Agent", "happydomain-checker-blacklist/1.0")
resp, err := sharedHTTPClient.Do(req)
if err != nil {
return nil, nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, nil, fmt.Errorf("oisd HTTP %d", resp.StatusCode)
}
domains := make([]string, 0, 262144)
byDomain := make(map[string]struct{}, 262144)
scanner := bufio.NewScanner(io.LimitReader(resp.Body, 64<<20))
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" || strings.HasPrefix(line, "!") || strings.HasPrefix(line, "#") {
continue
}
d := strings.TrimPrefix(strings.ToLower(line), "*.")
if d == "" {
continue
}
domains = append(domains, d)
byDomain[d] = struct{}{}
}
if err := scanner.Err(); err != nil {
return nil, nil, err
}
return domains, byDomain, nil
}

95
checker/oisd_test.go Normal file
View file

@ -0,0 +1,95 @@
package checker
import (
"context"
"net/http"
"net/http/httptest"
"testing"
sdk "git.happydns.org/checker-sdk-go/checker"
)
const oisdFakeFeed = `! OISD big domainswild
# comment line
*.evil.com
*.malware.example.org
*.c2.badactor.net
`
func TestOisdSource_Listed_ExactMatch(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte(oisdFakeFeed))
}))
defer srv.Close()
s := &oisdSource{bigCache: newOisdCache(srv.URL)}
r := s.Query(context.Background(), "evil.com", "evil.com", sdk.CheckerOptions{"enable_oisd": true})[0]
if !r.Enabled || r.Error != "" {
t.Fatalf("expected enabled and no error, got %+v", r)
}
if len(r.Evidence) != 1 || r.Evidence[0].Value != "evil.com" {
t.Errorf("expected evidence [evil.com], got %+v", r.Evidence)
}
if listed, sev := s.Evaluate(r); !listed || sev != SeverityCrit {
t.Errorf("expected (true, crit), got (%v, %q)", listed, sev)
}
}
func TestOisdSource_Listed_SubdomainInFeed(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte(oisdFakeFeed))
}))
defer srv.Close()
// Feed has "*.malware.example.org" → stored as "malware.example.org"; querying
// registered "example.org" should match via suffix check.
s := &oisdSource{bigCache: newOisdCache(srv.URL)}
r := s.Query(context.Background(), "sub.example.org", "example.org", sdk.CheckerOptions{"enable_oisd": true})[0]
if len(r.Evidence) != 1 || r.Evidence[0].Value != "malware.example.org" {
t.Errorf("expected subdomain match, got %+v", r.Evidence)
}
if listed, _ := s.Evaluate(r); !listed {
t.Error("expected listed=true")
}
}
func TestOisdSource_NotListed(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte(oisdFakeFeed))
}))
defer srv.Close()
s := &oisdSource{bigCache: newOisdCache(srv.URL)}
r := s.Query(context.Background(), "safe.com", "safe.com", sdk.CheckerOptions{"enable_oisd": true})[0]
if !r.Enabled || r.Error != "" || len(r.Evidence) != 0 {
t.Fatalf("expected clean result, got %+v", r)
}
if listed, _ := s.Evaluate(r); listed {
t.Error("expected listed=false")
}
}
func TestOisdSource_Disabled(t *testing.T) {
s := &oisdSource{bigCache: newOisdCache("http://nope")}
r := s.Query(context.Background(), "evil.com", "evil.com", sdk.CheckerOptions{"enable_oisd": false})[0]
if r.Enabled {
t.Errorf("expected disabled, got %+v", r)
}
}
func TestOisdSource_HTTPError(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
}))
defer srv.Close()
s := &oisdSource{bigCache: newOisdCache(srv.URL)}
r := s.Query(context.Background(), "evil.com", "evil.com", sdk.CheckerOptions{"enable_oisd": true})[0]
if r.Error == "" || r.Error != "oisd HTTP 500" {
t.Errorf("expected HTTP 500 error, got %q", r.Error)
}
}

View file

@ -6,9 +6,7 @@ import (
"fmt"
"io"
"net/http"
"net/url"
"strings"
"sync"
"time"
sdk "git.happydns.org/checker-sdk-go/checker"
@ -18,7 +16,7 @@ const openPhishFeedURL = "https://openphish.com/feed.txt"
func init() {
Register(&openPhishSource{
cache: newPhishCache(openPhishFeedURL, 1*time.Hour),
cache: newFeedCache(1*time.Hour, openPhishFetch(openPhishFeedURL)),
})
}
@ -27,7 +25,7 @@ func init() {
// every URL in the feed. The cache is per-source-instance so it lives
// for as long as the process.
type openPhishSource struct {
cache *phishCache
cache *feedCache
}
func (*openPhishSource) ID() string { return "openphish" }
@ -49,7 +47,7 @@ func (*openPhishSource) Options() SourceOptions {
func (s *openPhishSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult {
if !sdk.GetBoolOption(opts, "enable_openphish", true) || registered == "" {
return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: false}}
return disabledResult(s.ID(), s.Name())
}
urls, size, fetched, err := s.cache.lookup(ctx, registered)
@ -63,8 +61,6 @@ func (s *openPhishSource) Query(ctx context.Context, domain, registered string,
// Fall through with whatever the cache could provide.
}
if len(urls) > 0 {
res.Listed = true
res.Severity = SeverityCrit
res.Reasons = []string{"Phishing"}
for _, u := range urls {
res.Evidence = append(res.Evidence, Evidence{Label: "URL", Value: u})
@ -73,6 +69,10 @@ func (s *openPhishSource) Query(ctx context.Context, domain, registered string,
return []SourceResult{res}
}
func (*openPhishSource) Evaluate(r SourceResult) (bool, string) {
return evidenceEval(r, SeverityCrit)
}
func (*openPhishSource) Diagnose(res SourceResult) Diagnosis {
urls := make([]string, 0, len(res.Evidence))
for _, e := range res.Evidence {
@ -91,75 +91,14 @@ func (*openPhishSource) Diagnose(res SourceResult) Diagnosis {
}
}
// ---------- feed cache ----------
type phishCache struct {
mu sync.Mutex
urls []string
byHost map[string][]string
fetchedAt time.Time
lastAttemptAt time.Time
refreshing bool
ttl time.Duration
failBackoff time.Duration
feedURL string
}
func newPhishCache(feedURL string, ttl time.Duration) *phishCache {
if feedURL == "" {
feedURL = openPhishFeedURL
}
if ttl <= 0 {
ttl = 1 * time.Hour
}
return &phishCache{ttl: ttl, feedURL: feedURL, failBackoff: 1 * time.Minute}
}
func (c *phishCache) lookup(ctx context.Context, domain string) (urls []string, size int, fetchedAt time.Time, err error) {
domain = strings.ToLower(strings.TrimSuffix(domain, "."))
c.mu.Lock()
stale := c.byHost == nil || time.Since(c.fetchedAt) > c.ttl
doRefresh := stale && !c.refreshing && time.Since(c.lastAttemptAt) > c.failBackoff
if doRefresh {
c.refreshing = true
}
c.mu.Unlock()
if doRefresh {
// Fetch without holding the cache lock so concurrent lookups
// can still serve stale data. Only one refresh runs at a time.
newURLs, newByHost, ferr := c.fetch(ctx)
c.mu.Lock()
c.refreshing = false
c.lastAttemptAt = time.Now()
if ferr == nil {
c.urls = newURLs
c.byHost = newByHost
c.fetchedAt = c.lastAttemptAt
} else {
err = ferr
}
c.mu.Unlock()
}
c.mu.Lock()
for host, hostURLs := range c.byHost {
if host == domain || strings.HasSuffix(host, "."+domain) {
urls = append(urls, hostURLs...)
}
}
size = len(c.urls)
fetchedAt = c.fetchedAt
c.mu.Unlock()
return urls, size, fetchedAt, err
}
func (c *phishCache) fetch(ctx context.Context) ([]string, map[string][]string, error) {
// openPhishFetch returns a fetchFn that downloads and parses the
// OpenPhish plain-text feed at feedURL.
func openPhishFetch(feedURL string) func(context.Context) ([]string, map[string][]string, error) {
return func(ctx context.Context) ([]string, map[string][]string, error) {
reqCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, c.feedURL, nil)
req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, feedURL, nil)
if err != nil {
return nil, nil, err
}
@ -193,12 +132,5 @@ func (c *phishCache) fetch(ctx context.Context) ([]string, map[string][]string,
return nil, nil, err
}
return urls, byHost, nil
}
func hostOfURL(s string) string {
u, err := url.Parse(s)
if err != nil {
return ""
}
return strings.ToLower(u.Hostname())
}

208
checker/otx.go Normal file
View file

@ -0,0 +1,208 @@
package checker
import (
"bytes"
"context"
"encoding/json"
"fmt"
"html/template"
"net/http"
"time"
sdk "git.happydns.org/checker-sdk-go/checker"
)
const otxEndpoint = "https://otx.alienvault.com/api/v1/indicators/domain/"
func init() { Register(&otxSource{endpoint: otxEndpoint}) }
type otxSource struct{ endpoint string }
func (*otxSource) ID() string { return "otx" }
func (*otxSource) Name() string { return "AlienVault OTX" }
func (*otxSource) Options() SourceOptions {
return SourceOptions{
Admin: []sdk.CheckerOptionField{
{
Id: "otx_api_key",
Type: "string",
Label: "AlienVault OTX API key",
Description: "Free OTX API key from otx.alienvault.com. Leave empty to skip OTX lookups.",
Secret: true,
},
},
}
}
type otxDetails struct {
PulseCount int `json:"pulse_count"`
Reputation int `json:"reputation"`
Pulses []otxPulse `json:"pulses"`
}
type otxPulse struct {
Name string `json:"name"`
Tags []string `json:"tags,omitempty"`
MalwareFamilies []string `json:"malware_families,omitempty"`
Adversary string `json:"adversary,omitempty"`
Created string `json:"created,omitempty"`
}
func (s *otxSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult {
apiKey := stringOpt(opts, "otx_api_key")
if apiKey == "" {
return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: false}}
}
if registered == "" {
return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: true}}
}
res := SourceResult{
SourceID: s.ID(),
SourceName: s.Name(),
Enabled: true,
Reference: "https://otx.alienvault.com/indicator/domain/" + registered,
}
reqCtx, cancel := context.WithTimeout(ctx, 15*time.Second)
defer cancel()
req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, s.endpoint+registered+"/general", nil)
if err != nil {
res.Error = err.Error()
return []SourceResult{res}
}
req.Header.Set("X-OTX-API-KEY", apiKey)
req.Header.Set("Accept", "application/json")
body, status, err := httpDo(req, 4<<20)
if err != nil {
res.Error = err.Error()
return []SourceResult{res}
}
if status == http.StatusNotFound {
return []SourceResult{res}
}
if status != http.StatusOK {
res.Error = fmt.Sprintf("HTTP %d: %s", status, truncate(string(body), 200))
return []SourceResult{res}
}
var parsed struct {
Reputation int `json:"reputation"`
PulseInfo struct {
Count int `json:"count"`
Pulses []struct {
Name string `json:"name"`
Tags []string `json:"tags"`
MalwareFamilies []struct {
DisplayName string `json:"display_name"`
} `json:"malware_families"`
Adversary string `json:"adversary"`
Created string `json:"created"`
} `json:"pulses"`
} `json:"pulse_info"`
}
if err := json.Unmarshal(body, &parsed); err != nil {
res.Error = "decode: " + err.Error()
return []SourceResult{res}
}
d := otxDetails{
PulseCount: parsed.PulseInfo.Count,
Reputation: parsed.Reputation,
}
seenReason := map[string]bool{}
for _, p := range parsed.PulseInfo.Pulses {
pulse := otxPulse{
Name: p.Name,
Tags: p.Tags,
Adversary: p.Adversary,
Created: p.Created,
}
for _, mf := range p.MalwareFamilies {
pulse.MalwareFamilies = append(pulse.MalwareFamilies, mf.DisplayName)
if !seenReason[mf.DisplayName] {
seenReason[mf.DisplayName] = true
res.Reasons = append(res.Reasons, mf.DisplayName)
}
}
if p.Adversary != "" && !seenReason[p.Adversary] {
seenReason[p.Adversary] = true
res.Reasons = append(res.Reasons, p.Adversary)
}
d.Pulses = append(d.Pulses, pulse)
res.Evidence = append(res.Evidence, Evidence{
Label: "Pulse",
Value: p.Name,
Status: "threat",
})
}
res.Details = mustJSON(d)
return []SourceResult{res}
}
func (*otxSource) Evaluate(r SourceResult) (bool, string) {
if !r.Enabled || r.Error != "" || len(r.Evidence) == 0 {
return false, ""
}
var d otxDetails
_ = json.Unmarshal(r.Details, &d)
if d.Reputation < -1 {
return true, SeverityCrit
}
return true, SeverityWarn
}
func (*otxSource) Diagnose(res SourceResult) Diagnosis {
var d otxDetails
_ = json.Unmarshal(res.Details, &d)
detail := fmt.Sprintf(
"%d threat pulse(s) reference this domain (OTX reputation score: %d). Indicators: %s. "+
"Review the pulse details on AlienVault OTX to understand the threat context and take corrective action.",
d.PulseCount, d.Reputation, joinNonEmpty(res.Reasons, ", "),
)
sev := SeverityWarn
if d.Reputation < -1 {
sev = SeverityCrit
}
return Diagnosis{
Severity: sev,
Title: "Listed in AlienVault OTX threat pulses",
Detail: detail,
Fix: res.Reference,
FixIsURL: res.Reference != "",
}
}
func (*otxSource) RenderDetail(res SourceResult) (template.HTML, error) {
var d otxDetails
if len(res.Details) > 0 {
if err := json.Unmarshal(res.Details, &d); err != nil {
return "", fmt.Errorf("otx: decode details: %w", err)
}
}
if len(d.Pulses) == 0 {
return "", nil
}
var b bytes.Buffer
if err := otxDetailTpl.Execute(&b, d); err != nil {
return "", err
}
return template.HTML(b.String()), nil
}
var otxDetailTpl = template.Must(template.New("otx_detail").Parse(`
<p>OTX reputation score: <strong>{{.Reputation}}</strong>. Pulse count: <strong>{{.PulseCount}}</strong>.</p>
<table>
<thead><tr><th>Pulse</th><th>Malware families</th><th>Adversary</th><th>Tags</th><th>Created</th></tr></thead>
<tbody>{{range .Pulses}}<tr class="row-crit">
<td>{{.Name}}</td>
<td>{{range .MalwareFamilies}}<span>{{.}} </span>{{end}}</td>
<td>{{.Adversary}}</td>
<td>{{range .Tags}}<span>{{.}} </span>{{end}}</td>
<td><small>{{.Created}}</small></td>
</tr>{{end}}</tbody>
</table>`))

144
checker/otx_test.go Normal file
View file

@ -0,0 +1,144 @@
package checker
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
sdk "git.happydns.org/checker-sdk-go/checker"
)
func newOTXServer(t *testing.T, status int, body string) (string, func()) {
t.Helper()
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("X-OTX-API-KEY") == "" {
t.Errorf("missing X-OTX-API-KEY header")
}
w.WriteHeader(status)
_, _ = w.Write([]byte(body))
}))
return srv.URL + "/", srv.Close
}
func TestOTXSource_NoKey(t *testing.T) {
s := &otxSource{endpoint: otxEndpoint}
r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{})[0]
if r.Enabled {
t.Errorf("expected disabled without API key, got %+v", r)
}
}
func TestOTXSource_Listed(t *testing.T) {
body := `{
"reputation": 0,
"pulse_info": {
"count": 1,
"pulses": [{
"name": "Test Pulse",
"tags": ["phishing"],
"malware_families": [{"display_name": "Emotet"}],
"adversary": "",
"created": "2024-01-01T00:00:00.000Z"
}]
}
}`
endpoint, stop := newOTXServer(t, http.StatusOK, body)
defer stop()
s := &otxSource{endpoint: endpoint}
r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"otx_api_key": "k"})[0]
if len(r.Evidence) != 1 {
t.Fatalf("expected 1 evidence entry, got %d", len(r.Evidence))
}
if listed, severity := s.Evaluate(r); !listed || severity != SeverityWarn {
t.Errorf("expected Evaluate()=(true, warn), got (%v, %q)", listed, severity)
}
var d otxDetails
if err := json.Unmarshal(r.Details, &d); err != nil {
t.Fatalf("details decode: %v", err)
}
if d.PulseCount != 1 || d.Reputation != 0 {
t.Errorf("details wrong: %+v", d)
}
if len(d.Pulses) != 1 || len(d.Pulses[0].MalwareFamilies) != 1 || d.Pulses[0].MalwareFamilies[0] != "Emotet" {
t.Errorf("pulse details wrong: %+v", d.Pulses)
}
html, err := s.RenderDetail(r)
if err != nil || !strings.Contains(string(html), "Emotet") {
t.Errorf("RenderDetail html=%q err=%v", html, err)
}
}
func TestOTXSource_ListedCrit(t *testing.T) {
body := `{
"reputation": -2,
"pulse_info": {
"count": 5,
"pulses": [{"name": "APT Pulse", "tags": [], "malware_families": [], "adversary": "APT28", "created": ""}]
}
}`
endpoint, stop := newOTXServer(t, http.StatusOK, body)
defer stop()
s := &otxSource{endpoint: endpoint}
r := s.Query(context.Background(), "evil.com", "evil.com", sdk.CheckerOptions{"otx_api_key": "k"})[0]
if listed, severity := s.Evaluate(r); !listed || severity != SeverityCrit {
t.Errorf("expected Evaluate()=(true, crit) for reputation -2, got (%v, %q)", listed, severity)
}
}
func TestOTXSource_NotFound(t *testing.T) {
endpoint, stop := newOTXServer(t, http.StatusNotFound, `{"detail":"Not found"}`)
defer stop()
s := &otxSource{endpoint: endpoint}
r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"otx_api_key": "k"})[0]
if r.Error != "" {
t.Errorf("404 should be quiet not-listed, got Error=%q", r.Error)
}
if len(r.Evidence) != 0 {
t.Errorf("expected no evidence for 404, got %+v", r.Evidence)
}
if listed, _ := s.Evaluate(r); listed {
t.Errorf("Evaluate() on clean result should return false")
}
if !strings.Contains(r.Reference, "example.com") {
t.Errorf("reference URL missing domain: %+v", r)
}
}
func TestOTXSource_HTTPError(t *testing.T) {
endpoint, stop := newOTXServer(t, http.StatusInternalServerError, `{"error":"internal"}`)
defer stop()
s := &otxSource{endpoint: endpoint}
r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"otx_api_key": "k"})[0]
if r.Error == "" {
t.Errorf("expected non-empty Error for HTTP 500, got %+v", r)
}
}
func TestOTXSource_NoResults(t *testing.T) {
body := `{"reputation": 0, "pulse_info": {"count": 0, "pulses": []}}`
endpoint, stop := newOTXServer(t, http.StatusOK, body)
defer stop()
s := &otxSource{endpoint: endpoint}
r := s.Query(context.Background(), "clean.com", "clean.com", sdk.CheckerOptions{"otx_api_key": "k"})[0]
if len(r.Evidence) != 0 {
t.Errorf("expected no evidence for clean domain, got %+v", r.Evidence)
}
if listed, severity := s.Evaluate(r); listed || severity != "" {
t.Errorf("Evaluate() on clean domain = (%v, %q), want (false, \"\")", listed, severity)
}
}

174
checker/phishtank.go Normal file
View file

@ -0,0 +1,174 @@
package checker
import (
"bufio"
"compress/gzip"
"context"
"encoding/csv"
"fmt"
"io"
"net/http"
"strconv"
"strings"
"time"
sdk "git.happydns.org/checker-sdk-go/checker"
)
const (
phishTankFeedURL = "https://data.phishtank.com/data/online-valid.csv.gz"
phishTankDefaultTTL = 12 * time.Hour
)
var phishTankGlobalCache = newFeedCache(phishTankDefaultTTL, phishTankFetch)
func init() { Register(&phishTankSource{}) }
type phishTankSource struct{}
func (*phishTankSource) ID() string { return "phishtank" }
func (*phishTankSource) Name() string { return "PhishTank" }
func (*phishTankSource) Options() SourceOptions {
return SourceOptions{
Admin: []sdk.CheckerOptionField{
{
Id: "phishtank_refresh_hours",
Type: "string",
Label: "PhishTank feed refresh interval (hours)",
Description: "How often to re-download the PhishTank online-valid feed. Minimum: 1. Default: 12.",
Default: "12",
},
},
User: []sdk.CheckerOptionField{
{
Id: "enable_phishtank",
Type: "bool",
Label: "Use the PhishTank feed",
Description: "Download the PhishTank verified phishing list and check the domain against it.",
Default: true,
},
},
}
}
func (s *phishTankSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult {
if !sdk.GetBoolOption(opts, "enable_phishtank", true) || registered == "" {
return disabledResult(s.ID(), s.Name())
}
if ttlRaw, ok := sdk.GetOption[string](opts, "phishtank_refresh_hours"); ok && ttlRaw != "" {
if hours, err := strconv.Atoi(ttlRaw); err == nil && hours >= 1 {
phishTankGlobalCache.setTTL(time.Duration(hours) * time.Hour)
}
}
urls, size, fetched, err := phishTankGlobalCache.lookup(ctx, registered)
res := SourceResult{
SourceID: s.ID(), SourceName: s.Name(), Enabled: true,
Reference: "https://www.phishtank.com/",
Details: mustJSON(map[string]any{"feed_size": size, "fetched_at": fetched}),
}
if err != nil {
res.Error = err.Error()
}
if len(urls) > 0 {
res.Reasons = []string{"Phishing"}
for _, u := range urls {
res.Evidence = append(res.Evidence, Evidence{Label: "URL", Value: u})
}
}
return []SourceResult{res}
}
func (*phishTankSource) Evaluate(r SourceResult) (bool, string) {
return evidenceEval(r, SeverityCrit)
}
func (*phishTankSource) Diagnose(res SourceResult) Diagnosis {
urls := make([]string, 0, len(res.Evidence))
for _, e := range res.Evidence {
urls = append(urls, e.Value)
}
previewN := min(len(urls), 5)
return Diagnosis{
Severity: SeverityCrit,
Title: "Listed in the PhishTank phishing database",
Detail: fmt.Sprintf(
"%d URL(s) hosted on this domain are tracked as verified phishing by PhishTank. Examples: %s",
len(urls), joinNonEmpty(urls[:previewN], ", "),
),
Fix: "https://www.phishtank.com/developer_info.php",
FixIsURL: true,
}
}
// phishTankFetch downloads and parses the PhishTank gzip-compressed CSV feed.
func phishTankFetch(ctx context.Context) ([]string, map[string][]string, error) {
reqCtx, cancel := context.WithTimeout(ctx, 120*time.Second)
defer cancel()
req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, phishTankFeedURL, nil)
if err != nil {
return nil, nil, err
}
req.Header.Set("User-Agent", "happydomain-checker-blacklist/1.0")
resp, err := sharedHTTPClient.Do(req)
if err != nil {
return nil, nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, nil, fmt.Errorf("phishtank HTTP %d", resp.StatusCode)
}
gz, err := gzip.NewReader(io.LimitReader(resp.Body, 128<<20))
if err != nil {
return nil, nil, fmt.Errorf("phishtank gzip: %w", err)
}
defer gz.Close()
r := csv.NewReader(bufio.NewReader(gz))
r.ReuseRecord = true
header, err := r.Read()
if err != nil {
return nil, nil, fmt.Errorf("phishtank csv header: %w", err)
}
urlIdx := -1
for i, col := range header {
if col == "url" {
urlIdx = i
break
}
}
if urlIdx < 0 {
return nil, nil, fmt.Errorf("phishtank csv: no 'url' column in header")
}
urls := make([]string, 0, 32768)
byHost := make(map[string][]string, 32768)
for {
record, err := r.Read()
if err == io.EOF {
break
}
if err != nil {
return nil, nil, fmt.Errorf("phishtank csv: %w", err)
}
if urlIdx >= len(record) {
continue
}
u := strings.TrimSpace(record[urlIdx])
if u == "" {
continue
}
urls = append(urls, u)
if h := hostOfURL(u); h != "" {
byHost[h] = append(byHost[h], u)
}
}
return urls, byHost, nil
}

View file

@ -37,7 +37,7 @@ func (p *blacklistProvider) ExtractMetrics(ctx sdk.ReportContext, collectedAt ti
continue
}
v := 0.0
if r.Listed {
if listed, _ := EvaluateResult(r); listed {
v = 1
}
metrics = append(metrics, sdk.CheckMetric{

163
checker/pulsedive.go Normal file
View file

@ -0,0 +1,163 @@
package checker
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/url"
"time"
sdk "git.happydns.org/checker-sdk-go/checker"
)
const pulsediveEndpoint = "https://pulsedive.com/api/info.php"
func init() { Register(&pulsediveSource{endpoint: pulsediveEndpoint}) }
type pulsediveSource struct{ endpoint string }
func (*pulsediveSource) ID() string { return "pulsedive" }
func (*pulsediveSource) Name() string { return "Pulsedive" }
func (*pulsediveSource) Options() SourceOptions {
return SourceOptions{
Admin: []sdk.CheckerOptionField{
{
Id: "pulsedive_api_key",
Type: "string",
Label: "Pulsedive API key",
Description: "Pulsedive API key (free account at pulsedive.com). Leave empty to skip Pulsedive lookups.",
Secret: true,
},
},
}
}
type pulsediveDetails struct {
Risk string `json:"risk"`
Threats []pulsediveThreat `json:"threats"`
}
type pulsediveThreat struct {
Name string `json:"name"`
Category string `json:"category"`
Risk string `json:"risk,omitempty"`
}
func (s *pulsediveSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult {
apiKey := stringOpt(opts, "pulsedive_api_key")
if apiKey == "" || registered == "" {
return disabledResult(s.ID(), s.Name())
}
res := SourceResult{
SourceID: s.ID(),
SourceName: s.Name(),
Enabled: true,
Reference: "https://pulsedive.com/indicator/" + registered,
}
params := url.Values{
"indicator": {registered},
"key": {apiKey},
}
reqURL := s.endpoint + "?" + params.Encode()
reqCtx, cancel := context.WithTimeout(ctx, 15*time.Second)
defer cancel()
req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, reqURL, nil)
if err != nil {
res.Error = redactSecret(err.Error(), apiKey)
return []SourceResult{res}
}
req.Header.Set("User-Agent", "happydomain-checker-blacklist/1.0")
body, status, err := httpDo(req, 1<<20)
if err != nil {
res.Error = redactSecret(err.Error(), apiKey)
return []SourceResult{res}
}
if status != http.StatusOK {
res.Error = fmt.Sprintf("HTTP %d: %s", status, truncate(string(body), 200))
return []SourceResult{res}
}
// Check for "not found" before full parse — Pulsedive returns 200
// with {"error": "Indicator not found."} for unknown indicators.
var errEnvelope struct {
Error string `json:"error"`
}
if json.Unmarshal(body, &errEnvelope) == nil && errEnvelope.Error != "" {
return []SourceResult{res}
}
var parsed struct {
Risk string `json:"risk"`
Threats []struct {
Name string `json:"name"`
Category string `json:"category"`
Risk string `json:"risk"`
} `json:"threats"`
}
if err := json.Unmarshal(body, &parsed); err != nil {
res.Error = "decode: " + err.Error()
return []SourceResult{res}
}
d := pulsediveDetails{Risk: parsed.Risk}
seen := map[string]bool{}
for _, t := range parsed.Threats {
d.Threats = append(d.Threats, pulsediveThreat{
Name: t.Name,
Category: t.Category,
Risk: t.Risk,
})
if !seen[t.Name] {
seen[t.Name] = true
res.Reasons = append(res.Reasons, t.Name)
}
res.Evidence = append(res.Evidence, Evidence{
Label: "Threat",
Value: t.Name,
Status: t.Category,
})
}
res.Details = mustJSON(d)
return []SourceResult{res}
}
func (*pulsediveSource) Evaluate(r SourceResult) (bool, string) {
if !r.Enabled || r.Error != "" || len(r.Evidence) == 0 {
return false, ""
}
var d pulsediveDetails
_ = json.Unmarshal(r.Details, &d)
switch d.Risk {
case "critical", "high":
return true, SeverityCrit
default:
return true, SeverityWarn
}
}
func (*pulsediveSource) Diagnose(res SourceResult) Diagnosis {
var d pulsediveDetails
_ = json.Unmarshal(res.Details, &d)
previewN := min(len(d.Threats), 5)
names := make([]string, 0, previewN)
for _, t := range d.Threats[:previewN] {
names = append(names, t.Name)
}
return Diagnosis{
Severity: SeverityCrit,
Title: fmt.Sprintf("Pulsedive risk: %s — %d threat(s) associated", d.Risk, len(d.Threats)),
Detail: fmt.Sprintf(
"Pulsedive assigned a risk of %q to this domain. Associated threat(s): %s. Review the indicator page for feed context, related IPs, and historical activity, then follow up with the relevant threat's removal or remediation procedure.",
d.Risk, joinNonEmpty(names, ", "),
),
Fix: res.Reference,
FixIsURL: res.Reference != "",
}
}

103
checker/pulsedive_test.go Normal file
View file

@ -0,0 +1,103 @@
package checker
import (
"context"
"net/http"
"net/http/httptest"
"testing"
sdk "git.happydns.org/checker-sdk-go/checker"
)
func newPulsediveServer(t *testing.T, status int, body string) (string, func()) {
t.Helper()
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Query().Get("key") == "" {
t.Errorf("missing key query parameter")
}
w.WriteHeader(status)
_, _ = w.Write([]byte(body))
}))
return srv.URL + "/info.php", srv.Close
}
func TestPulsediveSource_NoKey(t *testing.T) {
s := &pulsediveSource{endpoint: pulsediveEndpoint}
r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{})[0]
if r.Enabled {
t.Errorf("expected disabled without API key, got %+v", r)
}
}
func TestPulsediveSource_Listed_High(t *testing.T) {
body := `{"risk":"high","threats":[{"name":"Emotet","category":"malware","risk":"high"}]}`
endpoint, stop := newPulsediveServer(t, http.StatusOK, body)
defer stop()
s := &pulsediveSource{endpoint: endpoint}
r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"pulsedive_api_key": "k"})[0]
if r.Error != "" {
t.Fatalf("unexpected error: %s", r.Error)
}
if len(r.Evidence) != 1 {
t.Fatalf("expected 1 evidence, got %d", len(r.Evidence))
}
if listed, severity := s.Evaluate(r); !listed || severity != SeverityCrit {
t.Errorf("expected (true, crit), got (%v, %q)", listed, severity)
}
}
func TestPulsediveSource_Listed_Medium(t *testing.T) {
body := `{"risk":"medium","threats":[{"name":"SomeSpam","category":"spam","risk":"medium"}]}`
endpoint, stop := newPulsediveServer(t, http.StatusOK, body)
defer stop()
s := &pulsediveSource{endpoint: endpoint}
r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"pulsedive_api_key": "k"})[0]
if listed, severity := s.Evaluate(r); !listed || severity != SeverityWarn {
t.Errorf("expected (true, warn), got (%v, %q)", listed, severity)
}
}
func TestPulsediveSource_NotFound(t *testing.T) {
endpoint, stop := newPulsediveServer(t, http.StatusOK, `{"error":"Indicator not found."}`)
defer stop()
s := &pulsediveSource{endpoint: endpoint}
r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"pulsedive_api_key": "k"})[0]
if r.Error != "" {
t.Errorf("not-found should be quiet, got error: %s", r.Error)
}
if len(r.Evidence) != 0 {
t.Errorf("expected no evidence, got %d", len(r.Evidence))
}
if listed, _ := s.Evaluate(r); listed {
t.Errorf("expected not listed for not-found domain")
}
}
func TestPulsediveSource_Clean(t *testing.T) {
body := `{"risk":"none","threats":[]}`
endpoint, stop := newPulsediveServer(t, http.StatusOK, body)
defer stop()
s := &pulsediveSource{endpoint: endpoint}
r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"pulsedive_api_key": "k"})[0]
if len(r.Evidence) != 0 {
t.Errorf("expected no evidence for clean domain, got %d", len(r.Evidence))
}
if listed, _ := s.Evaluate(r); listed {
t.Errorf("expected not listed for clean domain")
}
}
func TestPulsediveSource_HTTPError(t *testing.T) {
endpoint, stop := newPulsediveServer(t, http.StatusInternalServerError, `internal error`)
defer stop()
s := &pulsediveSource{endpoint: endpoint}
r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"pulsedive_api_key": "k"})[0]
if r.Error == "" {
t.Errorf("expected error on HTTP 500, got clean result")
}
}

127
checker/quad9.go Normal file
View file

@ -0,0 +1,127 @@
package checker
import (
"context"
"fmt"
"net"
"sync"
"time"
sdk "git.happydns.org/checker-sdk-go/checker"
)
func init() {
Register(&quad9Source{
secure: newDNSResolver("9.9.9.9:53"),
unsecured: newDNSResolver("9.9.9.10:53"),
})
}
type quad9Source struct {
secure *net.Resolver
unsecured *net.Resolver
}
func newDNSResolver(addr string) *net.Resolver {
return &net.Resolver{
PreferGo: true,
Dial: func(ctx context.Context, network, _ string) (net.Conn, error) {
d := &net.Dialer{Timeout: 5 * time.Second}
return d.DialContext(ctx, "udp", addr)
},
}
}
func (*quad9Source) ID() string { return "quad9" }
func (*quad9Source) Name() string { return "Quad9 secure DNS" }
func (*quad9Source) Options() SourceOptions {
return SourceOptions{
User: []sdk.CheckerOptionField{
{
Id: "enable_quad9",
Type: "bool",
Label: "Use Quad9 secure DNS check",
Description: "Compare Quad9's secure resolver (9.9.9.9) against its unsecured peer (9.9.9.10). A domain that resolves on the unsecured but returns NXDOMAIN on the secure resolver is blocked by Quad9's threat intelligence.",
Default: true,
},
},
}
}
func (s *quad9Source) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult {
if !sdk.GetBoolOption(opts, "enable_quad9", true) || registered == "" {
return disabledResult(s.ID(), s.Name())
}
res := SourceResult{
SourceID: s.ID(),
SourceName: s.Name(),
Enabled: true,
LookupURL: "https://www.quad9.net/result/?domain=" + registered,
}
var (
secureAddrs, unsecuredAddrs []net.IPAddr
secureErr, unsecuredErr error
wg sync.WaitGroup
)
queryCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
wg.Add(2)
go func() {
defer wg.Done()
secureAddrs, secureErr = s.secure.LookupIPAddr(queryCtx, registered)
}()
go func() {
defer wg.Done()
unsecuredAddrs, unsecuredErr = s.unsecured.LookupIPAddr(queryCtx, registered)
}()
wg.Wait()
// Suppress the unused variable warning — secureAddrs is only used as an
// existence check; the actual IPs are not surfaced.
_ = secureAddrs
// If the unsecured resolver can't resolve the domain either, it simply
// doesn't exist — not a Quad9 block.
if unsecuredErr != nil || len(unsecuredAddrs) == 0 {
if unsecuredErr != nil {
if dnsErr, ok := unsecuredErr.(*net.DNSError); !ok || !dnsErr.IsNotFound {
res.Error = "unsecured resolver: " + unsecuredErr.Error()
}
}
return []SourceResult{res}
}
// Domain resolves on unsecured. Check whether the secure resolver blocked it.
if dnsErr, ok := secureErr.(*net.DNSError); ok && dnsErr.IsNotFound {
res.Reasons = []string{"Blocked by Quad9 threat intelligence"}
res.Evidence = append(res.Evidence, Evidence{
Label: "Secure resolver (9.9.9.9)",
Value: "NXDOMAIN",
})
} else if secureErr != nil {
res.Error = "secure resolver: " + secureErr.Error()
}
return []SourceResult{res}
}
func (*quad9Source) Evaluate(r SourceResult) (bool, string) {
return evidenceEval(r, SeverityCrit)
}
func (*quad9Source) Diagnose(res SourceResult) Diagnosis {
return Diagnosis{
Severity: SeverityCrit,
Title: "Blocked by Quad9 secure DNS resolver",
Detail: fmt.Sprintf(
"This domain is on Quad9's threat intelligence blocklist: the secure resolver (9.9.9.9) returns NXDOMAIN while the unsecured peer (9.9.9.10) resolves normally. Quad9 aggregates feeds from 18+ threat intel partners (abuse.ch, Bambenek, CINS, DShield, etc.). Visitors whose ISP or device uses Quad9 cannot reach this domain. Submit a false-positive report at the Quad9 contact page if you believe the listing is incorrect.",
),
Fix: "https://www.quad9.net/support/contact/",
FixIsURL: true,
LookupURL: res.LookupURL,
}
}

View file

@ -101,7 +101,7 @@ func diagnose(d *BlacklistData) []Diagnosis {
var out []Diagnosis
for _, r := range d.Results {
if !r.Listed {
if listed, _ := EvaluateResult(r); !listed {
continue
}
if s, ok := byID[r.SourceID]; ok {
@ -189,7 +189,7 @@ func buildSections(d *BlacklistData) []sourceSection {
// subject sources have at most one). Plain sources skip this.
if dr, ok := byID[id].(DetailRenderer); ok {
for _, r := range results {
if !r.Listed && len(r.Details) == 0 {
if listed, _ := EvaluateResult(r); !listed && len(r.Details) == 0 {
continue
}
html, err := dr.RenderDetail(r)
@ -210,7 +210,7 @@ func sectionStatus(results []SourceResult) (string, string) {
if r.Enabled {
enabled++
}
if r.Listed {
if l, _ := EvaluateResult(r); l {
listed++
} else if r.Error != "" {
errs++
@ -238,11 +238,12 @@ func subjectStatusLabel(r SourceResult) string {
switch {
case !r.Enabled:
return "Disabled"
case r.Listed:
return "LISTED"
case r.Error != "":
return "Error"
}
if listed, _ := EvaluateResult(r); listed {
return "LISTED"
}
return "Clean"
}
@ -250,11 +251,12 @@ func subjectStatusClass(r SourceResult) string {
switch {
case !r.Enabled:
return "muted"
case r.Listed:
return r.Severity
case r.Error != "":
return "warn"
}
if listed, severity := EvaluateResult(r); listed {
return severity
}
return "ok"
}

View file

@ -15,8 +15,9 @@ func TestDiagnoseAndReportRender(t *testing.T) {
{
SourceID: "dnsbl", SourceName: "Spamhaus DBL",
Subject: "dbl.spamhaus.org",
Enabled: true, Listed: true, Severity: SeverityCrit,
Enabled: true,
Reasons: []string{"Phishing domain"},
Evidence: []Evidence{{Label: "Return code", Value: "127.0.1.4"}},
LookupURL: "https://check.spamhaus.org/results/?query=example.com",
RemovalURL: "https://www.spamhaus.org/dbl/removal/",
},
@ -27,7 +28,7 @@ func TestDiagnoseAndReportRender(t *testing.T) {
},
{
SourceID: "openphish", SourceName: "OpenPhish feed",
Enabled: true, Listed: true, Severity: SeverityCrit,
Enabled: true,
Evidence: []Evidence{{Label: "URL", Value: "http://example.com/login"}},
},
},
@ -66,7 +67,7 @@ func TestHeadline(t *testing.T) {
}
func TestSectionStatus(t *testing.T) {
if l, c := sectionStatus([]SourceResult{{Enabled: true, Listed: true, Severity: SeverityCrit}}); c != "crit" || !strings.HasPrefix(l, "LISTED") {
if l, c := sectionStatus([]SourceResult{{SourceID: "openphish", Enabled: true, Evidence: []Evidence{{Label: "URL", Value: "http://evil.com/"}}}}); c != "crit" || !strings.HasPrefix(l, "LISTED") {
t.Errorf("sectionStatus listed = %q/%q", l, c)
}
if l, c := sectionStatus([]SourceResult{{Enabled: true}}); c != "ok" || l != "Clean" {

View file

@ -40,18 +40,28 @@ func (*sourceRule) Evaluate(ctx context.Context, obs sdk.ObservationGetter, opts
}}
}
byID := make(map[string]Source, len(Sources()))
for _, s := range Sources() {
byID[s.ID()] = s
}
out := make([]sdk.CheckState, 0, len(data.Results))
for _, r := range data.Results {
out = append(out, evaluateOne(r))
src, ok := byID[r.SourceID]
if !ok {
src = unknownSource{}
}
out = append(out, evaluateOne(r, src))
}
return out
}
func evaluateOne(r SourceResult) sdk.CheckState {
func evaluateOne(r SourceResult, src Source) sdk.CheckState {
subj := r.SourceName
if r.Subject != "" && r.Subject != r.SourceName {
subj = r.SourceName + " / " + r.Subject
}
listed, severity := src.Evaluate(r)
switch {
case !r.Enabled:
return sdk.CheckState{
@ -72,9 +82,9 @@ func evaluateOne(r SourceResult) sdk.CheckState {
Message: subj + ": query failed: " + r.Error,
Code: "source_error",
}
case r.Listed:
case listed:
return sdk.CheckState{
Status: severityToStatus(r.Severity),
Status: severityToStatus(severity),
Subject: subj,
Message: fmt.Sprintf("Listed in %s: %s", subj, joinNonEmpty(r.Reasons, "; ")),
Code: "source_listed",
@ -95,6 +105,17 @@ func evaluateOne(r SourceResult) sdk.CheckState {
}
}
// unknownSource is a sentinel used when a SourceResult references a source ID
// that is no longer in the registry. Evaluate always returns (false, "").
type unknownSource struct{}
func (unknownSource) ID() string { return "" }
func (unknownSource) Name() string { return "unknown" }
func (unknownSource) Options() SourceOptions { return SourceOptions{} }
func (unknownSource) Query(_ context.Context, _, _ string, _ sdk.CheckerOptions) []SourceResult { return nil }
func (unknownSource) Diagnose(_ SourceResult) Diagnosis { return Diagnosis{} }
func (unknownSource) Evaluate(_ SourceResult) (bool, string) { return false, "" }
func severityToStatus(sev string) sdk.Status {
switch sev {
case SeverityCrit:

View file

@ -54,7 +54,7 @@ func (*safeBrowsingSource) Options() SourceOptions {
func (s *safeBrowsingSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult {
apiKey := stringOpt(opts, "google_safe_browsing_api_key")
if apiKey == "" {
return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: false}}
return disabledResult(s.ID(), s.Name())
}
if registered == "" {
return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: true}}
@ -124,8 +124,6 @@ func (s *safeBrowsingSource) Query(ctx context.Context, domain, registered strin
if len(parsed.Matches) == 0 {
return []SourceResult{res}
}
res.Listed = true
res.Severity = SeverityCrit
res.Reference = "https://transparencyreport.google.com/safe-browsing/search?url=" + registered
seenType := map[string]bool{}
for _, m := range parsed.Matches {
@ -143,6 +141,10 @@ func (s *safeBrowsingSource) Query(ctx context.Context, domain, registered strin
return []SourceResult{res}
}
func (*safeBrowsingSource) Evaluate(r SourceResult) (bool, string) {
return evidenceEval(r, SeverityCrit)
}
func (*safeBrowsingSource) Diagnose(res SourceResult) Diagnosis {
return Diagnosis{
Severity: SeverityCrit,

View file

@ -48,6 +48,13 @@ type Source interface {
// generic report wraps it with the title bar and severity styling.
// Called only when SourceResult.Listed is true.
Diagnose(res SourceResult) Diagnosis
// Evaluate inspects an already-collected SourceResult and returns
// whether the domain is considered listed and at what severity.
// Implementations must read observation fields only (Evidence,
// Reasons, Error, Enabled, BlockedQuery, Details) and must never
// consult r.Listed or r.Severity.
Evaluate(r SourceResult) (listed bool, severity string)
}
// DetailRenderer is an optional interface a Source can implement when
@ -144,3 +151,32 @@ func Sources() []Source {
copy(out, registry)
return out
}
// disabledResult returns the standard "source is disabled" sentinel slice.
func disabledResult(id, name string) []SourceResult {
return []SourceResult{{SourceID: id, SourceName: name, Enabled: false}}
}
// evidenceEval is the common Evaluate body: listed when there is at least
// one Evidence entry and no error.
func evidenceEval(r SourceResult, severity string) (bool, string) {
if r.Enabled && r.Error == "" && len(r.Evidence) > 0 {
return true, severity
}
return false, ""
}
// EvaluateResult looks up the source that produced r from the registry
// and delegates to its Evaluate method. Returns (false, "") when the
// source is not found — a safe default that never promotes a stale
// Listed=true value.
func EvaluateResult(r SourceResult) (bool, string) {
registryMu.RLock()
defer registryMu.RUnlock()
for _, s := range registry {
if s.ID() == r.SourceID {
return s.Evaluate(r)
}
}
return false, ""
}

View file

@ -24,7 +24,7 @@ func TestRegisteredSourcesAreSane(t *testing.T) {
}
}
// At least the built-in sources are present.
for _, want := range []string{"dnsbl", "google_safe_browsing", "openphish", "urlhaus", "virustotal"} {
for _, want := range []string{"dnsbl", "google_safe_browsing", "openphish", "pulsedive", "quad9", "urlhaus", "virustotal"} {
if !seen[want] {
t.Errorf("missing built-in source %q", want)
}

152
checker/threatfox.go Normal file
View file

@ -0,0 +1,152 @@
package checker
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"
"time"
sdk "git.happydns.org/checker-sdk-go/checker"
)
const threatFoxEndpoint = "https://threatfox-api.abuse.ch/api/v1/"
func init() { Register(&threatFoxSource{endpoint: threatFoxEndpoint}) }
type threatFoxSource struct {
endpoint string
}
func (*threatFoxSource) ID() string { return "threatfox" }
func (*threatFoxSource) Name() string { return "abuse.ch ThreatFox" }
func (*threatFoxSource) Options() SourceOptions {
return SourceOptions{
Admin: []sdk.CheckerOptionField{
{
Id: "threatfox_auth_key",
Type: "string",
Label: "ThreatFox Auth-Key",
Description: "abuse.ch ThreatFox Auth-Key (free, requires an abuse.ch account). Without this key the source is disabled.",
Secret: true,
},
},
User: []sdk.CheckerOptionField{
{
Id: "enable_threatfox",
Type: "bool",
Label: "Use abuse.ch ThreatFox",
Description: "Query ThreatFox for indicators of compromise (C2 servers, malware, phishing) associated with the domain.",
Default: true,
},
},
}
}
func (s *threatFoxSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult {
authKey := stringOpt(opts, "threatfox_auth_key")
if !sdk.GetBoolOption(opts, "enable_threatfox", true) || registered == "" || authKey == "" {
return disabledResult(s.ID(), s.Name())
}
res := SourceResult{
SourceID: s.ID(), SourceName: s.Name(), Enabled: true,
Reference: "https://threatfox.abuse.ch/browse/",
}
buf, err := json.Marshal(map[string]any{
"query": "search_ioc",
"search_term": registered,
})
if err != nil {
res.Error = err.Error()
return []SourceResult{res}
}
reqCtx, cancel := context.WithTimeout(ctx, 15*time.Second)
defer cancel()
req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, s.endpoint, bytes.NewReader(buf))
if err != nil {
res.Error = err.Error()
return []SourceResult{res}
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Auth-Key", authKey)
body, status, err := httpDo(req, 4<<20)
if err != nil {
res.Error = err.Error()
return []SourceResult{res}
}
if status != http.StatusOK {
res.Error = fmt.Sprintf("HTTP %d: %s", status, truncate(string(body), 200))
return []SourceResult{res}
}
var parsed struct {
QueryStatus string `json:"query_status"`
Data []struct {
IOCValue string `json:"ioc_value"`
IOCType string `json:"ioc_type"`
ThreatType string `json:"threat_type"`
MalwarePrintable string `json:"malware_printable"`
ConfidenceLevel int `json:"confidence_level"`
FirstSeen string `json:"first_seen"`
LastSeen string `json:"last_seen"`
} `json:"data"`
}
if err := json.Unmarshal(body, &parsed); err != nil {
res.Error = "decode: " + err.Error()
return []SourceResult{res}
}
switch parsed.QueryStatus {
case "ok":
threats := map[string]bool{}
for _, ioc := range parsed.Data {
label := ioc.MalwarePrintable
if label == "" {
label = ioc.ThreatType
}
if label != "" && !threats[label] {
threats[label] = true
res.Reasons = append(res.Reasons, label)
}
res.Evidence = append(res.Evidence, Evidence{
Label: "IOC",
Value: ioc.IOCValue,
Status: ioc.ThreatType,
Extra: map[string]string{
"malware": ioc.MalwarePrintable,
"confidence": fmt.Sprintf("%d%%", ioc.ConfidenceLevel),
"first_seen": ioc.FirstSeen,
},
})
}
case "no_result":
// Clean.
default:
res.Error = "query_status=" + parsed.QueryStatus
}
return []SourceResult{res}
}
func (*threatFoxSource) Evaluate(r SourceResult) (bool, string) {
return evidenceEval(r, SeverityCrit)
}
func (*threatFoxSource) Diagnose(res SourceResult) Diagnosis {
return Diagnosis{
Severity: SeverityCrit,
Title: "Listed in abuse.ch ThreatFox as a threat IOC",
Detail: fmt.Sprintf(
"%d IOC(s) match this domain; threat(s): %s. ThreatFox tracks indicators of compromise including C2 servers, malware distribution hosts, and phishing infrastructure. Treat the host as compromised or abused: investigate recent DNS changes, audit hosted content, then request removal through the ThreatFox reference page.",
len(res.Evidence), joinNonEmpty(res.Reasons, ", "),
),
Fix: res.Reference,
FixIsURL: res.Reference != "",
}
}

96
checker/threatfox_test.go Normal file
View file

@ -0,0 +1,96 @@
package checker
import (
"context"
"net/http"
"net/http/httptest"
"testing"
sdk "git.happydns.org/checker-sdk-go/checker"
)
func TestThreatFoxSource_NoResult(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"query_status":"no_result","data":[]}`))
}))
defer srv.Close()
s := &threatFoxSource{endpoint: srv.URL}
results := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_threatfox": true, "threatfox_auth_key": "k"})
if len(results) != 1 {
t.Fatalf("expected 1 result, got %d", len(results))
}
r := results[0]
listed, _ := s.Evaluate(r)
if !r.Enabled || listed || r.Error != "" {
t.Fatalf("expected enabled+clean, got %+v, Evaluate listed=%v", r, listed)
}
}
func TestThreatFoxSource_Listed(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Auth-Key") == "" {
t.Errorf("missing Auth-Key header")
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{
"query_status": "ok",
"data": [{
"ioc_value": "example.com:443",
"ioc_type": "domain",
"threat_type": "botnet_cc",
"malware_printable": "Emotet",
"confidence_level": 75,
"first_seen": "2024-01-01 00:00:00 UTC",
"last_seen": "2024-06-01 00:00:00 UTC"
}]
}`))
}))
defer srv.Close()
s := &threatFoxSource{endpoint: srv.URL}
r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_threatfox": true, "threatfox_auth_key": "k"})[0]
if len(r.Evidence) != 1 {
t.Fatalf("expected 1 evidence item, got %+v", r)
}
if listed, severity := s.Evaluate(r); !listed || severity != SeverityCrit {
t.Errorf("expected Evaluate()=(true, crit), got (%v, %q)", listed, severity)
}
if r.Evidence[0].Value != "example.com:443" {
t.Errorf("evidence value = %q", r.Evidence[0].Value)
}
if len(r.Reasons) != 1 || r.Reasons[0] != "Emotet" {
t.Errorf("reasons = %v", r.Reasons)
}
}
func TestThreatFoxSource_HTTPError(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusUnauthorized)
_, _ = w.Write([]byte("unauthorized"))
}))
defer srv.Close()
s := &threatFoxSource{endpoint: srv.URL}
r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_threatfox": true, "threatfox_auth_key": "k"})[0]
if r.Error == "" {
t.Errorf("expected error, got %+v", r)
}
}
func TestThreatFoxSource_Disabled(t *testing.T) {
s := &threatFoxSource{endpoint: "http://nope"}
r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_threatfox": false})[0]
if r.Enabled {
t.Errorf("expected disabled, got %+v", r)
}
}
func TestThreatFoxSource_NoAuthKey(t *testing.T) {
s := &threatFoxSource{endpoint: "http://nope"}
r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_threatfox": true})[0]
if r.Enabled {
t.Errorf("expected disabled when no auth key, got %+v", r)
}
}

View file

@ -31,7 +31,7 @@ type BlacklistData struct {
func (d *BlacklistData) TotalHits() int {
n := 0
for _, r := range d.Results {
if r.Listed {
if listed, _ := EvaluateResult(r); listed {
n++
}
}
@ -43,7 +43,7 @@ func (d *BlacklistData) TotalHits() int {
func (d *BlacklistData) FilterListed() []SourceResult {
out := make([]SourceResult, 0, len(d.Results))
for _, r := range d.Results {
if r.Listed {
if listed, _ := EvaluateResult(r); listed {
out = append(out, r)
}
}

View file

@ -67,7 +67,7 @@ type urlhausURL struct {
func (s *urlhausSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult {
authKey := stringOpt(opts, "urlhaus_auth_key")
if !sdk.GetBoolOption(opts, "enable_urlhaus", true) || registered == "" || authKey == "" {
return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: false}}
return disabledResult(s.ID(), s.Name())
}
res := SourceResult{SourceID: s.ID(), SourceName: s.Name(), Enabled: true}
@ -120,8 +120,6 @@ func (s *urlhausSource) Query(ctx context.Context, domain, registered string, op
if len(parsed.URLs) == 0 {
return []SourceResult{res}
}
res.Listed = true
res.Severity = SeverityCrit
threats := map[string]bool{}
details := urlhausDetails{}
for _, u := range parsed.URLs {
@ -148,6 +146,10 @@ func (s *urlhausSource) Query(ctx context.Context, domain, registered string, op
return []SourceResult{res}
}
func (*urlhausSource) Evaluate(r SourceResult) (bool, string) {
return evidenceEval(r, SeverityCrit)
}
func (*urlhausSource) Diagnose(res SourceResult) Diagnosis {
online := 0
for _, e := range res.Evidence {

View file

@ -24,8 +24,9 @@ func TestURLhausSource_NoResults(t *testing.T) {
t.Fatalf("expected 1 result, got %d", len(results))
}
r := results[0]
if !r.Enabled || r.Listed || r.Error != "" {
t.Fatalf("expected enabled+clean, got %+v", r)
listed, _ := s.Evaluate(r)
if !r.Enabled || listed || r.Error != "" {
t.Fatalf("expected enabled+clean, got %+v, Evaluate listed=%v", r, listed)
}
}
@ -48,8 +49,11 @@ func TestURLhausSource_Listed(t *testing.T) {
s := &urlhausSource{endpoint: srv.URL}
r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_urlhaus": true, "urlhaus_auth_key": "k"})[0]
if !r.Listed || len(r.Evidence) != 1 {
t.Fatalf("expected 1 listed evidence, got %+v", r)
if len(r.Evidence) != 1 {
t.Fatalf("expected 1 evidence item, got %+v", r)
}
if listed, severity := s.Evaluate(r); !listed || severity != SeverityCrit {
t.Errorf("expected Evaluate()=(true, crit), got (%v, %q)", listed, severity)
}
if r.Evidence[0].Status != "online" {
t.Errorf("evidence status = %q", r.Evidence[0].Status)

View file

@ -60,7 +60,7 @@ type vtVendorVerdict struct {
func (s *virusTotalSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult {
apiKey := stringOpt(opts, "virustotal_api_key")
if apiKey == "" {
return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: false}}
return disabledResult(s.ID(), s.Name())
}
if registered == "" {
return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: true}}
@ -146,17 +146,6 @@ func (s *virusTotalSource) Query(ctx context.Context, domain, registered string,
return d.Vendors[i].Engine < d.Vendors[j].Engine
})
res.Details = mustJSON(d)
if d.Malicious == 0 && d.Suspicious == 0 {
// Clean.
return []SourceResult{res}
}
res.Listed = true
if d.Malicious > 0 {
res.Severity = SeverityCrit
} else {
res.Severity = SeverityWarn
}
for _, v := range d.Vendors {
res.Reasons = append(res.Reasons, v.Engine)
res.Evidence = append(res.Evidence, Evidence{
@ -167,6 +156,23 @@ func (s *virusTotalSource) Query(ctx context.Context, domain, registered string,
return []SourceResult{res}
}
func (*virusTotalSource) Evaluate(r SourceResult) (bool, string) {
var d vtDetails
if len(r.Details) == 0 {
return false, ""
}
if err := json.Unmarshal(r.Details, &d); err != nil {
return false, ""
}
if d.Malicious == 0 && d.Suspicious == 0 {
return false, ""
}
if d.Malicious > 0 {
return true, SeverityCrit
}
return true, SeverityWarn
}
func (*virusTotalSource) Diagnose(res SourceResult) Diagnosis {
var d vtDetails
_ = json.Unmarshal(res.Details, &d)

View file

@ -46,8 +46,8 @@ func TestVTSource_Listed(t *testing.T) {
s := &virusTotalSource{endpoint: endpoint}
r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"virustotal_api_key": "k"})[0]
if !r.Listed || r.Severity != SeverityCrit {
t.Errorf("expected listed+crit, got %+v", r)
if listed, severity := s.Evaluate(r); !listed || severity != SeverityCrit {
t.Errorf("expected Evaluate()=(true, crit), got (%v, %q)", listed, severity)
}
var d vtDetails
if err := json.Unmarshal(r.Details, &d); err != nil {
@ -72,9 +72,12 @@ func TestVTSource_NotFound(t *testing.T) {
s := &virusTotalSource{endpoint: endpoint}
r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"virustotal_api_key": "k"})[0]
if r.Listed || r.Error != "" {
if r.Error != "" {
t.Errorf("404 should be quiet not-listed: %+v", r)
}
if listed, severity := s.Evaluate(r); listed || severity != "" {
t.Errorf("Evaluate() on clean result = (%v, %q), want (false, \"\")", listed, severity)
}
if !strings.Contains(r.Reference, "example.com") {
t.Errorf("reference URL missing: %+v", r)
}