diff --git a/.drone-manifest.yml b/.drone-manifest.yml deleted file mode 100644 index 9b05159..0000000 --- a/.drone-manifest.yml +++ /dev/null @@ -1,22 +0,0 @@ -image: happydomain/checker-blacklist:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}latest{{/if}} -{{#if build.tags}} -tags: -{{#each build.tags}} - - {{this}} -{{/each}} -{{/if}} -manifests: - - image: happydomain/checker-blacklist:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-amd64 - platform: - architecture: amd64 - os: linux - - image: happydomain/checker-blacklist:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm64 - platform: - architecture: arm64 - os: linux - variant: v8 - - image: happydomain/checker-blacklist:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm - platform: - architecture: arm - os: linux - variant: v7 diff --git a/.drone.yml b/.drone.yml deleted file mode 100644 index 41271dc..0000000 --- a/.drone.yml +++ /dev/null @@ -1,187 +0,0 @@ ---- -kind: pipeline -type: docker -name: build-amd64 - -platform: - os: linux - arch: amd64 - -steps: - - name: checker build - image: golang:1-alpine - commands: - - apk add --no-cache git make - - make - environment: - CHECKER_VERSION: "${DRONE_BRANCH}-${DRONE_COMMIT}" - CGO_ENABLED: 0 - when: - event: - exclude: - - tag - - - name: checker build tag - image: golang:1-alpine - commands: - - apk add --no-cache git make - - make - environment: - CHECKER_VERSION: "${DRONE_SEMVER}" - CGO_ENABLED: 0 - when: - event: - - tag - - - name: publish on Docker Hub - image: plugins/docker - settings: - repo: happydomain/checker-blacklist - auto_tag: true - auto_tag_suffix: ${DRONE_STAGE_OS}-${DRONE_STAGE_ARCH} - dockerfile: Dockerfile - build_args: - - CHECKER_VERSION=${DRONE_BRANCH}-${DRONE_COMMIT} - username: - from_secret: docker_username - password: - from_secret: docker_password - when: - event: - exclude: - - tag - - - name: publish on Docker Hub (tag) - image: plugins/docker - settings: - repo: happydomain/checker-blacklist - auto_tag: true - auto_tag_suffix: ${DRONE_STAGE_OS}-${DRONE_STAGE_ARCH} - dockerfile: Dockerfile - build_args: - - CHECKER_VERSION=${DRONE_SEMVER} - username: - from_secret: docker_username - password: - from_secret: docker_password - when: - event: - - tag - -trigger: - branch: - exclude: - - renovate/* - event: - - cron - - push - - tag - ---- -kind: pipeline -type: docker -name: build-arm64 - -platform: - os: linux - arch: arm64 - -steps: - - name: checker build - image: golang:1-alpine - commands: - - apk add --no-cache git make - - make - environment: - CHECKER_VERSION: "${DRONE_BRANCH}-${DRONE_COMMIT}" - CGO_ENABLED: 0 - when: - event: - exclude: - - tag - - - name: checker build tag - image: golang:1-alpine - commands: - - apk add --no-cache git make - - make - environment: - CHECKER_VERSION: "${DRONE_SEMVER}" - CGO_ENABLED: 0 - when: - event: - - tag - - - name: publish on Docker Hub - image: plugins/docker - settings: - repo: happydomain/checker-blacklist - auto_tag: true - auto_tag_suffix: ${DRONE_STAGE_OS}-${DRONE_STAGE_ARCH} - dockerfile: Dockerfile - build_args: - - CHECKER_VERSION=${DRONE_BRANCH}-${DRONE_COMMIT} - username: - from_secret: docker_username - password: - from_secret: docker_password - when: - event: - exclude: - - tag - - - name: publish on Docker Hub (tag) - image: plugins/docker - settings: - repo: happydomain/checker-blacklist - auto_tag: true - auto_tag_suffix: ${DRONE_STAGE_OS}-${DRONE_STAGE_ARCH} - dockerfile: Dockerfile - build_args: - - CHECKER_VERSION=${DRONE_SEMVER} - username: - from_secret: docker_username - password: - from_secret: docker_password - when: - event: - - tag - -trigger: - event: - - cron - - push - - tag - ---- -kind: pipeline -name: docker-manifest - -platform: - os: linux - arch: arm64 - -steps: - - name: publish on Docker Hub - image: plugins/manifest - settings: - auto_tag: true - ignore_missing: true - spec: .drone-manifest.yml - username: - from_secret: docker_username - password: - from_secret: docker_password - -trigger: - branch: - exclude: - - renovate/* - event: - - cron - - push - - tag - -depends_on: - - build-amd64 - - build-arm64 diff --git a/README.md b/README.md index 0058fd3..6c6c138 100644 --- a/README.md +++ b/README.md @@ -10,60 +10,11 @@ widely-used reputation systems. | Spamhaus DBL | DNS-based DBL | no | admin (default on) | | SURBL multi | DNS-based DBL | no | admin (default on) | | URIBL multi | DNS-based DBL | no | admin (default on) | -| NordSpam DBL | DNS-based DBL | no | admin (default on) | -| SpamEatingMonkey Fresh| DNS-based DBL | no | admin (default on) | -| Tiopan DBL | DNS-based DBL | no | admin (default on) | -| SORBS RHSBL | DNS-based DBL | no | admin (default on) | | Extra DNSBL zones | DNS-based DBL | no | admin | | Google Safe Browsing | HTTPS lookup | yes (admin) | admin | | OpenPhish public feed | downloaded list | no | user (default on) | -| PhishTank | downloaded list | no | user (default on) | -| abuse.ch URLhaus | HTTPS lookup | free Auth-Key (admin) | user (default on) | -| abuse.ch ThreatFox | HTTPS lookup | free Auth-Key (admin) | user (default on) | -| abuse.ch MalwareBazaar| HTTPS lookup | free Auth-Key (admin) | user (default on) | -| Botvrij.eu | downloaded list | no | user (default on) | -| Disconnect.me | downloaded list | no | user (default on) | -| OISD | downloaded list | no | user (default on) | +| abuse.ch URLhaus | HTTPS lookup | optional Auth-Key (admin) | user (default on) | | VirusTotal v3 | HTTPS lookup | yes (admin) | admin | -| AlienVault OTX | HTTPS lookup | free (admin) | admin | -| Pulsedive | HTTPS lookup | free (admin) | admin | -| Criminal IP | HTTPS lookup | yes (admin) | admin | -| Quad9 secure DNS | DNS comparison | no | user (default on) | - -### Obtaining API keys - -**Google Safe Browsing** (option: `google_safe_browsing_api_key`) -1. Go to the [Google Cloud Console](https://console.cloud.google.com/) and create or select a project. -2. Enable the *Safe Browsing API* under *APIs & Services → Library*. -3. Create an API key under *APIs & Services → Credentials*. -4. The free tier allows up to 10 000 queries/day with no billing required. - -**abuse.ch** (option: `urlhaus_auth_key` / `threatfox_auth_key` / `malwarebazaar_auth_key`) -1. Register a free account at [abuse.ch](https://abuse.ch/). -2. After login, retrieve your Auth-Key from your account profile page. -3. The same account and key works for URLhaus, ThreatFox, and MalwareBazaar — set it in each source option independently. -4. Free, no rate-limit tiers documented; the APIs are community-funded. - -**VirusTotal** (option: `virustotal_api_key`) -1. Create a free account at [virustotal.com](https://www.virustotal.com/). -2. Go to your profile and copy the API key. -3. Free tier: 4 requests/minute, 500 requests/day. No billing required. -4. The public API key is sufficient; premium keys unlock higher quotas. - -**AlienVault OTX** (option: `otx_api_key`) -1. Register a free account at [otx.alienvault.com](https://otx.alienvault.com/). -2. Go to *Settings → API Integration* to find your personal OTX key. -3. Free, no documented rate limits for the indicator lookup API. - -**Pulsedive** (option: `pulsedive_api_key`) -1. Register a free account at [pulsedive.com](https://pulsedive.com/). -2. Go to your profile and copy the API key shown under *API*. -3. Free tier available; higher quotas with a paid plan. - -**Criminal IP** (option: `criminal_ip_api_key`) -1. Register a free account at [criminalip.io](https://www.criminalip.io/). -2. Go to *My Information → API Key* to find your key. -3. Free tier: 100 requests/day. Paid plans unlock higher quotas. DNS-based blocklists are queried in parallel. The OpenPhish feed is downloaded once per hour by the provider and cached in memory. @@ -73,7 +24,7 @@ downloaded once per hour by the provider and cached in memory. The report opens with a diagnosis-first "Action required" section that lists the most common, high-impact problems with a one-shot remediation: -1. **Listed on Spamhaus DBL / SURBL / URIBL / NordSpam / SpamEatingMonkey / Tiopan / SORBS**: direct lookup link and +1. **Listed on Spamhaus DBL / SURBL / URIBL**: direct lookup link and removal procedure URL per operator. 2. **Flagged by Google Safe Browsing**: link to Google Search Console's security-issues review request. diff --git a/checker/botvrij.go b/checker/botvrij.go deleted file mode 100644 index dbc4348..0000000 --- a/checker/botvrij.go +++ /dev/null @@ -1,191 +0,0 @@ -package checker - -import ( - "bufio" - "context" - "fmt" - "io" - "net/http" - "strings" - "sync" - "time" - - sdk "git.happydns.org/checker-sdk-go/checker" -) - -const ( - botvrijFeedURL = "https://www.botvrij.eu/data/ioclist.domain.simple" - botvrijDefaultTTL = 6 * time.Hour - botvrijFailBackoff = 1 * time.Minute -) - -func init() { - Register(&botvrijSource{ - cache: newBotvrijCache(botvrijFeedURL), - }) -} - -type botvrijSource struct { - cache *botvrijCache -} - -func (*botvrijSource) ID() string { return "botvrij" } -func (*botvrijSource) Name() string { return "Botvrij.eu domain blocklist" } - -func (*botvrijSource) Options() SourceOptions { - return SourceOptions{ - User: []sdk.CheckerOptionField{ - { - Id: "enable_botvrij", - Type: "bool", - Label: "Use Botvrij.eu domain blocklist", - Description: "Download the Botvrij.eu public domain blocklist (refreshed every 6h) and check the domain against it.", - Default: true, - }, - }, - } -} - -func (s *botvrijSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult { - if !sdk.GetBoolOption(opts, "enable_botvrij", true) || registered == "" { - return disabledResult(s.ID(), s.Name()) - } - - matched, size, fetched, err := s.cache.lookup(ctx, registered) - res := SourceResult{ - SourceID: s.ID(), SourceName: s.Name(), Enabled: true, - Reference: "https://botvrij.eu/", - Details: mustJSON(map[string]any{"feed_size": size, "fetched_at": fetched}), - } - if err != nil { - res.Error = err.Error() - } - if len(matched) > 0 { - res.Reasons = []string{"Malicious domain"} - for _, d := range matched { - res.Evidence = append(res.Evidence, Evidence{Label: "Domain", Value: d}) - } - } - return []SourceResult{res} -} - -func (*botvrijSource) Evaluate(r SourceResult) (bool, string) { - return evidenceEval(r, SeverityCrit) -} - -func (*botvrijSource) Diagnose(res SourceResult) Diagnosis { - domains := make([]string, 0, len(res.Evidence)) - for _, e := range res.Evidence { - domains = append(domains, e.Value) - } - previewN := min(len(domains), 5) - return Diagnosis{ - Severity: SeverityCrit, - Title: "Listed in the Botvrij.eu domain blocklist", - Detail: fmt.Sprintf( - "%d domain(s) matching this registered domain appear in the Botvrij.eu IOC list; examples: %s. Botvrij.eu tracks domains associated with malware C2, exploit kits, and other malicious infrastructure. Investigate recent DNS changes and hosted content.", - len(domains), joinNonEmpty(domains[:previewN], ", "), - ), - Fix: "https://botvrij.eu/", - FixIsURL: true, - } -} - -// ---------- cache ---------- - -type botvrijCache struct { - mu sync.Mutex - domains []string // all domains in feed - byDomain map[string]struct{} // set for O(1) exact-match test - fetchedAt time.Time - lastAttemptAt time.Time - refreshing bool - ttl time.Duration - failBackoff time.Duration - feedURL string -} - -func newBotvrijCache(feedURL string) *botvrijCache { - return &botvrijCache{ - ttl: botvrijDefaultTTL, - failBackoff: botvrijFailBackoff, - feedURL: feedURL, - } -} - -func (c *botvrijCache) lookup(ctx context.Context, registered string) (matched []string, size int, fetchedAt time.Time, err error) { - registered = strings.ToLower(strings.TrimSuffix(registered, ".")) - - c.mu.Lock() - stale := c.byDomain == nil || time.Since(c.fetchedAt) > c.ttl - doRefresh := stale && !c.refreshing && time.Since(c.lastAttemptAt) > c.failBackoff - if doRefresh { - c.refreshing = true - } - c.mu.Unlock() - - if doRefresh { - newDomains, newByDomain, ferr := c.fetch(ctx) - c.mu.Lock() - c.refreshing = false - c.lastAttemptAt = time.Now() - if ferr == nil { - c.domains = newDomains - c.byDomain = newByDomain - c.fetchedAt = c.lastAttemptAt - } else { - err = ferr - } - c.mu.Unlock() - } - - c.mu.Lock() - suffix := "." + registered - for d := range c.byDomain { - if d == registered || strings.HasSuffix(d, suffix) { - matched = append(matched, d) - } - } - size = len(c.domains) - fetchedAt = c.fetchedAt - c.mu.Unlock() - return matched, size, fetchedAt, err -} - -func (c *botvrijCache) fetch(ctx context.Context) ([]string, map[string]struct{}, error) { - reqCtx, cancel := context.WithTimeout(ctx, 30*time.Second) - defer cancel() - - req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, c.feedURL, nil) - if err != nil { - return nil, nil, err - } - req.Header.Set("User-Agent", "happydomain-checker-blacklist/1.0") - - resp, err := sharedHTTPClient.Do(req) - if err != nil { - return nil, nil, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, nil, fmt.Errorf("botvrij HTTP %d", resp.StatusCode) - } - - domains := make([]string, 0, 4096) - byDomain := make(map[string]struct{}, 4096) - scanner := bufio.NewScanner(io.LimitReader(resp.Body, 16<<20)) - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) - if line == "" || strings.HasPrefix(line, "#") { - continue - } - d := strings.ToLower(line) - domains = append(domains, d) - byDomain[d] = struct{}{} - } - if err := scanner.Err(); err != nil { - return nil, nil, err - } - return domains, byDomain, nil -} diff --git a/checker/botvrij_test.go b/checker/botvrij_test.go deleted file mode 100644 index 74366b4..0000000 --- a/checker/botvrij_test.go +++ /dev/null @@ -1,94 +0,0 @@ -package checker - -import ( - "context" - "net/http" - "net/http/httptest" - "testing" - - sdk "git.happydns.org/checker-sdk-go/checker" -) - -const botvrijFakeFeed = `# Botvrij.eu IOC list - domains -# comment line -evil.com -malware.example.org -c2.badactor.net -` - -func TestBotvrijSource_Listed_ExactMatch(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = w.Write([]byte(botvrijFakeFeed)) - })) - defer srv.Close() - - s := &botvrijSource{cache: newBotvrijCache(srv.URL)} - r := s.Query(context.Background(), "evil.com", "evil.com", sdk.CheckerOptions{"enable_botvrij": true})[0] - - if !r.Enabled || r.Error != "" { - t.Fatalf("expected enabled and no error, got %+v", r) - } - if len(r.Evidence) != 1 || r.Evidence[0].Value != "evil.com" { - t.Errorf("expected evidence [evil.com], got %+v", r.Evidence) - } - if listed, sev := s.Evaluate(r); !listed || sev != SeverityCrit { - t.Errorf("expected (true, crit), got (%v, %q)", listed, sev) - } -} - -func TestBotvrijSource_Listed_SubdomainInFeed(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = w.Write([]byte(botvrijFakeFeed)) - })) - defer srv.Close() - - // Feed has "malware.example.org"; querying registered "example.org" should match. - s := &botvrijSource{cache: newBotvrijCache(srv.URL)} - r := s.Query(context.Background(), "sub.example.org", "example.org", sdk.CheckerOptions{"enable_botvrij": true})[0] - - if len(r.Evidence) != 1 || r.Evidence[0].Value != "malware.example.org" { - t.Errorf("expected subdomain match, got %+v", r.Evidence) - } - if listed, _ := s.Evaluate(r); !listed { - t.Error("expected listed=true") - } -} - -func TestBotvrijSource_NotListed(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = w.Write([]byte(botvrijFakeFeed)) - })) - defer srv.Close() - - s := &botvrijSource{cache: newBotvrijCache(srv.URL)} - r := s.Query(context.Background(), "safe.com", "safe.com", sdk.CheckerOptions{"enable_botvrij": true})[0] - - if !r.Enabled || r.Error != "" || len(r.Evidence) != 0 { - t.Fatalf("expected clean result, got %+v", r) - } - if listed, _ := s.Evaluate(r); listed { - t.Error("expected listed=false") - } -} - -func TestBotvrijSource_Disabled(t *testing.T) { - s := &botvrijSource{cache: newBotvrijCache("http://nope")} - r := s.Query(context.Background(), "evil.com", "evil.com", sdk.CheckerOptions{"enable_botvrij": false})[0] - if r.Enabled { - t.Errorf("expected disabled, got %+v", r) - } -} - -func TestBotvrijSource_HTTPError(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusInternalServerError) - })) - defer srv.Close() - - s := &botvrijSource{cache: newBotvrijCache(srv.URL)} - r := s.Query(context.Background(), "evil.com", "evil.com", sdk.CheckerOptions{"enable_botvrij": true})[0] - - if r.Error == "" || r.Error != "botvrij HTTP 500" { - t.Errorf("expected HTTP 500 error, got %q", r.Error) - } -} diff --git a/checker/criminalip.go b/checker/criminalip.go deleted file mode 100644 index d98f2b5..0000000 --- a/checker/criminalip.go +++ /dev/null @@ -1,189 +0,0 @@ -package checker - -import ( - "context" - "encoding/json" - "fmt" - "net/http" - "time" - - sdk "git.happydns.org/checker-sdk-go/checker" -) - -const criminalIPEndpoint = "https://api.criminalip.io/v1/domain/report?query=%s" - -func init() { Register(&criminalIPSource{endpoint: criminalIPEndpoint}) } - -type criminalIPSource struct{ endpoint string } - -func (*criminalIPSource) ID() string { return "criminal_ip" } -func (*criminalIPSource) Name() string { return "Criminal IP" } - -func (*criminalIPSource) Options() SourceOptions { - return SourceOptions{ - Admin: []sdk.CheckerOptionField{ - { - Id: "criminal_ip_api_key", - Type: "string", - Label: "Criminal IP API key", - Description: "API key for api.criminalip.io. Free tier: 100 req/day. Leave empty to skip Criminal IP lookups.", - Secret: true, - }, - }, - } -} - -func (s *criminalIPSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult { - apiKey := stringOpt(opts, "criminal_ip_api_key") - if apiKey == "" { - return disabledResult(s.ID(), s.Name()) - } - if registered == "" { - return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: true}} - } - - res := SourceResult{ - SourceID: s.ID(), - SourceName: s.Name(), - Enabled: true, - Reference: "https://www.criminalip.io/domain/report/" + registered, - } - - reqCtx, cancel := context.WithTimeout(ctx, 15*time.Second) - defer cancel() - - url := fmt.Sprintf(s.endpoint, registered) - req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, url, nil) - if err != nil { - res.Error = err.Error() - return []SourceResult{res} - } - req.Header.Set("x-api-key", apiKey) - - body, status, err := httpDo(req, 1<<20) - if err != nil { - res.Error = redactSecret(err.Error(), apiKey) - return []SourceResult{res} - } - if status != http.StatusOK { - res.Error = fmt.Sprintf("HTTP %d: %s", status, redactSecret(truncate(string(body), 200), apiKey)) - return []SourceResult{res} - } - - var parsed struct { - Status int `json:"status"` - Data struct { - Score struct { - Inbound string `json:"inbound"` - Outbound string `json:"outbound"` - } `json:"score"` - SummaryInfo struct { - MaliciousInfo []string `json:"malicious_info"` - } `json:"summary_info"` - IsMalicious bool `json:"is_malicious"` - IsPhishing bool `json:"is_phishing"` - IsSpam bool `json:"is_spam"` - } `json:"data"` - } - if err := json.Unmarshal(body, &parsed); err != nil { - res.Error = "decode: " + err.Error() - return []SourceResult{res} - } - - inbound := parsed.Data.Score.Inbound - outbound := parsed.Data.Score.Outbound - - // Only populate evidence when there is an elevated score. - if criminalIPScoreLevel(inbound) > 0 || criminalIPScoreLevel(outbound) > 0 { - res.Evidence = append(res.Evidence, - Evidence{Label: "Inbound score", Value: inbound}, - Evidence{Label: "Outbound score", Value: outbound}, - ) - seen := map[string]bool{} - for _, info := range parsed.Data.SummaryInfo.MaliciousInfo { - if info != "" && !seen[info] { - seen[info] = true - res.Reasons = append(res.Reasons, info) - } - } - if parsed.Data.IsMalicious && !seen["Malicious"] { - res.Reasons = append(res.Reasons, "Malicious") - } - if parsed.Data.IsPhishing && !seen["Phishing"] { - res.Reasons = append(res.Reasons, "Phishing") - } - if parsed.Data.IsSpam && !seen["Spam"] { - res.Reasons = append(res.Reasons, "Spam") - } - } - - res.Details = mustJSON(map[string]any{ - "inbound": inbound, - "outbound": outbound, - }) - return []SourceResult{res} -} - -// criminalIPScoreLevel maps a Criminal IP score string to a numeric level -// for comparison: 0=Safe, 1=Low, 2=Moderate, 3=High, 4=Critical. -func criminalIPScoreLevel(score string) int { - switch score { - case "Low": - return 1 - case "Moderate": - return 2 - case "High": - return 3 - case "Critical": - return 4 - } - return 0 // "Safe" or unknown -} - -func (*criminalIPSource) Evaluate(r SourceResult) (bool, string) { - if !r.Enabled || r.Error != "" || len(r.Evidence) == 0 { - return false, "" - } - var d struct { - Inbound string `json:"inbound"` - Outbound string `json:"outbound"` - } - if len(r.Details) > 0 { - _ = json.Unmarshal(r.Details, &d) - } - maxLevel := criminalIPScoreLevel(d.Inbound) - if l := criminalIPScoreLevel(d.Outbound); l > maxLevel { - maxLevel = l - } - switch { - case maxLevel >= 3: - return true, SeverityCrit - case maxLevel == 2: - return true, SeverityWarn - } - return false, "" -} - -func (*criminalIPSource) Diagnose(res SourceResult) Diagnosis { - var d struct { - Inbound string `json:"inbound"` - Outbound string `json:"outbound"` - } - if len(res.Details) > 0 { - _ = json.Unmarshal(res.Details, &d) - } - sev := SeverityWarn - if criminalIPScoreLevel(d.Inbound) >= 3 || criminalIPScoreLevel(d.Outbound) >= 3 { - sev = SeverityCrit - } - return Diagnosis{ - Severity: sev, - Title: fmt.Sprintf("Criminal IP: inbound %s / outbound %s risk", d.Inbound, d.Outbound), - Detail: fmt.Sprintf( - "Criminal IP rated this domain with an inbound score of %q and an outbound score of %q. Threat category/reason(s): %s. Review the full report for connected IPs and associated URLs, then investigate any compromised infrastructure.", - d.Inbound, d.Outbound, joinNonEmpty(res.Reasons, ", "), - ), - Fix: res.Reference, - FixIsURL: res.Reference != "", - } -} diff --git a/checker/disconnect.go b/checker/disconnect.go deleted file mode 100644 index ae93adf..0000000 --- a/checker/disconnect.go +++ /dev/null @@ -1,162 +0,0 @@ -package checker - -import ( - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "strings" - "time" - - sdk "git.happydns.org/checker-sdk-go/checker" -) - -const disconnectFeedURL = "https://s3.amazonaws.com/lists.disconnect.me/services.json" - -func init() { - Register(&disconnectSource{ - cache: newFeedCache(24*time.Hour, disconnectFetch(disconnectFeedURL)), - }) -} - -type disconnectSource struct{ cache *feedCache } - -func (*disconnectSource) ID() string { return "disconnect" } -func (*disconnectSource) Name() string { return "Disconnect.me" } - -func (*disconnectSource) Options() SourceOptions { - return SourceOptions{ - User: []sdk.CheckerOptionField{ - { - Id: "enable_disconnect", - Type: "bool", - Label: "Use the Disconnect.me tracking-protection list", - Description: "Check the domain against the Disconnect.me blocklist used by Firefox Enhanced Tracking Protection, Brave, and uBlock Origin.", - Default: true, - }, - }, - } -} - -func (s *disconnectSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult { - if !sdk.GetBoolOption(opts, "enable_disconnect", true) { - return disabledResult(s.ID(), s.Name()) - } - if registered == "" { - return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: true}} - } - - matches, size, fetched, err := s.cache.lookup(ctx, registered) - res := SourceResult{ - SourceID: s.ID(), SourceName: s.Name(), Enabled: true, - Reference: "https://disconnect.me/trackerprotection", - Details: mustJSON(map[string]any{"feed_size": size, "fetched_at": fetched}), - } - if err != nil { - res.Error = err.Error() - } - - seenCategory := map[string]bool{} - for _, m := range matches { - parts := strings.SplitN(m, "|", 2) - category, company := parts[0], "" - if len(parts) == 2 { - company = parts[1] - } - if !seenCategory[category] { - seenCategory[category] = true - res.Reasons = append(res.Reasons, category) - } - extra := map[string]string{} - if company != "" { - extra["company"] = company - } - res.Evidence = append(res.Evidence, Evidence{ - Label: "Category", - Value: category, - Status: strings.ToLower(category), - Extra: extra, - }) - } - return []SourceResult{res} -} - -func (*disconnectSource) Evaluate(r SourceResult) (bool, string) { - return evidenceEval(r, SeverityWarn) -} - -func (*disconnectSource) Diagnose(res SourceResult) Diagnosis { - return Diagnosis{ - Severity: SeverityWarn, - Title: "Listed in Disconnect.me tracking-protection blocklist", - Detail: fmt.Sprintf( - "Category: %s. This domain appears in the Disconnect.me list used by Firefox Enhanced Tracking Protection, Brave, and uBlock Origin. Browsers and privacy tools will block third-party requests to this domain, which may affect analytics, ad delivery, or embedded widgets. The list is maintained by Disconnect.me; contact them if you believe the classification is incorrect.", - joinNonEmpty(res.Reasons, ", "), - ), - Fix: "https://disconnect.me/contact", - FixIsURL: true, - } -} - -// disconnectJSON mirrors the structure of services.json. -type disconnectJSON struct { - Categories map[string][]map[string]map[string][]string `json:"categories"` -} - -func disconnectFetch(feedURL string) func(context.Context) ([]string, map[string][]string, error) { - return func(ctx context.Context) ([]string, map[string][]string, error) { - reqCtx, cancel := context.WithTimeout(ctx, 30*time.Second) - defer cancel() - - req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, feedURL, nil) - if err != nil { - return nil, nil, err - } - req.Header.Set("User-Agent", "happydomain-checker-blacklist/1.0") - - resp, err := sharedHTTPClient.Do(req) - if err != nil { - return nil, nil, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, nil, fmt.Errorf("disconnect HTTP %d", resp.StatusCode) - } - - raw, err := io.ReadAll(io.LimitReader(resp.Body, 32<<20)) - if err != nil { - return nil, nil, fmt.Errorf("disconnect read: %w", err) - } - - var data disconnectJSON - if err := json.Unmarshal(raw, &data); err != nil { - return nil, nil, fmt.Errorf("disconnect parse: %w", err) - } - - byHost := make(map[string][]string, 4096) - for category, entities := range data.Categories { - for _, entity := range entities { - for company, sites := range entity { - for _, domains := range sites { - for _, d := range domains { - d = strings.ToLower(strings.TrimSuffix(d, ".")) - if d == "" { - continue - } - entry := category + "|" + company - byHost[d] = append(byHost[d], entry) - } - } - } - } - } - - urls := make([]string, 0, len(byHost)) - for d := range byHost { - urls = append(urls, d) - } - return urls, byHost, nil - } -} diff --git a/checker/disconnect_test.go b/checker/disconnect_test.go deleted file mode 100644 index 7a5e325..0000000 --- a/checker/disconnect_test.go +++ /dev/null @@ -1,127 +0,0 @@ -package checker - -import ( - "context" - "net/http" - "net/http/httptest" - "testing" - "time" - - sdk "git.happydns.org/checker-sdk-go/checker" -) - -const disconnectFakeFeed = `{ - "categories": { - "Advertising": [ - { - "Evil Corp": { - "https://evilcorp.com": ["tracker.com", "sub.example.org"] - } - } - ], - "Analytics": [ - { - "Metrics Inc": { - "https://metrics.io": ["analytics.net"] - } - } - ] - } -}` - -func newDisconnectTestSource(srv *httptest.Server) *disconnectSource { - return &disconnectSource{ - cache: newFeedCache(time.Hour, disconnectFetch(srv.URL)), - } -} - -func TestDisconnectSource_Listed(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = w.Write([]byte(disconnectFakeFeed)) - })) - defer srv.Close() - - s := newDisconnectTestSource(srv) - r := s.Query(context.Background(), "tracker.com", "tracker.com", sdk.CheckerOptions{"enable_disconnect": true})[0] - - if !r.Enabled || r.Error != "" { - t.Fatalf("expected enabled with no error, got %+v", r) - } - if len(r.Evidence) == 0 { - t.Fatalf("expected evidence, got none") - } - found := false - for _, e := range r.Evidence { - if e.Value == "Advertising" { - found = true - if e.Extra["company"] != "Evil Corp" { - t.Errorf("expected company 'Evil Corp', got %q", e.Extra["company"]) - } - } - } - if !found { - t.Errorf("expected Advertising evidence, got %+v", r.Evidence) - } - if listed, sev := s.Evaluate(r); !listed || sev != SeverityWarn { - t.Errorf("expected (true, warn), got (%v, %q)", listed, sev) - } -} - -func TestDisconnectSource_SubdomainInFeed(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = w.Write([]byte(disconnectFakeFeed)) - })) - defer srv.Close() - - // Feed has "sub.example.org"; querying registered domain "example.org" should match. - s := newDisconnectTestSource(srv) - r := s.Query(context.Background(), "example.org", "example.org", sdk.CheckerOptions{"enable_disconnect": true})[0] - - if !r.Enabled || r.Error != "" { - t.Fatalf("expected enabled with no error, got %+v", r) - } - if len(r.Evidence) == 0 { - t.Errorf("expected subdomain match evidence, got none") - } -} - -func TestDisconnectSource_NotListed(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = w.Write([]byte(disconnectFakeFeed)) - })) - defer srv.Close() - - s := newDisconnectTestSource(srv) - r := s.Query(context.Background(), "clean.example.com", "clean.example.com", sdk.CheckerOptions{"enable_disconnect": true})[0] - - if !r.Enabled || r.Error != "" { - t.Fatalf("expected enabled with no error, got %+v", r) - } - if len(r.Evidence) != 0 { - t.Errorf("expected no evidence for clean domain, got %+v", r.Evidence) - } - if listed, _ := s.Evaluate(r); listed { - t.Errorf("expected not listed for clean domain") - } -} - -func TestDisconnectSource_Disabled(t *testing.T) { - s := &disconnectSource{cache: newFeedCache(time.Hour, disconnectFetch("http://nope"))} - r := s.Query(context.Background(), "tracker.com", "tracker.com", sdk.CheckerOptions{"enable_disconnect": false})[0] - if r.Enabled { - t.Errorf("expected disabled result, got %+v", r) - } -} - -func TestDisconnectSource_HTTPError(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusInternalServerError) - })) - defer srv.Close() - - s := newDisconnectTestSource(srv) - r := s.Query(context.Background(), "tracker.com", "tracker.com", sdk.CheckerOptions{"enable_disconnect": true})[0] - if r.Error == "" { - t.Errorf("expected error on HTTP 500, got empty error") - } -} diff --git a/checker/dnsbl.go b/checker/dnsbl.go index 02d7912..e0d7dff 100644 --- a/checker/dnsbl.go +++ b/checker/dnsbl.go @@ -88,40 +88,14 @@ var DefaultDNSBLZones = []DNSBLZone{ return v4 != nil && v4[3] == 1 }, }, - { - Zone: "dbl.nordspam.com", - Label: "NordSpam DBL", - LookupURL: "https://www.nordspam.com/", - RemovalURL: "https://www.nordspam.com/delist/", - Decode: decodeNordSpamDBL, - }, - { - Zone: "fresh.spameatingmonkey.net", - Label: "SpamEatingMonkey Fresh", - LookupURL: "https://spameatingmonkey.com/lookup", - RemovalURL: "https://spameatingmonkey.com/lookup", - Decode: decodeSEMFresh, - }, - { - Zone: "dbl.tiopan.com", - Label: "Tiopan DBL", - LookupURL: "http://www.tiopan.com/blacklist.php", - RemovalURL: "http://www.tiopan.com/blacklist.php", - Decode: decodeTiopanDBL, - }, - { - Zone: "rhsbl.sorbs.net", - Label: "SORBS RHSBL", - LookupURL: "http://www.sorbs.net/lookup.shtml", - RemovalURL: "http://www.sorbs.net/delisting/overview.shtml", - Decode: decodeSORBSRHSBL, - }, } func (s *dnsblSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult { zones := zonesFromOptions(opts) if registered == "" || len(zones) == 0 { - return disabledResult(s.ID(), s.Name()) + return []SourceResult{{ + SourceID: s.ID(), SourceName: s.Name(), Enabled: false, + }} } out := make([]SourceResult, len(zones)) @@ -186,6 +160,8 @@ func (s *dnsblSource) queryOne(ctx context.Context, registered string, z DNSBLZo res.BlockedQuery = true return res } + res.Listed = true + res.Severity = SeverityCrit if len(res.Reasons) == 0 { res.Reasons = append(res.Reasons, "Listed (no detail decoded)") } @@ -200,13 +176,6 @@ func (s *dnsblSource) queryOne(ctx context.Context, registered string, z DNSBLZo return res } -func (*dnsblSource) Evaluate(r SourceResult) (bool, string) { - if r.Enabled && !r.BlockedQuery && r.Error == "" && len(r.Evidence) > 0 { - return true, SeverityCrit - } - return false, "" -} - func (*dnsblSource) Diagnose(res SourceResult) Diagnosis { return Diagnosis{ Severity: SeverityCrit, @@ -343,37 +312,6 @@ func decodeSURBLMulti(ip net.IP) []string { return out } -func decodeNordSpamDBL(ip net.IP) []string { - if ip.String() == "127.0.0.2" { - return []string{"Listed (spam/phishing/scam/ransomware)"} - } - return []string{"Listed (code " + ip.String() + ")"} -} - -func decodeSEMFresh(ip net.IP) []string { - if ip.String() == "127.0.0.2" { - return []string{"Newly registered domain (within last 5 days)"} - } - return []string{"Listed (code " + ip.String() + ")"} -} - -func decodeTiopanDBL(ip net.IP) []string { - if ip.String() == "127.0.0.2" { - return []string{"Listed (spam/abuse)"} - } - return []string{"Listed (code " + ip.String() + ")"} -} - -func decodeSORBSRHSBL(ip net.IP) []string { - switch ip.String() { - case "127.0.0.11": - return []string{"BADCONF: domain has bad A/MX DNS records"} - case "127.0.0.12": - return []string{"NOMAIL: domain has no valid mail server"} - } - return []string{"Listed (code " + ip.String() + ")"} -} - func decodeURIBLMulti(ip net.IP) []string { v4 := ip.To4() if v4 == nil || v4[0] != 127 { diff --git a/checker/feedcache.go b/checker/feedcache.go deleted file mode 100644 index aeaf154..0000000 --- a/checker/feedcache.go +++ /dev/null @@ -1,89 +0,0 @@ -package checker - -import ( - "context" - "net/url" - "strings" - "sync" - "time" -) - -// feedCache is a generic URL-feed cache shared between phishing-feed -// sources (OpenPhish, PhishTank). It holds a hostname-indexed snapshot -// of the feed, refreshes on TTL expiry, and ensures only one refresh is -// in flight at a time so concurrent lookups still serve stale data -// during a refresh. -type feedCache struct { - mu sync.Mutex - urls []string - byHost map[string][]string - fetchedAt time.Time - lastAttemptAt time.Time - refreshing bool - ttl time.Duration - failBackoff time.Duration - fetchFn func(ctx context.Context) (urls []string, byHost map[string][]string, err error) -} - -func newFeedCache(ttl time.Duration, fetch func(context.Context) ([]string, map[string][]string, error)) *feedCache { - if ttl <= 0 { - ttl = time.Hour - } - return &feedCache{ - ttl: ttl, - failBackoff: time.Minute, - fetchFn: fetch, - } -} - -func (c *feedCache) setTTL(d time.Duration) { - c.mu.Lock() - c.ttl = d - c.mu.Unlock() -} - -func (c *feedCache) lookup(ctx context.Context, domain string) (urls []string, size int, fetchedAt time.Time, err error) { - domain = strings.ToLower(strings.TrimSuffix(domain, ".")) - - c.mu.Lock() - stale := c.byHost == nil || time.Since(c.fetchedAt) > c.ttl - doRefresh := stale && !c.refreshing && time.Since(c.lastAttemptAt) > c.failBackoff - if doRefresh { - c.refreshing = true - } - c.mu.Unlock() - - if doRefresh { - newURLs, newByHost, ferr := c.fetchFn(ctx) - c.mu.Lock() - c.refreshing = false - c.lastAttemptAt = time.Now() - if ferr == nil { - c.urls = newURLs - c.byHost = newByHost - c.fetchedAt = c.lastAttemptAt - } else { - err = ferr - } - c.mu.Unlock() - } - - c.mu.Lock() - for host, hostURLs := range c.byHost { - if host == domain || strings.HasSuffix(host, "."+domain) { - urls = append(urls, hostURLs...) - } - } - size = len(c.urls) - fetchedAt = c.fetchedAt - c.mu.Unlock() - return urls, size, fetchedAt, err -} - -func hostOfURL(s string) string { - u, err := url.Parse(s) - if err != nil { - return "" - } - return strings.ToLower(u.Hostname()) -} diff --git a/checker/malwarebazaar.go b/checker/malwarebazaar.go deleted file mode 100644 index 9e1af6b..0000000 --- a/checker/malwarebazaar.go +++ /dev/null @@ -1,146 +0,0 @@ -package checker - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "net/http" - "time" - - sdk "git.happydns.org/checker-sdk-go/checker" -) - -const malwareBazaarEndpoint = "https://mb-api.abuse.ch/api/v1/" - -func init() { Register(&malwareBazaarSource{endpoint: malwareBazaarEndpoint}) } - -type malwareBazaarSource struct { - endpoint string -} - -func (*malwareBazaarSource) ID() string { return "malwarebazaar" } -func (*malwareBazaarSource) Name() string { return "abuse.ch MalwareBazaar" } - -func (*malwareBazaarSource) Options() SourceOptions { - return SourceOptions{ - Admin: []sdk.CheckerOptionField{ - { - Id: "malwarebazaar_auth_key", - Type: "string", - Label: "MalwareBazaar Auth-Key", - Description: "abuse.ch MalwareBazaar Auth-Key (free, requires an abuse.ch account). Without this key the source is disabled.", - Secret: true, - }, - }, - User: []sdk.CheckerOptionField{ - { - Id: "enable_malwarebazaar", - Type: "bool", - Label: "Use abuse.ch MalwareBazaar", - Description: "Search MalwareBazaar for malware samples tagged with the domain (typically C2 infrastructure or delivery hosts).", - Default: true, - }, - }, - } -} - -func (s *malwareBazaarSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult { - authKey := stringOpt(opts, "malwarebazaar_auth_key") - if !sdk.GetBoolOption(opts, "enable_malwarebazaar", true) || registered == "" || authKey == "" { - return disabledResult(s.ID(), s.Name()) - } - - res := SourceResult{ - SourceID: s.ID(), SourceName: s.Name(), Enabled: true, - Reference: "https://bazaar.abuse.ch/browse/", - } - - buf, err := json.Marshal(map[string]any{ - "query": "search_tag", - "tag": registered, - }) - if err != nil { - res.Error = err.Error() - return []SourceResult{res} - } - - reqCtx, cancel := context.WithTimeout(ctx, 15*time.Second) - defer cancel() - - req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, s.endpoint, bytes.NewReader(buf)) - if err != nil { - res.Error = err.Error() - return []SourceResult{res} - } - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Auth-Key", authKey) - - body, status, err := httpDo(req, 4<<20) - if err != nil { - res.Error = err.Error() - return []SourceResult{res} - } - if status != http.StatusOK { - res.Error = fmt.Sprintf("HTTP %d: %s", status, truncate(string(body), 200)) - return []SourceResult{res} - } - - var parsed struct { - QueryStatus string `json:"query_status"` - Data []struct { - SHA256Hash string `json:"sha256_hash"` - FileName string `json:"file_name"` - FileType string `json:"file_type_mime"` - Signature string `json:"signature"` - FirstSeen string `json:"first_seen"` - } `json:"data"` - } - if err := json.Unmarshal(body, &parsed); err != nil { - res.Error = "decode: " + err.Error() - return []SourceResult{res} - } - - switch parsed.QueryStatus { - case "ok": - signatures := map[string]bool{} - for _, sample := range parsed.Data { - if sample.Signature != "" && !signatures[sample.Signature] { - signatures[sample.Signature] = true - res.Reasons = append(res.Reasons, sample.Signature) - } - res.Evidence = append(res.Evidence, Evidence{ - Label: "Sample", - Value: sample.SHA256Hash, - Status: sample.FileType, - Extra: map[string]string{ - "filename": sample.FileName, - "signature": sample.Signature, - "first_seen": sample.FirstSeen, - }, - }) - } - case "no_results", "illegal_search_term": - // Clean. - default: - res.Error = "query_status=" + parsed.QueryStatus - } - return []SourceResult{res} -} - -func (*malwareBazaarSource) Evaluate(r SourceResult) (bool, string) { - return evidenceEval(r, SeverityWarn) -} - -func (*malwareBazaarSource) Diagnose(res SourceResult) Diagnosis { - return Diagnosis{ - Severity: SeverityWarn, - Title: "Associated with malware samples in abuse.ch MalwareBazaar", - Detail: fmt.Sprintf( - "%d malware sample(s) are tagged with this domain; malware family/signature(s): %s. MalwareBazaar samples are tagged with their C2 domain or delivery host when known. Investigate the domain's hosting and DNS records to identify and remove malicious infrastructure.", - len(res.Evidence), joinNonEmpty(res.Reasons, ", "), - ), - Fix: res.Reference, - FixIsURL: res.Reference != "", - } -} diff --git a/checker/malwarebazaar_test.go b/checker/malwarebazaar_test.go deleted file mode 100644 index 62bc24a..0000000 --- a/checker/malwarebazaar_test.go +++ /dev/null @@ -1,94 +0,0 @@ -package checker - -import ( - "context" - "net/http" - "net/http/httptest" - "testing" - - sdk "git.happydns.org/checker-sdk-go/checker" -) - -func TestMalwareBazaarSource_NoResults(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"query_status":"no_results"}`)) - })) - defer srv.Close() - - s := &malwareBazaarSource{endpoint: srv.URL} - results := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_malwarebazaar": true, "malwarebazaar_auth_key": "k"}) - if len(results) != 1 { - t.Fatalf("expected 1 result, got %d", len(results)) - } - r := results[0] - listed, _ := s.Evaluate(r) - if !r.Enabled || listed || r.Error != "" { - t.Fatalf("expected enabled+clean, got %+v, Evaluate listed=%v", r, listed) - } -} - -func TestMalwareBazaarSource_Listed(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Header.Get("Auth-Key") == "" { - t.Errorf("missing Auth-Key header") - } - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{ - "query_status": "ok", - "data": [{ - "sha256_hash": "aaaa1111bbbb2222cccc3333dddd4444eeee5555ffff6666aaaa1111bbbb2222", - "file_name": "evil.exe", - "file_type_mime": "application/x-dosexec", - "signature": "Emotet", - "first_seen": "2024-01-01 00:00:00" - }] - }`)) - })) - defer srv.Close() - - s := &malwareBazaarSource{endpoint: srv.URL} - r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_malwarebazaar": true, "malwarebazaar_auth_key": "k"})[0] - if len(r.Evidence) != 1 { - t.Fatalf("expected 1 evidence item, got %+v", r) - } - if listed, severity := s.Evaluate(r); !listed || severity != SeverityWarn { - t.Errorf("expected Evaluate()=(true, warn), got (%v, %q)", listed, severity) - } - if r.Evidence[0].Status != "application/x-dosexec" { - t.Errorf("evidence status = %q", r.Evidence[0].Status) - } - if len(r.Reasons) != 1 || r.Reasons[0] != "Emotet" { - t.Errorf("reasons = %v", r.Reasons) - } -} - -func TestMalwareBazaarSource_HTTPError(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusUnauthorized) - _, _ = w.Write([]byte("unauthorized")) - })) - defer srv.Close() - - s := &malwareBazaarSource{endpoint: srv.URL} - r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_malwarebazaar": true, "malwarebazaar_auth_key": "k"})[0] - if r.Error == "" { - t.Errorf("expected error, got %+v", r) - } -} - -func TestMalwareBazaarSource_Disabled(t *testing.T) { - s := &malwareBazaarSource{endpoint: "http://nope"} - r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_malwarebazaar": false})[0] - if r.Enabled { - t.Errorf("expected disabled, got %+v", r) - } -} - -func TestMalwareBazaarSource_NoAuthKey(t *testing.T) { - s := &malwareBazaarSource{endpoint: "http://nope"} - r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_malwarebazaar": true})[0] - if r.Enabled { - t.Errorf("expected disabled when no auth key, got %+v", r) - } -} diff --git a/checker/oisd.go b/checker/oisd.go deleted file mode 100644 index 39773a1..0000000 --- a/checker/oisd.go +++ /dev/null @@ -1,212 +0,0 @@ -package checker - -import ( - "bufio" - "context" - "fmt" - "io" - "net/http" - "strings" - "sync" - "time" - - sdk "git.happydns.org/checker-sdk-go/checker" -) - -const ( - oisdBigFeedURL = "https://big.oisd.nl/domainswild" - oisdSmallFeedURL = "https://small.oisd.nl/domainswild" - oisdDefaultTTL = 24 * time.Hour - oisdFailBackoff = 1 * time.Minute -) - -func init() { - Register(&oisdSource{ - bigCache: newOisdCache(oisdBigFeedURL), - smallCache: newOisdCache(oisdSmallFeedURL), - }) -} - -type oisdSource struct { - bigCache *oisdCache - smallCache *oisdCache -} - -func (*oisdSource) ID() string { return "oisd" } -func (*oisdSource) Name() string { return "OISD domain blocklist" } - -func (*oisdSource) Options() SourceOptions { - return SourceOptions{ - User: []sdk.CheckerOptionField{ - { - Id: "enable_oisd", - Type: "bool", - Label: "Use the OISD domain blocklist", - Description: "Download the OISD domain blocklist (refreshed every 24h) and check the domain against it.", - Default: true, - }, - }, - Admin: []sdk.CheckerOptionField{ - { - Id: "oisd_variant", - Type: "string", - Label: "OISD blocklist variant", - Description: `Which OISD list to use: "big" (~250 k entries, recommended) or "small" (~50 k entries).`, - Default: "big", - }, - }, - } -} - -func (s *oisdSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult { - if !sdk.GetBoolOption(opts, "enable_oisd", true) || registered == "" { - return disabledResult(s.ID(), s.Name()) - } - - cache := s.bigCache - if stringOptDefault(opts, "oisd_variant", "big") == "small" { - cache = s.smallCache - } - - matched, size, fetched, err := cache.lookup(ctx, registered) - res := SourceResult{ - SourceID: s.ID(), SourceName: s.Name(), Enabled: true, - Reference: "https://oisd.nl/", - Details: mustJSON(map[string]any{"feed_size": size, "fetched_at": fetched}), - } - if err != nil { - res.Error = err.Error() - } - if len(matched) > 0 { - res.Reasons = []string{"Listed in OISD"} - for _, d := range matched { - res.Evidence = append(res.Evidence, Evidence{Label: "Domain", Value: d}) - } - } - return []SourceResult{res} -} - -func (*oisdSource) Evaluate(r SourceResult) (bool, string) { - return evidenceEval(r, SeverityCrit) -} - -func (*oisdSource) Diagnose(res SourceResult) Diagnosis { - domains := make([]string, 0, len(res.Evidence)) - for _, e := range res.Evidence { - domains = append(domains, e.Value) - } - previewN := min(len(domains), 5) - return Diagnosis{ - Severity: SeverityCrit, - Title: "Listed in the OISD domain blocklist", - Detail: fmt.Sprintf( - "%d domain(s) matching this registered domain appear in the OISD blocklist; examples: %s. OISD is a large curated blocklist used by DNS resolvers, ad blockers, and firewalls worldwide. Domains listed here are blocked for millions of users. Investigate whether the domain hosts ads, trackers, or malware, or whether it has been compromised.", - len(domains), joinNonEmpty(domains[:previewN], ", "), - ), - Fix: "https://oisd.nl/", - FixIsURL: true, - } -} - -// ---------- cache ---------- - -type oisdCache struct { - mu sync.Mutex - domains []string - byDomain map[string]struct{} - fetchedAt time.Time - lastAttemptAt time.Time - refreshing bool - ttl time.Duration - failBackoff time.Duration - feedURL string -} - -func newOisdCache(feedURL string) *oisdCache { - return &oisdCache{ - ttl: oisdDefaultTTL, - failBackoff: oisdFailBackoff, - feedURL: feedURL, - } -} - -func (c *oisdCache) lookup(ctx context.Context, registered string) (matched []string, size int, fetchedAt time.Time, err error) { - registered = strings.ToLower(strings.TrimSuffix(registered, ".")) - - c.mu.Lock() - stale := c.byDomain == nil || time.Since(c.fetchedAt) > c.ttl - doRefresh := stale && !c.refreshing && time.Since(c.lastAttemptAt) > c.failBackoff - if doRefresh { - c.refreshing = true - } - feedURL := c.feedURL - c.mu.Unlock() - - if doRefresh { - newDomains, newByDomain, ferr := c.fetch(ctx, feedURL) - c.mu.Lock() - c.refreshing = false - c.lastAttemptAt = time.Now() - if ferr == nil { - c.domains = newDomains - c.byDomain = newByDomain - c.fetchedAt = c.lastAttemptAt - } else { - err = ferr - } - c.mu.Unlock() - } - - c.mu.Lock() - suffix := "." + registered - for d := range c.byDomain { - if d == registered || strings.HasSuffix(d, suffix) { - matched = append(matched, d) - } - } - size = len(c.domains) - fetchedAt = c.fetchedAt - c.mu.Unlock() - return matched, size, fetchedAt, err -} - -func (c *oisdCache) fetch(ctx context.Context, feedURL string) ([]string, map[string]struct{}, error) { - reqCtx, cancel := context.WithTimeout(ctx, 60*time.Second) - defer cancel() - - req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, feedURL, nil) - if err != nil { - return nil, nil, err - } - req.Header.Set("User-Agent", "happydomain-checker-blacklist/1.0") - - resp, err := sharedHTTPClient.Do(req) - if err != nil { - return nil, nil, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, nil, fmt.Errorf("oisd HTTP %d", resp.StatusCode) - } - - domains := make([]string, 0, 262144) - byDomain := make(map[string]struct{}, 262144) - scanner := bufio.NewScanner(io.LimitReader(resp.Body, 64<<20)) - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) - if line == "" || strings.HasPrefix(line, "!") || strings.HasPrefix(line, "#") { - continue - } - d := strings.TrimPrefix(strings.ToLower(line), "*.") - if d == "" { - continue - } - domains = append(domains, d) - byDomain[d] = struct{}{} - } - if err := scanner.Err(); err != nil { - return nil, nil, err - } - return domains, byDomain, nil -} diff --git a/checker/oisd_test.go b/checker/oisd_test.go deleted file mode 100644 index 93655b2..0000000 --- a/checker/oisd_test.go +++ /dev/null @@ -1,95 +0,0 @@ -package checker - -import ( - "context" - "net/http" - "net/http/httptest" - "testing" - - sdk "git.happydns.org/checker-sdk-go/checker" -) - -const oisdFakeFeed = `! OISD big domainswild -# comment line -*.evil.com -*.malware.example.org -*.c2.badactor.net -` - -func TestOisdSource_Listed_ExactMatch(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = w.Write([]byte(oisdFakeFeed)) - })) - defer srv.Close() - - s := &oisdSource{bigCache: newOisdCache(srv.URL)} - r := s.Query(context.Background(), "evil.com", "evil.com", sdk.CheckerOptions{"enable_oisd": true})[0] - - if !r.Enabled || r.Error != "" { - t.Fatalf("expected enabled and no error, got %+v", r) - } - if len(r.Evidence) != 1 || r.Evidence[0].Value != "evil.com" { - t.Errorf("expected evidence [evil.com], got %+v", r.Evidence) - } - if listed, sev := s.Evaluate(r); !listed || sev != SeverityCrit { - t.Errorf("expected (true, crit), got (%v, %q)", listed, sev) - } -} - -func TestOisdSource_Listed_SubdomainInFeed(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = w.Write([]byte(oisdFakeFeed)) - })) - defer srv.Close() - - // Feed has "*.malware.example.org" → stored as "malware.example.org"; querying - // registered "example.org" should match via suffix check. - s := &oisdSource{bigCache: newOisdCache(srv.URL)} - r := s.Query(context.Background(), "sub.example.org", "example.org", sdk.CheckerOptions{"enable_oisd": true})[0] - - if len(r.Evidence) != 1 || r.Evidence[0].Value != "malware.example.org" { - t.Errorf("expected subdomain match, got %+v", r.Evidence) - } - if listed, _ := s.Evaluate(r); !listed { - t.Error("expected listed=true") - } -} - -func TestOisdSource_NotListed(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = w.Write([]byte(oisdFakeFeed)) - })) - defer srv.Close() - - s := &oisdSource{bigCache: newOisdCache(srv.URL)} - r := s.Query(context.Background(), "safe.com", "safe.com", sdk.CheckerOptions{"enable_oisd": true})[0] - - if !r.Enabled || r.Error != "" || len(r.Evidence) != 0 { - t.Fatalf("expected clean result, got %+v", r) - } - if listed, _ := s.Evaluate(r); listed { - t.Error("expected listed=false") - } -} - -func TestOisdSource_Disabled(t *testing.T) { - s := &oisdSource{bigCache: newOisdCache("http://nope")} - r := s.Query(context.Background(), "evil.com", "evil.com", sdk.CheckerOptions{"enable_oisd": false})[0] - if r.Enabled { - t.Errorf("expected disabled, got %+v", r) - } -} - -func TestOisdSource_HTTPError(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusInternalServerError) - })) - defer srv.Close() - - s := &oisdSource{bigCache: newOisdCache(srv.URL)} - r := s.Query(context.Background(), "evil.com", "evil.com", sdk.CheckerOptions{"enable_oisd": true})[0] - - if r.Error == "" || r.Error != "oisd HTTP 500" { - t.Errorf("expected HTTP 500 error, got %q", r.Error) - } -} diff --git a/checker/openphish.go b/checker/openphish.go index e175f73..065652a 100644 --- a/checker/openphish.go +++ b/checker/openphish.go @@ -6,7 +6,9 @@ import ( "fmt" "io" "net/http" + "net/url" "strings" + "sync" "time" sdk "git.happydns.org/checker-sdk-go/checker" @@ -16,7 +18,7 @@ const openPhishFeedURL = "https://openphish.com/feed.txt" func init() { Register(&openPhishSource{ - cache: newFeedCache(1*time.Hour, openPhishFetch(openPhishFeedURL)), + cache: newPhishCache(openPhishFeedURL, 1*time.Hour), }) } @@ -25,7 +27,7 @@ func init() { // every URL in the feed. The cache is per-source-instance so it lives // for as long as the process. type openPhishSource struct { - cache *feedCache + cache *phishCache } func (*openPhishSource) ID() string { return "openphish" } @@ -47,7 +49,7 @@ func (*openPhishSource) Options() SourceOptions { func (s *openPhishSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult { if !sdk.GetBoolOption(opts, "enable_openphish", true) || registered == "" { - return disabledResult(s.ID(), s.Name()) + return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: false}} } urls, size, fetched, err := s.cache.lookup(ctx, registered) @@ -61,6 +63,8 @@ func (s *openPhishSource) Query(ctx context.Context, domain, registered string, // Fall through with whatever the cache could provide. } if len(urls) > 0 { + res.Listed = true + res.Severity = SeverityCrit res.Reasons = []string{"Phishing"} for _, u := range urls { res.Evidence = append(res.Evidence, Evidence{Label: "URL", Value: u}) @@ -69,10 +73,6 @@ func (s *openPhishSource) Query(ctx context.Context, domain, registered string, return []SourceResult{res} } -func (*openPhishSource) Evaluate(r SourceResult) (bool, string) { - return evidenceEval(r, SeverityCrit) -} - func (*openPhishSource) Diagnose(res SourceResult) Diagnosis { urls := make([]string, 0, len(res.Evidence)) for _, e := range res.Evidence { @@ -91,46 +91,114 @@ func (*openPhishSource) Diagnose(res SourceResult) Diagnosis { } } -// openPhishFetch returns a fetchFn that downloads and parses the -// OpenPhish plain-text feed at feedURL. -func openPhishFetch(feedURL string) func(context.Context) ([]string, map[string][]string, error) { - return func(ctx context.Context) ([]string, map[string][]string, error) { - reqCtx, cancel := context.WithTimeout(ctx, 30*time.Second) - defer cancel() +// ---------- feed cache ---------- - req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, feedURL, nil) - if err != nil { - return nil, nil, err - } - req.Header.Set("User-Agent", "happydomain-checker-blacklist/1.0") - - resp, err := sharedHTTPClient.Do(req) - if err != nil { - return nil, nil, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, nil, fmt.Errorf("openphish HTTP %d", resp.StatusCode) - } - - urls := make([]string, 0, 8192) - byHost := make(map[string][]string, 8192) - scanner := bufio.NewScanner(io.LimitReader(resp.Body, 64<<20)) - scanner.Buffer(make([]byte, 0, 64*1024), 1<<20) - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) - if line == "" || strings.HasPrefix(line, "#") { - continue - } - urls = append(urls, line) - if h := hostOfURL(line); h != "" { - byHost[h] = append(byHost[h], line) - } - } - if err := scanner.Err(); err != nil { - return nil, nil, err - } - return urls, byHost, nil - } +type phishCache struct { + mu sync.Mutex + urls []string + byHost map[string][]string + fetchedAt time.Time + lastAttemptAt time.Time + refreshing bool + ttl time.Duration + failBackoff time.Duration + feedURL string +} + +func newPhishCache(feedURL string, ttl time.Duration) *phishCache { + if feedURL == "" { + feedURL = openPhishFeedURL + } + if ttl <= 0 { + ttl = 1 * time.Hour + } + return &phishCache{ttl: ttl, feedURL: feedURL, failBackoff: 1 * time.Minute} +} + +func (c *phishCache) lookup(ctx context.Context, domain string) (urls []string, size int, fetchedAt time.Time, err error) { + domain = strings.ToLower(strings.TrimSuffix(domain, ".")) + + c.mu.Lock() + stale := c.byHost == nil || time.Since(c.fetchedAt) > c.ttl + doRefresh := stale && !c.refreshing && time.Since(c.lastAttemptAt) > c.failBackoff + if doRefresh { + c.refreshing = true + } + c.mu.Unlock() + + if doRefresh { + // Fetch without holding the cache lock so concurrent lookups + // can still serve stale data. Only one refresh runs at a time. + newURLs, newByHost, ferr := c.fetch(ctx) + c.mu.Lock() + c.refreshing = false + c.lastAttemptAt = time.Now() + if ferr == nil { + c.urls = newURLs + c.byHost = newByHost + c.fetchedAt = c.lastAttemptAt + } else { + err = ferr + } + c.mu.Unlock() + } + + c.mu.Lock() + for host, hostURLs := range c.byHost { + if host == domain || strings.HasSuffix(host, "."+domain) { + urls = append(urls, hostURLs...) + } + } + size = len(c.urls) + fetchedAt = c.fetchedAt + c.mu.Unlock() + return urls, size, fetchedAt, err +} + +func (c *phishCache) fetch(ctx context.Context) ([]string, map[string][]string, error) { + reqCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, c.feedURL, nil) + if err != nil { + return nil, nil, err + } + req.Header.Set("User-Agent", "happydomain-checker-blacklist/1.0") + + resp, err := sharedHTTPClient.Do(req) + if err != nil { + return nil, nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, nil, fmt.Errorf("openphish HTTP %d", resp.StatusCode) + } + + urls := make([]string, 0, 8192) + byHost := make(map[string][]string, 8192) + scanner := bufio.NewScanner(io.LimitReader(resp.Body, 64<<20)) + scanner.Buffer(make([]byte, 0, 64*1024), 1<<20) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + urls = append(urls, line) + if h := hostOfURL(line); h != "" { + byHost[h] = append(byHost[h], line) + } + } + if err := scanner.Err(); err != nil { + return nil, nil, err + } + return urls, byHost, nil +} + +func hostOfURL(s string) string { + u, err := url.Parse(s) + if err != nil { + return "" + } + return strings.ToLower(u.Hostname()) } diff --git a/checker/otx.go b/checker/otx.go deleted file mode 100644 index d4e39cf..0000000 --- a/checker/otx.go +++ /dev/null @@ -1,208 +0,0 @@ -package checker - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "html/template" - "net/http" - "time" - - sdk "git.happydns.org/checker-sdk-go/checker" -) - -const otxEndpoint = "https://otx.alienvault.com/api/v1/indicators/domain/" - -func init() { Register(&otxSource{endpoint: otxEndpoint}) } - -type otxSource struct{ endpoint string } - -func (*otxSource) ID() string { return "otx" } -func (*otxSource) Name() string { return "AlienVault OTX" } - -func (*otxSource) Options() SourceOptions { - return SourceOptions{ - Admin: []sdk.CheckerOptionField{ - { - Id: "otx_api_key", - Type: "string", - Label: "AlienVault OTX API key", - Description: "Free OTX API key from otx.alienvault.com. Leave empty to skip OTX lookups.", - Secret: true, - }, - }, - } -} - -type otxDetails struct { - PulseCount int `json:"pulse_count"` - Reputation int `json:"reputation"` - Pulses []otxPulse `json:"pulses"` -} - -type otxPulse struct { - Name string `json:"name"` - Tags []string `json:"tags,omitempty"` - MalwareFamilies []string `json:"malware_families,omitempty"` - Adversary string `json:"adversary,omitempty"` - Created string `json:"created,omitempty"` -} - -func (s *otxSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult { - apiKey := stringOpt(opts, "otx_api_key") - if apiKey == "" { - return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: false}} - } - if registered == "" { - return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: true}} - } - - res := SourceResult{ - SourceID: s.ID(), - SourceName: s.Name(), - Enabled: true, - Reference: "https://otx.alienvault.com/indicator/domain/" + registered, - } - - reqCtx, cancel := context.WithTimeout(ctx, 15*time.Second) - defer cancel() - - req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, s.endpoint+registered+"/general", nil) - if err != nil { - res.Error = err.Error() - return []SourceResult{res} - } - req.Header.Set("X-OTX-API-KEY", apiKey) - req.Header.Set("Accept", "application/json") - - body, status, err := httpDo(req, 4<<20) - if err != nil { - res.Error = err.Error() - return []SourceResult{res} - } - if status == http.StatusNotFound { - return []SourceResult{res} - } - if status != http.StatusOK { - res.Error = fmt.Sprintf("HTTP %d: %s", status, truncate(string(body), 200)) - return []SourceResult{res} - } - - var parsed struct { - Reputation int `json:"reputation"` - PulseInfo struct { - Count int `json:"count"` - Pulses []struct { - Name string `json:"name"` - Tags []string `json:"tags"` - MalwareFamilies []struct { - DisplayName string `json:"display_name"` - } `json:"malware_families"` - Adversary string `json:"adversary"` - Created string `json:"created"` - } `json:"pulses"` - } `json:"pulse_info"` - } - if err := json.Unmarshal(body, &parsed); err != nil { - res.Error = "decode: " + err.Error() - return []SourceResult{res} - } - - d := otxDetails{ - PulseCount: parsed.PulseInfo.Count, - Reputation: parsed.Reputation, - } - - seenReason := map[string]bool{} - for _, p := range parsed.PulseInfo.Pulses { - pulse := otxPulse{ - Name: p.Name, - Tags: p.Tags, - Adversary: p.Adversary, - Created: p.Created, - } - for _, mf := range p.MalwareFamilies { - pulse.MalwareFamilies = append(pulse.MalwareFamilies, mf.DisplayName) - if !seenReason[mf.DisplayName] { - seenReason[mf.DisplayName] = true - res.Reasons = append(res.Reasons, mf.DisplayName) - } - } - if p.Adversary != "" && !seenReason[p.Adversary] { - seenReason[p.Adversary] = true - res.Reasons = append(res.Reasons, p.Adversary) - } - d.Pulses = append(d.Pulses, pulse) - res.Evidence = append(res.Evidence, Evidence{ - Label: "Pulse", - Value: p.Name, - Status: "threat", - }) - } - res.Details = mustJSON(d) - return []SourceResult{res} -} - -func (*otxSource) Evaluate(r SourceResult) (bool, string) { - if !r.Enabled || r.Error != "" || len(r.Evidence) == 0 { - return false, "" - } - var d otxDetails - _ = json.Unmarshal(r.Details, &d) - if d.Reputation < -1 { - return true, SeverityCrit - } - return true, SeverityWarn -} - -func (*otxSource) Diagnose(res SourceResult) Diagnosis { - var d otxDetails - _ = json.Unmarshal(res.Details, &d) - detail := fmt.Sprintf( - "%d threat pulse(s) reference this domain (OTX reputation score: %d). Indicators: %s. "+ - "Review the pulse details on AlienVault OTX to understand the threat context and take corrective action.", - d.PulseCount, d.Reputation, joinNonEmpty(res.Reasons, ", "), - ) - sev := SeverityWarn - if d.Reputation < -1 { - sev = SeverityCrit - } - return Diagnosis{ - Severity: sev, - Title: "Listed in AlienVault OTX threat pulses", - Detail: detail, - Fix: res.Reference, - FixIsURL: res.Reference != "", - } -} - -func (*otxSource) RenderDetail(res SourceResult) (template.HTML, error) { - var d otxDetails - if len(res.Details) > 0 { - if err := json.Unmarshal(res.Details, &d); err != nil { - return "", fmt.Errorf("otx: decode details: %w", err) - } - } - if len(d.Pulses) == 0 { - return "", nil - } - var b bytes.Buffer - if err := otxDetailTpl.Execute(&b, d); err != nil { - return "", err - } - return template.HTML(b.String()), nil -} - -var otxDetailTpl = template.Must(template.New("otx_detail").Parse(` -

OTX reputation score: {{.Reputation}}. Pulse count: {{.PulseCount}}.

- - -{{range .Pulses}} - - - - - -{{end}} -
PulseMalware familiesAdversaryTagsCreated
{{.Name}}{{range .MalwareFamilies}}{{.}} {{end}}{{.Adversary}}{{range .Tags}}{{.}} {{end}}{{.Created}}
`)) diff --git a/checker/otx_test.go b/checker/otx_test.go deleted file mode 100644 index 2da5846..0000000 --- a/checker/otx_test.go +++ /dev/null @@ -1,144 +0,0 @@ -package checker - -import ( - "context" - "encoding/json" - "net/http" - "net/http/httptest" - "strings" - "testing" - - sdk "git.happydns.org/checker-sdk-go/checker" -) - -func newOTXServer(t *testing.T, status int, body string) (string, func()) { - t.Helper() - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Header.Get("X-OTX-API-KEY") == "" { - t.Errorf("missing X-OTX-API-KEY header") - } - w.WriteHeader(status) - _, _ = w.Write([]byte(body)) - })) - return srv.URL + "/", srv.Close -} - -func TestOTXSource_NoKey(t *testing.T) { - s := &otxSource{endpoint: otxEndpoint} - r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{})[0] - if r.Enabled { - t.Errorf("expected disabled without API key, got %+v", r) - } -} - -func TestOTXSource_Listed(t *testing.T) { - body := `{ - "reputation": 0, - "pulse_info": { - "count": 1, - "pulses": [{ - "name": "Test Pulse", - "tags": ["phishing"], - "malware_families": [{"display_name": "Emotet"}], - "adversary": "", - "created": "2024-01-01T00:00:00.000Z" - }] - } - }` - endpoint, stop := newOTXServer(t, http.StatusOK, body) - defer stop() - - s := &otxSource{endpoint: endpoint} - r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"otx_api_key": "k"})[0] - - if len(r.Evidence) != 1 { - t.Fatalf("expected 1 evidence entry, got %d", len(r.Evidence)) - } - if listed, severity := s.Evaluate(r); !listed || severity != SeverityWarn { - t.Errorf("expected Evaluate()=(true, warn), got (%v, %q)", listed, severity) - } - - var d otxDetails - if err := json.Unmarshal(r.Details, &d); err != nil { - t.Fatalf("details decode: %v", err) - } - if d.PulseCount != 1 || d.Reputation != 0 { - t.Errorf("details wrong: %+v", d) - } - if len(d.Pulses) != 1 || len(d.Pulses[0].MalwareFamilies) != 1 || d.Pulses[0].MalwareFamilies[0] != "Emotet" { - t.Errorf("pulse details wrong: %+v", d.Pulses) - } - - html, err := s.RenderDetail(r) - if err != nil || !strings.Contains(string(html), "Emotet") { - t.Errorf("RenderDetail html=%q err=%v", html, err) - } -} - -func TestOTXSource_ListedCrit(t *testing.T) { - body := `{ - "reputation": -2, - "pulse_info": { - "count": 5, - "pulses": [{"name": "APT Pulse", "tags": [], "malware_families": [], "adversary": "APT28", "created": ""}] - } - }` - endpoint, stop := newOTXServer(t, http.StatusOK, body) - defer stop() - - s := &otxSource{endpoint: endpoint} - r := s.Query(context.Background(), "evil.com", "evil.com", sdk.CheckerOptions{"otx_api_key": "k"})[0] - - if listed, severity := s.Evaluate(r); !listed || severity != SeverityCrit { - t.Errorf("expected Evaluate()=(true, crit) for reputation -2, got (%v, %q)", listed, severity) - } -} - -func TestOTXSource_NotFound(t *testing.T) { - endpoint, stop := newOTXServer(t, http.StatusNotFound, `{"detail":"Not found"}`) - defer stop() - - s := &otxSource{endpoint: endpoint} - r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"otx_api_key": "k"})[0] - - if r.Error != "" { - t.Errorf("404 should be quiet not-listed, got Error=%q", r.Error) - } - if len(r.Evidence) != 0 { - t.Errorf("expected no evidence for 404, got %+v", r.Evidence) - } - if listed, _ := s.Evaluate(r); listed { - t.Errorf("Evaluate() on clean result should return false") - } - if !strings.Contains(r.Reference, "example.com") { - t.Errorf("reference URL missing domain: %+v", r) - } -} - -func TestOTXSource_HTTPError(t *testing.T) { - endpoint, stop := newOTXServer(t, http.StatusInternalServerError, `{"error":"internal"}`) - defer stop() - - s := &otxSource{endpoint: endpoint} - r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"otx_api_key": "k"})[0] - - if r.Error == "" { - t.Errorf("expected non-empty Error for HTTP 500, got %+v", r) - } -} - -func TestOTXSource_NoResults(t *testing.T) { - body := `{"reputation": 0, "pulse_info": {"count": 0, "pulses": []}}` - endpoint, stop := newOTXServer(t, http.StatusOK, body) - defer stop() - - s := &otxSource{endpoint: endpoint} - r := s.Query(context.Background(), "clean.com", "clean.com", sdk.CheckerOptions{"otx_api_key": "k"})[0] - - if len(r.Evidence) != 0 { - t.Errorf("expected no evidence for clean domain, got %+v", r.Evidence) - } - if listed, severity := s.Evaluate(r); listed || severity != "" { - t.Errorf("Evaluate() on clean domain = (%v, %q), want (false, \"\")", listed, severity) - } -} diff --git a/checker/phishtank.go b/checker/phishtank.go deleted file mode 100644 index d3505b4..0000000 --- a/checker/phishtank.go +++ /dev/null @@ -1,174 +0,0 @@ -package checker - -import ( - "bufio" - "compress/gzip" - "context" - "encoding/csv" - "fmt" - "io" - "net/http" - "strconv" - "strings" - "time" - - sdk "git.happydns.org/checker-sdk-go/checker" -) - -const ( - phishTankFeedURL = "https://data.phishtank.com/data/online-valid.csv.gz" - phishTankDefaultTTL = 12 * time.Hour -) - -var phishTankGlobalCache = newFeedCache(phishTankDefaultTTL, phishTankFetch) - -func init() { Register(&phishTankSource{}) } - -type phishTankSource struct{} - -func (*phishTankSource) ID() string { return "phishtank" } -func (*phishTankSource) Name() string { return "PhishTank" } - -func (*phishTankSource) Options() SourceOptions { - return SourceOptions{ - Admin: []sdk.CheckerOptionField{ - { - Id: "phishtank_refresh_hours", - Type: "string", - Label: "PhishTank feed refresh interval (hours)", - Description: "How often to re-download the PhishTank online-valid feed. Minimum: 1. Default: 12.", - Default: "12", - }, - }, - User: []sdk.CheckerOptionField{ - { - Id: "enable_phishtank", - Type: "bool", - Label: "Use the PhishTank feed", - Description: "Download the PhishTank verified phishing list and check the domain against it.", - Default: true, - }, - }, - } -} - -func (s *phishTankSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult { - if !sdk.GetBoolOption(opts, "enable_phishtank", true) || registered == "" { - return disabledResult(s.ID(), s.Name()) - } - - if ttlRaw, ok := sdk.GetOption[string](opts, "phishtank_refresh_hours"); ok && ttlRaw != "" { - if hours, err := strconv.Atoi(ttlRaw); err == nil && hours >= 1 { - phishTankGlobalCache.setTTL(time.Duration(hours) * time.Hour) - } - } - - urls, size, fetched, err := phishTankGlobalCache.lookup(ctx, registered) - res := SourceResult{ - SourceID: s.ID(), SourceName: s.Name(), Enabled: true, - Reference: "https://www.phishtank.com/", - Details: mustJSON(map[string]any{"feed_size": size, "fetched_at": fetched}), - } - if err != nil { - res.Error = err.Error() - } - if len(urls) > 0 { - res.Reasons = []string{"Phishing"} - for _, u := range urls { - res.Evidence = append(res.Evidence, Evidence{Label: "URL", Value: u}) - } - } - return []SourceResult{res} -} - -func (*phishTankSource) Evaluate(r SourceResult) (bool, string) { - return evidenceEval(r, SeverityCrit) -} - -func (*phishTankSource) Diagnose(res SourceResult) Diagnosis { - urls := make([]string, 0, len(res.Evidence)) - for _, e := range res.Evidence { - urls = append(urls, e.Value) - } - previewN := min(len(urls), 5) - return Diagnosis{ - Severity: SeverityCrit, - Title: "Listed in the PhishTank phishing database", - Detail: fmt.Sprintf( - "%d URL(s) hosted on this domain are tracked as verified phishing by PhishTank. Examples: %s", - len(urls), joinNonEmpty(urls[:previewN], ", "), - ), - Fix: "https://www.phishtank.com/developer_info.php", - FixIsURL: true, - } -} - -// phishTankFetch downloads and parses the PhishTank gzip-compressed CSV feed. -func phishTankFetch(ctx context.Context) ([]string, map[string][]string, error) { - reqCtx, cancel := context.WithTimeout(ctx, 120*time.Second) - defer cancel() - - req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, phishTankFeedURL, nil) - if err != nil { - return nil, nil, err - } - req.Header.Set("User-Agent", "happydomain-checker-blacklist/1.0") - - resp, err := sharedHTTPClient.Do(req) - if err != nil { - return nil, nil, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, nil, fmt.Errorf("phishtank HTTP %d", resp.StatusCode) - } - - gz, err := gzip.NewReader(io.LimitReader(resp.Body, 128<<20)) - if err != nil { - return nil, nil, fmt.Errorf("phishtank gzip: %w", err) - } - defer gz.Close() - - r := csv.NewReader(bufio.NewReader(gz)) - r.ReuseRecord = true - - header, err := r.Read() - if err != nil { - return nil, nil, fmt.Errorf("phishtank csv header: %w", err) - } - urlIdx := -1 - for i, col := range header { - if col == "url" { - urlIdx = i - break - } - } - if urlIdx < 0 { - return nil, nil, fmt.Errorf("phishtank csv: no 'url' column in header") - } - - urls := make([]string, 0, 32768) - byHost := make(map[string][]string, 32768) - for { - record, err := r.Read() - if err == io.EOF { - break - } - if err != nil { - return nil, nil, fmt.Errorf("phishtank csv: %w", err) - } - if urlIdx >= len(record) { - continue - } - u := strings.TrimSpace(record[urlIdx]) - if u == "" { - continue - } - urls = append(urls, u) - if h := hostOfURL(u); h != "" { - byHost[h] = append(byHost[h], u) - } - } - return urls, byHost, nil -} diff --git a/checker/provider.go b/checker/provider.go index de33c02..fd4e0ea 100644 --- a/checker/provider.go +++ b/checker/provider.go @@ -37,7 +37,7 @@ func (p *blacklistProvider) ExtractMetrics(ctx sdk.ReportContext, collectedAt ti continue } v := 0.0 - if listed, _ := EvaluateResult(r); listed { + if r.Listed { v = 1 } metrics = append(metrics, sdk.CheckMetric{ diff --git a/checker/pulsedive.go b/checker/pulsedive.go deleted file mode 100644 index edfeb1d..0000000 --- a/checker/pulsedive.go +++ /dev/null @@ -1,163 +0,0 @@ -package checker - -import ( - "context" - "encoding/json" - "fmt" - "net/http" - "net/url" - "time" - - sdk "git.happydns.org/checker-sdk-go/checker" -) - -const pulsediveEndpoint = "https://pulsedive.com/api/info.php" - -func init() { Register(&pulsediveSource{endpoint: pulsediveEndpoint}) } - -type pulsediveSource struct{ endpoint string } - -func (*pulsediveSource) ID() string { return "pulsedive" } -func (*pulsediveSource) Name() string { return "Pulsedive" } - -func (*pulsediveSource) Options() SourceOptions { - return SourceOptions{ - Admin: []sdk.CheckerOptionField{ - { - Id: "pulsedive_api_key", - Type: "string", - Label: "Pulsedive API key", - Description: "Pulsedive API key (free account at pulsedive.com). Leave empty to skip Pulsedive lookups.", - Secret: true, - }, - }, - } -} - -type pulsediveDetails struct { - Risk string `json:"risk"` - Threats []pulsediveThreat `json:"threats"` -} - -type pulsediveThreat struct { - Name string `json:"name"` - Category string `json:"category"` - Risk string `json:"risk,omitempty"` -} - -func (s *pulsediveSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult { - apiKey := stringOpt(opts, "pulsedive_api_key") - if apiKey == "" || registered == "" { - return disabledResult(s.ID(), s.Name()) - } - - res := SourceResult{ - SourceID: s.ID(), - SourceName: s.Name(), - Enabled: true, - Reference: "https://pulsedive.com/indicator/" + registered, - } - - params := url.Values{ - "indicator": {registered}, - "key": {apiKey}, - } - reqURL := s.endpoint + "?" + params.Encode() - - reqCtx, cancel := context.WithTimeout(ctx, 15*time.Second) - defer cancel() - - req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, reqURL, nil) - if err != nil { - res.Error = redactSecret(err.Error(), apiKey) - return []SourceResult{res} - } - req.Header.Set("User-Agent", "happydomain-checker-blacklist/1.0") - - body, status, err := httpDo(req, 1<<20) - if err != nil { - res.Error = redactSecret(err.Error(), apiKey) - return []SourceResult{res} - } - if status != http.StatusOK { - res.Error = fmt.Sprintf("HTTP %d: %s", status, truncate(string(body), 200)) - return []SourceResult{res} - } - - // Check for "not found" before full parse — Pulsedive returns 200 - // with {"error": "Indicator not found."} for unknown indicators. - var errEnvelope struct { - Error string `json:"error"` - } - if json.Unmarshal(body, &errEnvelope) == nil && errEnvelope.Error != "" { - return []SourceResult{res} - } - - var parsed struct { - Risk string `json:"risk"` - Threats []struct { - Name string `json:"name"` - Category string `json:"category"` - Risk string `json:"risk"` - } `json:"threats"` - } - if err := json.Unmarshal(body, &parsed); err != nil { - res.Error = "decode: " + err.Error() - return []SourceResult{res} - } - - d := pulsediveDetails{Risk: parsed.Risk} - seen := map[string]bool{} - for _, t := range parsed.Threats { - d.Threats = append(d.Threats, pulsediveThreat{ - Name: t.Name, - Category: t.Category, - Risk: t.Risk, - }) - if !seen[t.Name] { - seen[t.Name] = true - res.Reasons = append(res.Reasons, t.Name) - } - res.Evidence = append(res.Evidence, Evidence{ - Label: "Threat", - Value: t.Name, - Status: t.Category, - }) - } - res.Details = mustJSON(d) - return []SourceResult{res} -} - -func (*pulsediveSource) Evaluate(r SourceResult) (bool, string) { - if !r.Enabled || r.Error != "" || len(r.Evidence) == 0 { - return false, "" - } - var d pulsediveDetails - _ = json.Unmarshal(r.Details, &d) - switch d.Risk { - case "critical", "high": - return true, SeverityCrit - default: - return true, SeverityWarn - } -} - -func (*pulsediveSource) Diagnose(res SourceResult) Diagnosis { - var d pulsediveDetails - _ = json.Unmarshal(res.Details, &d) - previewN := min(len(d.Threats), 5) - names := make([]string, 0, previewN) - for _, t := range d.Threats[:previewN] { - names = append(names, t.Name) - } - return Diagnosis{ - Severity: SeverityCrit, - Title: fmt.Sprintf("Pulsedive risk: %s — %d threat(s) associated", d.Risk, len(d.Threats)), - Detail: fmt.Sprintf( - "Pulsedive assigned a risk of %q to this domain. Associated threat(s): %s. Review the indicator page for feed context, related IPs, and historical activity, then follow up with the relevant threat's removal or remediation procedure.", - d.Risk, joinNonEmpty(names, ", "), - ), - Fix: res.Reference, - FixIsURL: res.Reference != "", - } -} diff --git a/checker/pulsedive_test.go b/checker/pulsedive_test.go deleted file mode 100644 index 9018b49..0000000 --- a/checker/pulsedive_test.go +++ /dev/null @@ -1,103 +0,0 @@ -package checker - -import ( - "context" - "net/http" - "net/http/httptest" - "testing" - - sdk "git.happydns.org/checker-sdk-go/checker" -) - -func newPulsediveServer(t *testing.T, status int, body string) (string, func()) { - t.Helper() - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Query().Get("key") == "" { - t.Errorf("missing key query parameter") - } - w.WriteHeader(status) - _, _ = w.Write([]byte(body)) - })) - return srv.URL + "/info.php", srv.Close -} - -func TestPulsediveSource_NoKey(t *testing.T) { - s := &pulsediveSource{endpoint: pulsediveEndpoint} - r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{})[0] - if r.Enabled { - t.Errorf("expected disabled without API key, got %+v", r) - } -} - -func TestPulsediveSource_Listed_High(t *testing.T) { - body := `{"risk":"high","threats":[{"name":"Emotet","category":"malware","risk":"high"}]}` - endpoint, stop := newPulsediveServer(t, http.StatusOK, body) - defer stop() - - s := &pulsediveSource{endpoint: endpoint} - r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"pulsedive_api_key": "k"})[0] - if r.Error != "" { - t.Fatalf("unexpected error: %s", r.Error) - } - if len(r.Evidence) != 1 { - t.Fatalf("expected 1 evidence, got %d", len(r.Evidence)) - } - if listed, severity := s.Evaluate(r); !listed || severity != SeverityCrit { - t.Errorf("expected (true, crit), got (%v, %q)", listed, severity) - } -} - -func TestPulsediveSource_Listed_Medium(t *testing.T) { - body := `{"risk":"medium","threats":[{"name":"SomeSpam","category":"spam","risk":"medium"}]}` - endpoint, stop := newPulsediveServer(t, http.StatusOK, body) - defer stop() - - s := &pulsediveSource{endpoint: endpoint} - r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"pulsedive_api_key": "k"})[0] - if listed, severity := s.Evaluate(r); !listed || severity != SeverityWarn { - t.Errorf("expected (true, warn), got (%v, %q)", listed, severity) - } -} - -func TestPulsediveSource_NotFound(t *testing.T) { - endpoint, stop := newPulsediveServer(t, http.StatusOK, `{"error":"Indicator not found."}`) - defer stop() - - s := &pulsediveSource{endpoint: endpoint} - r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"pulsedive_api_key": "k"})[0] - if r.Error != "" { - t.Errorf("not-found should be quiet, got error: %s", r.Error) - } - if len(r.Evidence) != 0 { - t.Errorf("expected no evidence, got %d", len(r.Evidence)) - } - if listed, _ := s.Evaluate(r); listed { - t.Errorf("expected not listed for not-found domain") - } -} - -func TestPulsediveSource_Clean(t *testing.T) { - body := `{"risk":"none","threats":[]}` - endpoint, stop := newPulsediveServer(t, http.StatusOK, body) - defer stop() - - s := &pulsediveSource{endpoint: endpoint} - r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"pulsedive_api_key": "k"})[0] - if len(r.Evidence) != 0 { - t.Errorf("expected no evidence for clean domain, got %d", len(r.Evidence)) - } - if listed, _ := s.Evaluate(r); listed { - t.Errorf("expected not listed for clean domain") - } -} - -func TestPulsediveSource_HTTPError(t *testing.T) { - endpoint, stop := newPulsediveServer(t, http.StatusInternalServerError, `internal error`) - defer stop() - - s := &pulsediveSource{endpoint: endpoint} - r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"pulsedive_api_key": "k"})[0] - if r.Error == "" { - t.Errorf("expected error on HTTP 500, got clean result") - } -} diff --git a/checker/quad9.go b/checker/quad9.go deleted file mode 100644 index 7bf6094..0000000 --- a/checker/quad9.go +++ /dev/null @@ -1,127 +0,0 @@ -package checker - -import ( - "context" - "fmt" - "net" - "sync" - "time" - - sdk "git.happydns.org/checker-sdk-go/checker" -) - -func init() { - Register(&quad9Source{ - secure: newDNSResolver("9.9.9.9:53"), - unsecured: newDNSResolver("9.9.9.10:53"), - }) -} - -type quad9Source struct { - secure *net.Resolver - unsecured *net.Resolver -} - -func newDNSResolver(addr string) *net.Resolver { - return &net.Resolver{ - PreferGo: true, - Dial: func(ctx context.Context, network, _ string) (net.Conn, error) { - d := &net.Dialer{Timeout: 5 * time.Second} - return d.DialContext(ctx, "udp", addr) - }, - } -} - -func (*quad9Source) ID() string { return "quad9" } -func (*quad9Source) Name() string { return "Quad9 secure DNS" } - -func (*quad9Source) Options() SourceOptions { - return SourceOptions{ - User: []sdk.CheckerOptionField{ - { - Id: "enable_quad9", - Type: "bool", - Label: "Use Quad9 secure DNS check", - Description: "Compare Quad9's secure resolver (9.9.9.9) against its unsecured peer (9.9.9.10). A domain that resolves on the unsecured but returns NXDOMAIN on the secure resolver is blocked by Quad9's threat intelligence.", - Default: true, - }, - }, - } -} - -func (s *quad9Source) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult { - if !sdk.GetBoolOption(opts, "enable_quad9", true) || registered == "" { - return disabledResult(s.ID(), s.Name()) - } - - res := SourceResult{ - SourceID: s.ID(), - SourceName: s.Name(), - Enabled: true, - LookupURL: "https://www.quad9.net/result/?domain=" + registered, - } - - var ( - secureAddrs, unsecuredAddrs []net.IPAddr - secureErr, unsecuredErr error - wg sync.WaitGroup - ) - queryCtx, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - - wg.Add(2) - go func() { - defer wg.Done() - secureAddrs, secureErr = s.secure.LookupIPAddr(queryCtx, registered) - }() - go func() { - defer wg.Done() - unsecuredAddrs, unsecuredErr = s.unsecured.LookupIPAddr(queryCtx, registered) - }() - wg.Wait() - - // Suppress the unused variable warning — secureAddrs is only used as an - // existence check; the actual IPs are not surfaced. - _ = secureAddrs - - // If the unsecured resolver can't resolve the domain either, it simply - // doesn't exist — not a Quad9 block. - if unsecuredErr != nil || len(unsecuredAddrs) == 0 { - if unsecuredErr != nil { - if dnsErr, ok := unsecuredErr.(*net.DNSError); !ok || !dnsErr.IsNotFound { - res.Error = "unsecured resolver: " + unsecuredErr.Error() - } - } - return []SourceResult{res} - } - - // Domain resolves on unsecured. Check whether the secure resolver blocked it. - if dnsErr, ok := secureErr.(*net.DNSError); ok && dnsErr.IsNotFound { - res.Reasons = []string{"Blocked by Quad9 threat intelligence"} - res.Evidence = append(res.Evidence, Evidence{ - Label: "Secure resolver (9.9.9.9)", - Value: "NXDOMAIN", - }) - } else if secureErr != nil { - res.Error = "secure resolver: " + secureErr.Error() - } - - return []SourceResult{res} -} - -func (*quad9Source) Evaluate(r SourceResult) (bool, string) { - return evidenceEval(r, SeverityCrit) -} - -func (*quad9Source) Diagnose(res SourceResult) Diagnosis { - return Diagnosis{ - Severity: SeverityCrit, - Title: "Blocked by Quad9 secure DNS resolver", - Detail: fmt.Sprintf( - "This domain is on Quad9's threat intelligence blocklist: the secure resolver (9.9.9.9) returns NXDOMAIN while the unsecured peer (9.9.9.10) resolves normally. Quad9 aggregates feeds from 18+ threat intel partners (abuse.ch, Bambenek, CINS, DShield, etc.). Visitors whose ISP or device uses Quad9 cannot reach this domain. Submit a false-positive report at the Quad9 contact page if you believe the listing is incorrect.", - ), - Fix: "https://www.quad9.net/support/contact/", - FixIsURL: true, - LookupURL: res.LookupURL, - } -} diff --git a/checker/report.go b/checker/report.go index c823cf7..5649812 100644 --- a/checker/report.go +++ b/checker/report.go @@ -101,7 +101,7 @@ func diagnose(d *BlacklistData) []Diagnosis { var out []Diagnosis for _, r := range d.Results { - if listed, _ := EvaluateResult(r); !listed { + if !r.Listed { continue } if s, ok := byID[r.SourceID]; ok { @@ -189,7 +189,7 @@ func buildSections(d *BlacklistData) []sourceSection { // subject sources have at most one). Plain sources skip this. if dr, ok := byID[id].(DetailRenderer); ok { for _, r := range results { - if listed, _ := EvaluateResult(r); !listed && len(r.Details) == 0 { + if !r.Listed && len(r.Details) == 0 { continue } html, err := dr.RenderDetail(r) @@ -210,7 +210,7 @@ func sectionStatus(results []SourceResult) (string, string) { if r.Enabled { enabled++ } - if l, _ := EvaluateResult(r); l { + if r.Listed { listed++ } else if r.Error != "" { errs++ @@ -238,12 +238,11 @@ func subjectStatusLabel(r SourceResult) string { switch { case !r.Enabled: return "Disabled" + case r.Listed: + return "LISTED" case r.Error != "": return "Error" } - if listed, _ := EvaluateResult(r); listed { - return "LISTED" - } return "Clean" } @@ -251,12 +250,11 @@ func subjectStatusClass(r SourceResult) string { switch { case !r.Enabled: return "muted" + case r.Listed: + return r.Severity case r.Error != "": return "warn" } - if listed, severity := EvaluateResult(r); listed { - return severity - } return "ok" } diff --git a/checker/report_test.go b/checker/report_test.go index 3215745..1ff5fad 100644 --- a/checker/report_test.go +++ b/checker/report_test.go @@ -15,9 +15,8 @@ func TestDiagnoseAndReportRender(t *testing.T) { { SourceID: "dnsbl", SourceName: "Spamhaus DBL", Subject: "dbl.spamhaus.org", - Enabled: true, - Reasons: []string{"Phishing domain"}, - Evidence: []Evidence{{Label: "Return code", Value: "127.0.1.4"}}, + Enabled: true, Listed: true, Severity: SeverityCrit, + Reasons: []string{"Phishing domain"}, LookupURL: "https://check.spamhaus.org/results/?query=example.com", RemovalURL: "https://www.spamhaus.org/dbl/removal/", }, @@ -28,7 +27,7 @@ func TestDiagnoseAndReportRender(t *testing.T) { }, { SourceID: "openphish", SourceName: "OpenPhish feed", - Enabled: true, + Enabled: true, Listed: true, Severity: SeverityCrit, Evidence: []Evidence{{Label: "URL", Value: "http://example.com/login"}}, }, }, @@ -67,7 +66,7 @@ func TestHeadline(t *testing.T) { } func TestSectionStatus(t *testing.T) { - if l, c := sectionStatus([]SourceResult{{SourceID: "openphish", Enabled: true, Evidence: []Evidence{{Label: "URL", Value: "http://evil.com/"}}}}); c != "crit" || !strings.HasPrefix(l, "LISTED") { + if l, c := sectionStatus([]SourceResult{{Enabled: true, Listed: true, Severity: SeverityCrit}}); c != "crit" || !strings.HasPrefix(l, "LISTED") { t.Errorf("sectionStatus listed = %q/%q", l, c) } if l, c := sectionStatus([]SourceResult{{Enabled: true}}); c != "ok" || l != "Clean" { diff --git a/checker/rule.go b/checker/rule.go index 72d5c38..d389026 100644 --- a/checker/rule.go +++ b/checker/rule.go @@ -40,28 +40,18 @@ func (*sourceRule) Evaluate(ctx context.Context, obs sdk.ObservationGetter, opts }} } - byID := make(map[string]Source, len(Sources())) - for _, s := range Sources() { - byID[s.ID()] = s - } - out := make([]sdk.CheckState, 0, len(data.Results)) for _, r := range data.Results { - src, ok := byID[r.SourceID] - if !ok { - src = unknownSource{} - } - out = append(out, evaluateOne(r, src)) + out = append(out, evaluateOne(r)) } return out } -func evaluateOne(r SourceResult, src Source) sdk.CheckState { +func evaluateOne(r SourceResult) sdk.CheckState { subj := r.SourceName if r.Subject != "" && r.Subject != r.SourceName { subj = r.SourceName + " / " + r.Subject } - listed, severity := src.Evaluate(r) switch { case !r.Enabled: return sdk.CheckState{ @@ -82,9 +72,9 @@ func evaluateOne(r SourceResult, src Source) sdk.CheckState { Message: subj + ": query failed: " + r.Error, Code: "source_error", } - case listed: + case r.Listed: return sdk.CheckState{ - Status: severityToStatus(severity), + Status: severityToStatus(r.Severity), Subject: subj, Message: fmt.Sprintf("Listed in %s: %s", subj, joinNonEmpty(r.Reasons, "; ")), Code: "source_listed", @@ -105,17 +95,6 @@ func evaluateOne(r SourceResult, src Source) sdk.CheckState { } } -// unknownSource is a sentinel used when a SourceResult references a source ID -// that is no longer in the registry. Evaluate always returns (false, ""). -type unknownSource struct{} - -func (unknownSource) ID() string { return "" } -func (unknownSource) Name() string { return "unknown" } -func (unknownSource) Options() SourceOptions { return SourceOptions{} } -func (unknownSource) Query(_ context.Context, _, _ string, _ sdk.CheckerOptions) []SourceResult { return nil } -func (unknownSource) Diagnose(_ SourceResult) Diagnosis { return Diagnosis{} } -func (unknownSource) Evaluate(_ SourceResult) (bool, string) { return false, "" } - func severityToStatus(sev string) sdk.Status { switch sev { case SeverityCrit: diff --git a/checker/safebrowsing.go b/checker/safebrowsing.go index f04ec70..a9a9c03 100644 --- a/checker/safebrowsing.go +++ b/checker/safebrowsing.go @@ -54,7 +54,7 @@ func (*safeBrowsingSource) Options() SourceOptions { func (s *safeBrowsingSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult { apiKey := stringOpt(opts, "google_safe_browsing_api_key") if apiKey == "" { - return disabledResult(s.ID(), s.Name()) + return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: false}} } if registered == "" { return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: true}} @@ -124,6 +124,8 @@ func (s *safeBrowsingSource) Query(ctx context.Context, domain, registered strin if len(parsed.Matches) == 0 { return []SourceResult{res} } + res.Listed = true + res.Severity = SeverityCrit res.Reference = "https://transparencyreport.google.com/safe-browsing/search?url=" + registered seenType := map[string]bool{} for _, m := range parsed.Matches { @@ -141,10 +143,6 @@ func (s *safeBrowsingSource) Query(ctx context.Context, domain, registered strin return []SourceResult{res} } -func (*safeBrowsingSource) Evaluate(r SourceResult) (bool, string) { - return evidenceEval(r, SeverityCrit) -} - func (*safeBrowsingSource) Diagnose(res SourceResult) Diagnosis { return Diagnosis{ Severity: SeverityCrit, diff --git a/checker/source.go b/checker/source.go index 03f47fc..793d919 100644 --- a/checker/source.go +++ b/checker/source.go @@ -48,13 +48,6 @@ type Source interface { // generic report wraps it with the title bar and severity styling. // Called only when SourceResult.Listed is true. Diagnose(res SourceResult) Diagnosis - - // Evaluate inspects an already-collected SourceResult and returns - // whether the domain is considered listed and at what severity. - // Implementations must read observation fields only (Evidence, - // Reasons, Error, Enabled, BlockedQuery, Details) and must never - // consult r.Listed or r.Severity. - Evaluate(r SourceResult) (listed bool, severity string) } // DetailRenderer is an optional interface a Source can implement when @@ -151,32 +144,3 @@ func Sources() []Source { copy(out, registry) return out } - -// disabledResult returns the standard "source is disabled" sentinel slice. -func disabledResult(id, name string) []SourceResult { - return []SourceResult{{SourceID: id, SourceName: name, Enabled: false}} -} - -// evidenceEval is the common Evaluate body: listed when there is at least -// one Evidence entry and no error. -func evidenceEval(r SourceResult, severity string) (bool, string) { - if r.Enabled && r.Error == "" && len(r.Evidence) > 0 { - return true, severity - } - return false, "" -} - -// EvaluateResult looks up the source that produced r from the registry -// and delegates to its Evaluate method. Returns (false, "") when the -// source is not found — a safe default that never promotes a stale -// Listed=true value. -func EvaluateResult(r SourceResult) (bool, string) { - registryMu.RLock() - defer registryMu.RUnlock() - for _, s := range registry { - if s.ID() == r.SourceID { - return s.Evaluate(r) - } - } - return false, "" -} diff --git a/checker/source_test.go b/checker/source_test.go index c98c8b8..2bd1cd6 100644 --- a/checker/source_test.go +++ b/checker/source_test.go @@ -24,7 +24,7 @@ func TestRegisteredSourcesAreSane(t *testing.T) { } } // At least the built-in sources are present. - for _, want := range []string{"dnsbl", "google_safe_browsing", "openphish", "pulsedive", "quad9", "urlhaus", "virustotal"} { + for _, want := range []string{"dnsbl", "google_safe_browsing", "openphish", "urlhaus", "virustotal"} { if !seen[want] { t.Errorf("missing built-in source %q", want) } diff --git a/checker/threatfox.go b/checker/threatfox.go deleted file mode 100644 index 4df96fa..0000000 --- a/checker/threatfox.go +++ /dev/null @@ -1,152 +0,0 @@ -package checker - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "net/http" - "time" - - sdk "git.happydns.org/checker-sdk-go/checker" -) - -const threatFoxEndpoint = "https://threatfox-api.abuse.ch/api/v1/" - -func init() { Register(&threatFoxSource{endpoint: threatFoxEndpoint}) } - -type threatFoxSource struct { - endpoint string -} - -func (*threatFoxSource) ID() string { return "threatfox" } -func (*threatFoxSource) Name() string { return "abuse.ch ThreatFox" } - -func (*threatFoxSource) Options() SourceOptions { - return SourceOptions{ - Admin: []sdk.CheckerOptionField{ - { - Id: "threatfox_auth_key", - Type: "string", - Label: "ThreatFox Auth-Key", - Description: "abuse.ch ThreatFox Auth-Key (free, requires an abuse.ch account). Without this key the source is disabled.", - Secret: true, - }, - }, - User: []sdk.CheckerOptionField{ - { - Id: "enable_threatfox", - Type: "bool", - Label: "Use abuse.ch ThreatFox", - Description: "Query ThreatFox for indicators of compromise (C2 servers, malware, phishing) associated with the domain.", - Default: true, - }, - }, - } -} - -func (s *threatFoxSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult { - authKey := stringOpt(opts, "threatfox_auth_key") - if !sdk.GetBoolOption(opts, "enable_threatfox", true) || registered == "" || authKey == "" { - return disabledResult(s.ID(), s.Name()) - } - - res := SourceResult{ - SourceID: s.ID(), SourceName: s.Name(), Enabled: true, - Reference: "https://threatfox.abuse.ch/browse/", - } - - buf, err := json.Marshal(map[string]any{ - "query": "search_ioc", - "search_term": registered, - }) - if err != nil { - res.Error = err.Error() - return []SourceResult{res} - } - - reqCtx, cancel := context.WithTimeout(ctx, 15*time.Second) - defer cancel() - - req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, s.endpoint, bytes.NewReader(buf)) - if err != nil { - res.Error = err.Error() - return []SourceResult{res} - } - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Auth-Key", authKey) - - body, status, err := httpDo(req, 4<<20) - if err != nil { - res.Error = err.Error() - return []SourceResult{res} - } - if status != http.StatusOK { - res.Error = fmt.Sprintf("HTTP %d: %s", status, truncate(string(body), 200)) - return []SourceResult{res} - } - - var parsed struct { - QueryStatus string `json:"query_status"` - Data []struct { - IOCValue string `json:"ioc_value"` - IOCType string `json:"ioc_type"` - ThreatType string `json:"threat_type"` - MalwarePrintable string `json:"malware_printable"` - ConfidenceLevel int `json:"confidence_level"` - FirstSeen string `json:"first_seen"` - LastSeen string `json:"last_seen"` - } `json:"data"` - } - if err := json.Unmarshal(body, &parsed); err != nil { - res.Error = "decode: " + err.Error() - return []SourceResult{res} - } - - switch parsed.QueryStatus { - case "ok": - threats := map[string]bool{} - for _, ioc := range parsed.Data { - label := ioc.MalwarePrintable - if label == "" { - label = ioc.ThreatType - } - if label != "" && !threats[label] { - threats[label] = true - res.Reasons = append(res.Reasons, label) - } - res.Evidence = append(res.Evidence, Evidence{ - Label: "IOC", - Value: ioc.IOCValue, - Status: ioc.ThreatType, - Extra: map[string]string{ - "malware": ioc.MalwarePrintable, - "confidence": fmt.Sprintf("%d%%", ioc.ConfidenceLevel), - "first_seen": ioc.FirstSeen, - }, - }) - } - case "no_result": - // Clean. - default: - res.Error = "query_status=" + parsed.QueryStatus - } - return []SourceResult{res} -} - -func (*threatFoxSource) Evaluate(r SourceResult) (bool, string) { - return evidenceEval(r, SeverityCrit) -} - -func (*threatFoxSource) Diagnose(res SourceResult) Diagnosis { - return Diagnosis{ - Severity: SeverityCrit, - Title: "Listed in abuse.ch ThreatFox as a threat IOC", - Detail: fmt.Sprintf( - "%d IOC(s) match this domain; threat(s): %s. ThreatFox tracks indicators of compromise including C2 servers, malware distribution hosts, and phishing infrastructure. Treat the host as compromised or abused: investigate recent DNS changes, audit hosted content, then request removal through the ThreatFox reference page.", - len(res.Evidence), joinNonEmpty(res.Reasons, ", "), - ), - Fix: res.Reference, - FixIsURL: res.Reference != "", - } -} diff --git a/checker/threatfox_test.go b/checker/threatfox_test.go deleted file mode 100644 index 5e5f12f..0000000 --- a/checker/threatfox_test.go +++ /dev/null @@ -1,96 +0,0 @@ -package checker - -import ( - "context" - "net/http" - "net/http/httptest" - "testing" - - sdk "git.happydns.org/checker-sdk-go/checker" -) - -func TestThreatFoxSource_NoResult(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"query_status":"no_result","data":[]}`)) - })) - defer srv.Close() - - s := &threatFoxSource{endpoint: srv.URL} - results := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_threatfox": true, "threatfox_auth_key": "k"}) - if len(results) != 1 { - t.Fatalf("expected 1 result, got %d", len(results)) - } - r := results[0] - listed, _ := s.Evaluate(r) - if !r.Enabled || listed || r.Error != "" { - t.Fatalf("expected enabled+clean, got %+v, Evaluate listed=%v", r, listed) - } -} - -func TestThreatFoxSource_Listed(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Header.Get("Auth-Key") == "" { - t.Errorf("missing Auth-Key header") - } - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{ - "query_status": "ok", - "data": [{ - "ioc_value": "example.com:443", - "ioc_type": "domain", - "threat_type": "botnet_cc", - "malware_printable": "Emotet", - "confidence_level": 75, - "first_seen": "2024-01-01 00:00:00 UTC", - "last_seen": "2024-06-01 00:00:00 UTC" - }] - }`)) - })) - defer srv.Close() - - s := &threatFoxSource{endpoint: srv.URL} - r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_threatfox": true, "threatfox_auth_key": "k"})[0] - if len(r.Evidence) != 1 { - t.Fatalf("expected 1 evidence item, got %+v", r) - } - if listed, severity := s.Evaluate(r); !listed || severity != SeverityCrit { - t.Errorf("expected Evaluate()=(true, crit), got (%v, %q)", listed, severity) - } - if r.Evidence[0].Value != "example.com:443" { - t.Errorf("evidence value = %q", r.Evidence[0].Value) - } - if len(r.Reasons) != 1 || r.Reasons[0] != "Emotet" { - t.Errorf("reasons = %v", r.Reasons) - } -} - -func TestThreatFoxSource_HTTPError(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusUnauthorized) - _, _ = w.Write([]byte("unauthorized")) - })) - defer srv.Close() - - s := &threatFoxSource{endpoint: srv.URL} - r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_threatfox": true, "threatfox_auth_key": "k"})[0] - if r.Error == "" { - t.Errorf("expected error, got %+v", r) - } -} - -func TestThreatFoxSource_Disabled(t *testing.T) { - s := &threatFoxSource{endpoint: "http://nope"} - r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_threatfox": false})[0] - if r.Enabled { - t.Errorf("expected disabled, got %+v", r) - } -} - -func TestThreatFoxSource_NoAuthKey(t *testing.T) { - s := &threatFoxSource{endpoint: "http://nope"} - r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_threatfox": true})[0] - if r.Enabled { - t.Errorf("expected disabled when no auth key, got %+v", r) - } -} diff --git a/checker/types.go b/checker/types.go index 48ba06c..4022687 100644 --- a/checker/types.go +++ b/checker/types.go @@ -31,7 +31,7 @@ type BlacklistData struct { func (d *BlacklistData) TotalHits() int { n := 0 for _, r := range d.Results { - if listed, _ := EvaluateResult(r); listed { + if r.Listed { n++ } } @@ -43,7 +43,7 @@ func (d *BlacklistData) TotalHits() int { func (d *BlacklistData) FilterListed() []SourceResult { out := make([]SourceResult, 0, len(d.Results)) for _, r := range d.Results { - if listed, _ := EvaluateResult(r); listed { + if r.Listed { out = append(out, r) } } diff --git a/checker/urlhaus.go b/checker/urlhaus.go index 1bac676..4636ac8 100644 --- a/checker/urlhaus.go +++ b/checker/urlhaus.go @@ -67,7 +67,7 @@ type urlhausURL struct { func (s *urlhausSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult { authKey := stringOpt(opts, "urlhaus_auth_key") if !sdk.GetBoolOption(opts, "enable_urlhaus", true) || registered == "" || authKey == "" { - return disabledResult(s.ID(), s.Name()) + return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: false}} } res := SourceResult{SourceID: s.ID(), SourceName: s.Name(), Enabled: true} @@ -120,6 +120,8 @@ func (s *urlhausSource) Query(ctx context.Context, domain, registered string, op if len(parsed.URLs) == 0 { return []SourceResult{res} } + res.Listed = true + res.Severity = SeverityCrit threats := map[string]bool{} details := urlhausDetails{} for _, u := range parsed.URLs { @@ -146,10 +148,6 @@ func (s *urlhausSource) Query(ctx context.Context, domain, registered string, op return []SourceResult{res} } -func (*urlhausSource) Evaluate(r SourceResult) (bool, string) { - return evidenceEval(r, SeverityCrit) -} - func (*urlhausSource) Diagnose(res SourceResult) Diagnosis { online := 0 for _, e := range res.Evidence { diff --git a/checker/urlhaus_test.go b/checker/urlhaus_test.go index 9128efd..0a10921 100644 --- a/checker/urlhaus_test.go +++ b/checker/urlhaus_test.go @@ -24,9 +24,8 @@ func TestURLhausSource_NoResults(t *testing.T) { t.Fatalf("expected 1 result, got %d", len(results)) } r := results[0] - listed, _ := s.Evaluate(r) - if !r.Enabled || listed || r.Error != "" { - t.Fatalf("expected enabled+clean, got %+v, Evaluate listed=%v", r, listed) + if !r.Enabled || r.Listed || r.Error != "" { + t.Fatalf("expected enabled+clean, got %+v", r) } } @@ -49,11 +48,8 @@ func TestURLhausSource_Listed(t *testing.T) { s := &urlhausSource{endpoint: srv.URL} r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_urlhaus": true, "urlhaus_auth_key": "k"})[0] - if len(r.Evidence) != 1 { - t.Fatalf("expected 1 evidence item, got %+v", r) - } - if listed, severity := s.Evaluate(r); !listed || severity != SeverityCrit { - t.Errorf("expected Evaluate()=(true, crit), got (%v, %q)", listed, severity) + if !r.Listed || len(r.Evidence) != 1 { + t.Fatalf("expected 1 listed evidence, got %+v", r) } if r.Evidence[0].Status != "online" { t.Errorf("evidence status = %q", r.Evidence[0].Status) diff --git a/checker/virustotal.go b/checker/virustotal.go index 14d1132..5213df9 100644 --- a/checker/virustotal.go +++ b/checker/virustotal.go @@ -60,7 +60,7 @@ type vtVendorVerdict struct { func (s *virusTotalSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult { apiKey := stringOpt(opts, "virustotal_api_key") if apiKey == "" { - return disabledResult(s.ID(), s.Name()) + return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: false}} } if registered == "" { return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: true}} @@ -146,6 +146,17 @@ func (s *virusTotalSource) Query(ctx context.Context, domain, registered string, return d.Vendors[i].Engine < d.Vendors[j].Engine }) res.Details = mustJSON(d) + + if d.Malicious == 0 && d.Suspicious == 0 { + // Clean. + return []SourceResult{res} + } + res.Listed = true + if d.Malicious > 0 { + res.Severity = SeverityCrit + } else { + res.Severity = SeverityWarn + } for _, v := range d.Vendors { res.Reasons = append(res.Reasons, v.Engine) res.Evidence = append(res.Evidence, Evidence{ @@ -156,23 +167,6 @@ func (s *virusTotalSource) Query(ctx context.Context, domain, registered string, return []SourceResult{res} } -func (*virusTotalSource) Evaluate(r SourceResult) (bool, string) { - var d vtDetails - if len(r.Details) == 0 { - return false, "" - } - if err := json.Unmarshal(r.Details, &d); err != nil { - return false, "" - } - if d.Malicious == 0 && d.Suspicious == 0 { - return false, "" - } - if d.Malicious > 0 { - return true, SeverityCrit - } - return true, SeverityWarn -} - func (*virusTotalSource) Diagnose(res SourceResult) Diagnosis { var d vtDetails _ = json.Unmarshal(res.Details, &d) diff --git a/checker/virustotal_test.go b/checker/virustotal_test.go index e53d2cc..aba955c 100644 --- a/checker/virustotal_test.go +++ b/checker/virustotal_test.go @@ -46,8 +46,8 @@ func TestVTSource_Listed(t *testing.T) { s := &virusTotalSource{endpoint: endpoint} r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"virustotal_api_key": "k"})[0] - if listed, severity := s.Evaluate(r); !listed || severity != SeverityCrit { - t.Errorf("expected Evaluate()=(true, crit), got (%v, %q)", listed, severity) + if !r.Listed || r.Severity != SeverityCrit { + t.Errorf("expected listed+crit, got %+v", r) } var d vtDetails if err := json.Unmarshal(r.Details, &d); err != nil { @@ -72,12 +72,9 @@ func TestVTSource_NotFound(t *testing.T) { s := &virusTotalSource{endpoint: endpoint} r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"virustotal_api_key": "k"})[0] - if r.Error != "" { + if r.Listed || r.Error != "" { t.Errorf("404 should be quiet not-listed: %+v", r) } - if listed, severity := s.Evaluate(r); listed || severity != "" { - t.Errorf("Evaluate() on clean result = (%v, %q), want (false, \"\")", listed, severity) - } if !strings.Contains(r.Reference, "example.com") { t.Errorf("reference URL missing: %+v", r) }