From f14209d4fa03df8895122aa74fbf714843d1d232 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Thu, 4 Jun 2026 18:38:45 +0900 Subject: [PATCH] blacklist: add domain reputation check via checker-blacklist Integrates the checker-blacklist module behind a new POST /blacklist/domain endpoint that aggregates reputation/blocklist sources for a given domain, plus a SvelteKit UI under /blacklist/domain mirroring the existing IP blacklist flow. Per-source credentials (VirusTotal, Safe Browsing) are exposed as CLI flags; free sources run unconditionally. Closes: https://git.nemunai.re/happyDomain/happyDeliver/issues/96 --- api/openapi.yaml | 6 + api/schemas.yaml | 100 +++++++ go.mod | 2 + go.sum | 4 + internal/api/handlers.go | 50 +++- internal/app/server.go | 3 +- internal/config/blacklist.go | 40 +++ internal/config/cli.go | 2 + internal/config/config.go | 12 + .../lib/components/DomainBlacklistCard.svelte | 260 ++++++++++++++++++ web/src/lib/components/index.ts | 1 + web/src/routes/blacklist/+page.svelte | 4 + web/src/routes/domain/[domain]/+page.svelte | 192 ++++++++++++- 13 files changed, 655 insertions(+), 21 deletions(-) create mode 100644 internal/config/blacklist.go create mode 100644 web/src/lib/components/DomainBlacklistCard.svelte diff --git a/api/openapi.yaml b/api/openapi.yaml index 2dbf304..1250268 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -363,6 +363,12 @@ components: $ref: './schemas.yaml#/components/schemas/BlacklistCheckRequest' BlacklistCheckResponse: $ref: './schemas.yaml#/components/schemas/BlacklistCheckResponse' + DomainBlacklistResult: + $ref: './schemas.yaml#/components/schemas/DomainBlacklistResult' + DomainBlacklistSourceResult: + $ref: './schemas.yaml#/components/schemas/DomainBlacklistSourceResult' + DomainBlacklistEvidence: + $ref: './schemas.yaml#/components/schemas/DomainBlacklistEvidence' TestSummary: $ref: './schemas.yaml#/components/schemas/TestSummary' TestListResponse: diff --git a/api/schemas.yaml b/api/schemas.yaml index 042a3b3..1e33dac 100644 --- a/api/schemas.yaml +++ b/api/schemas.yaml @@ -1217,6 +1217,9 @@ components: example: "A" dns_results: $ref: '#/components/schemas/DNSResults' + blacklist: + $ref: '#/components/schemas/DomainBlacklistResult' + description: Domain reputation/blacklist aggregation (omitted when the check could not be run) BlacklistCheckRequest: type: object @@ -1268,6 +1271,103 @@ components: $ref: '#/components/schemas/BlacklistCheck' description: List of DNS whitelist check results (informational only) + DomainBlacklistResult: + type: object + required: + - registered_domain + - collected_at + - results + properties: + registered_domain: + type: string + description: eTLD+1 of the input domain + example: "example.com" + collected_at: + type: string + format: date-time + description: When the aggregation finished + score: + type: integer + minimum: 0 + maximum: 100 + description: Reputation score (0-100, higher is better). Omitted when the verdict is inconclusive (no usable source). + example: 100 + grade: + type: string + enum: [A+, A, B, C, D, E, F] + description: Letter grade derived from the score. Omitted when the verdict is inconclusive. + example: "A+" + results: + type: array + items: + $ref: '#/components/schemas/DomainBlacklistSourceResult' + description: One entry per registered source (disabled sources included with enabled=false) + + DomainBlacklistSourceResult: + type: object + required: + - source_id + - source_name + - enabled + - listed + properties: + source_id: + type: string + example: "quad9" + source_name: + type: string + example: "Quad9" + subject: + type: string + description: Per-zone identifier (DNSBL zones only) + enabled: + type: boolean + description: False when the source is disabled or missing credentials + listed: + type: boolean + description: Verdict from the source's Evaluate (false when disabled or errored) + blocked_query: + type: boolean + description: Resolver returned a block response (not a real listing) + severity: + type: string + description: Severity attached to the verdict (crit, warn, info, ok, or empty) + reasons: + type: array + items: { type: string } + evidence: + type: array + items: + $ref: '#/components/schemas/DomainBlacklistEvidence' + lookup_url: + type: string + removal_url: + type: string + reference: + type: string + error: + type: string + details: + type: object + additionalProperties: true + description: Source-specific structured data (free-form) + + DomainBlacklistEvidence: + type: object + required: + - label + - value + properties: + label: + type: string + value: + type: string + status: + type: string + extra: + type: object + additionalProperties: { type: string } + TestSummary: type: object required: diff --git a/go.mod b/go.mod index c638f4a..e1a37fe 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,8 @@ module git.happydns.org/happyDeliver go 1.25.0 require ( + git.happydns.org/checker-blacklist v0.4.0 + git.happydns.org/checker-sdk-go v1.9.0 github.com/JGLTechnologies/gin-rate-limit v1.5.8 github.com/emersion/go-smtp v0.24.0 github.com/getkin/kin-openapi v0.138.0 diff --git a/go.sum b/go.sum index f467434..1896d42 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,7 @@ +git.happydns.org/checker-blacklist v0.4.0 h1:mTOWz2tcMXGU2WFVM9VLxnSJ7mFXL2Lhq5NBq+lUU7g= +git.happydns.org/checker-blacklist v0.4.0/go.mod h1:huOwQWfAA+Wo+WbUbtyRIS/Y0eA+C3YrguGuJL+3qEE= +git.happydns.org/checker-sdk-go v1.9.0 h1:orBRymir+p6PMHVa4focryPKhTVWT7JAv6u9Ido5KF0= +git.happydns.org/checker-sdk-go v1.9.0/go.mod h1:aNAcfYFfbhvH9kJhE0Njp5GX0dQbxdRB0rJ0KvSC5nI= github.com/JGLTechnologies/gin-rate-limit v1.5.8 h1:KiaHIEbpYxHpDvjhpjIif8fnVmjdw/afCMdGoN1AsB0= github.com/JGLTechnologies/gin-rate-limit v1.5.8/go.mod h1:t9eLOUxikPI0TzKy0VYRbZJr7hBP2Qg9E3JigoxF70g= github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= diff --git a/internal/api/handlers.go b/internal/api/handlers.go index de2d5df..4a499dc 100644 --- a/internal/api/handlers.go +++ b/internal/api/handlers.go @@ -22,6 +22,7 @@ package api import ( + "context" "fmt" "net/http" "time" @@ -30,8 +31,10 @@ import ( "github.com/google/uuid" openapi_types "github.com/oapi-codegen/runtime/types" + sdk "git.happydns.org/checker-sdk-go/checker" "git.happydns.org/happyDeliver/internal/config" "git.happydns.org/happyDeliver/internal/model" + "git.happydns.org/happyDeliver/internal/reputation" "git.happydns.org/happyDeliver/internal/storage" "git.happydns.org/happyDeliver/internal/utils" "git.happydns.org/happyDeliver/internal/version" @@ -47,19 +50,21 @@ type EmailAnalyzer interface { // APIHandler implements the ServerInterface for handling API requests type APIHandler struct { - storage storage.Storage - config *config.Config - analyzer EmailAnalyzer - startTime time.Time + storage storage.Storage + config *config.Config + analyzer EmailAnalyzer + blacklistProvider sdk.ObservationProvider + startTime time.Time } // NewAPIHandler creates a new API handler -func NewAPIHandler(store storage.Storage, cfg *config.Config, analyzer EmailAnalyzer) *APIHandler { +func NewAPIHandler(store storage.Storage, cfg *config.Config, analyzer EmailAnalyzer, blacklistProvider sdk.ObservationProvider) *APIHandler { return &APIHandler{ - storage: store, - config: cfg, - analyzer: analyzer, - startTime: time.Now(), + storage: store, + config: cfg, + analyzer: analyzer, + blacklistProvider: blacklistProvider, + startTime: time.Now(), } } @@ -339,6 +344,7 @@ func (h *APIHandler) TestDomain(c *gin.Context) { Score: score, Grade: responseGrade, DnsResults: *dnsResults, + Blacklist: h.runDomainBlacklist(c.Request.Context(), request.Domain), } c.JSON(http.StatusOK, response) @@ -383,6 +389,32 @@ func (h *APIHandler) CheckBlacklist(c *gin.Context) { c.JSON(http.StatusOK, response) } +// runDomainBlacklist runs the checker-blacklist aggregation against a domain. +// It returns nil (and logs nothing fatal) when the check cannot be run, so the +// surrounding domain analysis still succeeds. +func (h *APIHandler) runDomainBlacklist(ctx context.Context, domain string) *model.DomainBlacklistResult { + opts := h.config.Analysis.Blacklist.AsCheckerOptions() + // "domain_name" is the option key the checker-blacklist provider reads + // (see checker/collect.go in the checker-blacklist module). + opts["domain_name"] = domain + + // Cap the aggregation: sources run concurrently, each with its own + // timeouts; this is the host-side ceiling. + timeout := h.config.Analysis.HTTPTimeout + if timeout <= 0 { + timeout = 30 * time.Second + } + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + raw, err := h.blacklistProvider.Collect(ctx, opts) + if err != nil { + return nil + } + + return reputation.FromObservation(raw) +} + // ListTests returns a paginated list of test summaries // (GET /tests) func (h *APIHandler) ListTests(c *gin.Context, params ListTestsParams) { diff --git a/internal/app/server.go b/internal/app/server.go index 7149f45..efcf8df 100644 --- a/internal/app/server.go +++ b/internal/app/server.go @@ -30,6 +30,7 @@ import ( ratelimit "github.com/JGLTechnologies/gin-rate-limit" "github.com/gin-gonic/gin" + blacklist "git.happydns.org/checker-blacklist/checker" "git.happydns.org/happyDeliver/internal/api" "git.happydns.org/happyDeliver/internal/config" "git.happydns.org/happyDeliver/internal/lmtp" @@ -70,7 +71,7 @@ func RunServer(cfg *config.Config) error { analyzerAdapter := analyzer.NewAPIAdapter(cfg) // Create API handler - handler := api.NewAPIHandler(store, cfg, analyzerAdapter) + handler := api.NewAPIHandler(store, cfg, analyzerAdapter, blacklist.Provider()) // Set up Gin router if os.Getenv("GIN_MODE") == "" { diff --git a/internal/config/blacklist.go b/internal/config/blacklist.go new file mode 100644 index 0000000..8185431 --- /dev/null +++ b/internal/config/blacklist.go @@ -0,0 +1,40 @@ +// This file is part of the happyDeliver (R) project. +// Copyright (c) 2025 happyDomain +// Authors: Pierre-Olivier Mercier, et al. +// +// This program is offered under a commercial and under the AGPL license. +// For commercial licensing, contact us at . +// +// For AGPL licensing: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package config + +import ( + sdk "git.happydns.org/checker-sdk-go/checker" +) + +// AsCheckerOptions returns the map that checker-blacklist sources read +// via stringOpt(). Empty values are omitted so sources that require a +// credential stay disabled rather than failing with an empty key. +func (b BlacklistConfig) AsCheckerOptions() sdk.CheckerOptions { + opts := sdk.CheckerOptions{} + if b.VirusTotalAPIKey != "" { + opts["virustotal_api_key"] = b.VirusTotalAPIKey + } + if b.SafeBrowsingAPIKey != "" { + opts["safebrowsing_api_key"] = b.SafeBrowsingAPIKey + } + return opts +} diff --git a/internal/config/cli.go b/internal/config/cli.go index fcc914f..9779c94 100644 --- a/internal/config/cli.go +++ b/internal/config/cli.go @@ -40,6 +40,8 @@ func declareFlags(o *Config) { flag.Var(&StringArray{&o.Analysis.RBLs}, "rbl", "Append a RBL (use this option multiple time to append multiple RBLs)") flag.BoolVar(&o.Analysis.CheckAllIPs, "check-all-ips", o.Analysis.CheckAllIPs, "Check all IPs found in email headers against RBLs (not just the first one)") flag.StringVar(&o.Analysis.RspamdAPIURL, "rspamd-api-url", o.Analysis.RspamdAPIURL, "rspamd API URL for symbol descriptions (default: use embedded list)") + flag.StringVar(&o.Analysis.Blacklist.VirusTotalAPIKey, "blacklist-virustotal-api-key", o.Analysis.Blacklist.VirusTotalAPIKey, "VirusTotal v3 API key for the domain blacklist checker") + flag.StringVar(&o.Analysis.Blacklist.SafeBrowsingAPIKey, "blacklist-safebrowsing-api-key", o.Analysis.Blacklist.SafeBrowsingAPIKey, "Google Safe Browsing API key for the domain blacklist checker") flag.DurationVar(&o.ReportRetention, "report-retention", o.ReportRetention, "How long to keep reports (e.g., 720h, 30d). 0 = keep forever") flag.UintVar(&o.RateLimit, "rate-limit", o.RateLimit, "API rate limit (requests per second per IP)") flag.Var(&URL{&o.SurveyURL}, "survey-url", "URL for user feedback survey") diff --git a/internal/config/config.go b/internal/config/config.go index b264994..6cf8110 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -75,6 +75,18 @@ type AnalysisConfig struct { DNSWLs []string CheckAllIPs bool // Check all IPs found in headers, not just the first one RspamdAPIURL string // rspamd API URL for fetching symbol descriptions (empty = use embedded list) + Blacklist BlacklistConfig +} + +// BlacklistConfig holds per-source credentials/options for the +// domain-oriented checker-blacklist provider. Keys must match the +// option IDs declared by each source in the checker-blacklist module +// (see checker/virustotal.go, checker/safebrowsing.go, …). Free sources +// (Quad9, OISD, URLhaus, OpenPhish, Disconnect, Botvrij, …) need no +// configuration. +type BlacklistConfig struct { + VirusTotalAPIKey string + SafeBrowsingAPIKey string } // DefaultConfig returns a configuration with sensible defaults diff --git a/web/src/lib/components/DomainBlacklistCard.svelte b/web/src/lib/components/DomainBlacklistCard.svelte new file mode 100644 index 0000000..2559446 --- /dev/null +++ b/web/src/lib/components/DomainBlacklistCard.svelte @@ -0,0 +1,260 @@ + + +
+
+

+ + Source Verdicts +

+
+
+
+ + + + + + + + + + + {#each sorted as r (rowKey(r))} + {@const key = rowKey(r)} + {@const open = openRows.has(key)} + {@const expandable = hasDetails(r)} + + + + + + + {#if expandable && open} + + + + + {/if} + {/each} + +
StatusSourceDetailLinks
+ {statusLabel(r)} + +
{r.source_name}
+ + {r.source_id} + {#if r.subject} + · {r.subject} + {/if} + + 		+ {firstReason(r)} + {#if expandable} + + {/if} + + {#if r.lookup_url} + + + + {/if} +
+ {#if r.reasons && r.reasons.length > 0} +
    + {#each r.reasons as reason} +
  • {reason}
    • + {/each} +
+ {/if} + {#if r.evidence && r.evidence.length > 0} + + + + + + + + + + {#each r.evidence as ev} + + + + + + {/each} + +
LabelValueStatus
{ev.label} + {ev.value} + + {#if ev.status} + {ev.status} + {:else} + — + {/if} +
+ {/if} + {#if r.reference} +

+ Reference: {r.reference} +

+ {/if} +
+
+
+
+ + diff --git a/web/src/lib/components/index.ts b/web/src/lib/components/index.ts index a593801..9d73db7 100644 --- a/web/src/lib/components/index.ts +++ b/web/src/lib/components/index.ts @@ -6,6 +6,7 @@ export { default as ContentAnalysisCard } from "./ContentAnalysisCard.svelte"; export { default as DkimRecordsDisplay } from "./DkimRecordsDisplay.svelte"; export { default as DmarcRecordDisplay } from "./DmarcRecordDisplay.svelte"; export { default as DnsRecordsCard } from "./DnsRecordsCard.svelte"; +export { default as DomainBlacklistCard } from "./DomainBlacklistCard.svelte"; export { default as EmailAddressDisplay } from "./EmailAddressDisplay.svelte"; export { default as EmailPathCard } from "./EmailPathCard.svelte"; export { default as ErrorDisplay } from "./ErrorDisplay.svelte"; diff --git a/web/src/routes/blacklist/+page.svelte b/web/src/routes/blacklist/+page.svelte index d2946b8..4d2e8e4 100644 --- a/web/src/routes/blacklist/+page.svelte +++ b/web/src/routes/blacklist/+page.svelte @@ -161,6 +161,10 @@ Send Test Email + + + Check a Domain + diff --git a/web/src/routes/domain/[domain]/+page.svelte b/web/src/routes/domain/[domain]/+page.svelte index d866e21..8c61cb2 100644 --- a/web/src/routes/domain/[domain]/+page.svelte +++ b/web/src/routes/domain/[domain]/+page.svelte @@ -4,7 +4,7 @@ import { testDomain } from "$lib/api"; import type { DomainTestResponse } from "$lib/api/types.gen"; - import { DnsRecordsCard, GradeDisplay, TinySurvey } from "$lib/components"; + import { DnsRecordsCard, DomainBlacklistCard, GradeDisplay, TinySurvey } from "$lib/components"; import { theme } from "$lib/stores/theme"; let domain = $derived(page.params.domain); @@ -12,6 +12,44 @@ let error = $state(null); let result = $state(null); + let blacklist = $derived(result?.blacklist ?? null); + + let blacklistSummary = $derived.by(() => { + if (!blacklist) return null; + const enabled = blacklist.results.filter((r) => r.enabled); + const disabled = blacklist.results.length - enabled.length; + const errored = enabled.filter((r) => r.error).length; + const listed = enabled.filter((r) => r.listed); + const critical = listed.filter((r) => r.severity === "crit").length; + return { + total: blacklist.results.length, + enabled: enabled.length, + disabled, + errored, + listed: listed.length, + critical, + }; + }); + + type Verdict = "danger" | "warn" | "inconclusive" | "ok"; + + let blacklistVerdict = $derived.by(() => { + const s = blacklistSummary; + if (!s) return null; + if (s.critical > 0) return "danger"; + if (s.listed > 0) return "warn"; + if (s.enabled > 0 && s.errored === s.enabled) return "inconclusive"; + return "ok"; + }); + + function formatCollectedAt(iso: string): string { + try { + return new Date(iso).toLocaleString(); + } catch { + return iso; + } + } + async function analyzeDomain() { loading = true; error = null; @@ -74,7 +112,9 @@

Analyzing {domain}...

-

Checking DNS records and configuration

+

+ Checking DNS records, configuration and domain reputation +

{:else if error} @@ -116,14 +156,31 @@

Domain Configuration Score

{/if} -
+
- - DNS +
+ + DNS +
+ {#if blacklist} +
+ + Reputation +
+ {/if}
@@ -144,6 +201,119 @@ domainOnly={true} /> + + {#if blacklist && blacklistSummary} +
+
+
+
+

+ + Domain Reputation +

+ {#if blacklist.registered_domain && blacklist.registered_domain !== result.domain} +

+ Registered domain: + {blacklist.registered_domain} +

+ {/if} + + {#if blacklistVerdict === "danger"} +
+ + Listed on {blacklistSummary.critical} high-severity + source{blacklistSummary.critical > 1 + ? "s" + : ""} +

+ This domain is reported by sources flagged + critical. Take action to delist. +

+
+ {:else if blacklistVerdict === "warn"} +
+ + Listed on {blacklistSummary.listed} source{blacklistSummary.listed > + 1 + ? "s" + : ""} +

+ Listed without critical severity — review the + source verdicts below. +

+
+ {:else if blacklistVerdict === "inconclusive"} +
+ + Inconclusive +

+ All enabled sources returned errors. Try again + later. +

+
+ {:else} +
+ + No source reports this domain +

+ Clean across all {blacklistSummary.enabled} enabled + source{blacklistSummary.enabled > 1 ? "s" : ""}. +

+
+ {/if} +
+
+
+
+
+
+ {blacklistSummary.enabled} +
+ Enabled +
+
+
+ {blacklistSummary.listed} +
+ Listed +
+
+
+ {blacklistSummary.disabled} +
+ Disabled +
+
+ {#if blacklistSummary.errored > 0} + + {blacklistSummary.errored} source{blacklistSummary.errored > + 1 + ? "s" + : ""} errored + + {/if} + + Collected {formatCollectedAt( + blacklist.collected_at, + )} + +
+
+
+
+
+ + + {/if} +
@@ -152,9 +322,9 @@ Want Complete Email Analysis?

- This domain-only test checks DNS configuration. For comprehensive - deliverability testing including DKIM verification, content - analysis, spam scoring, and blacklist checks: + This domain test checks DNS configuration and domain reputation. For + comprehensive deliverability testing including DKIM verification, + content analysis, spam scoring, and sending-IP blacklist checks: