diff --git a/api/openapi.yaml b/api/openapi.yaml index 1250268..2dbf304 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -363,12 +363,6 @@ components: $ref: './schemas.yaml#/components/schemas/BlacklistCheckRequest' BlacklistCheckResponse: $ref: './schemas.yaml#/components/schemas/BlacklistCheckResponse' - DomainBlacklistResult: - $ref: './schemas.yaml#/components/schemas/DomainBlacklistResult' - DomainBlacklistSourceResult: - $ref: './schemas.yaml#/components/schemas/DomainBlacklistSourceResult' - DomainBlacklistEvidence: - $ref: './schemas.yaml#/components/schemas/DomainBlacklistEvidence' TestSummary: $ref: './schemas.yaml#/components/schemas/TestSummary' TestListResponse: diff --git a/api/schemas.yaml b/api/schemas.yaml index 1e33dac..042a3b3 100644 --- a/api/schemas.yaml +++ b/api/schemas.yaml @@ -1217,9 +1217,6 @@ components: example: "A" dns_results: $ref: '#/components/schemas/DNSResults' - blacklist: - $ref: '#/components/schemas/DomainBlacklistResult' - description: Domain reputation/blacklist aggregation (omitted when the check could not be run) BlacklistCheckRequest: type: object @@ -1271,103 +1268,6 @@ components: $ref: '#/components/schemas/BlacklistCheck' description: List of DNS whitelist check results (informational only) - DomainBlacklistResult: - type: object - required: - - registered_domain - - collected_at - - results - properties: - registered_domain: - type: string - description: eTLD+1 of the input domain - example: "example.com" - collected_at: - type: string - format: date-time - description: When the aggregation finished - score: - type: integer - minimum: 0 - maximum: 100 - description: Reputation score (0-100, higher is better). Omitted when the verdict is inconclusive (no usable source). - example: 100 - grade: - type: string - enum: [A+, A, B, C, D, E, F] - description: Letter grade derived from the score. Omitted when the verdict is inconclusive. - example: "A+" - results: - type: array - items: - $ref: '#/components/schemas/DomainBlacklistSourceResult' - description: One entry per registered source (disabled sources included with enabled=false) - - DomainBlacklistSourceResult: - type: object - required: - - source_id - - source_name - - enabled - - listed - properties: - source_id: - type: string - example: "quad9" - source_name: - type: string - example: "Quad9" - subject: - type: string - description: Per-zone identifier (DNSBL zones only) - enabled: - type: boolean - description: False when the source is disabled or missing credentials - listed: - type: boolean - description: Verdict from the source's Evaluate (false when disabled or errored) - blocked_query: - type: boolean - description: Resolver returned a block response (not a real listing) - severity: - type: string - description: Severity attached to the verdict (crit, warn, info, ok, or empty) - reasons: - type: array - items: { type: string } - evidence: - type: array - items: - $ref: '#/components/schemas/DomainBlacklistEvidence' - lookup_url: - type: string - removal_url: - type: string - reference: - type: string - error: - type: string - details: - type: object - additionalProperties: true - description: Source-specific structured data (free-form) - - DomainBlacklistEvidence: - type: object - required: - - label - - value - properties: - label: - type: string - value: - type: string - status: - type: string - extra: - type: object - additionalProperties: { type: string } - TestSummary: type: object required: diff --git a/go.mod b/go.mod index e1a37fe..c638f4a 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,6 @@ module git.happydns.org/happyDeliver go 1.25.0 require ( - git.happydns.org/checker-blacklist v0.4.0 - git.happydns.org/checker-sdk-go v1.9.0 github.com/JGLTechnologies/gin-rate-limit v1.5.8 github.com/emersion/go-smtp v0.24.0 github.com/getkin/kin-openapi v0.138.0 diff --git a/go.sum b/go.sum index 1896d42..f467434 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,3 @@ -git.happydns.org/checker-blacklist v0.4.0 h1:mTOWz2tcMXGU2WFVM9VLxnSJ7mFXL2Lhq5NBq+lUU7g= -git.happydns.org/checker-blacklist v0.4.0/go.mod h1:huOwQWfAA+Wo+WbUbtyRIS/Y0eA+C3YrguGuJL+3qEE= -git.happydns.org/checker-sdk-go v1.9.0 h1:orBRymir+p6PMHVa4focryPKhTVWT7JAv6u9Ido5KF0= -git.happydns.org/checker-sdk-go v1.9.0/go.mod h1:aNAcfYFfbhvH9kJhE0Njp5GX0dQbxdRB0rJ0KvSC5nI= github.com/JGLTechnologies/gin-rate-limit v1.5.8 h1:KiaHIEbpYxHpDvjhpjIif8fnVmjdw/afCMdGoN1AsB0= github.com/JGLTechnologies/gin-rate-limit v1.5.8/go.mod h1:t9eLOUxikPI0TzKy0VYRbZJr7hBP2Qg9E3JigoxF70g= github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= diff --git a/internal/api/handlers.go b/internal/api/handlers.go index 4a499dc..de2d5df 100644 --- a/internal/api/handlers.go +++ b/internal/api/handlers.go @@ -22,7 +22,6 @@ package api import ( - "context" "fmt" "net/http" "time" @@ -31,10 +30,8 @@ import ( "github.com/google/uuid" openapi_types "github.com/oapi-codegen/runtime/types" - sdk "git.happydns.org/checker-sdk-go/checker" "git.happydns.org/happyDeliver/internal/config" "git.happydns.org/happyDeliver/internal/model" - "git.happydns.org/happyDeliver/internal/reputation" "git.happydns.org/happyDeliver/internal/storage" "git.happydns.org/happyDeliver/internal/utils" "git.happydns.org/happyDeliver/internal/version" @@ -50,21 +47,19 @@ type EmailAnalyzer interface { // APIHandler implements the ServerInterface for handling API requests type APIHandler struct { - storage storage.Storage - config *config.Config - analyzer EmailAnalyzer - blacklistProvider sdk.ObservationProvider - startTime time.Time + storage storage.Storage + config *config.Config + analyzer EmailAnalyzer + startTime time.Time } // NewAPIHandler creates a new API handler -func NewAPIHandler(store storage.Storage, cfg *config.Config, analyzer EmailAnalyzer, blacklistProvider sdk.ObservationProvider) *APIHandler { +func NewAPIHandler(store storage.Storage, cfg *config.Config, analyzer EmailAnalyzer) *APIHandler { return &APIHandler{ - storage: store, - config: cfg, - analyzer: analyzer, - blacklistProvider: blacklistProvider, - startTime: time.Now(), + storage: store, + config: cfg, + analyzer: analyzer, + startTime: time.Now(), } } @@ -344,7 +339,6 @@ func (h *APIHandler) TestDomain(c *gin.Context) { Score: score, Grade: responseGrade, DnsResults: *dnsResults, - Blacklist: h.runDomainBlacklist(c.Request.Context(), request.Domain), } c.JSON(http.StatusOK, response) @@ -389,32 +383,6 @@ func (h *APIHandler) CheckBlacklist(c *gin.Context) { c.JSON(http.StatusOK, response) } -// runDomainBlacklist runs the checker-blacklist aggregation against a domain. -// It returns nil (and logs nothing fatal) when the check cannot be run, so the -// surrounding domain analysis still succeeds. -func (h *APIHandler) runDomainBlacklist(ctx context.Context, domain string) *model.DomainBlacklistResult { - opts := h.config.Analysis.Blacklist.AsCheckerOptions() - // "domain_name" is the option key the checker-blacklist provider reads - // (see checker/collect.go in the checker-blacklist module). - opts["domain_name"] = domain - - // Cap the aggregation: sources run concurrently, each with its own - // timeouts; this is the host-side ceiling. - timeout := h.config.Analysis.HTTPTimeout - if timeout <= 0 { - timeout = 30 * time.Second - } - ctx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - - raw, err := h.blacklistProvider.Collect(ctx, opts) - if err != nil { - return nil - } - - return reputation.FromObservation(raw) -} - // ListTests returns a paginated list of test summaries // (GET /tests) func (h *APIHandler) ListTests(c *gin.Context, params ListTestsParams) { diff --git a/internal/app/server.go b/internal/app/server.go index efcf8df..7149f45 100644 --- a/internal/app/server.go +++ b/internal/app/server.go @@ -30,7 +30,6 @@ import ( ratelimit "github.com/JGLTechnologies/gin-rate-limit" "github.com/gin-gonic/gin" - blacklist "git.happydns.org/checker-blacklist/checker" "git.happydns.org/happyDeliver/internal/api" "git.happydns.org/happyDeliver/internal/config" "git.happydns.org/happyDeliver/internal/lmtp" @@ -71,7 +70,7 @@ func RunServer(cfg *config.Config) error { analyzerAdapter := analyzer.NewAPIAdapter(cfg) // Create API handler - handler := api.NewAPIHandler(store, cfg, analyzerAdapter, blacklist.Provider()) + handler := api.NewAPIHandler(store, cfg, analyzerAdapter) // Set up Gin router if os.Getenv("GIN_MODE") == "" { diff --git a/internal/config/blacklist.go b/internal/config/blacklist.go deleted file mode 100644 index 8185431..0000000 --- a/internal/config/blacklist.go +++ /dev/null @@ -1,40 +0,0 @@ -// This file is part of the happyDeliver (R) project. -// Copyright (c) 2025 happyDomain -// Authors: Pierre-Olivier Mercier, et al. -// -// This program is offered under a commercial and under the AGPL license. -// For commercial licensing, contact us at . -// -// For AGPL licensing: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package config - -import ( - sdk "git.happydns.org/checker-sdk-go/checker" -) - -// AsCheckerOptions returns the map that checker-blacklist sources read -// via stringOpt(). Empty values are omitted so sources that require a -// credential stay disabled rather than failing with an empty key. -func (b BlacklistConfig) AsCheckerOptions() sdk.CheckerOptions { - opts := sdk.CheckerOptions{} - if b.VirusTotalAPIKey != "" { - opts["virustotal_api_key"] = b.VirusTotalAPIKey - } - if b.SafeBrowsingAPIKey != "" { - opts["safebrowsing_api_key"] = b.SafeBrowsingAPIKey - } - return opts -} diff --git a/internal/config/cli.go b/internal/config/cli.go index 9779c94..fcc914f 100644 --- a/internal/config/cli.go +++ b/internal/config/cli.go @@ -40,8 +40,6 @@ func declareFlags(o *Config) { flag.Var(&StringArray{&o.Analysis.RBLs}, "rbl", "Append a RBL (use this option multiple time to append multiple RBLs)") flag.BoolVar(&o.Analysis.CheckAllIPs, "check-all-ips", o.Analysis.CheckAllIPs, "Check all IPs found in email headers against RBLs (not just the first one)") flag.StringVar(&o.Analysis.RspamdAPIURL, "rspamd-api-url", o.Analysis.RspamdAPIURL, "rspamd API URL for symbol descriptions (default: use embedded list)") - flag.StringVar(&o.Analysis.Blacklist.VirusTotalAPIKey, "blacklist-virustotal-api-key", o.Analysis.Blacklist.VirusTotalAPIKey, "VirusTotal v3 API key for the domain blacklist checker") - flag.StringVar(&o.Analysis.Blacklist.SafeBrowsingAPIKey, "blacklist-safebrowsing-api-key", o.Analysis.Blacklist.SafeBrowsingAPIKey, "Google Safe Browsing API key for the domain blacklist checker") flag.DurationVar(&o.ReportRetention, "report-retention", o.ReportRetention, "How long to keep reports (e.g., 720h, 30d). 0 = keep forever") flag.UintVar(&o.RateLimit, "rate-limit", o.RateLimit, "API rate limit (requests per second per IP)") flag.Var(&URL{&o.SurveyURL}, "survey-url", "URL for user feedback survey") diff --git a/internal/config/config.go b/internal/config/config.go index 6cf8110..b264994 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -75,18 +75,6 @@ type AnalysisConfig struct { DNSWLs []string CheckAllIPs bool // Check all IPs found in headers, not just the first one RspamdAPIURL string // rspamd API URL for fetching symbol descriptions (empty = use embedded list) - Blacklist BlacklistConfig -} - -// BlacklistConfig holds per-source credentials/options for the -// domain-oriented checker-blacklist provider. Keys must match the -// option IDs declared by each source in the checker-blacklist module -// (see checker/virustotal.go, checker/safebrowsing.go, …). Free sources -// (Quad9, OISD, URLhaus, OpenPhish, Disconnect, Botvrij, …) need no -// configuration. -type BlacklistConfig struct { - VirusTotalAPIKey string - SafeBrowsingAPIKey string } // DefaultConfig returns a configuration with sensible defaults diff --git a/web/src/lib/components/DomainBlacklistCard.svelte b/web/src/lib/components/DomainBlacklistCard.svelte deleted file mode 100644 index 2559446..0000000 --- a/web/src/lib/components/DomainBlacklistCard.svelte +++ /dev/null @@ -1,260 +0,0 @@ - - -
-
-

- - Source Verdicts -

-
-
-
- - - - - - - - - - - {#each sorted as r (rowKey(r))} - {@const key = rowKey(r)} - {@const open = openRows.has(key)} - {@const expandable = hasDetails(r)} - - - - - - - {#if expandable && open} - - - - - {/if} - {/each} - -
StatusSourceDetailLinks
- {statusLabel(r)} - -
{r.source_name}
- - {r.source_id} - {#if r.subject} - · {r.subject} - {/if} - - 		- {firstReason(r)} - {#if expandable} - - {/if} - - {#if r.lookup_url} - - - - {/if} -
- {#if r.reasons && r.reasons.length > 0} -
    - {#each r.reasons as reason} -
  • {reason}
    • - {/each} -
- {/if} - {#if r.evidence && r.evidence.length > 0} - - - - - - - - - - {#each r.evidence as ev} - - - - - - {/each} - -
LabelValueStatus
{ev.label} - {ev.value} - - {#if ev.status} - {ev.status} - {:else} - — - {/if} -
- {/if} - {#if r.reference} -

- Reference: {r.reference} -

- {/if} -
-
-
-
- - diff --git a/web/src/lib/components/index.ts b/web/src/lib/components/index.ts index 9d73db7..a593801 100644 --- a/web/src/lib/components/index.ts +++ b/web/src/lib/components/index.ts @@ -6,7 +6,6 @@ export { default as ContentAnalysisCard } from "./ContentAnalysisCard.svelte"; export { default as DkimRecordsDisplay } from "./DkimRecordsDisplay.svelte"; export { default as DmarcRecordDisplay } from "./DmarcRecordDisplay.svelte"; export { default as DnsRecordsCard } from "./DnsRecordsCard.svelte"; -export { default as DomainBlacklistCard } from "./DomainBlacklistCard.svelte"; export { default as EmailAddressDisplay } from "./EmailAddressDisplay.svelte"; export { default as EmailPathCard } from "./EmailPathCard.svelte"; export { default as ErrorDisplay } from "./ErrorDisplay.svelte"; diff --git a/web/src/routes/blacklist/+page.svelte b/web/src/routes/blacklist/+page.svelte index 4d2e8e4..d2946b8 100644 --- a/web/src/routes/blacklist/+page.svelte +++ b/web/src/routes/blacklist/+page.svelte @@ -161,10 +161,6 @@ Send Test Email - - - Check a Domain - diff --git a/web/src/routes/domain/[domain]/+page.svelte b/web/src/routes/domain/[domain]/+page.svelte index 8c61cb2..d866e21 100644 --- a/web/src/routes/domain/[domain]/+page.svelte +++ b/web/src/routes/domain/[domain]/+page.svelte @@ -4,7 +4,7 @@ import { testDomain } from "$lib/api"; import type { DomainTestResponse } from "$lib/api/types.gen"; - import { DnsRecordsCard, DomainBlacklistCard, GradeDisplay, TinySurvey } from "$lib/components"; + import { DnsRecordsCard, GradeDisplay, TinySurvey } from "$lib/components"; import { theme } from "$lib/stores/theme"; let domain = $derived(page.params.domain); @@ -12,44 +12,6 @@ let error = $state(null); let result = $state(null); - let blacklist = $derived(result?.blacklist ?? null); - - let blacklistSummary = $derived.by(() => { - if (!blacklist) return null; - const enabled = blacklist.results.filter((r) => r.enabled); - const disabled = blacklist.results.length - enabled.length; - const errored = enabled.filter((r) => r.error).length; - const listed = enabled.filter((r) => r.listed); - const critical = listed.filter((r) => r.severity === "crit").length; - return { - total: blacklist.results.length, - enabled: enabled.length, - disabled, - errored, - listed: listed.length, - critical, - }; - }); - - type Verdict = "danger" | "warn" | "inconclusive" | "ok"; - - let blacklistVerdict = $derived.by(() => { - const s = blacklistSummary; - if (!s) return null; - if (s.critical > 0) return "danger"; - if (s.listed > 0) return "warn"; - if (s.enabled > 0 && s.errored === s.enabled) return "inconclusive"; - return "ok"; - }); - - function formatCollectedAt(iso: string): string { - try { - return new Date(iso).toLocaleString(); - } catch { - return iso; - } - } - async function analyzeDomain() { loading = true; error = null; @@ -112,9 +74,7 @@

Analyzing {domain}...

-

- Checking DNS records, configuration and domain reputation -

+

Checking DNS records and configuration

{:else if error} @@ -156,31 +116,14 @@

Domain Configuration Score

{/if} -
+
-
- - DNS -
- {#if blacklist} -
- - Reputation -
- {/if} + + DNS
@@ -201,119 +144,6 @@ domainOnly={true} /> - - {#if blacklist && blacklistSummary} -
-
-
-
-

- - Domain Reputation -

- {#if blacklist.registered_domain && blacklist.registered_domain !== result.domain} -

- Registered domain: - {blacklist.registered_domain} -

- {/if} - - {#if blacklistVerdict === "danger"} -
- - Listed on {blacklistSummary.critical} high-severity - source{blacklistSummary.critical > 1 - ? "s" - : ""} -

- This domain is reported by sources flagged - critical. Take action to delist. -

-
- {:else if blacklistVerdict === "warn"} -
- - Listed on {blacklistSummary.listed} source{blacklistSummary.listed > - 1 - ? "s" - : ""} -

- Listed without critical severity — review the - source verdicts below. -

-
- {:else if blacklistVerdict === "inconclusive"} -
- - Inconclusive -

- All enabled sources returned errors. Try again - later. -

-
- {:else} -
- - No source reports this domain -

- Clean across all {blacklistSummary.enabled} enabled - source{blacklistSummary.enabled > 1 ? "s" : ""}. -

-
- {/if} -
-
-
-
-
-
- {blacklistSummary.enabled} -
- Enabled -
-
-
- {blacklistSummary.listed} -
- Listed -
-
-
- {blacklistSummary.disabled} -
- Disabled -
-
- {#if blacklistSummary.errored > 0} - - {blacklistSummary.errored} source{blacklistSummary.errored > - 1 - ? "s" - : ""} errored - - {/if} - - Collected {formatCollectedAt( - blacklist.collected_at, - )} - -
-
-
-
-
- - - {/if} -
@@ -322,9 +152,9 @@ Want Complete Email Analysis?

- This domain test checks DNS configuration and domain reputation. For - comprehensive deliverability testing including DKIM verification, - content analysis, spam scoring, and sending-IP blacklist checks: + This domain-only test checks DNS configuration. For comprehensive + deliverability testing including DKIM verification, content + analysis, spam scoring, and blacklist checks: