blacklist: add domain reputation check via checker-blacklist
Some checks reported errors
continuous-integration/drone/push Build was killed

Integrates the checker-blacklist module behind a new POST /blacklist/domain
endpoint that aggregates reputation/blocklist sources for a given domain,
plus a SvelteKit UI under /blacklist/domain mirroring the existing IP
blacklist flow. Per-source credentials (VirusTotal, Safe Browsing) are
exposed as CLI flags; free sources run unconditionally.

Closes: #96
This commit is contained in:
nemunaire 2026-06-04 18:38:45 +09:00
commit f14209d4fa
13 changed files with 655 additions and 21 deletions

View file

@ -22,6 +22,7 @@
package api
import (
"context"
"fmt"
"net/http"
"time"
@ -30,8 +31,10 @@ import (
"github.com/google/uuid"
openapi_types "github.com/oapi-codegen/runtime/types"
sdk "git.happydns.org/checker-sdk-go/checker"
"git.happydns.org/happyDeliver/internal/config"
"git.happydns.org/happyDeliver/internal/model"
"git.happydns.org/happyDeliver/internal/reputation"
"git.happydns.org/happyDeliver/internal/storage"
"git.happydns.org/happyDeliver/internal/utils"
"git.happydns.org/happyDeliver/internal/version"
@ -47,19 +50,21 @@ type EmailAnalyzer interface {
// APIHandler implements the ServerInterface for handling API requests
type APIHandler struct {
storage storage.Storage
config *config.Config
analyzer EmailAnalyzer
startTime time.Time
storage storage.Storage
config *config.Config
analyzer EmailAnalyzer
blacklistProvider sdk.ObservationProvider
startTime time.Time
}
// NewAPIHandler creates a new API handler
func NewAPIHandler(store storage.Storage, cfg *config.Config, analyzer EmailAnalyzer) *APIHandler {
func NewAPIHandler(store storage.Storage, cfg *config.Config, analyzer EmailAnalyzer, blacklistProvider sdk.ObservationProvider) *APIHandler {
return &APIHandler{
storage: store,
config: cfg,
analyzer: analyzer,
startTime: time.Now(),
storage: store,
config: cfg,
analyzer: analyzer,
blacklistProvider: blacklistProvider,
startTime: time.Now(),
}
}
@ -339,6 +344,7 @@ func (h *APIHandler) TestDomain(c *gin.Context) {
Score: score,
Grade: responseGrade,
DnsResults: *dnsResults,
Blacklist: h.runDomainBlacklist(c.Request.Context(), request.Domain),
}
c.JSON(http.StatusOK, response)
@ -383,6 +389,32 @@ func (h *APIHandler) CheckBlacklist(c *gin.Context) {
c.JSON(http.StatusOK, response)
}
// runDomainBlacklist runs the checker-blacklist aggregation against a domain.
// It returns nil (and logs nothing fatal) when the check cannot be run, so the
// surrounding domain analysis still succeeds.
func (h *APIHandler) runDomainBlacklist(ctx context.Context, domain string) *model.DomainBlacklistResult {
opts := h.config.Analysis.Blacklist.AsCheckerOptions()
// "domain_name" is the option key the checker-blacklist provider reads
// (see checker/collect.go in the checker-blacklist module).
opts["domain_name"] = domain
// Cap the aggregation: sources run concurrently, each with its own
// timeouts; this is the host-side ceiling.
timeout := h.config.Analysis.HTTPTimeout
if timeout <= 0 {
timeout = 30 * time.Second
}
ctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
raw, err := h.blacklistProvider.Collect(ctx, opts)
if err != nil {
return nil
}
return reputation.FromObservation(raw)
}
// ListTests returns a paginated list of test summaries
// (GET /tests)
func (h *APIHandler) ListTests(c *gin.Context, params ListTestsParams) {

View file

@ -30,6 +30,7 @@ import (
ratelimit "github.com/JGLTechnologies/gin-rate-limit"
"github.com/gin-gonic/gin"
blacklist "git.happydns.org/checker-blacklist/checker"
"git.happydns.org/happyDeliver/internal/api"
"git.happydns.org/happyDeliver/internal/config"
"git.happydns.org/happyDeliver/internal/lmtp"
@ -70,7 +71,7 @@ func RunServer(cfg *config.Config) error {
analyzerAdapter := analyzer.NewAPIAdapter(cfg)
// Create API handler
handler := api.NewAPIHandler(store, cfg, analyzerAdapter)
handler := api.NewAPIHandler(store, cfg, analyzerAdapter, blacklist.Provider())
// Set up Gin router
if os.Getenv("GIN_MODE") == "" {

View file

@ -0,0 +1,40 @@
// This file is part of the happyDeliver (R) project.
// Copyright (c) 2025 happyDomain
// Authors: Pierre-Olivier Mercier, et al.
//
// This program is offered under a commercial and under the AGPL license.
// For commercial licensing, contact us at <contact@happydomain.org>.
//
// For AGPL licensing:
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package config
import (
sdk "git.happydns.org/checker-sdk-go/checker"
)
// AsCheckerOptions returns the map that checker-blacklist sources read
// via stringOpt(). Empty values are omitted so sources that require a
// credential stay disabled rather than failing with an empty key.
func (b BlacklistConfig) AsCheckerOptions() sdk.CheckerOptions {
opts := sdk.CheckerOptions{}
if b.VirusTotalAPIKey != "" {
opts["virustotal_api_key"] = b.VirusTotalAPIKey
}
if b.SafeBrowsingAPIKey != "" {
opts["safebrowsing_api_key"] = b.SafeBrowsingAPIKey
}
return opts
}

View file

@ -40,6 +40,8 @@ func declareFlags(o *Config) {
flag.Var(&StringArray{&o.Analysis.RBLs}, "rbl", "Append a RBL (use this option multiple time to append multiple RBLs)")
flag.BoolVar(&o.Analysis.CheckAllIPs, "check-all-ips", o.Analysis.CheckAllIPs, "Check all IPs found in email headers against RBLs (not just the first one)")
flag.StringVar(&o.Analysis.RspamdAPIURL, "rspamd-api-url", o.Analysis.RspamdAPIURL, "rspamd API URL for symbol descriptions (default: use embedded list)")
flag.StringVar(&o.Analysis.Blacklist.VirusTotalAPIKey, "blacklist-virustotal-api-key", o.Analysis.Blacklist.VirusTotalAPIKey, "VirusTotal v3 API key for the domain blacklist checker")
flag.StringVar(&o.Analysis.Blacklist.SafeBrowsingAPIKey, "blacklist-safebrowsing-api-key", o.Analysis.Blacklist.SafeBrowsingAPIKey, "Google Safe Browsing API key for the domain blacklist checker")
flag.DurationVar(&o.ReportRetention, "report-retention", o.ReportRetention, "How long to keep reports (e.g., 720h, 30d). 0 = keep forever")
flag.UintVar(&o.RateLimit, "rate-limit", o.RateLimit, "API rate limit (requests per second per IP)")
flag.Var(&URL{&o.SurveyURL}, "survey-url", "URL for user feedback survey")

View file

@ -75,6 +75,18 @@ type AnalysisConfig struct {
DNSWLs []string
CheckAllIPs bool // Check all IPs found in headers, not just the first one
RspamdAPIURL string // rspamd API URL for fetching symbol descriptions (empty = use embedded list)
Blacklist BlacklistConfig
}
// BlacklistConfig holds per-source credentials/options for the
// domain-oriented checker-blacklist provider. Keys must match the
// option IDs declared by each source in the checker-blacklist module
// (see checker/virustotal.go, checker/safebrowsing.go, …). Free sources
// (Quad9, OISD, URLhaus, OpenPhish, Disconnect, Botvrij, …) need no
// configuration.
type BlacklistConfig struct {
VirusTotalAPIKey string
SafeBrowsingAPIKey string
}
// DefaultConfig returns a configuration with sensible defaults