blacklist: add domain reputation check via checker-blacklist
Some checks reported errors
continuous-integration/drone/push Build was killed
Some checks reported errors
continuous-integration/drone/push Build was killed
Integrates the checker-blacklist module behind a new POST /blacklist/domain endpoint that aggregates reputation/blocklist sources for a given domain, plus a SvelteKit UI under /blacklist/domain mirroring the existing IP blacklist flow. Per-source credentials (VirusTotal, Safe Browsing) are exposed as CLI flags; free sources run unconditionally. Closes: #96
This commit is contained in:
parent
d53c1b1e00
commit
f14209d4fa
13 changed files with 655 additions and 21 deletions
|
|
@ -22,6 +22,7 @@
|
|||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
|
@ -30,8 +31,10 @@ import (
|
|||
"github.com/google/uuid"
|
||||
openapi_types "github.com/oapi-codegen/runtime/types"
|
||||
|
||||
sdk "git.happydns.org/checker-sdk-go/checker"
|
||||
"git.happydns.org/happyDeliver/internal/config"
|
||||
"git.happydns.org/happyDeliver/internal/model"
|
||||
"git.happydns.org/happyDeliver/internal/reputation"
|
||||
"git.happydns.org/happyDeliver/internal/storage"
|
||||
"git.happydns.org/happyDeliver/internal/utils"
|
||||
"git.happydns.org/happyDeliver/internal/version"
|
||||
|
|
@ -47,19 +50,21 @@ type EmailAnalyzer interface {
|
|||
|
||||
// APIHandler implements the ServerInterface for handling API requests
|
||||
type APIHandler struct {
|
||||
storage storage.Storage
|
||||
config *config.Config
|
||||
analyzer EmailAnalyzer
|
||||
startTime time.Time
|
||||
storage storage.Storage
|
||||
config *config.Config
|
||||
analyzer EmailAnalyzer
|
||||
blacklistProvider sdk.ObservationProvider
|
||||
startTime time.Time
|
||||
}
|
||||
|
||||
// NewAPIHandler creates a new API handler
|
||||
func NewAPIHandler(store storage.Storage, cfg *config.Config, analyzer EmailAnalyzer) *APIHandler {
|
||||
func NewAPIHandler(store storage.Storage, cfg *config.Config, analyzer EmailAnalyzer, blacklistProvider sdk.ObservationProvider) *APIHandler {
|
||||
return &APIHandler{
|
||||
storage: store,
|
||||
config: cfg,
|
||||
analyzer: analyzer,
|
||||
startTime: time.Now(),
|
||||
storage: store,
|
||||
config: cfg,
|
||||
analyzer: analyzer,
|
||||
blacklistProvider: blacklistProvider,
|
||||
startTime: time.Now(),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -339,6 +344,7 @@ func (h *APIHandler) TestDomain(c *gin.Context) {
|
|||
Score: score,
|
||||
Grade: responseGrade,
|
||||
DnsResults: *dnsResults,
|
||||
Blacklist: h.runDomainBlacklist(c.Request.Context(), request.Domain),
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, response)
|
||||
|
|
@ -383,6 +389,32 @@ func (h *APIHandler) CheckBlacklist(c *gin.Context) {
|
|||
c.JSON(http.StatusOK, response)
|
||||
}
|
||||
|
||||
// runDomainBlacklist runs the checker-blacklist aggregation against a domain.
|
||||
// It returns nil (and logs nothing fatal) when the check cannot be run, so the
|
||||
// surrounding domain analysis still succeeds.
|
||||
func (h *APIHandler) runDomainBlacklist(ctx context.Context, domain string) *model.DomainBlacklistResult {
|
||||
opts := h.config.Analysis.Blacklist.AsCheckerOptions()
|
||||
// "domain_name" is the option key the checker-blacklist provider reads
|
||||
// (see checker/collect.go in the checker-blacklist module).
|
||||
opts["domain_name"] = domain
|
||||
|
||||
// Cap the aggregation: sources run concurrently, each with its own
|
||||
// timeouts; this is the host-side ceiling.
|
||||
timeout := h.config.Analysis.HTTPTimeout
|
||||
if timeout <= 0 {
|
||||
timeout = 30 * time.Second
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
|
||||
raw, err := h.blacklistProvider.Collect(ctx, opts)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
return reputation.FromObservation(raw)
|
||||
}
|
||||
|
||||
// ListTests returns a paginated list of test summaries
|
||||
// (GET /tests)
|
||||
func (h *APIHandler) ListTests(c *gin.Context, params ListTestsParams) {
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ import (
|
|||
ratelimit "github.com/JGLTechnologies/gin-rate-limit"
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
blacklist "git.happydns.org/checker-blacklist/checker"
|
||||
"git.happydns.org/happyDeliver/internal/api"
|
||||
"git.happydns.org/happyDeliver/internal/config"
|
||||
"git.happydns.org/happyDeliver/internal/lmtp"
|
||||
|
|
@ -70,7 +71,7 @@ func RunServer(cfg *config.Config) error {
|
|||
analyzerAdapter := analyzer.NewAPIAdapter(cfg)
|
||||
|
||||
// Create API handler
|
||||
handler := api.NewAPIHandler(store, cfg, analyzerAdapter)
|
||||
handler := api.NewAPIHandler(store, cfg, analyzerAdapter, blacklist.Provider())
|
||||
|
||||
// Set up Gin router
|
||||
if os.Getenv("GIN_MODE") == "" {
|
||||
|
|
|
|||
40
internal/config/blacklist.go
Normal file
40
internal/config/blacklist.go
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
// This file is part of the happyDeliver (R) project.
|
||||
// Copyright (c) 2025 happyDomain
|
||||
// Authors: Pierre-Olivier Mercier, et al.
|
||||
//
|
||||
// This program is offered under a commercial and under the AGPL license.
|
||||
// For commercial licensing, contact us at <contact@happydomain.org>.
|
||||
//
|
||||
// For AGPL licensing:
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
package config
|
||||
|
||||
import (
|
||||
sdk "git.happydns.org/checker-sdk-go/checker"
|
||||
)
|
||||
|
||||
// AsCheckerOptions returns the map that checker-blacklist sources read
|
||||
// via stringOpt(). Empty values are omitted so sources that require a
|
||||
// credential stay disabled rather than failing with an empty key.
|
||||
func (b BlacklistConfig) AsCheckerOptions() sdk.CheckerOptions {
|
||||
opts := sdk.CheckerOptions{}
|
||||
if b.VirusTotalAPIKey != "" {
|
||||
opts["virustotal_api_key"] = b.VirusTotalAPIKey
|
||||
}
|
||||
if b.SafeBrowsingAPIKey != "" {
|
||||
opts["safebrowsing_api_key"] = b.SafeBrowsingAPIKey
|
||||
}
|
||||
return opts
|
||||
}
|
||||
|
|
@ -40,6 +40,8 @@ func declareFlags(o *Config) {
|
|||
flag.Var(&StringArray{&o.Analysis.RBLs}, "rbl", "Append a RBL (use this option multiple time to append multiple RBLs)")
|
||||
flag.BoolVar(&o.Analysis.CheckAllIPs, "check-all-ips", o.Analysis.CheckAllIPs, "Check all IPs found in email headers against RBLs (not just the first one)")
|
||||
flag.StringVar(&o.Analysis.RspamdAPIURL, "rspamd-api-url", o.Analysis.RspamdAPIURL, "rspamd API URL for symbol descriptions (default: use embedded list)")
|
||||
flag.StringVar(&o.Analysis.Blacklist.VirusTotalAPIKey, "blacklist-virustotal-api-key", o.Analysis.Blacklist.VirusTotalAPIKey, "VirusTotal v3 API key for the domain blacklist checker")
|
||||
flag.StringVar(&o.Analysis.Blacklist.SafeBrowsingAPIKey, "blacklist-safebrowsing-api-key", o.Analysis.Blacklist.SafeBrowsingAPIKey, "Google Safe Browsing API key for the domain blacklist checker")
|
||||
flag.DurationVar(&o.ReportRetention, "report-retention", o.ReportRetention, "How long to keep reports (e.g., 720h, 30d). 0 = keep forever")
|
||||
flag.UintVar(&o.RateLimit, "rate-limit", o.RateLimit, "API rate limit (requests per second per IP)")
|
||||
flag.Var(&URL{&o.SurveyURL}, "survey-url", "URL for user feedback survey")
|
||||
|
|
|
|||
|
|
@ -75,6 +75,18 @@ type AnalysisConfig struct {
|
|||
DNSWLs []string
|
||||
CheckAllIPs bool // Check all IPs found in headers, not just the first one
|
||||
RspamdAPIURL string // rspamd API URL for fetching symbol descriptions (empty = use embedded list)
|
||||
Blacklist BlacklistConfig
|
||||
}
|
||||
|
||||
// BlacklistConfig holds per-source credentials/options for the
|
||||
// domain-oriented checker-blacklist provider. Keys must match the
|
||||
// option IDs declared by each source in the checker-blacklist module
|
||||
// (see checker/virustotal.go, checker/safebrowsing.go, …). Free sources
|
||||
// (Quad9, OISD, URLhaus, OpenPhish, Disconnect, Botvrij, …) need no
|
||||
// configuration.
|
||||
type BlacklistConfig struct {
|
||||
VirusTotalAPIKey string
|
||||
SafeBrowsingAPIKey string
|
||||
}
|
||||
|
||||
// DefaultConfig returns a configuration with sensible defaults
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue