diff --git a/Dockerfile b/Dockerfile index f6dc16a..9626813 100644 --- a/Dockerfile +++ b/Dockerfile @@ -170,7 +170,12 @@ RUN chmod +x /entrypoint.sh EXPOSE 25 8080 # Default configuration -ENV HAPPYDELIVER_DATABASE_TYPE=sqlite HAPPYDELIVER_DATABASE_DSN=/var/lib/happydeliver/happydeliver.db HAPPYDELIVER_DOMAIN=happydeliver.local HAPPYDELIVER_ADDRESS_PREFIX=test- HAPPYDELIVER_DNS_TIMEOUT=5s HAPPYDELIVER_HTTP_TIMEOUT=10s HAPPYDELIVER_RBL=zen.spamhaus.org,bl.spamcop.net,b.barracudacentral.org,dnsbl.sorbs.net,dnsbl-1.uceprotect.net,bl.mailspike.net +ENV HAPPYDELIVER_DATABASE_TYPE=sqlite \ + HAPPYDELIVER_DATABASE_DSN=/var/lib/happydeliver/happydeliver.db \ + HAPPYDELIVER_DOMAIN=happydeliver.local \ + HAPPYDELIVER_ADDRESS_PREFIX=test- \ + HAPPYDELIVER_DNS_TIMEOUT=5s \ + HAPPYDELIVER_HTTP_TIMEOUT=10s # Volume for persistent data VOLUME ["/var/lib/happydeliver", "/var/log/happydeliver"] diff --git a/api/openapi.yaml b/api/openapi.yaml index 5c628fd..e989261 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -350,6 +350,19 @@ components: listed: false - rbl: "bl.spamcop.net" listed: false + whitelists: + type: object + additionalProperties: + type: array + items: + $ref: '#/components/schemas/BlacklistCheck' + description: Map of IP addresses to their DNS whitelist check results (informational only) + example: + "192.0.2.1": + - rbl: "list.dnswl.org" + listed: false + - rbl: "swl.spamhaus.org" + listed: false content_analysis: $ref: '#/components/schemas/ContentAnalysis' header_analysis: @@ -776,7 +789,7 @@ components: properties: result: type: string - enum: [pass, fail, invalid, missing, none, neutral, softfail, temperror, permerror, declined, domain_pass, orgdomain_pass] + enum: [pass, fail, invalid, missing, none, neutral, softfail, temperror, permerror, declined, domain_pass, orgdomain_pass, skipped] description: Authentication result example: "pass" domain: @@ -969,6 +982,9 @@ components: name: "BAYES_HAM" score: -1.9 params: "0.02" + report: + type: string + description: Full rspamd report (raw X-Spamd-Result header) RspamdSymbol: type: object @@ -1330,7 +1346,7 @@ components: type: object required: - ip - - checks + - blacklists - listed_count - score - grade @@ -1339,7 +1355,7 @@ components: type: string description: The IP address that was checked example: "192.0.2.1" - checks: + blacklists: type: array items: $ref: '#/components/schemas/BlacklistCheck' @@ -1359,3 +1375,8 @@ components: enum: [A+, A, B, C, D, E, F] description: Letter grade representation of the score example: "A+" + whitelists: + type: array + items: + $ref: '#/components/schemas/BlacklistCheck' + description: List of DNS whitelist check results (informational only) diff --git a/internal/api/handlers.go b/internal/api/handlers.go index 80c8f9a..470136e 100644 --- a/internal/api/handlers.go +++ b/internal/api/handlers.go @@ -41,7 +41,7 @@ import ( type EmailAnalyzer interface { AnalyzeEmailBytes(rawEmail []byte, testID uuid.UUID) (reportJSON []byte, err error) AnalyzeDomain(domain string) (dnsResults *DNSResults, score int, grade string) - CheckBlacklistIP(ip string) (checks []BlacklistCheck, listedCount int, score int, grade string, err error) + CheckBlacklistIP(ip string) (checks []BlacklistCheck, whitelists []BlacklistCheck, listedCount int, score int, grade string, err error) } // APIHandler implements the ServerInterface for handling API requests @@ -359,7 +359,7 @@ func (h *APIHandler) CheckBlacklist(c *gin.Context) { } // Perform blacklist check using analyzer - checks, listedCount, score, grade, err := h.analyzer.CheckBlacklistIP(request.Ip) + checks, whitelists, listedCount, score, grade, err := h.analyzer.CheckBlacklistIP(request.Ip) if err != nil { c.JSON(http.StatusBadRequest, Error{ Error: "invalid_ip", @@ -372,7 +372,8 @@ func (h *APIHandler) CheckBlacklist(c *gin.Context) { // Build response response := BlacklistCheckResponse{ Ip: request.Ip, - Checks: checks, + Blacklists: checks, + Whitelists: &whitelists, ListedCount: listedCount, Score: score, Grade: BlacklistCheckResponseGrade(grade), diff --git a/internal/config/config.go b/internal/config/config.go index 4a335c9..468a2aa 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -65,6 +65,7 @@ type AnalysisConfig struct { DNSTimeout time.Duration HTTPTimeout time.Duration RBLs []string + DNSWLs []string CheckAllIPs bool // Check all IPs found in headers, not just the first one } @@ -88,6 +89,7 @@ func DefaultConfig() *Config { DNSTimeout: 5 * time.Second, HTTPTimeout: 10 * time.Second, RBLs: []string{}, + DNSWLs: []string{}, CheckAllIPs: false, // By default, only check the first IP }, } diff --git a/pkg/analyzer/analyzer.go b/pkg/analyzer/analyzer.go index e7ae561..a16829b 100644 --- a/pkg/analyzer/analyzer.go +++ b/pkg/analyzer/analyzer.go @@ -44,6 +44,7 @@ func NewEmailAnalyzer(cfg *config.Config) *EmailAnalyzer { cfg.Analysis.DNSTimeout, cfg.Analysis.HTTPTimeout, cfg.Analysis.RBLs, + cfg.Analysis.DNSWLs, cfg.Analysis.CheckAllIPs, ) @@ -120,22 +121,28 @@ func (a *APIAdapter) AnalyzeDomain(domain string) (*api.DNSResults, int, string) return dnsResults, score, grade } -// CheckBlacklistIP checks a single IP address against DNS blacklists -func (a *APIAdapter) CheckBlacklistIP(ip string) ([]api.BlacklistCheck, int, int, string, error) { +// CheckBlacklistIP checks a single IP address against DNS blacklists and whitelists +func (a *APIAdapter) CheckBlacklistIP(ip string) ([]api.BlacklistCheck, []api.BlacklistCheck, int, int, string, error) { // Check the IP against all configured RBLs checks, listedCount, err := a.analyzer.generator.rblChecker.CheckIP(ip) if err != nil { - return nil, 0, 0, "", err + return nil, nil, 0, 0, "", err } // Calculate score using the existing function // Create a minimal RBLResults structure for scoring - results := &RBLResults{ + results := &DNSListResults{ Checks: map[string][]api.BlacklistCheck{ip: checks}, IPsChecked: []string{ip}, ListedCount: listedCount, } - score, grade := a.analyzer.generator.rblChecker.CalculateRBLScore(results) + score, grade := a.analyzer.generator.rblChecker.CalculateScore(results) - return checks, listedCount, score, grade, nil + // Check the IP against all configured DNSWLs (informational only) + whitelists, _, err := a.analyzer.generator.dnswlChecker.CheckIP(ip) + if err != nil { + whitelists = nil + } + + return checks, whitelists, listedCount, score, grade, nil } diff --git a/pkg/analyzer/rbl.go b/pkg/analyzer/rbl.go index ff0e813..08d3b8f 100644 --- a/pkg/analyzer/rbl.go +++ b/pkg/analyzer/rbl.go @@ -27,18 +27,21 @@ import ( "net" "regexp" "strings" + "sync" "time" "git.happydns.org/happyDeliver/internal/api" ) -// RBLChecker checks IP addresses against DNS-based blacklists -type RBLChecker struct { +// DNSListChecker checks IP addresses against DNS-based block/allow lists. +// It handles both RBL (blacklist) and DNSWL (whitelist) semantics via flags. +type DNSListChecker struct { Timeout time.Duration - RBLs []string + Lists []string CheckAllIPs bool // Check all IPs found in headers, not just the first one + filterErrorCodes bool // When true (RBL mode), treat 127.255.255.253/254/255 as operational errors resolver *net.Resolver - informationalSet map[string]bool + informationalSet map[string]bool // Lists whose hits don't count toward the score } // DefaultRBLs is a list of commonly used RBL providers @@ -51,8 +54,6 @@ var DefaultRBLs = []string{ "dnsbl-1.uceprotect.net", // UCEPROTECT Level 1 "dnsbl-2.uceprotect.net", // UCEPROTECT Level 2 (informational) "dnsbl-3.uceprotect.net", // UCEPROTECT Level 3 (informational) - "spam.spamrats.com", // SpamRats SPAM - "dyna.spamrats.com", // SpamRats dynamic IPs "psbl.surriel.com", // PSBL "dnsbl.dronebl.org", // DroneBL "bl.mailspike.net", // Mailspike BL @@ -68,10 +69,16 @@ var DefaultInformationalRBLs = []string{ "dnsbl-3.uceprotect.net", // UCEPROTECT Level 3: entire ASes, too broad for scoring } +// DefaultDNSWLs is a list of commonly used DNSWL providers +var DefaultDNSWLs = []string{ + "list.dnswl.org", // DNSWL.org — the main DNS whitelist + "swl.spamhaus.org", // Spamhaus Safe Whitelist +} + // NewRBLChecker creates a new RBL checker with configurable timeout and RBL list -func NewRBLChecker(timeout time.Duration, rbls []string, checkAllIPs bool) *RBLChecker { +func NewRBLChecker(timeout time.Duration, rbls []string, checkAllIPs bool) *DNSListChecker { if timeout == 0 { - timeout = 5 * time.Second // Default timeout + timeout = 5 * time.Second } if len(rbls) == 0 { rbls = DefaultRBLs @@ -80,30 +87,48 @@ func NewRBLChecker(timeout time.Duration, rbls []string, checkAllIPs bool) *RBLC for _, rbl := range DefaultInformationalRBLs { informationalSet[rbl] = true } - return &RBLChecker{ + return &DNSListChecker{ Timeout: timeout, - RBLs: rbls, + Lists: rbls, CheckAllIPs: checkAllIPs, + filterErrorCodes: true, resolver: &net.Resolver{PreferGo: true}, informationalSet: informationalSet, } } -// RBLResults represents the results of RBL checks -type RBLResults struct { - Checks map[string][]api.BlacklistCheck // Map of IP -> list of RBL checks for that IP - IPsChecked []string - ListedCount int // Total listings including informational RBLs - RelevantListedCount int // Listings on scoring (non-informational) RBLs only +// NewDNSWLChecker creates a new DNSWL checker with configurable timeout and DNSWL list +func NewDNSWLChecker(timeout time.Duration, dnswls []string, checkAllIPs bool) *DNSListChecker { + if timeout == 0 { + timeout = 5 * time.Second + } + if len(dnswls) == 0 { + dnswls = DefaultDNSWLs + } + return &DNSListChecker{ + Timeout: timeout, + Lists: dnswls, + CheckAllIPs: checkAllIPs, + filterErrorCodes: false, + resolver: &net.Resolver{PreferGo: true}, + informationalSet: make(map[string]bool), + } } -// CheckEmail checks all IPs found in the email headers against RBLs -func (r *RBLChecker) CheckEmail(email *EmailMessage) *RBLResults { - results := &RBLResults{ +// DNSListResults represents the results of DNS list checks +type DNSListResults struct { + Checks map[string][]api.BlacklistCheck // Map of IP -> list of checks for that IP + IPsChecked []string + ListedCount int // Total listings including informational entries + RelevantListedCount int // Listings on scoring (non-informational) lists only +} + +// CheckEmail checks all IPs found in the email headers against the configured lists +func (r *DNSListChecker) CheckEmail(email *EmailMessage) *DNSListResults { + results := &DNSListResults{ Checks: make(map[string][]api.BlacklistCheck), } - // Extract IPs from Received headers ips := r.extractIPs(email) if len(ips) == 0 { return results @@ -111,20 +136,18 @@ func (r *RBLChecker) CheckEmail(email *EmailMessage) *RBLResults { results.IPsChecked = ips - // Check each IP against all RBLs for _, ip := range ips { - for _, rbl := range r.RBLs { - check := r.checkIP(ip, rbl) + for _, list := range r.Lists { + check := r.checkIP(ip, list) results.Checks[ip] = append(results.Checks[ip], check) if check.Listed { results.ListedCount++ - if !r.informationalSet[rbl] { + if !r.informationalSet[list] { results.RelevantListedCount++ } } } - // Only check the first IP unless CheckAllIPs is enabled if !r.CheckAllIPs { break } @@ -133,20 +156,26 @@ func (r *RBLChecker) CheckEmail(email *EmailMessage) *RBLResults { return results } -// CheckIP checks a single IP address against all configured RBLs -func (r *RBLChecker) CheckIP(ip string) ([]api.BlacklistCheck, int, error) { - // Validate that it's a valid IP address +// CheckIP checks a single IP address against all configured lists in parallel +func (r *DNSListChecker) CheckIP(ip string) ([]api.BlacklistCheck, int, error) { if !r.isPublicIP(ip) { return nil, 0, fmt.Errorf("invalid or non-public IP address: %s", ip) } - var checks []api.BlacklistCheck - listedCount := 0 + checks := make([]api.BlacklistCheck, len(r.Lists)) + var wg sync.WaitGroup - // Check the IP against all RBLs - for _, rbl := range r.RBLs { - check := r.checkIP(ip, rbl) - checks = append(checks, check) + for i, list := range r.Lists { + wg.Add(1) + go func(i int, list string) { + defer wg.Done() + checks[i] = r.checkIP(ip, list) + }(i, list) + } + wg.Wait() + + listedCount := 0 + for _, check := range checks { if check.Listed { listedCount++ } @@ -156,27 +185,19 @@ func (r *RBLChecker) CheckIP(ip string) ([]api.BlacklistCheck, int, error) { } // extractIPs extracts IP addresses from Received headers -func (r *RBLChecker) extractIPs(email *EmailMessage) []string { +func (r *DNSListChecker) extractIPs(email *EmailMessage) []string { var ips []string seenIPs := make(map[string]bool) - // Get all Received headers receivedHeaders := email.Header["Received"] - - // Regex patterns for IP addresses - // Match IPv4: xxx.xxx.xxx.xxx ipv4Pattern := regexp.MustCompile(`\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b`) - // Look for IPs in Received headers for _, received := range receivedHeaders { - // Find all IPv4 addresses matches := ipv4Pattern.FindAllString(received, -1) for _, match := range matches { - // Skip private/reserved IPs if !r.isPublicIP(match) { continue } - // Avoid duplicates if !seenIPs[match] { ips = append(ips, match) seenIPs[match] = true @@ -184,13 +205,10 @@ func (r *RBLChecker) extractIPs(email *EmailMessage) []string { } } - // If no IPs found in Received headers, try X-Originating-IP if len(ips) == 0 { originatingIP := email.Header.Get("X-Originating-IP") if originatingIP != "" { - // Extract IP from formats like "[192.0.2.1]" or "192.0.2.1" cleanIP := strings.TrimSuffix(strings.TrimPrefix(originatingIP, "["), "]") - // Remove any whitespace cleanIP = strings.TrimSpace(cleanIP) matches := ipv4Pattern.FindString(cleanIP) if matches != "" && r.isPublicIP(matches) { @@ -203,19 +221,16 @@ func (r *RBLChecker) extractIPs(email *EmailMessage) []string { } // isPublicIP checks if an IP address is public (not private, loopback, or reserved) -func (r *RBLChecker) isPublicIP(ipStr string) bool { +func (r *DNSListChecker) isPublicIP(ipStr string) bool { ip := net.ParseIP(ipStr) if ip == nil { return false } - // Check if it's a private network if ip.IsPrivate() || ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() { return false } - // Additional checks for reserved ranges - // 0.0.0.0/8, 192.0.0.0/24, 192.0.2.0/24 (TEST-NET-1), 198.51.100.0/24 (TEST-NET-2), 203.0.113.0/24 (TEST-NET-3) if ip.IsUnspecified() { return false } @@ -223,51 +238,43 @@ func (r *RBLChecker) isPublicIP(ipStr string) bool { return true } -// checkIP checks a single IP against a single RBL -func (r *RBLChecker) checkIP(ip, rbl string) api.BlacklistCheck { +// checkIP checks a single IP against a single DNS list +func (r *DNSListChecker) checkIP(ip, list string) api.BlacklistCheck { check := api.BlacklistCheck{ - Rbl: rbl, + Rbl: list, } - // Reverse the IP for DNSBL query reversedIP := r.reverseIP(ip) if reversedIP == "" { check.Error = api.PtrTo("Failed to reverse IP address") return check } - // Construct DNSBL query: reversed-ip.rbl-domain - query := fmt.Sprintf("%s.%s", reversedIP, rbl) + query := fmt.Sprintf("%s.%s", reversedIP, list) - // Perform DNS lookup with timeout ctx, cancel := context.WithTimeout(context.Background(), r.Timeout) defer cancel() addrs, err := r.resolver.LookupHost(ctx, query) if err != nil { - // Most likely not listed (NXDOMAIN) if dnsErr, ok := err.(*net.DNSError); ok { if dnsErr.IsNotFound { check.Listed = false return check } } - // Other DNS errors check.Error = api.PtrTo(fmt.Sprintf("DNS lookup failed: %v", err)) return check } - // If we got a response, check the return code if len(addrs) > 0 { - check.Response = api.PtrTo(addrs[0]) // Return code (e.g., 127.0.0.2) + check.Response = api.PtrTo(addrs[0]) - // Check for RBL error codes: 127.255.255.253, 127.255.255.254, 127.255.255.255 - // These indicate RBL operational issues, not actual listings - if addrs[0] == "127.255.255.253" || addrs[0] == "127.255.255.254" || addrs[0] == "127.255.255.255" { + // In RBL mode, 127.255.255.253/254/255 indicate operational errors, not real listings. + if r.filterErrorCodes && (addrs[0] == "127.255.255.253" || addrs[0] == "127.255.255.254" || addrs[0] == "127.255.255.255") { check.Listed = false - check.Error = api.PtrTo(fmt.Sprintf("RBL %s returned error code %s (RBL operational issue)", rbl, addrs[0])) + check.Error = api.PtrTo(fmt.Sprintf("RBL %s returned error code %s (RBL operational issue)", list, addrs[0])) } else { - // Normal listing response check.Listed = true } } @@ -275,50 +282,47 @@ func (r *RBLChecker) checkIP(ip, rbl string) api.BlacklistCheck { return check } -// reverseIP reverses an IPv4 address for DNSBL queries +// reverseIP reverses an IPv4 address for DNSBL/DNSWL queries // Example: 192.0.2.1 -> 1.2.0.192 -func (r *RBLChecker) reverseIP(ipStr string) string { +func (r *DNSListChecker) reverseIP(ipStr string) string { ip := net.ParseIP(ipStr) if ip == nil { return "" } - // Convert to IPv4 ipv4 := ip.To4() if ipv4 == nil { return "" // IPv6 not supported yet } - // Reverse the octets return fmt.Sprintf("%d.%d.%d.%d", ipv4[3], ipv4[2], ipv4[1], ipv4[0]) } -// CalculateRBLScore calculates the blacklist contribution to deliverability. -// Informational RBLs are not counted in the score. -func (r *RBLChecker) CalculateRBLScore(results *RBLResults) (int, string) { +// CalculateScore calculates the list contribution to deliverability. +// Informational lists are not counted in the score. +func (r *DNSListChecker) CalculateScore(results *DNSListResults) (int, string) { if results == nil || len(results.IPsChecked) == 0 { - // No IPs to check, give benefit of doubt return 100, "" } - scoringRBLCount := len(r.RBLs) - len(r.informationalSet) - if scoringRBLCount <= 0 { + scoringListCount := len(r.Lists) - len(r.informationalSet) + if scoringListCount <= 0 { return 100, "A+" } - percentage := 100 - results.RelevantListedCount*100/scoringRBLCount + percentage := 100 - results.RelevantListedCount*100/scoringListCount return percentage, ScoreToGrade(percentage) } -// GetUniqueListedIPs returns a list of unique IPs that are listed on at least one RBL -func (r *RBLChecker) GetUniqueListedIPs(results *RBLResults) []string { +// GetUniqueListedIPs returns a list of unique IPs that are listed on at least one entry +func (r *DNSListChecker) GetUniqueListedIPs(results *DNSListResults) []string { var listedIPs []string - for ip, rblChecks := range results.Checks { - for _, check := range rblChecks { + for ip, checks := range results.Checks { + for _, check := range checks { if check.Listed { listedIPs = append(listedIPs, ip) - break // Only add the IP once + break } } } @@ -326,17 +330,17 @@ func (r *RBLChecker) GetUniqueListedIPs(results *RBLResults) []string { return listedIPs } -// GetRBLsForIP returns all RBLs that list a specific IP -func (r *RBLChecker) GetRBLsForIP(results *RBLResults, ip string) []string { - var rbls []string +// GetListsForIP returns all lists that match a specific IP +func (r *DNSListChecker) GetListsForIP(results *DNSListResults, ip string) []string { + var lists []string - if rblChecks, exists := results.Checks[ip]; exists { - for _, check := range rblChecks { + if checks, exists := results.Checks[ip]; exists { + for _, check := range checks { if check.Listed { - rbls = append(rbls, check.Rbl) + lists = append(lists, check.Rbl) } } } - return rbls + return lists } diff --git a/pkg/analyzer/rbl_test.go b/pkg/analyzer/rbl_test.go index a1de270..1dd1262 100644 --- a/pkg/analyzer/rbl_test.go +++ b/pkg/analyzer/rbl_test.go @@ -59,8 +59,8 @@ func TestNewRBLChecker(t *testing.T) { if checker.Timeout != tt.expectedTimeout { t.Errorf("Timeout = %v, want %v", checker.Timeout, tt.expectedTimeout) } - if len(checker.RBLs) != tt.expectedRBLs { - t.Errorf("RBLs count = %d, want %d", len(checker.RBLs), tt.expectedRBLs) + if len(checker.Lists) != tt.expectedRBLs { + t.Errorf("RBLs count = %d, want %d", len(checker.Lists), tt.expectedRBLs) } if checker.resolver == nil { t.Error("Resolver should not be nil") @@ -265,7 +265,7 @@ func TestExtractIPs(t *testing.T) { func TestGetBlacklistScore(t *testing.T) { tests := []struct { name string - results *RBLResults + results *DNSListResults expectedScore int }{ { @@ -275,14 +275,14 @@ func TestGetBlacklistScore(t *testing.T) { }, { name: "No IPs checked", - results: &RBLResults{ + results: &DNSListResults{ IPsChecked: []string{}, }, expectedScore: 100, }, { name: "Not listed on any RBL", - results: &RBLResults{ + results: &DNSListResults{ IPsChecked: []string{"198.51.100.1"}, ListedCount: 0, }, @@ -290,7 +290,7 @@ func TestGetBlacklistScore(t *testing.T) { }, { name: "Listed on 1 RBL", - results: &RBLResults{ + results: &DNSListResults{ IPsChecked: []string{"198.51.100.1"}, ListedCount: 1, }, @@ -298,7 +298,7 @@ func TestGetBlacklistScore(t *testing.T) { }, { name: "Listed on 2 RBLs", - results: &RBLResults{ + results: &DNSListResults{ IPsChecked: []string{"198.51.100.1"}, ListedCount: 2, }, @@ -306,7 +306,7 @@ func TestGetBlacklistScore(t *testing.T) { }, { name: "Listed on 3 RBLs", - results: &RBLResults{ + results: &DNSListResults{ IPsChecked: []string{"198.51.100.1"}, ListedCount: 3, }, @@ -314,7 +314,7 @@ func TestGetBlacklistScore(t *testing.T) { }, { name: "Listed on 4+ RBLs", - results: &RBLResults{ + results: &DNSListResults{ IPsChecked: []string{"198.51.100.1"}, ListedCount: 4, }, @@ -326,7 +326,7 @@ func TestGetBlacklistScore(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - score, _ := checker.CalculateRBLScore(tt.results) + score, _ := checker.CalculateScore(tt.results) if score != tt.expectedScore { t.Errorf("GetBlacklistScore() = %v, want %v", score, tt.expectedScore) } @@ -335,7 +335,7 @@ func TestGetBlacklistScore(t *testing.T) { } func TestGetUniqueListedIPs(t *testing.T) { - results := &RBLResults{ + results := &DNSListResults{ Checks: map[string][]api.BlacklistCheck{ "198.51.100.1": { {Rbl: "zen.spamhaus.org", Listed: true}, @@ -363,7 +363,7 @@ func TestGetUniqueListedIPs(t *testing.T) { } func TestGetRBLsForIP(t *testing.T) { - results := &RBLResults{ + results := &DNSListResults{ Checks: map[string][]api.BlacklistCheck{ "198.51.100.1": { {Rbl: "zen.spamhaus.org", Listed: true}, @@ -402,7 +402,7 @@ func TestGetRBLsForIP(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - rbls := checker.GetRBLsForIP(results, tt.ip) + rbls := checker.GetListsForIP(results, tt.ip) if len(rbls) != len(tt.expectedRBLs) { t.Errorf("Got %d RBLs, want %d", len(rbls), len(tt.expectedRBLs)) diff --git a/pkg/analyzer/report.go b/pkg/analyzer/report.go index dc420fb..bd12960 100644 --- a/pkg/analyzer/report.go +++ b/pkg/analyzer/report.go @@ -35,7 +35,8 @@ type ReportGenerator struct { spamAnalyzer *SpamAssassinAnalyzer rspamdAnalyzer *RspamdAnalyzer dnsAnalyzer *DNSAnalyzer - rblChecker *RBLChecker + rblChecker *DNSListChecker + dnswlChecker *DNSListChecker contentAnalyzer *ContentAnalyzer headerAnalyzer *HeaderAnalyzer } @@ -45,6 +46,7 @@ func NewReportGenerator( dnsTimeout time.Duration, httpTimeout time.Duration, rbls []string, + dnswls []string, checkAllIPs bool, ) *ReportGenerator { return &ReportGenerator{ @@ -53,6 +55,7 @@ func NewReportGenerator( rspamdAnalyzer: NewRspamdAnalyzer(), dnsAnalyzer: NewDNSAnalyzer(dnsTimeout), rblChecker: NewRBLChecker(dnsTimeout, rbls, checkAllIPs), + dnswlChecker: NewDNSWLChecker(dnsTimeout, dnswls, checkAllIPs), contentAnalyzer: NewContentAnalyzer(httpTimeout), headerAnalyzer: NewHeaderAnalyzer(), } @@ -65,7 +68,8 @@ type AnalysisResults struct { Content *ContentResults DNS *api.DNSResults Headers *api.HeaderAnalysis - RBL *RBLResults + RBL *DNSListResults + DNSWL *DNSListResults SpamAssassin *api.SpamAssassinResult Rspamd *api.RspamdResult } @@ -81,6 +85,7 @@ func (r *ReportGenerator) AnalyzeEmail(email *EmailMessage) *AnalysisResults { results.Headers = r.headerAnalyzer.GenerateHeaderAnalysis(email, results.Authentication) results.DNS = r.dnsAnalyzer.AnalyzeDNS(email, results.Authentication, results.Headers) results.RBL = r.rblChecker.CheckEmail(email) + results.DNSWL = r.dnswlChecker.CheckEmail(email) results.SpamAssassin = r.spamAnalyzer.AnalyzeSpamAssassin(email) results.Rspamd = r.rspamdAnalyzer.AnalyzeRspamd(email) results.Content = r.contentAnalyzer.AnalyzeContent(email) @@ -135,7 +140,7 @@ func (r *ReportGenerator) GenerateReport(testID uuid.UUID, results *AnalysisResu blacklistScore := 0 var blacklistGrade string if results.RBL != nil { - blacklistScore, blacklistGrade = r.rblChecker.CalculateRBLScore(results.RBL) + blacklistScore, blacklistGrade = r.rblChecker.CalculateScore(results.RBL) } saScore, saGrade := r.spamAnalyzer.CalculateSpamAssassinScore(results.SpamAssassin) @@ -197,6 +202,11 @@ func (r *ReportGenerator) GenerateReport(testID uuid.UUID, results *AnalysisResu report.Blacklists = &results.RBL.Checks } + // Add whitelist checks as a map of IP -> array of BlacklistCheck (informational only) + if results.DNSWL != nil && len(results.DNSWL.Checks) > 0 { + report.Whitelists = &results.DNSWL.Checks + } + // Add SpamAssassin result with individual deliverability score if results.SpamAssassin != nil { saGradeTyped := api.SpamAssassinResultDeliverabilityGrade(saGrade) diff --git a/pkg/analyzer/report_test.go b/pkg/analyzer/report_test.go index 5a325b1..82e923e 100644 --- a/pkg/analyzer/report_test.go +++ b/pkg/analyzer/report_test.go @@ -32,7 +32,7 @@ import ( ) func TestNewReportGenerator(t *testing.T) { - gen := NewReportGenerator(10*time.Second, 10*time.Second, DefaultRBLs, false) + gen := NewReportGenerator(10*time.Second, 10*time.Second, DefaultRBLs, DefaultDNSWLs, false) if gen == nil { t.Fatal("Expected report generator, got nil") } @@ -55,7 +55,7 @@ func TestNewReportGenerator(t *testing.T) { } func TestAnalyzeEmail(t *testing.T) { - gen := NewReportGenerator(10*time.Second, 10*time.Second, DefaultRBLs, false) + gen := NewReportGenerator(10*time.Second, 10*time.Second, DefaultRBLs, DefaultDNSWLs, false) email := createTestEmail() @@ -75,7 +75,7 @@ func TestAnalyzeEmail(t *testing.T) { } func TestGenerateReport(t *testing.T) { - gen := NewReportGenerator(10*time.Second, 10*time.Second, DefaultRBLs, false) + gen := NewReportGenerator(10*time.Second, 10*time.Second, DefaultRBLs, DefaultDNSWLs, false) testID := uuid.New() email := createTestEmail() @@ -130,7 +130,7 @@ func TestGenerateReport(t *testing.T) { } func TestGenerateReportWithSpamAssassin(t *testing.T) { - gen := NewReportGenerator(10*time.Second, 10*time.Second, DefaultRBLs, false) + gen := NewReportGenerator(10*time.Second, 10*time.Second, DefaultRBLs, DefaultDNSWLs, false) testID := uuid.New() email := createTestEmailWithSpamAssassin() @@ -150,7 +150,7 @@ func TestGenerateReportWithSpamAssassin(t *testing.T) { } func TestGenerateRawEmail(t *testing.T) { - gen := NewReportGenerator(10*time.Second, 10*time.Second, DefaultRBLs, false) + gen := NewReportGenerator(10*time.Second, 10*time.Second, DefaultRBLs, DefaultDNSWLs, false) tests := []struct { name string diff --git a/pkg/analyzer/rspamd.go b/pkg/analyzer/rspamd.go index d394c62..f3f548b 100644 --- a/pkg/analyzer/rspamd.go +++ b/pkg/analyzer/rspamd.go @@ -58,6 +58,8 @@ func (a *RspamdAnalyzer) AnalyzeRspamd(email *EmailMessage) *api.RspamdResult { // Parse X-Spamd-Result header (primary source for score, threshold, and symbols) // Format: "default: False [-3.91 / 15.00];\n\tSYMBOL(score)[params]; ..." if spamdResult, ok := headers["X-Spamd-Result"]; ok { + report := strings.ReplaceAll(spamdResult, "; ", ";\n") + result.Report = &report a.parseSpamdResult(spamdResult, result) } @@ -111,8 +113,9 @@ func (a *RspamdAnalyzer) parseSpamdResult(header string, result *api.RspamdResul } // Parse symbols: SYMBOL(score)[params] - // Each symbol entry is separated by ";" - symbolRe := regexp.MustCompile(`(\w+)\((-?\d+\.?\d*)\)(?:\[([^\]]*)\])?`) + // Each symbol entry is separated by ";", so within each part we use a + // greedy match to capture params that may contain nested brackets. + symbolRe := regexp.MustCompile(`(\w+)\((-?\d+\.?\d*)\)(?:\[(.*)\])?`) for _, part := range strings.Split(header, ";") { part = strings.TrimSpace(part) matches := symbolRe.FindStringSubmatch(part) diff --git a/pkg/analyzer/rspamd_test.go b/pkg/analyzer/rspamd_test.go new file mode 100644 index 0000000..de98fe8 --- /dev/null +++ b/pkg/analyzer/rspamd_test.go @@ -0,0 +1,414 @@ +// This file is part of the happyDeliver (R) project. +// Copyright (c) 2026 happyDomain +// Authors: Pierre-Olivier Mercier, et al. +// +// This program is offered under a commercial and under the AGPL license. +// For commercial licensing, contact us at . +// +// For AGPL licensing: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package analyzer + +import ( + "bytes" + "net/mail" + "testing" + + "git.happydns.org/happyDeliver/internal/api" +) + +func TestAnalyzeRspamdNoHeaders(t *testing.T) { + analyzer := NewRspamdAnalyzer() + email := &EmailMessage{Header: make(mail.Header)} + + result := analyzer.AnalyzeRspamd(email) + + if result != nil { + t.Errorf("Expected nil for email without rspamd headers, got %+v", result) + } +} + +func TestParseSpamdResult(t *testing.T) { + tests := []struct { + name string + header string + expectedScore float32 + expectedThreshold float32 + expectedIsSpam bool + expectedSymbols map[string]float32 + expectedSymParams map[string]string + }{ + { + name: "Clean email negative score", + header: "default: False [-3.91 / 15.00];\n\tDATE_IN_PAST(0.10); ALL_TRUSTED(-1.00)[trusted]", + expectedScore: -3.91, + expectedThreshold: 15.00, + expectedIsSpam: false, + expectedSymbols: map[string]float32{ + "DATE_IN_PAST": 0.10, + "ALL_TRUSTED": -1.00, + }, + expectedSymParams: map[string]string{ + "ALL_TRUSTED": "trusted", + }, + }, + { + name: "Spam email True flag", + header: "default: True [16.50 / 15.00];\n\tBAYES_99(5.00)[1.00]; SPOOFED_SENDER(3.50)", + expectedScore: 16.50, + expectedThreshold: 15.00, + expectedIsSpam: true, + expectedSymbols: map[string]float32{ + "BAYES_99": 5.00, + "SPOOFED_SENDER": 3.50, + }, + expectedSymParams: map[string]string{ + "BAYES_99": "1.00", + }, + }, + { + name: "Zero threshold uses default", + header: "default: False [1.00 / 0.00]", + expectedScore: 1.00, + expectedThreshold: rspamdDefaultAddHeaderThreshold, + expectedIsSpam: false, + expectedSymbols: map[string]float32{}, + }, + { + name: "Symbol without params", + header: "default: False [2.00 / 15.00];\n\tMISSING_DATE(1.00)", + expectedScore: 2.00, + expectedThreshold: 15.00, + expectedIsSpam: false, + expectedSymbols: map[string]float32{ + "MISSING_DATE": 1.00, + }, + }, + { + name: "Case-insensitive true flag", + header: "default: true [8.00 / 6.00]", + expectedScore: 8.00, + expectedThreshold: 6.00, + expectedIsSpam: true, + expectedSymbols: map[string]float32{}, + }, + { + name: "Zero threshold with symbols containing nested brackets in params", + header: "default: False [0.90 / 0.00];\n" + + "\tARC_REJECT(1.00)[cannot verify 1 of 1 signatures: {[1] = sig:mail-tester.local:signature has incorrect length: 12}];\n" + + "\tMIME_GOOD(-0.10)[multipart/alternative,text/plain];\n" + + "\tMIME_TRACE(0.00)[0:+,1:+,2:~]", + expectedScore: 0.90, + expectedThreshold: rspamdDefaultAddHeaderThreshold, + expectedIsSpam: false, + expectedSymbols: map[string]float32{ + "ARC_REJECT": 1.00, + "MIME_GOOD": -0.10, + "MIME_TRACE": 0.00, + }, + expectedSymParams: map[string]string{ + "ARC_REJECT": "cannot verify 1 of 1 signatures: {[1] = sig:mail-tester.local:signature has incorrect length: 12}", + "MIME_GOOD": "multipart/alternative,text/plain", + "MIME_TRACE": "0:+,1:+,2:~", + }, + }, + } + + analyzer := NewRspamdAnalyzer() + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := &api.RspamdResult{ + Symbols: make(map[string]api.RspamdSymbol), + } + analyzer.parseSpamdResult(tt.header, result) + + if result.Score != tt.expectedScore { + t.Errorf("Score = %v, want %v", result.Score, tt.expectedScore) + } + if result.Threshold != tt.expectedThreshold { + t.Errorf("Threshold = %v, want %v", result.Threshold, tt.expectedThreshold) + } + if result.IsSpam != tt.expectedIsSpam { + t.Errorf("IsSpam = %v, want %v", result.IsSpam, tt.expectedIsSpam) + } + for symName, expectedScore := range tt.expectedSymbols { + sym, ok := result.Symbols[symName] + if !ok { + t.Errorf("Symbol %s not found", symName) + continue + } + if sym.Score != expectedScore { + t.Errorf("Symbol %s score = %v, want %v", symName, sym.Score, expectedScore) + } + } + for symName, expectedParam := range tt.expectedSymParams { + sym, ok := result.Symbols[symName] + if !ok { + t.Errorf("Symbol %s not found for params check", symName) + continue + } + if sym.Params == nil { + t.Errorf("Symbol %s params = nil, want %q", symName, expectedParam) + } else if *sym.Params != expectedParam { + t.Errorf("Symbol %s params = %q, want %q", symName, *sym.Params, expectedParam) + } + } + }) + } +} + +func TestAnalyzeRspamd(t *testing.T) { + tests := []struct { + name string + headers map[string]string + expectedScore float32 + expectedThreshold float32 + expectedIsSpam bool + expectedServer *string + expectedSymCount int + }{ + { + name: "Full headers clean email", + headers: map[string]string{ + "X-Spamd-Result": "default: False [-3.91 / 15.00];\n\tALL_TRUSTED(-1.00)[local]", + "X-Rspamd-Score": "-3.91", + "X-Rspamd-Server": "mail.example.com", + }, + expectedScore: -3.91, + expectedThreshold: 15.00, + expectedIsSpam: false, + expectedServer: func() *string { s := "mail.example.com"; return &s }(), + expectedSymCount: 1, + }, + { + name: "X-Rspamd-Score overrides spamd result score", + headers: map[string]string{ + "X-Spamd-Result": "default: False [2.00 / 15.00]", + "X-Rspamd-Score": "3.50", + }, + expectedScore: 3.50, + expectedThreshold: 15.00, + expectedIsSpam: false, + }, + { + name: "Spam email above threshold", + headers: map[string]string{ + "X-Spamd-Result": "default: True [16.00 / 15.00];\n\tBAYES_99(5.00)", + "X-Rspamd-Score": "16.00", + }, + expectedScore: 16.00, + expectedThreshold: 15.00, + expectedIsSpam: true, + expectedSymCount: 1, + }, + { + name: "No X-Spamd-Result, only X-Rspamd-Score below default threshold", + headers: map[string]string{ + "X-Rspamd-Score": "2.00", + }, + expectedScore: 2.00, + expectedIsSpam: false, + }, + { + name: "No X-Spamd-Result, X-Rspamd-Score above default add-header threshold", + headers: map[string]string{ + "X-Rspamd-Score": "7.00", + }, + expectedScore: 7.00, + expectedIsSpam: true, + }, + { + name: "Server header is trimmed", + headers: map[string]string{ + "X-Rspamd-Score": "1.00", + "X-Rspamd-Server": " rspamd-01 ", + }, + expectedScore: 1.00, + expectedServer: func() *string { s := "rspamd-01"; return &s }(), + }, + } + + analyzer := NewRspamdAnalyzer() + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + email := &EmailMessage{Header: make(mail.Header)} + for k, v := range tt.headers { + email.Header[k] = []string{v} + } + + result := analyzer.AnalyzeRspamd(email) + + if result == nil { + t.Fatal("Expected non-nil result") + } + if result.Score != tt.expectedScore { + t.Errorf("Score = %v, want %v", result.Score, tt.expectedScore) + } + if tt.expectedThreshold > 0 && result.Threshold != tt.expectedThreshold { + t.Errorf("Threshold = %v, want %v", result.Threshold, tt.expectedThreshold) + } + if result.IsSpam != tt.expectedIsSpam { + t.Errorf("IsSpam = %v, want %v", result.IsSpam, tt.expectedIsSpam) + } + if tt.expectedServer != nil { + if result.Server == nil { + t.Errorf("Server = nil, want %q", *tt.expectedServer) + } else if *result.Server != *tt.expectedServer { + t.Errorf("Server = %q, want %q", *result.Server, *tt.expectedServer) + } + } + if tt.expectedSymCount > 0 && len(result.Symbols) != tt.expectedSymCount { + t.Errorf("Symbol count = %d, want %d", len(result.Symbols), tt.expectedSymCount) + } + }) + } +} + +func TestCalculateRspamdScore(t *testing.T) { + tests := []struct { + name string + result *api.RspamdResult + expectedScore int + expectedGrade string + }{ + { + name: "Nil result (rspamd not installed)", + result: nil, + expectedScore: 100, + expectedGrade: "", + }, + { + name: "Score well below threshold", + result: &api.RspamdResult{ + Score: -3.91, + Threshold: 15.00, + }, + expectedScore: 100, + expectedGrade: "A+", + }, + { + name: "Score at zero", + result: &api.RspamdResult{ + Score: 0, + Threshold: 15.00, + }, + // 100 - round(0*100/30) = 100 → hits ScoreToGrade(100) = "A" + expectedScore: 100, + expectedGrade: "A", + }, + { + name: "Score at threshold (half of 2*threshold)", + result: &api.RspamdResult{ + Score: 15.00, + Threshold: 15.00, + }, + // 100 - round(15*100/(2*15)) = 100 - 50 = 50 + expectedScore: 50, + }, + { + name: "Score above 2*threshold", + result: &api.RspamdResult{ + Score: 31.00, + Threshold: 15.00, + }, + expectedScore: 0, + expectedGrade: "F", + }, + { + name: "Score exactly at 2*threshold", + result: &api.RspamdResult{ + Score: 30.00, + Threshold: 15.00, + }, + // 100 - round(30*100/30) = 100 - 100 = 0 + expectedScore: 0, + expectedGrade: "F", + }, + } + + analyzer := NewRspamdAnalyzer() + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + score, grade := analyzer.CalculateRspamdScore(tt.result) + + if score != tt.expectedScore { + t.Errorf("Score = %d, want %d", score, tt.expectedScore) + } + if tt.expectedGrade != "" && grade != tt.expectedGrade { + t.Errorf("Grade = %q, want %q", grade, tt.expectedGrade) + } + }) + } +} + +const sampleEmailWithRspamdHeaders = `X-Spamd-Result: default: False [-3.91 / 15.00]; + BAYES_HAM(-3.00)[99%]; + RCVD_IN_DNSWL_MED(-0.01)[1.2.3.4:from]; + R_DKIM_ALLOW(-0.20)[example.com:s=dkim]; + FROM_HAS_DN(0.00)[]; + MIME_GOOD(-0.10)[text/plain]; +X-Rspamd-Score: -3.91 +X-Rspamd-Server: rspamd-01.example.com +Date: Mon, 09 Mar 2026 10:00:00 +0000 +From: sender@example.com +To: test@happydomain.org +Subject: Test email +Message-ID: +MIME-Version: 1.0 +Content-Type: text/plain + +Hello world` + +func TestAnalyzeRspamdRealEmail(t *testing.T) { + email, err := ParseEmail(bytes.NewBufferString(sampleEmailWithRspamdHeaders)) + if err != nil { + t.Fatalf("Failed to parse email: %v", err) + } + + analyzer := NewRspamdAnalyzer() + result := analyzer.AnalyzeRspamd(email) + + if result == nil { + t.Fatal("Expected non-nil result") + } + if result.IsSpam { + t.Error("Expected IsSpam=false") + } + if result.Score != -3.91 { + t.Errorf("Score = %v, want -3.91", result.Score) + } + if result.Threshold != 15.00 { + t.Errorf("Threshold = %v, want 15.00", result.Threshold) + } + if result.Server == nil || *result.Server != "rspamd-01.example.com" { + t.Errorf("Server = %v, want \"rspamd-01.example.com\"", result.Server) + } + + expectedSymbols := []string{"BAYES_HAM", "RCVD_IN_DNSWL_MED", "R_DKIM_ALLOW", "FROM_HAS_DN", "MIME_GOOD"} + for _, sym := range expectedSymbols { + if _, ok := result.Symbols[sym]; !ok { + t.Errorf("Symbol %s not found", sym) + } + } + + score, _ := analyzer.CalculateRspamdScore(result) + if score != 100 { + t.Errorf("CalculateRspamdScore = %d, want 100", score) + } +} + diff --git a/web/src/lib/components/BlacklistCard.svelte b/web/src/lib/components/BlacklistCard.svelte index 7f9b7f2..bb80acb 100644 --- a/web/src/lib/components/BlacklistCard.svelte +++ b/web/src/lib/components/BlacklistCard.svelte @@ -1,23 +1,21 @@
-

+

Blacklist Checks @@ -35,11 +33,7 @@

- {#if receivedChain} - - {/if} - -
+
{#each Object.entries(blacklists) as [ip, checks]}
diff --git a/web/src/lib/components/EmailPathCard.svelte b/web/src/lib/components/EmailPathCard.svelte index 8dc57b0..a4fda45 100644 --- a/web/src/lib/components/EmailPathCard.svelte +++ b/web/src/lib/components/EmailPathCard.svelte @@ -1,5 +1,6 @@ {#if receivedChain && receivedChain.length > 0} -
-
Email Path (Received Chain)
-
+
+
+

+ + Email Path +

+
+
{#each receivedChain as hop, i}
@@ -30,7 +40,7 @@ : "-"}
- {#if hop.with || hop.id} + {#if hop.with || hop.id || hop.from}

{#if hop.with} diff --git a/web/src/lib/components/RspamdCard.svelte b/web/src/lib/components/RspamdCard.svelte index 2468f90..0db6378 100644 --- a/web/src/lib/components/RspamdCard.svelte +++ b/web/src/lib/components/RspamdCard.svelte @@ -17,8 +17,7 @@ const effectiveAction = $derived.by(() => { const rejectThreshold = rspamd.threshold > 0 ? rspamd.threshold : 15; - if (rspamd.score >= rejectThreshold) - return { label: "Reject", cls: "bg-danger" }; + if (rspamd.score >= rejectThreshold) return { label: "Reject", cls: "bg-danger" }; if (rspamd.score >= RSPAMD_ADD_HEADER_THRESHOLD) return { label: "Add header", cls: "bg-warning text-dark" }; if (rspamd.score >= RSPAMD_GREYLIST_THRESHOLD) @@ -31,7 +30,7 @@

- + rspamd Analysis @@ -108,10 +107,32 @@

{/if} + + {#if rspamd.report} +
+ Raw Report + 
{rspamd.report}
+
+ {/if}