happyDomain/happyDeliver
2
0
Fork 0
Code Issues 7 Pull requests 7 Projects Releases Packages Wiki Activity Actions

refactor: handle DNS whitelists

Browse source 
Introduce a single DNSListChecker struct with flags to avoid code
duplication with already existing RBL checker.
This commit is contained in:
nemunaire 2026-03-07 16:18:10 +07:00
commit ff013fe694
10 changed files with 225 additions and 107 deletions
Download patch file Download diff file Expand all files Collapse all files

5
pkg/analyzer/analyzer.go
View file

@ -44,6 +44,7 @@ func NewEmailAnalyzer(cfg *config.Config) *EmailAnalyzer {
cfg.Analysis.DNSTimeout,
cfg.Analysis.HTTPTimeout,
cfg.Analysis.RBLs,
cfg.Analysis.DNSWLs,
cfg.Analysis.CheckAllIPs,
)
@ -130,12 +131,12 @@ func (a *APIAdapter) CheckBlacklistIP(ip string) ([]api.BlacklistCheck, int, int
// Calculate score using the existing function
// Create a minimal RBLResults structure for scoring
results := &RBLResults{
results := &DNSListResults{
Checks: map[string][]api.BlacklistCheck{ip: checks},
IPsChecked: []string{ip},
ListedCount: listedCount,
}
score, grade := a.analyzer.generator.rblChecker.CalculateRBLScore(results)
score, grade := a.analyzer.generator.rblChecker.CalculateScore(results)
return checks, listedCount, score, grade, nil
}

165
pkg/analyzer/rbl.go
View file

@ -32,13 +32,15 @@ import (
"git.happydns.org/happyDeliver/internal/api"
)
// RBLChecker checks IP addresses against DNS-based blacklists
type RBLChecker struct {
// DNSListChecker checks IP addresses against DNS-based block/allow lists.
// It handles both RBL (blacklist) and DNSWL (whitelist) semantics via flags.
type DNSListChecker struct {
Timeout time.Duration
RBLs []string
Lists []string
CheckAllIPs bool // Check all IPs found in headers, not just the first one
filterErrorCodes bool // When true (RBL mode), treat 127.255.255.253/254/255 as operational errors
resolver *net.Resolver
informationalSet map[string]bool
informationalSet map[string]bool // Lists whose hits don't count toward the score
}
// DefaultRBLs is a list of commonly used RBL providers
@ -68,10 +70,16 @@ var DefaultInformationalRBLs = []string{
"dnsbl-3.uceprotect.net", // UCEPROTECT Level 3: entire ASes, too broad for scoring
}
// DefaultDNSWLs is a list of commonly used DNSWL providers
var DefaultDNSWLs = []string{
"list.dnswl.org", // DNSWL.org — the main DNS whitelist
"swl.spamhaus.org", // Spamhaus Safe Whitelist
}
// NewRBLChecker creates a new RBL checker with configurable timeout and RBL list
func NewRBLChecker(timeout time.Duration, rbls []string, checkAllIPs bool) *RBLChecker {
func NewRBLChecker(timeout time.Duration, rbls []string, checkAllIPs bool) *DNSListChecker {
if timeout == 0 {
timeout = 5 * time.Second // Default timeout
timeout = 5 * time.Second
}
if len(rbls) == 0 {
rbls = DefaultRBLs
@ -80,30 +88,48 @@ func NewRBLChecker(timeout time.Duration, rbls []string, checkAllIPs bool) *RBLC
for _, rbl := range DefaultInformationalRBLs {
informationalSet[rbl] = true
}
return &RBLChecker{
return &DNSListChecker{
Timeout: timeout,
RBLs: rbls,
Lists: rbls,
CheckAllIPs: checkAllIPs,
filterErrorCodes: true,
resolver: &net.Resolver{PreferGo: true},
informationalSet: informationalSet,
}
}
// RBLResults represents the results of RBL checks
type RBLResults struct {
Checks map[string][]api.BlacklistCheck // Map of IP -> list of RBL checks for that IP
IPsChecked []string
ListedCount int // Total listings including informational RBLs
RelevantListedCount int // Listings on scoring (non-informational) RBLs only
// NewDNSWLChecker creates a new DNSWL checker with configurable timeout and DNSWL list
func NewDNSWLChecker(timeout time.Duration, dnswls []string, checkAllIPs bool) *DNSListChecker {
if timeout == 0 {
timeout = 5 * time.Second
}
if len(dnswls) == 0 {
dnswls = DefaultDNSWLs
}
return &DNSListChecker{
Timeout: timeout,
Lists: dnswls,
CheckAllIPs: checkAllIPs,
filterErrorCodes: false,
resolver: &net.Resolver{PreferGo: true},
informationalSet: make(map[string]bool),
}
}
// CheckEmail checks all IPs found in the email headers against RBLs
func (r *RBLChecker) CheckEmail(email *EmailMessage) *RBLResults {
results := &RBLResults{
// DNSListResults represents the results of DNS list checks
type DNSListResults struct {
Checks map[string][]api.BlacklistCheck // Map of IP -> list of checks for that IP
IPsChecked []string
ListedCount int // Total listings including informational entries
RelevantListedCount int // Listings on scoring (non-informational) lists only
}
// CheckEmail checks all IPs found in the email headers against the configured lists
func (r *DNSListChecker) CheckEmail(email *EmailMessage) *DNSListResults {
results := &DNSListResults{
Checks: make(map[string][]api.BlacklistCheck),
}
// Extract IPs from Received headers
ips := r.extractIPs(email)
if len(ips) == 0 {
return results
@ -111,20 +137,18 @@ func (r *RBLChecker) CheckEmail(email *EmailMessage) *RBLResults {
results.IPsChecked = ips
// Check each IP against all RBLs
for _, ip := range ips {
for _, rbl := range r.RBLs {
check := r.checkIP(ip, rbl)
for _, list := range r.Lists {
check := r.checkIP(ip, list)
results.Checks[ip] = append(results.Checks[ip], check)
if check.Listed {
results.ListedCount++
if !r.informationalSet[rbl] {
if !r.informationalSet[list] {
results.RelevantListedCount++
}
}
}
// Only check the first IP unless CheckAllIPs is enabled
if !r.CheckAllIPs {
break
}
@ -133,9 +157,8 @@ func (r *RBLChecker) CheckEmail(email *EmailMessage) *RBLResults {
return results
}
// CheckIP checks a single IP address against all configured RBLs
func (r *RBLChecker) CheckIP(ip string) ([]api.BlacklistCheck, int, error) {
// Validate that it's a valid IP address
// CheckIP checks a single IP address against all configured lists
func (r *DNSListChecker) CheckIP(ip string) ([]api.BlacklistCheck, int, error) {
if !r.isPublicIP(ip) {
return nil, 0, fmt.Errorf("invalid or non-public IP address: %s", ip)
}
@ -143,9 +166,8 @@ func (r *RBLChecker) CheckIP(ip string) ([]api.BlacklistCheck, int, error) {
var checks []api.BlacklistCheck
listedCount := 0
// Check the IP against all RBLs
for _, rbl := range r.RBLs {
check := r.checkIP(ip, rbl)
for _, list := range r.Lists {
check := r.checkIP(ip, list)
checks = append(checks, check)
if check.Listed {
listedCount++
@ -156,27 +178,19 @@ func (r *RBLChecker) CheckIP(ip string) ([]api.BlacklistCheck, int, error) {
}
// extractIPs extracts IP addresses from Received headers
func (r *RBLChecker) extractIPs(email *EmailMessage) []string {
func (r *DNSListChecker) extractIPs(email *EmailMessage) []string {
var ips []string
seenIPs := make(map[string]bool)
// Get all Received headers
receivedHeaders := email.Header["Received"]
// Regex patterns for IP addresses
// Match IPv4: xxx.xxx.xxx.xxx
ipv4Pattern := regexp.MustCompile(`\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b`)
// Look for IPs in Received headers
for _, received := range receivedHeaders {
// Find all IPv4 addresses
matches := ipv4Pattern.FindAllString(received, -1)
for _, match := range matches {
// Skip private/reserved IPs
if !r.isPublicIP(match) {
continue
}
// Avoid duplicates
if !seenIPs[match] {
ips = append(ips, match)
seenIPs[match] = true
@ -184,13 +198,10 @@ func (r *RBLChecker) extractIPs(email *EmailMessage) []string {
}
}
// If no IPs found in Received headers, try X-Originating-IP
if len(ips) == 0 {
originatingIP := email.Header.Get("X-Originating-IP")
if originatingIP != "" {
// Extract IP from formats like "[192.0.2.1]" or "192.0.2.1"
cleanIP := strings.TrimSuffix(strings.TrimPrefix(originatingIP, "["), "]")
// Remove any whitespace
cleanIP = strings.TrimSpace(cleanIP)
matches := ipv4Pattern.FindString(cleanIP)
if matches != "" && r.isPublicIP(matches) {
@ -203,19 +214,16 @@ func (r *RBLChecker) extractIPs(email *EmailMessage) []string {
}
// isPublicIP checks if an IP address is public (not private, loopback, or reserved)
func (r *RBLChecker) isPublicIP(ipStr string) bool {
func (r *DNSListChecker) isPublicIP(ipStr string) bool {
ip := net.ParseIP(ipStr)
if ip == nil {
return false
}
// Check if it's a private network
if ip.IsPrivate() || ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() {
return false
}
// Additional checks for reserved ranges
// 0.0.0.0/8, 192.0.0.0/24, 192.0.2.0/24 (TEST-NET-1), 198.51.100.0/24 (TEST-NET-2), 203.0.113.0/24 (TEST-NET-3)
if ip.IsUnspecified() {
return false
}
@ -223,51 +231,43 @@ func (r *RBLChecker) isPublicIP(ipStr string) bool {
return true
}
// checkIP checks a single IP against a single RBL
func (r *RBLChecker) checkIP(ip, rbl string) api.BlacklistCheck {
// checkIP checks a single IP against a single DNS list
func (r *DNSListChecker) checkIP(ip, list string) api.BlacklistCheck {
check := api.BlacklistCheck{
Rbl: rbl,
Rbl: list,
}
// Reverse the IP for DNSBL query
reversedIP := r.reverseIP(ip)
if reversedIP == "" {
check.Error = api.PtrTo("Failed to reverse IP address")
return check
}
// Construct DNSBL query: reversed-ip.rbl-domain
query := fmt.Sprintf("%s.%s", reversedIP, rbl)
query := fmt.Sprintf("%s.%s", reversedIP, list)
// Perform DNS lookup with timeout
ctx, cancel := context.WithTimeout(context.Background(), r.Timeout)
defer cancel()
addrs, err := r.resolver.LookupHost(ctx, query)
if err != nil {
// Most likely not listed (NXDOMAIN)
if dnsErr, ok := err.(*net.DNSError); ok {
if dnsErr.IsNotFound {
check.Listed = false
return check
}
}
// Other DNS errors
check.Error = api.PtrTo(fmt.Sprintf("DNS lookup failed: %v", err))
return check
}
// If we got a response, check the return code
if len(addrs) > 0 {
check.Response = api.PtrTo(addrs[0]) // Return code (e.g., 127.0.0.2)
check.Response = api.PtrTo(addrs[0])
// Check for RBL error codes: 127.255.255.253, 127.255.255.254, 127.255.255.255
// These indicate RBL operational issues, not actual listings
if addrs[0] == "127.255.255.253" || addrs[0] == "127.255.255.254" || addrs[0] == "127.255.255.255" {
// In RBL mode, 127.255.255.253/254/255 indicate operational errors, not real listings.
if r.filterErrorCodes && (addrs[0] == "127.255.255.253" || addrs[0] == "127.255.255.254" || addrs[0] == "127.255.255.255") {
check.Listed = false
check.Error = api.PtrTo(fmt.Sprintf("RBL %s returned error code %s (RBL operational issue)", rbl, addrs[0]))
check.Error = api.PtrTo(fmt.Sprintf("RBL %s returned error code %s (RBL operational issue)", list, addrs[0]))
} else {
// Normal listing response
check.Listed = true
}
}
@ -275,50 +275,47 @@ func (r *RBLChecker) checkIP(ip, rbl string) api.BlacklistCheck {
return check
}
// reverseIP reverses an IPv4 address for DNSBL queries
// reverseIP reverses an IPv4 address for DNSBL/DNSWL queries
// Example: 192.0.2.1 -> 1.2.0.192
func (r *RBLChecker) reverseIP(ipStr string) string {
func (r *DNSListChecker) reverseIP(ipStr string) string {
ip := net.ParseIP(ipStr)
if ip == nil {
return ""
}
// Convert to IPv4
ipv4 := ip.To4()
if ipv4 == nil {
return "" // IPv6 not supported yet
}
// Reverse the octets
return fmt.Sprintf("%d.%d.%d.%d", ipv4[3], ipv4[2], ipv4[1], ipv4[0])
}
// CalculateRBLScore calculates the blacklist contribution to deliverability.
// Informational RBLs are not counted in the score.
func (r *RBLChecker) CalculateRBLScore(results *RBLResults) (int, string) {
// CalculateScore calculates the list contribution to deliverability.
// Informational lists are not counted in the score.
func (r *DNSListChecker) CalculateScore(results *DNSListResults) (int, string) {
if results == nil || len(results.IPsChecked) == 0 {
// No IPs to check, give benefit of doubt
return 100, ""
}
scoringRBLCount := len(r.RBLs) - len(r.informationalSet)
if scoringRBLCount <= 0 {
scoringListCount := len(r.Lists) - len(r.informationalSet)
if scoringListCount <= 0 {
return 100, "A+"
}
percentage := 100 - results.RelevantListedCount*100/scoringRBLCount
percentage := 100 - results.RelevantListedCount*100/scoringListCount
return percentage, ScoreToGrade(percentage)
}
// GetUniqueListedIPs returns a list of unique IPs that are listed on at least one RBL
func (r *RBLChecker) GetUniqueListedIPs(results *RBLResults) []string {
// GetUniqueListedIPs returns a list of unique IPs that are listed on at least one entry
func (r *DNSListChecker) GetUniqueListedIPs(results *DNSListResults) []string {
var listedIPs []string
for ip, rblChecks := range results.Checks {
for _, check := range rblChecks {
for ip, checks := range results.Checks {
for _, check := range checks {
if check.Listed {
listedIPs = append(listedIPs, ip)
break // Only add the IP once
break
}
}
}
@ -326,17 +323,17 @@ func (r *RBLChecker) GetUniqueListedIPs(results *RBLResults) []string {
return listedIPs
}
// GetRBLsForIP returns all RBLs that list a specific IP
func (r *RBLChecker) GetRBLsForIP(results *RBLResults, ip string) []string {
var rbls []string
// GetListsForIP returns all lists that match a specific IP
func (r *DNSListChecker) GetListsForIP(results *DNSListResults, ip string) []string {
var lists []string
if rblChecks, exists := results.Checks[ip]; exists {
for _, check := range rblChecks {
if checks, exists := results.Checks[ip]; exists {
for _, check := range checks {
if check.Listed {
rbls = append(rbls, check.Rbl)
lists = append(lists, check.Rbl)
}
}
}
return rbls
return lists
}

8
pkg/analyzer/rbl_test.go
View file

@ -59,8 +59,8 @@ func TestNewRBLChecker(t *testing.T) {
if checker.Timeout != tt.expectedTimeout {
t.Errorf("Timeout = %v, want %v", checker.Timeout, tt.expectedTimeout)
}
if len(checker.RBLs) != tt.expectedRBLs {
t.Errorf("RBLs count = %d, want %d", len(checker.RBLs), tt.expectedRBLs)
if len(checker.Lists) != tt.expectedRBLs {
t.Errorf("RBLs count = %d, want %d", len(checker.Lists), tt.expectedRBLs)
}
if checker.resolver == nil {
t.Error("Resolver should not be nil")
@ -326,7 +326,7 @@ func TestGetBlacklistScore(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
score, _ := checker.CalculateRBLScore(tt.results)
score, _ := checker.CalculateScore(tt.results)
if score != tt.expectedScore {
t.Errorf("GetBlacklistScore() = %v, want %v", score, tt.expectedScore)
}
@ -402,7 +402,7 @@ func TestGetRBLsForIP(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
rbls := checker.GetRBLsForIP(results, tt.ip)
rbls := checker.GetListsForIP(results, tt.ip)
if len(rbls) != len(tt.expectedRBLs) {
t.Errorf("Got %d RBLs, want %d", len(rbls), len(tt.expectedRBLs))

16
pkg/analyzer/report.go
View file

@ -35,7 +35,8 @@ type ReportGenerator struct {
spamAnalyzer *SpamAssassinAnalyzer
rspamdAnalyzer *RspamdAnalyzer
dnsAnalyzer *DNSAnalyzer
rblChecker *RBLChecker
rblChecker *DNSListChecker
dnswlChecker *DNSListChecker
contentAnalyzer *ContentAnalyzer
headerAnalyzer *HeaderAnalyzer
}
@ -45,6 +46,7 @@ func NewReportGenerator(
dnsTimeout time.Duration,
httpTimeout time.Duration,
rbls []string,
dnswls []string,
checkAllIPs bool,
) *ReportGenerator {
return &ReportGenerator{
@ -53,6 +55,7 @@ func NewReportGenerator(
rspamdAnalyzer: NewRspamdAnalyzer(),
dnsAnalyzer: NewDNSAnalyzer(dnsTimeout),
rblChecker: NewRBLChecker(dnsTimeout, rbls, checkAllIPs),
dnswlChecker: NewDNSWLChecker(dnsTimeout, dnswls, checkAllIPs),
contentAnalyzer: NewContentAnalyzer(httpTimeout),
headerAnalyzer: NewHeaderAnalyzer(),
}
@ -65,7 +68,8 @@ type AnalysisResults struct {
Content *ContentResults
DNS *api.DNSResults
Headers *api.HeaderAnalysis
RBL *RBLResults
RBL *DNSListResults
DNSWL *DNSListResults
SpamAssassin *api.SpamAssassinResult
Rspamd *api.RspamdResult
}
@ -81,6 +85,7 @@ func (r *ReportGenerator) AnalyzeEmail(email *EmailMessage) *AnalysisResults {
results.Headers = r.headerAnalyzer.GenerateHeaderAnalysis(email, results.Authentication)
results.DNS = r.dnsAnalyzer.AnalyzeDNS(email, results.Authentication, results.Headers)
results.RBL = r.rblChecker.CheckEmail(email)
results.DNSWL = r.dnswlChecker.CheckEmail(email)
results.SpamAssassin = r.spamAnalyzer.AnalyzeSpamAssassin(email)
results.Rspamd = r.rspamdAnalyzer.AnalyzeRspamd(email)
results.Content = r.contentAnalyzer.AnalyzeContent(email)
@ -135,7 +140,7 @@ func (r *ReportGenerator) GenerateReport(testID uuid.UUID, results *AnalysisResu
blacklistScore := 0
var blacklistGrade string
if results.RBL != nil {
blacklistScore, blacklistGrade = r.rblChecker.CalculateRBLScore(results.RBL)
blacklistScore, blacklistGrade = r.rblChecker.CalculateScore(results.RBL)
}
saScore, saGrade := r.spamAnalyzer.CalculateSpamAssassinScore(results.SpamAssassin)
@ -197,6 +202,11 @@ func (r *ReportGenerator) GenerateReport(testID uuid.UUID, results *AnalysisResu
report.Blacklists = &results.RBL.Checks
}
// Add whitelist checks as a map of IP -> array of BlacklistCheck (informational only)
if results.DNSWL != nil && len(results.DNSWL.Checks) > 0 {
report.Whitelists = &results.DNSWL.Checks
}
// Add SpamAssassin result with individual deliverability score
if results.SpamAssassin != nil {
saGradeTyped := api.SpamAssassinResultDeliverabilityGrade(saGrade)

10
pkg/analyzer/report_test.go
View file

@ -32,7 +32,7 @@ import (
)
func TestNewReportGenerator(t *testing.T) {
gen := NewReportGenerator(10*time.Second, 10*time.Second, DefaultRBLs, false)
gen := NewReportGenerator(10*time.Second, 10*time.Second, DefaultRBLs, DefaultDNSWLs, false)
if gen == nil {
t.Fatal("Expected report generator, got nil")
}
@ -55,7 +55,7 @@ func TestNewReportGenerator(t *testing.T) {
}
func TestAnalyzeEmail(t *testing.T) {
gen := NewReportGenerator(10*time.Second, 10*time.Second, DefaultRBLs, false)
gen := NewReportGenerator(10*time.Second, 10*time.Second, DefaultRBLs, DefaultDNSWLs, false)
email := createTestEmail()
@ -75,7 +75,7 @@ func TestAnalyzeEmail(t *testing.T) {
}
func TestGenerateReport(t *testing.T) {
gen := NewReportGenerator(10*time.Second, 10*time.Second, DefaultRBLs, false)
gen := NewReportGenerator(10*time.Second, 10*time.Second, DefaultRBLs, DefaultDNSWLs, false)
testID := uuid.New()
email := createTestEmail()
@ -130,7 +130,7 @@ func TestGenerateReport(t *testing.T) {
}
func TestGenerateReportWithSpamAssassin(t *testing.T) {
gen := NewReportGenerator(10*time.Second, 10*time.Second, DefaultRBLs, false)
gen := NewReportGenerator(10*time.Second, 10*time.Second, DefaultRBLs, DefaultDNSWLs, false)
testID := uuid.New()
email := createTestEmailWithSpamAssassin()
@ -150,7 +150,7 @@ func TestGenerateReportWithSpamAssassin(t *testing.T) {
}
func TestGenerateRawEmail(t *testing.T) {
gen := NewReportGenerator(10*time.Second, 10*time.Second, DefaultRBLs, false)
gen := NewReportGenerator(10*time.Second, 10*time.Second, DefaultRBLs, DefaultDNSWLs, false)
tests := []struct {
name string