Add rspamd symbol descriptions from embedded/API lookup
Embed rspamd-symbols.json in the binary to provide human-readable descriptions for rspamd symbols in reports. Optionally fetch fresh symbols from a configurable rspamd API URL (--rspamd-api-url flag), falling back to the embedded list on error. Update the frontend to display descriptions alongside symbol names and scores.
This commit is contained in:
parent
5c104f3c99
commit
7d3009d7d0
12 changed files with 6816 additions and 20 deletions
|
|
@ -175,7 +175,8 @@ ENV HAPPYDELIVER_DATABASE_TYPE=sqlite \
|
||||||
HAPPYDELIVER_DOMAIN=happydeliver.local \
|
HAPPYDELIVER_DOMAIN=happydeliver.local \
|
||||||
HAPPYDELIVER_ADDRESS_PREFIX=test- \
|
HAPPYDELIVER_ADDRESS_PREFIX=test- \
|
||||||
HAPPYDELIVER_DNS_TIMEOUT=5s \
|
HAPPYDELIVER_DNS_TIMEOUT=5s \
|
||||||
HAPPYDELIVER_HTTP_TIMEOUT=10s
|
HAPPYDELIVER_HTTP_TIMEOUT=10s \
|
||||||
|
HAPPYDELIVER_RSPAMD_API_URL=http://127.0.0.1:11334
|
||||||
|
|
||||||
# Volume for persistent data
|
# Volume for persistent data
|
||||||
VOLUME ["/var/lib/happydeliver", "/var/log/happydeliver"]
|
VOLUME ["/var/lib/happydeliver", "/var/log/happydeliver"]
|
||||||
|
|
|
||||||
|
|
@ -39,6 +39,7 @@ func declareFlags(o *Config) {
|
||||||
flag.DurationVar(&o.Analysis.HTTPTimeout, "http-timeout", o.Analysis.HTTPTimeout, "Timeout when performing HTTP query")
|
flag.DurationVar(&o.Analysis.HTTPTimeout, "http-timeout", o.Analysis.HTTPTimeout, "Timeout when performing HTTP query")
|
||||||
flag.Var(&StringArray{&o.Analysis.RBLs}, "rbl", "Append a RBL (use this option multiple time to append multiple RBLs)")
|
flag.Var(&StringArray{&o.Analysis.RBLs}, "rbl", "Append a RBL (use this option multiple time to append multiple RBLs)")
|
||||||
flag.BoolVar(&o.Analysis.CheckAllIPs, "check-all-ips", o.Analysis.CheckAllIPs, "Check all IPs found in email headers against RBLs (not just the first one)")
|
flag.BoolVar(&o.Analysis.CheckAllIPs, "check-all-ips", o.Analysis.CheckAllIPs, "Check all IPs found in email headers against RBLs (not just the first one)")
|
||||||
|
flag.StringVar(&o.Analysis.RspamdAPIURL, "rspamd-api-url", o.Analysis.RspamdAPIURL, "rspamd API URL for symbol descriptions (default: use embedded list)")
|
||||||
flag.DurationVar(&o.ReportRetention, "report-retention", o.ReportRetention, "How long to keep reports (e.g., 720h, 30d). 0 = keep forever")
|
flag.DurationVar(&o.ReportRetention, "report-retention", o.ReportRetention, "How long to keep reports (e.g., 720h, 30d). 0 = keep forever")
|
||||||
flag.UintVar(&o.RateLimit, "rate-limit", o.RateLimit, "API rate limit (requests per second per IP)")
|
flag.UintVar(&o.RateLimit, "rate-limit", o.RateLimit, "API rate limit (requests per second per IP)")
|
||||||
flag.Var(&URL{&o.SurveyURL}, "survey-url", "URL for user feedback survey")
|
flag.Var(&URL{&o.SurveyURL}, "survey-url", "URL for user feedback survey")
|
||||||
|
|
|
||||||
|
|
@ -72,7 +72,8 @@ type AnalysisConfig struct {
|
||||||
HTTPTimeout time.Duration
|
HTTPTimeout time.Duration
|
||||||
RBLs []string
|
RBLs []string
|
||||||
DNSWLs []string
|
DNSWLs []string
|
||||||
CheckAllIPs bool // Check all IPs found in headers, not just the first one
|
CheckAllIPs bool // Check all IPs found in headers, not just the first one
|
||||||
|
RspamdAPIURL string // rspamd API URL for fetching symbol descriptions (empty = use embedded list)
|
||||||
}
|
}
|
||||||
|
|
||||||
// DefaultConfig returns a configuration with sensible defaults
|
// DefaultConfig returns a configuration with sensible defaults
|
||||||
|
|
|
||||||
|
|
@ -47,6 +47,7 @@ func NewEmailAnalyzer(cfg *config.Config) *EmailAnalyzer {
|
||||||
cfg.Analysis.RBLs,
|
cfg.Analysis.RBLs,
|
||||||
cfg.Analysis.DNSWLs,
|
cfg.Analysis.DNSWLs,
|
||||||
cfg.Analysis.CheckAllIPs,
|
cfg.Analysis.CheckAllIPs,
|
||||||
|
cfg.Analysis.RspamdAPIURL,
|
||||||
)
|
)
|
||||||
|
|
||||||
return &EmailAnalyzer{
|
return &EmailAnalyzer{
|
||||||
|
|
|
||||||
|
|
@ -49,11 +49,12 @@ func NewReportGenerator(
|
||||||
rbls []string,
|
rbls []string,
|
||||||
dnswls []string,
|
dnswls []string,
|
||||||
checkAllIPs bool,
|
checkAllIPs bool,
|
||||||
|
rspamdAPIURL string,
|
||||||
) *ReportGenerator {
|
) *ReportGenerator {
|
||||||
return &ReportGenerator{
|
return &ReportGenerator{
|
||||||
authAnalyzer: NewAuthenticationAnalyzer(receiverHostname),
|
authAnalyzer: NewAuthenticationAnalyzer(receiverHostname),
|
||||||
spamAnalyzer: NewSpamAssassinAnalyzer(),
|
spamAnalyzer: NewSpamAssassinAnalyzer(),
|
||||||
rspamdAnalyzer: NewRspamdAnalyzer(),
|
rspamdAnalyzer: NewRspamdAnalyzer(LoadRspamdSymbols(rspamdAPIURL)),
|
||||||
dnsAnalyzer: NewDNSAnalyzer(dnsTimeout),
|
dnsAnalyzer: NewDNSAnalyzer(dnsTimeout),
|
||||||
rblChecker: NewRBLChecker(dnsTimeout, rbls, checkAllIPs),
|
rblChecker: NewRBLChecker(dnsTimeout, rbls, checkAllIPs),
|
||||||
dnswlChecker: NewDNSWLChecker(dnsTimeout, dnswls, checkAllIPs),
|
dnswlChecker: NewDNSWLChecker(dnsTimeout, dnswls, checkAllIPs),
|
||||||
|
|
|
||||||
|
|
@ -32,7 +32,7 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestNewReportGenerator(t *testing.T) {
|
func TestNewReportGenerator(t *testing.T) {
|
||||||
gen := NewReportGenerator("", 10*time.Second, 10*time.Second, DefaultRBLs, DefaultDNSWLs, false)
|
gen := NewReportGenerator("", 10*time.Second, 10*time.Second, DefaultRBLs, DefaultDNSWLs, false, "")
|
||||||
if gen == nil {
|
if gen == nil {
|
||||||
t.Fatal("Expected report generator, got nil")
|
t.Fatal("Expected report generator, got nil")
|
||||||
}
|
}
|
||||||
|
|
@ -55,7 +55,7 @@ func TestNewReportGenerator(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAnalyzeEmail(t *testing.T) {
|
func TestAnalyzeEmail(t *testing.T) {
|
||||||
gen := NewReportGenerator("", 10*time.Second, 10*time.Second, DefaultRBLs, DefaultDNSWLs, false)
|
gen := NewReportGenerator("", 10*time.Second, 10*time.Second, DefaultRBLs, DefaultDNSWLs, false, "")
|
||||||
|
|
||||||
email := createTestEmail()
|
email := createTestEmail()
|
||||||
|
|
||||||
|
|
@ -75,7 +75,7 @@ func TestAnalyzeEmail(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestGenerateReport(t *testing.T) {
|
func TestGenerateReport(t *testing.T) {
|
||||||
gen := NewReportGenerator("", 10*time.Second, 10*time.Second, DefaultRBLs, DefaultDNSWLs, false)
|
gen := NewReportGenerator("", 10*time.Second, 10*time.Second, DefaultRBLs, DefaultDNSWLs, false, "")
|
||||||
testID := uuid.New()
|
testID := uuid.New()
|
||||||
|
|
||||||
email := createTestEmail()
|
email := createTestEmail()
|
||||||
|
|
@ -130,7 +130,7 @@ func TestGenerateReport(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestGenerateReportWithSpamAssassin(t *testing.T) {
|
func TestGenerateReportWithSpamAssassin(t *testing.T) {
|
||||||
gen := NewReportGenerator("", 10*time.Second, 10*time.Second, DefaultRBLs, DefaultDNSWLs, false)
|
gen := NewReportGenerator("", 10*time.Second, 10*time.Second, DefaultRBLs, DefaultDNSWLs, false, "")
|
||||||
testID := uuid.New()
|
testID := uuid.New()
|
||||||
|
|
||||||
email := createTestEmailWithSpamAssassin()
|
email := createTestEmailWithSpamAssassin()
|
||||||
|
|
@ -150,7 +150,7 @@ func TestGenerateReportWithSpamAssassin(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestGenerateRawEmail(t *testing.T) {
|
func TestGenerateRawEmail(t *testing.T) {
|
||||||
gen := NewReportGenerator("", 10*time.Second, 10*time.Second, DefaultRBLs, DefaultDNSWLs, false)
|
gen := NewReportGenerator("", 10*time.Second, 10*time.Second, DefaultRBLs, DefaultDNSWLs, false, "")
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
|
|
|
||||||
21
pkg/analyzer/rspamd-symbols-README.md
Normal file
21
pkg/analyzer/rspamd-symbols-README.md
Normal file
|
|
@ -0,0 +1,21 @@
|
||||||
|
# rspamd-symbols.json
|
||||||
|
|
||||||
|
This file contains rspamd symbol descriptions, embedded into the binary at compile time as a fallback when no rspamd API URL is configured.
|
||||||
|
|
||||||
|
## How to update
|
||||||
|
|
||||||
|
Fetch the latest symbols from a running rspamd instance:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
curl http://127.0.0.1:11334/symbols > rspamd-symbols.json
|
||||||
|
```
|
||||||
|
|
||||||
|
Or with docker:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
docker run --rm --name rspamd --pull always rspamd/rspamd
|
||||||
|
docker exec -u 0 rspamd apt install -y curl
|
||||||
|
docker exec rspamd curl http://127.0.0.1:11334/symbols > rspamd-symbols.json
|
||||||
|
```
|
||||||
|
|
||||||
|
Then rebuild the project.
|
||||||
6646
pkg/analyzer/rspamd-symbols.json
Normal file
6646
pkg/analyzer/rspamd-symbols.json
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -37,11 +37,13 @@ const (
|
||||||
)
|
)
|
||||||
|
|
||||||
// RspamdAnalyzer analyzes rspamd results from email headers
|
// RspamdAnalyzer analyzes rspamd results from email headers
|
||||||
type RspamdAnalyzer struct{}
|
type RspamdAnalyzer struct {
|
||||||
|
symbols map[string]string
|
||||||
|
}
|
||||||
|
|
||||||
// NewRspamdAnalyzer creates a new rspamd analyzer
|
// NewRspamdAnalyzer creates a new rspamd analyzer with optional symbol descriptions
|
||||||
func NewRspamdAnalyzer() *RspamdAnalyzer {
|
func NewRspamdAnalyzer(symbols map[string]string) *RspamdAnalyzer {
|
||||||
return &RspamdAnalyzer{}
|
return &RspamdAnalyzer{symbols: symbols}
|
||||||
}
|
}
|
||||||
|
|
||||||
// AnalyzeRspamd extracts and analyzes rspamd results from email headers
|
// AnalyzeRspamd extracts and analyzes rspamd results from email headers
|
||||||
|
|
@ -83,6 +85,16 @@ func (a *RspamdAnalyzer) AnalyzeRspamd(email *EmailMessage) *api.RspamdResult {
|
||||||
result.Server = &server
|
result.Server = &server
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Populate symbol descriptions from the lookup map
|
||||||
|
if a.symbols != nil {
|
||||||
|
for name, sym := range result.Symbols {
|
||||||
|
if desc, ok := a.symbols[name]; ok {
|
||||||
|
sym.Description = &desc
|
||||||
|
result.Symbols[name] = sym
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Derive IsSpam from score vs reject threshold.
|
// Derive IsSpam from score vs reject threshold.
|
||||||
if result.Threshold > 0 {
|
if result.Threshold > 0 {
|
||||||
result.IsSpam = result.Score >= result.Threshold
|
result.IsSpam = result.Score >= result.Threshold
|
||||||
|
|
|
||||||
105
pkg/analyzer/rspamd_symbols.go
Normal file
105
pkg/analyzer/rspamd_symbols.go
Normal file
|
|
@ -0,0 +1,105 @@
|
||||||
|
// This file is part of the happyDeliver (R) project.
|
||||||
|
// Copyright (c) 2026 happyDomain
|
||||||
|
// Authors: Pierre-Olivier Mercier, et al.
|
||||||
|
//
|
||||||
|
// This program is offered under a commercial and under the AGPL license.
|
||||||
|
// For commercial licensing, contact us at <contact@happydomain.org>.
|
||||||
|
//
|
||||||
|
// For AGPL licensing:
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package analyzer
|
||||||
|
|
||||||
|
import (
|
||||||
|
_ "embed"
|
||||||
|
"encoding/json"
|
||||||
|
"io"
|
||||||
|
"log"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
//go:embed rspamd-symbols.json
|
||||||
|
var embeddedRspamdSymbols []byte
|
||||||
|
|
||||||
|
// rspamdSymbolGroup represents a group of rspamd symbols from the API/embedded JSON.
|
||||||
|
type rspamdSymbolGroup struct {
|
||||||
|
Group string `json:"group"`
|
||||||
|
Rules []rspamdSymbolEntry `json:"rules"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// rspamdSymbolEntry represents a single rspamd symbol entry.
|
||||||
|
type rspamdSymbolEntry struct {
|
||||||
|
Symbol string `json:"symbol"`
|
||||||
|
Description string `json:"description"`
|
||||||
|
Weight float64 `json:"weight"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseRspamdSymbolsJSON parses the rspamd symbols JSON into a name->description map.
|
||||||
|
func parseRspamdSymbolsJSON(data []byte) map[string]string {
|
||||||
|
var groups []rspamdSymbolGroup
|
||||||
|
if err := json.Unmarshal(data, &groups); err != nil {
|
||||||
|
log.Printf("Failed to parse rspamd symbols JSON: %v", err)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
symbols := make(map[string]string, len(groups)*10)
|
||||||
|
for _, g := range groups {
|
||||||
|
for _, r := range g.Rules {
|
||||||
|
if r.Description != "" {
|
||||||
|
symbols[r.Symbol] = r.Description
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return symbols
|
||||||
|
}
|
||||||
|
|
||||||
|
// LoadRspamdSymbols loads rspamd symbol descriptions.
|
||||||
|
// If apiURL is non-empty, it fetches from the rspamd API first, falling back to the embedded list on error.
|
||||||
|
func LoadRspamdSymbols(apiURL string) map[string]string {
|
||||||
|
if apiURL != "" {
|
||||||
|
if symbols := fetchRspamdSymbols(apiURL); symbols != nil {
|
||||||
|
return symbols
|
||||||
|
}
|
||||||
|
log.Printf("Failed to fetch rspamd symbols from %s, using embedded list", apiURL)
|
||||||
|
}
|
||||||
|
return parseRspamdSymbolsJSON(embeddedRspamdSymbols)
|
||||||
|
}
|
||||||
|
|
||||||
|
// fetchRspamdSymbols fetches symbol descriptions from the rspamd API.
|
||||||
|
func fetchRspamdSymbols(apiURL string) map[string]string {
|
||||||
|
url := strings.TrimRight(apiURL, "/") + "/symbols"
|
||||||
|
|
||||||
|
client := &http.Client{Timeout: 10 * time.Second}
|
||||||
|
resp, err := client.Get(url)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("Error fetching rspamd symbols: %v", err)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
log.Printf("rspamd API returned status %d", resp.StatusCode)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
body, err := io.ReadAll(resp.Body)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("Error reading rspamd symbols response: %v", err)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return parseRspamdSymbolsJSON(body)
|
||||||
|
}
|
||||||
|
|
@ -30,7 +30,7 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestAnalyzeRspamdNoHeaders(t *testing.T) {
|
func TestAnalyzeRspamdNoHeaders(t *testing.T) {
|
||||||
analyzer := NewRspamdAnalyzer()
|
analyzer := NewRspamdAnalyzer(nil)
|
||||||
email := &EmailMessage{Header: make(mail.Header)}
|
email := &EmailMessage{Header: make(mail.Header)}
|
||||||
|
|
||||||
result := analyzer.AnalyzeRspamd(email)
|
result := analyzer.AnalyzeRspamd(email)
|
||||||
|
|
@ -126,7 +126,7 @@ func TestParseSpamdResult(t *testing.T) {
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
analyzer := NewRspamdAnalyzer()
|
analyzer := NewRspamdAnalyzer(nil)
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
|
@ -241,7 +241,7 @@ func TestAnalyzeRspamd(t *testing.T) {
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
analyzer := NewRspamdAnalyzer()
|
analyzer := NewRspamdAnalyzer(nil)
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
|
@ -340,7 +340,7 @@ func TestCalculateRspamdScore(t *testing.T) {
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
analyzer := NewRspamdAnalyzer()
|
analyzer := NewRspamdAnalyzer(nil)
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
|
@ -380,7 +380,7 @@ func TestAnalyzeRspamdRealEmail(t *testing.T) {
|
||||||
t.Fatalf("Failed to parse email: %v", err)
|
t.Fatalf("Failed to parse email: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
analyzer := NewRspamdAnalyzer()
|
analyzer := NewRspamdAnalyzer(nil)
|
||||||
result := analyzer.AnalyzeRspamd(email)
|
result := analyzer.AnalyzeRspamd(email)
|
||||||
|
|
||||||
if result == nil {
|
if result == nil {
|
||||||
|
|
|
||||||
|
|
@ -75,7 +75,7 @@
|
||||||
<tr>
|
<tr>
|
||||||
<th>Symbol</th>
|
<th>Symbol</th>
|
||||||
<th class="text-end">Score</th>
|
<th class="text-end">Score</th>
|
||||||
<th>Parameters</th>
|
<th>Description</th>
|
||||||
</tr>
|
</tr>
|
||||||
</thead>
|
</thead>
|
||||||
<tbody>
|
<tbody>
|
||||||
|
|
@ -87,7 +87,14 @@
|
||||||
? "table-success"
|
? "table-success"
|
||||||
: ""}
|
: ""}
|
||||||
>
|
>
|
||||||
<td class="font-monospace">{symbolName}</td>
|
<td>
|
||||||
|
<span class="font-monospace">{symbolName}</span>
|
||||||
|
{#if symbol.params}
|
||||||
|
<small class="d-block text-muted">
|
||||||
|
{symbol.params}
|
||||||
|
</small>
|
||||||
|
{/if}
|
||||||
|
</td>
|
||||||
<td class="text-end">
|
<td class="text-end">
|
||||||
<span
|
<span
|
||||||
class={symbol.score > 0
|
class={symbol.score > 0
|
||||||
|
|
@ -99,7 +106,7 @@
|
||||||
{symbol.score > 0 ? "+" : ""}{symbol.score.toFixed(2)}
|
{symbol.score > 0 ? "+" : ""}{symbol.score.toFixed(2)}
|
||||||
</span>
|
</span>
|
||||||
</td>
|
</td>
|
||||||
<td class="small text-muted">{symbol.params ?? ""}</td>
|
<td class="small text-muted">{symbol.description ?? ""}</td>
|
||||||
</tr>
|
</tr>
|
||||||
{/each}
|
{/each}
|
||||||
</tbody>
|
</tbody>
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue