From 229e7a8f0227f37ce95b9f67f9cce6993452a2e2 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Fri, 15 May 2026 18:30:46 +0800 Subject: [PATCH] Add abuse.ch ThreatFox and MalwareBazaar blacklist sources ThreatFox queries the IOC database for domain indicators (C2 servers, malware distribution, phishing); MalwareBazaar searches for malware samples tagged with the domain. Both require a free abuse.ch Auth-Key. --- README.md | 2 + checker/malwarebazaar.go | 149 ++++++++++++++++++++++++++++++++ checker/malwarebazaar_test.go | 94 +++++++++++++++++++++ checker/threatfox.go | 155 ++++++++++++++++++++++++++++++++++ checker/threatfox_test.go | 96 +++++++++++++++++++++ 5 files changed, 496 insertions(+) create mode 100644 checker/malwarebazaar.go create mode 100644 checker/malwarebazaar_test.go create mode 100644 checker/threatfox.go create mode 100644 checker/threatfox_test.go diff --git a/README.md b/README.md index 0b541df..7b4e8e8 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,8 @@ widely-used reputation systems. | OpenPhish public feed | downloaded list | no | user (default on) | | PhishTank | downloaded list | no | user (default on) | | abuse.ch URLhaus | HTTPS lookup | free Auth-Key (admin) | user (default on) | +| abuse.ch ThreatFox | HTTPS lookup | free Auth-Key (admin) | user (default on) | +| abuse.ch MalwareBazaar| HTTPS lookup | free Auth-Key (admin) | user (default on) | | VirusTotal v3 | HTTPS lookup | yes (admin) | admin | ### Obtaining API keys diff --git a/checker/malwarebazaar.go b/checker/malwarebazaar.go new file mode 100644 index 0000000..1627e6a --- /dev/null +++ b/checker/malwarebazaar.go @@ -0,0 +1,149 @@ +package checker + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "time" + + sdk "git.happydns.org/checker-sdk-go/checker" +) + +const malwareBazaarEndpoint = "https://mb-api.abuse.ch/api/v1/" + +func init() { Register(&malwareBazaarSource{endpoint: malwareBazaarEndpoint}) } + +type malwareBazaarSource struct { + endpoint string +} + +func (*malwareBazaarSource) ID() string { return "malwarebazaar" } +func (*malwareBazaarSource) Name() string { return "abuse.ch MalwareBazaar" } + +func (*malwareBazaarSource) Options() SourceOptions { + return SourceOptions{ + Admin: []sdk.CheckerOptionField{ + { + Id: "malwarebazaar_auth_key", + Type: "string", + Label: "MalwareBazaar Auth-Key", + Description: "abuse.ch MalwareBazaar Auth-Key (free, requires an abuse.ch account). Without this key the source is disabled.", + Secret: true, + }, + }, + User: []sdk.CheckerOptionField{ + { + Id: "enable_malwarebazaar", + Type: "bool", + Label: "Use abuse.ch MalwareBazaar", + Description: "Search MalwareBazaar for malware samples tagged with the domain (typically C2 infrastructure or delivery hosts).", + Default: true, + }, + }, + } +} + +func (s *malwareBazaarSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult { + authKey := stringOpt(opts, "malwarebazaar_auth_key") + if !sdk.GetBoolOption(opts, "enable_malwarebazaar", true) || registered == "" || authKey == "" { + return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: false}} + } + + res := SourceResult{ + SourceID: s.ID(), SourceName: s.Name(), Enabled: true, + Reference: "https://bazaar.abuse.ch/browse/", + } + + buf, err := json.Marshal(map[string]any{ + "query": "search_tag", + "tag": registered, + }) + if err != nil { + res.Error = err.Error() + return []SourceResult{res} + } + + reqCtx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, s.endpoint, bytes.NewReader(buf)) + if err != nil { + res.Error = err.Error() + return []SourceResult{res} + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Auth-Key", authKey) + + body, status, err := httpDo(req, 4<<20) + if err != nil { + res.Error = err.Error() + return []SourceResult{res} + } + if status != http.StatusOK { + res.Error = fmt.Sprintf("HTTP %d: %s", status, truncate(string(body), 200)) + return []SourceResult{res} + } + + var parsed struct { + QueryStatus string `json:"query_status"` + Data []struct { + SHA256Hash string `json:"sha256_hash"` + FileName string `json:"file_name"` + FileType string `json:"file_type_mime"` + Signature string `json:"signature"` + FirstSeen string `json:"first_seen"` + } `json:"data"` + } + if err := json.Unmarshal(body, &parsed); err != nil { + res.Error = "decode: " + err.Error() + return []SourceResult{res} + } + + switch parsed.QueryStatus { + case "ok": + signatures := map[string]bool{} + for _, sample := range parsed.Data { + if sample.Signature != "" && !signatures[sample.Signature] { + signatures[sample.Signature] = true + res.Reasons = append(res.Reasons, sample.Signature) + } + res.Evidence = append(res.Evidence, Evidence{ + Label: "Sample", + Value: sample.SHA256Hash, + Status: sample.FileType, + Extra: map[string]string{ + "filename": sample.FileName, + "signature": sample.Signature, + "first_seen": sample.FirstSeen, + }, + }) + } + case "no_results", "illegal_search_term": + // Clean. + default: + res.Error = "query_status=" + parsed.QueryStatus + } + return []SourceResult{res} +} + +func (*malwareBazaarSource) Evaluate(r SourceResult) (bool, string) { + if r.Enabled && r.Error == "" && len(r.Evidence) > 0 { + return true, SeverityWarn + } + return false, "" +} + +func (*malwareBazaarSource) Diagnose(res SourceResult) Diagnosis { + return Diagnosis{ + Severity: SeverityWarn, + Title: "Associated with malware samples in abuse.ch MalwareBazaar", + Detail: fmt.Sprintf( + "%d malware sample(s) are tagged with this domain; malware family/signature(s): %s. MalwareBazaar samples are tagged with their C2 domain or delivery host when known. Investigate the domain's hosting and DNS records to identify and remove malicious infrastructure.", + len(res.Evidence), joinNonEmpty(res.Reasons, ", "), + ), + Fix: res.Reference, + FixIsURL: res.Reference != "", + } +} diff --git a/checker/malwarebazaar_test.go b/checker/malwarebazaar_test.go new file mode 100644 index 0000000..62bc24a --- /dev/null +++ b/checker/malwarebazaar_test.go @@ -0,0 +1,94 @@ +package checker + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + sdk "git.happydns.org/checker-sdk-go/checker" +) + +func TestMalwareBazaarSource_NoResults(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"query_status":"no_results"}`)) + })) + defer srv.Close() + + s := &malwareBazaarSource{endpoint: srv.URL} + results := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_malwarebazaar": true, "malwarebazaar_auth_key": "k"}) + if len(results) != 1 { + t.Fatalf("expected 1 result, got %d", len(results)) + } + r := results[0] + listed, _ := s.Evaluate(r) + if !r.Enabled || listed || r.Error != "" { + t.Fatalf("expected enabled+clean, got %+v, Evaluate listed=%v", r, listed) + } +} + +func TestMalwareBazaarSource_Listed(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Auth-Key") == "" { + t.Errorf("missing Auth-Key header") + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "query_status": "ok", + "data": [{ + "sha256_hash": "aaaa1111bbbb2222cccc3333dddd4444eeee5555ffff6666aaaa1111bbbb2222", + "file_name": "evil.exe", + "file_type_mime": "application/x-dosexec", + "signature": "Emotet", + "first_seen": "2024-01-01 00:00:00" + }] + }`)) + })) + defer srv.Close() + + s := &malwareBazaarSource{endpoint: srv.URL} + r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_malwarebazaar": true, "malwarebazaar_auth_key": "k"})[0] + if len(r.Evidence) != 1 { + t.Fatalf("expected 1 evidence item, got %+v", r) + } + if listed, severity := s.Evaluate(r); !listed || severity != SeverityWarn { + t.Errorf("expected Evaluate()=(true, warn), got (%v, %q)", listed, severity) + } + if r.Evidence[0].Status != "application/x-dosexec" { + t.Errorf("evidence status = %q", r.Evidence[0].Status) + } + if len(r.Reasons) != 1 || r.Reasons[0] != "Emotet" { + t.Errorf("reasons = %v", r.Reasons) + } +} + +func TestMalwareBazaarSource_HTTPError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusUnauthorized) + _, _ = w.Write([]byte("unauthorized")) + })) + defer srv.Close() + + s := &malwareBazaarSource{endpoint: srv.URL} + r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_malwarebazaar": true, "malwarebazaar_auth_key": "k"})[0] + if r.Error == "" { + t.Errorf("expected error, got %+v", r) + } +} + +func TestMalwareBazaarSource_Disabled(t *testing.T) { + s := &malwareBazaarSource{endpoint: "http://nope"} + r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_malwarebazaar": false})[0] + if r.Enabled { + t.Errorf("expected disabled, got %+v", r) + } +} + +func TestMalwareBazaarSource_NoAuthKey(t *testing.T) { + s := &malwareBazaarSource{endpoint: "http://nope"} + r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_malwarebazaar": true})[0] + if r.Enabled { + t.Errorf("expected disabled when no auth key, got %+v", r) + } +} diff --git a/checker/threatfox.go b/checker/threatfox.go new file mode 100644 index 0000000..67fae49 --- /dev/null +++ b/checker/threatfox.go @@ -0,0 +1,155 @@ +package checker + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "time" + + sdk "git.happydns.org/checker-sdk-go/checker" +) + +const threatFoxEndpoint = "https://threatfox-api.abuse.ch/api/v1/" + +func init() { Register(&threatFoxSource{endpoint: threatFoxEndpoint}) } + +type threatFoxSource struct { + endpoint string +} + +func (*threatFoxSource) ID() string { return "threatfox" } +func (*threatFoxSource) Name() string { return "abuse.ch ThreatFox" } + +func (*threatFoxSource) Options() SourceOptions { + return SourceOptions{ + Admin: []sdk.CheckerOptionField{ + { + Id: "threatfox_auth_key", + Type: "string", + Label: "ThreatFox Auth-Key", + Description: "abuse.ch ThreatFox Auth-Key (free, requires an abuse.ch account). Without this key the source is disabled.", + Secret: true, + }, + }, + User: []sdk.CheckerOptionField{ + { + Id: "enable_threatfox", + Type: "bool", + Label: "Use abuse.ch ThreatFox", + Description: "Query ThreatFox for indicators of compromise (C2 servers, malware, phishing) associated with the domain.", + Default: true, + }, + }, + } +} + +func (s *threatFoxSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult { + authKey := stringOpt(opts, "threatfox_auth_key") + if !sdk.GetBoolOption(opts, "enable_threatfox", true) || registered == "" || authKey == "" { + return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: false}} + } + + res := SourceResult{ + SourceID: s.ID(), SourceName: s.Name(), Enabled: true, + Reference: "https://threatfox.abuse.ch/browse/", + } + + buf, err := json.Marshal(map[string]any{ + "query": "search_ioc", + "search_term": registered, + }) + if err != nil { + res.Error = err.Error() + return []SourceResult{res} + } + + reqCtx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, s.endpoint, bytes.NewReader(buf)) + if err != nil { + res.Error = err.Error() + return []SourceResult{res} + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Auth-Key", authKey) + + body, status, err := httpDo(req, 4<<20) + if err != nil { + res.Error = err.Error() + return []SourceResult{res} + } + if status != http.StatusOK { + res.Error = fmt.Sprintf("HTTP %d: %s", status, truncate(string(body), 200)) + return []SourceResult{res} + } + + var parsed struct { + QueryStatus string `json:"query_status"` + Data []struct { + IOCValue string `json:"ioc_value"` + IOCType string `json:"ioc_type"` + ThreatType string `json:"threat_type"` + MalwarePrintable string `json:"malware_printable"` + ConfidenceLevel int `json:"confidence_level"` + FirstSeen string `json:"first_seen"` + LastSeen string `json:"last_seen"` + } `json:"data"` + } + if err := json.Unmarshal(body, &parsed); err != nil { + res.Error = "decode: " + err.Error() + return []SourceResult{res} + } + + switch parsed.QueryStatus { + case "ok": + threats := map[string]bool{} + for _, ioc := range parsed.Data { + label := ioc.MalwarePrintable + if label == "" { + label = ioc.ThreatType + } + if label != "" && !threats[label] { + threats[label] = true + res.Reasons = append(res.Reasons, label) + } + res.Evidence = append(res.Evidence, Evidence{ + Label: "IOC", + Value: ioc.IOCValue, + Status: ioc.ThreatType, + Extra: map[string]string{ + "malware": ioc.MalwarePrintable, + "confidence": fmt.Sprintf("%d%%", ioc.ConfidenceLevel), + "first_seen": ioc.FirstSeen, + }, + }) + } + case "no_result": + // Clean. + default: + res.Error = "query_status=" + parsed.QueryStatus + } + return []SourceResult{res} +} + +func (*threatFoxSource) Evaluate(r SourceResult) (bool, string) { + if r.Enabled && r.Error == "" && len(r.Evidence) > 0 { + return true, SeverityCrit + } + return false, "" +} + +func (*threatFoxSource) Diagnose(res SourceResult) Diagnosis { + return Diagnosis{ + Severity: SeverityCrit, + Title: "Listed in abuse.ch ThreatFox as a threat IOC", + Detail: fmt.Sprintf( + "%d IOC(s) match this domain; threat(s): %s. ThreatFox tracks indicators of compromise including C2 servers, malware distribution hosts, and phishing infrastructure. Treat the host as compromised or abused: investigate recent DNS changes, audit hosted content, then request removal through the ThreatFox reference page.", + len(res.Evidence), joinNonEmpty(res.Reasons, ", "), + ), + Fix: res.Reference, + FixIsURL: res.Reference != "", + } +} diff --git a/checker/threatfox_test.go b/checker/threatfox_test.go new file mode 100644 index 0000000..5e5f12f --- /dev/null +++ b/checker/threatfox_test.go @@ -0,0 +1,96 @@ +package checker + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + sdk "git.happydns.org/checker-sdk-go/checker" +) + +func TestThreatFoxSource_NoResult(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"query_status":"no_result","data":[]}`)) + })) + defer srv.Close() + + s := &threatFoxSource{endpoint: srv.URL} + results := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_threatfox": true, "threatfox_auth_key": "k"}) + if len(results) != 1 { + t.Fatalf("expected 1 result, got %d", len(results)) + } + r := results[0] + listed, _ := s.Evaluate(r) + if !r.Enabled || listed || r.Error != "" { + t.Fatalf("expected enabled+clean, got %+v, Evaluate listed=%v", r, listed) + } +} + +func TestThreatFoxSource_Listed(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Auth-Key") == "" { + t.Errorf("missing Auth-Key header") + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "query_status": "ok", + "data": [{ + "ioc_value": "example.com:443", + "ioc_type": "domain", + "threat_type": "botnet_cc", + "malware_printable": "Emotet", + "confidence_level": 75, + "first_seen": "2024-01-01 00:00:00 UTC", + "last_seen": "2024-06-01 00:00:00 UTC" + }] + }`)) + })) + defer srv.Close() + + s := &threatFoxSource{endpoint: srv.URL} + r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_threatfox": true, "threatfox_auth_key": "k"})[0] + if len(r.Evidence) != 1 { + t.Fatalf("expected 1 evidence item, got %+v", r) + } + if listed, severity := s.Evaluate(r); !listed || severity != SeverityCrit { + t.Errorf("expected Evaluate()=(true, crit), got (%v, %q)", listed, severity) + } + if r.Evidence[0].Value != "example.com:443" { + t.Errorf("evidence value = %q", r.Evidence[0].Value) + } + if len(r.Reasons) != 1 || r.Reasons[0] != "Emotet" { + t.Errorf("reasons = %v", r.Reasons) + } +} + +func TestThreatFoxSource_HTTPError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusUnauthorized) + _, _ = w.Write([]byte("unauthorized")) + })) + defer srv.Close() + + s := &threatFoxSource{endpoint: srv.URL} + r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_threatfox": true, "threatfox_auth_key": "k"})[0] + if r.Error == "" { + t.Errorf("expected error, got %+v", r) + } +} + +func TestThreatFoxSource_Disabled(t *testing.T) { + s := &threatFoxSource{endpoint: "http://nope"} + r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_threatfox": false})[0] + if r.Enabled { + t.Errorf("expected disabled, got %+v", r) + } +} + +func TestThreatFoxSource_NoAuthKey(t *testing.T) { + s := &threatFoxSource{endpoint: "http://nope"} + r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_threatfox": true})[0] + if r.Enabled { + t.Errorf("expected disabled when no auth key, got %+v", r) + } +}