happyDomain/checker-blacklist
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add abuse.ch ThreatFox and MalwareBazaar blacklist sources

Browse source 
ThreatFox queries the IOC database for domain indicators (C2 servers,
malware distribution, phishing); MalwareBazaar searches for malware
samples tagged with the domain. Both require a free abuse.ch Auth-Key.
This commit is contained in:
nemunaire 2026-05-15 18:30:46 +08:00
commit 229e7a8f02
5 changed files with 496 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

2
README.md
View file

@ -15,6 +15,8 @@ widely-used reputation systems.
| OpenPhish public feed | downloaded list | no | user (default on) | | OpenPhish public feed | downloaded list | no | user (default on) |
| PhishTank | downloaded list | no | user (default on) | | PhishTank | downloaded list | no | user (default on) |
| abuse.ch URLhaus | HTTPS lookup | free Auth-Key (admin) | user (default on) | | abuse.ch URLhaus | HTTPS lookup | free Auth-Key (admin) | user (default on) |
| abuse.ch ThreatFox | HTTPS lookup | free Auth-Key (admin) | user (default on) |
| abuse.ch MalwareBazaar| HTTPS lookup | free Auth-Key (admin) | user (default on) |
| VirusTotal v3 | HTTPS lookup | yes (admin) | admin | | VirusTotal v3 | HTTPS lookup | yes (admin) | admin |
### Obtaining API keys ### Obtaining API keys

149
checker/malwarebazaar.go Normal file
View file

@ -0,0 +1,149 @@
package checker
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"
"time"
sdk "git.happydns.org/checker-sdk-go/checker"
)
const malwareBazaarEndpoint = "https://mb-api.abuse.ch/api/v1/"
func init() { Register(&malwareBazaarSource{endpoint: malwareBazaarEndpoint}) }
type malwareBazaarSource struct {
endpoint string
}
func (*malwareBazaarSource) ID() string { return "malwarebazaar" }
func (*malwareBazaarSource) Name() string { return "abuse.ch MalwareBazaar" }
func (*malwareBazaarSource) Options() SourceOptions {
return SourceOptions{
Admin: []sdk.CheckerOptionField{
{
Id: "malwarebazaar_auth_key",
Type: "string",
Label: "MalwareBazaar Auth-Key",
Description: "abuse.ch MalwareBazaar Auth-Key (free, requires an abuse.ch account). Without this key the source is disabled.",
Secret: true,
},
},
User: []sdk.CheckerOptionField{
{
Id: "enable_malwarebazaar",
Type: "bool",
Label: "Use abuse.ch MalwareBazaar",
Description: "Search MalwareBazaar for malware samples tagged with the domain (typically C2 infrastructure or delivery hosts).",
Default: true,
},
},
}
}
func (s *malwareBazaarSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult {
authKey := stringOpt(opts, "malwarebazaar_auth_key")
if !sdk.GetBoolOption(opts, "enable_malwarebazaar", true) || registered == "" || authKey == "" {
return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: false}}
}
res := SourceResult{
SourceID: s.ID(), SourceName: s.Name(), Enabled: true,
Reference: "https://bazaar.abuse.ch/browse/",
}
buf, err := json.Marshal(map[string]any{
"query": "search_tag",
"tag": registered,
})
if err != nil {
res.Error = err.Error()
return []SourceResult{res}
}
reqCtx, cancel := context.WithTimeout(ctx, 15*time.Second)
defer cancel()
req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, s.endpoint, bytes.NewReader(buf))
if err != nil {
res.Error = err.Error()
return []SourceResult{res}
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Auth-Key", authKey)
body, status, err := httpDo(req, 4<<20)
if err != nil {
res.Error = err.Error()
return []SourceResult{res}
}
if status != http.StatusOK {
res.Error = fmt.Sprintf("HTTP %d: %s", status, truncate(string(body), 200))
return []SourceResult{res}
}
var parsed struct {
QueryStatus string `json:"query_status"`
Data []struct {
SHA256Hash string `json:"sha256_hash"`
FileName string `json:"file_name"`
FileType string `json:"file_type_mime"`
Signature string `json:"signature"`
FirstSeen string `json:"first_seen"`
} `json:"data"`
}
if err := json.Unmarshal(body, &parsed); err != nil {
res.Error = "decode: " + err.Error()
return []SourceResult{res}
}
switch parsed.QueryStatus {
case "ok":
signatures := map[string]bool{}
for _, sample := range parsed.Data {
if sample.Signature != "" && !signatures[sample.Signature] {
signatures[sample.Signature] = true
res.Reasons = append(res.Reasons, sample.Signature)
}
res.Evidence = append(res.Evidence, Evidence{
Label: "Sample",
Value: sample.SHA256Hash,
Status: sample.FileType,
Extra: map[string]string{
"filename": sample.FileName,
"signature": sample.Signature,
"first_seen": sample.FirstSeen,
},
})
}
case "no_results", "illegal_search_term":
// Clean.
default:
res.Error = "query_status=" + parsed.QueryStatus
}
return []SourceResult{res}
}
func (*malwareBazaarSource) Evaluate(r SourceResult) (bool, string) {
if r.Enabled && r.Error == "" && len(r.Evidence) > 0 {
return true, SeverityWarn
}
return false, ""
}
func (*malwareBazaarSource) Diagnose(res SourceResult) Diagnosis {
return Diagnosis{
Severity: SeverityWarn,
Title: "Associated with malware samples in abuse.ch MalwareBazaar",
Detail: fmt.Sprintf(
"%d malware sample(s) are tagged with this domain; malware family/signature(s): %s. MalwareBazaar samples are tagged with their C2 domain or delivery host when known. Investigate the domain's hosting and DNS records to identify and remove malicious infrastructure.",
len(res.Evidence), joinNonEmpty(res.Reasons, ", "),
),
Fix: res.Reference,
FixIsURL: res.Reference != "",
}
}

94
checker/malwarebazaar_test.go Normal file
View file

@ -0,0 +1,94 @@
package checker
import (
"context"
"net/http"
"net/http/httptest"
"testing"
sdk "git.happydns.org/checker-sdk-go/checker"
)
func TestMalwareBazaarSource_NoResults(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"query_status":"no_results"}`))
}))
defer srv.Close()
s := &malwareBazaarSource{endpoint: srv.URL}
results := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_malwarebazaar": true, "malwarebazaar_auth_key": "k"})
if len(results) != 1 {
t.Fatalf("expected 1 result, got %d", len(results))
}
r := results[0]
listed, _ := s.Evaluate(r)
if !r.Enabled || listed || r.Error != "" {
t.Fatalf("expected enabled+clean, got %+v, Evaluate listed=%v", r, listed)
}
}
func TestMalwareBazaarSource_Listed(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Auth-Key") == "" {
t.Errorf("missing Auth-Key header")
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{
"query_status": "ok",
"data": [{
"sha256_hash": "aaaa1111bbbb2222cccc3333dddd4444eeee5555ffff6666aaaa1111bbbb2222",
"file_name": "evil.exe",
"file_type_mime": "application/x-dosexec",
"signature": "Emotet",
"first_seen": "2024-01-01 00:00:00"
}]
}`))
}))
defer srv.Close()
s := &malwareBazaarSource{endpoint: srv.URL}
r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_malwarebazaar": true, "malwarebazaar_auth_key": "k"})[0]
if len(r.Evidence) != 1 {
t.Fatalf("expected 1 evidence item, got %+v", r)
}
if listed, severity := s.Evaluate(r); !listed || severity != SeverityWarn {
t.Errorf("expected Evaluate()=(true, warn), got (%v, %q)", listed, severity)
}
if r.Evidence[0].Status != "application/x-dosexec" {
t.Errorf("evidence status = %q", r.Evidence[0].Status)
}
if len(r.Reasons) != 1 || r.Reasons[0] != "Emotet" {
t.Errorf("reasons = %v", r.Reasons)
}
}
func TestMalwareBazaarSource_HTTPError(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusUnauthorized)
_, _ = w.Write([]byte("unauthorized"))
}))
defer srv.Close()
s := &malwareBazaarSource{endpoint: srv.URL}
r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_malwarebazaar": true, "malwarebazaar_auth_key": "k"})[0]
if r.Error == "" {
t.Errorf("expected error, got %+v", r)
}
}
func TestMalwareBazaarSource_Disabled(t *testing.T) {
s := &malwareBazaarSource{endpoint: "http://nope"}
r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_malwarebazaar": false})[0]
if r.Enabled {
t.Errorf("expected disabled, got %+v", r)
}
}
func TestMalwareBazaarSource_NoAuthKey(t *testing.T) {
s := &malwareBazaarSource{endpoint: "http://nope"}
r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_malwarebazaar": true})[0]
if r.Enabled {
t.Errorf("expected disabled when no auth key, got %+v", r)
}
}

155
checker/threatfox.go Normal file
View file

@ -0,0 +1,155 @@
package checker
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"
"time"
sdk "git.happydns.org/checker-sdk-go/checker"
)
const threatFoxEndpoint = "https://threatfox-api.abuse.ch/api/v1/"
func init() { Register(&threatFoxSource{endpoint: threatFoxEndpoint}) }
type threatFoxSource struct {
endpoint string
}
func (*threatFoxSource) ID() string { return "threatfox" }
func (*threatFoxSource) Name() string { return "abuse.ch ThreatFox" }
func (*threatFoxSource) Options() SourceOptions {
return SourceOptions{
Admin: []sdk.CheckerOptionField{
{
Id: "threatfox_auth_key",
Type: "string",
Label: "ThreatFox Auth-Key",
Description: "abuse.ch ThreatFox Auth-Key (free, requires an abuse.ch account). Without this key the source is disabled.",
Secret: true,
},
},
User: []sdk.CheckerOptionField{
{
Id: "enable_threatfox",
Type: "bool",
Label: "Use abuse.ch ThreatFox",
Description: "Query ThreatFox for indicators of compromise (C2 servers, malware, phishing) associated with the domain.",
Default: true,
},
},
}
}
func (s *threatFoxSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult {
authKey := stringOpt(opts, "threatfox_auth_key")
if !sdk.GetBoolOption(opts, "enable_threatfox", true) || registered == "" || authKey == "" {
return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: false}}
}
res := SourceResult{
SourceID: s.ID(), SourceName: s.Name(), Enabled: true,
Reference: "https://threatfox.abuse.ch/browse/",
}
buf, err := json.Marshal(map[string]any{
"query": "search_ioc",
"search_term": registered,
})
if err != nil {
res.Error = err.Error()
return []SourceResult{res}
}
reqCtx, cancel := context.WithTimeout(ctx, 15*time.Second)
defer cancel()
req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, s.endpoint, bytes.NewReader(buf))
if err != nil {
res.Error = err.Error()
return []SourceResult{res}
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Auth-Key", authKey)
body, status, err := httpDo(req, 4<<20)
if err != nil {
res.Error = err.Error()
return []SourceResult{res}
}
if status != http.StatusOK {
res.Error = fmt.Sprintf("HTTP %d: %s", status, truncate(string(body), 200))
return []SourceResult{res}
}
var parsed struct {
QueryStatus string `json:"query_status"`
Data []struct {
IOCValue string `json:"ioc_value"`
IOCType string `json:"ioc_type"`
ThreatType string `json:"threat_type"`
MalwarePrintable string `json:"malware_printable"`
ConfidenceLevel int `json:"confidence_level"`
FirstSeen string `json:"first_seen"`
LastSeen string `json:"last_seen"`
} `json:"data"`
}
if err := json.Unmarshal(body, &parsed); err != nil {
res.Error = "decode: " + err.Error()
return []SourceResult{res}
}
switch parsed.QueryStatus {
case "ok":
threats := map[string]bool{}
for _, ioc := range parsed.Data {
label := ioc.MalwarePrintable
if label == "" {
label = ioc.ThreatType
}
if label != "" && !threats[label] {
threats[label] = true
res.Reasons = append(res.Reasons, label)
}
res.Evidence = append(res.Evidence, Evidence{
Label: "IOC",
Value: ioc.IOCValue,
Status: ioc.ThreatType,
Extra: map[string]string{
"malware": ioc.MalwarePrintable,
"confidence": fmt.Sprintf("%d%%", ioc.ConfidenceLevel),
"first_seen": ioc.FirstSeen,
},
})
}
case "no_result":
// Clean.
default:
res.Error = "query_status=" + parsed.QueryStatus
}
return []SourceResult{res}
}
func (*threatFoxSource) Evaluate(r SourceResult) (bool, string) {
if r.Enabled && r.Error == "" && len(r.Evidence) > 0 {
return true, SeverityCrit
}
return false, ""
}
func (*threatFoxSource) Diagnose(res SourceResult) Diagnosis {
return Diagnosis{
Severity: SeverityCrit,
Title: "Listed in abuse.ch ThreatFox as a threat IOC",
Detail: fmt.Sprintf(
"%d IOC(s) match this domain; threat(s): %s. ThreatFox tracks indicators of compromise including C2 servers, malware distribution hosts, and phishing infrastructure. Treat the host as compromised or abused: investigate recent DNS changes, audit hosted content, then request removal through the ThreatFox reference page.",
len(res.Evidence), joinNonEmpty(res.Reasons, ", "),
),
Fix: res.Reference,
FixIsURL: res.Reference != "",
}
}

96
checker/threatfox_test.go Normal file
View file

@ -0,0 +1,96 @@
package checker
import (
"context"
"net/http"
"net/http/httptest"
"testing"
sdk "git.happydns.org/checker-sdk-go/checker"
)
func TestThreatFoxSource_NoResult(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"query_status":"no_result","data":[]}`))
}))
defer srv.Close()
s := &threatFoxSource{endpoint: srv.URL}
results := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_threatfox": true, "threatfox_auth_key": "k"})
if len(results) != 1 {
t.Fatalf("expected 1 result, got %d", len(results))
}
r := results[0]
listed, _ := s.Evaluate(r)
if !r.Enabled || listed || r.Error != "" {
t.Fatalf("expected enabled+clean, got %+v, Evaluate listed=%v", r, listed)
}
}
func TestThreatFoxSource_Listed(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Auth-Key") == "" {
t.Errorf("missing Auth-Key header")
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{
"query_status": "ok",
"data": [{
"ioc_value": "example.com:443",
"ioc_type": "domain",
"threat_type": "botnet_cc",
"malware_printable": "Emotet",
"confidence_level": 75,
"first_seen": "2024-01-01 00:00:00 UTC",
"last_seen": "2024-06-01 00:00:00 UTC"
}]
}`))
}))
defer srv.Close()
s := &threatFoxSource{endpoint: srv.URL}
r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_threatfox": true, "threatfox_auth_key": "k"})[0]
if len(r.Evidence) != 1 {
t.Fatalf("expected 1 evidence item, got %+v", r)
}
if listed, severity := s.Evaluate(r); !listed || severity != SeverityCrit {
t.Errorf("expected Evaluate()=(true, crit), got (%v, %q)", listed, severity)
}
if r.Evidence[0].Value != "example.com:443" {
t.Errorf("evidence value = %q", r.Evidence[0].Value)
}
if len(r.Reasons) != 1 || r.Reasons[0] != "Emotet" {
t.Errorf("reasons = %v", r.Reasons)
}
}
func TestThreatFoxSource_HTTPError(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusUnauthorized)
_, _ = w.Write([]byte("unauthorized"))
}))
defer srv.Close()
s := &threatFoxSource{endpoint: srv.URL}
r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_threatfox": true, "threatfox_auth_key": "k"})[0]
if r.Error == "" {
t.Errorf("expected error, got %+v", r)
}
}
func TestThreatFoxSource_Disabled(t *testing.T) {
s := &threatFoxSource{endpoint: "http://nope"}
r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_threatfox": false})[0]
if r.Enabled {
t.Errorf("expected disabled, got %+v", r)
}
}
func TestThreatFoxSource_NoAuthKey(t *testing.T) {
s := &threatFoxSource{endpoint: "http://nope"}
r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_threatfox": true})[0]
if r.Enabled {
t.Errorf("expected disabled when no auth key, got %+v", r)
}
}