Add abuse.ch ThreatFox and MalwareBazaar blacklist sources
ThreatFox queries the IOC database for domain indicators (C2 servers, malware distribution, phishing); MalwareBazaar searches for malware samples tagged with the domain. Both require a free abuse.ch Auth-Key.
This commit is contained in:
parent
6b08676ec5
commit
229e7a8f02
5 changed files with 496 additions and 0 deletions
|
|
@ -15,6 +15,8 @@ widely-used reputation systems.
|
|||
| OpenPhish public feed | downloaded list | no | user (default on) |
|
||||
| PhishTank | downloaded list | no | user (default on) |
|
||||
| abuse.ch URLhaus | HTTPS lookup | free Auth-Key (admin) | user (default on) |
|
||||
| abuse.ch ThreatFox | HTTPS lookup | free Auth-Key (admin) | user (default on) |
|
||||
| abuse.ch MalwareBazaar| HTTPS lookup | free Auth-Key (admin) | user (default on) |
|
||||
| VirusTotal v3 | HTTPS lookup | yes (admin) | admin |
|
||||
|
||||
### Obtaining API keys
|
||||
|
|
|
|||
149
checker/malwarebazaar.go
Normal file
149
checker/malwarebazaar.go
Normal file
|
|
@ -0,0 +1,149 @@
|
|||
package checker
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
sdk "git.happydns.org/checker-sdk-go/checker"
|
||||
)
|
||||
|
||||
const malwareBazaarEndpoint = "https://mb-api.abuse.ch/api/v1/"
|
||||
|
||||
func init() { Register(&malwareBazaarSource{endpoint: malwareBazaarEndpoint}) }
|
||||
|
||||
type malwareBazaarSource struct {
|
||||
endpoint string
|
||||
}
|
||||
|
||||
func (*malwareBazaarSource) ID() string { return "malwarebazaar" }
|
||||
func (*malwareBazaarSource) Name() string { return "abuse.ch MalwareBazaar" }
|
||||
|
||||
func (*malwareBazaarSource) Options() SourceOptions {
|
||||
return SourceOptions{
|
||||
Admin: []sdk.CheckerOptionField{
|
||||
{
|
||||
Id: "malwarebazaar_auth_key",
|
||||
Type: "string",
|
||||
Label: "MalwareBazaar Auth-Key",
|
||||
Description: "abuse.ch MalwareBazaar Auth-Key (free, requires an abuse.ch account). Without this key the source is disabled.",
|
||||
Secret: true,
|
||||
},
|
||||
},
|
||||
User: []sdk.CheckerOptionField{
|
||||
{
|
||||
Id: "enable_malwarebazaar",
|
||||
Type: "bool",
|
||||
Label: "Use abuse.ch MalwareBazaar",
|
||||
Description: "Search MalwareBazaar for malware samples tagged with the domain (typically C2 infrastructure or delivery hosts).",
|
||||
Default: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (s *malwareBazaarSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult {
|
||||
authKey := stringOpt(opts, "malwarebazaar_auth_key")
|
||||
if !sdk.GetBoolOption(opts, "enable_malwarebazaar", true) || registered == "" || authKey == "" {
|
||||
return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: false}}
|
||||
}
|
||||
|
||||
res := SourceResult{
|
||||
SourceID: s.ID(), SourceName: s.Name(), Enabled: true,
|
||||
Reference: "https://bazaar.abuse.ch/browse/",
|
||||
}
|
||||
|
||||
buf, err := json.Marshal(map[string]any{
|
||||
"query": "search_tag",
|
||||
"tag": registered,
|
||||
})
|
||||
if err != nil {
|
||||
res.Error = err.Error()
|
||||
return []SourceResult{res}
|
||||
}
|
||||
|
||||
reqCtx, cancel := context.WithTimeout(ctx, 15*time.Second)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, s.endpoint, bytes.NewReader(buf))
|
||||
if err != nil {
|
||||
res.Error = err.Error()
|
||||
return []SourceResult{res}
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Auth-Key", authKey)
|
||||
|
||||
body, status, err := httpDo(req, 4<<20)
|
||||
if err != nil {
|
||||
res.Error = err.Error()
|
||||
return []SourceResult{res}
|
||||
}
|
||||
if status != http.StatusOK {
|
||||
res.Error = fmt.Sprintf("HTTP %d: %s", status, truncate(string(body), 200))
|
||||
return []SourceResult{res}
|
||||
}
|
||||
|
||||
var parsed struct {
|
||||
QueryStatus string `json:"query_status"`
|
||||
Data []struct {
|
||||
SHA256Hash string `json:"sha256_hash"`
|
||||
FileName string `json:"file_name"`
|
||||
FileType string `json:"file_type_mime"`
|
||||
Signature string `json:"signature"`
|
||||
FirstSeen string `json:"first_seen"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &parsed); err != nil {
|
||||
res.Error = "decode: " + err.Error()
|
||||
return []SourceResult{res}
|
||||
}
|
||||
|
||||
switch parsed.QueryStatus {
|
||||
case "ok":
|
||||
signatures := map[string]bool{}
|
||||
for _, sample := range parsed.Data {
|
||||
if sample.Signature != "" && !signatures[sample.Signature] {
|
||||
signatures[sample.Signature] = true
|
||||
res.Reasons = append(res.Reasons, sample.Signature)
|
||||
}
|
||||
res.Evidence = append(res.Evidence, Evidence{
|
||||
Label: "Sample",
|
||||
Value: sample.SHA256Hash,
|
||||
Status: sample.FileType,
|
||||
Extra: map[string]string{
|
||||
"filename": sample.FileName,
|
||||
"signature": sample.Signature,
|
||||
"first_seen": sample.FirstSeen,
|
||||
},
|
||||
})
|
||||
}
|
||||
case "no_results", "illegal_search_term":
|
||||
// Clean.
|
||||
default:
|
||||
res.Error = "query_status=" + parsed.QueryStatus
|
||||
}
|
||||
return []SourceResult{res}
|
||||
}
|
||||
|
||||
func (*malwareBazaarSource) Evaluate(r SourceResult) (bool, string) {
|
||||
if r.Enabled && r.Error == "" && len(r.Evidence) > 0 {
|
||||
return true, SeverityWarn
|
||||
}
|
||||
return false, ""
|
||||
}
|
||||
|
||||
func (*malwareBazaarSource) Diagnose(res SourceResult) Diagnosis {
|
||||
return Diagnosis{
|
||||
Severity: SeverityWarn,
|
||||
Title: "Associated with malware samples in abuse.ch MalwareBazaar",
|
||||
Detail: fmt.Sprintf(
|
||||
"%d malware sample(s) are tagged with this domain; malware family/signature(s): %s. MalwareBazaar samples are tagged with their C2 domain or delivery host when known. Investigate the domain's hosting and DNS records to identify and remove malicious infrastructure.",
|
||||
len(res.Evidence), joinNonEmpty(res.Reasons, ", "),
|
||||
),
|
||||
Fix: res.Reference,
|
||||
FixIsURL: res.Reference != "",
|
||||
}
|
||||
}
|
||||
94
checker/malwarebazaar_test.go
Normal file
94
checker/malwarebazaar_test.go
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
package checker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
sdk "git.happydns.org/checker-sdk-go/checker"
|
||||
)
|
||||
|
||||
func TestMalwareBazaarSource_NoResults(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"query_status":"no_results"}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
s := &malwareBazaarSource{endpoint: srv.URL}
|
||||
results := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_malwarebazaar": true, "malwarebazaar_auth_key": "k"})
|
||||
if len(results) != 1 {
|
||||
t.Fatalf("expected 1 result, got %d", len(results))
|
||||
}
|
||||
r := results[0]
|
||||
listed, _ := s.Evaluate(r)
|
||||
if !r.Enabled || listed || r.Error != "" {
|
||||
t.Fatalf("expected enabled+clean, got %+v, Evaluate listed=%v", r, listed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMalwareBazaarSource_Listed(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Auth-Key") == "" {
|
||||
t.Errorf("missing Auth-Key header")
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{
|
||||
"query_status": "ok",
|
||||
"data": [{
|
||||
"sha256_hash": "aaaa1111bbbb2222cccc3333dddd4444eeee5555ffff6666aaaa1111bbbb2222",
|
||||
"file_name": "evil.exe",
|
||||
"file_type_mime": "application/x-dosexec",
|
||||
"signature": "Emotet",
|
||||
"first_seen": "2024-01-01 00:00:00"
|
||||
}]
|
||||
}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
s := &malwareBazaarSource{endpoint: srv.URL}
|
||||
r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_malwarebazaar": true, "malwarebazaar_auth_key": "k"})[0]
|
||||
if len(r.Evidence) != 1 {
|
||||
t.Fatalf("expected 1 evidence item, got %+v", r)
|
||||
}
|
||||
if listed, severity := s.Evaluate(r); !listed || severity != SeverityWarn {
|
||||
t.Errorf("expected Evaluate()=(true, warn), got (%v, %q)", listed, severity)
|
||||
}
|
||||
if r.Evidence[0].Status != "application/x-dosexec" {
|
||||
t.Errorf("evidence status = %q", r.Evidence[0].Status)
|
||||
}
|
||||
if len(r.Reasons) != 1 || r.Reasons[0] != "Emotet" {
|
||||
t.Errorf("reasons = %v", r.Reasons)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMalwareBazaarSource_HTTPError(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
_, _ = w.Write([]byte("unauthorized"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
s := &malwareBazaarSource{endpoint: srv.URL}
|
||||
r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_malwarebazaar": true, "malwarebazaar_auth_key": "k"})[0]
|
||||
if r.Error == "" {
|
||||
t.Errorf("expected error, got %+v", r)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMalwareBazaarSource_Disabled(t *testing.T) {
|
||||
s := &malwareBazaarSource{endpoint: "http://nope"}
|
||||
r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_malwarebazaar": false})[0]
|
||||
if r.Enabled {
|
||||
t.Errorf("expected disabled, got %+v", r)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMalwareBazaarSource_NoAuthKey(t *testing.T) {
|
||||
s := &malwareBazaarSource{endpoint: "http://nope"}
|
||||
r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_malwarebazaar": true})[0]
|
||||
if r.Enabled {
|
||||
t.Errorf("expected disabled when no auth key, got %+v", r)
|
||||
}
|
||||
}
|
||||
155
checker/threatfox.go
Normal file
155
checker/threatfox.go
Normal file
|
|
@ -0,0 +1,155 @@
|
|||
package checker
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
sdk "git.happydns.org/checker-sdk-go/checker"
|
||||
)
|
||||
|
||||
const threatFoxEndpoint = "https://threatfox-api.abuse.ch/api/v1/"
|
||||
|
||||
func init() { Register(&threatFoxSource{endpoint: threatFoxEndpoint}) }
|
||||
|
||||
type threatFoxSource struct {
|
||||
endpoint string
|
||||
}
|
||||
|
||||
func (*threatFoxSource) ID() string { return "threatfox" }
|
||||
func (*threatFoxSource) Name() string { return "abuse.ch ThreatFox" }
|
||||
|
||||
func (*threatFoxSource) Options() SourceOptions {
|
||||
return SourceOptions{
|
||||
Admin: []sdk.CheckerOptionField{
|
||||
{
|
||||
Id: "threatfox_auth_key",
|
||||
Type: "string",
|
||||
Label: "ThreatFox Auth-Key",
|
||||
Description: "abuse.ch ThreatFox Auth-Key (free, requires an abuse.ch account). Without this key the source is disabled.",
|
||||
Secret: true,
|
||||
},
|
||||
},
|
||||
User: []sdk.CheckerOptionField{
|
||||
{
|
||||
Id: "enable_threatfox",
|
||||
Type: "bool",
|
||||
Label: "Use abuse.ch ThreatFox",
|
||||
Description: "Query ThreatFox for indicators of compromise (C2 servers, malware, phishing) associated with the domain.",
|
||||
Default: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (s *threatFoxSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult {
|
||||
authKey := stringOpt(opts, "threatfox_auth_key")
|
||||
if !sdk.GetBoolOption(opts, "enable_threatfox", true) || registered == "" || authKey == "" {
|
||||
return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: false}}
|
||||
}
|
||||
|
||||
res := SourceResult{
|
||||
SourceID: s.ID(), SourceName: s.Name(), Enabled: true,
|
||||
Reference: "https://threatfox.abuse.ch/browse/",
|
||||
}
|
||||
|
||||
buf, err := json.Marshal(map[string]any{
|
||||
"query": "search_ioc",
|
||||
"search_term": registered,
|
||||
})
|
||||
if err != nil {
|
||||
res.Error = err.Error()
|
||||
return []SourceResult{res}
|
||||
}
|
||||
|
||||
reqCtx, cancel := context.WithTimeout(ctx, 15*time.Second)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, s.endpoint, bytes.NewReader(buf))
|
||||
if err != nil {
|
||||
res.Error = err.Error()
|
||||
return []SourceResult{res}
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Auth-Key", authKey)
|
||||
|
||||
body, status, err := httpDo(req, 4<<20)
|
||||
if err != nil {
|
||||
res.Error = err.Error()
|
||||
return []SourceResult{res}
|
||||
}
|
||||
if status != http.StatusOK {
|
||||
res.Error = fmt.Sprintf("HTTP %d: %s", status, truncate(string(body), 200))
|
||||
return []SourceResult{res}
|
||||
}
|
||||
|
||||
var parsed struct {
|
||||
QueryStatus string `json:"query_status"`
|
||||
Data []struct {
|
||||
IOCValue string `json:"ioc_value"`
|
||||
IOCType string `json:"ioc_type"`
|
||||
ThreatType string `json:"threat_type"`
|
||||
MalwarePrintable string `json:"malware_printable"`
|
||||
ConfidenceLevel int `json:"confidence_level"`
|
||||
FirstSeen string `json:"first_seen"`
|
||||
LastSeen string `json:"last_seen"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &parsed); err != nil {
|
||||
res.Error = "decode: " + err.Error()
|
||||
return []SourceResult{res}
|
||||
}
|
||||
|
||||
switch parsed.QueryStatus {
|
||||
case "ok":
|
||||
threats := map[string]bool{}
|
||||
for _, ioc := range parsed.Data {
|
||||
label := ioc.MalwarePrintable
|
||||
if label == "" {
|
||||
label = ioc.ThreatType
|
||||
}
|
||||
if label != "" && !threats[label] {
|
||||
threats[label] = true
|
||||
res.Reasons = append(res.Reasons, label)
|
||||
}
|
||||
res.Evidence = append(res.Evidence, Evidence{
|
||||
Label: "IOC",
|
||||
Value: ioc.IOCValue,
|
||||
Status: ioc.ThreatType,
|
||||
Extra: map[string]string{
|
||||
"malware": ioc.MalwarePrintable,
|
||||
"confidence": fmt.Sprintf("%d%%", ioc.ConfidenceLevel),
|
||||
"first_seen": ioc.FirstSeen,
|
||||
},
|
||||
})
|
||||
}
|
||||
case "no_result":
|
||||
// Clean.
|
||||
default:
|
||||
res.Error = "query_status=" + parsed.QueryStatus
|
||||
}
|
||||
return []SourceResult{res}
|
||||
}
|
||||
|
||||
func (*threatFoxSource) Evaluate(r SourceResult) (bool, string) {
|
||||
if r.Enabled && r.Error == "" && len(r.Evidence) > 0 {
|
||||
return true, SeverityCrit
|
||||
}
|
||||
return false, ""
|
||||
}
|
||||
|
||||
func (*threatFoxSource) Diagnose(res SourceResult) Diagnosis {
|
||||
return Diagnosis{
|
||||
Severity: SeverityCrit,
|
||||
Title: "Listed in abuse.ch ThreatFox as a threat IOC",
|
||||
Detail: fmt.Sprintf(
|
||||
"%d IOC(s) match this domain; threat(s): %s. ThreatFox tracks indicators of compromise including C2 servers, malware distribution hosts, and phishing infrastructure. Treat the host as compromised or abused: investigate recent DNS changes, audit hosted content, then request removal through the ThreatFox reference page.",
|
||||
len(res.Evidence), joinNonEmpty(res.Reasons, ", "),
|
||||
),
|
||||
Fix: res.Reference,
|
||||
FixIsURL: res.Reference != "",
|
||||
}
|
||||
}
|
||||
96
checker/threatfox_test.go
Normal file
96
checker/threatfox_test.go
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
package checker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
sdk "git.happydns.org/checker-sdk-go/checker"
|
||||
)
|
||||
|
||||
func TestThreatFoxSource_NoResult(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"query_status":"no_result","data":[]}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
s := &threatFoxSource{endpoint: srv.URL}
|
||||
results := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_threatfox": true, "threatfox_auth_key": "k"})
|
||||
if len(results) != 1 {
|
||||
t.Fatalf("expected 1 result, got %d", len(results))
|
||||
}
|
||||
r := results[0]
|
||||
listed, _ := s.Evaluate(r)
|
||||
if !r.Enabled || listed || r.Error != "" {
|
||||
t.Fatalf("expected enabled+clean, got %+v, Evaluate listed=%v", r, listed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestThreatFoxSource_Listed(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Auth-Key") == "" {
|
||||
t.Errorf("missing Auth-Key header")
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{
|
||||
"query_status": "ok",
|
||||
"data": [{
|
||||
"ioc_value": "example.com:443",
|
||||
"ioc_type": "domain",
|
||||
"threat_type": "botnet_cc",
|
||||
"malware_printable": "Emotet",
|
||||
"confidence_level": 75,
|
||||
"first_seen": "2024-01-01 00:00:00 UTC",
|
||||
"last_seen": "2024-06-01 00:00:00 UTC"
|
||||
}]
|
||||
}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
s := &threatFoxSource{endpoint: srv.URL}
|
||||
r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_threatfox": true, "threatfox_auth_key": "k"})[0]
|
||||
if len(r.Evidence) != 1 {
|
||||
t.Fatalf("expected 1 evidence item, got %+v", r)
|
||||
}
|
||||
if listed, severity := s.Evaluate(r); !listed || severity != SeverityCrit {
|
||||
t.Errorf("expected Evaluate()=(true, crit), got (%v, %q)", listed, severity)
|
||||
}
|
||||
if r.Evidence[0].Value != "example.com:443" {
|
||||
t.Errorf("evidence value = %q", r.Evidence[0].Value)
|
||||
}
|
||||
if len(r.Reasons) != 1 || r.Reasons[0] != "Emotet" {
|
||||
t.Errorf("reasons = %v", r.Reasons)
|
||||
}
|
||||
}
|
||||
|
||||
func TestThreatFoxSource_HTTPError(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
_, _ = w.Write([]byte("unauthorized"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
s := &threatFoxSource{endpoint: srv.URL}
|
||||
r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_threatfox": true, "threatfox_auth_key": "k"})[0]
|
||||
if r.Error == "" {
|
||||
t.Errorf("expected error, got %+v", r)
|
||||
}
|
||||
}
|
||||
|
||||
func TestThreatFoxSource_Disabled(t *testing.T) {
|
||||
s := &threatFoxSource{endpoint: "http://nope"}
|
||||
r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_threatfox": false})[0]
|
||||
if r.Enabled {
|
||||
t.Errorf("expected disabled, got %+v", r)
|
||||
}
|
||||
}
|
||||
|
||||
func TestThreatFoxSource_NoAuthKey(t *testing.T) {
|
||||
s := &threatFoxSource{endpoint: "http://nope"}
|
||||
r := s.Query(context.Background(), "example.com", "example.com", sdk.CheckerOptions{"enable_threatfox": true})[0]
|
||||
if r.Enabled {
|
||||
t.Errorf("expected disabled when no auth key, got %+v", r)
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue