diff --git a/README.md b/README.md index 0bf3f1a..f25ba23 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ widely-used reputation systems. | abuse.ch ThreatFox | HTTPS lookup | free Auth-Key (admin) | user (default on) | | abuse.ch MalwareBazaar| HTTPS lookup | free Auth-Key (admin) | user (default on) | | Botvrij.eu | downloaded list | no | user (default on) | +| Disconnect.me | downloaded list | no | user (default on) | | VirusTotal v3 | HTTPS lookup | yes (admin) | admin | ### Obtaining API keys diff --git a/checker/disconnect.go b/checker/disconnect.go new file mode 100644 index 0000000..ae93adf --- /dev/null +++ b/checker/disconnect.go @@ -0,0 +1,162 @@ +package checker + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + sdk "git.happydns.org/checker-sdk-go/checker" +) + +const disconnectFeedURL = "https://s3.amazonaws.com/lists.disconnect.me/services.json" + +func init() { + Register(&disconnectSource{ + cache: newFeedCache(24*time.Hour, disconnectFetch(disconnectFeedURL)), + }) +} + +type disconnectSource struct{ cache *feedCache } + +func (*disconnectSource) ID() string { return "disconnect" } +func (*disconnectSource) Name() string { return "Disconnect.me" } + +func (*disconnectSource) Options() SourceOptions { + return SourceOptions{ + User: []sdk.CheckerOptionField{ + { + Id: "enable_disconnect", + Type: "bool", + Label: "Use the Disconnect.me tracking-protection list", + Description: "Check the domain against the Disconnect.me blocklist used by Firefox Enhanced Tracking Protection, Brave, and uBlock Origin.", + Default: true, + }, + }, + } +} + +func (s *disconnectSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult { + if !sdk.GetBoolOption(opts, "enable_disconnect", true) { + return disabledResult(s.ID(), s.Name()) + } + if registered == "" { + return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: true}} + } + + matches, size, fetched, err := s.cache.lookup(ctx, registered) + res := SourceResult{ + SourceID: s.ID(), SourceName: s.Name(), Enabled: true, + Reference: "https://disconnect.me/trackerprotection", + Details: mustJSON(map[string]any{"feed_size": size, "fetched_at": fetched}), + } + if err != nil { + res.Error = err.Error() + } + + seenCategory := map[string]bool{} + for _, m := range matches { + parts := strings.SplitN(m, "|", 2) + category, company := parts[0], "" + if len(parts) == 2 { + company = parts[1] + } + if !seenCategory[category] { + seenCategory[category] = true + res.Reasons = append(res.Reasons, category) + } + extra := map[string]string{} + if company != "" { + extra["company"] = company + } + res.Evidence = append(res.Evidence, Evidence{ + Label: "Category", + Value: category, + Status: strings.ToLower(category), + Extra: extra, + }) + } + return []SourceResult{res} +} + +func (*disconnectSource) Evaluate(r SourceResult) (bool, string) { + return evidenceEval(r, SeverityWarn) +} + +func (*disconnectSource) Diagnose(res SourceResult) Diagnosis { + return Diagnosis{ + Severity: SeverityWarn, + Title: "Listed in Disconnect.me tracking-protection blocklist", + Detail: fmt.Sprintf( + "Category: %s. This domain appears in the Disconnect.me list used by Firefox Enhanced Tracking Protection, Brave, and uBlock Origin. Browsers and privacy tools will block third-party requests to this domain, which may affect analytics, ad delivery, or embedded widgets. The list is maintained by Disconnect.me; contact them if you believe the classification is incorrect.", + joinNonEmpty(res.Reasons, ", "), + ), + Fix: "https://disconnect.me/contact", + FixIsURL: true, + } +} + +// disconnectJSON mirrors the structure of services.json. +type disconnectJSON struct { + Categories map[string][]map[string]map[string][]string `json:"categories"` +} + +func disconnectFetch(feedURL string) func(context.Context) ([]string, map[string][]string, error) { + return func(ctx context.Context) ([]string, map[string][]string, error) { + reqCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, feedURL, nil) + if err != nil { + return nil, nil, err + } + req.Header.Set("User-Agent", "happydomain-checker-blacklist/1.0") + + resp, err := sharedHTTPClient.Do(req) + if err != nil { + return nil, nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, nil, fmt.Errorf("disconnect HTTP %d", resp.StatusCode) + } + + raw, err := io.ReadAll(io.LimitReader(resp.Body, 32<<20)) + if err != nil { + return nil, nil, fmt.Errorf("disconnect read: %w", err) + } + + var data disconnectJSON + if err := json.Unmarshal(raw, &data); err != nil { + return nil, nil, fmt.Errorf("disconnect parse: %w", err) + } + + byHost := make(map[string][]string, 4096) + for category, entities := range data.Categories { + for _, entity := range entities { + for company, sites := range entity { + for _, domains := range sites { + for _, d := range domains { + d = strings.ToLower(strings.TrimSuffix(d, ".")) + if d == "" { + continue + } + entry := category + "|" + company + byHost[d] = append(byHost[d], entry) + } + } + } + } + } + + urls := make([]string, 0, len(byHost)) + for d := range byHost { + urls = append(urls, d) + } + return urls, byHost, nil + } +} diff --git a/checker/disconnect_test.go b/checker/disconnect_test.go new file mode 100644 index 0000000..7a5e325 --- /dev/null +++ b/checker/disconnect_test.go @@ -0,0 +1,127 @@ +package checker + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" + + sdk "git.happydns.org/checker-sdk-go/checker" +) + +const disconnectFakeFeed = `{ + "categories": { + "Advertising": [ + { + "Evil Corp": { + "https://evilcorp.com": ["tracker.com", "sub.example.org"] + } + } + ], + "Analytics": [ + { + "Metrics Inc": { + "https://metrics.io": ["analytics.net"] + } + } + ] + } +}` + +func newDisconnectTestSource(srv *httptest.Server) *disconnectSource { + return &disconnectSource{ + cache: newFeedCache(time.Hour, disconnectFetch(srv.URL)), + } +} + +func TestDisconnectSource_Listed(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(disconnectFakeFeed)) + })) + defer srv.Close() + + s := newDisconnectTestSource(srv) + r := s.Query(context.Background(), "tracker.com", "tracker.com", sdk.CheckerOptions{"enable_disconnect": true})[0] + + if !r.Enabled || r.Error != "" { + t.Fatalf("expected enabled with no error, got %+v", r) + } + if len(r.Evidence) == 0 { + t.Fatalf("expected evidence, got none") + } + found := false + for _, e := range r.Evidence { + if e.Value == "Advertising" { + found = true + if e.Extra["company"] != "Evil Corp" { + t.Errorf("expected company 'Evil Corp', got %q", e.Extra["company"]) + } + } + } + if !found { + t.Errorf("expected Advertising evidence, got %+v", r.Evidence) + } + if listed, sev := s.Evaluate(r); !listed || sev != SeverityWarn { + t.Errorf("expected (true, warn), got (%v, %q)", listed, sev) + } +} + +func TestDisconnectSource_SubdomainInFeed(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(disconnectFakeFeed)) + })) + defer srv.Close() + + // Feed has "sub.example.org"; querying registered domain "example.org" should match. + s := newDisconnectTestSource(srv) + r := s.Query(context.Background(), "example.org", "example.org", sdk.CheckerOptions{"enable_disconnect": true})[0] + + if !r.Enabled || r.Error != "" { + t.Fatalf("expected enabled with no error, got %+v", r) + } + if len(r.Evidence) == 0 { + t.Errorf("expected subdomain match evidence, got none") + } +} + +func TestDisconnectSource_NotListed(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(disconnectFakeFeed)) + })) + defer srv.Close() + + s := newDisconnectTestSource(srv) + r := s.Query(context.Background(), "clean.example.com", "clean.example.com", sdk.CheckerOptions{"enable_disconnect": true})[0] + + if !r.Enabled || r.Error != "" { + t.Fatalf("expected enabled with no error, got %+v", r) + } + if len(r.Evidence) != 0 { + t.Errorf("expected no evidence for clean domain, got %+v", r.Evidence) + } + if listed, _ := s.Evaluate(r); listed { + t.Errorf("expected not listed for clean domain") + } +} + +func TestDisconnectSource_Disabled(t *testing.T) { + s := &disconnectSource{cache: newFeedCache(time.Hour, disconnectFetch("http://nope"))} + r := s.Query(context.Background(), "tracker.com", "tracker.com", sdk.CheckerOptions{"enable_disconnect": false})[0] + if r.Enabled { + t.Errorf("expected disabled result, got %+v", r) + } +} + +func TestDisconnectSource_HTTPError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer srv.Close() + + s := newDisconnectTestSource(srv) + r := s.Query(context.Background(), "tracker.com", "tracker.com", sdk.CheckerOptions{"enable_disconnect": true})[0] + if r.Error == "" { + t.Errorf("expected error on HTTP 500, got empty error") + } +}