checker-blacklist/checker/openphish.go

package checker

import (
	"bufio"
	"context"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"

	sdk "git.happydns.org/checker-sdk-go/checker"
)

const openPhishFeedURL = "https://openphish.com/feed.txt"

func init() {
	Register(&openPhishSource{
		cache: newPhishCache(openPhishFeedURL, 1*time.Hour),
	})
}

// openPhishSource downloads the public OpenPhish feed once per cache
// TTL and matches the registered domain (and all subdomains) against
// every URL in the feed. The cache is per-source-instance so it lives
// for as long as the process.
type openPhishSource struct {
	cache *phishCache
}

func (*openPhishSource) ID() string   { return "openphish" }
func (*openPhishSource) Name() string { return "OpenPhish feed" }

func (*openPhishSource) Options() SourceOptions {
	return SourceOptions{
		User: []sdk.CheckerOptionField{
			{
				Id:          "enable_openphish",
				Type:        "bool",
				Label:       "Use the OpenPhish public feed",
				Description: "Download the OpenPhish public feed (refreshed every 12h) and check the domain against it.",
				Default:     true,
			},
		},
	}
}

func (s *openPhishSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult {
	if !sdk.GetBoolOption(opts, "enable_openphish", true) || registered == "" {
		return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: false}}
	}

	urls, size, fetched, err := s.cache.lookup(ctx, registered)
	res := SourceResult{
		SourceID: s.ID(), SourceName: s.Name(), Enabled: true,
		Reference: "https://openphish.com/",
		Details:   mustJSON(map[string]any{"feed_size": size, "fetched_at": fetched}),
	}
	if err != nil {
		res.Error = err.Error()
		// Fall through with whatever the cache could provide.
	}
	if len(urls) > 0 {
		res.Listed = true
		res.Severity = SeverityCrit
		res.Reasons = []string{"Phishing"}
		for _, u := range urls {
			res.Evidence = append(res.Evidence, Evidence{Label: "URL", Value: u})
		}
	}
	return []SourceResult{res}
}

func (*openPhishSource) Diagnose(res SourceResult) Diagnosis {
	urls := make([]string, 0, len(res.Evidence))
	for _, e := range res.Evidence {
		urls = append(urls, e.Value)
	}
	previewN := min(len(urls), 5)
	return Diagnosis{
		Severity: SeverityCrit,
		Title:    "Listed in the OpenPhish phishing feed",
		Detail: fmt.Sprintf(
			"%d URL(s) hosted on this domain are tracked as phishing by OpenPhish. Treat the host as compromised: rotate credentials, audit recently-added files (look for /wp-includes/, /uploads/, lookalike admin paths), then request review at OpenPhish. Examples: %s",
			len(urls), joinNonEmpty(urls[:previewN], ", "),
		),
		Fix:      "https://openphish.com/feedback.html",
		FixIsURL: true,
	}
}

// ---------- feed cache ----------

type phishCache struct {
	mu            sync.Mutex
	urls          []string
	byHost        map[string][]string
	fetchedAt     time.Time
	lastAttemptAt time.Time
	refreshing    bool
	ttl           time.Duration
	failBackoff   time.Duration
	feedURL       string
}

func newPhishCache(feedURL string, ttl time.Duration) *phishCache {
	if feedURL == "" {
		feedURL = openPhishFeedURL
	}
	if ttl <= 0 {
		ttl = 1 * time.Hour
	}
	return &phishCache{ttl: ttl, feedURL: feedURL, failBackoff: 1 * time.Minute}
}

func (c *phishCache) lookup(ctx context.Context, domain string) (urls []string, size int, fetchedAt time.Time, err error) {
	domain = strings.ToLower(strings.TrimSuffix(domain, "."))

	c.mu.Lock()
	stale := c.byHost == nil || time.Since(c.fetchedAt) > c.ttl
	doRefresh := stale && !c.refreshing && time.Since(c.lastAttemptAt) > c.failBackoff
	if doRefresh {
		c.refreshing = true
	}
	c.mu.Unlock()

	if doRefresh {
		// Fetch without holding the cache lock so concurrent lookups
		// can still serve stale data. Only one refresh runs at a time.
		newURLs, newByHost, ferr := c.fetch(ctx)
		c.mu.Lock()
		c.refreshing = false
		c.lastAttemptAt = time.Now()
		if ferr == nil {
			c.urls = newURLs
			c.byHost = newByHost
			c.fetchedAt = c.lastAttemptAt
		} else {
			err = ferr
		}
		c.mu.Unlock()
	}

	c.mu.Lock()
	for host, hostURLs := range c.byHost {
		if host == domain || strings.HasSuffix(host, "."+domain) {
			urls = append(urls, hostURLs...)
		}
	}
	size = len(c.urls)
	fetchedAt = c.fetchedAt
	c.mu.Unlock()
	return urls, size, fetchedAt, err
}

func (c *phishCache) fetch(ctx context.Context) ([]string, map[string][]string, error) {
	reqCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
	defer cancel()

	req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, c.feedURL, nil)
	if err != nil {
		return nil, nil, err
	}
	req.Header.Set("User-Agent", "happydomain-checker-blacklist/1.0")

	resp, err := sharedHTTPClient.Do(req)
	if err != nil {
		return nil, nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return nil, nil, fmt.Errorf("openphish HTTP %d", resp.StatusCode)
	}

	urls := make([]string, 0, 8192)
	byHost := make(map[string][]string, 8192)
	scanner := bufio.NewScanner(io.LimitReader(resp.Body, 64<<20))
	scanner.Buffer(make([]byte, 0, 64*1024), 1<<20)
	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())
		if line == "" || strings.HasPrefix(line, "#") {
			continue
		}
		urls = append(urls, line)
		if h := hostOfURL(line); h != "" {
			byHost[h] = append(byHost[h], line)
		}
	}
	if err := scanner.Err(); err != nil {
		return nil, nil, err
	}
	return urls, byHost, nil
}

func hostOfURL(s string) string {
	u, err := url.Parse(s)
	if err != nil {
		return ""
	}
	return strings.ToLower(u.Hostname())
}