checker-blacklist/checker/urlhaus.go

package checker

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"html/template"
	"net/http"
	"net/url"
	"strings"
	"time"

	sdk "git.happydns.org/checker-sdk-go/checker"
)

const urlhausHostEndpoint = "https://urlhaus-api.abuse.ch/v1/host/"

func init() { Register(&urlhausSource{endpoint: urlhausHostEndpoint}) }

type urlhausSource struct {
	endpoint string
}

func (*urlhausSource) ID() string   { return "urlhaus" }
func (*urlhausSource) Name() string { return "abuse.ch URLhaus" }

func (*urlhausSource) Options() SourceOptions {
	return SourceOptions{
		User: []sdk.CheckerOptionField{
			{
				Id:          "enable_urlhaus",
				Type:        "bool",
				Label:       "Use abuse.ch URLhaus",
				Description: "Query the URLhaus host endpoint for active malware-distribution URLs hosted on the domain.",
				Default:     true,
			},
		},
		Admin: []sdk.CheckerOptionField{
			{
				Id:          "urlhaus_auth_key",
				Type:        "string",
				Label:       "URLhaus Auth-Key",
				Description: "abuse.ch URLhaus Auth-Key (free, requires an abuse.ch account). Required: the URLhaus API rejects anonymous requests with HTTP 401. Without this key the source is disabled.",
				Secret:      true,
			},
		},
	}
}

// urlhausDetails is the source-specific extras kept in
// SourceResult.Details so the rich detail renderer can show a per-URL
// table with online/offline state, threat type, tags and date added.
type urlhausDetails struct {
	URLs []urlhausURL `json:"urls"`
}

type urlhausURL struct {
	URL       string   `json:"url"`
	Status    string   `json:"status"`
	Threat    string   `json:"threat"`
	Tags      []string `json:"tags,omitempty"`
	DateAdded string   `json:"date_added,omitempty"`
	Reference string   `json:"reference,omitempty"`
}

func (s *urlhausSource) Query(ctx context.Context, domain, registered string, opts sdk.CheckerOptions) []SourceResult {
	authKey := stringOpt(opts, "urlhaus_auth_key")
	if !sdk.GetBoolOption(opts, "enable_urlhaus", true) || registered == "" || authKey == "" {
		return []SourceResult{{SourceID: s.ID(), SourceName: s.Name(), Enabled: false}}
	}

	res := SourceResult{SourceID: s.ID(), SourceName: s.Name(), Enabled: true}

	reqCtx, cancel := context.WithTimeout(ctx, 15*time.Second)
	defer cancel()

	form := url.Values{"host": {registered}}
	req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, s.endpoint, strings.NewReader(form.Encode()))
	if err != nil {
		res.Error = err.Error()
		return []SourceResult{res}
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	req.Header.Set("User-Agent", "happydomain-checker-blacklist/1.0")
	if authKey != "" {
		req.Header.Set("Auth-Key", authKey)
	}

	body, status, err := httpDo(req, 4<<20)
	if err != nil {
		res.Error = err.Error()
		return []SourceResult{res}
	}
	if status != http.StatusOK {
		res.Error = fmt.Sprintf("HTTP %d: %s", status, truncate(string(body), 200))
		return []SourceResult{res}
	}

	var parsed struct {
		QueryStatus string `json:"query_status"`
		Reference   string `json:"urlhaus_reference"`
		URLs        []struct {
			URL       string   `json:"url"`
			Status    string   `json:"url_status"`
			Threat    string   `json:"threat"`
			Tags      []string `json:"tags"`
			DateAdded string   `json:"date_added"`
			Reference string   `json:"urlhaus_reference"`
		} `json:"urls"`
	}
	if err := json.Unmarshal(body, &parsed); err != nil {
		res.Error = "decode: " + err.Error()
		return []SourceResult{res}
	}

	res.Reference = parsed.Reference
	switch parsed.QueryStatus {
	case "ok":
		if len(parsed.URLs) == 0 {
			return []SourceResult{res}
		}
		res.Listed = true
		res.Severity = SeverityCrit
		threats := map[string]bool{}
		details := urlhausDetails{}
		for _, u := range parsed.URLs {
			if u.Threat != "" && !threats[u.Threat] {
				threats[u.Threat] = true
				res.Reasons = append(res.Reasons, u.Threat)
			}
			res.Evidence = append(res.Evidence, Evidence{
				Label: "URL", Value: u.URL, Status: u.Status,
			})
			details.URLs = append(details.URLs, urlhausURL{
				URL: u.URL, Status: u.Status, Threat: u.Threat,
				Tags: u.Tags, DateAdded: u.DateAdded, Reference: u.Reference,
			})
		}
		res.Details = mustJSON(details)
	case "no_results":
		// Clean.
	case "invalid_host", "http_post_expected":
		res.Error = "rejected query: " + parsed.QueryStatus
	default:
		res.Error = "query_status=" + parsed.QueryStatus
	}
	return []SourceResult{res}
}

func (*urlhausSource) Diagnose(res SourceResult) Diagnosis {
	online := 0
	for _, e := range res.Evidence {
		if e.Status == "online" {
			online++
		}
	}
	return Diagnosis{
		Severity: SeverityCrit,
		Title:    "Listed in abuse.ch URLhaus (active malware distribution)",
		Detail: fmt.Sprintf(
			"%d URL(s) tracked, %d still online; threat type(s): %s. URLhaus indexes URLs that actively serve malware payloads. Treat the host as compromised: take the offending pages offline, audit the web stack (CMS plugins, recently-uploaded files, cron jobs), then submit a takedown notification through the URLhaus reference page.",
			len(res.Evidence), online, joinNonEmpty(res.Reasons, ", "),
		),
		Fix:      res.Reference,
		FixIsURL: res.Reference != "",
	}
}

// RenderDetail renders the URLhaus URL table. Implementing
// DetailRenderer keeps the rich per-source view alongside the source
// implementation rather than scattered in the report code.
func (*urlhausSource) RenderDetail(res SourceResult) (template.HTML, error) {
	var d urlhausDetails
	if len(res.Details) > 0 {
		if err := json.Unmarshal(res.Details, &d); err != nil {
			return "", fmt.Errorf("urlhaus: decode details: %w", err)
		}
	}
	if len(d.URLs) == 0 {
		return "", nil
	}

	var b bytes.Buffer
	if err := urlhausDetailTpl.Execute(&b, d); err != nil {
		return "", err
	}
	return template.HTML(b.String()), nil
}

var urlhausDetailTpl = template.Must(template.New("urlhaus_detail").Parse(`
<table>
<thead><tr><th>URL</th><th>Status</th><th>Threat</th><th>Tags</th><th>Added</th></tr></thead>
<tbody>{{range .URLs}}<tr class="row-crit">
<td><code>{{.URL}}</code>{{with .Reference}} <a href="{{.}}" target="_blank" rel="noreferrer">↗</a>{{end}}</td>
<td>{{.Status}}</td>
<td>{{.Threat}}</td>
<td>{{range .Tags}}<span>{{.}} </span>{{end}}</td>
<td><small>{{.DateAdded}}</small></td>
</tr>{{end}}</tbody>
</table>`))