checker-http/checker/collect.go

// This file is part of the happyDomain (R) project.
// Copyright (c) 2020-2026 happyDomain
// Authors: Pierre-Olivier Mercier, et al.

package checker

import (
	"bytes"
	"context"
	"crypto/tls"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	sdk "git.happydns.org/checker-sdk-go/checker"
	"golang.org/x/net/html"
)

// verboseLogging is enabled via the CHECKER_HTTP_VERBOSE environment variable;
// when off, per-probe logging is silenced to keep production logs clean.
var verboseLogging = os.Getenv("CHECKER_HTTP_VERBOSE") != ""

// Collect probes HTTP and HTTPS for every (IP, scheme) pair on the
// abstract.Server. The HTTP body of the primary HTTPS probe is parsed for
// SRI evaluation; secondary probes only retain headers/cookies/redirects.
func (p *httpProvider) Collect(ctx context.Context, opts sdk.CheckerOptions) (any, error) {
	server, err := resolveServer(opts)
	if err != nil {
		return nil, err
	}

	timeoutMs := sdk.GetIntOption(opts, OptionProbeTimeoutMs, DefaultProbeTimeoutMs)
	if timeoutMs <= 0 {
		timeoutMs = DefaultProbeTimeoutMs
	}
	timeout := time.Duration(timeoutMs) * time.Millisecond

	maxRedirects := sdk.GetIntOption(opts, OptionMaxRedirects, DefaultMaxRedirects)
	if maxRedirects < 0 {
		maxRedirects = DefaultMaxRedirects
	}
	userAgent := DefaultUserAgent
	if v, ok := sdk.GetOption[string](opts, OptionUserAgent); ok && v != "" {
		userAgent = v
	}

	host, ips := addressesFromServer(server)
	// abstract.Server only pins one A and one AAAA. Resolve the host
	// to pick up any additional records the authoritative DNS exposes,
	// so multi-IP deployments aren't silently under-probed. Failures
	// are non-fatal; the pinned IPs remain.
	seen := make(map[string]struct{}, len(ips)+4)
	for _, ip := range ips {
		seen[ip] = struct{}{}
	}
	ips = append(ips, discoverIPs(ctx, host, seen)...)
	if len(ips) == 0 {
		return nil, fmt.Errorf("abstract.Server has no A/AAAA records")
	}

	data := &HTTPData{
		Domain:      host,
		CollectedAt: time.Now(),
	}

	type job struct {
		scheme string
		port   uint16
		ip     string
		// parseHTML controls whether the HTML body is parsed and its
		// references kept on the probe. We only do this for the first
		// HTTPS probe to keep the payload bounded.
		parseHTML bool
	}

	var jobs []job
	htmlPicked := false
	for _, ip := range ips {
		jobs = append(jobs, job{scheme: "http", port: DefaultHTTPPort, ip: ip})
		j := job{scheme: "https", port: DefaultHTTPSPort, ip: ip}
		if !htmlPicked {
			j.parseHTML = true
			htmlPicked = true
		}
		jobs = append(jobs, j)
	}

	var mu sync.Mutex
	var wg sync.WaitGroup
	sem := make(chan struct{}, MaxConcurrentProbes)
	for _, j := range jobs {
		wg.Add(1)
		sem <- struct{}{}
		go func(j job) {
			defer wg.Done()
			defer func() { <-sem }()
			probe := runProbe(ctx, host, j.ip, j.scheme, j.port, timeout, maxRedirects, userAgent, j.parseHTML)
			if verboseLogging {
				log.Printf("checker-http: %s ip=%s status=%d redirects=%d err=%q",
					j.scheme, j.ip, probe.StatusCode, len(probe.RedirectChain), probe.Error)
			}
			mu.Lock()
			data.Probes = append(data.Probes, probe)
			mu.Unlock()
		}(j)
	}
	wg.Wait()

	return data, nil
}

func runProbe(ctx context.Context, host, ip, scheme string, port uint16, timeout time.Duration, maxRedirects int, ua string, parseHTML bool) HTTPProbe {
	addr := net.JoinHostPort(ip, fmt.Sprintf("%d", port))
	probe := HTTPProbe{
		Scheme:  scheme,
		Host:    host,
		IP:      ip,
		Port:    port,
		Address: addr,
		IsIPv6:  strings.Contains(ip, ":"),
	}

	dialer := &net.Dialer{Timeout: timeout}
	// tcpConnected is set the moment a dial succeeds, so we can
	// distinguish pure-TCP failures from later TLS/HTTP errors without
	// resorting to error-string matching.
	var tcpConnected atomic.Bool
	// Force every dial to the chosen IP, regardless of what hostname is
	// in the URL; that way we can attribute results to a specific A/AAAA
	// record and bypass local resolver oddities.
	transport := &http.Transport{
		DialContext: func(ctx context.Context, network, _ string) (net.Conn, error) {
			conn, err := dialer.DialContext(ctx, network, addr)
			if err != nil {
				return nil, err
			}
			tcpConnected.Store(true)
			return conn, nil
		},
		TLSClientConfig: &tls.Config{
			ServerName: host,
			// Deep TLS posture is delegated to checker-tls. We still want
			// HTTPS errors (expired cert, bad chain, ...) to surface as
			// probe errors, so verification stays enabled.
		},
		TLSHandshakeTimeout:   timeout,
		ResponseHeaderTimeout: timeout,
		DisableKeepAlives:     true,
	}
	defer transport.CloseIdleConnections()

	// Bound the whole probe (dial + TLS + headers + body across all
	// redirect hops) by a single per-probe deadline derived from ctx, so
	// a slow target can't pin a worker beyond the parent's lifetime and
	// outer cancellation propagates to in-flight I/O.
	probeBudget := timeout * time.Duration(maxRedirects+1)
	probeCtx, cancel := context.WithTimeout(ctx, probeBudget)
	defer cancel()

	var redirectChain []RedirectStep
	client := &http.Client{
		Transport: transport,
		// No client-level Timeout: probeCtx already bounds the request,
		// and a separate http.Client.Timeout would race with it.
		CheckRedirect: func(req *http.Request, via []*http.Request) error {
			prev := via[len(via)-1]
			redirectChain = append(redirectChain, RedirectStep{
				From:   prev.URL.String(),
				To:     req.URL.String(),
				Status: 0, // populated post-hoc below if available
			})
			// The transport's DialContext is pinned to the original
			// (ip, port) and TLS ServerName is pinned to the original
			// host. Following a redirect that changes host, scheme, or
			// port would silently route the request to the wrong
			// backend. Stop and return the 3xx so the caller can see
			// the Location, but don't follow it on this probe.
			if !strings.EqualFold(req.URL.Host, host) ||
				!strings.EqualFold(req.URL.Scheme, scheme) {
				return http.ErrUseLastResponse
			}
			if len(via) > maxRedirects {
				return fmt.Errorf("stopped after %d redirects", maxRedirects)
			}
			return nil
		},
	}

	target := &url.URL{Scheme: scheme, Host: host, Path: "/"}
	req, err := http.NewRequestWithContext(probeCtx, http.MethodGet, target.String(), nil)
	if err != nil {
		probe.Error = err.Error()
		return probe
	}
	req.Header.Set("User-Agent", ua)
	req.Header.Set("Accept", "text/html,application/xhtml+xml;q=0.9,*/*;q=0.5")

	start := time.Now()
	resp, err := client.Do(req)
	probe.ElapsedMS = time.Since(start).Milliseconds()
	if err != nil {
		probe.Error = err.Error()
		// The dialer wrapper sets tcpConnected the moment a TCP
		// connection is established, so we can attribute the failure
		// to a post-TCP layer (TLS, HTTP, redirect policy) without
		// any error-string heuristics.
		probe.TCPConnected = tcpConnected.Load()
		probe.RedirectChain = redirectChain
		return probe
	}
	defer resp.Body.Close()

	probe.TCPConnected = true
	probe.StatusCode = resp.StatusCode
	if resp.Request != nil && resp.Request.URL != nil {
		probe.FinalURL = resp.Request.URL.String()
	}

	// Per RFC 7230 §3.2.2, repeated headers (other than Set-Cookie) are
	// semantically equivalent to a single header whose value is the
	// comma-joined list; folding here preserves directives like a second
	// CSP or HSTS header that would otherwise be dropped. Set-Cookie is
	// excluded from the map since cookies are surfaced via resp.Cookies().
	probe.Headers = make(map[string]string, len(resp.Header))
	for k, v := range resp.Header {
		if len(v) == 0 {
			continue
		}
		lk := strings.ToLower(k)
		if lk == "set-cookie" {
			continue
		}
		probe.Headers[lk] = strings.Join(v, ", ")
	}

	for _, c := range resp.Cookies() {
		probe.Cookies = append(probe.Cookies, CookieInfo{
			Name:      c.Name,
			Domain:    c.Domain,
			Path:      c.Path,
			Secure:    c.Secure,
			HttpOnly:  c.HttpOnly,
			SameSite:  sameSiteString(c.SameSite),
			HasExpiry: !c.Expires.IsZero() || c.MaxAge > 0,
		})
	}
	probe.RedirectChain = redirectChain

	// Read one extra byte to detect whether we hit the cap. Anything
	// beyond MaxBodyBytes is dropped, but the probe surfaces
	// BodyTruncated so callers know SRI/HTML rules saw a partial view.
	body, err := io.ReadAll(io.LimitReader(resp.Body, MaxBodyBytes+1))
	if err == nil {
		if len(body) > MaxBodyBytes {
			body = body[:MaxBodyBytes]
			probe.BodyTruncated = true
		}
		probe.HTMLBytes = len(body)
		if parseHTML && isHTMLContent(probe.Headers["content-type"]) {
			probe.Resources = extractResources(body, host)
		}
	}

	return probe
}

func sameSiteString(s http.SameSite) string {
	switch s {
	case http.SameSiteLaxMode:
		return "Lax"
	case http.SameSiteStrictMode:
		return "Strict"
	case http.SameSiteNoneMode:
		return "None"
	default:
		return ""
	}
}

func isHTMLContent(ct string) bool {
	ct = strings.ToLower(ct)
	return strings.Contains(ct, "text/html") || strings.Contains(ct, "application/xhtml")
}

// extractResources walks the HTML body and collects <script src=...>,
// <link href=... rel="stylesheet"|"preload"...> and inline-eligible <img>
// references, with a flag for whether the resource is cross-origin
// (different host than the page); SRI is only meaningful in that case.
func extractResources(body []byte, pageHost string) []HTMLResource {
	doc, err := html.Parse(bytes.NewReader(body))
	if err != nil {
		return nil
	}
	var out []HTMLResource
	var walk func(*html.Node)
	walk = func(n *html.Node) {
		if n.Type == html.ElementNode {
			switch n.Data {
			case "script":
				if src, ok := attr(n, "src"); ok && src != "" {
					out = append(out, mkResource("script", src, n, pageHost))
				}
			case "link":
				rel, _ := attr(n, "rel")
				if href, ok := attr(n, "href"); ok && href != "" && relIsAsset(rel) {
					r := mkResource("link", href, n, pageHost)
					r.Rel = rel
					out = append(out, r)
				}
			}
		}
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			walk(c)
		}
	}
	walk(doc)
	return out
}

func relIsAsset(rel string) bool {
	rel = strings.ToLower(rel)
	return strings.Contains(rel, "stylesheet") || strings.Contains(rel, "preload") || strings.Contains(rel, "modulepreload")
}

func mkResource(tag, ref string, n *html.Node, pageHost string) HTMLResource {
	r := HTMLResource{Tag: tag, URL: ref}
	if integ, ok := attr(n, "integrity"); ok && integ != "" {
		r.Integrity = integ
	}
	if u, err := url.Parse(ref); err == nil && u.Host != "" && !strings.EqualFold(u.Host, pageHost) {
		r.CrossOrigin = true
	}
	return r
}

func attr(n *html.Node, key string) (string, bool) {
	for _, a := range n.Attr {
		if strings.EqualFold(a.Key, key) {
			return a.Val, true
		}
	}
	return "", false
}