From 93b0292109c804c07412d5ba5cfc046916d782c5 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sun, 14 Jun 2026 16:45:22 +0900 Subject: [PATCH] checker: probe well-known URIs concurrently robots.txt and security.txt are independent requests to the same host; fetch them in parallel so the collector's latency is the slower of the two rather than their sum (each pays its own TLS handshake under the keep-alive-disabled transport). Each goroutine writes its own variable and the map is assembled after both finish, so no locking is needed. --- checker/collector_wellknown.go | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/checker/collector_wellknown.go b/checker/collector_wellknown.go index 759f662..06b8361 100644 --- a/checker/collector_wellknown.go +++ b/checker/collector_wellknown.go @@ -9,6 +9,7 @@ import ( "fmt" "net/http" "strings" + "sync" ) // ObservationKeyWellKnown is the Extensions[] key under which @@ -54,16 +55,31 @@ func (wellknownCollector) Collect(ctx context.Context, t Target) (any, error) { defer cleanup() client := &http.Client{Transport: transport} - out := WellKnownData{URIs: make(map[string]WellKnownProbe, 2)} - + // The two URIs are independent requests to the same host, so probe them + // concurrently. Each goroutine writes its own variable; the map is + // assembled after both finish, so no locking is needed. + var robots, securityTxt WellKnownProbe + var wg sync.WaitGroup + wg.Add(2) // robots.txt: presence and status are all the (future) rule needs. - out.URIs["/robots.txt"] = WellKnownProbe{ - PathProbe: fetchHTTPSPath(ctx, client, t.Host, "/robots.txt", t.UserAgent, 64<<10), - } - + go func() { + defer wg.Done() + robots = WellKnownProbe{ + PathProbe: fetchHTTPSPath(ctx, client, t.Host, "/robots.txt", t.UserAgent, 64<<10), + } + }() // security.txt: read the body so the rule can tell a genuine RFC 9116 // file from a soft-404 page that merely returns 200. - out.URIs["/.well-known/security.txt"] = fetchSecurityTxt(ctx, client, t.Host, "/.well-known/security.txt", t.UserAgent, 64<<10) + go func() { + defer wg.Done() + securityTxt = fetchSecurityTxt(ctx, client, t.Host, "/.well-known/security.txt", t.UserAgent, 64<<10) + }() + wg.Wait() + + out := WellKnownData{URIs: map[string]WellKnownProbe{ + "/robots.txt": robots, + "/.well-known/security.txt": securityTxt, + }} return &out, nil }