perf(reader): reuse TCP connection across well-known feed URL probes

internal/reader/fetcher/request_builder.go

@@ -44,6 +44,10 @@ type RequestBuilder struct {
 	disableCompression bool
 	proxyRotator       *proxyrotator.ProxyRotator
 	feedProxyURL       string
+
+	// keepaliveClient is lazily built by the first ExecuteRequestKeepalive
+	// call and reused by subsequent calls. Close() releases it.
+	keepaliveClient *http.Client
 }
 
 func NewRequestBuilder() *RequestBuilder {
@@ -141,6 +145,54 @@ func (r *RequestBuilder) WithoutCompression() *RequestBuilder {
 }
 
 func (r *RequestBuilder) ExecuteRequest(requestURL string) (*http.Response, error) {
+	client, err := r.newClient()
+	if err != nil {
+		return nil, err
+	}
+	req, err := r.prepareRequest(requestURL)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Connection", "close")
+	return r.send(client, req)
+}
+
+// ExecuteRequestKeepalive sends a request using a builder-internal HTTP client
+// that is lazily built on the first call and reused by subsequent calls. The
+// underlying TCP/TLS connection can therefore be reused across calls (HTTP
+// keep-alive, HTTP/2 multiplexing), which nice when probing several URLs
+// against the same host, for example in during feeds URL discovery.
+//
+// When done with the connection pool, don't forget to call Close() to release it.
+// It's worth noting that because the client is built only once, the proxy
+// rotator advances only on the first call.
+func (r *RequestBuilder) ExecuteRequestKeepalive(requestURL string) (*http.Response, error) {
+	if r.keepaliveClient == nil {
+		client, err := r.newClient()
+		if err != nil {
+			return nil, err
+		}
+		r.keepaliveClient = client
+	}
+	req, err := r.prepareRequest(requestURL)
+	if err != nil {
+		return nil, err
+	}
+	return r.send(r.keepaliveClient, req)
+}
+
+// Close releases idle connections held by the keep-alive client, if any.
+// Safe to call even if ExecuteRequestKeepalive was never used.
+func (r *RequestBuilder) Close() {
+	if r.keepaliveClient != nil {
+		r.keepaliveClient.CloseIdleConnections()
+		r.keepaliveClient = nil
+	}
+}
+
+// newClient builds a configured *http.Client from the builder's settings.
+// The proxy rotator advances once per call.
+func (r *RequestBuilder) newClient() (*http.Client, error) {
 	var clientProxyURL *url.URL
 
 	switch {
@@ -231,14 +283,13 @@ func (r *RequestBuilder) ExecuteRequest(requestURL string) (*http.Response, erro
 		transport.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{}
 	}
 
-	var clientProxyURLRedacted string
 	if clientProxyURL != nil {
 		transport.Proxy = http.ProxyURL(clientProxyURL)
-		clientProxyURLRedacted = clientProxyURL.Redacted()
 	}
 
 	client := &http.Client{
-		Timeout: r.clientTimeout,
+		Timeout:   r.clientTimeout,
+		Transport: transport,
 	}
 
 	if r.withoutRedirects {
@@ -247,14 +298,28 @@ func (r *RequestBuilder) ExecuteRequest(requestURL string) (*http.Response, erro
 		}
 	}
 
-	client.Transport = transport
+	var clientProxyURLRedacted string
+	if clientProxyURL != nil {
+		clientProxyURLRedacted = clientProxyURL.Redacted()
+	}
+	slog.Debug("Built HTTP client", slog.Group("client",
+		slog.Bool("without_redirects", r.withoutRedirects),
+		slog.Bool("use_app_client_proxy", r.useClientProxy),
+		slog.String("client_proxy_url", clientProxyURLRedacted),
+		slog.Bool("ignore_tls_errors", r.ignoreTLSErrors),
+		slog.Bool("disable_http2", r.disableHTTP2),
+	))
 
+	return client, nil
+}
+
+func (r *RequestBuilder) prepareRequest(requestURL string) (*http.Request, error) {
 	req, err := http.NewRequest("GET", requestURL, nil)
 	if err != nil {
 		return nil, err
 	}
 
-	req.Header = r.headers
+	req.Header = r.headers.Clone()
 	if r.disableCompression {
 		req.Header.Set("Accept-Encoding", "identity")
 	} else {
@@ -267,19 +332,15 @@ func (r *RequestBuilder) ExecuteRequest(requestURL string) (*http.Response, erro
 		req.Header.Set("Accept", defaultAcceptHeader)
 	}
 
-	req.Header.Set("Connection", "close")
+	return req, nil
+}
 
+func (r *RequestBuilder) send(client *http.Client, req *http.Request) (*http.Response, error) {
 	slog.Debug("Making outgoing request", slog.Group("request",
 		slog.String("method", req.Method),
 		slog.String("url", req.URL.String()),
 		slog.Any("headers", req.Header),
-		slog.Bool("without_redirects", r.withoutRedirects),
-		slog.Bool("use_app_client_proxy", r.useClientProxy),
-		slog.String("client_proxy_url", clientProxyURLRedacted),
-		slog.Bool("ignore_tls_errors", r.ignoreTLSErrors),
-		slog.Bool("disable_http2", r.disableHTTP2),
 	))
-
 	return client.Do(req)
 }
 

internal/reader/fetcher/request_builder_test.go

@@ -4,10 +4,12 @@
 package fetcher // import "miniflux.app/v2/internal/reader/fetcher"
 
 import (
+	"io"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
@@ -336,6 +338,84 @@ func TestRequestBuilder_ConnectionCloseHeader(t *testing.T) {
 	defer resp.Body.Close()
 }
 
+func TestRequestBuilder_ExecuteRequestKeepaliveNoConnectionClose(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if got := r.Header.Get("Connection"); got == "close" {
+			t.Errorf("ExecuteRequestKeepalive should not send 'Connection: close', got %q", got)
+		}
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer server.Close()
+
+	builder := NewRequestBuilder()
+	defer builder.Close()
+
+	resp, err := builder.ExecuteRequestKeepalive(server.URL)
+	if err != nil {
+		t.Fatalf("ExecuteRequestKeepalive: %v", err)
+	}
+	resp.Body.Close()
+}
+
+func TestRequestBuilder_ExecuteRequestKeepaliveReusesConnection(t *testing.T) {
+	var (
+		mu       sync.Mutex
+		remotes  = map[string]struct{}{}
+		requests int
+	)
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		mu.Lock()
+		remotes[r.RemoteAddr] = struct{}{}
+		requests++
+		mu.Unlock()
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer server.Close()
+
+	builder := NewRequestBuilder()
+	defer builder.Close()
+
+	for range 5 {
+		resp, err := builder.ExecuteRequestKeepalive(server.URL)
+		if err != nil {
+			t.Fatalf("ExecuteRequestKeepalive: %v", err)
+		}
+		// Drain + close so the connection returns to the idle pool and the
+		// next iteration can reuse it.
+		io.Copy(io.Discard, resp.Body)
+		resp.Body.Close()
+	}
+
+	if requests != 5 {
+		t.Fatalf("expected 5 requests, got %d", requests)
+	}
+	if len(remotes) != 1 {
+		t.Errorf("expected a single reused TCP connection, got %d distinct remotes: %v", len(remotes), remotes)
+	}
+}
+
+func TestRequestBuilder_ExecuteRequestDoesNotMutateBuilderHeaders(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer server.Close()
+
+	builder := NewRequestBuilder()
+	resp, err := builder.ExecuteRequest(server.URL)
+	if err != nil {
+		t.Fatalf("ExecuteRequest: %v", err)
+	}
+	resp.Body.Close()
+
+	// Per-request headers (Accept, Accept-Encoding, Connection) must not leak
+	// back into the builder so that it can be reused for further requests.
+	for _, h := range []string{"Accept", "Accept-Encoding", "Connection"} {
+		if got := builder.headers.Get(h); got != "" {
+			t.Errorf("builder.headers[%q] leaked from ExecuteRequest: %q", h, got)
+		}
+	}
+}
+
 func TestRequestBuilder_WithCustomApplicationProxyURL(t *testing.T) {
 	proxyURL, _ := url.Parse("http://proxy.example.com:8080")
 	builder := NewRequestBuilder()

internal/reader/subscription/finder.go

@@ -195,6 +195,13 @@ func (f *subscriptionFinder) findSubscriptionsFromWellKnownURLs(websiteURL strin
 		baseURLs = append(baseURLs, websiteURL)
 	}
 
+	// Some websites redirects unknown URLs to the home page.
+	// As result, the list of known URLs is returned to the subscription list.
+	// We don't want the user to choose between invalid feed URLs.
+	f.requestBuilder.WithoutRedirects()
+
+	defer f.requestBuilder.Close()
+
 	var subscriptions Subscriptions
 	for _, baseURL := range baseURLs {
 		for knownURL, kind := range knownURLs {
@@ -203,12 +210,7 @@ func (f *subscriptionFinder) findSubscriptionsFromWellKnownURLs(websiteURL strin
 				continue
 			}
 
-			// Some websites redirects unknown URLs to the home page.
-			// As result, the list of known URLs is returned to the subscription list.
-			// We don't want the user to choose between invalid feed URLs.
-			f.requestBuilder.WithoutRedirects()
-
-			responseHandler := fetcher.NewResponseHandler(f.requestBuilder.ExecuteRequest(fullURL))
+			responseHandler := fetcher.NewResponseHandler(f.requestBuilder.ExecuteRequestKeepalive(fullURL))
 			localizedError := responseHandler.LocalizedError()
 			responseHandler.Close()