feat: add automatic S3 request retry with exponential backoff

api/src/main/java/io/minio/BaseS3Client.java

@@ -1,6 +1,6 @@
 /*
  * MinIO Java SDK for Amazon S3 Compatible Cloud Storage,
- * (C) 2025 MinIO, Inc.
+ * (C) 2025-2026 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -51,13 +51,11 @@
 import java.io.OutputStreamWriter;
 import java.io.PrintWriter;
 import java.nio.charset.StandardCharsets;
-import java.security.SecureRandom;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
-import java.util.Random;
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionException;
@@ -92,7 +90,6 @@ public abstract class BaseS3Client implements AutoCloseable {
       "ServerSideEncryptionConfigurationNotFoundError";
   // maximum allowed bucket policy size is 20KiB
   protected static final int MAX_BUCKET_POLICY_SIZE = 20 * 1024;
-  protected static final Random RANDOM = new Random(new SecureRandom().nextLong());
   protected static final ObjectMapper OBJECT_MAPPER =
       JsonMapper.builder()
           .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
@@ -104,6 +101,7 @@ public abstract class BaseS3Client implements AutoCloseable {
   private static final String UPLOAD_ID = "uploadId";
   private static final Set<String> TRACE_QUERY_PARAMS =
       ImmutableSet.of("retention", "legal-hold", "tagging", UPLOAD_ID, "acl", "attributes");
+
   private PrintWriter traceStream;
   protected final Map<String, String> regionCache = new ConcurrentHashMap<>();
   protected String userAgent = Utils.getDefaultUserAgent();
@@ -113,19 +111,59 @@ public abstract class BaseS3Client implements AutoCloseable {
   protected OkHttpClient httpClient;
   protected boolean closeHttpClient;
 
+  /**
+   * Maximum attempts per S3 request. Default {@link Retry#MAX_RETRY}. Read on every request via the
+   * retry interceptor's supplier so runtime tuning via {@link #setMaxRetries(int)} takes immediate
+   * effect. Package-private so the {@code Builder} classes in this package can seed it via the
+   * setter; subclasses must go through {@link #setMaxRetries(int)}.
+   */
+  volatile int maxRetries = Retry.MAX_RETRY;
+
   protected BaseS3Client(
       Http.BaseUrl baseUrl, Provider provider, OkHttpClient httpClient, boolean closeHttpClient) {
     this.baseUrl = baseUrl;
     this.provider = provider;
-    this.httpClient = httpClient;
     this.closeHttpClient = closeHttpClient;
+    this.httpClient = wrapWithRetry(httpClient);
   }
 
   protected BaseS3Client(BaseS3Client client) {
     this.baseUrl = client.baseUrl;
     this.provider = client.provider;
-    this.httpClient = client.httpClient;
     this.closeHttpClient = client.closeHttpClient;
+    this.maxRetries = client.maxRetries;
+    this.httpClient = wrapWithRetry(client.httpClient);
+  }
+
+  /**
+   * Re-wires the retry interceptor on {@code client} so it reads {@link #maxRetries} from this
+   * instance. Strips any prior {@link Http.RetryInterceptor} from BOTH the application-interceptor
+   * and network-interceptor chains (e.g. one bound to a different instance via the copy
+   * constructor, or accidentally registered on the network chain) before installing this one as an
+   * application interceptor.
+   *
+   * <p>Also forces {@code retryOnConnectionFailure(false)} on the underlying client. The SDK's own
+   * {@link Http.RetryInterceptor} is the single source of retry policy; layering OkHttp's
+   * connection-level retry on top would double-count attempts and obscure the {@link #maxRetries}
+   * budget. This silently overrides any {@code retryOnConnectionFailure(true)} that a
+   * caller-supplied client was configured with — by design.
+   */
+  private OkHttpClient wrapWithRetry(OkHttpClient client) {
+    OkHttpClient.Builder builder = client.newBuilder().retryOnConnectionFailure(false);
+    builder.interceptors().removeIf(i -> i instanceof Http.RetryInterceptor);
+    builder.networkInterceptors().removeIf(i -> i instanceof Http.RetryInterceptor);
+    return builder.addInterceptor(new Http.RetryInterceptor(() -> this.maxRetries)).build();
+  }
+
+  /**
+   * Sets the maximum number of attempts for transient HTTP failures. Pass {@code 1} to disable
+   * automatic retries. Defaults to {@code 10}.
+   *
+   * @param maxRetries maximum attempts (must be {@code >= 1}).
+   */
+  public void setMaxRetries(int maxRetries) {
+    if (maxRetries < 1) throw new IllegalArgumentException("maxRetries must be >= 1");
+    this.maxRetries = maxRetries;
   }
 
   /** Closes underneath HTTP client. */
@@ -182,6 +220,11 @@ public void setAppInfo(String name, String version) {
   /**
    * Enables HTTP call tracing and written to traceStream.
    *
+   * <p><b>Retry caveat.</b> Tracing happens at the SDK callback level, so only the final response
+   * of a retry sequence is recorded. Per-attempt traces (which the {@link Http.RetryInterceptor}
+   * sees and discards) are not surfaced here. To inspect individual retry attempts, register an
+   * OkHttp {@code HttpLoggingInterceptor} on a custom client.
+   *
    * @param traceStream {@link OutputStream} for writing HTTP call tracing.
    * @see #traceOff
    */
@@ -268,7 +311,13 @@ private String[] handleRedirectResponse(
     return new String[] {code, message};
   }
 
-  /** Execute HTTP request asynchronously for given parameters. */
+  /**
+   * Execute HTTP request asynchronously for given parameters. Retries on retryable IOException,
+   * HTTP status, and S3 error code are handled by {@link Http.RetryInterceptor}, which {@link
+   * #wrapWithRetry} installs on the underlying {@link OkHttpClient} regardless of whether the
+   * client is the default or caller-supplied. The attempt budget is read from {@link #maxRetries}
+   * on every request.
+   */
   protected CompletableFuture<Response> executeAsync(Http.S3Request s3request, String region) {
     Credentials credentials = (provider == null) ? null : provider.fetch();
     Http.Request request = null;
@@ -282,15 +331,9 @@ protected CompletableFuture<Response> executeAsync(Http.S3Request s3request, Str
     PrintWriter traceStream = this.traceStream;
     if (traceStream != null) traceStream.print(request.httpTraces());
 
-    OkHttpClient httpClient = this.httpClient;
-    // FIXME: enable retry for all request.
-    // if (!s3request.retryFailure()) {
-    //   httpClient = httpClient.newBuilder().retryOnConnectionFailure(false).build();
-    // }
-
     okhttp3.Request httpRequest = request.httpRequest();
     CompletableFuture<Response> completableFuture = newCompleteableFuture();
-    httpClient
+    this.httpClient
         .newCall(httpRequest)
         .enqueue(
             new Callback() {
@@ -469,9 +512,10 @@ private void onResponse(final Response response) throws IOException {
                           response.header("x-amz-id-2"));
                 }
 
-                // invalidate region cache if needed
-                if (errorResponse.code().equals(NO_SUCH_BUCKET)
-                    || errorResponse.code().equals(RETRY_HEAD)) {
+                // invalidate region cache if needed (bucket may be null for e.g. listBuckets)
+                if (s3request.bucket() != null
+                    && (errorResponse.code().equals(NO_SUCH_BUCKET)
+                        || errorResponse.code().equals(RETRY_HEAD))) {
                   regionCache.remove(s3request.bucket());
                 }
 
@@ -1223,6 +1267,15 @@ private Object[] createBody(PutObjectAPIBaseArgs args, MediaType contentType)
     boolean checksumHeader = headers.namePrefixAny("x-amz-checksum-");
     String md5Hash = headers.getFirst(Http.Headers.CONTENT_MD5);
 
+    long fileStartPos = 0;
+    if (args.file() != null) {
+      try {
+        fileStartPos = args.file().getFilePointer();
+      } catch (IOException e) {
+        throw new MinioException(e);
+      }
+    }
+
     if (sha256HexString == null && sha256Base64String == null) {
       if (!baseUrl.isHttps()) {
         Checksum.Hasher hasher = Checksum.Algorithm.SHA256.hasher();
@@ -1278,6 +1331,14 @@ private Object[] createBody(PutObjectAPIBaseArgs args, MediaType contentType)
       }
     }
 
+    if (args.file() != null) {
+      try {
+        args.file().seek(fileStartPos);
+      } catch (IOException e) {
+        throw new MinioException(e);
+      }
+    }
+
     Http.Body body = null;
     if (args.file() != null) {
       body = new Http.Body(args.file(), args.length(), contentType, sha256HexString, md5Hash);