feat: add automatic S3 request retry with exponential backoff

api/src/main/java/io/minio/BaseS3Client.java

@@ -51,13 +51,11 @@
 import java.io.OutputStreamWriter;
 import java.io.PrintWriter;
 import java.nio.charset.StandardCharsets;
-import java.security.SecureRandom;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
-import java.util.Random;
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionException;
@@ -92,7 +90,6 @@ public abstract class BaseS3Client implements AutoCloseable {
       "ServerSideEncryptionConfigurationNotFoundError";
   // maximum allowed bucket policy size is 20KiB
   protected static final int MAX_BUCKET_POLICY_SIZE = 20 * 1024;
-  protected static final Random RANDOM = new Random(new SecureRandom().nextLong());
   protected static final ObjectMapper OBJECT_MAPPER =
       JsonMapper.builder()
           .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
@@ -104,6 +101,7 @@ public abstract class BaseS3Client implements AutoCloseable {
   private static final String UPLOAD_ID = "uploadId";
   private static final Set<String> TRACE_QUERY_PARAMS =
       ImmutableSet.of("retention", "legal-hold", "tagging", UPLOAD_ID, "acl", "attributes");
+
   private PrintWriter traceStream;
   protected final Map<String, String> regionCache = new ConcurrentHashMap<>();
   protected String userAgent = Utils.getDefaultUserAgent();
@@ -268,7 +266,10 @@ private String[] handleRedirectResponse(
     return new String[] {code, message};
   }
 
-  /** Execute HTTP request asynchronously for given parameters. */
+  /**
+   * Execute HTTP request asynchronously for given parameters. Retries on retryable HTTP status
+   * codes are handled by {@link Http.RetryInterceptor} installed on the default HTTP client.
+   */
   protected CompletableFuture<Response> executeAsync(Http.S3Request s3request, String region) {
     Credentials credentials = (provider == null) ? null : provider.fetch();
     Http.Request request = null;
@@ -282,15 +283,9 @@ protected CompletableFuture<Response> executeAsync(Http.S3Request s3request, Str
     PrintWriter traceStream = this.traceStream;
     if (traceStream != null) traceStream.print(request.httpTraces());
 
-    OkHttpClient httpClient = this.httpClient;
-    // FIXME: enable retry for all request.
-    // if (!s3request.retryFailure()) {
-    //   httpClient = httpClient.newBuilder().retryOnConnectionFailure(false).build();
-    // }
-
     okhttp3.Request httpRequest = request.httpRequest();
     CompletableFuture<Response> completableFuture = newCompleteableFuture();
-    httpClient
+    this.httpClient
         .newCall(httpRequest)
         .enqueue(
             new Callback() {
@@ -469,9 +464,10 @@ private void onResponse(final Response response) throws IOException {
                           response.header("x-amz-id-2"));
                 }
 
-                // invalidate region cache if needed
-                if (errorResponse.code().equals(NO_SUCH_BUCKET)
-                    || errorResponse.code().equals(RETRY_HEAD)) {
+                // invalidate region cache if needed (bucket may be null for e.g. listBuckets)
+                if (s3request.bucket() != null
+                    && (errorResponse.code().equals(NO_SUCH_BUCKET)
+                        || errorResponse.code().equals(RETRY_HEAD))) {
                   regionCache.remove(s3request.bucket());
                 }
 
@@ -1223,6 +1219,15 @@ private Object[] createBody(PutObjectAPIBaseArgs args, MediaType contentType)
     boolean checksumHeader = headers.namePrefixAny("x-amz-checksum-");
     String md5Hash = headers.getFirst(Http.Headers.CONTENT_MD5);
 
+    long fileStartPos = 0;
+    if (args.file() != null) {
+      try {
+        fileStartPos = args.file().getFilePointer();
+      } catch (IOException e) {
+        throw new MinioException(e);
+      }
+    }
+
     if (sha256HexString == null && sha256Base64String == null) {
       if (!baseUrl.isHttps()) {
         Checksum.Hasher hasher = Checksum.Algorithm.SHA256.hasher();
@@ -1278,6 +1283,14 @@ private Object[] createBody(PutObjectAPIBaseArgs args, MediaType contentType)
       }
     }
 
+    if (args.file() != null) {
+      try {
+        args.file().seek(fileStartPos);
+      } catch (IOException e) {
+        throw new MinioException(e);
+      }
+    }
+
     Http.Body body = null;
     if (args.file() != null) {
       body = new Http.Body(args.file(), args.length(), contentType, sha256HexString, md5Hash);

api/src/main/java/io/minio/Http.java

@@ -52,6 +52,7 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.TimeUnit;
 import java.util.regex.Matcher;
 import java.util.stream.Collectors;
@@ -65,6 +66,7 @@
 import javax.net.ssl.TrustManager;
 import javax.net.ssl.TrustManagerFactory;
 import javax.net.ssl.X509TrustManager;
+import okhttp3.Interceptor;
 import okhttp3.MediaType;
 import okhttp3.OkHttpClient;
 import okhttp3.Protocol;
@@ -620,6 +622,7 @@ public static OkHttpClient newDefaultClient() {
             .writeTimeout(DEFAULT_TIMEOUT, TimeUnit.MILLISECONDS)
             .readTimeout(DEFAULT_TIMEOUT, TimeUnit.MILLISECONDS)
             .protocols(Arrays.asList(Protocol.HTTP_1_1))
+            .addInterceptor(new RetryInterceptor())
             .build();
     try {
       return enableExternalCertificatesFromEnv(client);
@@ -683,10 +686,57 @@ public static OkHttpClient setTimeout(
         .build();
   }
 
+  /**
+   * OkHttp interceptor that retries requests on retryable HTTP status codes with exponential
+   * backoff and full jitter.
+   */
+  public static class RetryInterceptor implements Interceptor {
+    private static final int MAX_RETRIES = 5;
+    private static final long BASE_DELAY_MS = 200L;
+    private static final long MAX_DELAY_MS = 30_000L;
+    private static final Set<Integer> RETRYABLE_STATUS_CODES =
+        ImmutableSet.of(
+            408, // Request Timeout
+            429, // Too Many Requests
+            499, // Client Closed Request (nginx)
+            500, // Internal Server Error
+            502, // Bad Gateway
+            503, // Service Unavailable
+            504, // Gateway Timeout
+            520); // Cloudflare unknown error
+
+    @Override
+    public okhttp3.Response intercept(Chain chain) throws IOException {
+      okhttp3.Request request = chain.request();
+      okhttp3.Response response = chain.proceed(request);
+
+      int tryCount = 0;
+      while (RETRYABLE_STATUS_CODES.contains(response.code()) && tryCount < MAX_RETRIES) {
+        tryCount++;
+        response.close();
+
+        // Cap exponent to avoid bit-shift overflow at high attempt counts.
+        int exp = Math.min(tryCount, 30);
+        long backoffCap = Math.min(MAX_DELAY_MS, BASE_DELAY_MS * (1L << exp));
+        long jittered = ThreadLocalRandom.current().nextLong(0, Math.max(1, backoffCap));
+        try {
+          Thread.sleep(jittered);
+        } catch (InterruptedException ie) {
+          Thread.currentThread().interrupt();
+          throw new IOException("retry interrupted", ie);
+        }
+
+        response = chain.proceed(request);
+      }
+      return response;
+    }
+  }
+
   /** HTTP body of {@link RandomAccessFile}, {@link ByteBuffer} or {@link byte} array. */
   public static class Body {
     private okhttp3.RequestBody requestBody;
     private RandomAccessFile file;
+    private long fileOffset;
     private ByteBuffer buffer;
     private byte[] data;
     private Long length;
@@ -710,6 +760,11 @@ public Body(
         String md5Hash) {
       if (length < 0) throw new IllegalArgumentException("valid length must be provided");
       this.file = file;
+      try {
+        this.fileOffset = file.getFilePointer();
+      } catch (IOException e) {
+        throw new IllegalStateException("failed to read file position", e);
+      }
       set(length, contentType, sha256Hash, md5Hash);
     }
 
@@ -783,7 +838,14 @@ public Headers headers() {
     /** Creates HTTP RequestBody for this body. */
     public RequestBody toRequestBody() throws MinioException {
       if (requestBody != null) return new RequestBody(requestBody);
-      if (file != null) return new RequestBody(file, length, contentType);
+      if (file != null) {
+        try {
+          file.seek(fileOffset);
+        } catch (IOException e) {
+          throw new MinioException(e);
+        }
+        return new RequestBody(file, length, contentType);
+      }
       if (buffer != null) return new RequestBody(buffer, contentType);
       return new RequestBody(data, length.intValue(), contentType);
     }
@@ -1500,6 +1562,11 @@ public String object() {
       return object;
     }
 
+    /** Returns the request body, or {@code null} if none was set. */
+    public Body body() {
+      return body;
+    }
+
     private Request toRequest(
         BaseUrl baseUrl, String region, Credentials credentials, Integer expiry)
         throws MinioException {

api/src/main/java/io/minio/MinioAsyncClient.java

@@ -93,6 +93,7 @@
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
+import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.atomic.AtomicBoolean;
 import okhttp3.HttpUrl;
 import okhttp3.MediaType;
@@ -3356,7 +3357,7 @@ public CompletableFuture<PutObjectFanOutResponse> putObjectFanOut(PutObjectFanOu
               // Build POST object data
               String objectName =
                   "fan-out-"
-                      + new BigInteger(32, RANDOM).toString(32)
+                      + new BigInteger(32, ThreadLocalRandom.current()).toString(32)
                       + "-"
                       + System.currentTimeMillis();
               PostPolicy policy =

api/src/test/java/io/minio/RetryTest.java

@@ -0,0 +1,89 @@
+/*
+ * MinIO Java SDK for Amazon S3 Compatible Cloud Storage, (C) 2026 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.minio;
+
+import io.minio.errors.ErrorResponseException;
+import io.minio.errors.MinioException;
+import java.io.IOException;
+import okhttp3.mockwebserver.MockResponse;
+import okhttp3.mockwebserver.MockWebServer;
+import okio.Buffer;
+import org.junit.Assert;
+import org.junit.Test;
+
+/** Integration tests for {@link Http.RetryInterceptor}. */
+public class RetryTest {
+  private static final String LIST_BUCKETS_OK =
+      "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
+          + "<ListAllMyBucketsResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
+          + "<Owner><ID>test</ID><DisplayName>test</DisplayName></Owner>"
+          + "<Buckets/></ListAllMyBucketsResult>";
+
+  private MockResponse successResponse() {
+    return new MockResponse()
+        .setResponseCode(200)
+        .setHeader("Content-Type", "application/xml")
+        .setBody(new Buffer().writeUtf8(LIST_BUCKETS_OK));
+  }
+
+  private MockResponse retryableServerError() {
+    return new MockResponse()
+        .setResponseCode(503)
+        .setHeader("Content-Type", "text/html")
+        .setBody(new Buffer().writeUtf8("<html>Service Unavailable</html>"));
+  }
+
+  @Test
+  public void testRetryOnRetryableStatusThenSuccess() throws IOException, MinioException {
+    try (MockWebServer server = new MockWebServer()) {
+      server.enqueue(retryableServerError());
+      server.enqueue(successResponse());
+      server.start();
+
+      MinioClient client = MinioClient.builder().endpoint(server.url("").toString()).build();
+      client.listBuckets();
+
+      Assert.assertEquals(2, server.getRequestCount());
+    }
+  }
+
+  @Test
+  public void testNoRetryOn4xx() throws IOException, MinioException {
+    String notFoundXml =
+        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
+            + "<Error><Code>NoSuchBucket</Code><Message>not found</Message>"
+            + "<Resource>/</Resource><RequestId>abc</RequestId></Error>";
+
+    try (MockWebServer server = new MockWebServer()) {
+      server.enqueue(
+          new MockResponse()
+              .setResponseCode(404)
+              .setHeader("Content-Type", "application/xml")
+              .setBody(new Buffer().writeUtf8(notFoundXml)));
+      server.start();
+
+      MinioClient client = MinioClient.builder().endpoint(server.url("").toString()).build();
+      try {
+        client.listBuckets();
+        Assert.fail("expected ErrorResponseException");
+      } catch (ErrorResponseException e) {
+        Assert.assertEquals("NoSuchBucket", e.errorResponse().code());
+      }
+      Assert.assertEquals(1, server.getRequestCount());
+    }
+  }
+}