Skip to content

HDDS-13921. Conditional Copy (CopyObject) #10207

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
peterxcli merged 8 commits into from
May 9, 2026
Merged

HDDS-13921. Conditional Copy (CopyObject) #10207

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
b815962
feat: add s3 copy source header constants.
YutaLin May 6, 2026
41b12c3
feat: evaluate copy resource precondition
YutaLin May 6, 2026
8523783
feat: handle conditional request in copy object
YutaLin May 6, 2026
1f067a4
test: conditional put request
YutaLin May 6, 2026
7db1431
test: conditional copy in S3SDKTests
YutaLin May 7, 2026
e320105
fix: test in S3SDKV1
YutaLin May 7, 2026
f4f9566
refactor: evaluate preconditions method
YutaLin May 8, 2026
a15aa35
test: add conditional copy destination test
YutaLin May 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 101 additions & 0 deletions ...tion-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@
import com.amazonaws.services.s3.model.CanonicalGrantee;
import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;
import com.amazonaws.services.s3.model.CopyObjectRequest;
import com.amazonaws.services.s3.model.CopyObjectResult;
import com.amazonaws.services.s3.model.CreateBucketRequest;
import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest;
import com.amazonaws.services.s3.model.GetObjectRequest;
Expand Down Expand Up @@ -494,6 +496,105 @@ public void testPutObjectIfMatchMissingKeyFail() {
assertEquals("NoSuchKey", missingKey.getErrorCode());
}

@Test
public void testCopyObject() {
final String sourceBucketName = getBucketName("source");
final String destBucketName = getBucketName("dest");
final String sourceKey = getKeyName("source");
final String destKey = getKeyName("dest");
final String content = "bar";
s3Client.createBucket(sourceBucketName);
s3Client.createBucket(destBucketName);

InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8));
PutObjectResult putResult = s3Client.putObject(sourceBucketName, sourceKey, is, new ObjectMetadata());
assertEquals("37b51d194a7513e45b56f6524f2d51f2", putResult.getETag());

CopyObjectResult copyResult = s3Client.copyObject(sourceBucketName, sourceKey, destBucketName, destKey);
assertEquals("37b51d194a7513e45b56f6524f2d51f2", copyResult.getETag());
}

@Test
public void testCopyObjectWithSourceIfMatch() {
final String sourceBucketName = getBucketName("source");
final String destBucketName = getBucketName("dest");
final String sourceKey = getKeyName("source");
final String destKey = getKeyName("dest");
final String content = "bar";
s3Client.createBucket(sourceBucketName);
s3Client.createBucket(destBucketName);

InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8));
PutObjectResult putResult = s3Client.putObject(sourceBucketName, sourceKey, is, new ObjectMetadata());
String sourceETag = putResult.getETag();

CopyObjectRequest copyRequest = new CopyObjectRequest(sourceBucketName, sourceKey, destBucketName, destKey)
.withMatchingETagConstraint(sourceETag);
CopyObjectResult copyResult = s3Client.copyObject(copyRequest);
assertEquals(sourceETag, copyResult.getETag());
}

@Test
public void testCopyObjectWithSourceIfMatchFail() {
final String sourceBucketName = getBucketName("source");
final String destBucketName = getBucketName("dest");
final String sourceKey = getKeyName("source");
final String destKey = getKeyName("dest");
final String content = "bar";
s3Client.createBucket(sourceBucketName);
s3Client.createBucket(destBucketName);

InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8));
s3Client.putObject(sourceBucketName, sourceKey, is, new ObjectMetadata());

CopyObjectRequest copyRequest = new CopyObjectRequest(sourceBucketName, sourceKey, destBucketName, destKey)
.withMatchingETagConstraint("wrong-etag");

CopyObjectResult copyResult = s3Client.copyObject(copyRequest);
assertNull(copyResult);
}

@Test
public void testCopyObjectWithSourceIfNoneMatch() {
final String sourceBucketName = getBucketName("source");
final String destBucketName = getBucketName("dest");
final String sourceKey = getKeyName("source");
final String destKey = getKeyName("dest");
final String content = "bar";
s3Client.createBucket(sourceBucketName);
s3Client.createBucket(destBucketName);

InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8));
PutObjectResult putResult = s3Client.putObject(sourceBucketName, sourceKey, is, new ObjectMetadata());
String sourceETag = putResult.getETag();

CopyObjectRequest copyRequest = new CopyObjectRequest(sourceBucketName, sourceKey, destBucketName, destKey)
.withNonmatchingETagConstraint("different-etag");
CopyObjectResult copyResult = s3Client.copyObject(copyRequest);
assertEquals(sourceETag, copyResult.getETag());
}

@Test
public void testCopyObjectWithSourceIfNoneMatchFail() {
final String sourceBucketName = getBucketName("source");
final String destBucketName = getBucketName("dest");
final String sourceKey = getKeyName("source");
final String destKey = getKeyName("dest");
final String content = "bar";
s3Client.createBucket(sourceBucketName);
s3Client.createBucket(destBucketName);

InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8));
PutObjectResult putResult = s3Client.putObject(sourceBucketName, sourceKey, is, new ObjectMetadata());
String sourceETag = putResult.getETag();

CopyObjectRequest copyRequest = new CopyObjectRequest(sourceBucketName, sourceKey, destBucketName, destKey)
.withNonmatchingETagConstraint(sourceETag);

CopyObjectResult copyResult = s3Client.copyObject(copyRequest);
assertNull(copyResult);
}

@Test
public void testPutObjectWithMD5Header() throws Exception {
final String bucketName = getBucketName();
Expand Down
223 changes: 223 additions & 0 deletions ...tion-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -992,6 +992,229 @@ public void testCopyObject() {
assertEquals("\"37b51d194a7513e45b56f6524f2d51f2\"", copyObjectResponse.copyObjectResult().eTag());
}

@Test
public void testCopyObjectWithSourceIfMatch() {
final String sourceBucketName = getBucketName("source");
final String destBucketName = getBucketName("dest");
final String sourceKey = getKeyName("source");
final String destKey = getKeyName("dest");
final String content = "bar";
s3Client.createBucket(b -> b.bucket(sourceBucketName));
s3Client.createBucket(b -> b.bucket(destBucketName));

PutObjectResponse putObjectResponse = s3Client.putObject(b -> b
.bucket(sourceBucketName)
.key(sourceKey),
RequestBody.fromString(content));

String sourceETag = putObjectResponse.eTag();

CopyObjectRequest copyReq = CopyObjectRequest.builder()
.sourceBucket(sourceBucketName)
.sourceKey(sourceKey)
.destinationBucket(destBucketName)
.destinationKey(destKey)
.copySourceIfMatch(sourceETag)
.build();

CopyObjectResponse copyObjectResponse = s3Client.copyObject(copyReq);
assertEquals(sourceETag, copyObjectResponse.copyObjectResult().eTag());
}

@Test
public void testCopyObjectWithSourceIfMatchFail() {
final String sourceBucketName = getBucketName("source");
final String destBucketName = getBucketName("dest");
final String sourceKey = getKeyName("source");
final String destKey = getKeyName("dest");
final String content = "bar";
s3Client.createBucket(b -> b.bucket(sourceBucketName));
s3Client.createBucket(b -> b.bucket(destBucketName));

s3Client.putObject(b -> b.bucket(sourceBucketName).key(sourceKey), RequestBody.fromString(content));

CopyObjectRequest copyReq = CopyObjectRequest.builder()
.sourceBucket(sourceBucketName)
.sourceKey(sourceKey)
.destinationBucket(destBucketName)
.destinationKey(destKey)
.copySourceIfMatch("wrong-etag")
.build();

S3Exception exception = assertThrows(S3Exception.class, () -> s3Client.copyObject(copyReq));
assertEquals(412, exception.statusCode());
assertEquals("PreconditionFailed", exception.awsErrorDetails().errorCode());
}

@Test
public void testCopyObjectWithSourceIfNoneMatch() {
final String sourceBucketName = getBucketName("source");
final String destBucketName = getBucketName("dest");
final String sourceKey = getKeyName("source");
final String destKey = getKeyName("dest");
final String content = "bar";
s3Client.createBucket(b -> b.bucket(sourceBucketName));
s3Client.createBucket(b -> b.bucket(destBucketName));

PutObjectResponse putObjectResponse = s3Client.putObject(b -> b
.bucket(sourceBucketName)
.key(sourceKey),
RequestBody.fromString(content));

String sourceETag = putObjectResponse.eTag();

CopyObjectRequest copyReq = CopyObjectRequest.builder()
.sourceBucket(sourceBucketName)
.sourceKey(sourceKey)
.destinationBucket(destBucketName)
.destinationKey(destKey)
.copySourceIfNoneMatch("different-etag")
.build();

CopyObjectResponse copyObjectResponse = s3Client.copyObject(copyReq);
assertEquals(sourceETag, copyObjectResponse.copyObjectResult().eTag());
}

@Test
public void testCopyObjectWithSourceIfNoneMatchFail() {
final String sourceBucketName = getBucketName("source");
final String destBucketName = getBucketName("dest");
final String sourceKey = getKeyName("source");
final String destKey = getKeyName("dest");
final String content = "bar";
s3Client.createBucket(b -> b.bucket(sourceBucketName));
s3Client.createBucket(b -> b.bucket(destBucketName));

PutObjectResponse putObjectResponse = s3Client.putObject(b -> b
.bucket(sourceBucketName)
.key(sourceKey),
RequestBody.fromString(content));

String sourceETag = putObjectResponse.eTag();

CopyObjectRequest copyReq = CopyObjectRequest.builder()
.sourceBucket(sourceBucketName)
.sourceKey(sourceKey)
.destinationBucket(destBucketName)
.destinationKey(destKey)
.copySourceIfNoneMatch(sourceETag)
.build();

S3Exception exception = assertThrows(S3Exception.class, () -> s3Client.copyObject(copyReq));
assertEquals(412, exception.statusCode());
assertEquals("PreconditionFailed", exception.awsErrorDetails().errorCode());
}

@Test
public void testCopyObjectWithDestinationIfNoneMatch() {
final String sourceBucketName = getBucketName("source");
final String destBucketName = getBucketName("dest");
final String sourceKey = getKeyName("source");
final String destKey = getKeyName("dest");
final String content = "bar";
s3Client.createBucket(b -> b.bucket(sourceBucketName));
s3Client.createBucket(b -> b.bucket(destBucketName));

PutObjectResponse putObjectResponse = s3Client.putObject(b -> b
.bucket(sourceBucketName)
.key(sourceKey),
RequestBody.fromString(content));

String sourceETag = putObjectResponse.eTag();

CopyObjectRequest copyReq = CopyObjectRequest.builder()
.sourceBucket(sourceBucketName)
.sourceKey(sourceKey)
.destinationBucket(destBucketName)
.destinationKey(destKey)
.ifNoneMatch("*")
.build();

CopyObjectResponse copyObjectResponse = s3Client.copyObject(copyReq);
assertEquals(sourceETag, copyObjectResponse.copyObjectResult().eTag());
}

@Test
public void testCopyObjectWithDestinationIfNoneMatchFail() {
final String sourceBucketName = getBucketName("source");
final String destBucketName = getBucketName("dest");
final String sourceKey = getKeyName("source");
final String destKey = getKeyName("dest");
final String content = "bar";
s3Client.createBucket(b -> b.bucket(sourceBucketName));
s3Client.createBucket(b -> b.bucket(destBucketName));

s3Client.putObject(b -> b.bucket(sourceBucketName).key(sourceKey), RequestBody.fromString(content));
s3Client.putObject(b -> b.bucket(destBucketName).key(destKey), RequestBody.fromString("existing"));

CopyObjectRequest copyReq = CopyObjectRequest.builder()
.sourceBucket(sourceBucketName)
.sourceKey(sourceKey)
.destinationBucket(destBucketName)
.destinationKey(destKey)
.ifNoneMatch("*")
.build();

S3Exception exception = assertThrows(S3Exception.class, () -> s3Client.copyObject(copyReq));
assertEquals(412, exception.statusCode());
assertEquals("PreconditionFailed", exception.awsErrorDetails().errorCode());
}

@Test
public void testCopyObjectWithDestinationIfMatch() {
final String sourceBucketName = getBucketName("source");
final String destBucketName = getBucketName("dest");
final String sourceKey = getKeyName("source");
final String destKey = getKeyName("dest");
final String content = "bar";
s3Client.createBucket(b -> b.bucket(sourceBucketName));
s3Client.createBucket(b -> b.bucket(destBucketName));

s3Client.putObject(b -> b.bucket(sourceBucketName).key(sourceKey), RequestBody.fromString(content));
PutObjectResponse destPutResponse = s3Client.putObject(b -> b
.bucket(destBucketName)
.key(destKey),
RequestBody.fromString("existing"));
String destETag = destPutResponse.eTag();

CopyObjectRequest copyReq = CopyObjectRequest.builder()
.sourceBucket(sourceBucketName)
.sourceKey(sourceKey)
.destinationBucket(destBucketName)
.destinationKey(destKey)
.ifMatch(destETag)
.build();

CopyObjectResponse copyObjectResponse = s3Client.copyObject(copyReq);
assertNotNull(copyObjectResponse.copyObjectResult().eTag());
}

@Test
public void testCopyObjectWithDestinationIfMatchFail() {
final String sourceBucketName = getBucketName("source");
final String destBucketName = getBucketName("dest");
final String sourceKey = getKeyName("source");
final String destKey = getKeyName("dest");
final String content = "bar";
s3Client.createBucket(b -> b.bucket(sourceBucketName));
s3Client.createBucket(b -> b.bucket(destBucketName));

s3Client.putObject(b -> b.bucket(sourceBucketName).key(sourceKey), RequestBody.fromString(content));
s3Client.putObject(b -> b.bucket(destBucketName).key(destKey), RequestBody.fromString("existing"));

CopyObjectRequest copyReq = CopyObjectRequest.builder()
.sourceBucket(sourceBucketName)
.sourceKey(sourceKey)
.destinationBucket(destBucketName)
.destinationKey(destKey)
.ifMatch("wrong-etag")
.build();

S3Exception exception = assertThrows(S3Exception.class, () -> s3Client.copyObject(copyReq));
assertEquals(412, exception.statusCode());
assertEquals("PreconditionFailed", exception.awsErrorDetails().errorCode());
}

@Test
public void testLowLevelMultipartUpload(@TempDir Path tempDir) throws Exception {
final String bucketName = getBucketName();
Expand Down
Loading