Skip to content

HDDS-13921. Conditional Copy (CopyObject) #10207

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
peterxcli merged 8 commits into from
May 9, 2026
Merged

HDDS-13921. Conditional Copy (CopyObject) #10207

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
b815962
feat: add s3 copy source header constants.
YutaLin May 6, 2026
41b12c3
feat: evaluate copy resource precondition
YutaLin May 6, 2026
8523783
feat: handle conditional request in copy object
YutaLin May 6, 2026
1f067a4
test: conditional put request
YutaLin May 6, 2026
7db1431
test: conditional copy in S3SDKTests
YutaLin May 7, 2026
e320105
fix: test in S3SDKV1
YutaLin May 7, 2026
f4f9566
refactor: evaluate preconditions method
YutaLin May 8, 2026
a15aa35
test: add conditional copy destination test
YutaLin May 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 101 additions & 0 deletions ...tion-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@
import com.amazonaws.services.s3.model.CanonicalGrantee;
import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;
import com.amazonaws.services.s3.model.CopyObjectRequest;
import com.amazonaws.services.s3.model.CopyObjectResult;
import com.amazonaws.services.s3.model.CreateBucketRequest;
import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest;
import com.amazonaws.services.s3.model.GetObjectRequest;
Expand Down Expand Up @@ -494,6 +496,105 @@ public void testPutObjectIfMatchMissingKeyFail() {
assertEquals("NoSuchKey", missingKey.getErrorCode());
}

@Test
public void testCopyObject() {
final String sourceBucketName = getBucketName("source");
final String destBucketName = getBucketName("dest");
final String sourceKey = getKeyName("source");
final String destKey = getKeyName("dest");
final String content = "bar";
s3Client.createBucket(sourceBucketName);
s3Client.createBucket(destBucketName);

InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8));
PutObjectResult putResult = s3Client.putObject(sourceBucketName, sourceKey, is, new ObjectMetadata());
assertEquals("37b51d194a7513e45b56f6524f2d51f2", putResult.getETag());

CopyObjectResult copyResult = s3Client.copyObject(sourceBucketName, sourceKey, destBucketName, destKey);
assertEquals("37b51d194a7513e45b56f6524f2d51f2", copyResult.getETag());
}

@Test
public void testCopyObjectWithSourceIfMatch() {
final String sourceBucketName = getBucketName("source");
final String destBucketName = getBucketName("dest");
final String sourceKey = getKeyName("source");
final String destKey = getKeyName("dest");
final String content = "bar";
s3Client.createBucket(sourceBucketName);
s3Client.createBucket(destBucketName);

InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8));
PutObjectResult putResult = s3Client.putObject(sourceBucketName, sourceKey, is, new ObjectMetadata());
String sourceETag = putResult.getETag();

CopyObjectRequest copyRequest = new CopyObjectRequest(sourceBucketName, sourceKey, destBucketName, destKey)
.withMatchingETagConstraint(sourceETag);
CopyObjectResult copyResult = s3Client.copyObject(copyRequest);
assertEquals(sourceETag, copyResult.getETag());
}

@Test
public void testCopyObjectWithSourceIfMatchFail() {
final String sourceBucketName = getBucketName("source");
final String destBucketName = getBucketName("dest");
final String sourceKey = getKeyName("source");
final String destKey = getKeyName("dest");
final String content = "bar";
s3Client.createBucket(sourceBucketName);
s3Client.createBucket(destBucketName);

InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8));
s3Client.putObject(sourceBucketName, sourceKey, is, new ObjectMetadata());

CopyObjectRequest copyRequest = new CopyObjectRequest(sourceBucketName, sourceKey, destBucketName, destKey)
.withMatchingETagConstraint("wrong-etag");

CopyObjectResult copyResult = s3Client.copyObject(copyRequest);
assertNull(copyResult);
}

@Test
public void testCopyObjectWithSourceIfNoneMatch() {
final String sourceBucketName = getBucketName("source");
final String destBucketName = getBucketName("dest");
final String sourceKey = getKeyName("source");
final String destKey = getKeyName("dest");
final String content = "bar";
s3Client.createBucket(sourceBucketName);
s3Client.createBucket(destBucketName);

InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8));
PutObjectResult putResult = s3Client.putObject(sourceBucketName, sourceKey, is, new ObjectMetadata());
String sourceETag = putResult.getETag();

CopyObjectRequest copyRequest = new CopyObjectRequest(sourceBucketName, sourceKey, destBucketName, destKey)
.withNonmatchingETagConstraint("different-etag");
CopyObjectResult copyResult = s3Client.copyObject(copyRequest);
assertEquals(sourceETag, copyResult.getETag());
}

@Test
public void testCopyObjectWithSourceIfNoneMatchFail() {
final String sourceBucketName = getBucketName("source");
final String destBucketName = getBucketName("dest");
final String sourceKey = getKeyName("source");
final String destKey = getKeyName("dest");
final String content = "bar";
s3Client.createBucket(sourceBucketName);
s3Client.createBucket(destBucketName);

InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8));
PutObjectResult putResult = s3Client.putObject(sourceBucketName, sourceKey, is, new ObjectMetadata());
String sourceETag = putResult.getETag();

CopyObjectRequest copyRequest = new CopyObjectRequest(sourceBucketName, sourceKey, destBucketName, destKey)
.withNonmatchingETagConstraint(sourceETag);

CopyObjectResult copyResult = s3Client.copyObject(copyRequest);
assertNull(copyResult);
}

@Test
public void testPutObjectWithMD5Header() throws Exception {
final String bucketName = getBucketName();
Expand Down
113 changes: 113 additions & 0 deletions ...tion-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -992,6 +992,119 @@ public void testCopyObject() {
assertEquals("\"37b51d194a7513e45b56f6524f2d51f2\"", copyObjectResponse.copyObjectResult().eTag());
}

@Test
public void testCopyObjectWithSourceIfMatch() {
final String sourceBucketName = getBucketName("source");
final String destBucketName = getBucketName("dest");
final String sourceKey = getKeyName("source");
final String destKey = getKeyName("dest");
final String content = "bar";
s3Client.createBucket(b -> b.bucket(sourceBucketName));
s3Client.createBucket(b -> b.bucket(destBucketName));

PutObjectResponse putObjectResponse = s3Client.putObject(b -> b
.bucket(sourceBucketName)
.key(sourceKey),
RequestBody.fromString(content));

String sourceETag = putObjectResponse.eTag();

CopyObjectRequest copyReq = CopyObjectRequest.builder()
.sourceBucket(sourceBucketName)
.sourceKey(sourceKey)
.destinationBucket(destBucketName)
.destinationKey(destKey)
.copySourceIfMatch(sourceETag)
.build();

CopyObjectResponse copyObjectResponse = s3Client.copyObject(copyReq);
assertEquals(sourceETag, copyObjectResponse.copyObjectResult().eTag());
}

@Test
public void testCopyObjectWithSourceIfMatchFail() {
final String sourceBucketName = getBucketName("source");
final String destBucketName = getBucketName("dest");
final String sourceKey = getKeyName("source");
final String destKey = getKeyName("dest");
final String content = "bar";
s3Client.createBucket(b -> b.bucket(sourceBucketName));
s3Client.createBucket(b -> b.bucket(destBucketName));

s3Client.putObject(b -> b.bucket(sourceBucketName).key(sourceKey), RequestBody.fromString(content));

CopyObjectRequest copyReq = CopyObjectRequest.builder()
.sourceBucket(sourceBucketName)
.sourceKey(sourceKey)
.destinationBucket(destBucketName)
.destinationKey(destKey)
.copySourceIfMatch("wrong-etag")
.build();

S3Exception exception = assertThrows(S3Exception.class, () -> s3Client.copyObject(copyReq));
assertEquals(412, exception.statusCode());
assertEquals("PreconditionFailed", exception.awsErrorDetails().errorCode());
}

@Test
public void testCopyObjectWithSourceIfNoneMatch() {
final String sourceBucketName = getBucketName("source");
final String destBucketName = getBucketName("dest");
final String sourceKey = getKeyName("source");
final String destKey = getKeyName("dest");
final String content = "bar";
s3Client.createBucket(b -> b.bucket(sourceBucketName));
s3Client.createBucket(b -> b.bucket(destBucketName));

PutObjectResponse putObjectResponse = s3Client.putObject(b -> b
.bucket(sourceBucketName)
.key(sourceKey),
RequestBody.fromString(content));

String sourceETag = putObjectResponse.eTag();

CopyObjectRequest copyReq = CopyObjectRequest.builder()
.sourceBucket(sourceBucketName)
.sourceKey(sourceKey)
.destinationBucket(destBucketName)
.destinationKey(destKey)
.copySourceIfNoneMatch("different-etag")
.build();

CopyObjectResponse copyObjectResponse = s3Client.copyObject(copyReq);
assertEquals(sourceETag, copyObjectResponse.copyObjectResult().eTag());
}

@Test
public void testCopyObjectWithSourceIfNoneMatchFail() {
final String sourceBucketName = getBucketName("source");
final String destBucketName = getBucketName("dest");
final String sourceKey = getKeyName("source");
final String destKey = getKeyName("dest");
final String content = "bar";
s3Client.createBucket(b -> b.bucket(sourceBucketName));
s3Client.createBucket(b -> b.bucket(destBucketName));

PutObjectResponse putObjectResponse = s3Client.putObject(b -> b
.bucket(sourceBucketName)
.key(sourceKey),
RequestBody.fromString(content));

String sourceETag = putObjectResponse.eTag();

CopyObjectRequest copyReq = CopyObjectRequest.builder()
.sourceBucket(sourceBucketName)
.sourceKey(sourceKey)
.destinationBucket(destBucketName)
.destinationKey(destKey)
.copySourceIfNoneMatch(sourceETag)
.build();

S3Exception exception = assertThrows(S3Exception.class, () -> s3Client.copyObject(copyReq));
assertEquals(412, exception.statusCode());
assertEquals("PreconditionFailed", exception.awsErrorDetails().errorCode());
}

@Test
public void testLowLevelMultipartUpload(@TempDir Path tempDir) throws Exception {
final String bucketName = getBucketName();
Expand Down
39 changes: 29 additions & 10 deletions hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -388,8 +388,8 @@ Response handleGetRequest(ObjectRequestContext context, String keyPath)

isFile(keyPath, keyDetails);

Response conditionalResponse = S3ConditionalRequest
.evaluateReadPreconditions(getHeaders(), keyPath, keyDetails);
Response conditionalResponse = S3ConditionalRequest.evaluatePreconditions(
getHeaders(), keyPath, keyDetails, S3ConditionalRequest.PreconditionContext.READ);
if (conditionalResponse != null) {
long metadataLatencyNs = getMetrics().updateGetKeyMetadataStats(
startNanos);
Expand Down Expand Up @@ -547,8 +547,8 @@ public Response head(
key = getClientProtocol().headS3Object(bucketName, keyPath);

isFile(keyPath, key);
Response conditionalResponse = S3ConditionalRequest
.evaluateReadPreconditions(getHeaders(), keyPath, key);
Response conditionalResponse = S3ConditionalRequest.evaluatePreconditions(
getHeaders(), keyPath, key, S3ConditionalRequest.PreconditionContext.READ);
if (conditionalResponse != null) {
getMetrics().updateHeadKeySuccessStats(startNanos);
auditReadSuccess(s3GAction);
Expand Down Expand Up @@ -976,20 +976,23 @@ void copy(OzoneVolume volume, DigestInputStream src, long srcKeyLen,
ReplicationConfig replication,
Map<String, String> metadata,
PerformanceStringBuilder perf, long startNanos,
Map<String, String> tags)
Map<String, String> tags,
S3ConditionalRequest.WriteConditions writeConditions)
throws IOException {
long copyLength;

if (isDatastreamEnabled() && !(replication != null &&
replication.getReplicationType() == EC) &&
srcKeyLen > getDatastreamMinLength()) {
perf.appendStreamMode();
copyLength = ObjectEndpointStreaming
.copyKeyWithStream(volume.getBucket(destBucket), destKey, srcKeyLen,
getChunkSize(), replication, metadata, src, perf, startNanos, tags);
getChunkSize(), replication, metadata, src, perf, startNanos, tags,
writeConditions);
} else {
try (OzoneOutputStream dest = getClientProtocol()
.createKey(volume.getName(), destBucket, destKey, srcKeyLen,
replication, metadata, tags)) {
try (OzoneOutputStream dest = openKeyForPut(
volume.getName(), destBucket, destKey, srcKeyLen,
replication, metadata, tags, writeConditions)) {
long metadataLatencyNs =
getMetrics().updateCopyKeyMetadataStats(startNanos);
perf.appendMetaLatencyNanos(metadataLatencyNs);
Expand Down Expand Up @@ -1050,6 +1053,14 @@ private CopyObjectResponse copyObject(OzoneVolume volume,
return copyObjectResponse;
}
}

String sourceKeyPath = sourceBucket + "/" + sourceKey;
S3ConditionalRequest.evaluatePreconditions(getHeaders(), sourceKeyPath,
sourceKeyDetails, S3ConditionalRequest.PreconditionContext.COPY_SOURCE);

S3ConditionalRequest.WriteConditions writeConditions =
S3ConditionalRequest.parseWriteConditions(getHeaders(), destkey);

long sourceKeyLen = sourceKeyDetails.getDataSize();

// Object tagging in copyObject with tagging directive
Expand Down Expand Up @@ -1091,7 +1102,7 @@ private CopyObjectResponse copyObject(OzoneVolume volume,
getMetrics().updateCopyKeyMetadataStats(startNanos);
sourceDigestInputStream = new DigestInputStream(src, getMD5DigestInstance());
copy(volume, sourceDigestInputStream, sourceKeyLen, destkey, destBucket, replicationConfig,
customMetadata, perf, startNanos, tags);
customMetadata, perf, startNanos, tags, writeConditions);
}

final OzoneKeyDetails destKeyDetails = getClientProtocol().getKeyDetails(
Expand All @@ -1104,9 +1115,17 @@ private CopyObjectResponse copyObject(OzoneVolume volume,
return copyObjectResponse;
} catch (OMException ex) {
if (ex.getResult() == ResultCodes.KEY_NOT_FOUND) {
if (getHeaders().getHeaderString(S3Consts.IF_MATCH_HEADER) != null) {
throw newError(PRECOND_FAILED, destkey, ex);
}
throw newError(S3ErrorTable.NO_SUCH_KEY, sourceKey, ex);
} else if (ex.getResult() == ResultCodes.BUCKET_NOT_FOUND) {
throw newError(S3ErrorTable.NO_SUCH_BUCKET, sourceBucket, ex);
} else if (ex.getResult() == ResultCodes.ATOMIC_WRITE_CONFLICT
|| ex.getResult() == ResultCodes.KEY_ALREADY_EXISTS
|| ex.getResult() == ResultCodes.ETAG_MISMATCH
|| ex.getResult() == ResultCodes.ETAG_NOT_AVAILABLE) {
throw newError(PRECOND_FAILED, destkey, ex);
}
throw newError(destBucket + "/" + destkey, ex);
} finally {
Expand Down
8 changes: 5 additions & 3 deletions .../s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,11 +186,13 @@ public static long copyKeyWithStream(
ReplicationConfig replicationConfig,
Map<String, String> keyMetadata,
DigestInputStream body, PerformanceStringBuilder perf, long startNanos,
Map<String, String> tags)
Map<String, String> tags,
S3ConditionalRequest.WriteConditions writeConditions)
throws IOException {
long writeLen;
try (OzoneDataStreamOutput streamOutput = bucket.createStreamKey(keyPath,
length, replicationConfig, keyMetadata, tags)) {
try (OzoneDataStreamOutput streamOutput = openStreamKeyForPut(bucket,
keyPath, length, replicationConfig, keyMetadata, tags,
writeConditions)) {
long metadataLatencyNs =
METRICS.updateCopyKeyMetadataStats(startNanos);
writeLen = writeToStreamOutput(streamOutput, body, bufferSize, length);
Expand Down
Loading