diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java index aa74bf01796f..f4484c02ae41 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java @@ -17,9 +17,6 @@ package org.apache.hadoop.hdds.protocol; -import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.HADOOP_PRC_PORTS_IN_DATANODEDETAILS; -import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.RATIS_DATASTREAM_PORT_IN_DATANODEDETAILS; -import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.WEBUI_PORTS_IN_DATANODEDETAILS; import static org.apache.hadoop.ozone.ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -46,7 +43,6 @@ import org.apache.hadoop.hdds.scm.net.NetConstants; import org.apache.hadoop.hdds.scm.net.NetUtils; import org.apache.hadoop.hdds.scm.net.NodeImpl; -import org.apache.hadoop.hdds.upgrade.BelongsToHDDSLayoutVersion; import org.apache.hadoop.hdds.utils.db.Codec; import org.apache.hadoop.hdds.utils.db.DelegatedCodec; import org.apache.hadoop.hdds.utils.db.Proto2Codec; @@ -1031,13 +1027,9 @@ public static final class Port { */ public enum Name { STANDALONE, RATIS, REST, REPLICATION, RATIS_ADMIN, RATIS_SERVER, - @BelongsToHDDSLayoutVersion(RATIS_DATASTREAM_PORT_IN_DATANODEDETAILS) RATIS_DATASTREAM, - @BelongsToHDDSLayoutVersion(WEBUI_PORTS_IN_DATANODEDETAILS) HTTP, - @BelongsToHDDSLayoutVersion(WEBUI_PORTS_IN_DATANODEDETAILS) HTTPS, - @BelongsToHDDSLayoutVersion(HADOOP_PRC_PORTS_IN_DATANODEDETAILS) CLIENT_RPC; public static final Set ALL_PORTS = ImmutableSet.copyOf( diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/BelongsToHDDSLayoutVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/BelongsToHDDSLayoutVersion.java deleted file mode 100644 index 374b1688b6e9..000000000000 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/BelongsToHDDSLayoutVersion.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hdds.upgrade; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -/** - * Annotation to mark a class or a field declaration that belongs to a specific - * HDDS Layout Version. - */ -@Target({ElementType.TYPE, ElementType.FIELD}) -@Retention(RetentionPolicy.RUNTIME) -public @interface BelongsToHDDSLayoutVersion { - HDDSLayoutFeature value(); -} diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/DatanodeIdYaml.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/DatanodeIdYaml.java index d45623f36bb1..cf9b1a68e09d 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/DatanodeIdYaml.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/DatanodeIdYaml.java @@ -20,21 +20,15 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; -import java.lang.reflect.Field; import java.nio.file.Files; -import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; import java.util.UUID; -import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.server.YamlUtils; -import org.apache.hadoop.hdds.upgrade.BelongsToHDDSLayoutVersion; -import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature; -import org.apache.hadoop.ozone.container.common.DatanodeLayoutStorage; +import org.apache.hadoop.ozone.container.upgrade.VersionedDatanodeFeatures; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.yaml.snakeyaml.DumperOptions; @@ -241,34 +235,6 @@ private static DatanodeDetailsYaml getDatanodeDetailsYaml( DatanodeDetails datanodeDetails, ConfigurationSource conf) throws IOException { - DatanodeLayoutStorage datanodeLayoutStorage - = new DatanodeLayoutStorage(conf, datanodeDetails.getUuidString()); - - Map portDetails = new LinkedHashMap<>(); - final List ports = datanodeDetails.getPorts(); - if (!CollectionUtils.isEmpty(ports)) { - for (DatanodeDetails.Port port : ports) { - Field f = null; - try { - f = DatanodeDetails.Port.Name.class - .getDeclaredField(port.getName().name()); - } catch (NoSuchFieldException e) { - LOG.error("There is no such field as {} in {}", port.getName().name(), - DatanodeDetails.Port.Name.class); - } - if (f != null - && f.isAnnotationPresent(BelongsToHDDSLayoutVersion.class)) { - HDDSLayoutFeature layoutFeature - = f.getAnnotation(BelongsToHDDSLayoutVersion.class).value(); - if (layoutFeature.layoutVersion() > - datanodeLayoutStorage.getApparentVersion()) { - continue; - } - } - portDetails.put(port.getName().toString(), port.getValue()); - } - } - String persistedOpString = null; if (datanodeDetails.getPersistedOpState() != null) { persistedOpString = datanodeDetails.getPersistedOpState().name(); @@ -281,7 +247,7 @@ private static DatanodeDetailsYaml getDatanodeDetailsYaml( datanodeDetails.getCertSerialId(), persistedOpString, datanodeDetails.getPersistedOpStateExpiryEpochSec(), - portDetails, + VersionedDatanodeFeatures.DatanodePorts.getPortsToPersist(datanodeDetails, conf), datanodeDetails.getInitialVersion(), datanodeDetails.getCurrentVersion()); } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/VersionedDatanodeFeatures.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/VersionedDatanodeFeatures.java index ffd038afdcc2..acaabdad6e48 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/VersionedDatanodeFeatures.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/VersionedDatanodeFeatures.java @@ -19,10 +19,17 @@ import java.io.File; import java.io.IOException; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.apache.commons.collections4.CollectionUtils; import org.apache.hadoop.hdds.conf.ConfigurationSource; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature; import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager; import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.container.common.DatanodeLayoutStorage; import org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration; import org.apache.hadoop.ozone.container.common.volume.StorageVolume; @@ -148,4 +155,51 @@ public static boolean isFinalizedAndEnabled(ConfigurationSource conf) { return false; } } + + /** + * Methods to handle persisting ports to be compatible with older Datanode versions that would fail on unknown port + * fields. + */ + public static class DatanodePorts { + /** + * Version that must be present before persisting each port name to disk. + * Ports omitted from this map are always persisted. + */ + private static final Map PORT_TO_VERSION = new HashMap<>(); + + static { + PORT_TO_VERSION.put(DatanodeDetails.Port.Name.RATIS_DATASTREAM, + HDDSLayoutFeature.RATIS_DATASTREAM_PORT_IN_DATANODEDETAILS); + PORT_TO_VERSION.put( + DatanodeDetails.Port.Name.HTTP, + HDDSLayoutFeature.WEBUI_PORTS_IN_DATANODEDETAILS); + PORT_TO_VERSION.put( + DatanodeDetails.Port.Name.HTTPS, + HDDSLayoutFeature.WEBUI_PORTS_IN_DATANODEDETAILS); + PORT_TO_VERSION.put( + DatanodeDetails.Port.Name.CLIENT_RPC, + HDDSLayoutFeature.HADOOP_PRC_PORTS_IN_DATANODEDETAILS); + } + + private static boolean shouldPersistPort(DatanodeDetails.Port port, DatanodeLayoutStorage datanodeLayoutStorage) { + HDDSLayoutFeature portVersion = PORT_TO_VERSION.get(port.getName()); + return portVersion == null || portVersion.isSupportedBy(datanodeLayoutStorage.getApparentVersion()); + } + + public static Map getPortsToPersist(DatanodeDetails datanodeDetails, ConfigurationSource conf) + throws IOException { + DatanodeLayoutStorage datanodeLayoutStorage = new DatanodeLayoutStorage(conf, datanodeDetails.getUuidString()); + Map portDetails = new LinkedHashMap<>(); + + final List ports = datanodeDetails.getPorts(); + if (!CollectionUtils.isEmpty(ports)) { + for (DatanodeDetails.Port port : ports) { + if (shouldPersistPort(port, datanodeLayoutStorage)) { + portDetails.put(port.getName().toString(), port.getValue()); + } + } + } + return portDetails; + } + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/BelongsToLayoutVersion.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/BelongsToLayoutVersion.java deleted file mode 100644 index 5b007a581cd5..000000000000 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/BelongsToLayoutVersion.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.om.upgrade; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -/** - * Annotation to mark a class that belongs to a specific Layout Version. - */ -@Target(ElementType.TYPE) -@Retention(RetentionPolicy.RUNTIME) -public @interface BelongsToLayoutVersion { - OMLayoutFeature value(); -} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeatureAspect.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeatureAspect.java index aeedb72d4ce6..d17cfe2d1f6d 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeatureAspect.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeatureAspect.java @@ -28,7 +28,6 @@ import org.aspectj.lang.JoinPoint; import org.aspectj.lang.annotation.Aspect; import org.aspectj.lang.annotation.Before; -import org.aspectj.lang.annotation.Pointcut; import org.aspectj.lang.reflect.MethodSignature; /** @@ -78,30 +77,6 @@ private void checkIsAllowed(String operationName, } } - @Pointcut("execution(* " + - "org.apache.hadoop.ozone.om.request.OMClientRequest+.preExecute(..)) " + - "&& @this(org.apache.hadoop.ozone.om.upgrade.BelongsToLayoutVersion)") - public void omRequestPointCut() { - } - - @Before("omRequestPointCut()") - public void beforeRequestApplyTxn(final JoinPoint joinPoint) - throws OMException { - - BelongsToLayoutVersion annotation = joinPoint.getTarget().getClass() - .getAnnotation(BelongsToLayoutVersion.class); - if (annotation == null) { - return; - } - - Object[] args = joinPoint.getArgs(); - OzoneManager om = (OzoneManager) args[0]; - - LayoutFeature lf = annotation.value(); - checkIsAllowed(joinPoint.getTarget().getClass().getSimpleName(), - om.getVersionManager(), lf); - } - /** * Note: Without this, it occasionally throws NoSuchMethodError when running * the test. diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/MockOmRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/MockOmRequest.java deleted file mode 100644 index eb2b04f6f968..000000000000 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/MockOmRequest.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.om.upgrade; - -import static org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature.INITIAL_VERSION; - -import org.apache.hadoop.ozone.om.OzoneManager; - -/** - * Mock OM Request class for testing annotation. - */ -@BelongsToLayoutVersion(INITIAL_VERSION) -public class MockOmRequest { - public void preExecute(OzoneManager om) { - } -} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutFeatureAspect.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutFeatureAspect.java index f3a389fa089d..14b3d3809f56 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutFeatureAspect.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutFeatureAspect.java @@ -86,25 +86,4 @@ public void testDisallowedUntilLayoutVersion() throws Throwable { assertThat(omException.getMessage()) .contains("cannot be invoked before finalization"); } - - @Test - public void testPreExecuteLayoutCheck() { - - OzoneManager om = mock(OzoneManager.class); - OMVersionManager ovm = mock(OMVersionManager.class); - when(ovm.isAllowed(any(ComponentVersion.class))).thenReturn(false); - when(om.getVersionManager()).thenReturn(ovm); - - MockOmRequest mockOmRequest = new MockOmRequest(); - OMLayoutFeatureAspect aspect = new OMLayoutFeatureAspect(); - - JoinPoint joinPoint = mock(JoinPoint.class); - when(joinPoint.getArgs()).thenReturn(new Object[]{om}); - when(joinPoint.getTarget()).thenReturn(mockOmRequest); - - OMException omException = assertThrows(OMException.class, - () -> aspect.beforeRequestApplyTxn(joinPoint)); - assertThat(omException.getMessage()) - .contains("cannot be invoked before finalization"); - } }