HDDS-15196. Split HDDS upgrade actions into specific SCM and Datanode actions.

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentUpgradeActionProvider.java

@@ -27,7 +27,7 @@
  * @param <A> concrete upgrade action type (for example OM-specific or HDDS-specific)
  */
 @FunctionalInterface
-public interface ComponentUpgradeActionProvider<A> {
+public interface ComponentUpgradeActionProvider<A extends UpgradeAction<?>> {
 
   /**
    * Returns all upgrade actions from this provider, keyed by component version.

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/upgrade/DatanodeUpgradeAction.java

@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.upgrade;
+
+import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
+import org.apache.hadoop.ozone.upgrade.UpgradeAction;
+
+/**
+ * Datanode Upgrade Action interface. An upgrade action is an operation that
+ * needs to be executed during finalization.
+ */
+public interface DatanodeUpgradeAction extends UpgradeAction<DatanodeStateMachine> {
+}

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/upgrade/DatanodeUpgradeActionProvider.java

@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.upgrade;
+
+import org.apache.hadoop.hdds.ComponentVersion;
+import org.apache.hadoop.ozone.upgrade.AbstractUpgradeActionProvider;
+import org.apache.hadoop.ozone.upgrade.UpgradeActionDatanode;
+
+/**
+ * Loads {@link DatanodeUpgradeAction} implementations annotated with {@link UpgradeActionDatanode}.
+ */
+public final class DatanodeUpgradeActionProvider extends AbstractUpgradeActionProvider<DatanodeUpgradeAction> {
+
+  public static final String DATANODE_UPGRADE_CLASS_PACKAGE = "org.apache.hadoop.ozone.container";
+
+  public DatanodeUpgradeActionProvider() {
+    super(UpgradeActionDatanode.class, DatanodeUpgradeAction.class, DATANODE_UPGRADE_CLASS_PACKAGE);
+  }
+
+  @Override
+  protected ComponentVersion extractVersion(Class<?> clazz) {
+    UpgradeActionDatanode annotation = clazz.getAnnotation(UpgradeActionDatanode.class);
+    return annotation.feature();
+  }
+}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSUpgradeAction.java → hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/upgrade/package-info.java

@@ -15,12 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.hdds.upgrade;
-
-import org.apache.hadoop.ozone.upgrade.UpgradeAction;
-
 /**
- * Upgrade Action for SCM and DataNodes.
+ * Provides Datanode upgrade actions and providers.
  */
-public interface HDDSUpgradeAction<T> extends UpgradeAction<T> {
-}
+package org.apache.hadoop.hdds.upgrade;

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java

@@ -43,6 +43,7 @@
 import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
 import org.apache.hadoop.hdds.security.symmetric.SecretKeyClient;
 import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
+import org.apache.hadoop.hdds.upgrade.DatanodeUpgradeActionProvider;
 import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager;
 import org.apache.hadoop.hdds.utils.HddsServerUtil;
 import org.apache.hadoop.hdds.utils.IOUtils;
@@ -168,7 +169,7 @@ public DatanodeStateMachine(HddsDatanodeService hddsDatanodeService,
         datanodeDetails.getUuidString());
 
     layoutVersionManager = new HDDSLayoutVersionManager(
-        layoutStorage.getApparentVersion());
+        layoutStorage.getApparentVersion(), null, new DatanodeUpgradeActionProvider());
     upgradeFinalizer = new DataNodeUpgradeFinalizer(layoutVersionManager);
     VersionedDatanodeFeatures.initialize(layoutVersionManager);
 

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/ContainerTableSchemaFinalizeAction.java

@@ -18,10 +18,9 @@
 package org.apache.hadoop.ozone.container.upgrade;
 
 import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.WITNESSED_CONTAINER_DB_PROTO_VALUE;
-import static org.apache.hadoop.ozone.upgrade.UpgradeActionHdds.Component.DATANODE;
 
 import org.apache.hadoop.hdds.scm.container.ContainerID;
-import org.apache.hadoop.hdds.upgrade.HDDSUpgradeAction;
+import org.apache.hadoop.hdds.upgrade.DatanodeUpgradeAction;
 import org.apache.hadoop.hdds.utils.db.BatchOperation;
 import org.apache.hadoop.hdds.utils.db.CodecException;
 import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
@@ -32,16 +31,16 @@
 import org.apache.hadoop.ozone.container.metadata.WitnessedContainerDBDefinition;
 import org.apache.hadoop.ozone.container.metadata.WitnessedContainerMetadataStore;
 import org.apache.hadoop.ozone.container.metadata.WitnessedContainerMetadataStoreImpl;
-import org.apache.hadoop.ozone.upgrade.UpgradeActionHdds;
+import org.apache.hadoop.ozone.upgrade.UpgradeActionDatanode;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
  * Upgrade Action for DataNode for update the table schema data of containerIds Table.
  */
-@UpgradeActionHdds(feature = WITNESSED_CONTAINER_DB_PROTO_VALUE, component = DATANODE)
+@UpgradeActionDatanode(feature = WITNESSED_CONTAINER_DB_PROTO_VALUE)
 public class ContainerTableSchemaFinalizeAction
-    implements HDDSUpgradeAction<DatanodeStateMachine> {
+    implements DatanodeUpgradeAction {
 
   private static final Logger LOG =
       LoggerFactory.getLogger(ContainerTableSchemaFinalizeAction.class);

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DatanodeSchemaV2FinalizeAction.java

@@ -18,20 +18,19 @@
 package org.apache.hadoop.ozone.container.upgrade;
 
 import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.DATANODE_SCHEMA_V2;
-import static org.apache.hadoop.ozone.upgrade.UpgradeActionHdds.Component.DATANODE;
 
-import org.apache.hadoop.hdds.upgrade.HDDSUpgradeAction;
+import org.apache.hadoop.hdds.upgrade.DatanodeUpgradeAction;
 import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
-import org.apache.hadoop.ozone.upgrade.UpgradeActionHdds;
+import org.apache.hadoop.ozone.upgrade.UpgradeActionDatanode;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
  * Upgrade Action for DataNode for the very first first Upgrade Version.
  */
-@UpgradeActionHdds(feature = DATANODE_SCHEMA_V2, component = DATANODE)
+@UpgradeActionDatanode(feature = DATANODE_SCHEMA_V2)
 public class DatanodeSchemaV2FinalizeAction
-    implements HDDSUpgradeAction<DatanodeStateMachine> {
+    implements DatanodeUpgradeAction {
 
   private static final Logger LOG =
       LoggerFactory.getLogger(DatanodeSchemaV2FinalizeAction.class);

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DatanodeSchemaV3FinalizeAction.java

@@ -18,26 +18,25 @@
 package org.apache.hadoop.ozone.container.upgrade;
 
 import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.DATANODE_SCHEMA_V3;
-import static org.apache.hadoop.ozone.upgrade.UpgradeActionHdds.Component.DATANODE;
 
-import org.apache.hadoop.hdds.upgrade.HDDSUpgradeAction;
+import org.apache.hadoop.hdds.upgrade.DatanodeUpgradeAction;
 import org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration;
 import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
 import org.apache.hadoop.ozone.container.common.utils.HddsVolumeUtil;
 import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
 import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
 import org.apache.hadoop.ozone.container.common.volume.StorageVolume;
-import org.apache.hadoop.ozone.upgrade.UpgradeActionHdds;
+import org.apache.hadoop.ozone.upgrade.UpgradeActionDatanode;
 import org.apache.ratis.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
  * Upgrade Action for DataNode for SCHEMA V3.
  */
-@UpgradeActionHdds(feature = DATANODE_SCHEMA_V3, component = DATANODE)
+@UpgradeActionDatanode(feature = DATANODE_SCHEMA_V3)
 public class DatanodeSchemaV3FinalizeAction
-    implements HDDSUpgradeAction<DatanodeStateMachine> {
+    implements DatanodeUpgradeAction {
 
   private static final Logger LOG =
       LoggerFactory.getLogger(DatanodeSchemaV3FinalizeAction.class);

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/ScmHAFinalizeUpgradeActionDatanode.java

@@ -18,28 +18,27 @@
 package org.apache.hadoop.ozone.container.upgrade;
 
 import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.SCM_HA;
-import static org.apache.hadoop.ozone.upgrade.UpgradeActionHdds.Component.DATANODE;
 
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Objects;
-import org.apache.hadoop.hdds.upgrade.HDDSUpgradeAction;
+import org.apache.hadoop.hdds.upgrade.DatanodeUpgradeAction;
 import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
 import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
 import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
 import org.apache.hadoop.ozone.container.common.volume.StorageVolume;
-import org.apache.hadoop.ozone.upgrade.UpgradeActionHdds;
+import org.apache.hadoop.ozone.upgrade.UpgradeActionDatanode;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
  * Action to run upgrade flow for SCM HA exactly once.
  */
-@UpgradeActionHdds(feature = SCM_HA, component = DATANODE)
+@UpgradeActionDatanode(feature = SCM_HA)
 public class ScmHAFinalizeUpgradeActionDatanode
-    implements HDDSUpgradeAction<DatanodeStateMachine> {
+    implements DatanodeUpgradeAction {
   private static final Logger LOG =
       LoggerFactory.getLogger(ScmHAFinalizeUpgradeActionDatanode.class);
 

hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/UpgradeActionHdds.java → hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/upgrade/UpgradeActionDatanode.java

@@ -24,23 +24,10 @@
 import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature;
 
 /**
- * Annotation to specify upgrade action run during HDDS (SCM or Datanode) finalization.
+ * Annotation to specify a Datanode upgrade action.
  */
 @Retention(RetentionPolicy.RUNTIME)
 @Target(ElementType.TYPE)
-public @interface UpgradeActionHdds {
+public @interface UpgradeActionDatanode {
   HDDSLayoutFeature feature();
-
-  Component component();
-
-  /**
-   * Simple enum to denote if an action is for the SCM or the DN.
-   */
-  enum Component {
-    SCM,
-    DATANODE;
-  }
 }
-
-
-

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/upgrade/package-info.java

@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Provides Datanode upgrade annotations.
+ */
+package org.apache.hadoop.ozone.upgrade;

hadoop-hdds/docs/content/design/upgrade-dev-primer.md

@@ -48,7 +48,7 @@ Method level annotation used to "disallow" an API if current layout version does
 ## @BelongsToLayoutVersion Annotation
 Annotation to mark an OM request class that it belongs to a specific Layout Version. Until that version is available post finalize, this request will not be supported. A newer version of an existing OM request can be created (by inheritance or a fully new class) and marked with a newer layout version. Until finalizing this layout version, the older request class is used. Post finalizing, the newer version of the request class is used.
 
-## Upgrade Action (UpgradeActionOm & UpgradeActionHdds)
+## Upgrade Action (UpgradeActionOm, UpgradeActionScm & UpgradeActionDatanode)
 Annotation to specify upgrade action run during finalization. Each layout feature can optionally define a single upgrade action that will be executed when the feature is finalized. This action should be idempotent and execute quickly. The action must complete for the feature to finish
 finalizing, so if there is an error executing the action it will be retried. This partial failure should not leave the component inoperable.
 

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java → hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java

@@ -27,6 +27,7 @@
 import org.apache.hadoop.hdds.ComponentVersion;
 import org.apache.hadoop.hdds.HDDSVersion;
 import org.apache.hadoop.ozone.upgrade.LayoutFeature;
+import org.apache.hadoop.ozone.upgrade.UpgradeAction;
 
 /**
  * List of HDDS Layout Features. All version management has been migrated to {@link HDDSVersion} and no new additions
@@ -64,8 +65,8 @@ public enum HDDSLayoutFeature implements LayoutFeature {
 
   private final int layoutVersion;
   private final String description;
-  private HDDSUpgradeAction scmAction;
-  private HDDSUpgradeAction datanodeAction;
+  private UpgradeAction<?> scmAction;
+  private UpgradeAction<?> datanodeAction;
 
   HDDSLayoutFeature(final int layoutVersion, String description) {
     this.layoutVersion = layoutVersion;
@@ -77,7 +78,7 @@ public enum HDDSLayoutFeature implements LayoutFeature {
    *
    * @param action The upgrade action to associate with this feature.
    */
-  public void addScmAction(HDDSUpgradeAction action) {
+  public void addScmAction(UpgradeAction<?> action) {
     // Required by SpotBugs since this setter exists in an enum.
     if (this.scmAction == null) {
       this.scmAction = action;
@@ -89,7 +90,7 @@ public void addScmAction(HDDSUpgradeAction action) {
    *
    * @param action The upgrade action to associate with this feature.
    */
-  public void addDatanodeAction(HDDSUpgradeAction action) {
+  public void addDatanodeAction(UpgradeAction<?> action) {
     // Required by SpotBugs since this setter exists in an enum.
     if (this.datanodeAction == null) {
       this.datanodeAction = action;
@@ -135,11 +136,11 @@ public String toString() {
     return name() + " (" + serialize() + ")";
   }
 
-  public Optional<HDDSUpgradeAction> scmAction() {
+  public Optional<UpgradeAction<?>> scmAction() {
     return Optional.ofNullable(scmAction);
   }
 
-  public Optional<HDDSUpgradeAction> datanodeAction() {
+  public Optional<UpgradeAction<?>> datanodeAction() {
     return Optional.ofNullable(datanodeAction);
   }
 }