diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneClusterConfig.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneClusterConfig.java new file mode 100644 index 000000000000..4a5d9d7ea206 --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneClusterConfig.java @@ -0,0 +1,327 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.local; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Duration; +import java.util.Locale; +import java.util.Objects; + +/** + * Configuration for a local Ozone cluster runtime. + * + *

The datanode count describes how many local datanode services should run + * on the same host. + */ +public final class LocalOzoneClusterConfig { + + private static final String DEFAULT_DATA_DIR_PARENT = ".ozone"; + private static final String DEFAULT_DATA_DIR_NAME = "local"; + + // Picocli annotation defaults require compile-time strings, so the CLI uses + // this expression while the typed default below uses the same path fragments. + static final String DEFAULT_DATA_DIR_VALUE = + "${sys:user.home}${sys:file.separator}" + DEFAULT_DATA_DIR_PARENT + + "${sys:file.separator}" + DEFAULT_DATA_DIR_NAME; + static final String DEFAULT_FORMAT_MODE_VALUE = "if-needed"; + static final String DEFAULT_DATANODES_VALUE = "1"; + static final String DEFAULT_PORT_VALUE = "0"; + static final String DEFAULT_S3G_ENABLED_VALUE = "true"; + static final String DEFAULT_EPHEMERAL_VALUE = "false"; + static final String DEFAULT_STARTUP_TIMEOUT_VALUE = "PT2M"; + + static final Path DEFAULT_DATA_DIR = + Paths.get(System.getProperty("user.home"), DEFAULT_DATA_DIR_PARENT, + DEFAULT_DATA_DIR_NAME) + .toAbsolutePath() + .normalize(); + static final FormatMode DEFAULT_FORMAT_MODE = + FormatMode.fromString(DEFAULT_FORMAT_MODE_VALUE); + static final int DEFAULT_DATANODES = + Integer.parseInt(DEFAULT_DATANODES_VALUE); + static final String DEFAULT_HOST = "127.0.0.1"; + static final String DEFAULT_BIND_HOST = "0.0.0.0"; + static final int DEFAULT_PORT = Integer.parseInt(DEFAULT_PORT_VALUE); + static final boolean DEFAULT_S3G_ENABLED = + Boolean.parseBoolean(DEFAULT_S3G_ENABLED_VALUE); + static final boolean DEFAULT_EPHEMERAL = + Boolean.parseBoolean(DEFAULT_EPHEMERAL_VALUE); + static final Duration DEFAULT_STARTUP_TIMEOUT = + Duration.parse(DEFAULT_STARTUP_TIMEOUT_VALUE); + static final String DEFAULT_S3_ACCESS_KEY = "admin"; + static final String DEFAULT_S3_SECRET_KEY = "admin123"; + static final String DEFAULT_S3_REGION = "us-east-1"; + + private final Path dataDir; + private final FormatMode formatMode; + private final int datanodes; + private final String host; + private final String bindHost; + private final int scmPort; + private final int omPort; + private final int s3gPort; + private final boolean s3gEnabled; + private final boolean ephemeral; + private final Duration startupTimeout; + private final String s3AccessKey; + private final String s3SecretKey; + private final String s3Region; + + private LocalOzoneClusterConfig(Builder builder) { + dataDir = Objects.requireNonNull(builder.dataDir, "dataDir") + .toAbsolutePath() + .normalize(); + formatMode = Objects.requireNonNull(builder.formatMode, "formatMode"); + datanodes = builder.datanodes; + host = Objects.requireNonNull(builder.host, "host"); + bindHost = Objects.requireNonNull(builder.bindHost, "bindHost"); + scmPort = builder.scmPort; + omPort = builder.omPort; + s3gPort = builder.s3gPort; + s3gEnabled = builder.s3gEnabled; + ephemeral = builder.ephemeral; + startupTimeout = Objects.requireNonNull(builder.startupTimeout, + "startupTimeout"); + s3AccessKey = Objects.requireNonNull(builder.s3AccessKey, "s3AccessKey"); + s3SecretKey = Objects.requireNonNull(builder.s3SecretKey, "s3SecretKey"); + s3Region = Objects.requireNonNull(builder.s3Region, "s3Region"); + } + + public Path getDataDir() { + return dataDir; + } + + public FormatMode getFormatMode() { + return formatMode; + } + + public int getDatanodes() { + return datanodes; + } + + public String getHost() { + return host; + } + + public String getBindHost() { + return bindHost; + } + + /** + * Returns the SCM client RPC port. Port {@code 0} asks the runtime to choose + * an available local port. + */ + public int getScmPort() { + return scmPort; + } + + /** + * Returns the OM RPC port. Port {@code 0} asks the runtime to choose + * an available local port. + */ + public int getOmPort() { + return omPort; + } + + /** + * Returns the S3 Gateway HTTP port. Port {@code 0} asks the runtime to + * choose an available local port. + */ + public int getS3gPort() { + return s3gPort; + } + + /** + * Returns whether the local runtime should include S3 Gateway. + */ + public boolean isS3gEnabled() { + return s3gEnabled; + } + + /** + * Returns whether the local runtime should remove its data directory when it + * shuts down. + */ + public boolean isEphemeral() { + return ephemeral; + } + + /** + * Returns how long the launcher should wait for local services to become + * ready before failing startup. + */ + public Duration getStartupTimeout() { + return startupTimeout; + } + + /** + * Returns the suggested local-only S3 access key printed for client setup. + */ + public String getS3AccessKey() { + return s3AccessKey; + } + + /** + * Returns the suggested local-only S3 secret key printed for client setup. + */ + public String getS3SecretKey() { + return s3SecretKey; + } + + /** + * Returns the suggested local-only S3 region printed for client setup. + */ + public String getS3Region() { + return s3Region; + } + + public static Builder builder() { + return new Builder(DEFAULT_DATA_DIR); + } + + public static Builder builder(Path dataDir) { + return new Builder(dataDir); + } + + /** + * Storage initialization mode for the local runtime. + */ + public enum FormatMode { + /** + * Initialize storage only when local metadata is missing or unformatted. + * Existing local data is reused. + */ + IF_NEEDED, + + /** + * Always format local storage before startup. Existing local data may be + * discarded. + */ + ALWAYS, + + /** + * Never format local storage. Startup should fail later if required storage + * is not already initialized. + */ + NEVER; + + public static FormatMode fromString(String value) { + if (value == null) { + throw new IllegalArgumentException("Format mode must not be null."); + } + String normalized = value.trim().toUpperCase(Locale.ROOT) + .replace('-', '_'); + return valueOf(normalized); + } + } + + /** + * Builder for {@link LocalOzoneClusterConfig}. + */ + public static final class Builder { + + private final Path dataDir; + private FormatMode formatMode = DEFAULT_FORMAT_MODE; + private int datanodes = DEFAULT_DATANODES; + private String host = DEFAULT_HOST; + private String bindHost = DEFAULT_BIND_HOST; + private int scmPort = DEFAULT_PORT; + private int omPort = DEFAULT_PORT; + private int s3gPort = DEFAULT_PORT; + private boolean s3gEnabled = DEFAULT_S3G_ENABLED; + private boolean ephemeral = DEFAULT_EPHEMERAL; + private Duration startupTimeout = DEFAULT_STARTUP_TIMEOUT; + private String s3AccessKey = DEFAULT_S3_ACCESS_KEY; + private String s3SecretKey = DEFAULT_S3_SECRET_KEY; + private String s3Region = DEFAULT_S3_REGION; + + private Builder(Path dataDir) { + this.dataDir = dataDir; + } + + public Builder setFormatMode(FormatMode value) { + formatMode = value; + return this; + } + + public Builder setDatanodes(int value) { + datanodes = value; + return this; + } + + public Builder setHost(String value) { + host = value; + return this; + } + + public Builder setBindHost(String value) { + bindHost = value; + return this; + } + + public Builder setScmPort(int value) { + scmPort = value; + return this; + } + + public Builder setOmPort(int value) { + omPort = value; + return this; + } + + public Builder setS3gPort(int value) { + s3gPort = value; + return this; + } + + public Builder setS3gEnabled(boolean value) { + s3gEnabled = value; + return this; + } + + public Builder setEphemeral(boolean value) { + ephemeral = value; + return this; + } + + public Builder setStartupTimeout(Duration value) { + startupTimeout = value; + return this; + } + + public Builder setS3AccessKey(String value) { + s3AccessKey = value; + return this; + } + + public Builder setS3SecretKey(String value) { + s3SecretKey = value; + return this; + } + + public Builder setS3Region(String value) { + s3Region = value; + return this; + } + + public LocalOzoneClusterConfig build() { + return new LocalOzoneClusterConfig(this); + } + } +} diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneRuntime.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneRuntime.java index 54a08e476ff6..3ed939044176 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneRuntime.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneRuntime.java @@ -18,22 +18,64 @@ package org.apache.hadoop.ozone.local; /** - * Runtime contract for local single-node Ozone commands. + * Runtime contract for local Ozone cluster commands. */ public interface LocalOzoneRuntime extends AutoCloseable { + /** + * Starts the local Ozone runtime. + * + *

Port and endpoint accessors return usable values after this method + * completes successfully.

+ * + * @throws Exception if the local runtime cannot be started + */ void start() throws Exception; + /** + * Returns the host name or address shown to users for connecting to this + * local runtime. + * + *

This is a user-facing host and may differ from the bind host used by + * individual services.

+ * + * @return user-facing host name or address + */ String getDisplayHost(); + /** + * Returns the SCM client port for this local runtime. + * + * @return SCM port + */ int getScmPort(); + /** + * Returns the OM client port for this local runtime. + * + * @return OM port + */ int getOmPort(); + /** + * Returns the S3 Gateway HTTP port for this local runtime. + * + * @return S3 Gateway port + */ int getS3gPort(); + /** + * Returns the full S3 Gateway endpoint shown to users. + * + * @return S3 Gateway endpoint, including scheme, host, and port + */ String getS3Endpoint(); + /** + * Stops the local runtime and releases resources created during startup. + * + * @throws Exception if shutdown fails + */ @Override void close() throws Exception; } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/OzoneLocal.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/OzoneLocal.java index bd19b9f5dbcf..6d1d4d1394f9 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/OzoneLocal.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/OzoneLocal.java @@ -17,17 +17,26 @@ package org.apache.hadoop.ozone.local; +import java.nio.file.Path; +import java.time.Duration; +import java.time.format.DateTimeParseException; import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; +import org.apache.hadoop.hdds.cli.AbstractSubcommand; import org.apache.hadoop.hdds.cli.GenericCli; import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.conf.TimeDurationUtil; +import picocli.CommandLine; import picocli.CommandLine.Command; +import picocli.CommandLine.ITypeConverter; +import picocli.CommandLine.Option; /** - * Internal CLI entry point for local single-node Ozone commands. + * Internal CLI entry point for local Ozone cluster commands. */ @Command(name = "ozone local", hidden = true, - description = "Internal commands for local single-node Ozone", + description = "Internal commands for local Ozone cluster runtime", versionProvider = HddsVersionProvider.class, mixinStandardHelpOptions = true, subcommands = { @@ -35,18 +44,235 @@ }) public class OzoneLocal extends GenericCli { + static final String ENV_DATA_DIR = "OZONE_LOCAL_DATA_DIR"; + static final String ENV_FORMAT = "OZONE_LOCAL_FORMAT"; + static final String ENV_DATANODES = "OZONE_LOCAL_DATANODES"; + static final String ENV_HOST = "OZONE_LOCAL_HOST"; + static final String ENV_BIND_HOST = "OZONE_LOCAL_BIND_HOST"; + static final String ENV_SCM_PORT = "OZONE_LOCAL_SCM_PORT"; + static final String ENV_OM_PORT = "OZONE_LOCAL_OM_PORT"; + static final String ENV_S3G_ENABLED = "OZONE_LOCAL_S3G_ENABLED"; + static final String ENV_S3G_PORT = "OZONE_LOCAL_S3G_PORT"; + static final String ENV_EPHEMERAL = "OZONE_LOCAL_EPHEMERAL"; + static final String ENV_STARTUP_TIMEOUT = "OZONE_LOCAL_STARTUP_TIMEOUT"; + static final String ENV_S3_ACCESS_KEY = "OZONE_LOCAL_S3_ACCESS_KEY"; + static final String ENV_S3_SECRET_KEY = "OZONE_LOCAL_S3_SECRET_KEY"; + static final String ENV_S3_REGION = "OZONE_LOCAL_S3_REGION"; + + private static final String DEFAULT_DATA_DIR_VALUE = "${env:" + ENV_DATA_DIR + + ":-" + LocalOzoneClusterConfig.DEFAULT_DATA_DIR_VALUE + "}"; + private static final String DEFAULT_FORMAT_VALUE = "${env:" + ENV_FORMAT + + ":-" + LocalOzoneClusterConfig.DEFAULT_FORMAT_MODE_VALUE + "}"; + private static final String DEFAULT_DATANODES_VALUE = "${env:" + + ENV_DATANODES + ":-" + LocalOzoneClusterConfig.DEFAULT_DATANODES_VALUE + + "}"; + private static final String DEFAULT_HOST_VALUE = "${env:" + ENV_HOST + + ":-" + LocalOzoneClusterConfig.DEFAULT_HOST + "}"; + private static final String DEFAULT_BIND_HOST_VALUE = "${env:" + + ENV_BIND_HOST + ":-" + LocalOzoneClusterConfig.DEFAULT_BIND_HOST + "}"; + private static final String DEFAULT_SCM_PORT_VALUE = "${env:" + + ENV_SCM_PORT + ":-" + LocalOzoneClusterConfig.DEFAULT_PORT_VALUE + + "}"; + private static final String DEFAULT_OM_PORT_VALUE = "${env:" + ENV_OM_PORT + + ":-" + LocalOzoneClusterConfig.DEFAULT_PORT_VALUE + "}"; + private static final String DEFAULT_S3G_ENABLED_VALUE = "${env:" + + ENV_S3G_ENABLED + ":-" + + LocalOzoneClusterConfig.DEFAULT_S3G_ENABLED_VALUE + "}"; + private static final String DEFAULT_S3G_PORT_VALUE = "${env:" + ENV_S3G_PORT + + ":-" + LocalOzoneClusterConfig.DEFAULT_PORT_VALUE + "}"; + private static final String DEFAULT_EPHEMERAL_VALUE = "${env:" + + ENV_EPHEMERAL + ":-" + + LocalOzoneClusterConfig.DEFAULT_EPHEMERAL_VALUE + "}"; + private static final String DEFAULT_STARTUP_TIMEOUT_VALUE = "${env:" + + ENV_STARTUP_TIMEOUT + ":-" + + LocalOzoneClusterConfig.DEFAULT_STARTUP_TIMEOUT_VALUE + "}"; + private static final String DEFAULT_S3_ACCESS_KEY_VALUE = "${env:" + + ENV_S3_ACCESS_KEY + ":-" + + LocalOzoneClusterConfig.DEFAULT_S3_ACCESS_KEY + "}"; + private static final String DEFAULT_S3_SECRET_KEY_VALUE = "${env:" + + ENV_S3_SECRET_KEY + ":-" + + LocalOzoneClusterConfig.DEFAULT_S3_SECRET_KEY + "}"; + private static final String DEFAULT_S3_REGION_VALUE = "${env:" + + ENV_S3_REGION + ":-" + LocalOzoneClusterConfig.DEFAULT_S3_REGION + "}"; + + public OzoneLocal() { + super(); + } + + OzoneLocal(CommandLine.IFactory factory) { + super(factory); + } + public static void main(String[] args) { new OzoneLocal().run(args); } @Command(name = "run", hidden = true, - description = "Internal placeholder for a local Ozone runtime") - static class RunCommand implements Callable { + description = "Resolve configuration for local Ozone runtime startup") + static class RunCommand extends AbstractSubcommand implements Callable { + + @Option(names = "--data-dir", + defaultValue = DEFAULT_DATA_DIR_VALUE, + description = "Persistent data directory for the local cluster") + private Path dataDir; + + @Option(names = "--format", + converter = FormatModeConverter.class, + defaultValue = DEFAULT_FORMAT_VALUE, + description = "Storage init mode: if-needed, always, never") + private LocalOzoneClusterConfig.FormatMode formatMode; + + @Option(names = "--datanodes", + defaultValue = DEFAULT_DATANODES_VALUE, + description = "Number of datanodes to start") + private int datanodes; + + @Option(names = "--host", + defaultValue = DEFAULT_HOST_VALUE, + description = "Advertised host to write into local service addresses") + private String host; + + @Option(names = "--bind-host", + defaultValue = DEFAULT_BIND_HOST_VALUE, + description = "Bind host for HTTP and RPC listeners") + private String bindHost; + + @Option(names = "--scm-port", + defaultValue = DEFAULT_SCM_PORT_VALUE, + description = "SCM client RPC port (0 means auto-allocate)") + private int scmPort; + + @Option(names = "--om-port", + defaultValue = DEFAULT_OM_PORT_VALUE, + description = "OM RPC port (0 means auto-allocate)") + private int omPort; + + @Option(names = "--s3g-port", + defaultValue = DEFAULT_S3G_PORT_VALUE, + description = "S3 Gateway HTTP port (0 means auto-allocate)") + private int s3gPort; + + @Option(names = "--s3g", + negatable = true, + defaultValue = DEFAULT_S3G_ENABLED_VALUE, + fallbackValue = "true", + description = "Enable S3 Gateway") + private boolean s3gEnabled; + + @Option(names = "--ephemeral", + negatable = true, + defaultValue = DEFAULT_EPHEMERAL_VALUE, + fallbackValue = "true", + description = "Delete the data directory on shutdown") + private boolean ephemeral; + + @Option(names = "--startup-timeout", + converter = DurationConverter.class, + defaultValue = DEFAULT_STARTUP_TIMEOUT_VALUE, + description = "How long to wait for the local cluster to become ready") + private Duration startupTimeout; + + @Option(names = "--s3-access-key", + defaultValue = DEFAULT_S3_ACCESS_KEY_VALUE, + description = "Suggested local AWS access key to print on startup") + private String s3AccessKey; + + @Option(names = "--s3-secret-key", + defaultValue = DEFAULT_S3_SECRET_KEY_VALUE, + description = "Suggested local AWS secret key to print on startup") + private String s3SecretKey; + + @Option(names = "--s3-region", + defaultValue = DEFAULT_S3_REGION_VALUE, + description = "Suggested local AWS region to print on startup") + private String s3Region; @Override public Void call() { + resolveConfig(); return null; } + + LocalOzoneClusterConfig resolveConfig() { + if (datanodes < 1) { + throw new IllegalArgumentException( + "Datanode count for --datanodes must be at least 1."); + } + validatePort(scmPort, "--scm-port"); + validatePort(omPort, "--om-port"); + validatePort(s3gPort, "--s3g-port"); + validateStartupTimeout(); + + return LocalOzoneClusterConfig.builder(dataDir) + .setFormatMode(formatMode) + .setDatanodes(datanodes) + .setHost(host) + .setBindHost(bindHost) + .setScmPort(scmPort) + .setOmPort(omPort) + .setS3gPort(s3gPort) + .setS3gEnabled(s3gEnabled) + .setEphemeral(ephemeral) + .setStartupTimeout(startupTimeout) + .setS3AccessKey(s3AccessKey) + .setS3SecretKey(s3SecretKey) + .setS3Region(s3Region) + .build(); + } + + private void validateStartupTimeout() { + if (startupTimeout.isZero() || startupTimeout.isNegative()) { + throw new IllegalArgumentException( + "Startup timeout for --startup-timeout must be greater than zero."); + } + } + + private static void validatePort(int value, String source) { + if (value < 0 || value > 65_535) { + throw new IllegalArgumentException("Port value for " + source + + " must be between 0 and 65535."); + } + } + + private static final class FormatModeConverter + implements ITypeConverter { + + @Override + public LocalOzoneClusterConfig.FormatMode convert(String value) { + try { + return LocalOzoneClusterConfig.FormatMode.fromString(value); + } catch (IllegalArgumentException ex) { + throw new CommandLine.TypeConversionException( + "Expected one of: if-needed, always, never."); + } + } + } + + private static final class DurationConverter + implements ITypeConverter { + + @Override + public Duration convert(String value) { + try { + return Duration.parse(value.trim()); + } catch (DateTimeParseException ignored) { + return parseHadoopStyleDuration(value); + } + } + + private static Duration parseHadoopStyleDuration(String value) { + try { + return TimeDurationUtil.getDuration("--startup-timeout", value, + TimeUnit.MILLISECONDS); + } catch (RuntimeException ex) { + throw new CommandLine.TypeConversionException(durationMessage()); + } + } + + private static String durationMessage() { + return "Use ISO-8601 like PT2M or Hadoop-style values like 120s."; + } + } } } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/package-info.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/package-info.java index ca9e55fd6cee..aa02e7fa5521 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/package-info.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/package-info.java @@ -16,6 +16,6 @@ */ /** - * Internal local single-node Ozone runtime support. + * Internal local Ozone cluster runtime support. */ package org.apache.hadoop.ozone.local; diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneClusterConfig.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneClusterConfig.java new file mode 100644 index 000000000000..7e7f82480246 --- /dev/null +++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneClusterConfig.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.local; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Paths; +import java.time.Duration; +import org.junit.jupiter.api.Test; + +/** + * Tests for {@link LocalOzoneClusterConfig}. + */ +class TestLocalOzoneClusterConfig { + + @Test + void builderProvidesLocalClusterDefaults() { + LocalOzoneClusterConfig config = LocalOzoneClusterConfig.builder().build(); + + assertEquals(LocalOzoneClusterConfig.DEFAULT_DATA_DIR, + config.getDataDir()); + assertEquals(LocalOzoneClusterConfig.FormatMode.IF_NEEDED, + config.getFormatMode()); + assertEquals(1, config.getDatanodes()); + assertEquals("127.0.0.1", config.getHost()); + assertEquals("0.0.0.0", config.getBindHost()); + assertEquals(0, config.getScmPort()); + assertEquals(0, config.getOmPort()); + assertEquals(0, config.getS3gPort()); + assertTrue(config.isS3gEnabled()); + assertFalse(config.isEphemeral()); + assertEquals(Duration.ofMinutes(2), config.getStartupTimeout()); + assertEquals("admin", config.getS3AccessKey()); + assertEquals("admin123", config.getS3SecretKey()); + assertEquals("us-east-1", config.getS3Region()); + } + + @Test + void typedDefaultsMatchSharedFallbackValues() { + assertEquals(LocalOzoneClusterConfig.FormatMode.fromString( + LocalOzoneClusterConfig.DEFAULT_FORMAT_MODE_VALUE), + LocalOzoneClusterConfig.DEFAULT_FORMAT_MODE); + assertEquals(Integer.parseInt( + LocalOzoneClusterConfig.DEFAULT_DATANODES_VALUE), + LocalOzoneClusterConfig.DEFAULT_DATANODES); + assertEquals(Integer.parseInt(LocalOzoneClusterConfig.DEFAULT_PORT_VALUE), + LocalOzoneClusterConfig.DEFAULT_PORT); + assertEquals(Boolean.parseBoolean( + LocalOzoneClusterConfig.DEFAULT_S3G_ENABLED_VALUE), + LocalOzoneClusterConfig.DEFAULT_S3G_ENABLED); + assertEquals(Boolean.parseBoolean( + LocalOzoneClusterConfig.DEFAULT_EPHEMERAL_VALUE), + LocalOzoneClusterConfig.DEFAULT_EPHEMERAL); + assertEquals(Duration.parse( + LocalOzoneClusterConfig.DEFAULT_STARTUP_TIMEOUT_VALUE), + LocalOzoneClusterConfig.DEFAULT_STARTUP_TIMEOUT); + } + + @Test + void builderAcceptsExplicitOverrides() { + LocalOzoneClusterConfig config = LocalOzoneClusterConfig.builder( + Paths.get("target", "custom-local-ozone")) + .setFormatMode(LocalOzoneClusterConfig.FormatMode.ALWAYS) + .setDatanodes(3) + .setHost("localhost") + .setBindHost("127.0.0.1") + .setScmPort(9860) + .setOmPort(9862) + .setS3gPort(9878) + .setS3gEnabled(false) + .setEphemeral(true) + .setStartupTimeout(Duration.ofSeconds(45)) + .setS3AccessKey("dev") + .setS3SecretKey("secret") + .setS3Region("ap-south-1") + .build(); + + assertEquals(Paths.get("target", "custom-local-ozone") + .toAbsolutePath().normalize(), config.getDataDir()); + assertEquals(LocalOzoneClusterConfig.FormatMode.ALWAYS, + config.getFormatMode()); + assertEquals(3, config.getDatanodes()); + assertEquals("localhost", config.getHost()); + assertEquals("127.0.0.1", config.getBindHost()); + assertEquals(9860, config.getScmPort()); + assertEquals(9862, config.getOmPort()); + assertEquals(9878, config.getS3gPort()); + assertFalse(config.isS3gEnabled()); + assertTrue(config.isEphemeral()); + assertEquals(Duration.ofSeconds(45), config.getStartupTimeout()); + assertEquals("dev", config.getS3AccessKey()); + assertEquals("secret", config.getS3SecretKey()); + assertEquals("ap-south-1", config.getS3Region()); + } + + @Test + void formatModeParsesUserFacingValues() { + assertEquals(LocalOzoneClusterConfig.FormatMode.IF_NEEDED, + LocalOzoneClusterConfig.FormatMode.fromString("if-needed")); + assertEquals(LocalOzoneClusterConfig.FormatMode.ALWAYS, + LocalOzoneClusterConfig.FormatMode.fromString(" always ")); + assertEquals(LocalOzoneClusterConfig.FormatMode.NEVER, + LocalOzoneClusterConfig.FormatMode.fromString("NEVER")); + } + + @Test + void formatModeRejectsUnknownValues() { + assertThrows(IllegalArgumentException.class, + () -> LocalOzoneClusterConfig.FormatMode.fromString("sometimes")); + } + + @Test + void formatModeRejectsNullValues() { + assertThrows(IllegalArgumentException.class, + () -> LocalOzoneClusterConfig.FormatMode.fromString(null)); + } +} diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestOzoneLocal.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestOzoneLocal.java index ff44f0ee5f68..219b3ae5d540 100644 --- a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestOzoneLocal.java +++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestOzoneLocal.java @@ -21,14 +21,23 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.ByteArrayOutputStream; import java.io.OutputStreamWriter; import java.io.PrintWriter; +import java.lang.reflect.Field; +import java.nio.file.Paths; +import java.time.Duration; import org.junit.jupiter.api.Test; import picocli.CommandLine; import picocli.CommandLine.Command; +import picocli.CommandLine.IDefaultValueProvider; +import picocli.CommandLine.Model.ArgSpec; +import picocli.CommandLine.Model.OptionSpec; +import picocli.CommandLine.Option; +import picocli.CommandLine.ParameterException; /** * Tests for {@link OzoneLocal}. @@ -54,14 +63,14 @@ void runCommandMetadataIsPresentAndHidden() { } @Test - void genericCliRegistersRunPlaceholder() { + void genericCliRegistersRunCommand() { OzoneLocal local = new OzoneLocal(); assertTrue(local.getCmd().getSubcommands().containsKey("run")); } @Test - void rootHelpHidesRunPlaceholder() throws Exception { + void rootHelpHidesRunCommand() throws Exception { OzoneLocal local = new OzoneLocal(); CommandLine commandLine = local.getCmd(); ByteArrayOutputStream out = new ByteArrayOutputStream(); @@ -76,12 +85,13 @@ void rootHelpHidesRunPlaceholder() throws Exception { String help = out.toString(UTF_8.name()); assertEquals(0, exitCode); assertTrue(help.contains("Usage: ozone local")); - assertFalse(help.contains("run")); + assertFalse(help.matches("(?s).*\\R\\s+run\\b.*"), help); assertEquals("", err.toString(UTF_8.name())); } @Test - void runCommandIsQuietNoOpPlaceholder() throws Exception { + void runCommandResolvesConfigurationQuietlyUntilRuntimeStartup() + throws Exception { OzoneLocal local = new OzoneLocal(); CommandLine commandLine = local.getCmd(); ByteArrayOutputStream out = new ByteArrayOutputStream(); @@ -97,4 +107,259 @@ void runCommandIsQuietNoOpPlaceholder() throws Exception { assertEquals("", out.toString(UTF_8.name())); assertEquals("", err.toString(UTF_8.name())); } + + @Test + void runCommandOptionsUseEnvironmentDefaults() throws Exception { + assertEnvDefault("dataDir", OzoneLocal.ENV_DATA_DIR, + LocalOzoneClusterConfig.DEFAULT_DATA_DIR_VALUE); + assertEnvDefault("formatMode", OzoneLocal.ENV_FORMAT, + LocalOzoneClusterConfig.DEFAULT_FORMAT_MODE_VALUE); + assertEnvDefault("datanodes", OzoneLocal.ENV_DATANODES, + LocalOzoneClusterConfig.DEFAULT_DATANODES_VALUE); + assertEnvDefault("host", OzoneLocal.ENV_HOST, + LocalOzoneClusterConfig.DEFAULT_HOST); + assertEnvDefault("bindHost", OzoneLocal.ENV_BIND_HOST, + LocalOzoneClusterConfig.DEFAULT_BIND_HOST); + assertEnvDefault("scmPort", OzoneLocal.ENV_SCM_PORT, + LocalOzoneClusterConfig.DEFAULT_PORT_VALUE); + assertEnvDefault("omPort", OzoneLocal.ENV_OM_PORT, + LocalOzoneClusterConfig.DEFAULT_PORT_VALUE); + assertEnvDefault("s3gEnabled", OzoneLocal.ENV_S3G_ENABLED, + LocalOzoneClusterConfig.DEFAULT_S3G_ENABLED_VALUE); + assertEnvDefault("s3gPort", OzoneLocal.ENV_S3G_PORT, + LocalOzoneClusterConfig.DEFAULT_PORT_VALUE); + assertEnvDefault("ephemeral", OzoneLocal.ENV_EPHEMERAL, + LocalOzoneClusterConfig.DEFAULT_EPHEMERAL_VALUE); + assertEnvDefault("startupTimeout", OzoneLocal.ENV_STARTUP_TIMEOUT, + LocalOzoneClusterConfig.DEFAULT_STARTUP_TIMEOUT_VALUE); + assertEnvDefault("s3AccessKey", OzoneLocal.ENV_S3_ACCESS_KEY, + LocalOzoneClusterConfig.DEFAULT_S3_ACCESS_KEY); + assertEnvDefault("s3SecretKey", OzoneLocal.ENV_S3_SECRET_KEY, + LocalOzoneClusterConfig.DEFAULT_S3_SECRET_KEY); + assertEnvDefault("s3Region", OzoneLocal.ENV_S3_REGION, + LocalOzoneClusterConfig.DEFAULT_S3_REGION); + } + + @Test + void resolveConfigUsesPicocliDefaults() { + LocalOzoneClusterConfig config = resolveWithFallbackDefaults(); + + assertEquals(LocalOzoneClusterConfig.DEFAULT_DATA_DIR, + config.getDataDir()); + assertEquals(LocalOzoneClusterConfig.FormatMode.IF_NEEDED, + config.getFormatMode()); + assertEquals(1, config.getDatanodes()); + assertEquals("127.0.0.1", config.getHost()); + assertEquals("0.0.0.0", config.getBindHost()); + assertEquals(0, config.getScmPort()); + assertEquals(0, config.getOmPort()); + assertEquals(0, config.getS3gPort()); + assertTrue(config.isS3gEnabled()); + assertFalse(config.isEphemeral()); + assertEquals(Duration.ofMinutes(2), config.getStartupTimeout()); + assertEquals("admin", config.getS3AccessKey()); + assertEquals("admin123", config.getS3SecretKey()); + assertEquals("us-east-1", config.getS3Region()); + } + + @Test + void resolveConfigUsesCliOverrides() { + LocalOzoneClusterConfig config = resolve( + "--data-dir", "target/cli-local", + "--format", "always", + "--datanodes", "3", + "--host", "cli-host", + "--bind-host", "127.0.0.1", + "--scm-port", "200", + "--om-port", "201", + "--s3g-port", "202", + "--no-s3g", + "--ephemeral", + "--startup-timeout", "45s", + "--s3-access-key", "cli-access", + "--s3-secret-key", "cli-secret", + "--s3-region", "cli-region"); + + assertEquals(Paths.get("target/cli-local").toAbsolutePath().normalize(), + config.getDataDir()); + assertEquals(LocalOzoneClusterConfig.FormatMode.ALWAYS, + config.getFormatMode()); + assertEquals(3, config.getDatanodes()); + assertEquals("cli-host", config.getHost()); + assertEquals("127.0.0.1", config.getBindHost()); + assertEquals(200, config.getScmPort()); + assertEquals(201, config.getOmPort()); + assertEquals(202, config.getS3gPort()); + assertFalse(config.isS3gEnabled()); + assertTrue(config.isEphemeral()); + assertEquals(Duration.ofSeconds(45), config.getStartupTimeout()); + assertEquals("cli-access", config.getS3AccessKey()); + assertEquals("cli-secret", config.getS3SecretKey()); + assertEquals("cli-region", config.getS3Region()); + } + + @Test + void resolveConfigParsesIsoStartupTimeout() { + LocalOzoneClusterConfig config = resolve("--startup-timeout", "PT45S"); + + assertEquals(Duration.ofSeconds(45), config.getStartupTimeout()); + } + + @Test + void resolveConfigAllowsS3gAndEphemeralToBeNegated() { + LocalOzoneClusterConfig config = resolve("--s3g", "--no-ephemeral"); + + assertTrue(config.isS3gEnabled()); + assertFalse(config.isEphemeral()); + } + + @Test + void resolveConfigRejectsInvalidFormat() { + assertParseError("--format", "sometimes", "--format"); + } + + @Test + void resolveConfigRejectsInvalidInteger() { + assertParseError("--datanodes", "two", "--datanodes"); + } + + @Test + void resolveConfigRejectsInvalidPort() { + assertConfigError("--scm-port", "65536", "--scm-port"); + } + + @Test + void resolveConfigRejectsDatanodeCountBelowOne() { + assertConfigError("--datanodes", "0", "--datanodes"); + } + + @Test + void resolveConfigRejectsInvalidDuration() { + assertParseError("--startup-timeout", "forever", "--startup-timeout"); + } + + @Test + void resolveConfigRejectsNonPositiveDuration() { + assertConfigError("--startup-timeout", "0s", "--startup-timeout"); + } + + @Test + void resolveConfigRejectsInvalidPath() { + assertParseError("--data-dir", "\0", "--data-dir"); + } + + @Test + void legacyWithoutS3gOptionIsNotAccepted() { + assertParseError("--without-s3g", "--without-s3g"); + } + + @Test + void genericCliErrorOutputIncludesOffendingConfigSource() + throws Exception { + OzoneLocal local = new OzoneLocal(); + ByteArrayOutputStream err = new ByteArrayOutputStream(); + local.getCmd().setErr(new PrintWriter(new OutputStreamWriter(err, UTF_8), + true)); + + int exitCode = local.execute(new String[] {"run", "--datanodes", "0"}); + + assertEquals(-1, exitCode); + assertTrue(err.toString(UTF_8.name()).contains("--datanodes")); + } + + private static LocalOzoneClusterConfig resolve(String... args) { + OzoneLocal.RunCommand command = new OzoneLocal.RunCommand(); + new CommandLine(command).parseArgs(args); + return command.resolveConfig(); + } + + private static LocalOzoneClusterConfig resolveWithFallbackDefaults( + String... args) { + OzoneLocal.RunCommand command = new OzoneLocal.RunCommand(); + new CommandLine(command) + .setDefaultValueProvider(new RunCommandFallbackDefaults()) + .parseArgs(args); + return command.resolveConfig(); + } + + private static void assertConfigError(String option, String value, + String expectedMessage) { + OzoneLocal.RunCommand command = new OzoneLocal.RunCommand(); + new CommandLine(command).parseArgs(option, value); + + IllegalArgumentException error = assertThrows(IllegalArgumentException.class, + command::resolveConfig); + + assertTrue(error.getMessage().contains(expectedMessage), + error.getMessage()); + } + + private static void assertParseError(String option, + String expectedMessage) { + OzoneLocal.RunCommand command = new OzoneLocal.RunCommand(); + ParameterException error = assertThrows(ParameterException.class, + () -> new CommandLine(command).parseArgs(option)); + + assertTrue(error.getMessage().contains(expectedMessage), + error.getMessage()); + } + + private static void assertParseError(String option, String value, + String expectedMessage) { + OzoneLocal.RunCommand command = new OzoneLocal.RunCommand(); + ParameterException error = assertThrows(ParameterException.class, + () -> new CommandLine(command).parseArgs(option, value)); + + assertTrue(error.getMessage().contains(expectedMessage), + error.getMessage()); + } + + private static void assertEnvDefault(String fieldName, + String environmentVariable, String fallback) throws Exception { + Field field = OzoneLocal.RunCommand.class.getDeclaredField(fieldName); + String defaultValue = field.getAnnotation(Option.class).defaultValue(); + + assertEquals("${env:" + environmentVariable + ":-" + fallback + "}", + defaultValue); + } + + private static final class RunCommandFallbackDefaults + implements IDefaultValueProvider { + + @Override + public String defaultValue(ArgSpec argSpec) { + if (!(argSpec instanceof OptionSpec)) { + return null; + } + String option = ((OptionSpec) argSpec).longestName(); + if ("--data-dir".equals(option)) { + return LocalOzoneClusterConfig.DEFAULT_DATA_DIR_VALUE; + } else if ("--format".equals(option)) { + return LocalOzoneClusterConfig.DEFAULT_FORMAT_MODE_VALUE; + } else if ("--datanodes".equals(option)) { + return LocalOzoneClusterConfig.DEFAULT_DATANODES_VALUE; + } else if ("--host".equals(option)) { + return LocalOzoneClusterConfig.DEFAULT_HOST; + } else if ("--bind-host".equals(option)) { + return LocalOzoneClusterConfig.DEFAULT_BIND_HOST; + } else if ("--scm-port".equals(option) + || "--om-port".equals(option) + || "--s3g-port".equals(option)) { + return LocalOzoneClusterConfig.DEFAULT_PORT_VALUE; + } else if ("--s3g".equals(option)) { + return LocalOzoneClusterConfig.DEFAULT_S3G_ENABLED_VALUE; + } else if ("--ephemeral".equals(option)) { + return LocalOzoneClusterConfig.DEFAULT_EPHEMERAL_VALUE; + } else if ("--startup-timeout".equals(option)) { + return LocalOzoneClusterConfig.DEFAULT_STARTUP_TIMEOUT_VALUE; + } else if ("--s3-access-key".equals(option)) { + return LocalOzoneClusterConfig.DEFAULT_S3_ACCESS_KEY; + } else if ("--s3-secret-key".equals(option)) { + return LocalOzoneClusterConfig.DEFAULT_S3_SECRET_KEY; + } else if ("--s3-region".equals(option)) { + return LocalOzoneClusterConfig.DEFAULT_S3_REGION; + } + return null; + } + } }