Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -54,14 +54,16 @@ public class ExportCommand extends ExportCommandOptions implements Callable<Inte
public Integer call() throws Exception {
validateDeprecatedOptions();
applyDeprecatedOptions();
resolveDefaults();

String scalarDbPropertiesFilePath = getScalarDbPropertiesFilePath();

try {
validateOutputDirectory();
FileUtils.validateFilePath(scalarDbPropertiesFilePath);
validatePositiveValue(
spec.commandLine(), dataChunkSize, DataLoaderError.INVALID_DATA_CHUNK_SIZE);
validatePositiveValue(spec.commandLine(), maxThreads, DataLoaderError.INVALID_MAX_THREADS);
validatePositiveValue(spec.commandLine(), threadCount, DataLoaderError.INVALID_THREAD_COUNT);

StorageFactory storageFactory = StorageFactory.create(scalarDbPropertiesFilePath);
TableMetadataService metaDataService =
Expand Down Expand Up @@ -126,6 +128,8 @@ private void validateDeprecatedOptions() {
DEPRECATED_END_EXCLUSIVE_OPTION,
END_INCLUSIVE_OPTION,
END_INCLUSIVE_OPTION_SHORT);
validateDeprecatedOptionPair(
spec.commandLine(), DEPRECATED_MAX_THREADS_OPTION, THREADS_OPTION, null);
}

private String getScalarDbPropertiesFilePath() {
Expand Down Expand Up @@ -170,7 +174,7 @@ private ExportOptions buildExportOptions(Key partitionKey, ScanRange scanRange)
.includeTransactionMetadata(includeTransactionMetadata)
.delimiter(delimiter)
.limit(limit)
.maxThreadCount(maxThreads)
.threadCount(threadCount)
.dataChunkSize(dataChunkSize)
.prettyPrintJson(prettyPrintJson)
.scanRange(scanRange);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ public class ExportCommandOptions {
public static final String END_INCLUSIVE_OPTION = "--end-inclusive";
public static final String END_INCLUSIVE_OPTION_SHORT = "-ei";
public static final String DEPRECATED_END_EXCLUSIVE_OPTION = "--end-exclusive";
public static final String THREADS_OPTION = "--threads";
public static final String DEPRECATED_MAX_THREADS_OPTION = "--max-threads";

@CommandLine.Option(
names = {"--config", "-c"},
Expand Down Expand Up @@ -73,11 +75,20 @@ public class ExportCommandOptions {
protected boolean includeTransactionMetadata;

@CommandLine.Option(
names = {"--max-threads", "-mt"},
paramLabel = "<MAX_THREADS>",
names = {"--threads"},
paramLabel = "<THREADS>",
description =
"Maximum number of threads to use for parallel processing (default: number of available processors)")
protected int maxThreads;
"Number of threads to use for parallel processing (default: number of available processors)")
protected int threadCount;

// Deprecated option - kept for backward compatibility
@CommandLine.Option(
names = {DEPRECATED_MAX_THREADS_OPTION, "-mt"},
paramLabel = "<MAX_THREADS>",
description = "Deprecated: Use --threads instead",
hidden = true)
@Deprecated
protected Integer maxThreadsDeprecated;

@CommandLine.Option(
names = {"--start-key", "-sk"},
Expand Down Expand Up @@ -184,5 +195,24 @@ public void applyDeprecatedOptions() {
if (endExclusiveDeprecated != null) {
scanEndInclusive = !endExclusiveDeprecated;
}

// If the deprecated option is set, use its value
if (maxThreadsDeprecated != null) {
threadCount = maxThreadsDeprecated;
}
}

/**
* Resolves default values for options.
*
* <p>This method is called AFTER applyDeprecatedOptions() to resolve any default values that
* depend on runtime information. For threadCount, a value of 0 indicates that the number of
* available processors should be used.
*/
public void resolveDefaults() {
// Resolve threadCount: 0 means use available processors
if (threadCount == 0) {
threadCount = Runtime.getRuntime().availableProcessors();
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -55,13 +55,15 @@ public class ImportCommand extends ImportCommandOptions implements Callable<Inte
public Integer call() throws Exception {
validateDeprecatedOptions();
applyDeprecatedOptions();
resolveDefaults();

validateImportTarget(controlFilePath, namespace, tableName);
validateLogDirectory(logDirectory);
validatePositiveValue(
spec.commandLine(), dataChunkSize, DataLoaderError.INVALID_DATA_CHUNK_SIZE);
validatePositiveValue(
spec.commandLine(), transactionSize, DataLoaderError.INVALID_TRANSACTION_SIZE);
validatePositiveValue(spec.commandLine(), maxThreads, DataLoaderError.INVALID_MAX_THREADS);
validatePositiveValue(spec.commandLine(), threadCount, DataLoaderError.INVALID_THREAD_COUNT);
validatePositiveValue(
spec.commandLine(), dataChunkQueueSize, DataLoaderError.INVALID_DATA_CHUNK_QUEUE_SIZE);
ControlFile controlFile = parseControlFileFromPath(controlFilePath).orElse(null);
Expand Down Expand Up @@ -282,10 +284,7 @@ private Optional<ControlFile> parseControlFileFromPath(String controlFilePath) {
*/
private void validateDeprecatedOptions() {
validateDeprecatedOptionPair(
spec.commandLine(),
DEPRECATED_THREADS_OPTION,
MAX_THREADS_OPTION,
MAX_THREADS_OPTION_SHORT);
spec.commandLine(), DEPRECATED_MAX_THREADS_OPTION, THREADS_OPTION, null);
}

/**
Expand All @@ -308,7 +307,7 @@ private ImportOptions createImportOptions(ControlFile controlFile) {
.namespace(namespace)
.dataChunkSize(dataChunkSize)
.transactionBatchSize(transactionSize)
.maxThreads(maxThreads)
.threadCount(threadCount)
.dataChunkQueueSize(dataChunkQueueSize)
.tableName(tableName);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,8 @@
public class ImportCommandOptions {

public static final String FILE_OPTION_NAME_LONG_FORMAT = "--file";
public static final String MAX_THREADS_OPTION = "--max-threads";
public static final String MAX_THREADS_OPTION_SHORT = "-mt";
public static final String DEPRECATED_THREADS_OPTION = "--threads";
public static final String THREADS_OPTION = "--threads";
public static final String DEPRECATED_MAX_THREADS_OPTION = "--max-threads";

@CommandLine.Option(
names = {"--mode", "-m"},
Expand All @@ -35,21 +34,20 @@ public class ImportCommandOptions {
protected String sourceFilePath;

@CommandLine.Option(
names = {"--max-threads", "-mt"},
paramLabel = "<MAX_THREADS>",
names = {"--threads"},
paramLabel = "<THREADS>",
description =
"Maximum number of threads to use for parallel processing (default: number of available processors)",
defaultValue = "16")
protected int maxThreads;
"Number of threads to use for parallel processing (default: number of available processors)")
protected int threadCount;

// Deprecated option - kept for backward compatibility
@CommandLine.Option(
names = {DEPRECATED_THREADS_OPTION},
paramLabel = "<THREADS>",
description = "Deprecated: Use --max-threads instead",
names = {DEPRECATED_MAX_THREADS_OPTION, "-mt"},
paramLabel = "<MAX_THREADS>",
description = "Deprecated: Use --threads instead",
hidden = true)
@Deprecated
protected Integer threadsDeprecated;
protected Integer maxThreadsDeprecated;

@CommandLine.Option(
names = {"--namespace", "-ns"},
Expand Down Expand Up @@ -180,8 +178,22 @@ public class ImportCommandOptions {
*/
public void applyDeprecatedOptions() {
// If the deprecated option is set, use its value
if (threadsDeprecated != null) {
maxThreads = threadsDeprecated;
if (maxThreadsDeprecated != null) {
threadCount = maxThreadsDeprecated;
}
}

/**
* Resolves default values for options.
*
* <p>This method is called AFTER applyDeprecatedOptions() to resolve any default values that
* depend on runtime information. For threadCount, a value of 0 indicates that the number of
* available processors should be used.
*/
public void resolveDefaults() {
// Resolve threadCount: 0 means use available processors
if (threadCount == 0) {
threadCount = Runtime.getRuntime().availableProcessors();
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ void removeFileIfCreated() {
void call_withBlankScalarDBConfigurationFile_shouldThrowException() {
exportCommand.configFilePath = "";
exportCommand.dataChunkSize = 100;
exportCommand.maxThreads = 4;
exportCommand.threadCount = 4;
exportCommand.namespace = "scalar";
exportCommand.table = "asset";
exportCommand.outputDirectory = "";
Expand All @@ -62,7 +62,7 @@ void call_withBlankScalarDBConfigurationFile_shouldThrowException() {
void call_withInvalidScalarDBConfigurationFile_shouldReturnOne() throws Exception {
exportCommand.configFilePath = "scalardb.properties";
exportCommand.dataChunkSize = 100;
exportCommand.maxThreads = 4;
exportCommand.threadCount = 4;
exportCommand.namespace = "scalar";
exportCommand.table = "asset";
exportCommand.outputDirectory = "";
Expand Down Expand Up @@ -190,4 +190,94 @@ void call_withOnlyDeprecatedEndExclusive_shouldApplyInvertedValue() {
// end-exclusive=false should become end-inclusive=true
assertEquals(true, command.scanEndInclusive);
}

@Test
void call_withBothThreadsAndMaxThreads_shouldThrowException() {
assertBothDeprecatedAndNewOptionsThrowException(
"--max-threads=8", "--threads=16", "--max-threads", "--threads");
}

@Test
void call_withOnlyDeprecatedMaxThreads_shouldApplyValue() {
// Simulate command line parsing with only deprecated option
String[] args = {
"--config",
"scalardb.properties",
"--namespace",
"scalar",
"--table",
"asset",
"--format",
"JSON",
"--max-threads",
"12"
};
ExportCommand command = new ExportCommand();
CommandLine cmd = new CommandLine(command);
cmd.parseArgs(args);

// Verify the deprecated value was parsed
assertEquals(12, command.maxThreadsDeprecated);

// Apply deprecated options
command.applyDeprecatedOptions();

// Verify the value was applied to threadCount
assertEquals(12, command.threadCount);
}

@Test
void call_withOnlyThreads_shouldUseValue() {
// Simulate command line parsing with only new --threads option
String[] args = {
"--config",
"scalardb.properties",
"--namespace",
"scalar",
"--table",
"asset",
"--format",
"JSON",
"--threads",
"20"
};
ExportCommand command = new ExportCommand();
CommandLine cmd = new CommandLine(command);
cmd.parseArgs(args);

// Verify the value was set to threadCount
assertEquals(20, command.threadCount);

// Verify the deprecated value was not set
assertEquals(null, command.maxThreadsDeprecated);
}

@Test
void call_withDeprecatedShortOption_shouldApplyValue() {
// Simulate command line parsing with deprecated short option -mt
String[] args = {
"--config",
"scalardb.properties",
"--namespace",
"scalar",
"--table",
"asset",
"--format",
"JSON",
"-mt",
"15"
};
ExportCommand command = new ExportCommand();
CommandLine cmd = new CommandLine(command);
cmd.parseArgs(args);

// Verify the deprecated value was parsed
assertEquals(15, command.maxThreadsDeprecated);

// Apply deprecated options
command.applyDeprecatedOptions();

// Verify the value was applied to threadCount
assertEquals(15, command.threadCount);
}
}
Loading
Loading