Skip to content

Commit 75037aa

Browse files
CodeByDrescherCodeByDrescher
authored
Merge pull request #837 from virtualcell/HDF5-Refactor-Part-1
Hdf5 refactor part 1
2 parents 702ee73 + a44f0ba commit 75037aa

File tree

111 files changed

+2803
-1267
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

111 files changed

+2803
-1267
lines changed

‎Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ LABEL \
1919
org.opencontainers.image.vendor="BioSimulators Team" \
2020
org.opencontainers.image.licenses="MIT" \
2121
\
22-
base_image="ubuntu:18.04" \
22+
base_image="ubuntu:20.04" \
2323
version="${SIMULATOR_VERSION}" \
2424
software="Virtual Cell" \
2525
software.version="${SIMULATOR_VERSION}" \

‎vcell-cli/src/main/java/org/vcell/cli/CLIPythonManager.java‎

Lines changed: 140 additions & 102 deletions
Large diffs are not rendered by default.

‎vcell-cli/src/main/java/org/vcell/cli/CLIRecorder.java‎

Lines changed: 72 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,17 @@
55
import java.nio.file.Paths;
66

77
import org.vcell.util.VCellUtilityHub;
8-
import org.vcell.util.recording.*;
8+
import org.vcell.util.recording.CLIRecordManager;
9+
import org.vcell.util.recording.Recorder;
10+
import org.vcell.util.recording.TextFileRecord;
911

12+
/**
13+
* Creates records for VCell CLI and the Record Management System (see `org.vcell.util.recording.*`)
14+
*/
1015
public class CLIRecorder extends Recorder implements CLIRecordable {
1116
protected final static boolean DEFAULT_SHOULD_PRINT_LOG_FILES = false, DEFAULT_SHOULD_FLUSH_LOG_FILES = false;
1217
protected boolean shouldPrintLogFiles, shouldFlushLogFiles;
13-
protected FileRecord detailedErrorLog, fullSuccessLog, errorLog, detailedResultsLog, spatialLog, importErrorLog;
18+
protected TextFileRecord detailedErrorLog, fullSuccessLog, errorLog, detailedResultsLog, spatialLog, importErrorLog;
1419
protected File outputDirectory;
1520

1621
// Note: this constructor is not public
@@ -31,14 +36,34 @@ protected CLIRecorder(boolean shouldPrintLogFiles, boolean shouldFlushLogFiles){
3136
this.shouldFlushLogFiles = shouldFlushLogFiles;
3237
}
3338

39+
/**
40+
* Basic constructor, will not override any functionality
41+
*
42+
* @param outputDirectoryPath where to put records (if applicable)
43+
* @throws IOException if there is a system IO issue breaking execution
44+
*/
3445
public CLIRecorder(File outputDirectoryPath) throws IOException {
3546
this(outputDirectoryPath, DEFAULT_SHOULD_PRINT_LOG_FILES);
3647
}
3748

49+
/**
50+
* Constructor that allows for forcing the creation of logs, even processing happens all in the same directory, allowing overrides.
51+
*
52+
* @param outputDirectory where to put records (if applicable)
53+
* @param forceLogFiles whether vcell should force log files to be created
54+
* @throws IOException if there is a system IO issue breaking execution.
55+
*/
3856
public CLIRecorder(File outputDirectory, boolean forceLogFiles) throws IOException {
3957
this(outputDirectory, forceLogFiles, DEFAULT_SHOULD_FLUSH_LOG_FILES);
4058
}
4159

60+
/**
61+
*
62+
* @param outputDirectory where to put records (if applicable)
63+
* @param forceLogFiles whether vcell should force log files to be created
64+
* @param shouldFlushLogFiles whether vcell should flush all changes to logs immediately to the system.
65+
* @throws IOException if there is a system IO issue breaking execution.
66+
*/
4267
public CLIRecorder(File outputDirectory, boolean forceLogFiles, boolean shouldFlushLogFiles) throws IOException {
4368
this(CLIUtils.isBatchExecution(outputDirectory.getAbsolutePath(), forceLogFiles), shouldFlushLogFiles);
4469
if (!outputDirectory.exists() && !outputDirectory.mkdirs()) {
@@ -49,46 +74,82 @@ public CLIRecorder(File outputDirectory, boolean forceLogFiles, boolean shouldFl
4974
}
5075
this.outputDirectory = outputDirectory;
5176

52-
RecordManager logManager = VCellUtilityHub.getLogManager();
53-
this.detailedErrorLog = logManager.requestNewFileLog(Paths.get(this.outputDirectory.getAbsolutePath(), "detailedErrorLog.txt").toString());
54-
this.fullSuccessLog = logManager.requestNewFileLog(Paths.get(this.outputDirectory.getAbsolutePath(), "fullSuccessLog.txt").toString());
55-
this.errorLog = logManager.requestNewFileLog(Paths.get(this.outputDirectory.getAbsolutePath(), "errorLog.txt").toString());
56-
this.detailedResultsLog = logManager.requestNewFileLog(Paths.get(this.outputDirectory.getAbsolutePath(), "detailedResultLog.txt").toString());
57-
this.spatialLog = logManager.requestNewFileLog(Paths.get(this.outputDirectory.getAbsolutePath(), "hasSpatialLog.txt").toString());
58-
this.importErrorLog = logManager.requestNewFileLog(Paths.get(this.outputDirectory.getAbsolutePath(), "importErrorLog.txt").toString());
77+
CLIRecordManager logManager;
78+
if (VCellUtilityHub.getLogManager() instanceof CLIRecordManager){
79+
logManager = (CLIRecordManager)VCellUtilityHub.getLogManager();
80+
} else {
81+
throw new RuntimeException("LogManager is not initalized to CLI-mode");
82+
}
83+
this.detailedErrorLog = logManager.requestNewRecord(Paths.get(this.outputDirectory.getAbsolutePath(), "detailedErrorLog.txt").toString());
84+
this.fullSuccessLog = logManager.requestNewRecord(Paths.get(this.outputDirectory.getAbsolutePath(), "fullSuccessLog.txt").toString());
85+
this.errorLog = logManager.requestNewRecord(Paths.get(this.outputDirectory.getAbsolutePath(), "errorLog.txt").toString());
86+
this.detailedResultsLog = logManager.requestNewRecord(Paths.get(this.outputDirectory.getAbsolutePath(), "detailedResultLog.txt").toString());
87+
this.spatialLog = logManager.requestNewRecord(Paths.get(this.outputDirectory.getAbsolutePath(), "hasSpatialLog.txt").toString());
88+
this.importErrorLog = logManager.requestNewRecord(Paths.get(this.outputDirectory.getAbsolutePath(), "importErrorLog.txt").toString());
5989

6090
this.createHeader();
6191
}
6292

6393
// Logging file methods
6494

65-
private void writeToFileLog(FileRecord log, String message) throws IOException {
95+
private void writeToFileLog(TextFileRecord log, String message) throws IOException {
6696
if (!this.shouldPrintLogFiles) return;
6797
log.print(message + "\n");
6898
if (this.shouldFlushLogFiles) log.flush();
6999
}
70100

101+
/**
102+
* Write to `detailedErrorLog.txt`
103+
*
104+
* @param message string to write to file
105+
*/
71106
public void writeDetailedErrorList(String message) throws IOException {
72107
this.writeToFileLog(this.detailedErrorLog, message);
73108
}
74109

110+
/**
111+
* Write to `fullSuccessLog.txt`
112+
*
113+
* @param message string to write to file
114+
*/
75115
public void writeFullSuccessList(String message) throws IOException {
76116
this.writeToFileLog(this.fullSuccessLog, message);
77117
}
78118

119+
/**
120+
* Write to `errorLog.txt`
121+
*
122+
* @param message string to write to file
123+
*/
79124
public void writeErrorList(String message) throws IOException {
80125
this.writeToFileLog(this.errorLog, message);
81126
}
82127

128+
/**
129+
* Write to `detailedResultLog.txt`
130+
*
131+
* @param message string to write to file
132+
*/
83133
public void writeDetailedResultList(String message) throws IOException {
84134
this.writeToFileLog(this.detailedResultsLog, message);
85135
}
86136

87-
// we make a list with the omex files that contain (some) spatial simulations (FVSolverStandalone solver)
137+
138+
/**
139+
* Write to `hasSpatialLog.txt`
140+
*
141+
* @param message string to write to file
142+
*/
88143
public void writeSpatialList(String message) throws IOException {
144+
// we make a list with the omex files that contain (some) spatial simulations (FVSolverStandalone solver)
89145
this.writeToFileLog(this.spatialLog, message);
90146
}
91147

148+
/**
149+
* Write to `importErrorLog.txt`
150+
*
151+
* @param message string to write to file
152+
*/
92153
public void writeImportErrorList(String message) throws IOException {
93154
this.writeToFileLog(this.importErrorLog, message);
94155
}

‎vcell-cli/src/main/java/org/vcell/cli/CLIUtils.java‎

Lines changed: 44 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
import org.apache.logging.log4j.Level;
66
import org.apache.logging.log4j.LogManager;
77
import org.apache.logging.log4j.core.LoggerContext;
8-
import org.apache.logging.log4j.core.config.Configuration;
98
import org.apache.logging.log4j.core.config.LoggerConfig;
109

1110
import org.apache.logging.log4j.Logger;
@@ -17,9 +16,20 @@
1716

1817
//import java.nio.file.Files;
1918

19+
/*
20+
* Static class for VCell CLI utility purposes.
21+
*/
2022
public class CLIUtils {
2123
private final static Logger logger = LogManager.getLogger(CLIUtils.class);
2224

25+
private CLIUtils(){}; // Static Class, no instances can be made
26+
27+
/**
28+
* Remove the directory and any contents inside
29+
*
30+
* @param file to recursively delete (if applicable)
31+
* @return success status of the operation
32+
*/
2333
public static boolean removeDirs(File f) {
2434
try {
2535
CLIUtils.deleteRecursively(f);
@@ -30,28 +40,12 @@ public static boolean removeDirs(File f) {
3040
return true;
3141
}
3242

33-
private static void deleteRecursively(File f) throws IOException {
34-
if (f.isDirectory()) {
35-
for (File c : Objects.requireNonNull(f.listFiles())) {
36-
CLIUtils.deleteRecursively(c);
37-
}
38-
}
39-
if (!f.delete()) {
40-
throw new FileNotFoundException("Failed to delete file: " + f);
41-
}
42-
}
43-
44-
// public static void cleanRootDir(File outdir){
45-
// // If this could be done without hard coding it'd be preferable.
46-
// String[] filesToDelete = {"detailedResultLog.txt", "fullSuccessLog.txt"};
47-
// for (String fileName : filesToDelete){
48-
// File instance = new File(outdir, fileName);
49-
// if (instance.exists()){
50-
// instance.delete();
51-
// }
52-
// }
53-
// }
54-
43+
/**
44+
* Changes the logging level at runtime for Vcell CLI
45+
*
46+
* @param ctx context to apply the logging change to
47+
* @param logLevel the level of logging to set to
48+
*/
5549
public static void setLogLevel(LoggerContext ctx, Level logLevel){
5650
org.apache.logging.log4j.core.config.Configuration config = ctx.getConfiguration();
5751

@@ -62,16 +56,32 @@ public static void setLogLevel(LoggerContext ctx, Level logLevel){
6256
ctx.updateLoggers(); // This causes all Loggers to refetch information from their LoggerConfig.
6357
}
6458

59+
/**
60+
* Determines whether a single file or a batch of files were submitted for processing in VCell CLI (or should be treated as a batch execution);
61+
*
62+
* @param outputBaseDir output dir of the execution
63+
* @param bForceKeepLogs whether VCell has been asked to force keeping logs
64+
* @return whether or not the exxecution should be treated as a batch execution
65+
*/
6566
public static boolean isBatchExecution(String outputBaseDir, boolean bForceKeepLogs) {
6667
Path path = Paths.get(outputBaseDir);
6768
boolean isDirectory = Files.isDirectory(path);
6869
return isDirectory || bForceKeepLogs;
6970
}
7071

72+
/**
73+
* Since VCell operates on Java 8, this is a string strip utility.
74+
*
75+
* @param str String to Strip whitespace from
76+
* @return the stripped string
77+
*/
7178
public static String stripString(String str){
7279
return str.replaceAll("^[ \t]+|[ \t]+$", ""); // replace whitespace at the front and back with nothing
7380
}
7481

82+
/**
83+
* Local Logger sub0class for `VCLogger` purposes
84+
*/
7585
public static class LocalLogger extends VCLogger {
7686
@Override
7787
public void sendMessage(Priority p, ErrorType et, String message) {
@@ -85,5 +95,16 @@ public boolean hasMessages() {
8595
return false;
8696
}
8797
}
98+
99+
private static void deleteRecursively(File f) throws IOException {
100+
if (f.isDirectory()) {
101+
for (File c : Objects.requireNonNull(f.listFiles())) {
102+
CLIUtils.deleteRecursively(c);
103+
}
104+
}
105+
if (!f.delete()) {
106+
throw new FileNotFoundException("Failed to delete file: " + f);
107+
}
108+
}
88109
}
89110

‎vcell-cli/src/main/java/org/vcell/cli/biosimulation/BiosimulationsCommand.java‎

Lines changed: 25 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -41,9 +41,14 @@ public class BiosimulationsCommand implements Callable<Integer> {
4141
private boolean help;
4242

4343
public Integer call() {
44-
CLIRecorder cliLogger = null;
44+
CLIRecorder cliRecorder = null;
45+
int returnCode;
46+
47+
if ((returnCode = this.noFurtherActionNeeded(bQuiet, bDebug, bVersion)) != -1)
48+
return returnCode;
49+
4550
try {
46-
cliLogger = new CLIRecorder(OUT_DIR); // CLILogger will throw an execption if our output dir isn't valid.
51+
cliRecorder = new CLIRecorder(OUT_DIR); // CLILogger will throw an execption if our output dir isn't valid.
4752
Level logLevel = logger.getLevel();
4853
if (!bQuiet && bDebug) {
4954
logLevel = Level.DEBUG;
@@ -68,19 +73,6 @@ public Integer call() {
6873
// Load properties before we need them below!
6974
PropertyLoader.loadProperties();
7075

71-
logger.debug("Validating CLI arguments");
72-
if (bDebug && bQuiet) {
73-
logger.error("cannot specify both debug and quiet, try --help for usage");
74-
return 1;
75-
}
76-
77-
if (bVersion) {
78-
String version = PropertyLoader.getRequiredProperty(PropertyLoader.vcellSoftwareVersion);
79-
logger.info("Outputing version:");
80-
System.out.println(version);
81-
return 0;
82-
}
83-
8476
logger.trace("Validating input");
8577
if (ARCHIVE == null) {
8678
logger.error("ARCHIVE file not specified, try --help for usage");
@@ -106,7 +98,7 @@ public Integer call() {
10698
logger.info("Beginning execution");
10799
try {
108100
CLIPythonManager.getInstance().instantiatePythonProcess();
109-
ExecuteImpl.singleMode(ARCHIVE, OUT_DIR, cliLogger);
101+
ExecuteImpl.singleMode(ARCHIVE, OUT_DIR, cliRecorder);
110102
return 0; // Does this prevent finally?
111103
} finally {
112104
try {
@@ -121,4 +113,21 @@ public Integer call() {
121113
return 1;
122114
}
123115
}
116+
117+
private int noFurtherActionNeeded(boolean bQuiet, boolean bDebug, boolean bVersion){
118+
logger.debug("Validating CLI arguments");
119+
if (bVersion) {
120+
String version = PropertyLoader.getRequiredProperty(PropertyLoader.vcellSoftwareVersion);
121+
logger.info("Outputing version:");
122+
System.out.println(version);
123+
return 0;
124+
}
125+
126+
if (bDebug && bQuiet) {
127+
logger.error("cannot specify both debug and quiet, try --help for usage");
128+
return 1;
129+
}
130+
131+
return -1;
132+
}
124133
}

‎vcell-cli/src/main/java/org/vcell/cli/run/ExecuteImpl.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
import cbit.vcell.parser.ExpressionException;
44
import cbit.vcell.solver.ode.ODESolverResultSet;
55

6-
import org.vcell.cli.CLIRecorder;
6+
import org.vcell.cli.CLIRecordable;
77
import org.vcell.cli.PythonStreamException;
88
import org.vcell.cli.vcml.VCMLHandler;
99
import org.vcell.util.FileUtils;
@@ -22,7 +22,7 @@ public class ExecuteImpl {
2222

2323
private final static Logger logger = LogManager.getLogger(ExecuteImpl.class);
2424

25-
public static void batchMode(File dirOfArchivesToProcess, File outputDir, CLIRecorder cliLogger,
25+
public static void batchMode(File dirOfArchivesToProcess, File outputDir, CLIRecordable cliLogger,
2626
boolean bKeepTempFiles, boolean bExactMatchOnly, boolean bSmallMeshOverride) {
2727
FilenameFilter filter = (f, name) -> name.endsWith(".omex") || name.endsWith(".vcml");
2828
File[] inputFiles = dirOfArchivesToProcess.listFiles(filter);
@@ -67,7 +67,7 @@ public static void batchMode(File dirOfArchivesToProcess, File outputDir, CLIRec
6767
}
6868
}
6969

70-
public static void singleMode(File inputFile, File rootOutputDir, CLIRecorder cliLogger,
70+
public static void singleMode(File inputFile, File rootOutputDir, CLIRecordable cliLogger,
7171
boolean bKeepTempFiles, boolean bExactMatchOnly, boolean bEncapsulateOutput, boolean bSmallMeshOverride) throws Exception {
7272
// Build statuses
7373
String bioModelBaseName = FileUtils.getBaseName(inputFile.getName()); // bioModelBaseName = input file without the path
@@ -81,7 +81,7 @@ public static void singleMode(File inputFile, File rootOutputDir, CLIRecorder cl
8181
ExecuteImpl.singleExecOmex(inputFile, rootOutputDir, cliLogger, bKeepTempFiles, bExactMatchOnly, bEncapsulateOutput, bSmallMeshOverride);
8282
}
8383

84-
public static void singleMode(File inputFile, File outputDir, CLIRecorder cliLogger) throws Exception {
84+
public static void singleMode(File inputFile, File outputDir, CLIRecordable cliLogger) throws Exception {
8585
final boolean bKeepTempFiles = false;
8686
final boolean bExactMatchOnly = false;
8787
final boolean bEncapsulateOutput = false;
@@ -91,7 +91,7 @@ public static void singleMode(File inputFile, File outputDir, CLIRecorder cliLog
9191
}
9292

9393
@Deprecated
94-
public static void singleExecVcml(File vcmlFile, File outputDir, CLIRecorder cliLogger) {
94+
public static void singleExecVcml(File vcmlFile, File outputDir, CLIRecordable cliLogger) {
9595
logger.warn("Using deprecated function to execute vcml");
9696
VCMLHandler.outputDir = outputDir.getAbsolutePath();
9797
logger.debug("Executing VCML file " + vcmlFile);
@@ -143,7 +143,7 @@ public static void singleExecVcml(File vcmlFile, File outputDir, CLIRecorder cli
143143
}
144144
}
145145

146-
private static void singleExecOmex(File inputFile, File rootOutputDir, CLIRecorder cliRecorder,
146+
private static void singleExecOmex(File inputFile, File rootOutputDir, CLIRecordable cliRecorder,
147147
boolean bKeepTempFiles, boolean bExactMatchOnly, boolean bEncapsulateOutput, boolean bSmallMeshOverride)
148148
throws ExecutionException, PythonStreamException, IOException, InterruptedException {
149149
ExecutionJob requestedExecution = new ExecutionJob(inputFile, rootOutputDir, cliRecorder,

0 commit comments

Comments
 (0)