Skip to content

Commit

Permalink
#810 Add the possibility to zip the inputs and outputs for N simulation
Browse files Browse the repository at this point in the history 
Signed-off-by: Dimitri Baron <dimitri.baron@rte-france.com>
  • Loading branch information
@barondim
barondim committed Dec 20, 2024
1 parent 901fea6 commit cc022b3
Show file tree
Hide file tree
Showing 7 changed files with 184 additions and 44 deletions.
177 changes: 143 additions & 34 deletions scripts/envDFL.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -494,120 +494,221 @@ build_tests_coverage() {
}

launch() {
if [ -n "$ZIP_ARCHIVE" ]; then
$DYNAFLOW_LAUNCHER_INSTALL_DIR/bin/DynaFlowLauncher \
--log-level $DYNAFLOW_LAUNCHER_LOG_LEVEL \
--network $2 \
--config $3 \
--input-archive $ZIP_ARCHIVE
else
if [ ! -f "$2" ]; then
error_exit "IIDM network file $2 doesn't exist"
error_exit "IIDM network file $2 doesn't exist"
fi
if [ ! -f "$3" ]; then
error_exit "DFL configuration file $3 doesn't exist"
error_exit "DFL configuration file $3 doesn't exist"
fi
$DYNAFLOW_LAUNCHER_INSTALL_DIR/bin/DynaFlowLauncher \
--log-level $DYNAFLOW_LAUNCHER_LOG_LEVEL \
--network $2 \
--config $3
fi
}

launch_gdb() {
if [ -n "$ZIP_ARCHIVE" ]; then
gdb --args $DYNAFLOW_LAUNCHER_INSTALL_DIR/bin/DynaFlowLauncher \
--log-level $DYNAFLOW_LAUNCHER_LOG_LEVEL \
--network $2 \
--config $3 \
--input-archive $ZIP_ARCHIVE
else
if [ ! -f "$2" ]; then
error_exit "IIDM network file $1 doesn't exist"
error_exit "IIDM network file $1 doesn't exist"
fi
if [ ! -f "$3" ]; then
error_exit "DFL configuration file $2 doesn't exist"
error_exit "DFL configuration file $2 doesn't exist"
fi
gdb --args $DYNAFLOW_LAUNCHER_INSTALL_DIR/bin/DynaFlowLauncher \
--log-level $DYNAFLOW_LAUNCHER_LOG_LEVEL \
--network $2 \
--config $3
fi
}

launch_valgrind() {
if [ -n "$ZIP_ARCHIVE" ]; then
valgrind --tool=callgrind --dump-instr=yes --collect-jumps=yes $DYNAFLOW_LAUNCHER_INSTALL_DIR/bin/DynaFlowLauncher \
--log-level $DYNAFLOW_LAUNCHER_LOG_LEVEL \
--network $2 \
--config $3 \
--input-archive $ZIP_ARCHIVE
else
if [ ! -f "$2" ]; then
error_exit "IIDM network file $1 doesn't exist"
error_exit "IIDM network file $1 doesn't exist"
fi
if [ ! -f "$3" ]; then
error_exit "DFL configuration file $2 doesn't exist"
error_exit "DFL configuration file $2 doesn't exist"
fi
valgrind --tool=callgrind --dump-instr=yes --collect-jumps=yes $DYNAFLOW_LAUNCHER_INSTALL_DIR/bin/DynaFlowLauncher \
--log-level $DYNAFLOW_LAUNCHER_LOG_LEVEL \
--network $2 \
--config $3
fi
}


launch_sa() {
if [ ! -f "$2" ]; then
export_preload
if [ "${DYNAWO_USE_MPI}" == "YES" ]; then
if [ -n "$ZIP_ARCHIVE" ]; then
"$MPIRUN_PATH" -np $NBPROCS $DYNAFLOW_LAUNCHER_INSTALL_DIR/bin/DynaFlowLauncher --log-level $DYNAFLOW_LAUNCHER_LOG_LEVEL \
--network $2 \
--config $3 \
--contingencies $4 \
--input-archive $ZIP_ARCHIVE
else
if [ ! -f "$2" ]; then
error_exit "IIDM network file $2 doesn't exist"
fi
if [ ! -f "$3" ]; then
fi
if [ ! -f "$3" ]; then
error_exit "DFL configuration file $3 doesn't exist"
fi
if [ ! -f "$4" ]; then
fi
if [ ! -f "$4" ]; then
error_exit "Security Analysis contingencies file $4 doesn't exist"
fi
export_preload
if [ "${DYNAWO_USE_MPI}" == "YES" ]; then
fi
"$MPIRUN_PATH" -np $NBPROCS $DYNAFLOW_LAUNCHER_INSTALL_DIR/bin/DynaFlowLauncher --log-level $DYNAFLOW_LAUNCHER_LOG_LEVEL \
--network $2 \
--config $3 \
--contingencies $4
fi
else
if [ -n "$ZIP_ARCHIVE" ]; then
$DYNAFLOW_LAUNCHER_INSTALL_DIR/bin/DynaFlowLauncher --log-level $DYNAFLOW_LAUNCHER_LOG_LEVEL \
--network $2 \
--config $3 \
--contingencies $4 \
--input-archive $ZIP_ARCHIVE
else
if [ ! -f "$2" ]; then
error_exit "IIDM network file $2 doesn't exist"
fi
if [ ! -f "$3" ]; then
error_exit "DFL configuration file $3 doesn't exist"
fi
if [ ! -f "$4" ]; then
error_exit "Security Analysis contingencies file $4 doesn't exist"
fi
$DYNAFLOW_LAUNCHER_INSTALL_DIR/bin/DynaFlowLauncher --log-level $DYNAFLOW_LAUNCHER_LOG_LEVEL \
--network $2 \
--config $3 \
--contingencies $4
fi
unset LD_PRELOAD
fi
unset LD_PRELOAD
}

launch_nsa() {
if [ ! -f "$2" ]; then
export_preload
if [ "${DYNAWO_USE_MPI}" == "YES" ]; then
if [ -n "$ZIP_ARCHIVE" ]; then
"$MPIRUN_PATH" -np $NBPROCS $DYNAFLOW_LAUNCHER_INSTALL_DIR/bin/DynaFlowLauncher --log-level $DYNAFLOW_LAUNCHER_LOG_LEVEL \
--network $2 \
--config $3 \
--contingencies $4 \
--input-archive $ZIP_ARCHIVE \
--nsa # Steady state calculations.
else
if [ ! -f "$2" ]; then
error_exit "IIDM network file $2 doesn't exist"
fi
if [ ! -f "$3" ]; then
fi
if [ ! -f "$3" ]; then
error_exit "DFL configuration file $3 doesn't exist"
fi
if [ ! -f "$4" ]; then
fi
if [ ! -f "$4" ]; then
error_exit "Security Analysis contingencies file $4 doesn't exist"
fi
export_preload
if [ "${DYNAWO_USE_MPI}" == "YES" ]; then
fi
"$MPIRUN_PATH" -np $NBPROCS $DYNAFLOW_LAUNCHER_INSTALL_DIR/bin/DynaFlowLauncher --log-level $DYNAFLOW_LAUNCHER_LOG_LEVEL \
--network $2 \
--config $3 \
--contingencies $4 \
--nsa # Steady state calculations.
fi
else
if [ -n "$ZIP_ARCHIVE" ]; then
$DYNAFLOW_LAUNCHER_INSTALL_DIR/bin/DynaFlowLauncher --log-level $DYNAFLOW_LAUNCHER_LOG_LEVEL \
--network $2 \
--config $3 \
--contingencies $4 \
--input-archive $ZIP_ARCHIVE \
--nsa # Steady state calculations.
else
if [ ! -f "$2" ]; then
error_exit "IIDM network file $2 doesn't exist"
fi
if [ ! -f "$3" ]; then
error_exit "DFL configuration file $3 doesn't exist"
fi
if [ ! -f "$4" ]; then
error_exit "Security Analysis contingencies file $4 doesn't exist"
fi
$DYNAFLOW_LAUNCHER_INSTALL_DIR/bin/DynaFlowLauncher --log-level $DYNAFLOW_LAUNCHER_LOG_LEVEL \
--network $2 \
--config $3 \
--contingencies $4 \
--nsa # Steady state calculations.
fi
unset LD_PRELOAD
fi
unset LD_PRELOAD
}

launch_sa_gdb() {
if [ ! -f "$2" ]; then
export_preload
if [ "${DYNAWO_USE_MPI}" == "YES" ]; then
if [ -n "$ZIP_ARCHIVE" ]; then
"$MPIRUN_PATH" -np $NBPROCS xterm -e gdb -q --args $DYNAFLOW_LAUNCHER_INSTALL_DIR/bin/DynaFlowLauncher --log-level $DYNAFLOW_LAUNCHER_LOG_LEVEL \
--network $2 \
--config $3 \
--contingencies $4 \
--input-archive $ZIP_ARCHIVE
else
if [ ! -f "$2" ]; then
error_exit "IIDM network file $2 doesn't exist"
fi
if [ ! -f "$3" ]; then
fi
if [ ! -f "$3" ]; then
error_exit "DFL configuration file $3 doesn't exist"
fi
if [ ! -f "$4" ]; then
fi
if [ ! -f "$4" ]; then
error_exit "Security Analysis contingencies file $4 doesn't exist"
fi
export_preload
if [ "${DYNAWO_USE_MPI}" == "YES" ]; then
fi
"$MPIRUN_PATH" -np $NBPROCS xterm -e gdb -q --args $DYNAFLOW_LAUNCHER_INSTALL_DIR/bin/DynaFlowLauncher --log-level $DYNAFLOW_LAUNCHER_LOG_LEVEL \
--network $2 \
--config $3 \
--contingencies $4
fi
else
if [ -n "$ZIP_ARCHIVE" ]; then
gdb -q --args $DYNAFLOW_LAUNCHER_INSTALL_DIR/bin/DynaFlowLauncher --log-level $DYNAFLOW_LAUNCHER_LOG_LEVEL \
--network $2 \
--config $3 \
--contingencies $4 \
--input-archive $ZIP_ARCHIVE
else
if [ ! -f "$2" ]; then
error_exit "IIDM network file $2 doesn't exist"
fi
if [ ! -f "$3" ]; then
error_exit "DFL configuration file $3 doesn't exist"
fi
if [ ! -f "$4" ]; then
error_exit "Security Analysis contingencies file $4 doesn't exist"
fi
gdb -q --args $DYNAFLOW_LAUNCHER_INSTALL_DIR/bin/DynaFlowLauncher --log-level $DYNAFLOW_LAUNCHER_LOG_LEVEL \
--network $2 \
--config $3 \
--contingencies $4
fi
unset LD_PRELOAD
fi
unset LD_PRELOAD
}

version() {
Expand Down Expand Up @@ -740,11 +841,19 @@ case "$1" in
;;
esac

# Nb Threads
# parse options
ARGS=""
while (($#)); do
key="$1"
case "$key" in
--input-archive)
ZIP_ARCHIVE=$2
shift 2 # pass argument and value
;;
--input-archive=*)
ZIP_ARCHIVE="${1#*=}"
shift # past value
;;
--nbThreads|-np)
NBPROCS=$2
shift 2 # pass argument and value
Expand Down
1 change: 1 addition & 0 deletions sources/Common/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ target_link_libraries(dfl_Common

PRIVATE
Boost::filesystem
libZIP::libZIP
)
add_library(DynaFlowLauncher::common ALIAS dfl_Common)
install(FILES ${CMAKE_SOURCE_DIR}/etc/Dictionaries/DFLLog_en_GB.dic ${CMAKE_SOURCE_DIR}/etc/Dictionaries/DFLError_en_GB.dic DESTINATION share)
Expand Down
2 changes: 1 addition & 1 deletion sources/Common/include/Options.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@

#pragma once

#include <boost/optional.hpp>
#include <boost/program_options.hpp>
#include <iostream>
#include <string>
Expand Down Expand Up @@ -43,6 +42,7 @@ class Options {
std::string networkFilePath; ///< Network filepath to process
std::string contingenciesFilePath; ///< Contingencies filepath for security analysis
std::string configPath; ///< Launcher configuration filepath
std::string zipArchivePath; ///< zip archive path to unzip to get input files
std::string dynawoLogLevel; ///< chosen log level
};

Expand Down
39 changes: 31 additions & 8 deletions sources/Common/src/Options.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,10 @@

#include "version.h"

#include <libzip/ZipInputStream.h>

#include <DYNFileSystemUtils.h>

#include <algorithm>
#include <boost/filesystem.hpp>
#include <sstream>
Expand Down Expand Up @@ -81,14 +85,18 @@ Options::basename(const std::string& filepath) {
return path.filename().replace_extension().generic_string();
}

Options::Options() : desc_{}, config_{"", "", "", "", defaultLogLevel_} {
desc_.add_options()("help,h", "Display help message")(
"log-level", po::value<ParsedLogLevel>(),
(std::string("Dynawo logger level (allowed values are ERROR, WARN, INFO, DEBUG): default is ") + defaultLogLevel_).c_str())(
"network", po::value<std::string>(&config_.networkFilePath)->required(), "Network file path to process (IIDM support only)")(
"contingencies", po::value<std::string>(&config_.contingenciesFilePath), "Contingencies file path to process (Security Analysis)")(
"config", po::value<std::string>(&config_.configPath)->required(), "launcher Configuration file to use")("version,v", "Display version")(
"nsa", "Run steady state calculation followed by security analysis. Requires contingencies file to be defined.");
Options::Options() : desc_{}, config_{"", "", "", "", "", defaultLogLevel_} {
desc_.add_options()
("help,h", "Display help message")
("version,v", "Display version")
("log-level", po::value<ParsedLogLevel>(),
(std::string("Dynawo logger level (allowed values are ERROR, WARN, INFO, DEBUG): default is ") + defaultLogLevel_).c_str())
("network", po::value<std::string>(&config_.networkFilePath)->required(), "Network file path to process (IIDM support only)")
("config", po::value<std::string>(&config_.configPath)->required(), "launcher Configuration file to use")
("contingencies", po::value<std::string>(&config_.contingenciesFilePath), "Contingencies file path to process (Security Analysis)")
("nsa", "Run steady state calculation followed by security analysis. Requires contingencies file to be defined.")
("input-archive", po::value<std::string>(&config_.zipArchivePath),
"Path to a ZIP archive containing input files for '--network', '--config', and '--contingencies'.");
}

Options::Request
Expand All @@ -108,6 +116,21 @@ Options::parse(int argc, char* argv[]) {

po::notify(vm);

if (vm.count("input-archive") > 0) {
boost::shared_ptr<zip::ZipFile> archive = zip::ZipInputStream::read(config_.zipArchivePath);
std::string archiveParentPath = boost::filesystem::path(config_.zipArchivePath).parent_path().string();
for (std::map<std::string, boost::shared_ptr<zip::ZipEntry>>::const_iterator archiveIt = archive->getEntries().begin();
archiveIt != archive->getEntries().end(); ++archiveIt) {
std::string name = archiveIt->first;
std::string data(archiveIt->second->getData());
std::ofstream file;
std::string filepath = createAbsolutePath(name, archiveParentPath);
file.open(createAbsolutePath(name, archiveParentPath).c_str(), std::ios::binary);
file << data;
file.close();
}
}

// These are not binded automatically
if (vm.count("log-level") > 0) {
config_.dynawoLogLevel = vm["log-level"].as<ParsedLogLevel>().logLevelDefinition;
Expand Down
6 changes: 5 additions & 1 deletion sources/Context.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -350,7 +350,11 @@ void Context::executeSecurityAnalysis() {
multipleJobs->setScenarios(scenarios);
auto saLauncher = boost::make_shared<DYNAlgorithms::SystematicAnalysisLauncher>();
saLauncher->setMultipleJobs(multipleJobs);
saLauncher->setOutputFile("aggregatedResults.xml");
if (def_.outputIsZip) {
saLauncher->setOutputFile("output.zip");
} else {
saLauncher->setOutputFile("aggregatedResults.xml");
}
saLauncher->setDirectory(config_.outputDir().generic_string());
saLauncher->init();
saLauncher->launch();
Expand Down
1 change: 1 addition & 0 deletions sources/Context.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ class Context {
boost::filesystem::path settingFilePath; ///< setting file path for dynamic data base
boost::filesystem::path assemblingFilePath; ///< assembling file path for dynamic data base
boost::filesystem::path contingenciesFilePath; ///< contigencies file path for Security Analysis simulation
bool outputIsZip; ///< true if the output is zip archive, false otherwise
std::string dynawoLogLevel; ///< string representation of the dynawo log level
boost::filesystem::path dynawoResDir; ///< DYNAWO resources
std::string locale; ///< localization
Expand Down
2 changes: 2 additions & 0 deletions sources/main.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,12 +53,14 @@ static inline double elapsed(const std::chrono::steady_clock::time_point &timePo

static boost::shared_ptr<dfl::Context> buildContext(dfl::inputs::SimulationParams const &params, dfl::inputs::Configuration &config) {
auto timeContextStart = std::chrono::steady_clock::now();
bool outputIsZip = !params.runtimeConfig->zipArchivePath.empty();
dfl::Context::ContextDef def{config.getStartingPointMode(),
params.simulationKind,
params.networkFilePath,
config.settingFilePath(),
config.assemblingFilePath(),
params.runtimeConfig->contingenciesFilePath,
outputIsZip,
params.runtimeConfig->dynawoLogLevel,
params.resourcesDirPath,
params.locale};
Expand Down

0 comments on commit cc022b3

Please sign in to comment.