diff --git a/CHANGELOG.md b/CHANGELOG.md index d2e2f7e..2aa64b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # FANS Changelog +## latest + +- Added MPI communicator abstraction [#139](https://github.com/DataAnalyticsEngineering/FANS/pull/139) + ## v0.6.2 - Fix bug demanding a leading slash in the datasetname and exiting ungracefully [#123](https://github.com/DataAnalyticsEngineering/FANS/pull/123) diff --git a/include/reader.h b/include/reader.h index cccff29..75a504b 100644 --- a/include/reader.h +++ b/include/reader.h @@ -17,6 +17,7 @@ class Reader { public: // Default constructor Reader() = default; + Reader(const MPI_Comm &comm); // Destructor to free allocated memory ~Reader(); @@ -53,8 +54,9 @@ class Reader { unsigned short *ms{nullptr}; // Micro-structure bool is_zyx = true; - int world_rank; - int world_size; + int world_rank; + int world_size; + MPI_Comm communicator; ptrdiff_t alloc_local; ptrdiff_t local_n0; @@ -63,7 +65,7 @@ class Reader { ptrdiff_t local_1_start; // void Setup(ptrdiff_t howmany); - void ReadInputFile(char input_fn[]); + void ReadInputFile(const std::string &input_fn); void ReadMS(int hm); void ComputeVolumeFractions(); // void ReadHDF5(char file_name[], char dset_name[]); diff --git a/include/solver.h b/include/solver.h index a058224..3858a42 100644 --- a/include/solver.h +++ b/include/solver.h @@ -18,6 +18,7 @@ class Solver : private MixedBCController { const int world_rank; const int world_size; + MPI_Comm communicator; const ptrdiff_t n_x, n_y, n_z; // NOTE: the order in the declaration is very important because it is the same order in which the later initialization via member initializer lists takes place @@ -109,6 +110,7 @@ Solver::Solver(Reader &reader, MaterialManager * matmanager(matmgr), world_rank(reader.world_rank), world_size(reader.world_size), + communicator(reader.communicator), n_x(reader.dims[0]), n_y(reader.dims[1]), n_z(reader.dims[2]), @@ -219,8 +221,8 @@ void Solver::CreateFFTWPlans(double *in, fftw_complex *transform // But, according to https://fftw.org/doc/MPI-Plan-Creation.html the BLOCK sizes must be the same: // "These must be the same block sizes as were passed to the corresponding ‘local_size’ function" const ptrdiff_t n[3] = {n_x, n_y, n_z}; - planfft = fftw_mpi_plan_many_dft_r2c(rank, n, howmany, iblock, oblock, in, transformed, MPI_COMM_WORLD, FFTW_MEASURE | FFTW_MPI_TRANSPOSED_OUT); - planifft = fftw_mpi_plan_many_dft_c2r(rank, n, howmany, iblock, oblock, transformed, out, MPI_COMM_WORLD, FFTW_MEASURE | FFTW_MPI_TRANSPOSED_IN); + planfft = fftw_mpi_plan_many_dft_r2c(rank, n, howmany, iblock, oblock, in, transformed, communicator, FFTW_MEASURE | FFTW_MPI_TRANSPOSED_OUT); + planifft = fftw_mpi_plan_many_dft_c2r(rank, n, howmany, iblock, oblock, transformed, out, communicator, FFTW_MEASURE | FFTW_MPI_TRANSPOSED_IN); // see https://eigen.tuxfamily.org/dox/group__TutorialMapClass.html#title3 new (&rhat) Map((std::complex *) transformed, local_n1 * n_x * (n_z / 2 + 1) * howmany); @@ -243,7 +245,7 @@ void Solver::compute_residual_basic(RealArray &r_matrix, RealArr // int MPI_Sendrecv(void *sendbuf, int sendcount, MPI_Datatype sendtype, int dest, int sendtag, void *recvbuf, // int recvcount, MPI_Datatype recvtype, int source, int recvtag, MPI_Comm comm, MPI_Status *status) MPI_Sendrecv(u, n_y * n_z * howmany, MPI_DOUBLE, (world_rank + world_size - 1) % world_size, 0, - u + local_n0 * n_y * n_z * howmany, n_y * n_z * howmany, MPI_DOUBLE, (world_rank + 1) % world_size, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); + u + local_n0 * n_y * n_z * howmany, n_y * n_z * howmany, MPI_DOUBLE, (world_rank + 1) % world_size, 0, communicator, MPI_STATUS_IGNORE); Matrix ue; @@ -263,7 +265,7 @@ void Solver::compute_residual_basic(RealArray &r_matrix, RealArr }); MPI_Sendrecv(r + local_n0 * n_y * (n_z + padding) * howmany, n_y * (n_z + padding) * howmany, MPI_DOUBLE, (world_rank + 1) % world_size, 0, - buffer_padding, n_y * (n_z + padding) * howmany, MPI_DOUBLE, (world_rank + world_size - 1) % world_size, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); + buffer_padding, n_y * (n_z + padding) * howmany, MPI_DOUBLE, (world_rank + world_size - 1) % world_size, 0, communicator, MPI_STATUS_IGNORE); RealArray b(buffer_padding, n_z * howmany, n_y, OuterStride<>((n_z + padding) * howmany)); // NOTE: for any padding of more than 2, the buffer_padding has to be extended @@ -428,7 +430,7 @@ double Solver::compute_error(RealArray &r) } double err; - MPI_Allreduce(&err_local, &err, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + MPI_Allreduce(&err_local, &err, 1, MPI_DOUBLE, MPI_MAX, communicator); err_all[iter] = err; double err0 = err_all[0]; @@ -496,7 +498,7 @@ void Solver::postprocess(Reader &reader, int load_idx, int time_ vector phase_counts(n_mat, 0); MPI_Sendrecv(v_u, n_y * n_z * howmany, MPI_DOUBLE, (world_rank + world_size - 1) % world_size, 0, - v_u + local_n0 * n_y * n_z * howmany, n_y * n_z * howmany, MPI_DOUBLE, (world_rank + 1) % world_size, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); + v_u + local_n0 * n_y * n_z * howmany, n_y * n_z * howmany, MPI_DOUBLE, (world_rank + 1) % world_size, 0, communicator, MPI_STATUS_IGNORE); Matrix ue; int phase_id; @@ -556,16 +558,16 @@ void Solver::postprocess(Reader &reader, int load_idx, int time_ }); } - MPI_Allreduce(MPI_IN_PLACE, stress_average.data(), n_str, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); - MPI_Allreduce(MPI_IN_PLACE, strain_average.data(), n_str, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + MPI_Allreduce(MPI_IN_PLACE, stress_average.data(), n_str, MPI_DOUBLE, MPI_SUM, communicator); + MPI_Allreduce(MPI_IN_PLACE, strain_average.data(), n_str, MPI_DOUBLE, MPI_SUM, communicator); stress_average /= (n_x * n_y * n_z); strain_average /= (n_x * n_y * n_z); // Reduce per-phase accumulations across all processes for (int mat_index = 0; mat_index < n_mat; ++mat_index) { - MPI_Allreduce(MPI_IN_PLACE, phase_stress_average[mat_index].data(), n_str, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); - MPI_Allreduce(MPI_IN_PLACE, phase_strain_average[mat_index].data(), n_str, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); - MPI_Allreduce(MPI_IN_PLACE, &phase_counts[mat_index], 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); + MPI_Allreduce(MPI_IN_PLACE, phase_stress_average[mat_index].data(), n_str, MPI_DOUBLE, MPI_SUM, communicator); + MPI_Allreduce(MPI_IN_PLACE, phase_strain_average[mat_index].data(), n_str, MPI_DOUBLE, MPI_SUM, communicator); + MPI_Allreduce(MPI_IN_PLACE, &phase_counts[mat_index], 1, MPI_INT, MPI_SUM, communicator); // Compute average for each phase if (phase_counts[mat_index] > 0) { @@ -716,7 +718,7 @@ VectorXd Solver::get_homogenized_stress() homogenized_stress = VectorXd::Zero(n_str); MPI_Sendrecv(v_u, n_y * n_z * howmany, MPI_DOUBLE, (world_rank + world_size - 1) % world_size, 0, - v_u + local_n0 * n_y * n_z * howmany, n_y * n_z * howmany, MPI_DOUBLE, (world_rank + 1) % world_size, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); + v_u + local_n0 * n_y * n_z * howmany, n_y * n_z * howmany, MPI_DOUBLE, (world_rank + 1) % world_size, 0, communicator, MPI_STATUS_IGNORE); Matrix ue; int phase_id; @@ -733,7 +735,7 @@ VectorXd Solver::get_homogenized_stress() homogenized_stress += stress.segment(n_str * idx[0], n_str); }); - MPI_Allreduce(MPI_IN_PLACE, homogenized_stress.data(), n_str, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + MPI_Allreduce(MPI_IN_PLACE, homogenized_stress.data(), n_str, MPI_DOUBLE, MPI_SUM, communicator); homogenized_stress /= (n_x * n_y * n_z); return homogenized_stress; diff --git a/include/solverCG.h b/include/solverCG.h index 842317c..2d07849 100644 --- a/include/solverCG.h +++ b/include/solverCG.h @@ -55,7 +55,7 @@ double SolverCG::dotProduct(RealArray &a, RealArray &b) { double local_value = (a * b).sum(); double result; - MPI_Allreduce(&local_value, &result, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + MPI_Allreduce(&local_value, &result, 1, MPI_DOUBLE, MPI_SUM, this->communicator); return result; } diff --git a/pixi.lock b/pixi.lock index e035181..3ce0791 100644 --- a/pixi.lock +++ b/pixi.lock @@ -1592,7 +1592,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/eigen-abi-5.0.1.80-hf414acd_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.3.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/executing-2.2.1-pyhd8ed1ab_0.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/fftw-3.3.10-mpi_openmpi_h76e6d66_12.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/fftw-3.3.11-mpi_openmpi_h76e6d66_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/filelock-3.29.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/fmt-12.1.0-hff5e90c_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-dejavu-sans-mono-2.37-hab24e00_0.tar.bz2 @@ -2000,7 +2000,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/eigen-abi-5.0.1.100-h9a8c16c_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.3.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/executing-2.2.1-pyhd8ed1ab_0.conda - - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/fftw-3.3.10-mpi_openmpi_h79548e3_12.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/fftw-3.3.11-mpi_openmpi_h79548e3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/filelock-3.29.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/fmt-12.1.0-h20c602a_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-dejavu-sans-mono-2.37-hab24e00_0.tar.bz2 @@ -2407,7 +2407,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-64/eigen-abi-5.0.1.80-h969a130_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.3.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/executing-2.2.1-pyhd8ed1ab_0.conda - - conda: https://conda.anaconda.org/conda-forge/osx-64/fftw-3.3.10-mpi_openmpi_h63b4274_12.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/fftw-3.3.11-mpi_openmpi_h63b4274_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/filelock-3.29.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/fmt-12.1.0-hda137b5_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-dejavu-sans-mono-2.37-hab24e00_0.tar.bz2 @@ -2768,7 +2768,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/eigen-abi-5.0.1.100-h485a483_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.3.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/executing-2.2.1-pyhd8ed1ab_0.conda - - conda: https://conda.anaconda.org/conda-forge/osx-arm64/fftw-3.3.10-mpi_openmpi_h83537c7_12.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/fftw-3.3.11-mpi_openmpi_h83537c7_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/filelock-3.29.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/fmt-12.1.0-h403dcb5_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/font-ttf-dejavu-sans-mono-2.37-hab24e00_0.tar.bz2 @@ -6402,7 +6402,7 @@ packages: depends: - libcxx >=22 - hdf5 >=1.14.6,<1.14.7.0a0 mpi_openmpi_* - - fftw >=3.3.10,<4.0a0 + - fftw >=3.3.11,<4.0a0 - fftw * mpi_openmpi_* - conda: . name: fans @@ -6416,7 +6416,7 @@ packages: - libstdcxx >=15 - libgcc >=15 - hdf5 >=1.14.6,<1.14.7.0a0 mpi_openmpi_* - - fftw >=3.3.10,<4.0a0 + - fftw >=3.3.11,<4.0a0 - fftw * mpi_openmpi_* - conda: . name: fans @@ -6429,7 +6429,7 @@ packages: depends: - libcxx >=22 - hdf5 >=1.14.6,<1.14.7.0a0 mpi_openmpi_* - - fftw >=3.3.10,<4.0a0 + - fftw >=3.3.11,<4.0a0 - fftw * mpi_openmpi_* - conda: . name: fans @@ -6442,7 +6442,7 @@ packages: - libstdcxx >=15 - libgcc >=15 - hdf5 >=1.14.6,<1.14.7.0a0 mpi_openmpi_* - - fftw >=3.3.10,<4.0a0 + - fftw >=3.3.11,<4.0a0 - fftw * mpi_openmpi_* - conda: https://conda.anaconda.org/conda-forge/linux-64/fftw-3.3.10-mpi_openmpi_h76e6d66_12.conda sha256: 66b0fe5b5e52565ee7120f5e897b1d206494c0440b7c00d4ec0ae09549e1cedc @@ -6471,6 +6471,20 @@ packages: license_family: GPL size: 1925113 timestamp: 1771754008607 +- conda: https://conda.anaconda.org/conda-forge/linux-64/fftw-3.3.11-mpi_openmpi_h76e6d66_0.conda + sha256: 0949915580511f07e46d5509d8d16e7ae52910c426aa026dfba51cd188f0d93f + md5: 1f27b20b2c508b341d2f2fffc038318b + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + - libgfortran + - libgfortran5 >=14.3.0 + - libstdcxx >=14 + - openmpi >=5.0.10,<6.0a0 + license: GPL-2.0-or-later + license_family: GPL + size: 2243005 + timestamp: 1776781947588 - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/fftw-3.3.10-mpi_openmpi_h79548e3_12.conda sha256: da93aca35dfc28cadf02d952c2e2edbbc82e13e6503058f525eda3d92f41cd3f md5: 862a16425e14f102cf30818160328013 @@ -6496,6 +6510,19 @@ packages: license_family: GPL size: 1446023 timestamp: 1771753652128 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/fftw-3.3.11-mpi_openmpi_h79548e3_0.conda + sha256: b209929f5c33756a3c84f8ef43507749b85be09ee0b3b975b5e0c2f3484e3e66 + md5: db9e364f3a2395debbbb5dbbdd6c7313 + depends: + - libgcc >=14 + - libgfortran + - libgfortran5 >=14.3.0 + - libstdcxx >=14 + - openmpi >=5.0.10,<6.0a0 + license: GPL-2.0-or-later + license_family: GPL + size: 1765261 + timestamp: 1776782018274 - conda: https://conda.anaconda.org/conda-forge/osx-64/fftw-3.3.10-mpi_openmpi_h63b4274_12.conda sha256: e5dbf27576c5773034560395e37220b95b4b279ed037da2a9af7d36981cfbedf md5: 6e7cd3e40705a769a1e1af286b668f70 @@ -6523,6 +6550,20 @@ packages: license_family: GPL size: 1807039 timestamp: 1771754664178 +- conda: https://conda.anaconda.org/conda-forge/osx-64/fftw-3.3.11-mpi_openmpi_h63b4274_0.conda + sha256: be9d79fe2fa453c6ce0b83a5b2c809c7e9a3bdf6af4b5a984b72add962331bca + md5: cd9442c07a68a171acc0c3eec57b0b85 + depends: + - __osx >=11.0 + - libcxx >=19 + - libgfortran + - libgfortran5 >=14.3.0 + - llvm-openmp >=19.1.7 + - openmpi >=5.0.10,<6.0a0 + license: GPL-2.0-or-later + license_family: GPL + size: 1861341 + timestamp: 1776783990081 - conda: https://conda.anaconda.org/conda-forge/osx-arm64/fftw-3.3.10-mpi_openmpi_h83537c7_12.conda sha256: aff483eea254aa370c76eec87dd19f981c791a0824dec9c2f0e01881d95e3d5e md5: 96b7104aadf94ad19b5592655da364ce @@ -6550,6 +6591,20 @@ packages: license_family: GPL size: 757377 timestamp: 1771753896425 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/fftw-3.3.11-mpi_openmpi_h83537c7_0.conda + sha256: 31353fa1ad22a539b788347687e9a93e40bb44c8653733edfe0de310d6fabced + md5: 3ce1481c6d215e40ade8e91265a37d54 + depends: + - __osx >=11.0 + - libcxx >=19 + - libgfortran + - libgfortran5 >=14.3.0 + - llvm-openmp >=19.1.7 + - openmpi >=5.0.10,<6.0a0 + license: GPL-2.0-or-later + license_family: GPL + size: 782914 + timestamp: 1776782206460 - conda: https://conda.anaconda.org/conda-forge/noarch/filelock-3.29.0-pyhd8ed1ab_0.conda sha256: 6b471a18372bbd52bdf32fc965f71de3bc1b5219418b8e6b3875a67a7b08c483 md5: 8fa8358d022a3a9bd101384a808044c6 diff --git a/pyfans/micro.cpp b/pyfans/micro.cpp index b9f3636..2e9f7b9 100644 --- a/pyfans/micro.cpp +++ b/pyfans/micro.cpp @@ -24,11 +24,40 @@ py::array_t merge_arrays(py::array_t array1, py::array_t return result; } -MicroSimulation::MicroSimulation(int sim_id, char *input_file) +PyFANSConfig load_config(const std::string &file_path) +{ + PyFANSConfig config{}; + + try { + ifstream i(file_path); + json j; + i >> j; + + if (j.contains("no_mpi") && j["no_mpi"].get()) + config.disable_mpi = true; + // ... more entries + } catch (const std::exception &e) { + printf("ERROR trying to read config file '%s' for pyFANS: %s\n", file_path.c_str(), e.what()); + printf("Falling back to default values.\n"); + } + + return config; +} + +MicroSimulation::MicroSimulation(int sim_id, const std::string &input_file, const std::string &config_file) { // initialize fftw mpi fftw_mpi_init(); + const auto config = load_config(config_file); + + // Avoiding reader re-initialization due to unnecessary buffer copies + reader.communicator = MPI_COMM_WORLD; + if (config.disable_mpi) + reader.communicator = MPI_COMM_SELF; + MPI_Comm_rank(reader.communicator, &reader.world_rank); + MPI_Comm_size(reader.communicator, &reader.world_size); + // Input file name is hardcoded. TODO: Make it configurable reader.ReadInputFile(input_file); diff --git a/pyfans/micro.hpp b/pyfans/micro.hpp index b6ebc6c..2dbe418 100644 --- a/pyfans/micro.hpp +++ b/pyfans/micro.hpp @@ -19,7 +19,7 @@ namespace py = pybind11; class MicroSimulation { public: - MicroSimulation(int sim_id, char *input_file = "input.json"); + MicroSimulation(int sim_id, const std::string &input_file = "input.json", const std::string &config_file = "pyfans-config.json"); py::dict solve(py::dict macro_write_data, double dt); private: @@ -30,3 +30,7 @@ class MicroSimulation { Solver<3, 6> *solver; double pert_param = 1e-6; // scalar strain perturbation parameter }; + +struct PyFANSConfig { + bool disable_mpi = false; +}; diff --git a/src/main.cpp b/src/main.cpp index 1df26f5..4832fc6 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -60,7 +60,7 @@ int main(int argc, char *argv[]) MPI_Init(NULL, NULL); fftw_mpi_init(); - Reader reader; + Reader reader{MPI_COMM_WORLD}; reader.ReadInputFile(argv[1]); reader.OpenResultsFile(argv[2]); diff --git a/src/reader.cpp b/src/reader.cpp index 13251cd..ea7dd7f 100644 --- a/src/reader.cpp +++ b/src/reader.cpp @@ -10,6 +10,13 @@ #include "H5FDmpi.h" #include "H5FDmpio.h" +Reader::Reader(const MPI_Comm &comm) + : communicator(comm) +{ + MPI_Comm_rank(communicator, &world_rank); + MPI_Comm_size(communicator, &world_size); +} + void Reader::ComputeVolumeFractions() { unsigned short local_max = 0; @@ -29,8 +36,8 @@ void Reader::ComputeVolumeFractions() // Find the global maximum and minimum material indices unsigned short global_max, global_min; - MPI_Allreduce(&local_max, &global_max, 1, MPI_UNSIGNED_SHORT, MPI_MAX, MPI_COMM_WORLD); - MPI_Allreduce(&local_min, &global_min, 1, MPI_UNSIGNED_SHORT, MPI_MIN, MPI_COMM_WORLD); + MPI_Allreduce(&local_max, &global_max, 1, MPI_UNSIGNED_SHORT, MPI_MAX, communicator); + MPI_Allreduce(&local_min, &global_min, 1, MPI_UNSIGNED_SHORT, MPI_MIN, communicator); // Calculate total number of materials n_mat = global_max - global_min + 1; @@ -52,7 +59,7 @@ void Reader::ComputeVolumeFractions() for (int i = 0; i < n_mat; i++) { long vf; - MPI_Allreduce(&(vol_frac[i]), &vf, 1, MPI_LONG, MPI_SUM, MPI_COMM_WORLD); + MPI_Allreduce(&(vol_frac[i]), &vf, 1, MPI_LONG, MPI_SUM, communicator); v_frac[i] = double(vf) / double(dims[0] * dims[1] * dims[2]); if (world_rank == 0) printf("# material %4u vol. frac. %10.4f%% \n", @@ -60,13 +67,9 @@ void Reader::ComputeVolumeFractions() } } -void Reader ::ReadInputFile(char input_fn[]) +void Reader ::ReadInputFile(const std::string &input_fn) { try { - - MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); - MPI_Comm_size(MPI_COMM_WORLD, &world_size); - ifstream i(input_fn); json j; i >> j; @@ -175,9 +178,8 @@ void Reader ::ReadInputFile(char input_fn[]) printf("# FANS Tolerance: \t %10.5e\n", errorParameters["tolerance"].get()); printf("# Max iterations: \t %6i\n", n_it); } - } catch (const std::exception &e) { - fprintf(stderr, "ERROR trying to read input file '%s' for FANS: %s\n", input_fn, e.what()); + fprintf(stderr, "ERROR trying to read input file '%s' for FANS: %s\n", input_fn.c_str(), e.what()); exit(10); } } @@ -236,10 +238,6 @@ void Reader ::ReadMS(int hm) hid_t plist_id; /* property list identifier */ herr_t status; - MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); - MPI_Comm_size(MPI_COMM_WORLD, &world_size); - MPI_Info info = MPI_INFO_NULL; - // Set up file access property list with parallel I/O access plist_id = H5Pcreate(H5P_FILE_ACCESS); // H5Pset_fapl_mpio(plist_id, MPI_COMM_WORLD, info); // "set File Access Property List" @@ -324,11 +322,11 @@ void Reader ::ReadMS(int hm) ptrdiff_t *local_n1, ptrdiff_t *local_1_start); \ */ - alloc_local = fftw_mpi_local_size_many_transposed(rank, n, hm, block0, block1, MPI_COMM_WORLD, &local_n0, &local_0_start, &local_n1, &local_1_start); + alloc_local = fftw_mpi_local_size_many_transposed(rank, n, hm, block0, block1, communicator, &local_n0, &local_0_start, &local_n1, &local_1_start); if (local_n0 < 4) throw std::runtime_error("[ FANS3D_Grid ] ERROR: Number of voxels in x-direction is less than 4 in process " + to_string(world_rank)); - MPI_Barrier(MPI_COMM_WORLD); + MPI_Barrier(communicator); hsize_t fcount[3], foffset[3]; if (is_zyx) { /* file layout Z Y X */ @@ -414,7 +412,7 @@ void Reader::OpenResultsFile(const char *output_fn) { std::snprintf(results_filename, sizeof(results_filename), "%s", output_fn); hid_t plist_id = H5Pcreate(H5P_FILE_ACCESS); - H5Pset_fapl_mpio(plist_id, MPI_COMM_WORLD, MPI_INFO_NULL); + H5Pset_fapl_mpio(plist_id, communicator, MPI_INFO_NULL); results_file_id = H5Fcreate(results_filename, H5F_ACC_TRUNC, H5P_DEFAULT, plist_id); H5Pclose(plist_id);