From 1389c7103a181f98f8e0ad69da571cd94ec4cbdb Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Tue, 25 Jun 2024 14:57:15 -0700 Subject: [PATCH 01/14] Changed MPI_ variables to SPHERAL_MPI_, added default communicator input for allReduce, moved some MPI functions from Utilities to Distributed, combined scan.hh with allReduce.hh and removed mpiUtilities --- src/Boundary/InflowOutflowBoundary.cc | 14 +-- src/Boundary/PlanarBoundary.cc | 1 - .../findNodesTouchingThroughPlanes.cc | 4 +- src/Damage/DamageModel.cc | 2 +- src/Damage/IvanoviSALEDamageModel.cc | 2 +- src/Damage/ProbabilisticDamageModel.cc | 20 ++-- .../weibullFlawDistributionBenzAsphaug.cc | 14 +-- src/Damage/weibullFlawDistributionOwen.cc | 18 ++-- src/DataBase/DataBase.cc | 44 ++++----- src/Distributed/CMakeLists.txt | 74 ++++++++------- src/{Utilities => Distributed}/Process.cc | 0 src/{Utilities => Distributed}/Process.hh | 0 .../RegisterMPIDataTypes.cc | 0 .../RegisterMPIDataTypes.hh | 0 .../RegisterMPIDataTypesInline.hh | 0 .../SpaceFillingCurveRedistributeNodes.cc | 14 +-- src/Distributed/VoronoiRedistributeNodes.cc | 16 ++-- src/{Utilities => Distributed}/allReduce.hh | 40 ++++++-- src/Field/FieldInline.hh | 8 +- src/Field/FieldListInline.hh | 8 +- src/FieldOperations/binFieldList2Lattice.cc | 8 +- src/FractalStruct/check_for_edge_trouble.cc | 92 +++++++++---------- src/Gravity/NBodyGravity.cc | 8 +- src/Gravity/PolyGravity.cc | 4 +- src/Gravity/Tree.cc | 6 +- src/Gravity/TreeGravity.cc | 8 +- src/Hydro/VoronoiHourglassControl.cc | 6 +- src/Integrator/Integrator.cc | 4 +- src/KernelIntegrator/FlatConnectivity.cc | 14 +-- src/KernelIntegrator/KernelIntegrator.cc | 4 - src/Mesh/LineMesh.cc | 8 +- src/Mesh/Mesh.cc | 19 ++-- src/Mesh/MeshInline.hh | 6 +- src/Mesh/PolyhedralMesh.cc | 4 +- src/Mesh/siloMeshDump.cc | 3 - src/Neighbor/ConnectivityMap.cc | 8 +- src/Neighbor/TreeNeighbor.cc | 6 +- .../centroidalRelaxNodesImpl.cc | 10 +- src/NodeGenerators/compactFacetedVolumes.cc | 10 +- src/NodeGenerators/fillFacetedVolume.cc | 4 +- .../generateCylDistributionFromRZ.cc | 20 +--- src/NodeGenerators/relaxNodeDistribution.cc | 11 +-- src/RK/computeVoronoiVolume.cc | 4 +- src/Utilities/CMakeLists.txt | 8 -- src/Utilities/DBC.hh | 2 +- src/Utilities/DataTypeTraits.hh | 2 +- src/Utilities/globalBoundingVolumes.cc | 15 +-- src/Utilities/iterateIdealH.cc | 4 +- src/Utilities/mpiUtilities.hh | 46 ---------- src/Utilities/nodeOrdering.cc | 36 +------- src/Utilities/scan.hh | 56 ----------- 51 files changed, 284 insertions(+), 431 deletions(-) rename src/{Utilities => Distributed}/Process.cc (100%) rename src/{Utilities => Distributed}/Process.hh (100%) rename src/{Utilities => Distributed}/RegisterMPIDataTypes.cc (100%) rename src/{Utilities => Distributed}/RegisterMPIDataTypes.hh (100%) rename src/{Utilities => Distributed}/RegisterMPIDataTypesInline.hh (100%) rename src/{Utilities => Distributed}/allReduce.hh (56%) delete mode 100644 src/Utilities/mpiUtilities.hh delete mode 100644 src/Utilities/scan.hh diff --git a/src/Boundary/InflowOutflowBoundary.cc b/src/Boundary/InflowOutflowBoundary.cc index ee74624ed..a1b6322c3 100644 --- a/src/Boundary/InflowOutflowBoundary.cc +++ b/src/Boundary/InflowOutflowBoundary.cc @@ -11,7 +11,7 @@ #include "Field/FieldBase.hh" #include "Hydro/HydroFieldNames.hh" #include "Geometry/GeometryRegistrar.hh" -#include "Utilities/allReduce.hh" +#include "Distributed/allReduce.hh" #include "Utilities/planarReflectingOperator.hh" #include "Utilities/DBC.hh" @@ -133,7 +133,7 @@ updateGhostNodes(NodeList& nodeList) { const auto xd = mPlane.signedDistance(pos[i]); xmin = std::min(xmin, xd); } - xmin = allReduce(xmin, MPI_MIN, Communicator::communicator()); + xmin = allReduce(xmin, SPHERAL_MPI_MIN); // CHECK(xmin >= 0.0); // Offset the current ghost points appropriately. @@ -309,8 +309,8 @@ InflowOutflowBoundary::initializeProblemStartup(const bool /*final*/) for (const auto i: nodeIDs) { vinflow += vel[i].dot(nhat); } - vinflow = (allReduce(vinflow, MPI_SUM, Communicator::communicator())/ - std::max(1.0e-30, allReduce(double(nodeIDs.size()), MPI_SUM, Communicator::communicator()))); // Negative implies outflow + vinflow = (allReduce(vinflow, SPHERAL_MPI_SUM)/ + std::max(1.0e-30, allReduce(double(nodeIDs.size()), SPHERAL_MPI_SUM))); // Negative implies outflow // Figure out a timestep limit such that we don't move more than the ghost // node thickness. @@ -320,8 +320,8 @@ InflowOutflowBoundary::initializeProblemStartup(const bool /*final*/) xmin = std::min(xmin, xd); xmax = std::max(xmax, xd); } - xmin = allReduce(xmin, MPI_MIN, Communicator::communicator()); - xmax = allReduce(xmax, MPI_MAX, Communicator::communicator()); + xmin = allReduce(xmin, SPHERAL_MPI_MIN); + xmax = allReduce(xmax, SPHERAL_MPI_MAX); mXmin[nodeList.name()] = xmin; mDT = std::min(mDT, std::abs(xmax - xmin)/std::max(1e-30, std::abs(vinflow))); // Protect from negative outflow velocity // cerr << "Timestep constraint: " << mDT << endl; @@ -458,7 +458,7 @@ InflowOutflowBoundary::finalize(const Scalar /*time*/, nodeList.neighbor().updateNodes(); } } - altered = (allReduce((altered ? 1 : 0), MPI_MAX, Communicator::communicator()) == 1); + altered = (allReduce((altered ? 1 : 0), SPHERAL_MPI_MAX) == 1); // If any NodeLists were altered, recompute the boundary conditions. if (altered) { diff --git a/src/Boundary/PlanarBoundary.cc b/src/Boundary/PlanarBoundary.cc index 74d6a90cc..f54bc4ca4 100644 --- a/src/Boundary/PlanarBoundary.cc +++ b/src/Boundary/PlanarBoundary.cc @@ -12,7 +12,6 @@ #include "NodeList/FluidNodeList.hh" #include "Mesh/Mesh.hh" #include "Utilities/DBC.hh" -#include "Utilities/allReduce.hh" #include "PlanarBoundary.hh" diff --git a/src/Boundary/findNodesTouchingThroughPlanes.cc b/src/Boundary/findNodesTouchingThroughPlanes.cc index f9cce1438..a79a11690 100644 --- a/src/Boundary/findNodesTouchingThroughPlanes.cc +++ b/src/Boundary/findNodesTouchingThroughPlanes.cc @@ -2,7 +2,7 @@ // Find the set of nodes that see through a pair of planes. //------------------------------------------------------------------------------ #include "findNodesTouchingThroughPlanes.hh" -#include "Utilities/allReduce.hh" +#include "Distributed/allReduce.hh" namespace Spheral { @@ -50,7 +50,7 @@ findNodesTouchingThroughPlanes(const NodeList& nodeList, const auto hmaxi = 1.0/Hi.eigenValues().minElement(); if (hmaxi > hmax and std::min(exitPlane.minimumDistance(ri), enterPlane.minimumDistance(ri)) < kernelExtent*hmaxi) hmax = hmaxi; } - hmax = allReduce(hmax, MPI_MAX, Communicator::communicator()); + hmax = allReduce(hmax, SPHERAL_MPI_MAX); // Now find all points within this range of the exit plane. if (hmax > 0.0) { diff --git a/src/Damage/DamageModel.cc b/src/Damage/DamageModel.cc index a49b892b8..995a9dbdb 100644 --- a/src/Damage/DamageModel.cc +++ b/src/Damage/DamageModel.cc @@ -196,7 +196,7 @@ finalize(const Scalar /*time*/, nD += nD_thread; } } - nD = allReduce(nD, MPI_SUM, Communicator::communicator()); + nD = allReduce(nD, SPHERAL_MPI_SUM); const auto ntot = std::max(1, dataBase.globalNumInternalNodes()); const auto dfrac = double(nD)/double(ntot); mComputeIntersectConnectivity = (dfrac > 0.2); // Should tune this number... diff --git a/src/Damage/IvanoviSALEDamageModel.cc b/src/Damage/IvanoviSALEDamageModel.cc index e50c2c98c..a6629ac8e 100644 --- a/src/Damage/IvanoviSALEDamageModel.cc +++ b/src/Damage/IvanoviSALEDamageModel.cc @@ -38,7 +38,7 @@ #include "Boundary/Boundary.hh" #include "Neighbor/Neighbor.hh" #include "Utilities/mortonOrderIndices.hh" -#include "Utilities/allReduce.hh" +#include "Distributed/allReduce.hh" #include "Utilities/uniform_random.hh" #include // hash_combine diff --git a/src/Damage/ProbabilisticDamageModel.cc b/src/Damage/ProbabilisticDamageModel.cc index 184373b05..05e99194e 100644 --- a/src/Damage/ProbabilisticDamageModel.cc +++ b/src/Damage/ProbabilisticDamageModel.cc @@ -29,7 +29,7 @@ #include "Boundary/Boundary.hh" #include "Neighbor/Neighbor.hh" #include "Utilities/mortonOrderIndices.hh" -#include "Utilities/allReduce.hh" +#include "Distributed/allReduce.hh" #include "Utilities/uniform_random.hh" #include // hash_combine @@ -121,7 +121,7 @@ initializeProblemStartupDependencies(DataBase& dataBase, for (auto i = 0u; i < mMask.numInternalElements(); ++i) { if (mMask[i] == 1) ++nused_local; } - const size_t nused_global = allReduce(nused_local, MPI_SUM, Communicator::communicator()); + const size_t nused_global = allReduce(nused_local, SPHERAL_MPI_SUM); // Compute the Morton-ordering for hashing with the global seed to seed each // point-wise random number generator. @@ -158,8 +158,8 @@ initializeProblemStartupDependencies(DataBase& dataBase, randomGenerators[i](); // Recommended to discard first value in sequence } } - mVmin = allReduce(mVmin, MPI_MIN, Communicator::communicator()); - mVmax = allReduce(mVmax, MPI_MAX, Communicator::communicator()); + mVmin = allReduce(mVmin, SPHERAL_MPI_MIN); + mVmax = allReduce(mVmax, SPHERAL_MPI_MAX); // Generate min/max ranges of flaws for each point. const auto mInv = 1.0/mmWeibull; @@ -200,12 +200,12 @@ initializeProblemStartupDependencies(DataBase& dataBase, // Some diagnostic output. if (nused_global > 0) { - minNumFlaws = allReduce(minNumFlaws, MPI_MIN, Communicator::communicator()); - maxNumFlaws = allReduce(maxNumFlaws, MPI_MAX, Communicator::communicator()); - totalNumFlaws = allReduce(totalNumFlaws, MPI_SUM, Communicator::communicator()); - epsMin = allReduce(epsMin, MPI_MIN, Communicator::communicator()); - epsMax = allReduce(epsMax, MPI_MAX, Communicator::communicator()); - numFlawsRatio = allReduce(numFlawsRatio, MPI_SUM, Communicator::communicator())/nused_global; + minNumFlaws = allReduce(minNumFlaws, SPHERAL_MPI_MIN); + maxNumFlaws = allReduce(maxNumFlaws, SPHERAL_MPI_MAX); + totalNumFlaws = allReduce(totalNumFlaws, SPHERAL_MPI_SUM); + epsMin = allReduce(epsMin, SPHERAL_MPI_MIN); + epsMax = allReduce(epsMax, SPHERAL_MPI_MAX); + numFlawsRatio = allReduce(numFlawsRatio, SPHERAL_MPI_SUM)/nused_global; if (Process::getRank() == 0) { cerr << "ProbabilisticDamageModel for " << nodes.name() << ":" << endl << " Min, max, max/min volumes: " << mVmin << " " << mVmax << " " << mVmax*safeInv(mVmin) << endl diff --git a/src/Damage/weibullFlawDistributionBenzAsphaug.cc b/src/Damage/weibullFlawDistributionBenzAsphaug.cc index 78e954dcc..9c3c2ec97 100644 --- a/src/Damage/weibullFlawDistributionBenzAsphaug.cc +++ b/src/Damage/weibullFlawDistributionBenzAsphaug.cc @@ -19,7 +19,7 @@ #include "Strength/SolidFieldNames.hh" #include "DataBase/State.hh" #include "Distributed/Communicator.hh" -#include "Utilities/allReduce.hh" +#include "Distributed/allReduce.hh" using std::unordered_map; using std::vector; @@ -99,7 +99,7 @@ weibullFlawDistributionBenzAsphaug(double volume, CHECK(rho(i) > 0.0); volume += mass(i)/rho(i); } - volume = allReduce(volume, MPI_SUM, Communicator::communicator()); + volume = allReduce(volume, SPHERAL_MPI_SUM); } volume = std::max(volume, 1e-100); CHECK(volume > 0.0); @@ -168,11 +168,11 @@ weibullFlawDistributionBenzAsphaug(double volume, // Prepare some diagnostic output. const auto nused = std::max(1, mask.sumElements()); - minNumFlaws = allReduce(minNumFlaws, MPI_MIN, Communicator::communicator()); - maxNumFlaws = allReduce(maxNumFlaws, MPI_MAX, Communicator::communicator()); - totalNumFlaws = allReduce(totalNumFlaws, MPI_SUM, Communicator::communicator()); - epsMax = allReduce(epsMax, MPI_MAX, Communicator::communicator()); - sumFlaws = allReduce(sumFlaws, MPI_SUM, Communicator::communicator()); + minNumFlaws = allReduce(minNumFlaws, SPHERAL_MPI_MIN); + maxNumFlaws = allReduce(maxNumFlaws, SPHERAL_MPI_MAX); + totalNumFlaws = allReduce(totalNumFlaws, SPHERAL_MPI_SUM); + epsMax = allReduce(epsMax, SPHERAL_MPI_MAX); + sumFlaws = allReduce(sumFlaws, SPHERAL_MPI_SUM); if (procID == 0) { cerr << "weibullFlawDistributionBenzAsphaug: Min num flaws per node: " << minNumFlaws << endl << " Max num flaws per node: " << maxNumFlaws << endl diff --git a/src/Damage/weibullFlawDistributionOwen.cc b/src/Damage/weibullFlawDistributionOwen.cc index ced8f61e0..9f3522d81 100644 --- a/src/Damage/weibullFlawDistributionOwen.cc +++ b/src/Damage/weibullFlawDistributionOwen.cc @@ -12,7 +12,7 @@ #include "Strength/SolidFieldNames.hh" #include "DataBase/State.hh" #include "Distributed/Communicator.hh" -#include "Utilities/allReduce.hh" +#include "Distributed/allReduce.hh" #include // hash_combine @@ -108,8 +108,8 @@ weibullFlawDistributionOwen(const unsigned seed, Vmax = max(Vmax, Vi); } } - Vmin = allReduce(Vmin*volumeMultiplier, MPI_MIN, Communicator::communicator()); - Vmax = allReduce(Vmax*volumeMultiplier, MPI_MAX, Communicator::communicator()); + Vmin = allReduce(Vmin*volumeMultiplier, SPHERAL_MPI_MIN); + Vmax = allReduce(Vmax*volumeMultiplier, SPHERAL_MPI_MAX); CHECK(Vmin > 0.0); CHECK(Vmax >= Vmin); @@ -157,12 +157,12 @@ weibullFlawDistributionOwen(const unsigned seed, // Some diagnostic output. const auto nused = std::max(1, mask.sumElements()); if (nglobal > 0) { - minNumFlaws = allReduce(minNumFlaws, MPI_MIN, Communicator::communicator()); - maxNumFlaws = allReduce(maxNumFlaws, MPI_MAX, Communicator::communicator()); - totalNumFlaws = allReduce(totalNumFlaws, MPI_SUM, Communicator::communicator()); - epsMin = allReduce(epsMin, MPI_MIN, Communicator::communicator()); - epsMax = allReduce(epsMax, MPI_MAX, Communicator::communicator()); - sumFlaws = allReduce(sumFlaws, MPI_SUM, Communicator::communicator()); + minNumFlaws = allReduce(minNumFlaws, SPHERAL_MPI_MIN); + maxNumFlaws = allReduce(maxNumFlaws, SPHERAL_MPI_MAX); + totalNumFlaws = allReduce(totalNumFlaws, SPHERAL_MPI_SUM); + epsMin = allReduce(epsMin, SPHERAL_MPI_MIN); + epsMax = allReduce(epsMax, SPHERAL_MPI_MAX); + sumFlaws = allReduce(sumFlaws, SPHERAL_MPI_SUM); } if (procID == 0) { cerr << "weibullFlawDistributionOwen: Min num flaws per node: " << minNumFlaws << endl diff --git a/src/DataBase/DataBase.cc b/src/DataBase/DataBase.cc index 9ade92327..7022a780a 100644 --- a/src/DataBase/DataBase.cc +++ b/src/DataBase/DataBase.cc @@ -17,7 +17,7 @@ #include "Hydro/HydroFieldNames.hh" #include "Utilities/globalBoundingVolumes.hh" #include "Utilities/globalNodeIDs.hh" -#include "Utilities/allReduce.hh" +#include "Distributed/allReduce.hh" #include "Distributed/Communicator.hh" #include "Utilities/DBC.hh" @@ -92,7 +92,7 @@ int DataBase::globalNumInternalNodes() const { int localResult = numInternalNodes(); int result = localResult; - result = allReduce(result, MPI_SUM, Communicator::communicator()); + result = allReduce(result, SPHERAL_MPI_SUM); return result; } @@ -101,7 +101,7 @@ int DataBase::globalNumGhostNodes() const { int localResult = numGhostNodes(); int result = localResult; - result = allReduce(result, MPI_SUM, Communicator::communicator()); + result = allReduce(result, SPHERAL_MPI_SUM); return result; } @@ -110,7 +110,7 @@ int DataBase::globalNumNodes() const { int localResult = numNodes(); int result = localResult; - result = allReduce(result, MPI_SUM, Communicator::communicator()); + result = allReduce(result, SPHERAL_MPI_SUM); return result; } @@ -122,7 +122,7 @@ int DataBase::globalNumFluidInternalNodes() const { int localResult = numFluidInternalNodes(); int result = localResult; - result = allReduce(result, MPI_SUM, Communicator::communicator()); + result = allReduce(result, SPHERAL_MPI_SUM); return result; } @@ -131,7 +131,7 @@ int DataBase::globalNumFluidGhostNodes() const { int localResult = numFluidGhostNodes(); int result = localResult; - result = allReduce(result, MPI_SUM, Communicator::communicator()); + result = allReduce(result, SPHERAL_MPI_SUM); return result; } @@ -140,7 +140,7 @@ int DataBase::globalNumFluidNodes() const { int localResult = numFluidNodes(); int result = localResult; - result = allReduce(result, MPI_SUM, Communicator::communicator()); + result = allReduce(result, SPHERAL_MPI_SUM); return result; } @@ -527,13 +527,13 @@ reinitializeNeighbors() const { // Find the global result across all processors. auto box = 0.0; for (auto i = 0; i != Dimension::nDim; ++i) { - xmin(i) = allReduce(xmin(i), MPI_MIN, Communicator::communicator()); - xmax(i) = allReduce(xmax(i), MPI_MAX, Communicator::communicator()); + xmin(i) = allReduce(xmin(i), SPHERAL_MPI_MIN); + xmax(i) = allReduce(xmax(i), SPHERAL_MPI_MAX); box = std::max(box, xmax(i) - xmin(i)); } - havg = allReduce(havg, MPI_SUM, Communicator::communicator()); - ntot = allReduce(ntot, MPI_SUM, Communicator::communicator()); - hmax = allReduce(hmax, MPI_MAX, Communicator::communicator()); + havg = allReduce(havg, SPHERAL_MPI_SUM); + ntot = allReduce(ntot, SPHERAL_MPI_SUM); + hmax = allReduce(hmax, SPHERAL_MPI_MAX); if (ntot > 0) { havg /= ntot; @@ -1831,8 +1831,8 @@ boundingBox(typename Dimension::Vector& xmin, // Now find the global bounds across all processors. for (int i = 0; i != Dimension::nDim; ++i) { - xmin(i) = allReduce(xmin(i), MPI_MIN, Communicator::communicator()); - xmax(i) = allReduce(xmax(i), MPI_MAX, Communicator::communicator()); + xmin(i) = allReduce(xmin(i), SPHERAL_MPI_MIN); + xmax(i) = allReduce(xmax(i), SPHERAL_MPI_MAX); } } @@ -1929,15 +1929,15 @@ globalSamplingBoundingVolume(typename Dimension::Vector& centroid, size_t nlocal = this->numInternalNodes(); centroid *= nlocal; for (int i = 0; i != Dimension::nDim; ++i) { - xminNodes(i) = allReduce(xminNodes(i), MPI_MIN, Communicator::communicator()); - xmaxNodes(i) = allReduce(xmaxNodes(i), MPI_MAX, Communicator::communicator()); - xminSample(i) = allReduce(xminSample(i), MPI_MIN, Communicator::communicator()); - xmaxSample(i) = allReduce(xmaxSample(i), MPI_MAX, Communicator::communicator()); - centroid(i) = allReduce(centroid(i), MPI_SUM, Communicator::communicator()); + xminNodes(i) = allReduce(xminNodes(i), SPHERAL_MPI_MIN); + xmaxNodes(i) = allReduce(xmaxNodes(i), SPHERAL_MPI_MAX); + xminSample(i) = allReduce(xminSample(i), SPHERAL_MPI_MIN); + xmaxSample(i) = allReduce(xmaxSample(i), SPHERAL_MPI_MAX); + centroid(i) = allReduce(centroid(i), SPHERAL_MPI_SUM); } // Fix up the centroid and radii. - size_t nglobal = allReduce((uint64_t) nlocal, MPI_SUM, Communicator::communicator()); + size_t nglobal = allReduce((uint64_t) nlocal, SPHERAL_MPI_SUM); if (nglobal > 0) { centroid /= nglobal; radiusNodes = 0.0; @@ -1957,8 +1957,8 @@ globalSamplingBoundingVolume(typename Dimension::Vector& centroid, radiusSample = max(radiusSample, drMag + 2.0*hi); } } - radiusNodes = allReduce(radiusNodes, MPI_MAX, Communicator::communicator()); - radiusSample = allReduce(radiusSample, MPI_MAX, Communicator::communicator()); + radiusNodes = allReduce(radiusNodes, SPHERAL_MPI_MAX); + radiusSample = allReduce(radiusSample, SPHERAL_MPI_MAX); const Vector delta = 0.001*(xmaxSample - xminSample); radiusNodes *= 1.001; radiusSample *= 1.001; diff --git a/src/Distributed/CMakeLists.txt b/src/Distributed/CMakeLists.txt index b7016cdcc..2eea61ce9 100644 --- a/src/Distributed/CMakeLists.txt +++ b/src/Distributed/CMakeLists.txt @@ -1,10 +1,26 @@ include_directories(.) +set(Distributed_sources + Communicator.cc + Process.cc + RegisterMPIDataTypes.cc + ) + +set(Distributed_inst ) + +set(Distributed_headers + allReduce.hh + Communicator.hh + Process.hh + RegisterMPIDataTypes.hh + RegisterMPIDataTypesInline.hh + ) + if (ENABLE_MPI) #---------------------------------------------------------------------------- # MPI parallel on #---------------------------------------------------------------------------- - set(Distributed_sources Communicator.cc waitAllWithDeadlockDetection.cc) + list(APPEND Distributed_sources waitAllWithDeadlockDetection.cc) if (ENABLE_1D) list(APPEND Distributed_sources SortAndDivideRedistributeNodes1d.cc) @@ -18,26 +34,23 @@ if (ENABLE_MPI) list(APPEND Distributed_sources SortAndDivideRedistributeNodes3d.cc) endif() - set(Distributed_inst - DistributedBoundary - NestedGridDistributedBoundary - TreeDistributedBoundary - BoundingVolumeDistributedBoundary - RedistributeNodes - DistributeByXPosition - SortAndDivideRedistributeNodes - SpaceFillingCurveRedistributeNodes - MortonOrderRedistributeNodes - PeanoHilbertOrderRedistributeNodes - VoronoiRedistributeNodes - ) - - instantiate(Distributed_inst Distributed_sources) + list(APPEND Distributed_inst + DistributedBoundary + NestedGridDistributedBoundary + TreeDistributedBoundary + BoundingVolumeDistributedBoundary + RedistributeNodes + DistributeByXPosition + SortAndDivideRedistributeNodes + SpaceFillingCurveRedistributeNodes + MortonOrderRedistributeNodes + PeanoHilbertOrderRedistributeNodes + VoronoiRedistributeNodes + ) - set(Distributed_headers + list(APPEND Distributed_headers BoundingVolumeDistributedBoundary.hh BoundingVolumeDistributedBoundaryInline.hh - Communicator.hh CompareDomainNodesByPosition.hh DistributeByXPosition.hh DistributedBoundary.hh @@ -65,25 +78,22 @@ if (ENABLE_MPI) waitAllWithDeadlockDetection.hh ) - if (NOT ENABLE_CXXONLY) - install(FILES mpi_mpi4py.py - DESTINATION ${SPHERAL_SITE_PACKAGES_PATH}/Spheral - RENAME mpi.py - ) - endif() - - spheral_add_obj_library(Distributed SPHERAL_OBJ_LIBS) + set(MPIPY_FILE_NAME "mpi_mpi4py.py") else() #---------------------------------------------------------------------------- # MPI parallel off #---------------------------------------------------------------------------- - if (NOT ENABLE_CXXONLY) - install(FILES fakempi.py - DESTINATION ${SPHERAL_SITE_PACKAGES_PATH}/Spheral - RENAME mpi.py - ) - endif() + set(MPIPY_FILE_NAME "fakempi.py") + +endif() +if (NOT ENABLE_CXXONLY) + install(FILES ${MPIPY_FILE_NAME} + DESTINATION ${SPHERAL_SITE_PACKAGES_PATH}/Spheral + RENAME mpi.py + ) endif() +instantiate(Distributed_inst Distributed_sources) +spheral_add_obj_library(Distributed SPHERAL_OBJ_LIBS) diff --git a/src/Utilities/Process.cc b/src/Distributed/Process.cc similarity index 100% rename from src/Utilities/Process.cc rename to src/Distributed/Process.cc diff --git a/src/Utilities/Process.hh b/src/Distributed/Process.hh similarity index 100% rename from src/Utilities/Process.hh rename to src/Distributed/Process.hh diff --git a/src/Utilities/RegisterMPIDataTypes.cc b/src/Distributed/RegisterMPIDataTypes.cc similarity index 100% rename from src/Utilities/RegisterMPIDataTypes.cc rename to src/Distributed/RegisterMPIDataTypes.cc diff --git a/src/Utilities/RegisterMPIDataTypes.hh b/src/Distributed/RegisterMPIDataTypes.hh similarity index 100% rename from src/Utilities/RegisterMPIDataTypes.hh rename to src/Distributed/RegisterMPIDataTypes.hh diff --git a/src/Utilities/RegisterMPIDataTypesInline.hh b/src/Distributed/RegisterMPIDataTypesInline.hh similarity index 100% rename from src/Utilities/RegisterMPIDataTypesInline.hh rename to src/Distributed/RegisterMPIDataTypesInline.hh diff --git a/src/Distributed/SpaceFillingCurveRedistributeNodes.cc b/src/Distributed/SpaceFillingCurveRedistributeNodes.cc index d2d30bd7a..c418d3bc2 100644 --- a/src/Distributed/SpaceFillingCurveRedistributeNodes.cc +++ b/src/Distributed/SpaceFillingCurveRedistributeNodes.cc @@ -20,7 +20,7 @@ #include "Utilities/globalNodeIDs.hh" #include "Utilities/bisectSearch.hh" #include "Utilities/RedistributionRegistrar.hh" -#include "Utilities/allReduce.hh" +#include "allReduce.hh" #include "Communicator.hh" #include "Utilities/DBC.hh" @@ -208,7 +208,7 @@ redistributeNodes(DataBase& dataBase, CHECK(count.size() == uniqueIndices.size()); CHECK(work.size() == uniqueIndices.size()); } - maxCount = allReduce(maxCount, MPI_MAX, Communicator::communicator()); + maxCount = allReduce(maxCount, SPHERAL_MPI_MAX); if (procID == 0) cerr << "SpaceFillingCurveRedistributeNodes: max redundancy is " << maxCount << endl; // // DEBUG @@ -446,7 +446,7 @@ numIndicesInRange(const vector: } // Globally reduce that sucker. - result = allReduce(result, MPI_SUM, Communicator::communicator()); + result = allReduce(result, SPHERAL_MPI_SUM); return result; } @@ -518,8 +518,8 @@ workAndNodesInRange(const vector >& nodes, const int numProcs, - MPI_Comm communicator) { + MPI_Comm communicator = Communicator::communicator()) { typedef typename Dimension::Vector Vector; // First find the local node closest to the center. @@ -387,7 +387,7 @@ redistributeNodes(DataBase& dataBase, // Now we can get the node distribution description. vector > nodeDistribution = this->currentDomainDecomposition(dataBase, globalIDs, workField); const size_t numNodes = nodeDistribution.size(); - const size_t numNodesGlobal = allReduce((uint64_t) numNodes, MPI_SUM, Communicator::communicator()); + const size_t numNodesGlobal = allReduce((uint64_t) numNodes, SPHERAL_MPI_SUM); const size_t avgNumNodes = numNodesGlobal/numProcs; CHECK(numNodes > 0); @@ -510,7 +510,7 @@ redistributeNodes(DataBase& dataBase, const Vector xmaxDaughter = daughterPositions[kdaughter] + dcell; if (numGensForDaughter[kdaughter] == 1) { generators[*genItr] = computeClosestNodePosition(0.5*(xminDaughter + xmaxDaughter), - nodeDistribution, numProcs, Communicator::communicator()); + nodeDistribution, numProcs); generatorBounds[*genItr] = make_pair(xminDaughter, xmaxDaughter); CHECK(testPointInBox(generators[*genItr], xminDaughter, xmaxDaughter)); } else if (numGensForDaughter[kdaughter] > 1) { @@ -749,7 +749,7 @@ computeCentroids(const vector >& nodes, // at least some work. for (size_t igen = 0; igen != numGenerators; ++igen) { generators[igen] = 0.25*generators[igen]*safeInv(normalization[igen]) + 0.75*generators0[igen]; - generators[igen] = computeClosestNodePosition(generators[igen], nodes, numProcs, Communicator::communicator()); + generators[igen] = computeClosestNodePosition(generators[igen], nodes, numProcs); } } @@ -867,8 +867,8 @@ cullGeneratorNodesByWork(const vector& generators, sort(distances.begin(), distances.end(), ComparePairsBySecondElement()); // Find the global range of distances from the generator. - double rmin = allReduce((distances.size() > 0 ? distances.front().second : DBL_MAX), MPI_MIN, Communicator::communicator()); - double rmax = allReduce((distances.size() > 0 ? distances.back().second : 0.0), MPI_MAX, Communicator::communicator()); + double rmin = allReduce((distances.size() > 0 ? distances.front().second : DBL_MAX), SPHERAL_MPI_MIN); + double rmax = allReduce((distances.size() > 0 ? distances.back().second : 0.0), SPHERAL_MPI_MAX); // Bisect for the appropriate radius to reject nodes. const double worktol = max(1.0e-10, 0.01*targetWork); @@ -883,7 +883,7 @@ cullGeneratorNodesByWork(const vector& generators, localWork += nodes[itr->first].work; ++itr; } - currentWork = allReduce(localWork, MPI_SUM, Communicator::communicator()); + currentWork = allReduce(localWork, SPHERAL_MPI_SUM); if (currentWork < targetWork) { rmin = rreject; } else { diff --git a/src/Utilities/allReduce.hh b/src/Distributed/allReduce.hh similarity index 56% rename from src/Utilities/allReduce.hh rename to src/Distributed/allReduce.hh index f172eea57..e61d952fe 100644 --- a/src/Utilities/allReduce.hh +++ b/src/Distributed/allReduce.hh @@ -16,18 +16,35 @@ //------------------------------------------------------------------------------ #include +#include "Communicator.hh" namespace Spheral { +#define SPHERAL_MPI_MIN MPI_MIN +#define SPHERAL_MPI_MAX MPI_MAX +#define SPHERAL_MPI_SUM MPI_SUM +#define SPHERAL_MPI_PROD MPI_PROD +#define SPHERAL_MPI_LAND MPI_LAND +#define SPHERAL_MPI_LOR MPI_LOR + template Value -allReduce(const Value& value, const MPI_Op op, const MPI_Comm comm) { +allReduce(const Value& value, const MPI_Op op, const MPI_Comm comm = Communicator::communicator()) { Value tmp = value; Value result; MPI_Allreduce(&tmp, &result, 1, DataTypeTraits::MpiDataType(), op, comm); return result; } +template +Value +scan(const Value& value, const MPI_Op op, const MPI_Comm comm = Communicator::communicator()) { + Value tmp = value; + Value result; + MPI_Scan(&tmp, &result, 1, DataTypeTraits::MpiDataType(), op, comm); + return result; +} + } #else @@ -37,17 +54,22 @@ allReduce(const Value& value, const MPI_Op op, const MPI_Comm comm) { namespace Spheral { -#define MPI_MIN 1 -#define MPI_MAX 2 -#define MPI_SUM 3 -#define MPI_PROD 4 -#define MPI_LAND 5 -#define MPI_LOR 6 -#define MPI_COMM_WORLD 0 +#define SPHERAL_MPI_MIN 1 +#define SPHERAL_MPI_MAX 2 +#define SPHERAL_MPI_SUM 3 +#define SPHERAL_MPI_PROD 4 +#define SPHERAL_MPI_LAND 5 +#define SPHERAL_MPI_LOR 6 + +template +Value +allReduce(const Value& value, const int /*op*/, const int comm = 0) { + return value; +} template Value -allReduce(const Value& value, const int /*op*/, const int /*comm*/) { +scan(const Value& value, const int /*op*/, const int comm = 0) { return value; } diff --git a/src/Field/FieldInline.hh b/src/Field/FieldInline.hh index 756ae4a9f..12a49a43d 100644 --- a/src/Field/FieldInline.hh +++ b/src/Field/FieldInline.hh @@ -5,7 +5,7 @@ #include "Utilities/packElement.hh" #include "Utilities/removeElements.hh" #include "Utilities/safeInv.hh" -#include "Utilities/allReduce.hh" +#include "Distributed/allReduce.hh" #include "Distributed/Communicator.hh" #include @@ -710,7 +710,7 @@ inline DataType Field:: sumElements() const { - return allReduce(this->localSumElements(), MPI_SUM, Communicator::communicator()); + return allReduce(this->localSumElements(), SPHERAL_MPI_SUM); } //------------------------------------------------------------------------------ @@ -721,7 +721,7 @@ inline DataType Field:: min() const { - return allReduce(this->localMin(), MPI_MIN, Communicator::communicator()); + return allReduce(this->localMin(), SPHERAL_MPI_MIN); } //------------------------------------------------------------------------------ @@ -732,7 +732,7 @@ inline DataType Field:: max() const { - return allReduce(this->localMax(), MPI_MAX, Communicator::communicator()); + return allReduce(this->localMax(), SPHERAL_MPI_MAX); } //------------------------------------------------------------------------------ diff --git a/src/Field/FieldListInline.hh b/src/Field/FieldListInline.hh index 835fe7e86..4909078a0 100644 --- a/src/Field/FieldListInline.hh +++ b/src/Field/FieldListInline.hh @@ -7,7 +7,7 @@ #include "Neighbor/Neighbor.hh" #include "Field/Field.hh" #include "Kernel/TableKernel.hh" -#include "Utilities/allReduce.hh" +#include "Distributed/allReduce.hh" #ifdef USE_MPI #include @@ -1349,7 +1349,7 @@ inline DataType FieldList:: sumElements() const { - return allReduce(this->localSumElements(), MPI_SUM, Communicator::communicator()); + return allReduce(this->localSumElements(), SPHERAL_MPI_SUM); } //------------------------------------------------------------------------------ @@ -1360,7 +1360,7 @@ inline DataType FieldList:: min() const { - return allReduce(this->localMin(), MPI_MIN, Communicator::communicator()); + return allReduce(this->localMin(), SPHERAL_MPI_MIN); } //------------------------------------------------------------------------------ @@ -1371,7 +1371,7 @@ inline DataType FieldList:: max() const { - return allReduce(this->localMax(), MPI_MAX, Communicator::communicator()); + return allReduce(this->localMax(), SPHERAL_MPI_MAX); } //------------------------------------------------------------------------------ diff --git a/src/FieldOperations/binFieldList2Lattice.cc b/src/FieldOperations/binFieldList2Lattice.cc index c66569d85..3042f637a 100644 --- a/src/FieldOperations/binFieldList2Lattice.cc +++ b/src/FieldOperations/binFieldList2Lattice.cc @@ -16,7 +16,7 @@ #include "NodeList/NodeList.hh" #include "Geometry/MathTraits.hh" #include "Utilities/testBoxIntersection.hh" -#include "Utilities/allReduce.hh" +#include "Distributed/allReduce.hh" #include "Distributed/BoundingVolumeDistributedBoundary.hh" #include "Distributed/Communicator.hh" @@ -227,8 +227,8 @@ binFieldList2Lattice(const FieldList& fieldList, result = vector(ntotal, DataTypeTraits::zero()); // Check that everyone agrees about the size. - CHECK(bufSize == allReduce(bufSize, MPI_MIN, Communicator::communicator())); - CHECK(bufSize == allReduce(bufSize, MPI_MAX, Communicator::communicator())); + CHECK(bufSize == allReduce(bufSize, SPHERAL_MPI_MIN)); + CHECK(bufSize == allReduce(bufSize, SPHERAL_MPI_MAX)); // Sum up everyone's contribution. for (auto sendProc = 0u; sendProc != numProcs; ++sendProc) { @@ -316,7 +316,7 @@ binFieldList2Lattice(const FieldList& fieldList, // Check that everyone agrees about the size. { int bufSizeMin; - MPI_Allreduce(&bufSize, &bufSizeMin, 1, MPI_INT, MPI_MIN, Communicator::communicator()); + MPI_Allreduce(&bufSize, &bufSizeMin, 1, MPI_INT, SPHERAL_MPI_MIN); CHECK(bufSizeMin == bufSize); } END_CONTRACT_SCOPE diff --git a/src/FractalStruct/check_for_edge_trouble.cc b/src/FractalStruct/check_for_edge_trouble.cc index 68b3b2a02..05b43db3c 100755 --- a/src/FractalStruct/check_for_edge_trouble.cc +++ b/src/FractalStruct/check_for_edge_trouble.cc @@ -1,46 +1,46 @@ -#include "libs.hh" -#include "classes.hh" -#include "headers.hh" -namespace FractalSpace -{ - void check_for_edge_trouble(Fractal& fractal) - { - ofstream& FileFractal=fractal.p_file->DUMPS; - // ofstream& FileFractal=fractal.p_file->FileFractal; - fractal.timing(-1,2); - //-------------------------------------------------------------------------------------------------------------------------------- - // Round off errors can cause trouble at the edge, move points a little - //-------------------------------------------------------------------------------------------------------------------------------- - FileFractal << "edge trouble " << "\n"; - double eps=DBL_EPSILON; - vector pos(3); - int outsiders=0; - for(int part=0; part < fractal.get_number_particles();part++) - { - Particle* p=fractal.particle_list[part]; - if(p->get_p_highest_level_group() == 0) - continue; - p->get_pos(pos); - bool outside=pos[0] >= 1.0 || pos[0] <=0.0 || - pos[1] >= 1.0 || pos[1] <=0.0 || - pos[2] >= 1.0 || pos[2] <=0.0; - if(!outside) continue; - outsiders++; - if(pos[0] >= 1.0) - pos[0]-=eps; - else if(pos[0] <= 0.0) - pos[0]+=eps; - if(pos[1] >= 1.0) - pos[1]-=eps; - else if(pos[1] <= 0.0) - pos[1]+=eps; - if(pos[2] >= 1.0) - pos[2]-=eps; - else if(pos[2] <= 0.0) - pos[2]+=eps; - p->set_pos(pos); - } - FileFractal << " Total Outsiders " << outsiders << "\n"; - fractal.timing(1,2); - } -} +#include "libs.hh" +#include "classes.hh" +#include "headers.hh" +namespace FractalSpace +{ + void check_for_edge_trouble(Fractal& fractal) + { + ofstream& FileFractal=fractal.p_file->DUMPS; + // ofstream& FileFractal=fractal.p_file->FileFractal; + fractal.timing(-1,2); + //-------------------------------------------------------------------------------------------------------------------------------- + // Round off errors can cause trouble at the edge, move points a little + //-------------------------------------------------------------------------------------------------------------------------------- + FileFractal << "edge trouble " << "\n"; + double eps=DBL_EPSILON; + vector pos(3); + int outsiders=0; + for(int part=0; part < fractal.get_number_particles();part++) + { + Particle* p=fractal.particle_list[part]; + if(p->get_p_highest_level_group() == 0) + continue; + p->get_pos(pos); + bool outside=pos[0] >= 1.0 || pos[0] <=0.0 || + pos[1] >= 1.0 || pos[1] <=0.0 || + pos[2] >= 1.0 || pos[2] <=0.0; + if(!outside) continue; + outsiders++; + if(pos[0] >= 1.0) + pos[0]-=eps; + else if(pos[0] <= 0.0) + pos[0]+=eps; + if(pos[1] >= 1.0) + pos[1]-=eps; + else if(pos[1] <= 0.0) + pos[1]+=eps; + if(pos[2] >= 1.0) + pos[2]-=eps; + else if(pos[2] <= 0.0) + pos[2]+=eps; + p->set_pos(pos); + } + FileFractal << " Total Outsiders " << outsiders << "\n"; + fractal.timing(1,2); + } +} diff --git a/src/Gravity/NBodyGravity.cc b/src/Gravity/NBodyGravity.cc index d0312874d..ce5fa3668 100644 --- a/src/Gravity/NBodyGravity.cc +++ b/src/Gravity/NBodyGravity.cc @@ -219,11 +219,11 @@ evaluateDerivatives(const typename Dimension::Scalar /*time*/, } } -#ifdef USE_MPI - mExtraEnergy = allReduce(mExtraEnergy, MPI_SUM, Communicator::communicator()); - mOldMaxAcceleration = allReduce(mOldMaxAcceleration, MPI_MAX, Communicator::communicator()); - mOldMaxVelocity = allReduce(mOldMaxVelocity, MPI_MAX, Communicator::communicator()); + mExtraEnergy = allReduce(mExtraEnergy, SPHERAL_MPI_SUM); + mOldMaxAcceleration = allReduce(mOldMaxAcceleration, SPHERAL_MPI_MAX); + mOldMaxVelocity = allReduce(mOldMaxVelocity, SPHERAL_MPI_MAX); +#ifdef USE_MPI // Wait until all our sends are complete. vector sendStatus(sendRequests.size()); MPI_Waitall(sendRequests.size(), &(*sendRequests.begin()), &(*sendStatus.begin())); diff --git a/src/Gravity/PolyGravity.cc b/src/Gravity/PolyGravity.cc index 17c3e1570..36127bd66 100644 --- a/src/Gravity/PolyGravity.cc +++ b/src/Gravity/PolyGravity.cc @@ -15,7 +15,7 @@ #include "DataBase/StateDerivatives.hh" #include "Utilities/globalBoundingVolumes.hh" #include "Utilities/packElement.hh" -#include "Utilities/allReduce.hh" +#include "Distributed/allReduce.hh" #include "Utilities/FastMath.hh" #include "Utilities/PairComparisons.hh" #include "Hydro/HydroFieldNames.hh" @@ -215,7 +215,7 @@ evaluateDerivatives(const typename Dimension::Scalar /*time*/, mDtMinAcc = min(mDtMinAcc, sqrt(hi/ai.magnitude())); // Similar to acceleration constraint from TreeGravity } } - mExtraEnergy = allReduce(mExtraEnergy, MPI_SUM, Communicator::communicator()); + mExtraEnergy = allReduce(mExtraEnergy, SPHERAL_MPI_SUM); } //------------------------------------------------------------------------------ diff --git a/src/Gravity/Tree.cc b/src/Gravity/Tree.cc index f0f92eae1..a95f88da0 100644 --- a/src/Gravity/Tree.cc +++ b/src/Gravity/Tree.cc @@ -11,7 +11,7 @@ #include "Tree.hh" #include "Utilities/globalBoundingVolumes.hh" #include "Utilities/packElement.hh" -#include "Utilities/allReduce.hh" +#include "Distributed/allReduce.hh" #include "Utilities/FastMath.hh" #include "Utilities/PairComparisons.hh" #include "Distributed/Communicator.hh" @@ -74,7 +74,7 @@ dumpTree(const bool globalTree) const { const unsigned rank = Process::getRank(); #endif unsigned nlevels = mLevels.size(); - if (globalTree) nlevels = allReduce(nlevels, MPI_MAX, Communicator::communicator()); + if (globalTree) nlevels = allReduce(nlevels, SPHERAL_MPI_MAX); ss << "Tree : nlevels = " << nlevels << "\n"; for (unsigned ilevel = 0; ilevel != nlevels; ++ilevel) { @@ -151,7 +151,7 @@ dumpTreeStatistics(const bool globalTree) const { const unsigned rank = Process::getRank(); #endif unsigned nlevels = mLevels.size(); - if (globalTree) nlevels = allReduce(nlevels, MPI_MAX, Communicator::communicator()); + if (globalTree) nlevels = allReduce(nlevels, SPHERAL_MPI_MAX); ss << "Tree : nlevels = " << nlevels << "\n"; for (unsigned ilevel = 0; ilevel != nlevels; ++ilevel) { diff --git a/src/Gravity/TreeGravity.cc b/src/Gravity/TreeGravity.cc index e81c49609..28c4a3cf7 100644 --- a/src/Gravity/TreeGravity.cc +++ b/src/Gravity/TreeGravity.cc @@ -12,7 +12,7 @@ #include "DataBase/StateDerivatives.hh" #include "Utilities/globalBoundingVolumes.hh" #include "Utilities/packElement.hh" -#include "Utilities/allReduce.hh" +#include "Distributed/allReduce.hh" #include "Utilities/FastMath.hh" #include "Utilities/PairComparisons.hh" #include "Hydro/HydroFieldNames.hh" @@ -321,7 +321,7 @@ evaluateDerivatives(const typename Dimension::Scalar /*time*/, } #ifdef USE_MPI - mExtraEnergy = allReduce(mExtraEnergy, MPI_SUM, Communicator::communicator()); + mExtraEnergy = allReduce(mExtraEnergy, SPHERAL_MPI_SUM); // Wait until all our sends are complete. if (not sendRequests.empty()) { @@ -556,7 +556,7 @@ dumpTree(const bool globalTree) const { const unsigned rank = Process::getRank(); #endif unsigned nlevels = mTree.size(); - if (globalTree) nlevels = allReduce(nlevels, MPI_MAX, Communicator::communicator()); + if (globalTree) nlevels = allReduce(nlevels, SPHERAL_MPI_MAX); ss << "Tree : nlevels = " << nlevels << "\n"; for (unsigned ilevel = 0; ilevel != nlevels; ++ilevel) { @@ -633,7 +633,7 @@ dumpTreeStatistics(const bool globalTree) const { const unsigned rank = Process::getRank(); #endif unsigned nlevels = mTree.size(); - if (globalTree) nlevels = allReduce(nlevels, MPI_MAX, Communicator::communicator()); + if (globalTree) nlevels = allReduce(nlevels, SPHERAL_MPI_MAX); ss << "Tree : nlevels = " << nlevels << "\n"; for (unsigned ilevel = 0; ilevel != nlevels; ++ilevel) { diff --git a/src/Hydro/VoronoiHourglassControl.cc b/src/Hydro/VoronoiHourglassControl.cc index 4eb435314..0c5adee55 100644 --- a/src/Hydro/VoronoiHourglassControl.cc +++ b/src/Hydro/VoronoiHourglassControl.cc @@ -14,7 +14,7 @@ #include "CRKSPH/computeCRKSPHCorrections.hh" #include "FieldOperations/monotonicallyLimitedGradient.hh" #include "Distributed/Communicator.hh" -#include "Utilities/allReduce.hh" +#include "Distributed/allReduce.hh" #include "Geometry/Dimension.hh" #include "Utilities/DBC.hh" @@ -247,8 +247,8 @@ finalize(const typename Dimension::Scalar time, } } CHECK(rhoZones.size() == mesh.numZones()); - rhoMin = allReduce(rhoMin, MPI_MIN, Communicator::communicator()); - rhoMax = allReduce(rhoMax, MPI_MAX, Communicator::communicator()); + rhoMin = allReduce(rhoMin, SPHERAL_MPI_MIN); + rhoMax = allReduce(rhoMax, SPHERAL_MPI_MAX); // Compute the CRKSPH limited gradient of the density if we're doing first order. if (mOrder > 0) { diff --git a/src/Integrator/Integrator.cc b/src/Integrator/Integrator.cc index f0d914c45..042e33fc4 100644 --- a/src/Integrator/Integrator.cc +++ b/src/Integrator/Integrator.cc @@ -16,7 +16,7 @@ #include "Hydro/HydroFieldNames.hh" #include "Utilities/range.hh" #include "Neighbor/ConnectivityMap.hh" -#include "Utilities/allReduce.hh" +#include "Distributed/allReduce.hh" #include "Distributed/Communicator.hh" #include "Utilities/DBC.hh" #include "Integrator.hh" @@ -228,7 +228,7 @@ selectDt(const typename Dimension::Scalar dtMin, dt.first >= dtMin and dt.first <= dtMax); // In the parallel case we need to find the minimum timestep across all processors. - const auto globalDt = allReduce(dt.first, MPI_MIN, Communicator::communicator()); + const auto globalDt = allReduce(dt.first, SPHERAL_MPI_MIN); // Are we verbose? if (dt.first == globalDt and diff --git a/src/KernelIntegrator/FlatConnectivity.cc b/src/KernelIntegrator/FlatConnectivity.cc index 4645a1476..fba50f751 100644 --- a/src/KernelIntegrator/FlatConnectivity.cc +++ b/src/KernelIntegrator/FlatConnectivity.cc @@ -6,17 +6,14 @@ #include "FlatConnectivity.hh" #include -#ifdef USE_MPI -#include "mpi.h" -#endif #include "Boundary/ConstantBoundary.hh" #include "Boundary/InflowOutflowBoundary.hh" #include "DataBase/DataBase.hh" #include "DataBase/State.hh" -#include "Distributed/Communicator.hh" #include "Geometry/CellFaceFlag.hh" #include "Hydro/HydroFieldNames.hh" +#include "Distributed/allReduce.hh" #include "Utilities/DBC.hh" #include "Utilities/globalNodeIDs.hh" @@ -329,17 +326,12 @@ computeGlobalIndices(const DataBase& dataBase, VERIFY(numInternalNodesDB == mNumInternalLocalNodes); // Get global indices manually -#ifdef USE_MPI - int globalScan = 0; - MPI_Scan(&mNumInternalLocalNodes, &globalScan, 1, MPI_INT, MPI_SUM, Communicator::communicator()); + int globalScan = scan(mNumInternalLocalNodes, SPHERAL_MPI_SUM); VERIFY(globalScan >= mNumInternalLocalNodes); mFirstGlobalIndex = globalScan - mNumInternalLocalNodes; mLastGlobalIndex = globalScan - 1; - MPI_Allreduce(&mNumInternalLocalNodes, &mNumGlobalNodes, 1, MPI_INT, MPI_SUM, Communicator::communicator()); + mNumGlobalNodes = allReduce(mNumInternalLocalNodes, SPHERAL_MPI_SUM); VERIFY(mNumGlobalNodes >= mNumInternalLocalNodes); -#else - mNumGlobalNodes = mNumInternalLocalNodes; -#endif VERIFY(mNumGlobalNodes == numGlobalNodesDB); // std::cout << Process::getRank() << "\t" << mNumInternalLocalNodes << "\t" << mNumGlobalNodes << "\t" << mFirstGlobalIndex << "\t" << mLastGlobalIndex << std::endl; diff --git a/src/KernelIntegrator/KernelIntegrator.cc b/src/KernelIntegrator/KernelIntegrator.cc index a3a838b2f..3eb6205f2 100644 --- a/src/KernelIntegrator/KernelIntegrator.cc +++ b/src/KernelIntegrator/KernelIntegrator.cc @@ -6,10 +6,6 @@ #include "KernelIntegrator.hh" #include "Hydro/HydroFieldNames.hh" -#include "Utilities/Process.hh" - -// // For debugging, can remove later -// #include "Utilities/Process.hh" namespace Spheral { diff --git a/src/Mesh/LineMesh.cc b/src/Mesh/LineMesh.cc index 8e7442ebd..37de442e8 100644 --- a/src/Mesh/LineMesh.cc +++ b/src/Mesh/LineMesh.cc @@ -6,7 +6,7 @@ #include "Mesh.hh" #include "Geometry/Dimension.hh" #include "Utilities/bisectSearch.hh" -#include "Utilities/allReduce.hh" +#include "Distributed/allReduce.hh" #include "Distributed/Communicator.hh" #include "Utilities/DBC.hh" @@ -56,7 +56,7 @@ reconstructInternal(const vector >::Vector>& localGenerators, const Mesh >::Vector& xmax) { // Is there anything to do? - if (allReduce(unsigned(localGenerators.size()), MPI_SUM, Communicator::communicator()) == 0) return; + if (allReduce(unsigned(localGenerators.size()), SPHERAL_MPI_SUM) == 0) return; // Pre-conditions. @@ -284,8 +284,8 @@ boundingSurface() const { xmin = std::min(xmin, mNodePositions[i].x()); xmax = std::max(xmax, mNodePositions[i].x()); } - xmin = allReduce(xmin, MPI_MIN, Communicator::communicator()); - xmax = allReduce(xmax, MPI_MAX, Communicator::communicator()); + xmin = allReduce(xmin, SPHERAL_MPI_MIN); + xmax = allReduce(xmax, SPHERAL_MPI_MAX); return FacetedVolume(Vector(0.5*(xmin + xmax)), 0.5*(xmax - xmin)); } diff --git a/src/Mesh/Mesh.cc b/src/Mesh/Mesh.cc index 532f87713..1b1bcd949 100644 --- a/src/Mesh/Mesh.cc +++ b/src/Mesh/Mesh.cc @@ -15,7 +15,7 @@ using std::abs; #include "MeshConstructionUtilities.hh" #include "Utilities/removeElements.hh" #include "Utilities/DBC.hh" -#include "Utilities/allReduce.hh" +#include "Distributed/allReduce.hh" #include "Utilities/boundingBox.hh" #include "Distributed/Communicator.hh" #include "NodeList/NodeList.hh" @@ -54,8 +54,7 @@ reduceToMaxString(const string& x, const unsigned rank, const unsigned numDomains) { unsigned badRank = allReduce((x.size() == 0 ? numDomains : rank), - MPI_MIN, - Communicator::communicator()); + SPHERAL_MPI_MIN); if (badRank == numDomains) { return ""; } else { @@ -474,7 +473,7 @@ removeZonesByMask(const vector& zoneMask) { removeElements(mSharedFaces, killDomains); // // Any pre-existing parallel info is now invalid. - // if (allReduce(mNeighborDomains.size(), MPI_MAX, Communicator::communicator()) > 0) { + // if (allReduce(mNeighborDomains.size(), SPHERAL_MPI_MAX) > 0) { // mNeighborDomains = vector(); // mSharedNodes = vector >(); // mSharedFaces = vector >(); @@ -749,8 +748,8 @@ generateDomainInfo() { // bit perfect consistency across processors. Vector boxInv; for (unsigned i = 0; i != Dimension::nDim; ++i) { - xmin(i) = allReduce(xmin(i) - dxhash, MPI_MIN, Communicator::communicator()); - xmax(i) = allReduce(xmax(i) + dxhash, MPI_MAX, Communicator::communicator()); + xmin(i) = allReduce(xmin(i) - dxhash, SPHERAL_MPI_MIN); + xmax(i) = allReduce(xmax(i) + dxhash, SPHERAL_MPI_MAX); boxInv(i) = safeInv(xmax(i) - xmin(i)); } @@ -1039,8 +1038,8 @@ generateParallelRind(vector& generators, // bit perfect consistency across processors. Vector boxInv; for (unsigned i = 0; i != Dimension::nDim; ++i) { - xmin(i) = allReduce(xmin(i) - dxhash, MPI_MIN, Communicator::communicator()); - xmax(i) = allReduce(xmax(i) + dxhash, MPI_MAX, Communicator::communicator()); + xmin(i) = allReduce(xmin(i) - dxhash, SPHERAL_MPI_MIN); + xmax(i) = allReduce(xmax(i) + dxhash, SPHERAL_MPI_MAX); boxInv(i) = safeInv(xmax(i) - xmin(i)); } @@ -1449,8 +1448,8 @@ boundingBox(typename Dimension::Vector& xmin, Spheral::boundingBox(mNodePositions, xmin, xmax); #ifdef USE_MPI for (unsigned i = 0; i != Dimension::nDim; ++i) { - xmin(i) = allReduce(xmin(i), MPI_MIN, Communicator::communicator()); - xmax(i) = allReduce(xmax(i), MPI_MAX, Communicator::communicator()); + xmin(i) = allReduce(xmin(i), SPHERAL_MPI_MIN); + xmax(i) = allReduce(xmax(i), SPHERAL_MPI_MAX); } #endif } diff --git a/src/Mesh/MeshInline.hh b/src/Mesh/MeshInline.hh index fa8428053..476ac5cda 100644 --- a/src/Mesh/MeshInline.hh +++ b/src/Mesh/MeshInline.hh @@ -9,7 +9,7 @@ #include "Utilities/removeElements.hh" #include "Utilities/safeInv.hh" -#include "Utilities/allReduce.hh" +#include "Distributed/allReduce.hh" #include "Distributed/Communicator.hh" #include "Utilities/DBC.hh" #include "MeshConstructionUtilities.hh" @@ -369,7 +369,7 @@ minimumScale() const { result = std::min(result, (face.position() - zonePosition).magnitude2()); } } - result = allReduce(0.5*sqrt(result), MPI_MIN, Communicator::communicator()); + result = allReduce(0.5*sqrt(result), SPHERAL_MPI_MIN); // That's it. ENSURE(result > 0.0); @@ -389,7 +389,7 @@ minimumScale() const { result = std::min(result, std::abs(mNodePositions[mZones[i].mNodeIDs[0]].x() - mNodePositions[mZones[i].mNodeIDs[1]].x())); } - result = allReduce(0.5*result, MPI_MIN, Communicator::communicator()); + result = allReduce(0.5*result, SPHERAL_MPI_MIN); // That's it. ENSURE(result > 0.0); diff --git a/src/Mesh/PolyhedralMesh.cc b/src/Mesh/PolyhedralMesh.cc index 7d496974d..5b5fed794 100644 --- a/src/Mesh/PolyhedralMesh.cc +++ b/src/Mesh/PolyhedralMesh.cc @@ -297,10 +297,10 @@ boundingSurface() const { for (domainID = 0; domainID != numDomains; ++domainID) { buffer = localBuffer; bufSize = localBuffer.size(); - MPI_Bcast(&bufSize, 1, MPI_UNSIGNED, domainID, Communicator::communicator()); + MPI_Bcast(&bufSize, 1, MPI_UNSIGNED, domainID); if (bufSize > 0) { buffer.resize(bufSize); - MPI_Bcast(&buffer.front(), bufSize, MPI_CHAR, domainID, Communicator::communicator()); + MPI_Bcast(&buffer.front(), bufSize, MPI_CHAR, domainID); bufItr = buffer.begin(); unpackElement(globalVertexPositions, bufItr, buffer.end()); unpackElement(nfacets, bufItr, buffer.end()); diff --git a/src/Mesh/siloMeshDump.cc b/src/Mesh/siloMeshDump.cc index f0d0d80f3..57ac0896e 100644 --- a/src/Mesh/siloMeshDump.cc +++ b/src/Mesh/siloMeshDump.cc @@ -9,9 +9,6 @@ #include #include -#ifdef USE_MPI -#include -#endif #include "silo.h" #include "Mesh.hh" diff --git a/src/Neighbor/ConnectivityMap.cc b/src/Neighbor/ConnectivityMap.cc index bebf3b41f..df0f9190b 100644 --- a/src/Neighbor/ConnectivityMap.cc +++ b/src/Neighbor/ConnectivityMap.cc @@ -1117,10 +1117,10 @@ computeConnectivity() { } // { - // tpre = allReduce(unsigned(tpre), MPI_SUM, Communicator::communicator()) / Process::getTotalNumberOfProcesses() / CLOCKS_PER_SEC; - // tmaster = allReduce(unsigned(tmaster), MPI_SUM, Communicator::communicator()) / Process::getTotalNumberOfProcesses() / CLOCKS_PER_SEC; - // trefine = allReduce(unsigned(trefine), MPI_SUM, Communicator::communicator()) / Process::getTotalNumberOfProcesses() / CLOCKS_PER_SEC; - // twalk = allReduce(unsigned(twalk), MPI_SUM, Communicator::communicator()) / Process::getTotalNumberOfProcesses() / CLOCKS_PER_SEC; + // tpre = allReduce(unsigned(tpre), SPHERAL_MPI_SUM) / Process::getTotalNumberOfProcesses() / CLOCKS_PER_SEC; + // tmaster = allReduce(unsigned(tmaster), SPHERAL_MPI_SUM) / Process::getTotalNumberOfProcesses() / CLOCKS_PER_SEC; + // trefine = allReduce(unsigned(trefine), SPHERAL_MPI_SUM) / Process::getTotalNumberOfProcesses() / CLOCKS_PER_SEC; + // twalk = allReduce(unsigned(twalk), SPHERAL_MPI_SUM) / Process::getTotalNumberOfProcesses() / CLOCKS_PER_SEC; // if (Process::getRank() == 0) { // std::cerr << "ConnectivityMap timings (pre, master, refine, walk) = " << tpre << " " << tmaster << " " << trefine << " " << twalk << std::endl; // } diff --git a/src/Neighbor/TreeNeighbor.cc b/src/Neighbor/TreeNeighbor.cc index de835d646..c19179a1d 100644 --- a/src/Neighbor/TreeNeighbor.cc +++ b/src/Neighbor/TreeNeighbor.cc @@ -13,7 +13,7 @@ #include "NodeList/NodeList.hh" #include "Utilities/globalBoundingVolumes.hh" #include "Utilities/packElement.hh" -#include "Utilities/allReduce.hh" +#include "Distributed/allReduce.hh" #include "Utilities/FastMath.hh" #include "Boundary/mapPositionThroughPlanes.hh" #include "Geometry/Dimension.hh" @@ -525,7 +525,7 @@ dumpTree(const Tree& tree, std::stringstream ss; CellKey ix, iy, iz; unsigned nlevels = tree.size(); - if (globalTree) nlevels = allReduce(nlevels, MPI_MAX, Communicator::communicator()); + if (globalTree) nlevels = allReduce(nlevels, SPHERAL_MPI_MAX); ss << "Tree : nlevels = " << nlevels << "\n"; for (unsigned ilevel = 0; ilevel != nlevels; ++ilevel) { @@ -607,7 +607,7 @@ dumpTreeStatistics(const Tree& tree, const bool globalTree) const { std::stringstream ss; unsigned nlevels = tree.size(); - if (globalTree) nlevels = allReduce(nlevels, MPI_MAX, Communicator::communicator()); + if (globalTree) nlevels = allReduce(nlevels, SPHERAL_MPI_MAX); ss << "Tree : nlevels = " << nlevels << "\n"; for (unsigned ilevel = 0; ilevel != nlevels; ++ilevel) { diff --git a/src/NodeGenerators/centroidalRelaxNodesImpl.cc b/src/NodeGenerators/centroidalRelaxNodesImpl.cc index a2132e6cb..f64772a5d 100644 --- a/src/NodeGenerators/centroidalRelaxNodesImpl.cc +++ b/src/NodeGenerators/centroidalRelaxNodesImpl.cc @@ -119,8 +119,8 @@ centroidalRelaxNodesImpl(DataBase& db, ntot += n; for (auto i = 0U; i != n; ++i) avgneighbors += cm.numNeighborsForNode(nodeListi, i); } - ntot = allReduce(ntot, MPI_SUM, Communicator::communicator()); - avgneighbors = allReduce(avgneighbors, MPI_SUM, Communicator::communicator())/ntot; + ntot = allReduce(ntot, SPHERAL_MPI_SUM); + avgneighbors = allReduce(avgneighbors, SPHERAL_MPI_SUM)/ntot; if (Process::getRank() == 0) cerr << "Avergage number of neighbors per node: " << avgneighbors << " " << ntot << endl; } // BLAGO @@ -185,9 +185,9 @@ centroidalRelaxNodesImpl(DataBase& db, if (vol(nodeListi, i) > 0.0) mass(nodeListi, i) = rhof(nodeListi,i)*vol(nodeListi,i); } } - avgdelta = (allReduce(avgdelta, MPI_SUM, Communicator::communicator())/ - allReduce(db.numInternalNodes(), MPI_SUM, Communicator::communicator())); - maxdelta = allReduce(maxdelta, MPI_MAX, Communicator::communicator()); + avgdelta = (allReduce(avgdelta, SPHERAL_MPI_SUM)/ + allReduce(db.numInternalNodes(), SPHERAL_MPI_SUM)); + maxdelta = allReduce(maxdelta, SPHERAL_MPI_MAX); if (Process::getRank() == 0) cerr << "centroidalRelaxNodes iteration " << iter << ", avg delta frac " << avgdelta << ", max delta frac " << maxdelta diff --git a/src/NodeGenerators/compactFacetedVolumes.cc b/src/NodeGenerators/compactFacetedVolumes.cc index 908b4c0e5..d48c57f21 100644 --- a/src/NodeGenerators/compactFacetedVolumes.cc +++ b/src/NodeGenerators/compactFacetedVolumes.cc @@ -9,7 +9,7 @@ #include "Material/GammaLawGas.hh" #include "DataBase/DataBase.hh" #include "Utilities/packElement.hh" -#include "Utilities/allReduce.hh" +#include "Distributed/allReduce.hh" namespace Spheral { @@ -40,7 +40,7 @@ unsigned compactFacetedVolumes(std::vector& s // Only proceed if there's work to do! int flagmax = *max_element(flags.begin(), flags.end()); - if (allReduce(flagmax, MPI_MAX, Communicator::communicator()) != 2) return 0; + if (allReduce(flagmax, SPHERAL_MPI_MAX) != 2) return 0; // Carve up the shapes range in parallel. // const size_t ndomain0 = nshapes/nprocs; @@ -276,9 +276,9 @@ unsigned compactFacetedVolumes(std::vector& s // CHECK(bufitr == buffer.end()); // } // } - // maxoverlap = allReduce(maxoverlap, MPI_MAX, Communicator::communicator()); + // maxoverlap = allReduce(maxoverlap, SPHERAL_MPI_MAX); // #endif - // double sumdisp = allReduce(std::accumulate(displacements.begin(), displacements.end(), 0.0, [](const double prior, const Vector& elemval) { return prior + elemval.magnitude(); }), MPI_SUM, Communicator::communicator()); + // double sumdisp = allReduce(std::accumulate(displacements.begin(), displacements.end(), 0.0, [](const double prior, const Vector& elemval) { return prior + elemval.magnitude(); }), SPHERAL_MPI_SUM); // if (rank == 0) { // cout << " Iteration " << iter // << ", maxoverlap " << maxoverlap @@ -291,7 +291,7 @@ unsigned compactFacetedVolumes(std::vector& s } // end of iteration } - iter = allReduce(iter, MPI_MAX, Communicator::communicator()); + iter = allReduce(iter, SPHERAL_MPI_MAX); // Any shapes we were unable to disentangle turn back to inactive, otherwise set the successful // survivors to flag=1. diff --git a/src/NodeGenerators/fillFacetedVolume.cc b/src/NodeGenerators/fillFacetedVolume.cc index 9151254b1..d81282b7d 100644 --- a/src/NodeGenerators/fillFacetedVolume.cc +++ b/src/NodeGenerators/fillFacetedVolume.cc @@ -2,7 +2,7 @@ #include "fillFacetedVolume.hh" #include "Utilities/rotationMatrix.hh" -#include "Utilities/allReduce.hh" +#include "Distributed/allReduce.hh" #include "Distributed/Communicator.hh" using std::vector; @@ -220,7 +220,7 @@ fillFacetedVolume10(const Dim<3>::FacetedVolume& outerBoundary0, } // If we didn't find anything, fall back to sampling on the surface. - if (allReduce(result.size(), MPI_SUM, Communicator::communicator()) == 0U) { + if (allReduce(result.size(), SPHERAL_MPI_SUM) == 0U) { if (Process::getRank() == 0) { cerr << "Falling back to surface points..." << endl; const size_t nexpect = size_t(std::max(1, std::min(int(verts.size()), int(outerBoundary.volume()/(dx*dx*dx) + 0.5)))); diff --git a/src/NodeGenerators/generateCylDistributionFromRZ.cc b/src/NodeGenerators/generateCylDistributionFromRZ.cc index 9ad17d002..ec2851cd6 100644 --- a/src/NodeGenerators/generateCylDistributionFromRZ.cc +++ b/src/NodeGenerators/generateCylDistributionFromRZ.cc @@ -2,14 +2,11 @@ // Helper method for the GenerateCylindricalNodeDistribution3d node generator // to generate the spun node distribution. //------------------------------------------------------------------------------ -#ifdef USE_MPI -#include -#endif #include "Boundary/CylindricalBoundary.hh" #include "Utilities/DBC.hh" #include "Geometry/Dimension.hh" -#include "Distributed/Communicator.hh" +#include "Distributed/allReduce.hh" #include #include @@ -142,19 +139,8 @@ generateCylDistributionFromRZ(vector& x, (int)globalIDs.size() == ndomain and (int)H.size() == ndomain); for (int ikey = 0; ikey != nextra; ++ikey) VERIFY((int)extraFields[ikey].size() == ndomain); -#ifdef USE_MPI - { - int nlocal = x.size(); - int nglobal; - if (nProcs > 1) { - MPI_Allreduce(&nlocal, &nglobal, 1, MPI_INT, MPI_SUM, Communicator::communicator()); - } - else { - nglobal = nlocal; - } - VERIFY(nglobal == ntot); - } -#endif + int nglobal = allReduce(x.size(), SPHERAL_MPI_SUM); + VERIFY(nglobal == ntot); } diff --git a/src/NodeGenerators/relaxNodeDistribution.cc b/src/NodeGenerators/relaxNodeDistribution.cc index e753f8231..fe8eb7480 100644 --- a/src/NodeGenerators/relaxNodeDistribution.cc +++ b/src/NodeGenerators/relaxNodeDistribution.cc @@ -5,12 +5,7 @@ #include "relaxNodeDistribution.hh" #include "Field/FieldList.hh" #include "Boundary/Boundary.hh" -#include "Utilities/allReduce.hh" - -#ifdef USE_MPI -#include -#include "Distributed/Communicator.hh" -#endif +#include "Distributed/allReduce.hh" #include using std::vector; @@ -144,9 +139,7 @@ relaxNodeDistribution(DataBase& dataBase, ++k; } } -#ifdef USE_MPI - Msum = allReduce(Msum, MPI_SUM, Communicator::communicator()); -#endif + Msum = allReduce(Msum, SPHERAL_MPI_SUM); // If needed, rescale the masses. if (targetMass > 0.0) { diff --git a/src/RK/computeVoronoiVolume.cc b/src/RK/computeVoronoiVolume.cc index 32cca254c..12ccbaf3e 100644 --- a/src/RK/computeVoronoiVolume.cc +++ b/src/RK/computeVoronoiVolume.cc @@ -6,7 +6,7 @@ #include "Field/FieldList.hh" #include "NodeList/NodeList.hh" #include "Neighbor/ConnectivityMap.hh" -#include "Utilities/allReduce.hh" +#include "Distributed/allReduce.hh" #include "Utilities/pointOnPolygon.hh" #include "Utilities/FastMath.hh" #include "Geometry/PolyClipperUtilities.hh" @@ -358,7 +358,7 @@ computeVoronoiVolume(const FieldList& pos const auto numGens = position.numNodes(); const auto numNodeLists = position.size(); - const auto numGensGlobal = allReduce(numGens, MPI_SUM, Communicator::communicator()); + const auto numGensGlobal = allReduce(numGens, SPHERAL_MPI_SUM); const auto haveFacetedBoundaries = facetedBoundaries.size() == numNodeLists; const auto haveWeights = weight.size() == numNodeLists; const auto haveDamage = false; // damage.size() == numNodeLists; // Suspending the idea of forcing surface based on damage diff --git a/src/Utilities/CMakeLists.txt b/src/Utilities/CMakeLists.txt index f8890cf2f..1acacbca6 100644 --- a/src/Utilities/CMakeLists.txt +++ b/src/Utilities/CMakeLists.txt @@ -19,10 +19,8 @@ set(Utilities_sources DBC.cc QuadraticInterpolator.cc CubicHermiteInterpolator.cc - Process.cc KeyTraits.cc PeanoHilbertTransform.cc - RegisterMPIDataTypes.cc RedistributionRegistrar.cc vectorBoundingBoxInst.cc lineSegmentIntersections.cc @@ -70,20 +68,16 @@ set(Utilities_headers OpenMP_wrapper.hh PairComparisons.hh PeanoHilbertTransform.hh - Process.hh RedistributionNotification.hh RedistributionNotificationHandle.hh RedistributionRegistrar.hh RedistributionRegistrarInline.hh - RegisterMPIDataTypes.hh - RegisterMPIDataTypesInline.hh SafeIndexMap.hh SpheralFunctions.hh SurfaceNodeCoupling.hh Timer.hh Tree.hh TreeInline.hh - allReduce.hh bisectRoot.hh bisectSearch.hh boundPointWithinBox.hh @@ -107,7 +101,6 @@ set(Utilities_headers log2.hh medianPosition.hh mortonOrderIndices.hh - mpiUtilities.hh newtonRaphson.hh nodeBoundingBoxes.hh nodeBoundingBoxesInline.hh @@ -127,7 +120,6 @@ set(Utilities_headers removeElements.hh rotationMatrix.hh safeInv.hh - scan.hh segmentIntersectEdges.hh simpsonsIntegration.hh size_t_bits.hh diff --git a/src/Utilities/DBC.hh b/src/Utilities/DBC.hh index a5a06d23f..9804ecfc9 100644 --- a/src/Utilities/DBC.hh +++ b/src/Utilities/DBC.hh @@ -10,7 +10,7 @@ #include #include #include -#include "Process.hh" +#include "Distributed/Process.hh" #ifndef DBC_FUNCTIONS_HH #define DBC_FUNCTIONS_HH diff --git a/src/Utilities/DataTypeTraits.hh b/src/Utilities/DataTypeTraits.hh index 1668141ea..6a4c6d1a5 100644 --- a/src/Utilities/DataTypeTraits.hh +++ b/src/Utilities/DataTypeTraits.hh @@ -16,7 +16,7 @@ #include #include "Geometry/Dimension.hh" #include "Geometry/PolyClipperUtilities.hh" -#include "RegisterMPIDataTypes.hh" +#include "Distributed/RegisterMPIDataTypes.hh" #include "Utilities/DomainNode.hh" #include "RK/RKCorrectionParams.hh" #include "RK/RKCoefficients.hh" diff --git a/src/Utilities/globalBoundingVolumes.cc b/src/Utilities/globalBoundingVolumes.cc index 7332151e9..a90493dc9 100644 --- a/src/Utilities/globalBoundingVolumes.cc +++ b/src/Utilities/globalBoundingVolumes.cc @@ -10,13 +10,8 @@ // Created by JMO, Sun Jan 31 19:53:36 PST 2010 //----------------------------------------------------------------------------// #include "DataBase/DataBase.hh" -#include "Utilities/allReduce.hh" +#include "Distributed/allReduce.hh" #include "Geometry/Dimension.hh" -#include "Distributed/Communicator.hh" - -#ifdef USE_MPI -#include -#endif #include #include @@ -108,8 +103,8 @@ globalBoundingBox(const Field& positions, // Now find the global bounds across all processors. for (unsigned i = 0; i != Dimension::nDim; ++i) { - xmin(i) = allReduce(xmin(i), MPI_MIN, Communicator::communicator()); - xmax(i) = allReduce(xmax(i), MPI_MAX, Communicator::communicator()); + xmin(i) = allReduce(xmin(i), SPHERAL_MPI_MIN); + xmax(i) = allReduce(xmax(i), SPHERAL_MPI_MAX); } } @@ -139,8 +134,8 @@ globalBoundingBox(const FieldList& positi // Now find the global bounds across all processors. for (unsigned i = 0; i != Dimension::nDim; ++i) { - xmin(i) = allReduce(xmin(i), MPI_MIN, Communicator::communicator()); - xmax(i) = allReduce(xmax(i), MPI_MAX, Communicator::communicator()); + xmin(i) = allReduce(xmin(i), SPHERAL_MPI_MIN); + xmax(i) = allReduce(xmax(i), SPHERAL_MPI_MAX); } } diff --git a/src/Utilities/iterateIdealH.cc b/src/Utilities/iterateIdealH.cc index 24560e972..e0b941da7 100644 --- a/src/Utilities/iterateIdealH.cc +++ b/src/Utilities/iterateIdealH.cc @@ -5,7 +5,7 @@ #include "iterateIdealH.hh" #include "Field/FieldList.hh" #include "NodeList/SmoothingScaleBase.hh" -#include "Utilities/allReduce.hh" +#include "Distributed/allReduce.hh" #include "Distributed/Communicator.hh" #include "Geometry/GeometryRegistrar.hh" @@ -275,7 +275,7 @@ iterateIdealH(DataBase& dataBase, H.assignFields(H1); // Globally reduce the max H change. - maxDeltaH = allReduce(maxDeltaH, MPI_MAX, Communicator::communicator()); + maxDeltaH = allReduce(maxDeltaH, SPHERAL_MPI_MAX); // Output the statitics. if (Process::getRank() == 0) diff --git a/src/Utilities/mpiUtilities.hh b/src/Utilities/mpiUtilities.hh deleted file mode 100644 index fd7cfb85f..000000000 --- a/src/Utilities/mpiUtilities.hh +++ /dev/null @@ -1,46 +0,0 @@ -//---------------------------------Spheral++----------------------------------// -// mpiUtilities -// -// Useful methods for doing MPI stuff. These methods are dummied to do the -// right thing if we're actually building serial as well. -// -// Created by JMO, Thu Aug 21 17:17:57 PDT 2008 -//----------------------------------------------------------------------------// -#ifndef __Spheral_mpiUtilities__ -#define __Spheral_mpiUtilities__ - -#ifdef USE_MPI -//============================================================================== -// MPI versions. -//============================================================================== -#include -#include "Distributed/Communicator.hh" - -namespace Spheral { - -inline -double -safeAllReduceMax(double val) { - double result; - MPI_Allreduce(&val, &result, 1, MPI_DOUBLE, MPI_MAX, Communicator::communicator()); - return result; -} - -} - -#else - -//============================================================================== -// Dummy serial versions. -//============================================================================== -namespace Spheral { -inline -double -safeAllReduceMax(double val) { - std::cerr << "Not here!" << std::endl; - return val; -} -} - -#endif -#endif diff --git a/src/Utilities/nodeOrdering.cc b/src/Utilities/nodeOrdering.cc index b96c45529..5fdc6fcb4 100644 --- a/src/Utilities/nodeOrdering.cc +++ b/src/Utilities/nodeOrdering.cc @@ -13,11 +13,7 @@ #include "Field/FieldList.hh" #include "Field/Field.hh" #include "NodeList/NodeList.hh" -#include "Distributed/Communicator.hh" - -#ifdef USE_MPI -#include -#endif +#include "Distributed/allReduce.hh" #include #include @@ -72,13 +68,7 @@ nodeOrdering(const FieldList& criteria) { // Find the total number of nodes. const int numLocalNodes = sortedList.size(); - int numGlobalNodes = numLocalNodes; -#ifdef USE_MPI - { - int tmp = numGlobalNodes; - MPI_Allreduce(&tmp, &numGlobalNodes, 1, MPI_INT, MPI_SUM, Communicator::communicator()); - } -#endif + int numGlobalNodes = allReduce(numLocalNodes, SPHERAL_MPI_SUM); // Iterate over the local nodes in order. int iLocal = 0; @@ -88,23 +78,12 @@ nodeOrdering(const FieldList& criteria) { if (iLocal < numLocalNodes) localKey = std::get<2>(sortedList[iLocal]); // Find the next key globally. - Key globalKey = localKey; -#ifdef USE_MPI - { - Key tmp = localKey; - MPI_Allreduce(&tmp, &globalKey, 1, DataTypeTraits::MpiDataType(), MPI_MIN, Communicator::communicator()); - } -#endif + Key globalKey = allReduce(localKey, SPHERAL_MPI_MIN); // If we have the next index, check for duplicates on other domains. int minProcID = numProcs + 1; if (localKey == globalKey) minProcID = procID; -#ifdef USE_MPI - { - int tmp = minProcID; - MPI_Allreduce(&tmp, &minProcID, 1, MPI_INT, MPI_MIN, Communicator::communicator()); - } -#endif + minProcID = allReduce(minProcID, SPHERAL_MPI_MIN); // Are we the next global key? if (localKey == globalKey and procID == minProcID) { @@ -134,12 +113,7 @@ nodeOrdering(const FieldList& criteria) { for (typename FieldList::const_iterator itr = result.begin(); itr != result.end(); ++itr, ++iNodeList) countGlobal += count((**itr).internalBegin(), (**itr).internalEnd(), iGlobal); -#ifdef USE_MPI - { - int tmp = countGlobal; - MPI_Allreduce(&tmp, &countGlobal, 1, MPI_INT, MPI_SUM, Communicator::communicator()); - } -#endif + countGlobal = allReduce(countGlobal, SPHERAL_MPI_SUM); ENSURE(countGlobal == 1); } } diff --git a/src/Utilities/scan.hh b/src/Utilities/scan.hh deleted file mode 100644 index 9c6de7d23..000000000 --- a/src/Utilities/scan.hh +++ /dev/null @@ -1,56 +0,0 @@ -//---------------------------------Spheral++----------------------------------// -// scan -// -// Hide (some) of the details about doing MPI scan -//----------------------------------------------------------------------------// -#ifndef __Spheral_scan__ -#define __Spheral_scan__ - -#include "Utilities/DataTypeTraits.hh" - -#ifdef USE_MPI -//------------------------------------------------------------------------------ -// MPI version -//------------------------------------------------------------------------------ - -#include - -namespace Spheral { - -template -Value -scan(const Value& value, const MPI_Op op, const MPI_Comm comm) { - Value tmp = value; - Value result; - MPI_Scan(&tmp, &result, 1, DataTypeTraits::MpiDataType(), op, comm); - return result; -} - -} - -#else -//------------------------------------------------------------------------------ -// Non-MPI version -//------------------------------------------------------------------------------ - -namespace Spheral { - -#define MPI_MIN 1 -#define MPI_MAX 2 -#define MPI_SUM 3 -#define MPI_PROD 4 -#define MPI_LAND 5 -#define MPI_LOR 6 -#define MPI_COMM_WORLD 0 - -template -Value -scan(const Value& value, const int op, const int comm) { - return value; -} - -} - -#endif -#endif - From d8ada328e3f2598b026ccf5a5f8c9bc7cda5bdbb Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Wed, 26 Jun 2024 16:35:52 -0700 Subject: [PATCH 02/14] Fix compiler errors --- src/Distributed/RegisterMPIDataTypes.cc | 2 +- src/FieldOperations/binFieldList2Lattice.cc | 3 +-- src/Mesh/PolyhedralMesh.cc | 4 ++-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/Distributed/RegisterMPIDataTypes.cc b/src/Distributed/RegisterMPIDataTypes.cc index b1ae641cc..f3da6a951 100644 --- a/src/Distributed/RegisterMPIDataTypes.cc +++ b/src/Distributed/RegisterMPIDataTypes.cc @@ -7,7 +7,7 @@ //----------------------------------------------------------------------------// #include "RegisterMPIDataTypes.hh" -#include "DataTypeTraits.hh" +#include "Utilities/DataTypeTraits.hh" #include "Geometry/Dimension.hh" namespace Spheral { diff --git a/src/FieldOperations/binFieldList2Lattice.cc b/src/FieldOperations/binFieldList2Lattice.cc index 3042f637a..f7d69f059 100644 --- a/src/FieldOperations/binFieldList2Lattice.cc +++ b/src/FieldOperations/binFieldList2Lattice.cc @@ -315,8 +315,7 @@ binFieldList2Lattice(const FieldList& fieldList, BEGIN_CONTRACT_SCOPE // Check that everyone agrees about the size. { - int bufSizeMin; - MPI_Allreduce(&bufSize, &bufSizeMin, 1, MPI_INT, SPHERAL_MPI_MIN); + int bufSizeMin = allReduce(bufSize, SPHERAL_MPI_MIN); CHECK(bufSizeMin == bufSize); } END_CONTRACT_SCOPE diff --git a/src/Mesh/PolyhedralMesh.cc b/src/Mesh/PolyhedralMesh.cc index 5b5fed794..7d496974d 100644 --- a/src/Mesh/PolyhedralMesh.cc +++ b/src/Mesh/PolyhedralMesh.cc @@ -297,10 +297,10 @@ boundingSurface() const { for (domainID = 0; domainID != numDomains; ++domainID) { buffer = localBuffer; bufSize = localBuffer.size(); - MPI_Bcast(&bufSize, 1, MPI_UNSIGNED, domainID); + MPI_Bcast(&bufSize, 1, MPI_UNSIGNED, domainID, Communicator::communicator()); if (bufSize > 0) { buffer.resize(bufSize); - MPI_Bcast(&buffer.front(), bufSize, MPI_CHAR, domainID); + MPI_Bcast(&buffer.front(), bufSize, MPI_CHAR, domainID, Communicator::communicator()); bufItr = buffer.begin(); unpackElement(globalVertexPositions, bufItr, buffer.end()); unpackElement(nfacets, bufItr, buffer.end()); From 8c8b2061c0242c4e73728abc84fa33ec68c86d7d Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Wed, 26 Jun 2024 16:54:19 -0700 Subject: [PATCH 03/14] Remove compiler warning --- src/FieldOperations/binFieldList2Lattice.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/FieldOperations/binFieldList2Lattice.cc b/src/FieldOperations/binFieldList2Lattice.cc index f7d69f059..41962341f 100644 --- a/src/FieldOperations/binFieldList2Lattice.cc +++ b/src/FieldOperations/binFieldList2Lattice.cc @@ -315,8 +315,7 @@ binFieldList2Lattice(const FieldList& fieldList, BEGIN_CONTRACT_SCOPE // Check that everyone agrees about the size. { - int bufSizeMin = allReduce(bufSize, SPHERAL_MPI_MIN); - CHECK(bufSizeMin == bufSize); + CHECK(bufSize == allReduce(bufSize, SPHERAL_MPI_MIN)); } END_CONTRACT_SCOPE From 31fd65a0d1ef51fec18a023b2a35d72601882a0b Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Mon, 1 Jul 2024 17:17:00 -0700 Subject: [PATCH 04/14] Replaced SPHERAL_MPI_ with SPHERAL_OP_ --- src/Boundary/InflowOutflowBoundary.cc | 12 +++--- .../findNodesTouchingThroughPlanes.cc | 2 +- src/Damage/DamageModel.cc | 2 +- src/Damage/ProbabilisticDamageModel.cc | 18 ++++---- .../weibullFlawDistributionBenzAsphaug.cc | 12 +++--- src/Damage/weibullFlawDistributionOwen.cc | 16 +++---- src/DataBase/DataBase.cc | 42 +++++++++---------- .../SpaceFillingCurveRedistributeNodes.cc | 12 +++--- src/Distributed/VoronoiRedistributeNodes.cc | 8 ++-- src/Distributed/allReduce.hh | 24 +++++------ src/Field/FieldInline.hh | 6 +-- src/Field/FieldListInline.hh | 6 +-- src/FieldOperations/binFieldList2Lattice.cc | 6 +-- src/Gravity/NBodyGravity.cc | 6 +-- src/Gravity/PolyGravity.cc | 2 +- src/Gravity/Tree.cc | 4 +- src/Gravity/TreeGravity.cc | 6 +-- src/Hydro/VoronoiHourglassControl.cc | 4 +- src/Integrator/Integrator.cc | 2 +- src/KernelIntegrator/FlatConnectivity.cc | 4 +- src/Mesh/LineMesh.cc | 6 +-- src/Mesh/Mesh.cc | 16 +++---- src/Mesh/MeshInline.hh | 4 +- src/Neighbor/ConnectivityMap.cc | 8 ++-- src/Neighbor/TreeNeighbor.cc | 4 +- .../centroidalRelaxNodesImpl.cc | 10 ++--- src/NodeGenerators/compactFacetedVolumes.cc | 8 ++-- src/NodeGenerators/fillFacetedVolume.cc | 2 +- .../generateCylDistributionFromRZ.cc | 2 +- src/NodeGenerators/relaxNodeDistribution.cc | 2 +- src/RK/computeVoronoiVolume.cc | 2 +- src/Utilities/globalBoundingVolumes.cc | 8 ++-- src/Utilities/iterateIdealH.cc | 2 +- src/Utilities/nodeOrdering.cc | 8 ++-- 34 files changed, 138 insertions(+), 138 deletions(-) diff --git a/src/Boundary/InflowOutflowBoundary.cc b/src/Boundary/InflowOutflowBoundary.cc index a1b6322c3..2a6b80250 100644 --- a/src/Boundary/InflowOutflowBoundary.cc +++ b/src/Boundary/InflowOutflowBoundary.cc @@ -133,7 +133,7 @@ updateGhostNodes(NodeList& nodeList) { const auto xd = mPlane.signedDistance(pos[i]); xmin = std::min(xmin, xd); } - xmin = allReduce(xmin, SPHERAL_MPI_MIN); + xmin = allReduce(xmin, SPHERAL_OP_MIN); // CHECK(xmin >= 0.0); // Offset the current ghost points appropriately. @@ -309,8 +309,8 @@ InflowOutflowBoundary::initializeProblemStartup(const bool /*final*/) for (const auto i: nodeIDs) { vinflow += vel[i].dot(nhat); } - vinflow = (allReduce(vinflow, SPHERAL_MPI_SUM)/ - std::max(1.0e-30, allReduce(double(nodeIDs.size()), SPHERAL_MPI_SUM))); // Negative implies outflow + vinflow = (allReduce(vinflow, SPHERAL_OP_SUM)/ + std::max(1.0e-30, allReduce(double(nodeIDs.size()), SPHERAL_OP_SUM))); // Negative implies outflow // Figure out a timestep limit such that we don't move more than the ghost // node thickness. @@ -320,8 +320,8 @@ InflowOutflowBoundary::initializeProblemStartup(const bool /*final*/) xmin = std::min(xmin, xd); xmax = std::max(xmax, xd); } - xmin = allReduce(xmin, SPHERAL_MPI_MIN); - xmax = allReduce(xmax, SPHERAL_MPI_MAX); + xmin = allReduce(xmin, SPHERAL_OP_MIN); + xmax = allReduce(xmax, SPHERAL_OP_MAX); mXmin[nodeList.name()] = xmin; mDT = std::min(mDT, std::abs(xmax - xmin)/std::max(1e-30, std::abs(vinflow))); // Protect from negative outflow velocity // cerr << "Timestep constraint: " << mDT << endl; @@ -458,7 +458,7 @@ InflowOutflowBoundary::finalize(const Scalar /*time*/, nodeList.neighbor().updateNodes(); } } - altered = (allReduce((altered ? 1 : 0), SPHERAL_MPI_MAX) == 1); + altered = (allReduce((altered ? 1 : 0), SPHERAL_OP_MAX) == 1); // If any NodeLists were altered, recompute the boundary conditions. if (altered) { diff --git a/src/Boundary/findNodesTouchingThroughPlanes.cc b/src/Boundary/findNodesTouchingThroughPlanes.cc index a79a11690..4bc7f5ff1 100644 --- a/src/Boundary/findNodesTouchingThroughPlanes.cc +++ b/src/Boundary/findNodesTouchingThroughPlanes.cc @@ -50,7 +50,7 @@ findNodesTouchingThroughPlanes(const NodeList& nodeList, const auto hmaxi = 1.0/Hi.eigenValues().minElement(); if (hmaxi > hmax and std::min(exitPlane.minimumDistance(ri), enterPlane.minimumDistance(ri)) < kernelExtent*hmaxi) hmax = hmaxi; } - hmax = allReduce(hmax, SPHERAL_MPI_MAX); + hmax = allReduce(hmax, SPHERAL_OP_MAX); // Now find all points within this range of the exit plane. if (hmax > 0.0) { diff --git a/src/Damage/DamageModel.cc b/src/Damage/DamageModel.cc index 995a9dbdb..3f6408d6f 100644 --- a/src/Damage/DamageModel.cc +++ b/src/Damage/DamageModel.cc @@ -196,7 +196,7 @@ finalize(const Scalar /*time*/, nD += nD_thread; } } - nD = allReduce(nD, SPHERAL_MPI_SUM); + nD = allReduce(nD, SPHERAL_OP_SUM); const auto ntot = std::max(1, dataBase.globalNumInternalNodes()); const auto dfrac = double(nD)/double(ntot); mComputeIntersectConnectivity = (dfrac > 0.2); // Should tune this number... diff --git a/src/Damage/ProbabilisticDamageModel.cc b/src/Damage/ProbabilisticDamageModel.cc index 05e99194e..bff2530fb 100644 --- a/src/Damage/ProbabilisticDamageModel.cc +++ b/src/Damage/ProbabilisticDamageModel.cc @@ -121,7 +121,7 @@ initializeProblemStartupDependencies(DataBase& dataBase, for (auto i = 0u; i < mMask.numInternalElements(); ++i) { if (mMask[i] == 1) ++nused_local; } - const size_t nused_global = allReduce(nused_local, SPHERAL_MPI_SUM); + const size_t nused_global = allReduce(nused_local, SPHERAL_OP_SUM); // Compute the Morton-ordering for hashing with the global seed to seed each // point-wise random number generator. @@ -158,8 +158,8 @@ initializeProblemStartupDependencies(DataBase& dataBase, randomGenerators[i](); // Recommended to discard first value in sequence } } - mVmin = allReduce(mVmin, SPHERAL_MPI_MIN); - mVmax = allReduce(mVmax, SPHERAL_MPI_MAX); + mVmin = allReduce(mVmin, SPHERAL_OP_MIN); + mVmax = allReduce(mVmax, SPHERAL_OP_MAX); // Generate min/max ranges of flaws for each point. const auto mInv = 1.0/mmWeibull; @@ -200,12 +200,12 @@ initializeProblemStartupDependencies(DataBase& dataBase, // Some diagnostic output. if (nused_global > 0) { - minNumFlaws = allReduce(minNumFlaws, SPHERAL_MPI_MIN); - maxNumFlaws = allReduce(maxNumFlaws, SPHERAL_MPI_MAX); - totalNumFlaws = allReduce(totalNumFlaws, SPHERAL_MPI_SUM); - epsMin = allReduce(epsMin, SPHERAL_MPI_MIN); - epsMax = allReduce(epsMax, SPHERAL_MPI_MAX); - numFlawsRatio = allReduce(numFlawsRatio, SPHERAL_MPI_SUM)/nused_global; + minNumFlaws = allReduce(minNumFlaws, SPHERAL_OP_MIN); + maxNumFlaws = allReduce(maxNumFlaws, SPHERAL_OP_MAX); + totalNumFlaws = allReduce(totalNumFlaws, SPHERAL_OP_SUM); + epsMin = allReduce(epsMin, SPHERAL_OP_MIN); + epsMax = allReduce(epsMax, SPHERAL_OP_MAX); + numFlawsRatio = allReduce(numFlawsRatio, SPHERAL_OP_SUM)/nused_global; if (Process::getRank() == 0) { cerr << "ProbabilisticDamageModel for " << nodes.name() << ":" << endl << " Min, max, max/min volumes: " << mVmin << " " << mVmax << " " << mVmax*safeInv(mVmin) << endl diff --git a/src/Damage/weibullFlawDistributionBenzAsphaug.cc b/src/Damage/weibullFlawDistributionBenzAsphaug.cc index 9c3c2ec97..2e9e99607 100644 --- a/src/Damage/weibullFlawDistributionBenzAsphaug.cc +++ b/src/Damage/weibullFlawDistributionBenzAsphaug.cc @@ -99,7 +99,7 @@ weibullFlawDistributionBenzAsphaug(double volume, CHECK(rho(i) > 0.0); volume += mass(i)/rho(i); } - volume = allReduce(volume, SPHERAL_MPI_SUM); + volume = allReduce(volume, SPHERAL_OP_SUM); } volume = std::max(volume, 1e-100); CHECK(volume > 0.0); @@ -168,11 +168,11 @@ weibullFlawDistributionBenzAsphaug(double volume, // Prepare some diagnostic output. const auto nused = std::max(1, mask.sumElements()); - minNumFlaws = allReduce(minNumFlaws, SPHERAL_MPI_MIN); - maxNumFlaws = allReduce(maxNumFlaws, SPHERAL_MPI_MAX); - totalNumFlaws = allReduce(totalNumFlaws, SPHERAL_MPI_SUM); - epsMax = allReduce(epsMax, SPHERAL_MPI_MAX); - sumFlaws = allReduce(sumFlaws, SPHERAL_MPI_SUM); + minNumFlaws = allReduce(minNumFlaws, SPHERAL_OP_MIN); + maxNumFlaws = allReduce(maxNumFlaws, SPHERAL_OP_MAX); + totalNumFlaws = allReduce(totalNumFlaws, SPHERAL_OP_SUM); + epsMax = allReduce(epsMax, SPHERAL_OP_MAX); + sumFlaws = allReduce(sumFlaws, SPHERAL_OP_SUM); if (procID == 0) { cerr << "weibullFlawDistributionBenzAsphaug: Min num flaws per node: " << minNumFlaws << endl << " Max num flaws per node: " << maxNumFlaws << endl diff --git a/src/Damage/weibullFlawDistributionOwen.cc b/src/Damage/weibullFlawDistributionOwen.cc index 9f3522d81..31a9318a9 100644 --- a/src/Damage/weibullFlawDistributionOwen.cc +++ b/src/Damage/weibullFlawDistributionOwen.cc @@ -108,8 +108,8 @@ weibullFlawDistributionOwen(const unsigned seed, Vmax = max(Vmax, Vi); } } - Vmin = allReduce(Vmin*volumeMultiplier, SPHERAL_MPI_MIN); - Vmax = allReduce(Vmax*volumeMultiplier, SPHERAL_MPI_MAX); + Vmin = allReduce(Vmin*volumeMultiplier, SPHERAL_OP_MIN); + Vmax = allReduce(Vmax*volumeMultiplier, SPHERAL_OP_MAX); CHECK(Vmin > 0.0); CHECK(Vmax >= Vmin); @@ -157,12 +157,12 @@ weibullFlawDistributionOwen(const unsigned seed, // Some diagnostic output. const auto nused = std::max(1, mask.sumElements()); if (nglobal > 0) { - minNumFlaws = allReduce(minNumFlaws, SPHERAL_MPI_MIN); - maxNumFlaws = allReduce(maxNumFlaws, SPHERAL_MPI_MAX); - totalNumFlaws = allReduce(totalNumFlaws, SPHERAL_MPI_SUM); - epsMin = allReduce(epsMin, SPHERAL_MPI_MIN); - epsMax = allReduce(epsMax, SPHERAL_MPI_MAX); - sumFlaws = allReduce(sumFlaws, SPHERAL_MPI_SUM); + minNumFlaws = allReduce(minNumFlaws, SPHERAL_OP_MIN); + maxNumFlaws = allReduce(maxNumFlaws, SPHERAL_OP_MAX); + totalNumFlaws = allReduce(totalNumFlaws, SPHERAL_OP_SUM); + epsMin = allReduce(epsMin, SPHERAL_OP_MIN); + epsMax = allReduce(epsMax, SPHERAL_OP_MAX); + sumFlaws = allReduce(sumFlaws, SPHERAL_OP_SUM); } if (procID == 0) { cerr << "weibullFlawDistributionOwen: Min num flaws per node: " << minNumFlaws << endl diff --git a/src/DataBase/DataBase.cc b/src/DataBase/DataBase.cc index 7022a780a..3ad7073e8 100644 --- a/src/DataBase/DataBase.cc +++ b/src/DataBase/DataBase.cc @@ -92,7 +92,7 @@ int DataBase::globalNumInternalNodes() const { int localResult = numInternalNodes(); int result = localResult; - result = allReduce(result, SPHERAL_MPI_SUM); + result = allReduce(result, SPHERAL_OP_SUM); return result; } @@ -101,7 +101,7 @@ int DataBase::globalNumGhostNodes() const { int localResult = numGhostNodes(); int result = localResult; - result = allReduce(result, SPHERAL_MPI_SUM); + result = allReduce(result, SPHERAL_OP_SUM); return result; } @@ -110,7 +110,7 @@ int DataBase::globalNumNodes() const { int localResult = numNodes(); int result = localResult; - result = allReduce(result, SPHERAL_MPI_SUM); + result = allReduce(result, SPHERAL_OP_SUM); return result; } @@ -122,7 +122,7 @@ int DataBase::globalNumFluidInternalNodes() const { int localResult = numFluidInternalNodes(); int result = localResult; - result = allReduce(result, SPHERAL_MPI_SUM); + result = allReduce(result, SPHERAL_OP_SUM); return result; } @@ -131,7 +131,7 @@ int DataBase::globalNumFluidGhostNodes() const { int localResult = numFluidGhostNodes(); int result = localResult; - result = allReduce(result, SPHERAL_MPI_SUM); + result = allReduce(result, SPHERAL_OP_SUM); return result; } @@ -140,7 +140,7 @@ int DataBase::globalNumFluidNodes() const { int localResult = numFluidNodes(); int result = localResult; - result = allReduce(result, SPHERAL_MPI_SUM); + result = allReduce(result, SPHERAL_OP_SUM); return result; } @@ -527,13 +527,13 @@ reinitializeNeighbors() const { // Find the global result across all processors. auto box = 0.0; for (auto i = 0; i != Dimension::nDim; ++i) { - xmin(i) = allReduce(xmin(i), SPHERAL_MPI_MIN); - xmax(i) = allReduce(xmax(i), SPHERAL_MPI_MAX); + xmin(i) = allReduce(xmin(i), SPHERAL_OP_MIN); + xmax(i) = allReduce(xmax(i), SPHERAL_OP_MAX); box = std::max(box, xmax(i) - xmin(i)); } - havg = allReduce(havg, SPHERAL_MPI_SUM); - ntot = allReduce(ntot, SPHERAL_MPI_SUM); - hmax = allReduce(hmax, SPHERAL_MPI_MAX); + havg = allReduce(havg, SPHERAL_OP_SUM); + ntot = allReduce(ntot, SPHERAL_OP_SUM); + hmax = allReduce(hmax, SPHERAL_OP_MAX); if (ntot > 0) { havg /= ntot; @@ -1831,8 +1831,8 @@ boundingBox(typename Dimension::Vector& xmin, // Now find the global bounds across all processors. for (int i = 0; i != Dimension::nDim; ++i) { - xmin(i) = allReduce(xmin(i), SPHERAL_MPI_MIN); - xmax(i) = allReduce(xmax(i), SPHERAL_MPI_MAX); + xmin(i) = allReduce(xmin(i), SPHERAL_OP_MIN); + xmax(i) = allReduce(xmax(i), SPHERAL_OP_MAX); } } @@ -1929,15 +1929,15 @@ globalSamplingBoundingVolume(typename Dimension::Vector& centroid, size_t nlocal = this->numInternalNodes(); centroid *= nlocal; for (int i = 0; i != Dimension::nDim; ++i) { - xminNodes(i) = allReduce(xminNodes(i), SPHERAL_MPI_MIN); - xmaxNodes(i) = allReduce(xmaxNodes(i), SPHERAL_MPI_MAX); - xminSample(i) = allReduce(xminSample(i), SPHERAL_MPI_MIN); - xmaxSample(i) = allReduce(xmaxSample(i), SPHERAL_MPI_MAX); - centroid(i) = allReduce(centroid(i), SPHERAL_MPI_SUM); + xminNodes(i) = allReduce(xminNodes(i), SPHERAL_OP_MIN); + xmaxNodes(i) = allReduce(xmaxNodes(i), SPHERAL_OP_MAX); + xminSample(i) = allReduce(xminSample(i), SPHERAL_OP_MIN); + xmaxSample(i) = allReduce(xmaxSample(i), SPHERAL_OP_MAX); + centroid(i) = allReduce(centroid(i), SPHERAL_OP_SUM); } // Fix up the centroid and radii. - size_t nglobal = allReduce((uint64_t) nlocal, SPHERAL_MPI_SUM); + size_t nglobal = allReduce((uint64_t) nlocal, SPHERAL_OP_SUM); if (nglobal > 0) { centroid /= nglobal; radiusNodes = 0.0; @@ -1957,8 +1957,8 @@ globalSamplingBoundingVolume(typename Dimension::Vector& centroid, radiusSample = max(radiusSample, drMag + 2.0*hi); } } - radiusNodes = allReduce(radiusNodes, SPHERAL_MPI_MAX); - radiusSample = allReduce(radiusSample, SPHERAL_MPI_MAX); + radiusNodes = allReduce(radiusNodes, SPHERAL_OP_MAX); + radiusSample = allReduce(radiusSample, SPHERAL_OP_MAX); const Vector delta = 0.001*(xmaxSample - xminSample); radiusNodes *= 1.001; radiusSample *= 1.001; diff --git a/src/Distributed/SpaceFillingCurveRedistributeNodes.cc b/src/Distributed/SpaceFillingCurveRedistributeNodes.cc index c418d3bc2..ee6f135f5 100644 --- a/src/Distributed/SpaceFillingCurveRedistributeNodes.cc +++ b/src/Distributed/SpaceFillingCurveRedistributeNodes.cc @@ -208,7 +208,7 @@ redistributeNodes(DataBase& dataBase, CHECK(count.size() == uniqueIndices.size()); CHECK(work.size() == uniqueIndices.size()); } - maxCount = allReduce(maxCount, SPHERAL_MPI_MAX); + maxCount = allReduce(maxCount, SPHERAL_OP_MAX); if (procID == 0) cerr << "SpaceFillingCurveRedistributeNodes: max redundancy is " << maxCount << endl; // // DEBUG @@ -446,7 +446,7 @@ numIndicesInRange(const vector: } // Globally reduce that sucker. - result = allReduce(result, SPHERAL_MPI_SUM); + result = allReduce(result, SPHERAL_OP_SUM); return result; } @@ -518,8 +518,8 @@ workAndNodesInRange(const vector& dataBase, // Now we can get the node distribution description. vector > nodeDistribution = this->currentDomainDecomposition(dataBase, globalIDs, workField); const size_t numNodes = nodeDistribution.size(); - const size_t numNodesGlobal = allReduce((uint64_t) numNodes, SPHERAL_MPI_SUM); + const size_t numNodesGlobal = allReduce((uint64_t) numNodes, SPHERAL_OP_SUM); const size_t avgNumNodes = numNodesGlobal/numProcs; CHECK(numNodes > 0); @@ -867,8 +867,8 @@ cullGeneratorNodesByWork(const vector& generators, sort(distances.begin(), distances.end(), ComparePairsBySecondElement()); // Find the global range of distances from the generator. - double rmin = allReduce((distances.size() > 0 ? distances.front().second : DBL_MAX), SPHERAL_MPI_MIN); - double rmax = allReduce((distances.size() > 0 ? distances.back().second : 0.0), SPHERAL_MPI_MAX); + double rmin = allReduce((distances.size() > 0 ? distances.front().second : DBL_MAX), SPHERAL_OP_MIN); + double rmax = allReduce((distances.size() > 0 ? distances.back().second : 0.0), SPHERAL_OP_MAX); // Bisect for the appropriate radius to reject nodes. const double worktol = max(1.0e-10, 0.01*targetWork); @@ -883,7 +883,7 @@ cullGeneratorNodesByWork(const vector& generators, localWork += nodes[itr->first].work; ++itr; } - currentWork = allReduce(localWork, SPHERAL_MPI_SUM); + currentWork = allReduce(localWork, SPHERAL_OP_SUM); if (currentWork < targetWork) { rmin = rreject; } else { diff --git a/src/Distributed/allReduce.hh b/src/Distributed/allReduce.hh index e61d952fe..8f67f5de1 100644 --- a/src/Distributed/allReduce.hh +++ b/src/Distributed/allReduce.hh @@ -20,12 +20,12 @@ namespace Spheral { -#define SPHERAL_MPI_MIN MPI_MIN -#define SPHERAL_MPI_MAX MPI_MAX -#define SPHERAL_MPI_SUM MPI_SUM -#define SPHERAL_MPI_PROD MPI_PROD -#define SPHERAL_MPI_LAND MPI_LAND -#define SPHERAL_MPI_LOR MPI_LOR +#define SPHERAL_OP_MIN MPI_MIN +#define SPHERAL_OP_MAX MPI_MAX +#define SPHERAL_OP_SUM MPI_SUM +#define SPHERAL_OP_PROD MPI_PROD +#define SPHERAL_OP_LAND MPI_LAND +#define SPHERAL_OP_LOR MPI_LOR template Value @@ -54,12 +54,12 @@ scan(const Value& value, const MPI_Op op, const MPI_Comm comm = Communicator::co namespace Spheral { -#define SPHERAL_MPI_MIN 1 -#define SPHERAL_MPI_MAX 2 -#define SPHERAL_MPI_SUM 3 -#define SPHERAL_MPI_PROD 4 -#define SPHERAL_MPI_LAND 5 -#define SPHERAL_MPI_LOR 6 +#define SPHERAL_OP_MIN 1 +#define SPHERAL_OP_MAX 2 +#define SPHERAL_OP_SUM 3 +#define SPHERAL_OP_PROD 4 +#define SPHERAL_OP_LAND 5 +#define SPHERAL_OP_LOR 6 template Value diff --git a/src/Field/FieldInline.hh b/src/Field/FieldInline.hh index 12a49a43d..7a9e6042d 100644 --- a/src/Field/FieldInline.hh +++ b/src/Field/FieldInline.hh @@ -710,7 +710,7 @@ inline DataType Field:: sumElements() const { - return allReduce(this->localSumElements(), SPHERAL_MPI_SUM); + return allReduce(this->localSumElements(), SPHERAL_OP_SUM); } //------------------------------------------------------------------------------ @@ -721,7 +721,7 @@ inline DataType Field:: min() const { - return allReduce(this->localMin(), SPHERAL_MPI_MIN); + return allReduce(this->localMin(), SPHERAL_OP_MIN); } //------------------------------------------------------------------------------ @@ -732,7 +732,7 @@ inline DataType Field:: max() const { - return allReduce(this->localMax(), SPHERAL_MPI_MAX); + return allReduce(this->localMax(), SPHERAL_OP_MAX); } //------------------------------------------------------------------------------ diff --git a/src/Field/FieldListInline.hh b/src/Field/FieldListInline.hh index 4909078a0..788850642 100644 --- a/src/Field/FieldListInline.hh +++ b/src/Field/FieldListInline.hh @@ -1349,7 +1349,7 @@ inline DataType FieldList:: sumElements() const { - return allReduce(this->localSumElements(), SPHERAL_MPI_SUM); + return allReduce(this->localSumElements(), SPHERAL_OP_SUM); } //------------------------------------------------------------------------------ @@ -1360,7 +1360,7 @@ inline DataType FieldList:: min() const { - return allReduce(this->localMin(), SPHERAL_MPI_MIN); + return allReduce(this->localMin(), SPHERAL_OP_MIN); } //------------------------------------------------------------------------------ @@ -1371,7 +1371,7 @@ inline DataType FieldList:: max() const { - return allReduce(this->localMax(), SPHERAL_MPI_MAX); + return allReduce(this->localMax(), SPHERAL_OP_MAX); } //------------------------------------------------------------------------------ diff --git a/src/FieldOperations/binFieldList2Lattice.cc b/src/FieldOperations/binFieldList2Lattice.cc index 41962341f..d76058189 100644 --- a/src/FieldOperations/binFieldList2Lattice.cc +++ b/src/FieldOperations/binFieldList2Lattice.cc @@ -227,8 +227,8 @@ binFieldList2Lattice(const FieldList& fieldList, result = vector(ntotal, DataTypeTraits::zero()); // Check that everyone agrees about the size. - CHECK(bufSize == allReduce(bufSize, SPHERAL_MPI_MIN)); - CHECK(bufSize == allReduce(bufSize, SPHERAL_MPI_MAX)); + CHECK(bufSize == allReduce(bufSize, SPHERAL_OP_MIN)); + CHECK(bufSize == allReduce(bufSize, SPHERAL_OP_MAX)); // Sum up everyone's contribution. for (auto sendProc = 0u; sendProc != numProcs; ++sendProc) { @@ -315,7 +315,7 @@ binFieldList2Lattice(const FieldList& fieldList, BEGIN_CONTRACT_SCOPE // Check that everyone agrees about the size. { - CHECK(bufSize == allReduce(bufSize, SPHERAL_MPI_MIN)); + CHECK(bufSize == allReduce(bufSize, SPHERAL_OP_MIN)); } END_CONTRACT_SCOPE diff --git a/src/Gravity/NBodyGravity.cc b/src/Gravity/NBodyGravity.cc index ce5fa3668..27848085f 100644 --- a/src/Gravity/NBodyGravity.cc +++ b/src/Gravity/NBodyGravity.cc @@ -219,9 +219,9 @@ evaluateDerivatives(const typename Dimension::Scalar /*time*/, } } - mExtraEnergy = allReduce(mExtraEnergy, SPHERAL_MPI_SUM); - mOldMaxAcceleration = allReduce(mOldMaxAcceleration, SPHERAL_MPI_MAX); - mOldMaxVelocity = allReduce(mOldMaxVelocity, SPHERAL_MPI_MAX); + mExtraEnergy = allReduce(mExtraEnergy, SPHERAL_OP_SUM); + mOldMaxAcceleration = allReduce(mOldMaxAcceleration, SPHERAL_OP_MAX); + mOldMaxVelocity = allReduce(mOldMaxVelocity, SPHERAL_OP_MAX); #ifdef USE_MPI // Wait until all our sends are complete. diff --git a/src/Gravity/PolyGravity.cc b/src/Gravity/PolyGravity.cc index 36127bd66..9879b1c8e 100644 --- a/src/Gravity/PolyGravity.cc +++ b/src/Gravity/PolyGravity.cc @@ -215,7 +215,7 @@ evaluateDerivatives(const typename Dimension::Scalar /*time*/, mDtMinAcc = min(mDtMinAcc, sqrt(hi/ai.magnitude())); // Similar to acceleration constraint from TreeGravity } } - mExtraEnergy = allReduce(mExtraEnergy, SPHERAL_MPI_SUM); + mExtraEnergy = allReduce(mExtraEnergy, SPHERAL_OP_SUM); } //------------------------------------------------------------------------------ diff --git a/src/Gravity/Tree.cc b/src/Gravity/Tree.cc index a95f88da0..23b1aa064 100644 --- a/src/Gravity/Tree.cc +++ b/src/Gravity/Tree.cc @@ -74,7 +74,7 @@ dumpTree(const bool globalTree) const { const unsigned rank = Process::getRank(); #endif unsigned nlevels = mLevels.size(); - if (globalTree) nlevels = allReduce(nlevels, SPHERAL_MPI_MAX); + if (globalTree) nlevels = allReduce(nlevels, SPHERAL_OP_MAX); ss << "Tree : nlevels = " << nlevels << "\n"; for (unsigned ilevel = 0; ilevel != nlevels; ++ilevel) { @@ -151,7 +151,7 @@ dumpTreeStatistics(const bool globalTree) const { const unsigned rank = Process::getRank(); #endif unsigned nlevels = mLevels.size(); - if (globalTree) nlevels = allReduce(nlevels, SPHERAL_MPI_MAX); + if (globalTree) nlevels = allReduce(nlevels, SPHERAL_OP_MAX); ss << "Tree : nlevels = " << nlevels << "\n"; for (unsigned ilevel = 0; ilevel != nlevels; ++ilevel) { diff --git a/src/Gravity/TreeGravity.cc b/src/Gravity/TreeGravity.cc index 28c4a3cf7..df3cfa0c8 100644 --- a/src/Gravity/TreeGravity.cc +++ b/src/Gravity/TreeGravity.cc @@ -321,7 +321,7 @@ evaluateDerivatives(const typename Dimension::Scalar /*time*/, } #ifdef USE_MPI - mExtraEnergy = allReduce(mExtraEnergy, SPHERAL_MPI_SUM); + mExtraEnergy = allReduce(mExtraEnergy, SPHERAL_OP_SUM); // Wait until all our sends are complete. if (not sendRequests.empty()) { @@ -556,7 +556,7 @@ dumpTree(const bool globalTree) const { const unsigned rank = Process::getRank(); #endif unsigned nlevels = mTree.size(); - if (globalTree) nlevels = allReduce(nlevels, SPHERAL_MPI_MAX); + if (globalTree) nlevels = allReduce(nlevels, SPHERAL_OP_MAX); ss << "Tree : nlevels = " << nlevels << "\n"; for (unsigned ilevel = 0; ilevel != nlevels; ++ilevel) { @@ -633,7 +633,7 @@ dumpTreeStatistics(const bool globalTree) const { const unsigned rank = Process::getRank(); #endif unsigned nlevels = mTree.size(); - if (globalTree) nlevels = allReduce(nlevels, SPHERAL_MPI_MAX); + if (globalTree) nlevels = allReduce(nlevels, SPHERAL_OP_MAX); ss << "Tree : nlevels = " << nlevels << "\n"; for (unsigned ilevel = 0; ilevel != nlevels; ++ilevel) { diff --git a/src/Hydro/VoronoiHourglassControl.cc b/src/Hydro/VoronoiHourglassControl.cc index 0c5adee55..cff10651f 100644 --- a/src/Hydro/VoronoiHourglassControl.cc +++ b/src/Hydro/VoronoiHourglassControl.cc @@ -247,8 +247,8 @@ finalize(const typename Dimension::Scalar time, } } CHECK(rhoZones.size() == mesh.numZones()); - rhoMin = allReduce(rhoMin, SPHERAL_MPI_MIN); - rhoMax = allReduce(rhoMax, SPHERAL_MPI_MAX); + rhoMin = allReduce(rhoMin, SPHERAL_OP_MIN); + rhoMax = allReduce(rhoMax, SPHERAL_OP_MAX); // Compute the CRKSPH limited gradient of the density if we're doing first order. if (mOrder > 0) { diff --git a/src/Integrator/Integrator.cc b/src/Integrator/Integrator.cc index 042e33fc4..4c95d581c 100644 --- a/src/Integrator/Integrator.cc +++ b/src/Integrator/Integrator.cc @@ -228,7 +228,7 @@ selectDt(const typename Dimension::Scalar dtMin, dt.first >= dtMin and dt.first <= dtMax); // In the parallel case we need to find the minimum timestep across all processors. - const auto globalDt = allReduce(dt.first, SPHERAL_MPI_MIN); + const auto globalDt = allReduce(dt.first, SPHERAL_OP_MIN); // Are we verbose? if (dt.first == globalDt and diff --git a/src/KernelIntegrator/FlatConnectivity.cc b/src/KernelIntegrator/FlatConnectivity.cc index fba50f751..1633c86be 100644 --- a/src/KernelIntegrator/FlatConnectivity.cc +++ b/src/KernelIntegrator/FlatConnectivity.cc @@ -326,11 +326,11 @@ computeGlobalIndices(const DataBase& dataBase, VERIFY(numInternalNodesDB == mNumInternalLocalNodes); // Get global indices manually - int globalScan = scan(mNumInternalLocalNodes, SPHERAL_MPI_SUM); + int globalScan = scan(mNumInternalLocalNodes, SPHERAL_OP_SUM); VERIFY(globalScan >= mNumInternalLocalNodes); mFirstGlobalIndex = globalScan - mNumInternalLocalNodes; mLastGlobalIndex = globalScan - 1; - mNumGlobalNodes = allReduce(mNumInternalLocalNodes, SPHERAL_MPI_SUM); + mNumGlobalNodes = allReduce(mNumInternalLocalNodes, SPHERAL_OP_SUM); VERIFY(mNumGlobalNodes >= mNumInternalLocalNodes); VERIFY(mNumGlobalNodes == numGlobalNodesDB); // std::cout << Process::getRank() << "\t" << mNumInternalLocalNodes << "\t" << mNumGlobalNodes << "\t" << mFirstGlobalIndex << "\t" << mLastGlobalIndex << std::endl; diff --git a/src/Mesh/LineMesh.cc b/src/Mesh/LineMesh.cc index 37de442e8..92f0555dc 100644 --- a/src/Mesh/LineMesh.cc +++ b/src/Mesh/LineMesh.cc @@ -56,7 +56,7 @@ reconstructInternal(const vector >::Vector>& localGenerators, const Mesh >::Vector& xmax) { // Is there anything to do? - if (allReduce(unsigned(localGenerators.size()), SPHERAL_MPI_SUM) == 0) return; + if (allReduce(unsigned(localGenerators.size()), SPHERAL_OP_SUM) == 0) return; // Pre-conditions. @@ -284,8 +284,8 @@ boundingSurface() const { xmin = std::min(xmin, mNodePositions[i].x()); xmax = std::max(xmax, mNodePositions[i].x()); } - xmin = allReduce(xmin, SPHERAL_MPI_MIN); - xmax = allReduce(xmax, SPHERAL_MPI_MAX); + xmin = allReduce(xmin, SPHERAL_OP_MIN); + xmax = allReduce(xmax, SPHERAL_OP_MAX); return FacetedVolume(Vector(0.5*(xmin + xmax)), 0.5*(xmax - xmin)); } diff --git a/src/Mesh/Mesh.cc b/src/Mesh/Mesh.cc index 1b1bcd949..17ef07fc9 100644 --- a/src/Mesh/Mesh.cc +++ b/src/Mesh/Mesh.cc @@ -54,7 +54,7 @@ reduceToMaxString(const string& x, const unsigned rank, const unsigned numDomains) { unsigned badRank = allReduce((x.size() == 0 ? numDomains : rank), - SPHERAL_MPI_MIN); + SPHERAL_OP_MIN); if (badRank == numDomains) { return ""; } else { @@ -473,7 +473,7 @@ removeZonesByMask(const vector& zoneMask) { removeElements(mSharedFaces, killDomains); // // Any pre-existing parallel info is now invalid. - // if (allReduce(mNeighborDomains.size(), SPHERAL_MPI_MAX) > 0) { + // if (allReduce(mNeighborDomains.size(), SPHERAL_OP_MAX) > 0) { // mNeighborDomains = vector(); // mSharedNodes = vector >(); // mSharedFaces = vector >(); @@ -748,8 +748,8 @@ generateDomainInfo() { // bit perfect consistency across processors. Vector boxInv; for (unsigned i = 0; i != Dimension::nDim; ++i) { - xmin(i) = allReduce(xmin(i) - dxhash, SPHERAL_MPI_MIN); - xmax(i) = allReduce(xmax(i) + dxhash, SPHERAL_MPI_MAX); + xmin(i) = allReduce(xmin(i) - dxhash, SPHERAL_OP_MIN); + xmax(i) = allReduce(xmax(i) + dxhash, SPHERAL_OP_MAX); boxInv(i) = safeInv(xmax(i) - xmin(i)); } @@ -1038,8 +1038,8 @@ generateParallelRind(vector& generators, // bit perfect consistency across processors. Vector boxInv; for (unsigned i = 0; i != Dimension::nDim; ++i) { - xmin(i) = allReduce(xmin(i) - dxhash, SPHERAL_MPI_MIN); - xmax(i) = allReduce(xmax(i) + dxhash, SPHERAL_MPI_MAX); + xmin(i) = allReduce(xmin(i) - dxhash, SPHERAL_OP_MIN); + xmax(i) = allReduce(xmax(i) + dxhash, SPHERAL_OP_MAX); boxInv(i) = safeInv(xmax(i) - xmin(i)); } @@ -1448,8 +1448,8 @@ boundingBox(typename Dimension::Vector& xmin, Spheral::boundingBox(mNodePositions, xmin, xmax); #ifdef USE_MPI for (unsigned i = 0; i != Dimension::nDim; ++i) { - xmin(i) = allReduce(xmin(i), SPHERAL_MPI_MIN); - xmax(i) = allReduce(xmax(i), SPHERAL_MPI_MAX); + xmin(i) = allReduce(xmin(i), SPHERAL_OP_MIN); + xmax(i) = allReduce(xmax(i), SPHERAL_OP_MAX); } #endif } diff --git a/src/Mesh/MeshInline.hh b/src/Mesh/MeshInline.hh index 476ac5cda..8d8f02857 100644 --- a/src/Mesh/MeshInline.hh +++ b/src/Mesh/MeshInline.hh @@ -369,7 +369,7 @@ minimumScale() const { result = std::min(result, (face.position() - zonePosition).magnitude2()); } } - result = allReduce(0.5*sqrt(result), SPHERAL_MPI_MIN); + result = allReduce(0.5*sqrt(result), SPHERAL_OP_MIN); // That's it. ENSURE(result > 0.0); @@ -389,7 +389,7 @@ minimumScale() const { result = std::min(result, std::abs(mNodePositions[mZones[i].mNodeIDs[0]].x() - mNodePositions[mZones[i].mNodeIDs[1]].x())); } - result = allReduce(0.5*result, SPHERAL_MPI_MIN); + result = allReduce(0.5*result, SPHERAL_OP_MIN); // That's it. ENSURE(result > 0.0); diff --git a/src/Neighbor/ConnectivityMap.cc b/src/Neighbor/ConnectivityMap.cc index df0f9190b..b6d89c49c 100644 --- a/src/Neighbor/ConnectivityMap.cc +++ b/src/Neighbor/ConnectivityMap.cc @@ -1117,10 +1117,10 @@ computeConnectivity() { } // { - // tpre = allReduce(unsigned(tpre), SPHERAL_MPI_SUM) / Process::getTotalNumberOfProcesses() / CLOCKS_PER_SEC; - // tmaster = allReduce(unsigned(tmaster), SPHERAL_MPI_SUM) / Process::getTotalNumberOfProcesses() / CLOCKS_PER_SEC; - // trefine = allReduce(unsigned(trefine), SPHERAL_MPI_SUM) / Process::getTotalNumberOfProcesses() / CLOCKS_PER_SEC; - // twalk = allReduce(unsigned(twalk), SPHERAL_MPI_SUM) / Process::getTotalNumberOfProcesses() / CLOCKS_PER_SEC; + // tpre = allReduce(unsigned(tpre), SPHERAL_OP_SUM) / Process::getTotalNumberOfProcesses() / CLOCKS_PER_SEC; + // tmaster = allReduce(unsigned(tmaster), SPHERAL_OP_SUM) / Process::getTotalNumberOfProcesses() / CLOCKS_PER_SEC; + // trefine = allReduce(unsigned(trefine), SPHERAL_OP_SUM) / Process::getTotalNumberOfProcesses() / CLOCKS_PER_SEC; + // twalk = allReduce(unsigned(twalk), SPHERAL_OP_SUM) / Process::getTotalNumberOfProcesses() / CLOCKS_PER_SEC; // if (Process::getRank() == 0) { // std::cerr << "ConnectivityMap timings (pre, master, refine, walk) = " << tpre << " " << tmaster << " " << trefine << " " << twalk << std::endl; // } diff --git a/src/Neighbor/TreeNeighbor.cc b/src/Neighbor/TreeNeighbor.cc index c19179a1d..f248d113a 100644 --- a/src/Neighbor/TreeNeighbor.cc +++ b/src/Neighbor/TreeNeighbor.cc @@ -525,7 +525,7 @@ dumpTree(const Tree& tree, std::stringstream ss; CellKey ix, iy, iz; unsigned nlevels = tree.size(); - if (globalTree) nlevels = allReduce(nlevels, SPHERAL_MPI_MAX); + if (globalTree) nlevels = allReduce(nlevels, SPHERAL_OP_MAX); ss << "Tree : nlevels = " << nlevels << "\n"; for (unsigned ilevel = 0; ilevel != nlevels; ++ilevel) { @@ -607,7 +607,7 @@ dumpTreeStatistics(const Tree& tree, const bool globalTree) const { std::stringstream ss; unsigned nlevels = tree.size(); - if (globalTree) nlevels = allReduce(nlevels, SPHERAL_MPI_MAX); + if (globalTree) nlevels = allReduce(nlevels, SPHERAL_OP_MAX); ss << "Tree : nlevels = " << nlevels << "\n"; for (unsigned ilevel = 0; ilevel != nlevels; ++ilevel) { diff --git a/src/NodeGenerators/centroidalRelaxNodesImpl.cc b/src/NodeGenerators/centroidalRelaxNodesImpl.cc index f64772a5d..3fbb47810 100644 --- a/src/NodeGenerators/centroidalRelaxNodesImpl.cc +++ b/src/NodeGenerators/centroidalRelaxNodesImpl.cc @@ -119,8 +119,8 @@ centroidalRelaxNodesImpl(DataBase& db, ntot += n; for (auto i = 0U; i != n; ++i) avgneighbors += cm.numNeighborsForNode(nodeListi, i); } - ntot = allReduce(ntot, SPHERAL_MPI_SUM); - avgneighbors = allReduce(avgneighbors, SPHERAL_MPI_SUM)/ntot; + ntot = allReduce(ntot, SPHERAL_OP_SUM); + avgneighbors = allReduce(avgneighbors, SPHERAL_OP_SUM)/ntot; if (Process::getRank() == 0) cerr << "Avergage number of neighbors per node: " << avgneighbors << " " << ntot << endl; } // BLAGO @@ -185,9 +185,9 @@ centroidalRelaxNodesImpl(DataBase& db, if (vol(nodeListi, i) > 0.0) mass(nodeListi, i) = rhof(nodeListi,i)*vol(nodeListi,i); } } - avgdelta = (allReduce(avgdelta, SPHERAL_MPI_SUM)/ - allReduce(db.numInternalNodes(), SPHERAL_MPI_SUM)); - maxdelta = allReduce(maxdelta, SPHERAL_MPI_MAX); + avgdelta = (allReduce(avgdelta, SPHERAL_OP_SUM)/ + allReduce(db.numInternalNodes(), SPHERAL_OP_SUM)); + maxdelta = allReduce(maxdelta, SPHERAL_OP_MAX); if (Process::getRank() == 0) cerr << "centroidalRelaxNodes iteration " << iter << ", avg delta frac " << avgdelta << ", max delta frac " << maxdelta diff --git a/src/NodeGenerators/compactFacetedVolumes.cc b/src/NodeGenerators/compactFacetedVolumes.cc index d48c57f21..6c3e5ea26 100644 --- a/src/NodeGenerators/compactFacetedVolumes.cc +++ b/src/NodeGenerators/compactFacetedVolumes.cc @@ -40,7 +40,7 @@ unsigned compactFacetedVolumes(std::vector& s // Only proceed if there's work to do! int flagmax = *max_element(flags.begin(), flags.end()); - if (allReduce(flagmax, SPHERAL_MPI_MAX) != 2) return 0; + if (allReduce(flagmax, SPHERAL_OP_MAX) != 2) return 0; // Carve up the shapes range in parallel. // const size_t ndomain0 = nshapes/nprocs; @@ -276,9 +276,9 @@ unsigned compactFacetedVolumes(std::vector& s // CHECK(bufitr == buffer.end()); // } // } - // maxoverlap = allReduce(maxoverlap, SPHERAL_MPI_MAX); + // maxoverlap = allReduce(maxoverlap, SPHERAL_OP_MAX); // #endif - // double sumdisp = allReduce(std::accumulate(displacements.begin(), displacements.end(), 0.0, [](const double prior, const Vector& elemval) { return prior + elemval.magnitude(); }), SPHERAL_MPI_SUM); + // double sumdisp = allReduce(std::accumulate(displacements.begin(), displacements.end(), 0.0, [](const double prior, const Vector& elemval) { return prior + elemval.magnitude(); }), SPHERAL_OP_SUM); // if (rank == 0) { // cout << " Iteration " << iter // << ", maxoverlap " << maxoverlap @@ -291,7 +291,7 @@ unsigned compactFacetedVolumes(std::vector& s } // end of iteration } - iter = allReduce(iter, SPHERAL_MPI_MAX); + iter = allReduce(iter, SPHERAL_OP_MAX); // Any shapes we were unable to disentangle turn back to inactive, otherwise set the successful // survivors to flag=1. diff --git a/src/NodeGenerators/fillFacetedVolume.cc b/src/NodeGenerators/fillFacetedVolume.cc index d81282b7d..9c3362f5c 100644 --- a/src/NodeGenerators/fillFacetedVolume.cc +++ b/src/NodeGenerators/fillFacetedVolume.cc @@ -220,7 +220,7 @@ fillFacetedVolume10(const Dim<3>::FacetedVolume& outerBoundary0, } // If we didn't find anything, fall back to sampling on the surface. - if (allReduce(result.size(), SPHERAL_MPI_SUM) == 0U) { + if (allReduce(result.size(), SPHERAL_OP_SUM) == 0U) { if (Process::getRank() == 0) { cerr << "Falling back to surface points..." << endl; const size_t nexpect = size_t(std::max(1, std::min(int(verts.size()), int(outerBoundary.volume()/(dx*dx*dx) + 0.5)))); diff --git a/src/NodeGenerators/generateCylDistributionFromRZ.cc b/src/NodeGenerators/generateCylDistributionFromRZ.cc index ec2851cd6..4592de46a 100644 --- a/src/NodeGenerators/generateCylDistributionFromRZ.cc +++ b/src/NodeGenerators/generateCylDistributionFromRZ.cc @@ -139,7 +139,7 @@ generateCylDistributionFromRZ(vector& x, (int)globalIDs.size() == ndomain and (int)H.size() == ndomain); for (int ikey = 0; ikey != nextra; ++ikey) VERIFY((int)extraFields[ikey].size() == ndomain); - int nglobal = allReduce(x.size(), SPHERAL_MPI_SUM); + int nglobal = allReduce(x.size(), SPHERAL_OP_SUM); VERIFY(nglobal == ntot); } diff --git a/src/NodeGenerators/relaxNodeDistribution.cc b/src/NodeGenerators/relaxNodeDistribution.cc index fe8eb7480..8c26079c2 100644 --- a/src/NodeGenerators/relaxNodeDistribution.cc +++ b/src/NodeGenerators/relaxNodeDistribution.cc @@ -139,7 +139,7 @@ relaxNodeDistribution(DataBase& dataBase, ++k; } } - Msum = allReduce(Msum, SPHERAL_MPI_SUM); + Msum = allReduce(Msum, SPHERAL_OP_SUM); // If needed, rescale the masses. if (targetMass > 0.0) { diff --git a/src/RK/computeVoronoiVolume.cc b/src/RK/computeVoronoiVolume.cc index 12ccbaf3e..2c37d3305 100644 --- a/src/RK/computeVoronoiVolume.cc +++ b/src/RK/computeVoronoiVolume.cc @@ -358,7 +358,7 @@ computeVoronoiVolume(const FieldList& pos const auto numGens = position.numNodes(); const auto numNodeLists = position.size(); - const auto numGensGlobal = allReduce(numGens, SPHERAL_MPI_SUM); + const auto numGensGlobal = allReduce(numGens, SPHERAL_OP_SUM); const auto haveFacetedBoundaries = facetedBoundaries.size() == numNodeLists; const auto haveWeights = weight.size() == numNodeLists; const auto haveDamage = false; // damage.size() == numNodeLists; // Suspending the idea of forcing surface based on damage diff --git a/src/Utilities/globalBoundingVolumes.cc b/src/Utilities/globalBoundingVolumes.cc index a90493dc9..01890f9d0 100644 --- a/src/Utilities/globalBoundingVolumes.cc +++ b/src/Utilities/globalBoundingVolumes.cc @@ -103,8 +103,8 @@ globalBoundingBox(const Field& positions, // Now find the global bounds across all processors. for (unsigned i = 0; i != Dimension::nDim; ++i) { - xmin(i) = allReduce(xmin(i), SPHERAL_MPI_MIN); - xmax(i) = allReduce(xmax(i), SPHERAL_MPI_MAX); + xmin(i) = allReduce(xmin(i), SPHERAL_OP_MIN); + xmax(i) = allReduce(xmax(i), SPHERAL_OP_MAX); } } @@ -134,8 +134,8 @@ globalBoundingBox(const FieldList& positi // Now find the global bounds across all processors. for (unsigned i = 0; i != Dimension::nDim; ++i) { - xmin(i) = allReduce(xmin(i), SPHERAL_MPI_MIN); - xmax(i) = allReduce(xmax(i), SPHERAL_MPI_MAX); + xmin(i) = allReduce(xmin(i), SPHERAL_OP_MIN); + xmax(i) = allReduce(xmax(i), SPHERAL_OP_MAX); } } diff --git a/src/Utilities/iterateIdealH.cc b/src/Utilities/iterateIdealH.cc index e0b941da7..612ca3b82 100644 --- a/src/Utilities/iterateIdealH.cc +++ b/src/Utilities/iterateIdealH.cc @@ -275,7 +275,7 @@ iterateIdealH(DataBase& dataBase, H.assignFields(H1); // Globally reduce the max H change. - maxDeltaH = allReduce(maxDeltaH, SPHERAL_MPI_MAX); + maxDeltaH = allReduce(maxDeltaH, SPHERAL_OP_MAX); // Output the statitics. if (Process::getRank() == 0) diff --git a/src/Utilities/nodeOrdering.cc b/src/Utilities/nodeOrdering.cc index 5fdc6fcb4..12ad45f83 100644 --- a/src/Utilities/nodeOrdering.cc +++ b/src/Utilities/nodeOrdering.cc @@ -68,7 +68,7 @@ nodeOrdering(const FieldList& criteria) { // Find the total number of nodes. const int numLocalNodes = sortedList.size(); - int numGlobalNodes = allReduce(numLocalNodes, SPHERAL_MPI_SUM); + int numGlobalNodes = allReduce(numLocalNodes, SPHERAL_OP_SUM); // Iterate over the local nodes in order. int iLocal = 0; @@ -78,12 +78,12 @@ nodeOrdering(const FieldList& criteria) { if (iLocal < numLocalNodes) localKey = std::get<2>(sortedList[iLocal]); // Find the next key globally. - Key globalKey = allReduce(localKey, SPHERAL_MPI_MIN); + Key globalKey = allReduce(localKey, SPHERAL_OP_MIN); // If we have the next index, check for duplicates on other domains. int minProcID = numProcs + 1; if (localKey == globalKey) minProcID = procID; - minProcID = allReduce(minProcID, SPHERAL_MPI_MIN); + minProcID = allReduce(minProcID, SPHERAL_OP_MIN); // Are we the next global key? if (localKey == globalKey and procID == minProcID) { @@ -113,7 +113,7 @@ nodeOrdering(const FieldList& criteria) { for (typename FieldList::const_iterator itr = result.begin(); itr != result.end(); ++itr, ++iNodeList) countGlobal += count((**itr).internalBegin(), (**itr).internalEnd(), iGlobal); - countGlobal = allReduce(countGlobal, SPHERAL_MPI_SUM); + countGlobal = allReduce(countGlobal, SPHERAL_OP_SUM); ENSURE(countGlobal == 1); } } From 8da88a56299f04a071d2c49ac0bd2a3a50bb603b Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Wed, 3 Jul 2024 12:19:24 -0700 Subject: [PATCH 05/14] Added constexpr to allreduce routines --- src/Distributed/allReduce.hh | 37 ++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/src/Distributed/allReduce.hh b/src/Distributed/allReduce.hh index 8f67f5de1..97c5743d9 100644 --- a/src/Distributed/allReduce.hh +++ b/src/Distributed/allReduce.hh @@ -9,16 +9,17 @@ #define __Spheral_allReduce__ #include "Utilities/DataTypeTraits.hh" +#include "Communicator.hh" #ifdef USE_MPI -//------------------------------------------------------------------------------ -// MPI version -//------------------------------------------------------------------------------ - #include -#include "Communicator.hh" +#endif namespace Spheral { +#ifdef USE_MPI +//------------------------------------------------------------------------------ +// MPI version +//------------------------------------------------------------------------------ #define SPHERAL_OP_MIN MPI_MIN #define SPHERAL_OP_MAX MPI_MAX @@ -27,33 +28,34 @@ namespace Spheral { #define SPHERAL_OP_LAND MPI_LAND #define SPHERAL_OP_LOR MPI_LOR +constexpr template Value -allReduce(const Value& value, const MPI_Op op, const MPI_Comm comm = Communicator::communicator()) { +allReduce(const Value& value, const MPI_Op op, + const Communicator comm = Communicator::communicator()) { Value tmp = value; Value result; - MPI_Allreduce(&tmp, &result, 1, DataTypeTraits::MpiDataType(), op, comm); + MPI_Allreduce(&tmp, &result, 1, + DataTypeTraits::MpiDataType(), op, comm); return result; } +constexpr template Value -scan(const Value& value, const MPI_Op op, const MPI_Comm comm = Communicator::communicator()) { +scan(const Value& value, const MPI_Op op, + const Communicator comm = Communicator::communicator()) { Value tmp = value; Value result; MPI_Scan(&tmp, &result, 1, DataTypeTraits::MpiDataType(), op, comm); return result; } - -} #else //------------------------------------------------------------------------------ // Non-MPI version //------------------------------------------------------------------------------ -namespace Spheral { - #define SPHERAL_OP_MIN 1 #define SPHERAL_OP_MAX 2 #define SPHERAL_OP_SUM 3 @@ -61,20 +63,23 @@ namespace Spheral { #define SPHERAL_OP_LAND 5 #define SPHERAL_OP_LOR 6 +constexpr template Value -allReduce(const Value& value, const int /*op*/, const int comm = 0) { +allReduce(const Value& value, const int /*op*/, + const Communicator comm = Communicator::communicator()) { return value; } +constexpr template Value -scan(const Value& value, const int /*op*/, const int comm = 0) { +scan(const Value& value, const int /*op*/, + const Communicator comm = Communicator::communicator()) { return value; } -} - #endif +} #endif From b3374ed46c7c421554392e3e5fdc6a3d0b0c4eeb Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Wed, 3 Jul 2024 13:15:23 -0700 Subject: [PATCH 06/14] Move constexpr to the correct place --- src/Distributed/allReduce.hh | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/Distributed/allReduce.hh b/src/Distributed/allReduce.hh index 97c5743d9..0437d3e69 100644 --- a/src/Distributed/allReduce.hh +++ b/src/Distributed/allReduce.hh @@ -28,9 +28,8 @@ namespace Spheral { #define SPHERAL_OP_LAND MPI_LAND #define SPHERAL_OP_LOR MPI_LOR -constexpr template -Value +constexpr Value allReduce(const Value& value, const MPI_Op op, const Communicator comm = Communicator::communicator()) { Value tmp = value; @@ -40,9 +39,8 @@ allReduce(const Value& value, const MPI_Op op, return result; } -constexpr template -Value +constexpr Value scan(const Value& value, const MPI_Op op, const Communicator comm = Communicator::communicator()) { Value tmp = value; @@ -63,17 +61,15 @@ scan(const Value& value, const MPI_Op op, #define SPHERAL_OP_LAND 5 #define SPHERAL_OP_LOR 6 -constexpr template -Value +constexpr Value allReduce(const Value& value, const int /*op*/, const Communicator comm = Communicator::communicator()) { return value; } -constexpr template -Value +constexpr Value scan(const Value& value, const int /*op*/, const Communicator comm = Communicator::communicator()) { return value; From 4170e5d3dd029ed1743fe8227b812f9f4c689af3 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Wed, 3 Jul 2024 13:33:35 -0700 Subject: [PATCH 07/14] Fixed communicator type for allreduce --- src/Distributed/allReduce.hh | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/Distributed/allReduce.hh b/src/Distributed/allReduce.hh index 0437d3e69..8bdef0ff7 100644 --- a/src/Distributed/allReduce.hh +++ b/src/Distributed/allReduce.hh @@ -31,7 +31,7 @@ namespace Spheral { template constexpr Value allReduce(const Value& value, const MPI_Op op, - const Communicator comm = Communicator::communicator()) { + const MPI_Comm comm = Communicator::communicator()) { Value tmp = value; Value result; MPI_Allreduce(&tmp, &result, 1, @@ -42,7 +42,7 @@ allReduce(const Value& value, const MPI_Op op, template constexpr Value scan(const Value& value, const MPI_Op op, - const Communicator comm = Communicator::communicator()) { + const MPI_Comm comm = Communicator::communicator()) { Value tmp = value; Value result; MPI_Scan(&tmp, &result, 1, DataTypeTraits::MpiDataType(), op, comm); @@ -63,15 +63,13 @@ scan(const Value& value, const MPI_Op op, template constexpr Value -allReduce(const Value& value, const int /*op*/, - const Communicator comm = Communicator::communicator()) { +allReduce(const Value& value, const int /*op*/, const int comm = 0) { return value; } template constexpr Value -scan(const Value& value, const int /*op*/, - const Communicator comm = Communicator::communicator()) { +scan(const Value& value, const int /*op*/, const int comm = 0) { return value; } From 33a0973d972a1bee035afe0731e1b86886552bc5 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Mon, 8 Jul 2024 11:21:21 -0700 Subject: [PATCH 08/14] Fixed some allreduce calls that were missed earlier --- src/Damage/computeFragmentField.cc | 51 ++++++---------------------- src/Distributed/Process.cc | 2 +- src/Utilities/globalNodeIDsInline.hh | 11 ++---- 3 files changed, 14 insertions(+), 50 deletions(-) diff --git a/src/Damage/computeFragmentField.cc b/src/Damage/computeFragmentField.cc index 6d8e7d806..193a1842b 100644 --- a/src/Damage/computeFragmentField.cc +++ b/src/Damage/computeFragmentField.cc @@ -71,9 +71,7 @@ globalReduceToUniqueElements(vector& x) { BEGIN_CONTRACT_SCOPE { int tmp = x.size(); - int sum; - MPI_Allreduce(&tmp, &sum, 1, MPI_INT, MPI_SUM, Communicator::communicator()); - ENSURE(sum == (int)x.size()*numProcs); + ENSURE(allReduce(tmp, SPHERAL_OP_SUM) == tmp*numProcs); } END_CONTRACT_SCOPE #endif @@ -132,12 +130,7 @@ computeFragmentField(const NodeList& nodes, gIDs.end()); int maxGlobalID = 0; if (maxGlobalItr != gIDs.end()) maxGlobalID = *maxGlobalItr; -#ifdef USE_MPI - { - int tmp = maxGlobalID; - MPI_Allreduce(&tmp, &maxGlobalID, 1, MPI_INT, MPI_MAX, Communicator::communicator()); - } -#endif + maxGlobalID = allReduce(maxGlobalID, SPHERAL_OP_MAX); maxGlobalID += 1; CHECK(maxGlobalID >= numGlobalNodesRemaining); @@ -161,12 +154,7 @@ computeFragmentField(const NodeList& nodes, } // Reduce the count of remaining nodes by the number of dust nodes. -#ifdef USE_MPI - { - int tmp = numDustNodes; - MPI_Allreduce(&tmp, &numDustNodes, 1, MPI_INT, MPI_SUM, Communicator::communicator()); - } -#endif + numDustNodes = allReduce(numDustNodes, SPHERAL_OP_SUM); CHECK(numDustNodes >= 0 && numDustNodes <= numGlobalNodesRemaining); numGlobalNodesRemaining -= numDustNodes; CHECK(numGlobalNodesRemaining >= 0); @@ -184,12 +172,7 @@ computeFragmentField(const NodeList& nodes, globalNodesRemaining.end()); int globalMinID = maxGlobalID; if (globalMinItr != globalNodesRemaining.end()) globalMinID = *globalMinItr; -#ifdef USE_MPI - { - int tmp = globalMinID; - MPI_Allreduce(&tmp, &globalMinID, 1, MPI_INT, MPI_MIN, Communicator::communicator()); - } -#endif + globalMinID = allReduce(globalMinID, SPHERAL_OP_MIN); CHECK(globalMinID < maxGlobalID); // Is this node on this domain? @@ -204,9 +187,7 @@ computeFragmentField(const NodeList& nodes, BEGIN_CONTRACT_SCOPE { int tmp = localNode ? 1 : 0; - int sum; - MPI_Allreduce(&tmp, &sum, 1, MPI_INT, MPI_SUM, Communicator::communicator()); - CHECK(sum == 1); + CHECK(allReduce(tmp, SPHERAL_OP_SUM) == 1); } END_CONTRACT_SCOPE int tmp = numProcs; @@ -216,13 +197,11 @@ computeFragmentField(const NodeList& nodes, tmp = procID; CHECK(result(ilocal) == maxGlobalID); } - MPI_Allreduce(&tmp, &nodeDomain, 1, MPI_INT, MPI_MIN, Communicator::communicator()); + nodeDomain = allReduce(tmp, SPHERAL_OP_MIN); CHECK(nodeDomain >= 0 && nodeDomain < numProcs); BEGIN_CONTRACT_SCOPE { - int tmp; - MPI_Allreduce(&nodeDomain, &tmp, 1, MPI_INT, MPI_SUM, Communicator::communicator()); - CHECK(tmp == numProcs*nodeDomain); + CHECK(allReduce(nodeDomain, SPHERAL_OP_SUM) == numProcs*nodeDomain); } END_CONTRACT_SCOPE #endif @@ -275,11 +254,8 @@ computeFragmentField(const NodeList& nodes, #ifdef USE_MPI BEGIN_CONTRACT_SCOPE { - int tmp; - MPI_Allreduce(&fragID, &tmp, 1, MPI_INT, MPI_SUM, Communicator::communicator()); - CHECK(tmp == numProcs*fragID); - MPI_Allreduce(&numFragments, &tmp, 1, MPI_INT, MPI_SUM, Communicator::communicator()); - CHECK(tmp == numProcs*numFragments); + CHECK(allReduce(fragID, SPHERAL_OP_SUM) == numProcs*fragID); + CHECK(allReduce(numFragments, SPHERAL_OP_SUM) == numProcs*numFragments); } END_CONTRACT_SCOPE #endif @@ -317,13 +293,7 @@ computeFragmentField(const NodeList& nodes, if (removeItr != globalNodesRemaining.end()) globalNodesRemaining.erase(removeItr); } - numGlobalNodesRemaining = globalNodesRemaining.size(); -#ifdef USE_MPI - { - int tmp = numGlobalNodesRemaining; - MPI_Allreduce(&tmp, &numGlobalNodesRemaining, 1, MPI_INT, MPI_SUM, Communicator::communicator()); - } -#endif + numGlobalNodesRemaining = allReduce(globalNodesRemaining.size(), SPHERAL_OP_SUM); BEGIN_CONTRACT_SCOPE { @@ -375,6 +345,7 @@ computeFragmentField(const NodeList& nodes, } } #ifdef USE_MPI + // LDO: Can the first reduce call be moved outside of the for loop? for (int i = 0; i != numFragments - 1; ++i) { double mtmp = mfrag[i]; Vector rtmp = rfrag[i]; diff --git a/src/Distributed/Process.cc b/src/Distributed/Process.cc index 3981a6703..63d3dc47e 100644 --- a/src/Distributed/Process.cc +++ b/src/Distributed/Process.cc @@ -8,7 +8,7 @@ #include #include "Process.hh" -#include "Distributed/Communicator.hh" +#include "Communicator.hh" #ifdef USE_MPI #include diff --git a/src/Utilities/globalNodeIDsInline.hh b/src/Utilities/globalNodeIDsInline.hh index 6f62acce2..cf8df747a 100644 --- a/src/Utilities/globalNodeIDsInline.hh +++ b/src/Utilities/globalNodeIDsInline.hh @@ -18,13 +18,13 @@ #include "Utilities/peanoHilbertOrderIndices.hh" #include "Utilities/KeyTraits.hh" #include "Utilities/DBC.hh" +#include "Distributed/allReduce.hh" #include #include #ifdef USE_MPI #include -#include "Distributed/Communicator.hh" #endif namespace Spheral { @@ -37,14 +37,7 @@ inline int numGlobalNodes(const NodeList& nodeList) { int localResult = nodeList.numInternalNodes(); -#ifdef USE_MPI - int result; - MPI_Allreduce(&localResult, &result, 1, MPI_INT, MPI_SUM, Communicator::communicator()); - CHECK(result >= localResult); -#else - const int result = localResult; -#endif - return result; + return allReduce(localResult, SPHERAL_OP_SUM); } //------------------------------------------------------------------------------ From 954859710565564bf0380837c7d5aa7b48eacde7 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Tue, 9 Jul 2024 09:41:09 -0700 Subject: [PATCH 09/14] Not sure why these are being caught when debug is turned off but hopefully this fixes it --- src/Damage/computeFragmentField.cc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Damage/computeFragmentField.cc b/src/Damage/computeFragmentField.cc index 193a1842b..a86e88882 100644 --- a/src/Damage/computeFragmentField.cc +++ b/src/Damage/computeFragmentField.cc @@ -70,8 +70,9 @@ globalReduceToUniqueElements(vector& x) { reduceToUniqueElements(x); BEGIN_CONTRACT_SCOPE { - int tmp = x.size(); - ENSURE(allReduce(tmp, SPHERAL_OP_SUM) == tmp*numProcs); + int tmp = (int)x.size(); + int sum = allReduce(tmp, SPHERAL_OP_SUM); + ENSURE(sum == tmp*numProcs); } END_CONTRACT_SCOPE #endif @@ -187,7 +188,8 @@ computeFragmentField(const NodeList& nodes, BEGIN_CONTRACT_SCOPE { int tmp = localNode ? 1 : 0; - CHECK(allReduce(tmp, SPHERAL_OP_SUM) == 1); + int sum = allReduce(tmp, SPHERAL_OP_SUM); + CHECK(sum == 1); } END_CONTRACT_SCOPE int tmp = numProcs; From 759cbdd8d133ea260f40177363c9dff008848c39 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Tue, 9 Jul 2024 10:26:38 -0700 Subject: [PATCH 10/14] Better fix for variable warnings --- src/Damage/computeFragmentField.cc | 31 +++++------------------------- 1 file changed, 5 insertions(+), 26 deletions(-) diff --git a/src/Damage/computeFragmentField.cc b/src/Damage/computeFragmentField.cc index a86e88882..1f7f15856 100644 --- a/src/Damage/computeFragmentField.cc +++ b/src/Damage/computeFragmentField.cc @@ -68,13 +68,7 @@ globalReduceToUniqueElements(vector& x) { copy(otherX.begin(), otherX.end(), back_inserter(x)); } reduceToUniqueElements(x); - BEGIN_CONTRACT_SCOPE - { - int tmp = (int)x.size(); - int sum = allReduce(tmp, SPHERAL_OP_SUM); - ENSURE(sum == tmp*numProcs); - } - END_CONTRACT_SCOPE + ENSURE(allReduce((int)x.size(), SPHERAL_OP_SUM) == (int)x.size()*numProcs); #endif } @@ -185,13 +179,7 @@ computeFragmentField(const NodeList& nodes, gIDs.end(), globalMinID); localNode = (ilocalItr != gIDs.end()); - BEGIN_CONTRACT_SCOPE - { - int tmp = localNode ? 1 : 0; - int sum = allReduce(tmp, SPHERAL_OP_SUM); - CHECK(sum == 1); - } - END_CONTRACT_SCOPE + CHECK(allReduce(localNode ? 1 : 0, SPHERAL_OP_SUM) == 1); int tmp = numProcs; if (localNode) { CHECK(ilocalItr != gIDs.end()); @@ -201,11 +189,7 @@ computeFragmentField(const NodeList& nodes, } nodeDomain = allReduce(tmp, SPHERAL_OP_MIN); CHECK(nodeDomain >= 0 && nodeDomain < numProcs); - BEGIN_CONTRACT_SCOPE - { - CHECK(allReduce(nodeDomain, SPHERAL_OP_SUM) == numProcs*nodeDomain); - } - END_CONTRACT_SCOPE + CHECK(allReduce(nodeDomain, SPHERAL_OP_SUM) == numProcs*nodeDomain); #endif // Get the position and H for this node. @@ -254,12 +238,8 @@ computeFragmentField(const NodeList& nodes, } CHECK(fragID >= 0 && fragID < numFragments); #ifdef USE_MPI - BEGIN_CONTRACT_SCOPE - { - CHECK(allReduce(fragID, SPHERAL_OP_SUM) == numProcs*fragID); - CHECK(allReduce(numFragments, SPHERAL_OP_SUM) == numProcs*numFragments); - } - END_CONTRACT_SCOPE + CHECK(allReduce(fragID, SPHERAL_OP_SUM) == numProcs*fragID); + CHECK(allReduce(numFragments, SPHERAL_OP_SUM) == numProcs*numFragments); #endif // Remove the known maxGlobalID from the stack of fragment IDs. @@ -347,7 +327,6 @@ computeFragmentField(const NodeList& nodes, } } #ifdef USE_MPI - // LDO: Can the first reduce call be moved outside of the for loop? for (int i = 0; i != numFragments - 1; ++i) { double mtmp = mfrag[i]; Vector rtmp = rfrag[i]; From 1df8ec86207a35a782936298d745cede9b834634 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Tue, 9 Jul 2024 13:19:12 -0700 Subject: [PATCH 11/14] Added nproc check to allreduce in generateCyl routine --- src/NodeGenerators/generateCylDistributionFromRZ.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/NodeGenerators/generateCylDistributionFromRZ.cc b/src/NodeGenerators/generateCylDistributionFromRZ.cc index 4592de46a..2dfd4a0cf 100644 --- a/src/NodeGenerators/generateCylDistributionFromRZ.cc +++ b/src/NodeGenerators/generateCylDistributionFromRZ.cc @@ -139,7 +139,10 @@ generateCylDistributionFromRZ(vector& x, (int)globalIDs.size() == ndomain and (int)H.size() == ndomain); for (int ikey = 0; ikey != nextra; ++ikey) VERIFY((int)extraFields[ikey].size() == ndomain); - int nglobal = allReduce(x.size(), SPHERAL_OP_SUM); + int nglobal = x.size(); + if (nProcs > 1) { + nglobal = allReduce(x.size(), SPHERAL_OP_SUM); + } VERIFY(nglobal == ntot); } From 2328760418e6aeee6defdc614afac6851066de54 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Thu, 18 Jul 2024 14:09:03 -0700 Subject: [PATCH 12/14] Renamed scan to distScan, reverted changes to check_for_edge_trouble.cc --- src/Distributed/allReduce.hh | 6 +- src/FractalStruct/check_for_edge_trouble.cc | 92 ++++++++++----------- src/KernelIntegrator/FlatConnectivity.cc | 2 +- 3 files changed, 50 insertions(+), 50 deletions(-) diff --git a/src/Distributed/allReduce.hh b/src/Distributed/allReduce.hh index 8bdef0ff7..d929a62a2 100644 --- a/src/Distributed/allReduce.hh +++ b/src/Distributed/allReduce.hh @@ -41,7 +41,7 @@ allReduce(const Value& value, const MPI_Op op, template constexpr Value -scan(const Value& value, const MPI_Op op, +distScan(const Value& value, const MPI_Op op, const MPI_Comm comm = Communicator::communicator()) { Value tmp = value; Value result; @@ -63,13 +63,13 @@ scan(const Value& value, const MPI_Op op, template constexpr Value -allReduce(const Value& value, const int /*op*/, const int comm = 0) { +allReduce(const Value& value, const int /*op*/, const int) { return value; } template constexpr Value -scan(const Value& value, const int /*op*/, const int comm = 0) { +distScan(const Value& value, const int /*op*/, const int) { return value; } diff --git a/src/FractalStruct/check_for_edge_trouble.cc b/src/FractalStruct/check_for_edge_trouble.cc index 05b43db3c..68b3b2a02 100755 --- a/src/FractalStruct/check_for_edge_trouble.cc +++ b/src/FractalStruct/check_for_edge_trouble.cc @@ -1,46 +1,46 @@ -#include "libs.hh" -#include "classes.hh" -#include "headers.hh" -namespace FractalSpace -{ - void check_for_edge_trouble(Fractal& fractal) - { - ofstream& FileFractal=fractal.p_file->DUMPS; - // ofstream& FileFractal=fractal.p_file->FileFractal; - fractal.timing(-1,2); - //-------------------------------------------------------------------------------------------------------------------------------- - // Round off errors can cause trouble at the edge, move points a little - //-------------------------------------------------------------------------------------------------------------------------------- - FileFractal << "edge trouble " << "\n"; - double eps=DBL_EPSILON; - vector pos(3); - int outsiders=0; - for(int part=0; part < fractal.get_number_particles();part++) - { - Particle* p=fractal.particle_list[part]; - if(p->get_p_highest_level_group() == 0) - continue; - p->get_pos(pos); - bool outside=pos[0] >= 1.0 || pos[0] <=0.0 || - pos[1] >= 1.0 || pos[1] <=0.0 || - pos[2] >= 1.0 || pos[2] <=0.0; - if(!outside) continue; - outsiders++; - if(pos[0] >= 1.0) - pos[0]-=eps; - else if(pos[0] <= 0.0) - pos[0]+=eps; - if(pos[1] >= 1.0) - pos[1]-=eps; - else if(pos[1] <= 0.0) - pos[1]+=eps; - if(pos[2] >= 1.0) - pos[2]-=eps; - else if(pos[2] <= 0.0) - pos[2]+=eps; - p->set_pos(pos); - } - FileFractal << " Total Outsiders " << outsiders << "\n"; - fractal.timing(1,2); - } -} +#include "libs.hh" +#include "classes.hh" +#include "headers.hh" +namespace FractalSpace +{ + void check_for_edge_trouble(Fractal& fractal) + { + ofstream& FileFractal=fractal.p_file->DUMPS; + // ofstream& FileFractal=fractal.p_file->FileFractal; + fractal.timing(-1,2); + //-------------------------------------------------------------------------------------------------------------------------------- + // Round off errors can cause trouble at the edge, move points a little + //-------------------------------------------------------------------------------------------------------------------------------- + FileFractal << "edge trouble " << "\n"; + double eps=DBL_EPSILON; + vector pos(3); + int outsiders=0; + for(int part=0; part < fractal.get_number_particles();part++) + { + Particle* p=fractal.particle_list[part]; + if(p->get_p_highest_level_group() == 0) + continue; + p->get_pos(pos); + bool outside=pos[0] >= 1.0 || pos[0] <=0.0 || + pos[1] >= 1.0 || pos[1] <=0.0 || + pos[2] >= 1.0 || pos[2] <=0.0; + if(!outside) continue; + outsiders++; + if(pos[0] >= 1.0) + pos[0]-=eps; + else if(pos[0] <= 0.0) + pos[0]+=eps; + if(pos[1] >= 1.0) + pos[1]-=eps; + else if(pos[1] <= 0.0) + pos[1]+=eps; + if(pos[2] >= 1.0) + pos[2]-=eps; + else if(pos[2] <= 0.0) + pos[2]+=eps; + p->set_pos(pos); + } + FileFractal << " Total Outsiders " << outsiders << "\n"; + fractal.timing(1,2); + } +} diff --git a/src/KernelIntegrator/FlatConnectivity.cc b/src/KernelIntegrator/FlatConnectivity.cc index 1633c86be..afb0fc3fa 100644 --- a/src/KernelIntegrator/FlatConnectivity.cc +++ b/src/KernelIntegrator/FlatConnectivity.cc @@ -326,7 +326,7 @@ computeGlobalIndices(const DataBase& dataBase, VERIFY(numInternalNodesDB == mNumInternalLocalNodes); // Get global indices manually - int globalScan = scan(mNumInternalLocalNodes, SPHERAL_OP_SUM); + int globalScan = distScan(mNumInternalLocalNodes, SPHERAL_OP_SUM); VERIFY(globalScan >= mNumInternalLocalNodes); mFirstGlobalIndex = globalScan - mNumInternalLocalNodes; mLastGlobalIndex = globalScan - 1; From 5ecb8df3617ccda1926f2dd00b6920ce2a4de94b Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Thu, 18 Jul 2024 14:27:33 -0700 Subject: [PATCH 13/14] Updated release notes --- RELEASE_NOTES.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index f3d0df693..e3f36e215 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,3 +1,22 @@ +Version vYYYY.MM.p -- Release date YYYY-MM-DD +============================================== + * Important Notes: + +Notable changes include: + + * New features/ API changes: + * MPI variables are now wrapped as + ``` + SPHERAL_OP_SUM, SPHERAL_OP_MAX, SPHERAL_OP_MIN + ``` + + * Build changes / improvements: + * Distributed source directory must always be built now + + * Bug Fixes / improvements: + * Wrappers for MPI calls are simplified and improved + + Version v2024.06.1 -- Release date 2024-07-09 ============================================== From 8f7540e004ef3ae0f1847fb7ca8586ee68731364 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Thu, 18 Jul 2024 15:09:46 -0700 Subject: [PATCH 14/14] More minor fixes --- src/Damage/IvanoviSALEDamageModel.cc | 1 - src/Distributed/allReduce.hh | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Damage/IvanoviSALEDamageModel.cc b/src/Damage/IvanoviSALEDamageModel.cc index a6629ac8e..81358209b 100644 --- a/src/Damage/IvanoviSALEDamageModel.cc +++ b/src/Damage/IvanoviSALEDamageModel.cc @@ -38,7 +38,6 @@ #include "Boundary/Boundary.hh" #include "Neighbor/Neighbor.hh" #include "Utilities/mortonOrderIndices.hh" -#include "Distributed/allReduce.hh" #include "Utilities/uniform_random.hh" #include // hash_combine diff --git a/src/Distributed/allReduce.hh b/src/Distributed/allReduce.hh index d929a62a2..b95af8a96 100644 --- a/src/Distributed/allReduce.hh +++ b/src/Distributed/allReduce.hh @@ -63,13 +63,13 @@ distScan(const Value& value, const MPI_Op op, template constexpr Value -allReduce(const Value& value, const int /*op*/, const int) { +allReduce(const Value& value, const int /*op*/, const int = 0) { return value; } template constexpr Value -distScan(const Value& value, const int /*op*/, const int) { +distScan(const Value& value, const int /*op*/, const int = 0) { return value; }