diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ff2582b5..e809444e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -13,3 +13,5 @@ add_subdirectory(phasicFlow) #add_subdirectory(Geometry) + +add_subdirectory(MPIParallelization) diff --git a/src/MPIParallelization/CMakeLists.txt b/src/MPIParallelization/CMakeLists.txt new file mode 100644 index 00000000..3bac9dbb --- /dev/null +++ b/src/MPIParallelization/CMakeLists.txt @@ -0,0 +1,33 @@ +#add Zoltan +set(Zoltan_Install_DIR) +if(DEFINED ENV{Zoltan_DIR}) + set(Zoltan_Install_DIR $ENV{Zoltan_DIR}/build) +else() + set(Zoltan_Install_DIR $ENV{HOME}/PhasicFlow/Zoltan/build) +endif() +message(STATUS "Zoltan install directory is ${Zoltan_Install_DIR}") + +set(ZOLTAN_PREFIX "${Zoltan_Install_DIR}" CACHE STRING "Zoltan install directory") + +find_path(ZOLTAN_INCLUDE_DIR zoltan.h PATHS "${ZOLTAN_PREFIX}/include") + +message(STATUS "Zoltan include path: ${ZOLTAN_INCLUDE_DIR}") + +find_library(ZOLTAN_LIBRARY zoltan PATHS "${ZOLTAN_PREFIX}/lib") +message(STATUS "Zoltan lib path: ${ZOLTAN_LIBRARY}") + + +set(SourceFiles partitioning) + +set(link_libs Kokkos::kokkos phasicFlow PRIVATE MPI::MPI_CXX ${ZOLTAN_LIBRARY} -lm ) + +pFlow_add_library_install(MPIParallelization SourceFiles link_libs) +target_include_directories(MPIParallelization PRIVATE ${ZOLTAN_INCLUDE_DIR}) + + + + + + + + diff --git a/src/MPIParallelization/partitioning.cpp b/src/MPIParallelization/partitioning.cpp new file mode 100644 index 00000000..a786a89c --- /dev/null +++ b/src/MPIParallelization/partitioning.cpp @@ -0,0 +1,92 @@ + +#include "zoltan_cpp.h" + +#include "partitioning.hpp" +#include "error.hpp" +#include "streams.hpp" + +void pFlow::partitioning::freeZoltan() +{ + if(validPointers_) + { + Zoltan::LB_Free_Part(&importGlobalGids_, &importLocalGids_, + &importProcs_, &importToPart_); + + Zoltan::LB_Free_Part(&exportGlobalGids_, &exportLocalGids_, + &exportProcs_, &exportToPart_); + validPointers_ = false; + } + + zoltan_.release(); +} + + +pFlow::partitioning::partitioning(int argc, char *argv[], const box& globalBox) +: + globalBox_(globalBox) +{ + if(!zoltanInitialized__) + { + auto rc = Zoltan_Initialize(argc, argv, &version_); + if (rc != ZOLTAN_OK) + { + fatalErrorInFunction<<"Cannot initialize zoltan"<(MPI::COMM_WORLD); + + zoltan_->Set_Param("DEBUG_LEVEL", "0"); + zoltan_->Set_Param("LB_METHOD", "RCB"); + zoltan_->Set_Param("NUM_GID_ENTRIES", "1"); + zoltan_->Set_Param("NUM_LID_ENTRIES", "1"); + zoltan_->Set_Param("OBJ_WEIGHT_DIM", "0"); + zoltan_->Set_Param("RETURN_LISTS", "ALL"); + +} + +pFlow::partitioning::~partitioning() +{ + freeZoltan(); +} + +void pFlow::partitioning::printBox()const +{ + pOutput<< "localBox:" << localBox_<(data); + + *ierr = ZOLTAN_OK; + + return obj->numberOfPOints(); +} + + +void partitioning::getPointList( + void *data, + int sizeGID, + int sizeLID, + ZOLTAN_ID_PTR globalID, + ZOLTAN_ID_PTR localID, + int wgt_dim, + float *obj_wgts, + int *ierr) +{ + + auto* obj = static_cast(data); + *ierr = ZOLTAN_OK; + + for (int i=0; i< obj->numberOfPOints(); i++) + { + globalID[i] = obj->globalIDList()[i]; + localID[i] = i; + } +}*/ \ No newline at end of file diff --git a/src/MPIParallelization/partitioning.hpp b/src/MPIParallelization/partitioning.hpp new file mode 100644 index 00000000..5d6da480 --- /dev/null +++ b/src/MPIParallelization/partitioning.hpp @@ -0,0 +1,110 @@ +/*------------------------------- phasicFlow --------------------------------- + O C enter of + O O E ngineering and + O O M ultiscale modeling of + OOOOOOO F luid flow +------------------------------------------------------------------------------ + Copyright (C): www.cemf.ir + email: hamid.r.norouzi AT gmail.com +------------------------------------------------------------------------------ +Licence: + This file is part of phasicFlow code. It is a free software for simulating + granular and multiphase flows. You can redistribute it and/or modify it under + the terms of GNU General Public License v3 or any other later versions. + + phasicFlow is distributed to help others in their research in the field of + granular and multiphase flows, but WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + +-----------------------------------------------------------------------------*/ + +#ifndef __rcbPartitioning_hpp__ +#define __rcbPartitioning_hpp__ + +#include "box.hpp" +#include "processors.hpp" + + + +namespace pFlow +{ + +class Zoltan; + + +class partitioning +{ + +protected: + + float version_ = 0.0; + + std::unique_ptr zoltan_ = nullptr; + + bool validPointers_ = false; + + const box& globalBox_; + + box localBox_; + + int32 changes_, numImport_, numExport_; + + id_t* importGlobalGids_, importLocalGids_, exportGlobalGids_, exportLocalGids_; + + int32*importProcs_, *importToPart_, *exportProcs_, *exportToPart_; + + static inline bool zoltanInitialized__ = false; + + void freeZoltan(); + +public: + + partitioning(int argc, char *argv[], const box& globalBox); + + virtual + ~partitioning(); + + + virtual + bool partition() = 0; + + inline + auto localBox()const + { + return localBox_; + } + + inline + const auto& globalBox()const + { + return globalBox_; + } + + inline + bool partitionsChanged()const + { + return changes_ == 1; + } + + void printBox()const; + + /*static + int getNumberOfPoints(void *data, int32 *ierr); + + static + void getPointList( + void *data, + int32 sizeGID, + int32 sizeLID, + id_t* globalID, + id_t* localID, + int32 wgt_dim, + float *obj_wgts, + int32 *ierr);*/ +}; + + +} + + +#endif //__rcbPartitioning_hpp__ \ No newline at end of file diff --git a/src/phasicFlow/types/basicTypes/builtinTypes.hpp b/src/phasicFlow/types/basicTypes/builtinTypes.hpp index 1abe89bd..340179ec 100755 --- a/src/phasicFlow/types/basicTypes/builtinTypes.hpp +++ b/src/phasicFlow/types/basicTypes/builtinTypes.hpp @@ -58,6 +58,8 @@ using uint32 = unsigned int; using uint64 = unsigned long long int; +using id_t = uint32; + using size_t = std::size_t; using word = std::string;