MPI part

src/CMakeLists.txt

@@ -13,3 +13,5 @@ add_subdirectory(phasicFlow)
 
 #add_subdirectory(Geometry)
 
+
+add_subdirectory(MPIParallelization)

src/MPIParallelization/CMakeLists.txt

@@ -0,0 +1,33 @@
+#add Zoltan
+set(Zoltan_Install_DIR)
+if(DEFINED ENV{Zoltan_DIR})
+   set(Zoltan_Install_DIR $ENV{Zoltan_DIR}/build)
+else()
+  set(Zoltan_Install_DIR $ENV{HOME}/PhasicFlow/Zoltan/build)
+endif()
+message(STATUS "Zoltan install directory is ${Zoltan_Install_DIR}")
+
+set(ZOLTAN_PREFIX "${Zoltan_Install_DIR}" CACHE STRING "Zoltan install directory")
+
+find_path(ZOLTAN_INCLUDE_DIR zoltan.h PATHS "${ZOLTAN_PREFIX}/include")
+
+message(STATUS "Zoltan include path: ${ZOLTAN_INCLUDE_DIR}")
+
+find_library(ZOLTAN_LIBRARY zoltan PATHS "${ZOLTAN_PREFIX}/lib")
+message(STATUS "Zoltan lib path: ${ZOLTAN_LIBRARY}")
+
+
+set(SourceFiles partitioning)
+
+set(link_libs Kokkos::kokkos phasicFlow PRIVATE MPI::MPI_CXX ${ZOLTAN_LIBRARY} -lm )
+
+pFlow_add_library_install(MPIParallelization SourceFiles link_libs)
+target_include_directories(MPIParallelization PRIVATE ${ZOLTAN_INCLUDE_DIR})
+
+
+
+
+
+
+
+

src/MPIParallelization/partitioning.cpp

@@ -0,0 +1,92 @@
+
+#include "zoltan_cpp.h"
+
+#include "partitioning.hpp"
+#include "error.hpp"
+#include "streams.hpp"
+
+void pFlow::partitioning::freeZoltan()
+{
+	if(validPointers_)
+	{
+		Zoltan::LB_Free_Part(&importGlobalGids_, &importLocalGids_, 
+                      	 	 &importProcs_, &importToPart_);
+
+		Zoltan::LB_Free_Part(&exportGlobalGids_, &exportLocalGids_, 
+                      		 &exportProcs_, &exportToPart_);
+        validPointers_ = false;
+	}
+	
+	zoltan_.release();
+}
+
+
+pFlow::partitioning::partitioning(int argc, char *argv[], const box& globalBox)
+:
+	globalBox_(globalBox)
+{
+	if(!zoltanInitialized__)
+	{
+		auto rc = Zoltan_Initialize(argc, argv, &version_);
+		if (rc != ZOLTAN_OK)
+		{
+			fatalErrorInFunction<<"Cannot initialize zoltan"<<endl;
+			fatalExit;
+		}
+		zoltanInitialized__ = true;
+	}
+
+	// Creates Zoltan object 
+	zoltan_ = std::make_unique<Zoltan>(MPI::COMM_WORLD);
+
+	zoltan_->Set_Param("DEBUG_LEVEL", "0");
+  	zoltan_->Set_Param("LB_METHOD", "RCB");
+  	zoltan_->Set_Param("NUM_GID_ENTRIES", "1"); 
+  	zoltan_->Set_Param("NUM_LID_ENTRIES", "1");
+  	zoltan_->Set_Param("OBJ_WEIGHT_DIM", "0");
+    zoltan_->Set_Param("RETURN_LISTS", "ALL");
+	
+}
+
+pFlow::partitioning::~partitioning()
+{
+	freeZoltan();
+}
+
+void pFlow::partitioning::printBox()const
+{
+	pOutput<< "localBox:" << localBox_<<endl;
+}
+
+
+ 
+/*int partitioning::getNumberOfPoints(void *data, int *ierr)
+{
+	auto* obj = static_cast<pointCollection *>(data);
+
+	*ierr = ZOLTAN_OK;
+
+	return obj->numberOfPOints();
+}
+
+
+void partitioning::getPointList(
+	void *data, 
+	int sizeGID, 
+	int sizeLID,
+    ZOLTAN_ID_PTR globalID, 
+    ZOLTAN_ID_PTR localID,
+    int wgt_dim, 
+    float *obj_wgts, 
+    int *ierr)
+{
+		
+	auto* obj = static_cast<pointCollection *>(data);
+	*ierr = ZOLTAN_OK;
+
+	for (int i=0; i< obj->numberOfPOints(); i++)
+	{
+		globalID[i] = obj->globalIDList()[i];
+		localID[i] = i;
+	}
+}*/

src/MPIParallelization/partitioning.hpp

@@ -0,0 +1,110 @@
+/*------------------------------- phasicFlow ---------------------------------
+      O        C enter of
+     O O       E ngineering and
+    O   O      M ultiscale modeling of
+   OOOOOOO     F luid flow       
+------------------------------------------------------------------------------
+  Copyright (C): www.cemf.ir
+  email: hamid.r.norouzi AT gmail.com
+------------------------------------------------------------------------------  
+Licence:
+  This file is part of phasicFlow code. It is a free software for simulating 
+  granular and multiphase flows. You can redistribute it and/or modify it under
+  the terms of GNU General Public License v3 or any other later versions. 
+ 
+  phasicFlow is distributed to help others in their research in the field of 
+  granular and multiphase flows, but WITHOUT ANY WARRANTY; without even the
+  implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+-----------------------------------------------------------------------------*/
+
+#ifndef __rcbPartitioning_hpp__
+#define __rcbPartitioning_hpp__
+
+#include "box.hpp"
+#include "processors.hpp"
+
+
+
+namespace pFlow
+{
+
+class Zoltan;
+
+
+class partitioning
+{
+
+protected:
+
+	float 					version_ 	= 0.0;
+
+	std::unique_ptr<Zoltan> zoltan_ 	= nullptr;
+
+	bool					validPointers_ = false;
+
+	const box& 				globalBox_;
+
+	box 					localBox_; 	
+
+	int32 	changes_, numImport_, numExport_;
+  	
+  	id_t* importGlobalGids_, importLocalGids_, exportGlobalGids_, exportLocalGids_; 
+  	
+  	int32*importProcs_, *importToPart_, *exportProcs_, *exportToPart_;
+
+	static inline bool 		zoltanInitialized__ = false;
+
+	void freeZoltan();
+
+public:
+
+	partitioning(int argc, char *argv[], const box& globalBox);
+
+	virtual 
+	~partitioning();
+
+		
+	virtual 
+	bool partition() = 0;
+
+	inline
+	auto localBox()const
+	{
+		return localBox_;
+	}
+
+	inline
+	const auto& globalBox()const
+	{
+		return globalBox_;
+	}
+
+	inline 
+	bool partitionsChanged()const
+	{
+		return changes_ == 1;
+	}
+
+	void printBox()const;
+
+	/*static 
+	int getNumberOfPoints(void *data, int32 *ierr);
+
+	static 
+	void getPointList(
+		void *data, 
+		int32 sizeGID, 
+		int32 sizeLID,
+        id_t* globalID, 
+        id_t* localID,
+        int32 wgt_dim, 
+        float *obj_wgts, 
+        int32 *ierr);*/
+};
+
+
+}
+
+
+#endif //__rcbPartitioning_hpp__

src/phasicFlow/types/basicTypes/builtinTypes.hpp

@@ -58,6 +58,8 @@ using uint32  = unsigned int;
 
 using uint64  = unsigned long long int;
 
+using id_t    = uint32;
+
 using size_t  = std::size_t;
 
 using word    = std::string;