VTK  9.3.0
vtkPStructuredGridConnectivity.h
Go to the documentation of this file.
1 // SPDX-FileCopyrightText: Copyright (c) Ken Martin, Will Schroeder, Bill Lorensen
2 // SPDX-License-Identifier: BSD-3-Clause
22 #ifndef vtkPStructuredGridConnectivity_h
23 #define vtkPStructuredGridConnectivity_h
24 
25 // VTK include directives
26 #include "vtkFiltersParallelGeometryModule.h" // For export macro
27 #include "vtkMPICommunicator.h" // Needed for vtkMPICommunicator::Request
29 
30 // C++ include directives
31 #include <vector> // For STL vector
32 
33 // Forward declarations
34 VTK_ABI_NAMESPACE_BEGIN
36 class vtkMPIController;
38 // class vtkMPICommunicator::Request;
39 
40 class VTKFILTERSPARALLELGEOMETRY_EXPORT vtkPStructuredGridConnectivity
42 {
43 public:
46  void PrintSelf(ostream& os, vtkIndent indent) override;
47 
49 
53  vtkGetObjectMacro(Controller, vtkMultiProcessController);
55 
59  void SetNumberOfGrids(unsigned int N) override;
60 
64  void RegisterGrid(int gridID, int extents[6], vtkUnsignedCharArray* nodesGhostArray,
65  vtkUnsignedCharArray* cellGhostArray, vtkPointData* pointData, vtkCellData* cellData,
66  vtkPoints* gridNodes) override;
67 
72  int GetNumberOfLocalGrids() { return static_cast<int>(this->GridIds.size()); }
73 
80  int GetGridRank(int gridID);
81 
85  bool IsGridRemote(int gridID);
86 
90  bool IsGridLocal(int gridID);
91 
97  void Initialize();
98 
104  void ComputeNeighbors() override;
105 
110  void CreateGhostLayers(int N = 1) override;
111 
112 protected:
115 
117  int Rank;
119 
120  std::vector<int> GridRanks; // Corresponding rank for each grid
121  std::vector<int> GridIds; // List of GridIds, owned by this process
122 
123  // Data structures to store the remote ghost data of each grid for each one
124  // of its neighbors. The first index is the global grid index. The second
125  // is the neighbor index.
126  std::vector<std::vector<vtkPoints*>> RemotePoints;
127  std::vector<std::vector<vtkPointData*>> RemotePointData;
128  std::vector<std::vector<vtkCellData*>> RemoteCellData;
129 
130  // Data structures to store the send/receive buffer sizes and corresponding
131  // persistent buffers. The first index is the global grid index. The second
132  // index is the neighbor index for the given grid.
133  std::vector<std::vector<unsigned int>> SendBufferSizes;
134  std::vector<std::vector<unsigned int>> RcvBufferSizes;
135  std::vector<std::vector<unsigned char*>> SendBuffers;
136  std::vector<std::vector<unsigned char*>> RcvBuffers;
137 
141 
142  // Array of MPI requests
144 
148  bool GridExtentsAreEqual(int rhs[6], int lhs[6]);
149 
153  bool HasPointData(int gridIdx);
154 
158  bool HasCellData(int gridIdx);
159 
163  bool HasPoints(int gridIdx);
164 
168  void InitializeMessageCounters();
169 
174  void ClearRemoteData();
175 
179  void ClearRawBuffers();
180 
185  void RegisterRemoteGrid(int gridID, int extents[6], int process);
186 
191  void TransferRemoteNeighborData(int gridIdx, int nei, const vtkStructuredNeighbor& Neighbor);
192 
198  void TransferGhostDataFromNeighbors(int gridID) override;
199 
204 
210 
215  void DeserializeBufferSizesForProcess(int* buffersizes, vtkIdType N, int processId);
216 
221  void SerializeBufferSizes(int*& sizesbuf, vtkIdType& N);
222 
228 
235 
240  void PostReceives();
241 
246  void PostSends();
247 
254 
261 
266 
271  void SerializeGhostPoints(int gridIdx, int ext[6], vtkMultiProcessStream& bytestream);
272 
277 
282  void SerializeFieldData(int sourceExtent[6], int targetExtent[6], vtkFieldData* fieldData,
283  vtkMultiProcessStream& bytestream);
284 
289  void SerializeGhostPointData(int gridIdx, int ext[6], vtkMultiProcessStream& bytestream);
290 
295  void SerializeGhostCellData(int gridIdx, int ext[6], vtkMultiProcessStream& bytestream);
296 
301  void DeserializeGhostPoints(int gridIdx, int nei, int ext[6], vtkMultiProcessStream& bytestream);
302 
306  void DeserializeDataArray(vtkDataArray*& dataArray, int dataType, int numberOfTuples,
307  int numberOfComponents, vtkMultiProcessStream& bytestream);
308 
313  void DeserializeFieldData(int ext[6], vtkFieldData* fieldData, vtkMultiProcessStream& bytestream);
314 
320  int gridIdx, int nei, int ext[6], vtkMultiProcessStream& bytestream);
321 
327  int gridIdx, int nei, int ext[6], vtkMultiProcessStream& bytestream);
328 
336  int sndGridID, int rcvGrid, int sndext[6], unsigned char*& buffer, unsigned int& size);
337 
342  void DeserializeGhostData(int gridID, int neiListID, int neiGridIdx, int rcvext[6],
343  unsigned char* buffer, unsigned int size);
344 
350 
355  void SerializeGridExtents(int*& sndbuffer, vtkIdType& N);
356 
361  void DeserializeGridExtentForProcess(int* rcvbuffer, vtkIdType& N, int processId);
362 
363 private:
365  void operator=(const vtkPStructuredGridConnectivity&) = delete;
366 };
367 
368 //=============================================================================
369 // INLINE METHODS
370 //=============================================================================
371 
372 inline bool vtkPStructuredGridConnectivity::GridExtentsAreEqual(int rhs[6], int lhs[6])
373 {
374  for (int i = 0; i < 6; ++i)
375  {
376  if (rhs[i] != lhs[i])
377  {
378  return false;
379  }
380  }
381  return true;
382 }
383 
384 //------------------------------------------------------------------------------
386 {
387  // Sanity check
388  assert("pre: grid index is out-of-bounds!" && (gridIdx >= 0) &&
389  (gridIdx < static_cast<int>(this->NumberOfGrids)));
390 
391  if ((this->GridPointData[gridIdx] != nullptr) &&
392  (this->GridPointData[gridIdx]->GetNumberOfArrays() > 0))
393  {
394  return true;
395  }
396  return false;
397 }
398 
399 //------------------------------------------------------------------------------
401 {
402  // Sanity check
403  assert("pre: grid index is out-of-bounds!" && (gridIdx >= 0) &&
404  (gridIdx < static_cast<int>(this->NumberOfGrids)));
405 
406  if ((this->GridCellData[gridIdx] != nullptr) &&
407  (this->GridCellData[gridIdx]->GetNumberOfArrays() > 0))
408  {
409  return true;
410  }
411  return false;
412 }
413 
414 //------------------------------------------------------------------------------
416 {
417  // Sanity check
418  assert("pre: grid index is out-of-bounds!" && (gridIdx >= 0) &&
419  (gridIdx < static_cast<int>(this->NumberOfGrids)));
420 
421  if (this->GridPoints[gridIdx] != nullptr)
422  {
423  return true;
424  }
425  return false;
426 }
427 
428 //------------------------------------------------------------------------------
430 {
431  this->TotalNumberOfMsgs = this->TotalNumberOfRcvs = this->TotalNumberOfSends = 0;
432 }
433 
434 //------------------------------------------------------------------------------
436 {
437  this->SendBufferSizes.clear();
438  this->RcvBufferSizes.clear();
439 
440  // STEP 0: Clear send buffers
441  for (unsigned int i = 0; i < this->SendBuffers.size(); ++i)
442  {
443  for (unsigned int j = 0; j < this->SendBuffers[i].size(); ++j)
444  {
445  delete[] this->SendBuffers[i][j];
446  } // END for all neighbors
447  this->SendBuffers[i].clear();
448  } // END for all grids
449  this->SendBuffers.clear();
450 
451  // STEP 1: Clear rcv buffers
452  for (unsigned int i = 0; i < this->RcvBuffers.size(); ++i)
453  {
454  for (unsigned int j = 0; j < this->RcvBuffers[i].size(); ++j)
455  {
456  delete[] this->RcvBuffers[i][j];
457  } // END for all neighbors
458  this->RcvBuffers[i].clear();
459  } // END for all grids
460  this->RcvBuffers.clear();
461 }
462 
463 //------------------------------------------------------------------------------
465 {
466  // STEP 0: Clear remote points
467  for (unsigned int i = 0; i < this->RemotePoints.size(); ++i)
468  {
469  for (unsigned int j = 0; j < this->RemotePoints[i].size(); ++j)
470  {
471  if (this->RemotePoints[i][j] != nullptr)
472  {
473  this->RemotePoints[i][j]->Delete();
474  }
475  } // END for all j
476  this->RemotePoints[i].clear();
477  } // END for all i
478  this->RemotePoints.clear();
479 
480  // STEP 1: Clear remote point data
481  for (unsigned int i = 0; i < this->RemotePointData.size(); ++i)
482  {
483  for (unsigned int j = 0; j < this->RemotePointData[i].size(); ++j)
484  {
485  if (this->RemotePointData[i][j] != nullptr)
486  {
487  this->RemotePointData[i][j]->Delete();
488  }
489  } // END for all j
490  this->RemotePointData[i].clear();
491  } // END for all i
492  this->RemotePointData.clear();
493 
494  // STEP 2: Clear remote cell data
495  for (unsigned int i = 0; i < this->RemoteCellData.size(); ++i)
496  {
497  for (unsigned int j = 0; j < this->RemoteCellData[i].size(); ++j)
498  {
499  if (this->RemoteCellData[i][j] != nullptr)
500  {
501  this->RemoteCellData[i][j]->Delete();
502  }
503  } // END for all j
504  this->RemoteCellData[i].clear();
505  }
506  this->RemoteCellData.clear();
507 }
508 
509 //------------------------------------------------------------------------------
511 {
512  return (!this->IsGridLocal(gridID));
513 }
514 
515 //------------------------------------------------------------------------------
517 {
518  assert("pre: Instance has not been initialized!" && this->Initialized);
519  assert("pre: gridID is out-of-bounds" && (gridID >= 0) &&
520  (gridID < static_cast<int>(this->NumberOfGrids)));
521  assert(
522  "pre: GridRanks is not properly allocated" && this->NumberOfGrids == this->GridRanks.size());
523  return ((this->GridRanks[gridID] == this->Rank));
524 }
525 
526 //------------------------------------------------------------------------------
528 {
529  assert("pre: Instance has not been initialized!" && this->Initialized);
530  assert("pre: gridID out-of-bounds!" &&
531  (gridID >= 0 && gridID < static_cast<int>(this->NumberOfGrids)));
532  return (this->GridRanks[gridID]);
533 }
534 VTK_ABI_NAMESPACE_END
535 #endif /* vtkPStructuredGridConnectivity_h */
std::vector< vtkCellData * > GridCellData
std::vector< vtkPointData * > GridPointData
std::vector< vtkPoints * > GridPoints
represent and manipulate cell attribute data
Definition: vtkCellData.h:31
abstract superclass for arrays of numeric data
Definition: vtkDataArray.h:45
represent and manipulate fields of data
Definition: vtkFieldData.h:52
a simple class to control print indentation
Definition: vtkIndent.h:29
Process communication using MPI.
Multiprocessing communication superclass.
stream used to pass data across processes using vtkMultiProcessController.
vtkPStructuredGridConnectivity inherits from vtkStructuredGridConnectivity and implements functionali...
int GetNumberOfLocalGrids()
Returns the number of local grids registers by the process that owns the current vtkPStructuredGridCo...
~vtkPStructuredGridConnectivity() override
std::vector< std::vector< unsigned int > > RcvBufferSizes
void TransferRemoteNeighborData(int gridIdx, int nei, const vtkStructuredNeighbor &Neighbor)
This method transfers all the remote neighbor data to the ghosted grid instance of the grid correspon...
void DeserializeDataArray(vtkDataArray *&dataArray, int dataType, int numberOfTuples, int numberOfComponents, vtkMultiProcessStream &bytestream)
Helper method to deserialize the data array from a bytestream.
virtual void SetController(vtkMultiProcessController *)
Set & Get the process controller.
bool GridExtentsAreEqual(int rhs[6], int lhs[6])
Returns true if the two extents are equal, otherwise false.
bool IsGridLocal(int gridID)
Returns true iff the grid corresponding to the given gridID is local.
void DeserializeGhostPointData(int gridIdx, int nei, int ext[6], vtkMultiProcessStream &bytestream)
Helper method to de-serialize the ghost point data received from a remote process.
void PrintSelf(ostream &os, vtkIndent indent) override
Methods invoked by print to print information about the object including superclasses.
void SetNumberOfGrids(unsigned int N) override
Sets the total number of domains distributed among processors.
void SerializeDataArray(vtkDataArray *dataArray, vtkMultiProcessStream &bytestream)
Serializes the data array into a bytestream.
void DeserializeFieldData(int ext[6], vtkFieldData *fieldData, vtkMultiProcessStream &bytestream)
Helper method to de-serialize field data.
void RegisterRemoteGrid(int gridID, int extents[6], int process)
Registers a remote grid with the given grid Id, structured extents and process.
void ClearRawBuffers()
Clears all raw send/rcv buffers.
void SerializeGhostCellData(int gridIdx, int ext[6], vtkMultiProcessStream &bytestream)
Helper method to serialize ghost cell data.
void ExchangeGhostDataPost()
Helper method for exchanging ghost data.
void DeserializeGhostPoints(int gridIdx, int nei, int ext[6], vtkMultiProcessStream &bytestream)
Helper method to de-serialize the ghost points received from a remote process.
bool HasPoints(int gridIdx)
Returns true iff the grid corresponding to the given ID has points.
std::vector< std::vector< unsigned char * > > SendBuffers
bool IsGridRemote(int gridID)
Returns true iff the grid is remote, otherwise false.
std::vector< std::vector< vtkPoints * > > RemotePoints
vtkMPICommunicator::Request * MPIRequests
int GetGridRank(int gridID)
Returns the rank of the given gridID.
void ExchangeGhostData()
Exchanges ghost data of the grids owned by this process.
void CreateGhostLayers(int N=1) override
Creates ghost layers on the grids owned by this process using data from both local and remote block n...
void SerializeGhostPoints(int gridIdx, int ext[6], vtkMultiProcessStream &bytestream)
Helper method to serialize the ghost points to send to a remote process.
static vtkPStructuredGridConnectivity * New()
void PostSends()
Helper method to communicate ghost data.
void CommunicateGhostData()
Helper method for exchanging ghost data.
bool HasCellData(int gridIdx)
Returns true iff the grid corresponding to the given ID has cell data.
void DeserializeGhostCellData(int gridIdx, int nei, int ext[6], vtkMultiProcessStream &bytestream)
Helper method to de-serialize the ghost cell data received from a remote process.
void SerializeGhostData(int sndGridID, int rcvGrid, int sndext[6], unsigned char *&buffer, unsigned int &size)
Given a grid ID and the corresponding send extent, this method serializes the grid and data within th...
void TransferGhostDataFromNeighbors(int gridID) override
This method transfers the fields (point data and cell data) to the ghost extents from the neighboring...
void ExchangeGridExtents()
Exchanges the grid extents among all processes and fully populates the GridExtents vector.
void DeserializeBufferSizesForProcess(int *buffersizes, vtkIdType N, int processId)
Helper method to deserialize the buffer sizes coming from the given process.
void ClearRemoteData()
Clears all internal VTK data-structures that are used to store the remote ghost data.
void PackGhostData()
Helper method to pack all the ghost data into send buffers.
void ExchangeBufferSizes()
Helper method to exchange buffer sizes.Each process sends the send buffer size of each grid to each o...
void UnpackGhostData()
Helper method to unpack the raw ghost data from the receive buffers in to the VTK remote point data-s...
void SerializeGridExtents(int *&sndbuffer, vtkIdType &N)
Serializes the grid extents and information in a buffer to send over MPI The data is serialized as fo...
void RegisterGrid(int gridID, int extents[6], vtkUnsignedCharArray *nodesGhostArray, vtkUnsignedCharArray *cellGhostArray, vtkPointData *pointData, vtkCellData *cellData, vtkPoints *gridNodes) override
See vtkStructuredGridConnectivity::RegisterGrid.
void SerializeBufferSizes(int *&sizesbuf, vtkIdType &N)
Helper method to serialize the buffer sizes for the grids of this process to neighboring grids.
void DeserializeGhostData(int gridID, int neiListID, int neiGridIdx, int rcvext[6], unsigned char *buffer, unsigned int size)
Given the raw buffer consisting of ghost data, this method deserializes the object and returns the gr...
bool HasPointData(int gridIdx)
Returns true iff the grid corresponding to the given ID has point data.
std::vector< std::vector< unsigned int > > SendBufferSizes
void ExchangeGhostDataInit()
Helper method for exchanging ghost data.
std::vector< std::vector< vtkCellData * > > RemoteCellData
void SerializeFieldData(int sourceExtent[6], int targetExtent[6], vtkFieldData *fieldData, vtkMultiProcessStream &bytestream)
Helper method to serialize field data.
void ComputeNeighbors() override
Computes the neighboring topology of a distributed structured grid data-set.
void PostReceives()
Helper method to communicate ghost data.
void Initialize()
Initializes this instance of vtkPStructuredGridConnectivity, essentially, the acquires the local proc...
void DeserializeGridExtentForProcess(int *rcvbuffer, vtkIdType &N, int processId)
Deserializes the received grid extent information to the GridExtents internal data-structures.
void SerializeGhostPointData(int gridIdx, int ext[6], vtkMultiProcessStream &bytestream)
Helper method to serialize ghost point data.
void InitializeMessageCounters()
Sets all message counters to 0.
std::vector< std::vector< vtkPointData * > > RemotePointData
std::vector< std::vector< unsigned char * > > RcvBuffers
represent and manipulate point attribute data
Definition: vtkPointData.h:30
represent and manipulate 3D points
Definition: vtkPoints.h:29
vtkStructuredGridConnectivity is a concrete instance of vtkObject that implements functionality for c...
An internal, light-weight class used to store neighbor information.
dynamic, self-adjusting array of unsigned char
@ size
Definition: vtkX3D.h:253
int vtkIdType
Definition: vtkType.h:315