1 #ifndef DUNE_FEM_CACHED_COMMUNICATION_MANAGER_HH
2 #define DUNE_FEM_CACHED_COMMUNICATION_MANAGER_HH
15 #include <dune/common/math.hh>
16 #include <dune/common/timer.hh>
17 #include <dune/common/visibility.hh>
20 #include <dune/grid/common/grid.hh>
21 #include <dune/grid/common/datahandleif.hh>
22 #include <dune/grid/utility/entitycommhelper.hh>
26 #include <dune/alugrid/3d/alu3dinclude.hh>
46 template<
class DiscreteFunctionSpace >
59 #if ALU3DGRID_PARALLEL
66 template<
class Space >
71 typedef Space SpaceType;
74 typedef typename SpaceType :: GridPartType GridPartType;
81 typedef IndexMapType *IndexMapVectorType;
84 typedef std :: set< int > LinkStorageType;
87 typedef ALU3DSPACE ObjectStream ObjectStreamType;
90 typedef ALU3DSPACE MpAccessLocal MPAccessInterfaceType;
92 typedef ALU3DSPACE MpAccessMPI MPAccessImplType;
95 typedef std :: vector< ObjectStreamType > ObjectStreamVectorType;
98 const SpaceType &space_;
99 const GridPartType &gridPart_;
101 const InterfaceType interface_;
102 const CommunicationDirection dir_;
107 LinkStorageType linkStorage_;
109 IndexMapType *recvIndexMap_;
110 IndexMapType *sendIndexMap_;
113 MPAccessInterfaceType *mpAccess_;
116 double exchangeTime_;
123 int nonBlockingObjects_ ;
126 template<
class LinkStorage,
class IndexMapVector, InterfaceType CommInterface >
133 class NonBlockingCommunication
135 typedef DependencyCache < Space > DependencyCacheType;
137 #if HAVE_DUNE_ALUGRID
138 typedef MPAccessInterfaceType :: NonBlockingExchange NonBlockingExchange;
140 template <
class DiscreteFunction>
141 class Pack :
public NonBlockingExchange :: DataHandleIF
144 NonBlockingCommunication& commObj_;
145 const DiscreteFunction& discreteFunction_;
148 Pack( NonBlockingCommunication& commObj,
const DiscreteFunction& df )
149 : commObj_( commObj ), discreteFunction_( df )
152 void pack(
const int link, ObjectStreamType& buffer )
154 commObj_.pack( link, buffer, discreteFunction_ );
157 void unpack(
const int link, ObjectStreamType& buffer )
159 DUNE_THROW(InvalidStateException,
"Pack::unpack should not be called!");
163 template <
class DiscreteFunction,
class Operation>
164 class Unpack :
public NonBlockingExchange :: DataHandleIF
167 NonBlockingCommunication& commObj_;
168 DiscreteFunction& discreteFunction_;
171 const Operation operation_;
174 Unpack( NonBlockingCommunication& commObj, DiscreteFunction& df )
175 : commObj_( commObj ), discreteFunction_( df ), operation_()
178 void pack(
const int link, ObjectStreamType& buffer )
180 DUNE_THROW(InvalidStateException,
"Unpack::pack should not be called!");
183 void unpack(
const int link, ObjectStreamType& buffer )
185 commObj_.unpack( link, buffer, discreteFunction_, operation_ );
189 typedef int NonBlockingExchange;
193 DUNE_EXPORT
static int getMessageTag()
195 enum { initial = 665 };
196 static int tagCounter = initial ;
198 int messageTag = tagCounter ;
203 messageTag = initial ;
204 tagCounter = initial ;
210 NonBlockingCommunication( DependencyCacheType& dependencyCache,
const int mySize )
211 : dependencyCache_( dependencyCache ),
212 nonBlockingExchange_(),
214 exchangeTime_( 0.0 ),
218 dependencyCache_.rebuild();
221 dependencyCache_.attachComm();
225 NonBlockingCommunication(
const NonBlockingCommunication& other )
226 : dependencyCache_( other.dependencyCache_ ),
227 nonBlockingExchange_(),
229 exchangeTime_( 0.0 ),
230 mySize_( other.mySize_ )
233 dependencyCache_.rebuild();
236 dependencyCache_.attachComm();
239 ~NonBlockingCommunication()
242 assert( ! nonBlockingExchange_ );
244 dependencyCache_.detachComm() ;
247 template <
class DiscreteFunctionSpace >
248 void send(
const PetscDiscreteFunction< DiscreteFunctionSpace >& discreteFunction )
253 template <
class DiscreteFunction >
254 void send(
const DiscreteFunction& discreteFunction )
257 assert( ! nonBlockingExchange_ );
260 if( mySize_ <= 1 )
return;
263 Dune::Timer sendTimer ;
266 const int nLinks = dependencyCache_.nlinks();
269 buffer_.resize( nLinks );
271 #if HAVE_DUNE_ALUGRID
273 nonBlockingExchange_.reset( dependencyCache_.mpAccess().nonBlockingExchange( getMessageTag() ) );
276 Pack< DiscreteFunction > packData( *
this, discreteFunction );
279 nonBlockingExchange_->send( buffer_, packData );
282 for(
int link = 0; link < nLinks; ++link )
283 pack( link, buffer_[ link ], discreteFunction );
287 exchangeTime_ = sendTimer.elapsed();
291 template <
class DiscreteFunctionSpace,
class Operation >
292 double receive( PetscDiscreteFunction< DiscreteFunctionSpace >& discreteFunction,
293 const Operation& operation )
296 Dune::Timer exchTimer;
299 discreteFunction.communicate();
301 return exchTimer.elapsed();
305 template <
class DiscreteFunction,
class Operation >
306 double receive( DiscreteFunction& discreteFunction,
const Operation& operation )
309 if( mySize_ <= 1 )
return 0.0;
312 Dune::Timer recvTimer ;
314 #if HAVE_DUNE_ALUGRID
316 Unpack< DiscreteFunction, Operation > unpackData( *
this, discreteFunction );
319 nonBlockingExchange_->receive( unpackData );
322 buffer_ = dependencyCache_.mpAccess().exchange( buffer_ );
325 const int nLinks = buffer_.size();
328 for(
int link = 0; link < nLinks; ++link )
329 unpack( link, buffer_[ link ], discreteFunction, operation );
333 exchangeTime_ += recvTimer.elapsed();
335 #if HAVE_DUNE_ALUGRID
337 nonBlockingExchange_.reset();
339 return exchangeTime_;
343 template <
class DiscreteFunction >
344 double receive( DiscreteFunction& discreteFunction )
347 typedef typename DiscreteFunction :: DiscreteFunctionSpaceType
348 :: template CommDataHandle< DiscreteFunction > :: OperationType DefaultOperationType;
349 DefaultOperationType operation;
350 return receive( discreteFunction, operation );
354 template <
class DiscreteFunction>
355 void pack(
const int link, ObjectStreamType& buffer,
const DiscreteFunction& discreteFunction )
360 dependencyCache_.writeBuffer( link, buffer, discreteFunction );
363 template <
class DiscreteFunction,
class Operation>
364 void unpack(
const int link, ObjectStreamType& buffer,
365 DiscreteFunction& discreteFunction,
const Operation& operation )
368 dependencyCache_.readBuffer( link, buffer, discreteFunction, operation );
372 DependencyCacheType& dependencyCache_;
373 std::unique_ptr< NonBlockingExchange > nonBlockingExchange_ ;
374 ObjectStreamVectorType buffer_;
375 double exchangeTime_ ;
380 typedef NonBlockingCommunication NonBlockingCommunicationType;
383 NonBlockingCommunicationType nonBlockingCommunication()
386 return NonBlockingCommunicationType( *
this, mySize_ );
393 DependencyCache(
const SpaceType &space,
const InterfaceType interface,
const CommunicationDirection dir )
395 gridPart_( space_.gridPart() ),
396 interface_( interface ),
398 myRank_( gridPart_.comm().rank() ),
399 mySize_( gridPart_.comm().size() ),
401 recvIndexMap_( new IndexMapType[ mySize_ ] ),
402 sendIndexMap_( new IndexMapType[ mySize_ ] ),
403 mpAccess_( new MPAccessImplType( MPIHelper::getCommunicator() ) ),
404 exchangeTime_( 0.0 ),
407 nonBlockingObjects_( 0 )
410 DependencyCache(
const DependencyCache & ) =
delete;
418 delete [] sendIndexMap_;
421 delete [] recvIndexMap_;
426 InterfaceType communicationInterface()
const
432 CommunicationDirection communicationDirection()
const
438 double buildTime()
const
444 double exchangeTime()
const
446 return exchangeTime_;
450 inline void buildMaps();
455 ++ nonBlockingObjects_;
461 --nonBlockingObjects_;
462 assert( nonBlockingObjects_ >= 0 );
465 bool noOpenCommunications()
const
471 inline void checkConsistency();
473 template<
class LS,
class IMV, InterfaceType CI >
474 inline void buildMaps( LinkBuilder< LS, IMV, CI > &handle );
478 inline int dest(
const int link )
const
480 return mpAccess().dest()[ link ];
484 inline int nlinks()
const
486 return mpAccess().nlinks();
490 inline void rebuild()
493 if( mySize_ <= 1 )
return;
496 assert( noOpenCommunications() );
500 int willRebuild = (sequence_ != space_.sequence()) ? 1 : 0;
501 const int myRebuild = willRebuild;
504 gridPart_.comm().broadcast( &willRebuild, 1 , 0);
506 assert( willRebuild == myRebuild );
510 if( sequence_ != space_.sequence() )
513 Dune::Timer buildTime;
516 sequence_ = space_.sequence();
519 buildTime_ = buildTime.elapsed();
524 template<
class DiscreteFunction,
class Operation >
525 inline void exchange( DiscreteFunction &discreteFunction,
const Operation& operation );
528 template<
class DiscreteFunction >
529 inline void writeBuffer( ObjectStreamVectorType &osv,
const DiscreteFunction &discreteFunction )
const;
532 template<
class DiscreteFunctionType,
class Operation >
533 inline void readBuffer( ObjectStreamVectorType &osv,
534 DiscreteFunctionType &discreteFunction,
535 const Operation& operation )
const;
538 inline MPAccessInterfaceType &mpAccess()
545 inline const MPAccessInterfaceType &mpAccess()
const
553 template<
class DiscreteFunctionSpace >
554 inline void writeBuffer(
const int link,
555 ObjectStreamType &str,
556 const PetscDiscreteFunction< DiscreteFunctionSpace > &discreteFunction )
const
558 DUNE_THROW(NotImplemented,
"writeBuffer not implemented for PetscDiscteteFunction" );
563 template<
class DiscreteFunction >
564 inline void writeBuffer(
const int link,
565 ObjectStreamType &str,
566 const DiscreteFunction &discreteFunction )
const
568 assert( sequence_ == space_.sequence() );
569 const auto &indexMap = sendIndexMap_[ dest( link ) ];
570 const int size = indexMap.size();
572 typedef typename DiscreteFunction :: DofType DofType;
575 typename DiscreteFunction::DiscreteFunctionSpaceType::LocalBlockIndices localBlockIndices;
576 str.reserve( size * Hybrid::size( localBlockIndices ) *
sizeof( DofType ) );
577 for(
int i = 0; i < size; ++i )
579 const auto &block = discreteFunction.dofVector()[ indexMap[ i ] ];
580 Hybrid::forEach( localBlockIndices, [ &str, &block ] (
auto &&k ) { str.writeUnchecked( block[ k ] ); } );
586 template<
class DiscreteFunctionSpace,
class Operation >
587 inline void readBuffer(
const int link,
588 ObjectStreamType &str,
589 PetscDiscreteFunction< DiscreteFunctionSpace > &discreteFunction,
590 const Operation& )
const
592 DUNE_THROW(NotImplemented,
"readBuffer not implemented for PetscDiscteteFunction" );
597 template<
class DiscreteFunction,
class Operation >
598 inline void readBuffer(
const int link,
599 ObjectStreamType &str,
600 DiscreteFunction &discreteFunction,
601 const Operation& operation )
const
603 static_assert( ! std::is_pointer< Operation > :: value,
604 "DependencyCache::readBuffer: Operation needs to be a reference!");
606 assert( sequence_ == space_.sequence() );
607 typedef typename DiscreteFunction :: DofType DofType;
610 const auto &indexMap = recvIndexMap_[ dest( link ) ];
612 const int size = indexMap.size();
614 typename DiscreteFunction::DiscreteFunctionSpaceType::LocalBlockIndices localBlockIndices;
615 assert(
static_cast< std::size_t
>( size * Hybrid::size( localBlockIndices ) *
sizeof( DofType ) ) <=
static_cast< std::size_t
>( str.size() ) );
616 for(
int i = 0; i < size; ++i )
618 auto &&block = discreteFunction.dofVector()[ indexMap[ i ] ];
619 Hybrid::forEach( localBlockIndices, [ &str, &operation, &block ] (
auto &&k ) {
621 #if HAVE_DUNE_ALUGRID
622 str.readUnchecked( value );
627 operation( value, block[ k ] );
634 template<
class Space >
635 template<
class LinkStorage,
class IndexMapVector, InterfaceType CommInterface >
636 class DependencyCache< Space > :: LinkBuilder
637 :
public CommDataHandleIF
638 < LinkBuilder< LinkStorage, IndexMapVector, CommInterface >,
639 typename SpaceType :: BlockMapperType :: GlobalKeyType >
642 typedef LinkStorage LinkStorageType;
644 typedef IndexMapVector IndexMapVectorType;
646 typedef typename SpaceType :: BlockMapperType BlockMapperType;
647 typedef typename BlockMapperType :: GlobalKeyType GlobalKeyType;
649 typedef GlobalKeyType DataType;
652 const GlobalKeyType myRank_;
653 const GlobalKeyType mySize_;
655 LinkStorageType &linkStorage_;
657 IndexMapVectorType &sendIndexMap_;
658 IndexMapVectorType &recvIndexMap_;
660 const SpaceType &space_;
661 const BlockMapperType &blockMapper_;
664 LinkBuilder( LinkStorageType &linkStorage,
665 IndexMapVectorType &sendIdxMap,
666 IndexMapVectorType &recvIdxMap,
667 const SpaceType &space )
668 : myRank_( space.gridPart().comm().rank() ),
669 mySize_( space.gridPart().comm().size() ),
670 linkStorage_( linkStorage ),
671 sendIndexMap_( sendIdxMap ),
672 recvIndexMap_( recvIdxMap ),
674 blockMapper_( space.blockMapper() )
678 void sendBackSendMaps()
681 MPAccessImplType mpAccess( MPIHelper::getCommunicator() );
684 mpAccess.removeLinkage();
686 mpAccess.insertRequestSymetric( linkStorage_ );
688 std::vector<int> dest = mpAccess.dest();
690 const int nlinks = mpAccess.nlinks();
693 ObjectStreamVectorType osv( nlinks );
703 for(
int link=0; link<nlinks; ++link)
704 sendIndexMap_[ dest[link] ].writeToBuffer( osv[link] );
707 osv = mpAccess.exchange( osv );
710 for(
int link=0; link<nlinks; ++link)
711 sendIndexMap_[ dest[link] ].readFromBuffer( osv[link] );
722 bool contains(
int dim,
int codim )
const
724 return space_.blockMapper().contains( codim );
728 bool fixedsize(
int dim,
int codim )
const
734 template<
class MessageBuffer,
class Entity >
735 void gather( MessageBuffer &buffer,
const Entity &entity )
const
738 const auto myPartitionType = entity.partitionType();
739 const bool send = EntityCommHelper< CommInterface > :: send( myPartitionType );
745 buffer.write( myRank_ );
747 const int numDofs = blockMapper_.numEntityDofs( entity );
750 typedef std::vector< GlobalKeyType > IndicesType ;
751 IndicesType indices( numDofs );
754 blockMapper_.mapEachEntityDof( entity, AssignFunctor< IndicesType >( indices ) );
757 for(
int i = 0; i < numDofs; ++i )
758 buffer.write( indices[ i ] );
763 template<
class MessageBuffer,
class Entity >
764 void scatter( MessageBuffer &buffer,
const Entity &entity,
const size_t dataSize )
772 assert( (rank >= 0) && (rank < mySize_) );
775 const auto myPartitionType = entity.partitionType();
776 const bool receive = EntityCommHelper< CommInterface > :: receive( myPartitionType );
779 linkStorage_.insert( rank );
782 typedef std::vector< GlobalKeyType > IndicesType ;
783 IndicesType indices( dataSize - 1 );
784 for(
size_t i=0; i<dataSize-1; ++i)
785 buffer.read( indices[i] );
800 sendIndexMap_[ rank ].insert( indices );
803 const int numDofs = blockMapper_.numEntityDofs( entity );
804 indices.resize( numDofs );
807 blockMapper_.mapEachEntityDof( entity, AssignFunctor< IndicesType >( indices ) );
810 recvIndexMap_[ rank ].insert( indices );
816 template<
class Entity >
817 size_t size(
const Entity &entity )
const
819 const PartitionType myPartitionType = entity.partitionType();
820 const bool send = EntityCommHelper< CommInterface > :: send( myPartitionType );
821 return (send) ? (blockMapper_.numEntityDofs( entity ) + 1) : 0;
827 template<
class Space >
828 inline void DependencyCache< Space > :: buildMaps()
830 if( interface_ == InteriorBorder_All_Interface )
832 LinkBuilder< LinkStorageType, IndexMapVectorType,
833 InteriorBorder_All_Interface >
834 handle( linkStorage_, sendIndexMap_, recvIndexMap_, space_ );
837 else if( interface_ == InteriorBorder_InteriorBorder_Interface )
839 LinkBuilder< LinkStorageType, IndexMapVectorType,
840 InteriorBorder_InteriorBorder_Interface >
841 handle( linkStorage_, sendIndexMap_, recvIndexMap_, space_ );
844 else if( interface_ == All_All_Interface )
846 LinkBuilder< LinkStorageType, IndexMapVectorType, All_All_Interface >
847 handle( linkStorage_, sendIndexMap_, recvIndexMap_, space_ );
851 DUNE_THROW( NotImplemented,
"DependencyCache for the given interface has not been implemented, yet." );
859 template<
class Space >
860 template<
class LS,
class IMV, InterfaceType CI >
861 inline void DependencyCache< Space > :: buildMaps( LinkBuilder< LS, IMV, CI > &handle )
863 linkStorage_.clear();
864 for(
int i = 0; i < mySize_; ++i )
866 recvIndexMap_[ i ].clear();
867 sendIndexMap_[ i ].clear();
871 gridPart_.communicate( handle, All_All_Interface , ForwardCommunication );
874 mpAccess().removeLinkage();
876 mpAccess().insertRequestSymetric( linkStorage_ );
879 template<
class Space >
880 inline void DependencyCache< Space > :: checkConsistency()
882 const int nLinks = nlinks();
884 ObjectStreamVectorType buffer( nLinks );
887 for(
int l=0; l<nLinks; ++l)
890 const int sendSize = sendIndexMap_[ dest( l ) ].size();
891 buffer[l].write( sendSize );
892 for(
int i=0; i<sendSize; ++i)
893 buffer[l].write( i );
897 buffer = mpAccess().exchange( buffer );
900 for(
int l=0; l<nLinks; ++l)
902 const int recvSize = recvIndexMap_[ dest( l ) ].size();
904 buffer[l].read( sendedSize );
907 if( recvSize != sendedSize )
909 DUNE_THROW(InvalidStateException,
"Sizes do not match!" << sendedSize <<
" o|r " << recvSize);
912 for(
int i=0; i<recvSize; ++i)
915 buffer[l].read( idx );
920 DUNE_THROW(InvalidStateException,
"Wrong ordering of send and recv maps!");
926 template<
class Space >
927 template<
class DiscreteFunction,
class Operation >
928 inline void DependencyCache< Space > :: exchange( DiscreteFunction &discreteFunction,
const Operation& operation )
931 if( mySize_ <= 1 )
return;
934 NonBlockingCommunicationType nbc( *
this, mySize_ );
937 nbc.send( discreteFunction );
940 exchangeTime_ = nbc.receive( discreteFunction, operation );
943 template<
class Space >
944 template<
class DiscreteFunction >
945 inline void DependencyCache< Space > :: writeBuffer( ObjectStreamVectorType &osv,
946 const DiscreteFunction &discreteFunction )
const
948 const int numLinks = nlinks();
949 for(
int link = 0; link < numLinks; ++link )
950 writeBuffer( link, osv[ link ], discreteFunction );
953 template<
class Space >
954 template<
class DiscreteFunction,
class Operation >
955 inline void DependencyCache< Space > :: readBuffer( ObjectStreamVectorType &osv, DiscreteFunction &discreteFunction,
956 const Operation& operation )
const
958 const int numLinks = nlinks();
959 for(
int link = 0; link < numLinks; ++link )
960 readBuffer( link, osv[ link ], discreteFunction, operation );
964 template <
class SpaceImp>
965 class CommManagerSingletonKey
967 const SpaceImp & space_;
968 const InterfaceType interface_;
969 const CommunicationDirection dir_;
972 CommManagerSingletonKey(
const SpaceImp & space,
973 const InterfaceType interface,
974 const CommunicationDirection dir)
975 : space_(space), interface_(interface), dir_(dir)
979 CommManagerSingletonKey(
const CommManagerSingletonKey & org)
980 : space_(org.space_), interface_(org.interface_), dir_(org.dir_)
984 bool operator == (
const CommManagerSingletonKey & otherKey)
const
987 return (&(space_.blockMapper()) == & (otherKey.space_.blockMapper()) );
991 const SpaceImp & space()
const
997 InterfaceType interface()
const
1003 CommunicationDirection direction()
const
1011 template <
class KeyImp,
class ObjectImp>
1012 class CommManagerFactory
1016 static ObjectImp * createObject(
const KeyImp & key )
1018 return new ObjectImp(key.space(), key.interface(), key.direction());
1022 static void deleteObject( ObjectImp * obj )
1029 template <
class SpaceImp>
1030 class CommunicationManager
1032 typedef CommunicationManager<SpaceImp>
ThisType;
1035 typedef DependencyCache<SpaceImp> DependencyCacheType;
1037 typedef CommManagerSingletonKey<SpaceImp> KeyType;
1038 typedef CommManagerFactory<KeyType, DependencyCacheType> FactoryType;
1040 typedef SingletonList< KeyType , DependencyCacheType , FactoryType > CommunicationProviderType;
1049 typedef ALU3DSPACE MpAccessLocal MPAccessInterfaceType;
1052 DependencyCacheType &cache_;
1053 CommunicationManager(
const ThisType& org);
1059 CommunicationManager(
const SpaceType & space,
1060 const InterfaceType interface,
1061 const CommunicationDirection dir)
1063 , key_(
space_,interface,dir)
1064 , mySize_(
space_.gridPart().comm().size())
1065 , cache_(CommunicationProviderType::getObject(key_))
1069 CommunicationManager(
const SpaceType & space)
1072 , mySize_(
space_.gridPart().comm().size())
1073 , cache_(CommunicationProviderType::getObject(key_))
1077 ~CommunicationManager()
1079 CommunicationProviderType::removeObject(cache_);
1085 return cache_.communicationInterface();
1091 return cache_.communicationDirection();
1097 return cache_.buildTime();
1103 return cache_.exchangeTime();
1106 MPAccessInterfaceType& mpAccess()
1108 return cache_.mpAccess();
1114 return cache_.nonBlockingCommunication();
1119 template <
class DiscreteFunctionType>
1120 void exchange(DiscreteFunctionType & df)
const
1123 typedef typename DiscreteFunctionType :: DiscreteFunctionSpaceType
1124 :: template CommDataHandle< DiscreteFunctionType > :: OperationType DefaultOperationType;
1127 DefaultOperationType operation;
1134 template <
class DiscreteFunctionType,
class Operation>
1135 void exchange(DiscreteFunctionType & df,
const Operation& operation )
const
1137 cache_.exchange( df, operation );
1141 template <
class ObjectStreamVectorType,
class DiscreteFunctionType>
1142 void writeBuffer(ObjectStreamVectorType& osv,
const DiscreteFunctionType & df)
const
1144 cache_.writeBuffer( osv, df );
1148 template <
class ObjectStreamVectorType,
class DiscreteFunctionType>
1149 void readBuffer(ObjectStreamVectorType& osv, DiscreteFunctionType & df)
const
1151 typedef typename DiscreteFunctionType :: DiscreteFunctionSpaceType
1152 :: template CommDataHandle<DiscreteFunctionType> :: OperationType OperationType;
1155 OperationType operation;
1157 readBuffer( osv, df, operation );
1161 template <
class ObjectStreamVectorType,
class DiscreteFunctionType,
class OperationType>
1162 void readBuffer(ObjectStreamVectorType& osv, DiscreteFunctionType & df,
const OperationType& operation)
const
1164 cache_.readBuffer( osv, df , operation);
1175 class CommunicationManagerList
1178 template <
class MPAccessType,
class ObjectStreamVectorType>
1179 class DiscreteFunctionCommunicatorInterface
1182 DiscreteFunctionCommunicatorInterface()
1185 virtual ~DiscreteFunctionCommunicatorInterface()
1188 virtual MPAccessType& mpAccess() = 0;
1189 virtual void writeBuffer(ObjectStreamVectorType&)
const = 0;
1190 virtual void readBuffer(ObjectStreamVectorType&) = 0;
1191 virtual void rebuildCache() = 0;
1193 virtual bool handles ( IsDiscreteFunction &df )
const = 0;
1199 template <
class DiscreteFunctionImp,
1201 class ObjectStreamVectorType,
1202 class OperationType >
1203 class DiscreteFunctionCommunicator
1204 :
public DiscreteFunctionCommunicatorInterface<MPAccessType,ObjectStreamVectorType>
1206 typedef DiscreteFunctionImp DiscreteFunctionType;
1207 typedef typename DiscreteFunctionType :: DiscreteFunctionSpaceType DiscreteFunctionSpaceType;
1209 typedef CommunicationManager<DiscreteFunctionSpaceType> CommunicationManagerType;
1212 DiscreteFunctionType& df_;
1214 CommunicationManagerType comm_;
1217 const OperationType operation_;
1220 DiscreteFunctionCommunicator(DiscreteFunctionType& df,
const OperationType& op )
1221 : df_(df), comm_(df_.space()), operation_( op )
1225 virtual MPAccessType& mpAccess()
1227 return comm_.mpAccess();
1231 virtual void writeBuffer(ObjectStreamVectorType& osv)
const
1233 comm_.writeBuffer(osv,df_);
1237 virtual void readBuffer(ObjectStreamVectorType& osv)
1239 comm_.readBuffer(osv, df_, operation_ );
1243 virtual void rebuildCache()
1245 comm_.rebuildCache();
1248 virtual bool handles ( IsDiscreteFunction &df )
const {
return (&
static_cast< IsDiscreteFunction &
>( df_ ) == &df); }
1252 typedef ALU3DSPACE ObjectStream ObjectStreamType;
1255 typedef std::vector< ObjectStreamType > ObjectStreamVectorType;
1258 typedef ALU3DSPACE MpAccessLocal MPAccessInterfaceType;
1261 typedef DiscreteFunctionCommunicatorInterface<MPAccessInterfaceType,ObjectStreamVectorType>
1262 CommObjInterfaceType;
1265 typedef std::list < std::unique_ptr< CommObjInterfaceType > > CommObjListType;
1266 CommObjListType objList_;
1273 template <
class CombinedObjectType>
1278 cObj.addToList(*
this);
1289 template <
class DiscreteFunctionImp,
class Operation>
1290 void addToList(DiscreteFunctionImp &df,
const Operation& operation )
1293 typedef DiscreteFunctionCommunicator<DiscreteFunctionImp,
1294 MPAccessInterfaceType,
1295 ObjectStreamVectorType,
1296 Operation > CommObj;
1297 CommObj * obj =
new CommObj(df, operation);
1298 objList_.push_back( std::unique_ptr< CommObjInterfaceType > (obj) );
1304 MPAccessInterfaceType& mpAccess = objList_.front()->mpAccess();
1307 mySize_ = mpAccess.psize();
1312 template <
class DiscreteFunctionImp>
1315 DFCommunicationOperation::Copy operation;
1319 template<
class DiscreteFunction >
1322 const auto handles = [ &df ] (
const std::unique_ptr< CommObjInterfaceType > &commObj ) { assert( commObj );
return commObj->handles( df ); };
1323 CommObjListType::reverse_iterator pos = std::find_if( objList_.rbegin(), objList_.rend(), handles );
1324 if( pos != objList_.rend() )
1325 objList_.erase( --pos.base() );
1327 DUNE_THROW( RangeError,
"Trying to remove discrete function that was never added" );
1335 if( mySize_ <= 1 ) return ;
1338 if(objList_.size() > 0)
1341 for(
auto& elem : objList_)
1342 elem->rebuildCache();
1345 auto& mpAccess = objList_.front()->mpAccess();
1348 ObjectStreamVectorType osv( mpAccess.nlinks() );
1351 for(
auto& elem : objList_)
1352 elem->writeBuffer(osv);
1355 osv = mpAccess.exchange(osv);
1358 for(
auto& elem : objList_)
1359 elem->readBuffer(osv);
const SpaceType & space_
Definition: communicationmanager.hh:161
double buildTime() const
return time needed for last build
Definition: communicationmanager.hh:199
InterfaceType communicationInterface() const
return communication interface
Definition: communicationmanager.hh:185
DefaultCommunicationManager< SpaceImp > ThisType
Definition: communicationmanager.hh:82
SpaceImp SpaceType
Definition: communicationmanager.hh:79
void exchange() const
Definition: communicationmanager.hh:373
void exchange(DiscreteFunction &discreteFunction) const
exchange data for a discrete function using the copy operation
Definition: communicationmanager.hh:221
CommunicationDirection communicationDirection() const
return communication direction
Definition: communicationmanager.hh:190
NonBlockingCommunication NonBlockingCommunicationType
Definition: communicationmanager.hh:169
NonBlockingCommunicationType nonBlockingCommunication() const
return object for non-blocking communication
Definition: communicationmanager.hh:211
double exchangeTime() const
return time needed for last exchange of data
Definition: communicationmanager.hh:205
void addToList(DiscreteFunctionImp &df, const Operation &operation)
add discrete function to communication list
Definition: communicationmanager.hh:345
void removeFromList(DiscreteFunction &df)
Definition: communicationmanager.hh:361
CommunicationManagerList()=default
Definition: bindguard.hh:11
bool operator==(const Double &a, const Double &b)
Definition: double.hh:601
static void forEach(IndexRange< T, sz > range, F &&f)
Definition: hybrid.hh:129
base class for determing whether a class is a discrete function or not
Definition: common/discretefunction.hh:52
Definition: cachedcommmanager.hh:47
Definition: commindexmap.hh:16