46 #ifndef MUELU_COORDINATESTRANSFER_FACTORY_KOKKOS_DEF_HPP 47 #define MUELU_COORDINATESTRANSFER_FACTORY_KOKKOS_DEF_HPP 51 #include <Xpetra_ImportFactory.hpp> 52 #include <Xpetra_IO.hpp> 53 #include <Xpetra_MultiVectorFactory.hpp> 54 #include <Xpetra_MapFactory.hpp> 56 #include "MueLu_Aggregates_kokkos.hpp" 57 #include "MueLu_CoarseMapFactory_kokkos.hpp" 60 #include "MueLu_Utilities_kokkos.hpp" 64 template <
class Scalar,
class LocalOrdinal,
class GlobalOrdinal,
class DeviceType>
65 RCP<const ParameterList> CoordinatesTransferFactory_kokkos<Scalar,LocalOrdinal,GlobalOrdinal,Kokkos::Compat::KokkosDeviceWrapperNode<DeviceType>>::GetValidParameterList()
const {
66 RCP<ParameterList> validParamList =
rcp(
new ParameterList());
68 validParamList->set< RCP<const FactoryBase> >(
"Coordinates", Teuchos::null,
"Factory for coordinates generation");
69 validParamList->set< RCP<const FactoryBase> >(
"Aggregates", Teuchos::null,
"Factory for coordinates generation");
70 validParamList->set< RCP<const FactoryBase> >(
"CoarseMap", Teuchos::null,
"Generating factory of the coarse map");
71 validParamList->set<
int > (
"write start", -1,
"First level at which coordinates should be written to file");
72 validParamList->set<
int > (
"write end", -1,
"Last level at which coordinates should be written to file");
74 return validParamList;
77 template <
class Scalar,
class LocalOrdinal,
class GlobalOrdinal,
class DeviceType>
78 void CoordinatesTransferFactory_kokkos<Scalar, LocalOrdinal, GlobalOrdinal, Kokkos::Compat::KokkosDeviceWrapperNode<DeviceType>>::DeclareInput(Level& fineLevel, Level& coarseLevel)
const {
79 static bool isAvailableCoords =
false;
82 isAvailableCoords = coarseLevel.IsAvailable(
"Coordinates",
this);
84 if (isAvailableCoords ==
false) {
85 Input(fineLevel,
"Coordinates");
86 Input(fineLevel,
"Aggregates");
87 Input(fineLevel,
"CoarseMap");
91 template <
class Scalar,
class LocalOrdinal,
class GlobalOrdinal,
class DeviceType>
92 void CoordinatesTransferFactory_kokkos<Scalar, LocalOrdinal, GlobalOrdinal, Kokkos::Compat::KokkosDeviceWrapperNode<DeviceType>>::Build(Level& fineLevel, Level& coarseLevel)
const {
93 FactoryMonitor m(*
this,
"Build", coarseLevel);
95 typedef Xpetra::MultiVector<double,LO,GO,NO> doubleMultiVector;
97 GetOStream(
Runtime0) <<
"Transferring coordinates" << std::endl;
99 if (coarseLevel.IsAvailable(
"Coordinates",
this)) {
100 GetOStream(
Runtime0) <<
"Reusing coordinates" << std::endl;
104 auto aggregates = Get<RCP<Aggregates_kokkos> >(fineLevel,
"Aggregates");
105 auto fineCoords = Get<RCP<doubleMultiVector> >(fineLevel,
"Coordinates");
106 auto coarseMap = Get<RCP<const Map> > (fineLevel,
"CoarseMap");
112 ArrayView<const GO> elementAList = coarseMap->getNodeElementList();
113 GO indexBase = coarseMap->getIndexBase();
116 if (rcp_dynamic_cast<const StridedMap>(coarseMap) != Teuchos::null)
117 blkSize = rcp_dynamic_cast<
const StridedMap>(coarseMap)->getFixedBlockSize();
119 Array<GO> elementList;
120 ArrayView<const GO> elementListView;
124 elementListView = elementAList;
127 auto numElements = elementAList.size() / blkSize;
129 elementList.resize(numElements);
132 for (LO i = 0; i < Teuchos::as<LO>(numElements); i++)
133 elementList[i] = (elementAList[i*blkSize]-indexBase)/blkSize + indexBase;
135 elementListView = elementList;
138 auto uniqueMap = fineCoords->getMap();
140 elementListView, indexBase, coarseMap->getComm());
141 RCP<doubleMultiVector> coarseCoords = Xpetra::MultiVectorFactory<double,LO,GO,NO>::Build(coarseCoordMap, fineCoords->getNumVectors());
144 RCP<doubleMultiVector> ghostedCoords = fineCoords;
145 if (aggregates->AggregatesCrossProcessors()) {
146 auto nonUniqueMap = aggregates->GetMap();
147 auto importer = ImportFactory::Build(uniqueMap, nonUniqueMap);
149 ghostedCoords = Xpetra::MultiVectorFactory<double,LO,GO,NO>::Build(nonUniqueMap, fineCoords->getNumVectors());
150 ghostedCoords->doImport(*fineCoords, *importer, Xpetra::INSERT);
155 auto aggGraph = aggregates->GetGraph();
156 auto numAggs = aggGraph.numRows();
158 auto fineCoordsView = fineCoords ->template getLocalView<DeviceType>();
159 auto coarseCoordsView = coarseCoords->template getLocalView<DeviceType>();
163 SubFactoryMonitor m2(*
this,
"AverageCoords", coarseLevel);
165 const auto dim = fineCoords->getNumVectors();
167 typename AppendTrait<decltype(fineCoordsView), Kokkos::RandomAccess>::type fineCoordsRandomView = fineCoordsView;
168 for (
size_t j = 0; j < dim; j++) {
169 Kokkos::parallel_for(
"MueLu:CoordinatesTransferF:Build:coord", Kokkos::RangePolicy<local_ordinal_type, execution_space>(0, numAggs),
170 KOKKOS_LAMBDA(
const LO i) {
174 auto aggregate = aggGraph.rowConst(i);
177 for (
size_t colID = 0; colID < static_cast<size_t>(aggregate.length); colID++)
178 sum += fineCoordsRandomView(aggregate(colID),j);
180 coarseCoordsView(i,j) = sum / aggregate.length;
185 Set<RCP<doubleMultiVector> >(coarseLevel,
"Coordinates", coarseCoords);
187 const ParameterList& pL = GetParameterList();
188 int writeStart = pL.get<
int>(
"write start"), writeEnd = pL.get<
int>(
"write end");
189 if (writeStart == 0 && fineLevel.GetLevelID() == 0 && writeStart <= writeEnd) {
190 std::string fileName =
"coordinates_before_rebalance_level_" +
toString(fineLevel.GetLevelID()) +
".m";
191 Xpetra::IO<double,LO,GO,NO>::Write(fileName, *fineCoords);
193 if (writeStart <= coarseLevel.GetLevelID() && coarseLevel.GetLevelID() <= writeEnd) {
194 std::string fileName =
"coordinates_before_rebalance_level_" +
toString(coarseLevel.GetLevelID()) +
".m";
195 Xpetra::IO<double,LO,GO,NO>::Write(fileName,*coarseCoords);
201 #endif // MUELU_COORDINATESTRANSFER_FACTORY_KOKKOS_DEF_HPP std::string toString(const T &what)
Little helper function to convert non-string types to strings.
One-liner description of what is happening.
Namespace for MueLu classes and methods.
TEUCHOS_DEPRECATED RCP< T > rcp(T *p, Dealloc_T dealloc, bool owns_mem)