45 #include "Teuchos_RCP.hpp" 46 #include "Teuchos_Comm.hpp" 47 #include "Teuchos_Assert.hpp" 49 #include "Tpetra_CrsMatrix.hpp" 50 #include "Tpetra_RowMatrixTransposer.hpp" 61 #include "Panzer_Workset_Builder.hpp" 64 #include "Phalanx_KokkosDeviceTypes.hpp" 67 #include <unordered_set> 68 #include <unordered_map> 81 PHX::View<panzer::GlobalOrdinal*> & globals)
84 std::vector<shards::CellTopology> elementBlockTopologies;
86 const shards::CellTopology & topology = elementBlockTopologies[0];
89 for(
const auto & other_topology : elementBlockTopologies){
90 TEUCHOS_ASSERT(other_topology.getKey() == topology.getKey());
93 Teuchos::RCP<panzer::FieldPattern> cell_pattern;
94 if(topology.getDimension() == 1){
96 }
else if(topology.getDimension() == 2){
98 }
else if(topology.getDimension() == 3){
106 std::vector<std::string> block_ids;
109 std::size_t totalSize = 0;
110 for (std::size_t which_blk=0;which_blk<block_ids.size();which_blk++) {
112 const std::vector<int> & localIDs = conn.
getElementBlock(block_ids[which_blk]);
113 totalSize += localIDs.size();
115 globals = PHX::View<panzer::GlobalOrdinal*>(
"global_cells",totalSize);
116 auto globals_h = Kokkos::create_mirror_view(globals);
118 for (std::size_t
id=0;
id<totalSize; ++id) {
121 TEUCHOS_ASSERT(n_conn==1);
124 globals_h(
id) = connectivity[0];
126 Kokkos::deep_copy(globals, globals_h);
138 std::vector<shards::CellTopology> elementBlockTopologies;
140 const shards::CellTopology & topology = elementBlockTopologies[0];
143 for(
const auto & other_topology : elementBlockTopologies){
144 TEUCHOS_ASSERT(other_topology.getKey() == topology.getKey());
151 std::vector<std::string> block_ids;
155 std::size_t totalCells=0, maxNodes=0;
156 for (std::size_t which_blk=0;which_blk<block_ids.size();which_blk++) {
158 const std::vector<int> & localIDs = conn.
getElementBlock(block_ids[which_blk]);
159 if ( localIDs.size() == 0 )
163 totalCells += localIDs.size();
164 maxNodes = maxNodes<Teuchos::as<std::size_t>(thisSize) ? Teuchos::as<std::size_t>(thisSize) : maxNodes;
166 globals = PHX::View<panzer::GlobalOrdinal**>(
"cell_to_node",totalCells,maxNodes);
167 auto globals_h = Kokkos::create_mirror_view(globals);
170 for (std::size_t
id=0;
id<totalCells; ++id) {
174 for(
int n=0;n<nodeCnt;n++)
175 globals_h(
id,n) = connectivity[n];
177 Kokkos::deep_copy(globals, globals_h);
182 Teuchos::RCP<const Tpetra::Map<panzer::LocalOrdinal,panzer::GlobalOrdinal,panzer::TpetraNodeType> >
183 buildNodeMap(
const Teuchos::RCP<
const Teuchos::Comm<int> > & comm,
184 PHX::View<const panzer::GlobalOrdinal**> cells_to_nodes)
201 typedef Tpetra::Map<panzer::LocalOrdinal,panzer::GlobalOrdinal,panzer::TpetraNodeType> map_type;
204 auto cells_to_nodes_h = Kokkos::create_mirror_view(cells_to_nodes);
205 Kokkos::deep_copy(cells_to_nodes_h, cells_to_nodes);
206 std::set<panzer::GlobalOrdinal> global_nodes;
207 for(
unsigned int i=0;i<cells_to_nodes.extent(0);i++)
208 for(
unsigned int j=0;j<cells_to_nodes.extent(1);j++)
209 global_nodes.insert(cells_to_nodes_h(i,j));
212 PHX::View<panzer::GlobalOrdinal*> node_ids(
"global_nodes",global_nodes.size());
213 auto node_ids_h = Kokkos::create_mirror_view(node_ids);
215 for(
auto itr=global_nodes.begin();itr!=global_nodes.end();++itr,++i)
216 node_ids_h(i) = *itr;
217 Kokkos::deep_copy(node_ids, node_ids_h);
222 return rcp(
new map_type(Teuchos::OrdinalTraits<Tpetra::global_size_t>::invalid(),node_ids,0,comm));
228 Teuchos::RCP<Tpetra::CrsMatrix<panzer::LocalOrdinal,panzer::LocalOrdinal,panzer::GlobalOrdinal,panzer::TpetraNodeType> >
229 buildNodeToCellMatrix(
const Teuchos::RCP<
const Teuchos::Comm<int> > & comm,
230 PHX::View<const panzer::GlobalOrdinal*> owned_cells,
231 PHX::View<const panzer::GlobalOrdinal**> owned_cells_to_nodes)
236 typedef Tpetra::Map<panzer::LocalOrdinal,panzer::GlobalOrdinal,panzer::TpetraNodeType> map_type;
237 typedef Tpetra::CrsMatrix<panzer::LocalOrdinal,panzer::LocalOrdinal,panzer::GlobalOrdinal,panzer::TpetraNodeType> crs_type;
238 typedef Tpetra::Import<panzer::LocalOrdinal,panzer::GlobalOrdinal,panzer::TpetraNodeType> import_type;
241 PANZER_FUNC_TIME_MONITOR_DIFF(
"panzer_stk::buildNodeToCellMatrix",BNTCM);
243 TEUCHOS_ASSERT(owned_cells.extent(0)==owned_cells_to_nodes.extent(0));
248 auto node_map = buildNodeMap(comm,owned_cells_to_nodes);
251 auto unique_node_map = Tpetra::createOneToOne(node_map);
254 RCP<map_type> cell_map = rcp(
new map_type(Teuchos::OrdinalTraits<Tpetra::global_size_t>::invalid(),owned_cells,0,comm));
258 RCP<crs_type> cell_to_node;
260 PANZER_FUNC_TIME_MONITOR_DIFF(
"Build matrix",BuildMatrix);
263 const unsigned int num_local_cells = owned_cells_to_nodes.extent(0);
264 const unsigned int num_nodes_per_cell = owned_cells_to_nodes.extent(1);
267 cell_to_node = rcp(
new crs_type(cell_map,num_nodes_per_cell,
268 Tpetra::StaticProfile));
270 std::vector<panzer::LocalOrdinal> local_node_indexes(num_nodes_per_cell);
271 std::vector<panzer::GlobalOrdinal> global_node_indexes(num_nodes_per_cell);
272 auto owned_cells_h = Kokkos::create_mirror_view(owned_cells);
273 auto owned_cells_to_nodes_h = Kokkos::create_mirror_view(owned_cells_to_nodes);
274 Kokkos::deep_copy(owned_cells_h, owned_cells);
275 Kokkos::deep_copy(owned_cells_to_nodes_h, owned_cells_to_nodes);
276 for(
unsigned int i=0;i<num_local_cells;i++) {
277 const panzer::GlobalOrdinal global_cell_index = owned_cells_h(i);
278 for(
unsigned int j=0;j<num_nodes_per_cell;j++) {
279 local_node_indexes[j] = Teuchos::as<panzer::LocalOrdinal>(j);
280 global_node_indexes[j] = owned_cells_to_nodes_h(i,j);
284 cell_to_node->insertGlobalValues(global_cell_index,global_node_indexes,local_node_indexes);
286 cell_to_node->fillComplete(unique_node_map,cell_map);
291 RCP<crs_type> node_to_cell;
293 PANZER_FUNC_TIME_MONITOR_DIFF(
"Tranpose matrix",TransposeMatrix);
296 Tpetra::RowMatrixTransposer<panzer::LocalOrdinal,panzer::LocalOrdinal,panzer::GlobalOrdinal,panzer::TpetraNodeType> transposer(cell_to_node);
299 auto trans = transposer.createTranspose();
303 RCP<import_type>
import = rcp(
new import_type(unique_node_map,node_map));
307 node_to_cell = rcp(
new crs_type(node_map,0));
310 node_to_cell->doImport(*trans,*
import,Tpetra::INSERT);
314 node_to_cell->fillComplete(cell_map,unique_node_map);
323 PHX::View<panzer::GlobalOrdinal*>
324 buildGhostedCellOneRing(
const Teuchos::RCP<
const Teuchos::Comm<int> > & comm,
325 PHX::View<const panzer::GlobalOrdinal*> cells,
326 PHX::View<const panzer::GlobalOrdinal**> cells_to_nodes)
329 PANZER_FUNC_TIME_MONITOR_DIFF(
"panzer_stk::buildGhostedCellOneRing",BGCOR);
330 typedef Tpetra::CrsMatrix<int,int,panzer::GlobalOrdinal,panzer::TpetraNodeType> crs_type;
359 Teuchos::RCP<crs_type> node_to_cell = buildNodeToCellMatrix(comm,cells,cells_to_nodes);
362 std::unordered_set<panzer::GlobalOrdinal> unique_cells;
366 auto cells_h = Kokkos::create_mirror_view(cells);
367 Kokkos::deep_copy(cells_h, cells);
368 for(
size_t i=0;i<cells.extent(0);i++) {
369 unique_cells.insert(cells_h(i));
373 std::set<panzer::GlobalOrdinal> ghstd_cells_set;
377 auto node_map = node_to_cell->getMap()->getMyGlobalIndices();
380 for(
size_t i=0;i<node_map.extent(0);i++) {
381 const panzer::GlobalOrdinal global_node_index = node_map(i);
382 size_t numEntries = node_to_cell->getNumEntriesInGlobalRow(node_map(i));
383 typename crs_type::nonconst_global_inds_host_view_type indices(
"indices", numEntries);
384 typename crs_type::nonconst_values_host_view_type values(
"values", numEntries);
387 node_to_cell->getGlobalRowCopy(global_node_index,indices,values,numEntries);
389 for(
size_t i=0; i<indices.extent(0); ++i) {
390 auto index = indices(i);
393 if(unique_cells.find(index)==unique_cells.end()) {
394 ghstd_cells_set.insert(index);
395 unique_cells.insert(index);
402 PHX::View<panzer::GlobalOrdinal*> ghstd_cells(
"ghstd_cells",ghstd_cells_set.size());
403 auto ghstd_cells_h = Kokkos::create_mirror_view(ghstd_cells);
404 for(
auto global_cell_index : ghstd_cells_set) {
405 ghstd_cells_h(indx) = global_cell_index;
410 Kokkos::deep_copy(ghstd_cells, ghstd_cells_h);
416 namespace partitioning_utilities
422 const std::vector<panzer::LocalOrdinal> & owned_parent_cells,
425 using GO = panzer::GlobalOrdinal;
426 using LO = panzer::LocalOrdinal;
428 PANZER_FUNC_TIME_MONITOR_DIFF(
"panzer::partitioning_utilities::setupSubLocalMeshInfo",setupSLMI);
442 const int num_owned_cells = owned_parent_cells.size();
443 TEUCHOS_TEST_FOR_EXCEPT_MSG(num_owned_cells == 0,
"panzer::partitioning_utilities::setupSubLocalMeshInfo : Input parent subcells must exist (owned_parent_cells)");
446 TEUCHOS_TEST_FOR_EXCEPT_MSG(num_parent_owned_cells <= 0,
"panzer::partitioning_utilities::setupSubLocalMeshInfo : Input parent info must contain owned cells");
453 TEUCHOS_ASSERT(static_cast<int>(parent_info.
cell_to_faces.extent(0)) == num_parent_total_cells);
454 const int num_faces_per_cell = parent_info.
cell_to_faces.extent(1);
458 std::vector<LO> ghstd_parent_cells;
459 std::vector<LO> virtual_parent_cells;
461 PANZER_FUNC_TIME_MONITOR_DIFF(
"Construct parent cell vector",ParentCell);
464 std::unordered_set<LO> owned_parent_cells_set(owned_parent_cells.begin(), owned_parent_cells.end());
471 const int virtual_parent_cell_offset = num_parent_owned_cells + num_parent_ghstd_cells;
473 std::unordered_set<LO> ghstd_parent_cells_set;
474 std::unordered_set<LO> virtual_parent_cells_set;
475 auto cell_to_faces_h = Kokkos::create_mirror_view(parent_info.
cell_to_faces);
476 auto face_to_cells_h = Kokkos::create_mirror_view(parent_info.
face_to_cells);
477 Kokkos::deep_copy(cell_to_faces_h, parent_info.
cell_to_faces);
478 Kokkos::deep_copy(face_to_cells_h, parent_info.
face_to_cells);
479 for(
int i=0;i<num_owned_cells;++i){
480 const LO parent_cell_index = owned_parent_cells[i];
481 for(
int local_face_index=0;local_face_index<num_faces_per_cell;++local_face_index){
482 const LO parent_face = cell_to_faces_h(parent_cell_index, local_face_index);
490 const LO neighbor_parent_side = (face_to_cells_h(parent_face,0) == parent_cell_index) ? 1 : 0;
493 const LO neighbor_parent_cell = face_to_cells_h(parent_face, neighbor_parent_side);
496 TEUCHOS_ASSERT(neighbor_parent_cell >= 0);
499 if(neighbor_parent_cell >= virtual_parent_cell_offset){
500 virtual_parent_cells_set.insert(neighbor_parent_cell);
501 }
else if(neighbor_parent_cell >= num_parent_owned_cells){
504 ghstd_parent_cells_set.insert(neighbor_parent_cell);
508 if(owned_parent_cells_set.find(neighbor_parent_cell) == owned_parent_cells_set.end()){
510 ghstd_parent_cells_set.insert(neighbor_parent_cell);
519 ghstd_parent_cells.assign(ghstd_parent_cells_set.begin(), ghstd_parent_cells_set.end());
520 virtual_parent_cells.assign(virtual_parent_cells_set.begin(), virtual_parent_cells_set.end());
524 const int num_ghstd_cells = ghstd_parent_cells.size();
525 const int num_virtual_cells = virtual_parent_cells.size();
526 const int num_real_cells = num_owned_cells + num_ghstd_cells;
527 const int num_total_cells = num_real_cells + num_virtual_cells;
529 std::vector<std::pair<LO, LO> > all_parent_cells(num_total_cells);
530 for (std::size_t i=0; i< owned_parent_cells.size(); ++i)
531 all_parent_cells[i] = std::pair<LO, LO>(owned_parent_cells[i], i);
533 for (std::size_t i=0; i< ghstd_parent_cells.size(); ++i) {
534 LO insert = owned_parent_cells.size()+i;
535 all_parent_cells[insert] = std::pair<LO, LO>(ghstd_parent_cells[i], insert);
538 for (std::size_t i=0; i< virtual_parent_cells.size(); ++i) {
539 LO insert = owned_parent_cells.size()+ ghstd_parent_cells.size()+i;
540 all_parent_cells[insert] = std::pair<LO, LO>(virtual_parent_cells[i], insert);
550 TEUCHOS_ASSERT(static_cast<int>(parent_info.
cell_vertices.extent(0)) == num_parent_total_cells);
551 TEUCHOS_ASSERT(static_cast<int>(parent_info.
local_cells.extent(0)) == num_parent_total_cells);
552 TEUCHOS_ASSERT(static_cast<int>(parent_info.
global_cells.extent(0)) == num_parent_total_cells);
554 const int num_vertices_per_cell = parent_info.
cell_vertices.extent(1);
558 sub_info.
global_cells = PHX::View<GO*>(
"global_cells", num_total_cells);
559 sub_info.
local_cells = PHX::View<LO*>(
"local_cells", num_total_cells);
560 sub_info.
cell_vertices = PHX::View<double***>(
"cell_vertices", num_total_cells, num_vertices_per_cell, num_dims);
561 auto global_cells_h = Kokkos::create_mirror_view(sub_info.
global_cells);
562 auto local_cells_h = Kokkos::create_mirror_view(sub_info.
local_cells);
563 auto cell_vertices_h = Kokkos::create_mirror_view(sub_info.
cell_vertices);
564 auto p_global_cells_h = Kokkos::create_mirror_view(parent_info.
global_cells);
565 auto p_local_cells_h = Kokkos::create_mirror_view(parent_info.
local_cells);
566 auto p_cell_vertices_h = Kokkos::create_mirror_view(parent_info.
cell_vertices);
567 Kokkos::deep_copy(p_global_cells_h,parent_info.
global_cells);
568 Kokkos::deep_copy(p_local_cells_h,parent_info.
local_cells);
569 Kokkos::deep_copy(p_cell_vertices_h,parent_info.
cell_vertices);
571 for (
int cell=0; cell<num_total_cells; ++cell) {
572 const LO parent_cell = all_parent_cells[cell].first;
573 global_cells_h(cell) = p_global_cells_h(parent_cell);
574 local_cells_h(cell) = p_local_cells_h(parent_cell);
575 for(
int vertex=0;vertex<num_vertices_per_cell;++vertex){
576 for(
int dim=0;dim<num_dims;++dim){
577 cell_vertices_h(cell,vertex,dim) = p_cell_vertices_h(parent_cell,vertex,dim);
581 Kokkos::deep_copy(sub_info.
global_cells, global_cells_h);
582 Kokkos::deep_copy(sub_info.
local_cells, local_cells_h);
591 face_t(LO c0, LO c1, LO sc0, LO sc1)
606 std::vector<face_t> faces;
608 PANZER_FUNC_TIME_MONITOR_DIFF(
"Create faces",CreateFaces);
610 std::unordered_map<LO,std::unordered_map<LO, std::pair<LO,LO> > > faces_set;
612 std::sort(all_parent_cells.begin(), all_parent_cells.end());
614 auto cell_to_faces_h = Kokkos::create_mirror_view(parent_info.
cell_to_faces);
615 auto face_to_cells_h = Kokkos::create_mirror_view(parent_info.
face_to_cells);
616 auto face_to_lidx_h = Kokkos::create_mirror_view(parent_info.
face_to_lidx);
617 Kokkos::deep_copy(cell_to_faces_h, parent_info.
cell_to_faces);
618 Kokkos::deep_copy(face_to_cells_h, parent_info.
face_to_cells);
619 Kokkos::deep_copy(face_to_lidx_h, parent_info.
face_to_lidx);
620 for(
int owned_cell=0;owned_cell<num_owned_cells;++owned_cell){
621 const LO owned_parent_cell = owned_parent_cells[owned_cell];
622 for(
int local_face=0;local_face<num_faces_per_cell;++local_face){
623 const LO parent_face = cell_to_faces_h(owned_parent_cell,local_face);
630 const LO neighbor_side = (face_to_cells_h(parent_face,0) == owned_parent_cell) ? 1 : 0;
632 const LO neighbor_parent_cell = face_to_cells_h(parent_face, neighbor_side);
633 const LO neighbor_subcell_index = face_to_lidx_h(parent_face, neighbor_side);
636 std::pair<LO, LO> search_point(neighbor_parent_cell, 0);
637 auto itr = std::lower_bound(all_parent_cells.begin(), all_parent_cells.end(), search_point);
639 TEUCHOS_TEST_FOR_EXCEPT_MSG(itr == all_parent_cells.end(),
"panzer_stk::setupSubLocalMeshInfo : Neighbor cell was not found in owned, ghosted, or virtual cells");
641 const LO neighbor_cell = itr->second;
643 LO cell_0, cell_1, subcell_index_0, subcell_index_1;
644 if(owned_cell < neighbor_cell){
646 subcell_index_0 = local_face;
647 cell_1 = neighbor_cell;
648 subcell_index_1 = neighbor_subcell_index;
651 subcell_index_1 = local_face;
652 cell_0 = neighbor_cell;
653 subcell_index_0 = neighbor_subcell_index;
657 faces_set[cell_0][subcell_index_0] = std::pair<LO,LO>(cell_1, subcell_index_1);
661 for(
const auto & cell_pair : faces_set){
662 const LO cell_0 = cell_pair.first;
663 for(
const auto & subcell_pair : cell_pair.second){
664 const LO subcell_index_0 = subcell_pair.first;
665 const LO cell_1 = subcell_pair.second.first;
666 const LO subcell_index_1 = subcell_pair.second.second;
667 faces.push_back(face_t(cell_0,cell_1,subcell_index_0,subcell_index_1));
672 const int num_faces = faces.size();
674 sub_info.
face_to_cells = PHX::View<LO*[2]>(
"face_to_cells", num_faces);
675 sub_info.
face_to_lidx = PHX::View<LO*[2]>(
"face_to_lidx", num_faces);
676 sub_info.
cell_to_faces = PHX::View<LO**>(
"cell_to_faces", num_total_cells, num_faces_per_cell);
677 auto cell_to_faces_h = Kokkos::create_mirror_view(sub_info.
cell_to_faces);
678 auto face_to_cells_h = Kokkos::create_mirror_view(sub_info.
face_to_cells);
679 auto face_to_lidx_h = Kokkos::create_mirror_view(sub_info.
face_to_lidx);
683 Kokkos::deep_copy(cell_to_faces_h, -1);
685 for(
int face_index=0;face_index<num_faces;++face_index){
686 const face_t & face = faces[face_index];
688 face_to_cells_h(face_index,0) = face.cell_0;
689 face_to_cells_h(face_index,1) = face.cell_1;
691 cell_to_faces_h(face.cell_0,face.subcell_index_0) = face_index;
692 cell_to_faces_h(face.cell_1,face.subcell_index_1) = face_index;
694 face_to_lidx_h(face_index,0) = face.subcell_index_0;
695 face_to_lidx_h(face_index,1) = face.subcell_index_1;
700 Kokkos::deep_copy(sub_info.
face_to_lidx, face_to_lidx_h);
707 const int splitting_size,
708 std::vector<panzer::LocalMeshPartition> & partitions)
711 using LO = panzer::LocalOrdinal;
720 std::vector<LO> partition_cells;
721 partition_cells.resize(base_partition_size);
727 LO partition_size = base_partition_size;
732 TEUCHOS_ASSERT(partition_size != 0);
735 if(partition_size != base_partition_size)
736 partition_cells.resize(partition_size);
739 for(LO i=0; i<partition_size; ++i)
740 partition_cells[i] = cell_count+i;
749 cell_count += partition_size;
759 std::vector<panzer::LocalMeshPartition> & partitions)
768 const std::string & element_block_name = description.
getElementBlock();
776 const auto & sideset_map = mesh_info.
sidesets.at(element_block_name);
778 const std::string & sideset_name = description.
getSideset();
781 if(sideset_map.find(sideset_name) == sideset_map.end())
789 for(
auto & partition : partitions){
790 partition.sideset_name = sideset_name;
791 partition.element_block_name = element_block_name;
793 partition.has_connectivity =
true;
816 for(
auto & partition : partitions){
817 partition.element_block_name = element_block_name;
819 partition.has_connectivity =
true;
829 PHX::View<panzer::GlobalOrdinal*> & owned_cells,
830 PHX::View<panzer::GlobalOrdinal*> & ghost_cells,
831 PHX::View<panzer::GlobalOrdinal*> & virtual_cells)
837 PHX::View<panzer::GlobalOrdinal**> owned_cell_to_nodes;
838 buildCellToNodes(conn, owned_cell_to_nodes);
841 buildCellGlobalIDs(conn, owned_cells);
844 ghost_cells = buildGhostedCellOneRing(comm,owned_cells,owned_cell_to_nodes);
851 faceToElement->initialize(conn);
852 auto elems_by_face = faceToElement->getFaceToElementsMap();
853 auto face_to_lidx = faceToElement->getFaceToCellLocalIdxMap();
855 const panzer::LocalOrdinal num_owned_cells = owned_cells.extent(0);
866 std::vector<int> all_boundary_faces;
867 const int num_faces = elems_by_face.extent(0);
868 auto elems_by_face_h = Kokkos::create_mirror_view(elems_by_face);
869 Kokkos::deep_copy(elems_by_face_h, elems_by_face);
870 for(
int face=0;face<num_faces;++face)
871 if(elems_by_face_h(face,0) < 0 or elems_by_face_h(face,1) < 0)
872 all_boundary_faces.push_back(face);
873 const panzer::LocalOrdinal num_virtual_cells = all_boundary_faces.size();
877 virtual_cells = PHX::View<panzer::GlobalOrdinal*>(
"virtual_cells",num_virtual_cells);
878 auto virtual_cells_h = Kokkos::create_mirror_view(virtual_cells);
880 PANZER_FUNC_TIME_MONITOR_DIFF(
"Initial global index creation",InitialGlobalIndexCreation);
882 const int num_ranks = comm->getSize();
883 const int rank = comm->getRank();
885 std::vector<panzer::GlobalOrdinal> owned_cell_distribution(num_ranks,0);
887 std::vector<panzer::GlobalOrdinal> my_owned_cell_distribution(num_ranks,0);
888 my_owned_cell_distribution[rank] = num_owned_cells;
890 Teuchos::reduceAll(*comm,Teuchos::REDUCE_SUM, num_ranks, my_owned_cell_distribution.data(),owned_cell_distribution.data());
893 std::vector<panzer::GlobalOrdinal> virtual_cell_distribution(num_ranks,0);
895 std::vector<panzer::GlobalOrdinal> my_virtual_cell_distribution(num_ranks,0);
896 my_virtual_cell_distribution[rank] = num_virtual_cells;
898 Teuchos::reduceAll(*comm,Teuchos::REDUCE_SUM, num_ranks, my_virtual_cell_distribution.data(),virtual_cell_distribution.data());
901 panzer::GlobalOrdinal num_global_real_cells=0;
902 for(
int i=0;i<num_ranks;++i){
903 num_global_real_cells+=owned_cell_distribution[i];
906 panzer::GlobalOrdinal global_virtual_start_idx = num_global_real_cells;
907 for(
int i=0;i<rank;++i){
908 global_virtual_start_idx += virtual_cell_distribution[i];
911 for(
int i=0;i<num_virtual_cells;++i){
912 virtual_cells_h(i) = global_virtual_start_idx + panzer::GlobalOrdinal(i);
915 Kokkos::deep_copy(virtual_cells, virtual_cells_h);
Backwards compatibility mode that ignores the worksetSize in the WorksetDescriptor.
std::map< std::string, std::map< std::string, LocalMeshSidesetInfo > > sidesets
Teuchos::RCP< const shards::CellTopology > cell_topology
PHX::View< panzer::LocalOrdinal * > local_cells
void fillLocalCellIDs(const Teuchos::RCP< const Teuchos::Comm< int >> &comm, panzer::ConnManager &conn, PHX::View< panzer::GlobalOrdinal *> &owned_cells, PHX::View< panzer::GlobalOrdinal *> &ghost_cells, PHX::View< panzer::GlobalOrdinal *> &virtual_cells)
Get the owned, ghost and virtual global cell ids for this process.
PHX::View< double *** > cell_vertices
void generateLocalMeshPartitions(const panzer::LocalMeshInfo &mesh_info, const panzer::WorksetDescriptor &description, std::vector< panzer::LocalMeshPartition > &partitions)
virtual LocalOrdinal getConnectivitySize(LocalOrdinal localElmtId) const =0
PHX::View< panzer::LocalOrdinal *[2]> face_to_cells
const std::string & getSideset(const int block=0) const
Get sideset name.
panzer::LocalOrdinal num_owned_cells
panzer::LocalOrdinal num_virtual_cells
Teuchos::RCP< const shards::CellTopology > cell_topology
virtual const std::vector< LocalOrdinal > & getElementBlock(const std::string &blockID) const =0
PHX::View< panzer::LocalOrdinal *[2]> face_to_lidx
bool requiresPartitioning() const
Do we need to partition the local mesh prior to generating worksets.
virtual void getElementBlockIds(std::vector< std::string > &elementBlockIds) const =0
Pure virtual base class for supplying mesh connectivity information to the DOF Manager.
virtual void getElementBlockTopologies(std::vector< shards::CellTopology > &elementBlockTopologies) const =0
int getWorksetSize() const
Get the requested workset size (default -2 (workset size is set elsewhere), -1 (largest possible work...
bool useSideset() const
This descriptor is for a side set.
panzer::LocalOrdinal num_ghstd_cells
std::map< std::string, LocalMeshBlockInfo > element_blocks
void splitMeshInfo(const panzer::LocalMeshInfoBase &mesh_info, const int splitting_size, std::vector< panzer::LocalMeshPartition > &partitions)
const std::string & getElementBlock(const int block=0) const
Get element block name.
void setupSubLocalMeshInfo(const panzer::LocalMeshInfoBase &parent_info, const std::vector< panzer::LocalOrdinal > &owned_parent_cells, panzer::LocalMeshInfoBase &sub_info)
PHX::View< panzer::GlobalOrdinal * > global_cells
virtual void buildConnectivity(const FieldPattern &fp)=0
Workset size is set to the total number of local elements in the MPI process.
PHX::View< panzer::LocalOrdinal ** > cell_to_faces
virtual const GlobalOrdinal * getConnectivity(LocalOrdinal localElmtId) const =0