From b8e3dd48dfe30d42435095379005632c4431c3cf Mon Sep 17 00:00:00 2001 From: Knapp Date: Fri, 23 Sep 2022 09:22:25 +0200 Subject: [PATCH 01/43] Add cmesh uniform bounds for hybrid meshes. Co-authored-by: holke johannes.holke@dlr.de --- src/t8_cmesh.h | 25 ++ src/t8_cmesh/t8_cmesh_cxx.cxx | 721 ++++++++++++++++++++++++++++++++++ 2 files changed, 746 insertions(+) diff --git a/src/t8_cmesh.h b/src/t8_cmesh.h index faa8812e9a..f124c35433 100644 --- a/src/t8_cmesh.h +++ b/src/t8_cmesh.h @@ -701,6 +701,31 @@ void t8_cmesh_uniform_bounds (t8_cmesh_t cmesh, int level, t8_gloidx_t *child_in_tree_end, int8_t *first_tree_shared); +/** + * Calculate the section of a uniform hybrid forest for the current rank + * + * \param [in] cmesh The cmesh to be considered. + * \param [in] level The uniform refinement level to be created. + * \param [in] scheme The element scheme for which to compute the bounds. + * \param [out] first_local_tree The first tree that contains elements belonging to the calling processor. + * \param [out] child_in_tree_begin The global index of the first element belonging to the calling processor. Not computed if NULL. + * \param [out] last_local_tree The last tree that contains elements belonging to the calling processor. + * \param [out] child_in_tree_end The global index of the first element that does not belonging to + * the calling processor anymore. Not computed if NULL. + * \param [out] first_tree_shared If not NULL, 1 or 0 is stored here depending on whether \a first_local_tree is the + * same as \a last_local_tree on the next process. + * \param [in] comm The communicator + */ +*/ +void t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, + int level, + t8_scheme_cxx_t * scheme, + t8_gloidx_t * first_local_tree, + t8_gloidx_t * child_in_tree_begin, + t8_gloidx_t * last_local_tree, + t8_gloidx_t * child_in_tree_end, + int8_t * first_tree_shared, sc_MPI_Comm comm) + /** Increase the reference counter of a cmesh. * \param [in,out] cmesh On input, this cmesh must exist with positive * reference count. It may be in any state. diff --git a/src/t8_cmesh/t8_cmesh_cxx.cxx b/src/t8_cmesh/t8_cmesh_cxx.cxx index facdac2631..78a1cb4a89 100644 --- a/src/t8_cmesh/t8_cmesh_cxx.cxx +++ b/src/t8_cmesh/t8_cmesh_cxx.cxx @@ -167,3 +167,724 @@ t8_cmesh_uniform_bounds (t8_cmesh_t cmesh, int level, } } + + +/* TODO: Empty processes, shared trees, binary search in offset-array to avoid recv_any, + * use partition_given to partition the cmesh*/ +void +t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, + t8_scheme_cxx_t * scheme, + t8_gloidx_t * first_local_tree, + t8_gloidx_t * child_in_tree_begin, + t8_gloidx_t * last_local_tree, + t8_gloidx_t * child_in_tree_end, + int8_t * first_tree_shared, sc_MPI_Comm comm) +{ + t8_gloidx_t cmesh_first_tree, local_num_children = 0, + elem_in_tree, first_child, last_child, *elem_index_pointer, igtree; + int send_first, send_last, send_first_nonempty, + num_procs_we_send_to = 0; + t8_cmesh_partition_query_t data; + t8_locidx_t tree_id, itree, pure_local_trees; + t8_shmem_array_t offset_array; + sc_array_t offset_partition, first_element_tree; + int ieclass; + int iproc; + int mpiret, num_messages_sent = 0; + int expect_start_message = 1; + int expect_end_message = 1; + sc_array send_requests, send_buffer; + int current_pos_in_send_buffer = 0; + t8_eclass_scheme_c *tree_scheme; + T8_ASSERT (cmesh != NULL); + const int first_tree_shared_shift = + cmesh->first_tree_shared ? 1 : 0; +#ifdef T8_ENABLE_DEBUG + int num_received_start_messages = 0; + int num_received_end_messages = 0; +#endif + + /* TODO: Clean up size_t and gloidx_t data types, ensure that each variables has the + * matching type. */ + + t8_debugf ("Into t8_cmesh_uniform_bounds_hybrid.\n"); + + if (t8_cmesh_is_empty (cmesh)) { + t8_cmesh_uniform_set_return_parameters_to_empty (first_local_tree, + child_in_tree_begin, + last_local_tree, + child_in_tree_end, + first_tree_shared); + return; + } + +#ifdef T8_ENABLE_DEBUG + { + /* Check that comm matches mpirank and size stored in cmesh */ + int mpirank, mpisize; + mpiret = sc_MPI_Comm_rank (comm, &mpirank); + SC_CHECK_MPI (mpiret); + mpiret = sc_MPI_Comm_size (comm, &mpisize); + SC_CHECK_MPI (mpiret); + T8_ASSERT (mpirank == cmesh->mpirank); + T8_ASSERT (mpisize == cmesh->mpisize); + } +#endif + /*Compute number of local elements. Ignore shared trees */ + for (ieclass = T8_ECLASS_ZERO; ieclass < T8_ECLASS_COUNT; ieclass++) { + tree_scheme = scheme->eclass_schemes[ieclass]; + local_num_children += cmesh->num_local_trees_per_eclass[ieclass] * + tree_scheme->t8_element_count_leafs_from_root (level); + } + + /* Do not consider shared trees */ + if (cmesh->first_tree_shared && cmesh->set_partition) { + ieclass = t8_cmesh_get_tree_class (cmesh, 0); + tree_scheme = scheme->eclass_schemes[ieclass]; + local_num_children -= + tree_scheme->t8_element_count_leafs_from_root (level); + } + + /* + * + * Cmesh is not partitioned + * + */ + /* If the initial cmesh is not partitioned, every process knows "everything" and we do not + * need any communication.*/ + if (!cmesh->set_partition) { + t8_gloidx_t current_tree_element_offset; + const t8_gloidx_t num_trees = t8_cmesh_get_num_trees (cmesh); + t8_debugf ("Cmesh is not partitioned.\n"); + /* Compute the first and last element of this process. Then loop over + * all trees to find the trees in which these are contained. + * We cast to long double and double to prevent overflow. */ + first_child = + t8_cmesh_get_first_element_of_process (cmesh->mpirank, cmesh->mpisize, + local_num_children); + last_child = + t8_cmesh_get_first_element_of_process (cmesh->mpirank + 1, + cmesh->mpisize, + local_num_children) - 1; + t8_debugf ("[H] hybrid fc = %li, lc = %li\n", first_child, last_child); + + /* Can't we optimize this linear loop by using a binary search? + * -> No, we cannot. Since we need in any case compute the t8_element_count_leafs_from_root + * for each tree. + */ + current_tree_element_offset = 0; + for (igtree = 0; igtree < num_trees; ++igtree) { + ieclass = t8_cmesh_get_tree_class (cmesh, (t8_locidx_t) igtree); + tree_scheme = scheme->eclass_schemes[ieclass]; + /* TODO: We can optimize by buffering the elem_in_tree value. Thus, if + the computation is expensive (may be for non-morton-type schemes), + we do it only once. */ + elem_in_tree = tree_scheme->t8_element_count_leafs_from_root (level); + /* Check if the first element is on the current tree */ + if (current_tree_element_offset <= first_child && + first_child < current_tree_element_offset + elem_in_tree) { + if (child_in_tree_begin != NULL) { + *child_in_tree_begin = first_child - current_tree_element_offset; + } + *first_local_tree = igtree; + /* If our first element is not the very first element in the tree, we share + * this tree with the previous process. */ + if (first_tree_shared != NULL) { + *first_tree_shared = + current_tree_element_offset < first_child ? 1 : 0; + } + } + if (last_child < first_child) { + /* This process is empty. We needed to identify the 'first_local_tree' since it is the + * first local tree of the next process, which we store in this case. + */ + t8_cmesh_uniform_set_return_parameters_to_empty (first_local_tree, + child_in_tree_begin, + last_local_tree, + child_in_tree_end, + first_tree_shared); + return; + } + /* Check if the last element is on the current tree */ + if (current_tree_element_offset <= last_child && + last_child < current_tree_element_offset + elem_in_tree) { + if (child_in_tree_end != NULL) { + *child_in_tree_end = last_child - current_tree_element_offset + 1; + } + *last_local_tree = igtree; + + /* We have found the last tree and can immediately return, + * ending the for loop. */ + T8_ASSERT (*first_local_tree <= *last_local_tree); + T8_ASSERT (0 <= *first_local_tree && *first_local_tree < num_trees); + T8_ASSERT (0 <= *last_local_tree && *last_local_tree < num_trees); + return; + } + current_tree_element_offset += elem_in_tree; + } + /* If we reach this part, we do not have any trees - the cmesh is empty */ + T8_ASSERT (num_trees == 0); + + t8_cmesh_uniform_set_return_parameters_to_empty (first_local_tree, + child_in_tree_begin, + last_local_tree, + child_in_tree_end, + first_tree_shared); + return; + } + + /* + * + * Cmesh is partitioned + * + */ + + t8_shmem_array_init (&offset_array, sizeof (t8_gloidx_t), + cmesh->mpisize + 1, comm); + /* Fill the offset array for each process the global index of its first element in + * the uniform partition. + * (0, l_n_c_0, l_n_c_0 + l_n_c_1, l_n_c_0 + l_n_c_1 + l_n_c_2, ...) */ + t8_shmem_prefix (&local_num_children, offset_array, 1, T8_MPI_GLOIDX, + sc_MPI_SUM); + + data.num_procs = cmesh->mpisize; + /* Get global number of elements */ + data.global_num_elements = + t8_shmem_array_get_gloidx (offset_array, cmesh->mpisize); + SC_CHECK_ABORTF (0 <= data.global_num_elements + && data.global_num_elements < T8_GLOIDX_MAX, + "Overflow in number of elements.\n"); + t8_debugf ("[H] global num %li\n", data.global_num_elements); + + /*Compute number of non-shared-trees and the local index of the first non-shared-tree */ + pure_local_trees = cmesh->num_local_trees - first_tree_shared_shift; + + if (pure_local_trees > 0) { + /* Compute which trees and elements to send to which process. + * We skip empty processes. */ + igtree = first_tree_shared_shift; + + sc_array_init_size (&first_element_tree, sizeof (t8_gloidx_t), + pure_local_trees + 1); + + /* Set the first entry of first_element_tree to the global index of + * the first element of our first pure local tree. */ + elem_index_pointer = + (t8_gloidx_t *) sc_array_index_int (&first_element_tree, 0); + *elem_index_pointer = + t8_shmem_array_get_gloidx (offset_array, cmesh->mpirank); + + cmesh_first_tree = t8_cmesh_get_first_treeid (cmesh); + tree_id = t8_cmesh_get_local_id (cmesh, cmesh_first_tree); + ieclass = t8_cmesh_get_tree_class (cmesh, tree_id); + + /* Compute the first element in every pure local tree. + * This array stores for each tree the global element index offset. + * Example: 2 local trees, each has 8 Elements. First element index 12: | 12 | 20 | 28 | */ + for (itree = 0; itree < pure_local_trees; ++itree, ++igtree) { + const t8_gloidx_t *first_element_of_tree = + (const t8_gloidx_t *) sc_array_index_int (&first_element_tree, itree); + t8_gloidx_t *first_element_of_next_tree = + (t8_gloidx_t *) sc_array_index_int (&first_element_tree, itree + 1); + ieclass = t8_cmesh_get_tree_class (cmesh, (t8_locidx_t) igtree); + tree_scheme = scheme->eclass_schemes[ieclass]; + /* Set the first element of the next tree by adding the number of element of the current tree. */ + *first_element_of_next_tree = + *first_element_of_tree + + tree_scheme->t8_element_count_leafs_from_root (level); + } + + /* Check that the last tree + 1 stores as offset the start of the next process */ + T8_ASSERT (*((t8_gloidx_t *) + t8_sc_array_index_locidx (&first_element_tree, + pure_local_trees)) == + t8_shmem_array_get_gloidx (offset_array, cmesh->mpirank + 1)); + + /* Compute the range of mpiranks to minimize the loop over processes */ + + /* Process offset must be 0 when calling t8_cmesh_determine_partition + * the first times out of sc_array_split. */ + data.process_offset = 0; + /* Compute the process that will own the first element of our first tree. */ + send_first_nonempty = + (t8_gloidx_t) t8_cmesh_determine_partition (&first_element_tree, 0, + &data); + if (!cmesh->first_tree_shared && t8_cmesh_get_global_id (cmesh, 0) == 0) { + /* If our first tree is 0 and not shared, then we need to send to all processes + * below the start process. Even if their partition is empty. */ + send_first = 0; + } + else { + send_first = send_first_nonempty; + } + + /* Compute the process that may own the last element of our first tree. */ + send_last = + (t8_gloidx_t) t8_cmesh_determine_partition (&first_element_tree, + pure_local_trees, &data); + if (send_last >= cmesh->mpisize) { + /* t8_cmesh_determine_partition will return mpisize if we plug in the number of global + * trees as tree index, which happens on the last process that has data. + * We need to correct by subtracting 1. */ + send_last = cmesh->mpisize - 1; + } + num_procs_we_send_to = send_last - send_first + 1; + + + sc_array_init_size (&offset_partition, sizeof (size_t), + num_procs_we_send_to); + /* In array split the 'types' that we compute are the processes we send to, + * but offset by the first process, so that they start at 0 and end at num_procs_we_send_to. */ + data.process_offset = send_first; + /* For each process that we send to find the first tree whose first element + * belongs to this process. + * These tree indices will be stored in offset_partition. */ + sc_array_split (&first_element_tree, &offset_partition, + (size_t) num_procs_we_send_to + 1, + t8_cmesh_determine_partition, (void *) &data); + // TODO: Is determination of last proc we send to wrong? + + /* We know: Lowest process and highest process we need to send trees to + Tree0 Tree1 TreeN + | ---- | --- | .... | --- | + send_first send_last + + We need the information: Given a process in range, in which tree do its elements start + and in which tree do they end. + Note: If proc i ends in tree j, then proc i+1 may start in tree j or tree j+1. + + offset_partition: + At position i for process send_first + i the first local tree whose first element + belongs to send_first+i. + If no such tree exists, then the index of the previous process is stored. + + Examples: + A B C + Proc 0 needs tree 0 and 1 Proc 0 needs tree 0 and 1 Proc 0 needs tree 0 and 1 + Proc 1 needs tree 1 and 2 Proc 1 needs tree 2 Proc 1 needs tree 1 + Proc 2 needs tree 1 and 2 + | 0 | 2 | 3 | | 0 | 2 | 3 | | 0 | 0 | 2 | 3 | + + Need to identify the first tree we send to proc i: + If a tree with the first element on proc i exists, then the first tree is either this tree + or the tree before. We can determine this by comparing the element_offset of the proc with the + element offset of the tree. + If no tree exists, then the process only requires elements from one tree (otherwise it would have the + first element of its second tree). + This tree must then be the last tree of the previous process. + + Need to identify last tree that a process has elements of. + Compute last element of process i via ((cmesh->mpirank+1)*data.global_num_elements)/data.num_procs - 1 + Get offset of first tree of process i+1: first_element_tree[offset_partition[i+1]] + If this index is <= to last element of process i then the last tree of i is the first tree of i+1, + otherwise this index is > and the last tree of i is the tree before the first tree of i+1. + + */ + + /* Initialize the array of MPI Requests */ + sc_array_init (&send_requests, sizeof (sc_MPI_Request)); + + /* Initialize the send buffer for all messages. + * Since we use MPI_Isend, we need to keep the buffers alive until + * we post the MPI_Wait. Since we first send all messages, we need to + * buffer them all. */ + sc_array_init (&send_buffer, sizeof (t8_gloidx_t)); + current_pos_in_send_buffer = 0; + + /* Iterate over offset_partition to find boundaries + * and send the MPI messages. */ + t8_debugf ("[H] sf: %i sl: %i\n", send_first, send_last); + + const t8_gloidx_t last_el_index_of_last_tree = + *(t8_gloidx_t *) t8_sc_array_index_gloidx (&first_element_tree, + pure_local_trees) - 1; + for (iproc = send_first; iproc <= send_last; iproc++) { + + const t8_gloidx_t first_element_index_of_current_proc = + t8_cmesh_get_first_element_of_process (iproc, data.num_procs, + data.global_num_elements); + const t8_gloidx_t last_element_index_of_current_proc = + t8_cmesh_get_first_element_of_process (iproc + 1, data.num_procs, + data.global_num_elements) - 1; + const int proc_is_empty = + last_element_index_of_current_proc < + first_element_index_of_current_proc; + int send_start_message = 1; + int send_end_message = 1; + + t8_locidx_t first_puretree_of_current_proc = -1; + t8_locidx_t last_puretree_of_current_proc = -1; + t8_gloidx_t first_element_in_tree_index_of_current_proc = -1; + t8_gloidx_t last_element_in_tree_index_of_current_proc = -2; + + if (!proc_is_empty) { + /* This process' partition is not empty. */ + const t8_locidx_t possibly_first_puretree_of_current_proc = + *((size_t *) + sc_array_index_int (&offset_partition, iproc - send_first)); + const t8_locidx_t possibly_first_puretree_of_next_proc = + *((size_t *) + sc_array_index_int (&offset_partition, iproc + 1 - send_first)); + const t8_gloidx_t first_el_index_of_first_tree = + *(t8_gloidx_t *) t8_sc_array_index_gloidx (&first_element_tree, + possibly_first_puretree_of_current_proc); + + if (first_element_index_of_current_proc >= + last_el_index_of_last_tree + 1) { + /* We do not send to this process at all. Its first element belongs + * to the next process. */ + send_start_message = send_end_message = 0; + first_puretree_of_current_proc = -1; + last_puretree_of_current_proc = -1; + } + if (send_start_message) { + /* Compute the first tree of this proc and whether we need to send a start message to it. */ + if (first_el_index_of_first_tree > + first_element_index_of_current_proc) { + /* The first element of this proc does lie on possibly_first_puretree_of_current_proc - 1. + * We check whether we own this tree and if not we do not send anything. */ + if (possibly_first_puretree_of_current_proc - 1 < 0) { + /* We do not send any Start message to the proc. */ + send_start_message = 0; + first_puretree_of_current_proc = -1; + } + else { + /* The first element of this proc lies on the previous tree. */ + first_puretree_of_current_proc = + possibly_first_puretree_of_current_proc - 1; + } + } + else { + first_puretree_of_current_proc = + possibly_first_puretree_of_current_proc; + } + } + /* Compute the last tree of this proc and whether we need to send an end message to it. */ + /* We know + * + * possibly_first_puretree_of_next_proc - The tree whose first element lies on the next process. + * + * + * If the next process is empty, then possibly_first_puretree_of_next_proc = possibly_first_puretree_of_current_proc + * and this is our last tree. + */ + if (send_end_message) { + if (last_el_index_of_last_tree < last_element_index_of_current_proc) { + /* The last element of this proc does not lie in our partition. + * We do not need to send any End information to this process. */ + send_end_message = 0; + last_puretree_of_current_proc = -1; + } + else { + if (iproc == send_last) { + /* The very last process must have our last tree as its last tree. */ + last_puretree_of_current_proc = pure_local_trees - 1; + } + else if (possibly_first_puretree_of_next_proc == + possibly_first_puretree_of_current_proc) { + /* The next process is empty. This can only happen if + * each process gets 0 or 1 element. + * Hence, the current process has only one element and it must lie on the first tree. */ + last_puretree_of_current_proc = first_puretree_of_current_proc; + } + else { + last_puretree_of_current_proc = + possibly_first_puretree_of_next_proc - 1; + } + } + } + t8_debugf ("[H] Set tree for %i: %i to %i, (#pure = %i)\n", iproc, + first_puretree_of_current_proc, + last_puretree_of_current_proc, pure_local_trees); +#ifdef T8_ENABLE_DEBUG + /* Check that the trees have valid values. */ + if (send_start_message) { + T8_ASSERT (0 <= first_puretree_of_current_proc + && first_puretree_of_current_proc < pure_local_trees); + } + else { + T8_ASSERT (first_puretree_of_current_proc == -1); + } + + if (send_end_message) { + T8_ASSERT (0 <= last_puretree_of_current_proc + && last_puretree_of_current_proc < pure_local_trees); + } + else { + T8_ASSERT (last_puretree_of_current_proc == -1); + } + if (first_puretree_of_current_proc != -1 + && last_puretree_of_current_proc != -1) { + T8_ASSERT (first_puretree_of_current_proc <= + last_puretree_of_current_proc); + } +#endif + if (send_start_message) { + /* Compute the index inside the tree of the first element. */ + const t8_gloidx_t first_el_of_first_tree = + *(t8_gloidx_t *) t8_sc_array_index_gloidx (&first_element_tree, + first_puretree_of_current_proc); + first_element_in_tree_index_of_current_proc = + first_element_index_of_current_proc - first_el_of_first_tree; + } + if (send_end_message) { + /* Compute the index inside the tree of the last element. */ + const t8_gloidx_t first_el_of_last_tree = *(t8_gloidx_t *) + t8_sc_array_index_locidx (&first_element_tree, + last_puretree_of_current_proc); + last_element_in_tree_index_of_current_proc = + last_element_index_of_current_proc - first_el_of_last_tree; + t8_debugf ("[H] Computing last element as %li - %li = %li\n\n", + last_element_index_of_current_proc, + first_el_of_last_tree, + last_element_in_tree_index_of_current_proc); + t8_debugf ("[H] Last pure tree = %i, first element in it %li\n", + last_puretree_of_current_proc, first_el_of_last_tree); + } + } + + /* + * + * MPI Communication starts here + * + */ + + /* Post the start message, we send the first tree id of the process + * and (if desired) the index of the first element in this tree. */ + if (send_start_message) { + const t8_gloidx_t global_id_of_first_tree = + proc_is_empty ? -1 : + first_puretree_of_current_proc + t8_cmesh_get_first_treeid (cmesh) + + first_tree_shared_shift; + + if (iproc != cmesh->mpirank) { + const int num_entries = 2; + t8_gloidx_t *message; + /* Allocate a new request */ + sc_MPI_Request *request = + (sc_MPI_Request *) sc_array_push (&send_requests); + + /* Grow total send buffer */ + sc_array_push_count (&send_buffer, num_entries); + /* Set message to current position in the buffer. */ + message = + (t8_gloidx_t *) sc_array_index_int (&send_buffer, + current_pos_in_send_buffer); + current_pos_in_send_buffer += num_entries; + /* Send the global id of the first tree of this process */ + message[0] = global_id_of_first_tree; + /* The index in the tree is the index of the element minus the offset of the tree. */ + message[1] = first_element_in_tree_index_of_current_proc; + t8_debugf ("[H] first e index %li, first in tree %li\n", + first_element_index_of_current_proc, + first_puretree_of_current_proc >= 0 ? *(t8_gloidx_t *) + t8_sc_array_index_gloidx (&first_element_tree, + first_puretree_of_current_proc) + : -1); + mpiret = + sc_MPI_Isend (message, num_entries, T8_MPI_GLOIDX, iproc, + T8_MPI_CMESH_UNIFORM_BOUNDS_START, comm, request); + SC_CHECK_MPI (mpiret); + t8_debugf + ("Sending start message (%li, %li) to %i (global num el %li)\n", + message[0], message[1], iproc, data.global_num_elements); + + T8_ASSERT (proc_is_empty || (0 <= message[0] + && message[0] < + t8_cmesh_get_num_trees (cmesh))); + T8_ASSERT (proc_is_empty || (0 <= message[1] + && message[1] < + data.global_num_elements)); + T8_ASSERT (!(proc_is_empty && message[0] != -1)); + T8_ASSERT (!(proc_is_empty && message[1] != -1)); + } + else { /* We are the current proc, so we just copy the data. */ + *first_local_tree = global_id_of_first_tree; + if (first_element_in_tree_index_of_current_proc > 0) { + /* The first tree is shared */ + if (first_tree_shared != NULL) { + *first_tree_shared = 1; + } + } + else { + if (first_tree_shared != NULL) { + *first_tree_shared = 0; + } + } + if (child_in_tree_begin != NULL) { + *child_in_tree_begin = + first_element_in_tree_index_of_current_proc; + } + /* We do not expect this message from another proc */ + expect_start_message = 0; +#ifdef T8_ENABLE_DEBUG + num_received_start_messages++; +#endif + + t8_debugf ("[H] Copied first tree %li element %li to self\n", + global_id_of_first_tree, + first_element_in_tree_index_of_current_proc); + } + } /* End sending of start message */ + + t8_debugf ("End message: %i\n", send_end_message); + /* Post the end message, we send the last tree id of the process + * and the index of the last element in this tree. */ + if (send_end_message) { + const t8_gloidx_t global_id_of_last_tree = + proc_is_empty ? -1 : + last_puretree_of_current_proc + t8_cmesh_get_first_treeid (cmesh) + + first_tree_shared_shift; + if (iproc != cmesh->mpirank) { + const int num_entries = 2; + t8_gloidx_t *message; + /* Allocate a new request */ + sc_MPI_Request *request = + (sc_MPI_Request *) sc_array_push (&send_requests); + + /* Grow total send buffer */ + sc_array_push_count (&send_buffer, num_entries); + /* Set message to current position in the buffer. */ + message = + (t8_gloidx_t *) sc_array_index_int (&send_buffer, + current_pos_in_send_buffer); + current_pos_in_send_buffer += num_entries; + + message[0] = global_id_of_last_tree; + /* The index in the tree is the index of the element minus the offset of the tree. */ + message[1] = last_element_in_tree_index_of_current_proc; + mpiret = + sc_MPI_Isend (message, num_entries, T8_MPI_GLOIDX, iproc, + T8_MPI_CMESH_UNIFORM_BOUNDS_END, comm, request); + SC_CHECK_MPI (mpiret); + t8_debugf ("Sending end message (%li, %li) to %i\n", message[0], + message[1], iproc); + T8_ASSERT (proc_is_empty || (0 <= message[0] + && message[0] < + t8_cmesh_get_num_trees (cmesh))); + T8_ASSERT (proc_is_empty || (0 <= message[1] + && message[1] < + data.global_num_elements)); + T8_ASSERT (!(proc_is_empty && message[0] != -1)); + T8_ASSERT (!(proc_is_empty && message[1] != -2)); + } + else { /* We are the current proc, so we just copy the data. */ + *last_local_tree = global_id_of_last_tree; + if (child_in_tree_end != NULL) { + *child_in_tree_end = + last_element_in_tree_index_of_current_proc + 1; + } + /* We do not expect this message from another proc */ + expect_end_message = 0; + t8_debugf ("[H] Copied last tree %li element %li to self\n", + global_id_of_last_tree, + last_element_in_tree_index_of_current_proc + 1); +#ifdef T8_ENABLE_DEBUG + num_received_end_messages++; +#endif + } + } /* End sending of end message */ + } /* End loop over processes */ + } /* if (pure_local_trees > 0) */ + + /* Post the receives. */ + if (expect_start_message) { + const int num_entries = 2; + t8_gloidx_t *message = T8_ALLOC (t8_gloidx_t, num_entries); + mpiret = + sc_MPI_Recv (message, num_entries, T8_MPI_GLOIDX, sc_MPI_ANY_SOURCE, + T8_MPI_CMESH_UNIFORM_BOUNDS_START, comm, + sc_MPI_STATUS_IGNORE); + SC_CHECK_MPI (mpiret); + +#ifdef T8_ENABLE_DEBUG + num_received_start_messages++; +#endif + /* Copy the received data to output parameters */ + *first_local_tree = message[0]; + T8_ASSERT (*first_local_tree == -1 || (0 <= *first_local_tree + && *first_local_tree < + t8_cmesh_get_num_trees (cmesh))); + if (message[1] > 0) { + /* The first tree is shared */ + if (first_tree_shared != NULL) { + *first_tree_shared = 1; + } + } + else { + if (first_tree_shared != NULL) { + *first_tree_shared = 0; + } + } + if (child_in_tree_begin != NULL) { + *child_in_tree_begin = message[1]; + t8_debugf ("[H] Received cit begin = %li\n", *child_in_tree_begin); + T8_ASSERT (*child_in_tree_begin == -1 || (0 <= *child_in_tree_begin + && *child_in_tree_begin < + data.global_num_elements)); + } + T8_FREE (message); + } /* End receiving start message */ + if (expect_end_message) { + const int num_entries = 2; + t8_gloidx_t *message = T8_ALLOC (t8_gloidx_t, num_entries); + mpiret = + sc_MPI_Recv (message, num_entries, T8_MPI_GLOIDX, sc_MPI_ANY_SOURCE, + T8_MPI_CMESH_UNIFORM_BOUNDS_END, comm, + sc_MPI_STATUS_IGNORE); + SC_CHECK_MPI (mpiret); +#ifdef T8_ENABLE_DEBUG + num_received_end_messages++; +#endif + t8_debugf ("Receiving end message (%li, %li) global num %li\n", + message[0], message[1], data.global_num_elements); + + /* Copy the received data to output parameters */ + *last_local_tree = message[0]; + T8_ASSERT (*last_local_tree == -1 || (0 <= *last_local_tree + && *last_local_tree <= + t8_cmesh_get_num_trees (cmesh))); + if (child_in_tree_end != NULL) { + *child_in_tree_end = message[1] + 1; + T8_ASSERT (*child_in_tree_end == -1 || (0 <= *child_in_tree_end + && *child_in_tree_end <= + data.global_num_elements)); + } + T8_FREE (message); + } /* End receiving end message */ + if (child_in_tree_begin != NULL && child_in_tree_end != NULL) { + /* Check for empty partition */ + if (*child_in_tree_end < 0 || *child_in_tree_begin < 0) { + /* This partition is empty */ + t8_debugf ("[H] EMPTY hybrid %li %li %li %li\n", *first_local_tree, + *last_local_tree, *child_in_tree_begin, *child_in_tree_end); + + t8_cmesh_uniform_set_return_parameters_to_empty (first_local_tree, + child_in_tree_begin, + last_local_tree, + child_in_tree_end, + first_tree_shared); + } + } + + mpiret = + sc_MPI_Waitall (num_messages_sent, (sc_MPI_Request *) send_requests.array, + sc_MPI_STATUSES_IGNORE); + SC_CHECK_MPI (mpiret); + T8_ASSERT (num_received_start_messages == 1); + T8_ASSERT (num_received_end_messages == 1); + + + if (pure_local_trees > 0) { + sc_array_reset (&first_element_tree); + sc_array_reset (&offset_partition); + sc_array_reset (&send_buffer); + sc_array_reset (&send_requests); + } + + t8_shmem_array_destroy (&offset_array); + + t8_debugf ("Done with t8_cmesh_uniform_bounds_hybrid.\n"); + return; + +} \ No newline at end of file From 553747c42e907a96967454f8607b6fa2f933fc05 Mon Sep 17 00:00:00 2001 From: Knapp Date: Fri, 23 Sep 2022 13:32:41 +0200 Subject: [PATCH 02/43] Add all missing functions to make code compilable --- src/t8.c | 7 ++ src/t8.h | 11 +++ src/t8_cmesh.h | 5 +- src/t8_cmesh/t8_cmesh_cxx.cxx | 142 ++++++++++++++++++++++++++++++++ src/t8_data/t8_shmem.c | 11 +++ src/t8_data/t8_shmem.h | 13 +++ src/t8_forest/t8_forest.c | 9 +- src/t8_forest/t8_forest_cxx.cxx | 4 +- 8 files changed, 190 insertions(+), 12 deletions(-) diff --git a/src/t8.c b/src/t8.c index 8200b28c10..d8dcef121c 100644 --- a/src/t8.c +++ b/src/t8.c @@ -166,3 +166,10 @@ t8_sc_array_index_locidx (sc_array_t *array, t8_locidx_t it) return array->array + array->elem_size * (size_t) it; } + +void * +t8_sc_array_index_gloidx (sc_array_t * array, t8_gloidx_t it) +{ + T8_ASSERT (it >= 0 && (size_t) it < array->elem_count); + return array->array + array->elem_size * (size_t) it; +} diff --git a/src/t8.h b/src/t8.h index ea28a89c6a..5a15b9ac67 100644 --- a/src/t8.h +++ b/src/t8.h @@ -107,6 +107,8 @@ typedef int64_t t8_gloidx_t; #define T8_MPI_GLOIDX sc_MPI_LONG_LONG_INT /** Macro to get the absolute value of a t8_gloidx_t */ #define T8_GLOIDX_ABS(x) ((t8_gloidx_t) llabs ((long long) (x))) +/** Maximum possible value of a t8_gloidx_t*/ +#define T8_GLOIDX_MAX INT64_MAX /** Comparison function for t8_gloidx_t */ #define t8_compare_gloidx(v,w) sc_int64_compare(v,w) @@ -129,6 +131,8 @@ typedef enum T8_MPI_PARTITION_FOREST, /**< Used for forest partitioning */ T8_MPI_GHOST_FOREST, /**< Used for for ghost layer creation */ T8_MPI_GHOST_EXC_FOREST, /**< Used for ghost data exchange */ + T8_MPI_CMESH_UNIFORM_BOUNDS_START, /**< Used for cmesh uniform bounds computation. */ + T8_MPI_CMESH_UNIFORM_BOUNDS_END, /**< Used for cmesh uniform bounds computation. */ T8_MPI_TAG_LAST } t8_MPI_tag_t; @@ -257,6 +261,13 @@ void t8_init (int log_threshold); void *t8_sc_array_index_locidx (sc_array_t *array, t8_locidx_t it); +/** Return a pointer to an array element indexed by a t8_gloidx_t. + * \param [in] index needs to be in [0]..[elem_count-1]. + * \return A void * pointing to entry \a it in \a array. + */ +void *t8_sc_array_index_gloidx (sc_array_t * array, + t8_gloidx_t it); + /* call this at the end of a header file to match T8_EXTERN_C_BEGIN (). */ T8_EXTERN_C_END (); diff --git a/src/t8_cmesh.h b/src/t8_cmesh.h index f124c35433..879bd41ffc 100644 --- a/src/t8_cmesh.h +++ b/src/t8_cmesh.h @@ -691,7 +691,7 @@ t8_shmem_array_t t8_cmesh_get_partition_table (t8_cmesh_t cmesh); * the calling processor anymore. Not computed if NULL. * \param [out] first_tree_shared If not NULL, 1 or 0 is stored here depending on whether \a first_local_tree is the * same as \a last_local_tree on the next process. - * \a cmesh must be committed before calling this function. * + * \a cmesh must be committed before calling this function. */ void t8_cmesh_uniform_bounds (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t *ts, @@ -716,7 +716,6 @@ void t8_cmesh_uniform_bounds (t8_cmesh_t cmesh, int level, * same as \a last_local_tree on the next process. * \param [in] comm The communicator */ -*/ void t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t * scheme, @@ -724,7 +723,7 @@ void t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, t8_gloidx_t * child_in_tree_begin, t8_gloidx_t * last_local_tree, t8_gloidx_t * child_in_tree_end, - int8_t * first_tree_shared, sc_MPI_Comm comm) + int8_t * first_tree_shared, sc_MPI_Comm comm); /** Increase the reference counter of a cmesh. * \param [in,out] cmesh On input, this cmesh must exist with positive diff --git a/src/t8_cmesh/t8_cmesh_cxx.cxx b/src/t8_cmesh/t8_cmesh_cxx.cxx index 78a1cb4a89..3caec878d2 100644 --- a/src/t8_cmesh/t8_cmesh_cxx.cxx +++ b/src/t8_cmesh/t8_cmesh_cxx.cxx @@ -23,6 +23,7 @@ #include #include #include "t8_cmesh_types.h" +#include /** \file t8_cmesh_cxx.cxx * This file collects all general cmesh routines that need c++ compilation. @@ -31,6 +32,83 @@ * TODO: document this file */ +/* This struct is used to temporarily store some context values + * when we determine the partition bounds of processes in + * t8_cmesh_determine_partition. */ +typedef struct +{ + int num_procs; /* Number of MPI processes. */ + int process_offset; /* The first process that this process sends to. */ + t8_gloidx_t global_num_elements; /* The global number of elements. */ +} t8_cmesh_partition_query_t; + +/* Helper function to set the correct return value in the case that + * the uniform partition is empty. + * We set all values to -1 and first_tree_shared to 0. */ +static void +t8_cmesh_uniform_set_return_parameters_to_empty (t8_gloidx_t * + first_local_tree, + t8_gloidx_t * + child_in_tree_begin, + t8_gloidx_t * + last_local_tree, + t8_gloidx_t * + child_in_tree_end, + int8_t * first_tree_shared) +{ + *first_local_tree = *last_local_tree = -1; + if (child_in_tree_begin != NULL) { + *child_in_tree_begin = -1; + } + if (child_in_tree_end != NULL) { + *child_in_tree_end = -1; + } + if (first_tree_shared != NULL) { + *first_tree_shared = 0; + } +} + +/* Helper function to compute (A*B)/C for large integers, when A*B might not fit in a + * 64-bit int anymore. Uses floor(floor(A/C)*B + ((A%C)/C)*B) instead. */ +static inline t8_gloidx_t +t8_A_times_B_over_C_gloidx (t8_gloidx_t A, t8_gloidx_t B, t8_gloidx_t C) +{ +#ifdef T8_ENABLE_DEBUG + { + /* We check wether computing A/C * B will cause an overflow. + * This can be achieved by checking if dividing the result by B + * yields A/C again. */ + const t8_gloidx_t a_over_c = A / C; + const t8_gloidx_t a_o_c_times_b = a_over_c * B; + T8_ASSERT (a_over_c == 0 || a_o_c_times_b / a_over_c == B); + } +#endif + return (t8_gloidx_t) ((A / C) * B + (((long double) (A % C)) / C) * B); +} + +/* Version of t8_A_times_B_over_C where A is an integer. + * We reuse the gloidx version but check before whether an int fits into + * a t8_gloidx_t. + * This function can also be used if B or C are ints. */ +static inline t8_gloidx_t +t8_A_times_B_over_C_intA (int A, t8_gloidx_t B, t8_gloidx_t C) +{ + T8_ASSERT (sizeof (int) <= sizeof (t8_gloidx_t)); + return t8_A_times_B_over_C_gloidx (A, B, C); +} + +/* Compute and return the first global element index of a process in the + * uniform partition. + * The first index of a process 0 <= p < P among E elements is + * floor ((p * E) / P) + */ +static inline t8_gloidx_t +t8_cmesh_get_first_element_of_process (int process, int mpisize, + t8_gloidx_t global_num_elements) +{ + return t8_A_times_B_over_C_intA (process, global_num_elements, mpisize); +} + void t8_cmesh_uniform_bounds (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t *ts, @@ -168,6 +246,70 @@ t8_cmesh_uniform_bounds (t8_cmesh_t cmesh, int level, } +/* Given an array (partition) storing the global index of the + * first element of each (pure) local tree and + * a (pure) local tree index, return the first process that + * will have elements of this tree in a uniform partition. + * The data pointer must point to a valid t8_cmesh_partition_query_t, + * storing the number of processe and the global number of elements. + * + * This function is used standalone and as callback of sc_array_split. */ +static size_t +t8_cmesh_determine_partition (sc_array_t * first_element_tree, + size_t pure_local_tree, void *data) +{ + T8_ASSERT (data != NULL); + T8_ASSERT (first_element_tree != NULL); + + const t8_cmesh_partition_query_t *query_data = + (const t8_cmesh_partition_query_t *) data; + size_t first_proc_adjusted; + const t8_gloidx_t element_index = + *(t8_gloidx_t *) sc_array_index (first_element_tree, pure_local_tree); + const t8_gloidx_t mirror_element_index = + query_data->global_num_elements - element_index - 1; + int first_proc_rank; + + t8_debugf ("[H] tree %li el %li mirror %li\n", pure_local_tree, + element_index, mirror_element_index); + if (element_index == query_data->global_num_elements) { + return query_data->num_procs - query_data->process_offset; + } + else { + first_proc_rank = + query_data->num_procs - 1 - + t8_A_times_B_over_C_intA (query_data->num_procs, mirror_element_index, + query_data->global_num_elements); + + // (int)(((long double)mirror_element_index/query_data->global_num_elements)*query_data->num_procs); + + /* If this is called by array split, we need to adjust for relative processes starting + * add the first process. */ + first_proc_adjusted = first_proc_rank - query_data->process_offset; + } + t8_debugf ("[H] ptree %zd, first element %li, on proc %zd\n", + pure_local_tree, element_index, first_proc_adjusted); + + /* Safety checks */ + T8_ASSERT (0 <= first_proc_rank + && (int) first_proc_rank < query_data->num_procs); + T8_ASSERT (0 <= first_proc_adjusted); + /* Check that the element lies in the partition of the computed proc. */ + T8_ASSERT (t8_cmesh_get_first_element_of_process + (first_proc_rank, query_data->num_procs, + query_data->global_num_elements) <= element_index); +#ifdef T8_ENABLE_DEBUG + if ((int) first_proc_rank != query_data->num_procs - 1) { + T8_ASSERT (t8_cmesh_get_first_element_of_process + (first_proc_rank + 1, query_data->num_procs, + query_data->global_num_elements) > element_index); + } + if (element_index == query_data->global_num_elements) { + T8_ASSERT ((int) first_proc_rank == query_data->num_procs); + } +#endif + return first_proc_adjusted; +} /* TODO: Empty processes, shared trees, binary search in offset-array to avoid recv_any, * use partition_given to partition the cmesh*/ diff --git a/src/t8_data/t8_shmem.c b/src/t8_data/t8_shmem.c index 1b7918e3f0..b8200ead21 100644 --- a/src/t8_data/t8_shmem.c +++ b/src/t8_data/t8_shmem.c @@ -218,6 +218,17 @@ t8_shmem_array_allgather (const void *sendbuf, int sendcount, recvcount, recvtype, recvarray->comm); } +void +t8_shmem_prefix (void *sendbuf, t8_shmem_array_t recvarray, int count, + sc_MPI_Datatype type, sc_MPI_Op op) +{ + T8_ASSERT (recvarray != NULL); + T8_ASSERT (recvarray->array != NULL); + + sc_shmem_prefix (sendbuf, recvarray->array, count, type, op, + recvarray->comm); +} + sc_MPI_Comm t8_shmem_array_get_comm (t8_shmem_array_t array) { diff --git a/src/t8_data/t8_shmem.h b/src/t8_data/t8_shmem.h index c551129b54..e1cd1b1f9e 100644 --- a/src/t8_data/t8_shmem.h +++ b/src/t8_data/t8_shmem.h @@ -149,6 +149,19 @@ void t8_shmem_array_allgather (const void *sendbuf, int recvcount, sc_MPI_Datatype recvtype); +/** Fill a shmem array with an allgather of the prefix op over all processes. + * + * The return array will be + * (0, send0, send0 op send1, send0 op send1 op send2, ...) + * \param[in] sendbuf the source from this process + * \param[in, out] recvarray the destination shmem array + * \param[in] count the number of items to allgather + * \param[in] type the type of items to allgather + * \param[in] op the operation to prefix + */ +void t8_shmem_prefix (void *sendbuf, t8_shmem_array_t recvarray, + int count, sc_MPI_Datatype type, sc_MPI_Op op); + /** Return the MPI communicator associated with a shmem array. * \param [in] array The shmem_array to be queried. * \return The MPI communicator stored at \a array. diff --git a/src/t8_forest/t8_forest.c b/src/t8_forest/t8_forest.c index 301a5cb06f..ef2209af9e 100644 --- a/src/t8_forest/t8_forest.c +++ b/src/t8_forest/t8_forest.c @@ -496,13 +496,8 @@ t8_forest_commit (t8_forest_t forest) t8_forest_compute_maxlevel (forest); T8_ASSERT (forest->set_level <= forest->maxlevel); /* populate a new forest with tree and quadrant objects */ - if (t8_forest_refines_irregular (forest) && forest->set_level > 0) { - /* On root level we will also use the normal algorithm */ - t8_forest_populate_irregular (forest); - } - else { - t8_forest_populate (forest); - } + t8_forest_populate (forest); + forest->global_num_trees = t8_cmesh_get_num_trees (forest->cmesh); } else { /* set_from != NULL */ diff --git a/src/t8_forest/t8_forest_cxx.cxx b/src/t8_forest/t8_forest_cxx.cxx index 441f1df2a4..e44b7333da 100644 --- a/src/t8_forest/t8_forest_cxx.cxx +++ b/src/t8_forest/t8_forest_cxx.cxx @@ -1282,11 +1282,11 @@ t8_forest_populate (t8_forest_t forest) SC_CHECK_ABORT (forest->set_level <= forest->maxlevel, "Given refinement level exceeds the maximum.\n"); /* TODO: create trees and quadrants according to uniform refinement */ - t8_cmesh_uniform_bounds (forest->cmesh, forest->set_level, + t8_cmesh_uniform_bounds_hybrid (forest->cmesh, forest->set_level, forest->scheme_cxx, &forest->first_local_tree, &child_in_tree_begin, &forest->last_local_tree, - &child_in_tree_end, NULL); + &child_in_tree_end, NULL, forest->mpicomm); /* True if the forest has no elements */ is_empty = forest->first_local_tree > forest->last_local_tree From 4616091e159f83503bcda87a97ba91169216ea8c Mon Sep 17 00:00:00 2001 From: Knapp Date: Fri, 23 Sep 2022 14:50:39 +0200 Subject: [PATCH 03/43] Add example --- example/cmesh/Makefile.am | 4 +- example/cmesh/t8_cmesh_hybrid_new.cxx | 335 ++++++++++++++++++++++++++ src/t8_cmesh/t8_cmesh_examples.c | 142 +++++++++++ 3 files changed, 480 insertions(+), 1 deletion(-) create mode 100644 example/cmesh/t8_cmesh_hybrid_new.cxx diff --git a/example/cmesh/Makefile.am b/example/cmesh/Makefile.am index df4b9b87b6..e377c7c4e6 100644 --- a/example/cmesh/Makefile.am +++ b/example/cmesh/Makefile.am @@ -5,8 +5,10 @@ bin_PROGRAMS += \ example/cmesh/t8_cmesh_partition \ example/cmesh/t8_cmesh_create_partitioned \ - example/cmesh/t8_cmesh_refine + example/cmesh/t8_cmesh_refine \ + example/cmesh/t8_cmesh_hybrid_new example_cmesh_t8_cmesh_partition_SOURCES = example/cmesh/t8_cmesh_partition.cxx example_cmesh_t8_cmesh_refine_SOURCES = example/cmesh/t8_cmesh_refine.cxx example_cmesh_t8_cmesh_create_partitioned_SOURCES = example/cmesh/t8_cmesh_create_partitioned.cxx +example_cmesh_t8_cmesh_hybrid_new_SOURCES = example/cmesh/t8_cmesh_hybrid_new.cxx diff --git a/example/cmesh/t8_cmesh_hybrid_new.cxx b/example/cmesh/t8_cmesh_hybrid_new.cxx new file mode 100644 index 0000000000..2d28077371 --- /dev/null +++ b/example/cmesh/t8_cmesh_hybrid_new.cxx @@ -0,0 +1,335 @@ +/* + This file is part of t8code. + t8code is a C library to manage a collection (a forest) of multiple + connected adaptive space-trees of general element types in parallel. + + Copyright (C) 2015 the developers + + t8code is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + t8code is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with t8code; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +static int +t8_basic_hybrid_refine(t8_forest_t forest, t8_forest_t forest_from, + t8_locidx_t which_tree, t8_locidx_t lelement_id, + t8_eclass_scheme_c * ts, int is_family, int num_elements, + t8_element_t * elements[]) +{ + int level, id; + level = ts->t8_element_level(elements[0]); + if (level >= *(int *) t8_forest_get_user_data (forest)) { + return 0; + } + else{ + switch (ts->t8_element_shape(elements[0])) { + case T8_ECLASS_HEX: + id=ts->t8_element_child_id(elements[0]); + return id%2==0?1:0; + case T8_ECLASS_TET: + id=ts->t8_element_child_id(elements[0]); + return id%2==0?1:0; + case T8_ECLASS_PRISM: + id=ts->t8_element_child_id(elements[0]); + return id%2==0?1:0; + case T8_ECLASS_PYRAMID: + return 1; + default: + return 1; + } + } +} + +static int +t8_basic_cake_refine(t8_forest_t forest, t8_forest_t forest_from, + t8_locidx_t which_tree, t8_locidx_t lelement_id, + t8_eclass_scheme_c * ts, int is_family, int num_elements, + t8_element_t * elements[]) +{ + int level, type; + level = ts->t8_element_level(elements[0]); + if (level >= *(int *) t8_forest_get_user_data (forest)) { + return 0; + } + else{ + int32_t h = T8_DPYRAMID_LEN(level); + switch (ts->t8_element_shape(elements[0])) { + case T8_ECLASS_TET: + type = ((t8_dtet_t *)elements[0])->type; + if(type == 0 || type == 2 || type == 4){ + return 1; + } + else{ + return 0; + } + case T8_ECLASS_PYRAMID: + + if(!(((t8_dpyramid_t *) elements[0])->x & h)){ + type = ((t8_dpyramid_t *) elements[0])->type; + return type == 6 ? 1:0; + } + else{ + return 0; + } + default: + return 1; + } + } +} + +static int +t8_basic_only_pyramid(t8_forest_t forest, t8_forest_t forest_from, + t8_locidx_t which_tree, t8_locidx_t lelement_id, + t8_eclass_scheme_c * ts, int is_family, int num_elements, + t8_element_t * elements[]) +{ + int level; + level = ts->t8_element_level(elements[0]); + if (level >= *(int *) t8_forest_get_user_data (forest)) { + return 0; + } + else if(ts->t8_element_shape(elements[0]) == T8_ECLASS_PYRAMID){ + return 1; + } + else{ + return 0; + } +} +static void +t8_basic_hybrid(int level, int endlvl, int do_vtk, t8_eclass_t eclass, + int num_elements, int mesh, int balance, const char* prefix, int part) +{ + t8_forest_t forest, forest_adapt, forest_partition; + t8_cmesh_t cmesh, cmesh_partition; + char vtuname[BUFSIZ], cmesh_file[BUFSIZ]; + int mpirank, mpiret; + double new_time = 0, adapt_time = 0, ghost_time = 0, partition_time = 0, + total_time = 0, balance_time = 0; + sc_statinfo_t times[6]; + int procs_sent, balance_rounds; + t8_locidx_t ghost_sent; + sc_stats_init(×[0], "new"); + sc_stats_init(×[1], "adapt"); + sc_stats_init(×[2], "ghost"); + sc_stats_init(×[3], "partition"); + sc_stats_init(×[4], "balance"); + sc_stats_init(×[5], "total"); + total_time -= sc_MPI_Wtime(); + switch (mesh) { + case 0: + t8_global_productionf ("Contructing cake mesh with %i pyramids.\n", + num_elements); + cmesh = t8_cmesh_new_pyramid_cake(sc_MPI_COMM_WORLD, num_elements); + break; + case 1: + t8_global_productionf ("Contructing full hybrid mesh.\n"); + cmesh = t8_cmesh_new_full_hybrid(sc_MPI_COMM_WORLD); + break; + case 2: + t8_global_productionf ("Contructing long brick out of %i pyramids.\n", + num_elements); + cmesh = t8_cmesh_new_long_brick_pyramid(sc_MPI_COMM_WORLD, num_elements); + break; + case 3: + t8_global_productionf ("Contructing mesh from: %s.\n", prefix); + cmesh = t8_cmesh_from_msh_file((char *) prefix, 0, sc_MPI_COMM_WORLD, 3, 0, 0); + break; + case 4: + t8_global_productionf("Constructing bigmesh with %i elements.\n", num_elements); + cmesh = t8_cmesh_new_bigmesh(eclass, num_elements, sc_MPI_COMM_WORLD); + break; + case 5: + t8_global_productionf ("Contructing a single %s.\n", t8_eclass_to_string[eclass]); + cmesh = t8_cmesh_new_from_class(eclass, sc_MPI_COMM_WORLD); + break; + default: + SC_ABORT("NO CMESH"); + return; + } + + snprintf(cmesh_file, BUFSIZ,"cmesh_hybrid"); + snprintf (vtuname, BUFSIZ, "cmesh_hybrid"); + if(mesh != 4){ + t8_cmesh_save(cmesh, cmesh_file); + if (t8_cmesh_vtk_write_file (cmesh, vtuname, 1.0) == 0) { + t8_debugf ("Output to %s\n", vtuname); + } + else { + t8_debugf ("Error in output\n"); + } + } + + mpiret = sc_MPI_Comm_rank (sc_MPI_COMM_WORLD, &mpirank); + SC_CHECK_MPI (mpiret); + if(part){ + t8_cmesh_init(&cmesh_partition); + t8_cmesh_set_derive(cmesh_partition, cmesh); + t8_cmesh_set_partition_uniform(cmesh_partition, level, t8_scheme_new_default_cxx()); + t8_cmesh_commit(cmesh_partition, sc_MPI_COMM_WORLD); + cmesh = cmesh_partition; + } + + t8_debugf("[D] start forest\n"); + t8_forest_init(&forest); + t8_forest_set_profiling(forest, 1); + t8_forest_set_cmesh(forest, cmesh, sc_MPI_COMM_WORLD); + t8_forest_set_scheme(forest, t8_scheme_new_default_cxx()); + t8_forest_set_level(forest, level); + new_time -= sc_MPI_Wtime(); + t8_forest_commit(forest); + new_time += sc_MPI_Wtime(); + if(do_vtk){ + snprintf (vtuname, BUFSIZ, "forest_hybrid"); + t8_forest_write_vtk (forest, vtuname); + t8_debugf("[D] output to %s\n", vtuname); + } + t8_forest_init(&forest_adapt); + t8_forest_set_user_data(forest_adapt, &endlvl); + t8_forest_set_profiling(forest_adapt, 1); + if(eclass == T8_ECLASS_PYRAMID){ + t8_debugf("Use cake-adapt\n"); + t8_forest_set_adapt(forest_adapt, forest, t8_basic_cake_refine, 1); + }else{ + t8_forest_set_adapt(forest_adapt, forest, t8_basic_hybrid_refine, 1); + } + //t8_forest_set_ghost_ext(forest_adapt, 1, T8_GHOST_FACES, 2); + adapt_time -= sc_MPI_Wtime(); + t8_forest_commit(forest_adapt); + adapt_time += sc_MPI_Wtime(); + //t8_debugf ("Successfully adapted forest.\n"); + //snprintf (vtuname, BUFSIZ, "forest_hybrid_refine"); + //t8_forest_write_vtk (forest_adapt, vtuname); + //t8_debugf ("Output to %s\n", vtuname); + //t8_forest_unref(&forest_adapt); + /* Ensure that the correct forest is passed to unref later */ + + t8_forest_init(&forest_partition); + t8_forest_set_partition(forest_partition, forest_adapt, 0); + t8_forest_set_ghost(forest_partition, 1, T8_GHOST_FACES); + if(balance){ + t8_forest_set_balance(forest_partition, forest_adapt,1); + } + t8_forest_set_profiling(forest_partition, 1); + t8_forest_commit(forest_partition); + if(do_vtk){ + snprintf (vtuname, BUFSIZ, "forest_hybrid_partition"); + t8_forest_write_vtk (forest_partition, vtuname); + t8_debugf ("Output to %s\n", vtuname); + } + t8_forest_print_profile(forest_partition); + partition_time += t8_forest_profile_get_partition_time(forest_partition, &procs_sent); + ghost_time += t8_forest_profile_get_ghost_time(forest_partition, &ghost_sent); + balance_time += t8_forest_profile_get_balance_time(forest_partition, &balance_rounds); + total_time += sc_MPI_Wtime(); + + sc_stats_accumulate(×[0], new_time); + sc_stats_accumulate(×[1], adapt_time); + sc_stats_accumulate(×[2], ghost_time); + sc_stats_accumulate(×[3], partition_time); + sc_stats_accumulate(×[4], balance_time); + sc_stats_accumulate(×[5], total_time); + sc_stats_compute(sc_MPI_COMM_WORLD, 6, times); + sc_stats_print(t8_get_package_id(), SC_LP_ESSENTIAL, 6, times, 1,1); + + t8_forest_unref(&forest_partition); +} + +int +main (int argc, char **argv) +{ + int mpiret, parsed; + int level, endlvl, helpme, do_vtk, eclass_int, mesh, elements, balance, part; + t8_eclass_t eclass; + sc_options_t *opt; + char usage[BUFSIZ]; + char help[BUFSIZ]; + const char *file; + + /* brief help message */ + snprintf (usage, BUFSIZ, "Usage:\t%s \n\t%s -h\t" + "for a brief overview of all options.", + basename (argv[0]), basename (argv[0])); + + /* long help message */ + snprintf (help, BUFSIZ, "ADD EXAMPLE DESCRIPTION.\n\n%s\n", usage); + mpiret = sc_MPI_Init (&argc, &argv); + SC_CHECK_MPI (mpiret); + + sc_init (sc_MPI_COMM_WORLD, 1, 1, NULL, SC_LP_ESSENTIAL); + t8_init (SC_LP_DEFAULT); + + opt = sc_options_new(argv[0]); + sc_options_add_switch (opt, 'h', "help", &helpme, + "Display a short help message."); + sc_options_add_int (opt, 'l', "level", &level, 0, + "The refinement level of the mesh."); + sc_options_add_int (opt, 'f', "final-level", &endlvl, 1, + "The final refinement level of the mesh."); + sc_options_add_switch(opt, 'v', "vtk", &do_vtk, "Enable vtk-output."); + sc_options_add_switch(opt, 'b', "balance", &balance, "Enable balance"); + sc_options_add_switch(opt, 'p', "partition", &part, "Enable cmesh-partition"); + sc_options_add_int(opt, 'e', "element", &eclass_int, 4, "Given an element-class, the programm will " + " construct a single element of this class. Is ignored, if the option cake is chosen." + "The type of elements to use.\n" + "\t\t4 - hexahedron\n" + "\t\t5 - tetrahedron\n\t\t6 - prism\n\t\t7 - pyramid"); + sc_options_add_int(opt, 'm', "mesh", &mesh, 5,"A mesh to choose from." + " The meshes to chose from are: \n" + "\t\t0 - cake of pyramids\n\t\t1 - hybrid mesh with all 3D elements" + "\n\t\t2 - a long row of bricks out ot pyramids" + "\n\t\t3 - user-specific mesh-file" + "\n\t\t4 - construct multiple elements of class given by -e" + "\n\t\t5 - a single element of class -e"); + sc_options_add_int(opt, 'n', "num_elements", &elements, 3, "The number of elements to use" + " if a m0 or m4 is build. Has to be larger than 2."); + sc_options_add_string(opt, 'p', "mshfile", &file, "NULL", "Prefix of the msh-file."); + + parsed = sc_options_parse(t8_get_package_id(), SC_LP_DEFAULT, opt, argc, argv); + if (helpme) { + /* display help message and usage */ + t8_global_productionf ("%s\n", help); + sc_options_print_usage (t8_get_package_id (), SC_LP_ERROR, opt, NULL); + } + else if(parsed >= 0 && 0 <= level && 4 <= eclass_int && eclass_int < T8_ECLASS_COUNT && + elements >= 2 && 0 <=mesh && mesh < 6){ + eclass = (t8_eclass_t) eclass_int; + t8_basic_hybrid (level, endlvl, do_vtk, eclass, elements, mesh, balance, file, part); + } + else { + /* wrong usage */ + t8_global_productionf ("\n\t ERROR: Wrong usage.\n\n"); + sc_options_print_usage (t8_get_package_id (), SC_LP_ERROR, opt, NULL); + } + sc_options_destroy(opt); + sc_finalize (); + + mpiret = sc_MPI_Finalize (); + SC_CHECK_MPI (mpiret); + + return 0; +} diff --git a/src/t8_cmesh/t8_cmesh_examples.c b/src/t8_cmesh/t8_cmesh_examples.c index cbe39139ed..f3f33aa79a 100644 --- a/src/t8_cmesh/t8_cmesh_examples.c +++ b/src/t8_cmesh/t8_cmesh_examples.c @@ -1924,3 +1924,145 @@ t8_cmesh_new_full_hybrid (sc_MPI_Comm comm) t8_cmesh_commit (cmesh, comm); return cmesh; } + +t8_cmesh_t +t8_cmesh_new_pyramid_cake (sc_MPI_Comm comm, int num_of_pyra) +{ + /*num_of_pyra pyras a 5 vertices a 3 coords */ + /* TODO: This seems too be a lot of memory, can we also get by with only + 5 * 3 doubles? */ + int i, j; + double *vertices = T8_ALLOC (double, num_of_pyra * 5 * 3); + t8_cmesh_t cmesh; + double degrees = 360. / num_of_pyra; + if (vertices) + T8_ASSERT (num_of_pyra > 2); + + for (i = 0; i < num_of_pyra; i++) { + for (j = 0; j < 5; j++) { + /*Get the edges at the unit circle */ + if (j == 4) { + vertices[i * 5 * 3 + j * 3] = 0; + vertices[i * 5 * 3 + j * 3 + 1] = 0; + vertices[i * 5 * 3 + j * 3 + 2] = 0; + } + else if (j == 1 || j == 3) { + vertices[i * 5 * 3 + j * 3] = cos (i * degrees * M_PI / 180); + vertices[i * 5 * 3 + j * 3 + 1] = sin (i * degrees * M_PI / 180); + vertices[i * 5 * 3 + j * 3 + 2] = (j == 3 ? 0.5 : -0.5); + } + else if (j == 0 || j == 2) { + vertices[i * 5 * 3 + j * 3] = + cos ((i * degrees + degrees) * M_PI / 180); + vertices[i * 5 * 3 + j * 3 + 1] = + sin ((i * degrees + degrees) * M_PI / 180); + vertices[i * 5 * 3 + j * 3 + 2] = (j == 2 ? 0.5 : -0.5); + } + } + } + t8_geometry_c *linear_geom = t8_geometry_linear_new (3); + + t8_cmesh_init (&cmesh); + + t8_cmesh_register_geometry (cmesh, linear_geom); + for (i = 0; i < num_of_pyra; i++) { + t8_cmesh_set_tree_class (cmesh, i, T8_ECLASS_PYRAMID); + } + + for (i = 0; i < num_of_pyra; i++) { + t8_cmesh_set_join (cmesh, i, (i == (num_of_pyra - 1) ? 0 : i + 1), 0, 1, + 0); + } + for (i = 0; i < num_of_pyra; i++) { + t8_cmesh_set_tree_vertices (cmesh, i, vertices + i * 15, 5); + } + t8_cmesh_commit (cmesh, comm); + T8_FREE (vertices); + + return cmesh; +} + +t8_cmesh_t +t8_cmesh_new_long_brick_pyramid (sc_MPI_Comm comm, int num_cubes) +{ + double vertices_coords[24] = { + 0, 0, 0, + 1, 0, 0, + 0, 1, 0, + 1, 1, 0, + 0, 0, 1, + 1, 0, 1, + 0, 1, 1, + 1, 1, 1 + }; + t8_locidx_t vertices[5]; + double attr_vertices[15]; + int i, j; + t8_cmesh_t cmesh; + T8_ASSERT (num_cubes > 0); + t8_geometry_c *linear_geom = t8_geometry_linear_new (3); + + t8_cmesh_init (&cmesh); + + t8_cmesh_register_geometry (cmesh, linear_geom); + for (i = 0; i < num_cubes; i++) { + for (j = 0; j < 3; j++) { + t8_cmesh_set_tree_class (cmesh, i * 3 + j, T8_ECLASS_PYRAMID); + } + /*in-cube face connection */ + if (i % 2 == 0) { + t8_cmesh_set_join (cmesh, i * 3, i * 3 + 1, 3, 2, 0); + t8_cmesh_set_join (cmesh, i * 3 + 1, i * 3 + 2, 0, 1, 0); + t8_cmesh_set_join (cmesh, i * 3 + 2, i * 3, 2, 0, 0); + } + else { + t8_cmesh_set_join (cmesh, i * 3, i * 3 + 1, 2, 2, 0); + t8_cmesh_set_join (cmesh, i * 3 + 1, i * 3 + 2, 1, 0, 0); + t8_cmesh_set_join (cmesh, i * 3 + 2, i * 3, 2, 3, 0); + } + } + /*over cube face connection */ + for (i = 0; i < num_cubes - 1; i++) { + if (i % 2 == 0) { + t8_cmesh_set_join (cmesh, i * 3, (i + 1) * 3, 2, 0, 0); + t8_cmesh_set_join (cmesh, i * 3 + 1, (i + 1) * 3 + 2, 3, 3, 0); + } + else { + t8_cmesh_set_join (cmesh, i * 3 + 1, (i + 1) * 3 + 2, 4, 4, 0); + } + } + /*vertices */ + for (i = 0; i < num_cubes; i++) { + vertices[0] = 1; + vertices[1] = 3; + vertices[2] = 0; + vertices[3] = 2; + vertices[4] = i % 2 == 0 ? 7 : 5; + t8_cmesh_new_translate_vertices_to_attributes (vertices, + vertices_coords, + attr_vertices, 5); + t8_cmesh_set_tree_vertices (cmesh, i * 3, attr_vertices, 5); + vertices[0] = i % 2 == 0 ? 0 : 2; + vertices[1] = i % 2 == 0 ? 2 : 3; + vertices[2] = i % 2 == 0 ? 4 : 6; + vertices[3] = i % 2 == 0 ? 6 : 7; + t8_cmesh_new_translate_vertices_to_attributes (vertices, + vertices_coords, + attr_vertices, 5); + t8_cmesh_set_tree_vertices (cmesh, i * 3 + 1, attr_vertices, 5); + vertices[0] = i % 2 == 0 ? 1 : 0; + vertices[1] = i % 2 == 0 ? 0 : 2; + vertices[2] = i % 2 == 0 ? 5 : 4; + vertices[3] = i % 2 == 0 ? 4 : 6; + t8_cmesh_new_translate_vertices_to_attributes (vertices, + vertices_coords, + attr_vertices, 5); + t8_cmesh_set_tree_vertices (cmesh, i * 3 + 2, attr_vertices, 5); + for (j = 0; j < 8; j++) { + vertices_coords[j * 3 + 1] += 1; + } + + } + t8_cmesh_commit (cmesh, comm); + return cmesh; +} From a853e402198a6145309d2cddbd10824d1a7f4407 Mon Sep 17 00:00:00 2001 From: Knapp Date: Fri, 23 Sep 2022 14:51:30 +0200 Subject: [PATCH 04/43] move is_irregular function and use hybrid_bounds only when a forest refines irregularly --- src/t8_forest/t8_forest.c | 37 ------------------------ src/t8_forest/t8_forest_cxx.cxx | 50 ++++++++++++++++++++++++++++++++- 2 files changed, 49 insertions(+), 38 deletions(-) diff --git a/src/t8_forest/t8_forest.c b/src/t8_forest/t8_forest.c index ef2209af9e..ad69b4a331 100644 --- a/src/t8_forest/t8_forest.c +++ b/src/t8_forest/t8_forest.c @@ -378,44 +378,7 @@ t8_forest_refine_everything (t8_forest_t forest, t8_forest_t forest_from, return 1; } -/** - * Check if any tree in a forest refines irregularly. - * An irregular refining tree is a tree with an element that does not - * refine into 2^dim children. For example the default implementation - * of pyramids. - * \note This function is MPI collective - * - * \param[in] forest The forest to check - * \return non-zero if any tree refines irregular - */ -static int -t8_forest_refines_irregular (t8_forest_t forest) -{ - int irregular = 0; - int irregular_all_procs = 0; /* Result over all procs */ - int int_eclass; - int mpiret; - t8_eclass_scheme_c *tscheme; - /* Iterate over all eclasses */ - for (int_eclass = (int) T8_ECLASS_ZERO; int_eclass < (int) T8_ECLASS_COUNT; - int_eclass++) { - /* If the forest has trees of the current eclass, check if elements of this - * eclass refine irregular. */ - if (forest->cmesh->num_local_trees_per_eclass[int_eclass] > 0) { - tscheme = - t8_forest_get_eclass_scheme_before_commit (forest, - (t8_eclass_t) int_eclass); - irregular = irregular || t8_element_refines_irregular (tscheme); - } - } - /* Combine the process-local results via a logic or and distribute the - * result over all procs (in the communicator).*/ - mpiret = sc_MPI_Allreduce (&irregular, &irregular_all_procs, 1, sc_MPI_INT, - sc_MPI_LOR, forest->mpicomm); - SC_CHECK_MPI (mpiret); - return irregular_all_procs; -} /**Algorithm to populate a forest, if any tree refines irregularly. * Create the elements on this process given a uniform partition diff --git a/src/t8_forest/t8_forest_cxx.cxx b/src/t8_forest/t8_forest_cxx.cxx index e44b7333da..4b85e70bf5 100644 --- a/src/t8_forest/t8_forest_cxx.cxx +++ b/src/t8_forest/t8_forest_cxx.cxx @@ -1259,6 +1259,45 @@ t8_forest_compute_desc (t8_forest_t forest) } } +/** + * Check if any tree in a forest refines irregularly. + * An irregular refining tree is a tree with an element that does not + * refine into 2^dim children. For example the default implementation + * of pyramids. + * \note This function is MPI collective + * + * \param[in] forest The forest to check + * \return non-zero if any tree refines irregular + */ +static int +t8_forest_refines_irregular (t8_forest_t forest) +{ + int irregular = 0; + int irregular_all_procs = 0; /* Result over all procs */ + int int_eclass; + int mpiret; + t8_eclass_scheme_c *tscheme; + /* Iterate over all eclasses */ + for (int_eclass = (int) T8_ECLASS_ZERO; int_eclass < (int) T8_ECLASS_COUNT; + int_eclass++) { + /* If the forest has trees of the current eclass, check if elements of this + * eclass refine irregular. */ + if (forest->cmesh->num_local_trees_per_eclass[int_eclass] > 0) { + tscheme = + t8_forest_get_eclass_scheme_before_commit (forest, + (t8_eclass_t) int_eclass); + irregular = irregular || tscheme->t8_element_refines_irregular(); + } + } + /* Combine the process-local results via a logic or and distribute the + * result over all procs (in the communicator).*/ + mpiret = sc_MPI_Allreduce (&irregular, &irregular_all_procs, 1, sc_MPI_INT, + sc_MPI_LOR, forest->mpicomm); + SC_CHECK_MPI (mpiret); + + return irregular_all_procs; +} + /* Create the elements on this process given a uniform partition * of the coarse mesh. */ void @@ -1282,11 +1321,20 @@ t8_forest_populate (t8_forest_t forest) SC_CHECK_ABORT (forest->set_level <= forest->maxlevel, "Given refinement level exceeds the maximum.\n"); /* TODO: create trees and quadrants according to uniform refinement */ - t8_cmesh_uniform_bounds_hybrid (forest->cmesh, forest->set_level, + if(t8_forest_refines_irregular(forest)){ + t8_cmesh_uniform_bounds_hybrid (forest->cmesh, forest->set_level, forest->scheme_cxx, &forest->first_local_tree, &child_in_tree_begin, &forest->last_local_tree, &child_in_tree_end, NULL, forest->mpicomm); + } + else{ + t8_cmesh_uniform_bounds (forest->cmesh, forest->set_level, + forest->scheme_cxx, + &forest->first_local_tree, + &child_in_tree_begin, &forest->last_local_tree, + &child_in_tree_end, NULL); + } /* True if the forest has no elements */ is_empty = forest->first_local_tree > forest->last_local_tree From 6021fe4fc9e8a2638b7dfa387f4ab79644621727 Mon Sep 17 00:00:00 2001 From: Knapp Date: Fri, 23 Sep 2022 14:52:09 +0200 Subject: [PATCH 05/43] Some optimization --- src/t8_cmesh/t8_cmesh_cxx.cxx | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/t8_cmesh/t8_cmesh_cxx.cxx b/src/t8_cmesh/t8_cmesh_cxx.cxx index 3caec878d2..12cec22516 100644 --- a/src/t8_cmesh/t8_cmesh_cxx.cxx +++ b/src/t8_cmesh/t8_cmesh_cxx.cxx @@ -322,8 +322,7 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, t8_gloidx_t * child_in_tree_end, int8_t * first_tree_shared, sc_MPI_Comm comm) { - t8_gloidx_t cmesh_first_tree, local_num_children = 0, - elem_in_tree, first_child, last_child, *elem_index_pointer, igtree; + t8_gloidx_t local_num_children = 0, first_child, last_child, *elem_index_pointer, igtree; int send_first, send_last, send_first_nonempty, num_procs_we_send_to = 0; t8_cmesh_partition_query_t data; @@ -421,7 +420,7 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, /* TODO: We can optimize by buffering the elem_in_tree value. Thus, if the computation is expensive (may be for non-morton-type schemes), we do it only once. */ - elem_in_tree = tree_scheme->t8_element_count_leafs_from_root (level); + const t8_gloidx_t elem_in_tree = tree_scheme->t8_element_count_leafs_from_root (level); /* Check if the first element is on the current tree */ if (current_tree_element_offset <= first_child && first_child < current_tree_element_offset + elem_in_tree) { @@ -516,7 +515,7 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, *elem_index_pointer = t8_shmem_array_get_gloidx (offset_array, cmesh->mpirank); - cmesh_first_tree = t8_cmesh_get_first_treeid (cmesh); + const t8_gloidx_t cmesh_first_tree = t8_cmesh_get_first_treeid (cmesh); tree_id = t8_cmesh_get_local_id (cmesh, cmesh_first_tree); ieclass = t8_cmesh_get_tree_class (cmesh, tree_id); From c4d4b3b14a067d491a702d76345f7acc8089949b Mon Sep 17 00:00:00 2001 From: Knapp Date: Fri, 23 Sep 2022 15:35:56 +0200 Subject: [PATCH 06/43] use t8_cmesh_uniform_bounds_hybrid in cmesh_partition --- example/cmesh/t8_cmesh_hybrid_new.cxx | 2 +- src/t8_cmesh/t8_cmesh_cxx.cxx | 7 ++++--- src/t8_cmesh/t8_cmesh_partition.c | 4 ++-- src/t8_forest/t8_forest_cxx.cxx | 28 +++++++++++---------------- 4 files changed, 18 insertions(+), 23 deletions(-) diff --git a/example/cmesh/t8_cmesh_hybrid_new.cxx b/example/cmesh/t8_cmesh_hybrid_new.cxx index 2d28077371..63e5b288e4 100644 --- a/example/cmesh/t8_cmesh_hybrid_new.cxx +++ b/example/cmesh/t8_cmesh_hybrid_new.cxx @@ -307,7 +307,7 @@ main (int argc, char **argv) "\n\t\t5 - a single element of class -e"); sc_options_add_int(opt, 'n', "num_elements", &elements, 3, "The number of elements to use" " if a m0 or m4 is build. Has to be larger than 2."); - sc_options_add_string(opt, 'p', "mshfile", &file, "NULL", "Prefix of the msh-file."); + sc_options_add_string(opt, 'g', "mshfile", &file, "NULL", "Prefix of the msh-file."); parsed = sc_options_parse(t8_get_package_id(), SC_LP_DEFAULT, opt, argc, argv); if (helpme) { diff --git a/src/t8_cmesh/t8_cmesh_cxx.cxx b/src/t8_cmesh/t8_cmesh_cxx.cxx index 12cec22516..3caec878d2 100644 --- a/src/t8_cmesh/t8_cmesh_cxx.cxx +++ b/src/t8_cmesh/t8_cmesh_cxx.cxx @@ -322,7 +322,8 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, t8_gloidx_t * child_in_tree_end, int8_t * first_tree_shared, sc_MPI_Comm comm) { - t8_gloidx_t local_num_children = 0, first_child, last_child, *elem_index_pointer, igtree; + t8_gloidx_t cmesh_first_tree, local_num_children = 0, + elem_in_tree, first_child, last_child, *elem_index_pointer, igtree; int send_first, send_last, send_first_nonempty, num_procs_we_send_to = 0; t8_cmesh_partition_query_t data; @@ -420,7 +421,7 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, /* TODO: We can optimize by buffering the elem_in_tree value. Thus, if the computation is expensive (may be for non-morton-type schemes), we do it only once. */ - const t8_gloidx_t elem_in_tree = tree_scheme->t8_element_count_leafs_from_root (level); + elem_in_tree = tree_scheme->t8_element_count_leafs_from_root (level); /* Check if the first element is on the current tree */ if (current_tree_element_offset <= first_child && first_child < current_tree_element_offset + elem_in_tree) { @@ -515,7 +516,7 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, *elem_index_pointer = t8_shmem_array_get_gloidx (offset_array, cmesh->mpirank); - const t8_gloidx_t cmesh_first_tree = t8_cmesh_get_first_treeid (cmesh); + cmesh_first_tree = t8_cmesh_get_first_treeid (cmesh); tree_id = t8_cmesh_get_local_id (cmesh, cmesh_first_tree); ieclass = t8_cmesh_get_tree_class (cmesh, tree_id); diff --git a/src/t8_cmesh/t8_cmesh_partition.c b/src/t8_cmesh/t8_cmesh_partition.c index 30486c0090..a7e3f73630 100644 --- a/src/t8_cmesh/t8_cmesh_partition.c +++ b/src/t8_cmesh/t8_cmesh_partition.c @@ -2610,10 +2610,10 @@ t8_cmesh_partition (t8_cmesh_t cmesh, sc_MPI_Comm comm) T8_ASSERT (cmesh->tree_offsets == NULL); ts = cmesh->set_partition_scheme; /* The refinement scheme */ T8_ASSERT (ts != NULL); - t8_cmesh_uniform_bounds (cmesh_from, cmesh->set_partition_level, + t8_cmesh_uniform_bounds_hybrid (cmesh_from, cmesh->set_partition_level, ts, &cmesh->first_tree, NULL, &last_tree, NULL, - &cmesh->first_tree_shared); + &cmesh->first_tree_shared, comm); cmesh->num_local_trees = last_tree - cmesh->first_tree + 1; /* Compute the tree offset */ t8_cmesh_gather_treecount_nocommit (cmesh, comm); diff --git a/src/t8_forest/t8_forest_cxx.cxx b/src/t8_forest/t8_forest_cxx.cxx index 4b85e70bf5..e2ae03ccf9 100644 --- a/src/t8_forest/t8_forest_cxx.cxx +++ b/src/t8_forest/t8_forest_cxx.cxx @@ -1321,20 +1321,11 @@ t8_forest_populate (t8_forest_t forest) SC_CHECK_ABORT (forest->set_level <= forest->maxlevel, "Given refinement level exceeds the maximum.\n"); /* TODO: create trees and quadrants according to uniform refinement */ - if(t8_forest_refines_irregular(forest)){ - t8_cmesh_uniform_bounds_hybrid (forest->cmesh, forest->set_level, - forest->scheme_cxx, - &forest->first_local_tree, - &child_in_tree_begin, &forest->last_local_tree, - &child_in_tree_end, NULL, forest->mpicomm); - } - else{ - t8_cmesh_uniform_bounds (forest->cmesh, forest->set_level, - forest->scheme_cxx, - &forest->first_local_tree, - &child_in_tree_begin, &forest->last_local_tree, - &child_in_tree_end, NULL); - } + t8_cmesh_uniform_bounds_hybrid (forest->cmesh, forest->set_level, + forest->scheme_cxx, + &forest->first_local_tree, + &child_in_tree_begin, &forest->last_local_tree, + &child_in_tree_end, NULL, forest->mpicomm); /* True if the forest has no elements */ is_empty = forest->first_local_tree > forest->last_local_tree @@ -1346,9 +1337,12 @@ t8_forest_populate (t8_forest_t forest) t8_cmesh_get_num_local_trees (forest->cmesh) - 1; if (!is_empty) { - SC_CHECK_ABORT (forest->first_local_tree >= cmesh_first_tree - && forest->last_local_tree <= cmesh_last_tree, - "cmesh partition does not match the planned forest partition"); + SC_CHECK_ABORTF (forest->first_local_tree >= cmesh_first_tree + && forest->last_local_tree <= cmesh_last_tree, + "Cmesh partition does not match the planned forest partition. " + "Forest range: %li to %li. Cmesh range: %li to %li", + forest->first_local_tree, forest->last_local_tree, + cmesh_first_tree, cmesh_last_tree); } forest->global_num_elements = forest->local_num_elements = 0; From f271d06326e650b8221ecece41515451c02e44f5 Mon Sep 17 00:00:00 2001 From: Knapp Date: Fri, 23 Sep 2022 16:25:36 +0200 Subject: [PATCH 07/43] Update variables according to their scope and made them const --- src/t8_cmesh/t8_cmesh_cxx.cxx | 35 ++++++++++++++--------------------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/src/t8_cmesh/t8_cmesh_cxx.cxx b/src/t8_cmesh/t8_cmesh_cxx.cxx index 3caec878d2..d0a9724b4e 100644 --- a/src/t8_cmesh/t8_cmesh_cxx.cxx +++ b/src/t8_cmesh/t8_cmesh_cxx.cxx @@ -322,16 +322,13 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, t8_gloidx_t * child_in_tree_end, int8_t * first_tree_shared, sc_MPI_Comm comm) { - t8_gloidx_t cmesh_first_tree, local_num_children = 0, - elem_in_tree, first_child, last_child, *elem_index_pointer, igtree; + t8_gloidx_t local_num_children = 0; + t8_gloidx_t *elem_index_pointer; int send_first, send_last, send_first_nonempty, num_procs_we_send_to = 0; t8_cmesh_partition_query_t data; - t8_locidx_t tree_id, itree, pure_local_trees; t8_shmem_array_t offset_array; sc_array_t offset_partition, first_element_tree; - int ieclass; - int iproc; int mpiret, num_messages_sent = 0; int expect_start_message = 1; int expect_end_message = 1; @@ -373,7 +370,7 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, } #endif /*Compute number of local elements. Ignore shared trees */ - for (ieclass = T8_ECLASS_ZERO; ieclass < T8_ECLASS_COUNT; ieclass++) { + for (int ieclass = T8_ECLASS_ZERO; ieclass < T8_ECLASS_COUNT; ieclass++) { tree_scheme = scheme->eclass_schemes[ieclass]; local_num_children += cmesh->num_local_trees_per_eclass[ieclass] * tree_scheme->t8_element_count_leafs_from_root (level); @@ -381,7 +378,7 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, /* Do not consider shared trees */ if (cmesh->first_tree_shared && cmesh->set_partition) { - ieclass = t8_cmesh_get_tree_class (cmesh, 0); + const int ieclass = t8_cmesh_get_tree_class (cmesh, 0); tree_scheme = scheme->eclass_schemes[ieclass]; local_num_children -= tree_scheme->t8_element_count_leafs_from_root (level); @@ -401,10 +398,10 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, /* Compute the first and last element of this process. Then loop over * all trees to find the trees in which these are contained. * We cast to long double and double to prevent overflow. */ - first_child = + const t8_gloidx_t first_child = t8_cmesh_get_first_element_of_process (cmesh->mpirank, cmesh->mpisize, local_num_children); - last_child = + const t8_gloidx_t last_child = t8_cmesh_get_first_element_of_process (cmesh->mpirank + 1, cmesh->mpisize, local_num_children) - 1; @@ -415,13 +412,13 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, * for each tree. */ current_tree_element_offset = 0; - for (igtree = 0; igtree < num_trees; ++igtree) { - ieclass = t8_cmesh_get_tree_class (cmesh, (t8_locidx_t) igtree); + for (t8_gloidx_t igtree = 0; igtree < num_trees; ++igtree) { + const int ieclass = t8_cmesh_get_tree_class (cmesh, (t8_locidx_t) igtree); tree_scheme = scheme->eclass_schemes[ieclass]; /* TODO: We can optimize by buffering the elem_in_tree value. Thus, if the computation is expensive (may be for non-morton-type schemes), we do it only once. */ - elem_in_tree = tree_scheme->t8_element_count_leafs_from_root (level); + const t8_gloidx_t elem_in_tree = tree_scheme->t8_element_count_leafs_from_root (level); /* Check if the first element is on the current tree */ if (current_tree_element_offset <= first_child && first_child < current_tree_element_offset + elem_in_tree) { @@ -499,12 +496,12 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, t8_debugf ("[H] global num %li\n", data.global_num_elements); /*Compute number of non-shared-trees and the local index of the first non-shared-tree */ - pure_local_trees = cmesh->num_local_trees - first_tree_shared_shift; + const t8_locidx_t pure_local_trees = cmesh->num_local_trees - first_tree_shared_shift; if (pure_local_trees > 0) { /* Compute which trees and elements to send to which process. * We skip empty processes. */ - igtree = first_tree_shared_shift; + t8_locidx_t igtree = first_tree_shared_shift; sc_array_init_size (&first_element_tree, sizeof (t8_gloidx_t), pure_local_trees + 1); @@ -516,19 +513,15 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, *elem_index_pointer = t8_shmem_array_get_gloidx (offset_array, cmesh->mpirank); - cmesh_first_tree = t8_cmesh_get_first_treeid (cmesh); - tree_id = t8_cmesh_get_local_id (cmesh, cmesh_first_tree); - ieclass = t8_cmesh_get_tree_class (cmesh, tree_id); - /* Compute the first element in every pure local tree. * This array stores for each tree the global element index offset. * Example: 2 local trees, each has 8 Elements. First element index 12: | 12 | 20 | 28 | */ - for (itree = 0; itree < pure_local_trees; ++itree, ++igtree) { + for (t8_locidx_t itree = 0; itree < pure_local_trees; ++itree, ++igtree) { const t8_gloidx_t *first_element_of_tree = (const t8_gloidx_t *) sc_array_index_int (&first_element_tree, itree); t8_gloidx_t *first_element_of_next_tree = (t8_gloidx_t *) sc_array_index_int (&first_element_tree, itree + 1); - ieclass = t8_cmesh_get_tree_class (cmesh, (t8_locidx_t) igtree); + const int ieclass = t8_cmesh_get_tree_class (cmesh, igtree); tree_scheme = scheme->eclass_schemes[ieclass]; /* Set the first element of the next tree by adding the number of element of the current tree. */ *first_element_of_next_tree = @@ -640,7 +633,7 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, const t8_gloidx_t last_el_index_of_last_tree = *(t8_gloidx_t *) t8_sc_array_index_gloidx (&first_element_tree, pure_local_trees) - 1; - for (iproc = send_first; iproc <= send_last; iproc++) { + for (int iproc = send_first; iproc <= send_last; iproc++) { const t8_gloidx_t first_element_index_of_current_proc = t8_cmesh_get_first_element_of_process (iproc, data.num_procs, From d15062ed156d352ed37c3a13626217df9ab3752b Mon Sep 17 00:00:00 2001 From: Knapp Date: Thu, 3 Nov 2022 13:35:33 +0100 Subject: [PATCH 08/43] Update to current main and indentation --- example/cmesh/t8_cmesh_hybrid_new.cxx | 473 +++++++++++++------------- 1 file changed, 245 insertions(+), 228 deletions(-) diff --git a/example/cmesh/t8_cmesh_hybrid_new.cxx b/example/cmesh/t8_cmesh_hybrid_new.cxx index 63e5b288e4..a0891e8560 100644 --- a/example/cmesh/t8_cmesh_hybrid_new.cxx +++ b/example/cmesh/t8_cmesh_hybrid_new.cxx @@ -34,241 +34,251 @@ #include #include - static int -t8_basic_hybrid_refine(t8_forest_t forest, t8_forest_t forest_from, - t8_locidx_t which_tree, t8_locidx_t lelement_id, - t8_eclass_scheme_c * ts, int is_family, int num_elements, - t8_element_t * elements[]) +t8_basic_hybrid_refine (t8_forest_t forest, t8_forest_t forest_from, + t8_locidx_t which_tree, t8_locidx_t lelement_id, + t8_eclass_scheme_c *ts, int is_family, + int num_elements, t8_element_t *elements[]) { - int level, id; - level = ts->t8_element_level(elements[0]); - if (level >= *(int *) t8_forest_get_user_data (forest)) { - return 0; - } - else{ - switch (ts->t8_element_shape(elements[0])) { - case T8_ECLASS_HEX: - id=ts->t8_element_child_id(elements[0]); - return id%2==0?1:0; - case T8_ECLASS_TET: - id=ts->t8_element_child_id(elements[0]); - return id%2==0?1:0; - case T8_ECLASS_PRISM: - id=ts->t8_element_child_id(elements[0]); - return id%2==0?1:0; - case T8_ECLASS_PYRAMID: - return 1; - default: - return 1; - } + int level, id; + level = ts->t8_element_level (elements[0]); + if (level >= *(int *) t8_forest_get_user_data (forest)) { + return 0; + } + else { + switch (ts->t8_element_shape (elements[0])) { + case T8_ECLASS_HEX: + id = ts->t8_element_child_id (elements[0]); + return id % 2 == 0 ? 1 : 0; + case T8_ECLASS_TET: + id = ts->t8_element_child_id (elements[0]); + return id % 2 == 0 ? 1 : 0; + case T8_ECLASS_PRISM: + id = ts->t8_element_child_id (elements[0]); + return id % 2 == 0 ? 1 : 0; + case T8_ECLASS_PYRAMID: + return 1; + default: + return 1; } + } } static int -t8_basic_cake_refine(t8_forest_t forest, t8_forest_t forest_from, - t8_locidx_t which_tree, t8_locidx_t lelement_id, - t8_eclass_scheme_c * ts, int is_family, int num_elements, - t8_element_t * elements[]) +t8_basic_cake_refine (t8_forest_t forest, t8_forest_t forest_from, + t8_locidx_t which_tree, t8_locidx_t lelement_id, + t8_eclass_scheme_c *ts, int is_family, int num_elements, + t8_element_t *elements[]) { - int level, type; - level = ts->t8_element_level(elements[0]); - if (level >= *(int *) t8_forest_get_user_data (forest)) { - return 0; - } - else{ - int32_t h = T8_DPYRAMID_LEN(level); - switch (ts->t8_element_shape(elements[0])) { - case T8_ECLASS_TET: - type = ((t8_dtet_t *)elements[0])->type; - if(type == 0 || type == 2 || type == 4){ - return 1; - } - else{ - return 0; - } - case T8_ECLASS_PYRAMID: + int level, type; + level = ts->t8_element_level (elements[0]); + if (level >= *(int *) t8_forest_get_user_data (forest)) { + return 0; + } + else { + int32_t h = T8_DPYRAMID_LEN (level); + switch (ts->t8_element_shape (elements[0])) { + case T8_ECLASS_TET: + type = ((t8_dtet_t *) elements[0])->type; + if (type == 0 || type == 2 || type == 4) { + return 1; + } + else { + return 0; + } + case T8_ECLASS_PYRAMID: - if(!(((t8_dpyramid_t *) elements[0])->x & h)){ - type = ((t8_dpyramid_t *) elements[0])->type; - return type == 6 ? 1:0; - } - else{ - return 0; - } - default: - return 1; - } + if (!(((t8_dpyramid_t *) elements[0])->pyramid.x & h)) { + type = ((t8_dpyramid_t *) elements[0])->pyramid.type; + return type == 6 ? 1 : 0; + } + else { + return 0; + } + default: + return 1; } + } } static int -t8_basic_only_pyramid(t8_forest_t forest, t8_forest_t forest_from, - t8_locidx_t which_tree, t8_locidx_t lelement_id, - t8_eclass_scheme_c * ts, int is_family, int num_elements, - t8_element_t * elements[]) +t8_basic_only_pyramid (t8_forest_t forest, t8_forest_t forest_from, + t8_locidx_t which_tree, t8_locidx_t lelement_id, + t8_eclass_scheme_c *ts, int is_family, + int num_elements, t8_element_t *elements[]) { - int level; - level = ts->t8_element_level(elements[0]); - if (level >= *(int *) t8_forest_get_user_data (forest)) { - return 0; - } - else if(ts->t8_element_shape(elements[0]) == T8_ECLASS_PYRAMID){ - return 1; - } - else{ - return 0; - } + int level; + level = ts->t8_element_level (elements[0]); + if (level >= *(int *) t8_forest_get_user_data (forest)) { + return 0; + } + else if (ts->t8_element_shape (elements[0]) == T8_ECLASS_PYRAMID) { + return 1; + } + else { + return 0; + } } + static void -t8_basic_hybrid(int level, int endlvl, int do_vtk, t8_eclass_t eclass, - int num_elements, int mesh, int balance, const char* prefix, int part) +t8_basic_hybrid (int level, int endlvl, int do_vtk, t8_eclass_t eclass, + int num_elements, int mesh, int balance, const char *prefix, + int part) { - t8_forest_t forest, forest_adapt, forest_partition; - t8_cmesh_t cmesh, cmesh_partition; - char vtuname[BUFSIZ], cmesh_file[BUFSIZ]; - int mpirank, mpiret; - double new_time = 0, adapt_time = 0, ghost_time = 0, partition_time = 0, - total_time = 0, balance_time = 0; - sc_statinfo_t times[6]; - int procs_sent, balance_rounds; - t8_locidx_t ghost_sent; - sc_stats_init(×[0], "new"); - sc_stats_init(×[1], "adapt"); - sc_stats_init(×[2], "ghost"); - sc_stats_init(×[3], "partition"); - sc_stats_init(×[4], "balance"); - sc_stats_init(×[5], "total"); - total_time -= sc_MPI_Wtime(); - switch (mesh) { - case 0: - t8_global_productionf ("Contructing cake mesh with %i pyramids.\n", - num_elements); - cmesh = t8_cmesh_new_pyramid_cake(sc_MPI_COMM_WORLD, num_elements); - break; - case 1: - t8_global_productionf ("Contructing full hybrid mesh.\n"); - cmesh = t8_cmesh_new_full_hybrid(sc_MPI_COMM_WORLD); - break; - case 2: - t8_global_productionf ("Contructing long brick out of %i pyramids.\n", - num_elements); - cmesh = t8_cmesh_new_long_brick_pyramid(sc_MPI_COMM_WORLD, num_elements); - break; - case 3: - t8_global_productionf ("Contructing mesh from: %s.\n", prefix); - cmesh = t8_cmesh_from_msh_file((char *) prefix, 0, sc_MPI_COMM_WORLD, 3, 0, 0); - break; - case 4: - t8_global_productionf("Constructing bigmesh with %i elements.\n", num_elements); - cmesh = t8_cmesh_new_bigmesh(eclass, num_elements, sc_MPI_COMM_WORLD); - break; - case 5: - t8_global_productionf ("Contructing a single %s.\n", t8_eclass_to_string[eclass]); - cmesh = t8_cmesh_new_from_class(eclass, sc_MPI_COMM_WORLD); - break; - default: - SC_ABORT("NO CMESH"); - return; - } + t8_forest_t forest, forest_adapt, forest_partition; + t8_cmesh_t cmesh, cmesh_partition; + char vtuname[BUFSIZ], cmesh_file[BUFSIZ]; + int mpirank, mpiret; + double new_time = 0, adapt_time = 0, ghost_time = + 0, partition_time = 0, total_time = 0, balance_time = 0; + sc_statinfo_t times[6]; + int procs_sent, balance_rounds; + t8_locidx_t ghost_sent; + sc_stats_init (×[0], "new"); + sc_stats_init (×[1], "adapt"); + sc_stats_init (×[2], "ghost"); + sc_stats_init (×[3], "partition"); + sc_stats_init (×[4], "balance"); + sc_stats_init (×[5], "total"); + total_time -= sc_MPI_Wtime (); + switch (mesh) { + case 0: + t8_global_productionf ("Contructing cake mesh with %i pyramids.\n", + num_elements); + cmesh = t8_cmesh_new_pyramid_cake (sc_MPI_COMM_WORLD, num_elements); + break; + case 1: + t8_global_productionf ("Contructing full hybrid mesh.\n"); + cmesh = t8_cmesh_new_full_hybrid (sc_MPI_COMM_WORLD); + break; + case 2: + t8_global_productionf ("Contructing long brick out of %i pyramids.\n", + num_elements); + cmesh = t8_cmesh_new_long_brick_pyramid (sc_MPI_COMM_WORLD, num_elements); + break; + case 3: + t8_global_productionf ("Contructing mesh from: %s.\n", prefix); + cmesh = + t8_cmesh_from_msh_file ((char *) prefix, 0, sc_MPI_COMM_WORLD, 3, 0, 0); + break; + case 4: + t8_global_productionf ("Constructing bigmesh with %i elements.\n", + num_elements); + cmesh = t8_cmesh_new_bigmesh (eclass, num_elements, sc_MPI_COMM_WORLD); + break; + case 5: + t8_global_productionf ("Contructing a single %s.\n", + t8_eclass_to_string[eclass]); + cmesh = t8_cmesh_new_from_class (eclass, sc_MPI_COMM_WORLD); + break; + default: + SC_ABORT ("NO CMESH"); + return; + } - snprintf(cmesh_file, BUFSIZ,"cmesh_hybrid"); - snprintf (vtuname, BUFSIZ, "cmesh_hybrid"); - if(mesh != 4){ - t8_cmesh_save(cmesh, cmesh_file); - if (t8_cmesh_vtk_write_file (cmesh, vtuname, 1.0) == 0) { - t8_debugf ("Output to %s\n", vtuname); - } - else { - t8_debugf ("Error in output\n"); - } + snprintf (cmesh_file, BUFSIZ, "cmesh_hybrid"); + snprintf (vtuname, BUFSIZ, "cmesh_hybrid"); + if (mesh != 4) { + t8_cmesh_save (cmesh, cmesh_file); + if (t8_cmesh_vtk_write_file (cmesh, vtuname, 1.0) == 0) { + t8_debugf ("Output to %s\n", vtuname); } - - mpiret = sc_MPI_Comm_rank (sc_MPI_COMM_WORLD, &mpirank); - SC_CHECK_MPI (mpiret); - if(part){ - t8_cmesh_init(&cmesh_partition); - t8_cmesh_set_derive(cmesh_partition, cmesh); - t8_cmesh_set_partition_uniform(cmesh_partition, level, t8_scheme_new_default_cxx()); - t8_cmesh_commit(cmesh_partition, sc_MPI_COMM_WORLD); - cmesh = cmesh_partition; + else { + t8_debugf ("Error in output\n"); } + } - t8_debugf("[D] start forest\n"); - t8_forest_init(&forest); - t8_forest_set_profiling(forest, 1); - t8_forest_set_cmesh(forest, cmesh, sc_MPI_COMM_WORLD); - t8_forest_set_scheme(forest, t8_scheme_new_default_cxx()); - t8_forest_set_level(forest, level); - new_time -= sc_MPI_Wtime(); - t8_forest_commit(forest); - new_time += sc_MPI_Wtime(); - if(do_vtk){ - snprintf (vtuname, BUFSIZ, "forest_hybrid"); - t8_forest_write_vtk (forest, vtuname); - t8_debugf("[D] output to %s\n", vtuname); - } - t8_forest_init(&forest_adapt); - t8_forest_set_user_data(forest_adapt, &endlvl); - t8_forest_set_profiling(forest_adapt, 1); - if(eclass == T8_ECLASS_PYRAMID){ - t8_debugf("Use cake-adapt\n"); - t8_forest_set_adapt(forest_adapt, forest, t8_basic_cake_refine, 1); - }else{ - t8_forest_set_adapt(forest_adapt, forest, t8_basic_hybrid_refine, 1); - } - //t8_forest_set_ghost_ext(forest_adapt, 1, T8_GHOST_FACES, 2); - adapt_time -= sc_MPI_Wtime(); - t8_forest_commit(forest_adapt); - adapt_time += sc_MPI_Wtime(); - //t8_debugf ("Successfully adapted forest.\n"); - //snprintf (vtuname, BUFSIZ, "forest_hybrid_refine"); - //t8_forest_write_vtk (forest_adapt, vtuname); - //t8_debugf ("Output to %s\n", vtuname); - //t8_forest_unref(&forest_adapt); - /* Ensure that the correct forest is passed to unref later */ + mpiret = sc_MPI_Comm_rank (sc_MPI_COMM_WORLD, &mpirank); + SC_CHECK_MPI (mpiret); + if (part) { + t8_cmesh_init (&cmesh_partition); + t8_cmesh_set_derive (cmesh_partition, cmesh); + t8_cmesh_set_partition_uniform (cmesh_partition, level, + t8_scheme_new_default_cxx ()); + t8_cmesh_commit (cmesh_partition, sc_MPI_COMM_WORLD); + cmesh = cmesh_partition; + } - t8_forest_init(&forest_partition); - t8_forest_set_partition(forest_partition, forest_adapt, 0); - t8_forest_set_ghost(forest_partition, 1, T8_GHOST_FACES); - if(balance){ - t8_forest_set_balance(forest_partition, forest_adapt,1); - } - t8_forest_set_profiling(forest_partition, 1); - t8_forest_commit(forest_partition); - if(do_vtk){ - snprintf (vtuname, BUFSIZ, "forest_hybrid_partition"); - t8_forest_write_vtk (forest_partition, vtuname); - t8_debugf ("Output to %s\n", vtuname); - } - t8_forest_print_profile(forest_partition); - partition_time += t8_forest_profile_get_partition_time(forest_partition, &procs_sent); - ghost_time += t8_forest_profile_get_ghost_time(forest_partition, &ghost_sent); - balance_time += t8_forest_profile_get_balance_time(forest_partition, &balance_rounds); - total_time += sc_MPI_Wtime(); + t8_debugf ("[D] start forest\n"); + t8_forest_init (&forest); + t8_forest_set_profiling (forest, 1); + t8_forest_set_cmesh (forest, cmesh, sc_MPI_COMM_WORLD); + t8_forest_set_scheme (forest, t8_scheme_new_default_cxx ()); + t8_forest_set_level (forest, level); + new_time -= sc_MPI_Wtime (); + t8_forest_commit (forest); + new_time += sc_MPI_Wtime (); + if (do_vtk) { + snprintf (vtuname, BUFSIZ, "forest_hybrid"); + t8_forest_write_vtk (forest, vtuname); + t8_debugf ("[D] output to %s\n", vtuname); + } + t8_forest_init (&forest_adapt); + t8_forest_set_user_data (forest_adapt, &endlvl); + t8_forest_set_profiling (forest_adapt, 1); + if (eclass == T8_ECLASS_PYRAMID) { + t8_debugf ("Use cake-adapt\n"); + t8_forest_set_adapt (forest_adapt, forest, t8_basic_cake_refine, 1); + } + else { + t8_forest_set_adapt (forest_adapt, forest, t8_basic_hybrid_refine, 1); + } + //t8_forest_set_ghost_ext(forest_adapt, 1, T8_GHOST_FACES, 2); + adapt_time -= sc_MPI_Wtime (); + t8_forest_commit (forest_adapt); + adapt_time += sc_MPI_Wtime (); + //t8_debugf ("Successfully adapted forest.\n"); + //snprintf (vtuname, BUFSIZ, "forest_hybrid_refine"); + //t8_forest_write_vtk (forest_adapt, vtuname); + //t8_debugf ("Output to %s\n", vtuname); + //t8_forest_unref(&forest_adapt); + /* Ensure that the correct forest is passed to unref later */ + + t8_forest_init (&forest_partition); + t8_forest_set_partition (forest_partition, forest_adapt, 0); + t8_forest_set_ghost (forest_partition, 1, T8_GHOST_FACES); + if (balance) { + t8_forest_set_balance (forest_partition, forest_adapt, 1); + } + t8_forest_set_profiling (forest_partition, 1); + t8_forest_commit (forest_partition); + if (do_vtk) { + snprintf (vtuname, BUFSIZ, "forest_hybrid_partition"); + t8_forest_write_vtk (forest_partition, vtuname); + t8_debugf ("Output to %s\n", vtuname); + } + t8_forest_print_profile (forest_partition); + partition_time += + t8_forest_profile_get_partition_time (forest_partition, &procs_sent); + ghost_time += + t8_forest_profile_get_ghost_time (forest_partition, &ghost_sent); + balance_time += + t8_forest_profile_get_balance_time (forest_partition, &balance_rounds); + total_time += sc_MPI_Wtime (); - sc_stats_accumulate(×[0], new_time); - sc_stats_accumulate(×[1], adapt_time); - sc_stats_accumulate(×[2], ghost_time); - sc_stats_accumulate(×[3], partition_time); - sc_stats_accumulate(×[4], balance_time); - sc_stats_accumulate(×[5], total_time); - sc_stats_compute(sc_MPI_COMM_WORLD, 6, times); - sc_stats_print(t8_get_package_id(), SC_LP_ESSENTIAL, 6, times, 1,1); + sc_stats_accumulate (×[0], new_time); + sc_stats_accumulate (×[1], adapt_time); + sc_stats_accumulate (×[2], ghost_time); + sc_stats_accumulate (×[3], partition_time); + sc_stats_accumulate (×[4], balance_time); + sc_stats_accumulate (×[5], total_time); + sc_stats_compute (sc_MPI_COMM_WORLD, 6, times); + sc_stats_print (t8_get_package_id (), SC_LP_ESSENTIAL, 6, times, 1, 1); - t8_forest_unref(&forest_partition); + t8_forest_unref (&forest_partition); } int main (int argc, char **argv) { int mpiret, parsed; - int level, endlvl, helpme, do_vtk, eclass_int, mesh, elements, balance, part; + int level, endlvl, helpme, do_vtk, eclass_int, mesh, + elements, balance, part; t8_eclass_t eclass; - sc_options_t *opt; + sc_options_t *opt; char usage[BUFSIZ]; char help[BUFSIZ]; - const char *file; + const char *file; /* brief help message */ snprintf (usage, BUFSIZ, "Usage:\t%s \n\t%s -h\t" @@ -283,49 +293,56 @@ main (int argc, char **argv) sc_init (sc_MPI_COMM_WORLD, 1, 1, NULL, SC_LP_ESSENTIAL); t8_init (SC_LP_DEFAULT); - opt = sc_options_new(argv[0]); + opt = sc_options_new (argv[0]); sc_options_add_switch (opt, 'h', "help", &helpme, "Display a short help message."); sc_options_add_int (opt, 'l', "level", &level, 0, "The refinement level of the mesh."); sc_options_add_int (opt, 'f', "final-level", &endlvl, 1, "The final refinement level of the mesh."); - sc_options_add_switch(opt, 'v', "vtk", &do_vtk, "Enable vtk-output."); - sc_options_add_switch(opt, 'b', "balance", &balance, "Enable balance"); - sc_options_add_switch(opt, 'p', "partition", &part, "Enable cmesh-partition"); - sc_options_add_int(opt, 'e', "element", &eclass_int, 4, "Given an element-class, the programm will " - " construct a single element of this class. Is ignored, if the option cake is chosen." - "The type of elements to use.\n" - "\t\t4 - hexahedron\n" - "\t\t5 - tetrahedron\n\t\t6 - prism\n\t\t7 - pyramid"); - sc_options_add_int(opt, 'm', "mesh", &mesh, 5,"A mesh to choose from." - " The meshes to chose from are: \n" - "\t\t0 - cake of pyramids\n\t\t1 - hybrid mesh with all 3D elements" - "\n\t\t2 - a long row of bricks out ot pyramids" - "\n\t\t3 - user-specific mesh-file" - "\n\t\t4 - construct multiple elements of class given by -e" - "\n\t\t5 - a single element of class -e"); - sc_options_add_int(opt, 'n', "num_elements", &elements, 3, "The number of elements to use" - " if a m0 or m4 is build. Has to be larger than 2."); - sc_options_add_string(opt, 'g', "mshfile", &file, "NULL", "Prefix of the msh-file."); + sc_options_add_switch (opt, 'v', "vtk", &do_vtk, "Enable vtk-output."); + sc_options_add_switch (opt, 'b', "balance", &balance, "Enable balance"); + sc_options_add_switch (opt, 'p', "partition", &part, + "Enable cmesh-partition"); + sc_options_add_int (opt, 'e', "element", &eclass_int, 4, + "Given an element-class, the programm will " + " construct a single element of this class. Is ignored, if the option cake is chosen." + "The type of elements to use.\n" "\t\t4 - hexahedron\n" + "\t\t5 - tetrahedron\n\t\t6 - prism\n\t\t7 - pyramid"); + sc_options_add_int (opt, 'm', "mesh", &mesh, 5, + "A mesh to choose from." + " The meshes to chose from are: \n" + "\t\t0 - cake of pyramids\n\t\t1 - hybrid mesh with all 3D elements" + "\n\t\t2 - a long row of bricks out ot pyramids" + "\n\t\t3 - user-specific mesh-file" + "\n\t\t4 - construct multiple elements of class given by -e" + "\n\t\t5 - a single element of class -e"); + sc_options_add_int (opt, 'n', "num_elements", &elements, 3, + "The number of elements to use" + " if a m0 or m4 is build. Has to be larger than 2."); + sc_options_add_string (opt, 'g', "mshfile", &file, "NULL", + "Prefix of the msh-file."); - parsed = sc_options_parse(t8_get_package_id(), SC_LP_DEFAULT, opt, argc, argv); + parsed = + sc_options_parse (t8_get_package_id (), SC_LP_DEFAULT, opt, argc, argv); if (helpme) { /* display help message and usage */ t8_global_productionf ("%s\n", help); sc_options_print_usage (t8_get_package_id (), SC_LP_ERROR, opt, NULL); } - else if(parsed >= 0 && 0 <= level && 4 <= eclass_int && eclass_int < T8_ECLASS_COUNT && - elements >= 2 && 0 <=mesh && mesh < 6){ - eclass = (t8_eclass_t) eclass_int; - t8_basic_hybrid (level, endlvl, do_vtk, eclass, elements, mesh, balance, file, part); + else if (parsed >= 0 && 0 <= level && 4 <= eclass_int + && eclass_int < T8_ECLASS_COUNT && elements >= 2 && 0 <= mesh + && mesh < 6) { + eclass = (t8_eclass_t) eclass_int; + t8_basic_hybrid (level, endlvl, do_vtk, eclass, elements, mesh, balance, + file, part); } else { /* wrong usage */ t8_global_productionf ("\n\t ERROR: Wrong usage.\n\n"); sc_options_print_usage (t8_get_package_id (), SC_LP_ERROR, opt, NULL); } - sc_options_destroy(opt); + sc_options_destroy (opt); sc_finalize (); mpiret = sc_MPI_Finalize (); From c6d505e32524a694055855507bdd479c6a94bebe Mon Sep 17 00:00:00 2001 From: Knapp Date: Mon, 20 Mar 2023 13:17:30 +0100 Subject: [PATCH 09/43] Merge Clean-up --- src/t8.c | 4 +-- src/t8.h | 2 +- src/t8_cmesh.h | 21 ++++++----- src/t8_cmesh/t8_cmesh_cxx.cxx | 58 +++++++++++++++---------------- src/t8_cmesh/t8_cmesh_examples.c | 6 +++- src/t8_cmesh/t8_cmesh_partition.c | 6 ++-- src/t8_data/t8_shmem.h | 5 +-- src/t8_forest/t8_forest.c | 2 -- src/t8_forest/t8_forest_cxx.cxx | 11 +++--- test/t8_gtest_vtk_linkage.cxx | 4 +-- 10 files changed, 64 insertions(+), 55 deletions(-) diff --git a/src/t8.c b/src/t8.c index d8dcef121c..1d04ced83a 100644 --- a/src/t8.c +++ b/src/t8.c @@ -167,8 +167,8 @@ t8_sc_array_index_locidx (sc_array_t *array, t8_locidx_t it) return array->array + array->elem_size * (size_t) it; } -void * -t8_sc_array_index_gloidx (sc_array_t * array, t8_gloidx_t it) +void * +t8_sc_array_index_gloidx (sc_array_t *array, t8_gloidx_t it) { T8_ASSERT (it >= 0 && (size_t) it < array->elem_count); return array->array + array->elem_size * (size_t) it; diff --git a/src/t8.h b/src/t8.h index 5a15b9ac67..6ee23bca2f 100644 --- a/src/t8.h +++ b/src/t8.h @@ -265,7 +265,7 @@ void *t8_sc_array_index_locidx (sc_array_t *array, * \param [in] index needs to be in [0]..[elem_count-1]. * \return A void * pointing to entry \a it in \a array. */ -void *t8_sc_array_index_gloidx (sc_array_t * array, +void *t8_sc_array_index_gloidx (sc_array_t *array, t8_gloidx_t it); /* call this at the end of a header file to match T8_EXTERN_C_BEGIN (). */ diff --git a/src/t8_cmesh.h b/src/t8_cmesh.h index 26fd33b722..00903f9fb4 100644 --- a/src/t8_cmesh.h +++ b/src/t8_cmesh.h @@ -697,14 +697,19 @@ void t8_cmesh_uniform_bounds (t8_cmesh_t cmesh, int level, * same as \a last_local_tree on the next process. * \param [in] comm The communicator */ -void t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, - int level, - t8_scheme_cxx_t * scheme, - t8_gloidx_t * first_local_tree, - t8_gloidx_t * child_in_tree_begin, - t8_gloidx_t * last_local_tree, - t8_gloidx_t * child_in_tree_end, - int8_t * first_tree_shared, sc_MPI_Comm comm); +void t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, + int level, + t8_scheme_cxx_t *scheme, + t8_gloidx_t + *first_local_tree, + t8_gloidx_t + *child_in_tree_begin, + t8_gloidx_t + *last_local_tree, + t8_gloidx_t + *child_in_tree_end, + int8_t *first_tree_shared, + sc_MPI_Comm comm); /** Increase the reference counter of a cmesh. * \param [in,out] cmesh On input, this cmesh must exist with positive diff --git a/src/t8_cmesh/t8_cmesh_cxx.cxx b/src/t8_cmesh/t8_cmesh_cxx.cxx index d0a9724b4e..ebd0ab8edf 100644 --- a/src/t8_cmesh/t8_cmesh_cxx.cxx +++ b/src/t8_cmesh/t8_cmesh_cxx.cxx @@ -46,15 +46,14 @@ typedef struct * the uniform partition is empty. * We set all values to -1 and first_tree_shared to 0. */ static void -t8_cmesh_uniform_set_return_parameters_to_empty (t8_gloidx_t * - first_local_tree, - t8_gloidx_t * - child_in_tree_begin, - t8_gloidx_t * - last_local_tree, - t8_gloidx_t * - child_in_tree_end, - int8_t * first_tree_shared) +t8_cmesh_uniform_set_return_parameters_to_empty (t8_gloidx_t + *first_local_tree, + t8_gloidx_t + *child_in_tree_begin, + t8_gloidx_t *last_local_tree, + t8_gloidx_t + *child_in_tree_end, + int8_t *first_tree_shared) { *first_local_tree = *last_local_tree = -1; if (child_in_tree_begin != NULL) { @@ -254,8 +253,8 @@ t8_cmesh_uniform_bounds (t8_cmesh_t cmesh, int level, * storing the number of processe and the global number of elements. * * This function is used standalone and as callback of sc_array_split. */ -static size_t -t8_cmesh_determine_partition (sc_array_t * first_element_tree, +static size_t +t8_cmesh_determine_partition (sc_array_t *first_element_tree, size_t pure_local_tree, void *data) { T8_ASSERT (data != NULL); @@ -315,15 +314,15 @@ t8_cmesh_determine_partition (sc_array_t * first_element_tree, * use partition_given to partition the cmesh*/ void t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, - t8_scheme_cxx_t * scheme, - t8_gloidx_t * first_local_tree, - t8_gloidx_t * child_in_tree_begin, - t8_gloidx_t * last_local_tree, - t8_gloidx_t * child_in_tree_end, - int8_t * first_tree_shared, sc_MPI_Comm comm) + t8_scheme_cxx_t *scheme, + t8_gloidx_t *first_local_tree, + t8_gloidx_t *child_in_tree_begin, + t8_gloidx_t *last_local_tree, + t8_gloidx_t *child_in_tree_end, + int8_t *first_tree_shared, sc_MPI_Comm comm) { t8_gloidx_t local_num_children = 0; - t8_gloidx_t *elem_index_pointer; + t8_gloidx_t *elem_index_pointer; int send_first, send_last, send_first_nonempty, num_procs_we_send_to = 0; t8_cmesh_partition_query_t data; @@ -378,7 +377,7 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, /* Do not consider shared trees */ if (cmesh->first_tree_shared && cmesh->set_partition) { - const int ieclass = t8_cmesh_get_tree_class (cmesh, 0); + const int ieclass = t8_cmesh_get_tree_class (cmesh, 0); tree_scheme = scheme->eclass_schemes[ieclass]; local_num_children -= tree_scheme->t8_element_count_leafs_from_root (level); @@ -398,10 +397,10 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, /* Compute the first and last element of this process. Then loop over * all trees to find the trees in which these are contained. * We cast to long double and double to prevent overflow. */ - const t8_gloidx_t first_child = + const t8_gloidx_t first_child = t8_cmesh_get_first_element_of_process (cmesh->mpirank, cmesh->mpisize, local_num_children); - const t8_gloidx_t last_child = + const t8_gloidx_t last_child = t8_cmesh_get_first_element_of_process (cmesh->mpirank + 1, cmesh->mpisize, local_num_children) - 1; @@ -413,12 +412,14 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, */ current_tree_element_offset = 0; for (t8_gloidx_t igtree = 0; igtree < num_trees; ++igtree) { - const int ieclass = t8_cmesh_get_tree_class (cmesh, (t8_locidx_t) igtree); + const int ieclass = + t8_cmesh_get_tree_class (cmesh, (t8_locidx_t) igtree); tree_scheme = scheme->eclass_schemes[ieclass]; /* TODO: We can optimize by buffering the elem_in_tree value. Thus, if the computation is expensive (may be for non-morton-type schemes), we do it only once. */ - const t8_gloidx_t elem_in_tree = tree_scheme->t8_element_count_leafs_from_root (level); + const t8_gloidx_t elem_in_tree = + tree_scheme->t8_element_count_leafs_from_root (level); /* Check if the first element is on the current tree */ if (current_tree_element_offset <= first_child && first_child < current_tree_element_offset + elem_in_tree) { @@ -496,12 +497,13 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, t8_debugf ("[H] global num %li\n", data.global_num_elements); /*Compute number of non-shared-trees and the local index of the first non-shared-tree */ - const t8_locidx_t pure_local_trees = cmesh->num_local_trees - first_tree_shared_shift; + const t8_locidx_t pure_local_trees = + cmesh->num_local_trees - first_tree_shared_shift; if (pure_local_trees > 0) { /* Compute which trees and elements to send to which process. * We skip empty processes. */ - t8_locidx_t igtree = first_tree_shared_shift; + t8_locidx_t igtree = first_tree_shared_shift; sc_array_init_size (&first_element_tree, sizeof (t8_gloidx_t), pure_local_trees + 1); @@ -521,7 +523,7 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, (const t8_gloidx_t *) sc_array_index_int (&first_element_tree, itree); t8_gloidx_t *first_element_of_next_tree = (t8_gloidx_t *) sc_array_index_int (&first_element_tree, itree + 1); - const int ieclass = t8_cmesh_get_tree_class (cmesh, igtree); + const int ieclass = t8_cmesh_get_tree_class (cmesh, igtree); tree_scheme = scheme->eclass_schemes[ieclass]; /* Set the first element of the next tree by adding the number of element of the current tree. */ *first_element_of_next_tree = @@ -565,7 +567,6 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, } num_procs_we_send_to = send_last - send_first + 1; - sc_array_init_size (&offset_partition, sizeof (size_t), num_procs_we_send_to); /* In array split the 'types' that we compute are the processes we send to, @@ -1009,7 +1010,6 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, T8_ASSERT (num_received_start_messages == 1); T8_ASSERT (num_received_end_messages == 1); - if (pure_local_trees > 0) { sc_array_reset (&first_element_tree); sc_array_reset (&offset_partition); @@ -1022,4 +1022,4 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, t8_debugf ("Done with t8_cmesh_uniform_bounds_hybrid.\n"); return; -} \ No newline at end of file +} diff --git a/src/t8_cmesh/t8_cmesh_examples.c b/src/t8_cmesh/t8_cmesh_examples.c index dec67a00c3..c36a7e89cd 100644 --- a/src/t8_cmesh/t8_cmesh_examples.c +++ b/src/t8_cmesh/t8_cmesh_examples.c @@ -2067,6 +2067,7 @@ t8_cmesh_new_long_brick_pyramid (sc_MPI_Comm comm, int num_cubes) return cmesh; } +t8_cmesh_t t8_cmesh_new_row_of_cubes (t8_locidx_t num_trees, const int set_attributes, sc_MPI_Comm comm) { @@ -2078,7 +2079,7 @@ t8_cmesh_new_row_of_cubes (t8_locidx_t num_trees, const int set_attributes, t8_cmesh_register_geometry (cmesh, linear_geom); /* Vertices of first cube in row. */ - double vertices_coords[24] = { + double vertices[24] = { 0, 0, 0, 1, 0, 0, 0, 1, 0, @@ -2113,3 +2114,6 @@ t8_cmesh_new_row_of_cubes (t8_locidx_t num_trees, const int set_attributes, for (t8_locidx_t tree_id = 0; tree_id < num_trees - 1; tree_id++) { t8_cmesh_set_join (cmesh, tree_id, tree_id + 1, 0, 1, 0); } + t8_cmesh_commit (cmesh, comm); + return cmesh; +} diff --git a/src/t8_cmesh/t8_cmesh_partition.c b/src/t8_cmesh/t8_cmesh_partition.c index 28f7ad6d13..747080433a 100644 --- a/src/t8_cmesh/t8_cmesh_partition.c +++ b/src/t8_cmesh/t8_cmesh_partition.c @@ -1777,9 +1777,9 @@ t8_cmesh_partition (t8_cmesh_t cmesh, sc_MPI_Comm comm) ts = cmesh->set_partition_scheme; /* The refinement scheme */ T8_ASSERT (ts != NULL); t8_cmesh_uniform_bounds_hybrid (cmesh_from, cmesh->set_partition_level, - ts, - &cmesh->first_tree, NULL, &last_tree, NULL, - &cmesh->first_tree_shared, comm); + ts, + &cmesh->first_tree, NULL, &last_tree, + NULL, &cmesh->first_tree_shared, comm); cmesh->num_local_trees = last_tree - cmesh->first_tree + 1; /* Compute the tree offset */ t8_cmesh_gather_treecount_nocommit (cmesh, comm); diff --git a/src/t8_data/t8_shmem.h b/src/t8_data/t8_shmem.h index 369eaf2828..00a587cc99 100644 --- a/src/t8_data/t8_shmem.h +++ b/src/t8_data/t8_shmem.h @@ -157,8 +157,9 @@ void t8_shmem_array_allgather (const void *sendbuf, * \param[in] type the type of items to allgather * \param[in] op the operation to prefix */ -void t8_shmem_prefix (void *sendbuf, t8_shmem_array_t recvarray, - int count, sc_MPI_Datatype type, sc_MPI_Op op); +void t8_shmem_prefix (void *sendbuf, + t8_shmem_array_t recvarray, int count, + sc_MPI_Datatype type, sc_MPI_Op op); /** Return the MPI communicator associated with a shmem array. * \param [in] array The shmem_array to be queried. diff --git a/src/t8_forest/t8_forest.c b/src/t8_forest/t8_forest.c index 6b30134807..4d3addae36 100644 --- a/src/t8_forest/t8_forest.c +++ b/src/t8_forest/t8_forest.c @@ -372,8 +372,6 @@ t8_forest_refine_everything (t8_forest_t forest, t8_forest_t forest_from, return 1; } - - /**Algorithm to populate a forest, if any tree refines irregularly. * Create the elements on this process given a uniform partition * of the coarse mesh. We can not use the function t8_forest_populate, because diff --git a/src/t8_forest/t8_forest_cxx.cxx b/src/t8_forest/t8_forest_cxx.cxx index 2256296d06..d8a849a5a3 100644 --- a/src/t8_forest/t8_forest_cxx.cxx +++ b/src/t8_forest/t8_forest_cxx.cxx @@ -1280,7 +1280,7 @@ t8_forest_refines_irregular (t8_forest_t forest) tscheme = t8_forest_get_eclass_scheme_before_commit (forest, (t8_eclass_t) int_eclass); - irregular = irregular || tscheme->t8_element_refines_irregular(); + irregular = irregular || tscheme->t8_element_refines_irregular (); } } /* Combine the process-local results via a logic or and distribute the @@ -1316,10 +1316,11 @@ t8_forest_populate (t8_forest_t forest) "Given refinement level exceeds the maximum.\n"); /* TODO: create trees and quadrants according to uniform refinement */ t8_cmesh_uniform_bounds_hybrid (forest->cmesh, forest->set_level, - forest->scheme_cxx, - &forest->first_local_tree, - &child_in_tree_begin, &forest->last_local_tree, - &child_in_tree_end, NULL, forest->mpicomm); + forest->scheme_cxx, + &forest->first_local_tree, + &child_in_tree_begin, + &forest->last_local_tree, + &child_in_tree_end, NULL, forest->mpicomm); /* True if the forest has no elements */ is_empty = forest->first_local_tree > forest->last_local_tree diff --git a/test/t8_gtest_vtk_linkage.cxx b/test/t8_gtest_vtk_linkage.cxx index cf63b1316b..0b0afc1f50 100644 --- a/test/t8_gtest_vtk_linkage.cxx +++ b/test/t8_gtest_vtk_linkage.cxx @@ -56,9 +56,9 @@ TEST (t8_gtest_vtk_linkage, t8_test_vtk_version_number) ASSERT_FALSE (strcmp (T8_VTK_VERSION_USED, vtk_version)) << "linked vtk version (" << vtk_version << ") does not equal the version t8code was configured with (" << - T (T8_VTK_VERSION_USED << ").\n"); + T8_VTK_VERSION_USED << ").\n"; if (!strcmp (T8_VTK_VERSION_USED, vtk_version)) { - t8_debugf ("Using vtk version %s.\n", vtk_verion); + t8_debugf ("Using vtk version %s.\n", vtk_version); } #endif } From 7c90788d4279ff76fa127f764cd8aced04953136 Mon Sep 17 00:00:00 2001 From: David Knapp Date: Mon, 25 Sep 2023 13:42:17 +0200 Subject: [PATCH 10/43] Add missing \ to Makefile --- example/cmesh/Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/cmesh/Makefile.am b/example/cmesh/Makefile.am index d4e9ffd9cc..0b8800ff60 100644 --- a/example/cmesh/Makefile.am +++ b/example/cmesh/Makefile.am @@ -6,7 +6,7 @@ bin_PROGRAMS += \ example/cmesh/t8_cmesh_partition \ example/cmesh/t8_cmesh_create_partitioned \ example/cmesh/t8_cmesh_refine \ - example/cmesh/t8_cmesh_hybrid_new + example/cmesh/t8_cmesh_hybrid_new \ example/cmesh/t8_cmesh_set_join_by_vertices \ example/cmesh/t8_cmesh_geometry_examples \ example/cmesh/t8_cmesh_hypercube_pad From ce05a7d10add157da2003c45072b322eed56509d Mon Sep 17 00:00:00 2001 From: David Knapp Date: Mon, 25 Sep 2023 15:57:20 +0200 Subject: [PATCH 11/43] Update headers used in example --- example/cmesh/t8_cmesh_hybrid_new.cxx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/example/cmesh/t8_cmesh_hybrid_new.cxx b/example/cmesh/t8_cmesh_hybrid_new.cxx index 4e2ef4fe9f..100ba53a1b 100644 --- a/example/cmesh/t8_cmesh_hybrid_new.cxx +++ b/example/cmesh/t8_cmesh_hybrid_new.cxx @@ -27,11 +27,11 @@ #include #include #include -#include +#include #include #include #include -#include +#include #include static int From 4add0d6b80ed0aeaf37bd43dd68939d220f75da5 Mon Sep 17 00:00:00 2001 From: David Knapp Date: Mon, 25 Sep 2023 15:58:13 +0200 Subject: [PATCH 12/43] Use hybrid_bounds algorithms --- src/t8_cmesh/t8_cmesh_partition.c | 4 ++-- src/t8_forest/t8_forest_cxx.cxx | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/t8_cmesh/t8_cmesh_partition.c b/src/t8_cmesh/t8_cmesh_partition.c index 181d137f11..4dc2e9751c 100644 --- a/src/t8_cmesh/t8_cmesh_partition.c +++ b/src/t8_cmesh/t8_cmesh_partition.c @@ -1571,8 +1571,8 @@ t8_cmesh_partition (t8_cmesh_t cmesh, sc_MPI_Comm comm) T8_ASSERT (cmesh->tree_offsets == NULL); ts = cmesh->set_partition_scheme; /* The refinement scheme */ T8_ASSERT (ts != NULL); - t8_cmesh_uniform_bounds (cmesh_from, cmesh->set_partition_level, ts, &cmesh->first_tree, NULL, &last_tree, NULL, - &cmesh->first_tree_shared); + t8_cmesh_uniform_bounds_hybrid (cmesh_from, cmesh->set_partition_level, ts, &cmesh->first_tree, NULL, &last_tree, + NULL, &cmesh->first_tree_shared, comm); cmesh->num_local_trees = last_tree - cmesh->first_tree + 1; /* Compute the tree offset */ t8_cmesh_gather_treecount_nocommit (cmesh, comm); diff --git a/src/t8_forest/t8_forest_cxx.cxx b/src/t8_forest/t8_forest_cxx.cxx index 2e2faf96af..dad338fd2e 100644 --- a/src/t8_forest/t8_forest_cxx.cxx +++ b/src/t8_forest/t8_forest_cxx.cxx @@ -1448,8 +1448,11 @@ t8_forest_populate (t8_forest_t forest) SC_CHECK_ABORT (forest->set_level <= forest->maxlevel, "Given refinement level exceeds the maximum.\n"); /* TODO: create trees and quadrants according to uniform refinement */ - t8_cmesh_uniform_bounds (forest->cmesh, forest->set_level, forest->scheme_cxx, &forest->first_local_tree, - &child_in_tree_begin, &forest->last_local_tree, &child_in_tree_end, NULL); + t8_cmesh_uniform_bounds_hybrid (forest->cmesh, forest->set_level, forest->scheme_cxx, &forest->first_local_tree, + &child_in_tree_begin, &forest->last_local_tree, &child_in_tree_end, NULL, + forest->mpicomm); + t8_debugf ("[D] flt: %li, citb: %li, llt: %li, cite: %li\n", forest->first_local_tree, child_in_tree_begin, + forest->last_local_tree, child_in_tree_end); /* True if the forest has no elements */ is_empty = forest->first_local_tree > forest->last_local_tree From 1ef18eda073c4765b3c5a8ac09eaa618faa84bde Mon Sep 17 00:00:00 2001 From: David Knapp Date: Mon, 23 Oct 2023 16:54:51 +0200 Subject: [PATCH 13/43] WIP: Debugging cmesh_uniform_bounds_hybrid --- example/cmesh/t8_cmesh_hybrid_new.cxx | 19 +++++++++---------- src/t8_cmesh/t8_cmesh_cxx.cxx | 6 +++++- src/t8_cmesh/t8_cmesh_offset.c | 5 +++++ test/t8_cmesh/t8_gtest_cmesh_partition.cxx | 1 + 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/example/cmesh/t8_cmesh_hybrid_new.cxx b/example/cmesh/t8_cmesh_hybrid_new.cxx index 100ba53a1b..ff8d72adb9 100644 --- a/example/cmesh/t8_cmesh_hybrid_new.cxx +++ b/example/cmesh/t8_cmesh_hybrid_new.cxx @@ -163,6 +163,15 @@ t8_basic_hybrid (int level, int endlvl, int do_vtk, t8_eclass_t eclass, int num_ return; } + mpiret = sc_MPI_Comm_rank (sc_MPI_COMM_WORLD, &mpirank); + SC_CHECK_MPI (mpiret); + if (part) { + t8_cmesh_init (&cmesh_partition); + t8_cmesh_set_derive (cmesh_partition, cmesh); + t8_cmesh_set_partition_uniform (cmesh_partition, level, t8_scheme_new_default_cxx ()); + t8_cmesh_commit (cmesh_partition, sc_MPI_COMM_WORLD); + cmesh = cmesh_partition; + } snprintf (cmesh_file, BUFSIZ, "cmesh_hybrid"); snprintf (vtuname, BUFSIZ, "cmesh_hybrid"); if (mesh != 4) { @@ -175,16 +184,6 @@ t8_basic_hybrid (int level, int endlvl, int do_vtk, t8_eclass_t eclass, int num_ } } - mpiret = sc_MPI_Comm_rank (sc_MPI_COMM_WORLD, &mpirank); - SC_CHECK_MPI (mpiret); - if (part) { - t8_cmesh_init (&cmesh_partition); - t8_cmesh_set_derive (cmesh_partition, cmesh); - t8_cmesh_set_partition_uniform (cmesh_partition, level, t8_scheme_new_default_cxx ()); - t8_cmesh_commit (cmesh_partition, sc_MPI_COMM_WORLD); - cmesh = cmesh_partition; - } - t8_debugf ("[D] start forest\n"); t8_forest_init (&forest); t8_forest_set_profiling (forest, 1); diff --git a/src/t8_cmesh/t8_cmesh_cxx.cxx b/src/t8_cmesh/t8_cmesh_cxx.cxx index ed37504a21..d1f4701ab4 100644 --- a/src/t8_cmesh/t8_cmesh_cxx.cxx +++ b/src/t8_cmesh/t8_cmesh_cxx.cxx @@ -50,6 +50,7 @@ t8_cmesh_uniform_set_return_parameters_to_empty (t8_gloidx_t *first_local_tree, t8_gloidx_t *last_local_tree, t8_gloidx_t *child_in_tree_end, int8_t *first_tree_shared) { + t8_debugf ("[D] proc is empty\n"); *first_local_tree = *last_local_tree = -1; if (child_in_tree_begin != NULL) { *child_in_tree_begin = -1; @@ -311,6 +312,7 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t *sc t8_debugf ("Into t8_cmesh_uniform_bounds_hybrid.\n"); if (t8_cmesh_is_empty (cmesh)) { + t8_debugf ("[D] cmesh_uniform_bounds_hybrid is empty\n"); t8_cmesh_uniform_set_return_parameters_to_empty (first_local_tree, child_in_tree_begin, last_local_tree, child_in_tree_end, first_tree_shared); return; @@ -356,6 +358,7 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t *sc /* Compute the first and last element of this process. Then loop over * all trees to find the trees in which these are contained. * We cast to long double and double to prevent overflow. */ + /* cmesh is replicated, therefore the computation of local_num_children equals the global number of children*/ const t8_gloidx_t first_child = t8_cmesh_get_first_element_of_process (cmesh->mpirank, cmesh->mpisize, local_num_children); const t8_gloidx_t last_child @@ -390,6 +393,7 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t *sc /* This process is empty. We needed to identify the 'first_local_tree' since it is the * first local tree of the next process, which we store in this case. */ + t8_debugf ("[D] lc < fc\n"); t8_cmesh_uniform_set_return_parameters_to_empty (first_local_tree, child_in_tree_begin, last_local_tree, child_in_tree_end, first_tree_shared); return; @@ -412,7 +416,7 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t *sc } /* If we reach this part, we do not have any trees - the cmesh is empty */ T8_ASSERT (num_trees == 0); - + t8_debugf ("[D] cmesh is empty\n"); t8_cmesh_uniform_set_return_parameters_to_empty (first_local_tree, child_in_tree_begin, last_local_tree, child_in_tree_end, first_tree_shared); return; diff --git a/src/t8_cmesh/t8_cmesh_offset.c b/src/t8_cmesh/t8_cmesh_offset.c index a2db11a4ec..230fc1ea3e 100644 --- a/src/t8_cmesh/t8_cmesh_offset.c +++ b/src/t8_cmesh/t8_cmesh_offset.c @@ -130,11 +130,13 @@ t8_offset_consistent (int mpisize, const t8_shmem_array_t offset_shmem, t8_gloid const t8_gloidx_t *offset = t8_shmem_array_get_gloidx_array (offset_shmem); ret = offset[0] == 0; + t8_debugf ("[D] init ret: %i\n", ret); last_tree = t8_offset_last (0, offset); /* stores the last tree of process i-1 */ for (i = 1; i < mpisize && ret; i++) { if (t8_offset_empty (i, offset)) { /* If the process is empty, then its first tree must not be shared */ ret &= offset[i] >= 0; + t8_debugf ("[LD] offset empty, proc: %i ret = %i\n", i, ret); } else { /* If the process is not empty its first local tree must be bigger or @@ -142,12 +144,15 @@ t8_offset_consistent (int mpisize, const t8_shmem_array_t offset_shmem, t8_gloid * Equality must only hold, when the first tree is shared, thus offset[i] < 0 */ if (offset[i] < 0) { ret &= t8_offset_first (i, offset) == last_tree; + t8_debugf ("[LD] shared tree, offset first == last_tree proc: %iret = %i\n", i, ret); } else { ret &= t8_offset_first (i, offset) > last_tree; + t8_debugf ("[LD] shared tree, offset_first > last_tree proc: %i ret = %i\n", i, ret); } last_tree = t8_offset_last (i, offset); ret &= (last_tree <= num_trees); + t8_debugf ("[LD] last_tree <= num_trees proc: %i ret = %i\n", i, ret); } } ret &= (offset[mpisize] == num_trees); diff --git a/test/t8_cmesh/t8_gtest_cmesh_partition.cxx b/test/t8_cmesh/t8_gtest_cmesh_partition.cxx index 3b8867907d..9197b4f38b 100644 --- a/test/t8_cmesh/t8_gtest_cmesh_partition.cxx +++ b/test/t8_cmesh/t8_gtest_cmesh_partition.cxx @@ -45,6 +45,7 @@ class t8_cmesh_partition_class: public testing::TestWithParam { } cmesh_original = t8_test_create_cmesh (cmesh_id); + t8_debugf ("[D] is partitioned: %i\n", cmesh_original->set_partition); } int cmesh_id; From 58892c864f8f7e1d8ec5ae79ba5467647e213950 Mon Sep 17 00:00:00 2001 From: "Dreyer, Lukas" Date: Wed, 25 Oct 2023 16:00:06 +0200 Subject: [PATCH 14/43] Fix special cases for empty procs --- src/t8_cmesh/t8_cmesh_cxx.cxx | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/t8_cmesh/t8_cmesh_cxx.cxx b/src/t8_cmesh/t8_cmesh_cxx.cxx index d1f4701ab4..b6e3505e94 100644 --- a/src/t8_cmesh/t8_cmesh_cxx.cxx +++ b/src/t8_cmesh/t8_cmesh_cxx.cxx @@ -396,6 +396,7 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t *sc t8_debugf ("[D] lc < fc\n"); t8_cmesh_uniform_set_return_parameters_to_empty (first_local_tree, child_in_tree_begin, last_local_tree, child_in_tree_end, first_tree_shared); + *first_local_tree = 0; return; } /* Check if the last element is on the current tree */ @@ -482,7 +483,8 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t *sc if (!cmesh->first_tree_shared && t8_cmesh_get_global_id (cmesh, 0) == 0) { /* If our first tree is 0 and not shared, then we need to send to all processes * below the start process. Even if their partition is empty. */ - send_first = 0; + t8_debugf("LLL: Would have set send_first to 0, instead: %i", send_first_nonempty); + send_first = send_first_nonempty; } else { send_first = send_first_nonempty; @@ -802,8 +804,30 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t *sc } /* End loop over processes */ } /* if (pure_local_trees > 0) */ + const t8_gloidx_t first_element_index_of_current_proc + = t8_cmesh_get_first_element_of_process (cmesh->mpirank, data.num_procs, data.global_num_elements); + const t8_gloidx_t last_element_index_of_current_proc + = t8_cmesh_get_first_element_of_process (cmesh->mpirank + 1, data.num_procs, data.global_num_elements) - 1; + + if(first_element_index_of_current_proc > last_element_index_of_current_proc){ + expect_start_message = 0; + expect_end_message = 0; + *first_local_tree = 0; + *last_local_tree = -1; + *first_tree_shared = 0; + if (child_in_tree_begin != NULL) { + *child_in_tree_begin = -1; + } + if (child_in_tree_end != NULL) { + *child_in_tree_end = - 1; + } + num_received_end_messages++; + num_received_start_messages++; + } + /* Post the receives. */ if (expect_start_message) { + t8_debugf("expect start message\n"); const int num_entries = 2; t8_gloidx_t *message = T8_ALLOC (t8_gloidx_t, num_entries); mpiret = sc_MPI_Recv (message, num_entries, T8_MPI_GLOIDX, sc_MPI_ANY_SOURCE, T8_MPI_CMESH_UNIFORM_BOUNDS_START, From 8ce611e83da9f142d8a3ba0df0894d45d5985a62 Mon Sep 17 00:00:00 2001 From: "Dreyer, Lukas" Date: Thu, 26 Oct 2023 10:16:10 +0200 Subject: [PATCH 15/43] Reintroduce cmeshes with pyramids --- test/t8_cmesh/t8_gtest_cmesh_partition.cxx | 5 ----- 1 file changed, 5 deletions(-) diff --git a/test/t8_cmesh/t8_gtest_cmesh_partition.cxx b/test/t8_cmesh/t8_gtest_cmesh_partition.cxx index 9197b4f38b..52302e9257 100644 --- a/test/t8_cmesh/t8_gtest_cmesh_partition.cxx +++ b/test/t8_cmesh/t8_gtest_cmesh_partition.cxx @@ -39,11 +39,6 @@ class t8_cmesh_partition_class: public testing::TestWithParam { SetUp () override { cmesh_id = GetParam (); - - if (cmesh_id == 89 || (237 <= cmesh_id && cmesh_id <= 256)) { - GTEST_SKIP (); - } - cmesh_original = t8_test_create_cmesh (cmesh_id); t8_debugf ("[D] is partitioned: %i\n", cmesh_original->set_partition); } From cbbb98086d80e75792e0345a15109a0b9b7d66a7 Mon Sep 17 00:00:00 2001 From: "Dreyer, Lukas" Date: Thu, 26 Oct 2023 10:17:12 +0200 Subject: [PATCH 16/43] Fix number of vertices for pyramid cmesh --- src/t8_cmesh/t8_cmesh_examples.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/t8_cmesh/t8_cmesh_examples.c b/src/t8_cmesh/t8_cmesh_examples.c index a272061aaf..3744af1795 100644 --- a/src/t8_cmesh/t8_cmesh_examples.c +++ b/src/t8_cmesh/t8_cmesh_examples.c @@ -325,7 +325,7 @@ t8_cmesh_new_pyramid (sc_MPI_Comm comm) /* Use linear geometry */ t8_cmesh_register_geometry (cmesh, linear_geom); t8_cmesh_set_tree_class (cmesh, 0, T8_ECLASS_PYRAMID); - t8_cmesh_set_tree_vertices (cmesh, 0, vertices, 15); + t8_cmesh_set_tree_vertices (cmesh, 0, vertices, 5); t8_cmesh_commit (cmesh, comm); return cmesh; } From 4c26c2ffb6b162bf7b1757bf4189113f6c75a2a6 Mon Sep 17 00:00:00 2001 From: "Dreyer, Lukas" Date: Thu, 26 Oct 2023 10:26:31 +0200 Subject: [PATCH 17/43] Fix compiler warnings --- example/cmesh/t8_cmesh_hybrid_new.cxx | 29 +++----------------- src/t8_forest/t8_forest.c | 38 --------------------------- 2 files changed, 4 insertions(+), 63 deletions(-) diff --git a/example/cmesh/t8_cmesh_hybrid_new.cxx b/example/cmesh/t8_cmesh_hybrid_new.cxx index ff8d72adb9..fb22a254e3 100644 --- a/example/cmesh/t8_cmesh_hybrid_new.cxx +++ b/example/cmesh/t8_cmesh_hybrid_new.cxx @@ -97,23 +97,6 @@ t8_basic_cake_refine (t8_forest_t forest, t8_forest_t forest_from, t8_locidx_t w } } -static int -t8_basic_only_pyramid (t8_forest_t forest, t8_forest_t forest_from, t8_locidx_t which_tree, t8_locidx_t lelement_id, - t8_eclass_scheme_c *ts, int is_family, int num_elements, t8_element_t *elements[]) -{ - int level; - level = ts->t8_element_level (elements[0]); - if (level >= *(int *) t8_forest_get_user_data (forest)) { - return 0; - } - else if (ts->t8_element_shape (elements[0]) == T8_ECLASS_PYRAMID) { - return 1; - } - else { - return 0; - } -} - static void t8_basic_hybrid (int level, int endlvl, int do_vtk, t8_eclass_t eclass, int num_elements, int mesh, int balance, const char *prefix, int part) @@ -257,18 +240,14 @@ main (int argc, char **argv) int level, endlvl, helpme, do_vtk, eclass_int, mesh, elements, balance, part; t8_eclass_t eclass; sc_options_t *opt; - char usage[BUFSIZ]; char help[BUFSIZ]; const char *file; - /* brief help message */ - snprintf (usage, BUFSIZ, - "Usage:\t%s \n\t%s -h\t" - "for a brief overview of all options.", - basename (argv[0]), basename (argv[0])); - /* long help message */ - snprintf (help, BUFSIZ, "ADD EXAMPLE DESCRIPTION.\n\n%s\n", usage); + snprintf (help, BUFSIZ, "ADD EXAMPLE DESCRIPTION.\n\n" + "Usage:\t%s \n\t%s -h\t" + "for a brief overview of all options.\n", + basename (argv[0]), basename (argv[0])); mpiret = sc_MPI_Init (&argc, &argv); SC_CHECK_MPI (mpiret); diff --git a/src/t8_forest/t8_forest.c b/src/t8_forest/t8_forest.c index b05da5de10..c7edf9bf6e 100644 --- a/src/t8_forest/t8_forest.c +++ b/src/t8_forest/t8_forest.c @@ -361,44 +361,6 @@ t8_forest_refine_everything (t8_forest_t forest, t8_forest_t forest_from, t8_loc return 1; } -/**Algorithm to populate a forest, if any tree refines irregularly. - * Create the elements on this process given a uniform partition - * of the coarse mesh. We can not use the function t8_forest_populate, because - * it assumes a regular refinement for all trees. - * \param[in] forest The forest to populate -*/ -static void -t8_forest_populate_irregular (t8_forest_t forest) -{ - t8_forest_t forest_zero; - t8_forest_t forest_tmp; - t8_forest_t forest_tmp_partition; - t8_cmesh_ref (forest->cmesh); - t8_scheme_cxx_ref (forest->scheme_cxx); - /* We start with a level 0 uniform refinement */ - t8_forest_init (&forest_zero); - t8_forest_set_level (forest_zero, 0); - t8_forest_set_cmesh (forest_zero, forest->cmesh, forest->mpicomm); - t8_forest_set_scheme (forest_zero, forest->scheme_cxx); - t8_forest_commit (forest_zero); - - /* Up to the specified level we refine every element. */ - for (int i = 1; i <= forest->set_level; i++) { - t8_forest_init (&forest_tmp); - t8_forest_set_level (forest_tmp, i); - t8_forest_set_adapt (forest_tmp, forest_zero, t8_forest_refine_everything, 0); - t8_forest_commit (forest_tmp); - /* Partition the forest to even the load */ - t8_forest_init (&forest_tmp_partition); - t8_forest_set_partition (forest_tmp_partition, forest_tmp, 0); - t8_forest_commit (forest_tmp_partition); - forest_zero = forest_tmp_partition; - } - /* Copy all elements over to the original forest. */ - t8_forest_copy_trees (forest, forest_zero, 1); - t8_forest_unref (&forest_tmp_partition); -} - void t8_forest_commit (t8_forest_t forest) { From 255fea78573404319d34d38b4a5d3b67f602aeb8 Mon Sep 17 00:00:00 2001 From: "Dreyer, Lukas" Date: Thu, 26 Oct 2023 10:32:47 +0200 Subject: [PATCH 18/43] Fix more compiler errors --- src/t8_cmesh/t8_cmesh_cxx.cxx | 2 ++ src/t8_forest/t8_forest.c | 8 -------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/src/t8_cmesh/t8_cmesh_cxx.cxx b/src/t8_cmesh/t8_cmesh_cxx.cxx index b6e3505e94..364eb519a4 100644 --- a/src/t8_cmesh/t8_cmesh_cxx.cxx +++ b/src/t8_cmesh/t8_cmesh_cxx.cxx @@ -821,8 +821,10 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t *sc if (child_in_tree_end != NULL) { *child_in_tree_end = - 1; } +#ifdef T8_ENABLE_DEBUG num_received_end_messages++; num_received_start_messages++; +#endif } /* Post the receives. */ diff --git a/src/t8_forest/t8_forest.c b/src/t8_forest/t8_forest.c index c7edf9bf6e..48ae3f4411 100644 --- a/src/t8_forest/t8_forest.c +++ b/src/t8_forest/t8_forest.c @@ -352,14 +352,6 @@ t8_forest_comm_global_num_elements (t8_forest_t forest) * pointer to one element. * \return Always return 1, to refine every element */ -static int -t8_forest_refine_everything (t8_forest_t forest, t8_forest_t forest_from, t8_locidx_t which_tree, - t8_locidx_t lelement_id, t8_eclass_scheme_c *ts, const int is_family, - const int num_elements, t8_element_t *elements[]) -{ - - return 1; -} void t8_forest_commit (t8_forest_t forest) From a8314dcd25fbd925b5bfe5c7c943b17cf21d319d Mon Sep 17 00:00:00 2001 From: David Knapp Date: Thu, 26 Oct 2023 12:30:10 +0200 Subject: [PATCH 19/43] Clean-up Removed debugging output Enhanced scope of variables More const params --- src/t8.c | 2 +- src/t8.h | 2 +- src/t8_cmesh/t8_cmesh_cxx.cxx | 123 ++++++++------------- src/t8_cmesh/t8_cmesh_offset.c | 5 - src/t8_forest/t8_forest_cxx.cxx | 3 - test/t8_cmesh/t8_gtest_cmesh_partition.cxx | 1 - 6 files changed, 49 insertions(+), 87 deletions(-) diff --git a/src/t8.c b/src/t8.c index 415acdb059..e2343542c6 100644 --- a/src/t8.c +++ b/src/t8.c @@ -167,7 +167,7 @@ t8_sc_array_index_locidx (sc_array_t *array, t8_locidx_t it) } void * -t8_sc_array_index_gloidx (sc_array_t *array, t8_gloidx_t it) +t8_sc_array_index_gloidx (const sc_array_t *array, const t8_gloidx_t it) { T8_ASSERT (it >= 0 && (size_t) it < array->elem_count); return array->array + array->elem_size * (size_t) it; diff --git a/src/t8.h b/src/t8.h index abdf56040e..2aee6e1f1a 100644 --- a/src/t8.h +++ b/src/t8.h @@ -275,7 +275,7 @@ t8_sc_array_index_locidx (sc_array_t *array, t8_locidx_t it); * \return A void * pointing to entry \a it in \a array. */ void * -t8_sc_array_index_gloidx (sc_array_t *array, t8_gloidx_t it); +t8_sc_array_index_gloidx (const sc_array_t *array, const t8_gloidx_t it); /* call this at the end of a header file to match T8_EXTERN_C_BEGIN (). */ T8_EXTERN_C_END (); diff --git a/src/t8_cmesh/t8_cmesh_cxx.cxx b/src/t8_cmesh/t8_cmesh_cxx.cxx index 364eb519a4..1c96f43a0a 100644 --- a/src/t8_cmesh/t8_cmesh_cxx.cxx +++ b/src/t8_cmesh/t8_cmesh_cxx.cxx @@ -50,7 +50,6 @@ t8_cmesh_uniform_set_return_parameters_to_empty (t8_gloidx_t *first_local_tree, t8_gloidx_t *last_local_tree, t8_gloidx_t *child_in_tree_end, int8_t *first_tree_shared) { - t8_debugf ("[D] proc is empty\n"); *first_local_tree = *last_local_tree = -1; if (child_in_tree_begin != NULL) { *child_in_tree_begin = -1; @@ -280,27 +279,14 @@ t8_cmesh_determine_partition (sc_array_t *first_element_tree, size_t pure_local_ return first_proc_adjusted; } -/* TODO: Empty processes, shared trees, binary search in offset-array to avoid recv_any, +/* TODO: Shared trees, binary search in offset-array to avoid recv_any, * use partition_given to partition the cmesh*/ void t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t *scheme, t8_gloidx_t *first_local_tree, t8_gloidx_t *child_in_tree_begin, t8_gloidx_t *last_local_tree, t8_gloidx_t *child_in_tree_end, int8_t *first_tree_shared, sc_MPI_Comm comm) { - t8_gloidx_t local_num_children = 0; - t8_gloidx_t *elem_index_pointer; - int send_first, send_last, send_first_nonempty, num_procs_we_send_to = 0; - t8_cmesh_partition_query_t data; - t8_shmem_array_t offset_array; - sc_array_t offset_partition, first_element_tree; - int mpiret, num_messages_sent = 0; - int expect_start_message = 1; - int expect_end_message = 1; - sc_array send_requests, send_buffer; - int current_pos_in_send_buffer = 0; - t8_eclass_scheme_c *tree_scheme; T8_ASSERT (cmesh != NULL); - const int first_tree_shared_shift = cmesh->first_tree_shared ? 1 : 0; #ifdef T8_ENABLE_DEBUG int num_received_start_messages = 0; int num_received_end_messages = 0; @@ -312,7 +298,6 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t *sc t8_debugf ("Into t8_cmesh_uniform_bounds_hybrid.\n"); if (t8_cmesh_is_empty (cmesh)) { - t8_debugf ("[D] cmesh_uniform_bounds_hybrid is empty\n"); t8_cmesh_uniform_set_return_parameters_to_empty (first_local_tree, child_in_tree_begin, last_local_tree, child_in_tree_end, first_tree_shared); return; @@ -322,7 +307,7 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t *sc { /* Check that comm matches mpirank and size stored in cmesh */ int mpirank, mpisize; - mpiret = sc_MPI_Comm_rank (comm, &mpirank); + int mpiret = sc_MPI_Comm_rank (comm, &mpirank); SC_CHECK_MPI (mpiret); mpiret = sc_MPI_Comm_size (comm, &mpisize); SC_CHECK_MPI (mpiret); @@ -330,9 +315,10 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t *sc T8_ASSERT (mpisize == cmesh->mpisize); } #endif + t8_gloidx_t local_num_children = 0; /*Compute number of local elements. Ignore shared trees */ for (int ieclass = T8_ECLASS_ZERO; ieclass < T8_ECLASS_COUNT; ieclass++) { - tree_scheme = scheme->eclass_schemes[ieclass]; + const t8_eclass_scheme_c *tree_scheme = scheme->eclass_schemes[ieclass]; local_num_children += cmesh->num_local_trees_per_eclass[ieclass] * tree_scheme->t8_element_count_leafs_from_root (level); } @@ -340,7 +326,7 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t *sc /* Do not consider shared trees */ if (cmesh->first_tree_shared && cmesh->set_partition) { const int ieclass = t8_cmesh_get_tree_class (cmesh, 0); - tree_scheme = scheme->eclass_schemes[ieclass]; + const t8_eclass_scheme_c *tree_scheme = scheme->eclass_schemes[ieclass]; local_num_children -= tree_scheme->t8_element_count_leafs_from_root (level); } @@ -352,7 +338,6 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t *sc /* If the initial cmesh is not partitioned, every process knows "everything" and we do not * need any communication.*/ if (!cmesh->set_partition) { - t8_gloidx_t current_tree_element_offset; const t8_gloidx_t num_trees = t8_cmesh_get_num_trees (cmesh); t8_debugf ("Cmesh is not partitioned.\n"); /* Compute the first and last element of this process. Then loop over @@ -363,16 +348,14 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t *sc = t8_cmesh_get_first_element_of_process (cmesh->mpirank, cmesh->mpisize, local_num_children); const t8_gloidx_t last_child = t8_cmesh_get_first_element_of_process (cmesh->mpirank + 1, cmesh->mpisize, local_num_children) - 1; - t8_debugf ("[H] hybrid fc = %li, lc = %li\n", first_child, last_child); - /* Can't we optimize this linear loop by using a binary search? * -> No, we cannot. Since we need in any case compute the t8_element_count_leafs_from_root * for each tree. */ - current_tree_element_offset = 0; + t8_gloidx_t current_tree_element_offset = 0; for (t8_gloidx_t igtree = 0; igtree < num_trees; ++igtree) { const int ieclass = t8_cmesh_get_tree_class (cmesh, (t8_locidx_t) igtree); - tree_scheme = scheme->eclass_schemes[ieclass]; + const t8_eclass_scheme_c *tree_scheme = scheme->eclass_schemes[ieclass]; /* TODO: We can optimize by buffering the elem_in_tree value. Thus, if the computation is expensive (may be for non-morton-type schemes), we do it only once. */ @@ -393,7 +376,6 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t *sc /* This process is empty. We needed to identify the 'first_local_tree' since it is the * first local tree of the next process, which we store in this case. */ - t8_debugf ("[D] lc < fc\n"); t8_cmesh_uniform_set_return_parameters_to_empty (first_local_tree, child_in_tree_begin, last_local_tree, child_in_tree_end, first_tree_shared); *first_local_tree = 0; @@ -417,7 +399,6 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t *sc } /* If we reach this part, we do not have any trees - the cmesh is empty */ T8_ASSERT (num_trees == 0); - t8_debugf ("[D] cmesh is empty\n"); t8_cmesh_uniform_set_return_parameters_to_empty (first_local_tree, child_in_tree_begin, last_local_tree, child_in_tree_end, first_tree_shared); return; @@ -428,33 +409,36 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t *sc * Cmesh is partitioned * */ - + t8_shmem_array_t offset_array; t8_shmem_array_init (&offset_array, sizeof (t8_gloidx_t), cmesh->mpisize + 1, comm); /* Fill the offset array for each process the global index of its first element in * the uniform partition. * (0, l_n_c_0, l_n_c_0 + l_n_c_1, l_n_c_0 + l_n_c_1 + l_n_c_2, ...) */ t8_shmem_prefix (&local_num_children, offset_array, 1, T8_MPI_GLOIDX, sc_MPI_SUM); + t8_cmesh_partition_query_t data; data.num_procs = cmesh->mpisize; /* Get global number of elements */ data.global_num_elements = t8_shmem_array_get_gloidx (offset_array, cmesh->mpisize); SC_CHECK_ABORTF (0 <= data.global_num_elements && data.global_num_elements < T8_GLOIDX_MAX, "Overflow in number of elements.\n"); - t8_debugf ("[H] global num %li\n", data.global_num_elements); /*Compute number of non-shared-trees and the local index of the first non-shared-tree */ + const int first_tree_shared_shift = cmesh->first_tree_shared ? 1 : 0; const t8_locidx_t pure_local_trees = cmesh->num_local_trees - first_tree_shared_shift; - + sc_array send_requests; + int expect_start_message = 1; + int expect_end_message = 1; if (pure_local_trees > 0) { /* Compute which trees and elements to send to which process. * We skip empty processes. */ t8_locidx_t igtree = first_tree_shared_shift; - + sc_array_t first_element_tree; sc_array_init_size (&first_element_tree, sizeof (t8_gloidx_t), pure_local_trees + 1); /* Set the first entry of first_element_tree to the global index of * the first element of our first pure local tree. */ - elem_index_pointer = (t8_gloidx_t *) sc_array_index_int (&first_element_tree, 0); + t8_gloidx_t *elem_index_pointer = (t8_gloidx_t *) sc_array_index_int (&first_element_tree, 0); *elem_index_pointer = t8_shmem_array_get_gloidx (offset_array, cmesh->mpirank); /* Compute the first element in every pure local tree. @@ -464,7 +448,7 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t *sc const t8_gloidx_t *first_element_of_tree = (const t8_gloidx_t *) sc_array_index_int (&first_element_tree, itree); t8_gloidx_t *first_element_of_next_tree = (t8_gloidx_t *) sc_array_index_int (&first_element_tree, itree + 1); const int ieclass = t8_cmesh_get_tree_class (cmesh, igtree); - tree_scheme = scheme->eclass_schemes[ieclass]; + const t8_eclass_scheme_c *tree_scheme = scheme->eclass_schemes[ieclass]; /* Set the first element of the next tree by adding the number of element of the current tree. */ *first_element_of_next_tree = *first_element_of_tree + tree_scheme->t8_element_count_leafs_from_root (level); } @@ -479,11 +463,13 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t *sc * the first times out of sc_array_split. */ data.process_offset = 0; /* Compute the process that will own the first element of our first tree. */ - send_first_nonempty = (t8_gloidx_t) t8_cmesh_determine_partition (&first_element_tree, 0, &data); + + const t8_gloidx_t send_first_nonempty = (t8_gloidx_t) t8_cmesh_determine_partition (&first_element_tree, 0, &data); + t8_gloidx_t send_first; if (!cmesh->first_tree_shared && t8_cmesh_get_global_id (cmesh, 0) == 0) { /* If our first tree is 0 and not shared, then we need to send to all processes * below the start process. Even if their partition is empty. */ - t8_debugf("LLL: Would have set send_first to 0, instead: %i", send_first_nonempty); + t8_debugf ("LLL: Would have set send_first to 0, instead: %li", send_first_nonempty); send_first = send_first_nonempty; } else { @@ -491,15 +477,15 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t *sc } /* Compute the process that may own the last element of our first tree. */ - send_last = (t8_gloidx_t) t8_cmesh_determine_partition (&first_element_tree, pure_local_trees, &data); + t8_gloidx_t send_last = (t8_gloidx_t) t8_cmesh_determine_partition (&first_element_tree, pure_local_trees, &data); if (send_last >= cmesh->mpisize) { /* t8_cmesh_determine_partition will return mpisize if we plug in the number of global * trees as tree index, which happens on the last process that has data. * We need to correct by subtracting 1. */ send_last = cmesh->mpisize - 1; } - num_procs_we_send_to = send_last - send_first + 1; - + const t8_gloidx_t num_procs_we_send_to = send_last - send_first + 1; + sc_array_t offset_partition; sc_array_init_size (&offset_partition, sizeof (size_t), num_procs_we_send_to); /* In array split the 'types' that we compute are the processes we send to, * but offset by the first process, so that they start at 0 and end at num_procs_we_send_to. */ @@ -555,16 +541,16 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t *sc * Since we use MPI_Isend, we need to keep the buffers alive until * we post the MPI_Wait. Since we first send all messages, we need to * buffer them all. */ + sc_array send_buffer; sc_array_init (&send_buffer, sizeof (t8_gloidx_t)); - current_pos_in_send_buffer = 0; + int current_pos_in_send_buffer = 0; /* Iterate over offset_partition to find boundaries * and send the MPI messages. */ - t8_debugf ("[H] sf: %i sl: %i\n", send_first, send_last); const t8_gloidx_t last_el_index_of_last_tree = *(t8_gloidx_t *) t8_sc_array_index_gloidx (&first_element_tree, pure_local_trees) - 1; - for (int iproc = send_first; iproc <= send_last; iproc++) { + for (t8_gloidx_t iproc = send_first; iproc <= send_last; iproc++) { const t8_gloidx_t first_element_index_of_current_proc = t8_cmesh_get_first_element_of_process (iproc, data.num_procs, data.global_num_elements); @@ -646,8 +632,6 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t *sc } } } - t8_debugf ("[H] Set tree for %i: %i to %i, (#pure = %i)\n", iproc, first_puretree_of_current_proc, - last_puretree_of_current_proc, pure_local_trees); #ifdef T8_ENABLE_DEBUG /* Check that the trees have valid values. */ if (send_start_message) { @@ -678,10 +662,6 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t *sc const t8_gloidx_t first_el_of_last_tree = *(t8_gloidx_t *) t8_sc_array_index_locidx (&first_element_tree, last_puretree_of_current_proc); last_element_in_tree_index_of_current_proc = last_element_index_of_current_proc - first_el_of_last_tree; - t8_debugf ("[H] Computing last element as %li - %li = %li\n\n", last_element_index_of_current_proc, - first_el_of_last_tree, last_element_in_tree_index_of_current_proc); - t8_debugf ("[H] Last pure tree = %i, first element in it %li\n", last_puretree_of_current_proc, - first_el_of_last_tree); } } @@ -714,14 +694,11 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t *sc message[0] = global_id_of_first_tree; /* The index in the tree is the index of the element minus the offset of the tree. */ message[1] = first_element_in_tree_index_of_current_proc; - t8_debugf ("[H] first e index %li, first in tree %li\n", first_element_index_of_current_proc, - first_puretree_of_current_proc >= 0 - ? *(t8_gloidx_t *) t8_sc_array_index_gloidx (&first_element_tree, first_puretree_of_current_proc) - : -1); - mpiret = sc_MPI_Isend (message, num_entries, T8_MPI_GLOIDX, iproc, T8_MPI_CMESH_UNIFORM_BOUNDS_START, comm, - request); + + const int mpiret = sc_MPI_Isend (message, num_entries, T8_MPI_GLOIDX, iproc, + T8_MPI_CMESH_UNIFORM_BOUNDS_START, comm, request); SC_CHECK_MPI (mpiret); - t8_debugf ("Sending start message (%li, %li) to %i (global num el %li)\n", message[0], message[1], iproc, + t8_debugf ("Sending start message (%li, %li) to %li (global num el %li)\n", message[0], message[1], iproc, data.global_num_elements); T8_ASSERT (proc_is_empty || (0 <= message[0] && message[0] < t8_cmesh_get_num_trees (cmesh))); @@ -750,9 +727,6 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t *sc #ifdef T8_ENABLE_DEBUG num_received_start_messages++; #endif - - t8_debugf ("[H] Copied first tree %li element %li to self\n", global_id_of_first_tree, - first_element_in_tree_index_of_current_proc); } } /* End sending of start message */ @@ -778,10 +752,10 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t *sc message[0] = global_id_of_last_tree; /* The index in the tree is the index of the element minus the offset of the tree. */ message[1] = last_element_in_tree_index_of_current_proc; - mpiret + const int mpiret = sc_MPI_Isend (message, num_entries, T8_MPI_GLOIDX, iproc, T8_MPI_CMESH_UNIFORM_BOUNDS_END, comm, request); SC_CHECK_MPI (mpiret); - t8_debugf ("Sending end message (%li, %li) to %i\n", message[0], message[1], iproc); + t8_debugf ("Sending end message (%li, %li) to %li\n", message[0], message[1], iproc); T8_ASSERT (proc_is_empty || (0 <= message[0] && message[0] < t8_cmesh_get_num_trees (cmesh))); T8_ASSERT (proc_is_empty || (0 <= message[1] && message[1] < data.global_num_elements)); T8_ASSERT (!(proc_is_empty && message[0] != -1)); @@ -794,22 +768,23 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t *sc } /* We do not expect this message from another proc */ expect_end_message = 0; - t8_debugf ("[H] Copied last tree %li element %li to self\n", global_id_of_last_tree, - last_element_in_tree_index_of_current_proc + 1); #ifdef T8_ENABLE_DEBUG num_received_end_messages++; #endif } } /* End sending of end message */ } /* End loop over processes */ - } /* if (pure_local_trees > 0) */ + sc_array_reset (&first_element_tree); + sc_array_reset (&offset_partition); + sc_array_reset (&send_buffer); + } /* if (pure_local_trees > 0) */ const t8_gloidx_t first_element_index_of_current_proc - = t8_cmesh_get_first_element_of_process (cmesh->mpirank, data.num_procs, data.global_num_elements); + = t8_cmesh_get_first_element_of_process (cmesh->mpirank, data.num_procs, data.global_num_elements); const t8_gloidx_t last_element_index_of_current_proc - = t8_cmesh_get_first_element_of_process (cmesh->mpirank + 1, data.num_procs, data.global_num_elements) - 1; + = t8_cmesh_get_first_element_of_process (cmesh->mpirank + 1, data.num_procs, data.global_num_elements) - 1; - if(first_element_index_of_current_proc > last_element_index_of_current_proc){ + if (first_element_index_of_current_proc > last_element_index_of_current_proc) { expect_start_message = 0; expect_end_message = 0; *first_local_tree = 0; @@ -819,7 +794,7 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t *sc *child_in_tree_begin = -1; } if (child_in_tree_end != NULL) { - *child_in_tree_end = - 1; + *child_in_tree_end = -1; } #ifdef T8_ENABLE_DEBUG num_received_end_messages++; @@ -829,11 +804,11 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t *sc /* Post the receives. */ if (expect_start_message) { - t8_debugf("expect start message\n"); + t8_debugf ("expect start message\n"); const int num_entries = 2; t8_gloidx_t *message = T8_ALLOC (t8_gloidx_t, num_entries); - mpiret = sc_MPI_Recv (message, num_entries, T8_MPI_GLOIDX, sc_MPI_ANY_SOURCE, T8_MPI_CMESH_UNIFORM_BOUNDS_START, - comm, sc_MPI_STATUS_IGNORE); + const int mpiret = sc_MPI_Recv (message, num_entries, T8_MPI_GLOIDX, sc_MPI_ANY_SOURCE, + T8_MPI_CMESH_UNIFORM_BOUNDS_START, comm, sc_MPI_STATUS_IGNORE); SC_CHECK_MPI (mpiret); #ifdef T8_ENABLE_DEBUG @@ -856,7 +831,6 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t *sc } if (child_in_tree_begin != NULL) { *child_in_tree_begin = message[1]; - t8_debugf ("[H] Received cit begin = %li\n", *child_in_tree_begin); T8_ASSERT (*child_in_tree_begin == -1 || (0 <= *child_in_tree_begin && *child_in_tree_begin < data.global_num_elements)); } @@ -865,8 +839,8 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t *sc if (expect_end_message) { const int num_entries = 2; t8_gloidx_t *message = T8_ALLOC (t8_gloidx_t, num_entries); - mpiret = sc_MPI_Recv (message, num_entries, T8_MPI_GLOIDX, sc_MPI_ANY_SOURCE, T8_MPI_CMESH_UNIFORM_BOUNDS_END, comm, - sc_MPI_STATUS_IGNORE); + const int mpiret = sc_MPI_Recv (message, num_entries, T8_MPI_GLOIDX, sc_MPI_ANY_SOURCE, + T8_MPI_CMESH_UNIFORM_BOUNDS_END, comm, sc_MPI_STATUS_IGNORE); SC_CHECK_MPI (mpiret); #ifdef T8_ENABLE_DEBUG num_received_end_messages++; @@ -894,16 +868,13 @@ t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t *sc child_in_tree_end, first_tree_shared); } } - - mpiret = sc_MPI_Waitall (num_messages_sent, (sc_MPI_Request *) send_requests.array, sc_MPI_STATUSES_IGNORE); + t8_gloidx_t num_messages_sent = 0; + const int mpiret = sc_MPI_Waitall (num_messages_sent, (sc_MPI_Request *) send_requests.array, sc_MPI_STATUSES_IGNORE); SC_CHECK_MPI (mpiret); T8_ASSERT (num_received_start_messages == 1); T8_ASSERT (num_received_end_messages == 1); if (pure_local_trees > 0) { - sc_array_reset (&first_element_tree); - sc_array_reset (&offset_partition); - sc_array_reset (&send_buffer); sc_array_reset (&send_requests); } diff --git a/src/t8_cmesh/t8_cmesh_offset.c b/src/t8_cmesh/t8_cmesh_offset.c index 230fc1ea3e..a2db11a4ec 100644 --- a/src/t8_cmesh/t8_cmesh_offset.c +++ b/src/t8_cmesh/t8_cmesh_offset.c @@ -130,13 +130,11 @@ t8_offset_consistent (int mpisize, const t8_shmem_array_t offset_shmem, t8_gloid const t8_gloidx_t *offset = t8_shmem_array_get_gloidx_array (offset_shmem); ret = offset[0] == 0; - t8_debugf ("[D] init ret: %i\n", ret); last_tree = t8_offset_last (0, offset); /* stores the last tree of process i-1 */ for (i = 1; i < mpisize && ret; i++) { if (t8_offset_empty (i, offset)) { /* If the process is empty, then its first tree must not be shared */ ret &= offset[i] >= 0; - t8_debugf ("[LD] offset empty, proc: %i ret = %i\n", i, ret); } else { /* If the process is not empty its first local tree must be bigger or @@ -144,15 +142,12 @@ t8_offset_consistent (int mpisize, const t8_shmem_array_t offset_shmem, t8_gloid * Equality must only hold, when the first tree is shared, thus offset[i] < 0 */ if (offset[i] < 0) { ret &= t8_offset_first (i, offset) == last_tree; - t8_debugf ("[LD] shared tree, offset first == last_tree proc: %iret = %i\n", i, ret); } else { ret &= t8_offset_first (i, offset) > last_tree; - t8_debugf ("[LD] shared tree, offset_first > last_tree proc: %i ret = %i\n", i, ret); } last_tree = t8_offset_last (i, offset); ret &= (last_tree <= num_trees); - t8_debugf ("[LD] last_tree <= num_trees proc: %i ret = %i\n", i, ret); } } ret &= (offset[mpisize] == num_trees); diff --git a/src/t8_forest/t8_forest_cxx.cxx b/src/t8_forest/t8_forest_cxx.cxx index a00c5c7468..2f25aa7207 100644 --- a/src/t8_forest/t8_forest_cxx.cxx +++ b/src/t8_forest/t8_forest_cxx.cxx @@ -1457,9 +1457,6 @@ t8_forest_populate (t8_forest_t forest) t8_cmesh_uniform_bounds_hybrid (forest->cmesh, forest->set_level, forest->scheme_cxx, &forest->first_local_tree, &child_in_tree_begin, &forest->last_local_tree, &child_in_tree_end, NULL, forest->mpicomm); - t8_debugf ("[D] flt: %li, citb: %li, llt: %li, cite: %li\n", forest->first_local_tree, child_in_tree_begin, - forest->last_local_tree, child_in_tree_end); - /* True if the forest has no elements */ is_empty = forest->first_local_tree > forest->last_local_tree || (forest->first_local_tree == forest->last_local_tree && child_in_tree_begin >= child_in_tree_end); diff --git a/test/t8_cmesh/t8_gtest_cmesh_partition.cxx b/test/t8_cmesh/t8_gtest_cmesh_partition.cxx index 52302e9257..b7a2d19dad 100644 --- a/test/t8_cmesh/t8_gtest_cmesh_partition.cxx +++ b/test/t8_cmesh/t8_gtest_cmesh_partition.cxx @@ -40,7 +40,6 @@ class t8_cmesh_partition_class: public testing::TestWithParam { { cmesh_id = GetParam (); cmesh_original = t8_test_create_cmesh (cmesh_id); - t8_debugf ("[D] is partitioned: %i\n", cmesh_original->set_partition); } int cmesh_id; From 1a2fbef32a0cdf52864a2cadf334dfc2dafab7ef Mon Sep 17 00:00:00 2001 From: David Knapp Date: Mon, 13 Nov 2023 09:55:59 +0100 Subject: [PATCH 20/43] Added comment to refinement criterion indentation --- example/cmesh/t8_cmesh_hybrid_new.cxx | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/example/cmesh/t8_cmesh_hybrid_new.cxx b/example/cmesh/t8_cmesh_hybrid_new.cxx index fb22a254e3..717e68447a 100644 --- a/example/cmesh/t8_cmesh_hybrid_new.cxx +++ b/example/cmesh/t8_cmesh_hybrid_new.cxx @@ -34,6 +34,21 @@ #include #include +/** + * Adaptation criterion for a forest containing hexahedra, tetrahedra, prisms and/or pryramids. + * Refines every second element, except for pyramids. + * Refines every pyramid + * + * \param [in] forest the forest + * \param [in] forest_from + * \param [in] which_tree The local id of the tree + * \param [in] lelement_id the local id of the element + * \param [in] ts the scheme to use + * \param [in] is_family flag, if the \a elements form a family + * \param [in] num_elements number of elements + * \param [in] elements A single element or a collection of \a num_elements + * \return int + */ static int t8_basic_hybrid_refine (t8_forest_t forest, t8_forest_t forest_from, t8_locidx_t which_tree, t8_locidx_t lelement_id, t8_eclass_scheme_c *ts, int is_family, int num_elements, t8_element_t *elements[]) @@ -244,10 +259,11 @@ main (int argc, char **argv) const char *file; /* long help message */ - snprintf (help, BUFSIZ, "ADD EXAMPLE DESCRIPTION.\n\n" - "Usage:\t%s \n\t%s -h\t" - "for a brief overview of all options.\n", - basename (argv[0]), basename (argv[0])); + snprintf (help, BUFSIZ, + "ADD EXAMPLE DESCRIPTION.\n\n" + "Usage:\t%s \n\t%s -h\t" + "for a brief overview of all options.\n", + basename (argv[0]), basename (argv[0])); mpiret = sc_MPI_Init (&argc, &argv); SC_CHECK_MPI (mpiret); From 69fdaf075dcdd8f97e47f6f7a3992961966ba309 Mon Sep 17 00:00:00 2001 From: David Knapp Date: Mon, 13 Nov 2023 09:59:19 +0100 Subject: [PATCH 21/43] Explain formulas in adapt-criterion --- example/cmesh/t8_cmesh_hybrid_new.cxx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/example/cmesh/t8_cmesh_hybrid_new.cxx b/example/cmesh/t8_cmesh_hybrid_new.cxx index 717e68447a..8908594d67 100644 --- a/example/cmesh/t8_cmesh_hybrid_new.cxx +++ b/example/cmesh/t8_cmesh_hybrid_new.cxx @@ -54,11 +54,13 @@ t8_basic_hybrid_refine (t8_forest_t forest, t8_forest_t forest_from, t8_locidx_t t8_eclass_scheme_c *ts, int is_family, int num_elements, t8_element_t *elements[]) { int level, id; + /*If the level is equal or higher than given by the user do not refine*/ level = ts->t8_element_level (elements[0]); if (level >= *(int *) t8_forest_get_user_data (forest)) { return 0; } else { + /*Refine every second element */ switch (ts->t8_element_shape (elements[0])) { case T8_ECLASS_HEX: id = ts->t8_element_child_id (elements[0]); @@ -70,6 +72,7 @@ t8_basic_hybrid_refine (t8_forest_t forest, t8_forest_t forest_from, t8_locidx_t id = ts->t8_element_child_id (elements[0]); return id % 2 == 0 ? 1 : 0; case T8_ECLASS_PYRAMID: + /* Refine every element. */ return 1; default: return 1; From 916cfdfe21f446435ce3885e905e25de46d57141 Mon Sep 17 00:00:00 2001 From: David Knapp Date: Mon, 13 Nov 2023 10:30:21 +0100 Subject: [PATCH 22/43] Comment adaptation criterion --- example/cmesh/t8_cmesh_hybrid_new.cxx | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/example/cmesh/t8_cmesh_hybrid_new.cxx b/example/cmesh/t8_cmesh_hybrid_new.cxx index 8908594d67..66bfb4a18f 100644 --- a/example/cmesh/t8_cmesh_hybrid_new.cxx +++ b/example/cmesh/t8_cmesh_hybrid_new.cxx @@ -80,17 +80,35 @@ t8_basic_hybrid_refine (t8_forest_t forest, t8_forest_t forest_from, t8_locidx_t } } +/** + * An adaptation criterion to refine a forest. Specialized for a "cake" of pyramids (t8_cmesh_new_pyramid_cake) + * For non-pyramid-elements it refines every element. For a pyramid all type 6 pyramids and all type (0|2|4)-tets + * are refined. + * + * \param [in] forest the forest + * \param [in] forest_from + * \param [in] which_tree The local id of the tree + * \param [in] lelement_id the local id of the element + * \param [in] ts the scheme to use + * \param [in] is_family flag, if the \a elements form a family + * \param [in] num_elements number of elements + * \param [in] elements A single element or a collection of \a num_elements + * \return int + */ static int t8_basic_cake_refine (t8_forest_t forest, t8_forest_t forest_from, t8_locidx_t which_tree, t8_locidx_t lelement_id, t8_eclass_scheme_c *ts, int is_family, int num_elements, t8_element_t *elements[]) { int level, type; + /*If the level is equal or higher than given by the user do not refine*/ level = ts->t8_element_level (elements[0]); if (level >= *(int *) t8_forest_get_user_data (forest)) { return 0; } else { int32_t h = T8_DPYRAMID_LEN (level); + /* If the shape of element is a tet or a pyramid refine type 6 pyras and type (0|2|4) tets + * Refine every other shape. */ switch (ts->t8_element_shape (elements[0])) { case T8_ECLASS_TET: type = ((t8_dtet_t *) elements[0])->type; From 6cda8b8b2a1f922596b617f8d1cef3d78ef94e43 Mon Sep 17 00:00:00 2001 From: David Knapp Date: Mon, 13 Nov 2023 10:39:02 +0100 Subject: [PATCH 23/43] Add description --- example/cmesh/t8_cmesh_hybrid_new.cxx | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/example/cmesh/t8_cmesh_hybrid_new.cxx b/example/cmesh/t8_cmesh_hybrid_new.cxx index 66bfb4a18f..6cf8d72dec 100644 --- a/example/cmesh/t8_cmesh_hybrid_new.cxx +++ b/example/cmesh/t8_cmesh_hybrid_new.cxx @@ -133,6 +133,19 @@ t8_basic_cake_refine (t8_forest_t forest, t8_forest_t forest_from, t8_locidx_t w } } +/** + * Creates a cmesh and forest using this cmesh and times all high-level algorithms (new, adapt, ghost, partition, ...) + * + * \param[in] level The initial level of forest + * \param[in] endlvl The maximal level of the forest + * \param[in] do_vtk Flag, if vtk-output should be produced + * \param[in] eclass if \a mesh is set to use bigmesh or new_from_class the eclass to use in this cmesh + * \param[in] num_elements if \a mesh is set to use bigmesh or new_long_brick_pyramid the number of elements to use in the this cmesh + * \param[in] mesh The mesh to use. + * \param[in] balance A flag if the forest should be balanced + * \param[in] prefix if \a do_vtk is set, the prefix of the vtk-output + * \param[in] part A flag if the forest should be partitioned + */ static void t8_basic_hybrid (int level, int endlvl, int do_vtk, t8_eclass_t eclass, int num_elements, int mesh, int balance, const char *prefix, int part) From f71b9d111fff71a379bbaeb2aead80a01637ef7b Mon Sep 17 00:00:00 2001 From: David Knapp Date: Mon, 13 Nov 2023 10:40:04 +0100 Subject: [PATCH 24/43] Update description --- example/cmesh/t8_cmesh_hybrid_new.cxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/cmesh/t8_cmesh_hybrid_new.cxx b/example/cmesh/t8_cmesh_hybrid_new.cxx index 6cf8d72dec..bb37826d0c 100644 --- a/example/cmesh/t8_cmesh_hybrid_new.cxx +++ b/example/cmesh/t8_cmesh_hybrid_new.cxx @@ -144,7 +144,7 @@ t8_basic_cake_refine (t8_forest_t forest, t8_forest_t forest_from, t8_locidx_t w * \param[in] mesh The mesh to use. * \param[in] balance A flag if the forest should be balanced * \param[in] prefix if \a do_vtk is set, the prefix of the vtk-output - * \param[in] part A flag if the forest should be partitioned + * \param[in] part A flag if the cmesh should be partitioned */ static void t8_basic_hybrid (int level, int endlvl, int do_vtk, t8_eclass_t eclass, int num_elements, int mesh, int balance, From 24020310b36c7ef58168cdae63892844eb0c6bfd Mon Sep 17 00:00:00 2001 From: David Knapp Date: Mon, 13 Nov 2023 11:01:48 +0100 Subject: [PATCH 25/43] Comment and clean up --- example/cmesh/t8_cmesh_hybrid_new.cxx | 67 ++++++++++++++++----------- 1 file changed, 40 insertions(+), 27 deletions(-) diff --git a/example/cmesh/t8_cmesh_hybrid_new.cxx b/example/cmesh/t8_cmesh_hybrid_new.cxx index bb37826d0c..88cbb6d78e 100644 --- a/example/cmesh/t8_cmesh_hybrid_new.cxx +++ b/example/cmesh/t8_cmesh_hybrid_new.cxx @@ -150,21 +150,18 @@ static void t8_basic_hybrid (int level, int endlvl, int do_vtk, t8_eclass_t eclass, int num_elements, int mesh, int balance, const char *prefix, int part) { - t8_forest_t forest, forest_adapt, forest_partition; - t8_cmesh_t cmesh, cmesh_partition; - char vtuname[BUFSIZ], cmesh_file[BUFSIZ]; - int mpirank, mpiret; - double new_time = 0, adapt_time = 0, ghost_time = 0, partition_time = 0, total_time = 0, balance_time = 0; + /* Create and initialize timing stats */ sc_statinfo_t times[6]; - int procs_sent, balance_rounds; - t8_locidx_t ghost_sent; sc_stats_init (×[0], "new"); sc_stats_init (×[1], "adapt"); sc_stats_init (×[2], "ghost"); sc_stats_init (×[3], "partition"); sc_stats_init (×[4], "balance"); sc_stats_init (×[5], "total"); - total_time -= sc_MPI_Wtime (); + double total_time = -sc_MPI_Wtime (); + + t8_cmesh_t cmesh; + /* Create the cmesh */ switch (mesh) { case 0: t8_global_productionf ("Constructing cake mesh with %i pyramids.\n", num_elements); @@ -195,15 +192,19 @@ t8_basic_hybrid (int level, int endlvl, int do_vtk, t8_eclass_t eclass, int num_ return; } - mpiret = sc_MPI_Comm_rank (sc_MPI_COMM_WORLD, &mpirank); - SC_CHECK_MPI (mpiret); if (part) { + /* Partition the cmesh */ + t8_cmesh_t cmesh_partition; t8_cmesh_init (&cmesh_partition); t8_cmesh_set_derive (cmesh_partition, cmesh); t8_cmesh_set_partition_uniform (cmesh_partition, level, t8_scheme_new_default_cxx ()); t8_cmesh_commit (cmesh_partition, sc_MPI_COMM_WORLD); cmesh = cmesh_partition; } + + /* Save the cmesh (except for big cmeshes) and create vtk output if the flag is set. */ + char vtuname[BUFSIZ]; + char cmesh_file[BUFSIZ]; snprintf (cmesh_file, BUFSIZ, "cmesh_hybrid"); snprintf (vtuname, BUFSIZ, "cmesh_hybrid"); if (mesh != 4) { @@ -216,60 +217,72 @@ t8_basic_hybrid (int level, int endlvl, int do_vtk, t8_eclass_t eclass, int num_ } } - t8_debugf ("[D] start forest\n"); + /* Create the forest and measure the time needed for the initial refinement*/ + t8_forest_t forest; t8_forest_init (&forest); t8_forest_set_profiling (forest, 1); t8_forest_set_cmesh (forest, cmesh, sc_MPI_COMM_WORLD); t8_forest_set_scheme (forest, t8_scheme_new_default_cxx ()); t8_forest_set_level (forest, level); - new_time -= sc_MPI_Wtime (); + double new_time = -sc_MPI_Wtime (); t8_forest_commit (forest); new_time += sc_MPI_Wtime (); + if (do_vtk) { + /* write the initially refined forest into a vtu-file*/ snprintf (vtuname, BUFSIZ, "forest_hybrid"); t8_forest_write_vtk (forest, vtuname); - t8_debugf ("[D] output to %s\n", vtuname); } + + /* Initialize the adapted forest*/ + t8_forest_t forest_adapt; t8_forest_init (&forest_adapt); t8_forest_set_user_data (forest_adapt, &endlvl); t8_forest_set_profiling (forest_adapt, 1); + + /* Set the adaptation criterion to use. */ if (eclass == T8_ECLASS_PYRAMID) { - t8_debugf ("Use cake-adapt\n"); t8_forest_set_adapt (forest_adapt, forest, t8_basic_cake_refine, 1); } else { t8_forest_set_adapt (forest_adapt, forest, t8_basic_hybrid_refine, 1); } - //t8_forest_set_ghost_ext(forest_adapt, 1, T8_GHOST_FACES, 2); - adapt_time -= sc_MPI_Wtime (); + /* Measure the time needed to adapt */ + double adapt_time = -sc_MPI_Wtime (); t8_forest_commit (forest_adapt); adapt_time += sc_MPI_Wtime (); - //t8_debugf ("Successfully adapted forest.\n"); - //snprintf (vtuname, BUFSIZ, "forest_hybrid_refine"); - //t8_forest_write_vtk (forest_adapt, vtuname); - //t8_debugf ("Output to %s\n", vtuname); - //t8_forest_unref(&forest_adapt); - /* Ensure that the correct forest is passed to unref later */ + /* Initialize the partitioned forest */ + t8_forest_t forest_partition; t8_forest_init (&forest_partition); t8_forest_set_partition (forest_partition, forest_adapt, 0); + + /* We want to compute the ghosts, too */ t8_forest_set_ghost (forest_partition, 1, T8_GHOST_FACES); if (balance) { + /* If set, the forest should be balanced */ t8_forest_set_balance (forest_partition, forest_adapt, 1); } + /* Use the profiling algorithm to measure the time needed and also how many ghost elements are sent, + * how many procs we sent to and how many rounds we need to balance the forest. */ t8_forest_set_profiling (forest_partition, 1); t8_forest_commit (forest_partition); + + /* Write vtk-output*/ if (do_vtk) { snprintf (vtuname, BUFSIZ, "forest_hybrid_partition"); t8_forest_write_vtk (forest_partition, vtuname); - t8_debugf ("Output to %s\n", vtuname); } - t8_forest_print_profile (forest_partition); - partition_time += t8_forest_profile_get_partition_time (forest_partition, &procs_sent); - ghost_time += t8_forest_profile_get_ghost_time (forest_partition, &ghost_sent); - balance_time += t8_forest_profile_get_balance_time (forest_partition, &balance_rounds); total_time += sc_MPI_Wtime (); + int procs_sent; /* Procs we sent to during forest_partition */ + int balance_rounds; /* Number of balance rounds */ + t8_locidx_t ghost_sent; /* Number of ghost-elements we sent */ + t8_forest_print_profile (forest_partition); + const double partition_time = t8_forest_profile_get_partition_time (forest_partition, &procs_sent); + const double ghost_time = t8_forest_profile_get_ghost_time (forest_partition, &ghost_sent); + const double balance_time = t8_forest_profile_get_balance_time (forest_partition, &balance_rounds); + /* Accumulate the timings over all procs and print the stats */ sc_stats_accumulate (×[0], new_time); sc_stats_accumulate (×[1], adapt_time); sc_stats_accumulate (×[2], ghost_time); From 2ce5b2ce23719d3ef6945752e388fa1f649b3523 Mon Sep 17 00:00:00 2001 From: David Knapp Date: Mon, 13 Nov 2023 11:04:33 +0100 Subject: [PATCH 26/43] Apply suggestions from code review Co-authored-by: Johannes Holke --- src/t8_cmesh.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/t8_cmesh.h b/src/t8_cmesh.h index 140646e4a1..ac0803feed 100644 --- a/src/t8_cmesh.h +++ b/src/t8_cmesh.h @@ -683,9 +683,9 @@ t8_cmesh_uniform_bounds (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t *ts, t8_gl * \param [in] cmesh The cmesh to be considered. * \param [in] level The uniform refinement level to be created. * \param [in] scheme The element scheme for which to compute the bounds. - * \param [out] first_local_tree The first tree that contains elements belonging to the calling processor. + * \param [out] first_local_tree The global index of the first tree that contains elements belonging to the calling processor. * \param [out] child_in_tree_begin The global index of the first element belonging to the calling processor. Not computed if NULL. - * \param [out] last_local_tree The last tree that contains elements belonging to the calling processor. + * \param [out] last_local_tree The global index of the last tree that contains elements belonging to the calling processor. * \param [out] child_in_tree_end The global index of the first element that does not belonging to * the calling processor anymore. Not computed if NULL. * \param [out] first_tree_shared If not NULL, 1 or 0 is stored here depending on whether \a first_local_tree is the From 49fb50cfdf354b40e97fcde5abc71f0e0f364906 Mon Sep 17 00:00:00 2001 From: David Knapp Date: Tue, 14 Nov 2023 10:02:24 +0100 Subject: [PATCH 27/43] WIP: Minimize example --- example/cmesh/t8_cmesh_hybrid_new.cxx | 191 +++++--------------------- 1 file changed, 33 insertions(+), 158 deletions(-) diff --git a/example/cmesh/t8_cmesh_hybrid_new.cxx b/example/cmesh/t8_cmesh_hybrid_new.cxx index 88cbb6d78e..b27fa37c81 100644 --- a/example/cmesh/t8_cmesh_hybrid_new.cxx +++ b/example/cmesh/t8_cmesh_hybrid_new.cxx @@ -80,75 +80,15 @@ t8_basic_hybrid_refine (t8_forest_t forest, t8_forest_t forest_from, t8_locidx_t } } -/** - * An adaptation criterion to refine a forest. Specialized for a "cake" of pyramids (t8_cmesh_new_pyramid_cake) - * For non-pyramid-elements it refines every element. For a pyramid all type 6 pyramids and all type (0|2|4)-tets - * are refined. - * - * \param [in] forest the forest - * \param [in] forest_from - * \param [in] which_tree The local id of the tree - * \param [in] lelement_id the local id of the element - * \param [in] ts the scheme to use - * \param [in] is_family flag, if the \a elements form a family - * \param [in] num_elements number of elements - * \param [in] elements A single element or a collection of \a num_elements - * \return int - */ -static int -t8_basic_cake_refine (t8_forest_t forest, t8_forest_t forest_from, t8_locidx_t which_tree, t8_locidx_t lelement_id, - t8_eclass_scheme_c *ts, int is_family, int num_elements, t8_element_t *elements[]) -{ - int level, type; - /*If the level is equal or higher than given by the user do not refine*/ - level = ts->t8_element_level (elements[0]); - if (level >= *(int *) t8_forest_get_user_data (forest)) { - return 0; - } - else { - int32_t h = T8_DPYRAMID_LEN (level); - /* If the shape of element is a tet or a pyramid refine type 6 pyras and type (0|2|4) tets - * Refine every other shape. */ - switch (ts->t8_element_shape (elements[0])) { - case T8_ECLASS_TET: - type = ((t8_dtet_t *) elements[0])->type; - if (type == 0 || type == 2 || type == 4) { - return 1; - } - else { - return 0; - } - case T8_ECLASS_PYRAMID: - - if (!(((t8_dpyramid_t *) elements[0])->pyramid.x & h)) { - type = ((t8_dpyramid_t *) elements[0])->pyramid.type; - return type == 6 ? 1 : 0; - } - else { - return 0; - } - default: - return 1; - } - } -} - /** * Creates a cmesh and forest using this cmesh and times all high-level algorithms (new, adapt, ghost, partition, ...) * * \param[in] level The initial level of forest * \param[in] endlvl The maximal level of the forest * \param[in] do_vtk Flag, if vtk-output should be produced - * \param[in] eclass if \a mesh is set to use bigmesh or new_from_class the eclass to use in this cmesh - * \param[in] num_elements if \a mesh is set to use bigmesh or new_long_brick_pyramid the number of elements to use in the this cmesh - * \param[in] mesh The mesh to use. - * \param[in] balance A flag if the forest should be balanced - * \param[in] prefix if \a do_vtk is set, the prefix of the vtk-output - * \param[in] part A flag if the cmesh should be partitioned */ static void -t8_basic_hybrid (int level, int endlvl, int do_vtk, t8_eclass_t eclass, int num_elements, int mesh, int balance, - const char *prefix, int part) +t8_basic_hybrid (const int level, int endlvl, const int do_vtk) { /* Create and initialize timing stats */ sc_statinfo_t times[6]; @@ -156,65 +96,33 @@ t8_basic_hybrid (int level, int endlvl, int do_vtk, t8_eclass_t eclass, int num_ sc_stats_init (×[1], "adapt"); sc_stats_init (×[2], "ghost"); sc_stats_init (×[3], "partition"); - sc_stats_init (×[4], "balance"); - sc_stats_init (×[5], "total"); + sc_stats_init (×[4], "total"); double total_time = -sc_MPI_Wtime (); t8_cmesh_t cmesh; /* Create the cmesh */ - switch (mesh) { - case 0: - t8_global_productionf ("Constructing cake mesh with %i pyramids.\n", num_elements); - cmesh = t8_cmesh_new_pyramid_cake (sc_MPI_COMM_WORLD, num_elements); - break; - case 1: - t8_global_productionf ("Constructing full hybrid mesh.\n"); - cmesh = t8_cmesh_new_full_hybrid (sc_MPI_COMM_WORLD); - break; - case 2: - t8_global_productionf ("Constructing long brick out of %i pyramids.\n", num_elements); - cmesh = t8_cmesh_new_long_brick_pyramid (sc_MPI_COMM_WORLD, num_elements); - break; - case 3: - t8_global_productionf ("Constructing mesh from: %s.\n", prefix); - cmesh = t8_cmesh_from_msh_file ((char *) prefix, 0, sc_MPI_COMM_WORLD, 3, 0, 0); - break; - case 4: - t8_global_productionf ("Constructing bigmesh with %i elements.\n", num_elements); - cmesh = t8_cmesh_new_bigmesh (eclass, num_elements, sc_MPI_COMM_WORLD); - break; - case 5: - t8_global_productionf ("Constructing a single %s.\n", t8_eclass_to_string[eclass]); - cmesh = t8_cmesh_new_from_class (eclass, sc_MPI_COMM_WORLD); - break; - default: - SC_ABORT ("NO CMESH"); - return; - } - - if (part) { - /* Partition the cmesh */ - t8_cmesh_t cmesh_partition; - t8_cmesh_init (&cmesh_partition); - t8_cmesh_set_derive (cmesh_partition, cmesh); - t8_cmesh_set_partition_uniform (cmesh_partition, level, t8_scheme_new_default_cxx ()); - t8_cmesh_commit (cmesh_partition, sc_MPI_COMM_WORLD); - cmesh = cmesh_partition; - } - - /* Save the cmesh (except for big cmeshes) and create vtk output if the flag is set. */ + t8_global_productionf ("Constructing full hybrid mesh.\n"); + cmesh = t8_cmesh_new_full_hybrid (sc_MPI_COMM_WORLD); + + /* Partition the cmesh */ + t8_cmesh_t cmesh_partition; + t8_cmesh_init (&cmesh_partition); + t8_cmesh_set_derive (cmesh_partition, cmesh); + t8_cmesh_set_partition_uniform (cmesh_partition, level, t8_scheme_new_default_cxx ()); + t8_cmesh_commit (cmesh_partition, sc_MPI_COMM_WORLD); + cmesh = cmesh_partition; + + /* Save the cmesh and create vtk output if the flag is set. */ char vtuname[BUFSIZ]; char cmesh_file[BUFSIZ]; snprintf (cmesh_file, BUFSIZ, "cmesh_hybrid"); snprintf (vtuname, BUFSIZ, "cmesh_hybrid"); - if (mesh != 4) { - t8_cmesh_save (cmesh, cmesh_file); - if (t8_cmesh_vtk_write_file (cmesh, vtuname, 1.0) == 0) { - t8_debugf ("Output to %s\n", vtuname); - } - else { - t8_debugf ("Error in output\n"); - } + t8_cmesh_save (cmesh, cmesh_file); + if (t8_cmesh_vtk_write_file (cmesh, vtuname, 1.0) == 0) { + t8_debugf ("Output to %s\n", vtuname); + } + else { + t8_debugf ("Error in output\n"); } /* Create the forest and measure the time needed for the initial refinement*/ @@ -241,12 +149,8 @@ t8_basic_hybrid (int level, int endlvl, int do_vtk, t8_eclass_t eclass, int num_ t8_forest_set_profiling (forest_adapt, 1); /* Set the adaptation criterion to use. */ - if (eclass == T8_ECLASS_PYRAMID) { - t8_forest_set_adapt (forest_adapt, forest, t8_basic_cake_refine, 1); - } - else { - t8_forest_set_adapt (forest_adapt, forest, t8_basic_hybrid_refine, 1); - } + t8_forest_set_adapt (forest_adapt, forest, t8_basic_hybrid_refine, 1); + /* Measure the time needed to adapt */ double adapt_time = -sc_MPI_Wtime (); t8_forest_commit (forest_adapt); @@ -259,10 +163,6 @@ t8_basic_hybrid (int level, int endlvl, int do_vtk, t8_eclass_t eclass, int num_ /* We want to compute the ghosts, too */ t8_forest_set_ghost (forest_partition, 1, T8_GHOST_FACES); - if (balance) { - /* If set, the forest should be balanced */ - t8_forest_set_balance (forest_partition, forest_adapt, 1); - } /* Use the profiling algorithm to measure the time needed and also how many ghost elements are sent, * how many procs we sent to and how many rounds we need to balance the forest. */ t8_forest_set_profiling (forest_partition, 1); @@ -275,22 +175,19 @@ t8_basic_hybrid (int level, int endlvl, int do_vtk, t8_eclass_t eclass, int num_ } total_time += sc_MPI_Wtime (); int procs_sent; /* Procs we sent to during forest_partition */ - int balance_rounds; /* Number of balance rounds */ t8_locidx_t ghost_sent; /* Number of ghost-elements we sent */ t8_forest_print_profile (forest_partition); const double partition_time = t8_forest_profile_get_partition_time (forest_partition, &procs_sent); const double ghost_time = t8_forest_profile_get_ghost_time (forest_partition, &ghost_sent); - const double balance_time = t8_forest_profile_get_balance_time (forest_partition, &balance_rounds); /* Accumulate the timings over all procs and print the stats */ sc_stats_accumulate (×[0], new_time); sc_stats_accumulate (×[1], adapt_time); sc_stats_accumulate (×[2], ghost_time); sc_stats_accumulate (×[3], partition_time); - sc_stats_accumulate (×[4], balance_time); - sc_stats_accumulate (×[5], total_time); - sc_stats_compute (sc_MPI_COMM_WORLD, 6, times); - sc_stats_print (t8_get_package_id (), SC_LP_ESSENTIAL, 6, times, 1, 1); + sc_stats_accumulate (×[4], total_time); + sc_stats_compute (sc_MPI_COMM_WORLD, 5, times); + sc_stats_print (t8_get_package_id (), SC_LP_ESSENTIAL, 5, times, 1, 1); t8_forest_unref (&forest_partition); } @@ -298,12 +195,12 @@ t8_basic_hybrid (int level, int endlvl, int do_vtk, t8_eclass_t eclass, int num_ int main (int argc, char **argv) { - int mpiret, parsed; - int level, endlvl, helpme, do_vtk, eclass_int, mesh, elements, balance, part; - t8_eclass_t eclass; + int level; + int endlvl; + int helpme; + int do_vtk; sc_options_t *opt; char help[BUFSIZ]; - const char *file; /* long help message */ snprintf (help, BUFSIZ, @@ -311,7 +208,7 @@ main (int argc, char **argv) "Usage:\t%s \n\t%s -h\t" "for a brief overview of all options.\n", basename (argv[0]), basename (argv[0])); - mpiret = sc_MPI_Init (&argc, &argv); + int mpiret = sc_MPI_Init (&argc, &argv); SC_CHECK_MPI (mpiret); sc_init (sc_MPI_COMM_WORLD, 1, 1, NULL, SC_LP_ESSENTIAL); @@ -322,37 +219,15 @@ main (int argc, char **argv) sc_options_add_int (opt, 'l', "level", &level, 0, "The refinement level of the mesh."); sc_options_add_int (opt, 'f', "final-level", &endlvl, 1, "The final refinement level of the mesh."); sc_options_add_switch (opt, 'v', "vtk", &do_vtk, "Enable vtk-output."); - sc_options_add_switch (opt, 'b', "balance", &balance, "Enable balance"); - sc_options_add_switch (opt, 'p', "partition", &part, "Enable cmesh-partition"); - sc_options_add_int (opt, 'e', "element", &eclass_int, 4, - "Given an element-class, the program will " - " construct a single element of this class. Is ignored, if the option cake is chosen." - "The type of elements to use.\n" - "\t\t4 - hexahedron\n" - "\t\t5 - tetrahedron\n\t\t6 - prism\n\t\t7 - pyramid"); - sc_options_add_int (opt, 'm', "mesh", &mesh, 5, - "A mesh to choose from." - " The meshes to chose from are: \n" - "\t\t0 - cake of pyramids\n\t\t1 - hybrid mesh with all 3D elements" - "\n\t\t2 - a long row of bricks out of pyramids" - "\n\t\t3 - user-specific mesh-file" - "\n\t\t4 - construct multiple elements of class given by -e" - "\n\t\t5 - a single element of class -e"); - sc_options_add_int (opt, 'n', "num_elements", &elements, 3, - "The number of elements to use" - " if a m0 or m4 is build. Has to be larger than 2."); - sc_options_add_string (opt, 'g', "mshfile", &file, "NULL", "Prefix of the msh-file."); - parsed = sc_options_parse (t8_get_package_id (), SC_LP_DEFAULT, opt, argc, argv); + const int parsed = sc_options_parse (t8_get_package_id (), SC_LP_DEFAULT, opt, argc, argv); if (helpme) { /* display help message and usage */ t8_global_productionf ("%s\n", help); sc_options_print_usage (t8_get_package_id (), SC_LP_ERROR, opt, NULL); } - else if (parsed >= 0 && 0 <= level && 4 <= eclass_int && eclass_int < T8_ECLASS_COUNT && elements >= 2 && 0 <= mesh - && mesh < 6) { - eclass = (t8_eclass_t) eclass_int; - t8_basic_hybrid (level, endlvl, do_vtk, eclass, elements, mesh, balance, file, part); + else if (parsed >= 0 && 0 <= level) { + t8_basic_hybrid (level, endlvl, do_vtk); } else { /* wrong usage */ From b2f81b909d431ab70afae3c69933a4cd315a32a5 Mon Sep 17 00:00:00 2001 From: David Knapp Date: Thu, 16 Nov 2023 10:33:16 +0100 Subject: [PATCH 28/43] Added description to t8_cmesh_uniform_bounds_hybrid --- src/t8_cmesh.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/t8_cmesh.h b/src/t8_cmesh.h index ac0803feed..bb951206b2 100644 --- a/src/t8_cmesh.h +++ b/src/t8_cmesh.h @@ -678,7 +678,10 @@ t8_cmesh_uniform_bounds (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t *ts, t8_gl int8_t *first_tree_shared); /** - * Calculate the section of a uniform hybrid forest for the current rank + * Calculate the section of a uniform hybrid forest for the current rank. Need for hybrid meshes, especially + * meshes where not all elements refine into 1:2^dim manner. The section is calculated without assuming such refinement + * and each process computes its number of elements on the given \var level communicates the number to other processes + * and the correct section is computed based on this information. * * \param [in] cmesh The cmesh to be considered. * \param [in] level The uniform refinement level to be created. From 800b716e96f4aabbefc5513c5c4ca9e94136fc25 Mon Sep 17 00:00:00 2001 From: David Knapp Date: Thu, 16 Nov 2023 10:44:01 +0100 Subject: [PATCH 29/43] More const --- src/t8_cmesh.h | 7 ++++--- src/t8_cmesh/t8_cmesh_cxx.cxx | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/t8_cmesh.h b/src/t8_cmesh.h index bb951206b2..e1864b1664 100644 --- a/src/t8_cmesh.h +++ b/src/t8_cmesh.h @@ -696,9 +696,10 @@ t8_cmesh_uniform_bounds (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t *ts, t8_gl * \param [in] comm The communicator */ void -t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t *scheme, t8_gloidx_t *first_local_tree, - t8_gloidx_t *child_in_tree_begin, t8_gloidx_t *last_local_tree, - t8_gloidx_t *child_in_tree_end, int8_t *first_tree_shared, sc_MPI_Comm comm); +t8_cmesh_uniform_bounds_hybrid (const t8_cmesh_t cmesh, const int level, const t8_scheme_cxx_t *scheme, + t8_gloidx_t *first_local_tree, t8_gloidx_t *child_in_tree_begin, + t8_gloidx_t *last_local_tree, t8_gloidx_t *child_in_tree_end, int8_t *first_tree_shared, + sc_MPI_Comm comm); /** Increase the reference counter of a cmesh. * \param [in,out] cmesh On input, this cmesh must exist with positive diff --git a/src/t8_cmesh/t8_cmesh_cxx.cxx b/src/t8_cmesh/t8_cmesh_cxx.cxx index 1c96f43a0a..7615f304bc 100644 --- a/src/t8_cmesh/t8_cmesh_cxx.cxx +++ b/src/t8_cmesh/t8_cmesh_cxx.cxx @@ -282,9 +282,10 @@ t8_cmesh_determine_partition (sc_array_t *first_element_tree, size_t pure_local_ /* TODO: Shared trees, binary search in offset-array to avoid recv_any, * use partition_given to partition the cmesh*/ void -t8_cmesh_uniform_bounds_hybrid (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t *scheme, t8_gloidx_t *first_local_tree, - t8_gloidx_t *child_in_tree_begin, t8_gloidx_t *last_local_tree, - t8_gloidx_t *child_in_tree_end, int8_t *first_tree_shared, sc_MPI_Comm comm) +t8_cmesh_uniform_bounds_hybrid (const t8_cmesh_t cmesh, const int level, const t8_scheme_cxx_t *scheme, + t8_gloidx_t *first_local_tree, t8_gloidx_t *child_in_tree_begin, + t8_gloidx_t *last_local_tree, t8_gloidx_t *child_in_tree_end, int8_t *first_tree_shared, + sc_MPI_Comm comm) { T8_ASSERT (cmesh != NULL); #ifdef T8_ENABLE_DEBUG From 408954d337abb3d814d4ed4e79fe77c1ac260973 Mon Sep 17 00:00:00 2001 From: David Knapp Date: Fri, 26 Jan 2024 14:47:01 +0100 Subject: [PATCH 30/43] Rename function, add assertion --- src/t8_cmesh/t8_cmesh_cxx.cxx | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/src/t8_cmesh/t8_cmesh_cxx.cxx b/src/t8_cmesh/t8_cmesh_cxx.cxx index 7615f304bc..69ce013cb0 100644 --- a/src/t8_cmesh/t8_cmesh_cxx.cxx +++ b/src/t8_cmesh/t8_cmesh_cxx.cxx @@ -103,9 +103,10 @@ t8_cmesh_get_first_element_of_process (int process, int mpisize, t8_gloidx_t glo } void -t8_cmesh_uniform_bounds (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t *ts, t8_gloidx_t *first_local_tree, - t8_gloidx_t *child_in_tree_begin, t8_gloidx_t *last_local_tree, t8_gloidx_t *child_in_tree_end, - int8_t *first_tree_shared) +t8_cmesh_uniform_bounds_equal_element_count (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t *ts, + t8_gloidx_t *first_local_tree, t8_gloidx_t *child_in_tree_begin, + t8_gloidx_t *last_local_tree, t8_gloidx_t *child_in_tree_end, + int8_t *first_tree_shared) { int is_empty; @@ -136,6 +137,23 @@ t8_cmesh_uniform_bounds (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t *ts, t8_gl /* Compute the number of children on level in each tree */ global_num_children = 0; +#ifdef T8_ENABLE_DEBUG + t8_gloidx_t children_per_tree_check = 0; + for (tree_class = T8_ECLASS_ZERO; tree_class < T8_ECLASS_COUNT; ++tree_class) { + /* Get the number of children on level of the first tree class that is used*/ + if (cmesh->num_trees_per_eclass[tree_class] > 0) { + children_per_tree_check = ts->eclass_schemes[tree_class]->t8_element_count_leafs_from_root (level); + break; + } + } + /* Compare it to the other used tree classes*/ + for (tree_class = T8_ECLASS_ZERO; tree_class < T8_ECLASS_COUNT; ++tree_class) { + if (cmesh->num_trees_per_eclass[tree_class] > 0) { + T8_ASSERT (children_per_tree_check == ts->eclass_schemes[tree_class]->t8_element_count_leafs_from_root (level)); + } + } + +#endif for (tree_class = T8_ECLASS_ZERO; tree_class < T8_ECLASS_COUNT; ++tree_class) { /* We iterate over each element class and get the number of children for this * tree class. From d0539e618e86190d5c43ab033ceaab1f3fb12c4d Mon Sep 17 00:00:00 2001 From: David Knapp Date: Fri, 26 Jan 2024 14:56:59 +0100 Subject: [PATCH 31/43] refactor function --- src/t8_cmesh/t8_cmesh_cxx.cxx | 136 ++++++++++++++++++---------------- 1 file changed, 73 insertions(+), 63 deletions(-) diff --git a/src/t8_cmesh/t8_cmesh_cxx.cxx b/src/t8_cmesh/t8_cmesh_cxx.cxx index 69ce013cb0..a7ed0baa4d 100644 --- a/src/t8_cmesh/t8_cmesh_cxx.cxx +++ b/src/t8_cmesh/t8_cmesh_cxx.cxx @@ -296,6 +296,77 @@ t8_cmesh_determine_partition (sc_array_t *first_element_tree, size_t pure_local_ #endif return first_proc_adjusted; } +void +t8_cmesh_partition_from_unpartioned (t8_cmesh_t cmesh, const t8_gloidx_t local_num_children, const int level, + const t8_scheme_cxx_t *scheme, t8_gloidx_t *first_local_tree, + t8_gloidx_t *child_in_tree_begin, t8_gloidx_t *last_local_tree, + t8_gloidx_t *child_in_tree_end, int8_t *first_tree_shared) +{ + const t8_gloidx_t num_trees = t8_cmesh_get_num_trees (cmesh); + t8_debugf ("Cmesh is not partitioned.\n"); + /* Compute the first and last element of this process. Then loop over + * all trees to find the trees in which these are contained. + * We cast to long double and double to prevent overflow. */ + /* cmesh is replicated, therefore the computation of local_num_children equals the global number of children*/ + const t8_gloidx_t first_child + = t8_cmesh_get_first_element_of_process (cmesh->mpirank, cmesh->mpisize, local_num_children); + const t8_gloidx_t last_child + = t8_cmesh_get_first_element_of_process (cmesh->mpirank + 1, cmesh->mpisize, local_num_children) - 1; + /* Can't we optimize this linear loop by using a binary search? + * -> No, we cannot. Since we need in any case compute the t8_element_count_leafs_from_root + * for each tree. + */ + t8_gloidx_t current_tree_element_offset = 0; + for (t8_gloidx_t igtree = 0; igtree < num_trees; ++igtree) { + const int ieclass = t8_cmesh_get_tree_class (cmesh, (t8_locidx_t) igtree); + const t8_eclass_scheme_c *tree_scheme = scheme->eclass_schemes[ieclass]; + /* TODO: We can optimize by buffering the elem_in_tree value. Thus, if + the computation is expensive (may be for non-morton-type schemes), + we do it only once. */ + const t8_gloidx_t elem_in_tree = tree_scheme->t8_element_count_leafs_from_root (level); + /* Check if the first element is on the current tree */ + if (current_tree_element_offset <= first_child && first_child < current_tree_element_offset + elem_in_tree) { + if (child_in_tree_begin != NULL) { + *child_in_tree_begin = first_child - current_tree_element_offset; + } + *first_local_tree = igtree; + /* If our first element is not the very first element in the tree, we share + * this tree with the previous process. */ + if (first_tree_shared != NULL) { + *first_tree_shared = current_tree_element_offset < first_child ? 1 : 0; + } + } + if (last_child < first_child) { + /* This process is empty. We needed to identify the 'first_local_tree' since it is the + * first local tree of the next process, which we store in this case. + */ + t8_cmesh_uniform_set_return_parameters_to_empty (first_local_tree, child_in_tree_begin, last_local_tree, + child_in_tree_end, first_tree_shared); + *first_local_tree = 0; + return; + } + /* Check if the last element is on the current tree */ + if (current_tree_element_offset <= last_child && last_child < current_tree_element_offset + elem_in_tree) { + if (child_in_tree_end != NULL) { + *child_in_tree_end = last_child - current_tree_element_offset + 1; + } + *last_local_tree = igtree; + + /* We have found the last tree and can immediately return, + * ending the for loop. */ + T8_ASSERT (*first_local_tree <= *last_local_tree); + T8_ASSERT (0 <= *first_local_tree && *first_local_tree < num_trees); + T8_ASSERT (0 <= *last_local_tree && *last_local_tree < num_trees); + return; + } + current_tree_element_offset += elem_in_tree; + } + /* If we reach this part, we do not have any trees - the cmesh is empty */ + T8_ASSERT (num_trees == 0); + t8_cmesh_uniform_set_return_parameters_to_empty (first_local_tree, child_in_tree_begin, last_local_tree, + child_in_tree_end, first_tree_shared); + return; +} /* TODO: Shared trees, binary search in offset-array to avoid recv_any, * use partition_given to partition the cmesh*/ @@ -357,69 +428,8 @@ t8_cmesh_uniform_bounds_hybrid (const t8_cmesh_t cmesh, const int level, const t /* If the initial cmesh is not partitioned, every process knows "everything" and we do not * need any communication.*/ if (!cmesh->set_partition) { - const t8_gloidx_t num_trees = t8_cmesh_get_num_trees (cmesh); - t8_debugf ("Cmesh is not partitioned.\n"); - /* Compute the first and last element of this process. Then loop over - * all trees to find the trees in which these are contained. - * We cast to long double and double to prevent overflow. */ - /* cmesh is replicated, therefore the computation of local_num_children equals the global number of children*/ - const t8_gloidx_t first_child - = t8_cmesh_get_first_element_of_process (cmesh->mpirank, cmesh->mpisize, local_num_children); - const t8_gloidx_t last_child - = t8_cmesh_get_first_element_of_process (cmesh->mpirank + 1, cmesh->mpisize, local_num_children) - 1; - /* Can't we optimize this linear loop by using a binary search? - * -> No, we cannot. Since we need in any case compute the t8_element_count_leafs_from_root - * for each tree. - */ - t8_gloidx_t current_tree_element_offset = 0; - for (t8_gloidx_t igtree = 0; igtree < num_trees; ++igtree) { - const int ieclass = t8_cmesh_get_tree_class (cmesh, (t8_locidx_t) igtree); - const t8_eclass_scheme_c *tree_scheme = scheme->eclass_schemes[ieclass]; - /* TODO: We can optimize by buffering the elem_in_tree value. Thus, if - the computation is expensive (may be for non-morton-type schemes), - we do it only once. */ - const t8_gloidx_t elem_in_tree = tree_scheme->t8_element_count_leafs_from_root (level); - /* Check if the first element is on the current tree */ - if (current_tree_element_offset <= first_child && first_child < current_tree_element_offset + elem_in_tree) { - if (child_in_tree_begin != NULL) { - *child_in_tree_begin = first_child - current_tree_element_offset; - } - *first_local_tree = igtree; - /* If our first element is not the very first element in the tree, we share - * this tree with the previous process. */ - if (first_tree_shared != NULL) { - *first_tree_shared = current_tree_element_offset < first_child ? 1 : 0; - } - } - if (last_child < first_child) { - /* This process is empty. We needed to identify the 'first_local_tree' since it is the - * first local tree of the next process, which we store in this case. - */ - t8_cmesh_uniform_set_return_parameters_to_empty (first_local_tree, child_in_tree_begin, last_local_tree, - child_in_tree_end, first_tree_shared); - *first_local_tree = 0; - return; - } - /* Check if the last element is on the current tree */ - if (current_tree_element_offset <= last_child && last_child < current_tree_element_offset + elem_in_tree) { - if (child_in_tree_end != NULL) { - *child_in_tree_end = last_child - current_tree_element_offset + 1; - } - *last_local_tree = igtree; - - /* We have found the last tree and can immediately return, - * ending the for loop. */ - T8_ASSERT (*first_local_tree <= *last_local_tree); - T8_ASSERT (0 <= *first_local_tree && *first_local_tree < num_trees); - T8_ASSERT (0 <= *last_local_tree && *last_local_tree < num_trees); - return; - } - current_tree_element_offset += elem_in_tree; - } - /* If we reach this part, we do not have any trees - the cmesh is empty */ - T8_ASSERT (num_trees == 0); - t8_cmesh_uniform_set_return_parameters_to_empty (first_local_tree, child_in_tree_begin, last_local_tree, - child_in_tree_end, first_tree_shared); + t8_cmesh_partition_from_unpartioned (cmesh, local_num_children, level, scheme, first_local_tree, + child_in_tree_begin, last_local_tree, child_in_tree_end, first_tree_shared); return; } From 69aae1383c4efd47341bf8adb801ca4f16b98a2f Mon Sep 17 00:00:00 2001 From: David Knapp Date: Mon, 29 Jan 2024 09:51:00 +0100 Subject: [PATCH 32/43] Apply suggestions from code review Co-authored-by: Johannes Holke --- example/cmesh/t8_cmesh_hybrid_new.cxx | 2 +- src/t8_cmesh/t8_cmesh_cxx.cxx | 28 +++++++++++++-------------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/example/cmesh/t8_cmesh_hybrid_new.cxx b/example/cmesh/t8_cmesh_hybrid_new.cxx index b27fa37c81..fcd579c16a 100644 --- a/example/cmesh/t8_cmesh_hybrid_new.cxx +++ b/example/cmesh/t8_cmesh_hybrid_new.cxx @@ -81,7 +81,7 @@ t8_basic_hybrid_refine (t8_forest_t forest, t8_forest_t forest_from, t8_locidx_t } /** - * Creates a cmesh and forest using this cmesh and times all high-level algorithms (new, adapt, ghost, partition, ...) + * Creates a cmesh and forest using this cmesh, executes all high-level algorithms (new, adapt, ghost, partition, ...) and measures their runtime * * \param[in] level The initial level of forest * \param[in] endlvl The maximal level of the forest diff --git a/src/t8_cmesh/t8_cmesh_cxx.cxx b/src/t8_cmesh/t8_cmesh_cxx.cxx index a7ed0baa4d..5e80516d16 100644 --- a/src/t8_cmesh/t8_cmesh_cxx.cxx +++ b/src/t8_cmesh/t8_cmesh_cxx.cxx @@ -67,17 +67,15 @@ t8_cmesh_uniform_set_return_parameters_to_empty (t8_gloidx_t *first_local_tree, static inline t8_gloidx_t t8_A_times_B_over_C_gloidx (t8_gloidx_t A, t8_gloidx_t B, t8_gloidx_t C) { -#ifdef T8_ENABLE_DEBUG - { - /* We check whether computing A/C * B will cause an overflow. - * This can be achieved by checking if dividing the result by B - * yields A/C again. */ - const t8_gloidx_t a_over_c = A / C; - const t8_gloidx_t a_o_c_times_b = a_over_c * B; - T8_ASSERT (a_over_c == 0 || a_o_c_times_b / a_over_c == B); - } -#endif - return (t8_gloidx_t) ((A / C) * B + (((long double) (A % C)) / C) * B); + const t8_gloidx_t a_over_c = A / C; + const t8_gloidx_t a_o_c_times_b = a_over_c * B; + + /* We check whether computing A/C * B will cause an overflow. + * This can be achieved by checking if dividing the result by B + * yields A/C again. */ + T8_ASSERT (a_over_c == 0 || a_o_c_times_b / a_over_c == B); + + return (t8_gloidx_t) (a_o_c_times_b + (((long double) (A % C)) / C) * B); } /* Version of t8_A_times_B_over_C where A is an integer. @@ -245,7 +243,7 @@ t8_cmesh_uniform_bounds_equal_element_count (t8_cmesh_t cmesh, int level, t8_sch * a (pure) local tree index, return the first process that * will have elements of this tree in a uniform partition. * The data pointer must point to a valid t8_cmesh_partition_query_t, - * storing the number of processe and the global number of elements. + * storing the number of processes and the global number of elements. * * This function is used standalone and as callback of sc_array_split. */ static size_t @@ -278,13 +276,13 @@ t8_cmesh_determine_partition (sc_array_t *first_element_tree, size_t pure_local_ t8_debugf ("[H] ptree %zd, first element %li, on proc %zd\n", pure_local_tree, element_index, first_proc_adjusted); /* Safety checks */ +#ifdef T8_ENABLE_DEBUG T8_ASSERT (0 <= first_proc_rank && (int) first_proc_rank < query_data->num_procs); T8_ASSERT (0 <= first_proc_adjusted); /* Check that the element lies in the partition of the computed proc. */ T8_ASSERT ( t8_cmesh_get_first_element_of_process (first_proc_rank, query_data->num_procs, query_data->global_num_elements) <= element_index); -#ifdef T8_ENABLE_DEBUG if ((int) first_proc_rank != query_data->num_procs - 1) { T8_ASSERT (t8_cmesh_get_first_element_of_process (first_proc_rank + 1, query_data->num_procs, query_data->global_num_elements) @@ -440,7 +438,7 @@ t8_cmesh_uniform_bounds_hybrid (const t8_cmesh_t cmesh, const int level, const t */ t8_shmem_array_t offset_array; t8_shmem_array_init (&offset_array, sizeof (t8_gloidx_t), cmesh->mpisize + 1, comm); - /* Fill the offset array for each process the global index of its first element in + /* Fill the offset array for each process with the global index of its first element in * the uniform partition. * (0, l_n_c_0, l_n_c_0 + l_n_c_1, l_n_c_0 + l_n_c_1 + l_n_c_2, ...) */ t8_shmem_prefix (&local_num_children, offset_array, 1, T8_MPI_GLOIDX, sc_MPI_SUM); @@ -452,7 +450,7 @@ t8_cmesh_uniform_bounds_hybrid (const t8_cmesh_t cmesh, const int level, const t SC_CHECK_ABORTF (0 <= data.global_num_elements && data.global_num_elements < T8_GLOIDX_MAX, "Overflow in number of elements.\n"); - /*Compute number of non-shared-trees and the local index of the first non-shared-tree */ + /* Compute number of non-shared-trees and the local index of the first non-shared-tree */ const int first_tree_shared_shift = cmesh->first_tree_shared ? 1 : 0; const t8_locidx_t pure_local_trees = cmesh->num_local_trees - first_tree_shared_shift; sc_array send_requests; From 229a1dcc3f21a6a5ba0a42fb4569258f448b7dd4 Mon Sep 17 00:00:00 2001 From: David Knapp Date: Mon, 29 Jan 2024 10:01:48 +0100 Subject: [PATCH 33/43] Indentation --- src/t8_cmesh/t8_cmesh_cxx.cxx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/t8_cmesh/t8_cmesh_cxx.cxx b/src/t8_cmesh/t8_cmesh_cxx.cxx index 5e80516d16..fa263af3cd 100644 --- a/src/t8_cmesh/t8_cmesh_cxx.cxx +++ b/src/t8_cmesh/t8_cmesh_cxx.cxx @@ -69,13 +69,13 @@ t8_A_times_B_over_C_gloidx (t8_gloidx_t A, t8_gloidx_t B, t8_gloidx_t C) { const t8_gloidx_t a_over_c = A / C; const t8_gloidx_t a_o_c_times_b = a_over_c * B; - + /* We check whether computing A/C * B will cause an overflow. * This can be achieved by checking if dividing the result by B * yields A/C again. */ T8_ASSERT (a_over_c == 0 || a_o_c_times_b / a_over_c == B); - - return (t8_gloidx_t) (a_o_c_times_b + (((long double) (A % C)) / C) * B); + + return (t8_gloidx_t) (a_o_c_times_b + (((long double) (A % C)) / C) * B); } /* Version of t8_A_times_B_over_C where A is an integer. From cf728e821cfa869a846f445aca03ff24e566c7b3 Mon Sep 17 00:00:00 2001 From: David Knapp Date: Tue, 13 Feb 2024 10:25:10 +0100 Subject: [PATCH 34/43] Resolve merge errors --- src/t8_cmesh/t8_cmesh_cxx.cxx | 12 ++++++------ test/Makefile.am | 10 ---------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/src/t8_cmesh/t8_cmesh_cxx.cxx b/src/t8_cmesh/t8_cmesh_cxx.cxx index 99f1473ce8..ad9d86cbaf 100644 --- a/src/t8_cmesh/t8_cmesh_cxx.cxx +++ b/src/t8_cmesh/t8_cmesh_cxx.cxx @@ -140,14 +140,14 @@ t8_cmesh_uniform_bounds_equal_element_count (t8_cmesh_t cmesh, int level, t8_sch for (tree_class = T8_ECLASS_ZERO; tree_class < T8_ECLASS_COUNT; ++tree_class) { /* Get the number of children on level of the first tree class that is used*/ if (cmesh->num_trees_per_eclass[tree_class] > 0) { - children_per_tree_check = ts->eclass_schemes[tree_class]->t8_element_count_leafs_from_root (level); + children_per_tree_check = ts->eclass_schemes[tree_class]->t8_element_count_leaves_from_root (level); break; } } /* Compare it to the other used tree classes*/ for (tree_class = T8_ECLASS_ZERO; tree_class < T8_ECLASS_COUNT; ++tree_class) { if (cmesh->num_trees_per_eclass[tree_class] > 0) { - T8_ASSERT (children_per_tree_check == ts->eclass_schemes[tree_class]->t8_element_count_leafs_from_root (level)); + T8_ASSERT (children_per_tree_check == ts->eclass_schemes[tree_class]->t8_element_count_leaves_from_root (level)); } } @@ -321,7 +321,7 @@ t8_cmesh_partition_from_unpartioned (t8_cmesh_t cmesh, const t8_gloidx_t local_n /* TODO: We can optimize by buffering the elem_in_tree value. Thus, if the computation is expensive (may be for non-morton-type schemes), we do it only once. */ - const t8_gloidx_t elem_in_tree = tree_scheme->t8_element_count_leafs_from_root (level); + const t8_gloidx_t elem_in_tree = tree_scheme->t8_element_count_leaves_from_root (level); /* Check if the first element is on the current tree */ if (current_tree_element_offset <= first_child && first_child < current_tree_element_offset + elem_in_tree) { if (child_in_tree_begin != NULL) { @@ -408,14 +408,14 @@ t8_cmesh_uniform_bounds_hybrid (const t8_cmesh_t cmesh, const int level, const t for (int ieclass = T8_ECLASS_ZERO; ieclass < T8_ECLASS_COUNT; ieclass++) { const t8_eclass_scheme_c *tree_scheme = scheme->eclass_schemes[ieclass]; local_num_children - += cmesh->num_local_trees_per_eclass[ieclass] * tree_scheme->t8_element_count_leafs_from_root (level); + += cmesh->num_local_trees_per_eclass[ieclass] * tree_scheme->t8_element_count_leaves_from_root (level); } /* Do not consider shared trees */ if (cmesh->first_tree_shared && cmesh->set_partition) { const int ieclass = t8_cmesh_get_tree_class (cmesh, 0); const t8_eclass_scheme_c *tree_scheme = scheme->eclass_schemes[ieclass]; - local_num_children -= tree_scheme->t8_element_count_leafs_from_root (level); + local_num_children -= tree_scheme->t8_element_count_leaves_from_root (level); } /* @@ -477,7 +477,7 @@ t8_cmesh_uniform_bounds_hybrid (const t8_cmesh_t cmesh, const int level, const t const int ieclass = t8_cmesh_get_tree_class (cmesh, igtree); const t8_eclass_scheme_c *tree_scheme = scheme->eclass_schemes[ieclass]; /* Set the first element of the next tree by adding the number of element of the current tree. */ - *first_element_of_next_tree = *first_element_of_tree + tree_scheme->t8_element_count_leafs_from_root (level); + *first_element_of_next_tree = *first_element_of_tree + tree_scheme->t8_element_count_leaves_from_root (level); } /* Check that the last tree + 1 stores as offset the start of the next process */ diff --git a/test/Makefile.am b/test/Makefile.am index 0bfd8c213f..b378b009fe 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -28,7 +28,6 @@ t8code_googletest_programs = \ test/t8_gtest_basics \ test/t8_schemes/t8_gtest_ancestor \ test/t8_cmesh/t8_gtest_hypercube \ - test/t8_cmesh/t8_gtest_cmesh_copy \ test/t8_schemes/t8_gtest_element_count_leaves \ test/t8_schemes/t8_gtest_element_ref_coords \ test/t8_geometry/t8_gtest_geometry \ @@ -134,10 +133,6 @@ test_t8_cmesh_t8_gtest_hypercube_SOURCES = \ test/t8_gtest_main.cxx \ test/t8_cmesh/t8_gtest_hypercube.cxx -test_t8_cmesh_t8_gtest_cmesh_copy_SOURCES = \ - test/t8_gtest_main.cxx \ - test/t8_cmesh/t8_gtest_cmesh_copy.cxx - test_t8_cmesh_t8_gtest_cmesh_set_join_by_vertices_SOURCES = \ test/t8_gtest_main.cxx \ test/t8_cmesh/t8_gtest_cmesh_set_join_by_vertices.cxx @@ -362,10 +357,6 @@ test_t8_cmesh_t8_gtest_hypercube_LDADD = $(t8_gtest_target_ld_add) test_t8_cmesh_t8_gtest_hypercube_LDFLAGS = $(t8_gtest_target_ld_flags) test_t8_cmesh_t8_gtest_hypercube_CPPFLAGS = $(t8_gtest_target_cpp_flags) -test_t8_cmesh_t8_gtest_cmesh_copy_LDADD = $(t8_gtest_target_ld_add) -test_t8_cmesh_t8_gtest_cmesh_copy_LDFLAGS = $(t8_gtest_target_ld_flags) -test_t8_cmesh_t8_gtest_cmesh_copy_CPPFLAGS = $(t8_gtest_target_cpp_flags) - test_t8_cmesh_t8_gtest_cmesh_set_join_by_vertices_LDADD = $(t8_gtest_target_ld_add) test_t8_cmesh_t8_gtest_cmesh_set_join_by_vertices_LDFLAGS = $(t8_gtest_target_ld_flags) test_t8_cmesh_t8_gtest_cmesh_set_join_by_vertices_CPPFLAGS = $(t8_gtest_target_cpp_flags) @@ -544,7 +535,6 @@ test_t8_schemes_t8_gtest_init_linear_id_CPPFLAGS += $(t8_gtest_target_mpi_cpp_fl test_t8_gtest_basics_CPPFLAGS += $(t8_gtest_target_mpi_cpp_flags) test_t8_schemes_t8_gtest_ancestor_CPPFLAGS += $(t8_gtest_target_mpi_cpp_flags) test_t8_cmesh_t8_gtest_hypercube_CPPFLAGS += $(t8_gtest_target_mpi_cpp_flags) -test_t8_cmesh_t8_gtest_cmesh_copy_CPPFLAGS += $(t8_gtest_target_mpi_cpp_flags) test_t8_cmesh_t8_gtest_cmesh_set_join_by_vertices_CPPFLAGS += $(t8_gtest_target_mpi_cpp_flags) test_t8_schemes_t8_gtest_element_count_leaves_CPPFLAGS += $(t8_gtest_target_mpi_cpp_flags) test_t8_schemes_t8_gtest_element_ref_coords_CPPFLAGS += $(t8_gtest_target_mpi_cpp_flags) From b92817da4d6d2bda47eb262894717dbd3928c7e2 Mon Sep 17 00:00:00 2001 From: David Knapp Date: Tue, 13 Feb 2024 10:33:18 +0100 Subject: [PATCH 35/43] Typo fix --- src/t8_cmesh/t8_cmesh_cxx.cxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/t8_cmesh/t8_cmesh_cxx.cxx b/src/t8_cmesh/t8_cmesh_cxx.cxx index ad9d86cbaf..2ba8e5bb07 100644 --- a/src/t8_cmesh/t8_cmesh_cxx.cxx +++ b/src/t8_cmesh/t8_cmesh_cxx.cxx @@ -311,7 +311,7 @@ t8_cmesh_partition_from_unpartioned (t8_cmesh_t cmesh, const t8_gloidx_t local_n const t8_gloidx_t last_child = t8_cmesh_get_first_element_of_process (cmesh->mpirank + 1, cmesh->mpisize, local_num_children) - 1; /* Can't we optimize this linear loop by using a binary search? - * -> No, we cannot. Since we need in any case compute the t8_element_count_leafs_from_root + * -> No, we cannot. Since we need in any case compute the t8_element_count_leaves_from_root * for each tree. */ t8_gloidx_t current_tree_element_offset = 0; From fa967f63c7f291c0109347472a4967ad1e912626 Mon Sep 17 00:00:00 2001 From: David Knapp Date: Tue, 13 Feb 2024 10:57:22 +0100 Subject: [PATCH 36/43] Rename t8_cmesh_uniform_bounds_hybrid --- src/t8_cmesh.h | 9 +++++---- src/t8_cmesh/t8_cmesh_cxx.cxx | 9 +++++---- src/t8_cmesh/t8_cmesh_partition.c | 4 ++-- src/t8_forest/t8_forest_cxx.cxx | 6 +++--- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/t8_cmesh.h b/src/t8_cmesh.h index 491696acc0..ebea67a252 100644 --- a/src/t8_cmesh.h +++ b/src/t8_cmesh.h @@ -738,10 +738,11 @@ t8_cmesh_uniform_bounds (t8_cmesh_t cmesh, int level, t8_scheme_cxx_t *ts, t8_gl * \param [in] comm The communicator */ void -t8_cmesh_uniform_bounds_hybrid (const t8_cmesh_t cmesh, const int level, const t8_scheme_cxx_t *scheme, - t8_gloidx_t *first_local_tree, t8_gloidx_t *child_in_tree_begin, - t8_gloidx_t *last_local_tree, t8_gloidx_t *child_in_tree_end, int8_t *first_tree_shared, - sc_MPI_Comm comm); +t8_cmesh_uniform_bounds_for_irregular_refinement (const t8_cmesh_t cmesh, const int level, + const t8_scheme_cxx_t *scheme, t8_gloidx_t *first_local_tree, + t8_gloidx_t *child_in_tree_begin, t8_gloidx_t *last_local_tree, + t8_gloidx_t *child_in_tree_end, int8_t *first_tree_shared, + sc_MPI_Comm comm); /** Increase the reference counter of a cmesh. * \param [in,out] cmesh On input, this cmesh must exist with positive diff --git a/src/t8_cmesh/t8_cmesh_cxx.cxx b/src/t8_cmesh/t8_cmesh_cxx.cxx index 2ba8e5bb07..6e555e7480 100644 --- a/src/t8_cmesh/t8_cmesh_cxx.cxx +++ b/src/t8_cmesh/t8_cmesh_cxx.cxx @@ -369,10 +369,11 @@ t8_cmesh_partition_from_unpartioned (t8_cmesh_t cmesh, const t8_gloidx_t local_n /* TODO: Shared trees, binary search in offset-array to avoid recv_any, * use partition_given to partition the cmesh*/ void -t8_cmesh_uniform_bounds_hybrid (const t8_cmesh_t cmesh, const int level, const t8_scheme_cxx_t *scheme, - t8_gloidx_t *first_local_tree, t8_gloidx_t *child_in_tree_begin, - t8_gloidx_t *last_local_tree, t8_gloidx_t *child_in_tree_end, int8_t *first_tree_shared, - sc_MPI_Comm comm) +t8_cmesh_uniform_bounds_for_irregular_refinement (const t8_cmesh_t cmesh, const int level, + const t8_scheme_cxx_t *scheme, t8_gloidx_t *first_local_tree, + t8_gloidx_t *child_in_tree_begin, t8_gloidx_t *last_local_tree, + t8_gloidx_t *child_in_tree_end, int8_t *first_tree_shared, + sc_MPI_Comm comm) { T8_ASSERT (cmesh != NULL); #ifdef T8_ENABLE_DEBUG diff --git a/src/t8_cmesh/t8_cmesh_partition.c b/src/t8_cmesh/t8_cmesh_partition.c index 031277fa7f..42ecb535a8 100644 --- a/src/t8_cmesh/t8_cmesh_partition.c +++ b/src/t8_cmesh/t8_cmesh_partition.c @@ -1559,8 +1559,8 @@ t8_cmesh_partition (t8_cmesh_t cmesh, sc_MPI_Comm comm) T8_ASSERT (cmesh->tree_offsets == NULL); ts = cmesh->set_partition_scheme; /* The refinement scheme */ T8_ASSERT (ts != NULL); - t8_cmesh_uniform_bounds_hybrid (cmesh_from, cmesh->set_partition_level, ts, &cmesh->first_tree, NULL, &last_tree, - NULL, &cmesh->first_tree_shared, comm); + t8_cmesh_uniform_bounds_for_irregular_refinement (cmesh_from, cmesh->set_partition_level, ts, &cmesh->first_tree, + NULL, &last_tree, NULL, &cmesh->first_tree_shared, comm); cmesh->num_local_trees = last_tree - cmesh->first_tree + 1; /* Compute the tree offset */ t8_cmesh_gather_treecount_nocommit (cmesh, comm); diff --git a/src/t8_forest/t8_forest_cxx.cxx b/src/t8_forest/t8_forest_cxx.cxx index 669739500b..4ee9a1e638 100644 --- a/src/t8_forest/t8_forest_cxx.cxx +++ b/src/t8_forest/t8_forest_cxx.cxx @@ -1470,9 +1470,9 @@ t8_forest_populate (t8_forest_t forest) SC_CHECK_ABORT (forest->set_level <= forest->maxlevel, "Given refinement level exceeds the maximum.\n"); /* TODO: create trees and quadrants according to uniform refinement */ - t8_cmesh_uniform_bounds_hybrid (forest->cmesh, forest->set_level, forest->scheme_cxx, &forest->first_local_tree, - &child_in_tree_begin, &forest->last_local_tree, &child_in_tree_end, NULL, - forest->mpicomm); + t8_cmesh_uniform_bounds_for_irregular_refinement ( + forest->cmesh, forest->set_level, forest->scheme_cxx, &forest->first_local_tree, &child_in_tree_begin, + &forest->last_local_tree, &child_in_tree_end, NULL, forest->mpicomm); /* True if the forest has no elements */ is_empty = forest->first_local_tree > forest->last_local_tree || (forest->first_local_tree == forest->last_local_tree && child_in_tree_begin >= child_in_tree_end); From f8b39d5fb1616959ab03fcad78758e23ae5f534d Mon Sep 17 00:00:00 2001 From: David Knapp Date: Wed, 14 Feb 2024 10:05:24 +0100 Subject: [PATCH 37/43] Remove comment --- src/t8_cmesh/t8_cmesh_cxx.cxx | 1 - 1 file changed, 1 deletion(-) diff --git a/src/t8_cmesh/t8_cmesh_cxx.cxx b/src/t8_cmesh/t8_cmesh_cxx.cxx index 6e555e7480..8d3150458e 100644 --- a/src/t8_cmesh/t8_cmesh_cxx.cxx +++ b/src/t8_cmesh/t8_cmesh_cxx.cxx @@ -497,7 +497,6 @@ t8_cmesh_uniform_bounds_for_irregular_refinement (const t8_cmesh_t cmesh, const if (!cmesh->first_tree_shared && t8_cmesh_get_global_id (cmesh, 0) == 0) { /* If our first tree is 0 and not shared, then we need to send to all processes * below the start process. Even if their partition is empty. */ - t8_debugf ("LLL: Would have set send_first to 0, instead: %li", send_first_nonempty); send_first = send_first_nonempty; } else { From ce649eb9bde3fa51937f648b15a7cf39b404336c Mon Sep 17 00:00:00 2001 From: David Knapp Date: Wed, 14 Feb 2024 10:27:14 +0100 Subject: [PATCH 38/43] Update to main --- example/cmesh/t8_cmesh_hybrid_new.cxx | 2 +- test/Makefile.am | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/example/cmesh/t8_cmesh_hybrid_new.cxx b/example/cmesh/t8_cmesh_hybrid_new.cxx index fcd579c16a..8de7cad91d 100644 --- a/example/cmesh/t8_cmesh_hybrid_new.cxx +++ b/example/cmesh/t8_cmesh_hybrid_new.cxx @@ -118,7 +118,7 @@ t8_basic_hybrid (const int level, int endlvl, const int do_vtk) snprintf (cmesh_file, BUFSIZ, "cmesh_hybrid"); snprintf (vtuname, BUFSIZ, "cmesh_hybrid"); t8_cmesh_save (cmesh, cmesh_file); - if (t8_cmesh_vtk_write_file (cmesh, vtuname, 1.0) == 0) { + if (t8_cmesh_vtk_write_file (cmesh, vtuname) == 0) { t8_debugf ("Output to %s\n", vtuname); } else { diff --git a/test/Makefile.am b/test/Makefile.am index 114d8ffbd0..6cc4c13064 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -290,6 +290,10 @@ test_t8_cmesh_generator_t8_gtest_cmesh_generator_test_SOURCES = \ test/t8_gtest_main.cxx \ test/t8_cmesh_generator/t8_gtest_cmesh_generator_test.cxx +test_t8_cmesh_t8_gtest_cmesh_copy_SOURCES = \ + test/t8_gtest_main.cxx \ + test/t8_cmesh/t8_gtest_cmesh_copy.cxx + #define ld and cpp flags for all targets t8_gtest_target_ld_add = $(LDADD) test/libgtest.la t8_gtest_target_ld_flags = $(AM_LDFLAGS) -pthread @@ -518,6 +522,9 @@ test_t8_cmesh_generator_t8_gtest_cmesh_generator_test_LDADD = $(t8_gtest_target_ test_t8_cmesh_generator_t8_gtest_cmesh_generator_test_LDFLAGS = $(t8_gtest_target_ld_flags) test_t8_cmesh_generator_t8_gtest_cmesh_generator_test_CPPFLAGS = $(t8_gtest_target_cpp_flags) +test_t8_cmesh_t8_gtest_cmesh_copy_LDADD = $(t8_gtest_target_ld_add) +test_t8_cmesh_t8_gtest_cmesh_copy_LDFLAGS = $(t8_gtest_target_ld_flags) +test_t8_cmesh_t8_gtest_cmesh_copy_CPPFLAGS = $(t8_gtest_target_cpp_flags) # If we did not configure t8code with MPI we need to build Googletest # without MPI support. if !T8_ENABLE_MPI @@ -576,6 +583,8 @@ test_t8_cmesh_t8_gtest_cmesh_tree_vertices_negative_volume_CPPFLAGS += $(t8_gtes test_t8_schemes_t8_gtest_default_CPPFLAGS += $(t8_gtest_target_mpi_cpp_flags) test_t8_schemes_t8_gtest_child_parent_face_CPPFLAGS += $(t8_gtest_target_mpi_cpp_flags) test_t8_cmesh_generator_t8_gtest_cmesh_generator_test_CPPFLAGS += $(t8_gtest_target_mpi_cpp_flags) +test_t8_cmesh_t8_gtest_cmesh_copy_CPPFLAGS += $(t8_gtest_target_mpi_cpp_flags) + endif From ef97012e3e9c88e32dcac44b64e0a38a02da9408 Mon Sep 17 00:00:00 2001 From: David Knapp Date: Wed, 14 Feb 2024 10:27:48 +0100 Subject: [PATCH 39/43] Clean-up --- src/t8_cmesh/t8_cmesh_cxx.cxx | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/t8_cmesh/t8_cmesh_cxx.cxx b/src/t8_cmesh/t8_cmesh_cxx.cxx index 8d3150458e..facd39fe26 100644 --- a/src/t8_cmesh/t8_cmesh_cxx.cxx +++ b/src/t8_cmesh/t8_cmesh_cxx.cxx @@ -258,7 +258,6 @@ t8_cmesh_determine_partition (sc_array_t *first_element_tree, size_t pure_local_ const t8_gloidx_t mirror_element_index = query_data->global_num_elements - element_index - 1; int first_proc_rank; - t8_debugf ("[H] tree %li el %li mirror %li\n", pure_local_tree, element_index, mirror_element_index); if (element_index == query_data->global_num_elements) { return query_data->num_procs - query_data->process_offset; } @@ -267,13 +266,10 @@ t8_cmesh_determine_partition (sc_array_t *first_element_tree, size_t pure_local_ = query_data->num_procs - 1 - t8_A_times_B_over_C_intA (query_data->num_procs, mirror_element_index, query_data->global_num_elements); - // (int)(((long double)mirror_element_index/query_data->global_num_elements)*query_data->num_procs); - /* If this is called by array split, we need to adjust for relative processes starting * add the first process. */ first_proc_adjusted = first_proc_rank - query_data->process_offset; } - t8_debugf ("[H] ptree %zd, first element %li, on proc %zd\n", pure_local_tree, element_index, first_proc_adjusted); /* Safety checks */ #ifdef T8_ENABLE_DEBUG @@ -522,7 +518,6 @@ t8_cmesh_uniform_bounds_for_irregular_refinement (const t8_cmesh_t cmesh, const * These tree indices will be stored in offset_partition. */ sc_array_split (&first_element_tree, &offset_partition, (size_t) num_procs_we_send_to + 1, t8_cmesh_determine_partition, (void *) &data); - // TODO: Is determination of last proc we send to wrong? /* We know: Lowest process and highest process we need to send trees to Tree0 Tree1 TreeN From 24ca4e941a8f3784fb3dbb344dc997f6c07bf3e0 Mon Sep 17 00:00:00 2001 From: David Knapp Date: Wed, 14 Feb 2024 10:27:58 +0100 Subject: [PATCH 40/43] Simplification of determining send_first --- src/t8_cmesh/t8_cmesh_cxx.cxx | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/src/t8_cmesh/t8_cmesh_cxx.cxx b/src/t8_cmesh/t8_cmesh_cxx.cxx index facd39fe26..97aa64d404 100644 --- a/src/t8_cmesh/t8_cmesh_cxx.cxx +++ b/src/t8_cmesh/t8_cmesh_cxx.cxx @@ -488,16 +488,7 @@ t8_cmesh_uniform_bounds_for_irregular_refinement (const t8_cmesh_t cmesh, const data.process_offset = 0; /* Compute the process that will own the first element of our first tree. */ - const t8_gloidx_t send_first_nonempty = (t8_gloidx_t) t8_cmesh_determine_partition (&first_element_tree, 0, &data); - t8_gloidx_t send_first; - if (!cmesh->first_tree_shared && t8_cmesh_get_global_id (cmesh, 0) == 0) { - /* If our first tree is 0 and not shared, then we need to send to all processes - * below the start process. Even if their partition is empty. */ - send_first = send_first_nonempty; - } - else { - send_first = send_first_nonempty; - } + const t8_gloidx_t send_first = (t8_gloidx_t) t8_cmesh_determine_partition (&first_element_tree, 0, &data); /* Compute the process that may own the last element of our first tree. */ t8_gloidx_t send_last = (t8_gloidx_t) t8_cmesh_determine_partition (&first_element_tree, pure_local_trees, &data); From e78a2624fd3b9f36f401459c3c173fb4169a51d7 Mon Sep 17 00:00:00 2001 From: David Knapp Date: Wed, 14 Feb 2024 10:36:26 +0100 Subject: [PATCH 41/43] break off uniform_bound_from_partition --- src/t8_cmesh/t8_cmesh_cxx.cxx | 152 +++++++++++++++++++--------------- 1 file changed, 83 insertions(+), 69 deletions(-) diff --git a/src/t8_cmesh/t8_cmesh_cxx.cxx b/src/t8_cmesh/t8_cmesh_cxx.cxx index 97aa64d404..f1c0f2b059 100644 --- a/src/t8_cmesh/t8_cmesh_cxx.cxx +++ b/src/t8_cmesh/t8_cmesh_cxx.cxx @@ -291,10 +291,10 @@ t8_cmesh_determine_partition (sc_array_t *first_element_tree, size_t pure_local_ return first_proc_adjusted; } void -t8_cmesh_partition_from_unpartioned (t8_cmesh_t cmesh, const t8_gloidx_t local_num_children, const int level, - const t8_scheme_cxx_t *scheme, t8_gloidx_t *first_local_tree, - t8_gloidx_t *child_in_tree_begin, t8_gloidx_t *last_local_tree, - t8_gloidx_t *child_in_tree_end, int8_t *first_tree_shared) +t8_cmesh_uniform_bounds_from_unpartioned (t8_cmesh_t cmesh, const t8_gloidx_t local_num_children, const int level, + const t8_scheme_cxx_t *scheme, t8_gloidx_t *first_local_tree, + t8_gloidx_t *child_in_tree_begin, t8_gloidx_t *last_local_tree, + t8_gloidx_t *child_in_tree_end, int8_t *first_tree_shared) { const t8_gloidx_t num_trees = t8_cmesh_get_num_trees (cmesh); t8_debugf ("Cmesh is not partitioned.\n"); @@ -362,77 +362,16 @@ t8_cmesh_partition_from_unpartioned (t8_cmesh_t cmesh, const t8_gloidx_t local_n return; } -/* TODO: Shared trees, binary search in offset-array to avoid recv_any, - * use partition_given to partition the cmesh*/ void -t8_cmesh_uniform_bounds_for_irregular_refinement (const t8_cmesh_t cmesh, const int level, - const t8_scheme_cxx_t *scheme, t8_gloidx_t *first_local_tree, - t8_gloidx_t *child_in_tree_begin, t8_gloidx_t *last_local_tree, - t8_gloidx_t *child_in_tree_end, int8_t *first_tree_shared, - sc_MPI_Comm comm) +t8_cmesh_uniform_bounds_from_partition (t8_cmesh_t cmesh, t8_gloidx_t local_num_children, const int level, + const t8_scheme_cxx_t *scheme, t8_gloidx_t *first_local_tree, + t8_gloidx_t *child_in_tree_begin, t8_gloidx_t *last_local_tree, + t8_gloidx_t *child_in_tree_end, int8_t *first_tree_shared, sc_MPI_Comm comm) { - T8_ASSERT (cmesh != NULL); #ifdef T8_ENABLE_DEBUG int num_received_start_messages = 0; int num_received_end_messages = 0; #endif - - /* TODO: Clean up size_t and gloidx_t data types, ensure that each variables has the - * matching type. */ - - t8_debugf ("Into t8_cmesh_uniform_bounds_hybrid.\n"); - - if (t8_cmesh_is_empty (cmesh)) { - t8_cmesh_uniform_set_return_parameters_to_empty (first_local_tree, child_in_tree_begin, last_local_tree, - child_in_tree_end, first_tree_shared); - return; - } - -#ifdef T8_ENABLE_DEBUG - { - /* Check that comm matches mpirank and size stored in cmesh */ - int mpirank, mpisize; - int mpiret = sc_MPI_Comm_rank (comm, &mpirank); - SC_CHECK_MPI (mpiret); - mpiret = sc_MPI_Comm_size (comm, &mpisize); - SC_CHECK_MPI (mpiret); - T8_ASSERT (mpirank == cmesh->mpirank); - T8_ASSERT (mpisize == cmesh->mpisize); - } -#endif - t8_gloidx_t local_num_children = 0; - /*Compute number of local elements. Ignore shared trees */ - for (int ieclass = T8_ECLASS_ZERO; ieclass < T8_ECLASS_COUNT; ieclass++) { - const t8_eclass_scheme_c *tree_scheme = scheme->eclass_schemes[ieclass]; - local_num_children - += cmesh->num_local_trees_per_eclass[ieclass] * tree_scheme->t8_element_count_leaves_from_root (level); - } - - /* Do not consider shared trees */ - if (cmesh->first_tree_shared && cmesh->set_partition) { - const int ieclass = t8_cmesh_get_tree_class (cmesh, 0); - const t8_eclass_scheme_c *tree_scheme = scheme->eclass_schemes[ieclass]; - local_num_children -= tree_scheme->t8_element_count_leaves_from_root (level); - } - - /* - * - * Cmesh is not partitioned - * - */ - /* If the initial cmesh is not partitioned, every process knows "everything" and we do not - * need any communication.*/ - if (!cmesh->set_partition) { - t8_cmesh_partition_from_unpartioned (cmesh, local_num_children, level, scheme, first_local_tree, - child_in_tree_begin, last_local_tree, child_in_tree_end, first_tree_shared); - return; - } - - /* - * - * Cmesh is partitioned - * - */ t8_shmem_array_t offset_array; t8_shmem_array_init (&offset_array, sizeof (t8_gloidx_t), cmesh->mpisize + 1, comm); /* Fill the offset array for each process with the global index of its first element in @@ -896,3 +835,78 @@ t8_cmesh_uniform_bounds_for_irregular_refinement (const t8_cmesh_t cmesh, const t8_debugf ("Done with t8_cmesh_uniform_bounds_hybrid.\n"); return; } + +/* TODO: Shared trees, binary search in offset-array to avoid recv_any, + * use partition_given to partition the cmesh*/ +void +t8_cmesh_uniform_bounds_for_irregular_refinement (const t8_cmesh_t cmesh, const int level, + const t8_scheme_cxx_t *scheme, t8_gloidx_t *first_local_tree, + t8_gloidx_t *child_in_tree_begin, t8_gloidx_t *last_local_tree, + t8_gloidx_t *child_in_tree_end, int8_t *first_tree_shared, + sc_MPI_Comm comm) +{ + T8_ASSERT (cmesh != NULL); + + /* TODO: Clean up size_t and gloidx_t data types, ensure that each variables has the + * matching type. */ + + t8_debugf ("Into t8_cmesh_uniform_bounds_hybrid.\n"); + + if (t8_cmesh_is_empty (cmesh)) { + t8_cmesh_uniform_set_return_parameters_to_empty (first_local_tree, child_in_tree_begin, last_local_tree, + child_in_tree_end, first_tree_shared); + return; + } + +#ifdef T8_ENABLE_DEBUG + { + /* Check that comm matches mpirank and size stored in cmesh */ + int mpirank, mpisize; + int mpiret = sc_MPI_Comm_rank (comm, &mpirank); + SC_CHECK_MPI (mpiret); + mpiret = sc_MPI_Comm_size (comm, &mpisize); + SC_CHECK_MPI (mpiret); + T8_ASSERT (mpirank == cmesh->mpirank); + T8_ASSERT (mpisize == cmesh->mpisize); + } +#endif + t8_gloidx_t local_num_children = 0; + /*Compute number of local elements. Ignore shared trees */ + for (int ieclass = T8_ECLASS_ZERO; ieclass < T8_ECLASS_COUNT; ieclass++) { + const t8_eclass_scheme_c *tree_scheme = scheme->eclass_schemes[ieclass]; + local_num_children + += cmesh->num_local_trees_per_eclass[ieclass] * tree_scheme->t8_element_count_leaves_from_root (level); + } + + /* Do not consider shared trees */ + if (cmesh->first_tree_shared && cmesh->set_partition) { + const int ieclass = t8_cmesh_get_tree_class (cmesh, 0); + const t8_eclass_scheme_c *tree_scheme = scheme->eclass_schemes[ieclass]; + local_num_children -= tree_scheme->t8_element_count_leaves_from_root (level); + } + + /* + * + * Cmesh is not partitioned + * + */ + /* If the initial cmesh is not partitioned, every process knows "everything" and we do not + * need any communication.*/ + if (!cmesh->set_partition) { + t8_cmesh_uniform_bounds_from_unpartioned (cmesh, local_num_children, level, scheme, first_local_tree, + child_in_tree_begin, last_local_tree, child_in_tree_end, + first_tree_shared); + return; + } + else { + /* + * + * Cmesh is partitioned + * + */ + t8_cmesh_uniform_bounds_from_partition (cmesh, local_num_children, level, scheme, first_local_tree, + child_in_tree_begin, last_local_tree, child_in_tree_end, first_tree_shared, + comm); + return; + } +} From 1cddf654014e13efe346d9e97af05ed745fda3cc Mon Sep 17 00:00:00 2001 From: David Knapp Date: Wed, 14 Feb 2024 11:28:15 +0100 Subject: [PATCH 42/43] Seperate sending start message --- src/t8_cmesh/t8_cmesh_cxx.cxx | 80 +++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/src/t8_cmesh/t8_cmesh_cxx.cxx b/src/t8_cmesh/t8_cmesh_cxx.cxx index f1c0f2b059..acabf980e3 100644 --- a/src/t8_cmesh/t8_cmesh_cxx.cxx +++ b/src/t8_cmesh/t8_cmesh_cxx.cxx @@ -362,6 +362,75 @@ t8_cmesh_uniform_bounds_from_unpartioned (t8_cmesh_t cmesh, const t8_gloidx_t lo return; } +int +t8_cmesh_uniform_bounds_send_start (t8_cmesh_t cmesh, const long int iproc, const int proc_is_empty, + const t8_locidx_t first_puretree_of_current_proc, const int first_tree_shared_shift, + sc_array_t send_requests, sc_array_t send_buffer, t8_gloidx_t *first_local_tree, + const t8_gloidx_t first_element_in_tree_index_of_current_proc, + int8_t *first_tree_shared, t8_gloidx_t *child_in_tree_begin, + t8_cmesh_partition_query_t *data, sc_MPI_Comm comm, + +#if T8_ENABLE_DEBUG + int *num_received_start_messages +#endif +) +{ + const t8_gloidx_t global_id_of_first_tree + = proc_is_empty ? -1 : first_puretree_of_current_proc + t8_cmesh_get_first_treeid (cmesh) + first_tree_shared_shift; + int current_pos_in_send_buffer = 0; + int expect_start_message = 1; + if (iproc != cmesh->mpirank) { + const int num_entries = 2; + t8_gloidx_t *message; + /* Allocate a new request */ + sc_MPI_Request *request = (sc_MPI_Request *) sc_array_push (&send_requests); + + /* Grow total send buffer */ + sc_array_push_count (&send_buffer, num_entries); + /* Set message to current position in the buffer. */ + message = (t8_gloidx_t *) sc_array_index_int (&send_buffer, current_pos_in_send_buffer); + current_pos_in_send_buffer += num_entries; + /* Send the global id of the first tree of this process */ + message[0] = global_id_of_first_tree; + /* The index in the tree is the index of the element minus the offset of the tree. */ + message[1] = first_element_in_tree_index_of_current_proc; + + const int mpiret + = sc_MPI_Isend (message, num_entries, T8_MPI_GLOIDX, iproc, T8_MPI_CMESH_UNIFORM_BOUNDS_START, comm, request); + SC_CHECK_MPI (mpiret); + t8_debugf ("Sending start message (%li, %li) to %li (global num el %li)\n", message[0], message[1], iproc, + data->global_num_elements); + + T8_ASSERT (proc_is_empty || (0 <= message[0] && message[0] < t8_cmesh_get_num_trees (cmesh))); + T8_ASSERT (proc_is_empty || (0 <= message[1] && message[1] < data->global_num_elements)); + T8_ASSERT (!(proc_is_empty && message[0] != -1)); + T8_ASSERT (!(proc_is_empty && message[1] != -1)); + } + else { /* We are the current proc, so we just copy the data. */ + *first_local_tree = global_id_of_first_tree; + if (first_element_in_tree_index_of_current_proc > 0) { + /* The first tree is shared */ + if (first_tree_shared != NULL) { + *first_tree_shared = 1; + } + } + else { + if (first_tree_shared != NULL) { + *first_tree_shared = 0; + } + } + if (child_in_tree_begin != NULL) { + *child_in_tree_begin = first_element_in_tree_index_of_current_proc; + } + /* We do not expect this message from another proc */ + expect_start_message = 0; +#ifdef T8_ENABLE_DEBUG + (*num_received_start_messages)++; +#endif + } + return expect_start_message; +} + void t8_cmesh_uniform_bounds_from_partition (t8_cmesh_t cmesh, t8_gloidx_t local_num_children, const int level, const t8_scheme_cxx_t *scheme, t8_gloidx_t *first_local_tree, @@ -626,6 +695,16 @@ t8_cmesh_uniform_bounds_from_partition (t8_cmesh_t cmesh, t8_gloidx_t local_num_ /* Post the start message, we send the first tree id of the process * and (if desired) the index of the first element in this tree. */ if (send_start_message) { + expect_start_message = t8_cmesh_uniform_bounds_send_start ( + cmesh, iproc, proc_is_empty, first_puretree_of_current_proc, first_tree_shared_shift, send_requests, + send_buffer, first_local_tree, first_element_in_tree_index_of_current_proc, first_tree_shared, + child_in_tree_begin, &data, comm, +#if T8_ENABLE_DEBUG + &num_received_start_messages + +#endif + ); +#if 0 const t8_gloidx_t global_id_of_first_tree = proc_is_empty ? -1 @@ -680,6 +759,7 @@ t8_cmesh_uniform_bounds_from_partition (t8_cmesh_t cmesh, t8_gloidx_t local_num_ num_received_start_messages++; #endif } +#endif } /* End sending of start message */ t8_debugf ("End message: %i\n", send_end_message); From 6ab9dfa7bc594fc37bae4a83bee771c9327493ab Mon Sep 17 00:00:00 2001 From: David Knapp Date: Thu, 15 Feb 2024 10:15:46 +0100 Subject: [PATCH 43/43] update cmesh_bounds_send_start --- src/t8_cmesh/t8_cmesh_cxx.cxx | 104 ++++++++-------------------------- 1 file changed, 23 insertions(+), 81 deletions(-) diff --git a/src/t8_cmesh/t8_cmesh_cxx.cxx b/src/t8_cmesh/t8_cmesh_cxx.cxx index acabf980e3..1a6db752c8 100644 --- a/src/t8_cmesh/t8_cmesh_cxx.cxx +++ b/src/t8_cmesh/t8_cmesh_cxx.cxx @@ -362,34 +362,33 @@ t8_cmesh_uniform_bounds_from_unpartioned (t8_cmesh_t cmesh, const t8_gloidx_t lo return; } -int -t8_cmesh_uniform_bounds_send_start (t8_cmesh_t cmesh, const long int iproc, const int proc_is_empty, - const t8_locidx_t first_puretree_of_current_proc, const int first_tree_shared_shift, - sc_array_t send_requests, sc_array_t send_buffer, t8_gloidx_t *first_local_tree, - const t8_gloidx_t first_element_in_tree_index_of_current_proc, - int8_t *first_tree_shared, t8_gloidx_t *child_in_tree_begin, - t8_cmesh_partition_query_t *data, sc_MPI_Comm comm, - -#if T8_ENABLE_DEBUG - int *num_received_start_messages +void +t8_cmesh_bounds_send_start (t8_cmesh_t cmesh, const int proc_is_empty, const t8_locidx_t first_puretree_of_current_proc, + const int first_tree_shared_shift, const t8_gloidx_t iproc, sc_array *send_requests, + sc_array *send_buffer, int *current_pos_in_send_buffer, + const t8_gloidx_t first_element_in_tree_index_of_current_proc, + t8_gloidx_t *first_local_tree, int8_t *first_tree_shared, t8_gloidx_t *child_in_tree_begin, + int *expect_start_message, t8_cmesh_partition_query_t *data, sc_MPI_Comm comm +#ifdef T8_ENABLE_DEBUG + , + int *num_received_start_messages #endif ) { const t8_gloidx_t global_id_of_first_tree = proc_is_empty ? -1 : first_puretree_of_current_proc + t8_cmesh_get_first_treeid (cmesh) + first_tree_shared_shift; - int current_pos_in_send_buffer = 0; - int expect_start_message = 1; + if (iproc != cmesh->mpirank) { const int num_entries = 2; t8_gloidx_t *message; /* Allocate a new request */ - sc_MPI_Request *request = (sc_MPI_Request *) sc_array_push (&send_requests); + sc_MPI_Request *request = (sc_MPI_Request *) sc_array_push (send_requests); /* Grow total send buffer */ - sc_array_push_count (&send_buffer, num_entries); + sc_array_push_count (send_buffer, num_entries); /* Set message to current position in the buffer. */ - message = (t8_gloidx_t *) sc_array_index_int (&send_buffer, current_pos_in_send_buffer); - current_pos_in_send_buffer += num_entries; + message = (t8_gloidx_t *) sc_array_index_int (send_buffer, *current_pos_in_send_buffer); + *current_pos_in_send_buffer += num_entries; /* Send the global id of the first tree of this process */ message[0] = global_id_of_first_tree; /* The index in the tree is the index of the element minus the offset of the tree. */ @@ -423,12 +422,11 @@ t8_cmesh_uniform_bounds_send_start (t8_cmesh_t cmesh, const long int iproc, cons *child_in_tree_begin = first_element_in_tree_index_of_current_proc; } /* We do not expect this message from another proc */ - expect_start_message = 0; + *expect_start_message = 0; #ifdef T8_ENABLE_DEBUG (*num_received_start_messages)++; #endif } - return expect_start_message; } void @@ -695,71 +693,16 @@ t8_cmesh_uniform_bounds_from_partition (t8_cmesh_t cmesh, t8_gloidx_t local_num_ /* Post the start message, we send the first tree id of the process * and (if desired) the index of the first element in this tree. */ if (send_start_message) { - expect_start_message = t8_cmesh_uniform_bounds_send_start ( - cmesh, iproc, proc_is_empty, first_puretree_of_current_proc, first_tree_shared_shift, send_requests, - send_buffer, first_local_tree, first_element_in_tree_index_of_current_proc, first_tree_shared, - child_in_tree_begin, &data, comm, -#if T8_ENABLE_DEBUG - &num_received_start_messages + t8_cmesh_bounds_send_start (cmesh, proc_is_empty, first_puretree_of_current_proc, first_tree_shared_shift, + iproc, &send_requests, &send_buffer, ¤t_pos_in_send_buffer, + first_element_in_tree_index_of_current_proc, first_local_tree, first_tree_shared, + child_in_tree_begin, &expect_start_message, &data, comm +#ifdef T8_ENABLE_DEBUG + , + &num_received_start_messages #endif ); -#if 0 - const t8_gloidx_t global_id_of_first_tree - = proc_is_empty - ? -1 - : first_puretree_of_current_proc + t8_cmesh_get_first_treeid (cmesh) + first_tree_shared_shift; - - if (iproc != cmesh->mpirank) { - const int num_entries = 2; - t8_gloidx_t *message; - /* Allocate a new request */ - sc_MPI_Request *request = (sc_MPI_Request *) sc_array_push (&send_requests); - - /* Grow total send buffer */ - sc_array_push_count (&send_buffer, num_entries); - /* Set message to current position in the buffer. */ - message = (t8_gloidx_t *) sc_array_index_int (&send_buffer, current_pos_in_send_buffer); - current_pos_in_send_buffer += num_entries; - /* Send the global id of the first tree of this process */ - message[0] = global_id_of_first_tree; - /* The index in the tree is the index of the element minus the offset of the tree. */ - message[1] = first_element_in_tree_index_of_current_proc; - - const int mpiret = sc_MPI_Isend (message, num_entries, T8_MPI_GLOIDX, iproc, - T8_MPI_CMESH_UNIFORM_BOUNDS_START, comm, request); - SC_CHECK_MPI (mpiret); - t8_debugf ("Sending start message (%li, %li) to %li (global num el %li)\n", message[0], message[1], iproc, - data.global_num_elements); - - T8_ASSERT (proc_is_empty || (0 <= message[0] && message[0] < t8_cmesh_get_num_trees (cmesh))); - T8_ASSERT (proc_is_empty || (0 <= message[1] && message[1] < data.global_num_elements)); - T8_ASSERT (!(proc_is_empty && message[0] != -1)); - T8_ASSERT (!(proc_is_empty && message[1] != -1)); - } - else { /* We are the current proc, so we just copy the data. */ - *first_local_tree = global_id_of_first_tree; - if (first_element_in_tree_index_of_current_proc > 0) { - /* The first tree is shared */ - if (first_tree_shared != NULL) { - *first_tree_shared = 1; - } - } - else { - if (first_tree_shared != NULL) { - *first_tree_shared = 0; - } - } - if (child_in_tree_begin != NULL) { - *child_in_tree_begin = first_element_in_tree_index_of_current_proc; - } - /* We do not expect this message from another proc */ - expect_start_message = 0; -#ifdef T8_ENABLE_DEBUG - num_received_start_messages++; -#endif - } -#endif } /* End sending of start message */ t8_debugf ("End message: %i\n", send_end_message); @@ -926,7 +869,6 @@ t8_cmesh_uniform_bounds_for_irregular_refinement (const t8_cmesh_t cmesh, const sc_MPI_Comm comm) { T8_ASSERT (cmesh != NULL); - /* TODO: Clean up size_t and gloidx_t data types, ensure that each variables has the * matching type. */