From 5d45866df884e14504372c2856f60ff5111e9e7c Mon Sep 17 00:00:00 2001
From: Corentin Prigent <corentin.prigent@u-bordeaux.fr>
Date: Thu, 16 May 2024 15:43:04 +0200
Subject: [PATCH] ignore MG_OLDPARBDY tag in MMG5_coltet and MMG5_coltetlag to
 ensure validity of if statements using integer comparaison

---
 src/mmg3d/mmg3d1.c | 25 ++++++++++++++++++-------
 src/mmg3d/mmg3d3.c |  5 ++++-
 2 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/src/mmg3d/mmg3d1.c b/src/mmg3d/mmg3d1.c
index a2c27c29d..7232ad096 100644
--- a/src/mmg3d/mmg3d1.c
+++ b/src/mmg3d/mmg3d1.c
@@ -899,7 +899,7 @@ static MMG5_int MMG5_coltet(MMG5_pMesh mesh,MMG5_pSol met,int8_t typchk) {
   MMG5_int        base,k,nc,nnm,lists[MMG3D_LMAX+2],refmin,refplus;
   int64_t         list[MMG3D_LMAX+2];
   int             l,kk,isloc,ifac1;
-  int16_t         tag,isnm,isnmint;
+  int16_t         tag,tag0,tag1,isnm,isnmint;
   int8_t          i,j,ip,iq;
   int             ier, bsret;   // function return values/error codes
 
@@ -938,9 +938,10 @@ static MMG5_int MMG5_coltet(MMG5_pMesh mesh,MMG5_pSol met,int8_t typchk) {
         }
         else {
           /* Ignore OLDPARBDY tag of p0 */
-          int16_t tag = p0->tag;
-          tag &= ~MG_OLDPARBDY;
-          if ( (tag > p1->tag) || (tag & MG_REQ) ) {
+          int16_t tag0 = p0->tag, tag1 = p1->tag;
+          tag0 &= ~MG_OLDPARBDY;
+          tag1 &= ~MG_OLDPARBDY;
+          if ( (tag0 > tag1) || (tag0 & MG_REQ) ) {
             /* Unable to merge edge */
             continue;
           }
@@ -956,8 +957,13 @@ static MMG5_int MMG5_coltet(MMG5_pMesh mesh,MMG5_pSol met,int8_t typchk) {
             tag = pxt->tag[MMG5_iarf[i][j]];
             isnm = (tag & MG_NOM);
             isnmint = ( p0->xp && mesh->xpoint[p0->xp].nnor );
+            /* Ignore MG_OLDPARBDY for tags: commit c51f5f4f86292bdb0e33adb07c21ed327b1e2ec0 (pull request #241)
+            updates edge tags in splits: MG_OLDPARBDY may appear in edge tags and not in corresponding points
+            which could cause a wrong treatment of the next if statement */
+            tag0 = p0->tag & ~MG_OLDPARBDY;
+            tag &= ~MG_OLDPARBDY;
 
-            if ( p0->tag > tag ) continue;
+            if ( tag0 > tag ) continue;
 
             /* Catch an exterior non manifold point by an external face */
             if ( isnm ) {
@@ -1096,7 +1102,10 @@ static MMG5_int MMG5_coltet(MMG5_pMesh mesh,MMG5_pSol met,int8_t typchk) {
             isnm = (tag & MG_NOM);
             isnmint = ( p0->xp && mesh->xpoint[p0->xp].nnor );
 
-            if ( p0->tag > tag ) continue;
+            tag0 = p0->tag & ~MG_OLDPARBDY;
+            tag &= ~MG_OLDPARBDY;
+
+            if ( tag0 > tag ) continue;
             if ( isnm ) {
               if ( isnmint ) {
                 assert( 0<=ip && ip<4 && "unexpected local index for vertex");
@@ -1130,7 +1139,9 @@ static MMG5_int MMG5_coltet(MMG5_pMesh mesh,MMG5_pSol met,int8_t typchk) {
         if ( pt->xt && (pxt->ftag[i] & MG_BDY) ) {
           tag = pxt->tag[MMG5_iarf[i][j]];
           tag |= MG_BDY;
-          if ( p0->tag > tag )  continue;
+          tag &= ~MG_OLDPARBDY;
+          tag0 &= ~MG_OLDPARBDY;
+          if ( tag0 > tag )  continue;
 
           isnm = ( tag & MG_NOM );
           isnmint = ( tag & MG_NOM ) ? ( p0->xp && mesh->xpoint[p0->xp].nnor ) : 0;
diff --git a/src/mmg3d/mmg3d3.c b/src/mmg3d/mmg3d3.c
index b69e196f4..23d247ce6 100644
--- a/src/mmg3d/mmg3d3.c
+++ b/src/mmg3d/mmg3d3.c
@@ -411,6 +411,7 @@ static MMG5_int MMG5_coltetlag(MMG5_pMesh mesh,MMG5_pSol met,int itdeg) {
   MMG5_int    k,nc,nnm,base;
   int         ier;
   int8_t      i,j,ip,iq,isnm;
+  int16_t     tag0, tag1;
 
   nc = nnm = 0;
   hmi2 = mesh->info.hmin*mesh->info.hmin;
@@ -435,9 +436,11 @@ static MMG5_int MMG5_coltetlag(MMG5_pMesh mesh,MMG5_pSol met,int itdeg) {
 
         p0 = &mesh->point[pt->v[ip]];
         p1 = &mesh->point[pt->v[iq]];
+        tag0 = p0->tag & ~MG_OLDPARBDY;
+        tag1 = p1->tag & ~MG_OLDPARBDY;
         if ( p0->flag == base )  continue;
         else if ( p0->tag & MG_BDY ) continue;
-        else if ( (p0->tag & MG_REQ) || (p0->tag > p1->tag) )  continue;
+        else if ( (p0->tag & MG_REQ) || (tag0 > tag1) )  continue;
 
         /* check length */
         ux = p1->c[0] - p0->c[0];