Fix Reference-LAPACK#1134 - Bug in Reference LAPACKE’s cunmlq and zunmlq for row-major layout and side right

Bug reported by @neil-lindquist. Thanks Neil!

On line 93 of both lapacke_cunmlq_work.c and lapacke_zunmlq_work.c, m is passed
to LAPACKE_zge_trans as the 2nd dimension. However, the 2nd dimension should be
r to handle side right correctly. All the other versions of {or,un}m{qr,lq} seem
to be correct.

Also line 52 of lapacke_cunmlq.c and lapacke_zunmlq.c have the same issue: m is
used for the column dimension instead of r. (Those files also do not currently
compute r). The real valued cases have the correct behavior.

Author: langou

LAPACKE/src/lapacke_cunmlq.c

@@ -49,7 +49,8 @@ lapack_int API_SUFFIX(LAPACKE_cunmlq)( int matrix_layout, char side, char trans,
 #ifndef LAPACK_DISABLE_NAN_CHECK
     if( LAPACKE_get_nancheck() ) {
         /* Optionally check input matrices for NaNs */
-        if( API_SUFFIX(LAPACKE_cge_nancheck)( matrix_layout, k, m, a, lda ) ) {
+        lapack_int r = API_SUFFIX(LAPACKE_lsame)( side, 'l' ) ? m : n;
+        if( API_SUFFIX(LAPACKE_cge_nancheck)( matrix_layout, k, r, a, lda ) ) {
             return -7;
         }
         if( API_SUFFIX(LAPACKE_cge_nancheck)( matrix_layout, m, n, c, ldc ) ) {

LAPACKE/src/lapacke_cunmlq_work.c

@@ -90,7 +90,7 @@ lapack_int API_SUFFIX(LAPACKE_cunmlq_work)( int matrix_layout, char side, char t
             goto exit_level_1;
         }
         /* Transpose input matrices */
-        API_SUFFIX(LAPACKE_cge_trans)( matrix_layout, k, m, a, lda, a_t, lda_t );
+        API_SUFFIX(LAPACKE_cge_trans)( matrix_layout, k, r, a, lda, a_t, lda_t );
         API_SUFFIX(LAPACKE_cge_trans)( matrix_layout, m, n, c, ldc, c_t, ldc_t );
         /* Call LAPACK function and adjust info */
         LAPACK_cunmlq( &side, &trans, &m, &n, &k, a_t, &lda_t, tau, c_t, &ldc_t,

LAPACKE/src/lapacke_zunmlq.c

@@ -49,7 +49,8 @@ lapack_int API_SUFFIX(LAPACKE_zunmlq)( int matrix_layout, char side, char trans,
 #ifndef LAPACK_DISABLE_NAN_CHECK
     if( LAPACKE_get_nancheck() ) {
         /* Optionally check input matrices for NaNs */
-        if( API_SUFFIX(LAPACKE_zge_nancheck)( matrix_layout, k, m, a, lda ) ) {
+        lapack_int r = API_SUFFIX(LAPACKE_lsame)( side, 'l' ) ? m : n;
+        if( API_SUFFIX(LAPACKE_zge_nancheck)( matrix_layout, k, r, a, lda ) ) {
             return -7;
         }
         if( API_SUFFIX(LAPACKE_zge_nancheck)( matrix_layout, m, n, c, ldc ) ) {

LAPACKE/src/lapacke_zunmlq_work.c

@@ -90,7 +90,7 @@ lapack_int API_SUFFIX(LAPACKE_zunmlq_work)( int matrix_layout, char side, char t
             goto exit_level_1;
         }
         /* Transpose input matrices */
-        API_SUFFIX(LAPACKE_zge_trans)( matrix_layout, k, m, a, lda, a_t, lda_t );
+        API_SUFFIX(LAPACKE_zge_trans)( matrix_layout, k, r, a, lda, a_t, lda_t );
         API_SUFFIX(LAPACKE_zge_trans)( matrix_layout, m, n, c, ldc, c_t, ldc_t );
         /* Call LAPACK function and adjust info */
         LAPACK_zunmlq( &side, &trans, &m, &n, &k, a_t, &lda_t, tau, c_t, &ldc_t,