Merge branch 'documentation' into 'derivgrind'

Improve documentation and fix compiler warnings

See merge request aehle/valgrind!34

Author: maxaehle

.github/workflows/regression.yml

@@ -21,20 +21,20 @@ jobs:
       - name: Configure and build
         run: |
           ./autogen.sh
-          ./configure --prefix=$PWD/install --enable-fortran --enable-python3 --enable-mlframeworks SHADOW_LAYERS_64=16,16,16,16
+          ./configure --prefix=$PWD/install --enable-fortran --enable-python --enable-mlframeworks SHADOW_LAYERS_64=16,16,16,16
           make install
       - name: Forward-mode regression tests
         run: python3 derivgrind/diff_tests/run_tests.py dot* --prefix=$PWD/install 2>&1 | tee forward_log 
       - name: Upload log of forward-mode regression tests
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: forward_log
           path: forward_log
           retention-days: 7
       - name: Reverse-mode regression tests
         run: python3 derivgrind/diff_tests/run_tests.py bar* --prefix=$PWD/install 2>&1 | tee reverse_log
       - name: Upload log of reverse-mode regression tests
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: reverse_log
           path: reverse_log

README.md

@@ -8,7 +8,7 @@ It has been implemented in the [Valgrind](https://valgrind.org/)
 framework for building dynamic analysis tools. 
 
 For more information beyond this [README.md](README.md), you may have a look at our papers:
-- M. Aehle, J. Blühdorn, M. Sagebaum, N. R. Gauger: *Forward-Mode Automatic Differentiation of Compiled Programs*. [arXiv:2209.01895](https://arxiv.org/abs/2209.01895), 2022.
+- M. Aehle, J. Blühdorn, M. Sagebaum, N. R. Gauger: *Forward-Mode Automatic Differentiation of Compiled Programs*. [ACM Transactions on Mathematical Software](https://doi.org/10.1145/3716309), 2025.
 - M. Aehle, J. Blühdorn, M. Sagebaum, N. R. Gauger: *Reverse-Mode Automatic Differentiation of Compiled Programs*. [arXiv:2212.13760](https://arxiv.org/abs/2212.13760), 2022.
 
 ## Building Derivgrind
@@ -19,11 +19,10 @@ this file to reproducibly obtain an environment that contains all dependencies.
 Clone this repository with `git clone --recursive`, and run the following commands in the root directory: 
 ```bash
 ./autogen.sh
-./configure --prefix=$PWD/install --enable-python --enable-fortran
+./configure --prefix=$PWD/install
 make install
 ```
-The flags `--enable-python` and `--enable-fortran` enable wrappers for client request (see below), which
-are necessary to run all unit tests successfully. For many applications, you may however leave them out.
+You may supply arguments `--enable-python --enable-fortran` to the `./configure` call to build client request wrappers for Python and Fortran. These wrappers allow to declare AD inputs and outputs from Python and Fortran code, and are required to run Derivgrind's testcases for these languages. Building these wrappers requires Python C extension headers (e.g. `python3-dev`) and a Fortran compiler (e.g. `f77`), respectively, to be installed on your system.
 
 ## Running Testcases
 
@@ -34,7 +33,7 @@ python3 run_tests.py dot_amd64_gcc_double_addition
 The names of the unit tests are composed of an AD mode, architecture, language/compiler, floating-point type and 
 arithmetic formula. You may use `*` as a wildcard to run several tests at once. You may specify the 
 Derivgrind installation directory with `--prefix=path`. Specify a directory with `--tempdir=path` if
-you want to inspect the temporary files created by Derivgrind and the test system.
+you want to inspect the temporary files created by Derivgrind and the test system. Derivgrind must have been configured with `--enable-python` and `--enable-fortran` to run the Python and Fortran tests.
 
 ## Differentiating a Simple C++ Program in Forward Mode
 Compile a simple C++ "client" program from 
@@ -141,7 +140,7 @@ Placing the directory on a ramdisk like `/dev/shm/` might speed the recording up
   in manifold ways. For example, a bitwise logical "and" can be used to set the
   sign bit to zero, and thereby compute the absolute value. Derivgrind recognizes only
   the most important of these constructs. More details can be found in our 
-  [forward-mode paper](https://arxiv.org/abs/2209.01895). Generally, avoid direct manipulation 
+  [forward-mode paper](https://doi.org/10.1145/3716309). Generally, avoid direct manipulation 
   of a floating-point number's binary representation in your program, and avoid the 
   differentiation of highly optimized numerical libraries.
 - While Valgrind supports many more platforms, only X86/Linux and AMD64/Linux 

derivgrind/bar/dg_bar.c

@@ -58,8 +58,10 @@ V256* dg_bar_shadow_mem_buffer;
 
 #define dg_rounding_mode IRExpr_Const(IRConst_U32(0))
 
-/* --- Define ExpressionHandling. --- */
-// some functions are not static because the trick-instrumention also needs them
+/* --- Define tape-recording ExpressionHandling. --- */
+/* See dg_expressionhandling.h for documentation of dg_bar_wrtmp, dg_bar_rdtmp etc. */
+
+/* some functions are not declared static because the trick-instrumention reuses them */
 
 void dg_bar_wrtmp(DiffEnv* diffenv, IRTemp temp, void* expr){
   IRStmt* spLo = IRStmt_WrTmp(temp+diffenv->tmp_offset, ((IRExpr**)expr)[0]);
@@ -106,15 +108,15 @@ void* dg_bar_geti(DiffEnv* diffenv, Int offset, IRType type, IRRegArray* descr,
  *  \param addr Address for memory location whose shadow should be written to.
  *  \param size Number of bytes per layer to be copied.
  */
-void dg_bar_x86g_amd64g_dirtyhelper_store(Addr addr, ULong size){
+static void dg_bar_x86g_amd64g_dirtyhelper_store(Addr addr, ULong size){
   dg_bar_shadowSet((void*)addr,dg_bar_shadow_mem_buffer,dg_bar_shadow_mem_buffer+1,size);
 }
 
 /*! Dirty call to copy shadow data from shadow memory into buffer.
  *  \param addr Address for memory location whose shadow should be read from.
  *  \param size Number of bytes per layer to be copied.
  */
-void dg_bar_x86g_amd64g_dirtyhelper_load(Addr addr, ULong size){
+static void dg_bar_x86g_amd64g_dirtyhelper_load(Addr addr, ULong size){
   dg_bar_shadowGet((void*)addr,dg_bar_shadow_mem_buffer,dg_bar_shadow_mem_buffer+1,size);
 }
 
@@ -166,11 +168,11 @@ void* dg_bar_load(DiffEnv* diffenv, IRExpr* addr, IRType type){
  *  It just writes the lower 4 bytes of the index to the beginning
  *  of the 80-bit blocks in the lower layer of shadow memory.
  */
-void dg_bar_x86g_amd64g_dirtyhelper_storeF80le_Lo ( Addr addrU, ULong i64 )
+static void dg_bar_x86g_amd64g_dirtyhelper_storeF80le_Lo ( Addr addrU, ULong i64 )
 {
   dg_bar_shadowSet((void*)addrU,(void*)&i64,NULL,4);
 }
-void dg_bar_x86g_amd64g_dirtyhelper_storeF80le_Hi ( Addr addrU, ULong i64 )
+static void dg_bar_x86g_amd64g_dirtyhelper_storeF80le_Hi ( Addr addrU, ULong i64 )
 {
   dg_bar_shadowSet((void*)addrU,NULL,(void*)&i64,4);
 }
@@ -180,20 +182,20 @@ void dg_bar_x86g_amd64g_dirtyhelper_storeF80le_Hi ( Addr addrU, ULong i64 )
  *  It just reads the lower 4 bytes of the index from the beginning
  *  of the 80-bit blocks in the lower layer of shadow memory.
  */
-ULong dg_bar_x86g_amd64g_dirtyhelper_loadF80le_Lo ( Addr addrU )
+static ULong dg_bar_x86g_amd64g_dirtyhelper_loadF80le_Lo ( Addr addrU )
 {
   ULong i64_Lo, i64_Hi;
   dg_bar_shadowGet((void*)addrU, (void*)&i64_Lo, (void*)&i64_Hi, 4);
   return i64_Lo;
 }
-ULong dg_bar_x86g_amd64g_dirtyhelper_loadF80le_Hi ( Addr addrU )
+static ULong dg_bar_x86g_amd64g_dirtyhelper_loadF80le_Hi ( Addr addrU )
 {
   ULong i64_Lo, i64_Hi;
   dg_bar_shadowGet((void*)addrU, (void*)&i64_Lo, (void*)&i64_Hi, 4);
   return i64_Hi;
 }
 
-void dg_bar_dirty_storeF80le(DiffEnv* diffenv, IRExpr* addr, void* expr){
+static void dg_bar_dirty_storeF80le(DiffEnv* diffenv, IRExpr* addr, void* expr){
   IRDirty* ddLo = unsafeIRDirty_0_N(
         0, "dg_bar_x86g_amd64g_dirtyhelper_storeF80le_Lo",
         &dg_bar_x86g_amd64g_dirtyhelper_storeF80le_Lo,
@@ -206,7 +208,7 @@ void dg_bar_dirty_storeF80le(DiffEnv* diffenv, IRExpr* addr, void* expr){
   addStmtToIRSB(diffenv->sb_out, IRStmt_Dirty(ddHi));
 }
 
-void dg_bar_dirty_loadF80le(DiffEnv* diffenv, IRExpr* addr, IRTemp temp){
+static void dg_bar_dirty_loadF80le(DiffEnv* diffenv, IRExpr* addr, IRTemp temp){
   IRDirty* ddLo = unsafeIRDirty_1_N(
         temp+diffenv->tmp_offset,
         0, "dg_bar_x86g_amd64g_dirtyhelper_loadF80le_Lo",
@@ -268,7 +270,7 @@ void* dg_bar_ite(DiffEnv* diffenv, IRExpr* cond, void* dtrue, void* dfalse){
   return (void*)mkIRExprVec_2(exLo,exHi);
 }
 
-ULong dg_bar_writeToTape_call(ULong index1Lo, ULong index1Hi, ULong index2Lo, ULong index2Hi, ULong diff1, ULong diff2){
+static ULong dg_bar_writeToTape_call(ULong index1Lo, ULong index1Hi, ULong index2Lo, ULong index2Hi, ULong diff1, ULong diff2){
   // assemble 8-byte indices from 4-byte beginnings in both shadow layers
   UInt index1[2], index2[2];
   index1[0] = *(UInt*)&index1Lo;
@@ -279,7 +281,7 @@ ULong dg_bar_writeToTape_call(ULong index1Lo, ULong index1Hi, ULong index2Lo, UL
   return returnindex;
 }
 
-void dg_bar_writeToTape_value_call(ULong value, ULong index){
+static void dg_bar_writeToTape_value_call(ULong value, ULong index){
   if(index!=0){
     valuesAddStatement(*(double*)&value);
   }
@@ -302,7 +304,7 @@ void dg_bar_writeToTape_value_call(ULong value, ULong index){
  *   new index assigned to the result.
  *
  */
-IRExpr** dg_bar_writeToTape(DiffEnv* diffenv, IRExpr* index1Lo, IRExpr* index1Hi, IRExpr* index2Lo, IRExpr* index2Hi, IRExpr* diff1, IRExpr* diff2, IRExpr* value){
+static IRExpr** dg_bar_writeToTape(DiffEnv* diffenv, IRExpr* index1Lo, IRExpr* index1Hi, IRExpr* index2Lo, IRExpr* index2Hi, IRExpr* diff1, IRExpr* diff2, IRExpr* value){
   IRTemp returnindex = newIRTemp(diffenv->sb_out->tyenv,Ity_I64);
   IRDirty* dd = unsafeIRDirty_1_N(
         returnindex,
@@ -328,7 +330,7 @@ IRExpr** dg_bar_writeToTape(DiffEnv* diffenv, IRExpr* index1Lo, IRExpr* index1Hi
   return mkIRExprVec_2(exLo,exHi);
 }
 
-void* dg_bar_operation(DiffEnv* diffenv, IROp op,
+static void* dg_bar_operation(DiffEnv* diffenv, IROp op,
                          IRExpr* arg1, IRExpr* arg2, IRExpr* arg3, IRExpr* arg4,
                          void* i1, void* i2, void* i3, void* i4){
   IRExpr *i1Lo=NULL, *i1Hi=NULL, *i2Lo=NULL, *i2Hi=NULL, *i3Lo=NULL, *i3Hi=NULL, *i4Lo=NULL, *i4Hi=NULL;
@@ -358,7 +360,7 @@ void* dg_bar_operation(DiffEnv* diffenv, IROp op,
   }
 }
 
-void* dg_bar_ccall(DiffEnv* diffenv, IRCallee* cee, IRType retty, IRExpr** args, void** modified_args){
+static void* dg_bar_ccall(DiffEnv* diffenv, IRCallee* cee, IRType retty, IRExpr** args, void** modified_args){
   return NULL;
 }
 

derivgrind/bar/dg_bar.h

@@ -53,20 +53,19 @@ void dg_bar_initialize(void);
  */
 void dg_bar_finalize(void);
 
-// Declarations of the tool functions are required by the bit-trick-finding instrumentation.
-#ifdef DG_BAR_H_INCLUDE_TOOL_FUNCTIONS
-  static void dg_bar_wrtmp(DiffEnv* diffenv, IRTemp temp, void* expr);
-  static void* dg_bar_rdtmp(DiffEnv* diffenv, IRTemp temp);
-  static void dg_bar_puti(DiffEnv* diffenv, Int offset, void* expr, IRRegArray* descr, IRExpr* ix);
-  static void* dg_bar_geti(DiffEnv* diffenv, Int offset, IRType type, IRRegArray* descr, IRExpr* ix);
-  static void dg_bar_store(DiffEnv* diffenv, IRExpr* addr, void* expr, IRExpr* guard);
-  static void* dg_bar_load(DiffEnv* diffenv, IRExpr* addr, IRType type);
-  static void* dg_bar_constant(DiffEnv* diffenv, IRConstTag type);
-  static void* dg_bar_default_(DiffEnv* diffenv, IRType type);
-  static IRExpr* dg_bar_compare(DiffEnv* diffenv, void* arg1, void* arg2);
-  static void* dg_bar_ite(DiffEnv* diffenv, IRExpr* cond, void* dtrue, void* dfalse);
-  extern V256* dg_bar_shadow_mem_buffer;
-#endif
+// some tool functions are reused by the bit-trick-finding instrumentation, thus declared here
+void dg_bar_wrtmp(DiffEnv* diffenv, IRTemp temp, void* expr);
+void* dg_bar_rdtmp(DiffEnv* diffenv, IRTemp temp);
+void dg_bar_puti(DiffEnv* diffenv, Int offset, void* expr, IRRegArray* descr, IRExpr* ix);
+void* dg_bar_geti(DiffEnv* diffenv, Int offset, IRType type, IRRegArray* descr, IRExpr* ix);
+void dg_bar_store(DiffEnv* diffenv, IRExpr* addr, void* expr, IRExpr* guard);
+void* dg_bar_load(DiffEnv* diffenv, IRExpr* addr, IRType type);
+void* dg_bar_constant(DiffEnv* diffenv, IRConstTag type);
+void* dg_bar_default_(DiffEnv* diffenv, IRType type);
+IRExpr* dg_bar_compare(DiffEnv* diffenv, void* arg1, void* arg2);
+void* dg_bar_ite(DiffEnv* diffenv, IRExpr* cond, void* dtrue, void* dfalse);
+extern V256* dg_bar_shadow_mem_buffer;
+
 
 
 #endif // DG_BAR_H

derivgrind/bar/dg_bar_shadow.cpp

@@ -33,6 +33,7 @@
 #include "externals/flexible-shadow/flexible-shadow-valgrindstdlib.hpp"
 #include <pub_tool_libcbase.h>
 #include "dg_utils.h"
+#include "dg_bar_shadow.h"
 
 #ifndef SHADOW_LAYERS_32
   #define SHADOW_LAYERS_32 18,14
@@ -48,8 +49,11 @@
 #endif
 
 struct ShadowLeafBar {
+  #pragma GCC diagnostic push
+  #pragma GCC diagnostic ignored "-Wunused-value" // see dg_dot_shadow.cpp
   UChar data_Lo[1ul<<(SHADOW_LAYERS)];
   UChar data_Hi[1ul<<(SHADOW_LAYERS)];
+  #pragma GCC diagnostic pop
   static ShadowLeafBar distinguished;
 };
 ShadowLeafBar ShadowLeafBar::distinguished;
@@ -58,7 +62,7 @@ using ShadowMapTypeBar = ShadowMap<Addr,ShadowLeafBar,ValgrindStandardLibraryInt
 
 ShadowMapTypeBar* sm_bar2;
 
-extern "C" void dg_bar_shadowGet(void* sm_address, void* real_address_Lo, void* real_address_Hi, int size){
+extern "C" void dg_bar_shadowGet(void* sm_address, void* real_address_Lo, void* real_address_Hi, unsigned int size){
   ShadowLeafBar* leaf = sm_bar2->leaf_for_read((Addr)sm_address);
   Addr contiguousSize = sm_bar2->contiguousElements((Addr)sm_address);
   ULong index = sm_bar2->index((Addr)sm_address);
@@ -80,7 +84,7 @@ extern "C" void dg_bar_shadowGet(void* sm_address, void* real_address_Lo, void*
   }
 }
 
-extern "C" void dg_bar_shadowSet(void* sm_address, void* real_address_Lo, void* real_address_Hi, int size){
+extern "C" void dg_bar_shadowSet(void* sm_address, void* real_address_Lo, void* real_address_Hi, unsigned int size){
   ShadowLeafBar* leaf = sm_bar2->leaf_for_write((Addr)sm_address);
   Addr contiguousSize = sm_bar2->contiguousElements((Addr)sm_address);
   ULong index = sm_bar2->index((Addr)sm_address);
@@ -103,10 +107,13 @@ extern "C" void dg_bar_shadowSet(void* sm_address, void* real_address_Lo, void*
 }
 
 extern "C" void dg_bar_shadowInit(){
+  #pragma GCC diagnostic push
+  #pragma GCC diagnostic ignored "-Wunused-value"
   for(Addr i=0; i<(1ul<<(SHADOW_LAYERS)); i++){
     ShadowLeafBar::distinguished.data_Lo[i] = 0;
     ShadowLeafBar::distinguished.data_Hi[i] = 0;
   }
+  #pragma GCC diagnostic pop
   sm_bar2 = (ShadowMapTypeBar*)VG_(malloc)("Space for primary map",sizeof(ShadowMapTypeBar));
   ShadowMapTypeBar::constructAt(sm_bar2);
 }

derivgrind/bar/dg_bar_shadow.h

@@ -37,8 +37,8 @@ extern "C" {
 #endif
 
 /*! */
-void dg_bar_shadowGet(void* sm_address, void* real_address_Lo, void* real_address_Hi, int size);
-void dg_bar_shadowSet(void* sm_address, void* real_address, void* real_address_Hi, int size);
+void dg_bar_shadowGet(void* sm_address, void* real_address_Lo, void* real_address_Hi, unsigned int size);
+void dg_bar_shadowSet(void* sm_address, void* real_address, void* real_address_Hi, unsigned int size);
 void dg_bar_shadowInit(void);
 void dg_bar_shadowFini(void);
 

derivgrind/derivgrind-recording.h

@@ -52,11 +52,36 @@
  * input and output variables in the client code.
  */
 
-static unsigned long long dg_indextmp, dg_indextmp2;
-static double dg_valtmp;
-static unsigned long long const dg_zero = 0;
-static double const dg_one = 1.;
-static unsigned long long const dg_ones = 0xfffffffffffffffful;
+#ifdef __GNUC__
+#define DG_UNUSED __attribute__((unused))
+#else
+#define DG_UNUSED
+#endif
+
+DG_UNUSED static unsigned long long* dg_indextmp(void){
+  static unsigned long long indextmp = 0;
+  return &indextmp;
+}
+DG_UNUSED static unsigned long long* dg_indextmp2(void){
+  static unsigned long long indextmp2 = 0;
+  return &indextmp2;
+}
+DG_UNUSED static double* dg_valtmp(void){
+  static double valtmp = 0.;
+  return &valtmp;
+}
+DG_UNUSED static unsigned long long const* dg_zero(void){
+  static unsigned long long const zero = 0;
+  return &zero;
+}
+DG_UNUSED static double const* dg_one(void){
+  static double const one = 1.;
+  return &one;
+}
+DG_UNUSED static unsigned long long const* dg_ones(void){
+  static unsigned long long const ones = 0xfffffffffffffffful;
+  return &ones;
+}
 
 /*! Mark variable as AD input and assign new 8-byte index to it.
  * 
@@ -67,11 +92,11 @@ static unsigned long long const dg_ones = 0xfffffffffffffffful;
  * to print the index from your client code. Or use the DG_INPUTF macro to
  * write it directly into a file.
  */
-#define DG_INPUT(var) (dg_valtmp=(double)(var),DG_NEW_INDEX_NOACTIVITYANALYSIS(&dg_zero,&dg_zero,&dg_zero,&dg_zero,&dg_indextmp,&dg_valtmp), DG_SET_INDEX(&var,&dg_indextmp), dg_indextmp)
+#define DG_INPUT(var) (*dg_valtmp()=(double)(var),DG_NEW_INDEX_NOACTIVITYANALYSIS(dg_zero(),dg_zero(),dg_zero(),dg_zero(),dg_indextmp(),dg_valtmp()), DG_SET_INDEX(&var,dg_indextmp()), *dg_indextmp())
 
 /*! Mark variable as AD input, assign new 8-byte index, and dump the index into a file.
  */
-#define DG_INPUTF(var) { dg_indextmp2 = DG_INPUT(var); DG_INDEX_TO_FILE(DG_INDEXFILE_INPUT, &dg_indextmp2); }
+#define DG_INPUTF(var) { *dg_indextmp2() = DG_INPUT(var); DG_INDEX_TO_FILE(DG_INDEXFILE_INPUT, dg_indextmp2()); }
 
 /*! Mark variable as AD output and retrieve its 8-byte index.
  * 
@@ -82,14 +107,14 @@ static unsigned long long const dg_ones = 0xfffffffffffffffful;
  * to print the index from your client code. Or use the DG_OUTPUTF macro to
  * write it directly into a file.
  */
-#define DG_OUTPUT(var) (dg_valtmp=(double)(var),DG_GET_INDEX(&var,&dg_indextmp2), DG_NEW_INDEX_NOACTIVITYANALYSIS(&dg_indextmp2,&dg_zero,&dg_one,&dg_zero,&dg_indextmp,&dg_valtmp), dg_indextmp)
+#define DG_OUTPUT(var) (*dg_valtmp()=(double)(var),DG_GET_INDEX(&var,dg_indextmp2()), DG_NEW_INDEX_NOACTIVITYANALYSIS(dg_indextmp2(),dg_zero(),dg_one(),dg_zero(),dg_indextmp(),dg_valtmp()), *dg_indextmp())
 
 /*! Mark variable as AD output, retrieve its 8-byte index, and dump the index into a file.
  */
-#define DG_OUTPUTF(var) { dg_indextmp2 = DG_OUTPUT(var); DG_INDEX_TO_FILE(DG_INDEXFILE_OUTPUT, &dg_indextmp2); }
+#define DG_OUTPUTF(var) { *dg_indextmp2() = DG_OUTPUT(var); DG_INDEX_TO_FILE(DG_INDEXFILE_OUTPUT, dg_indextmp2()); }
 
 /*! Mark variable as active floating-point number for the bit-trick finder.
  */
-#define DG_MARK_FLOAT(var) {DG_SET_FLAGS(&var, &dg_ones, (void*)0, sizeof(var));}
+#define DG_MARK_FLOAT(var) {DG_SET_FLAGS(&var, dg_ones(), (void*)0, sizeof(var));}
 
 #endif