Merge pull request #3 from Durganshu/develop

Restructuring and bug fixes in Mitosrun and OpenMP hooks

Author: stepanvanecek

.gitignore

@@ -8,3 +8,5 @@ cscope.out
 build/
 cmake-build-debug/
 .idea/
+inst-dir/
+.vscode/

CMakeLists.txt

@@ -9,8 +9,10 @@ set(MITOS_MAJOR_VERSION 0)
 set(MITOS_MINOR_VERSION 8)
 set(MITOS_PATCH_VERSION 1)
 set(MITOS_VERSION ${MITOS_MAJOR_VERSION}.${MITOS_MINOR_VERSION}.${MITOS_PATCH_VERSION})
-set(IBS_TYPE "OFF" "IBS_FETCH" "IBS_OP" CACHE STRING "Enabling Instruction-Based Sampling")
 option(MITOSHOOKS_OPEN_MP "Enable Mitoshooks OpenMP-Routing" OFF)
+
+# IBS (AMD) and PEBS (Intel) related options
+set(IBS_TYPE "OFF" "IBS_FETCH" "IBS_OP" CACHE STRING "Enabling Instruction-Based Sampling")
 set_property(CACHE IBS_TYPE PROPERTY STRINGS
         "OFF" "IBS_FETCH" "IBS_OP")
 
@@ -21,13 +23,14 @@ set_property(CACHE IBS_SAMPLING_MODE PROPERTY STRINGS
 if ((IBS_TYPE STREQUAL "IBS_FETCH") OR (IBS_TYPE STREQUAL "IBS_OP"))
     # define sampling mode for IBS
     if (IBS_SAMPLING_MODE STREQUAL "ALL_ON")
-        message(STATUS "IBS Sampling Mode ALL_ON Selected: Monitor all cores")
+	    message(STATUS "IBS (AMD) Sampling Mode ALL_ON Selected: Monitor all cores")
         add_definitions(-DUSE_IBS_ALL_ON)
     elseif (IBS_SAMPLING_MODE STREQUAL "THREAD_MIGRATION")
-        message(STATUS "IBS Sampling Mode THREAD_MIGRATION Selected: Monitor one thread; Migrate sampler at runtime from one core to another")
+	    message(STATUS "IBS (AMD) Sampling Mode THREAD_MIGRATION Selected: Monitor one thread; Migrate sampler at runtime from one core to another")
         add_definitions(-DUSE_IBS_THREAD_MIGRATION)
-    else()
-        message(STATUS "IBS not set")
+    # if no IBS options are selected, use PEBS
+    else() 
+        message(STATUS "Using PEBS (Intel)")
     endif ()
 endif ()
 
@@ -37,8 +40,9 @@ if (IBS_TYPE STREQUAL "IBS_FETCH")
 elseif (IBS_TYPE STREQUAL "IBS_OP")
     message(STATUS "Switch IBS_OP On")
     add_definitions(-DUSE_IBS_OP)
-else()
-    message(STATUS "IBS not set")
+    # If no IBS sampling mode is set, use PEBS
+    else()
+    message(STATUS "Using PEBS (Intel)")
 endif ()
 
 
@@ -66,6 +70,7 @@ endif()
 # C-Defines for non-essential packages
 if(MPI_FOUND)
     add_definitions(-DUSE_MPI)
+    message(STATUS "Mitoshooks MPI Enabled")
 endif()
 
 
@@ -97,4 +102,4 @@ export(PACKAGE Mitos)
 
 # Install cmake export
 install(EXPORT mitos-targets DESTINATION lib/Mitos)
-install(FILES mitos-config.cmake DESTINATION lib/Mitos)
+install(FILES mitos-config.cmake DESTINATION lib/Mitos)

README.md

@@ -16,7 +16,7 @@ Mitos requires:
   sampling.  This originated in the 3.10 Linux kernel, but is backported
   to some versions of [RHEL6.6](https://www.redhat.com/promo/Red_Hat_Enterprise_Linux6/).
 
-* [Dyninst](http://www.dyninst.org) version 8.2 or higher.
+* [Dyninst](http://www.dyninst.org) version 12.3.0 or higher.
 
 * [hwloc](http://www.open-mpi.org/projects/hwloc/)
 
@@ -26,11 +26,11 @@ Mitos requires:
    `CMAKE_PREFIX_PATH` environment variable.
 
 2. Run the following commands from the root of the MemAxes source:
-   ```
-   mkdir build && cd build
-   cmake -DCMAKE_INSTALL_PREFIX=/path/to/install/location ..
-   make
-   make install
+   ```bash
+   $> mkdir build && cd build
+   $> cmake -DCMAKE_INSTALL_PREFIX=/path/to/install/location ..
+   $> make
+   $> make install
    ```
 
 ## Running
@@ -40,18 +40,25 @@ Mitos requires:
 1. Find the `mitosrun` command in the `bin` directory in the install
    directory.
 
-2. Run any binary with `mitosrun` like this to generate a folder of
-   mitos output data. For example:
+2. Make sure that the path to the installation location of `mitos` and `Dyninst` is added to the `LD_LIBRARY_PATH`. For example:
 
+   ```bash
+   $> export LD_LIBRARY_PATH=/path/to/mitos/install/location/lib/
+   $> export LD_LIBRARY_PATH=/path/to/dyninst/lib/:$LD_LIBRARY_PATH
    ```
-   mitosrun ./examples/matmul
+
+3. Run any binary with `mitosrun` like this to generate a folder of
+   mitos output data. For example:
+
+   ```bash
+   $> mitosrun ./examples/matmul
    ```
 
    The above command will run the matmul example and create a folder
    called mitos_###, where ### is the number of seconds since the
    epoch. The folder will contain:
 
-   ```
+   ```bash
    mitos_###/
       data/
          samples.csv
@@ -67,18 +74,20 @@ Mitos requires:
 
    `mitosrun` can also be fine-tuned with the following parameters:
 
-   ```
+   ```bash
    [options]:
        -b sample buffer size (default 4096)
        -p sample period (default 4000)
        -t sample latency threshold (default 10)
+       -s top folder of source code to copy
    ```
+   **See the section on `Source Code Attribution` below for more details on how to save the source code information with the samples collected while running your application.**
 
-## IBS Configuration
-1. Configure CMAKE with IBS depending on the chosen executable and configure environment variables if necessary:
+## IBS (AMD) Configuration
+The default installation of `mitos` will be configured for Intel based Precise Event Based Sampling (PEBS). Additionally, `mitos` supports AMD based Instruction Based Sampling (IBS). Configure CMAKE with IBS depending on the chosen executable and configure environment variables if necessary:
 * `IBS_TYPE` 
   * Use IBS_FETCH or IBS_OP depending on the profiling use case (requires AMD processor with IBS support)
-  * IBS is not supported on Intel processors, therefore set variable to OFF
+  * IBS is not supported on Intel processors. By default, the variable is set to OFF.
 * `IBS_SAMPLING_MODE`
   * Mitosrun (with or without OpenMP): `IBS_ALL_ON` or `IBS_THREAD_MIGRATION`
   * Mitoshooks with OpenMP: 
@@ -90,19 +99,63 @@ Mitos requires:
   * NOTE: `IBS_ALL_ON` might also work, but this sampling method is not recommended.
 
 ## Mitoshooks with OpenMP Usage
-1. Requirements
-   * Compiler with OMPT support such as Clang (OpenMP feature since version 5.0)
-2. CMake Configuration
-   * OpenMP found
-   * `MITOSHOOKS_OPEN_MP`: ON
-3. Compilation
-   * Compilation of Mitoshooks library
-   * Compilation of OpenMP-application
-4. Execution
-   * Configuration of environment variable `OMP_TOOL_LIBRARIES` that points to mitoshooks-library:
-       * e.g. for omp_example.cpp: OMP_TOOL_LIBRARIES=./../src/libmitoshooks.so
-   * Launch OpenMP-application
-     * ./omp_example
+### 1. Requirements
+
+   Compiler with OMPT support such as Clang (OpenMP feature since version 5.0) is required. if clang is not the default compiler:
+   
+   ```bash
+   $> export CC=/path/to/clang*
+   $> export CXX=/path/to/clang++*
+   ```
+### 2. CMake Configuration
+   
+   OpenMP (version 5.0 and later) must be installed. If OpenMP is not installed at the default loaction, make sure to specify the paths so that `cmake` finds it.
+   
+   Enable `MITOSHOOKS_OPEN_MP` option by `-DMITOSHOOKS_OPEN_MP=ON`
+
+### 3. Compilation
+   * Build and install `mitos` and `mitoshooks`
+   * Depending on the build method (cmake/make/command-line), link the application with `mitos` and `mitoshooks` by specifying their paths.
+   * See the section on `Source Code Attribution` below for more details on how to save the source code information with the samples collected while running your application.
+   
+### 4. Execution
+   
+   Mitoshooks uses the OMPT interface to collect samples while each thread runs. Configure the environment variable `OMP_TOOL_LIBRARIES` that points to mitoshooks-library and use one of these methods to launch the OpenMP-application.
+
+   **Recommended method:** 
+   ```bash
+   $> env OMP_TOOL_LIBRARIES=/path/to/mitos-inst-dir/lib/libmitoshooks.so ./omp_example
+   ```
+   ***The following method should also work but is not recommended. Exporting the variable `OMP_TOOL_LIBRARIES` for the global enviornment may break other OpenMP applications that do not intend to use hooks.***
+   ```bash
+   $> export OMP_TOOL_LIBRARIES=/path/to/mitos-inst-dir/lib/libmitoshooks.so 
+   
+   $> ./omp_example
+   ```
+   `Mitoshooks` can be fine-tuned by setting these parameters:
+
+   ```bash
+   env MITOS_SAMPLING_PERIOD=1000 MITOS_LATENCY_THRESHOLD=10
+   ```
+   Default value of sampling period is 4000 and that of sample latency threshold is 3. When setting these values, the application can be executed by:
+
+   ```bash
+   $> env OMP_TOOL_LIBRARIES=/path/to/mitos-inst-dir/lib/libmitoshooks.so MITOS_SAMPLING_PERIOD=1000 MITOS_LATENCY_THRESHOLD=10 ./omp_example
+   ```
+
+# Source Code Attribution
+
+## Requirements
+* [Dyninst](http://www.dyninst.org) version 12.3.0 or higher.
+* The application must be compiled in Debug mode (only `-g` compiler flag sufficient).
+
+## Saving the virtual address
+
+The source code of the executable must save the virtual addess offset when the executable starts runnnig. This can be done by including the [virtual_address_writer.h](src/virtual_address_writer.h) and calling the function [save_virtual_address_offset("virt_address.txt")](src/virtual_address_writer.h#L18).
+
+See [matmul.cpp](examples/matmul.cpp) for reference.
+
+This saves the virtual address offset to `/tmp/virt_address.txt`. Dyninst will access this file and attribute the source code information when the samples are saved.
 
 # Authors
 

examples/CMakeLists.txt

@@ -6,8 +6,8 @@ set(CMAKE_BUILD_TYPE Debug)
 add_executable(api_test api_test.cpp)
 add_executable(matmul matmul.cpp)
 
-target_link_libraries(api_test mitos)
-target_link_libraries(matmul dl)
+target_link_libraries(api_test mitos mitoshooks)
+target_link_libraries(matmul mitos mitoshooks dl)
 
 if(Dyninst_FOUND)
     include_directories(${DYNINST_INCLUDE_DIR})
@@ -16,13 +16,16 @@ endif()
 if(MPI_FOUND)
     include_directories(${MPI_INCLUDE_PATH})
     add_executable(mpi_matmul mpi_matmul.cpp)
+    add_executable(mpi_jacobi mpi_jacobi.cpp)
     target_link_libraries(mpi_matmul mitos mitoshooks ${MPI_LIBRARIES})
+    target_link_libraries(mpi_jacobi mitos mitoshooks ${MPI_LIBRARIES})
 endif()
 
 if(OpenMP_CXX_FOUND)
     # env OMP_TOOL_LIBRARIES=$(PWD)/libinit.so
     # OMP_TOOL_LIBRARIES=./../src/libmitoshooks.so
     include_directories(./../src)
+    message(STATUS "Linking examples with mitos, mitoshooks and OpenMP")
     add_executable(omp_example omp_example.cpp)
     target_link_libraries(omp_example mitos mitoshooks OpenMP::OpenMP_CXX)
 endif ()

examples/api_test.cpp

@@ -4,6 +4,7 @@
 #include <cstdlib>
 
 #include "../src/Mitos.h"
+#include "src/virtual_address_writer.h"
 
 mitos_output mout;
 
@@ -25,7 +26,9 @@ void init_matrices(int N, double **a, double **b, double **c)
     *b = new double[N*N];
     *c = new double[N*N];
 
-    size_t dims[2] = {N,N};
+    size_t dims[2];
+    dims[0] = N; //static_cast<size_t>(N);
+    dims[1] = N; //static_cast<size_t>(N);
     Mitos_add_symbol("a",*a,sizeof(double),dims,2);
     Mitos_add_symbol("b",*b,sizeof(double),dims,2);
     Mitos_add_symbol("c",*c,sizeof(double),dims,2);
@@ -62,6 +65,7 @@ void matmul(int N, double *a, double *b, double *c)
 
 int main(int argc, char **argv)
 {
+    save_virtual_address_offset("virt_address.txt");
     int N = (argc == 2) ? atoi(argv[1]) : 1024;
 
     double *a,*b,*c;
@@ -74,9 +78,18 @@ int main(int argc, char **argv)
     Mitos_set_sample_latency_threshold(3);
     Mitos_set_sample_time_frequency(4000);
 
-    //Mitos_begin_sampler();
+    Mitos_begin_sampler();
     matmul(N,a,b,c);
-    //Mitos_end_sampler();
-
-    Mitos_post_process(argv[0],&mout);
-}
+    Mitos_end_sampler();
+    std::set<std::string> src_files;
+    Mitos_add_offsets("", &mout);
+    if(Mitos_openFile(argv[0], &mout))
+    {
+        std::cerr << "Error opening binary file!" << std::endl;
+        return 1;
+    }
+    if(Mitos_post_process(argv[0],&mout, src_files)){
+        std::cerr << "Error post processing!" << std::endl;
+        return 1;
+    }
+}