New features, bug fixes, and new test case framework for 4.16.0

.dockerignore

@@ -0,0 +1,8 @@
+cget
+venv
+.git
+private
+build
+clion-build-debug
+clion-build-release
+docker-compose.override.yml

.travis.yml

@@ -0,0 +1,55 @@
+language: cpp
+
+jobs:
+  include:
+    - os: linux
+      dist: bionic
+      compiler: gcc
+      env: LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8
+      before_install:
+        - |
+          sudo apt update
+          sudo apt install build-essential cmake gfortran locales zlib1g-dev liblzma-dev
+          sudo apt install python3 python3-setuptools python3-pip python3-virtualenv
+          sudo locale-gen en_US.UTF-8
+          sudo pip3 install cget
+          cget install -f requirements.txt
+
+    - os: osx
+      osx_image: xcode11.3
+      compiler: clang
+      before_install:
+        - |
+          brew install cmake zlib xz python3
+          brew cask install gfortran
+          sudo python3 -m pip install cget
+          sudo installer -pkg /Library/Developer/CommandLineTools/Packages/macOS_SDK_headers_for_macOS_10.14.pkg -target /
+          cget install -f requirements.txt
+
+    - os: osx
+      osx_image: xcode11.4
+      compiler: clang
+      before_install:
+        - |
+          brew install cmake zlib xz python3
+          brew cask install gfortran
+          sudo python3 -m pip install cget
+          export CPATH=$(xcrun --show-sdk-path)/usr/include
+          export CFLAGS="${CFLAGS} -Wno-nullability-completeness -Wno-expansion-to-defined -Wno-undef"
+          cget install -f requirements.txt
+
+cache: pip
+
+install:
+  - rm -rf build && mkdir build && cd build
+  - cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=../cget/cget/cget.cmake -DBUILD_TESTS=1 ..
+  - make
+  - cd ..
+
+script:
+  - build/raremetal/tests/bin/tests-raremetal
+  - cd build
+
+  # The raremetalworker tests only work on OSX. If we're on OSX, then run the tests and return the result.
+  # Otherwise, this statement should return true, to avoid completely failing the entire build/job.
+  - if [[ "${TRAVIS_OS_NAME}" == "osx" ]]; then ctest --verbose; else true; fi

CMakeLists.txt

@@ -11,6 +11,8 @@ find_library(LIB_HTS hts)
 find_library(LIB_Z z)
 find_library(LIB_RMATH Rmath)
 
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY bin)
+
 add_library(libraremetal
             libRareMetal/mvtnorm/randomF77.c
             libRareMetal/pdf/PDF.cpp
@@ -136,8 +138,8 @@ target_include_directories(raremetalworker PRIVATE otherLib/eigen-3.2.0)
 target_link_libraries(raremetalworker libraremetal ${LIB_SAVVY} ${LIB_ZSTD})
 target_compile_definitions(raremetalworker PRIVATE VERSION="${PROJECT_VERSION}")
 
-
 if(BUILD_TESTS)
     enable_testing()
     add_subdirectory(tests)
+    add_subdirectory(raremetal/tests)
 endif()

Dockerfile

@@ -0,0 +1,70 @@
+FROM ubuntu:18.04
+
+ARG BUILD_DATE
+ARG GIT_SHA
+ARG RAREMETAL_VERSION
+
+LABEL org.label-schema.name="RAREMETAL"
+LABEL org.label-schema.description="Meta-analysis of rare genetic variant aggregation tests"
+LABEL org.label-schema.vendor="University of Michigan, Center for Statistical Genetics"
+LABEL org.label-schema.url="https://github.com/statgen/RAREMETAL"
+LABEL org.label-schema.vcs-url="https://github.com/statgen/RAREMETAL"
+LABEL org.label-schema.schema-version="1.0"
+
+# Install required packages for swiss to install. Many of swiss' dependencies
+# require compiling C/C++ code.
+RUN apt-get update && apt-get install -y \
+  build-essential \
+  cmake \
+  gfortran \
+  python3 \
+  python3-pip \
+  zlib1g-dev \
+  liblzma-dev \
+  locales \
+  && rm -rf /var/lib/apt/lists/* \
+  && locale-gen en_US.UTF-8
+
+ENV LC_ALL en_US.UTF-8
+ENV LANG en_US.UTF-8
+
+# Install necessary python packages (backports.lzma needed for cget to extract .xz archives)
+RUN pip3 install cget
+
+# Create a group and user to execute as, then drop root
+ARG UID
+ARG GID
+RUN \
+  if [ -n "$GID" ]; then \
+    addgroup --gid $GID raremetal; \
+  else \
+    addgroup raremetal; \
+  fi && \
+  if [ -n "$UID" ]; then \
+    adduser --gecos "User for running raremetal as non-root" --shell /bin/bash --disabled-password --uid $UID --ingroup raremetal raremetal; \
+  else \
+    adduser --gecos "User for running raremetal as non-root" --shell /bin/bash --disabled-password --ingroup raremetal raremetal; \
+  fi
+
+WORKDIR /home/raremetal
+USER raremetal
+
+# Copy source
+COPY --chown=raremetal:raremetal . /home/raremetal/
+
+# Compile & run tests
+# We want the docker build to fail if the tests fail
+RUN cget install -f requirements.txt
+RUN mkdir build && \
+  cd build && \
+  cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=../cget/cget/cget.cmake -DBUILD_TESTS=1 .. && \
+  make && \
+  cd .. && \
+  build/raremetal/tests/bin/tests-raremetal
+
+ENTRYPOINT ["build/bin/raremetal"]
+
+# Frequently changing metadata here to avoid cache misses
+LABEL org.label-schema.version=$RAREMETAL_VERSION
+LABEL org.label-schema.vcs-ref=$GIT_SHA
+LABEL org.label-schema.build-date=$BUILD_DATE

README.md

@@ -1,43 +1,106 @@
-Raremetal: A tool for rare variants meta-analysis
+RAREMETAL: A tool for rare variants meta-analysis
 
 (c) 2012-2018 Shuang Feng, Dajiang Liu, Sai Chen, Gonçalo Abecasis
 
 For more information, please see the [RAREMETAL wiki](http://genome.sph.umich.edu/wiki/RAREMETAL).
 
 ## Installation
-1. Clone this repository
-2. Open a terminal and change to the directory containing this README file.
-3. Enter the following sequence of commands to download dependencies and build the code (with tests run automatically during build):
-
+
+### Requirements
+
+RAREMETAL requires the following libraries to build. These should already be installed on most scientific 
+computing clusters.
+
+#### Linux
+
+On Linux, the following packages are needed:
+
+- zlib
+- liblzma
+- cmake
+- gcc/g++
+- gfortran
+- python3
+- cget
+
+As an example in Ubuntu specifically, you can install with the following commands:
+
+```bash
+sudo apt update
+sudo apt install build-essential cmake gfortran locales zlib1g-dev liblzma-dev
+sudo apt install python3 python3-setuptools python3-pip python3-virtualenv
+sudo python3 -m pip install cget
+```
+
+#### MacOS
+
+Use Homebrew to install the necessary packages:
+
+```bash
+brew install cmake zlib xz python3
+brew cask install gfortran
+sudo python3 -m pip install cget
+```
+
+You may need to manually install the C++ headers on MacOS 10.4 (Mojave) by running the following command: 
+
 ```bash
-cget install -f requirements.txt
-mkdir build && cd build
-cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=../cget/cget/cget.cmake -DBUILD_TESTS=1 ..
-make
+sudo installer -pkg /Library/Developer/CommandLineTools/Packages/macOS_SDK_headers_for_macOS_10.14.pkg -target /
 ```
 
-To run unit tests, ensure that you have run cmake with `-DBUILD_TESTS=1` and built at least once, 
-then run `make` or `ctest --verbose`. 
+For Mac OS 10.5 (Catalina), C++ headers may be missing and you may receive excessive compiler warnings. The following settings may help:
+
+```bash
+# Specify path to MacOS C++ SDK header files
+export CPATH=$(xcrun --show-sdk-path)/usr/include
+
+# Disable compiler warnings introduced in Catalina
+export CFLAGS="${CFLAGS} -Wno-nullability-completeness -Wno-expansion-to-defined -Wno-undef"
+```
+
+#### Windows
+
+Windows is currently unsupported. You may have some luck with Windows Subsystem for Linux, or a VM.
+
+### Installing
+
+1. Clone this repository: `git clone https://github.com/statgen/raremetal.git`
+2. Open a terminal and change to the directory containing this README file.
+3. Enter the following sequence of commands to download dependencies and build the code:
+
+    ```bash
+    cget install -f requirements.txt
+    mkdir build && cd build
+    cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=../cget/cget/cget.cmake -DBUILD_TESTS=1 ..
+    make
+    ```
 
 If you encounter problems while building, see the [wiki instructions](https://genome.sph.umich.edu/wiki/RAREMETAL_DOWNLOAD_%26_BUILD) 
 and [FAQ](https://genome.sph.umich.edu/wiki/RAREMETAL_FAQ) for more information.
 
-### Requirements
-Raremetal requires the following libraries to build. These should already be installed on most scientific 
-computing clusters.
+### Running tests
 
-- libRMath (The R math library. If not installed, use `apt-get install r-mathlib` on ubuntu or `brew install r` 
-    on Mac OS)
-- zlib (`apt-get install zlib1g-dev` on Ubuntu)
+To run unit tests, ensure that you have run cmake with `-DBUILD_TESTS=1` and built at least once, then run: 
 
-On Mac OS, installing R as a standalone package is not sufficient to use libRMath in other executables. 
-Consider `brew install r` or [r-devel](https://r.research.att.com/) in that case.   
+```bash
+# To test RAREMETAL, run the following:
+build/raremetal/tests/bin/tests-raremetal
+
+# To test RAREMETALWORKER, run the following:
+cd build
+make test
+```
+
+Both sets of tests are automatically run by Travis CI as well, see `.travis.yml` for details.
 
 ## Changes
+
 Newest version: v.4.14.1 released at 07/10/2017
+
 View change log at: http://genome.sph.umich.edu/wiki/RAREMETAL_Change_Log
 
 ### RAREMETAL2
+
 We are also working on a beta version that provides new features and options- RAREMETAL2. 
 
 This code is still in the experimental stages, but a full release is planned in the future. See:

raremetal/src/Calculate_mvt_pvalue.h

@@ -6,6 +6,8 @@
 #include <stdio.h>
 #include "MathMatrix.h"
 #include "StringHash.h"
+#include "IntArray.h"
+#include "MathSVD.h"
 
 #define MATHLIB_STANDALONE
 
@@ -112,15 +114,9 @@ double CalculateMVTPvalue(Vector &score, Matrix &cov, double t_max)
                     //A new combination has been generated!
                     comb.Push(alpha[indx[owk]]);
                 }
-/*
-            printf("New combination:\n");
-            for(int i=0;i<comb.Length();i++)
-               printf("%g ",comb[i]);
-            printf("\n");
-*/
+
                 //Calculate P(A|BC)
                 GetConditionalDist(score, cov, comb, par);
-//            printf("pars are:%g %g\n",par[0],par[1]);
                 double chisq, condProb, prob;
                 if (par[1] == 0.0)
                 {
@@ -134,7 +130,6 @@ double CalculateMVTPvalue(Vector &score, Matrix &cov, double t_max)
                     }
                     condProb = pchisq(chisq, 1, 0, 0);
                 }
-//          printf("conditional prob is:%g\n",condProb);
                 String hashKey;
 
                 if (r == 2)
@@ -143,10 +138,7 @@ double CalculateMVTPvalue(Vector &score, Matrix &cov, double t_max)
                     hashKey += tmp;
                     tmp = comb[1];
                     hashKey += tmp;
-                    //              printf("hashKey is: %s\n",hashKey.c_str());
                     prob = condProb * uni;
-                    //prob = exp(log(condProb) + log(uni));
-                    //printf("joint prob is:%g\n",prob);
                     jointProbHash.SetDouble(hashKey, prob);
                     hashKey.Clear();
                 } else
@@ -156,13 +148,7 @@ double CalculateMVTPvalue(Vector &score, Matrix &cov, double t_max)
                         int tmp = comb[i];
                         hashKey += tmp;
                     }
-                    //              printf("hashKey is: %s\n",hashKey.c_str());
                     prob = jointProbHash.Double(hashKey);
-//	       printf("joint prob is:%g\n",prob);
-                    //if(prob == _NAN_)
-                    //{
-                    //         printf("Error! Joint probability %s is not hashed in properly.\n",hashKey);
-                    //     }
                     prob *= condProb; //This is the new joint prob.
                     String newKey;
                     int tmp = comb[0];
@@ -174,7 +160,6 @@ double CalculateMVTPvalue(Vector &score, Matrix &cov, double t_max)
                 }
 
                 pvalue -= prob;
-//	    printf("pvalue so far is:%g\n",pvalue);
                 comb.Clear();
                 for (int iwk = r - 1; iwk >= 0; iwk--)
                 {

raremetal/src/GroupFromAnnotation.cpp

@@ -5,15 +5,14 @@
 #include "InputFile.h"
 #include "QuickIndex.h"
 
-String GroupFromAnnotation::groupFile = "";
-String GroupFromAnnotation::vcfInput = "";
-String GroupFromAnnotation::function = "";
-String GroupFromAnnotation::mapFile = "../data/refFlat_hg19.txt";
-bool GroupFromAnnotation::labelHits = false;
-
 GroupFromAnnotation::GroupFromAnnotation()
 {
-    geneCount = 0;
+  geneCount = 0;
+  groupFile = "";
+  vcfInput = "";
+  function = "";
+  mapFile = "../data/refFlat_hg19.txt";
+  labelHits = false;
 }
 
 GroupFromAnnotation::~GroupFromAnnotation()