Merge pull request #1941 from hyperledger/dev

Hyperledger Iroha v1.0 Release Candidate 1
hyperledger-iroha · Dec 12, 2018 · 8677175 · 8677175
2 parents 38dd0a1 + 5a47e9f
commit 8677175
Show file tree

Hide file tree

Showing 976 changed files with 41,256 additions and 21,774 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "deploy/ansible/kubespray"]
+	path = deploy/ansible/kubespray
+	url = https://github.com/soramitsu/kubespray.git
diff --git a/.jenkinsci/artifacts.groovy b/.jenkinsci/artifacts.groovy
@@ -1,10 +1,8 @@
 #!/usr/bin/env groovy
 
-def uploadArtifacts(filePaths, uploadPath, artifactServers=['artifact.soramitsu.co.jp']) {
-  def baseUploadPath = 'files'
+def uploadArtifacts(filePaths, uploadPath, artifactServers=['nexus.iroha.tech']) {
   def filePathsConverted = []
   agentType = sh(script: 'uname', returnStdout: true).trim()
-  uploadPath = baseUploadPath + uploadPath
   filePaths.each {
     fp = sh(script: "ls -d ${it} | tr '\n' ','", returnStdout: true).trim()
     filePathsConverted.addAll(fp.split(','))
@@ -24,32 +22,22 @@ def uploadArtifacts(filePaths, uploadPath, artifactServers=['artifact.soramitsu.
       sh "gpg --yes --batch --no-tty --import ${CI_GPG_PRIVKEY} || true"
     }
     filePathsConverted.each {
-      sh "echo put ${it} $uploadPath >> \$(pwd)/batch.txt;"
+      sh "echo ${it} >> \$(pwd)/batch.txt;"
       sh "$shaSumBinary ${it} | cut -d' ' -f1 > \$(pwd)/\$(basename ${it}).sha256"
       sh "$md5SumBinary ${it} | cut -d' ' -f1 > \$(pwd)/\$(basename ${it}).md5"
       // TODO @bakhtin 30.05.18 IR-1384. Make gpg command options and paths compatible with Windows OS.
       if (!agentType.contains('MSYS_NT')) {
-        sh "echo \"${CI_GPG_MASTERKEY}\" | $gpgKeyBinary -o \$(pwd)/\$(basename ${it}).asc ${it}"
-        sh "echo put \$(pwd)/\$(basename ${it}).asc $uploadPath >> \$(pwd)/batch.txt;"
+        sh "echo \"${CI_GPG_MASTERKEY}\" | $gpgKeyBinary -o \$(pwd)/\$(basename ${it}).ascfile ${it}"
+        sh "echo \$(pwd)/\$(basename ${it}).ascfile >> \$(pwd)/batch.txt;"
       }
-      sh "echo put \$(pwd)/\$(basename ${it}).sha256 $uploadPath >> \$(pwd)/batch.txt;"
-      sh "echo put \$(pwd)/\$(basename ${it}).md5 $uploadPath >> \$(pwd)/batch.txt;"
+      sh "echo \$(pwd)/\$(basename ${it}).sha256 >> \$(pwd)/batch.txt;"
+      sh "echo \$(pwd)/\$(basename ${it}).md5 >> \$(pwd)/batch.txt;"
     }
   }
-  // mkdirs recursively
-  uploadPath = uploadPath.split('/')
-  def p = ''
-  sh "> \$(pwd)/mkdirs.txt"
-  uploadPath.each {
-    p += "/${it}"
-    sh("echo -mkdir $p >> \$(pwd)/mkdirs.txt")
-  }
 
-  sshagent(['jenkins-artifact']) {
-    sh "ssh-agent"
+  withCredentials([usernamePassword(credentialsId: 'ci_nexus', passwordVariable: 'NEXUS_PASS', usernameVariable: 'NEXUS_USER')]) {
     artifactServers.each {
-      sh "sftp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -b \$(pwd)/mkdirs.txt jenkins@${it} || true"
-      sh "sftp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -b \$(pwd)/batch.txt jenkins@${it}"
+      sh(script: "while read line; do curl -u ${NEXUS_USER}:${NEXUS_PASS} --upload-file \$line https://${it}/repository/artifacts/${uploadPath}/ ; done < \$(pwd)/batch.txt")
     }
   }
 }

diff --git a/.jenkinsci/debug-build.groovy b/.jenkinsci/debug-build.groovy
@@ -22,11 +22,11 @@ def doDebugBuild(coverageEnabled=false) {
   def iC = dPullOrBuild.dockerPullOrUpdate("${platform}-develop-build",
                                            "${env.GIT_RAW_BASE_URL}/${env.GIT_COMMIT}/docker/develop/Dockerfile",
                                            "${env.GIT_RAW_BASE_URL}/${previousCommit}/docker/develop/Dockerfile",
-                                           "${env.GIT_RAW_BASE_URL}/develop/docker/develop/Dockerfile",
+                                           "${env.GIT_RAW_BASE_URL}/dev/docker/develop/Dockerfile",
                                            ['PARALLELISM': parallelism])
   // push Docker image in case the current branch is develop,
   // or it is a commit into PR which base branch is develop (usually develop -> master)
-  if ((GIT_LOCAL_BRANCH == 'develop' || CHANGE_BRANCH_LOCAL == 'develop') && manifest.manifestSupportEnabled()) {
+  if ((GIT_LOCAL_BRANCH == 'develop' || CHANGE_BRANCH_LOCAL == 'develop' || GIT_LOCAL_BRANCH == 'dev' || CHANGE_BRANCH_LOCAL == 'dev') && manifest.manifestSupportEnabled()) {
     manifest.manifestCreate("${DOCKER_REGISTRY_BASENAME}:develop-build",
       ["${DOCKER_REGISTRY_BASENAME}:x86_64-develop-build",
        "${DOCKER_REGISTRY_BASENAME}:armv7l-develop-build",
@@ -45,72 +45,73 @@ def doDebugBuild(coverageEnabled=false) {
     }
   }
 
-  docker.image('postgres:9.5').withRun(""
-    + " -e POSTGRES_USER=${env.IROHA_POSTGRES_USER}"
-    + " -e POSTGRES_PASSWORD=${env.IROHA_POSTGRES_PASSWORD}"
-    + " --name ${env.IROHA_POSTGRES_HOST}"
-    + " --network=${env.IROHA_NETWORK}") {
-      iC.inside(""
-      + " -e IROHA_POSTGRES_HOST=${env.IROHA_POSTGRES_HOST}"
-      + " -e IROHA_POSTGRES_PORT=${env.IROHA_POSTGRES_PORT}"
-      + " -e IROHA_POSTGRES_USER=${env.IROHA_POSTGRES_USER}"
-      + " -e IROHA_POSTGRES_PASSWORD=${env.IROHA_POSTGRES_PASSWORD}"
-      + " --network=${env.IROHA_NETWORK}"
-      + " -v /var/jenkins/ccache:${CCACHE_DIR}"
-      + " -v /tmp/${GIT_COMMIT}-${BUILD_NUMBER}:/tmp/${GIT_COMMIT}") {
+  // enable prepared transactions so that 2 phase commit works
+  // we set it to 100 as a safe value
+  sh "docker run -td -e POSTGRES_USER=${env.IROHA_POSTGRES_USER} \
+  -e POSTGRES_PASSWORD=${env.IROHA_POSTGRES_PASSWORD} --name ${env.IROHA_POSTGRES_HOST} \
+  --network=${env.IROHA_NETWORK} postgres:9.5 -c 'max_prepared_transactions=100'"
+  iC.inside(""
+    + " -e IROHA_POSTGRES_HOST=${env.IROHA_POSTGRES_HOST}"
+    + " -e IROHA_POSTGRES_PORT=${env.IROHA_POSTGRES_PORT}"
+    + " -e IROHA_POSTGRES_USER=${env.IROHA_POSTGRES_USER}"
+    + " -e IROHA_POSTGRES_PASSWORD=${env.IROHA_POSTGRES_PASSWORD}"
+    + " --network=${env.IROHA_NETWORK}"
+    + " -v /var/jenkins/ccache:${CCACHE_DIR}"
+    + " -v /tmp/${GIT_COMMIT}-${BUILD_NUMBER}:/tmp/${GIT_COMMIT}") {
 
-      def scmVars = checkout scm
-      def cmakeOptions = ""
-      if ( coverageEnabled ) {
-        cmakeOptions = " -DCOVERAGE=ON "
-      }
-      env.IROHA_VERSION = "0x${scmVars.GIT_COMMIT}"
-      env.IROHA_HOME = "/opt/iroha"
-      env.IROHA_BUILD = "${env.IROHA_HOME}/build"
+    def scmVars = checkout scm
+    def cmakeOptions = ""
+    if ( coverageEnabled ) {
+      cmakeOptions = " -DCOVERAGE=ON "
+    }
+    env.IROHA_VERSION = "0x${scmVars.GIT_COMMIT}"
+    env.IROHA_HOME = "/opt/iroha"
+    env.IROHA_BUILD = "${env.IROHA_HOME}/build"
 
-      sh """
-        ccache --version
-        ccache --show-stats
-        ccache --zero-stats
-        ccache --max-size=5G
-      """
-      sh """
-        cmake \
-          -DTESTING=ON \
-          -H. \
-          -Bbuild \
-          -DCMAKE_BUILD_TYPE=Debug \
-          -DIROHA_VERSION=${env.IROHA_VERSION} \
-          ${cmakeOptions}
-      """
-      sh "cmake --build build -- -j${parallelism}"
-      sh "ccache --show-stats"
-      if ( coverageEnabled ) {
-        sh "cmake --build build --target coverage.init.info"
-      }
-      def testExitCode = sh(script: """cd build && ctest --output-on-failure""", returnStatus: true)
-      if (testExitCode != 0) {
-        currentBuild.result = "UNSTABLE"
-      }
-      if ( coverageEnabled ) {
-        sh "cmake --build build --target cppcheck"
-        // Sonar
-        if (env.CHANGE_ID != null) {
-          sh """
-            sonar-scanner \
-              -Dsonar.github.disableInlineComments \
-              -Dsonar.github.repository='${DOCKER_REGISTRY_BASENAME}' \
-              -Dsonar.analysis.mode=preview \
-              -Dsonar.login=${SONAR_TOKEN} \
-              -Dsonar.projectVersion=${BUILD_TAG} \
-              -Dsonar.github.oauth=${SORABOT_TOKEN} \
-              -Dsonar.github.pullRequest=${CHANGE_ID}
-          """
-        }
-        sh "cmake --build build --target coverage.info"
-        sh "python /tmp/lcov_cobertura.py build/reports/coverage.info -o build/reports/coverage.xml"
-        cobertura autoUpdateHealth: false, autoUpdateStability: false, coberturaReportFile: '**/build/reports/coverage.xml', conditionalCoverageTargets: '75, 50, 0', failUnhealthy: false, failUnstable: false, lineCoverageTargets: '75, 50, 0', maxNumberOfBuilds: 50, methodCoverageTargets: '75, 50, 0', onlyStable: false, zoomCoverageChart: false
+    sh """
+      ccache --version
+      ccache --show-stats
+      ccache --zero-stats
+      ccache --max-size=5G
+    """
+    sh """
+      cmake \
+        -DTESTING=ON \
+        -H. \
+        -Bbuild \
+        -DCMAKE_BUILD_TYPE=Debug \
+        -DIROHA_VERSION=${env.IROHA_VERSION} \
+        ${cmakeOptions}
+    """
+    sh "cmake --build build -- -j${parallelism}"
+    sh "ccache --show-stats"
+    if ( coverageEnabled ) {
+      sh "cmake --build build --target coverage.init.info"
+    }
+    sh "cd build; ctest --output-on-failure --no-compress-output -T Test || true"
+    sh 'python .jenkinsci/helpers/platform_tag.py "Linux \$(uname -m)" \$(ls build/Testing/*/Test.xml)'
+    // Mark build as UNSTABLE if there are any failed tests (threshold <100%)
+    xunit testTimeMargin: '3000', thresholdMode: 2, thresholds: [passed(unstableThreshold: '100')], \
+      tools: [CTest(deleteOutputFiles: true, failIfNotNew: false, \
+      pattern: 'build/Testing/**/Test.xml', skipNoTestFiles: false, stopProcessingIfError: true)]
+    if ( coverageEnabled ) {
+      sh "cmake --build build --target cppcheck"
+      // Sonar
+      if (env.CHANGE_ID != null) {
+        sh """
+          sonar-scanner \
+            -Dsonar.github.disableInlineComments \
+            -Dsonar.github.repository='${DOCKER_REGISTRY_BASENAME}' \
+            -Dsonar.analysis.mode=preview \
+            -Dsonar.login=${SONAR_TOKEN} \
+            -Dsonar.projectVersion=${BUILD_TAG} \
+            -Dsonar.github.oauth=${SORABOT_TOKEN} \
+            -Dsonar.github.pullRequest=${CHANGE_ID}
+        """
       }
+      sh "cmake --build build --target coverage.info"
+      sh "python /tmp/lcov_cobertura.py build/reports/coverage.info -o build/reports/coverage.xml"
+      cobertura autoUpdateHealth: false, autoUpdateStability: false, coberturaReportFile: '**/build/reports/coverage.xml', conditionalCoverageTargets: '75, 50, 0', failUnhealthy: false, failUnstable: false, lineCoverageTargets: '75, 50, 0', maxNumberOfBuilds: 50, methodCoverageTargets: '75, 50, 0', onlyStable: false, zoomCoverageChart: false
     }
   }
 }

diff --git a/.jenkinsci/docker-cleanup.groovy b/.jenkinsci/docker-cleanup.groovy
@@ -7,6 +7,7 @@ def doDockerCleanup() {
       # i.e., no other tags exist for this image
       docker rmi \$(docker images --no-trunc --format '{{.Repository}}:{{.Tag}}\\t{{.ID}}' | grep \$(docker images --no-trunc --format '{{.ID}}' ${iC.id}) | head -n -1 | cut -f 1) || true
       sleep 5
+      docker rm -f ${env.IROHA_POSTGRES_HOST} || true
       docker network rm $IROHA_NETWORK || true
       #remove folder with iroha.deb package and Dockerfiles
       rm -rf /tmp/${env.GIT_COMMIT}-${BUILD_NUMBER}

diff --git a/.jenkinsci/docker-pull-or-build.groovy b/.jenkinsci/docker-pull-or-build.groovy
@@ -46,7 +46,7 @@ def dockerPullOrUpdate(imageName, currentDockerfileURL, previousDockerfileURL, r
       }
     }
   }
-  if (GIT_LOCAL_BRANCH ==~ /develop|master/ || CHANGE_BRANCH_LOCAL == 'develop') {
+  if (GIT_LOCAL_BRANCH ==~ /develop|master|dev/ || CHANGE_BRANCH_LOCAL == 'develop' || CHANGE_BRANCH_LOCAL == 'dev') {
     docker.withRegistry('https://registry.hub.docker.com', 'docker-hub-credentials') {
       iC.push(imageName)
     }

diff --git a/.jenkinsci/doxygen.groovy b/.jenkinsci/doxygen.groovy
@@ -1,6 +1,9 @@
 #!/usr/bin/env groovy
 
 def doDoxygen() {
+  // TODO: Remove this comment once dev branch will return to develop
+  // I will not be changing branches here. It requires some rewriting
+  // Hope dev branch situation will be resolved soon
   if (env.GIT_LOCAL_BRANCH in ["master","develop"] || env.CHANGE_BRANCH_LOCAL == 'develop') {
     def branch = env.CHANGE_BRANCH_LOCAL == 'develop' ? env.CHANGE_BRANCH_LOCAL : env.GIT_LOCAL_BRANCH
     sh "doxygen Doxyfile"

diff --git a/.jenkinsci/helpers/platform_tag.py b/.jenkinsci/helpers/platform_tag.py
@@ -0,0 +1,14 @@
+#!/usr/env/python
+import xml.etree.ElementTree as ET
+import argparse
+
+parser = argparse.ArgumentParser(description='Tag test names in a JUnit report')
+for arg in ['tag', 'xml_report_file']:
+	parser.add_argument(arg)
+args = parser.parse_args()
+
+tree = ET.parse(args.xml_report_file)
+root = tree.getroot()
+for i in root.findall(".//Test/Name"):
+	i.text = "%s | %s" % (args.tag, i.text)
+tree.write(args.xml_report_file)
diff --git a/.jenkinsci/linux-post-step.groovy b/.jenkinsci/linux-post-step.groovy
@@ -1,7 +1,7 @@
 def linuxPostStep() {
   timeout(time: 600, unit: "SECONDS") {
     try {
-      if (currentBuild.currentResult == "SUCCESS" && GIT_LOCAL_BRANCH ==~ /(master|develop)/) {
+      if (currentBuild.currentResult == "SUCCESS" && GIT_LOCAL_BRANCH ==~ /(master|develop|dev)/) {
         def artifacts = load ".jenkinsci/artifacts.groovy"
         def commit = env.GIT_COMMIT
         def platform = sh(script: 'uname -m', returnStdout: true).trim()

diff --git a/.jenkinsci/release-build.groovy b/.jenkinsci/release-build.groovy
@@ -53,14 +53,17 @@ def doReleaseBuild() {
 
   sh "curl -L -o /tmp/${env.GIT_COMMIT}/Dockerfile --create-dirs ${env.GIT_RAW_BASE_URL}/${env.GIT_COMMIT}/docker/release/Dockerfile"
   sh "curl -L -o /tmp/${env.GIT_COMMIT}/entrypoint.sh ${env.GIT_RAW_BASE_URL}/${env.GIT_COMMIT}/docker/release/entrypoint.sh"
+  sh "curl -L -o /tmp/${env.GIT_COMMIT}/wait-for-it.sh ${env.GIT_RAW_BASE_URL}/${env.GIT_COMMIT}/docker/release/wait-for-it.sh"
   sh "mv /tmp/${GIT_COMMIT}-${BUILD_NUMBER}/iroha.deb /tmp/${env.GIT_COMMIT}"
-  sh "chmod +x /tmp/${env.GIT_COMMIT}/entrypoint.sh"
   iCRelease = docker.build("${DOCKER_REGISTRY_BASENAME}:${GIT_COMMIT}-${BUILD_NUMBER}-release", "--no-cache -f /tmp/${env.GIT_COMMIT}/Dockerfile /tmp/${env.GIT_COMMIT}")
 
   // push Docker image in case the current branch is develop,
   // or it is a commit into PR which base branch is develop (usually develop -> master)
-  if (GIT_LOCAL_BRANCH == 'develop' || CHANGE_BRANCH_LOCAL == 'develop') {
-    iCRelease.push("${platform}-develop")
+  checkTag = sh(script: 'git describe --tags --exact-match ${GIT_COMMIT}', returnStatus: true)
+  if (GIT_LOCAL_BRANCH == 'develop' || CHANGE_BRANCH_LOCAL == 'develop' || GIT_LOCAL_BRANCH == 'dev' || CHANGE_BRANCH_LOCAL == 'dev') {
+    docker.withRegistry('https://registry.hub.docker.com', 'docker-hub-credentials') {
+      iCRelease.push("${platform}-develop")
+    }
     if (manifest.manifestSupportEnabled()) {
       manifest.manifestCreate("${DOCKER_REGISTRY_BASENAME}:develop",
         ["${DOCKER_REGISTRY_BASENAME}:x86_64-develop",
@@ -81,7 +84,9 @@ def doReleaseBuild() {
     }
   }
   else if (GIT_LOCAL_BRANCH == 'master') {
-    iCRelease.push("${platform}-latest")
+    docker.withRegistry('https://registry.hub.docker.com', 'docker-hub-credentials') {
+      iCRelease.push("${platform}-latest")
+    }
     if (manifest.manifestSupportEnabled()) {
       manifest.manifestCreate("${DOCKER_REGISTRY_BASENAME}:latest",
         ["${DOCKER_REGISTRY_BASENAME}:x86_64-latest",
@@ -100,8 +105,25 @@ def doReleaseBuild() {
         manifest.manifestPush("${DOCKER_REGISTRY_BASENAME}:latest", login, password)
       }
     }
+    if (checkTag == 0) {
+      def tag = sh(script: 'git describe --tags --exact-match ${GIT_COMMIT}', returnStdout: true).trim().replaceAll('-','_')
+      docker.withRegistry('https://registry.hub.docker.com', 'docker-hub-credentials') {
+        iCRelease.push("${platform}-${tag}")
+      }
+      if (manifest.manifestSupportEnabled()) {
+        manifest.manifestCreate("${DOCKER_REGISTRY_BASENAME}:${tag}",
+          ["${DOCKER_REGISTRY_BASENAME}:x86_64-${tag}"])
+        manifest.manifestAnnotate("${DOCKER_REGISTRY_BASENAME}:${tag}",
+          [
+            [manifest: "${DOCKER_REGISTRY_BASENAME}:x86_64-${tag}",
+            arch: 'amd64', os: 'linux', osfeatures: [], variant: '']
+          ])
+        withCredentials([usernamePassword(credentialsId: 'docker-hub-credentials', usernameVariable: 'login', passwordVariable: 'password')]) {
+          manifest.manifestPush("${DOCKER_REGISTRY_BASENAME}:${tag}", login, password)
+        }
+      }
+    }
   }
-
   sh "docker rmi ${iCRelease.id}"
 }
 return this
diff --git a/.jenkinsci/selected-branches-coverage.groovy b/.jenkinsci/selected-branches-coverage.groovy
@@ -1,13 +1,7 @@
 #!/usr/bin/env groovy
 
-def selectedBranchesCoverage(branches, PRCoverage=true) {
-	// trigger coverage if branch is either develop or master, or it is a PR
-	if (PRCoverage) {
-		return env.GIT_LOCAL_BRANCH in branches || env.CHANGE_ID != null
-	}
-	else {
-		return env.GIT_LOCAL_BRANCH in branches
-	}
+def selectedBranchesCoverage(branches) {
+	return env.GIT_LOCAL_BRANCH in branches
 }
 
-return this
+return this
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,7 +1,7 @@
 # Copyright Soramitsu Co., Ltd. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-cmake_minimum_required(VERSION 3.5.1)
+cmake_minimum_required(VERSION 3.11.4)
 
 find_program(CCACHE_PROGRAM ccache)
 if(CCACHE_PROGRAM)
@@ -64,6 +64,7 @@ option(SWIG_JAVA    "Generate Swig Java bindings"    OFF)
 option(SUPPORT_PYTHON2 "ON if Python2, OFF if python3" OFF)
 option(SWIG_CSHARP  "Generate Swig C# bindings"      OFF)
 option(SWIG_NODE    "Generate Swig NodeJS"           OFF)
+option(USE_LIBIROHA "Use external model library"     OFF)
 
 
 if (NOT CMAKE_BUILD_TYPE)
@@ -108,10 +109,14 @@ set(SM_SCHEMA_DIR "${PROJECT_SOURCE_DIR}/shared_model/schema")
 set(SCHEMA_OUT_DIR ${CMAKE_BINARY_DIR}/schema)
 include_directories(
   ${PROJECT_SOURCE_DIR}/irohad
-  ${PROJECT_SOURCE_DIR}/shared_model
   ${PROJECT_SOURCE_DIR}/libs
 )
 
+# fallback to local shared_model
+if (NOT USE_LIBIROHA)
+  include_directories(${PROJECT_SOURCE_DIR}/shared_model)
+endif()
+
 SET(IROHA_ROOT_PROJECT ON)
 
 # Boost uses RTTI to perform some actions (such as type erasure).
@@ -132,7 +137,11 @@ add_subdirectory(schema)
 add_subdirectory(libs)
 add_subdirectory(irohad)
 add_subdirectory(iroha-cli)
-add_subdirectory(shared_model)
+
+# fallback to local shared_model
+if (NOT USE_LIBIROHA)
+  add_subdirectory(shared_model)
+endif()
 
 if(TESTING)
   enable_testing()