Eng 13947 Make generated tests in a smarter way. (#5442)

When we generate C++ unit tests, we print the test names to standard output. This can
break the build if someone puts unrelated printing into the planner.

These commits want to merge some changes to write the test names to a file, and then
read the file in CMake in order to generate the makefiles.

https://issues.voltdb.com/browse/ENG-13947

.gitignore

@@ -76,6 +76,7 @@ build.gradle
 /lib/python/vdm/tests/geb_vdm/.nb-gradle/
 tools/meshmonitor/build/
 tools/meshmonitor/dist/
+tools/ee/ee_auto_generated_unit_tests
 
 # IntelliJ excluded files.
 .idea/workspace.xml

build.xml

@@ -2474,6 +2474,8 @@ UTILITIES
         description="Remove other generated source files.">
     <delete file="${src.ee.dir}/org_voltdb_jni_ExecutionEngine.h" />
     <delete file="${src.ee.dir}/org_voltcore_utils_DBBPool.h" />
+    <!-- Delete the generated ee unit tests, but not if they don't exist. -->
+    <delete dir="${src.ee.test.dir}/ee_auto_generated_unit_tests" failonerror="false"/>
 </target>
 
 <!--

tests/ee/CMakeLists.txt

@@ -187,10 +187,21 @@ SET(VOLTDB_TEST_BUILD_TYPE "--build-type=${VOLTDB_BUILD_TYPE_LOWER}")
 FOREACH (CLASS ${VOLTDB_GENERATED_TEST_GENERATORS})
   LIST(APPEND VOLTDB_TEST_CLASS_LIST --test-class=${CLASS})
 ENDFOREACH()
+#
+# Generate the tests.
+#
 EXECUTE_PROCESS(
     COMMAND ${CMAKE_SOURCE_DIR}/tools/generate-ee-unit-tests.sh --voltdbroot=${CMAKE_SOURCE_DIR} ${VOLTDB_TEST_BUILD_TYPE} ${VOLTDB_TEST_CLASS_LIST}
-    OUTPUT_VARIABLE VOLTDB_GENERATED_TEST_PROGRAMS
-  )
+)
+#
+# The previous command left the test names in
+# This file.  Each line is a single file name.  We
+# need to do some CMake magic to convert this to a
+# CMake list.
+#
+FILE(READ ${CMAKE_SOURCE_DIR}/tests/ee/ee_auto_generated_unit_tests/generated_tests.txt VOLTDB_GENERATED_TEST_PROGRAMS)
+STRING(REPLACE "\n" ";" VOLTDB_GENERATED_TEST_PROGRAMS ${VOLTDB_GENERATED_TEST_PROGRAMS})
+
 STRING(STRIP "${VOLTDB_GENERATED_TEST_PROGRAMS}" VOLTDB_GENERATED_TEST_PROGRAMS)
 # MESSAGE( "VOLTDB_GENERATED_TEST_PROGRAMS is ${VOLTDB_GENERATED_TEST_PROGRAMS}" )
 

tests/frontend/org/voltdb/planner/eegentests/EEPlanGenerator.java

@@ -82,11 +82,7 @@
  * the License.
  */package org.voltdb.planner.eegentests;
 
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.PrintWriter;
+import java.io.*;
 import java.net.URL;
 import java.nio.file.Files;
 import java.nio.file.Paths;
@@ -210,10 +206,20 @@ public class EEPlanGenerator extends PlannerTestCase {
         "}\n";
 
     //
-    // Guess that we are started in the root.
+    // This holds the full path name of the directory into which we will
+    // put generated EE unit tests.
     //
-    private String m_VoltDBRootDirName = Paths.get(".").toAbsolutePath().normalize().toString();
-    private String m_testGenDir = "ee_auto_generated_unit_tests";
+    private String m_testGenPath;
+    //
+    // This is the last path component of the previous.
+    //
+    private String m_testGenDir;
+    //
+    // This holds the full path name of the file which contains names
+    // of generated tests.  The names are listed one test name per line.
+    // Each test name is the full path name to the test file.
+    //
+    private String m_testNamesFile;
 
     protected String getPlanString(String sqlStmt, int fragmentNumber) throws JSONException {
         boolean planForSinglePartition = (fragmentNumber == 0);
@@ -861,7 +867,6 @@ public int getPlanFragment() {
      * @throws Exception
      */
     protected void generateTests(String testFolder, String testClassName, DBConfig db) throws Exception {
-        System.out.printf("%s/%s/%s;", m_testGenDir, testFolder, testClassName);
         Map<String, String> params = new HashMap<>();
         params.put("SOURCE_PACKAGE_NAME",   db.getClassPackageName());
         params.put("SOURCE_CLASS_NAME",     db.getClassName());
@@ -879,6 +884,7 @@ protected void generateTests(String testFolder, String testClassName, DBConfig d
         params.put("ALL_TESTS",             db.getAllTests(params));
         params.put("DATABASE_CONFIG_BODY",  db.getDatabaseConfigBody(params));
         writeTestFile(testFolder, testClassName, params);
+        writeTestFileName(String.format("%s/%s/%s\n", m_testGenDir, testFolder, testClassName), true);
     }
 
     public static boolean typeMatch(Object elem, VoltType type, int size) {
@@ -940,22 +946,63 @@ private void writeFile(File path, String contents) throws Exception {
     protected void processArgs(String args[]) throws PlanningErrorException {
         for (int idx = 0; idx < args.length; idx += 1) {
             String arg = args[idx];
-            if (arg.startsWith("--test-source-dir=")) {
-                m_VoltDBRootDirName = arg.substring("--test-source-dir=".length());
-            } else if (arg.startsWith("--generated-source-dir=")) {
-                m_testGenDir = arg.substring("--generated-source-dir".length());
+            if (arg.startsWith("--generated-source-dir=")) {
+                m_testGenPath = arg.substring("--generated-source-dir=".length());
+                // Pull out the last component.  We
+                // need this later on, to put the source in the
+                // right directory.
+                String[] paths = m_testGenPath.split("/");
+                if (paths.length == 0) {
+                    throw new PlanningErrorException(
+                            String.format("--generated-source-dir argument \"%s\" is malformed.",
+                                          arg));
+                }
+                m_testGenDir = paths[paths.length - 1];
+            } else if (arg.startsWith("--test-names-file=")) {
+                m_testNamesFile = arg.substring("--test-names-file=".length());
+            } else {
+                throw new PlanningErrorException("Unknown generated sources argument: " + arg);
             }
         }
+        if (m_testGenPath == null) {
+            throw new PlanningErrorException("--generated-source-dir argument is missing in call to EEPlanGenerator.java");
+        }
+        if (m_testNamesFile == null) {
+            throw new PlanningErrorException("--test-names-file argument is missing in call to EEPlanGenerator.java");
+        }
+    }
+
+    /**
+     * Write the named string to the output file.
+     *
+     * @param name The string to write.
+     * @param append If true, then append to the file.  Otherwise truncate the file first.
+     * @throws FileNotFoundException
+     */
+    protected void writeTestFileName(String name,
+                                     boolean append) throws FileNotFoundException {
+        File outFileName = new File(m_testNamesFile);
+        if ( ! outFileName.exists() ) {
+            append = false;
+        }
+        try (PrintStream ps = new PrintStream(new BufferedOutputStream(new FileOutputStream(outFileName, append)))) {
+            ps.print(name);
+        }
     }
 
     private void writeTestFile(String testFolder, String testClassName, Map<String, String> params) throws Exception {
         String template = TESTFILE_TEMPLATE;
+        // This could be made much faster by looking for all the strings
+        // with a regular expression, rather than one at a time.
         for (Map.Entry<String, String> entry : params.entrySet()) {
             String pattern = "@" + entry.getKey() + "@";
             String value   = params.get(entry.getKey());
             template = template.replace(pattern, value);
         }
-        File outputDir = new File(String.format("%s/%s/%s", m_VoltDBRootDirName, m_testGenDir, testFolder));
+        if (template.isEmpty()) {
+            throw new PlanningErrorException("Cannot create C++ Unit Test source from template.  This is a bug.");
+        }
+        File outputDir = new File(String.format("%s/%s", m_testGenPath, testFolder));
         if (! outputDir.exists() && !outputDir.mkdirs()) {
             throw new IOException("Cannot make test source folder \"" + outputDir + "\"");
         }

tests/frontend/org/voltdb/planner/eegentests/GenerateEETests.java

@@ -66,11 +66,11 @@
 public class GenerateEETests extends EEPlanGenerator {
     private static final String DDL_FILENAME = "testplans-ee-generators.sql";
 
-    @Override
-    public void setUp() throws Exception {
+    public void setUp(String[] args) throws Exception {
         super.setUp(GenerateEETests.class.getResource(DDL_FILENAME),
                     "testplansgenerator",
                     true);
+        processArgs(args);
     }
 
 
@@ -775,9 +775,8 @@ protected void tearDown() throws Exception {
 
     public static void main(String args[]) {
         GenerateEETests tg = new GenerateEETests();
-        tg.processArgs(args);
         try {
-            tg.setUp();
+            tg.setUp(args);
             tg.generatedPlannerTest();
             tg.generatedInsertPolygonTest();
             tg.generatedCountPlan();

tools/generate-ee-unit-tests.sh

@@ -9,7 +9,7 @@
 # it's convenient and useful for debugging to put it here, so that the
 # tests can easily be generated manually.
 
-BUILD=release
+BUILD_TYPE=release
 VERBOSE=
 ECHO=+x
 
@@ -78,14 +78,24 @@ if [ -z "$TEST_CLASSES" ] ; then
     exit 100
 fi
 
-SRC_DIR="$VOLTDB_ROOT/tests/ee/"
-GENERATED_DIR='ee_auto_generated_unit_tests'
+SRC_DIR="$VOLTDB_ROOT/tests/ee"
+GENERATED_DIR_NAME='ee_auto_generated_unit_tests'
+GENERATED_DIR_PATH="${SRC_DIR}/${GENERATED_DIR_NAME}"
 OBJDIR="$VOLTDB_ROOT/obj/${BUILD_TYPE}"
+TEST_NAMES_FILE="${GENERATED_DIR_PATH}/generated_tests.txt"
+# echo "SRC_DIR=${SRC_DIR}"
+# echo "GENERATED_DIR_PATH=${GENERATED_DIR_PATH}"
+# echo "TEST_NAMES_FILE=${TEST_NAMES_FILE}"
+# echo "OBJDIR=${OBJDIR}"
+#
+# Empty out the test names file.
+#
+rm -f "$TEST_NAMES_FILE"
 for CLASS in $TEST_CLASSES; do
     (set $ECHO; java $VERBOSE \
                      -cp ${OBJDIR}/prod:${OBJDIR}/test:${VOLTDB_ROOT}/lib/\*:${VOLTDB_ROOT}/third_party/java/jars/\* \
                      -Dlog4j.configuration=file:${VOLTDB_ROOT}/tests/log4j-allconsole.xml $CLASS \
-                     --test-source-dir="$SRC_DIR" \
-                     --test-generated-dir="$GENERATED_DIR"
+                     --generated-source-dir="$GENERATED_DIR_PATH" \
+                     --test-names-file="$TEST_NAMES_FILE"
     )
 done