Parallel search

Author: p-e-w

README.md

@@ -84,6 +84,7 @@ case class Configuration(
 	recurrenceRelations: Boolean,
 	combinatorialFunctions: Boolean,
 	transcendentalFunctions: Boolean,
+	parallelSearch: Boolean,
 	numericalTest: Boolean,
 	printProgress: Boolean,
 	outputLaTeX: Boolean

build.sbt

@@ -1,6 +1,6 @@
 name := "Sequencer"
 
-version := "1.2.1"
+version := "1.3.0"
 
 organization := "com.worldwidemann"
 

src/main/scala/com/worldwidemann/sequencer/FormulaGenerator.scala

@@ -90,6 +90,9 @@ class FormulaGenerator(configuration: Configuration) {
         }))
       else List()))
 
+  // Ensures that TreeGenerator builds all trees required to accommodate the expressions above
+  def getMaxChildren = expressions.size - 1
+
   private def rollNodeExpression(node: Node) = {
     val allowedExpressions = expressions(node.children.size)
     node.expressionIndex = if (node.expressionIndex + 1 >= allowedExpressions.size) 0 else node.expressionIndex + 1
@@ -98,33 +101,28 @@ class FormulaGenerator(configuration: Configuration) {
     node.expressionIndex == 0
   }
 
+  // Generates all formulas based on the specified expression tree skeleton.
   // The callback pattern is much more efficient than returning a collection
   // because the number of formulas grows extremely fast with the number of nodes
-  def getFormulas(nodes: Int, formulaCallback: Node => Unit, progressCallback: Double => Unit) = {
-    val trees = new TreeGenerator(expressions.size - 1).getTrees(nodes)
-
-    for ((tree, i) <- trees.view.zipWithIndex) {
-      val nodes = tree.getTreeNodes
-      // Set initial expressions
-      nodes.foreach(rollNodeExpression)
+  def getFormulas(tree: Node, formulaCallback: Node => Unit) {
+    val nodes = tree.getTreeNodes
+    // Set initial expressions
+    nodes.foreach(rollNodeExpression)
 
-      var lastRolled = false
-      while (!lastRolled) {
-        // Pass the formula in its current form.
-        // The object is mutable, so any processing has to happen in the callback
-        formulaCallback(tree)
+    var lastRolled = false
+    while (!lastRolled) {
+      // Pass the formula in its current form.
+      // The object is mutable, so any processing has to happen in the callback
+      formulaCallback(tree)
 
-        var j = 0
-        var rolled = true
-        while (j < nodes.size && rolled) {
-          rolled = rollNodeExpression(nodes(j))
-          j += 1
-          if (j == nodes.size && rolled)
-            lastRolled = true
-        }
+      var i = 0
+      var rolled = true
+      while (i < nodes.size && rolled) {
+        rolled = rollNodeExpression(nodes(i))
+        i += 1
+        if (i == nodes.size && rolled)
+          lastRolled = true
       }
-
-      progressCallback((i + 1).toDouble / trees.size)
     }
   }
 }

src/main/scala/com/worldwidemann/sequencer/Sequencer.scala

@@ -10,54 +10,95 @@
 
 package com.worldwidemann.sequencer
 
-import scala.collection.mutable.ListBuffer
+import java.util.concurrent.atomic.AtomicInteger
+import scala.collection.mutable.SynchronizedQueue
+import scala.collection.generic.Growable
 
 case class Configuration(maximumComplexity: Int, maximumIdentifications: Int, predictionLength: Int,
                          recurrenceRelations: Boolean, combinatorialFunctions: Boolean, transcendentalFunctions: Boolean,
-                         numericalTest: Boolean, printProgress: Boolean, outputLaTeX: Boolean)
+                         parallelSearch: Boolean, numericalTest: Boolean, printProgress: Boolean, outputLaTeX: Boolean)
 
 case class SequenceIdentification(formula: String, continuation: Seq[String])
 
 class Sequencer(configuration: Configuration) {
+  // Note that neither of the two classes keeps a state,
+  // so sharing these objects is thread safe
+  private val formulaGenerator = new FormulaGenerator(configuration)
+  private val treeGenerator = new TreeGenerator(formulaGenerator.getMaxChildren)
+
   def identifySequence(sequence: Seq[String]): Seq[SequenceIdentification] = {
     val sequenceSimplified = sequence.map(Simplifier.simplify)
     val sequenceNumerical = sequenceSimplified.map(Utilities.getNumericalValue)
 
-    val identifications = new ListBuffer[SequenceIdentification]
+    val identifications = new SynchronizedQueue[SequenceIdentification]
 
     for (nodes <- 1 to configuration.maximumComplexity) {
-      new FormulaGenerator(configuration).getFormulas(nodes, formula => {
-        // Consider recurrence relations only if they predict at least one element
-        // of the sequence without referencing a seed value
-        if (sequence.size > 2 * (Utilities.getStartIndex(formula) - 1)) {
-          if (!configuration.numericalTest || Verifier.testFormula(formula, sequenceNumerical)) {
-            // Sequence matched numerically (or test skipped) => verify symbolically
-            if (Verifier.verifyFormula(formula, sequenceSimplified)) {
-              try {
-                val continuation = Predictor.predict(formula, sequenceSimplified, configuration.predictionLength)
-                  .map(element => if (configuration.outputLaTeX) Utilities.getLaTeX(element) else element)
-                identifications += SequenceIdentification(getFullFormula(formula, sequenceSimplified), continuation)
-                if (configuration.maximumIdentifications > 0 && identifications.distinct.size >= configuration.maximumIdentifications)
-                  return identifications.distinct.sortBy(_.formula.length)
-              } catch {
-                // Occasionally, simplification or prediction throw an exception although
-                // symbolic verification did not. This indicates a bug in Symja
-                // and is simply ignored
-                case e: Exception => {}
-              }
-            }
-          }
-        }
-      }, progress => {
-        if (configuration.printProgress)
-          print("\rTrying formulas with complexity " + nodes + "... " + "%3d".format((progress * 100).round) + " %")
+      val trees = treeGenerator.getTrees(nodes)
+      val progress = new AtomicInteger(0)
+
+      // TreeGenerator generates tree objects that are completely independent of each other,
+      // so searching for formulas based on them can be parallelized
+      (if (configuration.parallelSearch) trees.par else trees).foreach(tree => {
+        identifySequenceTask(tree, sequenceSimplified, sequenceNumerical, identifications)
+
+        if (configuration.printProgress && !maximumReached(identifications))
+          // TODO: Concurrency issues with interleaving print statements?
+          print("\rTrying formulas with complexity " + nodes + "... " +
+            "%3d".format(((progress.incrementAndGet.toDouble / trees.size) * 100).round) + " %")
       })
 
+      if (maximumReached(identifications))
+        return processIdentifications(identifications)
+
       if (configuration.printProgress)
         println
     }
 
-    identifications.distinct.sortBy(_.formula.length)
+    processIdentifications(identifications)
+  }
+
+  // Parallel execution unit.
+  // Finds formulas generating the sequence based on the specified expression tree skeleton
+  private def identifySequenceTask(tree: Node, sequence: Seq[String], sequenceNumerical: Seq[Double],
+                                   identifications: Seq[SequenceIdentification] with Growable[SequenceIdentification]) {
+    formulaGenerator.getFormulas(tree, formula => {
+      // Consider recurrence relations only if they predict at least one element
+      // of the sequence without referencing a seed value
+      if (sequence.size > 2 * (Utilities.getStartIndex(formula) - 1)) {
+        if (!configuration.numericalTest || Verifier.testFormula(formula, sequenceNumerical)) {
+          // Sequence matched numerically (or test skipped) => verify symbolically
+          if (Verifier.verifyFormula(formula, sequence)) {
+            try {
+              val continuation = Predictor.predict(formula, sequence, configuration.predictionLength)
+                .map(element => if (configuration.outputLaTeX) Utilities.getLaTeX(element) else element)
+              identifications += SequenceIdentification(getFullFormula(formula, sequence), continuation)
+            } catch {
+              // Occasionally, simplification or prediction throw an exception although
+              // symbolic verification did not. This indicates a bug in Symja
+              // and is simply ignored
+              case e: Exception => {}
+            }
+          }
+        }
+      }
+
+      if (maximumReached(identifications))
+        return
+    })
+  }
+
+  private def maximumReached(identifications: Seq[SequenceIdentification]) =
+    configuration.maximumIdentifications > 0 && identifications.distinct.size >= configuration.maximumIdentifications
+
+  private def processIdentifications(identifications: Seq[SequenceIdentification]) = {
+    // Sort alphabetically before sorting by length to get deterministic ordering independent of search order
+    val sortedIdentifications = identifications.distinct.sortBy(_.formula).sortBy(_.formula.length)
+    if (configuration.maximumIdentifications > 0)
+      // Necessary because the tasks might find additional formulas
+      // before they determine that the maximum has been reached and return
+      sortedIdentifications.take(configuration.maximumIdentifications)
+    else
+      sortedIdentifications
   }
 
   // Returns the full descriptive string form of the formula,

src/main/scala/com/worldwidemann/sequencer/SequencerRunner.scala

@@ -18,8 +18,8 @@ import scala.collection.mutable.ListBuffer
 import scopt.OptionParser
 
 object SequencerRunner {
-  def main(args: Array[String]): Unit = {
-    println("Sequencer 1.2.1 (https://github.com/p-e-w/sequencer)\n")
+  def main(args: Array[String]) {
+    println("Sequencer 1.3.0 (https://github.com/p-e-w/sequencer)\n")
 
     // Suppress annoying Symja console output (idea from http://stackoverflow.com/a/8363580).
     // This is a very brittle solution. In particular, if we do not use the Console stream
@@ -36,7 +36,10 @@ object SequencerRunner {
       } text ("search depth (maximum number of nodes in expression tree) [default: 6]")
       opt[Int]('r', "results") action { (x, c) =>
         c.copy(maximumIdentifications = x)
-      } text ("maximum number of formulas to return, 0 for unbounded [default: 5]")
+      } text ("maximum number of formulas to return, 0 for unlimited [default: 5]. " +
+        "Note that for parallel searches, the order in which formulas are found is not deterministic " +
+        "so limiting the number of search results can lead to unreproducible searches " +
+        "unless the --sequential option is also used.")
       opt[Int]('p', "predict") action { (x, c) =>
         c.copy(predictionLength = x)
       } text ("number of elements to predict in sequence continuation [default: 5]")
@@ -49,6 +52,9 @@ object SequencerRunner {
       opt[Unit]('t', "no-transcendentals") action { (x, c) =>
         c.copy(transcendentalFunctions = false)
       } text ("do not search for transcendental functions (speeds up search)")
+      opt[Unit]('q', "sequential") action { (x, c) =>
+        c.copy(parallelSearch = false)
+      } text ("disable search parallelization (single-threaded search)")
       opt[Unit]('s', "symbolic") action { (x, c) =>
         c.copy(numericalTest = false)
       } text ("skip numerical test (symbolic verification only; slows down search)")
@@ -63,7 +69,7 @@ object SequencerRunner {
       } text ("list of numbers to search for (symbolic expressions allowed)")
     }
 
-    parser.parse(args, Configuration(6, 5, 5, true, true, true, true, true, false)) match {
+    parser.parse(args, Configuration(6, 5, 5, true, true, true, true, true, true, false)) match {
       case Some(configuration) => {
         println("Searching for formulas for sequence " +
           (if (configuration.outputLaTeX)

src/main/scala/com/worldwidemann/sequencer/Utilities.scala

@@ -17,9 +17,8 @@ import org.matheclipse.core.eval.EvalUtilities
 import org.matheclipse.core.eval.TeXUtilities
 
 object Utilities {
-  private val evaluator = new EvalUtilities(false, true)
-
-  def evaluateSymja(expression: String) = evaluator.evaluate(expression).toString
+  // Symja isn't thread safe, so the EvalUtilities object can not be shared between calls
+  def evaluateSymja(expression: String) = new EvalUtilities(false, true).evaluate(expression).toString
 
   def isNumerical(value: Double) = !value.isNaN && !value.isInfinite
 
@@ -31,11 +30,9 @@ object Utilities {
     case e: Exception => false
   }
 
-  private val texUtilities = new TeXUtilities(new EvalEngine(true), true)
-
   def getLaTeX(expression: String) = {
     val writer = new StringWriter
-    texUtilities.toTeX(expression, writer)
+    new TeXUtilities(new EvalEngine(true), true).toTeX(expression, writer)
     writer.toString
   }