From b1cf4489b7eae22b223e932ddd53c031951e39a2 Mon Sep 17 00:00:00 2001 From: mandar2812 Date: Wed, 1 May 2019 13:57:40 +0200 Subject: [PATCH 1/3] Compatibility with ammonite 1.6.6 --- .../helios/core/timelag/package.scala | 14 ++++++------- .../PlasmaML/helios/fte/package.scala | 20 +++++++++---------- .../mandar2812/PlasmaML/helios/package.scala | 8 ++++---- project/Dependencies.scala | 4 ++-- 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/helios/src/main/scala/io/github/mandar2812/PlasmaML/helios/core/timelag/package.scala b/helios/src/main/scala/io/github/mandar2812/PlasmaML/helios/core/timelag/package.scala index 81554fd7..8fad80c4 100644 --- a/helios/src/main/scala/io/github/mandar2812/PlasmaML/helios/core/timelag/package.scala +++ b/helios/src/main/scala/io/github/mandar2812/PlasmaML/helios/core/timelag/package.scala @@ -733,7 +733,7 @@ package object timelag { browser_plots = false, identifier = identifier ) - mkdir ! directory / res.results.summary_dir.segments.last + mkdir ! directory / res.results.summary_dir.segments.toSeq.last }) //Create a manifest file @@ -750,7 +750,7 @@ package object timelag { s"${r.config.reg_type.getOrElse("NA")}," + s"${r.config.actual_input_shape.scalar.asInstanceOf[Int]}," + s"${r.config.input_shape.scalar.asInstanceOf[Int]}," + - s"${r.results.summary_dir.segments.last}" + s"${r.results.summary_dir.segments.toSeq.last}" ) .mkString("\n") @@ -769,14 +769,14 @@ package object timelag { val files_to_copy = ls ! res.results.summary_dir |? ( f => - f.segments.last.contains(".csv") || - f.segments.last.contains(".png") || - f.segments.last.contains(".pdf") || - f.segments.last.contains(".json") + f.segments.toSeq.last.contains(".csv") || + f.segments.toSeq.last.contains(".png") || + f.segments.toSeq.last.contains(".pdf") || + f.segments.toSeq.last.contains(".json") ) files_to_copy.foreach( - f => cp.into(f, directory / res.results.summary_dir.segments.last) + f => cp.into(f, directory / res.results.summary_dir.segments.toSeq.last) ) }) diff --git a/helios/src/main/scala/io/github/mandar2812/PlasmaML/helios/fte/package.scala b/helios/src/main/scala/io/github/mandar2812/PlasmaML/helios/fte/package.scala index bf9800a3..3d4a9925 100644 --- a/helios/src/main/scala/io/github/mandar2812/PlasmaML/helios/fte/package.scala +++ b/helios/src/main/scala/io/github/mandar2812/PlasmaML/helios/fte/package.scala @@ -547,9 +547,9 @@ package object fte { } val use_cached_data = if (use_cached_config) { - val training_data_files = ls ! tf_summary_dir |? (_.segments.last + val training_data_files = ls ! tf_summary_dir |? (_.segments.toSeq.last .contains("training_data_")) - val test_data_files = ls ! tf_summary_dir |? (_.segments.last + val test_data_files = ls ! tf_summary_dir |? (_.segments.toSeq.last .contains("training_data_")) training_data_files.length > 0 && test_data_files.length > 0 @@ -596,9 +596,9 @@ package object fte { val dataset = if (use_cached_config && use_cached_data) { println("Using previously cached data set") - val training_data_file = (ls ! tf_summary_dir |? (_.segments.last + val training_data_file = (ls ! tf_summary_dir |? (_.segments.toSeq.last .contains("training_data_"))).last - val test_data_file = (ls ! tf_summary_dir |? (_.segments.last + val test_data_file = (ls ! tf_summary_dir |? (_.segments.toSeq.last .contains("test_data_"))).last read_data_set( @@ -709,14 +709,14 @@ package object fte { val checkpoints = if (exists ! tf_summary_dir) - ls ! tf_summary_dir |? (_.isFile) |? (_.segments.last + ls ! tf_summary_dir |? (_.isFile) |? (_.segments.toSeq.last .contains("model.ckpt-")) else Seq() val checkpoint_max = if (checkpoints.isEmpty) 0 else - (checkpoints | (_.segments.last + (checkpoints | (_.segments.toSeq.last .split("-") .last .split('.') @@ -1089,9 +1089,9 @@ package object fte { } val use_cached_data = if (use_cached_config) { - val training_data_files = ls ! tf_summary_dir |? (_.segments.last + val training_data_files = ls ! tf_summary_dir |? (_.segments.toSeq.last .contains("training_data_")) - val test_data_files = ls ! tf_summary_dir |? (_.segments.last + val test_data_files = ls ! tf_summary_dir |? (_.segments.toSeq.last .contains("training_data_")) training_data_files.length > 0 && test_data_files.length > 0 @@ -1138,9 +1138,9 @@ package object fte { val dataset = if (use_cached_config && use_cached_data) { println("Using previously cached data set") - val training_data_file = (ls ! tf_summary_dir |? (_.segments.last + val training_data_file = (ls ! tf_summary_dir |? (_.segments.toSeq.last .contains("training_data_"))).last - val test_data_file = (ls ! tf_summary_dir |? (_.segments.last + val test_data_file = (ls ! tf_summary_dir |? (_.segments.toSeq.last .contains("test_data_"))).last read_data_set( diff --git a/helios/src/main/scala/io/github/mandar2812/PlasmaML/helios/package.scala b/helios/src/main/scala/io/github/mandar2812/PlasmaML/helios/package.scala index e67b937f..c11193a6 100644 --- a/helios/src/main/scala/io/github/mandar2812/PlasmaML/helios/package.scala +++ b/helios/src/main/scala/io/github/mandar2812/PlasmaML/helios/package.scala @@ -420,14 +420,14 @@ package object helios { val checkpoints = if (exists(tf_summary_dir)) - ls ! tf_summary_dir |? (_.isFile) |? (_.segments.last + ls ! tf_summary_dir |? (_.isFile) |? (_.segments.toSeq.last .contains("model.ckpt-")) else Seq() val checkpoint_max = if (checkpoints.isEmpty) 0 else - (checkpoints | (_.segments.last + (checkpoints | (_.segments.toSeq.last .split("-") .last .split('.') @@ -1339,14 +1339,14 @@ package object helios { val checkpoints = if (exists ! tf_summary_dir) - ls ! tf_summary_dir |? (_.isFile) |? (_.segments.last + ls ! tf_summary_dir |? (_.isFile) |? (_.segments.toSeq.last .contains("model.ckpt-")) else Seq() val checkpoint_max = if (checkpoints.isEmpty) 0 else - (checkpoints | (_.segments.last + (checkpoints | (_.segments.toSeq.last .split("-") .last .split('.') diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 0f96b248..91abe0ea 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -6,7 +6,7 @@ object Dependencies { val scala_major = 2.12 - val scala_minor = 4 + val scala_minor = 8 val scala = s"$scala_major.$scala_minor" @@ -28,7 +28,7 @@ object Dependencies { val dataDirectory = settingKey[File]("The directory holding the data files for running example scripts") val latest_dynaml_release = "v2.0-beta.1" - val latest_dynaml_dev_release = "v2.0" + val latest_dynaml_dev_release = "v2.0-amm-1.6.6" val dynaml_branch = "" From 8526f1bd95de5d7ea2718ff9e55c85d1c61ec16b Mon Sep 17 00:00:00 2001 From: mandar2812 Date: Wed, 1 May 2019 15:20:04 +0200 Subject: [PATCH 2/3] [helios]: Bug Fix --- helios/scripts/csss.sc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/helios/scripts/csss.sc b/helios/scripts/csss.sc index 5f637b59..90f33f82 100644 --- a/helios/scripts/csss.sc +++ b/helios/scripts/csss.sc @@ -2,13 +2,13 @@ import $exec.env import ammonite.ops.ImplicitWd._ def experiments() = - ls ! env.summary_dir |? (_.isDir) |? (_.segments.last.contains("fte")) + ls ! env.summary_dir |? (_.isDir) |? (_.segments.toSeq.last.contains("fte")) def scatter_plots_test(summary_dir: Path) = - ls ! summary_dir |? (_.segments.last.contains("scatter_test")) + ls ! summary_dir |? (_.segments.toSeq.last.contains("scatter_test")) def scatter_plots_train(summary_dir: Path) = - ls ! summary_dir |? (_.segments.last.contains("scatter_train")) + ls ! summary_dir |? (_.segments.toSeq.last.contains("scatter_train")) val script = pwd / 'helios / 'scripts / "visualise_tl.R" From 0ec3c71031d5fac1306f87685bd274f5516d02b4 Mon Sep 17 00:00:00 2001 From: mandar2812 Date: Wed, 1 May 2019 15:38:51 +0200 Subject: [PATCH 3/3] [helios]: Fix to fte data processing --- .../mandar2812/PlasmaML/helios/fte/data/package.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/helios/src/main/scala/io/github/mandar2812/PlasmaML/helios/fte/data/package.scala b/helios/src/main/scala/io/github/mandar2812/PlasmaML/helios/fte/data/package.scala index 70cf26b1..739c946b 100644 --- a/helios/src/main/scala/io/github/mandar2812/PlasmaML/helios/fte/data/package.scala +++ b/helios/src/main/scala/io/github/mandar2812/PlasmaML/helios/fte/data/package.scala @@ -803,7 +803,7 @@ package object data { def write_exp_config(config: FteOmniConfig, dir: Path): Unit = { if (!(exists ! dir / "config.json")) { val config_json = write_json(config) - write(dir / "config.json", config_json) + write(dir / "config.json", config_json, createFolders = true) } } @@ -832,14 +832,16 @@ package object data { line => write.append( directory / s"training_data_${identifier}.json", - s"${line}\n" + s"${line}\n", + createFolders = true ) val write_pattern_test: String => Unit = line => write.append( directory / s"test_data_${identifier}.json", - s"${line}\n" + s"${line}\n", + createFolders = true ) dataset.training_dataset