Added support for ram plotting in new CLI interface. (#229)

* Added support for `ramplot` command in new CLI interface.
* Fixed final f7 value pointing to a non-existent proof.
* Added some more plot validation work.
* Fixed issue with `diskplot --sizes` crash

.vscode/launch.json

@@ -33,10 +33,12 @@
             "args": [
                 "-f", "ade0cc43610ce7540ab96a524d0ab17f5df7866ef13d1221a7203e5d10ad2a4ae37f7b73f6cdfd6ddf4122e8a1c2f8ef",
                 "-p", "80a836a74b077cabaca7a76d1c3c9f269f7f3a8f2fa196a65ee8953eb81274eb8b7328d474982617af5a0fe71b47e9b8",
-                "-n", "2",
+                "-i", "c6b84729c23dc6d60c92f22c17083f47845c1179227c5509f07a5d2804a7b835",
+                // "-n", "1",
                 "-w",
                 "-v",
-                "${workspaceFolder}/.sandbox"
+                "ramplot",
+                "~/plot/tmp"
             ]
         },
 
@@ -73,33 +75,35 @@
                 "-p", "80a836a74b077cabaca7a76d1c3c9f269f7f3a8f2fa196a65ee8953eb81274eb8b7328d474982617af5a0fe71b47e9b8",
 //                 "-c", "xch1uf48n3f50xrs7zds0uek9wp9wmyza6crnex6rw8kwm3jnm39y82q5mvps6",
                 "-t", "62",
-                "-w",
+                // "-w",
                 // "-v",
-                "-n", "3",
+                // "-n", "3",
                 "-i", "c6b84729c23dc6d60c92f22c17083f47845c1179227c5509f07a5d2804a7b835",    // No overflow
                 "--memo", "80a836a74b077cabaca7a76d1c3c9f269f7f3a8f2fa196a65ee8953eb81274eb8b7328d474982617af5a0fe71b47e9b8ade0cc43610ce7540ab96a524d0ab17f5df7866ef13d1221a7203e5d10ad2a4ae37f7b73f6cdfd6ddf4122e8a1c2f8ef01b7bf8a22a9ac82a003e07b551c851ea683839f3e1beb8ac9ede57d2c020669",
 
                 // "-i", "7a709594087cca18cffa37be61bdecf9b6b465de91acb06ecb6dbe0f4a536f73",    // Yes overflow
                 // "--memo", "80a836a74b077cabaca7a76d1c3c9f269f7f3a8f2fa196a65ee8953eb81274eb8b7328d474982617af5a0fe71b47e9b8ade0cc43610ce7540ab96a524d0ab17f5df7866ef13d1221a7203e5d10ad2a4ae37f7b73f6cdfd6ddf4122e8a1c2f8ef207d52406afa2b6d7d92ea778f407205bd9dca40816c1b1cacfca2a6612b93eb",
 
                 "--show-memo",
+
                 "diskplot",
+
                 "-t1", "~/plot/tmp",
                 "--f1-threads", "24",
                 // "--fp-threads", "62",
                 "--c-threads", "28",
                 "--p2-threads", "24",
 
                 // "-a",
-                "--cache", "196G",
+                // "--cache", "110G",
                 // "--cache", "99G",
-                // "--cache", "200G",
+                "--cache", "200G",
                 // "--cache", "64G",
                 // "-s",
                 // "--k32-bounded",
-                // "-b", "64",
+                "-b", "64",
                 // "--sizes",
-                "-b", "128",
+                // "-b", "128",
                 // "-b", "256",
 
                 "--c-threads", "26",
@@ -172,18 +176,17 @@
 
             "args": [
                 "-b",
-                // "F1CompressToBits"
                 // "F1GenBucketized"
                 // "FxSort"
                 // "FxDisk"
                 // "F1Disk"
                 // "PairsAndMap"
-                "bucket-slice-write"
+                // "bucket-slice-write"
+                // "line-point-deltas"
             ]
         }
 
 
-
         ,{
             "name"       : "Plot Tool",
 
@@ -205,20 +208,23 @@
 
             "args": [
                 /// Validate
-                "-t", "32",
+                // "-t", "32",
                 // "-t", "1",
-                // "validate",
+                "validate",
+
+                "--f7", "2534554965",
+                "~/plot/tmp/plot-k32-2022-10-18-22-25-c6b84729c23dc6d60c92f22c17083f47845c1179227c5509f07a5d2804a7b835.plot"
+
                 // "-m",
                 // "-u",
-                // "-o", "99.7",
-                // "/mnt/p5510a/disk_tmp/plot-k32-2022-04-12-13-03-c6b84729c23dc6d60c92f22c17083f47845c1179227c5509f07a5d2804a7b835.plot.tmp",
+                // "~/plot/tmp/plot-k32-2022-10-17-15-05-c6b84729c23dc6d60c92f22c17083f47845c1179227c5509f07a5d2804a7b835.plot",
 
                 // "/mnt/p5510a/disk_tmp/plot.dat"
 
                 /// Compare
-                "plotcmp",
-                "/mnt/p5510a/disk_tmp/plot-k32-2022-04-12-13-53-c6b84729c23dc6d60c92f22c17083f47845c1179227c5509f07a5d2804a7b835.plot",
-                "/mnt/p5510a/disk_tmp/plot-k32-2022-04-12-13-03-c6b84729c23dc6d60c92f22c17083f47845c1179227c5509f07a5d2804a7b835.plot"
+                // "plotcmp",
+                // "/mnt/p5510a/disk_tmp/plot-k32-2022-04-12-13-53-c6b84729c23dc6d60c92f22c17083f47845c1179227c5509f07a5d2804a7b835.plot",
+                // "/mnt/p5510a/disk_tmp/plot-k32-2022-04-12-13-03-c6b84729c23dc6d60c92f22c17083f47845c1179227c5509f07a5d2804a7b835.plot"
             ]
         },
 

src/main.cpp

@@ -1,6 +1,7 @@
 #include "plotting/GlobalPlotConfig.h"
 #include "util/CliParser.h"
 #include "plotdisk/DiskPlotter.h"
+#include "plotmem/MemPlotter.h"
 #include "Version.h"
 
 #if PLATFORM_IS_UNIX
@@ -27,12 +28,20 @@ void PlotCompareMain( GlobalPlotConfig& gCfg, CliParser& cli );
 void PlotCompareMainPrintUsage();
 
 
+enum class PlotterType
+{
+    None = 0,
+    Ram,
+    Disk
+};
 
 struct Plotter 
 {
+    PlotterType type;
     union {
         void* _ptr;
         DiskPlotter* disk;
+        MemPlotter*  mem;
     };
 };
 
@@ -48,12 +57,12 @@ int main( int argc, const char* argv[] )
     Log::Line( "*** Warning: Debug mode is ENABLED ***" );
 #endif
 
-    ZeroMem( &_plotter );
+    _plotter = {};
 
     GlobalPlotConfig cfg;
     ParseCommandLine( cfg, --argc, ++argv );
 
-    FatalIf( !_plotter._ptr, "No command chosen." );
+    FatalIf( !_plotter._ptr, "No plot command chosen." );
 
 
     const int64 plotCount = cfg.plotCount > 0 ? (int64)cfg.plotCount : std::numeric_limits<int64>::max();
@@ -121,7 +130,20 @@ int main( int argc, const char* argv[] )
         }
         Log::Line( "" );
 
-        if( _plotter.disk )
+        if( _plotter.type == PlotterType::Ram )
+        {
+            auto& plotter = *_plotter.mem;
+
+            PlotRequest req = {};
+            req.plotId      = plotId;
+            req.memo        = plotMemo;
+            req.memoSize    = plotMemoSize;
+            req.outPath     = plotOutPath;
+            req.IsFinalPlot = i == plotCount-1;
+
+            plotter.Run( req );
+        }
+        else if( _plotter.type == PlotterType::Disk )
         {
             auto& plotter = *_plotter.disk;
 
@@ -132,6 +154,10 @@ int main( int argc, const char* argv[] )
             req.plotFileName = plotFileName;
             plotter.Plot( req );
         }
+        else
+        {
+            Fatal( "Unknown plotter type." );
+        }
     }
 }
 
@@ -144,6 +170,9 @@ void ParseCommandLine( GlobalPlotConfig& cfg, int argc, const char* argv[] )
     const char* poolPublicKey       = nullptr;
     const char* poolContractAddress = nullptr;
 
+    DiskPlotter::Config diskCfg = {};
+    MemPlotConfig       ramCfg  = {};
+
     while( cli.HasArgs() )
     {
         if( cli.ReadU32( cfg.threadCount, "-t", "--threads" ) )
@@ -239,12 +268,22 @@ void ParseCommandLine( GlobalPlotConfig& cfg, int argc, const char* argv[] )
                 }
             #endif
 
-            DiskPlotter::Config diskCfg;
+            // DiskPlotter::Config diskCfg;
             diskCfg.globalCfg = &cfg;
             DiskPlotter::ParseCommandLine( cli, diskCfg );
 
-            _plotter.disk = new DiskPlotter( diskCfg );
+            _plotter.type = PlotterType::Disk;
+            break;
+        }
+        else if( cli.ArgConsume( "ramplot" ) )
+        {
+            ramCfg.threadCount   = cfg.threadCount == 0 ? 
+                                    SysHost::GetLogicalCPUCount() : 
+                                    bbclamp( cfg.threadCount, 1u, SysHost::GetLogicalCPUCount() );
+            ramCfg.warmStart     = cfg.warmStart;
+            ramCfg.gCfg          = &cfg;
 
+            _plotter.type = PlotterType::Ram;
             break;
         }
         else if( cli.ArgConsume( "iotest" ) )
@@ -374,6 +413,12 @@ void ParseCommandLine( GlobalPlotConfig& cfg, int argc, const char* argv[] )
 
     // Config Summary
     Log::Line( "" );
+    Log::Line( "Bladebit Chia Plotter" );
+    Log::Line( "Version      : %s", BLADEBIT_VERSION_STR   );
+    Log::Line( "Git Commit   : %s", BLADEBIT_GIT_COMMIT    );
+    Log::Line( "Compiled With: %s", BBGetCompilerVersion() );
+    Log::Line( "" );
+
     Log::Line( "[Global Plotting Config]" );
     if( cfg.plotCount == 0 )
         Log::Line( " Will create plots indefinitely." );
@@ -396,6 +441,23 @@ void ParseCommandLine( GlobalPlotConfig& cfg, int argc, const char* argv[] )
     Log::Line( " Output path           : %s", cfg.outputFolder );
 
     Log::Line( "" );
+
+    // Create plotter
+    switch( _plotter.type )
+    {
+        case PlotterType::Disk:
+            _plotter.disk = new DiskPlotter( diskCfg );
+            break;
+
+        case PlotterType::Ram:
+            _plotter.mem  = new MemPlotter( ramCfg );
+            break;
+
+        default:
+            Fatal( "No plotter chosen." );
+            break;
+    }
+    Log::Line( "" );
 }
 
 
@@ -404,6 +466,7 @@ static const char* USAGE = "bladebit [GLOBAL_OPTIONS] <command> [COMMAND_OPTIONS
 R"(
 [COMMANDS]
  diskplot   : Create a plot by making use of a disk.
+ ramplot    : Create a plot completely in-ram.
  iotest     : Perform a write and read test on a specified disk.
  memtest    : Perform a memory (RAM) copy test.
  validate   : Validates all entries in a plot to ensure they all evaluate to a valid proof.

src/plotdisk/DiskPlotter.cpp

@@ -314,16 +314,14 @@ void DiskPlotter::ParseCommandLine( CliParser& cli, Config& cfg )
             FatalIf( cfg.numBuckets < BB_DP_MIN_BUCKET_COUNT || cfg.numBuckets > BB_DP_MAX_BUCKET_COUNT,
                 "Buckets must be between %u and %u, inclusive.", (uint)BB_DP_MIN_BUCKET_COUNT, (uint)BB_DP_MAX_BUCKET_COUNT );
             FatalIf( ( cfg.numBuckets & ( cfg.numBuckets - 1 ) ) != 0, "Buckets must be power of 2." );
+            FatalIf( !cfg.tmpPath, "Please specify at least 1 temporary path." );
+
+            const uint32 threadCount = bbclamp<uint32>( cfg.globalCfg->threadCount, 1u, SysHost::GetLogicalCPUCount() );
 
             size_t heapSize = 0;
-            if( cfg.tmpPath )
-            {
-                cfg.tmpPath2 = cfg.tmpPath2 ? cfg.tmpPath2 : cfg.tmpPath;
-                heapSize = GetRequiredSizeForBuckets( cfg.bounded, cfg.numBuckets, cfg.tmpPath2, cfg.tmpPath, BB_DP_MAX_JOBS );
-            }
-            else
-                heapSize = GetRequiredSizeForBuckets( cfg.bounded, cfg.numBuckets, 1, 1, BB_DP_MAX_JOBS );
-
+            cfg.tmpPath2 = cfg.tmpPath2 ? cfg.tmpPath2 : cfg.tmpPath;
+            heapSize = GetRequiredSizeForBuckets( cfg.bounded, cfg.numBuckets, cfg.tmpPath2, cfg.tmpPath, threadCount );
+
             Log::Line( "Buckets: %u | Heap Sizes: %.2lf GiB", cfg.numBuckets, (double)heapSize BtoGB );
             exit( 0 );
         }

src/plotdisk/k32/FxBounded.inl

@@ -37,27 +37,6 @@
 #endif
 
 
-typedef uint32 K32Meta1;
-typedef uint64 K32Meta2;
-// struct K32Meta3 { uint32 m0, m1, m2; };
-struct K32Meta3 { uint64 m0, m1; };
-struct K32Meta4 { uint64 m0, m1; };
-struct K32NoMeta {};
-
-template<TableId rTable>
-struct K32MetaType{};
-
-template<> struct K32MetaType<TableId::Table1>{ using In = K32NoMeta; using Out = K32Meta1;  };
-template<> struct K32MetaType<TableId::Table2>{ using In = K32Meta1;  using Out = K32Meta2;  };
-template<> struct K32MetaType<TableId::Table3>{ using In = K32Meta2;  using Out = K32Meta4;  };
-template<> struct K32MetaType<TableId::Table4>{ using In = K32Meta4;  using Out = K32Meta4;  };
-template<> struct K32MetaType<TableId::Table5>{ using In = K32Meta4;  using Out = K32Meta3;  };
-template<> struct K32MetaType<TableId::Table6>{ using In = K32Meta3;  using Out = K32Meta2;  };
-template<> struct K32MetaType<TableId::Table7>{ using In = K32Meta2;  using Out = K32NoMeta; };
-
-template<TableId rTable> struct K32TYOut { using Type = uint64; };
-template<>               struct K32TYOut<TableId::Table7> { using Type = uint32; };
-
 template<TableId rTable, uint32 _numBuckets>
 class DiskPlotFxBounded
 {

src/plotmem/MemPhase3.cpp

@@ -152,12 +152,6 @@ uint64 MemPhase3::ProcessTable( uint32* lEntries, uint64* lpBuffer, Pair* rTable
             lEntries,     lEntriesSortTmp,
             newLength );
 
-        // #NOTE: Because the C2 table size is inferred by substracting table pointers
-        //        in chiapos, we need to make sure we don't have any f7 entries with the
-        //        value of 0xFFFFFFFF. See WriteC12Parallel in Phase4 for more details.
-        while( newLength && cx.t7YBuffer[newLength-1] == 0xFFFFFFFF )
-            --newLength;
-
         cx.entryCount[(uint)TableId::Table7] = newLength;
     }
 

src/plotmem/MemPlotter.h

@@ -1,10 +1,12 @@
 #pragma once
 #include "PlotContext.h"
+#include "plotting/GlobalPlotConfig.h"
 
 struct NumaInfo;
 
 struct MemPlotConfig
 {
+    GlobalPlotConfig* gCfg;
     uint threadCount;
     bool warmStart;
     bool noNUMA;

src/plotting/PlotTools.h

@@ -2,6 +2,8 @@
 #include "ChiaConsts.h"
 #include "util/KeyTools.h"
 
+#define BB_PLOT_PROOF_X_COUNT 32
+
 #define BB_PLOT_ID_LEN 32
 #define BB_PLOT_ID_HEX_LEN (BB_PLOT_ID_LEN * 2)
 

src/plotting/PlotTypes.h

@@ -1,5 +1,4 @@
 #pragma once
-#include "threading/AutoResetSignal.h"
 
 struct Pairs
 {
@@ -28,4 +27,20 @@ struct Pair
         return *this;
     }
 };
-static_assert( sizeof( Pair ) == 8, "Invalid Pair struct." );
+static_assert( sizeof( Pair ) == 8, "Invalid Pair struct." );
+
+
+enum class PlotTable
+{
+    Table1 = 0,
+    Table2,
+    Table3,
+    Table4,
+    Table5,
+    Table6,
+    Table7,
+    C1,
+    C2,
+    C3,
+}; ImplementArithmeticOps( PlotTable );
+