diff --git a/.vscode/launch.json b/.vscode/launch.json index 20733749..246a5532 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -33,10 +33,12 @@ "args": [ "-f", "ade0cc43610ce7540ab96a524d0ab17f5df7866ef13d1221a7203e5d10ad2a4ae37f7b73f6cdfd6ddf4122e8a1c2f8ef", "-p", "80a836a74b077cabaca7a76d1c3c9f269f7f3a8f2fa196a65ee8953eb81274eb8b7328d474982617af5a0fe71b47e9b8", - "-n", "2", + "-i", "c6b84729c23dc6d60c92f22c17083f47845c1179227c5509f07a5d2804a7b835", + // "-n", "1", "-w", "-v", - "${workspaceFolder}/.sandbox" + "ramplot", + "~/plot/tmp" ] }, @@ -73,9 +75,9 @@ "-p", "80a836a74b077cabaca7a76d1c3c9f269f7f3a8f2fa196a65ee8953eb81274eb8b7328d474982617af5a0fe71b47e9b8", // "-c", "xch1uf48n3f50xrs7zds0uek9wp9wmyza6crnex6rw8kwm3jnm39y82q5mvps6", "-t", "62", - "-w", + // "-w", // "-v", - "-n", "3", + // "-n", "3", "-i", "c6b84729c23dc6d60c92f22c17083f47845c1179227c5509f07a5d2804a7b835", // No overflow "--memo", "80a836a74b077cabaca7a76d1c3c9f269f7f3a8f2fa196a65ee8953eb81274eb8b7328d474982617af5a0fe71b47e9b8ade0cc43610ce7540ab96a524d0ab17f5df7866ef13d1221a7203e5d10ad2a4ae37f7b73f6cdfd6ddf4122e8a1c2f8ef01b7bf8a22a9ac82a003e07b551c851ea683839f3e1beb8ac9ede57d2c020669", @@ -83,7 +85,9 @@ // "--memo", "80a836a74b077cabaca7a76d1c3c9f269f7f3a8f2fa196a65ee8953eb81274eb8b7328d474982617af5a0fe71b47e9b8ade0cc43610ce7540ab96a524d0ab17f5df7866ef13d1221a7203e5d10ad2a4ae37f7b73f6cdfd6ddf4122e8a1c2f8ef207d52406afa2b6d7d92ea778f407205bd9dca40816c1b1cacfca2a6612b93eb", "--show-memo", + "diskplot", + "-t1", "~/plot/tmp", "--f1-threads", "24", // "--fp-threads", "62", @@ -91,15 +95,15 @@ "--p2-threads", "24", // "-a", - "--cache", "196G", + // "--cache", "110G", // "--cache", "99G", - // "--cache", "200G", + "--cache", "200G", // "--cache", "64G", // "-s", // "--k32-bounded", - // "-b", "64", + "-b", "64", // "--sizes", - "-b", "128", + // "-b", "128", // "-b", "256", "--c-threads", "26", @@ -172,18 +176,17 @@ "args": [ "-b", - // "F1CompressToBits" // "F1GenBucketized" // "FxSort" // "FxDisk" // "F1Disk" // "PairsAndMap" - "bucket-slice-write" + // "bucket-slice-write" + // "line-point-deltas" ] } - ,{ "name" : "Plot Tool", @@ -205,20 +208,23 @@ "args": [ /// Validate - "-t", "32", + // "-t", "32", // "-t", "1", - // "validate", + "validate", + + "--f7", "2534554965", + "~/plot/tmp/plot-k32-2022-10-18-22-25-c6b84729c23dc6d60c92f22c17083f47845c1179227c5509f07a5d2804a7b835.plot" + // "-m", // "-u", - // "-o", "99.7", - // "/mnt/p5510a/disk_tmp/plot-k32-2022-04-12-13-03-c6b84729c23dc6d60c92f22c17083f47845c1179227c5509f07a5d2804a7b835.plot.tmp", + // "~/plot/tmp/plot-k32-2022-10-17-15-05-c6b84729c23dc6d60c92f22c17083f47845c1179227c5509f07a5d2804a7b835.plot", // "/mnt/p5510a/disk_tmp/plot.dat" /// Compare - "plotcmp", - "/mnt/p5510a/disk_tmp/plot-k32-2022-04-12-13-53-c6b84729c23dc6d60c92f22c17083f47845c1179227c5509f07a5d2804a7b835.plot", - "/mnt/p5510a/disk_tmp/plot-k32-2022-04-12-13-03-c6b84729c23dc6d60c92f22c17083f47845c1179227c5509f07a5d2804a7b835.plot" + // "plotcmp", + // "/mnt/p5510a/disk_tmp/plot-k32-2022-04-12-13-53-c6b84729c23dc6d60c92f22c17083f47845c1179227c5509f07a5d2804a7b835.plot", + // "/mnt/p5510a/disk_tmp/plot-k32-2022-04-12-13-03-c6b84729c23dc6d60c92f22c17083f47845c1179227c5509f07a5d2804a7b835.plot" ] }, diff --git a/src/main.cpp b/src/main.cpp index 2376edf6..7938fc3e 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1,6 +1,7 @@ #include "plotting/GlobalPlotConfig.h" #include "util/CliParser.h" #include "plotdisk/DiskPlotter.h" +#include "plotmem/MemPlotter.h" #include "Version.h" #if PLATFORM_IS_UNIX @@ -27,12 +28,20 @@ void PlotCompareMain( GlobalPlotConfig& gCfg, CliParser& cli ); void PlotCompareMainPrintUsage(); +enum class PlotterType +{ + None = 0, + Ram, + Disk +}; struct Plotter { + PlotterType type; union { void* _ptr; DiskPlotter* disk; + MemPlotter* mem; }; }; @@ -48,12 +57,12 @@ int main( int argc, const char* argv[] ) Log::Line( "*** Warning: Debug mode is ENABLED ***" ); #endif - ZeroMem( &_plotter ); + _plotter = {}; GlobalPlotConfig cfg; ParseCommandLine( cfg, --argc, ++argv ); - FatalIf( !_plotter._ptr, "No command chosen." ); + FatalIf( !_plotter._ptr, "No plot command chosen." ); const int64 plotCount = cfg.plotCount > 0 ? (int64)cfg.plotCount : std::numeric_limits::max(); @@ -121,7 +130,20 @@ int main( int argc, const char* argv[] ) } Log::Line( "" ); - if( _plotter.disk ) + if( _plotter.type == PlotterType::Ram ) + { + auto& plotter = *_plotter.mem; + + PlotRequest req = {}; + req.plotId = plotId; + req.memo = plotMemo; + req.memoSize = plotMemoSize; + req.outPath = plotOutPath; + req.IsFinalPlot = i == plotCount-1; + + plotter.Run( req ); + } + else if( _plotter.type == PlotterType::Disk ) { auto& plotter = *_plotter.disk; @@ -132,6 +154,10 @@ int main( int argc, const char* argv[] ) req.plotFileName = plotFileName; plotter.Plot( req ); } + else + { + Fatal( "Unknown plotter type." ); + } } } @@ -144,6 +170,9 @@ void ParseCommandLine( GlobalPlotConfig& cfg, int argc, const char* argv[] ) const char* poolPublicKey = nullptr; const char* poolContractAddress = nullptr; + DiskPlotter::Config diskCfg = {}; + MemPlotConfig ramCfg = {}; + while( cli.HasArgs() ) { if( cli.ReadU32( cfg.threadCount, "-t", "--threads" ) ) @@ -239,12 +268,22 @@ void ParseCommandLine( GlobalPlotConfig& cfg, int argc, const char* argv[] ) } #endif - DiskPlotter::Config diskCfg; + // DiskPlotter::Config diskCfg; diskCfg.globalCfg = &cfg; DiskPlotter::ParseCommandLine( cli, diskCfg ); - _plotter.disk = new DiskPlotter( diskCfg ); + _plotter.type = PlotterType::Disk; + break; + } + else if( cli.ArgConsume( "ramplot" ) ) + { + ramCfg.threadCount = cfg.threadCount == 0 ? + SysHost::GetLogicalCPUCount() : + bbclamp( cfg.threadCount, 1u, SysHost::GetLogicalCPUCount() ); + ramCfg.warmStart = cfg.warmStart; + ramCfg.gCfg = &cfg; + _plotter.type = PlotterType::Ram; break; } else if( cli.ArgConsume( "iotest" ) ) @@ -374,6 +413,12 @@ void ParseCommandLine( GlobalPlotConfig& cfg, int argc, const char* argv[] ) // Config Summary Log::Line( "" ); + Log::Line( "Bladebit Chia Plotter" ); + Log::Line( "Version : %s", BLADEBIT_VERSION_STR ); + Log::Line( "Git Commit : %s", BLADEBIT_GIT_COMMIT ); + Log::Line( "Compiled With: %s", BBGetCompilerVersion() ); + Log::Line( "" ); + Log::Line( "[Global Plotting Config]" ); if( cfg.plotCount == 0 ) Log::Line( " Will create plots indefinitely." ); @@ -396,6 +441,23 @@ void ParseCommandLine( GlobalPlotConfig& cfg, int argc, const char* argv[] ) Log::Line( " Output path : %s", cfg.outputFolder ); Log::Line( "" ); + + // Create plotter + switch( _plotter.type ) + { + case PlotterType::Disk: + _plotter.disk = new DiskPlotter( diskCfg ); + break; + + case PlotterType::Ram: + _plotter.mem = new MemPlotter( ramCfg ); + break; + + default: + Fatal( "No plotter chosen." ); + break; + } + Log::Line( "" ); } @@ -404,6 +466,7 @@ static const char* USAGE = "bladebit [GLOBAL_OPTIONS] [COMMAND_OPTIONS R"( [COMMANDS] diskplot : Create a plot by making use of a disk. + ramplot : Create a plot completely in-ram. iotest : Perform a write and read test on a specified disk. memtest : Perform a memory (RAM) copy test. validate : Validates all entries in a plot to ensure they all evaluate to a valid proof. diff --git a/src/plotdisk/DiskPlotter.cpp b/src/plotdisk/DiskPlotter.cpp index 0f55f143..7679ce12 100644 --- a/src/plotdisk/DiskPlotter.cpp +++ b/src/plotdisk/DiskPlotter.cpp @@ -314,16 +314,14 @@ void DiskPlotter::ParseCommandLine( CliParser& cli, Config& cfg ) FatalIf( cfg.numBuckets < BB_DP_MIN_BUCKET_COUNT || cfg.numBuckets > BB_DP_MAX_BUCKET_COUNT, "Buckets must be between %u and %u, inclusive.", (uint)BB_DP_MIN_BUCKET_COUNT, (uint)BB_DP_MAX_BUCKET_COUNT ); FatalIf( ( cfg.numBuckets & ( cfg.numBuckets - 1 ) ) != 0, "Buckets must be power of 2." ); + FatalIf( !cfg.tmpPath, "Please specify at least 1 temporary path." ); + + const uint32 threadCount = bbclamp( cfg.globalCfg->threadCount, 1u, SysHost::GetLogicalCPUCount() ); size_t heapSize = 0; - if( cfg.tmpPath ) - { - cfg.tmpPath2 = cfg.tmpPath2 ? cfg.tmpPath2 : cfg.tmpPath; - heapSize = GetRequiredSizeForBuckets( cfg.bounded, cfg.numBuckets, cfg.tmpPath2, cfg.tmpPath, BB_DP_MAX_JOBS ); - } - else - heapSize = GetRequiredSizeForBuckets( cfg.bounded, cfg.numBuckets, 1, 1, BB_DP_MAX_JOBS ); - + cfg.tmpPath2 = cfg.tmpPath2 ? cfg.tmpPath2 : cfg.tmpPath; + heapSize = GetRequiredSizeForBuckets( cfg.bounded, cfg.numBuckets, cfg.tmpPath2, cfg.tmpPath, threadCount ); + Log::Line( "Buckets: %u | Heap Sizes: %.2lf GiB", cfg.numBuckets, (double)heapSize BtoGB ); exit( 0 ); } diff --git a/src/plotdisk/k32/FxBounded.inl b/src/plotdisk/k32/FxBounded.inl index 03aa1934..fbfb2e4e 100644 --- a/src/plotdisk/k32/FxBounded.inl +++ b/src/plotdisk/k32/FxBounded.inl @@ -37,27 +37,6 @@ #endif -typedef uint32 K32Meta1; -typedef uint64 K32Meta2; -// struct K32Meta3 { uint32 m0, m1, m2; }; -struct K32Meta3 { uint64 m0, m1; }; -struct K32Meta4 { uint64 m0, m1; }; -struct K32NoMeta {}; - -template -struct K32MetaType{}; - -template<> struct K32MetaType{ using In = K32NoMeta; using Out = K32Meta1; }; -template<> struct K32MetaType{ using In = K32Meta1; using Out = K32Meta2; }; -template<> struct K32MetaType{ using In = K32Meta2; using Out = K32Meta4; }; -template<> struct K32MetaType{ using In = K32Meta4; using Out = K32Meta4; }; -template<> struct K32MetaType{ using In = K32Meta4; using Out = K32Meta3; }; -template<> struct K32MetaType{ using In = K32Meta3; using Out = K32Meta2; }; -template<> struct K32MetaType{ using In = K32Meta2; using Out = K32NoMeta; }; - -template struct K32TYOut { using Type = uint64; }; -template<> struct K32TYOut { using Type = uint32; }; - template class DiskPlotFxBounded { diff --git a/src/plotmem/MemPhase3.cpp b/src/plotmem/MemPhase3.cpp index bc7cf634..c9f1b7c3 100644 --- a/src/plotmem/MemPhase3.cpp +++ b/src/plotmem/MemPhase3.cpp @@ -152,12 +152,6 @@ uint64 MemPhase3::ProcessTable( uint32* lEntries, uint64* lpBuffer, Pair* rTable lEntries, lEntriesSortTmp, newLength ); - // #NOTE: Because the C2 table size is inferred by substracting table pointers - // in chiapos, we need to make sure we don't have any f7 entries with the - // value of 0xFFFFFFFF. See WriteC12Parallel in Phase4 for more details. - while( newLength && cx.t7YBuffer[newLength-1] == 0xFFFFFFFF ) - --newLength; - cx.entryCount[(uint)TableId::Table7] = newLength; } diff --git a/src/plotmem/MemPlotter.h b/src/plotmem/MemPlotter.h index a7411fed..fb931c19 100644 --- a/src/plotmem/MemPlotter.h +++ b/src/plotmem/MemPlotter.h @@ -1,10 +1,12 @@ #pragma once #include "PlotContext.h" +#include "plotting/GlobalPlotConfig.h" struct NumaInfo; struct MemPlotConfig { + GlobalPlotConfig* gCfg; uint threadCount; bool warmStart; bool noNUMA; diff --git a/src/plotting/PlotTools.h b/src/plotting/PlotTools.h index b5e339fa..63ce8f41 100644 --- a/src/plotting/PlotTools.h +++ b/src/plotting/PlotTools.h @@ -2,6 +2,8 @@ #include "ChiaConsts.h" #include "util/KeyTools.h" +#define BB_PLOT_PROOF_X_COUNT 32 + #define BB_PLOT_ID_LEN 32 #define BB_PLOT_ID_HEX_LEN (BB_PLOT_ID_LEN * 2) diff --git a/src/plotting/PlotTypes.h b/src/plotting/PlotTypes.h index 6817b9ac..379970cf 100644 --- a/src/plotting/PlotTypes.h +++ b/src/plotting/PlotTypes.h @@ -1,5 +1,4 @@ #pragma once -#include "threading/AutoResetSignal.h" struct Pairs { @@ -28,4 +27,20 @@ struct Pair return *this; } }; -static_assert( sizeof( Pair ) == 8, "Invalid Pair struct." ); \ No newline at end of file +static_assert( sizeof( Pair ) == 8, "Invalid Pair struct." ); + + +enum class PlotTable +{ + Table1 = 0, + Table2, + Table3, + Table4, + Table5, + Table6, + Table7, + C1, + C2, + C3, +}; ImplementArithmeticOps( PlotTable ); + diff --git a/src/plotting/Tables.h b/src/plotting/Tables.h index 53b3924d..3acf900d 100644 --- a/src/plotting/Tables.h +++ b/src/plotting/Tables.h @@ -41,7 +41,7 @@ template<> struct TableMetaType { using MetaIn = Meta3; using template<> struct TableMetaType { using MetaIn = uint64; using MetaOut = NoMeta; }; -/// Helper for obtaining the correct fx (y) output type per table + template struct YOut { using Type = uint64; }; template<> struct YOut { using Type = uint32; }; @@ -172,3 +172,26 @@ template<> struct TableMetaOut }; // static_assert( sizeof( NoMeta ) == 0, "Invalid NoMeta" ); + + +/// For bounded k32 +typedef uint32 K32Meta1; +typedef uint64 K32Meta2; +struct K32Meta3 { uint64 m0, m1; }; +struct K32Meta4 { uint64 m0, m1; }; +struct K32NoMeta {}; + +template +struct K32MetaType{}; + +template<> struct K32MetaType{ using In = K32NoMeta; using Out = K32Meta1; }; +template<> struct K32MetaType{ using In = K32Meta1; using Out = K32Meta2; }; +template<> struct K32MetaType{ using In = K32Meta2; using Out = K32Meta4; }; +template<> struct K32MetaType{ using In = K32Meta4; using Out = K32Meta4; }; +template<> struct K32MetaType{ using In = K32Meta4; using Out = K32Meta3; }; +template<> struct K32MetaType{ using In = K32Meta3; using Out = K32Meta2; }; +template<> struct K32MetaType{ using In = K32Meta2; using Out = K32NoMeta; }; + +/// Helper for obtaining the correct fx (y) output type per table +template struct K32TYOut { using Type = uint64; }; +template<> struct K32TYOut { using Type = uint32; }; diff --git a/src/tools/PlotReader.cpp b/src/tools/PlotReader.cpp index 38e6f5fd..bb4a84d6 100644 --- a/src/tools/PlotReader.cpp +++ b/src/tools/PlotReader.cpp @@ -4,6 +4,7 @@ #include "plotting/PlotTools.h" #include "plotting/CTables.h" #include "plotting/DTables.h" +#include "plotmem/LPGen.h" /// /// Plot Reader @@ -25,6 +26,15 @@ PlotReader::~PlotReader() { free( _parkBuffer ); free( _deltasBuffer ); + + if( _c2Entries.values ) + bbvirtfreebounded( _c2Entries.values ); + + if( _c1Buffer ) + bbvirtfreebounded( _c1Buffer ); + + if( _c3Buffer.Ptr() ) + bbvirtfreebounded( _c3Buffer.Ptr() ); } //----------------------------------------------------------- @@ -35,9 +45,7 @@ uint64 PlotReader::GetC3ParkCount() const // However, to make sure this is the case, we'll have to // read-in all C1 entries and ensure we hit an empty one, // to ensure we don't run into dead/alignment-space - const size_t c1TableSize = _plot.TableSize( PlotTable::C1 ); - const size_t f7Size = CDiv( _plot.K(), 8 ); - const uint64 c3ParkCount = std::max( c1TableSize / f7Size, (size_t)1 ) - 1; + const uint64 c3ParkCount = GetMaximumC1Entries(); // Or just do this: // Same thing, but we use it @@ -81,6 +89,59 @@ size_t PlotReader::GetTableParkCount( const PlotTable table ) const } } +//----------------------------------------------------------- +uint64 PlotReader::GetMaximumC1Entries() const +{ + // -1 because an extra 0 entry is added at the end + const size_t c1TableSize = _plot.TableSize( PlotTable::C1 ); + const size_t f7Size = CDiv( _plot.K(), 8 ); + const uint64 c3ParkCount = std::max( c1TableSize / f7Size, (size_t)1 ) - 1; + + return c3ParkCount; +} + +//----------------------------------------------------------- +bool PlotReader::GetActualC1EntryCount( uint64& outC1Count ) +{ + outC1Count = 0; + + const uint64 maxC1Entries = GetMaxF7EntryCount(); + + if( maxC1Entries < 1 ) + return true; + + const size_t f7SizeBytes = CDiv( _plot.K(), 8 ); + const uint64 c1Address = _plot.TableAddress( PlotTable::C1 ); + const size_t c1TableSize = _plot.TableSize( PlotTable::C1 ); + size_t c1ReadAddress = c1Address + c1TableSize - f7SizeBytes; + + // Read entries from the end of the table until the start, until we find an entry that is + // not zero/higher than the previous one + if( !_plot.Seek( SeekOrigin::Begin, (int64)c1ReadAddress ) ) + return false; + + const uint32 k = _plot.K(); + uint64 c1 = 0; + while( c1ReadAddress >= c1Address ) + { + uint64 newC1; + if( _plot.Read( f7SizeBytes, &newC1 ) != (ssize_t)f7SizeBytes ) + return false; + + newC1 = Swap64( newC1 ) >> ( 64 - k ); + if( newC1 > c1 ) + break; + + if( c1ReadAddress <= c1Address ) + return false; + + c1ReadAddress -= f7SizeBytes; + } + + outC1Count = ( c1ReadAddress - c1Address ) / f7SizeBytes; + return true; +} + //----------------------------------------------------------- int64 PlotReader::ReadC3Park( uint64 parkIndex, uint64* f7Buffer ) { @@ -368,6 +429,220 @@ bool PlotReader::FetchProofFromP7Entry( uint64 p7Entry, uint64 proof[32] ) return false; } +//----------------------------------------------------------- +Span PlotReader::GetP7IndicesForF7( const uint64 f7, Span indices ) +{ + if( indices.Length() == 0 ) + return {}; + + if( !LoadC2Entries() ) + return {}; + + uint64 c2Index = 0; + + for( uint64 i = 0; ; ) + { + const uint64 c2 = _c2Entries[i]; + + if( c2 > f7 || ++i >= _c2Entries.Length() ) + { + if( c2Index > 0 ) c2Index--; + break; + } + + c2Index++; + } + + const uint64 c1StartIndex = c2Index * kCheckpoint2Interval; + + const uint32 k = _plot.K(); + const size_t f7SizeBytes = CDiv( k, 8 ); + const size_t f7BitCount = f7SizeBytes * 8; + const uint64 c1TableAddress = _plot.TableAddress( PlotTable::C1 ); + const size_t c1TableSize = _plot.TableSize( PlotTable::C1 ); + const uint64 c1TableEnd = c1TableAddress + c1TableSize; + const uint64 c1EntryAddress = c1TableAddress + c1StartIndex * f7SizeBytes; + + + const uint64 c1EndAddress = std::min( c1EntryAddress + ( kCheckpoint1Interval * f7SizeBytes ), c1TableEnd ); + + const size_t readSize = c1EndAddress - c1EntryAddress; + const uint64 c1EntryCount = readSize / f7SizeBytes; + + if( c1EntryCount < 1 ) + return {}; + + if( !_plot.Seek( SeekOrigin::Begin, (int64)c1EntryAddress ) ) + { + Log::Error( "Seek to C1 address failed: %d", _plot.GetError() ); + return {}; + } + + // Read C1 entries until we find one equal or larger than the f7 we're looking for + if( !_c1Buffer ) + _c1Buffer = bbcvirtallocbounded( kCheckpoint1Interval * f7SizeBytes ); + + if( _plot.Read( readSize, _c1Buffer ) != (ssize_t)readSize ) + { + Log::Error( "Failed to read C1 entries: %d", _plot.GetError() ); + return {}; + } + + CPBitReader reader( _c1Buffer, readSize * 8 ); + uint64 c3Park = c1StartIndex; + uint64 c1 = 0; + + for( uint64 i = 0; ; ) + { + c1 = reader.Read64( (uint32)f7BitCount ); + + if( c1 >= f7 || ++i >= c1EntryCount ) + { + if( c3Park > 0 ) c3Park--; + break; + } + + c3Park++; + } + + const uint64 parkCount = c1 == f7 && c3Park > 0 ? 2 : 1; // If we got the same c1 as f7, then the previous + // needs to be read as well because we may have duplicate f7s + // in the previous park's last entries. + + if( _c3Buffer.Ptr() == nullptr ) + { + _c3Buffer.values = bbcvirtallocbounded( kCheckpoint1Interval * 2 ); + _c3Buffer.length = kCheckpoint1Interval * 2; + } + + uint64 c3Count = (uint64)ReadC3Park( c3Park, _c3Buffer.Ptr() ); + + if( parkCount > 1 ) + { + ASSERT( parkCount == 2 ); + c3Count += (uint64)ReadC3Park( c3Park+1, _c3Buffer.Ptr() + c3Count ); + } + + // Grab as many matches as we can + const Span c3Entries = _c3Buffer.SliceSize( (size_t)c3Count ); + const uint64 c3StartIndex = c3Park * kCheckpoint1Interval; + uint64 matchCount = 0; + + for( uint64 i = 0; i < c3Entries.Length(); i++ ) + { + if( c3Entries[i] == f7 ) + { + while( matchCount < indices.Length() && i < c3Count && c3Entries[i] == f7 ) + indices[matchCount++] = c3StartIndex + i++; + + return indices.SliceSize( matchCount ); + } + } + + return {}; +} + + +//----------------------------------------------------------- +bool PlotReader::FetchProof( const uint64 t6LPIndex, uint64 fullProofXs[BB_PLOT_PROOF_X_COUNT] ) +{ + uint64 lpIndices[2][BB_PLOT_PROOF_X_COUNT]; + + uint64* lpIdxSrc = lpIndices[0]; + uint64* lpIdxDst = lpIndices[1]; + + *lpIdxSrc = t6LPIndex; + + // Fetch line points to back pointers going through all our tables + // from 6 to 1, grabbing all of the x's that make up a proof. + uint32 lookupCount = 1; + + for( TableId table = TableId::Table6; table >= TableId::Table1; table-- ) + { + ASSERT( lookupCount <= 32 ); + + for( uint32 i = 0, dst = 0; i < lookupCount; i++, dst += 2 ) + { + const uint64 idx = lpIdxSrc[i]; + + uint128 lp = 0; + if( !ReadLP( table, idx, lp ) ) + return false; + + BackPtr ptr; + if( table < TableId::Table6 && _plot.K() <= 32 ) + ptr = LinePointToSquare64( (uint64)lp ); + else + ptr = LinePointToSquare( lp ); + + lpIdxDst[dst+0] = ptr.y; + lpIdxDst[dst+1] = ptr.x; + } + + lookupCount <<= 1; + + std::swap( lpIdxSrc, lpIdxDst ); + // memset( lpIdxDst, 0, sizeof( uint64 ) * PROOF_X_COUNT ); + } + + // Full proof x's will be at the src ptr + memcpy( fullProofXs, lpIdxSrc, sizeof( uint64 ) * BB_PLOT_PROOF_X_COUNT ); + return true; +} + +//----------------------------------------------------------- +bool PlotReader::LoadC2Entries() +{ + if( _c2Entries.Ptr() ) + return true; + + const size_t c2Size = _plot.TableSize( PlotTable::C2 ); + if( c2Size == 0 ) + return false; + + const size_t f7ByteSize = CDiv( _plot.K(), 8 ); + + const uint64 c2MaxEntries = c2Size / f7ByteSize; + if( c2MaxEntries < 1 ) + return false; + + if( !_plot.Seek( SeekOrigin::Begin, (int64)_plot.TableAddress( PlotTable::C2 ) ) ) + return false; + + + byte* buffer = bbvirtallocbounded( c2Size ); + + if( _plot.Read( c2Size, buffer ) != (ssize_t)c2Size ) + { + bbvirtfreebounded( buffer ); + return false; + } + + _c2Entries = bbcalloc_span( c2MaxEntries ); + + const size_t f7BitCount = f7ByteSize * 8; + CPBitReader reader( buffer, c2Size * 8 ); + + uint64 prevF7 = 0; + uint64 i; + for( i = 0; i < c2MaxEntries; i++ ) + { + const uint64 f7 = reader.Read64( (uint32)f7BitCount ); + + // Short circuit if we encounter an unsorted/out-of-order c2 entry + if( f7 < prevF7 ) + break; + + _c2Entries[i] = f7; + prevF7 = f7; + } + + _c2Entries.length = i; + + bbvirtfreebounded( buffer ); + return true; +} + /// /// Plot Files /// diff --git a/src/tools/PlotReader.h b/src/tools/PlotReader.h index 196e3bf5..9d051be8 100644 --- a/src/tools/PlotReader.h +++ b/src/tools/PlotReader.h @@ -1,25 +1,12 @@ #pragma once #include "plotting/PlotTools.h" +#include "plotting/PlotTypes.h" #include "io/FileStream.h" #include "util/Util.h" #include class CPBitReader; -enum class PlotTable -{ - Table1 = 0, - Table2, - Table3, - Table4, - Table5, - Table6, - Table7, - C1, - C2, - C3, -}; ImplementArithmeticOps( PlotTable ); - struct PlotHeader { byte id [BB_PLOT_ID_LEN] = { 0 }; @@ -175,6 +162,15 @@ class PlotReader size_t GetTableParkCount( const PlotTable table ) const; + uint64 GetMaximumC1Entries() const; + + // Returns the number of apparent C1 entries (same as C3 park count), + // excluding any empty space appearing after the table for alignment, + // This may not be accurate if the C1 entries are malformed and + // not sorted in ascending order. + // This method performs various reads until it find a seemingly valid C1 entries + bool GetActualC1EntryCount( uint64& outC1Count ); + // Read a whole C3 park into f7Buffer. // f7Buffer must hold at least as many as the amount of entries // required per C3Park. (kCheckpoint1Interval). @@ -187,6 +183,8 @@ class PlotReader uint64 GetFullProofForF7Index( uint64 f7Index, byte* fullProof ); + bool FetchProof( uint64 t6LPIndex, uint64 fullProofXs[BB_PLOT_PROOF_X_COUNT] ); + // void FindF7ParkIndices( uintt64 f7, std::vector indices ); bool ReadLPPark( TableId table, uint64 parkIndex, uint128 linePoints[kEntriesPerPark], uint64& outEntryCount ); @@ -196,17 +194,25 @@ class PlotReader inline IPlotFile& PlotFile() const { return _plot; } + Span GetP7IndicesForF7( const uint64 f7, Span indices ); + private: bool ReadLPParkComponents( TableId table, uint64 parkIndex, CPBitReader& outStubs, byte*& outDeltas, uint128& outBaseLinePoint, uint64& outDeltaCounts ); + bool LoadC2Entries(); private: IPlotFile& _plot; + uint32 _version; // size_t _parkBufferSize; - uint64* _parkBuffer ; // Buffer for loading compressed park data. - byte* _deltasBuffer ; // Buffer for decompressing deltas in parks that have delta. + uint64* _parkBuffer ; // Buffer for loading compressed park data. + byte* _deltasBuffer ; // Buffer for decompressing deltas in parks that have delta. + + Span _c2Entries; + byte* _c1Buffer = nullptr; + Span _c3Buffer; }; diff --git a/src/tools/PlotValidator.cpp b/src/tools/PlotValidator.cpp index c6c65806..fdbd5bbe 100644 --- a/src/tools/PlotValidator.cpp +++ b/src/tools/PlotValidator.cpp @@ -58,6 +58,8 @@ You can specify the thread count in the bladebit global option '-t'. it requires around 128GiB of RAM for k=32. This is only supported for plots with k=32 and below. + --f7 : Specify an f7 to find and validate in the plot. + -h, --help : Print this help message and exit. )"; @@ -98,6 +100,7 @@ static void GetProofF1( uint32 k, const byte plotId[BB_PLOT_ID_LEN], uint64 full template static bool FetchProof( PlotReader& plot, uint64 t6LPIndex, uint64 fullProofXs[PROOF_X_COUNT] ); +static void GetProofForChallenge( const char* plotPath, const char* challengeHex ); static bool ValidateFullProof( const uint32 k, const byte plotId[BB_PLOT_ID_LEN], uint64 fullProofXs[PROOF_X_COUNT], uint64& outF7 ); static void ReorderProof( PlotReader& plot, uint64 fullProofXs[PROOF_X_COUNT] ); static void GetProofF1( uint32 k, const byte plotId[BB_PLOT_ID_LEN], uint64 fullProofXs[PROOF_X_COUNT], uint64 fx[PROOF_X_COUNT] ); @@ -137,6 +140,9 @@ void PlotValidatorMain( GlobalPlotConfig& gCfg, CliParser& cli ) { ValidatePlotOptions opts; + const char* challenge = nullptr; + int64 f7 = -1; + while( cli.HasArgs() ) { if( cli.ReadSwitch( opts.inRAM, "-m", "--in-ram" ) ) @@ -145,6 +151,10 @@ void PlotValidatorMain( GlobalPlotConfig& gCfg, CliParser& cli ) continue; else if( cli.ReadF32( opts.startOffset, "-o", "--offset" ) ) continue; + else if( cli.ReadStr( challenge, "--prove" ) ) + continue; + else if( cli.ReadI64( f7, "--f7" ) ) // Same as proof, but the challenge is made from an f7 + continue; else if( cli.ArgConsume( "-h", "--help" ) ) { PlotValidatorPrintUsage(); @@ -160,6 +170,22 @@ void PlotValidatorMain( GlobalPlotConfig& gCfg, CliParser& cli ) } } + // Check for f7 + if( f7 >= 0 ) + { + challenge = new char[65]; + sprintf( (char*)challenge, "%08llx", f7 ); + memset( (void*)(challenge+8), '0', 64-8 ); + ((char*)challenge)[64] = 0; + } + + // Check for challenge + if( challenge != nullptr ) + { + GetProofForChallenge( opts.plotPath.c_str(), challenge ); + Exit( 0 ); + } + const uint32 maxThreads = SysHost::GetLogicalCPUCount(); opts.threadCount = gCfg.threadCount == 0 ? maxThreads : std::min( maxThreads, gCfg.threadCount ); @@ -208,11 +234,11 @@ bool ValidatePlot( const ValidatePlotOptions& options ) FatalIf( options.unpacked && plotFile->K() != 32, "Unpacked plots are only supported for k=32 plots." ); Log::Line( "Validating plot %s", options.plotPath.c_str() ); - Log::Line( "K : %u", plotFile->K() ); - Log::Line( "Unpacked: %s", options.unpacked? "true" : "false" );; + Log::Line( "K : %u", plotFile->K() ); + Log::Line( "Unpacked : %s", options.unpacked? "true" : "false" );; const uint64 plotC3ParkCount = plotFile->TableSize( PlotTable::C1 ) / sizeof( uint32 ) - 1; - Log::Line( "C3 Parks: %llu", plotC3ParkCount ); + Log::Line( "Maximum C3 Parks: %llu", plotC3ParkCount ); Log::Line( "" ); @@ -309,13 +335,31 @@ uint64 ValidateInMemory( UnpackedK32Plot& plot, ThreadPool& pool ) const uint32 expectedF7 = plot.f7[i]; if( expectedF7 != outF7 ) - failedCount++; + { + if( failedCount++ == 0 ) + { + Log( "Proof failed: Expected %llu but got %llu @ %llu (park %llu).", + (uint64)expectedF7, outF7, i, i / kCheckpoint1Interval ); + } + } } else - failedCount++; + { + if( failedCount++ == 0 ) + { + Log( "Proof failed: Validation error for f7 %llu @ %llu (park %llu).", + (uint64)plot.f7[i], i, i / kCheckpoint1Interval ); + } + } } else - failedCount++; + { + if( failedCount++ == 0 ) + { + Log( "Proof failed: Fetch failure for f7 %llu @ %llu (park %llu).", + (uint64)plot.f7[i], i, i / kCheckpoint1Interval ); + } + } const uint64 proofsChecked = i - offset; if( ( proofsChecked > 0 && proofsChecked % reportInterval == 0 ) || i + 1 == end ) @@ -476,6 +520,80 @@ void ValidateJob::Run() this->failCount = proofFailCount; } +// #TODO: Support K>32 +//----------------------------------------------------------- +void GetProofForChallenge( const char* plotPath, const char* challengeHex ) +{ + FatalIf( !plotPath || !*plotPath, "Invalid plot path." ); + FatalIf( !challengeHex || !*challengeHex, "Invalid challenge." ); + + const size_t lenChallenge = strlen( challengeHex ); + FatalIf( lenChallenge != 64, "Invalid challenge, should be 32 bytes." ); + + uint64 challenge[4] = {}; + HexStrToBytes( challengeHex, lenChallenge, (byte*)challenge, 32 ); + + FilePlot plot; + FatalIf( !plot.Open( plotPath ), "Failed to open plot at %s.", plotPath ); + FatalIf( plot.K() != 32, "Only k32 plots are supported." ); + + // Read F7 value + CPBitReader f7Reader( (byte*)challenge, sizeof( challenge ) * 8 ); + const uint32 f7 = (uint32)f7Reader.Read64( 32 ); + + // Find this f7 in the plot file + PlotReader reader( plot ); + + uint64 _indices[64] = {}; + Span indices( _indices, sizeof( _indices ) / sizeof( uint64 ) ); // #TODO: Should simply return the start index and count + + auto matches = reader.GetP7IndicesForF7( f7, indices ); + FatalIf( matches.Length() == 0, "Could not find f7 %llu in plot", f7 ); + + uint64 fullProofXs[PROOF_X_COUNT]; + uint64 proof [32] = {}; + char proofStr[513] = {}; + uint64 p7Entries[kEntriesPerPark] = {}; + + int64 prevP7Park = -1; + + for( uint64 i = 0; i < matches.Length(); i++ ) + { + const uint64 p7Index = matches[i]; + const uint64 p7Park = p7Index / kEntriesPerPark; + + // uint64 o = reader.GetFullProofForF7Index( matches[i], proof ); + if( (int64)p7Park != prevP7Park ) + { + FatalIf( !reader.ReadP7Entries( p7Park, p7Entries ), "Failed to read P7 %llu.", p7Park ); + } + + prevP7Park = (int64)p7Park; + + const uint64 localP7Index = p7Index - p7Park * kEntriesPerPark; + const uint64 t6Index = p7Entries[localP7Index]; + + const bool gotProof = FetchProof( reader, t6Index, fullProofXs ); + + if( gotProof ) + { + ReorderProof( reader, fullProofXs ); + + BitWriter writer( proof, sizeof( proof ) * 8 ); + + for( uint32 j = 0; j < PROOF_X_COUNT; j++ ) + writer.Write64BE( fullProofXs[j], 32 ); + + for( uint32 j = 0; j < PROOF_X_COUNT/2; j++ ) + proof[j] = Swap64( proof[j] ); + + size_t encoded; + BytesToHexStr( (byte*)proof, sizeof( proof ), proofStr, sizeof( proofStr ), encoded ); + // Log::Line( "[%llu] : %s", i, proofStr ); + Log::Line( proofStr ); + } + } +} //----------------------------------------------------------- UnpackedK32Plot UnpackedK32Plot::Load( IPlotFile** plotFile, ThreadPool& pool, uint32 threadCount ) @@ -495,6 +613,9 @@ UnpackedK32Plot UnpackedK32Plot::Load( IPlotFile** plotFile, ThreadPool& pool, u PlotReader& plotReader = readers[0]; + // uint64 c1EntryCount = 0; + // FatalIf( !plotReader.GetActualC1EntryCount( c1EntryCount ), "Failed to obtain C1 entry count." ); + uint64 f7Count = plotReader.GetMaxF7EntryCount(); FatalIf( f7Count < 1, "No F7s found." ); // Load F7s @@ -502,7 +623,7 @@ UnpackedK32Plot UnpackedK32Plot::Load( IPlotFile** plotFile, ThreadPool& pool, u Log::Line( "Unpacking f7 values..." ); uint32* f7 = bbcvirtallocboundednuma( f7Count ); - uint64 missingF7 = 0; + std::atomic sharedF7Count = 0; AnonMTJob::Run( pool, threadCount, [&]( AnonMTJob* self ) { @@ -513,8 +634,10 @@ UnpackedK32Plot UnpackedK32Plot::Load( IPlotFile** plotFile, ThreadPool& pool, u uint64 parkCount, parkOffset, parkEnd; GetThreadOffsets( self, plotParkCount, parkCount, parkOffset, parkEnd ); - uint64 f7Buffer[kCheckpoint1Interval]; - uint32* f7Writer = f7 + parkOffset * kCheckpoint1Interval; + uint64 f7Buffer[kCheckpoint1Interval]; + + uint32* f7Start = f7 + parkOffset * kCheckpoint1Interval; + uint32* f7Writer = f7Start; for( uint64 i = parkOffset; i < parkEnd; i++ ) { @@ -522,27 +645,34 @@ UnpackedK32Plot UnpackedK32Plot::Load( IPlotFile** plotFile, ThreadPool& pool, u FatalIf( entryCount == 0, "Empty C3 park @ %llu.", i ); - for( int64 e = 0; e < entryCount; e++ ) - f7Writer[e] = (uint32)f7Buffer[e]; - - f7Writer += entryCount; + if( entryCount > 0 ) + { + for( int64 e = 0; e < entryCount; e++ ) + f7Writer[e] = (uint32)f7Buffer[e]; + + f7Writer += entryCount; + } if( entryCount < kCheckpoint1Interval ) { - if( self->IsLastThread() && i + 1 == parkEnd ) + if( self->IsLastThread() ) { - missingF7 = kCheckpoint1Interval - entryCount; + // Short-circuit as soon as we find a partial park in the last thread break; } else - FatalErrorMsg( "C3 park %llu is not full and it is not the last park.", i ); + Fatal( "[%u/%u] C3 park %llu is not full and it is not the last park.", self->_jobId, self->_jobCount, i ); } } + + sharedF7Count += (uint64)(uintptr_t)(f7Writer - f7Start); }); - f7Count -= missingF7; + f7Count = sharedF7Count; plot.f7.length = f7Count; plot.f7.values = f7; + + Log::Line( "Actual C3 Parks : %llu", CDiv( f7Count, kCheckpoint1Interval ) ); } // Read Park 7 diff --git a/src/util/BitView.h b/src/util/BitView.h index 9d0ecdff..75a7ea93 100644 --- a/src/util/BitView.h +++ b/src/util/BitView.h @@ -449,6 +449,14 @@ class BitWriter _position += bitCount; } + //----------------------------------------------------------- + inline void Write64BE( const uint64 value, const uint32 bitCount ) + { + ASSERT( _capacity - _position >= bitCount ); + WriteBits64BE( _fields, _position, value, bitCount ); + _position += bitCount; + } + //----------------------------------------------------------- // dstOffset: Offset in bits as to where to start writing in fields //-----------------------------------------------------------