compact in-memory

Change-Id: I0e97de4133d67797a640ecc8c18ef62fb0ba3b55

.github/workflows/applications.yml

@@ -338,7 +338,7 @@ jobs:
     - name: Build chainweb library
       run: cabal build --ghc-options=-j2 lib:chainweb
     - name: Build chainweb applications
-      run: cabal build -j2 --ghc-options=-j2 exe:chainweb-node test:chainweb-tests exe:cwtool chainweb:bench:bench
+      run: cabal build -j2 --ghc-options=-j2 exe:chainweb-node exe:compact test:chainweb-tests exe:cwtool chainweb:bench:bench
 
     # Checks
     - name: Check that working directory tree is clean
@@ -360,6 +360,7 @@ jobs:
       run: |
         mkdir -p artifacts/chainweb
         cp $(cabal list-bin chainweb-node) artifacts/chainweb
+        cp $(cabal list-bin compact) artifacts/chainweb
         cp $(cabal list-bin chainweb-tests) artifacts/chainweb
         cp $(cabal list-bin cwtool) artifacts/chainweb
         cp $(cabal list-bin bench) artifacts/chainweb

.gitignore

@@ -53,3 +53,4 @@ rosetta/logs/*
 .ghci_history
 .direnv/
 .envrc
+massif.out.*

bench/Chainweb/Pact/Backend/ForkingBench.hs

@@ -301,15 +301,32 @@ withResources rdb trunkLength logLevel compact p f = C.envWithCleanup create des
         coinAccounts <- newMVar mempty
         nonceCounter <- newIORef 1
         txPerBlock <- newIORef 10
-        sqlEnv <- openSQLiteConnection "" {- temporary SQLite db -} chainwebBenchPragmas
         mp <- testMemPoolAccess txPerBlock coinAccounts
-        pactService <-
-          startPact testVer logger blockHeaderDb payloadDb mp sqlEnv
-        mainTrunkBlocks <-
-          playLine payloadDb blockHeaderDb trunkLength genesisBlock (snd pactService) nonceCounter
-        when (compact == DoCompact) $ do
-          C.withDefaultLogger Error $ \lgr -> do
-            void $ C.compact (BlockHeight trunkLength) lgr sqlEnv []
+        (sqlEnv, pactService, mainTrunkBlocks) <- do
+          srcSqlEnv <- openSQLiteConnection "" {- temporary SQLite db -} chainwebBenchPragmas
+          srcPactService <-
+            startPact testVer logger blockHeaderDb payloadDb mp srcSqlEnv
+          mainTrunkBlocks <-
+            playLine payloadDb blockHeaderDb trunkLength genesisBlock (snd srcPactService) nonceCounter
+
+          (sqlEnv, pactService) <- do
+            if compact == DoCompact
+            then do
+              targetSqlEnv <- openSQLiteConnection "" {- temporary SQLite db -} chainwebBenchPragmas
+              C.withDefaultLogger Error $ \lgr -> do
+                C.compactPactState lgr C.defaultRetainment (BlockHeight trunkLength) srcSqlEnv targetSqlEnv
+              targetPactService <-
+                startPact testVer logger blockHeaderDb payloadDb mp targetSqlEnv
+
+              -- Stop the previous pact service/close the sqlite connection
+              stopPact srcPactService
+              stopSqliteDb srcSqlEnv
+
+              pure (targetSqlEnv, targetPactService)
+            else do
+              pure (srcSqlEnv, srcPactService)
+
+          pure (sqlEnv, pactService, mainTrunkBlocks)
 
         return $ NoopNFData $ Resources {..}
 

cabal.project

@@ -87,8 +87,8 @@ source-repository-package
 source-repository-package
     type: git
     location: https://github.com/kadena-io/chainweb-storage.git
-    tag: 4b45c1ab9c070c6d16a058bcbab0c06ac0fb6d4e
-    --sha256: 0m6c7kl6x5a3k02q2i7qzfx91kxz19dzav0piqfxra52bq0x3sm6
+    tag: a5e06a8aa2da1e7981ff9fad91dfd41f7c39fc85
+    --sha256: sha256-3Zqsgrxa7SQUr8XzT2O5PLTELkI92NXZU4j2UUvlL4E=
 
 source-repository-package
     type: git

chainweb.cabal

@@ -379,6 +379,10 @@ library
         , crypton-connection >= 0.2 && < 0.4
         , containers >= 0.5
         , crypton >= 0.31
+        , crypton-connection >= 0.2 && < 0.4
+        , crypton-x509 >=1.7
+        , crypton-x509-system >=1.6
+        , crypton-x509-validation >=1.6
         , cuckoo >= 0.3
         , data-default >=0.7
         , data-dword >= 0.3
@@ -411,8 +415,8 @@ library
         , mmorph >= 1.1
         , monad-control >= 1.0
         , mtl >= 2.3
-        , mwc-random >= 0.13
         , mwc-probability >= 2.0 && <2.4
+        , mwc-random >= 0.13
         , network >= 3.1.2
         , optparse-applicative >= 0.14
         , pact >= 4.2.0.1
@@ -422,6 +426,7 @@ library
         , pem >=0.2
         , primitive >= 0.7.1.0
         , random >= 1.2
+        , rocksdb-haskell-kadena >= 1.1.0
         , rosetta >= 1.0
         , safe-exceptions >= 0.1
         , scheduler >= 1.4
@@ -436,32 +441,33 @@ library
         , streaming-commons >= 0.2
         , template-haskell >= 2.14
         , text >= 2.0
-        , trifecta >= 2.1
         , these >= 1.0
         , time >= 1.12.2
         , tls >=1.9
         , tls-session-manager >= 0.0
         , token-bucket >= 0.1
         , transformers >= 0.5
+        , trifecta >= 2.1
         , unliftio ^>= 0.2
         , unordered-containers >= 0.2.16
         , uuid >= 1.3.15
+        , vector >= 0.12.2
+        , vector-algorithms >= 0.7
         , wai >= 3.2.2.1
         , wai-app-static >= 3.1.6.3
         , wai-cors >= 0.2.7
         , wai-extra >= 3.0.28
-        , wai-middleware-validation
-        , vector >= 0.12.2
-        , vector-algorithms >= 0.7
         , wai-middleware-throttle >= 0.3
+        , wai-middleware-validation
         , warp >= 3.3.6
         , warp-tls >= 3.4
-        , crypton-x509 >=1.7
-        , crypton-x509-system >=1.6
-        , crypton-x509-validation >=1.6
         , yaml >= 0.11
         , yet-another-logger >= 0.4.1
 
+    if !os(windows)
+        build-depends:
+            unix >= 2.7
+
     if flag(ed25519)
         cpp-options: -DWITH_ED25519=1
     if flag(ghc-flags)
@@ -815,6 +821,40 @@ executable cwtool
     if flag(ed25519)
         cpp-options: -DWITH_ED25519=1
 
+executable compact
+    import: warning-flags, debugging-flags
+    default-language: Haskell2010
+    ghc-options:
+        -threaded
+        -rtsopts
+        "-with-rtsopts=-N -A4M --disable-delayed-os-memory-return -qn1"
+    hs-source-dirs:
+        compact
+    main-is: Main.hs
+    build-depends:
+        -- internal
+        , chainweb
+
+        -- external
+        , base >= 4.12 && < 5
+
+executable pact-diff
+    import: warning-flags, debugging-flags
+    default-language: Haskell2010
+    ghc-options:
+        -threaded
+        -rtsopts
+        "-with-rtsopts=-N -A4M --disable-delayed-os-memory-return -qn1"
+    hs-source-dirs:
+        pact-diff
+    main-is: Main.hs
+    build-depends:
+        -- internal
+        , chainweb
+
+        -- external
+        , base >= 4.12 && < 5
+
 benchmark bench
     import: warning-flags, debugging-flags
     default-language: Haskell2010

compact/Main.hs

@@ -0,0 +1,11 @@
+{-# language
+    ImportQualifiedPost
+#-}
+
+module Main (main) where
+
+import Chainweb.Pact.Backend.Compaction qualified as Compact
+
+main :: IO ()
+main = do
+  Compact.main

default.nix

@@ -76,6 +76,7 @@ let haskellSrc = with nix-filter.lib; filter {
       shell.buildInputs = with pkgs; [
         zlib
         pkg-config
+        sqlite
       ];
       modules = [
         {
@@ -102,17 +103,28 @@ let haskellSrc = with nix-filter.lib; filter {
       };
       pact = pactFromCached pkgs pact passthru.cached;
     };
-    default = pkgs.runCommandCC "chainweb" { inherit passthru; } ''
-      mkdir -pv $out/bin
-      cp ${flake.packages."chainweb:exe:chainweb-node"}/bin/chainweb-node $out/bin/chainweb-node
-      cp ${flake.packages."chainweb:exe:cwtool"}/bin/cwtool $out/bin/cwtool
-      chmod +w $out/bin/{cwtool,chainweb-node}
-      $STRIP $out/bin/chainweb-node
-      $STRIP $out/bin/cwtool
-      ${pkgs.lib.optionalString (pkgs.stdenv.isLinux) ''
-        patchelf --shrink-rpath $out/bin/{cwtool,chainweb-node}
-      ''}
-    '';
+    default =
+      let
+        exes = [
+          "chainweb-node"
+          "cwtool"
+          "compact"
+          "pact-diff"
+        ];
+
+        for = xs: f: builtins.map f xs;
+      in
+      pkgs.runCommandCC "chainweb" { inherit passthru; } ''
+        mkdir -pv $out/bin
+        ${builtins.concatStringsSep "\n" (for exes (exe: ''
+          cp ${flake.packages."chainweb:exe:${exe}"}/bin/${exe} $out/bin/${exe}
+          chmod +w $out/bin/${exe}
+          $STRIP $out/bin/${exe}
+          ${pkgs.lib.optionalString (pkgs.stdenv.isLinux) ''
+            patchelf --shrink-rpath $out/bin/${exe}
+          ''}
+        ''))}
+      '';
 in {
   # The Haskell project flake: Used by flake.nix
   inherit flake;

pact-diff/Main.hs

@@ -0,0 +1,11 @@
+{-# language
+    ImportQualifiedPost
+#-}
+
+module Main (main) where
+
+import Chainweb.Pact.Backend.PactState.Diff qualified as PactDiff
+
+main :: IO ()
+main = do
+  PactDiff.main