diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index acdcd5ccc..8d00bb966 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -19,7 +19,7 @@ jobs: fail-fast: false matrix: os: [macos-latest, ubuntu-latest, windows-latest] - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] + python-version: ['3.8', '3.9', '3.10', '3.11'] steps: - uses: actions/checkout@v3 @@ -55,7 +55,7 @@ jobs: - name: Build run: | - python -m pip install clvm_tools colorama blspy + python -m pip install clvm_tools colorama blspy chia-blockchain maturin develop --release -m wheel/Cargo.toml - name: python mypy diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 9a25098e0..5d5e1aa06 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -384,7 +384,7 @@ jobs: source venv/bin/activate git clone https://github.com/Chia-Network/clvm_tools.git --branch=main --single-branch pip install ./clvm_tools - pip install colorama maturin pytest + pip install colorama maturin pytest chia-blockchain maturin develop --release -m wheel/Cargo.toml pytest tests grcov . --binary-path target -s . --branch --ignore-not-existing --ignore='*/.cargo/*' --ignore='tests/*' --ignore='venv/*' -o rust_cov.info diff --git a/Cargo.lock b/Cargo.lock index 21353da74..f1bbc8f63 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -324,6 +324,7 @@ dependencies = [ "chia_py_streamable_macro", "chia_streamable_macro", "clvm-traits", + "clvm-utils", "clvmr", "hex", "pyo3", @@ -500,6 +501,7 @@ dependencies = [ "clvmr", "hex", "num-bigint", + "pyo3", "thiserror", ] @@ -1482,6 +1484,7 @@ dependencies = [ "inventory", "libc", "memoffset", + "num-bigint", "parking_lot", "pyo3-build-config", "pyo3-ffi", diff --git a/chia-protocol/Cargo.toml b/chia-protocol/Cargo.toml index f5bbf5c48..48918fe86 100644 --- a/chia-protocol/Cargo.toml +++ b/chia-protocol/Cargo.toml @@ -12,7 +12,7 @@ repository = "https://github.com/Chia-Network/chia_rs/chia-protocol/" py-bindings = ["dep:pyo3", "dep:chia_py_streamable_macro", "chia-traits/py-bindings"] [dependencies] -pyo3 = { version = ">=0.19.0", features = ["multiple-pymethods"], optional = true } +pyo3 = { version = ">=0.19.0", features = ["multiple-pymethods", "num-bigint"], optional = true } sha2 = "0.9.9" hex = "0.4.3" chia_streamable_macro = { version = "0.2.12", path = "../chia_streamable_macro" } @@ -20,6 +20,7 @@ chia_py_streamable_macro = { version = "0.2.12", path = "../chia_py_streamable_m clvmr = "0.3.0" chia-traits = { version = "0.2.12", path = "../chia-traits" } clvm-traits = { version = "0.2.12", path = "../clvm-traits", features = ["derive"] } +clvm-utils = { version = "0.2.12", path = "../clvm-utils" } chia-bls = { version = "0.2.12", path = "../chia-bls" } arbitrary = { version = "=1.3.0", features = ["derive"] } diff --git a/chia-protocol/src/program.rs b/chia-protocol/src/program.rs index 10bff1f1b..a65098a64 100644 --- a/chia-protocol/src/program.rs +++ b/chia-protocol/src/program.rs @@ -8,15 +8,6 @@ use clvmr::Allocator; use sha2::{Digest, Sha256}; use std::io::Cursor; -#[cfg(feature = "py-bindings")] -use chia_traits::{FromJsonDict, ToJsonDict}; - -#[cfg(feature = "py-bindings")] -use chia_py_streamable_macro::PyStreamable; - -#[cfg(feature = "py-bindings")] -use pyo3::prelude::*; - #[cfg_attr(feature = "py-bindings", pyclass, derive(PyStreamable))] #[derive(Hash, Debug, Clone, Eq, PartialEq)] pub struct Program(Bytes); @@ -59,6 +50,257 @@ impl Program { } } +impl Default for Program { + fn default() -> Self { + Self(vec![0x80_u8].into()) + } +} + +#[cfg(feature = "py-bindings")] +use crate::lazy_node::LazyNode; + +#[cfg(feature = "py-bindings")] +use chia_traits::{FromJsonDict, ToJsonDict}; + +#[cfg(feature = "py-bindings")] +use chia_py_streamable_macro::PyStreamable; + +#[cfg(feature = "py-bindings")] +use pyo3::prelude::*; + +#[cfg(feature = "py-bindings")] +use pyo3::types::{PyList, PyTuple}; + +#[cfg(feature = "py-bindings")] +use clvmr::serde::node_from_bytes_backrefs; + +#[cfg(feature = "py-bindings")] +use clvmr::allocator::SExp; + +#[cfg(feature = "py-bindings")] +use pyo3::exceptions::*; + +#[cfg(feature = "py-bindings")] +fn clvm_convert(a: &mut Allocator, o: &PyAny) -> PyResult { + // None + if o.is_none() { + Ok(a.null()) + // Program itself + } else if let Ok(prg) = o.extract::() { + Ok(node_from_bytes_backrefs(a, prg.0.as_slice())?) + // bytes + } else if let Ok(buffer) = o.extract::<&[u8]>() { + a.new_atom(buffer) + .map_err(|e| PyMemoryError::new_err(e.to_string())) + // str + } else if let Ok(text) = o.extract::() { + a.new_atom(text.as_bytes()) + .map_err(|e| PyMemoryError::new_err(e.to_string())) + // int + } else if let Ok(val) = o.extract::() { + a.new_number(val) + .map_err(|e| PyMemoryError::new_err(e.to_string())) + // Tuple (SExp-like) + } else if let Ok(pair) = o.downcast::() { + if pair.len() != 2 { + Err(PyValueError::new_err(format!( + "can't cast tuple of size {}", + pair.len() + ))) + } else { + let left = clvm_convert(a, pair.get_item(0)?)?; + let right = clvm_convert(a, pair.get_item(1)?)?; + a.new_pair(left, right) + .map_err(|e| PyMemoryError::new_err(e.to_string())) + } + // List + } else if let Ok(list) = o.downcast::() { + let mut rev = Vec::<&PyAny>::new(); + for py_item in list.iter() { + rev.push(py_item); + } + let mut ret = a.null(); + for py_item in rev.into_iter().rev() { + let item = clvm_convert(a, py_item)?; + ret = a + .new_pair(item, ret) + .map_err(|e| PyMemoryError::new_err(e.to_string()))?; + } + Ok(ret) + // SExp (such as clvm.SExp) + } else if let (Ok(atom), Ok(pair)) = (o.getattr("atom"), o.getattr("pair")) { + if atom.is_none() { + if pair.is_none() { + Err(PyTypeError::new_err(format!("invalid SExp item {o}"))) + } else { + let pair = pair.downcast::()?; + let left = clvm_convert(a, &pair[0])?; + let right = clvm_convert(a, &pair[1])?; + a.new_pair(left, right) + .map_err(|e| PyMemoryError::new_err(e.to_string())) + } + } else { + a.new_atom(atom.extract::<&[u8]>()?) + .map_err(|e| PyMemoryError::new_err(e.to_string())) + } + // anything convertible to bytes + } else if let Ok(fun) = o.getattr("__bytes__") { + let bytes = fun.call0()?; + let buffer = bytes.extract::<&[u8]>()?; + a.new_atom(buffer) + .map_err(|e| PyMemoryError::new_err(e.to_string())) + } else { + Err(PyTypeError::new_err(format!( + "unknown parameter to run_with_cost() {o}" + ))) + } +} + +#[cfg(feature = "py-bindings")] +fn to_program(py: Python<'_>, node: LazyNode) -> PyResult<&PyAny> { + use pyo3::types::PyDict; + let ctx: &PyDict = PyDict::new(py); + ctx.set_item("node", node)?; + py.run( + "from chia.types.blockchain_format.program import Program\n\ + ret = Program(node)\n", + None, + Some(ctx), + )?; + Ok(ctx.get_item("ret").unwrap()) +} + +#[cfg(feature = "py-bindings")] +#[pymethods] +impl Program { + #[pyo3(name = "default")] + #[staticmethod] + fn py_default() -> Self { + Self::default() + } + + fn get_tree_hash(&self) -> crate::Bytes32 { + let mut cursor = Cursor::new(self.0.as_ref()); + clvmr::serde::tree_hash_from_stream(&mut cursor) + .unwrap() + .into() + } + + #[staticmethod] + fn from_program(py: Python<'_>, p: PyObject) -> PyResult { + let buf = p.getattr(py, "__bytes__")?.call0(py)?; + let buf = buf.extract::<&[u8]>(py)?; + Ok(Self(buf.into())) + } + + #[staticmethod] + fn fromhex(h: String) -> Result { + let s = if let Some(st) = h.strip_prefix("0x") { + st + } else { + &h[..] + }; + Self::from_bytes(hex::decode(s).map_err(|_| Error::InvalidString)?.as_slice()) + } + + fn run_mempool_with_cost<'a>( + &self, + py: Python<'a>, + max_cost: u64, + args: &PyAny, + ) -> PyResult<(u64, &'a PyAny)> { + use clvmr::MEMPOOL_MODE; + self._run(py, max_cost, MEMPOOL_MODE, args) + } + + fn run_with_cost<'a>( + &self, + py: Python<'a>, + max_cost: u64, + args: &PyAny, + ) -> PyResult<(u64, &'a PyAny)> { + self._run(py, max_cost, 0, args) + } + + fn _run<'a>( + &self, + py: Python<'a>, + max_cost: u64, + flags: u32, + args: &PyAny, + ) -> PyResult<(u64, &'a PyAny)> { + use clvmr::reduction::Response; + use clvmr::run_program; + use clvmr::ChiaDialect; + use std::rc::Rc; + + let mut a = Allocator::new_limited(500000000, 62500000, 62500000); + let clvm_args = clvm_convert(&mut a, args)?; + + let r: Response = (|| -> PyResult { + let program = node_from_bytes_backrefs(&mut a, self.0.as_ref())?; + let dialect = ChiaDialect::new(flags); + + Ok(py.allow_threads(|| run_program(&mut a, &dialect, program, clvm_args, max_cost))) + })()?; + match r { + Ok(reduction) => { + let val = LazyNode::new(Rc::new(a), reduction.1); + Ok((reduction.0, to_program(py, val)?)) + } + Err(eval_err) => Err(PyValueError::new_err(eval_err.to_string())), + } + } + + fn to_program<'a>(&self, py: Python<'a>) -> PyResult<&'a PyAny> { + use std::rc::Rc; + let mut a = Allocator::new_limited(500000000, 62500000, 62500000); + let prg = node_from_bytes_backrefs(&mut a, self.0.as_ref())?; + let prg = LazyNode::new(Rc::new(a), prg); + to_program(py, prg) + } + + fn uncurry<'a>(&self, py: Python<'a>) -> PyResult<(&'a PyAny, &'a PyAny)> { + use clvm_utils::CurriedProgram; + use std::rc::Rc; + + let mut a = Allocator::new_limited(500000000, 62500000, 62500000); + let prg = node_from_bytes_backrefs(&mut a, self.0.as_ref())?; + let Ok(uncurried) = CurriedProgram::::from_clvm(&a, prg) else { + let a = Rc::new(a); + let prg = LazyNode::new(a.clone(), prg); + let ret = a.null(); + let ret = LazyNode::new(a, ret); + return Ok((to_program(py, prg)?, to_program(py, ret)?)); + }; + + let mut curried_args = Vec::::new(); + let mut args = uncurried.args; + loop { + if let SExp::Atom = a.sexp(args) { + break; + } + // the args of curried puzzles are in the form of: + // (c . ((q . ) . ( . ()))) + let (_, ((_, arg), (rest, _))) = + <( + clvm_traits::MatchByte<4>, + (clvm_traits::match_quote!(NodePtr), (NodePtr, ())), + ) as clvm_traits::FromClvm>::from_clvm(&a, args)?; + curried_args.push(arg); + args = rest; + } + let mut ret = a.null(); + for item in curried_args.into_iter().rev() { + ret = a.new_pair(item, ret).map_err(|_e| Error::EndOfBuffer)?; + } + let a = Rc::new(a); + let prg = LazyNode::new(a.clone(), uncurried.program); + let ret = LazyNode::new(a, ret); + Ok((to_program(py, prg)?, to_program(py, ret)?)) + } +} + impl Streamable for Program { fn update_digest(&self, digest: &mut Sha256) { digest.update(&self.0); diff --git a/clvm-traits/Cargo.toml b/clvm-traits/Cargo.toml index 6eeed53ad..2e13bd5be 100644 --- a/clvm-traits/Cargo.toml +++ b/clvm-traits/Cargo.toml @@ -13,8 +13,10 @@ features = ["derive"] [features] derive = ["dep:clvm-derive"] +py-bindings = ["dep:pyo3"] [dependencies] +pyo3 = { version = ">=0.19.0", optional = true } clvm-derive = { version = "0.2.12", path = "../clvm-derive", optional = true } clvmr = "0.3.0" num-bigint = "0.4.3" diff --git a/clvm-traits/src/error.rs b/clvm-traits/src/error.rs index 1068bd07b..2e3d57be4 100644 --- a/clvm-traits/src/error.rs +++ b/clvm-traits/src/error.rs @@ -32,3 +32,13 @@ impl From for Error { Self::Allocator(value) } } + +#[cfg(feature = "py-bindings")] +use pyo3::PyErr; + +#[cfg(feature = "py-bindings")] +impl std::convert::From for PyErr { + fn from(err: Error) -> PyErr { + pyo3::exceptions::PyValueError::new_err(err.to_string()) + } +} diff --git a/tests/test_program_fidelity.py b/tests/test_program_fidelity.py new file mode 100644 index 000000000..50fed79c3 --- /dev/null +++ b/tests/test_program_fidelity.py @@ -0,0 +1,84 @@ +from typing import List, Tuple, Optional + +import string +import chia_rs +from chia.types.blockchain_format.program import Program as ChiaProgram +from random import Random + +def rand_bytes(rnd: Random) -> bytes: + size = rnd.randint(0, 4) + ret = bytearray() + for _ in range(size): + ret.append(rnd.getrandbits(8)) + return bytes(ret) + +def rand_string(rnd: Random) -> str: + size = rnd.randint(1, 10) + return ''.join(rnd.choices(string.ascii_uppercase + string.digits, k=size)) + +def rand_int(rnd: Random) -> int: + return rnd.randint(0, 100000000000000) + +def rand_list(rnd: Random) -> List: + size = rnd.randint(0, 3) + ret = [] + for _ in range(size): + ret.append(rand_object(rnd)) + return ret + +def rand_program(rnd: Random) -> ChiaProgram: + return ChiaProgram.from_bytes(b"\xff\x01\xff\x04\x01") + +def rand_optional(rnd: Random) -> Optional[object]: + if rnd.randint(0, 1) == 0: + return None + return rand_object(rnd) + +def rand_object(rnd: Random) -> object: + types = [rand_optional, rand_int, rand_string, rand_bytes, rand_program, rand_list] + return rnd.sample(types, 1)[0](rnd) + +def test_run_program() -> None: + + rust_identity = chia_rs.Program.from_bytes(b"\x01") + py_identity = ChiaProgram.from_bytes(b"\x01") + + rnd = Random() + for _ in range(10000): + args = rand_object(rnd) + + py_ret = py_identity._run(10000, 0, args) + rust_ret = rust_identity._run(10000, 0, args) + + assert rust_ret == py_ret + +def test_tree_hash() -> None: + + rnd = Random() + for _ in range(10000): + py_prg = ChiaProgram.to(rand_object(rnd)) + rust_prg = chia_rs.Program.from_bytes(bytes(py_prg)) + + assert py_prg.get_tree_hash() == rust_prg.get_tree_hash() + +def test_uncurry() -> None: + + rnd = Random() + for _ in range(10000): + py_prg = ChiaProgram.to(rand_object(rnd)) + py_prg = py_prg.curry(rand_object(rnd)) + rust_prg = chia_rs.Program.from_program(py_prg) + assert py_prg.uncurry() == rust_prg.uncurry() + + py_prg = py_prg.curry(rand_object(rnd), rand_object(rnd)) + rust_prg = chia_rs.Program.from_program(py_prg) + assert py_prg.uncurry() == rust_prg.uncurry() + +def test_round_trip() -> None: + + rnd = Random() + for _ in range(10000): + py_prg = ChiaProgram.to(rand_object(rnd)) + rust_prg = chia_rs.Program.from_program(py_prg) + + assert py_prg == rust_prg.to_program() diff --git a/tests/test_run_puzzle.py b/tests/test_run_puzzle.py index f8ec171b9..e9aa20b0b 100644 --- a/tests/test_run_puzzle.py +++ b/tests/test_run_puzzle.py @@ -1,4 +1,5 @@ from chia_rs import run_puzzle, run_chia_program, ALLOW_BACKREFS +from chia.types.blockchain_format.sized_bytes import bytes32 from hashlib import sha256 import pytest from run_gen import print_spend_bundle_conditions @@ -96,7 +97,7 @@ def test_block_834752(flags: int, input_file: str) -> None: def test_failure(flags: int) -> None: output = "" - parent = b"1" * 32 + parent = bytes32(b"1" * 32) amount = 1337 # (mod (solution) (if (= solution (q . 1)) () (x solution))) puzzle = binutils.assemble("(a (i (= 2 (q . 1)) () (q 8 2)) 1)").as_bin() diff --git a/wheel/Cargo.toml b/wheel/Cargo.toml index 510b16d10..f2eec2678 100644 --- a/wheel/Cargo.toml +++ b/wheel/Cargo.toml @@ -24,3 +24,4 @@ chia-protocol = { version = "=0.2.12", path = "../chia-protocol", features = ["p chia-traits = { version = "=0.2.12", path = "../chia-traits", features = ["py-bindings"] } chia_py_streamable_macro = { version = "=0.2.12", path = "../chia_py_streamable_macro" } chia_streamable_macro = { version = "=0.2.12", path = "../chia_streamable_macro" } +clvm-traits = { version = "0.2.12", path = "../clvm-traits", features = ["derive", "py-bindings"] } diff --git a/wheel/chia_rs.pyi b/wheel/chia_rs.pyi index 6ec0596c3..d14d75c20 100644 --- a/wheel/chia_rs.pyi +++ b/wheel/chia_rs.pyi @@ -5,6 +5,7 @@ from typing import List, Optional, Sequence, Tuple from chia.types.blockchain_format.sized_bytes import bytes32 +from chia.types.blockchain_format.program import Program as ChiaProgram ReadableBuffer = Union[bytes, bytearray, memoryview] @@ -820,6 +821,18 @@ class PoolTarget: class Program: a0: bytes + def get_tree_hash(self) -> bytes32: ... + @staticmethod + def default() -> Program: ... + @staticmethod + def fromhex(hex) -> Program: ... + def run_mempool_with_cost(self, max_cost: int, args: object) -> Tuple[int, ChiaProgram]: ... + def run_with_cost(self, max_cost: int, args: object) -> Tuple[int, ChiaProgram]: ... + def _run(self, max_cost: int, flags: int, args: object) -> Tuple[int, ChiaProgram]: ... + @staticmethod + def from_program(p: ChiaProgram) -> Program: ... + def to_program(self) -> ChiaProgram: ... + def uncurry(self) -> Tuple[ChiaProgram, ChiaProgram]: ... def __init__( self, a0: bytes diff --git a/wheel/generate_type_stubs.py b/wheel/generate_type_stubs.py index c97f7a8ae..ecff37d66 100644 --- a/wheel/generate_type_stubs.py +++ b/wheel/generate_type_stubs.py @@ -171,6 +171,17 @@ def parse_rust_source(filename: str) -> List[Tuple[str, List[str]]]: "def is_end_of_slot(self) -> bool: ...", "def is_challenge(self) -> bool: ...", ], + "Program": [ + "def get_tree_hash(self) -> bytes32: ...", + "@staticmethod\n def default() -> Program: ...", + "@staticmethod\n def fromhex(hex) -> Program: ...", + "def run_mempool_with_cost(self, max_cost: int, args: object) -> Tuple[int, ChiaProgram]: ...", + "def run_with_cost(self, max_cost: int, args: object) -> Tuple[int, ChiaProgram]: ...", + "def _run(self, max_cost: int, flags: int, args: object) -> Tuple[int, ChiaProgram]: ...", + "@staticmethod\n def from_program(p: ChiaProgram) -> Program: ...", + "def to_program(self) -> ChiaProgram: ...", + "def uncurry(self) -> Tuple[ChiaProgram, ChiaProgram]: ...", + ], } classes = [] @@ -188,6 +199,7 @@ def parse_rust_source(filename: str) -> List[Tuple[str, List[str]]]: from typing import List, Optional, Sequence, Tuple from chia.types.blockchain_format.sized_bytes import bytes32 +from chia.types.blockchain_format.program import Program as ChiaProgram ReadableBuffer = Union[bytes, bytearray, memoryview]