From 37fba1c8c956b5056b42b95fe0ce2cd64ce1cf49 Mon Sep 17 00:00:00 2001 From: clearloop <26088946+clearloop@users.noreply.github.com> Date: Tue, 5 Dec 2023 19:27:05 +0800 Subject: [PATCH] feat(crates-io): publish gstd and gear-wasm-builder (#3515) --- .github/workflows/crates-io.yml | 15 +- Cargo.lock | 217 ++++++++++++++++------------ Cargo.toml | 15 +- galloc/Cargo.toml | 2 + gcli/Cargo.toml | 2 +- gcore/Cargo.toml | 1 + gstd/Cargo.toml | 7 +- gstd/codegen/Cargo.toml | 11 +- stack-buffer/Cargo.toml | 1 + utils/crates-io/Cargo.toml | 9 +- utils/crates-io/publish.rs | 171 ---------------------- utils/crates-io/src/lib.rs | 71 +++++++++ utils/crates-io/src/main.rs | 52 +++++++ utils/crates-io/src/manifest.rs | 68 +++++++++ utils/crates-io/src/publisher.rs | 139 ++++++++++++++++++ utils/crates-io/src/rename.rs | 73 ++++++++++ utils/crates-io/src/version.rs | 46 ++++++ utils/wasm-builder/Cargo.toml | 8 +- utils/wasm-builder/src/optimize.rs | 2 +- utils/wasm-builder/src/stack_end.rs | 8 +- 20 files changed, 625 insertions(+), 293 deletions(-) delete mode 100644 utils/crates-io/publish.rs create mode 100644 utils/crates-io/src/lib.rs create mode 100644 utils/crates-io/src/main.rs create mode 100644 utils/crates-io/src/manifest.rs create mode 100644 utils/crates-io/src/publisher.rs create mode 100644 utils/crates-io/src/rename.rs create mode 100644 utils/crates-io/src/version.rs diff --git a/.github/workflows/crates-io.yml b/.github/workflows/crates-io.yml index 8d03d121436..c899782ddb9 100644 --- a/.github/workflows/crates-io.yml +++ b/.github/workflows/crates-io.yml @@ -2,6 +2,13 @@ name: Crates IO on: workflow_dispatch: + inputs: + publish: + description: "If publish packages" + type: boolean + default: false + pull_request: + branches: [master] env: CARGO_INCREMENTAL: 0 @@ -10,7 +17,7 @@ env: TERM: xterm-256color jobs: - publish: + check: runs-on: ubuntu-latest env: CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} @@ -21,5 +28,9 @@ jobs: - name: "Install: Rust toolchain" uses: dsherret/rust-toolchain-file@v1 + - name: "Check packages" + run: cargo run --release -p crates-io-manager check + - name: "Publish packages" - run: cargo run --release -p crates-io-manager + if: ${{ github.event_name == 'workflow_dispatch' && inputs.publish }} + run: cargo run --release -p crates-io-manager publish diff --git a/Cargo.lock b/Cargo.lock index 5dd0748c599..4fb67215546 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1611,20 +1611,6 @@ dependencies = [ "wasmtime-types", ] -[[package]] -name = "crates-io" -version = "0.37.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "876aa69b4afca5f2eb5e23daa3445930faf829bcb67075a20ffa884f11f8c57c" -dependencies = [ - "anyhow", - "curl", - "percent-encoding", - "serde", - "serde_json", - "url", -] - [[package]] name = "crates-io-manager" version = "1.0.3" @@ -1632,8 +1618,9 @@ dependencies = [ "anyhow", "cargo_metadata 0.18.1", "cargo_toml", - "crates-io", - "curl", + "clap 4.4.10", + "reqwest", + "serde", "toml 0.7.8", ] @@ -1781,36 +1768,6 @@ dependencies = [ "cipher 0.4.4", ] -[[package]] -name = "curl" -version = "0.4.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "509bd11746c7ac09ebd19f0b17782eae80aadee26237658a6b4808afb5c11a22" -dependencies = [ - "curl-sys", - "libc", - "openssl-probe", - "openssl-sys", - "schannel", - "socket2 0.4.9", - "winapi", -] - -[[package]] -name = "curl-sys" -version = "0.4.68+curl-8.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4a0d18d88360e374b16b2273c832b5e57258ffc1d4aa4f96b108e0738d5752f" -dependencies = [ - "cc", - "libc", - "libz-sys", - "openssl-sys", - "pkg-config", - "vcpkg", - "windows-sys 0.48.0", -] - [[package]] name = "curve25519-dalek" version = "2.1.3" @@ -2841,18 +2798,6 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86e3bdc80eee6e16b2b6b0f87fbc98c04bee3455e35174c0de1a125d0688c632" -[[package]] -name = "dlmalloc" -version = "0.1.4" -source = "git+https://github.com/gear-tech/dlmalloc-rust.git#15352f969112faa463302f2490bbb7f6e1cb904d" -dependencies = [ - "libc", - "libc-print", - "page_size", - "static_assertions", - "str-buf", -] - [[package]] name = "document-features" version = "0.2.7" @@ -3393,6 +3338,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "fork-tree" version = "3.0.0" @@ -3907,7 +3867,7 @@ dependencies = [ name = "galloc" version = "1.0.3" dependencies = [ - "dlmalloc", + "gear-dlmalloc", ] [[package]] @@ -4211,6 +4171,19 @@ dependencies = [ "scale-info", ] +[[package]] +name = "gear-dlmalloc" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5c1d3d820d57acd0c4129b3551f5fe5b01e61c8cb2863bad2b183a4b32dc978" +dependencies = [ + "libc", + "libc-print", + "page_size", + "static_assertions", + "str-buf", +] + [[package]] name = "gear-key-finder" version = "1.0.3" @@ -4328,6 +4301,17 @@ dependencies = [ "vara-runtime", ] +[[package]] +name = "gear-pwasm-utils" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ea09901ce41f64d1be7369b8c2efc774e89b200ac434496ab24e6af96e68f97" +dependencies = [ + "byteorder", + "gear-wasm", + "log", +] + [[package]] name = "gear-replay-cli" version = "1.0.0" @@ -4566,7 +4550,7 @@ checksum = "bbfbfa701dc65e683fcd2fb24f046bcef22634acbdf47ad14724637dc39ad05b" [[package]] name = "gear-wasm-builder" -version = "0.1.2" +version = "1.0.3" dependencies = [ "anyhow", "cargo_metadata 0.18.1", @@ -4574,12 +4558,12 @@ dependencies = [ "colored", "dirs", "gear-core", + "gear-pwasm-utils", "gear-wasm-instrument", "gmeta", "log", "once_cell", "pathdiff", - "pwasm-utils", "regex", "thiserror", "toml 0.7.8", @@ -4900,7 +4884,7 @@ dependencies = [ [[package]] name = "gstd-codegen" -version = "0.1.0" +version = "1.0.3" dependencies = [ "gstd", "proc-macro2", @@ -5249,6 +5233,19 @@ dependencies = [ "webpki-roots 0.23.1", ] +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "iana-time-zone" version = "0.1.57" @@ -7001,6 +6998,24 @@ dependencies = [ "rand 0.8.5", ] +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "nb-connect" version = "1.2.0" @@ -7370,6 +7385,32 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" +[[package]] +name = "openssl" +version = "0.10.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79a4c6c3a2b158f7f8f2a2fc5a969fa3a068df6fc9dbb4a43845436e3af7c800" +dependencies = [ + "bitflags 2.4.0", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + [[package]] name = "openssl-probe" version = "0.1.5" @@ -7378,9 +7419,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.95" +version = "0.9.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40a4130519a360279579c2053038317e40eff64d13fd3f004f9e1b72b8a6aaf9" +checksum = "3812c071ba60da8b5677cc12bcb1d42989a65553772897a7e0355545a819838f" dependencies = [ "cc", "libc", @@ -7796,7 +7837,7 @@ dependencies = [ "pallet-gear-scheduler", "pallet-timestamp", "parity-scale-codec", - "parity-wasm 0.45.0", + "parity-wasm", "primitive-types", "scale-info", "serde", @@ -7854,7 +7895,7 @@ dependencies = [ "pallet-timestamp", "pallet-transaction-payment", "parity-scale-codec", - "parity-wasm 0.45.0", + "parity-wasm", "primitive-types", "scale-info", "serde", @@ -8024,7 +8065,7 @@ dependencies = [ "log", "pallet-balances", "parity-scale-codec", - "parity-wasm 0.45.0", + "parity-wasm", "primitive-types", "scale-info", "serde", @@ -8505,12 +8546,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa9777aa91b8ad9dd5aaa04a9b6bcb02c7f1deb952fca5a66034d5e63afc5c6f" -[[package]] -name = "parity-wasm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be5e13c266502aadf83426d87d81a0f5d1ef45b8027f5a471c360abfe4bfae92" - [[package]] name = "parity-wasm" version = "0.45.0" @@ -9117,17 +9152,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "pwasm-utils" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ecdabd73c8beaf98c66e45aff3032b56260ee49eb5d0d1222ecce269bfafda7" -dependencies = [ - "byteorder", - "log", - "parity-wasm 0.42.2", -] - [[package]] name = "quick-error" version = "1.2.3" @@ -9531,10 +9555,12 @@ dependencies = [ "http-body", "hyper", "hyper-rustls 0.24.1", + "hyper-tls", "ipnet", "js-sys", "log", "mime", + "native-tls", "once_cell", "percent-encoding", "pin-project-lite 0.2.13", @@ -9545,6 +9571,7 @@ dependencies = [ "serde_urlencoded", "system-configuration", "tokio", + "tokio-native-tls", "tokio-rustls 0.24.1", "tower-service", "url", @@ -12355,7 +12382,7 @@ source = "git+https://github.com/gear-tech/substrate.git?branch=gear-polkadot-v0 dependencies = [ "impl-serde", "parity-scale-codec", - "parity-wasm 0.45.0", + "parity-wasm", "scale-info", "serde", "sp-core-hashing-proc-macro", @@ -13132,6 +13159,16 @@ dependencies = [ "syn 2.0.39", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.23.4" @@ -13979,7 +14016,7 @@ version = "0.1.0" dependencies = [ "clap 4.4.10", "hex", - "parity-wasm 0.45.0", + "parity-wasm", ] [[package]] @@ -13987,7 +14024,7 @@ name = "wasm-instrument" version = "0.3.0" source = "git+https://github.com/gear-tech/wasm-instrument?branch=v0.3.0-sign-ext#7d5226a805b39ed297e78427c21ee2cb39795139" dependencies = [ - "parity-wasm 0.45.0", + "parity-wasm", ] [[package]] @@ -14039,7 +14076,7 @@ dependencies = [ "env_logger", "gear-wasm-builder", "log", - "parity-wasm 0.45.0", + "parity-wasm", ] [[package]] @@ -14338,7 +14375,7 @@ name = "wasmi" version = "0.13.2" source = "git+https://github.com/gear-tech/wasmi?branch=v0.13.2-sign-ext#3a0b1022377919e62aadf4d78b762abd1c3e9a04" dependencies = [ - "parity-wasm 0.45.0", + "parity-wasm", "wasmi-validation", "wasmi_core 0.2.1 (git+https://github.com/gear-tech/wasmi?branch=v0.13.2-sign-ext)", ] @@ -14386,7 +14423,7 @@ name = "wasmi-validation" version = "0.5.0" source = "git+https://github.com/gear-tech/wasmi?branch=v0.13.2-sign-ext#3a0b1022377919e62aadf4d78b762abd1c3e9a04" dependencies = [ - "parity-wasm 0.45.0", + "parity-wasm", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index f2093a5a799..f89971465f0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -103,14 +103,13 @@ byteorder = { version = "1.5.0", default-features = false } blake2-rfc = { version = "0.2.18", default-features = false } bs58 = { version = "0.5.0", default-features = false } cargo_metadata = "0.18.1" -clap = "4.4.8" +clap = "4.4.10" codec = { package = "parity-scale-codec", version = "3.6.4", default-features = false } color-eyre = "0.6.2" colored = "2.0.0" const-str = "0.5" derive_more = "0.99.17" dirs = "4.0.0" -dlmalloc = { git = "https://github.com/gear-tech/dlmalloc-rust.git" } dyn-clonable = "0.9.0" enum-iterator = "1.4.0" env_logger = "0.10" @@ -168,15 +167,12 @@ tempfile = "3.8.1" # Published deps # -# fork of `parity-wasm` with sign-ext enabled by default. -# -# https://github.com/gear-tech/parity-wasm/tree/v0.45.1-sign-ext -# gear-wasm = "0.45.1" -# -# fork of `wasm-instrument` -# +# https://github.com/gear-tech/dlmalloc-rust/tree/v0.1.4 +dlmalloc = { package = "gear-dlmalloc", version = "0.1.4" } # https://github.com/gear-tech/wasm-instrument/tree/v0.2.3-sign-ext gwasm-instrument = { version = "0.2.3", default-features = false } +# https://github.com/gear-tech/wasm-utils/tree/v0.19.0 +pwasm-utils = { version = "0.19.0", package = "gear-pwasm-utils" } # Internal deps numerated = { path = "common/numerated" } @@ -432,7 +428,6 @@ mimalloc = { version = "0.1.39", default-features = false } # node nacl = "0.5.3" # gcli nonempty = "0.8.1" # utils/utils libfuzzer-sys = "0.4" # utils/runtime-fuzzer/fuzz -pwasm-utils = "0.19.0" # utils/wasm-builder page_size = { version = "0.6", default-features = false } # pallets/gear pathdiff = { version = "0.2.1", default-features = false } # utils/wasm-builder rand_pcg = "0.3.1" # pallets/gear diff --git a/galloc/Cargo.toml b/galloc/Cargo.toml index fa5f19ddc84..f8fedb3528a 100644 --- a/galloc/Cargo.toml +++ b/galloc/Cargo.toml @@ -1,5 +1,7 @@ [package] name = "galloc" +description = "Gear Wasm allocator" +keywords = ["gear", "allocator", "wasm"] version.workspace = true authors.workspace = true edition.workspace = true diff --git a/gcli/Cargo.toml b/gcli/Cargo.toml index a562d8cefff..13cd71cc71e 100644 --- a/gcli/Cargo.toml +++ b/gcli/Cargo.toml @@ -36,7 +36,7 @@ scale-info.workspace = true schnorrkel.workspace = true serde.workspace = true serde_json.workspace = true -clap = { workspace = true, features = ["derive"] } +clap = { workspace = true, features = ["std", "derive"] } thiserror.workspace = true tokio = { workspace = true, features = [ "full" ] } whoami.workspace = true diff --git a/gcore/Cargo.toml b/gcore/Cargo.toml index dff1d9db8bf..f0e4ac0fdea 100644 --- a/gcore/Cargo.toml +++ b/gcore/Cargo.toml @@ -1,5 +1,6 @@ [package] name = "gcore" +description = "Gear programs core library" version.workspace = true authors.workspace = true edition.workspace = true diff --git a/gstd/Cargo.toml b/gstd/Cargo.toml index ab8ddff724f..aa6c6ae6d5d 100644 --- a/gstd/Cargo.toml +++ b/gstd/Cargo.toml @@ -1,9 +1,14 @@ [package] name = "gstd" +description = "Gear programs standard library" +keywords = ["gear", "std", "no-std", "wasm", "smart-contracts"] +categories = ["api-bindings"] version.workspace = true -authors.workspace = true edition.workspace = true +authors.workspace = true license.workspace = true +homepage.workspace = true +repository.workspace = true [dependencies] document-features = { version = "0.2.7", optional = true } diff --git a/gstd/codegen/Cargo.toml b/gstd/codegen/Cargo.toml index 3f99d9d2985..5b790f762fb 100644 --- a/gstd/codegen/Cargo.toml +++ b/gstd/codegen/Cargo.toml @@ -1,15 +1,20 @@ [package] name = "gstd-codegen" -version = "0.1.0" -authors.workspace = true +description = "Code generation library for the Gear standard library" +keywords = ["gear", "codegen", "std"] +categories = ["api-bindings"] +version.workspace = true edition.workspace = true +authors.workspace = true license.workspace = true +homepage.workspace = true +repository.workspace = true [lib] proc-macro = true [dependencies] -syn = { workspace = true, features = ["full"] } +syn = { workspace = true, features = ["default", "full"] } quote.workspace = true proc-macro2.workspace = true diff --git a/stack-buffer/Cargo.toml b/stack-buffer/Cargo.toml index df253031675..550d2df76cb 100644 --- a/stack-buffer/Cargo.toml +++ b/stack-buffer/Cargo.toml @@ -1,5 +1,6 @@ [package] name = "gear-stack-buffer" +description = "Gear stack buffer implementation" version.workspace = true authors.workspace = true edition.workspace = true diff --git a/utils/crates-io/Cargo.toml b/utils/crates-io/Cargo.toml index 834d1e80a86..cdac3d49a29 100644 --- a/utils/crates-io/Cargo.toml +++ b/utils/crates-io/Cargo.toml @@ -3,14 +3,11 @@ name = "crates-io-manager" version.workspace = true edition.workspace = true -[[bin]] -name = "publish" -path = "publish.rs" - [dependencies] anyhow.workspace = true cargo_metadata.workspace = true cargo_toml.workspace = true -crates-io.workspace = true -curl.workspace = true +clap = { workspace = true, features = ["derive"] } +reqwest = { workspace = true, features = ["blocking", "json", "default-tls"] } +serde = { workspace = true, features = ["derive"] } toml.workspace = true diff --git a/utils/crates-io/publish.rs b/utils/crates-io/publish.rs deleted file mode 100644 index f8e121abd08..00000000000 --- a/utils/crates-io/publish.rs +++ /dev/null @@ -1,171 +0,0 @@ -//! mini-program for publishing packages to crates.io. -use anyhow::Result; -use cargo_metadata::MetadataCommand; -use cargo_toml::{Dependency, Manifest, Value}; -use crates_io::Registry; -use curl::easy::Easy; -use std::{ - collections::{BTreeMap, HashMap}, - env, fs, - path::PathBuf, - process::{Command, ExitStatus}, - thread, - time::Duration, -}; - -/// Packages need to be published. -const PACKAGES: [&str; 14] = [ - // Packages without local dependencies. - "actor-system-error", - "gear-common-codegen", - "gear-core-errors", - "gear-wasm-instrument", - "gmeta-codegen", - "gsdk-codegen", - "gsys", - // The packages below have local dependencies, - // and should be published in order. - "gmeta", - "gear-core", - "gear-utils", - "gear-common", - "gsdk", - "gcli", - "gclient", -]; - -/// Packages need to be patched in dependencies. -const PATCHED_PACKAGES: [&str; 1] = ["sp-arithmetic"]; - -struct CratesIo { - registry: Registry, -} - -impl CratesIo { - /// Create a new instance of `CratesIo`. - pub fn new() -> Result { - let mut handle = Easy::new(); - handle.useragent("gear-crates-io-manager")?; - - Ok(Self { - registry: Registry::new_handle("https://crates.io".into(), None, handle, false), - }) - } - - /// Verify if the package is published to crates.io. - pub fn verify(&mut self, package: &str, version: &str) -> Result { - // Here using limit = 1 since we are searching explicit - // packages here. - let (crates, _total) = self.registry.search(package, 1)?; - if crates.len() != 1 { - return Ok(false); - } - - Ok(crates[0].max_version == version) - } -} - -fn main() -> Result<()> { - let mut validator = CratesIo::new()?; - let metadata = MetadataCommand::new().no_deps().exec()?; - let mut graph = BTreeMap::new(); - let index = HashMap::::from_iter( - PACKAGES.into_iter().enumerate().map(|(i, p)| (p.into(), i)), - ); - - let workspace_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("../../Cargo.toml") - .canonicalize()?; - let workspace = Manifest::from_path(&workspace_path)?; - - for p in metadata.packages.into_iter() { - if !index.contains_key(&p.name) { - continue; - } - - let version = p.version.to_string(); - if validator.verify(&p.name, &version)? { - println!("Package {}@{} already published.", &p.name, &version); - continue; - } - - let path = p.manifest_path.into_std_path_buf(); - let mut manifest = Manifest::::from_slice_with_metadata(&fs::read(&path)?)?; - manifest.complete_from_path_and_workspace(&path, Some((&workspace, &workspace_path)))?; - - // NOTE: This is a bug inside of crate cargo_toml, it should - // not append crate-type = ["rlib"] to proc-macro crates, fixing - // it by hacking it now. - if p.name.ends_with("-codegen") { - if let Some(mut product) = manifest.lib { - product.crate_type = vec![]; - manifest.lib = Some(product); - } - } - - for (name, dep) in manifest.dependencies.iter_mut() { - // No need to update dependencies for packages without - // local dependencies. - if !index.contains_key(name) && !PATCHED_PACKAGES.contains(&name.as_str()) { - continue; - } - - let mut detail = if let Dependency::Detailed(detail) = &dep { - detail.clone() - } else { - continue; - }; - - match name.as_ref() { - // NOTE: the required version of sp-arithmetic is 6.0.0 in - // git repo, but 7.0.0 in crates.io, so we need to fix it. - "sp-arithmetic" => { - detail.version = Some("7.0.0".into()); - detail.branch = None; - detail.git = None; - } - _ => detail.version = Some(version.to_string()), - } - - *dep = Dependency::Detailed(detail); - } - - graph.insert(index.get(&p.name), (path, manifest)); - } - - for (path, manifest) in graph.values() { - println!("Publishing {:?}", path); - fs::write(path, toml::to_string_pretty(manifest)?)?; - - let path = path.to_string_lossy(); - let status = publish(&path)?; - if !status.success() { - println!( - "Failed to publish package {}...\nRetry after 11 mins...", - &path - ); - // The most likely reason for failure is that - // we have reached the rate limit of crates.io. - // - // Need to wait for 10 mins and try again. here - // we use 11 mins to be safe. - // - // Only retry for once, if it still fails, we - // will just give up. - thread::sleep(Duration::from_secs(660)); - publish(&path)?; - } - } - - Ok(()) -} - -fn publish(manifest: &str) -> Result { - Command::new("cargo") - .arg("publish") - .arg("--manifest-path") - .arg(manifest) - .arg("--allow-dirty") - .status() - .map_err(Into::into) -} diff --git a/utils/crates-io/src/lib.rs b/utils/crates-io/src/lib.rs new file mode 100644 index 00000000000..209e0d5d711 --- /dev/null +++ b/utils/crates-io/src/lib.rs @@ -0,0 +1,71 @@ +// This file is part of Gear. + +// Copyright (C) 2021-2023 Gear Technologies Inc. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! crates-io-manager library +#![deny(missing_docs)] + +mod manifest; +mod publisher; +pub mod rename; +mod version; + +pub use self::{manifest::ManifestWithPath, publisher::Publisher, version::verify}; +use anyhow::Result; +use std::process::{Command, ExitStatus}; + +/// Required Packages without local dependencies. +pub const SAFE_DEPENDENCIES: [&str; 10] = [ + "actor-system-error", + "galloc", + "gear-stack-buffer", + "gear-core-errors", + "gear-common-codegen", + "gear-wasm-instrument", + "gmeta-codegen", + "gsdk-codegen", + "gstd-codegen", + "gsys", +]; + +/// Required packages with local dependencies. +pub const STACKED_DEPENDENCIES: [&str; 5] = + ["gcore", "gmeta", "gear-core", "gear-utils", "gear-common"]; + +/// Packages need to be published. +pub const PACKAGES: [&str; 5] = ["gear-wasm-builder", "gstd", "gsdk", "gclient", "gcli"]; + +/// Check the input package +pub fn check(manifest: &str) -> Result { + Command::new("cargo") + .arg("check") + .arg("--manifest-path") + .arg(manifest) + .status() + .map_err(Into::into) +} + +/// Publish the input package +pub fn publish(manifest: &str) -> Result { + Command::new("cargo") + .arg("publish") + .arg("--manifest-path") + .arg(manifest) + .arg("--allow-dirty") + .status() + .map_err(Into::into) +} diff --git a/utils/crates-io/src/main.rs b/utils/crates-io/src/main.rs new file mode 100644 index 00000000000..1cf6443480e --- /dev/null +++ b/utils/crates-io/src/main.rs @@ -0,0 +1,52 @@ +// This file is part of Gear. + +// Copyright (C) 2021-2023 Gear Technologies Inc. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! mini-program for publishing packages to crates.io. + +use anyhow::Result; +use clap::Parser; +use crates_io_manager::Publisher; + +/// The command to run. +#[derive(Clone, Debug, Parser)] +enum Command { + /// Check packages that to be published. + Check, + /// Publish packages. + Publish, +} + +/// Gear crates-io manager command line interface +/// +/// NOTE: this binary should not be used locally +/// but run in CI. +#[derive(Debug, Parser)] +pub struct Opt { + #[clap(subcommand)] + command: Command, +} + +fn main() -> Result<()> { + let Opt { command } = Opt::parse(); + + let publisher = Publisher::new()?.build()?; + match command { + Command::Check => publisher.check(), + Command::Publish => publisher.publish(), + } +} diff --git a/utils/crates-io/src/manifest.rs b/utils/crates-io/src/manifest.rs new file mode 100644 index 00000000000..ca1d62f68e5 --- /dev/null +++ b/utils/crates-io/src/manifest.rs @@ -0,0 +1,68 @@ +// This file is part of Gear. + +// Copyright (C) 2021-2023 Gear Technologies Inc. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! Manifest utils for crates-io-manager + +use anyhow::Result; +use cargo_toml::{Manifest, Value}; +use std::{ + fs, + path::{Path, PathBuf}, +}; + +/// Cargo manifest with path +pub struct ManifestWithPath { + /// Crate name + pub name: String, + /// Cargo manifest + pub manifest: Manifest, + /// Path of the manifest + pub path: PathBuf, +} + +impl ManifestWithPath { + /// Get the workspace manifest + pub fn workspace() -> Result { + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .ancestors() + .nth(2) + .map(|workspace_dir| workspace_dir.join("Cargo.toml")) + .ok_or_else(|| anyhow::anyhow!("Could not find workspace manifest"))? + .canonicalize()?; + + Ok(Self { + name: "__gear_workspace".into(), + manifest: Manifest::from_path(&path)?, + path, + }) + } + + /// Complete the manifest of the specified crate from + /// the current manifest + pub fn manifest(&self, path: impl AsRef) -> Result { + let mut manifest = Manifest::::from_slice_with_metadata(&fs::read(&path)?)?; + manifest + .complete_from_path_and_workspace(path.as_ref(), Some((&self.manifest, &self.path)))?; + + Ok(Self { + name: manifest.package.clone().map(|p| p.name).unwrap_or_default(), + manifest, + path: path.as_ref().to_path_buf(), + }) + } +} diff --git a/utils/crates-io/src/publisher.rs b/utils/crates-io/src/publisher.rs new file mode 100644 index 00000000000..11211fd1215 --- /dev/null +++ b/utils/crates-io/src/publisher.rs @@ -0,0 +1,139 @@ +// This file is part of Gear. + +// Copyright (C) 2021-2023 Gear Technologies Inc. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! Packages publisher + +use crate::{rename, ManifestWithPath, PACKAGES, SAFE_DEPENDENCIES, STACKED_DEPENDENCIES}; +use anyhow::Result; +use cargo_metadata::{Metadata, MetadataCommand}; +use std::{ + collections::{BTreeMap, HashMap}, + fs, +}; + +/// crates-io packages publisher. +pub struct Publisher { + metadata: Metadata, + graph: BTreeMap, ManifestWithPath>, + index: HashMap, +} + +impl Publisher { + /// Create a new publisher. + pub fn new() -> Result { + let metadata = MetadataCommand::new().no_deps().exec()?; + let graph = BTreeMap::new(); + let index = HashMap::::from_iter( + [ + SAFE_DEPENDENCIES.to_vec(), + STACKED_DEPENDENCIES.into(), + PACKAGES.into(), + ] + .concat() + .into_iter() + .enumerate() + .map(|(i, p)| (p.into(), i)), + ); + + Ok(Self { + metadata, + graph, + index, + }) + } + + /// Build package graphs + /// + /// 1. Replace git dependencies to crates-io dependencies. + /// 2. Rename version of all local packages + /// 3. Patch dependencies if needed + pub fn build(mut self) -> Result { + let workspace = ManifestWithPath::workspace()?; + for p in &self.metadata.packages { + if !self.index.contains_key(&p.name) { + continue; + } + + let version = p.version.to_string(); + println!("Verifying {}@{} ...", &p.name, &version); + if crate::verify(&p.name, &version)? { + println!("Package {}@{} already published .", &p.name, &version); + continue; + } + + let mut manifest = workspace.manifest(&p.manifest_path)?; + rename::package(p, &mut manifest.manifest)?; + rename::deps(&mut manifest.manifest, self.index.keys().collect(), version)?; + + self.graph + .insert(self.index.get(&p.name).cloned(), manifest); + } + + Ok(self) + } + + /// Check the to-be-published packages + /// + /// TODO: Complete the check process (#3565) + pub fn check(&self) -> Result<()> { + self.flush()?; + + let mut failed = Vec::new(); + for ManifestWithPath { path, name, .. } in self.graph.values() { + if !PACKAGES.contains(&name.as_str()) { + continue; + } + + println!("Checking {path:?}"); + let status = crate::check(&path.to_string_lossy())?; + if !status.success() { + failed.push(path); + } + } + + if !failed.is_empty() { + panic!("Packages {failed:?} failed to pass the check ..."); + } + + Ok(()) + } + + /// Publish packages + pub fn publish(&self) -> Result<()> { + self.flush()?; + + for ManifestWithPath { path, .. } in self.graph.values() { + println!("Publishing {path:?}"); + let status = crate::publish(&path.to_string_lossy())?; + if !status.success() { + panic!("Failed to publish package {path:?} ..."); + } + } + + Ok(()) + } + + /// Flush new manifests to disk + fn flush(&self) -> Result<()> { + for ManifestWithPath { path, manifest, .. } in self.graph.values() { + fs::write(path, toml::to_string_pretty(&manifest)?)?; + } + + Ok(()) + } +} diff --git a/utils/crates-io/src/rename.rs b/utils/crates-io/src/rename.rs new file mode 100644 index 00000000000..46eadc9a3fd --- /dev/null +++ b/utils/crates-io/src/rename.rs @@ -0,0 +1,73 @@ +// This file is part of Gear. + +// Copyright (C) 2021-2023 Gear Technologies Inc. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! Renaming handler + +use anyhow::Result; +use cargo_metadata::Package; +use cargo_toml::{Dependency, Manifest}; + +/// Rename a package +pub fn package(pkg: &Package, manifest: &mut Manifest) -> Result<()> { + // NOTE: This is a bug inside of crate cargo_toml, it should + // not append crate-type = ["rlib"] to proc-macro crates, fixing + // it by hacking it now. + if pkg.name.ends_with("-codegen") { + if let Some(product) = manifest.lib.as_mut() { + product.crate_type = vec![]; + } + } + + Ok(()) +} + +/// Rename a dependencies +pub fn deps(map: &mut Manifest, index: Vec<&String>, version: String) -> Result<()> { + for (name, dep) in map.dependencies.iter_mut() { + // No need to update dependencies for packages without + // local dependencies. + if !index.contains(&name) && !name.starts_with("sp-") { + continue; + } + + let mut detail = if let Dependency::Detailed(detail) = &dep { + detail.clone() + } else { + continue; + }; + + match name.as_ref() { + // NOTE: the required version of sp-arithmetic is 6.0.0 in + // git repo, but 7.0.0 in crates.io, so we need to fix it. + "sp-arithmetic" => { + detail.version = Some("7.0.0".into()); + detail.branch = None; + detail.git = None; + } + sp if sp.starts_with("sp-") => { + detail.branch = None; + detail.git = None; + } + _ => detail.version = Some(version.clone()), + } + + *dep = Dependency::Detailed(detail); + } + + Ok(()) +} diff --git a/utils/crates-io/src/version.rs b/utils/crates-io/src/version.rs new file mode 100644 index 00000000000..d22924818ef --- /dev/null +++ b/utils/crates-io/src/version.rs @@ -0,0 +1,46 @@ +// This file is part of Gear. + +// Copyright (C) 2021-2023 Gear Technologies Inc. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! Crate verifier + +use anyhow::Result; +use serde::Deserialize; + +#[derive(Debug, Deserialize)] +struct Resp { + pub versions: Vec, +} + +#[derive(Debug, Deserialize)] +struct Version { + pub num: String, +} + +/// Verify if the package has already been published. +pub fn verify(name: &str, version: &str) -> Result { + let client = reqwest::blocking::Client::builder() + .user_agent("gear-crates-io-manager") + .build()?; + + let resp = client + .get(format!("https://crates.io/api/v1/crates/{name}/versions")) + .send()? + .json::()?; + + Ok(resp.versions.into_iter().any(|v| v.num == version)) +} diff --git a/utils/wasm-builder/Cargo.toml b/utils/wasm-builder/Cargo.toml index 79bad58298c..d9952c4dfd1 100644 --- a/utils/wasm-builder/Cargo.toml +++ b/utils/wasm-builder/Cargo.toml @@ -1,11 +1,11 @@ [package] name = "gear-wasm-builder" -version = "0.1.2" +version.workspace = true edition.workspace = true -license = "GPL-3.0-or-later WITH Classpath-exception-2.0" +license.workspace = true +repository.workspace = true description = "Utility for building Gear programs" -include = ["src/**/*", "Cargo.toml", "README.md"] -repository = "https://github.com/gear-tech/gear" +include = ["build.rs", "src/**/*", "Cargo.toml", "README.md"] readme = "README.md" [dependencies] diff --git a/utils/wasm-builder/src/optimize.rs b/utils/wasm-builder/src/optimize.rs index dcff479eb3f..376636dd64d 100644 --- a/utils/wasm-builder/src/optimize.rs +++ b/utils/wasm-builder/src/optimize.rs @@ -78,7 +78,7 @@ impl Optimizer { } pub fn flush_to_file(self, path: &PathBuf) { - fs::write(path, self.module.to_bytes().unwrap()).unwrap(); + fs::write(path, self.module.into_bytes().unwrap()).unwrap(); } /// Process optimization. diff --git a/utils/wasm-builder/src/stack_end.rs b/utils/wasm-builder/src/stack_end.rs index 529ec48fd52..e98986587a6 100644 --- a/utils/wasm-builder/src/stack_end.rs +++ b/utils/wasm-builder/src/stack_end.rs @@ -37,7 +37,7 @@ use pwasm_utils::parity_wasm::{ pub fn insert_stack_end_export(module: &mut Module) -> Result<(), &'static str> { let module_bytes = module .clone() - .to_bytes() + .into_bytes() .map_err(|_| "cannot get code from module")?; let stack_pointer_index = @@ -141,7 +141,7 @@ pub fn insert_start_call_in_export_funcs(module: &mut Module) -> Result<(), &'st pub fn move_mut_globals_to_static(module: &mut Module) -> Result<(), &'static str> { let module_bytes = module .clone() - .to_bytes() + .into_bytes() .map_err(|_| "cannot get code from module")?; // Identify stack pointer and data end globals @@ -477,7 +477,7 @@ mod test { // Insert `_start` call in `handle` code and check that it works as expected. let mut module = parity_wasm::deserialize_buffer(binary.as_ref()).unwrap(); insert_start_call_in_export_funcs(&mut module).unwrap(); - check(&module.to_bytes().unwrap(), 12); + check(&module.into_bytes().unwrap(), 12); } #[test] @@ -556,6 +556,6 @@ mod test { // their values after first execution, and second execution will return another result. let mut module = parity_wasm::deserialize_buffer(binary.as_ref()).unwrap(); move_mut_globals_to_static(&mut module).unwrap(); - check(&module.to_bytes().unwrap(), 111, 113); + check(&module.into_bytes().unwrap(), 111, 113); } }