diff --git a/.devcontainer/post_create.sh b/.devcontainer/post_create.sh index cbc54dfeabe..987da1a8c58 100644 --- a/.devcontainer/post_create.sh +++ b/.devcontainer/post_create.sh @@ -69,3 +69,6 @@ opam install -y dune ounit2 ocamlformat # Setup for Cpp binding sudo apt install -y ninja-build + +# Setup for D binding +sudo apt install -y dmd dub diff --git a/.gitattributes b/.gitattributes index 51806da0353..cf8f7fa77cd 100644 --- a/.gitattributes +++ b/.gitattributes @@ -8,6 +8,7 @@ integrations export-ignore bindings/c export-ignore bindings/cpp export-ignore +bindings/d export-ignore bindings/dotnet export-ignore bindings/go export-ignore bindings/haskell export-ignore diff --git a/.github/workflows/ci_bindings_d.yml b/.github/workflows/ci_bindings_d.yml new file mode 100644 index 00000000000..9835868b59c --- /dev/null +++ b/.github/workflows/ci_bindings_d.yml @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Bindings D CI + +on: + push: + branches: + - main + tags: + - "*" + pull_request: + branches: + - main + paths: + - "core/**" + - "bindings/c/**" + - "bindings/d/**" + - ".github/workflows/ci_bindings_d.yml" + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + test: + strategy: + matrix: + # dmd: base (self-hosting) compiler (frontend & backend) + # ldc2/ldmd2: (dmd-frontend + LLVM backend) - recommended for MacOS ARM64 + dlang: ["ldc-latest", "dmd-latest"] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dlang-community/setup-dlang@v2 + with: + compiler: ${{ matrix.dlang }} + + - name: Setup Rust toolchain + uses: ./.github/actions/setup + + - name: Build D binding + working-directory: bindings/d + run: dub build + + - name: Check diff + run: git diff --exit-code + + - name: Check + working-directory: bindings/d + run: dub lint + + - name: Run tests + working-directory: bindings/d + run: dub test && cd test && dub diff --git a/.typos.toml b/.typos.toml index 9c58e5d2d9e..7d504be990a 100644 --- a/.typos.toml +++ b/.typos.toml @@ -36,4 +36,6 @@ extend-exclude = [ # Generated pnpm locks. "website/pnpm-lock.yaml", "CHANGELOG.md", + # dscanner config + "bindings/d/dscanner.ini", ] diff --git a/README.md b/README.md index 4e74c654eaf..d721fa888b6 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ OpenDAL offers a unified data access layer, empowering users to seamlessly and e | [Rust Core] | [![Rust Core Image]][Rust Core Link] | [![Docs Release]][Rust Core Release Docs] [![Docs Dev]][Rust Core Dev Docs] | | [C Binding] | - | [![Docs Dev]][C Binding Dev Docs] | | [Cpp Binding] | - | [![Docs Dev]][Cpp Binding Dev Docs] | +| [D Binding] | - | - | | [Dotnet Binding] | - | - | | [Go Binding] | [![Go Binding Image]][Go Binding Link] | [![Docs Release]][Go Release Docs] | | [Haskell Binding] | - | - | @@ -38,6 +39,7 @@ OpenDAL offers a unified data access layer, empowering users to seamlessly and e [C Binding Dev Docs]: https://opendal.apache.org/docs/c/ [Cpp Binding]: bindings/cpp/README.md [Cpp Binding Dev Docs]: https://opendal.apache.org/docs/cpp/ +[D Binding]: bindings/d/README.md [Dotnet Binding]: bindings/dotnet/README.md [Go Binding]: bindings/go/README.md [Go Binding Image]: https://badge.fury.io/go/github.com%2Fapache%2Fopendal%2Fbindings%2Fgo.svg diff --git a/bin/oli/Cargo.lock b/bin/oli/Cargo.lock index fc969b52789..672cc75e590 100644 --- a/bin/oli/Cargo.lock +++ b/bin/oli/Cargo.lock @@ -671,6 +671,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8" dependencies = [ "clap_builder", + "clap_derive", ] [[package]] @@ -685,6 +686,18 @@ dependencies = [ "strsim", ] +[[package]] +name = "clap_derive" +version = "4.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "clap_lex" version = "0.7.2" @@ -1663,7 +1676,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" dependencies = [ "cfg-if", - "windows-targets 0.52.6", + "windows-targets 0.48.5", ] [[package]] diff --git a/bin/oli/Cargo.toml b/bin/oli/Cargo.toml index 924250ded3d..4030cffee2e 100644 --- a/bin/oli/Cargo.toml +++ b/bin/oli/Cargo.toml @@ -55,7 +55,7 @@ services-sled = ["opendal/services-sled"] [dependencies] anyhow = "1" -clap = { version = "4", features = ["cargo", "string"] } +clap = { version = "4", features = ["cargo", "string", "derive", "deprecated"] } dirs = "5.0.1" futures = "0.3" opendal = { version = "0.50.0", path = "../../core", features = [ diff --git a/bin/oli/src/bin/oli.rs b/bin/oli/src/bin/oli.rs index 56e281e314a..99ba3d27630 100644 --- a/bin/oli/src/bin/oli.rs +++ b/bin/oli/src/bin/oli.rs @@ -28,28 +28,13 @@ use std::path::PathBuf; use anyhow::anyhow; use anyhow::Result; -use clap::value_parser; -use clap::Arg; -use clap::Command; -use dirs::config_dir; +use oli::commands::OliSubcommand; -fn new_cmd(name: &'static str) -> Result { - let d = config_dir().ok_or_else(|| anyhow!("unknown config dir"))?; - let default_config_path = d.join("oli/config.toml").as_os_str().to_owned(); - - Ok(Command::new(name) - .version(env!("CARGO_PKG_VERSION")) - .arg( - Arg::new("config") - .long("config") - .help("Path to the config file") - .global(true) - .default_value(default_config_path) - .value_parser(value_parser!(PathBuf)) - .required(false), - ) - .subcommand_required(true) - .arg_required_else_help(true)) +#[derive(Debug, clap::Parser)] +#[command(about, version)] +pub struct Oli { + #[command(subcommand)] + subcommand: OliSubcommand, } #[tokio::main] @@ -66,28 +51,28 @@ async fn main() -> Result<()> { .and_then(OsStr::to_str) { Some("oli") => { - let cmd = oli::commands::cli::cli(new_cmd("oli")?); - oli::commands::cli::main(&cmd.get_matches()).await?; + let cmd: Oli = clap::Parser::parse(); + cmd.subcommand.run().await?; } Some("ocat") => { - let cmd = oli::commands::cat::cli(new_cmd("ocat")?); - oli::commands::cat::main(&cmd.get_matches()).await?; + let cmd: oli::commands::cat::CatCmd = clap::Parser::parse(); + cmd.run().await?; } Some("ocp") => { - let cmd = oli::commands::cp::cli(new_cmd("ocp")?); - oli::commands::cp::main(&cmd.get_matches()).await?; + let cmd: oli::commands::cp::CopyCmd = clap::Parser::parse(); + cmd.run().await?; } Some("ols") => { - let cmd = oli::commands::ls::cli(new_cmd("ols")?); - oli::commands::ls::main(&cmd.get_matches()).await?; + let cmd: oli::commands::ls::LsCmd = clap::Parser::parse(); + cmd.run().await?; } Some("orm") => { - let cmd = oli::commands::rm::cli(new_cmd("orm")?); - oli::commands::rm::main(&cmd.get_matches()).await?; + let cmd: oli::commands::rm::RmCmd = clap::Parser::parse(); + cmd.run().await?; } Some("ostat") => { - let cmd = oli::commands::stat::cli(new_cmd("ostat")?); - oli::commands::stat::main(&cmd.get_matches()).await?; + let cmd: oli::commands::stat::StatCmd = clap::Parser::parse(); + cmd.run().await?; } Some(v) => { println!("{v} is not supported") diff --git a/bin/oli/src/commands/cat.rs b/bin/oli/src/commands/cat.rs index 4bfe8190f7f..fa892be6680 100644 --- a/bin/oli/src/commands/cat.rs +++ b/bin/oli/src/commands/cat.rs @@ -15,39 +15,38 @@ // specific language governing permissions and limitations // under the License. -use std::path::PathBuf; - -use anyhow::anyhow; use anyhow::Result; -use clap::Arg; -use clap::ArgMatches; -use clap::Command; use futures::io; use crate::config::Config; +use crate::params::config::ConfigParams; -pub async fn main(args: &ArgMatches) -> Result<()> { - let config_path = args - .get_one::("config") - .ok_or_else(|| anyhow!("missing config path"))?; - let cfg = Config::load(config_path)?; +#[derive(Debug, clap::Parser)] +#[command( + name = "cat", + about = "Display object content", + disable_version_flag = true +)] +pub struct CatCmd { + #[command(flatten)] + pub config_params: ConfigParams, + #[arg()] + pub target: String, +} - let target = args - .get_one::("target") - .ok_or_else(|| anyhow!("missing target"))?; - let (op, path) = cfg.parse_location(target)?; +impl CatCmd { + pub async fn run(&self) -> Result<()> { + let cfg = Config::load(&self.config_params.config)?; - let reader = op.reader(&path).await?; - let meta = op.stat(&path).await?; - let mut buf_reader = reader - .into_futures_async_read(0..meta.content_length()) - .await?; - let mut stdout = io::AllowStdIo::new(std::io::stdout()); - io::copy_buf(&mut buf_reader, &mut stdout).await?; - Ok(()) -} + let (op, path) = cfg.parse_location(&self.target)?; -pub fn cli(cmd: Command) -> Command { - cmd.about("display object content") - .arg(Arg::new("target").required(true)) + let reader = op.reader(&path).await?; + let meta = op.stat(&path).await?; + let mut buf_reader = reader + .into_futures_async_read(0..meta.content_length()) + .await?; + let mut stdout = io::AllowStdIo::new(std::io::stdout()); + io::copy_buf(&mut buf_reader, &mut stdout).await?; + Ok(()) + } } diff --git a/bin/oli/src/commands/cli.rs b/bin/oli/src/commands/cli.rs deleted file mode 100644 index 320ed155d21..00000000000 --- a/bin/oli/src/commands/cli.rs +++ /dev/null @@ -1,47 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use anyhow::anyhow; -use anyhow::Result; -use clap::ArgMatches; -use clap::Command; - -pub async fn main(args: &ArgMatches) -> Result<()> { - match args.subcommand() { - Some(("cat", sub_args)) => super::cat::main(sub_args).await?, - Some(("cp", sub_args)) => super::cp::main(sub_args).await?, - Some(("ls", sub_args)) => super::ls::main(sub_args).await?, - Some(("rm", sub_args)) => super::rm::main(sub_args).await?, - Some(("stat", sub_args)) => super::stat::main(sub_args).await?, - _ => return Err(anyhow!("not handled")), - } - - Ok(()) -} - -fn new_cmd(name: &'static str) -> Command { - Command::new(name).disable_version_flag(true) -} - -pub fn cli(cmd: Command) -> Command { - cmd.about("OpenDAL Command Line Interface") - .subcommand(super::cat::cli(new_cmd("cat"))) - .subcommand(super::cp::cli(new_cmd("cp"))) - .subcommand(super::ls::cli(new_cmd("ls"))) - .subcommand(super::rm::cli(new_cmd("rm"))) - .subcommand(super::stat::cli(new_cmd("stat"))) -} diff --git a/bin/oli/src/commands/cp.rs b/bin/oli/src/commands/cp.rs index d8ea084df3b..e96f65097c3 100644 --- a/bin/oli/src/commands/cp.rs +++ b/bin/oli/src/commands/cp.rs @@ -16,90 +16,77 @@ // under the License. use std::path::Path; -use std::path::PathBuf; -use anyhow::anyhow; use anyhow::Result; -use clap::Arg; -use clap::ArgAction; -use clap::ArgMatches; -use clap::Command; use futures::AsyncWriteExt; use futures::TryStreamExt; use crate::config::Config; +use crate::params::config::ConfigParams; -pub async fn main(args: &ArgMatches) -> Result<()> { - let config_path = args - .get_one::("config") - .ok_or_else(|| anyhow!("missing config path"))?; - let cfg = Config::load(config_path)?; - let recursive = args.get_flag("recursive"); +#[derive(Debug, clap::Parser)] +#[command(name = "cp", about = "Copy object", disable_version_flag = true)] +pub struct CopyCmd { + #[command(flatten)] + pub config_params: ConfigParams, + #[arg()] + pub source: String, + #[arg()] + pub destination: String, + /// Copy objects recursively. + #[arg(short = 'r', long)] + pub recursive: bool, +} - let src = args - .get_one::("source") - .ok_or_else(|| anyhow!("missing source"))?; - let (src_op, src_path) = cfg.parse_location(src)?; +impl CopyCmd { + pub async fn run(&self) -> Result<()> { + let cfg = Config::load(&self.config_params.config)?; - let dst = args - .get_one::("destination") - .ok_or_else(|| anyhow!("missing target"))?; - let (dst_op, dst_path) = cfg.parse_location(dst)?; + let (src_op, src_path) = cfg.parse_location(&self.source)?; - if !recursive { - let mut dst_w = dst_op.writer(&dst_path).await?.into_futures_async_write(); - let src_meta = src_op.stat(&src_path).await?; - let reader = src_op.reader_with(&src_path).chunk(8 * 1024 * 1024).await?; - let buf_reader = reader - .into_futures_async_read(0..src_meta.content_length()) - .await?; - futures::io::copy_buf(buf_reader, &mut dst_w).await?; - // flush data - dst_w.close().await?; - return Ok(()); - } + let (dst_op, dst_path) = cfg.parse_location(&self.destination)?; - let dst_root = Path::new(&dst_path); - let mut ds = src_op.lister_with(&src_path).recursive(true).await?; - let prefix = src_path.strip_prefix('/').unwrap_or(src_path.as_str()); - while let Some(de) = ds.try_next().await? { - let meta = de.metadata(); - if meta.mode().is_dir() { - continue; + if !self.recursive { + let mut dst_w = dst_op.writer(&dst_path).await?.into_futures_async_write(); + let src_meta = src_op.stat(&src_path).await?; + let reader = src_op.reader_with(&src_path).chunk(8 * 1024 * 1024).await?; + let buf_reader = reader + .into_futures_async_read(0..src_meta.content_length()) + .await?; + futures::io::copy_buf(buf_reader, &mut dst_w).await?; + // flush data + dst_w.close().await?; + return Ok(()); } - let depath = de.path(); - let fp = depath - .strip_prefix('/') - .unwrap_or(depath) - .strip_prefix(prefix) - .expect("invalid path"); - let reader = src_op.reader_with(de.path()).chunk(8 * 1024 * 1024).await?; - let buf_reader = reader - .into_futures_async_read(0..meta.content_length()) - .await?; - let mut writer = dst_op - .writer(&dst_root.join(fp).to_string_lossy()) - .await? - .into_futures_async_write(); + let dst_root = Path::new(&dst_path); + let mut ds = src_op.lister_with(&src_path).recursive(true).await?; + let prefix = src_path.strip_prefix('/').unwrap_or(src_path.as_str()); + while let Some(de) = ds.try_next().await? { + let meta = de.metadata(); + if meta.mode().is_dir() { + continue; + } + let depath = de.path(); + let fp = depath + .strip_prefix('/') + .unwrap_or(depath) + .strip_prefix(prefix) + .expect("invalid path"); + let reader = src_op.reader_with(de.path()).chunk(8 * 1024 * 1024).await?; + let buf_reader = reader + .into_futures_async_read(0..meta.content_length()) + .await?; - println!("Copying {}", de.path()); - futures::io::copy_buf(buf_reader, &mut writer).await?; - writer.close().await?; - } - Ok(()) -} + let mut writer = dst_op + .writer(&dst_root.join(fp).to_string_lossy()) + .await? + .into_futures_async_write(); -pub fn cli(cmd: Command) -> Command { - cmd.about("copy") - .arg(Arg::new("source").required(true)) - .arg(Arg::new("destination").required(true)) - .arg( - Arg::new("recursive") - .required(false) - .long("recursive") - .short('r') - .help("Copy files under source recursively to destination") - .action(ArgAction::SetTrue), - ) + println!("Copying {}", de.path()); + futures::io::copy_buf(buf_reader, &mut writer).await?; + writer.close().await?; + } + Ok(()) + } } diff --git a/bin/oli/src/commands/ls.rs b/bin/oli/src/commands/ls.rs index 50c60c56ec6..3e7eb102688 100644 --- a/bin/oli/src/commands/ls.rs +++ b/bin/oli/src/commands/ls.rs @@ -15,53 +15,42 @@ // specific language governing permissions and limitations // under the License. -use std::path::PathBuf; - -use anyhow::anyhow; use anyhow::Result; -use clap::Arg; -use clap::ArgAction; -use clap::ArgMatches; -use clap::Command; use futures::TryStreamExt; use crate::config::Config; +use crate::params::config::ConfigParams; + +#[derive(Debug, clap::Parser)] +#[command(name = "ls", about = "List object", disable_version_flag = true)] +pub struct LsCmd { + #[command(flatten)] + pub config_params: ConfigParams, + #[arg()] + pub target: String, + /// List objects recursively. + #[arg(short, long)] + pub recursive: bool, +} -pub async fn main(args: &ArgMatches) -> Result<()> { - let config_path = args - .get_one::("config") - .ok_or_else(|| anyhow!("missing config path"))?; - let cfg = Config::load(config_path)?; +impl LsCmd { + pub async fn run(&self) -> Result<()> { + let cfg = Config::load(&self.config_params.config)?; - let recursive = args.get_flag("recursive"); + let (op, path) = cfg.parse_location(&self.target)?; - let target = args - .get_one::("target") - .ok_or_else(|| anyhow!("missing target"))?; - let (op, path) = cfg.parse_location(target)?; + if !self.recursive { + let mut ds = op.lister(&path).await?; + while let Some(de) = ds.try_next().await? { + println!("{}", de.name()); + } + return Ok(()); + } - if !recursive { - let mut ds = op.lister(&path).await?; + let mut ds = op.lister_with(&path).recursive(true).await?; while let Some(de) = ds.try_next().await? { - println!("{}", de.name()); + println!("{}", de.path()); } - return Ok(()); + Ok(()) } - - let mut ds = op.lister_with(&path).recursive(true).await?; - while let Some(de) = ds.try_next().await? { - println!("{}", de.path()); - } - Ok(()) -} - -pub fn cli(cmd: Command) -> Command { - cmd.about("ls").arg(Arg::new("target").required(true)).arg( - Arg::new("recursive") - .required(false) - .long("recursive") - .short('r') - .help("List recursively") - .action(ArgAction::SetTrue), - ) } diff --git a/bin/oli/src/commands/mod.rs b/bin/oli/src/commands/mod.rs index 749af022a40..e70d0c101a6 100644 --- a/bin/oli/src/commands/mod.rs +++ b/bin/oli/src/commands/mod.rs @@ -15,27 +15,31 @@ // specific language governing permissions and limitations // under the License. -//! Commands provides the implementation of each commands. -//! -//! Each submodule represents a single command, and should export 2 functions respectively. -//! The signature of those should be like the following: -//! -//! ```ignore -//! pub async fn main(args: &ArgMatches) -> Result<()> { -//! // the main logic -//! } -//! -//! // cli is used to customize the command, like setting the arguments. -//! // As each command can be invoked like a separate binary, -//! // we will pass a command with different name to get the complete command. -//! pub fn cli(cmd: Command) -> Command { -//! // set the arguments, help message, etc. -//! } -//! ``` +//! Provides the implementation of each command. pub mod cat; -pub mod cli; pub mod cp; pub mod ls; pub mod rm; pub mod stat; + +#[derive(Debug, clap::Subcommand)] +pub enum OliSubcommand { + Cat(cat::CatCmd), + Cp(cp::CopyCmd), + Ls(ls::LsCmd), + Rm(rm::RmCmd), + Stat(stat::StatCmd), +} + +impl OliSubcommand { + pub async fn run(&self) -> anyhow::Result<()> { + match self { + Self::Cat(cmd) => cmd.run().await, + Self::Cp(cmd) => cmd.run().await, + Self::Ls(cmd) => cmd.run().await, + Self::Rm(cmd) => cmd.run().await, + Self::Stat(cmd) => cmd.run().await, + } + } +} diff --git a/bin/oli/src/commands/rm.rs b/bin/oli/src/commands/rm.rs index 87fbe913aed..04bc258d48c 100644 --- a/bin/oli/src/commands/rm.rs +++ b/bin/oli/src/commands/rm.rs @@ -15,50 +15,37 @@ // specific language governing permissions and limitations // under the License. -use std::path::PathBuf; - -use anyhow::anyhow; use anyhow::Result; -use clap::Arg; -use clap::ArgAction; -use clap::ArgMatches; -use clap::Command; use crate::config::Config; +use crate::params::config::ConfigParams; + +#[derive(Debug, clap::Parser)] +#[command(name = "rm", about = "Remove object", disable_version_flag = true)] +pub struct RmCmd { + #[command(flatten)] + pub config_params: ConfigParams, + #[arg()] + pub target: String, + /// Remove objects recursively. + #[arg(short, long)] + pub recursive: bool, +} -pub async fn main(args: &ArgMatches) -> Result<()> { - let config_path = args - .get_one::("config") - .ok_or_else(|| anyhow!("missing config path"))?; - let cfg = Config::load(config_path)?; +impl RmCmd { + pub async fn run(&self) -> Result<()> { + let cfg = Config::load(&self.config_params.config)?; - let recursive = args.get_flag("recursive"); + let (op, path) = cfg.parse_location(&self.target)?; - let target = args - .get_one::("target") - .ok_or_else(|| anyhow!("missing target"))?; - let (op, path) = cfg.parse_location(target)?; + if !self.recursive { + println!("Delete: {path}"); + op.delete(&path).await?; + return Ok(()); + } - if !recursive { - println!("Delete: {path}"); - op.delete(&path).await?; - return Ok(()); + println!("Delete all: {path}"); + op.remove_all(&path).await?; + Ok(()) } - - println!("Delete all: {path}"); - op.remove_all(&path).await?; - Ok(()) -} - -pub fn cli(cmd: Command) -> Command { - cmd.about("remove object") - .arg(Arg::new("target").required(true)) - .arg( - Arg::new("recursive") - .required(false) - .long("recursive") - .short('r') - .help("List recursively") - .action(ArgAction::SetTrue), - ) } diff --git a/bin/oli/src/commands/stat.rs b/bin/oli/src/commands/stat.rs index d4efacae485..8f1051f1a1c 100644 --- a/bin/oli/src/commands/stat.rs +++ b/bin/oli/src/commands/stat.rs @@ -15,47 +15,47 @@ // specific language governing permissions and limitations // under the License. -use std::path::PathBuf; - -use anyhow::anyhow; use anyhow::Result; -use clap::Arg; -use clap::ArgMatches; -use clap::Command; use crate::config::Config; - -pub async fn main(args: &ArgMatches) -> Result<()> { - let config_path = args - .get_one::("config") - .ok_or_else(|| anyhow!("missing config path"))?; - let cfg = Config::load(config_path)?; - - let target = args - .get_one::("target") - .ok_or_else(|| anyhow!("missing target"))?; - let (op, path) = cfg.parse_location(target)?; - - let meta = op.stat(&path).await?; - println!("path: {target}"); - let size = meta.content_length(); - println!("size: {size}"); - if let Some(etag) = meta.etag() { - println!("etag: {etag}"); - } - let file_type = meta.mode(); - println!("type: {file_type}"); - if let Some(content_type) = meta.content_type() { - println!("content-type: {content_type}"); - } - if let Some(last_modified) = meta.last_modified() { - println!("last-modified: {last_modified}"); - } - - Ok(()) +use crate::params::config::ConfigParams; + +#[derive(Debug, clap::Parser)] +#[command( + name = "stat", + about = "Show object metadata", + disable_version_flag = true +)] +pub struct StatCmd { + #[command(flatten)] + pub config_params: ConfigParams, + #[arg()] + pub target: String, } -pub fn cli(cmd: Command) -> Command { - cmd.about("show object metadata") - .arg(Arg::new("target").required(true)) +impl StatCmd { + pub async fn run(&self) -> Result<()> { + let cfg = Config::load(&self.config_params.config)?; + + let target = &self.target; + println!("path: {target}"); + let (op, path) = cfg.parse_location(target)?; + + let meta = op.stat(&path).await?; + let size = meta.content_length(); + println!("size: {size}"); + if let Some(etag) = meta.etag() { + println!("etag: {etag}"); + } + let file_type = meta.mode(); + println!("type: {file_type}"); + if let Some(content_type) = meta.content_type() { + println!("content-type: {content_type}"); + } + if let Some(last_modified) = meta.last_modified() { + println!("last-modified: {last_modified}"); + } + + Ok(()) + } } diff --git a/bin/oli/src/config/mod.rs b/bin/oli/src/config/mod.rs index 93d888e4ff7..d15ea884a49 100644 --- a/bin/oli/src/config/mod.rs +++ b/bin/oli/src/config/mod.rs @@ -30,7 +30,6 @@ use opendal::services; use opendal::Operator; use opendal::Scheme; use serde::Deserialize; -use toml; use url::Url; #[derive(Deserialize, Default)] diff --git a/bin/oli/src/lib.rs b/bin/oli/src/lib.rs index 7f0fce63955..e829973afc5 100644 --- a/bin/oli/src/lib.rs +++ b/bin/oli/src/lib.rs @@ -17,3 +17,4 @@ pub mod commands; pub mod config; +pub mod params; diff --git a/bin/oli/src/params/config.rs b/bin/oli/src/params/config.rs new file mode 100644 index 00000000000..3ceb8f951bf --- /dev/null +++ b/bin/oli/src/params/config.rs @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::ffi::OsString; +use std::path::PathBuf; + +#[derive(Debug, clap::Args)] +pub struct ConfigParams { + /// Path to the config file. + #[arg( + long, + default_value = default_config_path(), + value_parser = clap::value_parser!(PathBuf) + )] + pub config: PathBuf, +} + +fn default_config_path() -> OsString { + let d = dirs::config_dir().expect("unknown config dir"); + d.join("oli/config.toml").as_os_str().to_owned() +} diff --git a/bin/oli/src/params/mod.rs b/bin/oli/src/params/mod.rs new file mode 100644 index 00000000000..207d7e279fb --- /dev/null +++ b/bin/oli/src/params/mod.rs @@ -0,0 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Provides the implementation of common parameters. + +pub mod config; diff --git a/bindings/README.md b/bindings/README.md index 21d32c7d5e2..2ad662e693a 100644 --- a/bindings/README.md +++ b/bindings/README.md @@ -13,6 +13,7 @@ This folder contains the bindings for OpenDAL. Currently, we support the followi * [C](c/README.md) * [C++](cpp/README.md) * [C#](dotnet/README.md) +* [D](d/README.md) * [Go](go/README.md) * [Haskell](haskell/README.md) * [Lua](lua/README.md) diff --git a/bindings/d/.gitignore b/bindings/d/.gitignore new file mode 100644 index 00000000000..3acdc490d35 --- /dev/null +++ b/bindings/d/.gitignore @@ -0,0 +1,20 @@ +.dub +docs.json +__dummy.html +docs/ +/d +d.so +d.dylib +d.dll +d.a +d.lib +d-test-* +*.exe +*.pdb +*.o +*.obj +*.lst +*.h +*.a +*.ninja_log +tests* diff --git a/bindings/d/CONTRIBUTING.md b/bindings/d/CONTRIBUTING.md new file mode 100644 index 00000000000..c64e596ad5c --- /dev/null +++ b/bindings/d/CONTRIBUTING.md @@ -0,0 +1,58 @@ +# Contributing + +- [Contributing](#contributing) + - [Setup](#setup) + - [Using a dev container environment](#using-a-dev-container-environment) + - [Bring your own toolbox](#bring-your-own-toolbox) + - [Build](#build) + - [Test](#test) + +## Setup + +### Using a dev container environment + +OpenDAL provides a pre-configured [dev container](https://containers.dev/) that could be used in [GitHub Codespaces](https://github.com/features/codespaces), [VSCode](https://code.visualstudio.com/), [JetBrains](https://www.jetbrains.com/remote-development/gateway/), [JupyterLab](https://jupyterlab.readthedocs.io/en/stable/). Please pick up your favourite runtime environment. + +The fastest way is: + +[![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://codespaces.new/apache/opendal?quickstart=1&machine=standardLinux32gb) + +### Bring your own toolbox + +To build OpenDAL D binding locally, you need: + +- [dmd/ldc/gdc](https://dlang.org/download) + + +## Build + +First, build the C bindings: + +```shell +dub build -b release +``` + +> **Note**: +> +> - `dub build` adds the header file `opendal.h` under `../c/include` +> - The library is under `../../target/debug` or `../../target/release` after building. + +## Test + +To build and run the tests. + +```shell +$ dub test + Generating test runner configuration 'opendal-test-unittest' for 'unittest' (library). + Starting Performing "unittest" build using /usr/bin/ldc2 for x86_64. + Building opendal ~master: building configuration [opendal-test-unittest] + Pre-build Running commands + Finished `release` profile [optimized] target(s) in 0.08s +Cargo build completed successfully + Linking opendal-test-unittest + Running opendal-test-unittest +Basic Operator creation and write test passed +1 modules passed unittests +``` + + diff --git a/bindings/d/DEPENDENCIES.md b/bindings/d/DEPENDENCIES.md new file mode 100644 index 00000000000..99097232086 --- /dev/null +++ b/bindings/d/DEPENDENCIES.md @@ -0,0 +1,4 @@ +# Dependencies + +OpenDAL D Binding is based on the C Binding. +There are no extra runtime dependencies except those conveyed from C Binding. diff --git a/bindings/d/DEPENDENCIES.rust.tsv b/bindings/d/DEPENDENCIES.rust.tsv new file mode 100644 index 00000000000..e69de29bb2d diff --git a/bindings/d/README.md b/bindings/d/README.md new file mode 100644 index 00000000000..bb1a9e40fdf --- /dev/null +++ b/bindings/d/README.md @@ -0,0 +1,24 @@ +# Apache OpenDAL™ D Binding (WIP) + +![](https://img.shields.io/badge/status-unreleased-red) + +![](https://github.com/apache/opendal/assets/5351546/87bbf6e5-f19e-449a-b368-3e283016c887) + +## Build + +To compile OpenDAL from source code, you need: + +- [dmd/ldc/gdc](https://dlang.org/download) + +```bash +# build libopendal_c (underneath call make -C ../c) +dub build -b release +# build and run unit tests +dub test +``` + +## License and Trademarks + +Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + +Apache OpenDAL, OpenDAL, and Apache are either registered trademarks or trademarks of the Apache Software Foundation. diff --git a/bindings/d/build.d b/bindings/d/build.d new file mode 100644 index 00000000000..9b5387a804b --- /dev/null +++ b/bindings/d/build.d @@ -0,0 +1,42 @@ +module build; +import std.stdio: writeln; +import std.path: buildPath; +import std.process: spawnShell, wait; +import std.exception; +import std.file: copy, mkdir, exists; +import std.conv: to; + +version (Windows) + enum staticlib = "opendal_c.lib"; +else + enum staticlib = "libopendal_c.a"; + +void main (string[] args) +{ + bool isRelease = args.length > 1 && args[1] == "release"; + string buildType = isRelease ? "release" : "debug"; + + // Run cargo build + auto cargoCmd = "cargo build --manifest-path " ~ buildPath( + "..", "c", "Cargo.toml") ~ (isRelease ? " --release" : ""); + + auto status = wait(spawnShell(cargoCmd)); + if (status != 0) + { + throw new Exception("Cargo build failed with status: " ~ status.to!string); + } + else + { + writeln("Cargo build completed successfully"); + } + + // Get opendal.h + copy(buildPath("..", "c", "include", "opendal.h"), buildPath("source", "opendal", "opendal.h")); + + // Get libopendal_c.a + auto libPath = buildPath("..", "c", "target", buildType, staticlib); + writeln("Copying ", libPath, " to ", buildPath("lib", staticlib)); + if (!exists("lib")) + mkdir ("lib"); + copy(libPath, buildPath("lib", staticlib)); +} diff --git a/bindings/d/dscanner.ini b/bindings/d/dscanner.ini new file mode 100644 index 00000000000..48472d548c2 --- /dev/null +++ b/bindings/d/dscanner.ini @@ -0,0 +1,105 @@ +; Configure which static analysis checks are enabled +[analysis.config.StaticAnalysisConfig] +; Check variable, class, struct, interface, union, and function names against +; the Phobos style guide +style_check="disabled" +; Check for array literals that cause unnecessary allocation +enum_array_literal_check="enabled" +; Check for poor exception handling practices +exception_check="enabled" +; Check for use of the deprecated 'delete' keyword +delete_check="enabled" +; Check for use of the deprecated floating point operators +float_operator_check="enabled" +; Check underscores to improve number constant readability +number_style_check="enabled" +; Checks that opEquals, opCmp, toHash, and toString are either const, immutable +; , or inout. +object_const_check="enabled" +; Checks for .. expressions where the left side is larger than the right. +backwards_range_check="enabled" +; Checks for if statements whose 'then' block is the same as the 'else' block +if_else_same_check="enabled" +; Checks for some problems with constructors +constructor_check="enabled" +; Checks for unused function parameters +unused_parameter_check="disabled" +; Checks for unused variables +unused_variable_check="enabled" +; Checks for unused labels +unused_label_check="enabled" +; Checks for duplicate attributes +duplicate_attribute="enabled" +; Checks that opEquals and toHash are both defined or neither are defined +opequals_tohash_check="disabled" +; Checks for subtraction from .length properties +length_subtraction_check="disabled" +; Checks for methods or properties whose names conflict with built-in propertie +; s +builtin_property_names_check="enabled" +; Checks for confusing code in inline asm statements +asm_style_check="enabled" +; Checks for confusing logical operator precedence +logical_precedence_check="disabled" +; Checks for undocumented public declarations +undocumented_declaration_check="disabled" +; Checks for poor placement of function attributes +function_attribute_check="enabled" +; Checks for use of the comma operator +comma_expression_check="enabled" +; Checks for local imports that are too broad +local_import_check="disabled" +; Checks for variables that could be declared immutable +could_be_immutable_check="disabled" +; Checks for redundant expressions in if statements +redundant_if_check="enabled" +; Checks for redundant parenthesis +redundant_parens_check="disabled" +; Checks for mismatched argument and parameter names +mismatched_args_check="disabled" +; Checks for labels with the same name as variables +label_var_same_name_check="disabled" +; Checks for lines longer than 120 characters +long_line_check="disabled" +; Checks for assignment to auto-ref function parameters +auto_ref_assignment_check="disabled" +; Checks for incorrect infinite range definitions +incorrect_infinite_range_check="enabled" +; Checks for asserts that are always true +useless_assert_check="enabled" +; Check for uses of the old-style alias syntax +alias_syntax_check="enabled" +; Checks for else if that should be else static if +static_if_else_check="enabled" +; Check for unclear lambda syntax +lambda_return_check="enabled" +; Check for auto function without return statement +auto_function_check="enabled" +; Check for sortedness of imports +imports_sortedness="disabled" +; Check for explicitly annotated unittests +explicitly_annotated_unittests="disabled" +; Check for properly documented public functions (Returns, Params) +properly_documented_public_functions="disabled" +; Check for useless usage of the final attribute +final_attribute_check="enabled" +; Check for virtual calls in the class constructors +vcall_in_ctor="enabled" +; Check for useless user defined initializers +useless_initializer="disabled" +; Check allman brace style +allman_braces_check="disabled" +; Check for redundant attributes +redundant_attributes_check="disabled" +; Check for public declarations without a documented unittest +has_public_example="disabled" +; Check for asserts without an explanatory message +assert_without_msg="disabled" +; Check indent of if constraints +if_constraints_indent="enabled" +; Check for @trusted applied to a bigger scope than a single function +trust_too_much="disabled" +; Check for redundant storage classes on variable declarations") +redundant_storage_classes="enabled" +; Check for unused function return values +unused_result="disabled" \ No newline at end of file diff --git a/bindings/d/dub.json b/bindings/d/dub.json new file mode 100644 index 00000000000..668fc63f163 --- /dev/null +++ b/bindings/d/dub.json @@ -0,0 +1,37 @@ +{ + "name": "opendal", + "license": "Apache-2.0", + "description": "Apache OpenDAL D bindings", + "excludedSourceFiles": ["source/opendal/package.d"], + "toolchainRequirements": { + "frontend": ">=2.105.3" + }, + "buildTypes": { + "debug": { + "buildOptions": ["debugMode", "debugInfo"] + }, + "release": { + "buildOptions": ["releaseMode", "optimize", "inline"] + } + }, + "preBuildCommands": ["\"$DC\" -run $PACKAGE_DIR/build.d -- release"], + "configurations": [ + { + "name": "opendal", + "targetType": "library", + "targetName": "opendal", + "sourceFiles": ["source/opendal/package.d"], + "targetPath": "lib", + "lflags-posix": ["-L$PACKAGE_DIR/lib"], + "lflags-windows": ["/LIBPATH:$PACKAGE_DIR\\lib"], + "libs": ["opendal_c"] + }, + { + "name": "unittest", + "sourceFiles": ["source/opendal/package.d"], + "lflags-posix": ["-L$PACKAGE_DIR/lib"], + "lflags-windows": ["/LIBPATH:$PACKAGE_DIR\\lib"], + "libs": ["opendal_c"] + } + ] +} \ No newline at end of file diff --git a/bindings/d/source/opendal/opendal_c.c b/bindings/d/source/opendal/opendal_c.c new file mode 100644 index 00000000000..4f0d59e888b --- /dev/null +++ b/bindings/d/source/opendal/opendal_c.c @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/// OpenDAL - ImportC generate D bindings +#pragma attribute(push, nogc, nothrow) +#include "opendal.h" +#pragma attribute(pop) \ No newline at end of file diff --git a/bindings/d/source/opendal/operator.d b/bindings/d/source/opendal/operator.d new file mode 100644 index 00000000000..a3fe616fbf0 --- /dev/null +++ b/bindings/d/source/opendal/operator.d @@ -0,0 +1,229 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +module opendal.operator; + +import std.string: toStringz; +import std.exception: enforce; +import std.conv: to; +import std.parallelism: task, TaskPool; + +/// OpenDAL-C binding for D. (unsafe/@system) +private import opendal.opendal_c; + +struct Operator +{ + private opendal_operator* op; + private TaskPool taskPool; + private bool enabledParallelism; + + this(string scheme, OperatorOptions options, bool useParallel = false) @trusted + { + auto result = opendal_operator_new(scheme.toStringz, options.options); + enforce(result.op !is null, "Failed to create Operator"); + enforce(result.error is null, "Error in Operator"); + op = result.op; + enabledParallelism = useParallel; + + if (enabledParallelism) + taskPool = new TaskPool(); + } + + void write(string path, ubyte[] data) @trusted + { + opendal_bytes bytes = opendal_bytes(data.ptr, data.length, data.length); + auto error = opendal_operator_write(op, path.toStringz, &bytes); + enforce(error is null, "Error writing data"); + } + + void writeParallel(string path, ubyte[] data) @safe + { + auto t = task!((Operator* op, string p, ubyte[] d) { op.write(p, d); })(&this, path, data); + taskPool.put(t); + t.yieldForce(); + } + + ubyte[] readParallel(string path) @trusted + { + auto t = task!((Operator* op, string p) { return op.read(p); })(&this, path); + taskPool.put(t); + return t.yieldForce(); + } + + Entry[] listParallel(string path) @trusted + { + auto t = task!((Operator* op, string p) { return op.list(p); })(&this, path); + taskPool.put(t); + return t.yieldForce(); + } + + ubyte[] read(string path) @trusted + { + auto result = opendal_operator_read(op, path.toStringz); + enforce(result.error is null, "Error reading data"); + scope (exit) + opendal_bytes_free(&result.data); + return result.data.data[0 .. result.data.len].dup; + } + + void remove(string path) @trusted + { + auto error = opendal_operator_delete(op, path.toStringz); + enforce(error is null, "Error deleting object"); + } + + bool exists(string path) @trusted + { + auto result = opendal_operator_exists(op, path.toStringz); + enforce(result.error is null, "Error checking existence"); + return result.exists; + } + + Metadata stat(string path) @trusted + { + auto result = opendal_operator_stat(op, path.toStringz); + enforce(result.error is null, "Error getting metadata"); + return Metadata(result.meta); + } + + Entry[] list(string path) @trusted + { + auto result = opendal_operator_list(op, path.toStringz); + enforce(result.error is null, "Error listing objects"); + + Entry[] entries; + while (true) + { + auto next = opendal_lister_next(result.lister); + if (next.entry is null) + break; + entries ~= Entry(next.entry); + } + return entries; + } + + void createDir(string path) @trusted + { + auto error = opendal_operator_create_dir(op, path.toStringz); + enforce(error is null, "Error creating directory"); + } + + void rename(string src, string dest) @trusted + { + auto error = opendal_operator_rename(op, src.toStringz, dest.toStringz); + enforce(error is null, "Error renaming object"); + } + + void copy(string src, string dest) @trusted + { + auto error = opendal_operator_copy(op, src.toStringz, dest.toStringz); + enforce(error is null, "Error copying object"); + } + + ~this() @trusted + { + if (op !is null) + opendal_operator_free(op); + if (enabledParallelism) + taskPool.stop(); + } +} + +class OperatorOptions +{ + private opendal_operator_options* options; + + this() @trusted + { + options = opendal_operator_options_new(); + } + + void set(string key, string value) @trusted + { + opendal_operator_options_set(options, key.toStringz, value.toStringz); + } + + ~this() @trusted + { + if (options !is null) + opendal_operator_options_free(options); + } +} + +struct Metadata +{ + private opendal_metadata* meta; + + this(scope opendal_metadata* meta) @trusted pure + { + this.meta = meta; + } + + ulong contentLength() @trusted + { + return opendal_metadata_content_length(meta); + } + + bool isFile() @trusted + { + return opendal_metadata_is_file(meta); + } + + bool isDir() @trusted + { + return opendal_metadata_is_dir(meta); + } + + long lastModified() @trusted + { + return opendal_metadata_last_modified_ms(meta); + } + + ~this() @trusted + { + if (meta !is null) + opendal_metadata_free(meta); + } +} + +struct Entry +{ + private opendal_entry* entry; + + this(opendal_entry* entry) pure @nogc + { + this.entry = entry; + } + + string path() + { + return to!string(opendal_entry_path(entry)); + } + + string name() + { + return to!string(opendal_entry_name(entry)); + } + + ~this() + { + if (entry !is null) + opendal_entry_free(entry); + } +} diff --git a/bindings/d/source/opendal/package.d b/bindings/d/source/opendal/package.d new file mode 100644 index 00000000000..523b1ed8c62 --- /dev/null +++ b/bindings/d/source/opendal/package.d @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +module opendal; + +public import opendal.operator; + +version (unittest) +{ + @("Test basic Operator creation") + @safe unittest + { + /* Initialize a operator for "memory" backend, with no options */ + OperatorOptions options = new OperatorOptions(); + Operator op = Operator("memory", options); + + /* Prepare some data to be written */ + string data = "this_string_length_is_24"; + + /* Write this into path "/testpath" */ + op.write("/testpath", cast(ubyte[])data.dup); + + /* We can read it out, make sure the data is the same */ + auto read_bytes = op.read("/testpath"); + assert(read_bytes.length == 24); + assert(cast(string)read_bytes.idup == data); + } + + @("Benchmark parallel and normal functions") + @safe unittest + { + import std.exception: assertNotThrown; + import std.file: tempDir; + import std.path: buildPath; + import std.datetime.stopwatch: StopWatch; + import std.stdio: writeln; + + auto options = new OperatorOptions(); + options.set("root", tempDir); + auto op = Operator("fs", options, true); + + auto testPath = buildPath(tempDir, "benchmark_test.txt"); + auto testData = cast(ubyte[])"Benchmarking OpenDAL async and normal functions".dup; + + // Benchmark write operations + StopWatch sw; + + sw.start(); + assertNotThrown(op.write(testPath, testData)); + sw.stop(); + auto normalWriteTime = sw.peek(); + + sw.reset(); + sw.start(); + assertNotThrown(op.writeParallel(testPath, testData)); + sw.stop(); + auto parallelWriteTime = sw.peek(); + + // Benchmark read operations + sw.reset(); + sw.start(); + auto normalReadData = op.read(testPath); + sw.stop(); + auto normalReadTime = sw.peek(); + + sw.reset(); + sw.start(); + auto parallelReadData = op.readParallel(testPath); + sw.stop(); + auto parallelReadTime = sw.peek(); + + // Benchmark list operations + sw.reset(); + sw.start(); + op.list(tempDir); + sw.stop(); + auto normalListTime = sw.peek(); + + sw.reset(); + sw.start(); + op.listParallel(tempDir); + sw.stop(); + auto parallelListTime = sw.peek(); + + // Print benchmark results + writeln("Write benchmark:"); + writeln(" Normal: ", normalWriteTime); + writeln(" Parallel: ", parallelWriteTime); + + writeln("Read benchmark:"); + writeln(" Normal: ", normalReadTime); + writeln(" Parallel: ", parallelReadTime); + + writeln("List benchmark:"); + writeln(" Normal: ", normalListTime); + writeln(" Parallel: ", parallelListTime); + + // Verify data integrity + assert(normalReadData == testData); + assert(parallelReadData == testData); + + // Clean up + op.remove(testPath); + assert(!op.exists(testPath)); + } + +} diff --git a/bindings/d/test/dub.json b/bindings/d/test/dub.json new file mode 100644 index 00000000000..d21ccb1079c --- /dev/null +++ b/bindings/d/test/dub.json @@ -0,0 +1,14 @@ +{ + "name": "tests", + "targetType": "executable", + "dflags": ["-preview=all"], + "dependencies": {"opendal":{"path": ".."}}, + "buildTypes": { + "debug": { + "buildOptions": ["debugMode", "debugInfo"] + }, + "release": { + "buildOptions": ["releaseMode", "optimize", "inline"] + } + } +} \ No newline at end of file diff --git a/bindings/d/test/source/bdd.d b/bindings/d/test/source/bdd.d new file mode 100644 index 00000000000..2658f40307a --- /dev/null +++ b/bindings/d/test/source/bdd.d @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +module bdd; + +import opendal; +import std.stdio: writeln; + +class OperatorContext +{ + Operator op; + + this() @trusted + { + auto options = new OperatorOptions(); + op = Operator("memory", options); + } +} + +class WriteScenario +{ + OperatorContext context; + string data; + string path; + + this(OperatorContext context) @trusted + { + this.context = context; + } + + WriteScenario givenSomeData(string data) + { + this.data = data; + return this; + } + + WriteScenario whenWritingToPath(string path) + { + this.path = path; + context.op.write(path, cast(ubyte[])data.dup); + return this; + } + + void thenDataShouldBeReadable() + { + auto read_bytes = context.op.read(path); + assert(read_bytes.length == data.length, "Read data length does not match written data length"); + assert(cast(string)read_bytes.idup == data, "Read data does not match written data"); + } +} + +void main() @safe +{ + auto context = new OperatorContext(); + + describe("Operator memory backend", { + it("should write and read data correctly", { + new WriteScenario(context) + .givenSomeData("this_string_length_is_24") + .whenWritingToPath("/testpath") + .thenDataShouldBeReadable(); + }); + + it("should print the read data", { + auto read_bytes = context.op.read("/testpath"); + writeln(cast(string)read_bytes.idup); + }); + }); +} + +void describe(string description, void delegate() tests) @trusted +{ + writeln("Describe: ", description); + tests(); +} + +void it(string description, void delegate() test) +{ + writeln(" It ", description); + test(); + writeln(" - Passed"); +} diff --git a/bindings/java/Cargo.toml b/bindings/java/Cargo.toml index 3e4123caa31..78df098dafb 100644 --- a/bindings/java/Cargo.toml +++ b/bindings/java/Cargo.toml @@ -73,8 +73,7 @@ services-all = [ "services-persy", "services-postgresql", "services-koofr", - # Workaround for https://github.com/apache/opendal/issues/5000 - # "services-mysql", + "services-mysql", "services-redb", "services-redis", # FIXME: rocksdb will lead to "cannot allocate memory in static TLS block" while linking. diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index 52a92b04da5..28f5725767d 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -69,8 +69,7 @@ services-all = [ "services-onedrive", "services-persy", "services-postgresql", - # Workaround for https://github.com/apache/opendal/issues/5000 - # "services-mysql", + "services-mysql", "services-redb", "services-redis", # FIXME how to support rocksdb services in bindings? diff --git a/website/community/release/release.md b/website/community/release/release.md index 4a5ad971f7b..7e113d94bb2 100644 --- a/website/community/release/release.md +++ b/website/community/release/release.md @@ -356,12 +356,10 @@ Checklist for reference: Use our verify.py to assist in the verify process: -```shell -svn co https://dist.apache.org/repos/dist/dev/opendal/${release_version}/ opendal-dev -cd opendal-dev -curl -sSL https://github.com/apache/opendal/raw/v${release_version}/scripts/verify.py -o verify.py -python verify.py -``` + svn co https://dist.apache.org/repos/dist/dev/opendal/${release_version}/ opendal-dev + cd opendal-dev + curl -sSL https://github.com/apache/opendal/raw/v${release_version}/scripts/verify.py -o verify.py + python verify.py Thanks