Skip to content

Commit 10bab19

Browse files
gaius-qigaius-qi
authored
fix: rocksdb panic caused by out of bounds (#860)
Signed-off-by: Gaius <[email protected]>
1 parent 88b1f7e commit 10bab19

File tree

13 files changed

+235
-223
lines changed

13 files changed

+235
-223
lines changed

‎Cargo.lock

Lines changed: 10 additions & 10 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ members = [
1212
]
1313

1414
[workspace.package]
15-
version = "0.1.117"
15+
version = "0.1.118"
1616
authors = ["The Dragonfly Developers"]
1717
homepage = "https://d7y.io/"
1818
repository = "https://github.com/dragonflyoss/client.git"
@@ -22,15 +22,15 @@ readme = "README.md"
2222
edition = "2021"
2323

2424
[workspace.dependencies]
25-
dragonfly-client = { path = "dragonfly-client", version = "0.1.117" }
26-
dragonfly-client-core = { path = "dragonfly-client-core", version = "0.1.117" }
27-
dragonfly-client-config = { path = "dragonfly-client-config", version = "0.1.117" }
28-
dragonfly-client-storage = { path = "dragonfly-client-storage", version = "0.1.117" }
29-
dragonfly-client-backend = { path = "dragonfly-client-backend", version = "0.1.117" }
30-
dragonfly-client-util = { path = "dragonfly-client-util", version = "0.1.117" }
31-
dragonfly-client-init = { path = "dragonfly-client-init", version = "0.1.117" }
25+
dragonfly-client = { path = "dragonfly-client", version = "0.1.118" }
26+
dragonfly-client-core = { path = "dragonfly-client-core", version = "0.1.118" }
27+
dragonfly-client-config = { path = "dragonfly-client-config", version = "0.1.118" }
28+
dragonfly-client-storage = { path = "dragonfly-client-storage", version = "0.1.118" }
29+
dragonfly-client-backend = { path = "dragonfly-client-backend", version = "0.1.118" }
30+
dragonfly-client-util = { path = "dragonfly-client-util", version = "0.1.118" }
31+
dragonfly-client-init = { path = "dragonfly-client-init", version = "0.1.118" }
3232
thiserror = "1.0"
33-
dragonfly-api = "=2.0.171"
33+
dragonfly-api = "=2.0.173"
3434
reqwest = { version = "0.12.4", features = ["stream", "native-tls", "default-tls", "rustls-tls"] }
3535
rcgen = { version = "0.12.1", features = ["x509-parser"] }
3636
hyper = { version = "1.5", features = ["full"] }

‎dragonfly-client-config/src/dfdaemon.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -805,11 +805,11 @@ pub struct Storage {
805805
#[serde(default = "default_storage_keep")]
806806
pub keep: bool,
807807

808-
/// write_buffer_size is the buffer size for writing piece to disk, default is 4KB.
808+
/// write_buffer_size is the buffer size for writing piece to disk, default is 128KB.
809809
#[serde(default = "default_storage_write_buffer_size")]
810810
pub write_buffer_size: usize,
811811

812-
/// read_buffer_size is the buffer size for reading piece from disk, default is 4KB.
812+
/// read_buffer_size is the buffer size for reading piece from disk, default is 128KB.
813813
#[serde(default = "default_storage_read_buffer_size")]
814814
pub read_buffer_size: usize,
815815
}

‎dragonfly-client-storage/src/content.rs

Lines changed: 19 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ use std::cmp::{max, min};
2121
use std::path::{Path, PathBuf};
2222
use std::sync::Arc;
2323
use tokio::fs::{self, File, OpenOptions};
24-
use tokio::io::{self, AsyncRead, AsyncReadExt, AsyncSeekExt, BufReader, SeekFrom};
24+
use tokio::io::{self, AsyncRead, AsyncReadExt, AsyncSeekExt, BufReader, BufWriter, SeekFrom};
2525
use tokio_util::io::InspectReader;
2626
use tracing::{error, info, instrument, warn};
2727

@@ -249,35 +249,29 @@ impl Content {
249249
range: Option<Range>,
250250
) -> Result<impl AsyncRead> {
251251
let task_path = self.get_task_path(task_id);
252-
if let Some(range) = range {
252+
let mut f = File::open(task_path.as_path()).await.map_err(|err| {
253+
error!("open {:?} failed: {}", task_path, err);
254+
err
255+
})?;
256+
257+
// Calculate the target offset and length based on the range.
258+
let (target_offset, target_length) = if let Some(range) = range {
253259
let target_offset = max(offset, range.start);
254260
let target_length =
255261
min(offset + length - 1, range.start + range.length - 1) - target_offset + 1;
262+
(target_offset, target_length)
263+
} else {
264+
(offset, length)
265+
};
256266

257-
let mut f = File::open(task_path.as_path()).await.map_err(|err| {
258-
error!("open {:?} failed: {}", task_path, err);
267+
f.seek(SeekFrom::Start(target_offset))
268+
.await
269+
.map_err(|err| {
270+
error!("seek {:?} failed: {}", task_path, err);
259271
err
260272
})?;
261273

262-
f.seek(SeekFrom::Start(target_offset))
263-
.await
264-
.map_err(|err| {
265-
error!("seek {:?} failed: {}", task_path, err);
266-
err
267-
})?;
268-
return Ok(f.take(target_length));
269-
}
270-
271-
let mut f = File::open(task_path.as_path()).await.map_err(|err| {
272-
error!("open {:?} failed: {}", task_path, err);
273-
err
274-
})?;
275-
276-
f.seek(SeekFrom::Start(offset)).await.map_err(|err| {
277-
error!("seek {:?} failed: {}", task_path, err);
278-
err
279-
})?;
280-
Ok(f.take(length))
274+
Ok(f.take(target_length))
281275
}
282276

283277
/// write_piece writes the piece to the content.
@@ -318,7 +312,8 @@ impl Content {
318312
})?;
319313

320314
// Copy the piece to the file.
321-
let length = io::copy(&mut tee, &mut f).await.map_err(|err| {
315+
let mut writer = BufWriter::with_capacity(self.config.storage.write_buffer_size, f);
316+
let length = io::copy(&mut tee, &mut writer).await.map_err(|err| {
322317
error!("copy {:?} failed: {}", task_path, err);
323318
err
324319
})?;

‎dragonfly-client-storage/src/lib.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ use std::path::PathBuf;
2424
use std::sync::Arc;
2525
use std::time::Duration;
2626
use tokio::io::AsyncRead;
27-
use tracing::{debug, error, info, instrument};
27+
use tracing::{debug, error, instrument};
2828

2929
pub mod content;
3030
pub mod metadata;
@@ -449,7 +449,7 @@ impl Storage {
449449

450450
// If the piece is finished, return.
451451
if piece.is_finished() {
452-
info!("wait piece finished success");
452+
debug!("wait piece finished success");
453453
return Ok(piece);
454454
}
455455

‎dragonfly-client-storage/src/metadata.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ pub struct Task {
4646
pub response_header: HashMap<String, String>,
4747

4848
/// uploading_count is the count of the task being uploaded by other peers.
49-
pub uploading_count: u64,
49+
pub uploading_count: i64,
5050

5151
/// uploaded_count is the count of the task has been uploaded by other peers.
5252
pub uploaded_count: u64,
@@ -244,7 +244,7 @@ pub struct Piece {
244244
pub parent_id: Option<String>,
245245

246246
/// uploading_count is the count of the piece being uploaded by other peers.
247-
pub uploading_count: u64,
247+
pub uploading_count: i64,
248248

249249
/// uploaded_count is the count of the piece has been uploaded by other peers.
250250
pub uploaded_count: u64,

‎dragonfly-client/src/grpc/dfdaemon_download.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -354,7 +354,7 @@ impl DfdaemonDownload for DfdaemonDownloadServerHandler {
354354
let download_clone = download.clone();
355355
let task_manager_clone = task_manager.clone();
356356
let task_clone = task.clone();
357-
let (out_stream_tx, out_stream_rx) = mpsc::channel(1024 * 10);
357+
let (out_stream_tx, out_stream_rx) = mpsc::channel(10 * 1024);
358358
tokio::spawn(
359359
async move {
360360
match task_manager_clone
@@ -743,7 +743,7 @@ impl DfdaemonDownload for DfdaemonDownloadServerHandler {
743743
let request_clone = request.clone();
744744
let task_manager_clone = task_manager.clone();
745745
let task_clone = task.clone();
746-
let (out_stream_tx, out_stream_rx) = mpsc::channel(1024 * 10);
746+
let (out_stream_tx, out_stream_rx) = mpsc::channel(10 * 1024);
747747
tokio::spawn(
748748
async move {
749749
match task_manager_clone

‎dragonfly-client/src/grpc/dfdaemon_upload.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -347,7 +347,7 @@ impl DfdaemonUpload for DfdaemonUploadServerHandler {
347347
let download_clone = download.clone();
348348
let task_manager_clone = task_manager.clone();
349349
let task_clone = task.clone();
350-
let (out_stream_tx, out_stream_rx) = mpsc::channel(1024 * 10);
350+
let (out_stream_tx, out_stream_rx) = mpsc::channel(10 * 1024);
351351
tokio::spawn(
352352
async move {
353353
match task_manager_clone
@@ -644,7 +644,7 @@ impl DfdaemonUpload for DfdaemonUploadServerHandler {
644644
let task_manager = self.task.clone();
645645

646646
// Initialize stream channel.
647-
let (out_stream_tx, out_stream_rx) = mpsc::channel(1024 * 10);
647+
let (out_stream_tx, out_stream_rx) = mpsc::channel(10 * 1024);
648648
tokio::spawn(
649649
async move {
650650
loop {
@@ -940,7 +940,7 @@ impl DfdaemonUpload for DfdaemonUploadServerHandler {
940940
let request_clone = request.clone();
941941
let task_manager_clone = task_manager.clone();
942942
let task_clone = task.clone();
943-
let (out_stream_tx, out_stream_rx) = mpsc::channel(1024 * 10);
943+
let (out_stream_tx, out_stream_rx) = mpsc::channel(10 * 1024);
944944
tokio::spawn(
945945
async move {
946946
match task_manager_clone

‎dragonfly-client/src/grpc/scheduler.rs

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -527,19 +527,21 @@ impl SchedulerClient {
527527
#[instrument(skip(self))]
528528
async fn update_available_scheduler_addrs(&self) -> Result<()> {
529529
// Get the endpoints of available schedulers.
530-
let data = self.dynconfig.data.read().await;
531-
let data_available_schedulers_clone = data.available_schedulers.clone();
532-
drop(data);
530+
let data_available_schedulers_clone = {
531+
let data = self.dynconfig.data.read().await;
532+
data.available_schedulers.clone()
533+
};
533534

534535
// Check if the available schedulers is empty.
535536
if data_available_schedulers_clone.is_empty() {
536537
return Err(Error::AvailableSchedulersNotFound);
537538
}
538539

539540
// Get the available schedulers.
540-
let available_schedulers = self.available_schedulers.read().await;
541-
let available_schedulers_clone = available_schedulers.clone();
542-
drop(available_schedulers);
541+
let available_schedulers_clone = {
542+
let available_schedulers = self.available_schedulers.read().await;
543+
available_schedulers.clone()
544+
};
543545

544546
// Check if the available schedulers is not changed.
545547
if data_available_schedulers_clone.len() == available_schedulers_clone.len()
@@ -574,13 +576,11 @@ impl SchedulerClient {
574576
new_available_schedulers.push(available_scheduler.clone());
575577

576578
// Add the scheduler address to the addresses of available schedulers.
577-
new_available_scheduler_addrs
578-
.push(SocketAddr::new(ip, available_scheduler.port as u16));
579+
let socket_addr = SocketAddr::new(ip, available_scheduler.port as u16);
580+
new_available_scheduler_addrs.push(socket_addr);
579581

580582
// Add the scheduler to the hashring.
581-
new_hashring.add(VNode {
582-
addr: SocketAddr::new(ip, available_scheduler.port as u16),
583-
});
583+
new_hashring.add(VNode { addr: socket_addr });
584584
}
585585

586586
// Update the available schedulers.

0 commit comments

Comments
 (0)