feat: add cache store for proxy (#870)

Signed-off-by: Gaius <[email protected]>

Cargo.toml

@@ -12,7 +12,7 @@ members = [
 ]
 
 [workspace.package]
-version = "0.1.118"
+version = "0.1.119"
 authors = ["The Dragonfly Developers"]
 homepage = "https://d7y.io/"
 repository = "https://github.com/dragonflyoss/client.git"
@@ -22,13 +22,13 @@ readme = "README.md"
 edition = "2021"
 
 [workspace.dependencies]
-dragonfly-client = { path = "dragonfly-client", version = "0.1.118" }
-dragonfly-client-core = { path = "dragonfly-client-core", version = "0.1.118" }
-dragonfly-client-config = { path = "dragonfly-client-config", version = "0.1.118" }
-dragonfly-client-storage = { path = "dragonfly-client-storage", version = "0.1.118" }
-dragonfly-client-backend = { path = "dragonfly-client-backend", version = "0.1.118" }
-dragonfly-client-util = { path = "dragonfly-client-util", version = "0.1.118" }
-dragonfly-client-init = { path = "dragonfly-client-init", version = "0.1.118" }
+dragonfly-client = { path = "dragonfly-client", version = "0.1.119" }
+dragonfly-client-core = { path = "dragonfly-client-core", version = "0.1.119" }
+dragonfly-client-config = { path = "dragonfly-client-config", version = "0.1.119" }
+dragonfly-client-storage = { path = "dragonfly-client-storage", version = "0.1.119" }
+dragonfly-client-backend = { path = "dragonfly-client-backend", version = "0.1.119" }
+dragonfly-client-util = { path = "dragonfly-client-util", version = "0.1.119" }
+dragonfly-client-init = { path = "dragonfly-client-init", version = "0.1.119" }
 thiserror = "1.0"
 dragonfly-api = "=2.0.173"
 reqwest = { version = "0.12.4", features = ["stream", "native-tls", "default-tls", "rustls-tls"] }
@@ -92,6 +92,7 @@ percent-encoding = "2.3.1"
 tempfile = "3.14.0"
 tokio-rustls = "0.25.0-alpha.4"
 serde_json = "1.0.132"
+lru = "0.12.5"
 
 [profile.release]
 opt-level = "z"

dragonfly-client-config/src/dfdaemon.rs

@@ -212,6 +212,13 @@ pub fn default_proxy_server_port() -> u16 {
     4001
 }
 
+/// default_proxy_cache_capacity is the default cache capacity for the proxy server, default is
+/// 150.
+#[inline]
+pub fn default_proxy_cache_capacity() -> usize {
+    150
+}
+
 /// default_proxy_read_buffer_size is the default buffer size for reading piece, default is 32KB.
 #[inline]
 pub fn default_proxy_read_buffer_size() -> usize {
@@ -1082,6 +1089,13 @@ pub struct Proxy {
     /// prefetch pre-downloads full of the task when download with range request.
     pub prefetch: bool,
 
+    /// cache_capacity is the capacity of the cache by LRU algorithm for HTTP proxy, default is 150.
+    /// The cache is used to store the hot piece content of the task, piece length is 4MB~16MB.
+    /// If the capacity is 150, the cache size is 600MB~2.4GB, need to adjust according to the
+    /// memory size of the host.
+    #[serde(default = "default_proxy_cache_capacity")]
+    pub cache_capacity: usize,
+
     /// read_buffer_size is the buffer size for reading piece from disk, default is 1KB.
     #[serde(default = "default_proxy_read_buffer_size")]
     pub read_buffer_size: usize,
@@ -1096,6 +1110,7 @@ impl Default for Proxy {
             registry_mirror: RegistryMirror::default(),
             disable_back_to_source: false,
             prefetch: false,
+            cache_capacity: default_proxy_cache_capacity(),
             read_buffer_size: default_proxy_read_buffer_size(),
         }
     }

dragonfly-client-core/src/error/mod.rs

@@ -126,6 +126,11 @@ pub enum DFError {
     #[error("invalid parameter")]
     InvalidParameter,
 
+    /// Infallible is the error for infallible.
+    #[error(transparent)]
+    Infallible(#[from] std::convert::Infallible),
+
+    /// Utf8 is the error for utf8.
     #[error(transparent)]
     Utf8(#[from] std::str::Utf8Error),
 

dragonfly-client-storage/src/lib.rs

@@ -441,6 +441,7 @@ impl Storage {
     }
 
     /// piece_id returns the piece id.
+    #[inline]
     #[instrument(skip_all)]
     pub fn piece_id(&self, task_id: &str, number: u32) -> String {
         self.metadata.piece_id(task_id, number)

dragonfly-client-storage/src/metadata.rs

@@ -829,6 +829,7 @@ impl<E: StorageEngineOwned> Metadata<E> {
     }
 
     /// piece_id returns the piece id.
+    #[inline]
     #[instrument(skip_all)]
     pub fn piece_id(&self, task_id: &str, number: u32) -> String {
         format!("{}-{}", task_id, number)

dragonfly-client-util/src/id_generator/mod.rs

@@ -57,6 +57,7 @@ impl IDGenerator {
     }
 
     /// host_id generates the host id.
+    #[inline]
     #[instrument(skip_all)]
     pub fn host_id(&self) -> String {
         if self.is_seed_peer {
@@ -67,6 +68,7 @@ impl IDGenerator {
     }
 
     /// task_id generates the task id.
+    #[inline]
     #[instrument(skip_all)]
     pub fn task_id(
         &self,
@@ -116,6 +118,7 @@ impl IDGenerator {
     }
 
     /// persistent_cache_task_id generates the persistent cache task id.
+    #[inline]
     #[instrument(skip_all)]
     pub fn persistent_cache_task_id(
         &self,
@@ -145,6 +148,7 @@ impl IDGenerator {
     }
 
     /// peer_id generates the peer id.
+    #[inline]
     #[instrument(skip_all)]
     pub fn peer_id(&self) -> String {
         if self.is_seed_peer {

dragonfly-client/Cargo.toml

@@ -63,6 +63,7 @@ uuid.workspace = true
 percent-encoding.workspace = true
 tokio-rustls.workspace = true
 serde_json.workspace = true
+lru.workspace = true
 lazy_static = "1.5"
 tracing-log = "0.2"
 tracing-subscriber = { version = "0.3", features = ["env-filter", "time", "chrono"] }

dragonfly-client/src/metrics/mod.rs

@@ -178,6 +178,21 @@ lazy_static! {
             &[]
         ).expect("metric can be created");
 
+    /// PROXY_REQUEST_VIA_DFDAEMON_COUNT is used to count the number of proxy requset via dfdaemon.
+    pub static ref PROXY_REQUEST_VIA_DFDAEMON_COUNT: IntCounterVec =
+        IntCounterVec::new(
+            Opts::new("proxy_request_by_dfdaemon_total", "Counter of the number of the proxy request by dfdaemon.").namespace(dragonfly_client_config::SERVICE_NAME).subsystem(dragonfly_client_config::NAME),
+            &[]
+        ).expect("metric can be created");
+
+    /// PROXY_REQUEST_VIA_DFDAEMON_AND_CACHE_HITS_COUNT is used to count the number of proxy request via
+    /// dfdaemon and cache hits.
+    pub static ref PROXY_REQUEST_VIA_DFDAEMON_AND_CACHE_HITS_COUNT: IntCounterVec =
+        IntCounterVec::new(
+            Opts::new("proxy_request_via_dfdaemon_and_cache_hits_total", "Counter of the number of cache hits of the proxy request via dfdaemon.").namespace(dragonfly_client_config::SERVICE_NAME).subsystem(dragonfly_client_config::NAME),
+            &[]
+        ).expect("metric can be created");
+
     /// STAT_TASK_COUNT is used to count the number of stat tasks.
     pub static ref STAT_TASK_COUNT: IntCounterVec =
         IntCounterVec::new(
@@ -312,6 +327,16 @@ fn register_custom_metrics() {
         .register(Box::new(PROXY_REQUEST_FAILURE_COUNT.clone()))
         .expect("metric can be registered");
 
+    REGISTRY
+        .register(Box::new(PROXY_REQUEST_VIA_DFDAEMON_COUNT.clone()))
+        .expect("metric can be registered");
+
+    REGISTRY
+        .register(Box::new(
+            PROXY_REQUEST_VIA_DFDAEMON_AND_CACHE_HITS_COUNT.clone(),
+        ))
+        .expect("metric can be registered");
+
     REGISTRY
         .register(Box::new(STAT_TASK_COUNT.clone()))
         .expect("metric can be registered");
@@ -371,6 +396,8 @@ fn reset_custom_metrics() {
     BACKEND_REQUEST_DURATION.reset();
     PROXY_REQUEST_COUNT.reset();
     PROXY_REQUEST_FAILURE_COUNT.reset();
+    PROXY_REQUEST_VIA_DFDAEMON_COUNT.reset();
+    PROXY_REQUEST_VIA_DFDAEMON_AND_CACHE_HITS_COUNT.reset();
     STAT_TASK_COUNT.reset();
     STAT_TASK_FAILURE_COUNT.reset();
     DELETE_TASK_COUNT.reset();
@@ -704,6 +731,21 @@ pub fn collect_proxy_request_failure_metrics() {
     PROXY_REQUEST_FAILURE_COUNT.with_label_values(&[]).inc();
 }
 
+/// collect_proxy_request_via_dfdaemon_metrics collects the proxy request via dfdaemon metrics.
+pub fn collect_proxy_request_via_dfdaemon_metrics() {
+    PROXY_REQUEST_VIA_DFDAEMON_COUNT
+        .with_label_values(&[])
+        .inc();
+}
+
+/// collect_proxy_request_via_dfdaemon_and_cache_hits_metrics collects the proxy request via
+/// dfdaemon and cache hits metrics.
+pub fn collect_proxy_request_via_dfdaemon_and_cache_hits_metrics() {
+    PROXY_REQUEST_VIA_DFDAEMON_AND_CACHE_HITS_COUNT
+        .with_label_values(&[])
+        .inc();
+}
+
 /// collect_stat_task_started_metrics collects the stat task started metrics.
 pub fn collect_stat_task_started_metrics(typ: i32) {
     STAT_TASK_COUNT