feat: Bump OpenDAL to 0.50.1 (#16661)

* Save work Signed-off-by: Xuanwo <github@xuanwo.io> * feat: Bump OpenDAL to 0.50.1 Signed-off-by: Xuanwo <github@xuanwo.io> * Fix cargo.lock Signed-off-by: Xuanwo <github@xuanwo.io> * Refactor metrics Signed-off-by: Xuanwo <github@xuanwo.io> * Fix build Signed-off-by: Xuanwo <github@xuanwo.io> * Fix header Signed-off-by: Xuanwo <github@xuanwo.io> * Fix build Signed-off-by: Xuanwo <github@xuanwo.io> * Format toml Signed-off-by: Xuanwo <github@xuanwo.io> * Update cargo lock Signed-off-by: Xuanwo <github@xuanwo.io> * Fix services for opendal Signed-off-by: Xuanwo <github@xuanwo.io> * Fix opendal 0.49 Signed-off-by: Xuanwo <github@xuanwo.io> * Fix opendal compat Signed-off-by: Xuanwo <github@xuanwo.io> * Fix version mismatch Signed-off-by: Xuanwo <github@xuanwo.io> * Fix patch Signed-off-by: Xuanwo <github@xuanwo.io> * Add ping support in redis mock Signed-off-by: Xuanwo <github@xuanwo.io> * Fix hang on auto detect Signed-off-by: Xuanwo <github@xuanwo.io> * Ignore other errors Signed-off-by: Xuanwo <github@xuanwo.io> * Remove unexpected check Signed-off-by: Xuanwo <github@xuanwo.io> * Fix compat Signed-off-by: Xuanwo <github@xuanwo.io> * Format toml Signed-off-by: Xuanwo <github@xuanwo.io> * Use main branch of opendal_compat Signed-off-by: Xuanwo <github@xuanwo.io> * ignore the dir itself Signed-off-by: Xuanwo <github@xuanwo.io> * Fix path handling Signed-off-by: Xuanwo <github@xuanwo.io> * Try make sure location always end with / Signed-off-by: Xuanwo <github@xuanwo.io> * Revert "Try make sure location always end with /" This reverts commit dc7eea8. * Fix hive may return same files Signed-off-by: Xuanwo <github@xuanwo.io> --------- Signed-off-by: Xuanwo <github@xuanwo.io> Co-authored-by: Bohu <overred.shuttler@gmail.com>

Author: Xuanwo

Cargo.lock

@@ -229,7 +229,7 @@ criterion = "0.5"
 ctor = "0.2"
 dashmap = "5.4.0"
 deepsize = { version = "0.2.0" }
-deltalake = "0.18"
+deltalake = "0.20"
 derive-visitor = { version = "0.4.0", features = ["std-types-drive"] }
 derive_more = "0.99.17"
 enumflags2 = { version = "0.7.7", features = ["serde"] }
@@ -256,7 +256,7 @@ mysql_async = { version = "0.34", default-features = false, features = ["native-
 object_store_opendal = "0.46"
 once_cell = "1.15.0"
 openai_api_rust = "0.1"
-opendal = { version = "0.49.0", features = [
+opendal = { version = "0.50.1", features = [
  "layers-fastrace",
  "layers-prometheus-client",
  "layers-async-backtrace",
@@ -275,6 +275,7 @@ opendal = { version = "0.49.0", features = [
  "services-huggingface",
  "services-redis",
 ] }
+opendal_compat = { version = "1" }
 openraft = { git = "https://github.com/drmingdrmer/openraft", tag = "v0.10.0-alpha.6", features = [
  "serde",
  "tracing-log",
@@ -413,14 +414,18 @@ backtrace = { git = "https://github.com/rust-lang/backtrace-rs.git", rev = "7226
  "serialize-serde",
 ] }
 color-eyre = { git = "https://github.com/eyre-rs/eyre.git", rev = "e5d92c3" }
-deltalake = { git = "https://github.com/delta-io/delta-rs", rev = "57795da" }
 ethnum = { git = "https://github.com/datafuse-extras/ethnum-rs", rev = "4cb05f1" }
 jsonb = { git = "https://github.com/databendlabs/jsonb", rev = "ada713c" }
 openai_api_rust = { git = "https://github.com/datafuse-extras/openai-api", rev = "819a0ed" }
+opendal_compat = { git = "https://github.com/apache/opendal", rev = "f6e60f6" }
 orc-rust = { git = "https://github.com/datafuse-extras/datafusion-orc", rev = "03372b97" }
 recursive = { git = "https://github.com/datafuse-extras/recursive.git", rev = "6af35a1" }
 sled = { git = "https://github.com/datafuse-extras/sled", tag = "v0.34.7-datafuse.1" }
 tantivy = { git = "https://github.com/datafuse-extras/tantivy", rev = "7502370" }
 tantivy-common = { git = "https://github.com/datafuse-extras/tantivy", rev = "7502370", package = "tantivy-common" }
 tantivy-jieba = { git = "https://github.com/datafuse-extras/tantivy-jieba", rev = "0e300e9" }
 xorfilter-rs = { git = "https://github.com/datafuse-extras/xorfilter", tag = "databend-alpha.4" }
+
+# This patch makes sure opendal_compat also used the same version of opendal instead of git one.
+[patch.'https://github.com/apache/opendal']
+opendal = { version = "0.50.1" }

Cargo.toml

@@ -50,6 +50,8 @@ prometheus-parse = "0.2.3"
 rand = { workspace = true, features = ["serde1"] }
 regex = { workspace = true }
 replace_with = "0.1.7"
+reqwest = { workspace = true }
+reqwest-hickory-resolver = { workspace = true }
 rustix = "0.38.37"
 semver = { workspace = true }
 serde = { workspace = true }

src/common/base/Cargo.toml

@@ -0,0 +1,88 @@
+// Copyright 2021 Datafuse Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::env;
+use std::sync::Arc;
+use std::sync::LazyLock;
+use std::time::Duration;
+
+use reqwest_hickory_resolver::HickoryResolver;
+
+/// Global shared hickory resolver.
+static GLOBAL_HICKORY_RESOLVER: LazyLock<Arc<HickoryResolver>> =
+ LazyLock::new(|| Arc::new(HickoryResolver::default()));
+
+/// Global shared http client.
+///
+/// Please create your own http client if you want dedicated http connection pool.
+pub static GLOBAL_HTTP_CLIENT: LazyLock<HttpClient> = LazyLock::new(HttpClient::new);
+
+/// HttpClient that used by databend.
+pub struct HttpClient {
+ client: reqwest::Client,
+}
+
+impl Default for HttpClient {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl HttpClient {
+ /// Create a new http client.
+ ///
+ /// # Notes
+ ///
+ /// This client is optimized for interact with storage services.
+ /// Please tune the settings if you want to use it for other purposes.
+ pub fn new() -> Self {
+ let mut builder = reqwest::ClientBuilder::new();
+
+ // Disable http2 for better performance.
+ builder = builder.http1_only();
+
+ // Set dns resolver.
+ builder = builder.dns_resolver(GLOBAL_HICKORY_RESOLVER.clone());
+
+ // Pool max idle per host controls connection pool size.
+ // Default to no limit, set to `0` for disable it.
+ let pool_max_idle_per_host = env::var("_DATABEND_INTERNAL_POOL_MAX_IDLE_PER_HOST")
+ .ok()
+ .and_then(|v| v.parse::<usize>().ok())
+ .unwrap_or(usize::MAX);
+ builder = builder.pool_max_idle_per_host(pool_max_idle_per_host);
+
+ // Connect timeout default to 30s.
+ let connect_timeout = env::var("_DATABEND_INTERNAL_CONNECT_TIMEOUT")
+ .ok()
+ .and_then(|v| v.parse::<u64>().ok())
+ .unwrap_or(30);
+ builder = builder.connect_timeout(Duration::from_secs(connect_timeout));
+
+ // Enable TCP keepalive if set.
+ if let Ok(v) = env::var("_DATABEND_INTERNAL_TCP_KEEPALIVE") {
+ if let Ok(v) = v.parse::<u64>() {
+ builder = builder.tcp_keepalive(Duration::from_secs(v));
+ }
+ }
+
+ let client = builder.build().expect("http client must be created");
+ HttpClient { client }
+ }
+
+ /// Get the inner reqwest client.
+ pub fn inner(&self) -> reqwest::Client {
+ self.client.clone()
+ }
+}

src/common/base/src/http_client.rs

@@ -31,6 +31,7 @@ pub mod containers;
 pub mod display;
 pub mod future;
 pub mod headers;
+pub mod http_client;
 pub mod mem_allocator;
 pub mod rangemap;
 pub mod runtime;

src/common/base/src/lib.rs

@@ -29,8 +29,6 @@ opendal = { workspace = true }
 parquet = { workspace = true }
 prometheus-client = { workspace = true }
 regex = { workspace = true }
-reqwest = { workspace = true }
-reqwest-hickory-resolver = { workspace = true }
 serde = { workspace = true }
 thiserror = { workspace = true }