This is an automated email from the ASF dual-hosted git repository. xushiyan pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/hudi-rs.git
The following commit(s) were added to refs/heads/main by this push: new 62da0d2 feat: add internal config to skip validation (#51) 62da0d2 is described below commit 62da0d28b717a905f59eab059af1c1def83a1878 Author: Shiyan Xu <2701446+xushi...@users.noreply.github.com> AuthorDate: Sat Jul 6 01:57:14 2024 -0500 feat: add internal config to skip validation (#51) - add internal.rs to host hudi internal configs - add `hoodie.internal.skip.config.validation` to allow skipping --- crates/core/src/config/internal.rs | 62 ++++++++++++++++++++++++++++++++++++++ crates/core/src/config/mod.rs | 1 + crates/core/src/table/mod.rs | 24 +++++++-------- 3 files changed, 74 insertions(+), 13 deletions(-) diff --git a/crates/core/src/config/internal.rs b/crates/core/src/config/internal.rs new file mode 100644 index 0000000..d6ad814 --- /dev/null +++ b/crates/core/src/config/internal.rs @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use std::collections::HashMap; +use std::str::FromStr; + +use anyhow::{anyhow, Result}; +use strum_macros::EnumIter; + +use crate::config::{ConfigParser, HudiConfigValue}; + +#[derive(Clone, Debug, PartialEq, Eq, Hash, EnumIter)] +pub enum HudiInternalConfig { + SkipConfigValidation, +} + +impl AsRef<str> for HudiInternalConfig { + fn as_ref(&self) -> &str { + match self { + Self::SkipConfigValidation => "hoodie.internal.skip.config.validation", + } + } +} + +impl ConfigParser for HudiInternalConfig { + type Output = HudiConfigValue; + + fn default_value(&self) -> Option<HudiConfigValue> { + match self { + Self::SkipConfigValidation => Some(HudiConfigValue::Boolean(false)), + } + } + + fn parse_value(&self, configs: &HashMap<String, String>) -> Result<Self::Output> { + let get_result = configs + .get(self.as_ref()) + .map(|v| v.as_str()) + .ok_or(anyhow!("Config '{}' not found", self.as_ref())); + + match self { + Self::SkipConfigValidation => get_result + .and_then(|v| bool::from_str(v).map_err(|e| anyhow!(e))) + .map(HudiConfigValue::Boolean), + } + } +} diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rs index 2399f93..f8975cf 100644 --- a/crates/core/src/config/mod.rs +++ b/crates/core/src/config/mod.rs @@ -22,6 +22,7 @@ use std::sync::Arc; use anyhow::Result; +pub mod internal; pub mod read; pub mod table; diff --git a/crates/core/src/table/mod.rs b/crates/core/src/table/mod.rs index 1493394..db50525 100644 --- a/crates/core/src/table/mod.rs +++ b/crates/core/src/table/mod.rs @@ -28,9 +28,11 @@ use arrow_schema::Schema; use strum::IntoEnumIterator; use url::Url; +use HudiInternalConfig::SkipConfigValidation; use HudiTableConfig::{DropsPartitionFields, TableType, TableVersion}; use TableTypeValue::CopyOnWrite; +use crate::config::internal::HudiInternalConfig; use crate::config::read::HudiReadConfig; use crate::config::table::{HudiTableConfig, TableTypeValue}; use crate::config::HudiConfigs; @@ -111,17 +113,13 @@ impl Table { } let hudi_configs = HudiConfigs::new(hudi_options); - Self::validate_configs(&hudi_configs, &extra_options).map(|_| (hudi_configs, extra_options)) + Self::validate_configs(&hudi_configs).map(|_| (hudi_configs, extra_options)) } - fn validate_configs( - hudi_configs: &HudiConfigs, - extra_options: &HashMap<String, String>, - ) -> Result<()> { - if extra_options - .get("hoodie_internal.skip.config.validation") - .and_then(|v| bool::from_str(v).ok()) - .unwrap_or(false) + fn validate_configs(hudi_configs: &HudiConfigs) -> Result<()> { + if hudi_configs + .get_or_default(SkipConfigValidation) + .to::<bool>() { return Ok(()); } @@ -389,7 +387,7 @@ mod tests { let table = Table::new( base_url.as_str(), HashMap::from_iter(vec![( - "hoodie_internal.skip.config.validation".to_string(), + "hoodie.internal.skip.config.validation".to_string(), "true".to_string(), )]), ) @@ -448,7 +446,7 @@ mod tests { let table = Table::new( base_url.as_str(), HashMap::from_iter(vec![( - "hoodie_internal.skip.config.validation".to_string(), + "hoodie.internal.skip.config.validation".to_string(), "true".to_string(), )]), ) @@ -480,7 +478,7 @@ mod tests { let table = Table::new( base_url.as_str(), HashMap::from_iter(vec![( - "hoodie_internal.skip.config.validation".to_string(), + "hoodie.internal.skip.config.validation".to_string(), "true".to_string(), )]), ) @@ -518,7 +516,7 @@ mod tests { let table = Table::new( base_url.as_str(), HashMap::from_iter(vec![( - "hoodie_internal.skip.config.validation".to_string(), + "hoodie.internal.skip.config.validation".to_string(), "true".to_string(), )]), )