This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 02f0553eb85 HIVE-29066: PARTITION_NAME_WHITELIST_PATTERN is not
honoring session level configuration (#5943)
02f0553eb85 is described below
commit 02f0553eb85deb55e94fd881fc2faec917326656
Author: Raghav Aggarwal <[email protected]>
AuthorDate: Sat Aug 2 13:51:19 2025 +0530
HIVE-29066: PARTITION_NAME_WHITELIST_PATTERN is not honoring session level
configuration (#5943)
---
.../apache/hive/common/util/HiveStringUtils.java | 25 ++++---
.../hadoop/hive/ql/exec/FileSinkOperator.java | 35 ++++++----
.../hadoop/hive/ql/plan/DynamicPartitionCtx.java | 4 +-
.../dynamic_partitions_with_whitelist.q | 2 +-
.../session_partition_with_whitelist.q | 9 +++
.../add_partition_with_whitelist.q.out | 2 +-
.../alter_partition_with_whitelist.q.out | 2 +-
.../dynamic_partitions_with_whitelist.q.out | 8 +--
.../session_partition_with_whitelist.q.out | 20 ++++++
.../apache/hadoop/hive/metastore/HMSHandler.java | 38 +++++------
.../apache/hadoop/hive/metastore/ObjectStore.java | 14 +---
.../hive/metastore/utils/MetaStoreServerUtils.java | 77 ++++++++++++----------
12 files changed, 130 insertions(+), 106 deletions(-)
diff --git a/common/src/java/org/apache/hive/common/util/HiveStringUtils.java
b/common/src/java/org/apache/hive/common/util/HiveStringUtils.java
index d703f9f41f7..1b4b34f1a82 100644
--- a/common/src/java/org/apache/hive/common/util/HiveStringUtils.java
+++ b/common/src/java/org/apache/hive/common/util/HiveStringUtils.java
@@ -18,6 +18,7 @@
package org.apache.hive.common.util;
+import com.google.common.base.Splitter;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.net.InetAddress;
@@ -38,8 +39,6 @@
import java.util.Map;
import java.util.StringTokenizer;
import java.util.regex.Pattern;
-
-import com.google.common.base.Splitter;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.text.translate.CharSequenceTranslator;
import org.apache.commons.lang3.text.translate.EntityArrays;
@@ -1080,19 +1079,17 @@ public static boolean commentProvided(char[] chars) {
return false;
}
- public static String getPartitionValWithInvalidCharacter(List<String>
partVals,
- Pattern partitionValidationPattern) {
- if (partitionValidationPattern == null) {
- return null;
+ public static String getPartitionValWithInvalidCharacter(
+ List<String> partVals, Pattern partitionValidationPattern) {
+ String result = null;
+ if (partitionValidationPattern != null) {
+ result =
+ partVals.stream()
+ .filter(partVal ->
!partitionValidationPattern.matcher(partVal).matches())
+ .findFirst()
+ .orElse(null);
}
-
- for (String partVal : partVals) {
- if (!partitionValidationPattern.matcher(partVal).matches()) {
- return partVal;
- }
- }
-
- return null;
+ return result;
}
/**
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
index cfa2608a93c..6a57672207f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
@@ -24,6 +24,7 @@
import static
org.apache.hadoop.hive.ql.security.authorization.HiveCustomStorageHandlerUtils.setWriteOperation;
import static
org.apache.hadoop.hive.ql.security.authorization.HiveCustomStorageHandlerUtils.setWriteOperationIsSorted;
+import com.google.common.collect.Lists;
import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
@@ -38,20 +39,17 @@
import java.util.Properties;
import java.util.Set;
import java.util.function.BiFunction;
-
-import com.google.common.collect.Lists;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-
import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConfUtil;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.conf.HiveConfUtil;
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.ql.CompilationOpContext;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.exec.Utilities.MissingBucketsContext;
@@ -81,7 +79,11 @@
import org.apache.hadoop.hive.ql.plan.api.OperatorType;
import org.apache.hadoop.hive.ql.stats.StatsCollectionContext;
import org.apache.hadoop.hive.ql.stats.StatsPublisher;
-import org.apache.hadoop.hive.serde2.*;
+import org.apache.hadoop.hive.serde2.AbstractSerDe;
+import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.SerDeStats;
+import org.apache.hadoop.hive.serde2.Serializer;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import
org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
@@ -92,13 +94,11 @@
import org.apache.hadoop.hive.shims.HadoopShims.StoragePolicyShim;
import org.apache.hadoop.hive.shims.HadoopShims.StoragePolicyValue;
import org.apache.hadoop.hive.shims.ShimLoader;
-
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.util.ReflectionUtils;
-
import org.apache.hive.common.util.HiveStringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -1125,12 +1125,19 @@ public void process(Object row, int tag) throws
HiveException {
}
}
- String invalidPartitionVal;
- if((invalidPartitionVal =
HiveStringUtils.getPartitionValWithInvalidCharacter(dpVals,
dpCtx.getWhiteListPattern()))!=null) {
- throw new HiveFatalException("Partition value '" +
invalidPartitionVal +
- "' contains a character not matched by whitelist pattern '" +
- dpCtx.getWhiteListPattern().toString() + "'. " + "(configure
with " +
-
HiveConf.ConfVars.METASTORE_PARTITION_NAME_WHITELIST_PATTERN.varname + ")");
+ String invalidPartitionVal =
+ HiveStringUtils.getPartitionValWithInvalidCharacter(
+ dpVals, dpCtx.getWhiteListPattern());
+
+ if (invalidPartitionVal != null) {
+ String errorMsg =
+ ("Partition value '%s' contains a character not matched by
whitelist pattern '%s'. Configure with %s")
+ .formatted(
+ invalidPartitionVal,
+ dpCtx.getWhiteListPattern().toString(),
+
MetastoreConf.ConfVars.PARTITION_NAME_WHITELIST_PATTERN.getVarname());
+
+ throw new HiveFatalException(errorMsg);
}
fpaths = getDynOutPaths(dpVals, lbDirName);
dynamicPartitionSpecs.add(fpaths.dpDirForCounters);
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPartitionCtx.java
b/ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPartitionCtx.java
index c90f6dfb491..a83f91f2f42 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPartitionCtx.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPartitionCtx.java
@@ -91,7 +91,7 @@ public DynamicPartitionCtx(List<String> partColNames, String
defaultPartName,
this.spPath = null;
String confVal;
try {
- confVal =
Hive.get().getMetaConf(ConfVars.PARTITION_NAME_WHITELIST_PATTERN.getHiveName());
+ confVal =
Hive.get().getMetaConf(ConfVars.PARTITION_NAME_WHITELIST_PATTERN.getVarname());
} catch (HiveException e) {
throw new SemanticException(e);
}
@@ -126,7 +126,7 @@ public DynamicPartitionCtx(Map<String, String> partSpec,
String defaultPartName,
}
String confVal;
try {
- confVal =
Hive.get().getMetaConf(ConfVars.PARTITION_NAME_WHITELIST_PATTERN.getHiveName());
+ confVal =
Hive.get().getMetaConf(ConfVars.PARTITION_NAME_WHITELIST_PATTERN.getVarname());
} catch (HiveException e) {
throw new SemanticException(e);
}
diff --git
a/ql/src/test/queries/clientnegative/dynamic_partitions_with_whitelist.q
b/ql/src/test/queries/clientnegative/dynamic_partitions_with_whitelist.q
index 8b126c6d71a..4d324f5c77b 100644
--- a/ql/src/test/queries/clientnegative/dynamic_partitions_with_whitelist.q
+++ b/ql/src/test/queries/clientnegative/dynamic_partitions_with_whitelist.q
@@ -2,7 +2,6 @@
set hive.strict.checks.bucketing=false;
set hive.mapred.mode=nonstrict;
-SET hive.metastore.partition.name.whitelist.pattern=[^9]*;
set
hive.exec.failure.hooks=org.apache.hadoop.hive.ql.hooks.VerifyTableDirectoryIsEmptyHook;
set hive.exec.dynamic.partition=true;
@@ -17,4 +16,5 @@ load data local inpath '../../data/files/bmj/000000_0' INTO
TABLE source_table p
-- If the directory is not empty the hook will throw an error, instead the
error should come from the metastore
-- This shows that no dynamic partitions were created and left behind or had
directories created
+SET metaconf:metastore.partition.name.whitelist.pattern=[^9]*;
insert overwrite table dest_table partition (ds, hr) select key, hr, ds, value
from source_table where ds='2008-04-08' and value='val_129' order by value asc;
diff --git
a/ql/src/test/queries/clientnegative/session_partition_with_whitelist.q
b/ql/src/test/queries/clientnegative/session_partition_with_whitelist.q
new file mode 100644
index 00000000000..fd6073b0603
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/session_partition_with_whitelist.q
@@ -0,0 +1,9 @@
+set metaconf:metastore.partition.name.whitelist.pattern;
+
+create table t1 (id int) partitioned by (pcol string);
+alter table t1 add partition (pCol='2025-06-09');
+
+set metaconf:metastore.partition.name.whitelist.pattern=[^9]*;
+alter table t1 add partition (pCol='2025-06-19');
+show partitions t1;
+
diff --git
a/ql/src/test/results/clientnegative/add_partition_with_whitelist.q.out
b/ql/src/test/results/clientnegative/add_partition_with_whitelist.q.out
index 6aa789b6f72..dfacd50fcd2 100644
--- a/ql/src/test/results/clientnegative/add_partition_with_whitelist.q.out
+++ b/ql/src/test/results/clientnegative/add_partition_with_whitelist.q.out
@@ -15,4 +15,4 @@ POSTHOOK: Input: default@part_whitelist_test
PREHOOK: query: ALTER TABLE part_whitelist_test ADD PARTITION (ds='1,2,3,4')
PREHOOK: type: ALTERTABLE_ADDPARTS
PREHOOK: Output: default@part_whitelist_test
-FAILED: Execution Error, return code 40000 from
org.apache.hadoop.hive.ql.ddl.DDLTask. MetaException(message:Partition value
'1,2,3,4' contains a character not matched by whitelist pattern
'[\\x20-\\x7E&&[^,]]*'. (configure with
metastore.partition.name.whitelist.pattern))
+FAILED: Execution Error, return code 40000 from
org.apache.hadoop.hive.ql.ddl.DDLTask. MetaException(message:Partition value
'1,2,3,4' contains a character not matched by whitelist pattern
'[\\x20-\\x7E&&[^,]]*'. Configure with
metastore.partition.name.whitelist.pattern)
diff --git
a/ql/src/test/results/clientnegative/alter_partition_with_whitelist.q.out
b/ql/src/test/results/clientnegative/alter_partition_with_whitelist.q.out
index 8a7857ee821..0ae9cf8e01e 100644
--- a/ql/src/test/results/clientnegative/alter_partition_with_whitelist.q.out
+++ b/ql/src/test/results/clientnegative/alter_partition_with_whitelist.q.out
@@ -23,4 +23,4 @@ PREHOOK: query: ALTER TABLE part_whitelist_test PARTITION
(ds='1') rename to par
PREHOOK: type: ALTERTABLE_RENAMEPART
PREHOOK: Input: default@part_whitelist_test
PREHOOK: Output: default@part_whitelist_test@ds=1
-FAILED: Execution Error, return code 40000 from
org.apache.hadoop.hive.ql.ddl.DDLTask. Unable to rename partition. Partition
value '1,2,3' contains a character not matched by whitelist pattern
'[\\x20-\\x7E&&[^,]]*'. (configure with
metastore.partition.name.whitelist.pattern)
+FAILED: Execution Error, return code 40000 from
org.apache.hadoop.hive.ql.ddl.DDLTask. Unable to rename partition. Partition
value '1,2,3' contains a character not matched by whitelist pattern
'[\\x20-\\x7E&&[^,]]*'. Configure with
metastore.partition.name.whitelist.pattern
diff --git
a/ql/src/test/results/clientnegative/dynamic_partitions_with_whitelist.q.out
b/ql/src/test/results/clientnegative/dynamic_partitions_with_whitelist.q.out
index a9ca39db8c7..e2d004e4e2a 100644
--- a/ql/src/test/results/clientnegative/dynamic_partitions_with_whitelist.q.out
+++ b/ql/src/test/results/clientnegative/dynamic_partitions_with_whitelist.q.out
@@ -33,13 +33,13 @@ Vertex failed, vertexName=Map 1, vertexId=vertex_#ID#,
diagnostics=[Task failed,
#### A masked pattern was here ####
Caused by: java.lang.RuntimeException: Hive Runtime Error while closing
operators
#### A masked pattern was here ####
-Caused by: org.apache.hadoop.hive.ql.metadata.HiveFatalException: Partition
value 'val_129' contains a character not matched by whitelist pattern '[^9]*'.
(configure with hive.metastore.partition.name.whitelist.pattern)
+Caused by: org.apache.hadoop.hive.ql.metadata.HiveFatalException: Partition
value 'val_129' contains a character not matched by whitelist pattern '[^9]*'.
Configure with metastore.partition.name.whitelist.pattern
#### A masked pattern was here ####
], TaskAttempt 1 failed, info=[Error: Error while running task ( failure ) :
attempt_#ID#:java.lang.RuntimeException: java.lang.RuntimeException: Hive
Runtime Error while closing operators
#### A masked pattern was here ####
Caused by: java.lang.RuntimeException: Hive Runtime Error while closing
operators
#### A masked pattern was here ####
-Caused by: org.apache.hadoop.hive.ql.metadata.HiveFatalException: Partition
value 'val_129' contains a character not matched by whitelist pattern '[^9]*'.
(configure with hive.metastore.partition.name.whitelist.pattern)
+Caused by: org.apache.hadoop.hive.ql.metadata.HiveFatalException: Partition
value 'val_129' contains a character not matched by whitelist pattern '[^9]*'.
Configure with metastore.partition.name.whitelist.pattern
#### A masked pattern was here ####
]], Vertex did not succeed due to OWN_TASK_FAILURE, failedTasks:1
killedTasks:0, Vertex vertex_#ID# [Map 1] killed/failed due to:OWN_TASK_FAILURE]
[Masked Vertex killed due to OTHER_VERTEX_FAILURE]
@@ -48,12 +48,12 @@ FAILED: Execution Error, return code 2 from
org.apache.hadoop.hive.ql.exec.tez.T
#### A masked pattern was here ####
Caused by: java.lang.RuntimeException: Hive Runtime Error while closing
operators
#### A masked pattern was here ####
-Caused by: org.apache.hadoop.hive.ql.metadata.HiveFatalException: Partition
value 'val_129' contains a character not matched by whitelist pattern '[^9]*'.
(configure with hive.metastore.partition.name.whitelist.pattern)
+Caused by: org.apache.hadoop.hive.ql.metadata.HiveFatalException: Partition
value 'val_129' contains a character not matched by whitelist pattern '[^9]*'.
Configure with metastore.partition.name.whitelist.pattern
#### A masked pattern was here ####
], TaskAttempt 1 failed, info=[Error: Error while running task ( failure ) :
attempt_#ID#:java.lang.RuntimeException: java.lang.RuntimeException: Hive
Runtime Error while closing operators
#### A masked pattern was here ####
Caused by: java.lang.RuntimeException: Hive Runtime Error while closing
operators
#### A masked pattern was here ####
-Caused by: org.apache.hadoop.hive.ql.metadata.HiveFatalException: Partition
value 'val_129' contains a character not matched by whitelist pattern '[^9]*'.
(configure with hive.metastore.partition.name.whitelist.pattern)
+Caused by: org.apache.hadoop.hive.ql.metadata.HiveFatalException: Partition
value 'val_129' contains a character not matched by whitelist pattern '[^9]*'.
Configure with metastore.partition.name.whitelist.pattern
#### A masked pattern was here ####
]], Vertex did not succeed due to OWN_TASK_FAILURE, failedTasks:1
killedTasks:0, Vertex vertex_#ID# [Map 1] killed/failed due
to:OWN_TASK_FAILURE][Masked Vertex killed due to OTHER_VERTEX_FAILURE]DAG did
not succeed due to VERTEX_FAILURE. failedVertices:1 killedVertices:1
diff --git
a/ql/src/test/results/clientnegative/session_partition_with_whitelist.q.out
b/ql/src/test/results/clientnegative/session_partition_with_whitelist.q.out
new file mode 100644
index 00000000000..197600e99fd
--- /dev/null
+++ b/ql/src/test/results/clientnegative/session_partition_with_whitelist.q.out
@@ -0,0 +1,20 @@
+metaconf:metastore.partition.name.whitelist.pattern=
+PREHOOK: query: create table t1 (id int) partitioned by (pcol string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t1
+POSTHOOK: query: create table t1 (id int) partitioned by (pcol string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t1
+PREHOOK: query: alter table t1 add partition (pCol='2025-06-09')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@t1
+POSTHOOK: query: alter table t1 add partition (pCol='2025-06-09')
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@t1
+POSTHOOK: Output: default@t1@pcol=2025-06-09
+PREHOOK: query: alter table t1 add partition (pCol='2025-06-19')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@t1
+FAILED: Execution Error, return code 40000 from
org.apache.hadoop.hive.ql.ddl.DDLTask. MetaException(message:Partition value
'2025-06-19' contains a character not matched by whitelist pattern '[^9]*'.
Configure with metastore.partition.name.whitelist.pattern)
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
index e4bcb21d6a4..086766de41b 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
@@ -296,7 +296,6 @@ public Configuration getHiveConf() {
private MetaStoreFilterHook filterHook;
private boolean isServerFilterEnabled = false;
- private Pattern partitionValidationPattern;
private final boolean isInTest;
@Override
@@ -371,11 +370,7 @@ public void init(Warehouse wh) throws MetaException {
endFunctionListeners = MetaStoreServerUtils.getMetaStoreListeners(
MetaStoreEndFunctionListener.class, conf, MetastoreConf.getVar(conf,
ConfVars.END_FUNCTION_LISTENERS));
- String partitionValidationRegex =
- MetastoreConf.getVar(conf, ConfVars.PARTITION_NAME_WHITELIST_PATTERN);
- if (partitionValidationRegex != null &&
!partitionValidationRegex.isEmpty()) {
- partitionValidationPattern = Pattern.compile(partitionValidationRegex);
- }
+
expressionProxy = PartFilterExprUtil.createExpressionProxy(conf);
fileMetadataManager = new FileMetadataManager(this.getMS(), conf);
@@ -472,13 +467,13 @@ public void setConf(Configuration conf) {
@Override
public Configuration getConf() {
- Configuration conf = HMSHandlerContext.getConfiguration()
- .orElseGet(() -> {
- Configuration configuration = new Configuration(this.conf);
- HMSHandlerContext.setConfiguration(configuration);
- return configuration;
- });
- return conf;
+ return HMSHandlerContext.getConfiguration()
+ .orElseGet(
+ () -> {
+ Configuration configuration = new Configuration(this.conf);
+ HMSHandlerContext.setConfiguration(configuration);
+ return configuration;
+ });
}
@Override
@@ -4052,7 +4047,7 @@ private Partition append_partition_common(RawStore ms,
String catName, String db
part.setTableName(tableName);
part.setValues(part_vals);
- MetaStoreServerUtils.validatePartitionNameCharacters(part_vals,
partitionValidationPattern);
+ MetaStoreServerUtils.validatePartitionNameCharacters(part_vals,
getConf());
tbl = ms.getTable(part.getCatName(), part.getDbName(),
part.getTableName(), null);
if (tbl == null) {
@@ -4465,8 +4460,7 @@ private void cleanupPartitionFolders(final
Map<PartValEqWrapperLite, Boolean> ad
private void validatePartition(final Partition part, final String catName,
final String tblName, final String dbName,
final Set<PartValEqWrapperLite> partsToAdd)
throws MetaException, TException {
- MetaStoreServerUtils.validatePartitionNameCharacters(part.getValues(),
- partitionValidationPattern);
+ MetaStoreServerUtils.validatePartitionNameCharacters(part.getValues(),
getConf());
if (part.getDbName() == null || part.getTableName() == null) {
throw new MetaException("The database and table name must be set in the
partition.");
}
@@ -6015,8 +6009,7 @@ private void alter_partition_core(String catName, String
db_name, String tbl_nam
firePreEvent(new PreAlterPartitionEvent(db_name, tbl_name, table,
part_vals, new_part, this));
if (part_vals != null && !part_vals.isEmpty()) {
-
MetaStoreServerUtils.validatePartitionNameCharacters(new_part.getValues(),
- partitionValidationPattern);
+
MetaStoreServerUtils.validatePartitionNameCharacters(new_part.getValues(),
getConf());
}
oldPart = alterHandler.alterPartition(getMS(), wh, catName, db_name,
tbl_name,
@@ -8654,18 +8647,17 @@ public List<String> set_ugi(String username,
List<String> groupNames) throws TEx
}
@Override
- public boolean partition_name_has_valid_characters(List<String> part_vals,
- boolean throw_exception)
throws TException {
+ public boolean partition_name_has_valid_characters(
+ List<String> part_vals, boolean throw_exception) throws TException {
startFunction("partition_name_has_valid_characters");
boolean ret;
Exception ex = null;
try {
if (throw_exception) {
- MetaStoreServerUtils.validatePartitionNameCharacters(part_vals,
partitionValidationPattern);
+ MetaStoreServerUtils.validatePartitionNameCharacters(part_vals,
getConf());
ret = true;
} else {
- ret = MetaStoreServerUtils.partitionNameHasValidCharacters(part_vals,
- partitionValidationPattern);
+ ret = MetaStoreServerUtils.partitionNameHasValidCharacters(part_vals,
getConf());
}
} catch (Exception e) {
ex = e;
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index c1ca07678fb..7836ad0f0fb 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -343,7 +343,6 @@ private enum TXN_STATUS {
private volatile int openTrasactionCalls = 0;
private Transaction currentTransaction = null;
private TXN_STATUS transactionStatus = TXN_STATUS.NO_STATE;
- private Pattern partitionValidationPattern;
private Counter directSqlErrors;
private boolean areTxnStatsSupported = false;
private PropertyStore propertyStore;
@@ -384,15 +383,7 @@ public void setConf(Configuration conf) {
transactionStatus = TXN_STATUS.NO_STATE;
initialize();
-
- String partitionValidationRegex =
- MetastoreConf.getVar(this.conf,
ConfVars.PARTITION_NAME_WHITELIST_PATTERN);
- if (partitionValidationRegex != null &&
!partitionValidationRegex.isEmpty()) {
- partitionValidationPattern = Pattern.compile(partitionValidationRegex);
- } else {
- partitionValidationPattern = null;
- }
-
+
// Note, if metrics have not been initialized this will return null, which
means we aren't
// using metrics. Thus we should always check whether this is non-null
before using.
MetricRegistry registry = Metrics.getRegistry();
@@ -2773,8 +2764,7 @@ protected String describeResult() {
private boolean isValidPartition(
Partition part, List<FieldSchema> partitionKeys, boolean ifNotExists)
throws MetaException {
- MetaStoreServerUtils.validatePartitionNameCharacters(part.getValues(),
- partitionValidationPattern);
+ MetaStoreServerUtils.validatePartitionNameCharacters(part.getValues(),
conf);
boolean doesExist = doesPartitionExist(part.getCatName(),
part.getDbName(), part.getTableName(), partitionKeys,
part.getValues());
if (doesExist && !ifNotExists) {
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java
index 69ec400017a..fd2e1926e43 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java
@@ -17,6 +17,13 @@
*/
package org.apache.hadoop.hive.metastore.utils;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableListMultimap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Multimaps;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
import java.beans.PropertyDescriptor;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
@@ -51,14 +58,7 @@
import java.util.concurrent.Future;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Predicates;
-import com.google.common.collect.ImmutableListMultimap;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Multimaps;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import javax.annotation.Nullable;
import org.apache.commons.beanutils.PropertyUtils;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.ListUtils;
@@ -114,8 +114,6 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.annotation.Nullable;
-
/**
* Utility methods used by Hive standalone metastore server.
*/
@@ -132,7 +130,7 @@ public class MetaStoreServerUtils {
= new com.google.common.base.Function<String, String>() {
@Override
public String apply(@Nullable String string) {
- return org.apache.commons.lang3.StringUtils.defaultString(string);
+ return StringUtils.defaultString(string);
}
};
@@ -255,31 +253,43 @@ public static double decimalToDouble(Decimal decimal) {
return new BigDecimal(new BigInteger(decimal.getUnscaled()),
decimal.getScale()).doubleValue();
}
- public static void validatePartitionNameCharacters(List<String> partVals,
- Pattern
partitionValidationPattern) throws MetaException {
+ private static Pattern getPartitionValidationRegex(Configuration conf) {
+ return Optional.ofNullable(
+ MetastoreConf.getVar(conf,
MetastoreConf.ConfVars.PARTITION_NAME_WHITELIST_PATTERN))
+ .filter(StringUtils::isNotBlank)
+ .map(Pattern::compile)
+ .orElse(null);
+ }
+
+ public static void validatePartitionNameCharacters(List<String> partVals,
Configuration conf)
+ throws MetaException {
+ Pattern partitionValidationPattern = getPartitionValidationRegex(conf);
String invalidPartitionVal = getPartitionValWithInvalidCharacter(partVals,
partitionValidationPattern);
+
if (invalidPartitionVal != null) {
- throw new MetaException("Partition value '" + invalidPartitionVal +
- "' contains a character " + "not matched by whitelist pattern '" +
- partitionValidationPattern.toString() + "'. " + "(configure with " +
- MetastoreConf.ConfVars.PARTITION_NAME_WHITELIST_PATTERN.getVarname()
+ ")");
- }
- }
+ String errorMsg =
+ ("Partition value '%s' contains a character not matched by whitelist
pattern '%s'. Configure with %s")
+ .formatted(
+ invalidPartitionVal,
+ partitionValidationPattern.toString(),
+
MetastoreConf.ConfVars.PARTITION_NAME_WHITELIST_PATTERN.getVarname());
- private static String getPartitionValWithInvalidCharacter(List<String>
partVals,
- Pattern
partitionValidationPattern) {
- if (partitionValidationPattern == null) {
- return null;
+ throw new MetaException(errorMsg);
}
+ }
- for (String partVal : partVals) {
- if (!partitionValidationPattern.matcher(partVal).matches()) {
- return partVal;
- }
+ private static String getPartitionValWithInvalidCharacter(
+ List<String> partVals, Pattern partitionValidationPattern) {
+ String result = null;
+ if (partitionValidationPattern != null) {
+ result =
+ partVals.stream()
+ .filter(partVal ->
!partitionValidationPattern.matcher(partVal).matches())
+ .findFirst()
+ .orElse(null);
}
-
- return null;
+ return result;
}
/**
@@ -351,7 +361,7 @@ public static synchronized byte[]
hashStorageDescriptor(StorageDescriptor sd, Me
SortedSet<String> sortedOuterList = new TreeSet<>();
for (List<String> innerList : skewed.getSkewedColValues()) {
SortedSet<String> sortedInnerList = new TreeSet<>(innerList);
-
sortedOuterList.add(org.apache.commons.lang3.StringUtils.join(sortedInnerList,
"."));
+ sortedOuterList.add(StringUtils.join(sortedInnerList, "."));
}
for (String colval : sortedOuterList) {
md.update(colval.getBytes(ENCODING));
@@ -361,7 +371,7 @@ public static synchronized byte[]
hashStorageDescriptor(StorageDescriptor sd, Me
SortedMap<String, String> sortedMap = new TreeMap<>();
for (Map.Entry<List<String>, String> smap :
skewed.getSkewedColValueLocationMaps().entrySet()) {
SortedSet<String> sortedKey = new TreeSet<>(smap.getKey());
- sortedMap.put(org.apache.commons.lang3.StringUtils.join(sortedKey,
"."), smap.getValue());
+ sortedMap.put(StringUtils.join(sortedKey, "."), smap.getValue());
}
for (Map.Entry<String, String> e : sortedMap.entrySet()) {
md.update(e.getKey().getBytes(ENCODING));
@@ -775,9 +785,8 @@ public static String
validateSkewedColNamesSubsetCol(List<String> skewedColNames
return copySkewedColNames.toString();
}
- public static boolean partitionNameHasValidCharacters(List<String> partVals,
- Pattern
partitionValidationPattern) {
- return getPartitionValWithInvalidCharacter(partVals,
partitionValidationPattern) == null;
+ public static boolean partitionNameHasValidCharacters(List<String> partVals,
Configuration conf) {
+ return getPartitionValWithInvalidCharacter(partVals,
getPartitionValidationRegex(conf)) == null;
}
public static void getMergableCols(ColumnStatistics csNew, Map<String,
String> parameters) {
