This is an automated email from the ASF dual-hosted git repository. pvary pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new c4dc616a1c HIVE-26355: Column compare should be case insensitive for name (Wechar Yu reviewed by Peter Vary) (#3406) c4dc616a1c is described below commit c4dc616a1c86d4564a385d8c988138942b7853a9 Author: Wechar Yu <yuwq1...@gmail.com> AuthorDate: Fri Jul 1 13:28:02 2022 +0800 HIVE-26355: Column compare should be case insensitive for name (Wechar Yu reviewed by Peter Vary) (#3406) --- .../hive/metastore/utils/MetaStoreServerUtils.java | 31 +++++++++++++++++++--- .../metastore/utils/TestMetaStoreServerUtils.java | 28 +++++++++++++++++++ 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java index 0d5aaad455..8793c8c7c6 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java @@ -505,21 +505,46 @@ public class MetaStoreServerUtils { params.remove(StatsSetupConst.NUM_ERASURE_CODED_FILES); } + /** + * Compare the names, types and comments of two lists of {@link FieldSchema}. + * <p> + * The name of {@link FieldSchema} is compared in the case-insensitive mode + * because all names in Hive are case-insensitive. + * + * @param oldCols old columns + * @param newCols new columns + * @return true if the two columns are the same, false otherwise + */ public static boolean areSameColumns(List<FieldSchema> oldCols, List<FieldSchema> newCols) { - return ListUtils.isEqualList(oldCols, newCols); + if (oldCols == newCols) { + return true; + } + if (oldCols == null || newCols == null || oldCols.size() != newCols.size()) { + return false; + } + // We should ignore the case of field names, because some computing engines are case-sensitive, such as Spark. + List<FieldSchema> transformedOldCols = oldCols.stream() + .map(col -> new FieldSchema(col.getName().toLowerCase(), col.getType(), col.getComment())) + .collect(Collectors.toList()); + List<FieldSchema> transformedNewCols = newCols.stream() + .map(col -> new FieldSchema(col.getName().toLowerCase(), col.getType(), col.getComment())) + .collect(Collectors.toList()); + return ListUtils.isEqualList(transformedOldCols, transformedNewCols); } /** * Returns true if p is a prefix of s. + * <p> + * The compare of {@link FieldSchema} is the same as {@link #areSameColumns(List, List)}. */ public static boolean arePrefixColumns(List<FieldSchema> p, List<FieldSchema> s) { if (p == s) { return true; } - if (p.size() > s.size()) { + if (p == null || s == null || p.size() > s.size()) { return false; } - return ListUtils.isEqualList(p, s.subList(0, p.size())); + return areSameColumns(p, s.subList(0, p.size())); } public static void updateBasicState(EnvironmentContext environmentContext, Map<String,String> diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/utils/TestMetaStoreServerUtils.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/utils/TestMetaStoreServerUtils.java index c6597eb94b..7a90e1571d 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/utils/TestMetaStoreServerUtils.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/utils/TestMetaStoreServerUtils.java @@ -884,5 +884,33 @@ public class TestMetaStoreServerUtils { sd.setLocation("s3a://bucket/other_path"); Assert.assertTrue(MetaStoreUtils.validateTblStorage(sd)); } + + @Test + public void testSameColumns() { + FieldSchema col1 = new FieldSchema("col1", "string", "col1 comment"); + FieldSchema Col1 = new FieldSchema("Col1", "string", "col1 comment"); + FieldSchema col2 = new FieldSchema("col2", "string", "col2 comment"); + Assert.assertTrue(MetaStoreServerUtils.areSameColumns(null, null)); + Assert.assertFalse(MetaStoreServerUtils.areSameColumns(Arrays.asList(col1), null)); + Assert.assertFalse(MetaStoreServerUtils.areSameColumns(null, Arrays.asList(col1))); + Assert.assertTrue(MetaStoreServerUtils.areSameColumns(Arrays.asList(col1), Arrays.asList(col1))); + Assert.assertTrue(MetaStoreServerUtils.areSameColumns(Arrays.asList(col1, col2), Arrays.asList(col1, col2))); + Assert.assertTrue(MetaStoreServerUtils.areSameColumns(Arrays.asList(Col1, col2), Arrays.asList(col1, col2))); + } + + @Test + public void testPrefixColumns() { + FieldSchema col1 = new FieldSchema("col1", "string", "col1 comment"); + FieldSchema Col1 = new FieldSchema("Col1", "string", "col1 comment"); + FieldSchema col2 = new FieldSchema("col2", "string", "col2 comment"); + FieldSchema col3 = new FieldSchema("col3", "string", "col3 comment"); + Assert.assertTrue(MetaStoreServerUtils.arePrefixColumns(null, null)); + Assert.assertFalse(MetaStoreServerUtils.arePrefixColumns(Arrays.asList(col1), null)); + Assert.assertFalse(MetaStoreServerUtils.arePrefixColumns(null, Arrays.asList(col1))); + Assert.assertTrue(MetaStoreServerUtils.arePrefixColumns(Arrays.asList(col1), Arrays.asList(col1))); + Assert.assertTrue(MetaStoreServerUtils.arePrefixColumns(Arrays.asList(col1, col2), Arrays.asList(col1, col2, col3))); + Assert.assertTrue(MetaStoreServerUtils.arePrefixColumns(Arrays.asList(Col1, col2), Arrays.asList(col1, col2, col3))); + Assert.assertFalse(MetaStoreServerUtils.arePrefixColumns(Arrays.asList(col1, col2, col3), Arrays.asList(col1, col2))); + } }