This is an automated email from the ASF dual-hosted git repository. nagarwal pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push: new 23dbc09 [MINOR] Removing un-used files and references (#3150) 23dbc09 is described below commit 23dbc09a0d5d677ed3728dbd5357320590ca2833 Author: n3nash <nagar...@uber.com> AuthorDate: Thu Jun 24 22:17:40 2021 -0700 [MINOR] Removing un-used files and references (#3150) --- .../org/apache/hudi/hive/TestHiveSyncTool.java | 133 --------------------- .../hive-global-commit-config.xml | 27 ----- 2 files changed, 160 deletions(-) diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java index e88c46a..3be00c4 100644 --- a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java +++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java @@ -943,137 +943,4 @@ public class TestHiveSyncTool { .containsValue("BIGINT"), errorMsg); hiveClient.updateHiveSQL(dropTableSql); } - /* - private void verifyOldParquetFileTest(HoodieHiveClient hiveClient, String emptyCommitTime) throws Exception { - assertTrue(hiveClient.doesTableExist(HiveTestUtil.hiveSyncConfig.tableName), - "Table " + HiveTestUtil.hiveSyncConfig.tableName + " should exist after sync completes"); - assertEquals(hiveClient.getTableSchema(HiveTestUtil.hiveSyncConfig.tableName).size(), - hiveClient.getDataSchema().getColumns().size() + 1, - "Hive Schema should match the table schema + partition field"); - assertEquals(1, - hiveClient.scanTablePartitions(HiveTestUtil.hiveSyncConfig.tableName).size(), - "Table partitions should match the number of partitions we wrote"); - assertEquals("The last commit that was sycned should be updated in the TBLPROPERTIES", emptyCommitTime, - hiveClient.getLastCommitTimeSynced(HiveTestUtil.hiveSyncConfig.tableName).get()); - - // make sure correct schema is picked - Schema schema = SchemaTestUtil.getSimpleSchema(); - for (Field field : schema.getFields()) { - assertEquals(String.format("Hive Schema Field %s was added", field), field.schema().getType().getName(), - hiveClient.getTableSchema(HiveTestUtil.hiveSyncConfig.tableName).get(field.name()).toLowerCase()); - } - assertEquals("Hive Schema Field datestr was added", "string", - hiveClient.getTableSchema(HiveTestUtil.hiveSyncConfig.tableName).get("datestr").toLowerCase()); - assertEquals(schema.getFields().size() + 1, - hiveClient.getTableSchema(HiveTestUtil.hiveSyncConfig.tableName).size(), - "Hive Schema fields size"); - } - - @ParameterizedTest - @MethodSource("useJdbc") - public void testPickingOlderParquetFileIfLatestIsEmptyCommit(boolean useJdbc) throws Exception { - HiveTestUtil.hiveSyncConfig.useJdbc = useJdbc; - final String commitTime = "100"; - HiveTestUtil.createCOWTable(commitTime, 1, false); - HoodieCommitMetadata commitMetadata = new HoodieCommitMetadata(); - // create empty commit - final String emptyCommitTime = "200"; - HiveTestUtil.createCommitFile(commitMetadata, emptyCommitTime); - HoodieHiveClient hiveClient = - new HoodieHiveClient(HiveTestUtil.hiveSyncConfig, HiveTestUtil.getHiveConf(), HiveTestUtil.fileSystem); - assertFalse(hiveClient.doesTableExist(HiveTestUtil.hiveSyncConfig.tableName), - "Table " + HiveTestUtil.hiveSyncConfig.tableName + " should not exist initially"); - - HiveSyncTool tool = new HiveSyncTool(HiveTestUtil.hiveSyncConfig, HiveTestUtil.getHiveConf(), HiveTestUtil.fileSystem); - tool.syncHoodieTable(); - - verifyOldParquetFileTest(hiveClient, emptyCommitTime); - } - - @ParameterizedTest - @MethodSource("useJdbc") - public void testNotPickingOlderParquetFileWhenLatestCommitReadFails(boolean useJdbc) throws Exception { - HiveTestUtil.hiveSyncConfig.useJdbc = useJdbc; - final String commitTime = "100"; - HiveTestUtil.createCOWTable(commitTime, 1, false); - HoodieCommitMetadata commitMetadata = new HoodieCommitMetadata(); - - // evolve the schema - DateTime dateTime = DateTime.now().plusDays(6); - String commitTime2 = "101"; - HiveTestUtil.addCOWPartitions(1, false, false, dateTime, commitTime2); - - // create empty commit - final String emptyCommitTime = "200"; - HiveTestUtil.createCommitFile(commitMetadata, emptyCommitTime); - - HoodieHiveClient hiveClient = - new HoodieHiveClient(HiveTestUtil.hiveSyncConfig, HiveTestUtil.getHiveConf(), HiveTestUtil.fileSystem); - assertFalse(hiveClient.doesTableExist(HiveTestUtil.hiveSyncConfig.tableName), - "Table " + HiveTestUtil.hiveSyncConfig.tableName + " should not exist initially"); - - HiveSyncTool tool = new HiveSyncTool(HiveTestUtil.hiveSyncConfig, HiveTestUtil.getHiveConf(), HiveTestUtil.fileSystem); - - // now delete the evolved commit instant - Path fullPath = new Path(HiveTestUtil.hiveSyncConfig.basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" - + hiveClient.getActiveTimeline().getInstants() - .filter(inst -> inst.getTimestamp().equals(commitTime2)) - .findFirst().get().getFileName()); - assertTrue(HiveTestUtil.fileSystem.delete(fullPath, false)); - - try { - tool.syncHoodieTable(); - } catch (RuntimeException e) { - // we expect the table sync to fail - } - - // table should not be synced yet - assertFalse(hiveClient.doesTableExist(HiveTestUtil.hiveSyncConfig.tableName), - "Table " + HiveTestUtil.hiveSyncConfig.tableName + " should not exist at all"); - } - - @ParameterizedTest - @MethodSource("useJdbc") - public void testNotPickingOlderParquetFileWhenLatestCommitReadFailsForExistingTable(boolean useJdbc) throws Exception { - HiveTestUtil.hiveSyncConfig.useJdbc = useJdbc; - final String commitTime = "100"; - HiveTestUtil.createCOWTable(commitTime, 1, false); - HoodieCommitMetadata commitMetadata = new HoodieCommitMetadata(); - // create empty commit - final String emptyCommitTime = "200"; - HiveTestUtil.createCommitFile(commitMetadata, emptyCommitTime); - HoodieHiveClient hiveClient = - new HoodieHiveClient(HiveTestUtil.hiveSyncConfig, HiveTestUtil.getHiveConf(), HiveTestUtil.fileSystem); - assertFalse(hiveClient.doesTableExist(HiveTestUtil.hiveSyncConfig.tableName), - "Table " + HiveTestUtil.hiveSyncConfig.tableName + " should not exist initially"); - - HiveSyncTool tool = new HiveSyncTool(HiveTestUtil.hiveSyncConfig, HiveTestUtil.getHiveConf(), HiveTestUtil.fileSystem); - tool.syncHoodieTable(); - - verifyOldParquetFileTest(hiveClient, emptyCommitTime); - - // evolve the schema - DateTime dateTime = DateTime.now().plusDays(6); - String commitTime2 = "301"; - HiveTestUtil.addCOWPartitions(1, false, false, dateTime, commitTime2); - HiveTestUtil.createCommitFile(commitMetadata, "400"); // create another empty commit - - tool = new HiveSyncTool(HiveTestUtil.hiveSyncConfig, HiveTestUtil.getHiveConf(), HiveTestUtil.fileSystem); - HoodieHiveClient hiveClientLatest = new HoodieHiveClient(HiveTestUtil.hiveSyncConfig, HiveTestUtil.getHiveConf(), HiveTestUtil.fileSystem); - // now delete the evolved commit instant - Path fullPath = new Path(HiveTestUtil.hiveSyncConfig.basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" - + hiveClientLatest.getActiveTimeline().getInstants() - .filter(inst -> inst.getTimestamp().equals(commitTime2)) - .findFirst().get().getFileName()); - assertTrue(HiveTestUtil.fileSystem.delete(fullPath, false)); - - try { - tool.syncHoodieTable(); - } catch (RuntimeException e) { - // we expect the table sync to fail - } - - // old sync values should be left intact - verifyOldParquetFileTest(hiveClient, emptyCommitTime); - }*/ } diff --git a/hudi-utilities/src/test/resources/delta-streamer-config/hive-global-commit-config.xml b/hudi-utilities/src/test/resources/delta-streamer-config/hive-global-commit-config.xml deleted file mode 100644 index e1a8347..0000000 --- a/hudi-utilities/src/test/resources/delta-streamer-config/hive-global-commit-config.xml +++ /dev/null @@ -1,27 +0,0 @@ -<?xml version="1.0" encoding="UTF-8" standalone="no"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - --> -<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd"> -<properties> -<comment>hi</comment> -<entry key="hivesyncglobal.retry_attempts">1</entry> -<entry key="hivesyncglobal.remote_hive_site_uri">/home/hive-remote-site.xml</entry> -<entry key="hivesyncglobal.remote_base_path">hdfs://hadoop-cluster2:9000/tmp/hudi_trips_cow</entry> -<entry key="hivesyncglobal.local_hive_site_uri">/home/hive/packaging/target/apache-hive-2.3.4-uber-51-SNAPSHOT-bin/apache-hive-2.3.4-uber-51-SNAPSHOT-bin/conf/hive-site.xml</entry> -<entry key="hivesyncglobal.remote_hs2_jdbc_urls">jdbc:hive2://hadoop-cluster2:10000/default;transportMode=http;httpPath=hs2</entry> -</properties>