SENTRY-888: Exceptions in Callable tasks in MetaStoreCacheInitializer are being dropped ( Sravya Tirukkovalur, Reviewed by: Lenni Kuff)
Project: http://git-wip-us.apache.org/repos/asf/incubator-sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-sentry/commit/89a906a8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-sentry/tree/89a906a8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-sentry/diff/89a906a8 Branch: refs/heads/hive_plugin_v2 Commit: 89a906a83aef8246812a2091c31de8c6dc8c1ab5 Parents: 7a61d56 Author: Sravya Tirukkovalur <[email protected]> Authored: Fri Oct 9 16:17:37 2015 -0700 Committer: Sun Dapeng <[email protected]> Committed: Mon Nov 2 16:37:01 2015 +0800 ---------------------------------------------------------------------- .../sentry/hdfs/MetastoreCacheInitializer.java | 11 ++--- .../hdfs/TestMetastoreCacheInitializer.java | 42 ++++++++++++++++++++ 2 files changed, 48 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/89a906a8/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/MetastoreCacheInitializer.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/MetastoreCacheInitializer.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/MetastoreCacheInitializer.java index f1e28e9..eb85d45 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/MetastoreCacheInitializer.java +++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/MetastoreCacheInitializer.java @@ -46,7 +46,7 @@ class MetastoreCacheInitializer implements Closeable { final Exception failure; CallResult(Exception ex) { - failure = null; + failure = ex; } } @@ -56,16 +56,18 @@ class MetastoreCacheInitializer implements Closeable { @Override public CallResult call() throws Exception { + Exception e = null; try { doTask(); } catch (Exception ex) { // Ignore if object requested does not exists - return new CallResult( - (ex instanceof NoSuchObjectException) ? null : ex); + if (!(ex instanceof NoSuchObjectException) ){ + e = ex; + } } finally { taskCounter.decrementAndGet(); } - return new CallResult(null); + return new CallResult(e); } abstract void doTask() throws Exception; @@ -225,7 +227,6 @@ class MetastoreCacheInitializer implements Closeable { String[]{"/"}); PathsUpdate tempUpdate = new PathsUpdate(-1, false); List<String> allDbStr = hmsHandler.get_all_databases(); - List<Future<CallResult>> results = new ArrayList<Future<CallResult>>(); for (String dbName : allDbStr) { Callable<CallResult> dbTask = new DbTask(tempUpdate, dbName); results.add(threadPool.submit(dbTask)); http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/89a906a8/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestMetastoreCacheInitializer.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestMetastoreCacheInitializer.java b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestMetastoreCacheInitializer.java index a5a165a..f1e729f 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestMetastoreCacheInitializer.java +++ b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestMetastoreCacheInitializer.java @@ -130,4 +130,46 @@ public class TestMetastoreCacheInitializer { cacheInitializer.close(); } + + // Make sure exceptions in initializer parallel tasks are propagated well + @Test + public void testExceptionInTask() throws Exception { + //Set up mocks: db1.tb1, with tb1 returning a wrong dbname (db2) + Database db1 = Mockito.mock(Database.class); + Mockito.when(db1.getName()).thenReturn("db1"); + Mockito.when(db1.getLocationUri()).thenReturn("hdfs:///db1"); + + Table tab1 = Mockito.mock(Table.class); + //Return a wrong db name, so that this triggers an exception + Mockito.when(tab1.getDbName()).thenReturn("db2"); + Mockito.when(tab1.getTableName()).thenReturn("tab1"); + + IHMSHandler hmsHandler = Mockito.mock(IHMSHandler.class); + Mockito.when(hmsHandler.get_all_databases()).thenReturn(Lists + .newArrayList("db1")); + Mockito.when(hmsHandler.get_database("db1")).thenReturn(db1); + Mockito.when(hmsHandler.get_table_objects_by_name("db1", + Lists.newArrayList("tab1"))) + .thenReturn(Lists.newArrayList(tab1)); + Mockito.when(hmsHandler.get_all_tables("db1")).thenReturn(Lists + .newArrayList("tab1")); + + Configuration conf = new Configuration(); + conf.setInt(ServiceConstants.ServerConfig + .SENTRY_HDFS_SYNC_METASTORE_CACHE_MAX_PART_PER_RPC, 1); + conf.setInt(ServiceConstants.ServerConfig + .SENTRY_HDFS_SYNC_METASTORE_CACHE_MAX_TABLES_PER_RPC, 1); + conf.setInt(ServiceConstants.ServerConfig + .SENTRY_HDFS_SYNC_METASTORE_CACHE_INIT_THREADS, 1); + + try { + MetastoreCacheInitializer cacheInitializer = new + MetastoreCacheInitializer(hmsHandler, conf); + UpdateableAuthzPaths update = cacheInitializer.createInitialUpdate(); + Assert.fail("Expected cacheInitializer to fail"); + } catch (Exception e) { + Assert.assertTrue(e instanceof RuntimeException); + } + + } }
