[ https://issues.apache.org/jira/browse/HIVE-20977?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16732361#comment-16732361 ]
Hive QA commented on HIVE-20977: -------------------------------- Here are the results of testing the latest attachment: https://issues.apache.org/jira/secure/attachment/12953518/HIVE-20977.2.patch {color:green}SUCCESS:{color} +1 due to 2 test(s) being added or modified. {color:red}ERROR:{color} -1 due to 55 failed/errored test(s), 15761 tests executed *Failed tests:* {noformat} org.apache.hadoop.hive.metastore.TestEmbeddedHiveMetaStore.testPartitionFilter (batchId=223) org.apache.hadoop.hive.metastore.TestRemoteHiveMetaStore.testPartitionFilter (batchId=224) org.apache.hadoop.hive.metastore.TestRemoteHiveMetaStoreZK.testPartitionFilter (batchId=226) org.apache.hadoop.hive.metastore.TestRemoteHiveMetaStoreZKBindHost.testPartitionFilter (batchId=230) org.apache.hadoop.hive.metastore.TestSetUGIOnBothClientServer.testPartitionFilter (batchId=221) org.apache.hadoop.hive.metastore.TestSetUGIOnOnlyClient.testPartitionFilter (batchId=219) org.apache.hadoop.hive.metastore.TestSetUGIOnOnlyServer.testPartitionFilter (batchId=229) org.apache.hadoop.hive.metastore.client.TestGetPartitions.getPartitionsByNamesBogusCatalog[Embedded] (batchId=222) org.apache.hadoop.hive.metastore.client.TestGetPartitions.getPartitionsByNamesBogusCatalog[Remote] (batchId=222) org.apache.hadoop.hive.metastore.client.TestGetPartitions.testGetPartitionWithAuthInfoNullDbName[Embedded] (batchId=222) org.apache.hadoop.hive.metastore.client.TestGetPartitions.testGetPartitionWithAuthInfoNullDbName[Remote] (batchId=222) org.apache.hadoop.hive.metastore.client.TestGetPartitions.testGetPartitionWithAuthInfoNullTblName[Embedded] (batchId=222) org.apache.hadoop.hive.metastore.client.TestGetPartitions.testGetPartitionWithAuthInfoNullTblName[Remote] (batchId=222) org.apache.hadoop.hive.metastore.client.TestGetPartitions.testGetPartitionsByNamesNoDbName[Embedded] (batchId=222) org.apache.hadoop.hive.metastore.client.TestGetPartitions.testGetPartitionsByNamesNoDbName[Remote] (batchId=222) org.apache.hadoop.hive.metastore.client.TestGetPartitions.testGetPartitionsByNamesNoDb[Embedded] (batchId=222) org.apache.hadoop.hive.metastore.client.TestGetPartitions.testGetPartitionsByNamesNoDb[Remote] (batchId=222) org.apache.hadoop.hive.metastore.client.TestGetPartitions.testGetPartitionsByNamesNoTable[Embedded] (batchId=222) org.apache.hadoop.hive.metastore.client.TestGetPartitions.testGetPartitionsByNamesNoTable[Remote] (batchId=222) org.apache.hadoop.hive.metastore.client.TestGetPartitions.testGetPartitionsByNamesNoTblName[Embedded] (batchId=222) org.apache.hadoop.hive.metastore.client.TestGetPartitions.testGetPartitionsByNamesNoTblName[Remote] (batchId=222) org.apache.hadoop.hive.metastore.client.TestGetPartitions.testGetPartitionsByNamesNullDbName[Embedded] (batchId=222) org.apache.hadoop.hive.metastore.client.TestGetPartitions.testGetPartitionsByNamesNullDbName[Remote] (batchId=222) org.apache.hadoop.hive.metastore.client.TestGetPartitions.testGetPartitionsByNamesNullTblName[Embedded] (batchId=222) org.apache.hadoop.hive.metastore.client.TestGetPartitions.testGetPartitionsByNamesNullTblName[Remote] (batchId=222) org.apache.hadoop.hive.metastore.client.TestListPartitions.listPartitionNamesBogusCatalog[Embedded] (batchId=220) org.apache.hadoop.hive.metastore.client.TestListPartitions.listPartitionNamesBogusCatalog[Remote] (batchId=220) org.apache.hadoop.hive.metastore.client.TestListPartitions.testListPartitionNamesByValuesNullDbName[Embedded] (batchId=220) org.apache.hadoop.hive.metastore.client.TestListPartitions.testListPartitionNamesByValuesNullDbName[Remote] (batchId=220) org.apache.hadoop.hive.metastore.client.TestListPartitions.testListPartitionNamesByValuesNullTblName[Embedded] (batchId=220) org.apache.hadoop.hive.metastore.client.TestListPartitions.testListPartitionNamesByValuesNullTblName[Remote] (batchId=220) org.apache.hadoop.hive.metastore.client.TestListPartitions.testListPartitionNamesNoDbName[Embedded] (batchId=220) org.apache.hadoop.hive.metastore.client.TestListPartitions.testListPartitionNamesNoDbName[Remote] (batchId=220) org.apache.hadoop.hive.metastore.client.TestListPartitions.testListPartitionNamesNoDb[Embedded] (batchId=220) org.apache.hadoop.hive.metastore.client.TestListPartitions.testListPartitionNamesNoDb[Remote] (batchId=220) org.apache.hadoop.hive.metastore.client.TestListPartitions.testListPartitionNamesNoTable[Embedded] (batchId=220) org.apache.hadoop.hive.metastore.client.TestListPartitions.testListPartitionNamesNoTable[Remote] (batchId=220) org.apache.hadoop.hive.metastore.client.TestListPartitions.testListPartitionNamesNoTblName[Embedded] (batchId=220) org.apache.hadoop.hive.metastore.client.TestListPartitions.testListPartitionNamesNoTblName[Remote] (batchId=220) org.apache.hadoop.hive.metastore.client.TestListPartitions.testListPartitionNamesNullDbName[Embedded] (batchId=220) org.apache.hadoop.hive.metastore.client.TestListPartitions.testListPartitionNamesNullDbName[Remote] (batchId=220) org.apache.hadoop.hive.metastore.client.TestListPartitions.testListPartitionNamesNullTblName[Embedded] (batchId=220) org.apache.hadoop.hive.metastore.client.TestListPartitions.testListPartitionNamesNullTblName[Remote] (batchId=220) org.apache.hadoop.hive.metastore.client.TestListPartitions.testListPartitionsAllNullDbName[Embedded] (batchId=220) org.apache.hadoop.hive.metastore.client.TestListPartitions.testListPartitionsAllNullDbName[Remote] (batchId=220) org.apache.hadoop.hive.metastore.client.TestListPartitions.testListPartitionsAllNullTblName[Embedded] (batchId=220) org.apache.hadoop.hive.metastore.client.TestListPartitions.testListPartitionsAllNullTblName[Remote] (batchId=220) org.apache.hadoop.hive.metastore.client.TestListPartitions.testListPartitionsByFilterNullDbName[Embedded] (batchId=220) org.apache.hadoop.hive.metastore.client.TestListPartitions.testListPartitionsByFilterNullDbName[Remote] (batchId=220) org.apache.hadoop.hive.metastore.client.TestListPartitions.testListPartitionsByFilterNullTblName[Embedded] (batchId=220) org.apache.hadoop.hive.metastore.client.TestListPartitions.testListPartitionsByFilterNullTblName[Remote] (batchId=220) org.apache.hadoop.hive.metastore.client.TestListPartitions.testListPartitionsWithAuthByValuesNullDbName[Embedded] (batchId=220) org.apache.hadoop.hive.metastore.client.TestListPartitions.testListPartitionsWithAuthByValuesNullDbName[Remote] (batchId=220) org.apache.hadoop.hive.metastore.client.TestListPartitions.testListPartitionsWithAuthByValuesNullTblName[Embedded] (batchId=220) org.apache.hadoop.hive.metastore.client.TestListPartitions.testListPartitionsWithAuthByValuesNullTblName[Remote] (batchId=220) {noformat} Test results: https://builds.apache.org/job/PreCommit-HIVE-Build/15455/testReport Console output: https://builds.apache.org/job/PreCommit-HIVE-Build/15455/console Test logs: http://104.198.109.242/logs/PreCommit-HIVE-Build-15455/ Messages: {noformat} Executing org.apache.hive.ptest.execution.TestCheckPhase Executing org.apache.hive.ptest.execution.PrepPhase Executing org.apache.hive.ptest.execution.YetusPhase Executing org.apache.hive.ptest.execution.ExecutionPhase Executing org.apache.hive.ptest.execution.ReportingPhase Tests exited with: TestsFailedException: 55 tests failed {noformat} This message is automatically generated. ATTACHMENT ID: 12953518 - PreCommit-HIVE-Build > Lazy evaluate the table object in PreReadTableEvent to improve get_partition > performance > ---------------------------------------------------------------------------------------- > > Key: HIVE-20977 > URL: https://issues.apache.org/jira/browse/HIVE-20977 > Project: Hive > Issue Type: Improvement > Reporter: Karthik Manamcheri > Assignee: Karthik Manamcheri > Priority: Minor > Attachments: HIVE-20977.1.patch, HIVE-20977.2.patch > > > The PreReadTableEvent is generated for non-table operations (such as > get_partitions), but only if there is an event listener attached. However, > this is also not necessary if the event listener is not interested in the > read table event. > For example, the TransactionalValidationListener's onEvent looks like this > {code:java} > @Override > public void onEvent(PreEventContext context) throws MetaException, > NoSuchObjectException, > InvalidOperationException { > switch (context.getEventType()) { > case CREATE_TABLE: > handle((PreCreateTableEvent) context); > break; > case ALTER_TABLE: > handle((PreAlterTableEvent) context); > break; > default: > //no validation required.. > } > }{code} > > Note that for read table events it is a no-op. The problem is that the > get_table is evaluated when creating the PreReadTableEvent finally to be just > ignored! > Look at the code below.. {{getMS().getTable(..)}} is evaluated irrespective > of if the listener uses it or not. > {code:java} > private void fireReadTablePreEvent(String catName, String dbName, String > tblName) > throws MetaException, NoSuchObjectException { > if(preListeners.size() > 0) { > // do this only if there is a pre event listener registered (avoid > unnecessary > // metastore api call) > Table t = getMS().getTable(catName, dbName, tblName); > if (t == null) { > throw new NoSuchObjectException(TableName.getQualified(catName, dbName, > tblName) > + " table not found"); > } > firePreEvent(new PreReadTableEvent(t, this)); > } > } > {code} > This can be improved by using a {{Supplier}} and lazily evaluating the table > when needed (once when the first time it is called, memorized after that). > *Motivation* > Whenever a partition call occurs (get_partition, etc.), we fire the > PreReadTableEvent. This affects performance since it fetches the table even > if it is not being used. This change will improve performance on the > get_partition calls. > -- This message was sent by Atlassian JIRA (v7.6.3#76005)