[ https://issues.apache.org/jira/browse/HIVE-20823?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16682791#comment-16682791 ]
Hive QA commented on HIVE-20823: -------------------------------- Here are the results of testing the latest attachment: https://issues.apache.org/jira/secure/attachment/12947727/HIVE-20823.09.patch {color:green}SUCCESS:{color} +1 due to 15 test(s) being added or modified. {color:red}ERROR:{color} -1 due to 67 failed/errored test(s), 15531 tests executed *Failed tests:* {noformat} org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver[cbo_limit] (batchId=182) org.apache.hadoop.hive.ql.TestAcidOnTez.testInsertWithRemoveUnion (batchId=242) org.apache.hadoop.hive.ql.TestTxnCommands2.testNonAcidToAcidConversion3 (batchId=307) org.apache.hadoop.hive.ql.TestTxnCommands2WithSplitUpdateAndVectorization.testNonAcidToAcidConversion3 (batchId=319) org.apache.hadoop.hive.ql.TestTxnCommandsWithSplitUpdateAndVectorization.testNonAcidToAcidConversion01 (batchId=307) org.apache.hadoop.hive.ql.TestTxnConcatenate.testConcatenate (batchId=292) org.apache.hadoop.hive.ql.TestTxnConcatenate.testConcatenatePart (batchId=292) org.apache.hadoop.hive.ql.TestTxnExIm.testImport (batchId=305) org.apache.hadoop.hive.ql.TestTxnExIm.testImportNoTarget (batchId=305) org.apache.hadoop.hive.ql.TestTxnLoadData.loadData (batchId=283) org.apache.hadoop.hive.ql.TestTxnLoadData.loadDataNonAcid2AcidConversion (batchId=283) org.apache.hadoop.hive.ql.TestTxnLoadData.loadDataUpdate (batchId=283) org.apache.hadoop.hive.ql.TestTxnLoadData.testMultiStatement (batchId=283) org.apache.hadoop.hive.ql.TestTxnNoBucketsVectorized.testNoBuckets (batchId=307) org.apache.hadoop.hive.ql.io.TestAcidUtils.deleteDeltasWithOpenTxnInRead (batchId=294) org.apache.hadoop.hive.ql.io.TestAcidUtils.deltasWithOpenTxnInRead (batchId=294) org.apache.hadoop.hive.ql.io.TestAcidUtils.deltasWithOpenTxnInRead2 (batchId=294) org.apache.hadoop.hive.ql.io.TestAcidUtils.testParsing (batchId=294) org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.majorCompactAfterAbort[0] (batchId=243) org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.majorCompactAfterAbort[1] (batchId=243) org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.majorCompactWhileStreamingForSplitUpdate[0] (batchId=243) org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.majorCompactWhileStreamingForSplitUpdate[1] (batchId=243) org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.majorCompactWhileStreaming[0] (batchId=243) org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.majorCompactWhileStreaming[1] (batchId=243) org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.minorCompactAfterAbort[0] (batchId=243) org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.minorCompactAfterAbort[1] (batchId=243) org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.minorCompactWhileStreamingWithSplitUpdate[0] (batchId=243) org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.minorCompactWhileStreamingWithSplitUpdate[1] (batchId=243) org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.minorCompactWhileStreaming[0] (batchId=243) org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.minorCompactWhileStreaming[1] (batchId=243) org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.testMinorCompactionForSplitUpdateWithInsertsAndDeletes[0] (batchId=243) org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.testMinorCompactionForSplitUpdateWithInsertsAndDeletes[1] (batchId=243) org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.testMinorCompactionForSplitUpdateWithOnlyInserts[0] (batchId=243) org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.testMinorCompactionForSplitUpdateWithOnlyInserts[1] (batchId=243) org.apache.hadoop.hive.ql.txn.compactor.TestWorker.majorNoBaseLotsOfDeltas (batchId=293) org.apache.hadoop.hive.ql.txn.compactor.TestWorker.majorPartitionWithBase (batchId=293) org.apache.hadoop.hive.ql.txn.compactor.TestWorker.majorPartitionWithBaseMissingBuckets (batchId=293) org.apache.hadoop.hive.ql.txn.compactor.TestWorker.majorTableLegacy (batchId=293) org.apache.hadoop.hive.ql.txn.compactor.TestWorker.majorTableNoBase (batchId=293) org.apache.hadoop.hive.ql.txn.compactor.TestWorker.majorTableWithBase (batchId=293) org.apache.hadoop.hive.ql.txn.compactor.TestWorker.majorWithAborted (batchId=293) org.apache.hadoop.hive.ql.txn.compactor.TestWorker.majorWithOpenInMiddle (batchId=293) org.apache.hadoop.hive.ql.txn.compactor.TestWorker.minorNoBaseLotsOfDeltas (batchId=293) org.apache.hadoop.hive.ql.txn.compactor.TestWorker.minorPartitionWithBase (batchId=293) org.apache.hadoop.hive.ql.txn.compactor.TestWorker.minorTableLegacy (batchId=293) org.apache.hadoop.hive.ql.txn.compactor.TestWorker.minorTableNoBase (batchId=293) org.apache.hadoop.hive.ql.txn.compactor.TestWorker.minorTableWithBase (batchId=293) org.apache.hadoop.hive.ql.txn.compactor.TestWorker.minorWithAborted (batchId=293) org.apache.hadoop.hive.ql.txn.compactor.TestWorker.minorWithOpenInMiddle (batchId=293) org.apache.hadoop.hive.ql.txn.compactor.TestWorker2.majorNoBaseLotsOfDeltas (batchId=293) org.apache.hadoop.hive.ql.txn.compactor.TestWorker2.majorPartitionWithBase (batchId=293) org.apache.hadoop.hive.ql.txn.compactor.TestWorker2.majorPartitionWithBaseMissingBuckets (batchId=293) org.apache.hadoop.hive.ql.txn.compactor.TestWorker2.majorTableLegacy (batchId=293) org.apache.hadoop.hive.ql.txn.compactor.TestWorker2.majorTableNoBase (batchId=293) org.apache.hadoop.hive.ql.txn.compactor.TestWorker2.majorTableWithBase (batchId=293) org.apache.hadoop.hive.ql.txn.compactor.TestWorker2.majorWithAborted (batchId=293) org.apache.hadoop.hive.ql.txn.compactor.TestWorker2.majorWithOpenInMiddle (batchId=293) org.apache.hadoop.hive.ql.txn.compactor.TestWorker2.minorNoBaseLotsOfDeltas (batchId=293) org.apache.hadoop.hive.ql.txn.compactor.TestWorker2.minorPartitionWithBase (batchId=293) org.apache.hadoop.hive.ql.txn.compactor.TestWorker2.minorTableLegacy (batchId=293) org.apache.hadoop.hive.ql.txn.compactor.TestWorker2.minorTableNoBase (batchId=293) org.apache.hadoop.hive.ql.txn.compactor.TestWorker2.minorTableWithBase (batchId=293) org.apache.hadoop.hive.ql.txn.compactor.TestWorker2.minorWithAborted (batchId=293) org.apache.hadoop.hive.ql.txn.compactor.TestWorker2.minorWithOpenInMiddle (batchId=293) org.apache.hive.hcatalog.streaming.TestStreaming.testNoBuckets (batchId=217) org.apache.hive.streaming.TestStreaming.testAutoRollTransactionBatch (batchId=331) org.apache.hive.streaming.TestStreaming.testNoBuckets (batchId=331) {noformat} Test results: https://builds.apache.org/job/PreCommit-HIVE-Build/14874/testReport Console output: https://builds.apache.org/job/PreCommit-HIVE-Build/14874/console Test logs: http://104.198.109.242/logs/PreCommit-HIVE-Build-14874/ Messages: {noformat} Executing org.apache.hive.ptest.execution.TestCheckPhase Executing org.apache.hive.ptest.execution.PrepPhase Executing org.apache.hive.ptest.execution.YetusPhase Executing org.apache.hive.ptest.execution.ExecutionPhase Executing org.apache.hive.ptest.execution.ReportingPhase Tests exited with: TestsFailedException: 67 tests failed {noformat} This message is automatically generated. ATTACHMENT ID: 12947727 - PreCommit-HIVE-Build > Make Compactor run in a transaction > ----------------------------------- > > Key: HIVE-20823 > URL: https://issues.apache.org/jira/browse/HIVE-20823 > Project: Hive > Issue Type: Improvement > Components: Transactions > Affects Versions: 3.0.0 > Reporter: Eugene Koifman > Assignee: Eugene Koifman > Priority: Major > Attachments: HIVE-20823.01.patch, HIVE-20823.03.patch, > HIVE-20823.04.patch, HIVE-20823.05.patch, HIVE-20823.07.patch, > HIVE-20823.08.patch, HIVE-20823.09.patch > > > Have compactor open a transaction and run the job in that transaction. > # make compactor produced base/delta include this txn id in the folder name, > e.g. base_7_c17 where 17 is the txnid. > # add {{CQ_TXN_ID bigint}} to COMPACTION_QUEUE and COMPLETED_COMPACTIONS to > record this txn id > # make sure {{AcidUtils.getAcidState()}} pays attention to this transaction > on read and ignores this dir if this txn id is not committed in the current > snapshot > ## this means not only validWriteIdList but ValidTxnIdList should be passed > along in config (if it isn't yet) > # once this is done, {{CompactorMR.createCompactorMarker()}} can be > eliminated and {{AcidUtils.isValidBase}} modified accordingly > # modify Cleaner so that it doesn't clean old files until new file is visible > to all readers > # -- This message was sent by Atlassian JIRA (v7.6.3#76005)