kadirozde commented on a change in pull request #915:
URL: https://github.com/apache/phoenix/pull/915#discussion_r504922136
##########
File path:
phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GlobalIndexRegionScanner.java
##########
@@ -240,4 +320,1033 @@ protected boolean isColumnIncluded(Cell cell) {
byte[] qualifier = CellUtil.cloneQualifier(cell);
return set.contains(qualifier);
}
+ @VisibleForTesting
+ public boolean shouldVerify(IndexTool.IndexVerifyType verifyType,
+ byte[] indexRowKey, Scan scan, Region region,
IndexMaintainer indexMaintainer,
+ IndexVerificationResultRepository
verificationResultRepository, boolean shouldVerifyCheckDone) throws IOException
{
+ this.verifyType = verifyType;
+ this.indexRowKeyforReadRepair = indexRowKey;
+ this.scan = scan;
+ this.region = region;
+ this.indexMaintainer = indexMaintainer;
+ this.verificationResultRepository = verificationResultRepository;
+ this.shouldVerifyCheckDone = shouldVerifyCheckDone;
+ return shouldVerify();
+ }
+
+ protected boolean shouldVerify() throws IOException {
+ // In case of read repair, proceed with rebuild
+ // All other types of rebuilds/verification should be incrementally
performed if appropriate param is passed
+ byte[] lastVerifyTimeValue =
scan.getAttribute(UngroupedAggregateRegionObserver.INDEX_RETRY_VERIFY);
+ Long lastVerifyTime = lastVerifyTimeValue == null ? 0 :
Bytes.toLong(lastVerifyTimeValue);
+ if(indexRowKeyforReadRepair != null || lastVerifyTime == 0 ||
shouldVerifyCheckDone) {
+ return true;
+ }
+
+ IndexToolVerificationResult verificationResultTemp =
verificationResultRepository
+ .getVerificationResult(lastVerifyTime, scan, region,
indexMaintainer.getIndexTableName()) ;
+ if(verificationResultTemp != null) {
+ verificationResult = verificationResultTemp;
+ }
+ shouldVerifyCheckDone = true;
+ return verificationResultTemp == null;
+ }
+
+ @Override
+ public HRegionInfo getRegionInfo() {
+ return region.getRegionInfo();
+ }
+
+ @Override
+ public boolean isFilterDone() {
+ return false;
+ }
+
+ private void closeTables() throws IOException {
+ hTableFactory.shutdown();
+ if (indexHTable != null) {
+ indexHTable.close();
+ }
+ if (dataHTable != null) {
+ dataHTable.close();
+ }
+ }
+ @Override
+ public void close() throws IOException {
+ innerScanner.close();
+ if (indexRowKeyforReadRepair != null) {
+ closeTables();
+ return;
+ }
+ if (verify) {
+ try {
+ if (verificationResultRepository != null) {
+
verificationResultRepository.logToIndexToolResultTable(verificationResult,
+ verifyType,
region.getRegionInfo().getRegionName(), skipped);
+ }
+ } finally {
+ this.pool.stop("IndexRegionObserverRegionScanner is closing");
+ closeTables();
+ if (verificationResultRepository != null) {
+ verificationResultRepository.close();
+ }
+ if (verificationOutputRepository != null) {
+ verificationOutputRepository.close();
+ }
+ }
+ }
+ else {
+ this.pool.stop("IndexRegionObserverRegionScanner is closing");
+ closeTables();
+ }
+ }
+
+ @VisibleForTesting
+ public int setIndexTableTTL(int ttl) {
+ indexTableTTL = ttl;
+ return 0;
+ }
+
+ @VisibleForTesting
+ public int setIndexMaintainer(IndexMaintainer indexMaintainer) {
+ this.indexMaintainer = indexMaintainer;
+ return 0;
+ }
+
+ @VisibleForTesting
+ public long setMaxLookBackInMills(long maxLookBackInMills) {
+ this.maxLookBackInMills = maxLookBackInMills;
+ return 0;
+ }
+
+ public void logToIndexToolOutputTable(byte[] dataRowKey, byte[]
indexRowKey, long dataRowTs, long indexRowTs,
+ String errorMsg, boolean
isBeforeRebuild,
+
IndexVerificationOutputRepository.IndexVerificationErrorType errorType) throws
IOException {
+ logToIndexToolOutputTable(dataRowKey, indexRowKey, dataRowTs,
indexRowTs, errorMsg, null,
+ null, isBeforeRebuild, errorType);
+ }
+
+ @VisibleForTesting
+ public void logToIndexToolOutputTable(byte[] dataRowKey, byte[]
indexRowKey, long dataRowTs, long indexRowTs,
+ String errorMsg, byte[]
expectedVaue, byte[] actualValue, boolean isBeforeRebuild,
+
IndexVerificationOutputRepository.IndexVerificationErrorType errorType) throws
IOException {
+ ungroupedAggregateRegionObserver.checkForRegionClosing();
+ verificationOutputRepository.logToIndexToolOutputTable(dataRowKey,
indexRowKey, dataRowTs, indexRowTs,
+ errorMsg, expectedVaue, actualValue,
scan.getTimeRange().getMax(),
+ region.getRegionInfo().getTable().getName(), isBeforeRebuild,
errorType);
+ }
+
+ private static Cell getCell(Mutation m, byte[] family, byte[] qualifier) {
+ List<Cell> cellList = m.getFamilyCellMap().get(family);
+ if (cellList == null) {
+ return null;
+ }
+ for (Cell cell : cellList) {
+ if (CellUtil.matchingQualifier(cell, qualifier)) {
+ return cell;
+ }
+ }
+ return null;
+ }
+
+ private void logMismatch(Mutation expected, Mutation actual, int
iteration, IndexToolVerificationResult.PhaseResult verificationPhaseResult,
boolean isBeforeRebuild) throws IOException {
+ if (getTimestamp(expected) != getTimestamp(actual)) {
+ String errorMsg = "Not matching timestamp";
+ byte[] dataKey = indexMaintainer.buildDataRowKey(new
ImmutableBytesWritable(expected.getRow()), viewConstants);
+ logToIndexToolOutputTable(dataKey, expected.getRow(),
getTimestamp(expected), getTimestamp(actual),
+ errorMsg, null, null, isBeforeRebuild, INVALID_ROW);
+ return;
+ }
+ int expectedCellCount = 0;
+ for (List<Cell> cells : expected.getFamilyCellMap().values()) {
+ if (cells == null) {
+ continue;
+ }
+ for (Cell expectedCell : cells) {
+ expectedCellCount++;
+ byte[] family = CellUtil.cloneFamily(expectedCell);
+ byte[] qualifier = CellUtil.cloneQualifier(expectedCell);
+ Cell actualCell = getCell(actual, family, qualifier);
+ if (actualCell == null ||
+ !CellUtil.matchingType(expectedCell, actualCell)) {
+ byte[] dataKey = indexMaintainer.buildDataRowKey(new
ImmutableBytesWritable(expected.getRow()), viewConstants);
+ String errorMsg = "Missing cell (in iteration " +
iteration + ") " + Bytes.toString(family) + ":" + Bytes.toString(qualifier);
+ logToIndexToolOutputTable(dataKey, expected.getRow(),
getTimestamp(expected),
+ getTimestamp(actual), errorMsg, isBeforeRebuild,
INVALID_ROW);
+
verificationPhaseResult.setIndexHasMissingCellsCount(verificationPhaseResult.getIndexHasMissingCellsCount()
+ 1);
+ return;
+ }
+ if (!CellUtil.matchingValue(actualCell, expectedCell)) {
+ String errorMsg = "Not matching value (in iteration " +
iteration + ") for " + Bytes.toString(family) + ":" + Bytes.toString(qualifier);
Review comment:
Did you mean 5923? Single cell format for indexes will be optional.
Also, please note that there will be a separate cell for each family.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
