>From Ian Maxon <[email protected]>:
Ian Maxon has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17601 )
Change subject: [ASTERIXDB-3204][SQL][MTD] ANALYZE DATASET *
......................................................................
[ASTERIXDB-3204][SQL][MTD] ANALYZE DATASET *
- user model changes: yes
- storage format changes: no
- interface changes: no
Details:
Introduce the syntax in the title that allows one
to collect samples of all datasets in a dataverse
without specifying them all.
Furthermore, modify the test harness for running
all execution tests with samples to use this in
lieu of regexes to determine which datasets to
analyze.
Change-Id: Ib5e1b2efefe38332ea1f8f7328c63ace12ac9ceb
---
M
asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
M
asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/fulltext/stopwords-full-text-filter-1/stopwords-full-text-filter-1.7.adm
M
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/AnalyzingTestExecutor.java
M
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppAnalyzedExecutionTest.java
M asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
M
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataLockUtil.java
6 files changed, 325 insertions(+), 195 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/01/17601/1
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 28a6d2d..0b171c3 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -275,6 +275,8 @@
private static final Logger LOGGER = LogManager.getLogger();
public static final boolean IS_DEBUG_MODE = false;// true
+ public static final String STAR = "*";
+
protected final List<Statement> statements;
protected final ICcApplicationContext appCtx;
protected final SessionOutput sessionOutput;
@@ -4075,17 +4077,213 @@
}
}
+ //TODO:make this a record when we require jdk17
+ private final class PreparedSampleIndexInfo {
+ final String newIndexName;
+ final Index existingIndex;
+ final Index newIndexPendingAdd;
+ final JobSpecification sampleIdxToCreate;
+
+ public PreparedSampleIndexInfo(String newIndexName, Index
existingIndex, Index newIndexPendingAdd,
+ JobSpecification sampleIdxToCreate) {
+ this.newIndexName = newIndexName;
+ this.existingIndex = existingIndex;
+ this.newIndexPendingAdd = newIndexPendingAdd;
+ this.sampleIdxToCreate = sampleIdxToCreate;
+ }
+ }
+
+ private PreparedSampleIndexInfo createAndPrepareSampleIdx(Dataset ds,
DataverseName dataverseName,
+ MetadataProvider metadataProvider, SourceLocation sourceLoc, int
sampleCardinalityTarget, long sampleSeed)
+ throws Exception {
+ if (ds == null) {
+ throw new
CompilationException(ErrorCode.UNKNOWN_DATASET_IN_DATAVERSE, sourceLoc,
ds.getDatasetName(),
+ dataverseName);
+ }
+ if (ds.getDatasetType() == DatasetType.INTERNAL) {
+ validateDatasetState(metadataProvider, ds, sourceLoc);
+ } else {
+ throw new CompilationException(ErrorCode.OPERATION_NOT_SUPPORTED,
sourceLoc);
+ }
+
+ IndexType sampleIndexType = IndexType.SAMPLE;
+ Pair<String, String> sampleIndexNames =
IndexUtil.getSampleIndexNames(ds.getDatasetName());
+ String newIndexName;
+ Index existingIndex =
MetadataManager.INSTANCE.getIndex(metadataProvider.getMetadataTxnContext(),
dataverseName,
+ ds.getDatasetName(), sampleIndexNames.first);
+ if (existingIndex != null) {
+ newIndexName = sampleIndexNames.second;
+ } else {
+ existingIndex =
MetadataManager.INSTANCE.getIndex(metadataProvider.getMetadataTxnContext(),
dataverseName,
+ ds.getDatasetName(), sampleIndexNames.second);
+ newIndexName = sampleIndexNames.first;
+ }
+
+ InternalDatasetDetails dsDetails = (InternalDatasetDetails)
ds.getDatasetDetails();
+
+ Index.SampleIndexDetails newIndexDetailsPendingAdd = new
Index.SampleIndexDetails(dsDetails.getPrimaryKey(),
+ dsDetails.getKeySourceIndicator(),
dsDetails.getPrimaryKeyType(), sampleCardinalityTarget, 0, 0,
+ sampleSeed, Collections.emptyMap());
+ Index newIndexPendingAdd = new Index(dataverseName,
ds.getDatasetName(), newIndexName, sampleIndexType,
+ newIndexDetailsPendingAdd, false, false,
MetadataUtil.PENDING_ADD_OP);
+
+ // #. add a new index with PendingAddOp
+
MetadataManager.INSTANCE.addIndex(metadataProvider.getMetadataTxnContext(),
newIndexPendingAdd);
+ // #. prepare to create the index artifact in NC.
+ JobSpecification spec =
+ IndexUtil.buildSecondaryIndexCreationJobSpec(ds,
newIndexPendingAdd, metadataProvider, sourceLoc);
+ if (spec == null) {
+ throw new CompilationException(ErrorCode.COMPILATION_ERROR,
sourceLoc,
+ "Failed to create job spec for creating index '" +
ds.getDatasetName() + "."
+ + newIndexPendingAdd.getIndexName() + "'");
+ }
+ return new PreparedSampleIndexInfo(newIndexName, existingIndex,
newIndexPendingAdd, spec);
+ }
+
+ private Index runLoadJobAndFinalizeIndex(Dataset ds, DataverseName
dataverseName, String newIndexName,
+ JobSpecification spec, IHyracksClientConnection hcc,
SourceLocation sourceLoc, int sampleCardinalityTarget,
+ long sampleSeed) throws Exception {
+ List<IOperatorStats> opStats = runJob(hcc, spec, jobFlags,
+
Collections.singletonList(SampleOperationsHelper.DATASET_STATS_OPERATOR_NAME));
+ if (opStats == null || opStats.size() == 0) {
+ throw new
CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, "", sourceLoc);
+ }
+ DatasetStreamStats stats = new DatasetStreamStats(opStats.get(0));
+ InternalDatasetDetails dsDetails = (InternalDatasetDetails)
ds.getDatasetDetails();
+ Index.SampleIndexDetails newIndexDetailsFinal = new
Index.SampleIndexDetails(dsDetails.getPrimaryKey(),
+ dsDetails.getKeySourceIndicator(),
dsDetails.getPrimaryKeyType(), sampleCardinalityTarget,
+ stats.getCardinality(), stats.getAvgTupleSize(), sampleSeed,
stats.getIndexesStats());
+ Index newIndexFinal = new Index(dataverseName, ds.getDatasetName(),
newIndexName, IndexType.SAMPLE,
+ newIndexDetailsFinal, false, false,
MetadataUtil.PENDING_NO_OP);
+ return newIndexFinal;
+ }
+
+ private void dropExistingSamples(Map<Dataset, PreparedSampleIndexInfo>
preparedIndexes,
+ Map<Dataset, JobSpecification> existingIndexDropSpecs,
+ Map<Dataset, ProgressState> progressExistingIndexDrop,
IHyracksClientConnection hcc,
+ MetadataProvider metadataProvider, MetadataTransactionContext
mdTxnCtx, MutableBoolean bActiveTxn,
+ SourceLocation sourceLoc) throws Exception {
+ mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+ bActiveTxn.setTrue();
+ metadataProvider.setMetadataTxnContext(mdTxnCtx);
+ Map<Dataset, PreparedSampleIndexInfo> indexesToReplace =
+ preparedIndexes.entrySet().stream().filter(e ->
e.getValue().existingIndex != null)
+ .collect(Collectors.toMap(e -> e.getKey(), e ->
e.getValue()));
+ for (Map.Entry<Dataset, PreparedSampleIndexInfo> info :
indexesToReplace.entrySet()) {
+ Index existingIndex = info.getValue().existingIndex;
+ Dataset ds = info.getKey();
+
MetadataManager.INSTANCE.dropIndex(metadataProvider.getMetadataTxnContext(),
+ existingIndex.getDataverseName(),
existingIndex.getDatasetName(), existingIndex.getIndexName());
+ existingIndex.setPendingOp(MetadataUtil.PENDING_DROP_OP);
+
MetadataManager.INSTANCE.addIndex(metadataProvider.getMetadataTxnContext(),
existingIndex);
+ JobSpecification existingIndexDropSpec =
+ IndexUtil.buildDropIndexJobSpec(existingIndex,
metadataProvider, ds, sourceLoc);
+ existingIndexDropSpecs.put(info.getKey(), existingIndexDropSpec);
+ }
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+ bActiveTxn.setFalse();
+
+ // #. set existing index to PendingDropOp because we'll be dropping it
next
+ indexesToReplace.keySet().stream()
+ .map(ds -> progressExistingIndexDrop.put(ds,
ProgressState.ADDED_PENDINGOP_RECORD_TO_METADATA));
+
+ // #. drop existing index on NCs
+ for (JobSpecification existingIndexDropSpec :
existingIndexDropSpecs.values()) {
+ runJob(hcc, existingIndexDropSpec);
+ }
+
+ // #. drop existing index metadata
+ mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+ bActiveTxn.setTrue();
+ metadataProvider.setMetadataTxnContext(mdTxnCtx);
+ for (Map.Entry<Dataset, PreparedSampleIndexInfo> info :
indexesToReplace.entrySet()) {
+ Index existingIndex = info.getValue().existingIndex;
+ Dataset ds = info.getKey();
+
MetadataManager.INSTANCE.dropIndex(metadataProvider.getMetadataTxnContext(),
+ existingIndex.getDataverseName(),
existingIndex.getDatasetName(), existingIndex.getIndexName());
+ progressExistingIndexDrop.put(ds, ProgressState.NO_PROGRESS);
+ }
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+ bActiveTxn.setFalse();
+ indexesToReplace.keySet().stream().map(ds ->
progressExistingIndexDrop.put(ds, ProgressState.NO_PROGRESS));
+ }
+
+ private void abortPendingSampleDrop(Dataset ds, Index existingIndex,
JobSpecification dropSpec, Exception e,
+ IHyracksClientConnection hcc, MetadataProvider metadataProvider,
MetadataTransactionContext mdTxnCtx)
+ throws RemoteException {
+ // #. execute compensation operations remove the index in NCs
+ try {
+ runJob(hcc, dropSpec);
+ } catch (Exception e2) {
+ // do no throw exception since still the metadata needs to be
compensated.
+ e.addSuppressed(e2);
+ }
+ // #. remove the record from the metadata.
+ mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+ metadataProvider.setMetadataTxnContext(mdTxnCtx);
+ try {
+
MetadataManager.INSTANCE.dropIndex(metadataProvider.getMetadataTxnContext(),
+ existingIndex.getDataverseName(),
existingIndex.getDatasetName(), existingIndex.getIndexName());
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+ } catch (Exception e2) {
+ e.addSuppressed(e2);
+ abort(e, e2, mdTxnCtx);
+ throw new IllegalStateException("System is inconsistent state:
pending index("
+ + existingIndex.getDataverseName() + "." +
existingIndex.getDatasetName() + "."
+ + existingIndex.getIndexName() + ") couldn't be removed
from the metadata", e);
+ }
+ }
+
+ private void abortPendingSampleCreate(Dataset ds, Index
newIndexPendingAdd, MetadataProvider metadataProvider,
+ MetadataTransactionContext mdTxnCtx, MutableBoolean bActiveTxn,
IHyracksClientConnection hcc,
+ SourceLocation sourceLoc, Exception e) throws RemoteException {
+ // #. execute compensation operations remove the index in NCs
+ mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+ bActiveTxn.setTrue();
+ metadataProvider.setMetadataTxnContext(mdTxnCtx);
+ try {
+ JobSpecification jobSpec =
+ IndexUtil.buildDropIndexJobSpec(newIndexPendingAdd,
metadataProvider, ds, sourceLoc);
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+ bActiveTxn.setFalse();
+ runJob(hcc, jobSpec);
+ } catch (Exception e2) {
+ e.addSuppressed(e2);
+ if (bActiveTxn.booleanValue()) {
+ abort(e, e2, mdTxnCtx);
+ }
+ }
+ // #. remove the record from the metadata.
+ mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+ metadataProvider.setMetadataTxnContext(mdTxnCtx);
+ try {
+
MetadataManager.INSTANCE.dropIndex(metadataProvider.getMetadataTxnContext(),
+ newIndexPendingAdd.getDataverseName(),
newIndexPendingAdd.getDatasetName(),
+ newIndexPendingAdd.getIndexName());
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+ } catch (Exception e2) {
+ e.addSuppressed(e2);
+ abort(e, e2, mdTxnCtx);
+ throw new IllegalStateException("System is in inconsistent state:
pending index("
+ + newIndexPendingAdd.getDataverseName() + "." +
newIndexPendingAdd.getDatasetName() + "."
+ + newIndexPendingAdd.getIndexName() + ") couldn't be
removed from the metadata", e);
+ }
+ }
+
protected void doAnalyzeDataset(MetadataProvider metadataProvider,
AnalyzeStatement stmtAnalyze,
- DataverseName dataverseName, String datasetName,
IHyracksClientConnection hcc,
+ DataverseName dataverseName, String datasetNameOrStar,
IHyracksClientConnection hcc,
IRequestParameters requestParameters) throws Exception {
SourceLocation sourceLoc = stmtAnalyze.getSourceLocation();
- ProgressState progressNewIndexCreate = ProgressState.NO_PROGRESS;
- ProgressState progressExistingIndexDrop = ProgressState.NO_PROGRESS;
- Dataset ds = null;
- Index existingIndex = null, newIndexPendingAdd = null;
- JobSpecification existingIndexDropSpec = null;
+ Map<Dataset, ProgressState> progressNewIndexCreate = new HashMap<>(),
+ progressExistingIndexDrop = new HashMap<>();
+ List<Dataset> dses = null;
+ Map<Dataset, Index> newIndexesPendingAdd = new HashMap<>(),
newFinalizedIndexes = new HashMap<>();
+ Map<Dataset, JobSpecification> existingIndexDropSpecs = new
HashMap<>(), sampleLoadSpecs = new HashMap<>();
+ Map<Dataset, PreparedSampleIndexInfo> preparedIndexes = new
HashMap<>();
+ int sampleCardinalityTarget = stmtAnalyze.getSampleSize();
+ long sampleSeed = stmtAnalyze.getOrCreateSampleSeed();
MetadataTransactionContext mdTxnCtx =
MetadataManager.INSTANCE.beginTransaction();
- boolean bActiveTxn = true;
+ MutableBoolean bActiveTxn = new MutableBoolean(true);
metadataProvider.setMetadataTxnContext(mdTxnCtx);
try {
// Check if the dataverse exists
@@ -4093,187 +4291,98 @@
if (dv == null) {
throw new CompilationException(ErrorCode.UNKNOWN_DATAVERSE,
sourceLoc, dataverseName);
}
- // Check if the dataset exists
- ds = metadataProvider.findDataset(dataverseName, datasetName);
- if (ds == null) {
- throw new
CompilationException(ErrorCode.UNKNOWN_DATASET_IN_DATAVERSE, sourceLoc,
datasetName,
- dataverseName);
- }
- if (ds.getDatasetType() == DatasetType.INTERNAL) {
- validateDatasetState(metadataProvider, ds, sourceLoc);
+ if (STAR.equals(datasetNameOrStar)) {
+ //Get all datasets, and only take the internal ones to be
sampled.
+ dses = MetadataManager.INSTANCE.getDataverseDatasets(mdTxnCtx,
dataverseName);
+ dses = dses.stream().filter(dataset ->
dataset.getDatasetType() == DatasetType.INTERNAL)
+ .collect(Collectors.toList());
} else {
- throw new
CompilationException(ErrorCode.OPERATION_NOT_SUPPORTED, sourceLoc);
+ // Check if the dataset exists, and if it does, make it a list
of one value.
+ dses = List.of(MetadataManager.INSTANCE.getDataset(mdTxnCtx,
dataverseName, datasetNameOrStar));
}
-
- IndexType sampleIndexType = IndexType.SAMPLE;
- Pair<String, String> sampleIndexNames =
IndexUtil.getSampleIndexNames(datasetName);
- String newIndexName;
- existingIndex =
MetadataManager.INSTANCE.getIndex(metadataProvider.getMetadataTxnContext(),
dataverseName,
- datasetName, sampleIndexNames.first);
- if (existingIndex != null) {
- newIndexName = sampleIndexNames.second;
- } else {
- existingIndex =
MetadataManager.INSTANCE.getIndex(metadataProvider.getMetadataTxnContext(),
- dataverseName, datasetName, sampleIndexNames.second);
- newIndexName = sampleIndexNames.first;
- }
-
- InternalDatasetDetails dsDetails = (InternalDatasetDetails)
ds.getDatasetDetails();
- int sampleCardinalityTarget = stmtAnalyze.getSampleSize();
- long sampleSeed = stmtAnalyze.getOrCreateSampleSeed();
-
- Index.SampleIndexDetails newIndexDetailsPendingAdd = new
Index.SampleIndexDetails(dsDetails.getPrimaryKey(),
- dsDetails.getKeySourceIndicator(),
dsDetails.getPrimaryKeyType(), sampleCardinalityTarget, 0, 0,
- sampleSeed, Collections.emptyMap());
- newIndexPendingAdd = new Index(dataverseName, datasetName,
newIndexName, sampleIndexType,
- newIndexDetailsPendingAdd, false, false,
MetadataUtil.PENDING_ADD_OP);
-
- // #. add a new index with PendingAddOp
-
MetadataManager.INSTANCE.addIndex(metadataProvider.getMetadataTxnContext(),
newIndexPendingAdd);
- // #. prepare to create the index artifact in NC.
- JobSpecification spec =
- IndexUtil.buildSecondaryIndexCreationJobSpec(ds,
newIndexPendingAdd, metadataProvider, sourceLoc);
- if (spec == null) {
- throw new CompilationException(ErrorCode.COMPILATION_ERROR,
sourceLoc,
- "Failed to create job spec for creating index '" +
ds.getDatasetName() + "."
- + newIndexPendingAdd.getIndexName() + "'");
+ //Add the pending index to metadata, and prepare the sample index
creation job
+ for (Dataset ds : dses) {
+ PreparedSampleIndexInfo info = createAndPrepareSampleIdx(ds,
dataverseName, metadataProvider, sourceLoc,
+ sampleCardinalityTarget, sampleSeed);
+ preparedIndexes.put(ds, info);
+ progressNewIndexCreate.put(ds,
ProgressState.ADDED_PENDINGOP_RECORD_TO_METADATA);
}
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
- bActiveTxn = false;
- progressNewIndexCreate =
ProgressState.ADDED_PENDINGOP_RECORD_TO_METADATA;
+ bActiveTxn.setFalse();
// #. create the index artifact in NC.
- runJob(hcc, spec);
+ for (PreparedSampleIndexInfo info : preparedIndexes.values()) {
+ runJob(hcc, info.sampleIdxToCreate);
+ }
- // #. flush dataset
- FlushDatasetUtil.flushDataset(hcc, metadataProvider,
dataverseName, datasetName);
+ // #. flush datasets
+ for (Dataset ds : dses) {
+ FlushDatasetUtil.flushDataset(hcc, metadataProvider,
dataverseName, ds.getDatasetName());
+ }
mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
- bActiveTxn = true;
+ bActiveTxn.setTrue();
metadataProvider.setMetadataTxnContext(mdTxnCtx);
-
- // #. load data into the index in NC.
- spec = IndexUtil.buildSecondaryIndexLoadingJobSpec(ds,
newIndexPendingAdd, metadataProvider, sourceLoc);
- MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
- bActiveTxn = false;
-
- List<IOperatorStats> opStats = runJob(hcc, spec, jobFlags,
-
Collections.singletonList(SampleOperationsHelper.DATASET_STATS_OPERATOR_NAME));
- if (opStats == null || opStats.size() == 0) {
- throw new
CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, "", sourceLoc);
+ for (Dataset ds : dses) {
+ // #. load data into the index in NC.
+ JobSpecification spec =
IndexUtil.buildSecondaryIndexLoadingJobSpec(ds,
+ preparedIndexes.get(ds).newIndexPendingAdd,
metadataProvider, sourceLoc);
+ sampleLoadSpecs.put(ds, spec);
}
- DatasetStreamStats stats = new DatasetStreamStats(opStats.get(0));
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+ bActiveTxn.setFalse();
- Index.SampleIndexDetails newIndexDetailsFinal = new
Index.SampleIndexDetails(dsDetails.getPrimaryKey(),
- dsDetails.getKeySourceIndicator(),
dsDetails.getPrimaryKeyType(), sampleCardinalityTarget,
- stats.getCardinality(), stats.getAvgTupleSize(),
sampleSeed, stats.getIndexesStats());
- Index newIndexFinal = new Index(dataverseName, datasetName,
newIndexName, sampleIndexType,
- newIndexDetailsFinal, false, false,
MetadataUtil.PENDING_NO_OP);
-
+ //load the indexes, and finalize the entries in metadata
+ for (Map.Entry<Dataset, JobSpecification> finalSpecs :
sampleLoadSpecs.entrySet()) {
+ Dataset ds = finalSpecs.getKey();
+ JobSpecification spec = finalSpecs.getValue();
+ Index finalizedIndex =
+ runLoadJobAndFinalizeIndex(ds, dataverseName,
preparedIndexes.get(ds).newIndexName, spec, hcc,
+ sourceLoc, sampleCardinalityTarget,
sampleSeed);
+ newFinalizedIndexes.put(ds, finalizedIndex);
+ }
// #. begin new metadataTxn
mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
- bActiveTxn = true;
+ bActiveTxn.setTrue();
metadataProvider.setMetadataTxnContext(mdTxnCtx);
// #. add same new index with PendingNoOp after deleting its entry
with PendingAddOp
-
MetadataManager.INSTANCE.dropIndex(metadataProvider.getMetadataTxnContext(),
- newIndexPendingAdd.getDataverseName(),
newIndexPendingAdd.getDatasetName(),
- newIndexPendingAdd.getIndexName());
-
MetadataManager.INSTANCE.addIndex(metadataProvider.getMetadataTxnContext(),
newIndexFinal);
- MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
- bActiveTxn = false;
- progressNewIndexCreate = ProgressState.NO_PROGRESS;
-
- if (existingIndex != null) {
- // #. set existing index to PendingDropOp because we'll be
dropping it next
- mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
- bActiveTxn = true;
- metadataProvider.setMetadataTxnContext(mdTxnCtx);
+ for (Map.Entry<Dataset, PreparedSampleIndexInfo> e :
preparedIndexes.entrySet()) {
+ Index newIndexPendingAdd = e.getValue().newIndexPendingAdd;
MetadataManager.INSTANCE.dropIndex(metadataProvider.getMetadataTxnContext(),
- existingIndex.getDataverseName(),
existingIndex.getDatasetName(), existingIndex.getIndexName());
- existingIndex.setPendingOp(MetadataUtil.PENDING_DROP_OP);
-
MetadataManager.INSTANCE.addIndex(metadataProvider.getMetadataTxnContext(),
existingIndex);
- existingIndexDropSpec =
IndexUtil.buildDropIndexJobSpec(existingIndex, metadataProvider, ds, sourceLoc);
- MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
- progressExistingIndexDrop =
ProgressState.ADDED_PENDINGOP_RECORD_TO_METADATA;
- bActiveTxn = false;
-
- // #. drop existing index on NCs
- runJob(hcc, existingIndexDropSpec);
-
- // #. drop existing index metadata
- mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
- bActiveTxn = true;
- metadataProvider.setMetadataTxnContext(mdTxnCtx);
-
MetadataManager.INSTANCE.dropIndex(metadataProvider.getMetadataTxnContext(),
- existingIndex.getDataverseName(),
existingIndex.getDatasetName(), existingIndex.getIndexName());
- MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
- bActiveTxn = false;
- progressExistingIndexDrop = ProgressState.NO_PROGRESS;
+ newIndexPendingAdd.getDataverseName(),
newIndexPendingAdd.getDatasetName(),
+ newIndexPendingAdd.getIndexName());
+
MetadataManager.INSTANCE.addIndex(metadataProvider.getMetadataTxnContext(),
+ newFinalizedIndexes.get(e.getKey()));
+ progressNewIndexCreate.put(e.getKey(),
ProgressState.NO_PROGRESS);
}
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+ bActiveTxn.setFalse();
+ //now, drop all the old samples
+ dropExistingSamples(preparedIndexes, existingIndexDropSpecs,
progressExistingIndexDrop, hcc,
+ metadataProvider, mdTxnCtx, bActiveTxn, sourceLoc);
} catch (Exception e) {
LOGGER.error("failed to analyze dataset; executing compensating
operations", e);
- if (bActiveTxn) {
+ //abort the open transaction, if there is one
+ if (bActiveTxn.booleanValue()) {
abort(e, e, mdTxnCtx);
}
-
- if (progressExistingIndexDrop ==
ProgressState.ADDED_PENDINGOP_RECORD_TO_METADATA) {
- // #. execute compensation operations remove the index in NCs
- try {
- runJob(hcc, existingIndexDropSpec);
- } catch (Exception e2) {
- // do no throw exception since still the metadata needs to
be compensated.
- e.addSuppressed(e2);
- }
- // #. remove the record from the metadata.
- mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
- metadataProvider.setMetadataTxnContext(mdTxnCtx);
- try {
-
MetadataManager.INSTANCE.dropIndex(metadataProvider.getMetadataTxnContext(),
- existingIndex.getDataverseName(),
existingIndex.getDatasetName(),
- existingIndex.getIndexName());
- MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
- } catch (Exception e2) {
- e.addSuppressed(e2);
- abort(e, e2, mdTxnCtx);
- throw new IllegalStateException("System is inconsistent
state: pending index("
- + existingIndex.getDataverseName() + "." +
existingIndex.getDatasetName() + "."
- + existingIndex.getIndexName() + ") couldn't be
removed from the metadata", e);
- }
- } else if (progressNewIndexCreate ==
ProgressState.ADDED_PENDINGOP_RECORD_TO_METADATA) {
- // #. execute compensation operations remove the index in NCs
- mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
- bActiveTxn = true;
- metadataProvider.setMetadataTxnContext(mdTxnCtx);
- try {
- JobSpecification jobSpec =
-
IndexUtil.buildDropIndexJobSpec(newIndexPendingAdd, metadataProvider, ds,
sourceLoc);
- MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
- bActiveTxn = false;
- runJob(hcc, jobSpec);
- } catch (Exception e2) {
- e.addSuppressed(e2);
- if (bActiveTxn) {
- abort(e, e2, mdTxnCtx);
- }
- }
- // #. remove the record from the metadata.
- mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
- metadataProvider.setMetadataTxnContext(mdTxnCtx);
- try {
-
MetadataManager.INSTANCE.dropIndex(metadataProvider.getMetadataTxnContext(),
- newIndexPendingAdd.getDataverseName(),
newIndexPendingAdd.getDatasetName(),
- newIndexPendingAdd.getIndexName());
- MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
- } catch (Exception e2) {
- e.addSuppressed(e2);
- abort(e, e2, mdTxnCtx);
- throw new IllegalStateException("System is in inconsistent
state: pending index("
- + newIndexPendingAdd.getDataverseName() + "." +
newIndexPendingAdd.getDatasetName() + "."
- + newIndexPendingAdd.getIndexName() + ") couldn't
be removed from the metadata", e);
+ //compensate all in-progress index drops
+ for (Map.Entry<Dataset, ProgressState> dropInFlight :
progressExistingIndexDrop.entrySet()) {
+ if (dropInFlight.getValue() ==
ProgressState.ADDED_PENDINGOP_RECORD_TO_METADATA) {
+ Dataset ds = dropInFlight.getKey();
+ abortPendingSampleDrop(ds,
preparedIndexes.get(ds).existingIndex, existingIndexDropSpecs.get(ds), e,
+ hcc, metadataProvider, mdTxnCtx);
}
}
-
+ //compensate all in-progress creates
+ for (Map.Entry<Dataset, ProgressState> createInFlight :
progressNewIndexCreate.entrySet()) {
+ if (createInFlight.getValue() ==
ProgressState.ADDED_PENDINGOP_RECORD_TO_METADATA) {
+ Index newIndexPendingAdd =
newIndexesPendingAdd.get(createInFlight.getKey());
+ abortPendingSampleCreate(createInFlight.getKey(),
newIndexPendingAdd, metadataProvider, mdTxnCtx,
+ bActiveTxn, hcc, sourceLoc, e);
+ }
+ }
throw e;
}
}
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/AnalyzingTestExecutor.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/AnalyzingTestExecutor.java
index 2658804..0618da2 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/AnalyzingTestExecutor.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/AnalyzingTestExecutor.java
@@ -32,9 +32,7 @@
public class AnalyzingTestExecutor extends TestExecutor {
- private Pattern loadPattern =
Pattern.compile("(load)\\s+(dataset|collection)\\s+([a-zA-z0-9\\.`]+)\\s+using",
- Pattern.MULTILINE | Pattern.CASE_INSENSITIVE);
- private Pattern upsertPattern =
Pattern.compile("^(upsert|insert)\\s+into\\s+([a-zA-z0-9\\.`]+)\\s*(\\(|as)?",
+ private Pattern connectFeedPattern =
Pattern.compile("(connect|start)\\s+(feed)\\s+([a-zA-z0-9\\.`]+)",
Pattern.MULTILINE | Pattern.CASE_INSENSITIVE);
private Pattern usePattern =
Pattern.compile("use\\s+(dataverse\\s+)?([a-zA-z0-9\\.`]+)\\s*;",
Pattern.MULTILINE | Pattern.CASE_INSENSITIVE);
@@ -49,35 +47,31 @@
Matcher dvMatcher = usePattern.matcher(statement);
String dv = "";
if (dvMatcher.find()) {
- dv = dvMatcher.group(2) + ".";
+ dv = dvMatcher.group(2);
}
- Matcher dsMatcher = loadPattern.matcher(statement);
- Matcher upsertMatcher = upsertPattern.matcher(statement);
ExtractedResult res = super.executeUpdateOrDdl(statement,
outputFormat, getQueryServiceUri(SQLPP));
- analyzeFromRegex(dsMatcher, dv, 3);
- analyzeFromRegex(upsertMatcher, dv, 2);
+ Matcher connectFeedMatcher = connectFeedPattern.matcher(statement);
+ if (!connectFeedMatcher.find()) {
+ analyzeEverything(dv);
+ }
return res;
}
- private void analyzeFromRegex(Matcher m, String dv, int pos) throws
Exception {
- while (m.find()) {
- String ds = m.group(pos);
- StringBuilder analyzeStmt = new StringBuilder();
- analyzeStmt.append("ANALYZE DATASET ");
- if (!ds.contains(".")) {
- analyzeStmt.append(dv);
- }
- analyzeStmt.append(ds);
- analyzeStmt.append(" WITH {\"sample-seed\": \"1000\"}");
- analyzeStmt.append(";");
- InputStream resultStream =
executeQueryService(analyzeStmt.toString(), getQueryServiceUri(SQLPP),
- TestCaseContext.OutputFormat.CLEAN_JSON);
- String resultStr = IOUtils.toString(resultStream, UTF_8);
- JsonNode result = RESULT_NODE_READER.<ObjectNode>
readValue(resultStr).get("status");
- if (!"success".equals(result.asText())) {
- JsonNode error = RESULT_NODE_READER.<ObjectNode>
readValue(resultStr).get("errors");
- throw new IllegalStateException("ANALYZE DATASET failed with
error: " + error);
- }
+ private void analyzeEverything(String dv) throws Exception {
+ StringBuilder analyzeStmt = new StringBuilder();
+ if (!"".equals(dv)) {
+ analyzeStmt.append("USE " + dv + ";");
+ }
+ analyzeStmt.append("ANALYZE DATASET *");
+ analyzeStmt.append(" WITH {\"sample-seed\": \"1000\"}");
+ analyzeStmt.append(";");
+ InputStream resultStream = executeQueryService(analyzeStmt.toString(),
getQueryServiceUri(SQLPP),
+ TestCaseContext.OutputFormat.CLEAN_JSON);
+ String resultStr = IOUtils.toString(resultStream, UTF_8);
+ JsonNode result = RESULT_NODE_READER.<ObjectNode>
readValue(resultStr).get("status");
+ if (!"success".equals(result.asText())) {
+ JsonNode error = RESULT_NODE_READER.<ObjectNode>
readValue(resultStr).get("errors");
+ throw new IllegalStateException("ANALYZE DATASET failed with
error: " + error);
}
}
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppAnalyzedExecutionTest.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppAnalyzedExecutionTest.java
index 10a22b4..be97bcb 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppAnalyzedExecutionTest.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppAnalyzedExecutionTest.java
@@ -44,7 +44,7 @@
public class SqlppAnalyzedExecutionTest {
protected static final String TEST_CONFIG_FILE_NAME =
"src/test/resources/cc-analyze.conf";
private final String[] denyList = { "synonym: synonym-01", "ddl:
analyze-dataset-1", "misc: dump_index",
- "array-index: composite-index-queries", "filters: upsert",
"column: analyze-dataset",
+ "array-index: composite-index-queries", "filters: upsert",
"filters: delete", "column: analyze-dataset",
"ddl: analyze-dataset-with-indexes", "warnings:
cardinality-hint-warning" };
@BeforeClass
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/fulltext/stopwords-full-text-filter-1/stopwords-full-text-filter-1.7.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/fulltext/stopwords-full-text-filter-1/stopwords-full-text-filter-1.7.adm
index 16635ef..38b9409 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/fulltext/stopwords-full-text-filter-1/stopwords-full-text-filter-1.7.adm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/fulltext/stopwords-full-text-filter-1/stopwords-full-text-filter-1.7.adm
@@ -2,4 +2,4 @@
{ "DataverseName": "MyDataVerse", "DatasetName": "MyMessageDataset",
"IndexName": "message_ft_index_0", "IndexStructure":
"SINGLE_PARTITION_WORD_INVIX", "SearchKey": [ [ "myMessageBody" ] ],
"IsPrimary": false, "Timestamp": "Wed Mar 08 15:49:09 PST 2023", "PendingOp": 0
}
{ "DataverseName": "MyDataVerse", "DatasetName": "MyMessageDataset",
"IndexName": "message_ft_index_1", "IndexStructure":
"SINGLE_PARTITION_WORD_INVIX", "SearchKey": [ [ "myMessageBody" ] ],
"IsPrimary": false, "Timestamp": "Wed Mar 08 15:49:10 PST 2023", "PendingOp":
0, "FullTextConfig": "my_first_stopword_config" }
{ "DataverseName": "MyDataVerse", "DatasetName": "MyMessageDataset",
"IndexName": "message_ft_index_2", "IndexStructure":
"SINGLE_PARTITION_WORD_INVIX", "SearchKey": [ [ "myMessageBody" ] ],
"IsPrimary": false, "Timestamp": "Wed Mar 08 15:49:10 PST 2023", "PendingOp":
0, "FullTextConfig": "my_second_stopword_config" }
-{ "DataverseName": "MyDataVerse", "DatasetName": "MyMessageDataset",
"IndexName": "sample_idx_2_MyMessageDataset", "IndexStructure": "SAMPLE",
"SearchKey": [ [ "myMessageId" ] ], "IsPrimary": false, "Timestamp": "Wed Mar
08 15:49:09 PST 2023", "PendingOp": 0, "SampleCardinalityTarget": 1063,
"SourceCardinality": 2, "SourceAvgItemSize": 62 }
+{ "DataverseName": "MyDataVerse", "DatasetName": "MyMessageDataset",
"IndexName": "sample_idx_1_MyMessageDataset", "IndexStructure": "SAMPLE",
"SearchKey": [ [ "myMessageId" ] ], "IsPrimary": false, "Timestamp": "Wed Mar
08 15:49:09 PST 2023", "PendingOp": 0, "SampleCardinalityTarget": 1063,
"SourceCardinality": 2, "SourceAvgItemSize": 62 }
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
index 9e3dea4..5fcfb64 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
+++ b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
@@ -2802,7 +2802,7 @@
Pair<DataverseName,Identifier> nameComponents = null;
}
{
- <ANALYZE> { startToken = token; } DatasetToken() nameComponents =
QualifiedName()
+ <ANALYZE> { startToken = token; } DatasetToken() (nameComponents =
QualifiedName() | <MUL> { nameComponents = new Pair(null,new Identifier("*"));
})
(
stmt = AnalyzeDatasetDropStatement(startToken, nameComponents.first,
nameComponents.second)
| stmt = AnalyzeDatasetStatement(startToken, nameComponents.first,
nameComponents.second)
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataLockUtil.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataLockUtil.java
index 8ee8f11..4156777 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataLockUtil.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataLockUtil.java
@@ -313,8 +313,12 @@
@Override
public void analyzeDatasetBegin(IMetadataLockManager lockMgr, LockList
locks, DataverseName dataverseName,
String datasetName) throws AlgebricksException {
- lockMgr.acquireDataverseReadLock(locks, dataverseName);
- lockMgr.acquireDatasetCreateIndexLock(locks, dataverseName,
datasetName);
+ if ("*".equals(datasetName)) {
+ lockMgr.acquireDataverseWriteLock(locks, dataverseName);
+ } else {
+ lockMgr.acquireDataverseReadLock(locks, dataverseName);
+ lockMgr.acquireDatasetCreateIndexLock(locks, dataverseName,
datasetName);
+ }
}
@Override
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17601
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: Ib5e1b2efefe38332ea1f8f7328c63ace12ac9ceb
Gerrit-Change-Number: 17601
Gerrit-PatchSet: 1
Gerrit-Owner: Ian Maxon <[email protected]>
Gerrit-MessageType: newchange
