Murtadha Hubail has submitted this change and it was merged. Change subject: [NO ISSUE][TX] Maintain Txn Accessed Datasets ......................................................................
[NO ISSUE][TX] Maintain Txn Accessed Datasets - user model changes: no - storage format changes: no - interface changes: no Details: - For every transaction, maintain a list of datasets the transaction is attempting to access. Change-Id: I54cb04ac313d73981dd43d575875fa003fcd3bb9 Reviewed-on: https://asterix-gerrit.ics.uci.edu/3180 Sonar-Qube: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Contrib: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> Reviewed-by: Michael Blow <[email protected]> Reviewed-by: Till Westmann <[email protected]> --- M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/sqlpp/ParserTestExecutor.java M asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/VariableCheckAndRewriteVisitor.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java 3 files changed, 44 insertions(+), 11 deletions(-) Approvals: Anon. E. Moose #1000171: Till Westmann: Looks good to me, but someone else must approve Jenkins: Verified; No violations found; ; Verified Michael Blow: Looks good to me, approved diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/sqlpp/ParserTestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/sqlpp/ParserTestExecutor.java index e4e300a..e929ef0 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/sqlpp/ParserTestExecutor.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/sqlpp/ParserTestExecutor.java @@ -55,6 +55,8 @@ import org.apache.asterix.testframework.xml.TestCase.CompilationUnit; import org.apache.asterix.testframework.xml.TestGroup; import org.junit.Assert; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; import junit.extensions.PA; @@ -131,7 +133,16 @@ when(metadataProvider.getDefaultDataverseName()).thenReturn(dvName); when(metadataProvider.getConfig()).thenReturn(config); when(config.get(FunctionUtil.IMPORT_PRIVATE_FUNCTIONS)).thenReturn("true"); - when(metadataProvider.findDataset(anyString(), anyString())).thenReturn(mock(Dataset.class)); + when(metadataProvider.findDataset(anyString(), anyString())).thenAnswer(new Answer<Dataset>() { + @Override + public Dataset answer(InvocationOnMock invocation) { + Object[] args = invocation.getArguments(); + final Dataset mockDataset = mock(Dataset.class); + String fullyQualifiedName = args[0] != null ? args[0] + "." + args[1] : (String) args[1]; + when(mockDataset.getFullyQualifiedName()).thenReturn(fullyQualifiedName); + return mockDataset; + } + }); for (Statement st : statements) { if (st.getKind() == Statement.Kind.QUERY) { diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/VariableCheckAndRewriteVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/VariableCheckAndRewriteVisitor.java index 50b13e2..45a18e8 100644 --- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/VariableCheckAndRewriteVisitor.java +++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/VariableCheckAndRewriteVisitor.java @@ -44,6 +44,7 @@ import org.apache.asterix.lang.sqlpp.visitor.CheckDatasetOnlyResolutionVisitor; import org.apache.asterix.lang.sqlpp.visitor.base.AbstractSqlppExpressionScopingVisitor; import org.apache.asterix.metadata.declared.MetadataProvider; +import org.apache.asterix.metadata.entities.Dataset; import org.apache.asterix.om.functions.BuiltinFunctions; import org.apache.commons.lang3.StringUtils; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; @@ -146,12 +147,13 @@ private Expression resolveAsDataset(String dataverseName, String datasetName, SourceLocation sourceLoc) throws CompilationException { - if (!datasetExists(dataverseName, datasetName, sourceLoc)) { + Dataset dataset = findDataset(dataverseName, datasetName, sourceLoc); + if (dataset == null) { throwUnresolvableError(dataverseName, datasetName, sourceLoc); } - String fullyQualifiedName = dataverseName == null ? datasetName : dataverseName + "." + datasetName; + metadataProvider.addAccessedDataset(dataset); List<Expression> argList = new ArrayList<>(1); - argList.add(new LiteralExpr(new StringLiteral(fullyQualifiedName))); + argList.add(new LiteralExpr(new StringLiteral(dataset.getFullyQualifiedName()))); CallExpr callExpr = new CallExpr(new FunctionSignature(BuiltinFunctions.DATASET), argList); callExpr.setSourceLocation(sourceLoc); return callExpr; @@ -186,22 +188,28 @@ return parent.accept(visitor, originalExpressionWithUndefinedIdentifier); } - private boolean datasetExists(String dataverseName, String datasetName, SourceLocation sourceLoc) + private Dataset findDataset(String dataverseName, String datasetName, SourceLocation sourceLoc) throws CompilationException { try { - return metadataProvider.findDataset(dataverseName, datasetName) != null - || fullyQualifiedDatasetNameExists(datasetName); + Dataset dataset = metadataProvider.findDataset(dataverseName, datasetName); + if (dataset != null) { + return dataset; + } + return findDatasetByFullyQualifiedName(datasetName); } catch (AlgebricksException e) { throw new CompilationException(ErrorCode.COMPILATION_ERROR, e, sourceLoc, e.getMessage()); } } - private boolean fullyQualifiedDatasetNameExists(String name) throws AlgebricksException { + private Dataset findDatasetByFullyQualifiedName(String name) throws AlgebricksException { if (name.indexOf('.') < 0) { - return false; + return null; } String[] path = StringUtils.split(name, '.'); - return path.length == 2 && metadataProvider.findDataset(path[0], path[1]) != null; + if (path.length != 2) { + return null; + } + return metadataProvider.findDataset(path[0], path[1]); } @Override diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java index 85beb95..adfaa89 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java @@ -21,10 +21,13 @@ import java.io.File; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.asterix.common.cluster.IClusterStateManager; import org.apache.asterix.common.config.DatasetConfig.DatasetType; @@ -124,9 +127,9 @@ import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer; import org.apache.hyracks.api.dataflow.value.ITypeTraits; import org.apache.hyracks.api.dataflow.value.RecordDescriptor; -import org.apache.hyracks.api.result.ResultSetId; import org.apache.hyracks.api.io.FileSplit; import org.apache.hyracks.api.job.JobSpecification; +import org.apache.hyracks.api.result.ResultSetId; import org.apache.hyracks.data.std.primitive.ShortPointable; import org.apache.hyracks.dataflow.common.data.marshalling.ShortSerializerDeserializer; import org.apache.hyracks.dataflow.std.file.IFileSplitProvider; @@ -151,6 +154,7 @@ private final IFunctionManager functionManager; private final LockList locks; private final Map<String, Object> config; + private final Set<Dataset> txnAccessedDatasets; private Dataverse defaultDataverse; private MetadataTransactionContext mdTxnCtx; @@ -173,6 +177,7 @@ functionManager = ((IFunctionExtensionManager) appCtx.getExtensionManager()).getFunctionManager(); locks = new LockList(); config = new HashMap<>(); + txnAccessedDatasets = new HashSet<>(); } @SuppressWarnings("unchecked") @@ -233,6 +238,7 @@ public void setMetadataTxnContext(MetadataTransactionContext mdTxnCtx) { this.mdTxnCtx = mdTxnCtx; + txnAccessedDatasets.clear(); } public MetadataTransactionContext getMetadataTxnContext() { @@ -1621,4 +1627,12 @@ public ICompressionManager getCompressionManager() { return appCtx.getCompressionManager(); } + + public void addAccessedDataset(Dataset dataset) { + txnAccessedDatasets.add(dataset); + } + + public Set<Dataset> getAccssedDatasets() { + return Collections.unmodifiableSet(txnAccessedDatasets); + } } -- To view, visit https://asterix-gerrit.ics.uci.edu/3180 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: I54cb04ac313d73981dd43d575875fa003fcd3bb9 Gerrit-PatchSet: 4 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Murtadha Hubail <[email protected]> Gerrit-Reviewer: Ali Alsuliman <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Dmitry Lychagin <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Michael Blow <[email protected]> Gerrit-Reviewer: Murtadha Hubail <[email protected]> Gerrit-Reviewer: Till Westmann <[email protected]>
