This is an automated email from the ASF dual-hosted git repository.
dlych pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit fae6e3b7ae81bb06818ef4e909db1e7604d54edb
Author: Ali Alsuliman <ali.al.solai...@gmail.com>
AuthorDate: Tue Dec 15 11:13:24 2020 -0800
[NO ISSUE] Force flag for dataset drop & query service servlet refactoring
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
Expose the force flag for dataset drop to extensions.
Change-Id: I0cf4afa65752e3faa4351d51098f9f78412ba88c
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/9325
Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu>
Reviewed-by: Ali Alsuliman <ali.al.solai...@gmail.com>
Reviewed-by: Dmitry Lychagin <dmitry.lycha...@couchbase.com>
---
.../apache/asterix/translator/IRequestParameters.java | 6 ++++++
.../asterix/api/http/server/QueryServiceServlet.java | 19 ++++++++++++++-----
.../asterix/app/translator/QueryTranslator.java | 13 ++++++++-----
.../asterix/app/translator/RequestParameters.java | 17 +++++++++++++++++
.../java/org/apache/asterix/utils/RebalanceUtil.java | 2 +-
.../org/apache/asterix/metadata/MetadataManager.java | 6 +++---
.../org/apache/asterix/metadata/MetadataNode.java | 4 ----
.../apache/asterix/metadata/api/IMetadataManager.java | 4 +++-
.../apache/asterix/metadata/api/IMetadataNode.java | 4 +++-
.../asterix/metadata/bootstrap/MetadataBootstrap.java | 2 +-
.../org/apache/asterix/metadata/entities/Dataset.java | 8 ++++----
11 files changed, 60 insertions(+), 25 deletions(-)
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IRequestParameters.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IRequestParameters.java
index 417a130..4ad1040 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IRequestParameters.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IRequestParameters.java
@@ -67,4 +67,10 @@ public interface IRequestParameters extends
ICommonRequestParameters {
* {@code 0} if all categories are allowed
*/
int getStatementCategoryRestrictionMask();
+
+ /**
+ * @return true if DROP DATASET statements in the request should force
drop datasets which could make the metadata
+ * inconsistent.
+ */
+ boolean isForceDropDataset();
}
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
index d43e611..6de91d8 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
@@ -90,6 +90,7 @@ import org.apache.hyracks.api.application.IServiceContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.HyracksException;
import org.apache.hyracks.api.exceptions.Warning;
+import org.apache.hyracks.api.result.IResultSet;
import org.apache.hyracks.control.common.controllers.CCConfig;
import org.apache.hyracks.http.api.IServletRequest;
import org.apache.hyracks.http.api.IServletResponse;
@@ -269,7 +270,7 @@ public class QueryServiceServlet extends
AbstractQueryApiServlet {
SessionOutput sessionOutput = createSessionOutput(httpWriter);
ResponsePrinter responsePrinter = new ResponsePrinter(sessionOutput);
ResultDelivery delivery = ResultDelivery.IMMEDIATE;
- QueryServiceRequestParameters param = newRequestParameters();
+ QueryServiceRequestParameters param = newQueryRequestParameters();
RequestExecutionState executionState = newRequestExecutionState();
try {
// buffer the output until we are ready to set the status of the
response message correctly
@@ -411,9 +412,8 @@ public class QueryServiceServlet extends
AbstractQueryApiServlet {
int stmtCategoryRestriction =
org.apache.asterix.app.translator.RequestParameters
.getStatementCategoryRestrictionMask(param.isReadOnly());
IRequestParameters requestParameters =
- new
org.apache.asterix.app.translator.RequestParameters(requestReference,
statementsText,
- getResultSet(), resultProperties, stats,
statementProperties, null, param.getClientContextID(),
- optionalParameters, stmtParams,
param.isMultiStatement(), stmtCategoryRestriction);
+ newRequestParameters(param, requestReference, statementsText,
getResultSet(), resultProperties, stats,
+ statementProperties, optionalParameters, stmtParams,
stmtCategoryRestriction);
translator.compileAndExecute(getHyracksClientConnection(),
requestParameters);
executionState.end();
translator.getWarnings(warnings, maxWarnings - warnings.size());
@@ -492,10 +492,19 @@ public class QueryServiceServlet extends
AbstractQueryApiServlet {
responsePrinter.addResultPrinter(new
ErrorsPrinter(Collections.singletonList(executionError)));
}
- protected QueryServiceRequestParameters newRequestParameters() {
+ protected QueryServiceRequestParameters newQueryRequestParameters() {
return new QueryServiceRequestParameters();
}
+ protected IRequestParameters
newRequestParameters(QueryServiceRequestParameters param,
+ IRequestReference requestReference, String statementsText,
IResultSet resultSet,
+ ResultProperties resultProperties, Stats stats,
IStatementExecutor.StatementProperties statementProperties,
+ Map<String, String> optionalParameters, Map<String, IAObject>
stmtParams, int stmtCategoryRestriction) {
+ return new RequestParameters(requestReference, statementsText,
resultSet, resultProperties, stats,
+ statementProperties, null, param.getClientContextID(),
optionalParameters, stmtParams,
+ param.isMultiStatement(), stmtCategoryRestriction);
+ }
+
protected static boolean isPrintingProfile(IStatementExecutor.Stats stats)
{
return stats.getProfileType() == Stats.ProfileType.FULL &&
stats.getJobProfile() != null;
}
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 2ba5c0e..c000d8b 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -857,7 +857,8 @@ public class QueryTranslator extends AbstractLangTranslator
implements IStatemen
// #. add a new dataset with PendingNoOp after deleting the
dataset with
// PendingAddOp
-
MetadataManager.INSTANCE.dropDataset(metadataProvider.getMetadataTxnContext(),
dataverseName, datasetName);
+
MetadataManager.INSTANCE.dropDataset(metadataProvider.getMetadataTxnContext(),
dataverseName, datasetName,
+ requestParameters.isForceDropDataset());
dataset.setPendingOp(MetadataUtil.PENDING_NO_OP);
MetadataManager.INSTANCE.addDataset(metadataProvider.getMetadataTxnContext(),
dataset);
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
@@ -894,7 +895,8 @@ public class QueryTranslator extends AbstractLangTranslator
implements IStatemen
mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
metadataProvider.setMetadataTxnContext(mdTxnCtx);
try {
- MetadataManager.INSTANCE.dropDataset(mdTxnCtx,
dataverseName, datasetName);
+ MetadataManager.INSTANCE.dropDataset(mdTxnCtx,
dataverseName, datasetName,
+ requestParameters.isForceDropDataset());
if (itemTypeAdded) {
MetadataManager.INSTANCE.dropDatatype(mdTxnCtx,
itemTypeEntity.getDataverseName(),
itemTypeEntity.getDatatypeName());
@@ -1692,7 +1694,7 @@ public class QueryTranslator extends
AbstractLangTranslator implements IStatemen
validateDatasetState(metadataProvider, ds, sourceLoc);
ds.drop(metadataProvider, mdTxnCtx, jobsToExecute, bActiveTxn,
progress, hcc, dropCorrespondingNodeGroup,
- sourceLoc, Collections.emptySet());
+ sourceLoc, Collections.emptySet(),
requestParameters.isForceDropDataset());
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx.getValue());
return true;
@@ -1709,7 +1711,8 @@ public class QueryTranslator extends
AbstractLangTranslator implements IStatemen
if (ds != null) {
jobsToExecute.clear();
ds.drop(metadataProvider, mdTxnCtx, jobsToExecute,
bActiveTxn, progress, hcc,
- dropCorrespondingNodeGroup, sourceLoc,
EnumSet.of(DropOption.IF_EXISTS));
+ dropCorrespondingNodeGroup, sourceLoc,
EnumSet.of(DropOption.IF_EXISTS),
+ requestParameters.isForceDropDataset());
}
for (JobSpecification jobSpec : jobsToExecute) {
JobUtils.runJob(hcc, jobSpec, true);
@@ -1724,7 +1727,7 @@ public class QueryTranslator extends
AbstractLangTranslator implements IStatemen
metadataProvider.setMetadataTxnContext(mdTxnCtx.getValue());
try {
MetadataManager.INSTANCE.dropDataset(metadataProvider.getMetadataTxnContext(),
dataverseName,
- datasetName);
+ datasetName,
requestParameters.isForceDropDataset());
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx.getValue());
} catch (Exception e2) {
e.addSuppressed(e2);
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/RequestParameters.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/RequestParameters.java
index 5ebc9ba..d5ea685 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/RequestParameters.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/RequestParameters.java
@@ -58,6 +58,7 @@ public class RequestParameters implements IRequestParameters {
private final boolean multiStatement;
private final int statementCategoryRestrictionMask;
private final String statement;
+ private final boolean forceDropDataset;
public RequestParameters(IRequestReference requestReference, String
statement, IResultSet resultSet,
ResultProperties resultProperties, Stats stats,
StatementProperties statementProperties,
@@ -72,6 +73,16 @@ public class RequestParameters implements IRequestParameters
{
IStatementExecutor.ResultMetadata outMetadata, String
clientContextId,
Map<String, String> optionalParameters, Map<String, IAObject>
statementParameters, boolean multiStatement,
int statementCategoryRestrictionMask) {
+ this(requestReference, statement, resultSet, resultProperties, stats,
statementProperties, outMetadata,
+ clientContextId, optionalParameters, statementParameters,
multiStatement,
+ statementCategoryRestrictionMask, false);
+ }
+
+ public RequestParameters(IRequestReference requestReference, String
statement, IResultSet resultSet,
+ ResultProperties resultProperties, Stats stats,
StatementProperties statementProperties,
+ IStatementExecutor.ResultMetadata outMetadata, String
clientContextId,
+ Map<String, String> optionalParameters, Map<String, IAObject>
statementParameters, boolean multiStatement,
+ int statementCategoryRestrictionMask, boolean forceDropDataset) {
this.requestReference = requestReference;
this.statement = statement;
this.resultSet = resultSet;
@@ -84,6 +95,7 @@ public class RequestParameters implements IRequestParameters {
this.statementParameters = statementParameters;
this.multiStatement = multiStatement;
this.statementCategoryRestrictionMask =
statementCategoryRestrictionMask;
+ this.forceDropDataset = forceDropDataset;
}
@Override
@@ -127,6 +139,11 @@ public class RequestParameters implements
IRequestParameters {
}
@Override
+ public boolean isForceDropDataset() {
+ return forceDropDataset;
+ }
+
+ @Override
public int getStatementCategoryRestrictionMask() {
return statementCategoryRestrictionMask;
}
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RebalanceUtil.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RebalanceUtil.java
index d32bfb2..7ad90ad 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RebalanceUtil.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RebalanceUtil.java
@@ -419,7 +419,7 @@ public class RebalanceUtil {
// drop dataset entry from metadata
runMetadataTransaction(metadataProvider,
() ->
MetadataManager.INSTANCE.dropDataset(metadataProvider.getMetadataTxnContext(),
- dataset.getDataverseName(),
dataset.getDatasetName()));
+ dataset.getDataverseName(),
dataset.getDatasetName(), false));
MetadataManager.INSTANCE.commitTransaction(metadataProvider.getMetadataTxnContext());
// try to drop the dataset's node group
runMetadataTransaction(metadataProvider, () ->
tryDropDatasetNodegroup(dataset, metadataProvider));
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java
index 268a9c0..a481f99 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java
@@ -260,10 +260,10 @@ public abstract class MetadataManager implements
IMetadataManager {
}
@Override
- public void dropDataset(MetadataTransactionContext ctx, DataverseName
dataverseName, String datasetName)
- throws AlgebricksException {
+ public void dropDataset(MetadataTransactionContext ctx, DataverseName
dataverseName, String datasetName,
+ boolean force) throws AlgebricksException {
try {
- metadataNode.dropDataset(ctx.getTxnId(), dataverseName,
datasetName);
+ metadataNode.dropDataset(ctx.getTxnId(), dataverseName,
datasetName, force);
} catch (RemoteException e) {
throw new
MetadataException(ErrorCode.REMOTE_EXCEPTION_WHEN_CALLING_METADATA_NODE, e);
}
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
index 88098f8..c80f7ad 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
@@ -605,10 +605,6 @@ public class MetadataNode implements IMetadataNode {
}
@Override
- public void dropDataset(TxnId txnId, DataverseName dataverseName, String
datasetName) throws AlgebricksException {
- dropDataset(txnId, dataverseName, datasetName, false);
- }
-
public void dropDataset(TxnId txnId, DataverseName dataverseName, String
datasetName, boolean force)
throws AlgebricksException {
if (!force) {
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java
index 1646a93..139216e 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java
@@ -212,10 +212,12 @@ public interface IMetadataManager extends
IMetadataBootstrap {
* Name of dataverse which holds the given dataset.
* @param datasetName
* Name of dataset to delete.
+ * @param force
+ * If true, forces drop the dataset. Setting it to true could
make the metadata inconsistent.
* @throws AlgebricksException
* For example, if the dataset and/or dataverse does not exist.
*/
- void dropDataset(MetadataTransactionContext ctx, DataverseName
dataverseName, String datasetName)
+ void dropDataset(MetadataTransactionContext ctx, DataverseName
dataverseName, String datasetName, boolean force)
throws AlgebricksException;
/**
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java
index bca4171..08ed0a3 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java
@@ -217,11 +217,13 @@ public interface IMetadataNode extends Remote,
Serializable {
* Name of dataverse which holds the given dataset.
* @param datasetName
* Name of dataset to delete.
+ * @param force
+ * If true, forces drop the dataset. Setting it to true could
make the metadata inconsistent.
* @throws AlgebricksException
* For example, if the dataset and/or dataverse does not exist.
* @throws RemoteException
*/
- void dropDataset(TxnId txnId, DataverseName dataverseName, String
datasetName)
+ void dropDataset(TxnId txnId, DataverseName dataverseName, String
datasetName, boolean force)
throws AlgebricksException, RemoteException;
/**
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
index 2aca994..8cfd146 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
@@ -471,7 +471,7 @@ public class MetadataBootstrap {
throws AlgebricksException {
if (dataset.getPendingOp() != MetadataUtil.PENDING_NO_OP) {
// drop pending dataset
- MetadataManager.INSTANCE.dropDataset(mdTxnCtx,
dataset.getDataverseName(), dataset.getDatasetName());
+ MetadataManager.INSTANCE.dropDataset(mdTxnCtx,
dataset.getDataverseName(), dataset.getDatasetName(), true);
if (LOGGER.isInfoEnabled()) {
LOGGER.info(
"Dropped a pending dataset: " +
dataset.getDataverseName() + "." + dataset.getDatasetName());
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
index 4e556a2..dd3f67d 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
@@ -347,7 +347,7 @@ public class Dataset implements IMetadataEntity<Dataset>,
IDataset {
public void drop(MetadataProvider metadataProvider,
MutableObject<MetadataTransactionContext> mdTxnCtx,
List<JobSpecification> jobsToExecute, MutableBoolean bActiveTxn,
MutableObject<ProgressState> progress,
IHyracksClientConnection hcc, boolean dropCorrespondingNodeGroup,
SourceLocation sourceLoc,
- Set<DropOption> options) throws Exception {
+ Set<DropOption> options, boolean force) throws Exception {
Map<FeedConnectionId, Pair<JobSpecification, Boolean>>
disconnectJobList = new HashMap<>();
if (getDatasetType() == DatasetType.INTERNAL) {
// #. prepare jobs to drop the datatset and the indexes in NC
@@ -361,7 +361,7 @@ public class Dataset implements IMetadataEntity<Dataset>,
IDataset {
}
jobsToExecute.add(DatasetUtil.dropDatasetJobSpec(this,
metadataProvider, options));
// #. mark the existing dataset as PendingDropOp
- MetadataManager.INSTANCE.dropDataset(mdTxnCtx.getValue(),
dataverseName, datasetName);
+ MetadataManager.INSTANCE.dropDataset(mdTxnCtx.getValue(),
dataverseName, datasetName, force);
MetadataManager.INSTANCE.addDataset(mdTxnCtx.getValue(),
new Dataset(dataverseName, datasetName,
getItemTypeDataverseName(), getItemTypeName(),
getMetaItemTypeDataverseName(),
getMetaItemTypeName(), getNodeGroupName(),
@@ -401,7 +401,7 @@ public class Dataset implements IMetadataEntity<Dataset>,
IDataset {
}
// #. mark the existing dataset as PendingDropOp
- MetadataManager.INSTANCE.dropDataset(mdTxnCtx.getValue(),
dataverseName, datasetName);
+ MetadataManager.INSTANCE.dropDataset(mdTxnCtx.getValue(),
dataverseName, datasetName, force);
MetadataManager.INSTANCE.addDataset(mdTxnCtx.getValue(),
new Dataset(dataverseName, datasetName,
getItemTypeDataverseName(), getItemTypeName(),
getNodeGroupName(), getCompactionPolicy(),
getCompactionPolicyProperties(),
@@ -425,7 +425,7 @@ public class Dataset implements IMetadataEntity<Dataset>,
IDataset {
}
// #. finally, delete the dataset.
- MetadataManager.INSTANCE.dropDataset(mdTxnCtx.getValue(),
dataverseName, datasetName);
+ MetadataManager.INSTANCE.dropDataset(mdTxnCtx.getValue(),
dataverseName, datasetName, force);
// drop inline types
if (TypeUtil.isDatasetInlineTypeName(this, recordTypeDataverseName,
recordTypeName)) {
