>From Ian Maxon <[email protected]>: Ian Maxon has submitted this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17962 )
Change subject: [ASTERIXDB-3313][COMP] Defuse profile when not profiling ...................................................................... [ASTERIXDB-3313][COMP] Defuse profile when not profiling - user model changes: no - storage format changes: no - interface changes: no Details: The logical2physical map is time consuming to produce, so it should not be computed unless it's needed for a profile. Likewise keeping the last plan in memory is wasteful and useless unless a profile is needed. Lastly in the pipeline assembler, look at the profile flag first instead of the instanceof check to save time. Change-Id: Ia9a84eedbcb33c29f03155a8605bb82af372f1f3 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17962 Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Ian Maxon <[email protected]> Reviewed-by: Ali Alsuliman <[email protected]> --- M hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobBuilder.java M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/PlanCompiler.java M hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/ICompiler.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java M hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/operators/meta/PipelineAssembler.java 7 files changed, 76 insertions(+), 8 deletions(-) Approvals: Ian Maxon: Looks good to me, but someone else must approve Ali Alsuliman: Looks good to me, approved Jenkins: Verified; Verified Anon. E. Moose #1000171: diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java index 3c0b01d..0cc2790 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java @@ -25,6 +25,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.EnumSet; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -107,6 +108,7 @@ import org.apache.hyracks.api.exceptions.HyracksException; import org.apache.hyracks.api.exceptions.IWarningCollector; import org.apache.hyracks.api.exceptions.SourceLocation; +import org.apache.hyracks.api.job.JobFlag; import org.apache.hyracks.api.job.JobId; import org.apache.hyracks.api.job.JobSpecification; import org.apache.hyracks.api.job.resource.IClusterCapacity; @@ -204,7 +206,7 @@ public JobSpecification compileQuery(IClusterInfoCollector clusterInfoCollector, MetadataProvider metadataProvider, Query query, int varCounter, String outputDatasetName, SessionOutput output, ICompiledDmlStatement statement, Map<VarIdentifier, IAObject> externalVars, IResponsePrinter printer, - IWarningCollector warningCollector, IRequestParameters requestParameters) + IWarningCollector warningCollector, IRequestParameters requestParameters, EnumSet<JobFlag> runtimeFlags) throws AlgebricksException, ACIDException { // establish facts @@ -325,7 +327,7 @@ JobEventListenerFactory jobEventListenerFactory = new JobEventListenerFactory(txnId, metadataProvider.isWriteTransaction()); - JobSpecification spec = compiler.createJob(ccAppContext, jobEventListenerFactory); + JobSpecification spec = compiler.createJob(ccAppContext, jobEventListenerFactory, runtimeFlags); if (isQuery) { if (!compiler.skipJobCapacityAssignment()) { @@ -347,7 +349,10 @@ if (isQuery || isLoad) { generateOptimizedLogicalPlan(plan, spec.getLogical2PhysicalMap(), output.config().getPlanFormat(), cboMode); - lastPlan = new PlanInfo(plan, spec.getLogical2PhysicalMap(), cboMode, output.config().getPlanFormat()); + if (runtimeFlags.contains(JobFlag.PROFILE_RUNTIME)) { + lastPlan = + new PlanInfo(plan, spec.getLogical2PhysicalMap(), cboMode, output.config().getPlanFormat()); + } } } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java index 8b635a2..e602b76 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java @@ -3663,7 +3663,7 @@ loadStmt.getDatasetName(), loadStmt.getAdapter(), properties, loadStmt.dataIsAlreadySorted()); cls.setSourceLocation(stmt.getSourceLocation()); JobSpecification spec = apiFramework.compileQuery(hcc, metadataProvider, null, 0, null, sessionOutput, cls, - null, responsePrinter, warningCollector, null); + null, responsePrinter, warningCollector, null, jobFlags); afterCompile(); MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); bActiveTxn = false; @@ -3794,7 +3794,7 @@ // Query Compilation (happens under the same ongoing metadata transaction) return apiFramework.compileQuery(clusterInfoCollector, metadataProvider, (Query) rewrittenResult.first, rewrittenResult.second, stmt == null ? null : stmt.getDatasetName(), sessionOutput, stmt, externalVars, - responsePrinter, warningCollector, requestParameters); + responsePrinter, warningCollector, requestParameters, jobFlags); } protected JobSpecification rewriteCompileInsertUpsert(IClusterInfoCollector clusterInfoCollector, @@ -3835,7 +3835,7 @@ // transaction) return apiFramework.compileQuery(clusterInfoCollector, metadataProvider, rewrittenInsertUpsert.getQuery(), rewrittenResult.second, datasetName, sessionOutput, clfrqs, externalVars, responsePrinter, - warningCollector, null); + warningCollector, null, jobFlags); } protected void handleCreateFeedStatement(MetadataProvider metadataProvider, Statement stmt) throws Exception { diff --git a/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java b/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java index 85910aa..a44e1be 100644 --- a/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java +++ b/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java @@ -18,6 +18,7 @@ */ package org.apache.hyracks.algebricks.compiler.api; +import java.util.EnumSet; import java.util.List; import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint; @@ -47,6 +48,7 @@ import org.apache.hyracks.algebricks.runtime.writers.SerializedDataWriterFactory; import org.apache.hyracks.api.exceptions.IWarningCollector; import org.apache.hyracks.api.job.IJobletEventListenerFactory; +import org.apache.hyracks.api.job.JobFlag; import org.apache.hyracks.api.job.JobSpecification; public class HeuristicCompilerFactoryBuilder extends AbstractCompilerFactoryBuilder { @@ -172,6 +174,17 @@ } @Override + public JobSpecification createJob(Object appContext, IJobletEventListenerFactory jobEventListenerFactory, + EnumSet<JobFlag> runtimeFlags) throws AlgebricksException { + AlgebricksConfig.ALGEBRICKS_LOGGER.trace("Starting Job Generation.\n"); + PlanCompiler pc = factory.createPlanCompiler(oc, appContext, writerFactory); + if (runtimeFlags.contains(JobFlag.PROFILE_RUNTIME)) { + pc.enableLog2PhysMapping(); + } + return pc.compilePlan(plan, jobEventListenerFactory); + } + + @Override public boolean skipJobCapacityAssignment() { return oc.skipJobCapacityAssignment(); } diff --git a/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/ICompiler.java b/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/ICompiler.java index 0f5798e..7de0adb 100644 --- a/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/ICompiler.java +++ b/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/ICompiler.java @@ -18,8 +18,11 @@ */ package org.apache.hyracks.algebricks.compiler.api; +import java.util.EnumSet; + import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.api.job.IJobletEventListenerFactory; +import org.apache.hyracks.api.job.JobFlag; import org.apache.hyracks.api.job.JobSpecification; public interface ICompiler { @@ -28,5 +31,8 @@ public JobSpecification createJob(Object appContext, IJobletEventListenerFactory jobEventListenerFactory) throws AlgebricksException; + JobSpecification createJob(Object appContext, IJobletEventListenerFactory jobletEventListenerFactory, + EnumSet<JobFlag> runtimeFlags) throws AlgebricksException; + boolean skipJobCapacityAssignment(); } diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobBuilder.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobBuilder.java index 99ef8d0..ca26515 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobBuilder.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobBuilder.java @@ -74,6 +74,8 @@ private int aodCounter = 0; + private boolean genLog2PhysMap = false; + public JobBuilder(JobSpecification jobSpec, AlgebricksAbsolutePartitionConstraint clusterLocations) { this.jobSpec = jobSpec; this.clusterLocations = clusterLocations; @@ -96,6 +98,10 @@ new String[] { clusterLocations.getLocations()[Math.abs(jobSpec.hashCode() % nPartitions)] }); } + public void enableLog2PhysMapping() { + this.genLog2PhysMap = true; + } + @Override public void contributeMicroOperator(ILogicalOperator op, IPushRuntimeFactory runtime, RecordDescriptor recDesc) { contributeMicroOperator(op, runtime, recDesc, null); @@ -214,7 +220,11 @@ jobSpec.addRoot(opDesc); } setAllPartitionConstraints(tgtConstraints); - jobSpec.setLogical2PhysicalMap(getLogical2PhysicalMap()); + if (genLog2PhysMap) { + jobSpec.setLogical2PhysicalMap(getLogical2PhysicalMap()); + } else { + jobSpec.setLogical2PhysicalMap(Collections.emptyMap()); + } } public List<IOperatorDescriptor> getGeneratedMetaOps() { diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/PlanCompiler.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/PlanCompiler.java index 2ef5c6c..f8bdfa5 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/PlanCompiler.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/PlanCompiler.java @@ -39,6 +39,8 @@ private JobGenContext context; private Map<Mutable<ILogicalOperator>, List<Mutable<ILogicalOperator>>> operatorVisitedToParents = new HashMap<>(); + boolean genLog2PhysMap = false; + public PlanCompiler(JobGenContext context) { this.context = context; } @@ -47,6 +49,10 @@ return context; } + public void enableLog2PhysMapping() { + this.genLog2PhysMap = true; + } + public JobSpecification compilePlan(ILogicalPlan plan, IJobletEventListenerFactory jobEventListenerFactory) throws AlgebricksException { return compilePlanImpl(plan, false, null, jobEventListenerFactory); @@ -66,6 +72,9 @@ } List<ILogicalOperator> rootOps = new ArrayList<>(); JobBuilder builder = new JobBuilder(spec, context.getClusterLocations()); + if (genLog2PhysMap) { + builder.enableLog2PhysMapping(); + } for (Mutable<ILogicalOperator> opRef : plan.getRoots()) { compileOpRef(opRef, spec, builder, outerPlanSchema); rootOps.add(opRef.getValue()); diff --git a/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/operators/meta/PipelineAssembler.java b/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/operators/meta/PipelineAssembler.java index 0ce2ff5..202c087 100644 --- a/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/operators/meta/PipelineAssembler.java +++ b/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/operators/meta/PipelineAssembler.java @@ -77,7 +77,7 @@ for (int j = 0; j < newRuntimes.length; j++) { //ETS is wrapped externally, and doesn't need the micro-op wrapper since it isn't a pipeline //we also want to avoid any instances of NoOp stats in the pipeline that snuck in somehow - boolean shouldProfile = !(runtimeFactory instanceof EmptyTupleSourceRuntimeFactory) && profile + boolean shouldProfile = profile && !(runtimeFactory instanceof EmptyTupleSourceRuntimeFactory) && microOpStats.containsKey(runtimeFactory); if (shouldProfile) { ProfiledPushRuntime profiled; -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17962 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: trinity Gerrit-Change-Id: Ia9a84eedbcb33c29f03155a8605bb82af372f1f3 Gerrit-Change-Number: 17962 Gerrit-PatchSet: 3 Gerrit-Owner: Ian Maxon <[email protected]> Gerrit-Reviewer: Ali Alsuliman <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Ian Maxon <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-MessageType: merged
