Repository: aurora Updated Branches: refs/heads/master 17ddc484f -> 9340d468f
Increase throughput of DbTaskStore Profiling master indicated that the bottleneck was MyBatis populating ResultSets and populating the resulting objects. This patch removes subselects, which reduces the number of ResultSets and removes the population of an object via a constructor which is slower than populating an object via setters. Reviewed at https://reviews.apache.org/r/43457/ Project: http://git-wip-us.apache.org/repos/asf/aurora/repo Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/9340d468 Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/9340d468 Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/9340d468 Branch: refs/heads/master Commit: 9340d468f446020250151d3da1c0d7c0e371b5bc Parents: 17ddc48 Author: Zameer Manji <zma...@apache.org> Authored: Fri Feb 12 13:32:54 2016 -0800 Committer: Zameer Manji <zma...@apache.org> Committed: Fri Feb 12 13:32:54 2016 -0800 ---------------------------------------------------------------------- .../storage/db/views/DbAssginedPort.java | 30 +++++++++++ .../storage/db/views/DbAssignedTask.java | 13 +++-- .../storage/db/views/DbScheduledTask.java | 12 +++++ .../scheduler/storage/db/TaskConfigMapper.xml | 34 +++++------- .../aurora/scheduler/storage/db/TaskMapper.xml | 54 ++++++++------------ 5 files changed, 87 insertions(+), 56 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/aurora/blob/9340d468/src/main/java/org/apache/aurora/scheduler/storage/db/views/DbAssginedPort.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/views/DbAssginedPort.java b/src/main/java/org/apache/aurora/scheduler/storage/db/views/DbAssginedPort.java new file mode 100644 index 0000000..fdfff77 --- /dev/null +++ b/src/main/java/org/apache/aurora/scheduler/storage/db/views/DbAssginedPort.java @@ -0,0 +1,30 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.aurora.scheduler.storage.db.views; + +public final class DbAssginedPort { + private String name; + private Integer port; + + private DbAssginedPort() { + } + + public Integer getPort() { + return port; + } + + public String getName() { + return name; + } +} http://git-wip-us.apache.org/repos/asf/aurora/blob/9340d468/src/main/java/org/apache/aurora/scheduler/storage/db/views/DbAssignedTask.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/views/DbAssignedTask.java b/src/main/java/org/apache/aurora/scheduler/storage/db/views/DbAssignedTask.java index 9372239..cbc6a0c 100644 --- a/src/main/java/org/apache/aurora/scheduler/storage/db/views/DbAssignedTask.java +++ b/src/main/java/org/apache/aurora/scheduler/storage/db/views/DbAssignedTask.java @@ -14,8 +14,10 @@ package org.apache.aurora.scheduler.storage.db.views; import java.util.List; +import java.util.Map; + +import com.google.common.collect.Maps; -import org.apache.aurora.common.collections.Pair; import org.apache.aurora.gen.AssignedTask; public final class DbAssignedTask { @@ -23,19 +25,24 @@ public final class DbAssignedTask { private String slaveId; private String slaveHost; private DbTaskConfig task; - private List<Pair<String, Integer>> assignedPorts; + private List<DbAssginedPort> assignedPorts; private int instanceId; private DbAssignedTask() { } AssignedTask toThrift() { + Map<String, Integer> ports = Maps.newHashMap(); + for (DbAssginedPort port: assignedPorts) { + ports.put(port.getName(), port.getPort()); + } + return new AssignedTask() .setTaskId(taskId) .setSlaveId(slaveId) .setSlaveHost(slaveHost) .setTask(task.toThrift()) - .setAssignedPorts(Pairs.toMap(assignedPorts)) + .setAssignedPorts(ports) .setInstanceId(instanceId); } } http://git-wip-us.apache.org/repos/asf/aurora/blob/9340d468/src/main/java/org/apache/aurora/scheduler/storage/db/views/DbScheduledTask.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/views/DbScheduledTask.java b/src/main/java/org/apache/aurora/scheduler/storage/db/views/DbScheduledTask.java index 502a1fa..8d0dab0 100644 --- a/src/main/java/org/apache/aurora/scheduler/storage/db/views/DbScheduledTask.java +++ b/src/main/java/org/apache/aurora/scheduler/storage/db/views/DbScheduledTask.java @@ -13,8 +13,12 @@ */ package org.apache.aurora.scheduler.storage.db.views; +import java.util.Collections; import java.util.List; +import com.google.common.collect.Ordering; +import com.google.common.primitives.Longs; + import org.apache.aurora.gen.ScheduleStatus; import org.apache.aurora.gen.ScheduledTask; import org.apache.aurora.gen.TaskEvent; @@ -31,6 +35,7 @@ public final class DbScheduledTask { } public IScheduledTask toImmutable() { + Collections.sort(taskEvents, BY_TIMESTAMP); return IScheduledTask.build( new ScheduledTask() .setAssignedTask(assignedTask.toThrift()) @@ -39,4 +44,11 @@ public final class DbScheduledTask { .setTaskEvents(taskEvents) .setAncestorId(ancestorId)); } + + private static final Ordering<TaskEvent> BY_TIMESTAMP = new Ordering<TaskEvent>() { + @Override + public int compare(TaskEvent left, TaskEvent right) { + return Longs.compare(left.getTimestamp(), right.getTimestamp()); + } + }; } http://git-wip-us.apache.org/repos/asf/aurora/blob/9340d468/src/main/resources/org/apache/aurora/scheduler/storage/db/TaskConfigMapper.xml ---------------------------------------------------------------------- diff --git a/src/main/resources/org/apache/aurora/scheduler/storage/db/TaskConfigMapper.xml b/src/main/resources/org/apache/aurora/scheduler/storage/db/TaskConfigMapper.xml index b1394cf..fd272cc 100644 --- a/src/main/resources/org/apache/aurora/scheduler/storage/db/TaskConfigMapper.xml +++ b/src/main/resources/org/apache/aurora/scheduler/storage/db/TaskConfigMapper.xml @@ -79,22 +79,6 @@ <association property="constraint" resultMap="taskConstraintMap"/> </resultMap> - <select id="selectConstraints" resultMap="constraintMap"> - SELECT - tc.id AS id, - tc.name AS name, - tlc.id AS l_id, - tlc.value AS l_limit, - tvc.id AS v_id, - tvc.negated AS v_negated, - tvcv.value AS v_v_value - FROM task_constraints AS tc - LEFT OUTER JOIN limit_constraints AS tlc ON tlc.constraint_id = tc.id - LEFT OUTER JOIN value_constraints AS tvc ON tvc.constraint_id = tc.id - LEFT OUTER JOIN value_constraint_values AS tvcv ON tvcv.value_constraint_id = tvc.id - WHERE tc.task_config_id = #{id} - </select> - <resultMap id="dockerParameterMap" type="org.apache.aurora.gen.DockerParameter"> <id column="id"/> </resultMap> @@ -138,9 +122,8 @@ <association property="container" resultMap="containerMap" columnPrefix="c_"/> <collection property="constraints" - select="selectConstraints" - column="id" - foreignColumn="task_config_id"/> + columnPrefix="constraint_" + resultMap="constraintMap"/> <collection property="requestedPorts" ofType="String" columnPrefix="p_"> <result column="port_name" /> </collection> @@ -175,12 +158,23 @@ d.image AS c_image, m.id AS m_id, m.key AS m_key, - m.value AS m_value + m.value AS m_value, + tc.id AS constraint_id, + tc.name AS constraint_name, + tlc.id AS constraint_l_id, + tlc.value AS constraint_l_limit, + tvc.id AS constraint_v_id, + tvc.negated AS constraint_v_negated, + tvcv.value as constraint_v_v_value FROM task_configs AS c INNER JOIN job_keys AS j ON j.id = c.job_key_id LEFT OUTER JOIN task_config_requested_ports AS p ON p.task_config_id = c.id LEFT OUTER JOIN task_config_docker_containers AS d ON d.task_config_id = c.id LEFT OUTER JOIN task_config_metadata AS m ON m.task_config_id = c.id + LEFT OUTER JOIN task_constraints AS tc ON tc.task_config_id = c.id + LEFT OUTER JOIN limit_constraints as tlc ON tlc.constraint_id = tc.id + LEFT OUTER JOIN value_constraints as tvc ON tvc.constraint_id = tc.id + LEFT OUTER JOIN value_constraint_values AS tvcv ON tvcv.value_constraint_id = tvc.id </sql> <select id="selectConfig" resultMap="taskConfigMap"> http://git-wip-us.apache.org/repos/asf/aurora/blob/9340d468/src/main/resources/org/apache/aurora/scheduler/storage/db/TaskMapper.xml ---------------------------------------------------------------------- diff --git a/src/main/resources/org/apache/aurora/scheduler/storage/db/TaskMapper.xml b/src/main/resources/org/apache/aurora/scheduler/storage/db/TaskMapper.xml index ea469cc..684614f 100644 --- a/src/main/resources/org/apache/aurora/scheduler/storage/db/TaskMapper.xml +++ b/src/main/resources/org/apache/aurora/scheduler/storage/db/TaskMapper.xml @@ -48,20 +48,11 @@ <result property="status" column="status" typeHandler="org.apache.aurora.scheduler.storage.db.typehandlers.ScheduleStatusTypeHandler" /> + <result column="timestamp_ms" property="timestamp" /> + <result column="message" property="message" /> + <result column="scheduler_host" property="scheduler" /> </resultMap> - <select id="selectTaskEvents" resultMap="taskEventMap"> - SELECT - e.id AS id, - e.timestamp_ms AS timestamp, - e.status AS status, - e.message AS message, - e.scheduler_host AS scheduler - FROM task_events AS e - WHERE e.task_row_id = (SELECT id FROM tasks WHERE task_id = #{task_id}) - ORDER BY e.timestamp_ms ASC - </select> - <resultMap id="scheduledTaskMap" type="org.apache.aurora.scheduler.storage.db.views.DbScheduledTask"> <id column="row_id"/> <result property="status" @@ -78,14 +69,13 @@ foreignColumn="row_id"/> <collection property="assignedTask.assignedPorts" - select="selectPorts" - column="task_id" - foreignColumn="task_row_id"/> + resultMap="portMap" + columnPrefix="tp_"/> <collection property="taskEvents" - select="selectTaskEvents" - column="task_id" - foreignColumn="task_row_id"/> + resultMap="taskEventMap" + columnPrefix="te_" + notNullColumn="status"/> </resultMap> <sql id="unscopedSelect"> @@ -101,10 +91,18 @@ j.environment AS c_j_environment, j.name AS c_j_name, h.slave_id AS slave_id, - h.host AS slave_host + h.host AS slave_host, + tp.name as tp_name, + tp.port as tp_port, + te.timestamp_ms as te_timestamp, + te.status as te_status, + te.message as te_message, + te.scheduler_host as te_scheduler FROM tasks AS t INNER JOIN task_configs as c ON c.id = t.task_config_row_id INNER JOIN job_keys AS j ON j.id = c.job_key_id + LEFT OUTER JOIN task_ports as tp ON tp.task_row_id = t.id + LEFT OUTER JOIN task_events as te ON te.task_row_id = t.id LEFT OUTER JOIN host_attributes AS h ON h.id = t.slave_row_id </sql> @@ -208,22 +206,12 @@ ) </insert> - <resultMap id="portMap" type="org.apache.aurora.common.collections.Pair"> - <constructor> - <arg column="name"/> - <arg column="port"/> - </constructor> + <resultMap id="portMap" type="org.apache.aurora.scheduler.storage.db.views.DbAssginedPort"> + <id column="id"/> + <result column="name" property="name" /> + <result column="port" property="port" /> </resultMap> - <select id="selectPorts" resultMap="portMap"> - SELECT - id, - name, - port - FROM task_ports - WHERE task_row_id = (SELECT id FROM tasks WHERE task_id = #{task_id}) - </select> - <delete id="truncate"> <!-- This assumes cascading deletes will clean up all references. Also, once the job store is
