Repository: aurora Updated Branches: refs/heads/master 656cf9ac5 -> cc2aa46f7
Fix schema to allow multiple task volumes per task. The original commit adding this feature added an artifical constraint to the schema that prevented more than one task volume per task. This is because there was a `UNIQUE` constraint between the volumes table and the task config table, preventing a task config from being associated with more than one volume. This patch removes that constraint. As a result some of the MyBatis mappers had to change and a new migration was added. Bugs closed: AURORA-1914 Reviewed at https://reviews.apache.org/r/58066/ Project: http://git-wip-us.apache.org/repos/asf/aurora/repo Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/cc2aa46f Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/cc2aa46f Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/cc2aa46f Branch: refs/heads/master Commit: cc2aa46f7ad8590e201621ffe2799299959ef7eb Parents: 656cf9a Author: Zameer Manji <zma...@apache.org> Authored: Fri Apr 7 14:18:33 2017 +0200 Committer: Zameer Manji <zma...@apache.org> Committed: Fri Apr 7 14:18:33 2017 +0200 ---------------------------------------------------------------------- .../migration/V010_RemoveUniqueConstraint.java | 41 ++++++++++++++++++++ .../scheduler/storage/db/views/DbContainer.java | 6 +-- .../storage/db/views/DbTaskConfig.java | 12 +++++- .../scheduler/storage/db/TaskConfigMapper.xml | 14 ++++--- .../aurora/scheduler/storage/db/schema.sql | 2 - .../storage/AbstractTaskStoreTest.java | 7 +++- .../apache/aurora/e2e/http/http_example.aurora | 5 ++- 7 files changed, 71 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/aurora/blob/cc2aa46f/src/main/java/org/apache/aurora/scheduler/storage/db/migration/V010_RemoveUniqueConstraint.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/migration/V010_RemoveUniqueConstraint.java b/src/main/java/org/apache/aurora/scheduler/storage/db/migration/V010_RemoveUniqueConstraint.java new file mode 100644 index 0000000..08c38f3 --- /dev/null +++ b/src/main/java/org/apache/aurora/scheduler/storage/db/migration/V010_RemoveUniqueConstraint.java @@ -0,0 +1,41 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.aurora.scheduler.storage.db.migration; + +import java.math.BigDecimal; + +import org.apache.ibatis.migration.MigrationScript; + +public class V010_RemoveUniqueConstraint implements MigrationScript { + @Override + public BigDecimal getId() { + return BigDecimal.valueOf(10L); + } + + @Override + public String getDescription() { + return "Remove unique constraint in task_config_volumes"; + } + + @Override + public String getUpScript() { + // The constraint name is taken from the schema so it is always constant. + return "ALTER TABLE IF EXISTS task_config_volumes DROP CONSTRAINT IF EXISTS CONSTRAINT_654B;"; + } + + @Override + public String getDownScript() { + return "ALTER TABLE IF EXISTS task_config_volumes ADD UNIQUE(task_config_id);"; + } +} http://git-wip-us.apache.org/repos/asf/aurora/blob/cc2aa46f/src/main/java/org/apache/aurora/scheduler/storage/db/views/DbContainer.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/views/DbContainer.java b/src/main/java/org/apache/aurora/scheduler/storage/db/views/DbContainer.java index 6a36e27..8d4d7ec 100644 --- a/src/main/java/org/apache/aurora/scheduler/storage/db/views/DbContainer.java +++ b/src/main/java/org/apache/aurora/scheduler/storage/db/views/DbContainer.java @@ -13,17 +13,13 @@ */ package org.apache.aurora.scheduler.storage.db.views; -import java.util.List; - import org.apache.aurora.gen.Container; import org.apache.aurora.gen.DockerContainer; import org.apache.aurora.gen.MesosContainer; -import org.apache.aurora.gen.Volume; public final class DbContainer { private DockerContainer docker; private DbImage image; - private List<Volume> volumes; private DbContainer() { } @@ -34,7 +30,7 @@ public final class DbContainer { } if (image != null) { - return Container.mesos(new MesosContainer().setImage(image.toThrift()).setVolumes(volumes)); + return Container.mesos(new MesosContainer().setImage(image.toThrift())); } return Container.mesos(new MesosContainer()); http://git-wip-us.apache.org/repos/asf/aurora/blob/cc2aa46f/src/main/java/org/apache/aurora/scheduler/storage/db/views/DbTaskConfig.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/views/DbTaskConfig.java b/src/main/java/org/apache/aurora/scheduler/storage/db/views/DbTaskConfig.java index cf65a28..138cd53 100644 --- a/src/main/java/org/apache/aurora/scheduler/storage/db/views/DbTaskConfig.java +++ b/src/main/java/org/apache/aurora/scheduler/storage/db/views/DbTaskConfig.java @@ -26,6 +26,7 @@ import org.apache.aurora.gen.MesosContainer; import org.apache.aurora.gen.MesosFetcherURI; import org.apache.aurora.gen.Metadata; import org.apache.aurora.gen.TaskConfig; +import org.apache.aurora.gen.Volume; import org.apache.aurora.scheduler.storage.entities.ITaskConfig; import static org.apache.aurora.GuavaUtils.toImmutableSet; @@ -49,6 +50,7 @@ public final class DbTaskConfig { private List<Metadata> metadata; private List<MesosFetcherURI> mesosFetcherUris; private DbContainer container; + private List<Volume> volumes; private String tier; private List<DBResource> resources; @@ -60,7 +62,7 @@ public final class DbTaskConfig { } TaskConfig toThrift() { - return new TaskConfig() + TaskConfig builder = new TaskConfig() .setJob(job) .setOwner(owner) .setIsService(isService) @@ -83,6 +85,14 @@ public final class DbTaskConfig { .setContainer( container == null ? Container.mesos(new MesosContainer()) : container.toThrift()) .setResources(resources.stream().map(DBResource::toThrift).collect(toImmutableSet())); + + // In the DB Layer volumes are associated with a task config, since containers are not + // modelled as tables. + if (builder.getContainer().isSetMesos()) { + builder.getContainer().getMesos().setVolumes(volumes); + } + + return builder; } public ITaskConfig toImmutable() { http://git-wip-us.apache.org/repos/asf/aurora/blob/cc2aa46f/src/main/resources/org/apache/aurora/scheduler/storage/db/TaskConfigMapper.xml ---------------------------------------------------------------------- diff --git a/src/main/resources/org/apache/aurora/scheduler/storage/db/TaskConfigMapper.xml b/src/main/resources/org/apache/aurora/scheduler/storage/db/TaskConfigMapper.xml index 97f6b80..5422183 100644 --- a/src/main/resources/org/apache/aurora/scheduler/storage/db/TaskConfigMapper.xml +++ b/src/main/resources/org/apache/aurora/scheduler/storage/db/TaskConfigMapper.xml @@ -118,9 +118,12 @@ </resultMap> <resultMap id="containerMap" type="org.apache.aurora.scheduler.storage.db.views.DbContainer"> + <!--NOTE: Do not put any collections under here. ContainerMap doesn't correspond to a table, it's a syntheic map + for assocations. Since it doesn't have a table, there is no unique id. Since there is no unique id, MyBatis nested + result map collections fail. + --> <association property="docker" resultMap="dockerContainerMap" /> <association property="image" columnPrefix="image_" resultMap="imageMap" /> - <collection property="volumes" columnPrefix="volume_" resultMap="volumeMap" notNullColumn="id" /> </resultMap> <resultMap id="metadataMap" type="org.apache.aurora.gen.Metadata"> @@ -153,6 +156,7 @@ resultMap="org.apache.aurora.scheduler.storage.db.JobKeyMapper.jobKeyMap" columnPrefix="j_"/> <association property="container" resultMap="containerMap" columnPrefix="c_"/> + <collection property="volumes" columnPrefix="volume_" resultMap="volumeMap" notNullColumn="id" /> <collection property="constraints" columnPrefix="constraint_" @@ -207,10 +211,10 @@ di.tag as c_image_docker_tag, ai.name as c_image_appc_name, ai.image_id as c_image_appc_image_id, - v.id as c_volume_id, - v.host_path as c_volume_host_path, - v.container_path as c_volume_container_path, - v.mode as c_volume_mode, + v.id as volume_id, + v.host_path as volume_host_path, + v.container_path as volume_container_path, + v.mode as volume_mode, tc.id AS constraint_id, tc.name AS constraint_name, tlc.id AS constraint_l_id, http://git-wip-us.apache.org/repos/asf/aurora/blob/cc2aa46f/src/main/resources/org/apache/aurora/scheduler/storage/db/schema.sql ---------------------------------------------------------------------- diff --git a/src/main/resources/org/apache/aurora/scheduler/storage/db/schema.sql b/src/main/resources/org/apache/aurora/scheduler/storage/db/schema.sql index ff1623a..7a86f47 100644 --- a/src/main/resources/org/apache/aurora/scheduler/storage/db/schema.sql +++ b/src/main/resources/org/apache/aurora/scheduler/storage/db/schema.sql @@ -235,8 +235,6 @@ CREATE TABLE task_config_volumes( host_path VARCHAR NOT NULL, container_path VARCHAR NOT NULL, mode INT NOT NULL REFERENCES volume_modes(id), - - UNIQUE(task_config_id) ); CREATE TABLE task_states( http://git-wip-us.apache.org/repos/asf/aurora/blob/cc2aa46f/src/test/java/org/apache/aurora/scheduler/storage/AbstractTaskStoreTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/storage/AbstractTaskStoreTest.java b/src/test/java/org/apache/aurora/scheduler/storage/AbstractTaskStoreTest.java index 80be339..69c35b1 100644 --- a/src/test/java/org/apache/aurora/scheduler/storage/AbstractTaskStoreTest.java +++ b/src/test/java/org/apache/aurora/scheduler/storage/AbstractTaskStoreTest.java @@ -197,7 +197,9 @@ public abstract class AbstractTaskStoreTest extends TearDownTestCase { public void testSaveWithContainerVolumes() { ScheduledTask builder = TASK_B.newBuilder(); Image image = Image.docker(new DockerImage().setName("some-name").setTag("some-tag")); - List<Volume> volumes = ImmutableList.of(new Volume("container", "host", Mode.RO)); + List<Volume> volumes = ImmutableList.of( + new Volume("container2", "host2", Mode.RW), + new Volume("container", "host", Mode.RO)); builder.getAssignedTask().getTask().getContainer().getMesos().setImage(image) .setVolumes(volumes); @@ -683,7 +685,8 @@ public abstract class AbstractTaskStoreTest extends TearDownTestCase { private void assertQueryResults(Query.Builder query, Set<IScheduledTask> tasks) { Iterable<IScheduledTask> result = fetchTasks(query); assertQueryHasNoDupes(result); - assertEquals(tasks, ImmutableSet.copyOf(fetchTasks(query))); + Set<IScheduledTask> set = ImmutableSet.copyOf(result); + assertEquals(tasks, set); } private void assertQueryHasNoDupes(Iterable<IScheduledTask> result) { http://git-wip-us.apache.org/repos/asf/aurora/blob/cc2aa46f/src/test/sh/org/apache/aurora/e2e/http/http_example.aurora ---------------------------------------------------------------------- diff --git a/src/test/sh/org/apache/aurora/e2e/http/http_example.aurora b/src/test/sh/org/apache/aurora/e2e/http/http_example.aurora index be43a09..0361b36 100644 --- a/src/test/sh/org/apache/aurora/e2e/http/http_example.aurora +++ b/src/test/sh/org/apache/aurora/e2e/http/http_example.aurora @@ -127,7 +127,10 @@ job = Service( ), ) -volumes = [Volume(container_path="etc/rsyslog.d.container", host_path="/etc/rsyslog.d", mode=Mode("RO"))] +volumes = [ + Volume(container_path="etc/rsyslog.d.container", host_path="/etc/rsyslog.d", mode=Mode("RO")), + Volume(container_path="etc/w3m.container", host_path="/etc/w3m", mode=Mode("RO")) +] jobs = [ job(