AMBARI-20875. Removing A Service Causes DB Verification To Produce Warnings (dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f942bb22 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f942bb22 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f942bb22 Branch: refs/heads/branch-feature-AMBARI-12556 Commit: f942bb22e30133c2fefd7a32cf981b7c088e9db7 Parents: 7d24bfc Author: Lisnichenko Dmitro <dlysniche...@hortonworks.com> Authored: Wed May 3 16:32:18 2017 +0300 Committer: Lisnichenko Dmitro <dlysniche...@hortonworks.com> Committed: Wed May 3 16:32:18 2017 +0300 ---------------------------------------------------------------------- .../checks/DatabaseConsistencyCheckHelper.java | 9 +- .../orm/entities/ClusterConfigEntity.java | 11 ++ .../apache/ambari/server/state/ServiceImpl.java | 2 +- .../server/upgrade/SchemaUpgradeHelper.java | 1 + .../server/upgrade/UpgradeCatalog252.java | 94 +++++++++++ .../main/resources/Ambari-DDL-Derby-CREATE.sql | 1 + .../main/resources/Ambari-DDL-MySQL-CREATE.sql | 1 + .../main/resources/Ambari-DDL-Oracle-CREATE.sql | 1 + .../resources/Ambari-DDL-Postgres-CREATE.sql | 1 + .../resources/Ambari-DDL-SQLAnywhere-CREATE.sql | 1 + .../resources/Ambari-DDL-SQLServer-CREATE.sql | 1 + .../server/upgrade/UpgradeCatalog252Test.java | 166 +++++++++++++++++++ 12 files changed, 286 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/f942bb22/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java index 0b7f122..7c40c83 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java @@ -630,6 +630,9 @@ public class DatabaseConsistencyCheckHelper { List<ClusterConfigEntity> notMappedClusterConfigs = getNotMappedClusterConfigsToService(); for (ClusterConfigEntity clusterConfigEntity : notMappedClusterConfigs){ + if (!clusterConfigEntity.isServiceDeleted()){ + continue; // skip clusterConfigs that did not leave after service deletion + } List<String> types = new ArrayList<>(); String type = clusterConfigEntity.getType(); types.add(type); @@ -662,9 +665,11 @@ public class DatabaseConsistencyCheckHelper { Set<String> nonMappedConfigs = new HashSet<>(); for (ClusterConfigEntity clusterConfigEntity : notMappedClasterConfigs) { - nonMappedConfigs.add(clusterConfigEntity.getType() + '-' + clusterConfigEntity.getTag()); + if (!clusterConfigEntity.isServiceDeleted()){ + nonMappedConfigs.add(clusterConfigEntity.getType() + '-' + clusterConfigEntity.getTag()); + } } - if (!notMappedClasterConfigs.isEmpty()){ + if (!nonMappedConfigs.isEmpty()){ warning("You have config(s): {} that is(are) not mapped (in serviceconfigmapping table) to any service!", StringUtils.join(nonMappedConfigs, ",")); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/f942bb22/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ClusterConfigEntity.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ClusterConfigEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ClusterConfigEntity.java index 876063d..1092ac5 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ClusterConfigEntity.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ClusterConfigEntity.java @@ -129,6 +129,9 @@ public class ClusterConfigEntity { @ManyToMany(mappedBy = "clusterConfigEntities") private Collection<ServiceConfigEntity> serviceConfigEntities; + @Column(name = "service_deleted", nullable = false, insertable = true, updatable = true) + private short serviceDeleted = 0; + /** * Unidirectional one-to-one association to {@link StackEntity} */ @@ -136,6 +139,14 @@ public class ClusterConfigEntity { @JoinColumn(name = "stack_id", unique = false, nullable = false, insertable = true, updatable = true) private StackEntity stack; + public boolean isServiceDeleted() { + return serviceDeleted != 0; + } + + public void setServiceDeleted(boolean serviceDeleted) { + this.serviceDeleted = (short)(serviceDeleted ? 1 : 0); + } + public Long getConfigId() { return configId; } http://git-wip-us.apache.org/repos/asf/ambari/blob/f942bb22/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java index a0c0db1..61f44cc 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java @@ -492,12 +492,12 @@ public class ServiceImpl implements Service { void deleteAllServiceConfigs() throws AmbariException { long clusterId = getClusterId(); ServiceConfigEntity lastServiceConfigEntity = serviceConfigDAO.findMaxVersion(clusterId, getName()); - // de-select every configuration from the service if (lastServiceConfigEntity != null) { for (ClusterConfigEntity serviceConfigEntity : lastServiceConfigEntity.getClusterConfigEntities()) { LOG.info("Disabling configuration {}", serviceConfigEntity); serviceConfigEntity.setSelected(false); + serviceConfigEntity.setServiceDeleted(true); clusterDAO.merge(serviceConfigEntity); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/f942bb22/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java index d022f1f..48cf5f6 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java @@ -190,6 +190,7 @@ public class SchemaUpgradeHelper { catalogBinder.addBinding().to(UpgradeCatalog242.class); catalogBinder.addBinding().to(UpgradeCatalog250.class); catalogBinder.addBinding().to(UpgradeCatalog251.class); + catalogBinder.addBinding().to(UpgradeCatalog252.class); catalogBinder.addBinding().to(UpgradeCatalog300.class); catalogBinder.addBinding().to(FinalUpgradeCatalog.class); http://git-wip-us.apache.org/repos/asf/ambari/blob/f942bb22/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog252.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog252.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog252.java new file mode 100644 index 0000000..c25dec1 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog252.java @@ -0,0 +1,94 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.upgrade; + +import java.sql.SQLException; + +import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.orm.DBAccessor.DBColumnInfo; + +import com.google.inject.Inject; +import com.google.inject.Injector; + +/** + * The {@link org.apache.ambari.server.upgrade.UpgradeCatalog252} upgrades Ambari from 2.5.1 to 2.5.2. + */ +public class UpgradeCatalog252 extends AbstractUpgradeCatalog { + + static final String CLUSTERCONFIG_TABLE = "clusterconfig"; + static final String SERVICE_DELETED_COLUMN = "service_deleted"; + + /** + * Constructor. + * + * @param injector + */ + @Inject + public UpgradeCatalog252(Injector injector) { + super(injector); + } + + /** + * {@inheritDoc} + */ + @Override + public String getSourceVersion() { + return "2.5.1"; + } + + /** + * {@inheritDoc} + */ + @Override + public String getTargetVersion() { + return "2.5.2"; + } + + /** + * {@inheritDoc} + */ + @Override + protected void executeDDLUpdates() throws AmbariException, SQLException { + addServiceDeletedColumnToClusterConfigTable(); + } + + /** + * {@inheritDoc} + */ + @Override + protected void executePreDMLUpdates() throws AmbariException, SQLException { + } + + /** + * {@inheritDoc} + */ + @Override + protected void executeDMLUpdates() throws AmbariException, SQLException { + } + + /** + * Adds the {@value #SERVICE_DELETED_COLUMN} column to the + * {@value #CLUSTERCONFIG_TABLE} table. + * + * @throws java.sql.SQLException + */ + private void addServiceDeletedColumnToClusterConfigTable() throws SQLException { + dbAccessor.addColumn(CLUSTERCONFIG_TABLE, + new DBColumnInfo(SERVICE_DELETED_COLUMN, Short.class, null, 0, false)); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/f942bb22/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql index b241dc2..ece6600 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql @@ -76,6 +76,7 @@ CREATE TABLE clusterconfig ( config_data VARCHAR(3000) NOT NULL, config_attributes VARCHAR(3000), create_timestamp BIGINT NOT NULL, + service_deleted SMALLINT NOT NULL DEFAULT 0, selected_timestamp BIGINT NOT NULL DEFAULT 0, CONSTRAINT PK_clusterconfig PRIMARY KEY (config_id), CONSTRAINT FK_clusterconfig_cluster_id FOREIGN KEY (cluster_id) REFERENCES clusters (cluster_id), http://git-wip-us.apache.org/repos/asf/ambari/blob/f942bb22/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql index 670bf17..e0f2ef9 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql @@ -96,6 +96,7 @@ CREATE TABLE clusterconfig ( config_data LONGTEXT NOT NULL, config_attributes LONGTEXT, create_timestamp BIGINT NOT NULL, + service_deleted SMALLINT NOT NULL DEFAULT 0, selected_timestamp BIGINT NOT NULL DEFAULT 0, CONSTRAINT PK_clusterconfig PRIMARY KEY (config_id), CONSTRAINT FK_clusterconfig_cluster_id FOREIGN KEY (cluster_id) REFERENCES clusters (cluster_id), http://git-wip-us.apache.org/repos/asf/ambari/blob/f942bb22/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql index 00b3248..6d0f856 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql @@ -76,6 +76,7 @@ CREATE TABLE clusterconfig ( config_data CLOB NOT NULL, config_attributes CLOB, create_timestamp NUMBER(19) NOT NULL, + service_deleted SMALLINT NOT NULL DEFAULT 0, selected_timestamp NUMBER(19) DEFAULT 0 NOT NULL, CONSTRAINT PK_clusterconfig PRIMARY KEY (config_id), CONSTRAINT FK_clusterconfig_cluster_id FOREIGN KEY (cluster_id) REFERENCES clusters (cluster_id), http://git-wip-us.apache.org/repos/asf/ambari/blob/f942bb22/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql index f6af968..49b956b 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql @@ -76,6 +76,7 @@ CREATE TABLE clusterconfig ( config_data TEXT NOT NULL, config_attributes TEXT, create_timestamp BIGINT NOT NULL, + service_deleted SMALLINT NOT NULL DEFAULT 0, selected_timestamp BIGINT NOT NULL DEFAULT 0, CONSTRAINT PK_clusterconfig PRIMARY KEY (config_id), CONSTRAINT FK_clusterconfig_cluster_id FOREIGN KEY (cluster_id) REFERENCES clusters (cluster_id), http://git-wip-us.apache.org/repos/asf/ambari/blob/f942bb22/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql index 64a0137..c594a2e 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql @@ -75,6 +75,7 @@ CREATE TABLE clusterconfig ( config_data TEXT NOT NULL, config_attributes TEXT, create_timestamp NUMERIC(19) NOT NULL, + service_deleted SMALLINT NOT NULL DEFAULT 0, selected_timestamp NUMERIC(19) NOT NULL DEFAULT 0, CONSTRAINT PK_clusterconfig PRIMARY KEY (config_id), CONSTRAINT FK_clusterconfig_cluster_id FOREIGN KEY (cluster_id) REFERENCES clusters (cluster_id), http://git-wip-us.apache.org/repos/asf/ambari/blob/f942bb22/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql index 22b2c3d..77459a6 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql @@ -89,6 +89,7 @@ CREATE TABLE clusterconfig ( config_data VARCHAR(MAX) NOT NULL, config_attributes VARCHAR(MAX), create_timestamp BIGINT NOT NULL, + service_deleted SMALLINT NOT NULL DEFAULT 0, selected_timestamp BIGINT NOT NULL DEFAULT 0, CONSTRAINT PK_clusterconfig PRIMARY KEY CLUSTERED (config_id), CONSTRAINT FK_clusterconfig_cluster_id FOREIGN KEY (cluster_id) REFERENCES clusters (cluster_id), http://git-wip-us.apache.org/repos/asf/ambari/blob/f942bb22/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog252Test.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog252Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog252Test.java new file mode 100644 index 0000000..484ca5b --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog252Test.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.upgrade; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; + +import javax.persistence.EntityManager; + +import org.apache.ambari.server.actionmanager.ActionManager; +import org.apache.ambari.server.configuration.Configuration; +import org.apache.ambari.server.controller.KerberosHelper; +import org.apache.ambari.server.controller.MaintenanceStateHelper; +import org.apache.ambari.server.orm.DBAccessor; +import org.apache.ambari.server.orm.DBAccessor.DBColumnInfo; +import org.apache.ambari.server.state.Cluster; +import org.apache.ambari.server.state.Clusters; +import org.apache.ambari.server.state.Config; +import org.apache.ambari.server.state.Service; +import org.apache.ambari.server.state.stack.OsFamily; +import org.easymock.Capture; +import org.easymock.EasyMockRunner; +import org.easymock.Mock; +import org.easymock.MockType; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +import com.google.gson.Gson; +import com.google.inject.Binder; +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.Module; +import com.google.inject.Provider; + +import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.capture; +import static org.easymock.EasyMock.eq; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.newCapture; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.reset; +import static org.easymock.EasyMock.verify; + +/** + * {@link org.apache.ambari.server.upgrade.UpgradeCatalog252} unit tests. + */ +@RunWith(EasyMockRunner.class) +public class UpgradeCatalog252Test { + + // private Injector injector; + @Mock(type = MockType.STRICT) + private Provider<EntityManager> entityManagerProvider; + + @Mock(type = MockType.NICE) + private EntityManager entityManager; + + @Mock(type = MockType.NICE) + private DBAccessor dbAccessor; + + @Mock(type = MockType.NICE) + private Configuration configuration; + + @Mock(type = MockType.NICE) + private Connection connection; + + @Mock(type = MockType.NICE) + private Statement statement; + + @Mock(type = MockType.NICE) + private ResultSet resultSet; + + @Mock(type = MockType.NICE) + private OsFamily osFamily; + + @Mock(type = MockType.NICE) + private KerberosHelper kerberosHelper; + + @Mock(type = MockType.NICE) + private ActionManager actionManager; + + @Mock(type = MockType.NICE) + private Config config; + + @Mock(type = MockType.STRICT) + private Service service; + + @Mock(type = MockType.NICE) + private Clusters clusters; + + @Mock(type = MockType.NICE) + private Cluster cluster; + + @Mock(type = MockType.NICE) + private Injector injector; + + @Before + public void init() { + reset(entityManagerProvider, injector); + + expect(entityManagerProvider.get()).andReturn(entityManager).anyTimes(); + + expect(injector.getInstance(Gson.class)).andReturn(null).anyTimes(); + expect(injector.getInstance(MaintenanceStateHelper.class)).andReturn(null).anyTimes(); + expect(injector.getInstance(KerberosHelper.class)).andReturn(kerberosHelper).anyTimes(); + + replay(entityManagerProvider, injector); + } + + @After + public void tearDown() { + } + + @Test + public void testExecuteDDLUpdates() throws Exception { + Capture<DBColumnInfo> hrcBackgroundColumnCapture = newCapture(); + dbAccessor.addColumn(eq(UpgradeCatalog252.CLUSTERCONFIG_TABLE), capture(hrcBackgroundColumnCapture)); + + expect(dbAccessor.getConnection()).andReturn(connection).anyTimes(); + expect(connection.createStatement()).andReturn(statement).anyTimes(); + expect(statement.executeQuery(anyObject(String.class))).andReturn(resultSet).anyTimes(); + expect(configuration.getDatabaseType()).andReturn(Configuration.DatabaseType.POSTGRES).anyTimes(); + + replay(dbAccessor, configuration, connection, statement, resultSet); + + Module module = new Module() { + @Override + public void configure(Binder binder) { + binder.bind(DBAccessor.class).toInstance(dbAccessor); + binder.bind(OsFamily.class).toInstance(osFamily); + binder.bind(EntityManager.class).toInstance(entityManager); + binder.bind(Configuration.class).toInstance(configuration); + } + }; + + Injector injector = Guice.createInjector(module); + UpgradeCatalog252 upgradeCatalog252 = injector.getInstance(UpgradeCatalog252.class); + upgradeCatalog252.executeDDLUpdates(); + + verify(dbAccessor); + + DBColumnInfo captured = hrcBackgroundColumnCapture.getValue(); + Assert.assertEquals(UpgradeCatalog252.SERVICE_DELETED_COLUMN, captured.getName()); + Assert.assertEquals(false, captured.getDefaultValue()); + Assert.assertEquals(Boolean.class, captured.getType()); + } +}