AMBARI-21795. Auto fix service config mapping issue discovered during ambari-upgrade.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ca5fd787 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ca5fd787 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ca5fd787 Branch: refs/heads/feature-branch-AMBARI-21307 Commit: ca5fd78778345d74d885cd547c44917cf0feb03f Parents: 698293d Author: Vitaly Brodetskyi <vbrodets...@hortonworks.com> Authored: Fri Aug 25 22:54:13 2017 +0300 Committer: Vitaly Brodetskyi <vbrodets...@hortonworks.com> Committed: Fri Aug 25 22:54:13 2017 +0300 ---------------------------------------------------------------------- .../checks/DatabaseConsistencyCheckHelper.java | 4 +-- .../orm/entities/ClusterConfigEntity.java | 12 +++---- .../apache/ambari/server/state/ServiceImpl.java | 2 +- .../server/upgrade/UpgradeCatalog260.java | 38 ++++++++++++++++++++ .../main/resources/Ambari-DDL-Derby-CREATE.sql | 2 +- .../main/resources/Ambari-DDL-MySQL-CREATE.sql | 2 +- .../main/resources/Ambari-DDL-Oracle-CREATE.sql | 2 +- .../resources/Ambari-DDL-Postgres-CREATE.sql | 2 +- .../resources/Ambari-DDL-SQLAnywhere-CREATE.sql | 2 +- .../resources/Ambari-DDL-SQLServer-CREATE.sql | 2 +- .../server/upgrade/UpgradeCatalog260Test.java | 9 +++++ 11 files changed, 62 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/ca5fd787/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java index aae34ae..1e2dac5 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java @@ -570,7 +570,7 @@ public class DatabaseConsistencyCheckHelper { List<ClusterConfigEntity> notMappedClusterConfigs = getNotMappedClusterConfigsToService(); for (ClusterConfigEntity clusterConfigEntity : notMappedClusterConfigs){ - if (!clusterConfigEntity.isServiceDeleted()){ + if (!clusterConfigEntity.isUnmapped()){ continue; // skip clusterConfigs that did not leave after service deletion } List<String> types = new ArrayList<>(); @@ -605,7 +605,7 @@ public class DatabaseConsistencyCheckHelper { Set<String> nonMappedConfigs = new HashSet<>(); for (ClusterConfigEntity clusterConfigEntity : notMappedClasterConfigs) { - if (!clusterConfigEntity.isServiceDeleted()){ + if (!clusterConfigEntity.isUnmapped()){ nonMappedConfigs.add(clusterConfigEntity.getType() + '-' + clusterConfigEntity.getTag()); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/ca5fd787/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ClusterConfigEntity.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ClusterConfigEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ClusterConfigEntity.java index 3a74367..0c5276c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ClusterConfigEntity.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ClusterConfigEntity.java @@ -132,8 +132,8 @@ public class ClusterConfigEntity { @ManyToMany(mappedBy = "clusterConfigEntities") private Collection<ServiceConfigEntity> serviceConfigEntities; - @Column(name = "service_deleted", nullable = false, insertable = true, updatable = true) - private short serviceDeleted = 0; + @Column(name = "unmapped", nullable = false, insertable = true, updatable = true) + private short unmapped = 0; /** * Unidirectional one-to-one association to {@link StackEntity} @@ -142,12 +142,12 @@ public class ClusterConfigEntity { @JoinColumn(name = "stack_id", unique = false, nullable = false, insertable = true, updatable = true) private StackEntity stack; - public boolean isServiceDeleted() { - return serviceDeleted != 0; + public boolean isUnmapped() { + return unmapped != 0; } - public void setServiceDeleted(boolean serviceDeleted) { - this.serviceDeleted = (short)(serviceDeleted ? 1 : 0); + public void setUnmapped(boolean unmapped) { + this.unmapped = (short)(unmapped ? 1 : 0); } public Long getConfigId() { http://git-wip-us.apache.org/repos/asf/ambari/blob/ca5fd787/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java index 0247774..1104d19 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java @@ -504,7 +504,7 @@ public class ServiceImpl implements Service { for (ClusterConfigEntity serviceConfigEntity : lastServiceConfigEntity.getClusterConfigEntities()) { LOG.info("Disabling configuration {}", serviceConfigEntity); serviceConfigEntity.setSelected(false); - serviceConfigEntity.setServiceDeleted(true); + serviceConfigEntity.setUnmapped(true); clusterDAO.merge(serviceConfigEntity); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/ca5fd787/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog260.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog260.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog260.java index 9e1b55b..b4e7a02 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog260.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog260.java @@ -21,8 +21,12 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.List; +import javax.persistence.EntityManager; +import javax.persistence.Query; + import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.orm.DBAccessor; +import org.apache.ambari.server.orm.entities.ClusterConfigEntity; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,6 +48,8 @@ public class UpgradeCatalog260 extends AbstractUpgradeCatalog { public static final String CLUSTER_CONFIG_TABLE = "clusterconfig"; public static final String SELECTED_COLUMN = "selected"; + public static final String SERVICE_DELETED_COLUMN = "service_deleted"; + public static final String UNMAPPED_COLUMN = "unmapped"; public static final String SELECTED_TIMESTAMP_COLUMN = "selected_timestamp"; public static final String SERVICE_COMPONENT_DESIRED_STATE_TABLE = "servicecomponentdesiredstate"; @@ -149,6 +155,37 @@ public class UpgradeCatalog260 extends AbstractUpgradeCatalog { createUpgradeHistoryTable(); dropStaleTables(); updateRepositoryVersionTable(); + renameServiceDeletedColumn(); + } + + private void renameServiceDeletedColumn() throws AmbariException, SQLException { + if (dbAccessor.tableHasColumn(CLUSTER_CONFIG_TABLE, SERVICE_DELETED_COLUMN)) { + dbAccessor.renameColumn(CLUSTER_CONFIG_TABLE, SERVICE_DELETED_COLUMN, new DBAccessor.DBColumnInfo(UNMAPPED_COLUMN, Short.class, null, 0, false)); + } + } + + /* + * This method, search for configs which are not linked with any service + * and set "unmapped" to true. We need that because in case when + * config is not mapped and "unmapped" flag = false, db consistency check + * will show warning about not mapped config. (AMBARI-21795) + * */ + private void setUnmappedForOrphanedConfigs() { + executeInTransaction(new Runnable() { + @Override + public void run() { + EntityManager entityManager = getEntityManagerProvider().get(); + Query query = entityManager.createNamedQuery("ClusterConfigEntity.findNotMappedClusterConfigsToService",ClusterConfigEntity.class); + + List<ClusterConfigEntity> notMappedConfigs = (List<ClusterConfigEntity>) query.getResultList(); + if (notMappedConfigs != null) { + for (ClusterConfigEntity clusterConfigEntity : notMappedConfigs) { + clusterConfigEntity.setUnmapped(true); + entityManager.merge(clusterConfigEntity); + } + } + } + }); } private void createUpgradeHistoryTable() throws SQLException { @@ -329,6 +366,7 @@ public class UpgradeCatalog260 extends AbstractUpgradeCatalog { @Override protected void executeDMLUpdates() throws AmbariException, SQLException { addNewConfigurationsFromXml(); + setUnmappedForOrphanedConfigs(); removeSupersetFromDruid(); } http://git-wip-us.apache.org/repos/asf/ambari/blob/ca5fd787/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql index 8a55336..dc7f79a 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql @@ -76,7 +76,7 @@ CREATE TABLE clusterconfig ( config_data VARCHAR(3000) NOT NULL, config_attributes VARCHAR(3000), create_timestamp BIGINT NOT NULL, - service_deleted SMALLINT NOT NULL DEFAULT 0, + unmapped SMALLINT NOT NULL DEFAULT 0, selected_timestamp BIGINT NOT NULL DEFAULT 0, CONSTRAINT PK_clusterconfig PRIMARY KEY (config_id), CONSTRAINT FK_clusterconfig_cluster_id FOREIGN KEY (cluster_id) REFERENCES clusters (cluster_id), http://git-wip-us.apache.org/repos/asf/ambari/blob/ca5fd787/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql index a0916bf..0c28012 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql @@ -96,7 +96,7 @@ CREATE TABLE clusterconfig ( config_data LONGTEXT NOT NULL, config_attributes LONGTEXT, create_timestamp BIGINT NOT NULL, - service_deleted SMALLINT NOT NULL DEFAULT 0, + unmapped SMALLINT NOT NULL DEFAULT 0, selected_timestamp BIGINT NOT NULL DEFAULT 0, CONSTRAINT PK_clusterconfig PRIMARY KEY (config_id), CONSTRAINT FK_clusterconfig_cluster_id FOREIGN KEY (cluster_id) REFERENCES clusters (cluster_id), http://git-wip-us.apache.org/repos/asf/ambari/blob/ca5fd787/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql index 5204ebb..6cd330c 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql @@ -76,7 +76,7 @@ CREATE TABLE clusterconfig ( config_data CLOB NOT NULL, config_attributes CLOB, create_timestamp NUMBER(19) NOT NULL, - service_deleted SMALLINT DEFAULT 0 NOT NULL, + unmapped SMALLINT DEFAULT 0 NOT NULL, selected_timestamp NUMBER(19) DEFAULT 0 NOT NULL, CONSTRAINT PK_clusterconfig PRIMARY KEY (config_id), CONSTRAINT FK_clusterconfig_cluster_id FOREIGN KEY (cluster_id) REFERENCES clusters (cluster_id), http://git-wip-us.apache.org/repos/asf/ambari/blob/ca5fd787/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql index dee2f87..8c8ed5c 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql @@ -76,7 +76,7 @@ CREATE TABLE clusterconfig ( config_data TEXT NOT NULL, config_attributes TEXT, create_timestamp BIGINT NOT NULL, - service_deleted SMALLINT NOT NULL DEFAULT 0, + unmapped SMALLINT NOT NULL DEFAULT 0, selected_timestamp BIGINT NOT NULL DEFAULT 0, CONSTRAINT PK_clusterconfig PRIMARY KEY (config_id), CONSTRAINT FK_clusterconfig_cluster_id FOREIGN KEY (cluster_id) REFERENCES clusters (cluster_id), http://git-wip-us.apache.org/repos/asf/ambari/blob/ca5fd787/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql index 654bac8..59abd8b 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql @@ -75,7 +75,7 @@ CREATE TABLE clusterconfig ( config_data TEXT NOT NULL, config_attributes TEXT, create_timestamp NUMERIC(19) NOT NULL, - service_deleted SMALLINT NOT NULL DEFAULT 0, + unmapped SMALLINT NOT NULL DEFAULT 0, selected_timestamp NUMERIC(19) NOT NULL DEFAULT 0, CONSTRAINT PK_clusterconfig PRIMARY KEY (config_id), CONSTRAINT FK_clusterconfig_cluster_id FOREIGN KEY (cluster_id) REFERENCES clusters (cluster_id), http://git-wip-us.apache.org/repos/asf/ambari/blob/ca5fd787/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql index 238dc1c..ea92256 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql @@ -89,7 +89,7 @@ CREATE TABLE clusterconfig ( config_data VARCHAR(MAX) NOT NULL, config_attributes VARCHAR(MAX), create_timestamp BIGINT NOT NULL, - service_deleted SMALLINT NOT NULL DEFAULT 0, + unmapped SMALLINT NOT NULL DEFAULT 0, selected_timestamp BIGINT NOT NULL DEFAULT 0, CONSTRAINT PK_clusterconfig PRIMARY KEY CLUSTERED (config_id), CONSTRAINT FK_clusterconfig_cluster_id FOREIGN KEY (cluster_id) REFERENCES clusters (cluster_id), http://git-wip-us.apache.org/repos/asf/ambari/blob/ca5fd787/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog260Test.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog260Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog260Test.java index 2e38686..b6c323e 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog260Test.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog260Test.java @@ -145,6 +145,7 @@ public class UpgradeCatalog260Test { expect(connection.createStatement()).andReturn(statement).anyTimes(); expect(statement.executeQuery(anyObject(String.class))).andReturn(resultSet).anyTimes(); expect(configuration.getDatabaseType()).andReturn(Configuration.DatabaseType.POSTGRES).anyTimes(); + expect(dbAccessor.tableHasColumn(UpgradeCatalog260.CLUSTER_CONFIG_TABLE, UpgradeCatalog260.SERVICE_DELETED_COLUMN)).andReturn(true).anyTimes(); Capture<String[]> scdcaptureKey = newCapture(); @@ -182,6 +183,9 @@ public class UpgradeCatalog260Test { Capture<DBColumnInfo> repoVersionHiddenColumnCapture = newCapture(); expectUpdateRepositoryVersionTableTable(repoVersionHiddenColumnCapture); + Capture<DBColumnInfo> unapped = newCapture(); + expectRenameServiceDeletedColumn(unapped); + replay(dbAccessor, configuration, connection, statement, resultSet); Module module = new Module() { @@ -218,6 +222,11 @@ public class UpgradeCatalog260Test { expectLastCall().once(); } + public void expectRenameServiceDeletedColumn(Capture<DBColumnInfo> unmapped) throws SQLException { + dbAccessor.renameColumn(eq(UpgradeCatalog260.CLUSTER_CONFIG_TABLE), eq(UpgradeCatalog260.SERVICE_DELETED_COLUMN), capture(unmapped)); + expectLastCall().once(); + } + public void verifyCreateUpgradeHistoryTable(Capture<List<DBColumnInfo>> columns) { List<DBColumnInfo> columnsValue = columns.getValue(); Assert.assertEquals(columnsValue.size(), 6);
