Repository: ambari Updated Branches: refs/heads/trunk 1e631ce38 -> bdf5effbe
AMBARI-10580 - Ambari Database Upgrades For Stack Entity Changes (jonathanhurley) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/bdf5effb Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/bdf5effb Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/bdf5effb Branch: refs/heads/trunk Commit: bdf5effbe899c93b7da408417474f1c8cdbb4b2b Parents: 1e631ce Author: Jonathan Hurley <jhur...@hortonworks.com> Authored: Fri Apr 17 17:36:26 2015 -0400 Committer: Jonathan Hurley <jhur...@hortonworks.com> Committed: Fri Apr 17 17:56:15 2015 -0400 ---------------------------------------------------------------------- .../server/alerts/StaleAlertRunnable.java | 11 +- .../ambari/server/stack/StackManager.java | 9 + .../services/AmbariServerAlertService.java | 1 + .../server/upgrade/AbstractUpgradeCatalog.java | 8 + .../server/upgrade/SchemaUpgradeHelper.java | 19 ++ .../ambari/server/upgrade/UpgradeCatalog.java | 25 ++- .../server/upgrade/UpgradeCatalog200.java | 2 +- .../server/upgrade/UpgradeCatalog210.java | 217 ++++++++++++++++++- .../server/alerts/StaleAlertRunnableTest.java | 41 +++- .../server/upgrade/UpgradeCatalog210Test.java | 74 +++---- .../ambari/server/upgrade/UpgradeTest.java | 2 - 11 files changed, 342 insertions(+), 67 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/bdf5effb/ambari-server/src/main/java/org/apache/ambari/server/alerts/StaleAlertRunnable.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/alerts/StaleAlertRunnable.java b/ambari-server/src/main/java/org/apache/ambari/server/alerts/StaleAlertRunnable.java index 40e6fb2..e23dd62 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/alerts/StaleAlertRunnable.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/alerts/StaleAlertRunnable.java @@ -47,8 +47,10 @@ import com.google.inject.Provider; /** * The {@link StaleAlertRunnable} is used by the - * {@link AmbariServerAlertService} to check agent heartbeats and fire alert - * events when an agent host changes state. + * {@link AmbariServerAlertService} to check the last time that an alert was + * checked and determine if it seems to no longer be running. It will produce a + * single alert with {@link AlertState#CRITICAL} along with a textual + * description of the alerts that are stale. */ public class StaleAlertRunnable implements Runnable { @@ -154,6 +156,11 @@ public class StaleAlertRunnable implements Runnable { continue; } + // skip this alert (who watches the watchers) + if (definition.getDefinitionName().equals(STALE_ALERT_DEFINITION_NAME)) { + continue; + } + // convert minutes to milliseconds for the definition's interval long intervalInMillis = definition.getScheduleInterval() * MINUTE_TO_MS_CONVERSION; http://git-wip-us.apache.org/repos/asf/ambari/blob/bdf5effb/ambari-server/src/main/java/org/apache/ambari/server/stack/StackManager.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackManager.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackManager.java index 1026245..832370d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackManager.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackManager.java @@ -104,6 +104,8 @@ public class StackManager { ActionMetadata actionMetadata, StackDAO stackDao) throws AmbariException { + LOG.info("Initializing the stack manager..."); + validateStackDirectory(stackRoot); validateCommonServicesDirectory(commonServicesRoot); @@ -125,6 +127,8 @@ public class StackManager { String stackVersion = stack.getVersion(); if (stackDao.find(stackName, stackVersion) == null) { + LOG.info("Adding stack {}-{} to the database", stackName, stackVersion); + StackEntity stackEntity = new StackEntity(); stackEntity.setStackName(stackName); stackEntity.setStackVersion(stackVersion); @@ -236,6 +240,9 @@ public class StackManager { */ private void validateCommonServicesDirectory(File commonServicesRoot) throws AmbariException { if(commonServicesRoot != null) { + LOG.info("Validating common services directory {} ...", + commonServicesRoot); + String commonServicesRootAbsolutePath = commonServicesRoot.getAbsolutePath(); if (LOG.isDebugEnabled()) { LOG.debug("Loading common services information" @@ -257,6 +264,8 @@ public class StackManager { * @throws AmbariException if the specified stack root directory is invalid */ private void validateStackDirectory(File stackRoot) throws AmbariException { + LOG.info("Validating stack directory {} ...", stackRoot); + String stackRootAbsPath = stackRoot.getAbsolutePath(); if (LOG.isDebugEnabled()) { LOG.debug("Loading stack information" http://git-wip-us.apache.org/repos/asf/ambari/blob/bdf5effb/ambari-server/src/main/java/org/apache/ambari/server/state/services/AmbariServerAlertService.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/services/AmbariServerAlertService.java b/ambari-server/src/main/java/org/apache/ambari/server/state/services/AmbariServerAlertService.java index 6669e7b..a13a8c7 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/services/AmbariServerAlertService.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/services/AmbariServerAlertService.java @@ -46,6 +46,7 @@ import com.google.inject.Provider; /** * The {@link AmbariServerAlertService} is used to manage the dynamically loaded + * {@link Runnable}s which perform server-side alert checks. */ @AmbariService public class AmbariServerAlertService extends AbstractScheduledService { http://git-wip-us.apache.org/repos/asf/ambari/blob/bdf5effb/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java index 863b5da..940be93 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java @@ -391,4 +391,12 @@ public abstract class AbstractUpgradeCatalog implements UpgradeCatalog { return "{ upgradeCatalog: sourceVersion = " + getSourceVersion() + ", " + "targetVersion = " + getTargetVersion() + " }"; } + + /** + * {@inheritDoc} + */ + @Override + public void onPostUpgrade() throws AmbariException, SQLException { + // NOOP + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/bdf5effb/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java index 61482c9..1d6095b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java @@ -235,6 +235,23 @@ public class SchemaUpgradeHelper { } } + public void executeOnPostUpgrade(List<UpgradeCatalog> upgradeCatalogs) + throws AmbariException { + LOG.info("Finalizing catalog upgrade."); + + if (upgradeCatalogs != null && !upgradeCatalogs.isEmpty()) { + for (UpgradeCatalog upgradeCatalog : upgradeCatalogs) { + try { + upgradeCatalog.onPostUpgrade(); + ; + } catch (Exception e) { + LOG.error("Upgrade failed. ", e); + throw new AmbariException(e.getMessage(), e); + } + } + } + } + public void resetUIState() throws AmbariException { LOG.info("Resetting UI state."); try { @@ -279,6 +296,8 @@ public class SchemaUpgradeHelper { schemaUpgradeHelper.executeDMLUpdates(upgradeCatalogs); + schemaUpgradeHelper.executeOnPostUpgrade(upgradeCatalogs); + schemaUpgradeHelper.resetUIState(); LOG.info("Upgrade successful."); http://git-wip-us.apache.org/repos/asf/ambari/blob/bdf5effb/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java index a10b21c..f482dc2 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java @@ -17,10 +17,10 @@ */ package org.apache.ambari.server.upgrade; -import org.apache.ambari.server.AmbariException; - import java.sql.SQLException; +import org.apache.ambari.server.AmbariException; + /** * Interface for upgrading Ambari DB */ @@ -30,20 +30,31 @@ public interface UpgradeCatalog { * to the new version. * @throws AmbariException */ - public void upgradeSchema() throws AmbariException, SQLException; + void upgradeSchema() throws AmbariException, SQLException; /** * perform data updates as necessary, requires started persist service * @throws AmbariException * @throws SQLException */ - public void upgradeData() throws AmbariException, SQLException; + void upgradeData() throws AmbariException, SQLException; + + /** + * Called after {@link #upgradeSchema()} and {@link #upgradeData()}, this + * method is used to perform any operations after the catalog has finished. + * Usually, this is cleanup work that does not directly affect the upgrade. + * + * @throws AmbariException + * @throws SQLException + */ + void onPostUpgrade() throws AmbariException, SQLException; /** * Return the version that will be upgraded to + * * @return */ - public abstract String getTargetVersion(); + String getTargetVersion(); /** * Return latest source version that can be upgraded from. @@ -51,11 +62,11 @@ public interface UpgradeCatalog { * * @return null : default */ - public String getSourceVersion(); + String getSourceVersion(); /** * Returns a list of versions using simplified regex of the Ambari versions that allow running this UpgradeCatalog. * @return null : default */ - public String[] getCompatibleVersions(); + String[] getCompatibleVersions(); } http://git-wip-us.apache.org/repos/asf/ambari/blob/bdf5effb/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog200.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog200.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog200.java index e577aa4..4117a1c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog200.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog200.java @@ -100,7 +100,7 @@ public class UpgradeCatalog200 extends AbstractUpgradeCatalog { */ @Override public String[] getCompatibleVersions() { - return new String[] {"2.0.*"}; + return new String[] { "2.0.*", "2.1.*" }; } /** http://git-wip-us.apache.org/repos/asf/ambari/blob/bdf5effb/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java index 884032e..8897657 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java @@ -20,24 +20,29 @@ package org.apache.ambari.server.upgrade; import java.sql.ResultSet; import java.sql.SQLException; +import java.text.MessageFormat; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.configuration.Configuration; import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.orm.DBAccessor.DBColumnInfo; -import org.apache.ambari.server.orm.dao.ClusterDAO; -import org.apache.ambari.server.orm.dao.ClusterServiceDAO; -import org.apache.ambari.server.orm.entities.ClusterEntity; +import org.apache.ambari.server.orm.dao.StackDAO; +import org.apache.ambari.server.orm.entities.StackEntity; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Clusters; import org.apache.ambari.server.state.Service; +import org.apache.ambari.server.state.StackId; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.gson.Gson; import com.google.inject.Inject; import com.google.inject.Injector; import com.google.inject.persist.Transactional; @@ -47,7 +52,6 @@ import com.google.inject.persist.Transactional; * Upgrade catalog for version 2.1.0. */ public class UpgradeCatalog210 extends AbstractUpgradeCatalog { - private static final String CLUSTERS_TABLE = "clusters"; private static final String HOSTS_TABLE = "hosts"; private static final String HOST_COMPONENT_DESIRED_STATE_TABLE = "hostcomponentdesiredstate"; @@ -64,10 +68,21 @@ public class UpgradeCatalog210 extends AbstractUpgradeCatalog { private static final String WIDGET_LAYOUT_USER_WIDGET_TABLE = "widget_layout_user_widget"; private static final String VIEW_INSTANCE_TABLE = "viewinstance"; private static final String VIEW_PARAMETER_TABLE = "viewparameter"; - private static final String STACK_TABLE_DEFINITION = "stack"; + private static final String STACK_TABLE = "stack"; + private static final String REPO_VERSION_TABLE = "repo_version"; private static final String HOST_ID_COL = "host_id"; + // constants for stack table changes + private static final String STACK_ID_COLUMN_NAME = "stack_id"; + private static final String DESIRED_STACK_ID_COLUMN_NAME = "desired_stack_id"; + private static final String CURRENT_STACK_ID_COLUMN_NAME = "current_stack_id"; + private static final String DESIRED_STACK_VERSION_COLUMN_NAME = "desired_stack_version"; + private static final String CURRENT_STACK_VERSION_COLUMN_NAME = "current_stack_version"; + private static final DBColumnInfo DESIRED_STACK_ID_COLUMN = new DBColumnInfo(DESIRED_STACK_ID_COLUMN_NAME, Long.class, null, null, true); + private static final DBColumnInfo CURRENT_STACK_ID_COLUMN = new DBColumnInfo(CURRENT_STACK_ID_COLUMN_NAME, Long.class, null, null, true); + private static final DBColumnInfo STACK_ID_COLUMN = new DBColumnInfo(STACK_ID_COLUMN_NAME, Long.class, null, null, true); + /** * {@inheritDoc} */ @@ -87,6 +102,7 @@ public class UpgradeCatalog210 extends AbstractUpgradeCatalog { /** * {@inheritDoc} */ + @Override public String[] getCompatibleVersions() { return new String[] {"*"}; } @@ -122,6 +138,14 @@ public class UpgradeCatalog210 extends AbstractUpgradeCatalog { } /** + * {@inheritDoc} + */ + @Override + public void onPostUpgrade() throws AmbariException, SQLException { + cleanupStackUpdates(); + } + + /** * Execute all of the hosts DDL updates. * * @throws org.apache.ambari.server.AmbariException @@ -301,7 +325,7 @@ public class UpgradeCatalog210 extends AbstractUpgradeCatalog { dbAccessor.dropColumn(HOST_COMPONENT_DESIRED_STATE_TABLE, "host_name"); dbAccessor.dropColumn(HOST_STATE_TABLE, "host_name"); // TODO, include other tables. - + // view columns for cluster association dbAccessor.addColumn(VIEW_INSTANCE_TABLE, new DBColumnInfo("cluster_handle", String.class, 255, null, true)); dbAccessor.addColumn(VIEW_PARAMETER_TABLE, new DBColumnInfo("cluster_config", String.class, 255, null, true)); @@ -348,24 +372,155 @@ public class UpgradeCatalog210 extends AbstractUpgradeCatalog { } /** - * Adds the stack table and constraints. + * Adds the stack table, FKs, and constraints. */ private void executeStackDDLUpdates() throws AmbariException, SQLException { - // alert_definition + // stack table creation ArrayList<DBColumnInfo> columns = new ArrayList<DBColumnInfo>(); columns.add(new DBColumnInfo("stack_id", Long.class, null, null, false)); columns.add(new DBColumnInfo("stack_name", String.class, 255, null, false)); columns.add(new DBColumnInfo("stack_version", String.class, 255, null, false)); - dbAccessor.createTable(STACK_TABLE_DEFINITION, columns, "stack_id"); + dbAccessor.createTable(STACK_TABLE, columns, "stack_id"); - dbAccessor.executeQuery("ALTER TABLE " + STACK_TABLE_DEFINITION + dbAccessor.executeQuery("ALTER TABLE " + STACK_TABLE + " ADD CONSTRAINT unq_stack UNIQUE (stack_name,stack_version)", false); dbAccessor.executeQuery( "INSERT INTO ambari_sequences(sequence_name, sequence_value) VALUES('stack_id_seq', 0)", false); + + // create the new stack ID columns NULLABLE for now since we need to insert + // data into them later on (we'll change them to NOT NULL after that) + dbAccessor.addColumn(CLUSTERS_TABLE, DESIRED_STACK_ID_COLUMN); + dbAccessor.addColumn("hostcomponentdesiredstate", DESIRED_STACK_ID_COLUMN); + dbAccessor.addColumn("servicecomponentdesiredstate", DESIRED_STACK_ID_COLUMN); + dbAccessor.addColumn("servicedesiredstate", DESIRED_STACK_ID_COLUMN); + + dbAccessor.addFKConstraint(CLUSTERS_TABLE, "fk_clusters_desired_stack_id", DESIRED_STACK_ID_COLUMN_NAME, STACK_TABLE, STACK_ID_COLUMN_NAME, true); + dbAccessor.addFKConstraint("hostcomponentdesiredstate", "fk_hcds_desired_stack_id", DESIRED_STACK_ID_COLUMN_NAME, STACK_TABLE, STACK_ID_COLUMN_NAME, true); + dbAccessor.addFKConstraint("servicecomponentdesiredstate", "fk_scds_desired_stack_id", DESIRED_STACK_ID_COLUMN_NAME, STACK_TABLE, STACK_ID_COLUMN_NAME, true); + dbAccessor.addFKConstraint("servicedesiredstate", "fk_sds_desired_stack_id", DESIRED_STACK_ID_COLUMN_NAME, STACK_TABLE, STACK_ID_COLUMN_NAME, true); + + dbAccessor.addColumn("clusterstate", CURRENT_STACK_ID_COLUMN); + dbAccessor.addColumn("hostcomponentstate", CURRENT_STACK_ID_COLUMN); + + dbAccessor.addFKConstraint("clusterstate", "fk_cs_current_stack_id", CURRENT_STACK_ID_COLUMN_NAME, STACK_TABLE, STACK_ID_COLUMN_NAME, true); + dbAccessor.addFKConstraint("hostcomponentstate", "fk_hcs_current_stack_id", CURRENT_STACK_ID_COLUMN_NAME, STACK_TABLE, STACK_ID_COLUMN_NAME, true); + + dbAccessor.addColumn("clusterconfig", STACK_ID_COLUMN); + dbAccessor.addColumn("serviceconfig", STACK_ID_COLUMN); + dbAccessor.addColumn("blueprint", STACK_ID_COLUMN); + dbAccessor.addColumn(REPO_VERSION_TABLE, STACK_ID_COLUMN); + + dbAccessor.addFKConstraint("clusterconfig", "fk_clusterconfig_stack_id", STACK_ID_COLUMN_NAME, STACK_TABLE, STACK_ID_COLUMN_NAME, true); + dbAccessor.addFKConstraint("serviceconfig", "fk_serviceconfig_stack_id", STACK_ID_COLUMN_NAME, STACK_TABLE, STACK_ID_COLUMN_NAME, true); + dbAccessor.addFKConstraint("blueprint", "fk_blueprint_stack_id", STACK_ID_COLUMN_NAME, STACK_TABLE, STACK_ID_COLUMN_NAME, true); + dbAccessor.addFKConstraint(REPO_VERSION_TABLE, "fk_repoversion_stack_id", STACK_ID_COLUMN_NAME, STACK_TABLE, STACK_ID_COLUMN_NAME, true); + + // drop the unique constraint for the old column and add the new one + dbAccessor.dropConstraint(REPO_VERSION_TABLE, "uq_repo_version_stack_version"); + dbAccessor.executeQuery("ALTER TABLE repo_version ADD CONSTRAINT uq_repo_version_stack_id UNIQUE (stack_id, version)"); + } + + /** + * Adds the stack table and constraints. + */ + protected void executeStackDMLUpdates() throws AmbariException, SQLException { + Gson gson = new Gson(); + + injector.getInstance(AmbariMetaInfo.class); + + StackDAO stackDAO = injector.getInstance(StackDAO.class); + List<StackEntity> stacks = stackDAO.findAll(); + Map<Long,String> entityToJsonMap = new HashMap<Long, String>(); + + // build a mapping of stack entity to old-school JSON + for( StackEntity stack : stacks ){ + StackId stackId = new StackId(stack.getStackName(), + stack.getStackVersion()); + String stackJson = gson.toJson(stackId); + entityToJsonMap.put(stack.getStackId(), stackJson); + } + + // use a bulk update on all tables to populate the new FK columns + String UPDATE_TEMPLATE = "UPDATE {0} SET {1} = {2} WHERE {3} = ''{4}''"; + String UPDATE_BLUEPRINT_TEMPLATE = "UPDATE blueprint SET stack_id = {0} WHERE stack_name = ''{1}'' AND stack_version = ''{2}''"; + + Set<Long> stackEntityIds = entityToJsonMap.keySet(); + for (Long stackEntityId : stackEntityIds) { + StackEntity stackEntity = stackDAO.findById(stackEntityId); + String outdatedJson = entityToJsonMap.get(stackEntityId); + + String clustersSQL = MessageFormat.format(UPDATE_TEMPLATE, "clusters", + DESIRED_STACK_ID_COLUMN_NAME, stackEntityId, + DESIRED_STACK_VERSION_COLUMN_NAME, outdatedJson); + + String hostComponentDesiredStateSQL = MessageFormat.format( + UPDATE_TEMPLATE, "hostcomponentdesiredstate", + DESIRED_STACK_ID_COLUMN_NAME, stackEntityId, + DESIRED_STACK_VERSION_COLUMN_NAME, outdatedJson); + + String serviceComponentDesiredStateSQL = MessageFormat.format( + UPDATE_TEMPLATE, "servicecomponentdesiredstate", + DESIRED_STACK_ID_COLUMN_NAME, stackEntityId, + DESIRED_STACK_VERSION_COLUMN_NAME, outdatedJson); + + String serviceDesiredStateSQL = MessageFormat.format(UPDATE_TEMPLATE, + "servicedesiredstate", + DESIRED_STACK_ID_COLUMN_NAME, stackEntityId, + DESIRED_STACK_VERSION_COLUMN_NAME, outdatedJson); + + String clusterStateSQL = MessageFormat.format(UPDATE_TEMPLATE, + "clusterstate", CURRENT_STACK_ID_COLUMN_NAME, stackEntityId, + CURRENT_STACK_VERSION_COLUMN_NAME, outdatedJson); + + String hostComponentStateSQL = MessageFormat.format(UPDATE_TEMPLATE, + "hostcomponentstate", CURRENT_STACK_ID_COLUMN_NAME, stackEntityId, + CURRENT_STACK_VERSION_COLUMN_NAME, outdatedJson); + + String blueprintSQL = MessageFormat.format(UPDATE_BLUEPRINT_TEMPLATE, + stackEntityId, stackEntity.getStackName(), + stackEntity.getStackVersion()); + + String repoVersionSQL = MessageFormat.format(UPDATE_TEMPLATE, + REPO_VERSION_TABLE, STACK_ID_COLUMN_NAME, stackEntityId, "stack", + outdatedJson); + + dbAccessor.executeQuery(clustersSQL); + dbAccessor.executeQuery(hostComponentDesiredStateSQL); + dbAccessor.executeQuery(serviceComponentDesiredStateSQL); + dbAccessor.executeQuery(serviceDesiredStateSQL); + dbAccessor.executeQuery(clusterStateSQL); + dbAccessor.executeQuery(hostComponentStateSQL); + dbAccessor.executeQuery(blueprintSQL); + dbAccessor.executeQuery(repoVersionSQL); + } + + // for the tables with no prior stack, set these based on the cluster's + // stack for each cluster defined + String INSERT_STACK_ID_TEMPLATE = "UPDATE {0} SET {1} = {2} WHERE cluster_id = {3}"; + ResultSet resultSet = dbAccessor.executeSelect("SELECT * FROM clusters"); + while (resultSet.next()) { + long clusterId = resultSet.getLong("cluster_id"); + String stackJson = resultSet.getString(DESIRED_STACK_VERSION_COLUMN_NAME); + StackId stackId = gson.fromJson(stackJson, StackId.class); + + StackEntity stackEntity = stackDAO.find(stackId.getStackName(), + stackId.getStackVersion()); + + String clusterConfigSQL = MessageFormat.format(INSERT_STACK_ID_TEMPLATE, + "clusterconfig", STACK_ID_COLUMN_NAME, stackEntity.getStackId(), + clusterId); + + String serviceConfigSQL = MessageFormat.format(INSERT_STACK_ID_TEMPLATE, + "serviceconfig", STACK_ID_COLUMN_NAME, stackEntity.getStackId(), + clusterId); + + dbAccessor.executeQuery(clusterConfigSQL); + dbAccessor.executeQuery(serviceConfigSQL); + } } /** @@ -462,7 +617,49 @@ public class UpgradeCatalog210 extends AbstractUpgradeCatalog { @Override protected void executeDMLUpdates() throws AmbariException, SQLException { addNewConfigurationsFromXml(); + // Initialize all default widgets and widget layouts initializeClusterAndServiceWidgets(); + + // populate new stack ID columns + executeStackDMLUpdates(); + } + + /** + * Adds non NULL constraints and drops outdated columns no longer needed after + * the column data migration. + */ + private void cleanupStackUpdates() throws SQLException { + DESIRED_STACK_ID_COLUMN.setNullable(false); + CURRENT_STACK_ID_COLUMN.setNullable(false); + STACK_ID_COLUMN.setNullable(false); + + // make all stack columns NOT NULL now that they are filled in + dbAccessor.alterColumn(CLUSTERS_TABLE, DESIRED_STACK_ID_COLUMN); + dbAccessor.alterColumn("hostcomponentdesiredstate", DESIRED_STACK_ID_COLUMN); + dbAccessor.alterColumn("servicecomponentdesiredstate", DESIRED_STACK_ID_COLUMN); + dbAccessor.alterColumn("servicedesiredstate", DESIRED_STACK_ID_COLUMN); + + dbAccessor.alterColumn("clusterstate", CURRENT_STACK_ID_COLUMN); + dbAccessor.alterColumn("hostcomponentstate", CURRENT_STACK_ID_COLUMN); + + dbAccessor.alterColumn("clusterconfig", STACK_ID_COLUMN); + dbAccessor.alterColumn("serviceconfig", STACK_ID_COLUMN); + dbAccessor.alterColumn("blueprint", STACK_ID_COLUMN); + dbAccessor.alterColumn(REPO_VERSION_TABLE, STACK_ID_COLUMN); + + // drop unused JSON columns + dbAccessor.dropColumn(CLUSTERS_TABLE, DESIRED_STACK_VERSION_COLUMN_NAME); + dbAccessor.dropColumn("hostcomponentdesiredstate", DESIRED_STACK_VERSION_COLUMN_NAME); + dbAccessor.dropColumn("servicecomponentdesiredstate", DESIRED_STACK_VERSION_COLUMN_NAME); + dbAccessor.dropColumn("servicedesiredstate", DESIRED_STACK_VERSION_COLUMN_NAME); + + dbAccessor.dropColumn("clusterstate", CURRENT_STACK_VERSION_COLUMN_NAME); + dbAccessor.dropColumn("hostcomponentstate", CURRENT_STACK_VERSION_COLUMN_NAME); + + dbAccessor.dropColumn("blueprint", "stack_name"); + dbAccessor.dropColumn("blueprint", "stack_version"); + + dbAccessor.dropColumn(REPO_VERSION_TABLE, "stack"); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/bdf5effb/ambari-server/src/test/java/org/apache/ambari/server/alerts/StaleAlertRunnableTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/alerts/StaleAlertRunnableTest.java b/ambari-server/src/test/java/org/apache/ambari/server/alerts/StaleAlertRunnableTest.java index f3e3fa6..94a0bc2 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/alerts/StaleAlertRunnableTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/alerts/StaleAlertRunnableTest.java @@ -150,11 +150,19 @@ public class StaleAlertRunnableTest { @Test public void testAllAlertsAreCurrent() { // create current alerts that are not stale + AlertDefinitionEntity definition1 = new AlertDefinitionEntity(); + definition1.setClusterId(CLUSTER_ID); + definition1.setDefinitionName("foo-definition"); + definition1.setServiceName("HDFS"); + definition1.setComponentName("NAMENODE"); + definition1.setEnabled(true); + definition1.setScheduleInterval(1); + AlertCurrentEntity current1 = createNiceMock(AlertCurrentEntity.class); AlertHistoryEntity history1 = createNiceMock(AlertHistoryEntity.class); expect(current1.getAlertHistory()).andReturn(history1).atLeastOnce(); - expect(history1.getAlertDefinition()).andReturn(m_definition).atLeastOnce(); + expect(history1.getAlertDefinition()).andReturn(definition1).atLeastOnce(); expect(current1.getMaintenanceState()).andReturn(MaintenanceState.OFF).atLeastOnce(); expect(current1.getLatestTimestamp()).andReturn(System.currentTimeMillis()).atLeastOnce(); @@ -187,8 +195,7 @@ public class StaleAlertRunnableTest { assertEquals(AlertState.OK, alert.getState()); assertEquals(DEFINITION_NAME, alert.getName()); - verify(m_definition, m_cluster, m_clusters, - m_definitionDao); + verify(m_cluster, m_clusters, m_definitionDao); } /** @@ -196,12 +203,21 @@ public class StaleAlertRunnableTest { */ @Test public void testStaleAlert() { + // create current alerts that are not stale + AlertDefinitionEntity definition1 = new AlertDefinitionEntity(); + definition1.setClusterId(CLUSTER_ID); + definition1.setDefinitionName("foo-definition"); + definition1.setServiceName("HDFS"); + definition1.setComponentName("NAMENODE"); + definition1.setEnabled(true); + definition1.setScheduleInterval(1); + // create current alerts that are stale AlertCurrentEntity current1 = createNiceMock(AlertCurrentEntity.class); AlertHistoryEntity history1 = createNiceMock(AlertHistoryEntity.class); expect(current1.getAlertHistory()).andReturn(history1).atLeastOnce(); - expect(history1.getAlertDefinition()).andReturn(m_definition).atLeastOnce(); + expect(history1.getAlertDefinition()).andReturn(definition1).atLeastOnce(); // a really old timestampt to trigger the alert expect(current1.getMaintenanceState()).andReturn(MaintenanceState.OFF).atLeastOnce(); @@ -235,7 +251,7 @@ public class StaleAlertRunnableTest { assertEquals(AlertState.CRITICAL, alert.getState()); assertEquals(DEFINITION_NAME, alert.getName()); - verify(m_definition, m_cluster, m_clusters, m_definitionDao); + verify(m_cluster, m_clusters, m_definitionDao); } /** @@ -243,6 +259,15 @@ public class StaleAlertRunnableTest { */ @Test public void testStaleAlertInMaintenaceMode() { + // create current alerts that are not stale + AlertDefinitionEntity definition1 = new AlertDefinitionEntity(); + definition1.setClusterId(CLUSTER_ID); + definition1.setDefinitionName("foo-definition"); + definition1.setServiceName("HDFS"); + definition1.setComponentName("NAMENODE"); + definition1.setEnabled(true); + definition1.setScheduleInterval(1); + // create current alerts where 1 is stale but in maintence mode AlertCurrentEntity current1 = createNiceMock(AlertCurrentEntity.class); AlertHistoryEntity history1 = createNiceMock(AlertHistoryEntity.class); @@ -250,10 +275,10 @@ public class StaleAlertRunnableTest { AlertHistoryEntity history2 = createNiceMock(AlertHistoryEntity.class); expect(current1.getAlertHistory()).andReturn(history1).atLeastOnce(); - expect(history1.getAlertDefinition()).andReturn(m_definition).atLeastOnce(); + expect(history1.getAlertDefinition()).andReturn(definition1).atLeastOnce(); expect(current2.getAlertHistory()).andReturn(history2).atLeastOnce(); - expect(history2.getAlertDefinition()).andReturn(m_definition).atLeastOnce(); + expect(history2.getAlertDefinition()).andReturn(definition1).atLeastOnce(); // maintenance mode with a really old timestamp expect(current1.getMaintenanceState()).andReturn(MaintenanceState.ON).atLeastOnce(); @@ -292,7 +317,7 @@ public class StaleAlertRunnableTest { assertEquals(AlertState.OK, alert.getState()); assertEquals(DEFINITION_NAME, alert.getName()); - verify(m_definition, m_cluster, m_clusters, m_definitionDao); + verify(m_cluster, m_clusters, m_definitionDao); } /** http://git-wip-us.apache.org/repos/asf/ambari/blob/bdf5effb/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java index 32d6d1e..714b6b7 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java @@ -18,14 +18,30 @@ package org.apache.ambari.server.upgrade; -import com.google.inject.AbstractModule; -import com.google.inject.Binder; -import com.google.inject.Guice; -import com.google.inject.Injector; -import com.google.inject.Module; -import com.google.inject.Provider; -import com.google.inject.persist.PersistService; -import org.apache.ambari.server.api.services.AmbariMetaInfo; +import static junit.framework.Assert.assertEquals; +import static org.easymock.EasyMock.capture; +import static org.easymock.EasyMock.createMockBuilder; +import static org.easymock.EasyMock.createNiceMock; +import static org.easymock.EasyMock.createStrictMock; +import static org.easymock.EasyMock.eq; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.expectLastCall; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.reset; +import static org.easymock.EasyMock.verify; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.persistence.EntityManager; + import org.apache.ambari.server.configuration.Configuration; import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.orm.DBAccessor; @@ -34,42 +50,20 @@ import org.apache.ambari.server.orm.GuiceJpaInitializer; import org.apache.ambari.server.orm.InMemoryDefaultTestModule; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Clusters; -import org.apache.ambari.server.state.OperatingSystemInfo; -import org.apache.ambari.server.state.RepositoryInfo; import org.apache.ambari.server.state.Service; -import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.stack.OsFamily; import org.easymock.Capture; -import org.easymock.EasyMockSupport; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import javax.persistence.EntityManager; -import java.lang.reflect.Field; -import java.lang.reflect.Method; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; - -import static org.easymock.EasyMock.capture; - -import static junit.framework.Assert.assertEquals; -import static org.easymock.EasyMock.createMockBuilder; -import static org.easymock.EasyMock.createNiceMock; -import static org.easymock.EasyMock.createStrictMock; -import static org.easymock.EasyMock.eq; -import static org.easymock.EasyMock.expect; -import static org.easymock.EasyMock.expectLastCall; -import static org.easymock.EasyMock.replay; -import static org.easymock.EasyMock.reset; -import static org.easymock.EasyMock.verify; +import com.google.inject.Binder; +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.Module; +import com.google.inject.Provider; +import com.google.inject.persist.PersistService; /** * {@link org.apache.ambari.server.upgrade.UpgradeCatalog210} unit tests. @@ -136,13 +130,16 @@ public class UpgradeCatalog210Test { public void testExecuteDMLUpdates() throws Exception { Method addNewConfigurationsFromXml = AbstractUpgradeCatalog.class.getDeclaredMethod("addNewConfigurationsFromXml"); + Method initializeClusterAndServiceWidgets = UpgradeCatalog210.class.getDeclaredMethod("initializeClusterAndServiceWidgets"); + Method executeStackDMLUpdates = UpgradeCatalog210.class.getDeclaredMethod("executeStackDMLUpdates"); + UpgradeCatalog210 upgradeCatalog210 = createMockBuilder(UpgradeCatalog210.class) .addMockedMethod(addNewConfigurationsFromXml) .addMockedMethod(initializeClusterAndServiceWidgets) - .createMock(); + .addMockedMethod( executeStackDMLUpdates) .createMock(); upgradeCatalog210.addNewConfigurationsFromXml(); expectLastCall().once(); @@ -150,6 +147,9 @@ public class UpgradeCatalog210Test { upgradeCatalog210.initializeClusterAndServiceWidgets(); expectLastCall().once(); + upgradeCatalog210.executeStackDMLUpdates(); + expectLastCall().once(); + replay(upgradeCatalog210); upgradeCatalog210.executeDMLUpdates(); http://git-wip-us.apache.org/repos/asf/ambari/blob/bdf5effb/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeTest.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeTest.java index c1090c8..7455706 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeTest.java @@ -61,7 +61,6 @@ import org.apache.ambari.server.orm.dao.ViewInstanceDAO; import org.apache.ambari.server.utils.VersionUtils; import org.apache.ambari.server.view.ViewRegistry; import org.easymock.EasyMock; -import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -74,7 +73,6 @@ import com.google.inject.Key; import com.google.inject.TypeLiteral; import com.google.inject.persist.PersistService; -@Ignore @RunWith(Parameterized.class) public class UpgradeTest { private static final Logger LOG = LoggerFactory.getLogger(UpgradeTest.class);
