Repository: ambari Updated Branches: refs/heads/trunk b11a1f0ba -> f4877c8be
AMBARI-20858. Reupload missing config files for Log Search collections (oleewere) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f4877c8b Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f4877c8b Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f4877c8b Branch: refs/heads/trunk Commit: f4877c8be7dfc456c1126123dfe3a8e064670d65 Parents: b11a1f0 Author: oleewere <oleew...@gmail.com> Authored: Thu Apr 27 13:19:36 2017 +0200 Committer: oleewere <oleew...@gmail.com> Committed: Thu Apr 27 13:20:13 2017 +0200 ---------------------------------------------------------------------- .../configurer/SolrCollectionConfigurer.java | 6 +- .../ambari/logsearch/dao/AuditSolrDao.java | 2 +- .../logsearch/dao/ServiceLogsSolrDao.java | 2 +- .../ambari/logsearch/dao/UserConfigSolrDao.java | 2 +- .../handler/AbstractSolrConfigHandler.java | 6 ++ .../handler/UploadConfigurationHandler.java | 60 +++++++++++++++----- 6 files changed, 59 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/f4877c8b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java index 2f56812..f2d022e 100644 --- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java @@ -54,9 +54,11 @@ public class SolrCollectionConfigurer implements Configurer { private static final int CONNECTION_TIMEOUT = 30000; private final SolrDaoBase solrDaoBase; + private final boolean hasEnumConfig; // enumConfig.xml for solr collection - public SolrCollectionConfigurer(final SolrDaoBase solrDaoBase) { + public SolrCollectionConfigurer(final SolrDaoBase solrDaoBase, final boolean hasEnumConfig) { this.solrDaoBase = solrDaoBase; + this.hasEnumConfig = hasEnumConfig; } @Override @@ -100,7 +102,7 @@ public class SolrCollectionConfigurer implements Configurer { } private boolean uploadConfigurationsIfNeeded(CloudSolrClient cloudSolrClient, File configSetFolder, SolrCollectionState state, SolrPropsConfig solrPropsConfig) throws Exception { - boolean reloadCollectionNeeded = new UploadConfigurationHandler(configSetFolder).handle(cloudSolrClient, solrPropsConfig); + boolean reloadCollectionNeeded = new UploadConfigurationHandler(configSetFolder, hasEnumConfig).handle(cloudSolrClient, solrPropsConfig); if (!state.isConfigurationUploaded()) { state.setConfigurationUploaded(true); } http://git-wip-us.apache.org/repos/asf/ambari/blob/f4877c8b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java index d058383..3eea08f 100644 --- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java @@ -69,7 +69,7 @@ public class AuditSolrDao extends SolrDaoBase { String rangerAuditCollection = solrAuditLogPropsConfig.getRangerCollection(); try { - new SolrCollectionConfigurer(this).start(); + new SolrCollectionConfigurer(this, true).start(); boolean createAlias = (aliasNameIn != null && StringUtils.isNotBlank(rangerAuditCollection)); if (createAlias) { new SolrAuditAliasConfigurer(this).start(); http://git-wip-us.apache.org/repos/asf/ambari/blob/f4877c8b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java index 37375dc..308ef1f 100644 --- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java @@ -65,7 +65,7 @@ public class ServiceLogsSolrDao extends SolrDaoBase { public void postConstructor() { LOG.info("postConstructor() called."); try { - new SolrCollectionConfigurer(this).start(); + new SolrCollectionConfigurer(this, true).start(); } catch (Exception e) { LOG.error("error while connecting to Solr for service logs : solrUrl=" + solrServiceLogPropsConfig.getSolrUrl() + ", zkConnectString=" + solrServiceLogPropsConfig.getZkConnectString() http://git-wip-us.apache.org/repos/asf/ambari/blob/f4877c8b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java index 256ddae..8fb27a7 100644 --- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java @@ -98,7 +98,7 @@ public class UserConfigSolrDao extends SolrDaoBase { String collection = solrUserConfig.getCollection(); try { - new SolrCollectionConfigurer(this).start(); + new SolrCollectionConfigurer(this, false).start(); new LogfeederFilterConfigurer(this).start(); } catch (Exception e) { LOG.error("error while connecting to Solr for history logs : solrUrl=" + solrUrl + ", zkConnectString=" + zkConnectString + http://git-wip-us.apache.org/repos/asf/ambari/blob/f4877c8b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/AbstractSolrConfigHandler.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/AbstractSolrConfigHandler.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/AbstractSolrConfigHandler.java index 8693157..6f3d8ca 100644 --- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/AbstractSolrConfigHandler.java +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/AbstractSolrConfigHandler.java @@ -55,9 +55,11 @@ public abstract class AbstractSolrConfigHandler implements SolrZkRequestHandler< ZkConfigManager zkConfigManager = new ZkConfigManager(zkClient); boolean configExists = zkConfigManager.configExists(solrPropsConfig.getConfigName()); if (configExists) { + uploadMissingConfigFiles(zkClient, zkConfigManager, solrPropsConfig.getConfigName()); reloadCollectionNeeded = doIfConfigExists(solrPropsConfig, zkClient, separator, downloadFolderLocation, tmpDir); } else { doIfConfigNotExist(solrPropsConfig, zkConfigManager); + uploadMissingConfigFiles(zkClient, zkConfigManager, solrPropsConfig.getConfigName()); } } catch (Exception e) { throw new RuntimeException(String.format("Cannot upload configurations to zk. (collection: %s, config set folder: %s)", @@ -90,6 +92,10 @@ public abstract class AbstractSolrConfigHandler implements SolrZkRequestHandler< // Do nothing }; + public void uploadMissingConfigFiles(SolrZkClient zkClient, ZkConfigManager zkConfigManager, String configName) throws IOException { + // do Nothing + } + public boolean doIfConfigExists(SolrPropsConfig solrPropsConfig, SolrZkClient zkClient, String separator, String downloadFolderLocation, File tmpDir) throws IOException { boolean result = false; LOG.info("Config set exists for '{}' collection. Refreshing it if needed...", solrPropsConfig.getCollection()); http://git-wip-us.apache.org/repos/asf/ambari/blob/f4877c8b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/UploadConfigurationHandler.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/UploadConfigurationHandler.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/UploadConfigurationHandler.java index 6964951..f48b7db 100644 --- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/UploadConfigurationHandler.java +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/UploadConfigurationHandler.java @@ -28,17 +28,25 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.nio.file.FileSystems; public class UploadConfigurationHandler extends AbstractSolrConfigHandler { private static final Logger LOG = LoggerFactory.getLogger(UploadConfigurationHandler.class); private static final String SOLR_CONFIG_FILE = "solrconfig.xml"; + private static final String[] configFiles = { + "admin-extra.html", "admin-extra.menu-bottom.html", "admin-extra.menu-top.html", + "elevate.xml", "enumsConfig.xml", "managed-schema", "solrconfig.xml" + }; + private boolean hasEnumConfig; - public UploadConfigurationHandler(File configSetFolder) { + public UploadConfigurationHandler(File configSetFolder, boolean hasEnumConfig) { super(configSetFolder); + this.hasEnumConfig = hasEnumConfig; } @Override @@ -51,19 +59,7 @@ public class UploadConfigurationHandler extends AbstractSolrConfigHandler { zkConfigManager.uploadConfigDir(getConfigSetFolder().toPath(), solrPropsConfig.getConfigName()); String filePath = String.format("%s%s%s", getConfigSetFolder(), separator, getConfigFileName()); String configsPath = String.format("/%s/%s/%s", "configs", solrPropsConfig.getConfigName(), getConfigFileName()); - InputStream is = new FileInputStream(filePath); - try { - if (zkClient.exists(configsPath, true)) { - zkClient.setData(configsPath, IOUtils.toByteArray(is), true); - } else { - zkClient.create(configsPath, IOUtils.toByteArray(is), CreateMode.PERSISTENT, true); - } - } catch (Exception e) { - throw new IllegalStateException(e); - } - finally { - IOUtils.closeQuietly(is); - } + uploadFileToZk(zkClient, filePath, configsPath); result = true; } return result; @@ -82,4 +78,40 @@ public class UploadConfigurationHandler extends AbstractSolrConfigHandler { public String getConfigFileName() { return SOLR_CONFIG_FILE; } + + @Override + public void uploadMissingConfigFiles(SolrZkClient zkClient, ZkConfigManager zkConfigManager, String configName) throws IOException { + LOG.info("Check any of the configs files are missing for config ({})", configName); + for (String configFile : configFiles) { + if ("enumsConfig.xml".equals(configFile) && !hasEnumConfig) { + LOG.info("Config file ({}) is not needed for {}", configFile, configName); + continue; + } + String zkPath = String.format("%s/%s", configName, configFile); + if (zkConfigManager.configExists(zkPath)) { + LOG.info("Config file ({}) has already uploaded properly.", configFile); + } else { + LOG.info("Config file ({}) is missing. Reupload...", configFile); + FileSystems.getDefault().getSeparator(); + uploadFileToZk(zkClient, + String.format("%s%s%s", getConfigSetFolder(), FileSystems.getDefault().getSeparator(), configFile), + String.format("%s%s", "/configs/", zkPath)); + } + } + } + + private void uploadFileToZk(SolrZkClient zkClient, String filePath, String configsPath) throws FileNotFoundException { + InputStream is = new FileInputStream(filePath); + try { + if (zkClient.exists(configsPath, true)) { + zkClient.setData(configsPath, IOUtils.toByteArray(is), true); + } else { + zkClient.create(configsPath, IOUtils.toByteArray(is), CreateMode.PERSISTENT, true); + } + } catch (Exception e) { + throw new IllegalStateException(e); + } finally { + IOUtils.closeQuietly(is); + } + } }