KNOX-1144
Project: http://git-wip-us.apache.org/repos/asf/knox/repo Commit: http://git-wip-us.apache.org/repos/asf/knox/commit/a438bcc1 Tree: http://git-wip-us.apache.org/repos/asf/knox/tree/a438bcc1 Diff: http://git-wip-us.apache.org/repos/asf/knox/diff/a438bcc1 Branch: refs/heads/KNOX-998-Package_Restructuring Commit: a438bcc1e4dec613f078ecea25bb177a3904a3a7 Parents: 7e03a9c Author: Phil Zampino <pzamp...@gmail.com> Authored: Tue Dec 12 10:11:25 2017 -0500 Committer: Phil Zampino <pzamp...@apache.org> Committed: Wed Jan 3 14:06:29 2018 -0500 ---------------------------------------------------------------------- .../topology/impl/DefaultTopologyService.java | 40 +++++++++++++++----- .../DefaultRemoteConfigurationMonitor.java | 22 ++++++++++- .../ZooKeeperConfigurationMonitorTest.java | 17 ++++++++- 3 files changed, 65 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/knox/blob/a438bcc1/gateway-server/src/main/java/org/apache/hadoop/gateway/services/topology/impl/DefaultTopologyService.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/services/topology/impl/DefaultTopologyService.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/services/topology/impl/DefaultTopologyService.java index aded6cd..398f3e9 100644 --- a/gateway-server/src/main/java/org/apache/hadoop/gateway/services/topology/impl/DefaultTopologyService.java +++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/services/topology/impl/DefaultTopologyService.java @@ -52,6 +52,8 @@ import org.apache.hadoop.gateway.topology.builder.TopologyBuilder; import org.apache.hadoop.gateway.topology.discovery.ClusterConfigurationMonitor; import org.apache.hadoop.gateway.topology.monitor.RemoteConfigurationMonitor; import org.apache.hadoop.gateway.topology.monitor.RemoteConfigurationMonitorFactory; +import org.apache.hadoop.gateway.topology.simple.SimpleDescriptor; +import org.apache.hadoop.gateway.topology.simple.SimpleDescriptorFactory; import org.apache.hadoop.gateway.topology.simple.SimpleDescriptorHandler; import org.apache.hadoop.gateway.topology.validation.TopologyValidator; import org.apache.hadoop.gateway.topology.xml.AmbariFormatXmlTopologyRules; @@ -592,18 +594,39 @@ public class DefaultTopologyService initListener(sharedProvidersDirectory, spm, spm); log.monitoringProviderConfigChangesInDirectory(sharedProvidersDirectory.getAbsolutePath()); - // For all the descriptors currently in the descriptors dir at start-up time, trigger topology generation. + // For all the descriptors currently in the descriptors dir at start-up time, determine if topology regeneration + // is required. // This happens prior to the start-up loading of the topologies. String[] descriptorFilenames = descriptorsDirectory.list(); if (descriptorFilenames != null) { for (String descriptorFilename : descriptorFilenames) { if (DescriptorsMonitor.isDescriptorFile(descriptorFilename)) { + String topologyName = FilenameUtils.getBaseName(descriptorFilename); + File existingDescriptorFile = getExistingFile(descriptorsDirectory, topologyName); + // If there isn't a corresponding topology file, or if the descriptor has been modified since the // corresponding topology file was generated, then trigger generation of one - File matchingTopologyFile = getExistingFile(topologiesDirectory, FilenameUtils.getBaseName(descriptorFilename)); - if (matchingTopologyFile == null || - matchingTopologyFile.lastModified() < (new File(descriptorsDirectory, descriptorFilename)).lastModified()) { - descriptorsMonitor.onFileChange(new File(descriptorsDirectory, descriptorFilename)); + File matchingTopologyFile = getExistingFile(topologiesDirectory, topologyName); + if (matchingTopologyFile == null || matchingTopologyFile.lastModified() < existingDescriptorFile.lastModified()) { + descriptorsMonitor.onFileChange(existingDescriptorFile); + } else { + // If regeneration is NOT required, then we at least need to report the provider configuration + // reference relationship (KNOX-1144) + String normalizedDescriptorPath = FilenameUtils.normalize(existingDescriptorFile.getAbsolutePath()); + + // Parse the descriptor to determine the provider config reference + SimpleDescriptor sd = SimpleDescriptorFactory.parse(normalizedDescriptorPath); + if (sd != null) { + File referencedProviderConfig = + getExistingFile(sharedProvidersDirectory, FilenameUtils.getBaseName(sd.getProviderConfig())); + if (referencedProviderConfig != null) { + List<String> references = + descriptorsMonitor.getReferencingDescriptors(referencedProviderConfig.getAbsolutePath()); + if (!references.contains(normalizedDescriptorPath)) { + references.add(normalizedDescriptorPath); + } + } + } } } } @@ -711,11 +734,8 @@ public class DefaultTopologyService } List<String> getReferencingDescriptors(String providerConfigPath) { - List<String> result = providerConfigReferences.get(FilenameUtils.normalize(providerConfigPath)); - if (result == null) { - result = Collections.emptyList(); - } - return result; + String normalizedPath = FilenameUtils.normalize(providerConfigPath); + return providerConfigReferences.computeIfAbsent(normalizedPath, p -> new ArrayList<>()); } @Override http://git-wip-us.apache.org/repos/asf/knox/blob/a438bcc1/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/monitor/DefaultRemoteConfigurationMonitor.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/monitor/DefaultRemoteConfigurationMonitor.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/monitor/DefaultRemoteConfigurationMonitor.java index af60058..3bf3330 100644 --- a/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/monitor/DefaultRemoteConfigurationMonitor.java +++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/monitor/DefaultRemoteConfigurationMonitor.java @@ -29,6 +29,7 @@ import org.apache.zookeeper.ZooDefs; import java.io.File; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -112,6 +113,19 @@ class DefaultRemoteConfigurationMonitor implements RemoteConfigurationMonitor { if (providerConfigs == null) { // Either the ZNode does not exist, or there is an authentication problem throw new IllegalStateException("Unable to access remote path: " + NODE_KNOX_PROVIDERS); + } else { + // Download any existing provider configs in the remote registry, which either do not exist locally, or have + // been modified, so that they are certain to be present when this monitor downloads any descriptors that + // reference them. + for (String providerConfig : providerConfigs) { + File localFile = new File(providersDir, providerConfig); + + byte[] remoteContent = client.getEntryData(NODE_KNOX_PROVIDERS + "/" + providerConfig).getBytes(); + if (!localFile.exists() || !Arrays.equals(remoteContent, FileUtils.readFileToByteArray(localFile))) { + FileUtils.writeByteArrayToFile(localFile, remoteContent); + log.downloadedRemoteConfigFile(providersDir.getName(), providerConfig); + } + } } // Confirm access to the remote descriptors directory znode @@ -213,8 +227,12 @@ class DefaultRemoteConfigurationMonitor implements RemoteConfigurationMonitor { File localFile = new File(localDir, path.substring(path.lastIndexOf("/"))); if (data != null) { try { - FileUtils.writeByteArrayToFile(localFile, data); - log.downloadedRemoteConfigFile(localDir.getName(), localFile.getName()); + // If there is no corresponding local file, or the content is different from the existing local + // file, write the data to the local file. + if (!localFile.exists() || !Arrays.equals(FileUtils.readFileToByteArray(localFile), data)) { + FileUtils.writeByteArrayToFile(localFile, data); + log.downloadedRemoteConfigFile(localDir.getName(), localFile.getName()); + } } catch (IOException e) { log.errorDownloadingRemoteConfiguration(path, e); } http://git-wip-us.apache.org/repos/asf/knox/blob/a438bcc1/gateway-server/src/test/java/org/apache/hadoop/gateway/topology/monitor/ZooKeeperConfigurationMonitorTest.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/test/java/org/apache/hadoop/gateway/topology/monitor/ZooKeeperConfigurationMonitorTest.java b/gateway-server/src/test/java/org/apache/hadoop/gateway/topology/monitor/ZooKeeperConfigurationMonitorTest.java index 1c4ed6e..ecf5b70 100644 --- a/gateway-server/src/test/java/org/apache/hadoop/gateway/topology/monitor/ZooKeeperConfigurationMonitorTest.java +++ b/gateway-server/src/test/java/org/apache/hadoop/gateway/topology/monitor/ZooKeeperConfigurationMonitorTest.java @@ -113,10 +113,10 @@ public class ZooKeeperConfigurationMonitorTest { client.create().creatingParentsIfNeeded().withMode(CreateMode.PERSISTENT).withACL(acls).forPath(PATH_KNOX_DESCRIPTORS); assertNotNull("Failed to create node:" + PATH_KNOX_DESCRIPTORS, - client.checkExists().forPath(PATH_KNOX_DESCRIPTORS)); + client.checkExists().forPath(PATH_KNOX_DESCRIPTORS)); client.create().creatingParentsIfNeeded().withMode(CreateMode.PERSISTENT).withACL(acls).forPath(PATH_KNOX_PROVIDERS); assertNotNull("Failed to create node:" + PATH_KNOX_PROVIDERS, - client.checkExists().forPath(PATH_KNOX_PROVIDERS)); + client.checkExists().forPath(PATH_KNOX_PROVIDERS)); } @AfterClass @@ -164,12 +164,25 @@ public class ZooKeeperConfigurationMonitorTest { DefaultRemoteConfigurationMonitor cm = new DefaultRemoteConfigurationMonitor(gc, clientService); + // Create a provider configuration in the test ZK, prior to starting the monitor, to make sure that the monitor + // will download existing entries upon starting. + final String preExistingProviderConfig = getProviderPath("pre-existing-providers.xml"); + client.create().withMode(CreateMode.PERSISTENT).forPath(preExistingProviderConfig, + TEST_PROVIDERS_CONFIG_1.getBytes()); + File preExistingProviderConfigLocalFile = new File(providersDir, "pre-existing-providers.xml"); + assertFalse("This file should not exist locally prior to monitor starting.", + preExistingProviderConfigLocalFile.exists()); + try { cm.start(); } catch (Exception e) { fail("Failed to start monitor: " + e.getMessage()); } + assertTrue("This file should exist locally immediately after monitor starting.", + preExistingProviderConfigLocalFile.exists()); + + try { final String pc_one_znode = getProviderPath("providers-config1.xml"); final File pc_one = new File(providersDir, "providers-config1.xml");