Repository: hive Updated Branches: refs/heads/llap baddfa5cc -> 2024c962d
HIVE-10653 : LLAP: registry logs strange lines on daemons (Sergey Shelukhin) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2611ddb0 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2611ddb0 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2611ddb0 Branch: refs/heads/llap Commit: 2611ddb09b4e0b0d9345a2bee803702ee142fbf4 Parents: baddfa5 Author: Sergey Shelukhin <ser...@apache.org> Authored: Tue May 26 13:22:42 2015 -0700 Committer: Sergey Shelukhin <ser...@apache.org> Committed: Tue May 26 13:22:42 2015 -0700 ---------------------------------------------------------------------- .../hive/llap/daemon/impl/LlapDaemon.java | 2 +- .../registry/impl/LlapRegistryService.java | 11 +++---- .../registry/impl/LlapYarnRegistryImpl.java | 34 ++++++++++---------- .../dag/app/rm/LlapTaskSchedulerService.java | 2 +- 4 files changed, 24 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/2611ddb0/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java ---------------------------------------------------------------------- diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java index fabacf7..d55b7cf 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java @@ -152,7 +152,7 @@ public class LlapDaemon extends AbstractService implements ContainerRunner, Llap executorMemoryBytes, metrics); - this.registry = new LlapRegistryService(); + this.registry = new LlapRegistryService(true); } private void printAsciiArt() { http://git-wip-us.apache.org/repos/asf/hive/blob/2611ddb0/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapRegistryService.java ---------------------------------------------------------------------- diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapRegistryService.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapRegistryService.java index 79042c5..9f5702d 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapRegistryService.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapRegistryService.java @@ -43,27 +43,26 @@ import com.google.common.base.Preconditions; public class LlapRegistryService extends AbstractService { private static final Logger LOG = Logger.getLogger(LlapRegistryService.class); - + private ServiceRegistry registry = null; + private final boolean isDaemon; - public LlapRegistryService() { + public LlapRegistryService(boolean isDaemon) { super("LlapRegistryService"); + this.isDaemon = isDaemon; } @Override public void serviceInit(Configuration conf) { String hosts = conf.getTrimmed(LlapConfiguration.LLAP_DAEMON_SERVICE_HOSTS); if (hosts.startsWith("@")) { - registry = initRegistry(hosts.substring(1), conf); + registry = new LlapYarnRegistryImpl(hosts.substring(1), conf, isDaemon); } else { registry = new LlapFixedRegistryImpl(hosts, conf); } LOG.info("Using LLAP registry type " + registry); } - private ServiceRegistry initRegistry(String instanceName, Configuration conf) { - return new LlapYarnRegistryImpl(instanceName, conf); - } @Override public void serviceStart() throws Exception { http://git-wip-us.apache.org/repos/asf/hive/blob/2611ddb0/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapYarnRegistryImpl.java ---------------------------------------------------------------------- diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapYarnRegistryImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapYarnRegistryImpl.java index a903369..ace080e 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapYarnRegistryImpl.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapYarnRegistryImpl.java @@ -54,8 +54,7 @@ public class LlapYarnRegistryImpl implements ServiceRegistry { private static final Logger LOG = Logger.getLogger(LlapYarnRegistryImpl.class); - private RegistryOperationsService client; - private final String instanceName; + private final RegistryOperationsService client; private final Configuration conf; private final ServiceRecordMarshal encoder; private final String path; @@ -72,6 +71,7 @@ public class LlapYarnRegistryImpl implements ServiceRegistry { final ScheduledExecutorService refresher = Executors.newScheduledThreadPool(1, new ThreadFactoryBuilder().setDaemon(true).setNameFormat("LlapYarnRegistryRefresher").build()); final long refreshDelay; + private final boolean isDaemon; static { String localhost = "localhost"; @@ -83,11 +83,10 @@ public class LlapYarnRegistryImpl implements ServiceRegistry { hostname = localhost; } - public LlapYarnRegistryImpl(String instanceName, Configuration conf) { + public LlapYarnRegistryImpl(String instanceName, Configuration conf, boolean isDaemon) { LOG.info("Llap Registry is enabled with registryid: " + instanceName); this.conf = new Configuration(conf); - this.instanceName = instanceName; conf.addResource(YarnConfiguration.YARN_SITE_CONFIGURATION_FILE); // registry reference client = (RegistryOperationsService) RegistryOperationsFactory.createInstance(conf); @@ -97,6 +96,7 @@ public class LlapYarnRegistryImpl implements ServiceRegistry { refreshDelay = conf.getInt(LlapConfiguration.LLAP_DAEMON_SERVICE_REFRESH_INTERVAL, LlapConfiguration.LLAP_DAEMON_SERVICE_REFRESH_INTERVAL_DEFAULT); + this.isDaemon = isDaemon; Preconditions.checkArgument(refreshDelay > 0, "Refresh delay for registry has to be positive = %d", refreshDelay); } @@ -278,7 +278,7 @@ public class LlapYarnRegistryImpl implements ServiceRegistry { if (instances != null) { // deep-copy before modifying - Set<String> oldKeys = new HashSet(instances.keySet()); + Set<String> oldKeys = new HashSet<>(instances.keySet()); if (oldKeys.removeAll(latestKeys)) { // This is all the records which have not checked in, and are effectively dead. for (String k : oldKeys) { @@ -335,19 +335,19 @@ public class LlapYarnRegistryImpl implements ServiceRegistry { @Override public void start() { - if (client != null) { - client.start(); - refresher.scheduleWithFixedDelay(new Runnable() { - @Override - public void run() { - try { - instances.refresh(); - } catch (IOException ioe) { - LOG.warn("Could not refresh hosts during scheduled refresh", ioe); - } + if (client == null) return; + client.start(); + if (isDaemon) return; + refresher.scheduleWithFixedDelay(new Runnable() { + @Override + public void run() { + try { + instances.refresh(); + } catch (IOException ioe) { + LOG.warn("Could not refresh hosts during scheduled refresh", ioe); } - }, 0, refreshDelay, TimeUnit.SECONDS); - } + } + }, 0, refreshDelay, TimeUnit.SECONDS); } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/2611ddb0/llap-server/src/java/org/apache/tez/dag/app/rm/LlapTaskSchedulerService.java ---------------------------------------------------------------------- diff --git a/llap-server/src/java/org/apache/tez/dag/app/rm/LlapTaskSchedulerService.java b/llap-server/src/java/org/apache/tez/dag/app/rm/LlapTaskSchedulerService.java index fff4142..d355029 100644 --- a/llap-server/src/java/org/apache/tez/dag/app/rm/LlapTaskSchedulerService.java +++ b/llap-server/src/java/org/apache/tez/dag/app/rm/LlapTaskSchedulerService.java @@ -132,7 +132,7 @@ public class LlapTaskSchedulerService extends TaskSchedulerService { // Per Executor Thread private final Resource resourcePerExecutor; - private final LlapRegistryService registry = new LlapRegistryService(); + private final LlapRegistryService registry = new LlapRegistryService(false); private volatile ListenableFuture<Void> nodeEnablerFuture;
