HIVE-9911. LLAP: Clean up structures and intermediate data when a query completes. (Siddharth Seth)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/cae3ec16 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/cae3ec16 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/cae3ec16 Branch: refs/heads/llap Commit: cae3ec16b2fad8d6b99d315f1a4c40165d492683 Parents: 6547c0c Author: Siddharth Seth <ss...@apache.org> Authored: Wed Apr 29 08:21:47 2015 -0700 Committer: Siddharth Seth <ss...@apache.org> Committed: Wed Apr 29 08:21:47 2015 -0700 ---------------------------------------------------------------------- .../llap/configuration/LlapConfiguration.java | 6 + .../daemon/rpc/LlapDaemonProtocolProtos.java | 2765 +++++++++++++++++- .../org/apache/hadoop/hive/llap/LlapNodeId.java | 5 +- .../hive/llap/daemon/ContainerRunner.java | 7 + .../llap/daemon/impl/ContainerRunnerImpl.java | 70 +- .../hive/llap/daemon/impl/LlapDaemon.java | 19 +- .../impl/LlapDaemonProtocolClientImpl.java | 27 + .../impl/LlapDaemonProtocolServerImpl.java | 20 + .../hive/llap/daemon/impl/QueryFileCleaner.java | 94 + .../hive/llap/daemon/impl/QueryTracker.java | 143 + .../hive/llap/shufflehandler/DirWatcher.java | 22 +- .../llap/shufflehandler/ShuffleHandler.java | 62 +- .../llap/tezplugins/LlapTaskCommunicator.java | 56 +- .../hive/llap/tezplugins/TaskCommunicator.java | 40 +- .../src/protobuf/LlapDaemonProtocol.proto | 23 + 15 files changed, 3270 insertions(+), 89 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/cae3ec16/llap-client/src/java/org/apache/hadoop/hive/llap/configuration/LlapConfiguration.java ---------------------------------------------------------------------- diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/configuration/LlapConfiguration.java b/llap-client/src/java/org/apache/hadoop/hive/llap/configuration/LlapConfiguration.java index f03c807..f6ec119 100644 --- a/llap-client/src/java/org/apache/hadoop/hive/llap/configuration/LlapConfiguration.java +++ b/llap-client/src/java/org/apache/hadoop/hive/llap/configuration/LlapConfiguration.java @@ -65,8 +65,14 @@ public class LlapConfiguration extends Configuration { public static final String LLAP_DAEMON_VCPUS_PER_INSTANCE = LLAP_DAEMON_PREFIX + "vcpus.per.instance"; public static final int LLAP_DAEMON_VCPUS_PER_INSTANCE_DEFAULT = 4; + public static final String LLAP_DAEMON_NUM_FILE_CLEANER_THREADS = LLAP_DAEMON_PREFIX + "num.file.cleaner.threads"; + public static final int LLAP_DAEMON_NUM_FILE_CLEANER_THREADS_DEFAULT = 1; + // Section for configs used in the AM // + public static final String LLAP_FILE_CLEANUP_DELAY_SECONDS = LLAP_PREFIX + "file.cleanup.delay-seconds"; + public static final long LLAP_FILE_CLEANUP_DELAY_SECONDS_DEFAULT = 300; // 5 minutes by default + public static final String LLAP_DAEMON_SERVICE_HOSTS = LLAP_DAEMON_PREFIX + "service.hosts"; public static final String LLAP_DAEMON_SERVICE_REFRESH_INTERVAL = LLAP_DAEMON_PREFIX + "service.refresh.interval";
