This is an automated email from the ASF dual-hosted git repository. anishek pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new d4af384 HIVE-24014: Need to delete DumpDirectoryCleanerTask (Arko Sharma, reviewed by Aasha Medhi) d4af384 is described below commit d4af3840f89408edea886a28ee4ae7d79a6f16f8 Author: Anishek Agarwal <anis...@gmail.com> AuthorDate: Tue Aug 11 16:48:10 2020 +0530 HIVE-24014: Need to delete DumpDirectoryCleanerTask (Arko Sharma, reviewed by Aasha Medhi) --- .../java/org/apache/hadoop/hive/conf/HiveConf.java | 6 -- .../hive/metastore/repl/DumpDirCleanerTask.java | 74 ---------------------- .../hadoop/hive/metastore/conf/MetastoreConf.java | 1 - 3 files changed, 81 deletions(-) diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 2e7247b..ab46865 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -536,12 +536,6 @@ public class HiveConf extends Configuration { REPL_FILE_LIST_CACHE_SIZE("hive.repl.file.list.cache.size", 10000, "This config indicates threshold for the maximum number of data copy locations to be kept in memory. \n" + "When the config 'hive.repl.data.copy.lazy' is set to true, this config is not considered."), - REPL_DUMPDIR_CLEAN_FREQ("hive.repl.dumpdir.clean.freq", "0s", - new TimeValidator(TimeUnit.SECONDS), - "Frequency at which timer task runs to purge expired dump dirs."), - REPL_DUMPDIR_TTL("hive.repl.dumpdir.ttl", "7d", - new TimeValidator(TimeUnit.DAYS), - "TTL of dump dirs before cleanup."), REPL_DUMP_METADATA_ONLY("hive.repl.dump.metadata.only", false, "Indicates whether replication dump only metadata information or data + metadata. \n" + "This config makes hive.repl.include.external.tables config ineffective."), diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/repl/DumpDirCleanerTask.java b/metastore/src/java/org/apache/hadoop/hive/metastore/repl/DumpDirCleanerTask.java deleted file mode 100644 index daf67e4..0000000 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/repl/DumpDirCleanerTask.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.metastore.repl; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.metastore.MetastoreTaskThread; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.concurrent.TimeUnit; - -public class DumpDirCleanerTask implements MetastoreTaskThread { - public static final Logger LOG = LoggerFactory.getLogger(DumpDirCleanerTask.class); - private Configuration conf; - private Path dumpRoot; - private long ttl; - - @Override - public void setConf(Configuration conf) { - this.conf = conf; - dumpRoot = new Path(HiveConf.getVar(conf, ConfVars.REPLDIR)); - ttl = HiveConf.getTimeVar(conf, ConfVars.REPL_DUMPDIR_TTL, TimeUnit.MILLISECONDS); - } - - @Override - public Configuration getConf() { - return conf; - } - - @Override - public long runFrequency(TimeUnit unit) { - return HiveConf.getTimeVar(conf, ConfVars.REPL_DUMPDIR_CLEAN_FREQ, unit); - } - - @Override - public void run() { - LOG.debug("Trying to delete old dump dirs"); - try { - FileSystem fs = FileSystem.get(dumpRoot.toUri(), conf); - FileStatus[] statuses = fs.listStatus(dumpRoot); - for (FileStatus status : statuses) - { - if (status.getModificationTime() < System.currentTimeMillis() - ttl) - { - fs.delete(status.getPath(), true); - LOG.info("Deleted old dump dir: " + status.getPath()); - } - } - } catch (IOException e) { - LOG.error("Error while trying to delete dump dir", e); - } - } -} diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index cc4e68f..0583208 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -1091,7 +1091,6 @@ public class MetastoreConf { + " quoted table names.\nThe default value is true."), TASK_THREADS_ALWAYS("metastore.task.threads.always", "metastore.task.threads.always", EVENT_CLEANER_TASK_CLASS + "," + RUNTIME_STATS_CLEANER_TASK_CLASS + "," + - "org.apache.hadoop.hive.metastore.repl.DumpDirCleanerTask" + "," + "org.apache.hadoop.hive.metastore.HiveProtoEventsCleanerTask" + "," + "org.apache.hadoop.hive.metastore.ScheduledQueryExecutionsMaintTask" + "," + "org.apache.hadoop.hive.metastore.ReplicationMetricsMaintTask",