This is an automated email from the ASF dual-hosted git repository.
shengkai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new 5654eb798c7 [FLINK-32731][e2e] Backup the hive-metastore, hdfs
namenode and mysql logs (#23155)
5654eb798c7 is described below
commit 5654eb798c744c924aff93d68ec3c4e413e75232
Author: Shengkai <33114724+fsk...@users.noreply.github.com>
AuthorDate: Wed Aug 9 15:46:07 2023 +0800
[FLINK-32731][e2e] Backup the hive-metastore, hdfs namenode and mysql logs
(#23155)
---
.../flink/tests/hive/containers/HiveContainer.java | 58 +++++++++++++++++++++-
.../table/gateway/containers/HiveContainer.java | 57 +++++++++++++++++++++
2 files changed, 113 insertions(+), 2 deletions(-)
diff --git
a/flink-end-to-end-tests/flink-end-to-end-tests-hive/src/test/java/org/apache/flink/tests/hive/containers/HiveContainer.java
b/flink-end-to-end-tests/flink-end-to-end-tests-hive/src/test/java/org/apache/flink/tests/hive/containers/HiveContainer.java
index 716f40d48e1..6edb7f46e8c 100644
---
a/flink-end-to-end-tests/flink-end-to-end-tests-hive/src/test/java/org/apache/flink/tests/hive/containers/HiveContainer.java
+++
b/flink-end-to-end-tests/flink-end-to-end-tests-hive/src/test/java/org/apache/flink/tests/hive/containers/HiveContainer.java
@@ -19,8 +19,10 @@
package org.apache.flink.tests.hive.containers;
import org.apache.flink.table.catalog.hive.client.HiveShimLoader;
+import org.apache.flink.test.parameters.ParameterProperty;
import org.apache.flink.util.DockerImageVersions;
+import org.junit.runner.Description;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testcontainers.containers.GenericContainer;
@@ -31,19 +33,31 @@ import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.List;
+import java.util.UUID;
/** Test container for Hive. */
public class HiveContainer extends GenericContainer<HiveContainer> {
private static final Logger LOG =
LoggerFactory.getLogger(HiveContainer.class);
public static final String HOST_NAME = "hadoop-master";
+ public static final int HIVE_METASTORE_PORT = 9083;
private static final boolean HIVE_310_OR_LATER =
HiveShimLoader.getHiveVersion().compareTo(HiveShimLoader.HIVE_VERSION_V3_1_0)
>= 0;
- private String hiveWarehouseDir;
- public static final int HIVE_METASTORE_PORT = 9083;
+ // Detailed log paths are from
+ //
https://github.com/prestodb/docker-images/tree/master/prestodb/hdp2.6-hive/files/etc/supervisord.d
+ //
https://github.com/prestodb/docker-images/blob/master/prestodb/hive3.1-hive/files/etc/supervisord.conf
+ private static final String NAME_NODE_LOG_PATH =
+ "/var/log/hadoop-hdfs/hadoop-hdfs-namenode.log";
+ private static final String METASTORE_LOG_PATH = "/tmp/hive/hive.log";
+ private static final String MYSQL_METASTORE_LOG_PATH =
"/var/log/mysqld.log";
+ private static final ParameterProperty<Path>
DISTRIBUTION_LOG_BACKUP_DIRECTORY =
+ new ParameterProperty<>("logBackupDir", Paths::get);
+
+ private String hiveWarehouseDir;
public HiveContainer(List<String> initTableNames) {
super(
@@ -62,6 +76,12 @@ public class HiveContainer extends
GenericContainer<HiveContainer> {
}
}
+ @Override
+ protected void finished(Description description) {
+ backupLogs();
+ super.finished(description);
+ }
+
public String getHiveMetastoreURL() {
return String.format("thrift://%s:%s", getHost(),
getMappedPort(HIVE_METASTORE_PORT));
}
@@ -101,4 +121,38 @@ public class HiveContainer extends
GenericContainer<HiveContainer> {
file.setWritable(true, false);
file.setExecutable(true, false);
}
+
+ private void backupLogs() {
+ Path path = DISTRIBUTION_LOG_BACKUP_DIRECTORY.get().orElse(null);
+ if (path == null) {
+ LOG.warn(
+ "Property {} not set, logs will not be backed up.",
+ DISTRIBUTION_LOG_BACKUP_DIRECTORY.getPropertyName());
+ return;
+ }
+ try {
+ Path dir =
+ Files.createDirectory(
+ Paths.get(
+ String.valueOf(path.toAbsolutePath()),
+ "hive-" + UUID.randomUUID()));
+ copyFileFromContainer(
+ NAME_NODE_LOG_PATH,
+
Files.createFile(Paths.get(dir.toAbsolutePath().toString(), "namenode.log"))
+ .toAbsolutePath()
+ .toString());
+ copyFileFromContainer(
+ METASTORE_LOG_PATH,
+
Files.createFile(Paths.get(dir.toAbsolutePath().toString(), "metastore.log"))
+ .toAbsolutePath()
+ .toString());
+ copyFileFromContainer(
+ MYSQL_METASTORE_LOG_PATH,
+
Files.createFile(Paths.get(dir.toAbsolutePath().toString(), "mysql.log"))
+ .toAbsolutePath()
+ .toString());
+ } catch (Throwable e) {
+ LOG.warn("Failed to backup logs...", e);
+ }
+ }
}
diff --git
a/flink-end-to-end-tests/flink-sql-gateway-test/src/test/java/org/apache/flink/table/gateway/containers/HiveContainer.java
b/flink-end-to-end-tests/flink-sql-gateway-test/src/test/java/org/apache/flink/table/gateway/containers/HiveContainer.java
index 35dc6325779..c16aa41b8db 100644
---
a/flink-end-to-end-tests/flink-sql-gateway-test/src/test/java/org/apache/flink/table/gateway/containers/HiveContainer.java
+++
b/flink-end-to-end-tests/flink-sql-gateway-test/src/test/java/org/apache/flink/table/gateway/containers/HiveContainer.java
@@ -17,14 +17,21 @@
package org.apache.flink.table.gateway.containers;
+import org.apache.flink.test.parameters.ParameterProperty;
import org.apache.flink.util.DockerImageVersions;
+import org.junit.runner.Description;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.output.Slf4jLogConsumer;
import org.testcontainers.utility.DockerImageName;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.UUID;
+
/** Test Container for hive. */
public class HiveContainer extends GenericContainer<HiveContainer> {
@@ -33,6 +40,16 @@ public class HiveContainer extends
GenericContainer<HiveContainer> {
public static final String HOST_NAME = "hadoop-master";
public static final int HIVE_METASTORE_PORT = 9083;
+ // Detailed log paths are from
+ //
https://github.com/prestodb/docker-images/tree/master/prestodb/hdp2.6-hive/files/etc/supervisord.d
+ private static final String NAME_NODE_LOG_PATH =
+ "/var/log/hadoop-hdfs/hadoop-hdfs-namenode.log";
+ private static final String METASTORE_LOG_PATH = "/tmp/hive/hive.log";
+ private static final String MYSQL_METASTORE_LOG_PATH =
"/var/log/mysqld.log";
+
+ private static final ParameterProperty<Path>
DISTRIBUTION_LOG_BACKUP_DIRECTORY =
+ new ParameterProperty<>("logBackupDir", Paths::get);
+
public HiveContainer() {
super(DockerImageName.parse(DockerImageVersions.HIVE2));
withExtraHost(HOST_NAME, "127.0.0.1");
@@ -47,7 +64,47 @@ public class HiveContainer extends
GenericContainer<HiveContainer> {
}
}
+ @Override
+ protected void finished(Description description) {
+ backupLogs();
+ super.finished(description);
+ }
+
public String getHiveMetastoreURI() {
return String.format("thrift://%s:%s", getHost(),
getMappedPort(HIVE_METASTORE_PORT));
}
+
+ private void backupLogs() {
+ Path path = DISTRIBUTION_LOG_BACKUP_DIRECTORY.get().orElse(null);
+ if (path == null) {
+ LOG.warn(
+ "Property {} not set, logs will not be backed up.",
+ DISTRIBUTION_LOG_BACKUP_DIRECTORY.getPropertyName());
+ return;
+ }
+ try {
+ Path dir =
+ Files.createDirectory(
+ Paths.get(
+ String.valueOf(path.toAbsolutePath()),
+ "hive-" + UUID.randomUUID()));
+ copyFileFromContainer(
+ NAME_NODE_LOG_PATH,
+
Files.createFile(Paths.get(dir.toAbsolutePath().toString(), "namenode.log"))
+ .toAbsolutePath()
+ .toString());
+ copyFileFromContainer(
+ METASTORE_LOG_PATH,
+
Files.createFile(Paths.get(dir.toAbsolutePath().toString(), "metastore.log"))
+ .toAbsolutePath()
+ .toString());
+ copyFileFromContainer(
+ MYSQL_METASTORE_LOG_PATH,
+
Files.createFile(Paths.get(dir.toAbsolutePath().toString(), "mysql.log"))
+ .toAbsolutePath()
+ .toString());
+ } catch (Throwable e) {
+ LOG.warn("Failed to backup logs...", e);
+ }
+ }
}
