This is an automated email from the ASF dual-hosted git repository.
shengkai pushed a commit to branch release-1.18
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 3dcdc7f29384bc399e65ce46253975570e93481f
Author: Shengkai <33114724+fsk...@users.noreply.github.com>
AuthorDate: Mon Aug 28 09:38:20 2023 +0800
[FLINK-32731][e2e] Add retry mechanism when fails to start the namenode
(#23267)
---
.../flink/tests/hive/containers/HiveContainer.java | 30 ++++++++++++++++++++++
.../table/gateway/containers/HiveContainer.java | 30 ++++++++++++++++++++++
2 files changed, 60 insertions(+)
diff --git
a/flink-end-to-end-tests/flink-end-to-end-tests-hive/src/test/java/org/apache/flink/tests/hive/containers/HiveContainer.java
b/flink-end-to-end-tests/flink-end-to-end-tests-hive/src/test/java/org/apache/flink/tests/hive/containers/HiveContainer.java
index 6edb7f46e8c..d4ca4080023 100644
---
a/flink-end-to-end-tests/flink-end-to-end-tests-hive/src/test/java/org/apache/flink/tests/hive/containers/HiveContainer.java
+++
b/flink-end-to-end-tests/flink-end-to-end-tests-hive/src/test/java/org/apache/flink/tests/hive/containers/HiveContainer.java
@@ -22,6 +22,11 @@ import
org.apache.flink.table.catalog.hive.client.HiveShimLoader;
import org.apache.flink.test.parameters.ParameterProperty;
import org.apache.flink.util.DockerImageVersions;
+import com.github.dockerjava.api.command.InspectContainerResponse;
+import okhttp3.FormBody;
+import okhttp3.OkHttpClient;
+import okhttp3.Request;
+import okhttp3.Response;
import org.junit.runner.Description;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -56,6 +61,7 @@ public class HiveContainer extends
GenericContainer<HiveContainer> {
private static final String MYSQL_METASTORE_LOG_PATH =
"/var/log/mysqld.log";
private static final ParameterProperty<Path>
DISTRIBUTION_LOG_BACKUP_DIRECTORY =
new ParameterProperty<>("logBackupDir", Paths::get);
+ private static final int NAME_NODE_WEB_PORT = 50070;
private String hiveWarehouseDir;
@@ -64,7 +70,9 @@ public class HiveContainer extends
GenericContainer<HiveContainer> {
DockerImageName.parse(
HIVE_310_OR_LATER ? DockerImageVersions.HIVE3 :
DockerImageVersions.HIVE2));
withExtraHost(HOST_NAME, "127.0.0.1");
+ withStartupAttempts(3);
addExposedPort(HIVE_METASTORE_PORT);
+ addExposedPort(NAME_NODE_WEB_PORT);
mountHiveWarehouseDirToContainer(initTableNames);
}
@@ -82,6 +90,28 @@ public class HiveContainer extends
GenericContainer<HiveContainer> {
super.finished(description);
}
+ @Override
+ protected void containerIsStarted(InspectContainerResponse containerInfo) {
+ super.containerIsStarted(containerInfo);
+ final Request request =
+ new Request.Builder()
+ .post(new FormBody.Builder().build())
+ .url(
+ String.format(
+ "http://127.0.0.1:%s";,
getMappedPort(NAME_NODE_WEB_PORT)))
+ .build();
+ OkHttpClient client = new OkHttpClient();
+ try (Response response = client.newCall(request).execute()) {
+ if (!response.isSuccessful()) {
+ throw new RuntimeException(
+ String.format(
+ "The rest request is not successful: %s",
response.message()));
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
public String getHiveMetastoreURL() {
return String.format("thrift://%s:%s", getHost(),
getMappedPort(HIVE_METASTORE_PORT));
}
diff --git
a/flink-end-to-end-tests/flink-sql-gateway-test/src/test/java/org/apache/flink/table/gateway/containers/HiveContainer.java
b/flink-end-to-end-tests/flink-sql-gateway-test/src/test/java/org/apache/flink/table/gateway/containers/HiveContainer.java
index c16aa41b8db..cb44cd4e517 100644
---
a/flink-end-to-end-tests/flink-sql-gateway-test/src/test/java/org/apache/flink/table/gateway/containers/HiveContainer.java
+++
b/flink-end-to-end-tests/flink-sql-gateway-test/src/test/java/org/apache/flink/table/gateway/containers/HiveContainer.java
@@ -20,6 +20,11 @@ package org.apache.flink.table.gateway.containers;
import org.apache.flink.test.parameters.ParameterProperty;
import org.apache.flink.util.DockerImageVersions;
+import com.github.dockerjava.api.command.InspectContainerResponse;
+import okhttp3.FormBody;
+import okhttp3.OkHttpClient;
+import okhttp3.Request;
+import okhttp3.Response;
import org.junit.runner.Description;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,6 +44,7 @@ public class HiveContainer extends
GenericContainer<HiveContainer> {
public static final String HOST_NAME = "hadoop-master";
public static final int HIVE_METASTORE_PORT = 9083;
+ private static final int NAME_NODE_WEB_PORT = 50070;
// Detailed log paths are from
//
https://github.com/prestodb/docker-images/tree/master/prestodb/hdp2.6-hive/files/etc/supervisord.d
@@ -53,7 +59,9 @@ public class HiveContainer extends
GenericContainer<HiveContainer> {
public HiveContainer() {
super(DockerImageName.parse(DockerImageVersions.HIVE2));
withExtraHost(HOST_NAME, "127.0.0.1");
+ withStartupAttempts(3);
addExposedPort(HIVE_METASTORE_PORT);
+ addExposedPort(NAME_NODE_WEB_PORT);
}
@Override
@@ -64,6 +72,28 @@ public class HiveContainer extends
GenericContainer<HiveContainer> {
}
}
+ @Override
+ protected void containerIsStarted(InspectContainerResponse containerInfo) {
+ super.containerIsStarted(containerInfo);
+ final Request request =
+ new Request.Builder()
+ .post(new FormBody.Builder().build())
+ .url(
+ String.format(
+ "http://127.0.0.1:%s";,
getMappedPort(NAME_NODE_WEB_PORT)))
+ .build();
+ OkHttpClient client = new OkHttpClient();
+ try (Response response = client.newCall(request).execute()) {
+ if (!response.isSuccessful()) {
+ throw new RuntimeException(
+ String.format(
+ "The rest request is not successful: %s",
response.message()));
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
@Override
protected void finished(Description description) {
backupLogs();
