This is an automated email from the ASF dual-hosted git repository.
jin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hugegraph-toolchain.git
The following commit(s) were added to refs/heads/master by this push:
new 39e46922 chore(loader): improve the perf in loader-ci (#724)
39e46922 is described below
commit 39e46922b719e7183eda4d95847d812b680c6cf7
Author: lokidundun <[email protected]>
AuthorDate: Wed Apr 15 14:18:24 2026 +0800
chore(loader): improve the perf in loader-ci (#724)
---
.github/workflows/loader-ci.yml | 21 ++++++++-
hugegraph-loader/assembly/travis/install-hadoop.sh | 55 +++++++++++++++++-----
.../travis/install-hugegraph-from-source.sh | 29 ++++++++----
hugegraph-loader/assembly/travis/install-mysql.sh | 18 +++++--
.../hugegraph/loader/source/jdbc/JDBCVendor.java | 5 ++
.../hugegraph/loader/test/functional/DBUtil.java | 4 +-
6 files changed, 107 insertions(+), 25 deletions(-)
diff --git a/.github/workflows/loader-ci.yml b/.github/workflows/loader-ci.yml
index b3e62b2f..d992117b 100644
--- a/.github/workflows/loader-ci.yml
+++ b/.github/workflows/loader-ci.yml
@@ -35,6 +35,14 @@ jobs:
strategy:
matrix:
JAVA_VERSION: ['11']
+ services:
+ mysql:
+ image: mysql:8.0
+ env:
+ MYSQL_ROOT_PASSWORD: root
+ ports:
+ - 3306:3306
+ options: --health-cmd="mysqladmin ping" --health-interval=10s
--health-timeout=5s --health-retries=3
steps:
- name: Fetch Code
@@ -55,6 +63,18 @@ jobs:
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
restore-keys: ${{ runner.os }}-m2
+ - name: Cache Hadoop
+ uses: actions/cache@v4
+ with:
+ path: ~/hadoop-3.3.6.tar.gz
+ key: ${{ runner.os }}-hadoop-3.3.6
+
+ - name: Cache HugeGraph Server
+ uses: actions/cache@v4
+ with:
+ path: ~/hugegraph-cache-${{ env.COMMIT_ID }}
+ key: ${{ runner.os }}-hugegraph-server-${{ env.COMMIT_ID }}
+
- name: use staged maven repo settings
if: ${{ env.USE_STAGE == 'true' }}
run: |
@@ -68,7 +88,6 @@ jobs:
- name: Prepare env and service
run: |
$TRAVIS_DIR/install-hadoop.sh
- $TRAVIS_DIR/install-mysql.sh ${{ env.DB_DATABASE }} ${{ env.DB_PASS
}}
$TRAVIS_DIR/install-hugegraph-from-source.sh $COMMIT_ID
- name: Run test
diff --git a/hugegraph-loader/assembly/travis/install-hadoop.sh
b/hugegraph-loader/assembly/travis/install-hadoop.sh
index cae279ae..e439411c 100755
--- a/hugegraph-loader/assembly/travis/install-hadoop.sh
+++ b/hugegraph-loader/assembly/travis/install-hadoop.sh
@@ -17,22 +17,52 @@
#
set -ev
-sudo wget
https://archive.apache.org/dist/hadoop/common/hadoop-2.8.5/hadoop-2.8.5.tar.gz
+# Upgrade stable version to 3.3.6
+HADOOP_VERSION="3.3.6"
+HADOOP_TARBALL="hadoop-${HADOOP_VERSION}.tar.gz"
+HADOOP_HOME="/usr/local/hadoop"
+HADOOP_TARBALL_PATH="${HOME}/${HADOOP_TARBALL}"
-sudo tar -zxf hadoop-2.8.5.tar.gz -C /usr/local
-cd /usr/local
-sudo mv hadoop-2.8.5 hadoop
-#sudo chown -R travis ./hadoop
-cd hadoop
+if [[ ! -d "${HADOOP_HOME}" ]]; then
+ if [[ ! -f "${HADOOP_TARBALL_PATH}" ]]; then
+ echo "Downloading Hadoop ${HADOOP_VERSION}..."
+ wget -O "${HADOOP_TARBALL_PATH}"
"https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/${HADOOP_TARBALL}";
+ else
+ echo "Using cached Hadoop tarball at ${HADOOP_TARBALL_PATH}"
+ fi
+ echo "Extracting Hadoop to ${HADOOP_HOME}..."
+ sudo tar -zxf "${HADOOP_TARBALL_PATH}" -C /usr/local
+ cd /usr/local
+ sudo mv "hadoop-${HADOOP_VERSION}" hadoop
+ sudo chown -R "$(whoami):$(whoami)" "${HADOOP_HOME}"
+else
+ echo "Hadoop already installed at ${HADOOP_HOME}, skipping download and
extraction."
+fi
+
+cd "${HADOOP_HOME}"
pwd
-echo "export HADOOP_HOME=/usr/local/hadoop" >> ~/.bashrc
-echo "export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native" >> ~/.bashrc
-echo "export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin" >> ~/.bashrc
+# Export for GitHub Actions subsequent steps
+if [[ -n "${GITHUB_ENV:-}" ]]; then
+ echo "HADOOP_HOME=${HADOOP_HOME}" >> "${GITHUB_ENV}"
+ echo "HADOOP_COMMON_LIB_NATIVE_DIR=${HADOOP_HOME}/lib/native" >>
"${GITHUB_ENV}"
+ echo "PATH=${PATH}:${HADOOP_HOME}/bin:${HADOOP_HOME}/sbin" >>
"${GITHUB_ENV}"
+fi
+
+if ! grep -qxF "export HADOOP_HOME=${HADOOP_HOME}" ~/.bashrc; then
+ echo "export HADOOP_HOME=${HADOOP_HOME}" >> ~/.bashrc
+fi
+if ! grep -qxF "export HADOOP_COMMON_LIB_NATIVE_DIR=${HADOOP_HOME}/lib/native"
~/.bashrc; then
+ echo "export HADOOP_COMMON_LIB_NATIVE_DIR=${HADOOP_HOME}/lib/native" >>
~/.bashrc
+fi
+if ! grep -qxF "export PATH=\$PATH:${HADOOP_HOME}/bin:${HADOOP_HOME}/sbin"
~/.bashrc; then
+ echo "export PATH=\$PATH:${HADOOP_HOME}/bin:${HADOOP_HOME}/sbin" >>
~/.bashrc
+fi
source ~/.bashrc
-tee etc/hadoop/core-site.xml <<EOF
+if [[ ! -f etc/hadoop/core-site.xml ]] || ! grep -q "hdfs://localhost:8020"
etc/hadoop/core-site.xml; then
+ sudo tee etc/hadoop/core-site.xml > /dev/null <<EOF
<configuration>
<property>
<name>fs.defaultFS</name>
@@ -40,8 +70,10 @@ tee etc/hadoop/core-site.xml <<EOF
</property>
</configuration>
EOF
+fi
-tee etc/hadoop/hdfs-site.xml <<EOF
+if [[ ! -f etc/hadoop/hdfs-site.xml ]] || ! grep -q "/opt/hdfs/name"
etc/hadoop/hdfs-site.xml; then
+ sudo tee etc/hadoop/hdfs-site.xml > /dev/null <<EOF
<configuration>
<property>
<name>dfs.namenode.name.dir</name>
@@ -61,6 +93,7 @@ tee etc/hadoop/hdfs-site.xml <<EOF
</property>
</configuration>
EOF
+fi
bin/hdfs namenode -format
sbin/hadoop-daemon.sh start namenode
diff --git a/hugegraph-loader/assembly/travis/install-hugegraph-from-source.sh
b/hugegraph-loader/assembly/travis/install-hugegraph-from-source.sh
index 354acd95..8884c126 100755
--- a/hugegraph-loader/assembly/travis/install-hugegraph-from-source.sh
+++ b/hugegraph-loader/assembly/travis/install-hugegraph-from-source.sh
@@ -23,15 +23,28 @@ fi
COMMIT_ID=$1
HUGEGRAPH_GIT_URL="https://github.com/apache/hugegraph.git";
+CACHE_DIR="${HOME}/hugegraph-cache-${COMMIT_ID}"
+
+mkdir -p "${CACHE_DIR}"
+
+CACHED_TARBALL=$(find "${CACHE_DIR}" -maxdepth 1 -name
"apache-hugegraph-*.tar.gz" -print -quit)
+
+if [[ -f "${CACHED_TARBALL}" ]]; then
+ echo "Found cached HugeGraph server tarball, skipping build."
+ cp "${CACHED_TARBALL}" ./
+else
+ echo "Building HugeGraph server from source (commit ${COMMIT_ID})..."
+ git clone --depth 150 ${HUGEGRAPH_GIT_URL} hugegraph
+ cd hugegraph
+ git checkout "${COMMIT_ID}"
+ mvn package -DskipTests -Dmaven.javadoc.skip=true -ntp
+ cd hugegraph-server
+ mv apache-hugegraph-*.tar.gz ../../
+ cd ../../
+ rm -rf hugegraph
+ cp apache-hugegraph-*.tar.gz "${CACHE_DIR}"/
+fi
-git clone --depth 150 ${HUGEGRAPH_GIT_URL} hugegraph
-cd hugegraph
-git checkout "${COMMIT_ID}"
-mvn package -DskipTests -Dmaven.javadoc.skip=true -ntp
-cd hugegraph-server
-mv apache-hugegraph-*.tar.gz ../../
-cd ../../
-rm -rf hugegraph
tar zxf apache-hugegraph-*.tar.gz
HTTPS_SERVER_DIR="hugegraph_https"
diff --git a/hugegraph-loader/assembly/travis/install-mysql.sh
b/hugegraph-loader/assembly/travis/install-mysql.sh
index de1aef2f..779fb2f0 100755
--- a/hugegraph-loader/assembly/travis/install-mysql.sh
+++ b/hugegraph-loader/assembly/travis/install-mysql.sh
@@ -15,6 +15,10 @@
# License for the specific language governing permissions and limitations
# under the License.
#
+# NOTE: This script is NO LONGER actively used in CI.
+# It has been replaced by the MySQL service container in:
+# .github/workflows/loader-ci.yml
+# Kept temporarily as a backup / for legacy Travis compatibility only.
set -ev
TRAVIS_DIR=$(dirname $0)
@@ -23,12 +27,20 @@ TRAVIS_DIR=$(dirname $0)
CONF=hugegraph-test/src/main/resources/hugegraph.properties
MYSQL_USERNAME=root
-# Set MySQL configurations
+DB_NAME="${1:-mysql}"
+DB_PASS="${2:-root}"
+
+# Skip if MySQL is already running (e.g., provided by GitHub Actions service)
+if (echo >/dev/tcp/localhost/3306) >/dev/null 2>&1; then
+ echo "MySQL is already reachable on port 3306, skipping setup."
+ exit 0
+fi
+# Set MySQL configurations
# Keep for upgrade in future
-docker pull mysql:5.7
-docker run -p 3306:3306 --name "$1" -e MYSQL_ROOT_PASSWORD="$2" -d mysql:5.7
+docker pull mysql:8.0
+docker run -p 3306:3306 --name "${DB_NAME}" -e
MYSQL_ROOT_PASSWORD="${DB_PASS}" -d mysql:8.0
# Old Version
diff --git
a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/source/jdbc/JDBCVendor.java
b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/source/jdbc/JDBCVendor.java
index dc5833b6..a9a84eed 100644
---
a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/source/jdbc/JDBCVendor.java
+++
b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/source/jdbc/JDBCVendor.java
@@ -53,6 +53,11 @@ public enum JDBCVendor {
return super.checkSchema(source);
}
+ @Override
+ public String buildUrl(JDBCSource source) {
+ return super.buildUrl(source) + "&allowPublicKeyRetrieval=true";
+ }
+
@Override
public String buildGetHeaderSql(JDBCSource source) {
return String.format("SELECT COLUMN_NAME " +
diff --git
a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/functional/DBUtil.java
b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/functional/DBUtil.java
index 49b41db7..67fa0d54 100644
---
a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/functional/DBUtil.java
+++
b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/functional/DBUtil.java
@@ -43,7 +43,7 @@ public class DBUtil {
public void connect(boolean useSSL) {
try {
Class.forName(this.driver);
- String url = String.format("%s?useSSL=%s", this.url, useSSL);
+ String url =
String.format("%s?useSSL=%s&allowPublicKeyRetrieval=true", this.url, useSSL);
this.conn = DriverManager.getConnection(url, this.user, this.pass);
} catch (ClassNotFoundException e) {
throw new LoadException("Invalid driver class '%s'", e,
this.driver);
@@ -54,7 +54,7 @@ public class DBUtil {
public void connect(String database, boolean useSSL) {
this.close();
- String url = String.format("%s/%s?useSSL=%s", this.url, database,
useSSL);
+ String url =
String.format("%s/%s?useSSL=%s&allowPublicKeyRetrieval=true", this.url,
database, useSSL);
try {
Class.forName(this.driver);
this.conn = DriverManager.getConnection(url, this.user, this.pass);
