This is an automated email from the ASF dual-hosted git repository.
xianjin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git
The following commit(s) were added to refs/heads/master by this push:
new 7f74dafc2 [#1854] feat(script): Support launching
coordinator/shuffle-server/dashboard using IntelliJ IDEA (#1855)
7f74dafc2 is described below
commit 7f74dafc2050edc45b2b0ce4b05cf31e91977a10
Author: maobaolong <baoloong...@tencent.com>
AuthorDate: Thu Jul 4 23:07:58 2024 +0800
[#1854] feat(script): Support launching
coordinator/shuffle-server/dashboard using IntelliJ IDEA (#1855)
### What changes were proposed in this pull request?
Involve a install script to copy launch configurations to intellij project
work area.
### Why are the changes needed?
Fix: #1854
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
<img width="1234" alt="image"
src="https://github.com/apache/incubator-uniffle/assets/17329931/8c177ac5-8761-4a73-a64a-e036815679ad";>
<img width="578" alt="image"
src="https://github.com/apache/incubator-uniffle/assets/17329931/e1fbcc50-605b-4183-b1b4-5c8081556dd7";>
<img width="269" alt="image"
src="https://github.com/apache/incubator-uniffle/assets/17329931/8f8b35be-1006-4cd0-80db-0756a7c17d75";>
<img width="269" alt="image"
src="https://github.com/apache/incubator-uniffle/assets/17329931/afa68182-6be1-4a40-adc0-eb1a6b528cc2";>
Enjoy it.
---
conf/local_dev/coordinator.conf | 23 ++++++++++
conf/local_dev/dashboard.conf | 20 +++++++++
conf/local_dev/log4j2.xml | 49 ++++++++++++++++++++++
conf/local_dev/server.conf | 32 ++++++++++++++
dev/intellij/install-runconfig.sh | 28 +++++++++++++
.../runConfigurations/CoordinatorServer.run.xml | 34 +++++++++++++++
.../runConfigurations/JettyServerFront.run.xml | 34 +++++++++++++++
.../runConfigurations/ShuffleServer.run.xml | 34 +++++++++++++++
8 files changed, 254 insertions(+)
diff --git a/conf/local_dev/coordinator.conf b/conf/local_dev/coordinator.conf
new file mode 100644
index 000000000..9bcad0921
--- /dev/null
+++ b/conf/local_dev/coordinator.conf
@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+rss.rpc.server.port 19999
+rss.jetty.http.port 19998
+rss.coordinator.server.heartbeat.timeout 30000
+rss.coordinator.app.expired 60000
+rss.coordinator.shuffle.nodes.max 13
+rss.coordinator.select.partition.strategy CONTINUOUS
diff --git a/conf/local_dev/dashboard.conf b/conf/local_dev/dashboard.conf
new file mode 100644
index 000000000..9663fe251
--- /dev/null
+++ b/conf/local_dev/dashboard.conf
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+rss.dashboard.http.port 19997
+# change to the actual coordinator web addresses
+coordinator.web.address http://localhost:19998,http://localhost:19998
diff --git a/conf/local_dev/log4j2.xml b/conf/local_dev/log4j2.xml
new file mode 100644
index 000000000..0ad07dd0c
--- /dev/null
+++ b/conf/local_dev/log4j2.xml
@@ -0,0 +1,49 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+-->
+<Configuration status="WARN" monitorInterval="30">
+ <Appenders>
+ <Console name="console" target="SYSTEM_ERR">
+ <PatternLayout pattern="[%d{yyyy-MM-dd HH:mm:ss.SSS}] [%t] [%p] %c{1}.%M
- %m%n%ex"/>
+ </Console>
+ <RollingFile name="RollingAppender" fileName="${sys:log.path}"
filePattern="${sys:log.path}.%i">
+ <PatternLayout pattern="[%d{yyyy-MM-dd HH:mm:ss.SSS}] [%t] [%p] %c{1}.%M
- %m%n%ex"/>
+ <Policies>
+ <SizeBasedTriggeringPolicy size="2GB"/>
+ </Policies>
+ <DefaultRolloverStrategy max="10"/>
+ </RollingFile>
+ </Appenders>
+ <Loggers>
+ <Root level="info">
+ <AppenderRef ref="console"/>
+ <AppenderRef ref="RollingAppender"/>
+ </Root>
+ <Logger name="io.grpc.netty.shaded.io.grpc.netty" level="info"
additivity="false">
+ <AppenderRef ref="console"/>
+ <AppenderRef ref="RollingAppender"/>
+ </Logger>
+ <Logger name="org.apache.hadoop" level="info" additivity="false">
+ <AppenderRef ref="console"/>
+ <AppenderRef ref="RollingAppender"/>
+ </Logger>
+ <Logger name="org.eclipse.jetty" level="info" additivity="false">
+ <AppenderRef ref="console"/>
+ <AppenderRef ref="RollingAppender"/>
+ </Logger>
+ </Loggers>
+</Configuration>
diff --git a/conf/local_dev/server.conf b/conf/local_dev/server.conf
new file mode 100644
index 000000000..688364b34
--- /dev/null
+++ b/conf/local_dev/server.conf
@@ -0,0 +1,32 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+rss.rpc.server.port 19999
+rss.jetty.http.port 19978
+# change to the actual disk path
+rss.storage.basePath /tmp/dir0,/tmp/dir1
+rss.storage.type MEMORY_LOCALFILE_HDFS
+# change to actual coordinator rpc addresses
+rss.coordinator.quorum localhost:19999,localhost:19999
+rss.server.buffer.capacity 40gb
+rss.server.read.buffer.capacity 20gb
+rss.server.flush.thread.alive 5
+rss.server.flush.localfile.threadPool.size 10
+rss.server.flush.hadoop.threadPool.size 60
+rss.server.disk.capacity 1g
+rss.server.single.buffer.flush.enabled true
+rss.server.single.buffer.flush.threshold 128m
diff --git a/dev/intellij/install-runconfig.sh
b/dev/intellij/install-runconfig.sh
new file mode 100755
index 000000000..e743ed684
--- /dev/null
+++ b/dev/intellij/install-runconfig.sh
@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -o pipefail
+set -o nounset # exit the script if you try to use an uninitialised variable
+set -o errexit # exit the script if any statement returns a non-true return
value
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+SRC_DIR="$SCRIPT_DIR/runConfigurations"
+DEST_DIR="$SCRIPT_DIR/../../.idea/runConfigurations/"
+mkdir -p "$DEST_DIR"
+ls -1 "$SRC_DIR" | xargs -n1 -I FILE cp "$SRC_DIR/FILE" "$DEST_DIR"
diff --git a/dev/intellij/runConfigurations/CoordinatorServer.run.xml
b/dev/intellij/runConfigurations/CoordinatorServer.run.xml
new file mode 100644
index 000000000..c0e7bc74d
--- /dev/null
+++ b/dev/intellij/runConfigurations/CoordinatorServer.run.xml
@@ -0,0 +1,34 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+-->
+<component name="ProjectRunConfigurationManager">
+ <configuration default="false" name="CoordinatorServer" type="Application"
factoryName="Application" nameIsGenerated="true">
+ <option name="INCLUDE_PROVIDED_SCOPE" value="true" />
+ <option name="MAIN_CLASS_NAME"
value="org.apache.uniffle.coordinator.CoordinatorServer" />
+ <module name="coordinator" />
+ <option name="PROGRAM_PARAMETERS" value="--conf
$PROJECT_DIR$/conf/local_dev/coordinator.conf" />
+ <option name="VM_PARAMETERS"
value="-Dlog4j2.configurationFile=file:$PROJECT_DIR$/conf/local_dev/log4j2.xml
-Dlog.path=$PROJECT_DIR$/logs/coordinator.log" />
+ <extension name="coverage">
+ <pattern>
+ <option name="PATTERN" value="org.apache.uniffle.coordinator.*" />
+ <option name="ENABLED" value="true" />
+ </pattern>
+ </extension>
+ <method v="2">
+ <option name="Make" enabled="true" />
+ </method>
+ </configuration>
+</component>
diff --git a/dev/intellij/runConfigurations/JettyServerFront.run.xml
b/dev/intellij/runConfigurations/JettyServerFront.run.xml
new file mode 100644
index 000000000..087904539
--- /dev/null
+++ b/dev/intellij/runConfigurations/JettyServerFront.run.xml
@@ -0,0 +1,34 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+-->
+<component name="ProjectRunConfigurationManager">
+ <configuration default="false" name="JettyServerFront" type="Application"
factoryName="Application" nameIsGenerated="true">
+ <option name="INCLUDE_PROVIDED_SCOPE" value="true" />
+ <option name="MAIN_CLASS_NAME"
value="org.apache.uniffle.dashboard.web.JettyServerFront" />
+ <module name="dashboard" />
+ <option name="PROGRAM_PARAMETERS" value="--conf
$PROJECT_DIR$/conf/local_dev/dashboard.conf" />
+ <option name="VM_PARAMETERS"
value="-Dlog4j2.configurationFile=file:$PROJECT_DIR$/conf/local_dev/log4j2.xml
-Dlog.path=$PROJECT_DIR$/logs/dashboard.log" />
+ <extension name="coverage">
+ <pattern>
+ <option name="PATTERN" value="org.apache.uniffle.dashboard.web.*" />
+ <option name="ENABLED" value="true" />
+ </pattern>
+ </extension>
+ <method v="2">
+ <option name="Make" enabled="true" />
+ </method>
+ </configuration>
+</component>
diff --git a/dev/intellij/runConfigurations/ShuffleServer.run.xml
b/dev/intellij/runConfigurations/ShuffleServer.run.xml
new file mode 100644
index 000000000..4a05a752e
--- /dev/null
+++ b/dev/intellij/runConfigurations/ShuffleServer.run.xml
@@ -0,0 +1,34 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+-->
+<component name="ProjectRunConfigurationManager">
+ <configuration default="false" name="ShuffleServer" type="Application"
factoryName="Application" nameIsGenerated="true">
+ <option name="INCLUDE_PROVIDED_SCOPE" value="true" />
+ <option name="MAIN_CLASS_NAME"
value="org.apache.uniffle.server.ShuffleServer" />
+ <module name="shuffle-server" />
+ <option name="PROGRAM_PARAMETERS" value="--conf
$PROJECT_DIR$/conf/local_dev/server.conf" />
+ <option name="VM_PARAMETERS"
value="-Dlog4j2.configurationFile=file:$PROJECT_DIR$/conf/local_dev/log4j2.xml
-Dlog.path=$PROJECT_DIR$/logs/shuffle_server.log" />
+ <extension name="coverage">
+ <pattern>
+ <option name="PATTERN" value="org.apache.uniffle.server.*" />
+ <option name="ENABLED" value="true" />
+ </pattern>
+ </extension>
+ <method v="2">
+ <option name="Make" enabled="true" />
+ </method>
+ </configuration>
+</component>
