This is an automated email from the ASF dual-hosted git repository.
peacewong pushed a commit to branch dev-1.3.0
in repository https://gitbox.apache.org/repos/asf/incubator-linkis.git
The following commit(s) were added to refs/heads/dev-1.3.0 by this push:
new 753e81ac3 [Feature-3610]Optime for docker of linkis ldh (#3611)
753e81ac3 is described below
commit 753e81ac3637428a48d9e6c4ce2dd1b356564dc8
Author: Casion <[email protected]>
AuthorDate: Tue Oct 11 16:08:44 2022 +0800
[Feature-3610]Optime for docker of linkis ldh (#3611)
---
README.md | 10 +-
.../impl/BmlResourceLocalizationService.scala | 22 ++-
.../ecm/server/spring/ECMSpringConfiguration.scala | 6 +
linkis-dist/deploy-config/linkis-env.sh | 3 +
linkis-dist/docker/ldh.Dockerfile | 2 +-
linkis-dist/docker/linkis.Dockerfile | 14 +-
.../linkis/templates/configmap-init-sql.yaml | 2 +-
.../linkis/templates/configmap-linkis-config.yaml | 26 +--
.../templates/linkis-cg-engineconnmanager.yaml | 19 +-
.../linkis/templates/linkis-cg-engineplugin.yaml | 8 +-
.../linkis/templates/linkis-cg-entrance.yaml | 8 +-
.../linkis/templates/linkis-cg-linkismanager.yaml | 8 +-
.../charts/linkis/templates/linkis-mg-eureka.yaml | 8 +-
.../charts/linkis/templates/linkis-mg-gateway.yaml | 8 +-
.../linkis/templates/linkis-ps-publicservice.yaml | 8 +-
linkis-dist/helm/charts/linkis/values.yaml | 6 +-
linkis-dist/helm/scripts/create-kind-cluster.sh | 6 +
linkis-dist/helm/scripts/login-pod.sh | 33 +++-
.../helm/scripts/{login-pod.sh => logs-pod.sh} | 28 ++-
linkis-dist/helm/scripts/prepare-for-spark.sh | 51 ++++++
linkis-dist/helm/scripts/remote-debug-proxy.sh | 97 -----------
linkis-dist/helm/scripts/remote-proxy.sh | 129 ++++++++++++++
.../helm/scripts/resources/kind-cluster.yaml | 2 +
.../resources/ldh/configmaps/configmap-hadoop.yaml | 191 +++++++--------------
.../resources/ldh/configmaps/configmap-spark.yaml | 34 ++++
linkis-dist/package/sbin/ext/linkis-common-start | 10 +-
linkis-dist/package/sbin/ext/linkis-mg-gateway | 8 +-
27 files changed, 481 insertions(+), 266 deletions(-)
diff --git a/README.md b/README.md
index c7047c3c6..524bcc087 100644
--- a/README.md
+++ b/README.md
@@ -132,23 +132,29 @@ Please go to the [Linkis Releases
Page](https://linkis.apache.org/download/main)
```shell
+Note: If you want use `-Dlinkis.build.web=true` to build linkis-web image,
you need to compile linkis-web first.
+
## compile backend
### Mac OS/Linux
+
# 1. When compiling for the first time, execute the following command first
./mvnw -N install
+
# 2. make the linkis distribution package
# - Option 1: make the linkis distribution package only
./mvnw clean install -Dmaven.javadoc.skip=true -Dmaven.test.skip=true
+
# - Option 2: make the linkis distribution package and docker image
# - Option 2.1: image without mysql jdbc jars
./mvnw clean install -Pdocker -Dmaven.javadoc.skip=true -Dmaven.test.skip=true
# - Option 2.2: image with mysql jdbc jars
./mvnw clean install -Pdocker -Dmaven.javadoc.skip=true -Dmaven.test.skip=true
-Dlinkis.build.with.jdbc=true
+
# - Option 3: linkis distribution package and docker image (included web)
./mvnw clean install -Pdocker -Dmaven.javadoc.skip=true -Dmaven.test.skip=true
-Dlinkis.build.web=true
-# - Option 4: linkis distribution package and docker image (included web and
ldh (hadoop all in one for test))
-./mvnw clean install -Pdocker -Dmaven.javadoc.skip=true -Dmaven.test.skip=true
-Dlinkis.build.web=true -Dlinkis.build.ldh=true
+# - Option 4: linkis distribution package and docker image (included web and
ldh (hadoop all in one for test))
+./mvnw clean install -Pdocker -Dmaven.javadoc.skip=true -Dmaven.test.skip=true
-Dlinkis.build.web=true -Dlinkis.build.ldh=true -Dlinkis.build.with.jdbc=true
### Windows
mvnw.cmd -N install
diff --git
a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/BmlResourceLocalizationService.scala
b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/BmlResourceLocalizationService.scala
index 7d1190d91..c67a3cc19 100644
---
a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/BmlResourceLocalizationService.scala
+++
b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/BmlResourceLocalizationService.scala
@@ -18,6 +18,7 @@
package org.apache.linkis.ecm.server.service.impl
import org.apache.linkis.DataWorkCloudApplication
+import org.apache.linkis.common.conf.Configuration
import org.apache.linkis.common.io.FsPath
import org.apache.linkis.common.utils.{Logging, Utils, ZipUtils}
import org.apache.linkis.ecm.core.conf.ECMErrorCode
@@ -35,6 +36,8 @@ import org.apache.linkis.storage.FSFactory
import org.apache.linkis.storage.fs.FileSystem
import org.apache.linkis.storage.utils.{FileSystemUtils, StorageUtils}
+import org.springframework.core.env.Environment
+
import java.io.File
import java.nio.file.Paths
@@ -54,9 +57,14 @@ class BmlResourceLocalizationService extends
ResourceLocalizationService with Lo
private var localDirsHandleService: LocalDirsHandleService = _
+ private var springEnv: Environment = _
+
def setLocalDirsHandleService(localDirsHandleService:
LocalDirsHandleService): Unit =
this.localDirsHandleService = localDirsHandleService
+ def setSpringEnv(springEnv: Environment): Unit =
+ this.springEnv = springEnv
+
override def handleInitEngineConnResources(
request: EngineConnLaunchRequest,
engineConn: EngineConn
@@ -87,7 +95,19 @@ class BmlResourceLocalizationService extends
ResourceLocalizationService with Lo
override val engineConnWorkDir: String = workDir
override val engineConnLogDirs: String = logDirs
override val engineConnTempDirs: String = tmpDirs
- override val engineConnManagerHost: String = Utils.getComputerName
+ override val engineConnManagerHost: String = {
+ var hostName = Utils.getComputerName
+ val eurekaPreferIp = Configuration.EUREKA_PREFER_IP
+ logger.info("eurekaPreferIp:" + eurekaPreferIp)
+ if (eurekaPreferIp) {
+ hostName =
springEnv.getProperty("spring.cloud.client.ip-address")
+ logger.info("hostName:" + hostName)
+ logger.info(
+ "using ip address replace hostname,beacause
eureka.instance.prefer-ip-address:" + eurekaPreferIp
+ )
+ }
+ hostName
+ }
override val engineConnManagerPort: String =
DataWorkCloudApplication.getApplicationContext.getEnvironment.getProperty(
"server.port"
diff --git
a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/spring/ECMSpringConfiguration.scala
b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/spring/ECMSpringConfiguration.scala
index e74f041a8..4a684bbec 100644
---
a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/spring/ECMSpringConfiguration.scala
+++
b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/spring/ECMSpringConfiguration.scala
@@ -22,12 +22,17 @@ import
org.apache.linkis.ecm.server.context.{DefaultECMContext, ECMContext}
import org.apache.linkis.ecm.server.service._
import org.apache.linkis.ecm.server.service.impl._
+import org.springframework.beans.factory.annotation.Autowired
import
org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean
import org.springframework.context.annotation.{Bean, Configuration}
+import org.springframework.core.env.Environment
@Configuration
class ECMSpringConfiguration {
+ @Autowired
+ private var env: Environment = _;
+
@Bean
@ConditionalOnMissingBean
def getDefaultEngineConnManagerContext: ECMContext = {
@@ -48,6 +53,7 @@ class ECMSpringConfiguration {
): ResourceLocalizationService = {
val service: BmlResourceLocalizationService = new
BmlResourceLocalizationService
service.setLocalDirsHandleService(localDirsHandleService)
+ service.setSpringEnv(env)
service
}
diff --git a/linkis-dist/deploy-config/linkis-env.sh
b/linkis-dist/deploy-config/linkis-env.sh
index 857fca76f..14242e1fc 100644
--- a/linkis-dist/deploy-config/linkis-env.sh
+++ b/linkis-dist/deploy-config/linkis-env.sh
@@ -164,6 +164,9 @@ export SERVER_HEAP_SIZE="512M"
##The decompression directory and the installation directory need to be
inconsistent
#LINKIS_HOME=/appcom/Install/LinkisInstall
+##The extended lib such mysql-connector-java-*.jar
+#LINKIS_EXTENDED_LIB=/appcom/common/linkisExtendedLib
+
LINKIS_VERSION=1.3.0
# for install
diff --git a/linkis-dist/docker/ldh.Dockerfile
b/linkis-dist/docker/ldh.Dockerfile
index f6404f3b5..c5513a28c 100644
--- a/linkis-dist/docker/ldh.Dockerfile
+++ b/linkis-dist/docker/ldh.Dockerfile
@@ -43,7 +43,7 @@ RUN useradd -r -s /bin/bash -u 100001 -g root -G wheel hadoop
# TODO: remove install mysql client when schema-init-tools is ready
RUN yum install -y \
- less ls vim unzip curl sudo krb5-workstation sssd crontabs net-tools
python-pip glibc-common \
+ less vim unzip curl sudo krb5-workstation sssd crontabs net-tools
python-pip glibc-common \
java-${JDK_VERSION}-${JDK_BUILD_REVISION} \
java-${JDK_VERSION}-devel-${JDK_BUILD_REVISION} \
mysql \
diff --git a/linkis-dist/docker/linkis.Dockerfile
b/linkis-dist/docker/linkis.Dockerfile
index 83118cd8a..4a9fa9886 100644
--- a/linkis-dist/docker/linkis.Dockerfile
+++ b/linkis-dist/docker/linkis.Dockerfile
@@ -31,7 +31,7 @@ ARG JDK_BUILD_REVISION=1.8.0.332.b09-1.el7_9
# TODO: remove install mysql client when schema-init-tools is ready
RUN yum install -y \
- less ls vim unzip curl sudo krb5-workstation sssd crontabs python-pip
glibc-common \
+ less vim unzip curl sudo krb5-workstation sssd crontabs net-tools
python-pip glibc-common \
java-${JDK_VERSION}-${JDK_BUILD_REVISION} \
java-${JDK_VERSION}-devel-${JDK_BUILD_REVISION} \
mysql \
@@ -56,6 +56,7 @@ ARG LINKIS_SYSTEM_UID="9001"
ARG LINKIS_HOME=/opt/linkis
ARG LINKIS_CONF_DIR=/etc/linkis-conf
ARG LINKIS_LOG_DIR=/var/logs/linkis
+ARG LDH_HOME=/opt/ldh
WORKDIR ${LINKIS_HOME}
@@ -64,7 +65,8 @@ RUN sed -i "s#^%wheel.*#%wheel ALL=(ALL)
NOPASSWD: ALL#g" /etc/sudo
RUN mkdir -p /opt/tmp \
&& mkdir -p ${LINKIS_CONF_DIR} \
- && mkdir -p ${LINKIS_LOG_DIR}
+ && mkdir -p ${LINKIS_LOG_DIR} \
+ && mkdir -p ${LDH_HOME}
ENV JAVA_HOME /etc/alternatives/jre
ENV LINKIS_CONF_DIR ${LINKIS_CONF_DIR}
@@ -75,13 +77,17 @@ ENV LINKIS_HOME ${LINKIS_HOME}
ADD apache-linkis-${LINKIS_VERSION}-incubating-bin /opt/tmp/
RUN mv /opt/tmp/linkis-package/* ${LINKIS_HOME}/ \
- && mv /opt/tmp/release-docs/LICENSE ${LINKIS_HOME}/LICENSE \
- && mv /opt/tmp/release-docs/NOTICE ${LINKIS_HOME}/NOTICE \
+ && mv /opt/tmp/LICENSE ${LINKIS_HOME}/ \
+ && mv /opt/tmp/NOTICE ${LINKIS_HOME}/ \
+ && mv /opt/tmp/DISCLAIMER ${LINKIS_HOME}/ \
+ && mv /opt/tmp/README.md ${LINKIS_HOME}/ \
+ && mv /opt/tmp/README_CN.md ${LINKIS_HOME}/ \
&& rm -rf /opt/tmp
RUN chmod g+w -R ${LINKIS_HOME} && chown
${LINKIS_SYSTEM_USER}:${LINKIS_SYSTEM_GROUP} -R ${LINKIS_HOME} \
&& chmod g+w -R ${LINKIS_CONF_DIR} && chown
${LINKIS_SYSTEM_USER}:${LINKIS_SYSTEM_GROUP} -R ${LINKIS_CONF_DIR} \
&& chmod g+w -R ${LINKIS_LOG_DIR} && chown
${LINKIS_SYSTEM_USER}:${LINKIS_SYSTEM_GROUP} -R ${LINKIS_LOG_DIR} \
+ && chmod g+w -R ${LDH_HOME} && chown
${LINKIS_SYSTEM_USER}:${LINKIS_SYSTEM_GROUP} -R ${LDH_HOME} \
&& chmod a+x ${LINKIS_HOME}/bin/* \
&& chmod a+x ${LINKIS_HOME}/sbin/*
diff --git a/linkis-dist/helm/charts/linkis/templates/configmap-init-sql.yaml
b/linkis-dist/helm/charts/linkis/templates/configmap-init-sql.yaml
index ba0c369e6..8b76c0ae6 100644
--- a/linkis-dist/helm/charts/linkis/templates/configmap-init-sql.yaml
+++ b/linkis-dist/helm/charts/linkis/templates/configmap-init-sql.yaml
@@ -992,7 +992,7 @@ data:
SET @OPENLOOKENG_IDE=CONCAT('*-IDE,',@OPENLOOKENG_LABEL);
-- Global Settings
- INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`,
`name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`,
`is_advanced`, `level`, `treeName`) VALUES ('wds.linkis.rm.yarnqueue',
'yarn队列名', 'yarn队列名', 'ide', 'None', NULL, '0', '0', '1', '队列资源');
+ INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`,
`name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`,
`is_advanced`, `level`, `treeName`) VALUES ('wds.linkis.rm.yarnqueue',
'yarn队列名', 'yarn队列名', 'default', 'None', NULL, '0', '0', '1', '队列资源');
INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`,
`name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`,
`is_advanced`, `level`, `treeName`) VALUES
('wds.linkis.rm.yarnqueue.instance.max', '取值范围：1-128，单位：个', '队列实例最大个数', '30',
'Regex', '^(?:[1-9]\\d?|[1234]\\d{2}|128)$', '0', '0', '1', '队列资源');
INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`,
`name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`,
`is_advanced`, `level`, `treeName`) VALUES
('wds.linkis.rm.yarnqueue.cores.max', '取值范围：1-500，单位：个', '队列CPU使用上限', '150',
'Regex', '^(?:[1-9]\\d?|[1234]\\d{2}|500)$', '0', '0', '1', '队列资源');
INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`,
`name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`,
`is_advanced`, `level`, `treeName`) VALUES
('wds.linkis.rm.yarnqueue.memory.max', '取值范围：1-1000，单位：G', '队列内存使用上限', '300G',
'Regex', '^([1-9]\\d{0,2}|1000)(G|g)$', '0', '0', '1', '队列资源');
diff --git
a/linkis-dist/helm/charts/linkis/templates/configmap-linkis-config.yaml
b/linkis-dist/helm/charts/linkis/templates/configmap-linkis-config.yaml
index 650cc453b..9775d9159 100644
--- a/linkis-dist/helm/charts/linkis/templates/configmap-linkis-config.yaml
+++ b/linkis-dist/helm/charts/linkis/templates/configmap-linkis-config.yaml
@@ -36,8 +36,8 @@ data:
log4j2.xml: |
<?xml version="1.0" encoding="UTF-8"?>
- <configuration status="error" monitorInterval="30">
- <appenders>
+ <Configuration status="error" monitorInterval="30">
+ <Appenders>
<RollingFile name="RollingFile" append="true"
fileName="${env:LINKIS_LOG_DIR}/${sys:serviceName}.log"
filePattern="${env:LINKIS_LOG_DIR}/$${date:yyyy-MM}/${sys:serviceName}/linkis-log-%d{yyyy-MM-dd-hh}-%i.log">
<PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} [%-5level]
[%-40t] %c{1.} (%L) [%M] - %msg%xEx%n"/>
@@ -46,17 +46,17 @@ data:
</Policies>
<DefaultRolloverStrategy max="10"/>
</RollingFile>
- </appenders>
- <loggers>
- <root level="INFO">
- <appender-ref ref="RollingFile"/>
- </root>
-
- <logger
name="com.netflix.loadbalancer.DynamicServerListLoadBalancer" level="warn"
additivity="true">
- <appender-ref ref="RollingFile"/>
- </logger>
- </loggers>
- </configuration>
+ <Console name="Console" target="SYSTEM_OUT">
+ <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level
%logger{36} - %msg%n"/>
+ </Console>
+ </Appenders>
+ <Loggers>
+ <Root level="INFO">
+ <AppenderRef ref="RollingFile"/>
+ <AppenderRef ref="Console"/>
+ </Root>
+ </Loggers>
+ </Configuration>
linkis-env.sh: |
export LINKIS_VERSION={{ .Chart.AppVersion }}
diff --git
a/linkis-dist/helm/charts/linkis/templates/linkis-cg-engineconnmanager.yaml
b/linkis-dist/helm/charts/linkis/templates/linkis-cg-engineconnmanager.yaml
index 6a9d91e2e..14f810a1f 100644
--- a/linkis-dist/helm/charts/linkis/templates/linkis-cg-engineconnmanager.yaml
+++ b/linkis-dist/helm/charts/linkis/templates/linkis-cg-engineconnmanager.yaml
@@ -98,6 +98,10 @@ spec:
name: {{ include "linkis.fullname" . }}-linkis-config
- name: log
emptyDir: {}
+ - name: common
+ hostPath:
+ path: {{ .Values.linkis.locations.commonDir }}
+ type: DirectoryOrCreate
- name: runtime
{{- if .Values.linkis.featureGates.localMode }}
hostPath:
@@ -106,12 +110,23 @@ spec:
{{- else }}
emptyDir: {}
{{- end }}
+
{{- if eq .Values.linkis.featureGates.localMode false }}
- name: hadoop-conf
projected:
sources:
- configMap:
name: {{ .Values.linkis.deps.hadoop.configMapName }}
+ - name: hive-conf
+ projected:
+ sources:
+ - configMap:
+ name: {{ .Values.linkis.deps.hive.configMapName }}
+ - name: spark-conf
+ projected:
+ sources:
+ - configMap:
+ name: {{ .Values.linkis.deps.spark.configMapName }}
{{- end }}
serviceAccountName: {{ include "linkis.serviceAccountName" . }}
securityContext:
@@ -126,7 +141,7 @@ spec:
- /bin/bash
- -ecx
- >-
- RUN_IN_FOREGROUND=true {{ .Values.linkis.locations.homeDir
}}/sbin/linkis-daemon.sh start cg-engineconnmanager
+ RUN_IN_FOREGROUND=true
LINKIS_EXTENDED_LIB={{.Values.linkis.locations.commonDir}}/extendlib {{
.Values.linkis.locations.homeDir }}/sbin/linkis-daemon.sh start
cg-engineconnmanager
ports:
- name: "http"
containerPort: {{ .Values.cgEngineConnManager.port }}
@@ -191,6 +206,8 @@ spec:
mountPath: {{ .Values.linkis.locations.logDir }}
- name: runtime
mountPath: {{ .Values.linkis.locations.runtimeDir }}
+ - name: common
+ mountPath: {{ .Values.linkis.locations.commonDir }}
resources:
{{- toYaml .Values.cgEngineConnManager.resources | nindent 12 }}
{{- with .Values.cgEngineConnManager.nodeSelector }}
diff --git
a/linkis-dist/helm/charts/linkis/templates/linkis-cg-engineplugin.yaml
b/linkis-dist/helm/charts/linkis/templates/linkis-cg-engineplugin.yaml
index d97d6b107..130745f1e 100644
--- a/linkis-dist/helm/charts/linkis/templates/linkis-cg-engineplugin.yaml
+++ b/linkis-dist/helm/charts/linkis/templates/linkis-cg-engineplugin.yaml
@@ -98,6 +98,10 @@ spec:
name: {{ include "linkis.fullname" . }}-linkis-config
- name: log
emptyDir: {}
+ - name: common
+ hostPath:
+ path: {{ .Values.linkis.locations.commonDir }}
+ type: DirectoryOrCreate
- name: runtime
{{- if .Values.linkis.featureGates.localMode }}
hostPath:
@@ -136,7 +140,7 @@ spec:
- /bin/bash
- -ecx
- >-
- RUN_IN_FOREGROUND=true {{ .Values.linkis.locations.homeDir
}}/sbin/linkis-daemon.sh start cg-engineplugin
+ RUN_IN_FOREGROUND=true
LINKIS_EXTENDED_LIB={{.Values.linkis.locations.commonDir}}/extendlib {{
.Values.linkis.locations.homeDir }}/sbin/linkis-daemon.sh start cg-engineplugin
ports:
- name: "http"
containerPort: {{ .Values.cgEnginePlugin.port }}
@@ -201,6 +205,8 @@ spec:
mountPath: {{ .Values.linkis.locations.logDir }}
- name: runtime
mountPath: {{ .Values.linkis.locations.runtimeDir }}
+ - name: common
+ mountPath: {{ .Values.linkis.locations.commonDir }}
resources:
{{- toYaml .Values.cgEnginePlugin.resources | nindent 12 }}
{{- with .Values.cgEnginePlugin.nodeSelector }}
diff --git a/linkis-dist/helm/charts/linkis/templates/linkis-cg-entrance.yaml
b/linkis-dist/helm/charts/linkis/templates/linkis-cg-entrance.yaml
index 7b3e9d16f..426fb105d 100644
--- a/linkis-dist/helm/charts/linkis/templates/linkis-cg-entrance.yaml
+++ b/linkis-dist/helm/charts/linkis/templates/linkis-cg-entrance.yaml
@@ -98,6 +98,10 @@ spec:
name: {{ include "linkis.fullname" . }}-linkis-config
- name: log
emptyDir: {}
+ - name: common
+ hostPath:
+ path: {{ .Values.linkis.locations.commonDir }}
+ type: DirectoryOrCreate
- name: runtime
{{- if .Values.linkis.featureGates.localMode }}
hostPath:
@@ -126,7 +130,7 @@ spec:
- /bin/bash
- -ecx
- >-
- RUN_IN_FOREGROUND=true {{ .Values.linkis.locations.homeDir
}}/sbin/linkis-daemon.sh start cg-entrance
+ RUN_IN_FOREGROUND=true
LINKIS_EXTENDED_LIB={{.Values.linkis.locations.commonDir}}/extendlib {{
.Values.linkis.locations.homeDir }}/sbin/linkis-daemon.sh start cg-entrance
ports:
- name: "http"
containerPort: {{ .Values.cgEntrance.port }}
@@ -183,6 +187,8 @@ spec:
mountPath: {{ .Values.linkis.locations.logDir }}
- name: runtime
mountPath: {{ .Values.linkis.locations.runtimeDir }}
+ - name: common
+ mountPath: {{ .Values.linkis.locations.commonDir }}
resources:
{{- toYaml .Values.cgEntrance.resources | nindent 12 }}
{{- with .Values.cgEntrance.nodeSelector }}
diff --git
a/linkis-dist/helm/charts/linkis/templates/linkis-cg-linkismanager.yaml
b/linkis-dist/helm/charts/linkis/templates/linkis-cg-linkismanager.yaml
index ad429ce06..2167ae897 100644
--- a/linkis-dist/helm/charts/linkis/templates/linkis-cg-linkismanager.yaml
+++ b/linkis-dist/helm/charts/linkis/templates/linkis-cg-linkismanager.yaml
@@ -98,6 +98,10 @@ spec:
name: {{ include "linkis.fullname" . }}-linkis-config
- name: log
emptyDir: {}
+ - name: common
+ hostPath:
+ path: {{ .Values.linkis.locations.commonDir }}
+ type: DirectoryOrCreate
- name: runtime
{{- if .Values.linkis.featureGates.localMode }}
hostPath:
@@ -119,7 +123,7 @@ spec:
- /bin/bash
- -ecx
- >-
- RUN_IN_FOREGROUND=true {{ .Values.linkis.locations.homeDir
}}/sbin/linkis-daemon.sh start cg-linkismanager
+ RUN_IN_FOREGROUND=true
LINKIS_EXTENDED_LIB={{.Values.linkis.locations.commonDir}}/extendlib {{
.Values.linkis.locations.homeDir }}/sbin/linkis-daemon.sh start cg-linkismanager
ports:
- name: "http"
containerPort: {{ .Values.cgLinkisManager.port }}
@@ -168,6 +172,8 @@ spec:
mountPath: {{ .Values.linkis.locations.logDir }}
- name: runtime
mountPath: {{ .Values.linkis.locations.runtimeDir }}
+ - name: common
+ mountPath: {{ .Values.linkis.locations.commonDir }}
resources:
{{- toYaml .Values.cgLinkisManager.resources | nindent 12 }}
{{- with .Values.cgLinkisManager.nodeSelector }}
diff --git a/linkis-dist/helm/charts/linkis/templates/linkis-mg-eureka.yaml
b/linkis-dist/helm/charts/linkis/templates/linkis-mg-eureka.yaml
index f57be4339..301bcec0a 100644
--- a/linkis-dist/helm/charts/linkis/templates/linkis-mg-eureka.yaml
+++ b/linkis-dist/helm/charts/linkis/templates/linkis-mg-eureka.yaml
@@ -100,6 +100,10 @@ spec:
name: {{ include "linkis.fullname" . }}-linkis-config
- name: log
emptyDir: {}
+ - name: common
+ hostPath:
+ path: {{ .Values.linkis.locations.commonDir }}
+ type: DirectoryOrCreate
{{- if .Values.imagePullSecrets }}
imagePullSecrets:
{{ toYaml .Values.imagePullSecrets | indent 8 }}
@@ -116,7 +120,7 @@ spec:
- /bin/bash
- -ecx
- >-
- RUN_IN_FOREGROUND=true {{ .Values.linkis.locations.homeDir
}}/sbin/linkis-daemon.sh start mg-eureka
+ RUN_IN_FOREGROUND=true
LINKIS_EXTENDED_LIB={{.Values.linkis.locations.commonDir}}/extendlib {{
.Values.linkis.locations.homeDir }}/sbin/linkis-daemon.sh start mg-eureka
ports:
- name: "http"
containerPort: {{ .Values.mgEureka.port }}
@@ -166,3 +170,5 @@ spec:
mountPath: {{ .Values.linkis.locations.confDir }}
- name: log
mountPath: {{ .Values.linkis.locations.logDir }}
+ - name: common
+ mountPath: {{ .Values.linkis.locations.commonDir }}
diff --git a/linkis-dist/helm/charts/linkis/templates/linkis-mg-gateway.yaml
b/linkis-dist/helm/charts/linkis/templates/linkis-mg-gateway.yaml
index 2b69291f5..06d3a8c1f 100644
--- a/linkis-dist/helm/charts/linkis/templates/linkis-mg-gateway.yaml
+++ b/linkis-dist/helm/charts/linkis/templates/linkis-mg-gateway.yaml
@@ -98,6 +98,10 @@ spec:
name: {{ include "linkis.fullname" . }}-linkis-config
- name: log
emptyDir: {}
+ - name: common
+ hostPath:
+ path: {{ .Values.linkis.locations.commonDir }}
+ type: DirectoryOrCreate
- name: runtime
{{- if .Values.linkis.featureGates.localMode }}
hostPath:
@@ -119,7 +123,7 @@ spec:
- /bin/bash
- -ecx
- >-
- RUN_IN_FOREGROUND=true {{ .Values.linkis.locations.homeDir
}}/sbin/linkis-daemon.sh start mg-gateway
+ RUN_IN_FOREGROUND=true
LINKIS_EXTENDED_LIB={{.Values.linkis.locations.commonDir}}/extendlib {{
.Values.linkis.locations.homeDir }}/sbin/linkis-daemon.sh start mg-gateway
ports:
- name: "http"
containerPort: {{ .Values.mgGateway.port }}
@@ -168,6 +172,8 @@ spec:
mountPath: {{ .Values.linkis.locations.logDir }}
- name: runtime
mountPath: {{ .Values.linkis.locations.runtimeDir }}
+ - name: common
+ mountPath: {{ .Values.linkis.locations.commonDir }}
resources:
{{- toYaml .Values.mgGateway.resources | nindent 12 }}
{{- with .Values.mgGateway.nodeSelector }}
diff --git
a/linkis-dist/helm/charts/linkis/templates/linkis-ps-publicservice.yaml
b/linkis-dist/helm/charts/linkis/templates/linkis-ps-publicservice.yaml
index c219c6161..3b3e87c3e 100644
--- a/linkis-dist/helm/charts/linkis/templates/linkis-ps-publicservice.yaml
+++ b/linkis-dist/helm/charts/linkis/templates/linkis-ps-publicservice.yaml
@@ -98,6 +98,10 @@ spec:
name: {{ include "linkis.fullname" . }}-linkis-config
- name: log
emptyDir: {}
+ - name: common
+ hostPath:
+ path: {{ .Values.linkis.locations.commonDir }}
+ type: DirectoryOrCreate
- name: runtime
{{- if .Values.linkis.featureGates.localMode }}
hostPath:
@@ -131,7 +135,7 @@ spec:
- /bin/bash
- -ecx
- >-
- RUN_IN_FOREGROUND=true {{ .Values.linkis.locations.homeDir
}}/sbin/linkis-daemon.sh start ps-publicservice
+ RUN_IN_FOREGROUND=true
LINKIS_EXTENDED_LIB={{.Values.linkis.locations.commonDir}}/extendlib {{
.Values.linkis.locations.homeDir }}/sbin/linkis-daemon.sh start ps-publicservice
ports:
- name: "http"
containerPort: {{ .Values.psPublicService.port }}
@@ -192,6 +196,8 @@ spec:
mountPath: {{ .Values.linkis.locations.logDir }}
- name: runtime
mountPath: {{ .Values.linkis.locations.runtimeDir }}
+ - name: common
+ mountPath: {{ .Values.linkis.locations.commonDir }}
resources:
{{- toYaml .Values.psPublicService.resources | nindent 12 }}
{{- with .Values.psPublicService.nodeSelector }}
diff --git a/linkis-dist/helm/charts/linkis/values.yaml
b/linkis-dist/helm/charts/linkis/values.yaml
index 16fa1fce9..638f75134 100644
--- a/linkis-dist/helm/charts/linkis/values.yaml
+++ b/linkis-dist/helm/charts/linkis/values.yaml
@@ -36,6 +36,7 @@ image:
# Overrides the image tag whose default is the chart appVersion.
tag: ""
imagePullSecrets: []
+withJdbcDriverJar: false
serviceAccount:
create: true
annotations: {}
@@ -85,6 +86,7 @@ linkis:
confDir: /etc/linkis-conf
logDir: /var/logs/linkis
runtimeDir: /opt/linkis-runtime
+ commonDir: /opt/common
# hostPath is a directory on the host machine.
# In KinD cluster, it is actual a directory in KinD's
# node container, which is specialized in KinD cluster
@@ -112,7 +114,7 @@ linkis:
version: 2.7.2
configMapName: hadoop-conf
yarn:
- restfulUrl: http://localhost:8080
+ restfulUrl: http://ldh.ldh.svc.cluster.local:8088
authEnable: false
authUser: hadoop
authPassword: "123456"
@@ -287,7 +289,7 @@ psPublicService:
Web:
replicas: 1
- port: 8087
+ port: 8088
prometheus:
metricsPath: metrics
annotations: {}
diff --git a/linkis-dist/helm/scripts/create-kind-cluster.sh
b/linkis-dist/helm/scripts/create-kind-cluster.sh
index a804b6c4b..1d9804f8e 100755
--- a/linkis-dist/helm/scripts/create-kind-cluster.sh
+++ b/linkis-dist/helm/scripts/create-kind-cluster.sh
@@ -27,11 +27,17 @@ set -e
echo "# Creating KinD cluster ..."
# create data dir for KinD cluster
KIND_CLUSTER_HOST_PATH=${TMP_DIR}/data
+
+# Shared storage of some common dependent packages such as
mysql-connector-java-*.jar
+# Values.linkis.locations.commonDir -> kind docker /data/common -> vm
${KIND_COMMON_PATH}
+KIND_COMMON_PATH=/data/common/
+
mkdir -p ${KIND_CLUSTER_HOST_PATH}
# create kind cluster conf
KIND_CLUSTER_CONF_TPL=${RESOURCE_DIR}/kind-cluster.yaml
KIND_CLUSTER_CONF_FILE=${TMP_DIR}/kind-cluster.yaml
KIND_CLUSTER_HOST_PATH=${KIND_CLUSTER_HOST_PATH} envsubst <
${KIND_CLUSTER_CONF_TPL} > ${KIND_CLUSTER_CONF_FILE}
+KIND_COMMON_PATH=${KIND_COMMON_PATH} envsubst < ${KIND_CLUSTER_CONF_TPL} >
${KIND_CLUSTER_CONF_FILE}
echo "- kind cluster config: ${KIND_CLUSTER_CONF_FILE}"
cat ${KIND_CLUSTER_CONF_FILE}
diff --git a/linkis-dist/helm/scripts/login-pod.sh
b/linkis-dist/helm/scripts/login-pod.sh
index 1bb391928..075291b5e 100755
--- a/linkis-dist/helm/scripts/login-pod.sh
+++ b/linkis-dist/helm/scripts/login-pod.sh
@@ -17,6 +17,8 @@
WORK_DIR=`cd $(dirname $0); pwd -P`
+. ${WORK_DIR}/common.sh
+
COMPONENT_NAME=$1
LINKIS_KUBE_NAMESPACE=linkis
@@ -29,4 +31,33 @@ login() {
kubectl exec -it -n ${LINKIS_KUBE_NAMESPACE} ${POD_NAME} -- bash
}
-login ${COMPONENT_NAME}
+login_ldh() {
+
+ echo "- login [ldh]'s bash ..."
+ POD_NAME=`kubectl get pods -n ldh -l app=ldh -o
jsonpath='{.items[0].metadata.name}'`
+ kubectl exec -it -n ldh ${POD_NAME} -- bash
+
+}
+
+login_mysql() {
+ echo "- login [mysql]'s bash ..."
+ POD_NAME=`kubectl get pods -n mysql -l app=mysql -o
jsonpath='{.items[0].metadata.name}'`
+ kubectl exec -it -n mysql ${POD_NAME} -- bash
+}
+login_kind(){
+ echo "- login [kind]'s bash ..."
+ DOCKER_ID=`docker ps -aqf name=${KIND_CLUSTER_NAME}-control-plane`
+ docker exec -it ${DOCKER_ID} bash
+}
+
+if [ "${COMPONENT_NAME}" == "ldh" ]; then
+ login_ldh ${COMPONENT_NAME}
+elif [ "${COMPONENT_NAME}" == "mysql" ]; then
+ login_mysql ${COMPONENT_NAME}
+elif [ "${COMPONENT_NAME}" == "kind" ]; then
+ login_kind ${COMPONENT_NAME}
+else
+ login ${COMPONENT_NAME}
+fi
+
+
diff --git a/linkis-dist/helm/scripts/login-pod.sh
b/linkis-dist/helm/scripts/logs-pod.sh
old mode 100755
new mode 100644
similarity index 66%
copy from linkis-dist/helm/scripts/login-pod.sh
copy to linkis-dist/helm/scripts/logs-pod.sh
index 1bb391928..eaed1381f
--- a/linkis-dist/helm/scripts/login-pod.sh
+++ b/linkis-dist/helm/scripts/logs-pod.sh
@@ -22,11 +22,31 @@ COMPONENT_NAME=$1
LINKIS_KUBE_NAMESPACE=linkis
LINKIS_INSTANCE_NAME=linkis-demo
-login() {
+logs() {
component_name=$1
- echo "- login [${component_name}]'s bash ..."
POD_NAME=`kubectl get pods -n ${LINKIS_KUBE_NAMESPACE} -l
app.kubernetes.io/instance=${LINKIS_INSTANCE_NAME}-${component_name} -o
jsonpath='{.items[0].metadata.name}'`
- kubectl exec -it -n ${LINKIS_KUBE_NAMESPACE} ${POD_NAME} -- bash
+ kubectl logs -n ${LINKIS_KUBE_NAMESPACE} ${POD_NAME} -f
}
-login ${COMPONENT_NAME}
+logs_ldh() {
+ POD_NAME=`kubectl get pods -n ldh -l app=ldh -o
jsonpath='{.items[0].metadata.name}'`
+ kubectl logs -n ldh ${POD_NAME} -f
+
+}
+
+logs_mysql() {
+
+ POD_NAME=`kubectl get pods -n mysql -l app=mysql -o
jsonpath='{.items[0].metadata.name}'`
+ kubectl logs -n mysql ${POD_NAME} -f
+}
+
+
+if [ "${COMPONENT_NAME}" == "ldh" ]; then
+ logs_ldh ${COMPONENT_NAME}
+elif [ "${COMPONENT_NAME}" == "mysql" ]; then
+ logs_mysql ${COMPONENT_NAME}
+else
+ logs ${COMPONENT_NAME}
+fi
+
+
diff --git a/linkis-dist/helm/scripts/prepare-for-spark.sh
b/linkis-dist/helm/scripts/prepare-for-spark.sh
new file mode 100644
index 000000000..2bbd1123a
--- /dev/null
+++ b/linkis-dist/helm/scripts/prepare-for-spark.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+## Temporarily solve the problem that the spark submit client fails
+## and resolve the domain name of the yarn callback ecm when executing the
spark on yarn task
+
+WORK_DIR=`cd $(dirname $0); pwd -P`
+
+## copy spark resource from ldh to linkis-cg-engineconnmanager
+
+LDH_POD_NAME=`kubectl get pods -n ldh -l app=ldh -o
jsonpath='{.items[0].metadata.name}'`
+kubectl cp -n ldh ${LDH_POD_NAME}:/opt/ldh/ ./ldh
+
+ECM_POD_NAME=`kubectl get pods -n linkis -l
app.kubernetes.io/instance=linkis-demo-cg-engineconnmanager -o
jsonpath='{.items[0].metadata.name}'`
+kubectl cp ./ldh -n linkis ${ECM_POD_NAME}:/opt/ ;
+
+
+kubectl exec -it -n linkis ${ECM_POD_NAME} -- bash -c "chmod +x
/opt/ldh/1.3.0/spark-2.4.3-bin-hadoop2.7/bin/*"
+kubectl exec -it -n linkis ${ECM_POD_NAME} -- bash -c "ln -s
/opt/ldh/1.3.0/spark-2.4.3-bin-hadoop2.7 /opt/ldh/current/spark"
+kubectl exec -it -n linkis ${ECM_POD_NAME} -- bash -c "ln -s
/opt/ldh/1.3.0/hadoop-2.7.2 /opt/ldh/current/hadoop"
+kubectl exec -it -n linkis ${ECM_POD_NAME} -- bash -c "ln -s
/opt/ldh/1.3.0/apache-hive-2.3.3-bin /opt/ldh/current/hive"
+
+
+kubectl exec -it -n linkis ${ECM_POD_NAME} -- bash -c "echo 'export
SPARK_HOME=/opt/ldh/current/spark' |sudo tee --append /etc/profile"
+kubectl exec -it -n linkis ${ECM_POD_NAME} -- bash -c "echo 'export
PATH=\$SPARK_HOME/bin:\$PATH' |sudo tee --append /etc/profile"
+kubectl exec -it -n linkis ${ECM_POD_NAME} -- bash -c "source /etc/profile"
+
+# add ecm dns for ldh pod
+ECM_POD_IP=`kubectl get pods -n linkis -l
app.kubernetes.io/instance=linkis-demo-cg-engineconnmanager -o
jsonpath='{.items[0].status.podIP}'`
+
+ECM_POD_SUBDOMAIN=`kubectl get pods -n linkis -l
app.kubernetes.io/instance=linkis-demo-cg-engineconnmanager -o
jsonpath='{.items[0].spec.subdomain}'`
+
+ECM_DNS="${ECM_POD_IP}
${ECM_POD_NAME}.${ECM_POD_SUBDOMAIN}.linkis.svc.cluster.local"
+
+kubectl exec -it -n ldh ${LDH_POD_NAME} -- bash -c "echo ${ECM_DNS} |sudo tee
--append /etc/hosts"
+
+
+rm -rf ldh;
\ No newline at end of file
diff --git a/linkis-dist/helm/scripts/remote-debug-proxy.sh
b/linkis-dist/helm/scripts/remote-debug-proxy.sh
deleted file mode 100755
index 81606f093..000000000
--- a/linkis-dist/helm/scripts/remote-debug-proxy.sh
+++ /dev/null
@@ -1,97 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-# http://www.apache.org/licenses/LICENSE-2.0
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-#
-
-WORK_DIR=`cd $(dirname $0); pwd -P`
-
-ACTION=$1
-
-LINKIS_KUBE_NAMESPACE=linkis
-LINKIS_INSTANCE_NAME=linkis-demo
-
-LINKIS_PORT_MAP_WEB="8087:8087"
-LINKIS_PORT_MAP_MG_EUREKA="5001:5005"
-LINKIS_PORT_MAP_MG_GATEWAY="5002:5005"
-
-LINKIS_PORT_MAP_PS_PUBLICSERVICE="5004:5005"
-
-LINKIS_PORT_MAP_CG_LINKISMANAGER="5007:5005"
-LINKIS_PORT_MAP_CG_ENTRANCE="5008:5005"
-LINKIS_PORT_MAP_CG_ENGINECONNMANAGER="5009:5005"
-LINKIS_PORT_MAP_CG_ENGINEPLUGIN="5010:5005"
-
-start_port_forward() {
- component_name=$1
- port_map=$2
- echo "- starting port-forwad for [${component_name}] with mapping
[local->${port_map}->pod] ..."
- POD_NAME=`kubectl get pods -n ${LINKIS_KUBE_NAMESPACE} -l
app.kubernetes.io/instance=${LINKIS_INSTANCE_NAME}-${component_name} -o
jsonpath='{.items[0].metadata.name}'`
- kubectl port-forward -n ${LINKIS_KUBE_NAMESPACE} pod/${POD_NAME} ${port_map}
--address='0.0.0.0' >/dev/null &
-}
-
-stop_port_forward() {
- component_name=$1
- port_map=$2
- echo "- stopping port-forward for [${component_name}] with mapping
[local->${port_map}->pod] ..."
-
- pid=`ps aux |grep "port-forward" | grep " ${LINKIS_KUBE_NAMESPACE} " | grep
"${component_name}" | grep "${port_map}" | awk -F ' ' '{print $2}'`
- if [ "X$pid" != "X" ]; then
- kill -9 $pid
- fi
-}
-
-start_port_forward_all() {
- start_port_forward web ${LINKIS_PORT_MAP_WEB}
-
- start_port_forward mg-eureka ${LINKIS_PORT_MAP_MG_EUREKA}
- start_port_forward mg-gateway ${LINKIS_PORT_MAP_MG_GATEWAY}
-
- start_port_forward ps-publicservice ${LINKIS_PORT_MAP_PS_PUBLICSERVICE}
-
- start_port_forward cg-linkismanager ${LINKIS_PORT_MAP_CG_LINKISMANAGER}
- start_port_forward cg-entrance ${LINKIS_PORT_MAP_CG_ENTRANCE}
- start_port_forward cg-engineconnmanager
${LINKIS_PORT_MAP_CG_ENGINECONNMANAGER}
- start_port_forward cg-engineplugin ${LINKIS_PORT_MAP_CG_ENGINEPLUGIN}
-}
-
-stop_port_forward_all() {
- stop_port_forward web ${LINKIS_PORT_MAP_WEB}
-
- stop_port_forward mg-eureka ${LINKIS_PORT_MAP_MG_EUREKA}
- stop_port_forward mg-gateway ${LINKIS_PORT_MAP_MG_GATEWAY}
-
- stop_port_forward ps-publicservice ${LINKIS_PORT_MAP_PS_PUBLICSERVICE}
-
- stop_port_forward cg-linkismanager ${LINKIS_PORT_MAP_CG_LINKISMANAGER}
- stop_port_forward cg-entrance ${LINKIS_PORT_MAP_CG_ENTRANCE}
- stop_port_forward cg-engineconnmanager
${LINKIS_PORT_MAP_CG_ENGINECONNMANAGER}
- stop_port_forward cg-engineplugin ${LINKIS_PORT_MAP_CG_ENGINEPLUGIN}
-}
-
-case $ACTION in
- "start")
- start_port_forward_all
- ;;
- "stop")
- stop_port_forward_all
- ;;
- "list")
- ps aux |grep "port-forward" | grep " ${LINKIS_KUBE_NAMESPACE} " | grep
"${LINKIS_INSTANCE_NAME}"
- ;;
- *)
- echo "invalid arguments, only start,stop,list are accepted"
- exit -1
- ;;
-esac
diff --git a/linkis-dist/helm/scripts/remote-proxy.sh
b/linkis-dist/helm/scripts/remote-proxy.sh
new file mode 100644
index 000000000..5b5f95507
--- /dev/null
+++ b/linkis-dist/helm/scripts/remote-proxy.sh
@@ -0,0 +1,129 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+
+WORK_DIR=`cd $(dirname $0); pwd -P`
+
+ACTION=$1
+
+DEBUG=$2
+
+LINKIS_KUBE_NAMESPACE=linkis
+LINKIS_INSTANCE_NAME=linkis-demo
+
+LINKIS_PORT_MAP_WEB="8088:8088"
+LINKIS_PORT_MAP_MG_EUREKA="20303:20303"
+
+LINKIS_PORT_MAP_MG_GATEWAY="9001:9001"
+
+#debug port
+
+LINKIS_DEBUG_PORT_MAP_MG_EUREKA="22101:5005"
+LINKIS_DEBUG_PORT_MAP_MG_GATEWAY="22102:5005"
+
+LINKIS_DEBUG_PORT_MAP_PS_PUBLICSERVICE="22103:5005"
+
+LINKIS_DEBUG_PORT_MAP_CG_LINKISMANAGER="22104:5005"
+LINKIS_DEBUG_PORT_MAP_CG_ENTRANCE="22105:5005"
+LINKIS_DEBUG_PORT_MAP_CG_ENGINECONNMANAGER="22106:5005"
+LINKIS_DEBUG_PORT_MAP_CG_ENGINEPLUGIN="22107:5005"
+
+
+start_port_forward() {
+ component_name=$1
+ port_map=$2
+ echo "- starting port-forwad for [${component_name}] with mapping
[local->${port_map}->pod] ..."
+ POD_NAME=`kubectl get pods -n ${LINKIS_KUBE_NAMESPACE} -l
app.kubernetes.io/instance=${LINKIS_INSTANCE_NAME}-${component_name} -o
jsonpath='{.items[0].metadata.name}'`
+ kubectl port-forward -n ${LINKIS_KUBE_NAMESPACE} pod/${POD_NAME} ${port_map}
--address='0.0.0.0' >/dev/null &
+}
+
+stop_port_forward() {
+ component_name=$1
+ port_map=$2
+ echo "- stopping port-forward for [${component_name}] with mapping
[local->${port_map}->pod] ..."
+
+ pid=`ps aux |grep "port-forward" | grep " ${LINKIS_KUBE_NAMESPACE} " | grep
"${component_name}" | grep "${port_map}" | awk -F ' ' '{print $2}'`
+ if [ "X$pid" != "X" ]; then
+ kill -9 $pid
+ fi
+}
+
+start_port_forward_all() {
+ DEBUG=$1
+
+ start_port_forward web ${LINKIS_PORT_MAP_WEB}
+ start_port_forward mg-eureka ${LINKIS_PORT_MAP_MG_EUREKA}
+ start_port_forward mg-gateway ${LINKIS_PORT_MAP_MG_GATEWAY}
+
+ if [ "${DEBUG}" == "true" ]; then
+
+ start_port_forward mg-eureka
${LINKIS_DEBUG_PORT_MAP_MG_EUREKA}
+ start_port_forward mg-gateway
${LINKIS_DEBUG_PORT_MAP_MG_GATEWAY}
+
+ start_port_forward ps-publicservice
${LINKIS_DEBUG_PORT_MAP_PS_PUBLICSERVICE}
+
+ start_port_forward cg-linkismanager
${LINKIS_DEBUG_PORT_MAP_CG_LINKISMANAGER}
+ start_port_forward cg-entrance
${LINKIS_DEBUG_PORT_MAP_CG_ENTRANCE}
+ start_port_forward cg-engineconnmanager
${LINKIS_DEBUG_PORT_MAP_CG_ENGINECONNMANAGER}
+ start_port_forward cg-engineplugin
${LINKIS_DEBUG_PORT_MAP_CG_ENGINEPLUGIN}
+ fi
+}
+
+stop_port_forward_all() {
+ DEBUG=$1
+
+ stop_port_forward web ${LINKIS_PORT_MAP_WEB}
+ stop_port_forward mg-eureka ${LINKIS_PORT_MAP_MG_EUREKA}
+ stop_port_forward mg-gateway ${LINKIS_PORT_MAP_MG_GATEWAY}
+
+ if [ "${DEBUG}" == "true" ]; then
+
+ stop_port_forward mg-eureka ${LINKIS_DEBUG_PORT_MAP_MG_EUREKA}
+ stop_port_forward mg-gateway
${LINKIS_DEBUG_PORT_MAP_MG_GATEWAY}
+
+ stop_port_forward ps-publicservice
${LINKIS_DEBUG_PORT_MAP_PS_PUBLICSERVICE}
+
+ stop_port_forward cg-linkismanager
${LINKIS_DEBUG_PORT_MAP_CG_LINKISMANAGER}
+ stop_port_forward cg-entrance
${LINKIS_DEBUG_PORT_MAP_CG_ENTRANCE}
+ stop_port_forward cg-engineconnmanager
${LINKIS_DEBUG_PORT_MAP_CG_ENGINECONNMANAGER}
+ stop_port_forward cg-engineplugin
${LINKIS_DEBUG_PORT_MAP_CG_ENGINEPLUGIN}
+
+ fi
+
+
+}
+
+case $ACTION in
+ "start")
+ start_port_forward_all
+ ;;
+ "stop")
+ stop_port_forward_all
+ ;;
+ "start-with-debug")
+ start_port_forward_all true
+ ;;
+ "stop-with-debug")
+ stop_port_forward_all true
+ ;;
+ "list")
+ ps aux |grep "port-forward" | grep " ${LINKIS_KUBE_NAMESPACE} " | grep
"${LINKIS_INSTANCE_NAME}"
+ ;;
+ *)
+ echo "invalid arguments, only
start,start-with-debug,stop,stop-with-debug,list are accepted"
+ exit -1
+ ;;
+esac
diff --git a/linkis-dist/helm/scripts/resources/kind-cluster.yaml
b/linkis-dist/helm/scripts/resources/kind-cluster.yaml
index 1c071fa16..338dd13db 100644
--- a/linkis-dist/helm/scripts/resources/kind-cluster.yaml
+++ b/linkis-dist/helm/scripts/resources/kind-cluster.yaml
@@ -20,6 +20,8 @@ nodes:
extraMounts:
- hostPath: ${KIND_CLUSTER_HOST_PATH}
containerPath: /data
+ - hostPath: ${KIND_COMMON_PATH}
+ containerPath: /opt/common
# - role: worker
# extraMounts:
# - hostPath: ${KIND_CLUSTER_HOST_PATH}
diff --git
a/linkis-dist/helm/scripts/resources/ldh/configmaps/configmap-hadoop.yaml
b/linkis-dist/helm/scripts/resources/ldh/configmaps/configmap-hadoop.yaml
index aa5ff2e32..fa74a304c 100644
--- a/linkis-dist/helm/scripts/resources/ldh/configmaps/configmap-hadoop.yaml
+++ b/linkis-dist/helm/scripts/resources/ldh/configmaps/configmap-hadoop.yaml
@@ -530,136 +530,65 @@ data:
#export YARN_ROUTER_OPTS=
yarn-site.xml: |
<configuration>
- <property>
- <name>yarn.nodemanager.aux-services</name>
- <value>mapreduce_shuffle</value>
- </property>
- <!--
- <property>
- <name>yarn.nodemanager.disk-health-checker.enable</name>
- <value>false</value>
- </property>
- -->
- </configuration>
- capacity-scheduler.xml: |
- <configuration>
-
- <property>
- <name>yarn.scheduler.capacity.maximum-applications</name>
- <value>10000</value>
- <description>
- Maximum number of applications that can be pending and running.
- </description>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.maximum-am-resource-percent</name>
- <value>0.1</value>
- <description>
- Maximum percent of resources in the cluster which can be used to run
- application masters i.e. controls number of concurrent running
- applications.
- </description>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.resource-calculator</name>
-
<value>org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator</value>
- <description>
- The ResourceCalculator implementation to be used to compare
- Resources in the scheduler.
- The default i.e. DefaultResourceCalculator only uses Memory while
- DominantResourceCalculator uses dominant-resource to compare
- multi-dimensional resources such as Memory, CPU etc.
- </description>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.queues</name>
- <value>default</value>
- <description>
- The queues at the this level (root is the root queue).
- </description>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.default.capacity</name>
- <value>100</value>
- <description>Default queue target capacity.</description>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.default.user-limit-factor</name>
- <value>1</value>
- <description>
- Default queue user limit a percentage from 0.0 to 1.0.
- </description>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.default.maximum-capacity</name>
- <value>100</value>
- <description>
- The maximum capacity of the default queue.
- </description>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.default.state</name>
- <value>RUNNING</value>
- <description>
- The state of the default queue. State can be one of RUNNING or
STOPPED.
- </description>
- </property>
-
- <property>
-
<name>yarn.scheduler.capacity.root.default.acl_submit_applications</name>
- <value>*</value>
- <description>
- The ACL of who can submit jobs to the default queue.
- </description>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.default.acl_administer_queue</name>
- <value>*</value>
- <description>
- The ACL of who can administer jobs on the default queue.
- </description>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.node-locality-delay</name>
- <value>40</value>
- <description>
- Number of missed scheduling opportunities after which the
CapacityScheduler
- attempts to schedule rack-local containers.
- Typically this should be set to number of nodes in the cluster, By
default is setting
- approximately number of nodes in one rack which is 40.
- </description>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.queue-mappings</name>
- <value></value>
- <description>
- A list of mappings that will be used to assign jobs to queues
- The syntax for this list is [u|g]:[name]:[queue_name][,next mapping]*
- Typically this list will be used to map users to queues,
- for example, u:%user:%user maps all users to queues with the same
name
- as the user.
- </description>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.queue-mappings-override.enable</name>
- <value>false</value>
- <description>
- If a queue mapping is present, will it override the value specified
- by the user? This can be used by administrators to place jobs in
queues
- that are different than the one specified by the user.
- The default is false.
- </description>
- </property>
+ <property>
+ <name>yarn.resourcemanager.scheduler.class</name>
+
<value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler</value>
+ <description>The class to use as the resource
scheduler.</description>
+ </property>
+ <property>
+ <name>yarn.scheduler.fair.allocation.file</name>
+ <value>fair-scheduler.xml</value>
+ <description>Whether to use the username associated with the
allocation as the default queue name, in the event that a queue name is not
specified.</description>
+ </property>
+ <property>
+ <name>yarn.nodemanager.resource.memory-mb</name>
+ <value>56080</value>
+ <description>Amount of physical memory, in MB, that can be
allocated for containers.</description>
+ </property>
+ <property>
+ <name>yarn.nodemanager.resource.cpu-vcores</name>
+ <value>30</value>
+ <description>Number of CPU cores that can be allocated for
containers.</description>
+ </property>
+ <property>
+ <name>yarn.nodemanager.vmem-check-enabled</name>
+ <value>false</value>
+ <description>Whether virtual memory limits will be enforced for
containers</description>
+ </property>
+ <property>
+ <name>yarn.nodemanager.vmem-pmem-ratio</name>
+ <value>10.1</value>
+ <description>Ratio between virtual memory to physical memory when
+ setting memory limits for containers. Container allocations are
+ expressed in terms of physical memory, and virtual memory usage
+ is allowed to exceed this allocation by this
ratio.</description>
+ </property>
+ <property>
+ <name>yarn.nodemanager.aux-services</name>
+ <value>mapreduce_shuffle</value>
+ <description>the valid service name should only contain a-zA-Z0-9_
and can not start with numbers</description>
+ </property>
+ <property>
+ <name>yarn.nodemanager.disk-health-checker.enable</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.address</name>
+ <value>ldh.ldh.svc.cluster.local:8032</value>
+ </property>
</configuration>
+
+ fair-scheduler.xml: |
+ <allocations>
+ <queueMaxAMShareDefault>0.5</queueMaxAMShareDefault>
+ <queue name="root">
+ <queue name="default">
+ <maxRunningApps>3</maxRunningApps>
+ <weight>2</weight>
+ <schedulingPolicy>fair</schedulingPolicy>
+ <aclSubmitApps>*</aclSubmitApps>
+ <aclAdministerApps>*</aclAdministerApps>
+ </queue>
+ </queue>
+ </allocations>
\ No newline at end of file
diff --git
a/linkis-dist/helm/scripts/resources/ldh/configmaps/configmap-spark.yaml
b/linkis-dist/helm/scripts/resources/ldh/configmaps/configmap-spark.yaml
index e1de98841..5c95e795c 100644
--- a/linkis-dist/helm/scripts/resources/ldh/configmaps/configmap-spark.yaml
+++ b/linkis-dist/helm/scripts/resources/ldh/configmaps/configmap-spark.yaml
@@ -80,6 +80,40 @@ data:
# - MKL_NUM_THREADS=1 Disable multi-threading of Intel MKL
# - OPENBLAS_NUM_THREADS=1 Disable multi-threading of OpenBLAS
+ hive-site.xml: |
+ <?xml version="1.0" encoding="UTF-8" standalone="no"?>
+ <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+ <configuration>
+ <property>
+ <name>javax.jdo.option.ConnectionURL</name>
+
<value>jdbc:mysql://mysql.mysql.svc.cluster.local:3306/hive_metadata?&createDatabaseIfNotExist=true&characterEncoding=UTF-8&useSSL=false</value>
+ </property>
+ <property>
+ <name>javax.jdo.option.ConnectionUserName</name>
+ <value>root</value>
+ </property>
+ <property>
+ <name>javax.jdo.option.ConnectionPassword</name>
+ <value>123456</value>
+ </property>
+ <property>
+ <name>javax.jdo.option.ConnectionDriverName</name>
+ <value>com.mysql.jdbc.Driver</value>
+ </property>
+ <property>
+ <name>datanucleus.schema.autoCreateAll</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>hive.metastore.schema.verification</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>hive.metastore.uris</name>
+ <value>thrift://ldh.ldh.svc.cluster.local:9083</value>
+ </property>
+ </configuration>
+
spark-defaults.conf: |
# Default system properties included when running spark-submit.
# This is useful for setting default environmental settings.
diff --git a/linkis-dist/package/sbin/ext/linkis-common-start
b/linkis-dist/package/sbin/ext/linkis-common-start
index c93ce958c..30390f425 100644
--- a/linkis-dist/package/sbin/ext/linkis-common-start
+++ b/linkis-dist/package/sbin/ext/linkis-common-start
@@ -69,8 +69,16 @@ if [ ! -r "$SERVER_LIB" ] ; then
exit 1
fi
+##The extended lib such mysql-connector-java-*.jar
+export LINKIS_EXTENDED_LIB=$LINKIS_EXTENDED_LIB
+if [ -r "$LINKIS_EXTENDED_LIB" ] ; then
+ echo "add extended lib $LINKIS_EXTENDED_LIB"
+fi
+
+
+
## set class path
-export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$LINKIS_COMMONS_LIB/*:$SERVER_LIB/*
+export
SERVER_CLASS_PATH=$SERVER_CONF_PATH:$LINKIS_COMMONS_LIB/*:$SERVER_LIB/*:$LINKIS_EXTENDED_LIB/*
echo "=====Java Start Command====="
diff --git a/linkis-dist/package/sbin/ext/linkis-mg-gateway
b/linkis-dist/package/sbin/ext/linkis-mg-gateway
index bc51387bd..9b44e7968 100644
--- a/linkis-dist/package/sbin/ext/linkis-mg-gateway
+++ b/linkis-dist/package/sbin/ext/linkis-mg-gateway
@@ -72,8 +72,14 @@ if [ ! -r "$SERVER_LIB" ] ; then
exit 1
fi
+##The extended lib such mysql-connector-java-*.jar
+export LINKIS_EXTENDED_LIB=$LINKIS_EXTENDED_LIB
+if [ -r "$LINKIS_EXTENDED_LIB" ] ; then
+ echo "add extended lib $LINKIS_EXTENDED_LIB"
+fi
+
## set class path
-export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$SERVER_LIB/*
+export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$SERVER_LIB/*:$LINKIS_EXTENDED_LIB/*
echo "=====Java Start Command====="
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
