This is an automated email from the ASF dual-hosted git repository. kinghao pushed a commit to branch master-feature in repository https://gitbox.apache.org/repos/asf/linkis.git
commit b87f07a7539e1ef9d15d596d28dbf46c4ad9ddbd Merge: e3c84aa44f 2a3d1b1566 Author: kinghao <[email protected]> AuthorDate: Fri Nov 21 22:19:29 2025 +0800 merge 1.9.0 .github/workflows/dlc.json | 11 +- .github/workflows/integration-test.yml | 15 +- README.md | 109 ++- README_CN.md | 109 ++- codecheck.ignore | 28 + linkis-commons/linkis-common/pom.xml | 16 +- .../org/apache/linkis/common/utils/AESUtils.java | 175 ++++ .../org/apache/linkis/common/utils/SHAUtils.java | 97 +++ .../apache/linkis/common/conf/Configuration.scala | 39 +- .../linkis/common/utils/CodeAndRunTypeUtils.scala | 4 +- .../org/apache/linkis/common/utils/LDAPUtils.scala | 2 +- .../org/apache/linkis/common/utils/RSAUtils.scala | 118 ++- .../org/apache/linkis/common/utils/Utils.scala | 9 + .../linkis/common/variable/DateTypeUtils.scala | 4 + .../linkis/common/utils/SecurityUtilsTest.java | 15 + .../common/variable/VariableOperationTest.java | 4 +- .../common/utils/CodeAndRunTypeUtilsTest.scala | 2 +- .../linkis/hadoop/common/conf/HadoopConf.scala | 4 +- .../linkis/hadoop/common/utils/HDFSUtils.scala | 41 +- .../linkis/httpclient/AbstractHttpClient.scala | 17 +- .../linkis/mybatis/conf/MybatisConfiguration.java | 2 +- .../mybatis/conf/MybatisConfigurationTest.java | 2 +- .../linkis/protocol/constants/TaskConstant.java | 2 + .../linkis/scheduler/util/SchedulerUtils.java | 84 ++ .../scheduler/conf/SchedulerConfiguration.scala | 18 + .../scheduler/executer/ExecuteResponse.scala | 3 + .../scheduler/listener/JobRetryListener.scala | 16 +- .../org/apache/linkis/scheduler/queue/Job.scala | 23 +- .../scheduler/queue/PriorityLoopArrayQueue.scala | 270 ++++++ .../linkis/scheduler/queue/SchedulerEvent.scala | 12 + .../queue/fifoqueue/FIFOConsumerManager.scala | 5 +- .../queue/fifoqueue/FIFOUserConsumer.scala | 2 + .../parallelqueue/ParallelConsumerManager.scala | 16 +- .../queue/PriorityLoopArrayQueueTest.java | 234 +++++ .../org/apache/linkis/scheduler/queue/Test.scala | 31 +- .../linkis/scheduler/util/TestSchedulerUtils.scala | 73 ++ .../org/apache/linkis/storage/fs/FileSystem.java | 18 +- .../storage/fs/impl/AzureBlobFileSystem.java | 2 +- .../linkis/storage/fs/impl/HDFSFileSystem.java | 58 +- .../linkis/storage/fs/impl/LocalFileSystem.java | 17 +- .../linkis/storage/fs/impl/OSSFileSystem.java | 5 - .../linkis/storage/fs/impl/S3FileSystem.java | 5 - .../linkis/storage/conf/LinkisStorageConf.scala | 7 +- .../linkis/storage/domain/MethodEntity.scala | 7 +- .../org/apache/linkis/ujes/client/UJESClient.scala | 9 +- .../ujes/client/request/EncryptTokenAction.scala | 59 ++ .../ujes/client/response/EncryptTokenResult.scala | 18 +- .../ujes/client/response/JobInfoResult.scala | 2 + .../ujes/client/response/JobInfoResultTest.java | 93 ++ .../governance/common/entity/job/JobAiRequest.java | 195 +++++ .../common/protocol/conf/TenantConf.scala | 4 +- .../common/protocol/job/JobReqProcotol.scala | 4 +- .../common/protocol/task/ResponseTaskExecute.scala | 9 + .../governance/common/utils/GovernanceUtils.scala | 27 +- .../ecm/core/launch/ProcessEngineConnLaunch.scala | 28 +- .../executor/conf/ComputationExecutorConf.scala | 21 +- .../executor/execute/ComputationExecutor.scala | 122 ++- .../executor/execute/EngineExecutionContext.scala | 23 +- .../service/TaskExecutionServiceImpl.scala | 49 +- .../engineconn/common/conf/EngineConnConf.scala | 2 + .../common/conf/EnvConfiguration.scala | 2 +- .../entrance/conf/EntranceSpringConfiguration.java | 2 + .../errorcode/EntranceErrorCodeSummary.java | 7 +- .../linkis/entrance/job/EntranceExecutionJob.java | 2 +- .../entrance/parser/AbstractEntranceParser.java | 11 + .../apache/linkis/entrance/parser/ParserUtils.java | 57 +- .../persistence/QueryPersistenceEngine.java | 5 +- .../persistence/QueryPersistenceManager.java | 155 ++++ .../entrance/server/DefaultEntranceServer.java | 6 +- .../apache/linkis/entrance/EntranceServer.scala | 213 +++-- .../entrance/conf/EntranceConfiguration.scala | 144 ++- .../entrance/execute/DefaultEntranceExecutor.scala | 106 ++- .../linkis/entrance/execute/EntranceExecutor.scala | 8 + .../linkis/entrance/execute/EntranceJob.scala | 5 + .../impl/AISQLTransformInterceptor.scala | 182 ++++ .../impl/SensitiveCheckInterceptor.scala | 107 +++ .../interceptor/impl/TemplateConfUtils.scala | 184 +++- .../EntranceUserParallelOrchestratorPlugin.scala | 6 +- .../entrance/parser/CommonEntranceParser.scala | 140 ++- .../entrance/persistence/PersistenceManager.scala | 8 +- .../entrance/scheduler/EntranceGroupFactory.scala | 45 +- .../linkis/entrance/utils/CommonLogPathUtils.scala | 47 +- .../linkis/entrance/utils/EntranceUtils.scala | 346 +++++++- .../linkis/entrance/utils/JobHistoryHelper.scala | 18 +- .../apache/linkis/entrance/utils/SafeUtils.scala | 92 ++ .../interceptor/impl/TemplateConfUtilsTest.java | 62 +- .../linkis/ujes/jdbc/UJESSQLDriverMain.scala | 14 +- .../apache/linkis/ujes/jdbc/UJESSQLResultSet.scala | 25 +- .../linkis/manager/am/conf/AMConfiguration.java | 40 +- .../am/converter/DefaultMetricsConverter.java | 1 + .../am/manager/DefaultEngineNodeManager.java | 25 +- .../manager/am/pointer/AbstractNodePointer.java | 5 + .../linkis/manager/am/restful/EMRestfulApi.java | 1 + .../manager/am/restful/EngineRestfulApi.java | 14 +- .../am/service/impl/ECResourceInfoServiceImpl.java | 7 +- .../rm/external/yarn/YarnResourceRequester.java | 136 ++- .../conf/EngineConnPluginConfiguration.scala | 3 + .../am/selector/rule/HotspotExclusionRule.scala | 16 +- .../am/service/em/DefaultEMInfoService.scala | 30 +- .../engine/DefaultEngineAskEngineService.scala | 204 +++-- .../engine/DefaultEngineCreateService.scala | 33 +- .../service/engine/DefaultEngineInfoService.scala | 28 +- .../service/engine/DefaultEngineReuseService.scala | 184 +++- .../service/engine/DefaultEngineStopService.scala | 40 +- .../apache/linkis/manager/am/utils/AMUtils.scala | 116 ++- .../manager/label/service/NodeLabelService.scala | 5 + .../service/impl/DefaultNodeLabelService.scala | 72 +- .../manager/rm/entity/LabelResourceMap.scala | 5 +- .../manager/rm/entity/ResourceOperationType.scala | 2 +- .../linkis/manager/rm/restful/RMMonitorRest.scala | 2 +- .../manager/rm/service/ResourceLockService.scala | 2 +- .../rm/service/impl/DefaultResourceManager.scala | 237 +++-- .../selector/rule/HotspotExclusionRuleTest.scala | 4 +- .../label/builder/CombinedLabelBuilder.java | 2 +- .../manager/label/conf/LabelCommonConfig.java | 29 +- .../manager/label/entity/cluster/ClusterLabel.java | 5 + .../label/utils/EngineTypeLabelCreator.java | 25 +- .../manager/common/conf/RMConfiguration.java | 17 +- .../manager/common/entity/node/AMEngineNode.java | 12 + .../manager/common/entity/node/EngineNode.java | 4 + .../common/entity/persistence/PersistenceNode.java | 13 + .../entity/persistence/PersistenceResource.java | 20 + .../entity/resource/LoadInstanceResource.java | 16 +- .../common/entity/resource/LoadResource.java | 2 +- .../common/entity/resource/MemoryResource.java | 2 +- .../common/entity/resource/SpecialResource.java | 5 +- .../linkis/manager/common/utils/ResourceUtils.java | 3 + .../linkis/manager/dao/LabelManagerMapper.java | 10 + .../persistence/ResourceManagerPersistence.java | 6 + .../impl/DefaultLabelManagerPersistence.java | 77 +- .../impl/DefaultNodeManagerPersistence.java | 4 + .../impl/DefaultNodeMetricManagerPersistence.java | 1 + .../impl/DefaultResourceManagerPersistence.java | 5 + .../resources/mapper/common/LabelManagerMapper.xml | 60 +- .../resources/mapper/common/NodeManagerMapper.xml | 3 + linkis-dist/bin/install-linkis-to-kubernetes.sh | 2 + .../ldh-with-mysql-jdbc.Dockerfile} | 39 +- linkis-dist/docker/ldh.Dockerfile | 9 +- .../scripts/make-ldh-image-with-mysql-jdbc.sh | 51 ++ .../linkis/templates/configmap-linkis-config.yaml | 6 +- linkis-dist/helm/charts/linkis/templates/jobs.yaml | 11 +- linkis-dist/helm/scripts/install-ldh.sh | 5 +- linkis-dist/helm/scripts/prepare-for-spark.sh | 25 +- .../resources/ldh/configmaps/configmap-hadoop.yaml | 125 ++- .../package/admin/clear_bml_resources_task.sh | 57 ++ linkis-dist/package/admin/clear_history_task.sh | 45 +- linkis-dist/package/admin/linkis_task_archive.sh | 82 ++ .../admin/linkis_udf_get_python_methods.py} | 44 +- linkis-dist/package/admin/tools/linkis-analyze.sh | 601 +++++++++++++ linkis-dist/package/admin/tools/linkis-log-tool.sh | 61 ++ linkis-dist/package/admin/tools/remote-engine.sh | 53 ++ linkis-dist/package/conf/application-eureka.yml | 6 +- linkis-dist/package/db/linkis_ddl.sql | 143 ++- .../db/upgrade/1.4.1_schema/mysql/linkis_ddl.sql | 96 ++ linkis-dist/release-docs/NOTICE | 671 +++++++++++++- .../release-docs/licenses/LICENSE-azure.txt | 21 + .../release-docs/licenses/LICENSE-content-type.txt | 202 +++++ .../release-docs/licenses/LICENSE-lang-tag.txt | 202 +++++ .../release-docs/licenses/LICENSE-msal4j.txt | 21 + .../licenses/LICENSE-oauth2-oidc-sdk.txt | 202 +++++ .../hive/src/main/assembly/distribution.xml | 4 + .../hive/src/main/resources/log4j2.xml | 11 +- .../hive/conf/HiveEngineConfiguration.scala | 12 + .../hive/executor/HiveEngineConnExecutor.scala | 49 +- .../HiveProcessEngineConnLaunchBuilder.scala | 12 + .../io/executor/IoEngineConnExecutor.scala | 5 +- .../manager/engineplugin/io/utils/IOHelp.scala | 8 +- linkis-engineconn-plugins/jdbc/pom.xml | 6 - .../engineplugin/jdbc/ConnectionManager.java | 5 + .../jdbc/constant/JDBCEngineConnConstant.java | 11 +- .../engineplugin/jdbc/conf/JDBCConfiguration.scala | 8 +- .../jdbc/executor/JDBCEngineConnExecutor.scala | 94 +- .../jdbc/executor/JDBCMultiDatasourceParser.scala | 61 +- .../nebula/executor/NebulaEngineConnExecutor.java | 4 +- .../AccessibleExecutorConfiguration.scala | 46 + linkis-engineconn-plugins/spark/pom.xml | 133 +++ .../spark/client/deployment/util/FileUtils.java | 3 +- .../client/deployment/util/KubernetesHelper.java | 2 +- .../engineplugin/spark/common/MultiTreeNode.java | 66 ++ .../spark/errorcode/SparkErrorCodeSummary.java | 1 + .../engineplugin/spark/utils/PlanParseUtil.java | 262 ++++++ .../spark/src/main/resources/log4j2.xml | 8 +- .../spark/config/SparkConfiguration.scala | 4 + .../spark/exception/RuleCheckFailedException.scala | 12 +- .../engineplugin/spark/executor/SQLSession.scala | 14 + .../spark/executor/SparkEngineConnExecutor.scala | 121 ++- .../spark/executor/SparkSqlExecutor.scala | 1 + .../spark/extension/SparkUDFCheckRule.scala | 69 ++ .../monitor/config/ApplicationConfiguration.java | 2 - .../monitor/scheduled/JobHistoryMonitor.java | 7 + .../linkis/monitor/scheduled/TaskArchiveClear.java | 7 +- .../monitor/scheduled/UserDepartmentInfoSync.java | 2 +- .../apache/linkis/monitor/until/ThreadUtils.java | 9 +- .../linkis/monitor/constants/Constants.scala | 1 + .../jobhistory/jobtime/JobTimeExceedRule.scala | 16 +- .../jobtime/StarrocksTimeExceedAlterSender.scala | 84 ++ .../jobtime/StarrocksTimeExceedRule.scala | 20 + .../jobtime/StarrocksTimeKillHitEvent.scala} | 0 ...torResourceAction.scala => MonitorAction.scala} | 0 .../io/iteraceptor/IOMethodInterceptor.scala | 21 +- .../computation/monitor/EngineConnMonitor.scala | 20 +- .../physical/CodeLogicalUnitExecTask.scala | 2 + .../service/ComputationTaskExecutionReceiver.scala | 10 + .../conf/OrchestratorConfiguration.scala | 5 +- .../execution/impl/DefaultFailedTaskResponse.scala | 6 + .../orchestrator/plans/physical/ExecTask.scala | 10 + .../plans/physical/PhysicalContextImpl.scala | 40 +- .../configuration/dao/DepartmentTenantMapper.java | 5 +- .../configuration/dao/TemplateConfigKeyMapper.java | 4 + .../restful/api/ConfigurationRestfulApi.java | 68 +- .../restful/api/TemplateRestfulApi.java | 36 +- .../restful/api/TenantConfigrationRestfulApi.java | 8 +- .../service/TemplateConfigKeyService.java | 2 + .../configuration/service/TenantConfigService.java | 9 +- .../service/impl/TemplateConfigKeyServiceImpl.java | 59 ++ .../service/impl/TenantConfigServiceImpl.java | 25 +- .../service/impl/TenantServiceImpl.java | 11 +- .../resources/mapper/common/DepartmentMapper.xml | 2 +- .../mapper/common/DepartmentTenantMapper.xml | 2 + .../mapper/common/TemplateConfigKeyMapper.xml | 15 + .../linkis/configuration/conf/Configuration.scala | 14 + .../service/ConfigurationService.scala | 8 +- .../core/conf/DatasourceConf.java | 10 +- .../datasourcemanager/core/dao/DataSourceDao.java | 4 + .../core/dao/DataSourceVersionDao.java | 2 + .../core/receivers/DsmReceiver.java | 11 +- .../core/restful/DataSourceCoreRestfulApi.java | 523 ++++++++++- .../core/restful/DataSourceOperateRestfulApi.java | 18 + .../core/restful/RestfulApiHelper.java | 56 +- .../core/service/DataSourceInfoService.java | 14 +- .../service/impl/DataSourceInfoServiceImpl.java | 52 ++ .../query/server/restful/MetadataQueryRestful.java | 56 -- .../resources/mapper/mysql/DataSouceMapper.xml | 9 + .../mapper/mysql/DataSourceVersionMapper.xml | 4 + .../metadata/query/service/EsMetaService.java | 6 +- .../query/service/AbstractSqlConnection.java | 176 ++-- .../query/service/ClickhouseMetaService.java | 24 +- .../metadata/query/service/Db2MetaService.java | 16 - .../metadata/query/service/DmMetaService.java | 24 +- .../query/service/GreenplumMetaService.java | 16 - .../query/service/KingbaseMetaService.java | 16 - .../metadata/query/service/MysqlMetaService.java | 16 - .../metadata/query/service/OracleMetaService.java | 16 - .../query/service/PostgresqlMetaService.java | 16 - .../query/service/SqlserverMetaService.java | 16 - .../query/service/clickhouse/SqlConnection.java | 102 ++- .../metadata/query/service/db2/SqlConnection.java | 165 +++- .../metadata/query/service/dm/SqlConnection.java | 116 ++- .../query/service/greenplum/SqlConnection.java | 148 +++- .../query/service/kingbase/SqlConnection.java | 137 ++- .../query/service/mysql/SqlConnection.java | 181 ++-- .../query/service/oracle/SqlConnection.java | 133 ++- .../query/service/postgres/SqlConnection.java | 148 +++- .../query/service/sqlserver/SqlConnection.java | 123 ++- .../query/service/starrocks/SqlConnection.java | 251 ++++++ .../service/impl/MetadataQueryServiceImpl.java | 521 +++++++++++ .../query/service/AbstractSqlConnection.java | 66 -- .../query/service/clickhouse/SqlConnection.java | 113 ++- .../metadata/query/service/db2/SqlConnection.java} | 175 ++-- .../metadata/query/service/dm/SqlConnection.java | 104 ++- .../query/service/greenplum/SqlConnection.java} | 185 ++-- .../query/service/kingbase/SqlConnection.java | 101 ++- .../query/service/mysql/SqlConnection.java | 148 +++- .../query/service/oracle/SqlConnection.java | 126 ++- .../query/service/postgres}/SqlConnection.java | 165 ++-- .../query/service/sqlserver/SqlConnection.java | 109 ++- .../query/service/starrocks/SqlConnection.java | 251 ++++++ .../domain/mdq/po/RangerBaseModelObject.java | 126 +++ .../metadata/domain/mdq/po/RangerPolicy.java | 798 +++++++++++++++++ .../domain/mdq/vo/MdqTableStatisticInfoDTO.java | 68 ++ .../apache/linkis/metadata/hive/config/DSEnum.java | 3 +- .../linkis/metadata/hive/dao/HiveMetaDao.java | 64 +- .../apache/linkis/metadata/hive/dao/RangerDao.java | 14 +- .../metadata/restful/api/DataSourceRestfulApi.java | 82 +- .../metadata/restful/api/MdqTableRestfulApi.java | 79 +- .../restful/remote/DataSourceRestfulRemote.java | 5 +- .../linkis/metadata/service/DataSourceService.java | 26 +- .../service/HiveMetaWithPermissionService.java | 2 +- .../apache/linkis/metadata/service/MdqService.java | 9 +- ...onService.java => RangerPermissionService.java} | 12 +- .../service/impl/DataSourceServiceImpl.java | 313 ++++++- .../impl/HiveMetaWithPermissionServiceImpl.java | 146 +++- .../metadata/service/impl/MdqServiceImpl.java | 89 +- .../service/impl/RangerPermissionServiceImpl.java | 125 +++ .../org/apache/linkis/metadata/util/DWSConfig.java | 15 +- .../main/resources/mapper/common/HiveMetaDao.xml | 252 ++++++ .../src/main/resources/mapper/mysql/RangerDao.xml} | 43 +- .../resources/mapper/postgresql/HiveMetaDao.xml | 552 +++++++++--- .../linkis/metadata/conf/MdqConfiguration.scala | 9 + .../linkis/metadata/hive/dao/HiveMetaDaoTest.java | 14 +- .../linkis/jobhistory/dao/JobAiHistoryMapper.java | 28 +- .../linkis/jobhistory/dao/JobDiagnosisMapper.java | 12 +- .../linkis/jobhistory/dao/JobHistoryMapper.java | 11 +- .../linkis/jobhistory/entity/JobAiHistory.java | 204 +++++ .../linkis/jobhistory/entity/JobDiagnosis.java | 101 +++ .../linkis/jobhistory/entity/QueryTaskVO.java | 28 + .../jobhistory/restful/api/QueryRestfulApi.java | 141 ++- .../linkis/jobhistory/transitional/TaskStatus.java | 11 + .../linkis/jobhistory/util/JobhistoryUtils.java | 43 +- .../resources/mapper/common/JobAiHistoryMapper.xml | 59 ++ .../resources/mapper/common/JobDiagnosisMapper.xml | 45 + .../resources/mapper/mysql/JobHistoryMapper.xml | 17 + .../jobhistory/conversions/TaskConversions.scala | 77 +- .../jobhistory/service/JobAiHistoryService.java | 17 +- .../service/JobHistoryDiagnosisService.java | 25 +- .../jobhistory/service/JobHistoryQueryService.java | 4 +- .../service/impl/JobAiHistoryServiceImpl.scala | 58 ++ .../impl/JobHistoryDiagnosisServicelmpl.scala | 52 ++ .../service/impl/JobHistoryQueryServiceImpl.scala | 116 ++- .../apache/linkis/jobhistory/util/QueryUtils.scala | 6 + .../jobhistory/dao/JobHistoryMapperTest.java | 7 +- .../service/JobHistoryQueryServiceTest.java | 1 + .../org/apache/linkis/bml/client/BmlClient.java | 2 + .../DatasourceClientErrorCodeSummary.java | 4 + .../linkis/bml/client/impl/HttpBmlClient.scala | 11 + .../datasource/client/DataSourceRemoteClient.scala | 4 + .../client/impl/LinkisDataSourceRemoteClient.scala | 5 + .../request/GetDataSourceByTypeNameAction.scala | 85 ++ .../GetDataSourceKeyDefineByTypeAction.scala | 65 ++ .../GetInfoPublishedByUserIpPortAction.scala | 121 +++ .../response/GetDataSourceByTypeNameResult.scala | 44 + .../GetDataSourceKeyDefineByTypeResult.scala | 47 + .../GetInfoPublishedByUserIpPortResult.scala | 27 +- .../common/domain/DataSource.java | 9 + .../query/common/domain/GenerateSqlInfo.java | 0 .../query/common/service/SparkDdlSQlTemplate.java | 26 +- .../linkis/udf/entity/PythonModuleInfoVO.java | 11 + .../apache/linkis/bml/protocol/BmlProtocol.scala | 6 + .../server/dao/GatewayAuthTokenRelationMapper.java | 11 +- .../domain/GatewayAuthRelationTokenEntity.java | 145 ++++ .../server/domain/GatewayAuthTokenEntity.java | 16 + .../server/restful/GatewayAuthTokenRestfulApi.java | 160 +++- ...e.java => GatewayAuthTokenRelationService.java} | 9 +- .../server/service/GatewayAuthTokenService.java | 6 + .../GatewayAuthTokenRelationServicelmpl.java} | 22 +- .../service/impl/GatewayAuthTokenServiceImpl.java | 14 + .../filesystem/conf/WorkSpaceConfiguration.java | 6 + .../filesystem/constant/WorkSpaceConstants.java | 5 +- .../exception/WorkspaceExceptionManager.java | 5 + .../filesystem/restful/api/FsRestfulApi.java | 453 ++++++---- .../linkis/filesystem/util/FilesystemUtils.java | 78 ++ .../linkis/filesystem/util/WorkspaceUtil.java | 4 +- .../mapper/common/GatewayAuthTokenMapper.xml | 3 +- .../common/GatewayAuthTokenRelationMapper.xml | 24 + .../org/apache/linkis/udf/api/UDFRestfulApi.java | 200 ++++- .../java/org/apache/linkis/udf/conf/Constants.java | 37 + .../apache/linkis/udf/entity/PythonModuleInfo.java | 11 + .../apache/linkis/udf/exception/UdfException.java | 26 +- .../java/org/apache/linkis/udf/utils/UdfUtils.java | 341 ++++++++ .../mapper/common/PythonModuleInfoMapper.xml | 6 +- .../apache/linkis/udf/api/rpc/UdfReceiver.scala | 2 + .../src/main/assembly/distribution.xml | 9 +- .../gateway/authentication/dao/TokenDao.java | 4 + .../gateway/authentication/entity/TokenEntity.java | 9 + .../main/resources/mapper/common/TokenMapper.xml | 20 + .../authentication/conf/TokenConfiguration.scala | 2 +- .../service/CachedTokenService.scala | 51 +- .../gateway/authentication/dao/TokenDaoTest.java | 2 +- .../service/CachedTokenServiceTest.java | 2 +- .../src/test/resources/create.sql | 2 +- .../src/test/resources/create_pg.sql | 2 +- .../gateway/config/GatewayConfiguration.scala | 5 + .../linkis/gateway/security/UserRestful.scala | 28 + .../security/token/TokenAuthentication.scala | 33 +- .../StaticAuthenticationStrategy.scala | 3 + .../ujes/parser/EntranceRequestGatewayParser.scala | 71 +- .../src/apps/PythonModule/.env | 13 +- .../src/apps/PythonModule/.fes.js | 23 +- .../src/apps/PythonModule/.fes.prod.js | 16 +- .../src/apps/PythonModule/.fes.test.js | 16 +- linkis-web/src/apps/PythonModule/index.html | 31 + .../src/apps/PythonModule/public/tutorial.html | 104 +++ linkis-web/src/apps/PythonModule/src/app.jsx | 68 ++ .../src/apps/PythonModule/src/global.less | 21 +- .../apps/PythonModule/src/letgo/cache-control.js | 248 ++++++ .../src/letgo/components/pageLoading.vue | 39 + .../src/apps/PythonModule/src/letgo/global.css | 15 +- .../src/apps/PythonModule/src/letgo/globalBase.js | 34 +- .../apps/PythonModule/src/letgo/letgoConstants.js | 25 +- .../apps/PythonModule/src/letgo/letgoRequest.js | 15 +- .../src/apps/PythonModule/src/letgo/pageBase.js | 18 +- .../src/apps/PythonModule/src/letgo/reactive.js | 90 ++ .../src/apps/PythonModule/src/letgo/shared.js | 52 ++ .../src/apps/PythonModule/src/letgo/useComputed.js | 24 +- .../src/apps/PythonModule/src/letgo/useInstance.js | 47 +- .../src/apps/PythonModule/src/letgo/useJSQuery.js | 140 +++ .../apps/PythonModule/src/letgo/useLetgoGlobal.js | 94 ++ .../PythonModule/src/letgo/useTemporaryState.js | 36 +- .../src/apps/PythonModule/src/locales/en-US.js | 79 ++ .../src/apps/PythonModule/src/locales/zh-CN.js | 78 ++ .../apps/PythonModule/src/pages/index/index.jsx | 964 +++++++++++++++++++++ .../src/apps/PythonModule/src/pages/index/main.js | 447 ++++++++++ .../src/apps/linkis/components/tag/index.vue | 15 +- .../src/apps/linkis/module/pythonModule/index.vue | 30 + .../linkis/module/resourceManagement/index.vue | 4 +- pom.xml | 1 + tool/dependencies/known-dependencies.txt | 17 + 397 files changed, 20846 insertions(+), 3100 deletions(-) diff --cc linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/JobHistoryMonitor.java index c1a7a16d06,712ae62899..1d96819e09 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/JobHistoryMonitor.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/JobHistoryMonitor.java @@@ -29,7 -27,10 +29,11 @@@ import org.apache.linkis.monitor.jobhis import org.apache.linkis.monitor.jobhistory.errorcode.JobHistoryErrorCodeAlertSender; import org.apache.linkis.monitor.jobhistory.index.JobIndexRule; import org.apache.linkis.monitor.jobhistory.index.JobIndexSender; +import org.apache.linkis.monitor.jobhistory.jobtime.*; + import org.apache.linkis.monitor.jobhistory.jobtime.JobTimeExceedAlertSender; + import org.apache.linkis.monitor.jobhistory.jobtime.JobTimeExceedRule; + import org.apache.linkis.monitor.jobhistory.jobtime.StarrocksTimeExceedAlterSender; + import org.apache.linkis.monitor.jobhistory.jobtime.StarrocksTimeExceedRule; import org.apache.linkis.monitor.jobhistory.labels.JobHistoryLabelsAlertSender; import org.apache.linkis.monitor.jobhistory.labels.JobHistoryLabelsRule; import org.apache.linkis.monitor.jobhistory.runtime.CommonJobRunTimeRule; @@@ -222,52 -215,9 +226,55 @@@ public class JobHistoryMonitor jobTimeAlerts.keySet(), new JobTimeExceedAlertSender(jobTimeAlerts)); scanner.addScanRule(jobTimeExceedRule); } + StarrocksTimeExceedRule starrocksTimeExceedRule = + new StarrocksTimeExceedRule(new StarrocksTimeExceedAlterSender()); + scanner.addScanRule(starrocksTimeExceedRule); JobMonitorUtils.run(scanner, fetchers, shouldStart); } + + /** * 每10分钟扫描一次,扫描两个小时之内的任务，告警要求：管理台配置告警相关参数 */ + @Scheduled(cron = "${linkis.monitor.jdbc.timeout.alert.cron:0 0/10 0 * * ?}") + public void jdbcUnfinishedAlertScan() { + long id = + Optional.ofNullable(CacheUtils.cacheBuilder.getIfPresent("jdbcUnfinishedAlertScan")) + .orElse(MonitorConfig.JOB_HISTORY_TIME_EXCEED.getValue()); + long intervalMs = 7200 * 1000L; + long maxIntervalMs = Constants.ERRORCODE_MAX_INTERVALS_SECONDS() * 1000; + long endTime = System.currentTimeMillis(); + long startTime = endTime - intervalMs; + AnomalyScanner scanner = new DefaultScanner(); + List<DataFetcher> fetchers = + JobMonitorUtils.generateFetchers(startTime, endTime, maxIntervalMs, id, ""); + if (fetchers.isEmpty()) { + logger.warn("jdbcUnfinishedScan generated 0 dataFetchers, plz check input"); + return; + } + StarrocksTimeExceedRule starrocksTimeExceedRule = + new StarrocksTimeExceedRule(new StarrocksTimeExceedAlertSender()); + scanner.addScanRule(starrocksTimeExceedRule); + JobMonitorUtils.run(scanner, fetchers, true); + } + + /** * 每10分钟扫描一次,扫描两个小时之内的任务，满足要求触发kill kill要求：数据源配置kill参数 */ + @Scheduled(cron = "${linkis.monitor.jdbc.timeout.kill.cron:0 0/10 0 * * ?}") + public void jdbcUnfinishedKillScan() { + long id = + Optional.ofNullable(CacheUtils.cacheBuilder.getIfPresent("jdbcUnfinishedKillScan")) + .orElse(MonitorConfig.JOB_HISTORY_TIME_EXCEED.getValue()); + long intervalMs = 7200 * 1000L; + long maxIntervalMs = Constants.ERRORCODE_MAX_INTERVALS_SECONDS() * 1000; + long endTime = System.currentTimeMillis(); + long startTime = endTime - intervalMs; + AnomalyScanner scanner = new DefaultScanner(); + List<DataFetcher> fetchers = + JobMonitorUtils.generateFetchers(startTime, endTime, maxIntervalMs, id, ""); + if (fetchers.isEmpty()) { + logger.warn("jdbcUnfinishedScan generated 0 dataFetchers, plz check input"); + return; + } + StarrocksTimeKillRule starrocksTimeKillRule = + new StarrocksTimeKillRule(new StarrocksTimeKillAlertSender()); + scanner.addScanRule(starrocksTimeKillRule); + JobMonitorUtils.run(scanner, fetchers, true); + } } diff --cc linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/constants/Constants.scala index affa0ccb83,983b08f909..995cb9bb0a --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/constants/Constants.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/constants/Constants.scala @@@ -113,4 -113,6 +113,5 @@@ object Constants val JDBC_ENGINE = "jdbc" + val TASK_ARCHIVE_SH = "linkis_task_archive.sh" - } diff --cc linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/jobtime/StarrocksTimeExceedRule.scala index b616c5c02c,8521af6b57..d1d0820db8 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/jobtime/StarrocksTimeExceedRule.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/jobtime/StarrocksTimeExceedRule.scala @@@ -83,10 -80,27 +83,30 @@@ class StarrocksTimeExceedRule(hitObserv alertData.add(job) } } + } + if (taskMinID == 0L || taskMinID > job.getId) { + taskMinID = job.getId + scanRuleList.put("jdbcUnfinishedAlertScan", taskMinID) + // 获取超时kill配置信息 + if (StringUtils.isNotBlank(job.getParams)) { + val connectParamsMap = MapUtils.getMap( + datasourceConfMap, + "connectParams", + new util.HashMap[AnyRef, AnyRef] + ) + val killTime = MapUtils.getString(connectParamsMap, "kill_task_time", "") + logger.info("starock killTime: {}", killTime) + if (StringUtils.isNotBlank(killTime) && elapse > killTime.toLong * 60 * 1000) { + if (StringUtils.isNotBlank(killTime)) { + val timeoutInSeconds = timeValue.toDouble + val timeoutInMillis = (timeoutInSeconds * 60 * 1000).toLong + if (elapse > timeoutInMillis) { + // 触发kill任务 + HttpsUntils.killJob(job) + } + } + } + } -// } } case _ => logger.warn( diff --cc linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-authentication/src/test/resources/create.sql index b9bd3fc575,67e389f806..8fc39b1e56 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-authentication/src/test/resources/create.sql +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-authentication/src/test/resources/create.sql @@@ -37,4 -37,4 +37,4 @@@ DELETE FROM linkis_mg_gateway_auth_toke -- ---------------------------- -- Default Tokens -- ---------------------------- - INSERT INTO `linkis_mg_gateway_auth_token`(`token_name`,`legal_users`,`legal_hosts`,`business_owner`,`create_time`,`update_time`,`elapse_day`,`update_by`) VALUES ('LINKIS-UNAVAILABLE-TOKE','test','127.0.0.1','BDP',curdate(),curdate(),-1,'LINKIS'); -INSERT INTO `linkis_mg_gateway_auth_token`(`token_name`,`legal_users`,`legal_hosts`,`business_owner`,`create_time`,`update_time`,`elapse_day`,`update_by`) VALUES ('LINKIS-UNAVAILABLE-TOKEN','test','127.0.0.1','BDP',curdate(),curdate(),-1,'LINKIS'); ++INSERT INTO `linkis_mg_gateway_auth_token`(`token_name`,`legal_users`,`legal_hosts`,`business_owner`,`create_time`,`update_time`,`elapse_day`,`update_by`) VALUES ('LINKIS-UNAVAILABLE-TOKEN','test','127.0.0.1','BDP',curdate(),curdate(),-1,'LINKIS'); diff --cc linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-authentication/src/test/resources/create_pg.sql index a2f532ce15,c68dc66459..725ed37bf1 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-authentication/src/test/resources/create_pg.sql +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-authentication/src/test/resources/create_pg.sql @@@ -34,4 -34,4 +34,4 @@@ delete from linkis_mg_gateway_auth_toke -- ---------------------------- -- Default Tokens -- ---------------------------- - INSERT INTO "linkis_mg_gateway_auth_token"("token_name","legal_users","legal_hosts","business_owner","create_time","update_time","elapse_day","update_by") VALUES ('LINKIS-UNAVAILABLE-TOKE','test','127.0.0.1','BDP',now(),now(),-1,'LINKIS'); -INSERT INTO "linkis_mg_gateway_auth_token"("token_name","legal_users","legal_hosts","business_owner","create_time","update_time","elapse_day","update_by") VALUES ('LINKIS-UNAVAILABLE-TOKEN','test','127.0.0.1','BDP',now(),now(),-1,'LINKIS'); ++INSERT INTO "linkis_mg_gateway_auth_token"("token_name","legal_users","legal_hosts","business_owner","create_time","update_time","elapse_day","update_by") VALUES ('LINKIS-UNAVAILABLE-TOKEN','test','127.0.0.1','BDP',now(),now(),-1,'LINKIS'); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
