This is an automated email from the ASF dual-hosted git repository. kirs pushed a commit to branch dev in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git
commit df251bf47773fa2f80c2d7008d9ff94b88c8c912 Merge: 770bade75 1b706c16a Author: Kirs <[email protected]> AuthorDate: Fri Jul 1 16:26:43 2022 +0800 Merge pull request #2083 from apache/api-draft [API-DRAFT] [MERGE] Merge api-draft to dev branch .github/workflows/backend.yml | 2 +- .licenserc.yaml | 2 +- LICENSE | 1 + config/flink.batch.conf.template | 2 + config/spark.streaming.conf.template | 2 +- docs/en/connector/sink/Assert.md | 47 +- docs/en/images/seatunnel_architecture.png | Bin 0 -> 778394 bytes docs/en/images/seatunnel_starter.png | Bin 0 -> 423840 bytes .../en/{connector => new-connector}/sink/Assert.md | 51 +- docs/en/new-connector/sink/Clickhouse.md | 104 ++++ docs/en/new-connector/sink/ClickhouseFile.md | 106 ++++ docs/en/new-connector/sink/Jdbc.md | 96 ++++ docs/en/new-connector/sink/common-options.md | 45 ++ docs/en/new-connector/source/Clickhouse.md | 63 ++ docs/en/new-connector/source/Jdbc.md | 75 +++ docs/en/new-connector/source/common-options.md | 33 ++ ...mapping.properties => plugin-mapping.properties | 17 + pom.xml | 105 ++-- {seatunnel-core => seatunnel-api}/pom.xml | 23 +- .../api/common/PluginIdentifierInterface.java | 21 +- .../seatunnel/api/common/PrepareFailException.java | 30 +- .../seatunnel/api/common/SeaTunnelContext.java | 87 +++ .../api/common/SeaTunnelPluginLifeCycle.java | 35 +- .../api/serialization/DefaultSerializer.java | 31 +- .../api/serialization/DeserializationSchema.java | 36 +- .../api/serialization/SerializationSchema.java | 21 +- .../seatunnel/api/serialization/Serializer.java | 29 +- .../api/sink/DefaultSinkWriterContext.java | 36 +- .../apache/seatunnel/api/sink/SeaTunnelSink.java | 122 ++++ .../api/sink/SinkAggregatedCommitter.java | 66 +++ .../apache/seatunnel/api/sink/SinkCommitter.java | 50 ++ .../org/apache/seatunnel/api/sink/SinkWriter.java | 95 +++ .../apache/seatunnel/api/source/Boundedness.java | 22 +- .../org/apache/seatunnel/api/source/Collector.java | 20 +- .../api/source/SeaTunnelRuntimeEnvironment.java | 19 +- .../seatunnel/api/source/SeaTunnelSource.java | 99 ++++ .../apache/seatunnel/api/source/SourceEvent.java | 18 +- .../apache/seatunnel/api/source/SourceReader.java | 119 ++++ .../apache/seatunnel/api/source/SourceSplit.java | 18 +- .../api/source/SourceSplitEnumerator.java | 124 ++++ .../seatunnel/api/source/SupportCoordinate.java | 18 +- .../seatunnel/api/state/CheckpointListener.java | 18 +- .../seatunnel/api/table/catalog/Catalog.java | 116 ++++ .../seatunnel/api/table/catalog/CatalogTable.java | 93 +++ .../apache/seatunnel/api/table/catalog/Column.java | 223 ++++++++ .../api/table/catalog/TableIdentifier.java | 81 +++ .../seatunnel/api/table/catalog/TablePath.java | 86 +++ .../seatunnel/api/table/catalog/TableSchema.java | 132 +++++ .../table/catalog/exception/CatalogException.java | 31 +- .../exception/DatabaseNotExistException.java | 32 ++ .../catalog/exception/TableNotExistException.java | 35 ++ .../api/table/connector/DeserializationFormat.java | 28 +- .../api/table/connector/SerializationFormat.java | 15 +- .../table/connector/SupportReadingMetadata.java | 21 +- .../seatunnel/api/table/connector/TableSink.java | 15 +- .../seatunnel/api/table/connector/TableSource.java | 16 +- .../api/table/factory/CatalogFactory.java | 19 +- .../factory/DeserializationFormatFactory.java | 16 +- .../seatunnel/api/table/factory/Factory.java | 21 +- .../api/table/factory/FactoryException.java | 15 +- .../seatunnel/api/table/factory/FactoryUtil.java | 155 +++++ .../table/factory/SerializationFormatFactory.java | 16 +- .../api/table/factory/SupportMultipleTable.java | 60 ++ .../api/table/factory/TableFactoryContext.java | 58 ++ .../api/table/factory/TableSinkFactory.java | 20 +- .../api/table/factory/TableSourceFactory.java | 31 +- .../apache/seatunnel/api/table/type/ArrayType.java | 87 +++ .../apache/seatunnel/api/table/type/BasicType.java | 79 +++ .../seatunnel/api/table/type/CompositeType.java | 15 +- .../seatunnel/api/table/type/DecimalType.java | 65 +++ .../seatunnel/api/table/type/LocalTimeType.java | 72 +++ .../apache/seatunnel/api/table/type/MapType.java | 104 ++++ .../api/table/type/PrimitiveByteArrayType.java | 38 +- .../apache/seatunnel/api/table/type/RowKind.java | 117 ++++ .../api/table/type/SeaTunnelDataType.java | 19 +- .../seatunnel/api/table/type/SeaTunnelRow.java | 106 ++++ .../seatunnel/api/table/type/SeaTunnelRowType.java | 121 ++++ .../apache/seatunnel/api/table/type/SqlType.java | 34 +- .../apache/seatunnel/apis/base/env/RuntimeEnv.java | 4 +- .../apache/seatunnel/apis/base/plugin/Plugin.java | 14 +- .../apache/seatunnel/flink/FlinkEnvironment.java | 18 +- .../apache/seatunnel/flink/util/SchemaUtil.java | 2 + .../apache/seatunnel/spark/SparkEnvironment.java | 4 +- seatunnel-common/pom.xml | 9 + .../apache/seatunnel/common/PropertiesUtil.java | 37 ++ .../org/apache/seatunnel/common/config/Common.java | 33 +- .../common/constants/CollectionConstants.java | 15 +- .../seatunnel/common/constants}/PluginType.java | 6 +- .../seatunnel/common/utils/ReflectionUtils.java | 49 ++ .../seatunnel/common/utils/SerializationUtils.java | 49 ++ .../apache/seatunnel/common/config/CommonTest.java | 2 +- .../common/utils/VariablesSubstituteTest.java | 7 +- seatunnel-connectors-v2-dist/pom.xml | 108 ++++ seatunnel-connectors-v2/README.md | 84 +++ seatunnel-connectors-v2/README.zh.md | 64 +++ .../connector-assert}/pom.xml | 33 +- .../assertion/excecutor/AssertExecutor.java | 108 ++++ .../seatunnel/assertion/rule/AssertFieldRule.java | 65 +++ .../seatunnel/assertion/rule/AssertRuleParser.java | 83 +++ .../seatunnel/assertion/sink/AssertSink.java | 85 +++ .../seatunnel/assertion/sink/AssertSinkWriter.java | 55 ++ .../flink/assertion/AssertExecutorTest.java | 110 ++++ .../flink/assertion/rule/AssertRuleParserTest.java | 85 +++ .../connector-clickhouse/pom.xml | 70 +++ .../config/ClickhouseFileCopyMethod.java | 34 +- .../seatunnel/clickhouse/config/Config.java | 96 ++++ .../clickhouse/config/FileReaderOption.java | 113 ++++ .../seatunnel/clickhouse/config/ReaderOption.java | 94 +++ .../seatunnel/clickhouse/shard/Shard.java | 98 ++++ .../seatunnel/clickhouse/shard/ShardMetadata.java | 145 +++++ .../clickhouse/sink/DistributedEngine.java | 58 ++ .../sink/client/ClickhouseBatchStatement.java | 52 ++ .../clickhouse/sink/client/ClickhouseProxy.java | 212 +++++++ .../clickhouse/sink/client/ClickhouseSink.java | 172 ++++++ .../sink/client/ClickhouseSinkWriter.java | 230 ++++++++ .../clickhouse/sink/client/ShardRouter.java | 97 ++++ .../clickhouse/sink/file/ClickhouseFileSink.java | 145 +++++ .../sink/file/ClickhouseFileSinkWriter.java | 237 ++++++++ .../clickhouse/sink/file/ClickhouseTable.java | 117 ++++ .../clickhouse/sink/file/FileTransfer.java | 19 +- .../clickhouse/sink/file/FileTransferFactory.java | 36 +- .../clickhouse/sink/file/RsyncFileTransfer.java | 144 +++++ .../clickhouse/sink/file/ScpFileTransfer.java | 124 ++++ .../sink/inject/ArrayInjectFunction.java | 28 +- .../sink/inject/BigDecimalInjectFunction.java | 28 +- .../sink/inject/ClickhouseFieldInjectFunction.java | 46 ++ .../clickhouse/sink/inject/DateInjectFunction.java | 37 +- .../sink/inject/DateTimeInjectFunction.java | 34 +- .../sink/inject/DoubleInjectFunction.java | 31 +- .../sink/inject/FloatInjectFunction.java | 34 +- .../clickhouse/sink/inject/IntInjectFunction.java | 47 ++ .../clickhouse/sink/inject/LongInjectFunction.java | 29 +- .../sink/inject/StringInjectFunction.java | 18 +- .../clickhouse/source/ClickhouseSource.java | 132 +++++ .../clickhouse/source/ClickhouseSourceReader.java | 108 ++++ .../clickhouse/source/ClickhouseSourceSplit.java | 17 +- .../source/ClickhouseSourceSplitEnumerator.java | 100 ++++ .../clickhouse/state/CKAggCommitInfo.java | 15 +- .../seatunnel/clickhouse/state/CKCommitInfo.java | 15 +- .../clickhouse/state/ClickhouseSinkState.java | 15 +- .../clickhouse/state/ClickhouseSourceState.java | 15 +- .../seatunnel/clickhouse/tool/IntHolder.java | 19 +- .../seatunnel/clickhouse/util/ClickhouseUtil.java | 40 ++ .../seatunnel/clickhouse/util/TypeConvertUtil.java | 100 ++++ .../connector-common}/pom.xml | 23 +- .../seatunnel/common/sink/AbstractSimpleSink.java | 59 ++ .../seatunnel/common/sink/AbstractSinkWriter.java | 19 +- .../common/source/AbstractSingleSplitReader.java | 67 +++ .../common/source/AbstractSingleSplitSource.java | 58 ++ .../seatunnel/common/source/SingleSplit.java | 21 +- .../common/source/SingleSplitEnumerator.java | 94 +++ .../common/source/SingleSplitEnumeratorState.java | 15 +- .../common/source/SingleSplitReaderContext.java | 26 +- .../connector-console}/pom.xml | 23 +- .../seatunnel/console/sink/ConsoleSink.java | 63 ++ .../seatunnel/console/sink/ConsoleSinkWriter.java | 37 +- .../connector-fake}/pom.xml | 21 +- .../seatunnel/fake/source/FakeSource.java | 73 +++ .../seatunnel/fake/source/FakeSourceReader.java | 73 +++ .../connector-hive}/pom.xml | 25 +- .../connectors/seatunnel/hive/config/Config.java | 15 +- .../hive/sink/HiveAggregatedCommitInfo.java | 25 +- .../seatunnel/hive/sink/HiveCommitInfo.java | 25 +- .../connectors/seatunnel/hive/sink/HiveSink.java | 94 +++ .../hive/sink/HiveSinkAggregatedCommitter.java | 88 +++ .../seatunnel/hive/sink/HiveSinkConfig.java | 106 ++++ .../seatunnel/hive/sink/HiveSinkState.java | 19 +- .../seatunnel/hive/sink/HiveSinkWriter.java | 99 ++++ .../hive/sink/file/writer/AbstractFileWriter.java | 155 +++++ .../hive/sink/file/writer/FileWriter.java | 49 ++ .../hive/sink/file/writer/HdfsTxtFileWriter.java | 151 +++++ .../seatunnel/hive/sink/file/writer/HdfsUtils.java | 96 ++++ .../connector-http}/pom.xml | 23 +- .../seatunnel/http/client/HttpClientProvider.java | 283 +++++++++ .../seatunnel/http/client/HttpResponse.java | 74 +++ .../connectors/seatunnel/http/config/Config.java | 18 +- .../seatunnel/http/source/HttpSource.java | 109 ++++ .../seatunnel/http/source/HttpSourceParameter.java | 70 +++ .../seatunnel/http/source/HttpSourceReader.java | 79 +++ .../connector-jdbc}/pom.xml | 26 +- .../jdbc/catalog/AbstractJdbcCatalog.java | 211 +++++++ .../seatunnel/jdbc/catalog/MySqlCatalog.java | 230 ++++++++ .../seatunnel/jdbc/config/JdbcConfig.java | 102 ++++ .../seatunnel/jdbc/config/JdbcSinkOptions.java | 35 +- .../seatunnel/jdbc/config/JdbcSourceOptions.java | 77 +++ .../seatunnel/jdbc/internal/JdbcInputFormat.java | 220 +++++++ .../seatunnel/jdbc/internal/JdbcOutputFormat.java | 257 +++++++++ .../jdbc/internal/connection/DataSourceUtils.java | 104 ++++ .../connection/JdbcConnectionProvider.java | 70 +++ .../connection/SimpleJdbcConnectionProvider.java | 154 +++++ .../converter/AbstractJdbcRowConverter.java | 97 ++++ .../jdbc/internal/converter/JdbcRowConverter.java | 25 +- .../jdbc/internal/dialect/JdbcDialect.java | 52 ++ .../jdbc/internal/dialect/JdbcDialectFactory.java | 25 +- .../jdbc/internal/dialect/JdbcDialectLoader.java | 102 ++++ .../internal/dialect/JdbcDialectTypeMapper.java | 23 +- .../dialect/mysql/MySqlDialectFactory.java | 29 +- .../internal/dialect/mysql/MySqlTypeMapper.java | 170 ++++++ .../jdbc/internal/dialect/mysql/MysqlDialect.java | 30 +- .../dialect/mysql/MysqlJdbcRowConverter.java | 31 +- .../internal/dialect/psql/PostgresDialect.java | 30 +- .../dialect/psql/PostgresDialectFactory.java | 29 +- .../dialect/psql/PostgresJdbcRowConverter.java | 31 +- .../internal/dialect/psql/PostgresTypeMapper.java | 159 +++++ .../internal/executor/BiConsumerWithException.java | 61 ++ .../executor/JdbcBatchStatementExecutor.java | 29 +- .../internal/executor/JdbcStatementBuilder.java | 22 +- .../executor/SimpleBatchStatementExecutor.java | 79 +++ .../jdbc/internal/options/JdbcConfig.java | 55 ++ .../internal/options/JdbcConnectionOptions.java | 200 +++++++ .../internal/options/JdbcConnectorOptions.java | 253 ++++++++ .../split/JdbcGenericParameterValuesProvider.java | 30 +- .../JdbcNumericBetweenParametersProvider.java | 116 ++++ .../split/JdbcParameterValuesProvider.java | 32 +- .../jdbc/internal/xa/GroupXaOperationResult.java | 80 +++ .../jdbc/internal/xa/SemanticXidGenerator.java | 113 ++++ .../seatunnel/jdbc/internal/xa/XaFacade.java | 113 ++++ .../jdbc/internal/xa/XaFacadeImplAutoLoad.java | 477 +++++++++++++++ .../seatunnel/jdbc/internal/xa/XaGroupOps.java | 43 ++ .../seatunnel/jdbc/internal/xa/XaGroupOpsImpl.java | 151 +++++ .../seatunnel/jdbc/internal/xa/XidGenerator.java | 62 ++ .../seatunnel/jdbc/internal/xa/XidImpl.java | 136 +++++ .../jdbc/sink/JdbcExactlyOnceSinkWriter.java | 190 ++++++ .../connectors/seatunnel/jdbc/sink/JdbcSink.java | 143 +++++ .../jdbc/sink/JdbcSinkAggregatedCommitter.java | 91 +++ .../seatunnel/jdbc/sink/JdbcSinkCommitter.java | 70 +++ .../seatunnel/jdbc/sink/JdbcSinkWriter.java | 96 ++++ .../seatunnel/jdbc/source/JdbcSource.java | 216 +++++++ .../seatunnel/jdbc/source/JdbcSourceReader.java | 97 ++++ .../seatunnel/jdbc/source/JdbcSourceSplit.java | 22 +- .../jdbc/source/JdbcSourceSplitEnumerator.java | 100 ++++ .../seatunnel/jdbc/source/PartitionParameter.java | 20 +- .../jdbc/state/JdbcAggregatedCommitInfo.java | 20 +- .../seatunnel/jdbc/state/JdbcSinkState.java | 19 +- .../seatunnel/jdbc/state/JdbcSourceState.java | 15 +- .../connectors/seatunnel/jdbc/state/XidInfo.java | 23 +- .../seatunnel/jdbc/utils/ExceptionUtils.java | 53 ++ .../connectors/seatunnel/jdbc/utils/JdbcUtils.java | 158 +++++ .../seatunnel/jdbc/utils/ThrowingRunnable.java | 52 ++ .../jdbc/catalog/AbstractJdbcCatalogTest.java | 48 ++ .../connector-kafka}/pom.xml | 25 +- .../connectors/seatunnel/kafka/config/Config.java | 53 ++ .../seatunnel/kafka/config/KafkaSemantics.java | 27 +- .../serialize/DefaultSeaTunnelRowSerializer.java | 50 ++ .../kafka/serialize/SeaTunnelRowSerializer.java | 22 +- .../kafka/sink/KafkaInternalProducer.java | 164 ++++++ .../kafka/sink/KafkaNoTransactionSender.java | 83 +++ .../seatunnel/kafka/sink/KafkaProduceSender.java | 64 +++ .../connectors/seatunnel/kafka/sink/KafkaSink.java | 95 +++ .../seatunnel/kafka/sink/KafkaSinkCommitter.java | 85 +++ .../seatunnel/kafka/sink/KafkaSinkWriter.java | 161 ++++++ .../kafka/sink/KafkaTransactionSender.java | 126 ++++ .../seatunnel/kafka/source/ConsumerMetadata.java | 82 +++ .../seatunnel/kafka/source/KafkaSource.java | 132 +++++ .../seatunnel/kafka/source/KafkaSourceReader.java | 171 ++++++ .../seatunnel/kafka/source/KafkaSourceSplit.java | 72 +++ .../kafka/source/KafkaSourceSplitEnumerator.java | 167 ++++++ .../kafka/state/KafkaAggregatedCommitInfo.java | 19 +- .../seatunnel/kafka/state/KafkaCommitInfo.java | 23 +- .../seatunnel/kafka/state/KafkaSinkState.java | 23 +- .../seatunnel/kafka/state/KafkaSourceState.java | 35 +- seatunnel-connectors-v2/connector-pulsar/pom.xml | 99 ++++ .../seatunnel/pulsar/config/BasePulsarConfig.java | 27 +- .../seatunnel/pulsar/config/PulsarAdminConfig.java | 76 +++ .../pulsar/config/PulsarClientConfig.java | 80 +++ .../seatunnel/pulsar/config/PulsarConfigUtil.java | 81 +++ .../pulsar/config/PulsarConsumerConfig.java | 60 ++ .../seatunnel/pulsar/config/SourceProperties.java | 119 ++++ .../seatunnel/pulsar/source/PulsarSource.java | 281 +++++++++ .../source/enumerator/PulsarSplitEnumerator.java | 281 +++++++++ .../enumerator/PulsarSplitEnumeratorState.java | 36 ++ .../cursor/start/MessageIdStartCursor.java | 65 +++ .../enumerator/cursor/start/StartCursor.java | 78 +++ .../cursor/start/SubscriptionStartCursor.java | 63 ++ .../cursor/start/TimestampStartCursor.java | 40 ++ .../cursor/stop/LatestMessageStopCursor.java | 59 ++ .../cursor/stop/MessageIdStopCursor.java | 58 ++ .../enumerator/cursor/stop/NeverStopCursor.java | 49 ++ .../source/enumerator/cursor/stop/StopCursor.java | 63 ++ .../cursor/stop/TimestampStopCursor.java | 44 ++ .../enumerator/discoverer/PulsarDiscoverer.java | 48 ++ .../enumerator/discoverer/TopicListDiscoverer.java | 63 ++ .../discoverer/TopicPatternDiscoverer.java | 98 ++++ .../source/enumerator/topic/TopicPartition.java | 91 +++ .../seatunnel/pulsar/source/reader/Handover.java | 88 +++ .../pulsar/source/reader/PulsarSourceReader.java | 221 +++++++ .../source/reader/PulsarSplitReaderThread.java | 143 +++++ .../pulsar/source/reader/RecordWithSplitId.java | 22 +- .../pulsar/source/split/PulsarPartitionSplit.java | 94 +++ .../connector-socket}/pom.xml | 21 +- .../seatunnel/socket/source/SocketSource.java | 72 +++ .../socket/source/SocketSourceParameter.java | 32 +- .../socket/source/SocketSourceReader.java | 91 +++ .../seatunnel/socket/state/SocketState.java | 15 +- .../pom.xml | 40 +- .../seatunnel-connector-flink-clickhouse/pom.xml | 5 - .../flink/clickhouse/sink/ClickhouseBatchSink.java | 3 - .../clickhouse/sink/ClickhouseOutputFormat.java | 2 +- .../org/apache/seatunnel/flink/fake/Config.java | 2 +- .../flink/fake/source/FakeSourceStream.java | 2 +- seatunnel-core/README.md | 8 + seatunnel-core/pom.xml | 3 + seatunnel-core/seatunnel-core-base/pom.xml | 6 + .../org/apache/seatunnel/core/base/Seatunnel.java | 3 +- .../core/base/command/AbstractCommandArgs.java | 1 + .../core/base/command/BaseTaskExecuteCommand.java | 5 +- .../seatunnel/core/base/command/Command.java | 3 +- ...nContext.java => AbstractExecutionContext.java} | 43 +- .../seatunnel/core/base/config/ConfigBuilder.java | 21 +- .../seatunnel/core/base/config/ConfigChecker.java | 36 +- .../core/base/config/EnvironmentFactory.java | 2 + .../core/base/config/ExecutionFactory.java | 4 +- .../seatunnel/core/base/config/PluginFactory.java | 237 -------- .../CommandException.java} | 16 +- .../CommandExecuteException.java} | 16 +- .../ConfigCheckException.java} | 16 +- .../core/base/config/PluginFactoryTest.java | 65 --- .../seatunnel/core/base/utils/FileUtilsTest.java | 1 + .../apache/seatunnel/core/sql/FlinkSqlStarter.java | 6 +- .../apache/seatunnel/core/sql/job/Executor.java | 16 +- seatunnel-core/seatunnel-core-flink/pom.xml | 24 +- .../apache/seatunnel/core/flink/FlinkStarter.java | 4 +- .../seatunnel/core/flink/SeatunnelFlink.java | 3 +- .../core/flink/args/FlinkCommandArgs.java | 4 +- ...mmand.java => FlinkApiConfValidateCommand.java} | 14 +- ...ommand.java => FlinkApiTaskExecuteCommand.java} | 37 +- .../core/flink/command/FlinkCommandBuilder.java | 9 +- .../core/flink/config/FlinkApiConfigChecker.java | 44 ++ .../core/flink/config/FlinkExecutionContext.java | 104 ++++ .../core/flink/utils/CommandLineUtils.java | 17 +- .../flink/command/FlinkTaskExecuteCommandTest.java | 2 +- .../core/flink/utils/CommandLineUtilsTest.java | 17 +- seatunnel-core/seatunnel-core-spark/pom.xml | 26 + .../seatunnel/core/spark/SeatunnelSpark.java | 3 +- .../apache/seatunnel/core/spark/SparkStarter.java | 36 +- .../core/spark/command/SparkCommandBuilder.java | 5 +- .../spark/command/SparkConfValidateCommand.java | 8 +- .../spark/command/SparkTaskExecuteCommand.java | 59 +- .../core/spark/config/SparkApiConfigChecker.java | 44 ++ .../core/spark/config/SparkExecutionContext.java | 103 ++++ seatunnel-core/seatunnel-core-starter/README.md | 12 + .../pom.xml | 7 +- .../apache/seatunnel/core/starter}/Seatunnel.java | 9 +- .../apache/seatunnel/core/starter/Starter.java} | 16 +- .../core/starter}/command/AbstractCommandArgs.java | 6 +- .../seatunnel/core/starter}/command/Command.java | 8 +- .../core/starter/command/CommandArgs.java} | 18 +- .../core/starter/command/CommandBuilder.java} | 16 +- .../core/starter/command/DeployModeConverter.java} | 20 +- .../starter/config/AbstractExecutionContext.java} | 45 +- .../core/starter}/config/ConfigBuilder.java | 22 +- .../core/starter/config/ConfigChecker.java | 36 +- .../seatunnel/core/starter/config/EngineType.java} | 18 +- .../core/starter}/config/EnvironmentFactory.java | 3 +- .../core/starter}/config/ExecutionFactory.java | 6 +- .../seatunnel/core/starter}/config/PluginType.java | 2 +- .../starter/constants/CommonParamConstants.java} | 16 +- .../core/starter/exception/CommandException.java} | 16 +- .../exception/CommandExecuteException.java} | 16 +- .../starter/exception/ConfigCheckException.java} | 16 +- .../starter/exception/TaskExecuteException.java} | 15 +- .../core/starter/execution/TaskExecution.java} | 15 +- .../core/starter/utils/AsciiArtUtils.java | 72 +++ .../core/starter/utils/CompressionUtils.java | 158 +++++ .../seatunnel/core/starter/utils/FileUtils.java | 70 +++ .../core/starter/utils/CompressionUtilsTest.java | 50 ++ .../core/starter}/utils/FileUtilsTest.java | 4 +- .../src/test/resources/flink.batch.conf | 5 +- .../pom.xml | 64 ++- .../bin/start-seatunnel-flink-new-connector.sh | 46 ++ .../src/main/docker/Dockerfile | 37 ++ .../core/starter}/flink/FlinkStarter.java | 16 +- .../core/starter}/flink/SeatunnelFlink.java | 17 +- .../core/starter}/flink/args/FlinkCommandArgs.java | 9 +- .../command/FlinkApiConfValidateCommand.java} | 27 +- .../flink/command/FlinkApiTaskExecuteCommand.java | 61 ++ .../flink/command/FlinkCommandBuilder.java | 18 +- .../flink/config/FlinkApiConfigChecker.java} | 17 +- .../starter/flink/config/FlinkApiEnvironment.java | 75 +++ .../core/starter/flink/config/FlinkJobType.java} | 12 +- .../core/starter/flink/config/FlinkRunMode.java} | 22 +- .../starter/flink/constant/FlinkConstant.java} | 16 +- .../core/starter/flink/env}/FlinkEnvironment.java | 63 +- .../execution/AbstractPluginExecuteProcessor.java | 76 +++ .../starter/flink/execution/FlinkExecution.java | 73 +++ .../flink/execution/PluginExecuteProcessor.java} | 25 +- .../flink/execution/SinkExecuteProcessor.java | 84 +++ .../flink/execution/SourceExecuteProcessor.java | 121 ++++ .../flink/execution/TransformExecuteProcessor.java | 86 +++ .../starter}/flink/utils/CommandLineUtils.java | 28 +- .../core/starter/flink/FlinkStarterTest.java | 65 +++ .../starter/flink/args/FlinkCommandArgsTest.java | 30 +- .../pom.xml | 58 +- .../bin/start-seatunnel-spark-new-connector.sh | 45 ++ .../src/main/docker/Dockerfile | 37 ++ .../core/starter}/spark/SeatunnelSpark.java | 15 +- .../core/starter}/spark/SparkStarter.java | 68 ++- .../core/starter/spark/args/SparkCommandArgs.java | 62 ++ .../command/SparkApiConfValidateCommand.java} | 29 +- .../spark/command/SparkApiTaskExecuteCommand.java | 60 ++ .../spark/command/SparkCommandBuilder.java | 26 +- .../spark/config/SparkApiConfigChecker.java} | 17 +- .../starter/spark/config/SparkEnvironment.java | 45 +- .../execution/AbstractPluginExecuteProcessor.java | 68 +++ .../spark/execution/PluginExecuteProcessor.java} | 25 +- .../spark/execution/SinkExecuteProcessor.java | 84 +++ .../spark/execution/SourceExecuteProcessor.java | 81 +++ .../starter/spark/execution/SparkExecution.java | 63 ++ .../spark/execution/TransformExecuteProcessor.java | 80 +++ .../core/starter/spark/utils/CommandLineUtils.java | 29 +- .../core/starter/spark/SparkStarterTest.java | 25 +- .../starter/spark/args/SparkCommandArgsTest.java | 67 +++ .../starter/spark/utils/CommandLineUtilsTest.java | 27 +- .../src/test/resources/spark_application.conf | 66 +++ seatunnel-dist/release-docs/LICENSE | 52 +- seatunnel-dist/release-docs/NOTICE | 20 + .../release-docs/licenses/LICENSE-bouncycastle.txt | 7 + .../release-docs/licenses/LICENSE-jaxrs-api.txt | 637 +++++++++++++++++++++ seatunnel-dist/src/main/assembly/assembly-bin.xml | 48 +- seatunnel-e2e/pom.xml | 4 +- .../apache/seatunnel/e2e/flink/FlinkContainer.java | 2 +- .../seatunnel-flink-new-connector-e2e}/pom.xml | 27 +- .../apache/seatunnel/e2e/flink/FlinkContainer.java | 44 +- .../e2e/flink/assertion/FakeSourceToAssertIT.java | 24 +- .../e2e/flink/fake/FakeSourceToConsoleIT.java | 25 +- .../resources/assertion/fakesource_to_assert.conf | 88 +++ .../test/resources/fake/fakesource_to_console.conf | 35 +- .../src/test/resources/log4j.properties | 23 + .../apache/seatunnel/e2e/spark/SparkContainer.java | 2 +- .../seatunnel-spark-new-connector-e2e}/pom.xml | 28 +- .../apache/seatunnel/e2e/spark/SparkContainer.java | 21 +- .../e2e/spark/fake/FakeSourceToConsoleIT.java | 29 +- .../test/resources/fake/fakesource_to_console.conf | 34 +- .../src/test/resources/log4j.properties | 22 + seatunnel-examples/pom.xml | 2 + .../seatunnel-flink-examples/pom.xml | 6 + .../seatunnel/example/flink/LocalFlinkExample.java | 3 +- .../pom.xml | 22 +- .../example/flink/SeaTunnelApiExample.java} | 15 +- .../main/resources/examples/fake_to_console.conf | 35 +- .../src/main/resources/log4j.properties | 22 + .../seatunnel-flink-sql-examples/pom.xml | 2 +- .../seatunnel/example/spark/LocalSparkExample.java | 3 +- .../seatunnel-spark-new-connector-example}/pom.xml | 54 +- .../example/spark/SeaTunnelApiExample.java} | 33 +- .../src/main/resources/examples/spark.batch.conf | 33 +- {seatunnel-e2e => seatunnel-formats}/pom.xml | 8 +- .../seatunnel-format-json}/pom.xml | 28 +- .../format/json/JsonDeserializationSchema.java | 122 ++++ .../seatunnel/format/json/JsonToRowConverters.java | 281 +++++++++ .../pom.xml | 28 +- .../plugin/discovery/AbstractPluginDiscovery.java | 181 ++++++ .../plugin/discovery/PluginDiscovery.java | 70 +++ .../plugin/discovery/PluginIdentifier.java | 89 +++ .../discovery/flink/FlinkSinkPluginDiscovery.java | 17 +- .../flink/FlinkSourcePluginDiscovery.java | 18 +- .../flink/FlinkTransformPluginDiscovery.java | 40 +- .../SeaTunnelFlinkTransformPluginDiscovery.java | 29 +- .../seatunnel/SeaTunnelSinkPluginDiscovery.java | 17 +- .../seatunnel/SeaTunnelSourcePluginDiscovery.java | 18 +- .../SeaTunnelSparkTransformPluginDiscovery.java | 18 +- .../discovery/spark/SparkSinkPluginDiscovery.java | 17 +- .../spark/SparkSourcePluginDiscovery.java | 17 +- .../spark/SparkTransformPluginDiscovery.java | 39 +- {seatunnel-e2e => seatunnel-translation}/pom.xml | 8 +- .../seatunnel-translation-base}/pom.xml | 18 +- .../translation/serialization/RowConverter.java | 128 +++++ .../serialization/SerializerConverter.java | 20 +- .../sink/SinkAggregatedCommitterConverter.java | 19 +- .../translation/sink/SinkCommitterConverter.java | 20 +- .../seatunnel/translation/sink/SinkConverter.java | 22 +- .../translation/sink/SinkWriterConverter.java | 14 +- .../translation/source/BaseSourceFunction.java | 24 +- .../source/CoordinatedEnumeratorContext.java | 60 ++ .../source/CoordinatedReaderContext.java | 62 ++ .../translation/source/CoordinatedSource.java | 273 +++++++++ .../source/ParallelEnumeratorContext.java | 77 +++ .../translation/source/ParallelReaderContext.java | 63 ++ .../translation/source/ParallelSource.java | 201 +++++++ .../util/ThreadPoolExecutorFactory.java | 31 +- .../seatunnel-translation-flink}/pom.xml | 45 +- .../serialization/CommitWrapperSerializer.java | 66 +++ .../flink/serialization/FlinkRowConverter.java | 115 ++++ .../FlinkSimpleVersionedSerializer.java | 33 +- .../serialization/FlinkWriterStateSerializer.java | 68 +++ .../flink/serialization/KryoTypeInfo.java | 41 ++ .../flink/serialization/WrappedRow.java | 40 +- .../translation/flink/sink/CommitWrapper.java | 16 +- .../translation/flink/sink/FlinkCommitter.java | 56 ++ .../flink/sink/FlinkGlobalCommitter.java | 72 +++ .../translation/flink/sink/FlinkSink.java | 90 +++ .../translation/flink/sink/FlinkSinkWriter.java | 75 +++ .../translation/flink/sink/FlinkWriterState.java | 33 +- .../flink/source/BaseSeaTunnelSourceFunction.java | 136 +++++ .../translation/flink/source/RowCollector.java | 55 ++ .../flink/source/SeaTunnelCoordinatedSource.java | 43 ++ .../flink/source/SeaTunnelParallelSource.java | 49 ++ .../flink/utils/TypeConverterUtils.java | 146 +++++ .../flink/utils/TypeConverterUtilsTest.java | 129 +++++ .../seatunnel-translation-spark}/pom.xml | 44 +- .../spark/serialization/InternalRowConverter.java | 178 ++++++ .../spark/sink/SparkDataSourceWriter.java | 95 +++ .../translation/spark/sink/SparkDataWriter.java | 101 ++++ .../spark/sink/SparkDataWriterFactory.java | 62 ++ .../translation/spark/sink/SparkSink.java | 76 +++ .../translation/spark/sink/SparkSinkInjector.java | 49 ++ .../translation/spark/sink/SparkStreamWriter.java | 73 +++ .../spark/sink/SparkWriterCommitMessage.java | 23 +- .../translation/spark/source/Handover.java | 88 +++ .../spark/source/InternalRowCollector.java | 53 ++ .../translation/spark/source/ReaderState.java | 32 +- .../spark/source/SeaTunnelSourceSupport.java | 92 +++ .../spark/source/batch/BatchPartition.java | 47 ++ .../spark/source/batch/BatchSourceReader.java | 62 ++ .../batch/CoordinatedBatchPartitionReader.java | 97 ++++ .../source/batch/ParallelBatchPartitionReader.java | 147 +++++ .../source/continnous/ContinuousPartition.java | 53 ++ .../source/continnous/ContinuousSourceReader.java | 116 ++++ .../spark/source/continnous/CoordinationState.java | 49 ++ .../ParallelContinuousPartitionReader.java | 78 +++ .../CoordinatedMicroBatchPartitionReader.java | 138 +++++ .../spark/source/micro/MicroBatchPartition.java | 64 +++ .../spark/source/micro/MicroBatchSourceReader.java | 111 ++++ .../spark/source/micro/MicroBatchState.java | 27 +- .../micro/ParallelMicroBatchPartitionReader.java | 182 ++++++ .../spark/utils/TypeConverterUtils.java | 173 ++++++ ...org.apache.spark.sql.sources.DataSourceRegister | 18 + tools/dependencies/known-dependencies.txt | 30 +- 528 files changed, 28870 insertions(+), 3208 deletions(-)
