This is an automated email from the ASF dual-hosted git repository.
kirs pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new b318b3166 [Feature][Connector-V2][OneSignal]Add OneSignal source
conector (#3454)
b318b3166 is described below
commit b318b3166f8b303c29a31dd35fcd678695def440
Author: TaoZex <[email protected]>
AuthorDate: Mon Nov 21 21:25:52 2022 +0800
[Feature][Connector-V2][OneSignal]Add OneSignal source conector (#3454)
* [Feature][Connector-V2][OneSignal]Add OneSignal source conector
* update pom
---
docs/en/connector-v2/source/OneSignal.md | 182 +++++++++++++++++++++
plugin-mapping.properties | 1 +
.../{ => connector-http-onesignal}/pom.xml | 21 ++-
.../onesignal/source/OneSignalSource.java | 61 +++++++
.../onesignal/source/OneSignalSourceFactory.java | 55 +++++++
.../source/config/OneSignalSourceConfig.java | 34 ++++
.../source/config/OneSignalSourceParameter.java | 35 ++++
seatunnel-connectors-v2/connector-http/pom.xml | 1 +
seatunnel-dist/pom.xml | 6 +
.../connector-http-e2e/pom.xml | 6 +
.../e2e/connector/http/HttpOneSignalIT.java | 76 +++++++++
.../resources/mockserver-onesignal-config.json | 174 ++++++++++++++++++++
.../test/resources/onesignal_json_to_assert.conf | 112 +++++++++++++
13 files changed, 753 insertions(+), 11 deletions(-)
diff --git a/docs/en/connector-v2/source/OneSignal.md
b/docs/en/connector-v2/source/OneSignal.md
new file mode 100644
index 000000000..9e0286537
--- /dev/null
+++ b/docs/en/connector-v2/source/OneSignal.md
@@ -0,0 +1,182 @@
+# OneSignal
+
+> OneSignal source connector
+
+## Description
+
+Used to read data from OneSignal.
+
+## Key features
+
+- [x] [batch](../../concept/connector-v2-features.md)
+- [ ] [stream](../../concept/connector-v2-features.md)
+- [ ] [exactly-once](../../concept/connector-v2-features.md)
+- [x] [schema projection](../../concept/connector-v2-features.md)
+- [ ] [parallelism](../../concept/connector-v2-features.md)
+- [ ] [support user-defined split](../../concept/connector-v2-features.md)
+
+## Options
+
+| name | type | required | default value |
+| --------------------------- | ------ | -------- | ------------- |
+| url | String | Yes | - |
+| password | String | Yes | - |
+| method | String | No | get |
+| schema | Config | No | - |
+| schema.fields | Config | No | - |
+| format | String | No | json |
+| params | Map | No | - |
+| body | String | No | - |
+| poll_interval_ms | int | No | - |
+| retry | int | No | - |
+| retry_backoff_multiplier_ms | int | No | 100 |
+| retry_backoff_max_ms | int | No | 10000 |
+| common-options | config | No | - |
+
+### url [String]
+
+http request url
+
+### password [String]
+
+Auth key for login, you can get more detail at this link:
+
+https://documentation.onesignal.com/docs/accounts-and-keys#user-auth-key
+
+### method [String]
+
+http request method, only supports GET, POST method
+
+### params [Map]
+
+http params
+
+### body [String]
+
+http body
+
+### poll_interval_ms [int]
+
+request http api interval(millis) in stream mode
+
+### retry [int]
+
+The max retry times if request http return to `IOException`
+
+### retry_backoff_multiplier_ms [int]
+
+The retry-backoff times(millis) multiplier if request http failed
+
+### retry_backoff_max_ms [int]
+
+The maximum retry-backoff times(millis) if request http failed
+
+### format [String]
+
+the format of upstream data, now only support `json` `text`, default `json`.
+
+when you assign format is `json`, you should also assign schema option, for
example:
+
+upstream data is the following:
+
+```json
+
+{"code": 200, "data": "get success", "success": true}
+
+```
+
+you should assign schema as the following:
+
+```hocon
+
+schema {
+ fields {
+ code = int
+ data = string
+ success = boolean
+ }
+}
+
+```
+
+connector will generate data as the following:
+
+| code | data | success |
+|------|-------------|---------|
+| 200 | get success | true |
+
+when you assign format is `text`, connector will do nothing for upstream data,
for example:
+
+upstream data is the following:
+
+```json
+
+{"code": 200, "data": "get success", "success": true}
+
+```
+
+connector will generate data as the following:
+
+| content |
+|---------|
+| {"code": 200, "data": "get success", "success": true} |
+
+### schema [Config]
+
+#### fields [Config]
+
+the schema fields of upstream data
+
+### common options
+
+Source plugin common parameters, please refer to [Source Common
Options](common-options.md) for details
+
+## Example
+
+```hocon
+
+OneSignal {
+ url = "https://onesignal.com/api/v1/apps";
+ password = "Seatunnel-test"
+ schema = {
+ fields {
+ id = string
+ name = string
+ gcm_key = string
+ chrome_key = string
+ chrome_web_key = string
+ chrome_web_origin = string
+ chrome_web_gcm_sender_id = string
+ chrome_web_default_notification_icon = string
+ chrome_web_sub_domain = string
+ apns_env = string
+ apns_certificates = string
+ apns_p8 = string
+ apns_team_id = string
+ apns_key_id = string
+ apns_bundle_id = string
+ safari_apns_certificate = string
+ safari_site_origin = string
+ safari_push_id = string
+ safari_icon_16_16 = string
+ safari_icon_32_32 = string
+ safari_icon_64_64 = string
+ safari_icon_128_128 = string
+ safari_icon_256_256 = string
+ site_name = string
+ created_at = string
+ updated_at = string
+ players = int
+ messageable_players = int
+ basic_auth_key = string
+ additional_data_is_root_payload = string
+ }
+ }
+}
+```
+
+## Changelog
+
+### next version
+
+- Add OneSignal Source Connector
diff --git a/plugin-mapping.properties b/plugin-mapping.properties
index 60b6502fc..65271a258 100644
--- a/plugin-mapping.properties
+++ b/plugin-mapping.properties
@@ -150,3 +150,4 @@ seatunnel.source.GoogleSheets = connector-google-sheets
seatunnel.sink.Tablestore = connector-tablestore
seatunnel.source.Lemlist = connector-http-lemlist
seatunnel.source.Klaviyo = connector-http-klaviyo
+seatunnel.source.OneSignal = connector-http-onesignal
diff --git a/seatunnel-connectors-v2/connector-http/pom.xml
b/seatunnel-connectors-v2/connector-http/connector-http-onesignal/pom.xml
similarity index 73%
copy from seatunnel-connectors-v2/connector-http/pom.xml
copy to seatunnel-connectors-v2/connector-http/connector-http-onesignal/pom.xml
index bcc6df222..e87ad5acc 100644
--- a/seatunnel-connectors-v2/connector-http/pom.xml
+++ b/seatunnel-connectors-v2/connector-http/connector-http-onesignal/pom.xml
@@ -21,21 +21,20 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
<parent>
- <artifactId>seatunnel-connectors-v2</artifactId>
+ <artifactId>connector-http</artifactId>
<groupId>org.apache.seatunnel</groupId>
<version>${revision}</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <artifactId>connector-http</artifactId>
- <packaging>pom</packaging>
- <modules>
- <module>connector-http-base</module>
- <module>connector-http-feishu</module>
- <module>connector-http-wechat</module>
- <module>connector-http-myhours</module>
- <module>connector-http-lemlist</module>
- <module>connector-http-klaviyo</module>
- </modules>
+ <artifactId>connector-http-onesignal</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>connector-http-base</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
</project>
\ No newline at end of file
diff --git
a/seatunnel-connectors-v2/connector-http/connector-http-onesignal/src/main/java/org/apache/seatunnel/connectors/seatunnel/onesignal/source/OneSignalSource.java
b/seatunnel-connectors-v2/connector-http/connector-http-onesignal/src/main/java/org/apache/seatunnel/connectors/seatunnel/onesignal/source/OneSignalSource.java
new file mode 100644
index 000000000..c59687d75
--- /dev/null
+++
b/seatunnel-connectors-v2/connector-http/connector-http-onesignal/src/main/java/org/apache/seatunnel/connectors/seatunnel/onesignal/source/OneSignalSource.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.onesignal.source;
+
+import org.apache.seatunnel.api.common.PrepareFailException;
+import org.apache.seatunnel.api.source.SeaTunnelSource;
+import org.apache.seatunnel.api.table.type.SeaTunnelRow;
+import org.apache.seatunnel.common.config.CheckConfigUtil;
+import org.apache.seatunnel.common.config.CheckResult;
+import org.apache.seatunnel.common.constants.PluginType;
+import
org.apache.seatunnel.connectors.seatunnel.common.source.AbstractSingleSplitReader;
+import
org.apache.seatunnel.connectors.seatunnel.common.source.SingleSplitReaderContext;
+import org.apache.seatunnel.connectors.seatunnel.http.source.HttpSource;
+import org.apache.seatunnel.connectors.seatunnel.http.source.HttpSourceReader;
+import
org.apache.seatunnel.connectors.seatunnel.onesignal.source.config.OneSignalSourceConfig;
+import
org.apache.seatunnel.connectors.seatunnel.onesignal.source.config.OneSignalSourceParameter;
+
+import org.apache.seatunnel.shade.com.typesafe.config.Config;
+
+import com.google.auto.service.AutoService;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@AutoService(SeaTunnelSource.class)
+public class OneSignalSource extends HttpSource {
+ private final OneSignalSourceParameter oneSignalSourceParameter = new
OneSignalSourceParameter();
+ @Override
+ public String getPluginName() {
+ return "OneSignal";
+ }
+
+ @Override
+ public void prepare(Config pluginConfig) throws PrepareFailException {
+ CheckResult result = CheckConfigUtil.checkAllExists(pluginConfig,
OneSignalSourceConfig.URL.key(), OneSignalSourceConfig.PASSWORD.key());
+ if (!result.isSuccess()) {
+ throw new PrepareFailException(getPluginName(), PluginType.SOURCE,
result.getMsg());
+ }
+ oneSignalSourceParameter.buildWithConfig(pluginConfig);
+ buildSchemaWithConfig(pluginConfig);
+ }
+
+ @Override
+ public AbstractSingleSplitReader<SeaTunnelRow>
createReader(SingleSplitReaderContext readerContext) throws Exception {
+ return new HttpSourceReader(this.oneSignalSourceParameter,
readerContext, this.deserializationSchema);
+ }
+}
diff --git
a/seatunnel-connectors-v2/connector-http/connector-http-onesignal/src/main/java/org/apache/seatunnel/connectors/seatunnel/onesignal/source/OneSignalSourceFactory.java
b/seatunnel-connectors-v2/connector-http/connector-http-onesignal/src/main/java/org/apache/seatunnel/connectors/seatunnel/onesignal/source/OneSignalSourceFactory.java
new file mode 100644
index 000000000..ff2b40b23
--- /dev/null
+++
b/seatunnel-connectors-v2/connector-http/connector-http-onesignal/src/main/java/org/apache/seatunnel/connectors/seatunnel/onesignal/source/OneSignalSourceFactory.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.onesignal.source;
+
+import org.apache.seatunnel.api.configuration.util.Condition;
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.api.table.factory.Factory;
+import org.apache.seatunnel.api.table.factory.TableSourceFactory;
+import org.apache.seatunnel.connectors.seatunnel.common.schema.SeaTunnelSchema;
+import org.apache.seatunnel.connectors.seatunnel.http.config.HttpConfig;
+import org.apache.seatunnel.connectors.seatunnel.http.config.HttpRequestMethod;
+import
org.apache.seatunnel.connectors.seatunnel.onesignal.source.config.OneSignalSourceConfig;
+
+import com.google.auto.service.AutoService;
+
+@AutoService(Factory.class)
+public class OneSignalSourceFactory implements TableSourceFactory {
+ @Override
+ public String factoryIdentifier() {
+ return "OneSignal";
+ }
+
+ @Override
+ public OptionRule optionRule() {
+ return OptionRule.builder()
+ .required(OneSignalSourceConfig.URL)
+ .required(OneSignalSourceConfig.PASSWORD)
+ .optional(OneSignalSourceConfig.METHOD)
+ .optional(OneSignalSourceConfig.HEADERS)
+ .optional(OneSignalSourceConfig.PARAMS)
+ .conditional(Condition.of(HttpConfig.METHOD,
HttpRequestMethod.POST), OneSignalSourceConfig.BODY)
+ .conditional(Condition.of(HttpConfig.FORMAT, "json"),
SeaTunnelSchema.SCHEMA)
+ .optional(OneSignalSourceConfig.FORMAT)
+ .optional(OneSignalSourceConfig.POLL_INTERVAL_MILLS)
+ .optional(OneSignalSourceConfig.RETRY)
+ .optional(OneSignalSourceConfig.RETRY_BACKOFF_MAX_MS)
+ .optional(OneSignalSourceConfig.RETRY_BACKOFF_MULTIPLIER_MS)
+ .build();
+ }
+}
diff --git
a/seatunnel-connectors-v2/connector-http/connector-http-onesignal/src/main/java/org/apache/seatunnel/connectors/seatunnel/onesignal/source/config/OneSignalSourceConfig.java
b/seatunnel-connectors-v2/connector-http/connector-http-onesignal/src/main/java/org/apache/seatunnel/connectors/seatunnel/onesignal/source/config/OneSignalSourceConfig.java
new file mode 100644
index 000000000..551a05023
--- /dev/null
+++
b/seatunnel-connectors-v2/connector-http/connector-http-onesignal/src/main/java/org/apache/seatunnel/connectors/seatunnel/onesignal/source/config/OneSignalSourceConfig.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.onesignal.source.config;
+
+import org.apache.seatunnel.api.configuration.Option;
+import org.apache.seatunnel.api.configuration.Options;
+import org.apache.seatunnel.connectors.seatunnel.http.config.HttpConfig;
+
+public class OneSignalSourceConfig extends HttpConfig {
+ public static final String AUTHORIZATION = "Authorization";
+ public static final String CONTENT_TYPE = "Content-Type";
+ public static final String APPLICATION_JSON = "application/json";
+ public static final String BASIC = "Basic";
+
+ public static final Option<String> PASSWORD = Options.key("password")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("OneSignal login auth key");
+}
diff --git
a/seatunnel-connectors-v2/connector-http/connector-http-onesignal/src/main/java/org/apache/seatunnel/connectors/seatunnel/onesignal/source/config/OneSignalSourceParameter.java
b/seatunnel-connectors-v2/connector-http/connector-http-onesignal/src/main/java/org/apache/seatunnel/connectors/seatunnel/onesignal/source/config/OneSignalSourceParameter.java
new file mode 100644
index 000000000..4fa79cce1
--- /dev/null
+++
b/seatunnel-connectors-v2/connector-http/connector-http-onesignal/src/main/java/org/apache/seatunnel/connectors/seatunnel/onesignal/source/config/OneSignalSourceParameter.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.onesignal.source.config;
+
+import org.apache.seatunnel.connectors.seatunnel.http.config.HttpParameter;
+
+import org.apache.seatunnel.shade.com.typesafe.config.Config;
+
+import java.util.HashMap;
+
+public class OneSignalSourceParameter extends HttpParameter {
+ public void buildWithConfig(Config pluginConfig) {
+ super.buildWithConfig(pluginConfig);
+ // put authorization in headers
+ this.headers = this.getHeaders() == null ? new HashMap<>() :
this.getHeaders();
+ this.headers.put(OneSignalSourceConfig.CONTENT_TYPE,
OneSignalSourceConfig.APPLICATION_JSON);
+ this.headers.put(OneSignalSourceConfig.AUTHORIZATION,
OneSignalSourceConfig.BASIC + " " +
pluginConfig.getString(OneSignalSourceConfig.PASSWORD.key()));
+ this.setHeaders(this.headers);
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-http/pom.xml
b/seatunnel-connectors-v2/connector-http/pom.xml
index bcc6df222..a12dc508a 100644
--- a/seatunnel-connectors-v2/connector-http/pom.xml
+++ b/seatunnel-connectors-v2/connector-http/pom.xml
@@ -36,6 +36,7 @@
<module>connector-http-myhours</module>
<module>connector-http-lemlist</module>
<module>connector-http-klaviyo</module>
+ <module>connector-http-onesignal</module>
</modules>
</project>
\ No newline at end of file
diff --git a/seatunnel-dist/pom.xml b/seatunnel-dist/pom.xml
index dd3f5f99d..bbdac1d8b 100644
--- a/seatunnel-dist/pom.xml
+++ b/seatunnel-dist/pom.xml
@@ -171,6 +171,12 @@
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>connector-http-onesignal</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
<dependency>
<groupId>org.apache.seatunnel</groupId>
<artifactId>connector-jdbc</artifactId>
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/pom.xml
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/pom.xml
index ebdceefc9..57e17acc9 100644
--- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/pom.xml
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/pom.xml
@@ -56,6 +56,12 @@
<version>${project.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>connector-http-onesignal</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
\ No newline at end of file
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpOneSignalIT.java
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpOneSignalIT.java
new file mode 100644
index 000000000..0e0a446b2
--- /dev/null
+++
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpOneSignalIT.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.e2e.connector.http;
+
+import org.apache.seatunnel.e2e.common.TestResource;
+import org.apache.seatunnel.e2e.common.TestSuiteBase;
+import org.apache.seatunnel.e2e.common.container.TestContainer;
+
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.TestTemplate;
+import org.testcontainers.containers.Container;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.output.Slf4jLogConsumer;
+import org.testcontainers.containers.wait.strategy.HttpWaitStrategy;
+import org.testcontainers.lifecycle.Startables;
+import org.testcontainers.utility.DockerImageName;
+import org.testcontainers.utility.DockerLoggerFactory;
+import org.testcontainers.utility.MountableFile;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.stream.Stream;
+
+public class HttpOneSignalIT extends TestSuiteBase implements TestResource {
+
+ private static final String IMAGE = "mockserver/mockserver:5.14.0";
+
+ private GenericContainer<?> mockserverContainer;
+
+ @BeforeAll
+ @Override
+ public void startUp() {
+ this.mockserverContainer = new
GenericContainer<>(DockerImageName.parse(IMAGE))
+ .withNetwork(NETWORK)
+ .withNetworkAliases("mockserver")
+ .withExposedPorts(1080)
+ .withCopyFileToContainer(MountableFile.forHostPath(new
File(HttpIT.class.getResource(
+
"/mockserver-onesignal-config.json").getPath()).getAbsolutePath()),
+ "/tmp/mockserver-onesignal-config.json")
+ .withEnv("MOCKSERVER_INITIALIZATION_JSON_PATH",
"/tmp/mockserver-onesignal-config.json")
+ .withLogConsumer(new
Slf4jLogConsumer(DockerLoggerFactory.getLogger(IMAGE)))
+ .waitingFor(new
HttpWaitStrategy().forPath("/").forStatusCode(404));
+ Startables.deepStart(Stream.of(mockserverContainer)).join();
+ }
+
+ @AfterAll
+ @Override
+ public void tearDown() {
+ if (mockserverContainer != null) {
+ mockserverContainer.stop();
+ }
+ }
+
+ @TestTemplate
+ public void testHttpOneSignalSourceToAssertSink(TestContainer container)
throws IOException, InterruptedException {
+ Container.ExecResult execResult =
container.executeJob("/onesignal_json_to_assert.conf");
+ Assertions.assertEquals(0, execResult.getExitCode());
+ }
+}
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/mockserver-onesignal-config.json
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/mockserver-onesignal-config.json
new file mode 100644
index 000000000..c7dac3936
--- /dev/null
+++
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/mockserver-onesignal-config.json
@@ -0,0 +1,174 @@
+//
https://www.mock-server.com/mock_server/getting_started.html#request_matchers
+
+[
+ {
+ "httpRequest": {
+ "method" : "GET",
+ "path": "/api/v1/apps"
+ },
+ "httpResponse": {
+ "body": [
+ {
+ "id": "8151f1be-63f2-4c8c-9348-7ad6fda73b3d",
+ "name": "enjoy life",
+ "gcm_key": null,
+ "chrome_key": null,
+ "chrome_web_key": null,
+ "chrome_web_origin": null,
+ "chrome_web_gcm_sender_id": null,
+ "chrome_web_default_notification_icon": null,
+ "chrome_web_sub_domain": null,
+ "apns_env": null,
+ "apns_certificates": null,
+ "apns_p8": null,
+ "apns_team_id": null,
+ "apns_key_id": null,
+ "apns_bundle_id": null,
+ "safari_apns_certificate": null,
+ "safari_site_origin": null,
+ "safari_push_id": null,
+ "safari_icon_16_16":
"public/safari_packages/8151f1be-63f2-4c8c-9348-7ad6fda73b3d/icons/16x16.png",
+ "safari_icon_32_32":
"public/safari_packages/8151f1be-63f2-4c8c-9348-7ad6fda73b3d/icons/[email protected]",
+ "safari_icon_64_64":
"public/safari_packages/8151f1be-63f2-4c8c-9348-7ad6fda73b3d/icons/[email protected]",
+ "safari_icon_128_128":
"public/safari_packages/8151f1be-63f2-4c8c-9348-7ad6fda73b3d/icons/128x128.png",
+ "safari_icon_256_256":
"public/safari_packages/8151f1be-63f2-4c8c-9348-7ad6fda73b3d/icons/[email protected]",
+ "site_name": null,
+ "created_at": "2022-10-30T14:48:14.688Z",
+ "updated_at": "2022-10-30T14:48:14.953Z",
+ "players": 100,
+ "messageable_players": 0,
+ "basic_auth_key": "Y2EyZjI5NzgtMzU1NC00NTU3LWIwNWItXGQ0MzQ4MzQ2ZjY2",
+ "additional_data_is_root_payload": false
+ },
+ {
+ "id": "6386d2ea-d98f-468b-8f01-116d920a1e42",
+ "name": "test",
+ "gcm_key": null,
+ "chrome_key": null,
+ "chrome_web_key": null,
+ "chrome_web_origin": null,
+ "chrome_web_gcm_sender_id": null,
+ "chrome_web_default_notification_icon": null,
+ "chrome_web_sub_domain": null,
+ "apns_env": null,
+ "apns_certificates": null,
+ "apns_p8": null,
+ "apns_team_id": null,
+ "apns_key_id": null,
+ "apns_bundle_id": null,
+ "safari_apns_certificate": null,
+ "safari_site_origin": null,
+ "safari_push_id": null,
+ "safari_icon_16_16":
"public/safari_packages/6386d2ea-d98f-468b-8f01-116d920a1e42/icons/16x16.png",
+ "safari_icon_32_32":
"public/safari_packages/6386d2ea-d98f-468b-8f01-116d920a1e42/icons/[email protected]",
+ "safari_icon_64_64":
"public/safari_packages/6386d2ea-d98f-468b-8f01-116d920a1e42/icons/[email protected]",
+ "safari_icon_128_128":
"public/safari_packages/6386d2ea-d98f-468b-8f01-116d920a1e42/icons/128x128.png",
+ "safari_icon_256_256":
"public/safari_packages/6386d2ea-d98f-468b-8f01-116d920a1e42/icons/[email protected]",
+ "site_name": null,
+ "created_at": "2022-10-30T14:50:24.711Z",
+ "updated_at": "2022-10-30T14:50:24.849Z",
+ "players": 1,
+ "messageable_players": 0,
+ "basic_auth_key": "ODFiZWJiZDItZWYwZC00ODYzLWE4YmUtYTRmY2ZjNTU5NTVi",
+ "additional_data_is_root_payload": false
+ },
+ {
+ "id": "63844g53-d98f-468b-8f01-11632320a1e42",
+ "name": "game",
+ "gcm_key": null,
+ "chrome_key": null,
+ "chrome_web_key": null,
+ "chrome_web_origin": null,
+ "chrome_web_gcm_sender_id": null,
+ "chrome_web_default_notification_icon": null,
+ "chrome_web_sub_domain": null,
+ "apns_env": null,
+ "apns_certificates": null,
+ "apns_p8": null,
+ "apns_team_id": null,
+ "apns_key_id": null,
+ "apns_bundle_id": null,
+ "safari_apns_certificate": null,
+ "safari_site_origin": null,
+ "safari_push_id": null,
+ "safari_icon_16_16":
"public/safari_packages/63844g53-d98f-468b-8f01-11632320a1e42/icons/16x16.png",
+ "safari_icon_32_32":
"public/safari_packages/63844g53-d98f-468b-8f01-11632320a1e42/icons/[email protected]",
+ "safari_icon_64_64":
"public/safari_packages/63844g53-d98f-468b-8f01-11632320a1e42/icons/[email protected]",
+ "safari_icon_128_128":
"public/safari_packages/63844g53-d98f-468b-8f01-11632320a1e42/icons/128x128.png",
+ "safari_icon_256_256":
"public/safari_packages/63844g53-d98f-468b-8f01-11632320a1e42/icons/[email protected]",
+ "site_name": null,
+ "created_at": "2022-10-30T14:50:24.711Z",
+ "updated_at": "2022-10-30T14:50:24.849Z",
+ "players": 16,
+ "messageable_players": 0,
+ "basic_auth_key": "ODFiZWJiZDItZWYwZC00ODYzLWE4MmUtYTRmY2ZjNTU5NTVi",
+ "additional_data_is_root_payload": false
+ },
+ {
+ "id": "632332a-d9f-43238b-8f2301-11a1e42",
+ "name": "metting",
+ "gcm_key": null,
+ "chrome_key": null,
+ "chrome_web_key": null,
+ "chrome_web_origin": null,
+ "chrome_web_gcm_sender_id": null,
+ "chrome_web_default_notification_icon": null,
+ "chrome_web_sub_domain": null,
+ "apns_env": null,
+ "apns_certificates": null,
+ "apns_p8": null,
+ "apns_team_id": null,
+ "apns_key_id": null,
+ "apns_bundle_id": null,
+ "safari_apns_certificate": null,
+ "safari_site_origin": null,
+ "safari_push_id": null,
+ "safari_icon_16_16":
"public/safari_packages/632332a-d9f-43238b-8f2301-11a1e42/icons/16x16.png",
+ "safari_icon_32_32":
"public/safari_packages/632332a-d9f-43238b-8f2301-11a1e42/icons/[email protected]",
+ "safari_icon_64_64":
"public/safari_packages/632332a-d9f-43238b-8f2301-11a1e42/icons/[email protected]",
+ "safari_icon_128_128":
"public/safari_packages/632332a-d9f-43238b-8f2301-11a1e42/icons/128x128.png",
+ "safari_icon_256_256":
"public/safari_packages/632332a-d9f-43238b-8f2301-11a1e42/icons/[email protected]",
+ "site_name": null,
+ "created_at": "2022-10-30T14:50:24.711Z",
+ "updated_at": "2022-10-30T14:50:24.849Z",
+ "players": 0,
+ "messageable_players": 0,
+ "basic_auth_key": "ODFiZWJiZDItZWYwZgbvODYzLWE4MmUtYTRmY2ZjNTU5NTVi",
+ "additional_data_is_root_payload": false
+ },
+ {
+ "id": "6386d2ea-d98f-468b-8f01-116d23r0a1e42",
+ "name": "app test",
+ "gcm_key": null,
+ "chrome_key": null,
+ "chrome_web_key": null,
+ "chrome_web_origin": null,
+ "chrome_web_gcm_sender_id": null,
+ "chrome_web_default_notification_icon": null,
+ "chrome_web_sub_domain": null,
+ "apns_env": null,
+ "apns_certificates": null,
+ "apns_p8": null,
+ "apns_team_id": null,
+ "apns_key_id": null,
+ "apns_bundle_id": null,
+ "safari_apns_certificate": null,
+ "safari_site_origin": null,
+ "safari_push_id": null,
+ "safari_icon_16_16":
"public/safari_packages/6386d2ea-d98f-468b-8f01-116d23r0a1e42/icons/16x16.png",
+ "safari_icon_32_32":
"public/safari_packages/6386d2ea-d98f-468b-8f01-116d23r0a1e42/icons/[email protected]",
+ "safari_icon_64_64":
"public/safari_packages/6386d2ea-d98f-468b-8f01-116d23r0a1e42/icons/[email protected]",
+ "safari_icon_128_128":
"public/safari_packages/6386d2ea-d98f-468b-8f01-116d23r0a1e42/icons/128x128.png",
+ "safari_icon_256_256":
"public/safari_packages/6386d2ea-d98f-468b-8f01-116d23r0a1e42/icons/[email protected]",
+ "site_name": null,
+ "created_at": "2022-10-30T14:50:24.711Z",
+ "updated_at": "2022-10-30T14:50:24.849Z",
+ "players": 2,
+ "messageable_players": 0,
+ "basic_auth_key": "ODFiZWJiZDItZWYwZC00ODYzLWE4MmUtYgreY2ZjNTU5NTVi",
+ "additional_data_is_root_payload": false
+ }
+ ]
+ }
+ }
+]
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/onesignal_json_to_assert.conf
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/onesignal_json_to_assert.conf
new file mode 100644
index 000000000..1da25e22b
--- /dev/null
+++
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/onesignal_json_to_assert.conf
@@ -0,0 +1,112 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+env {
+ execution.parallelism = 1
+ job.mode = "BATCH"
+}
+
+source {
+ OneSignal {
+ url = "http://mockserver:1080/api/v1/apps";
+ password = "Seatunnel-test"
+ method = "GET"
+ format = "json"
+ schema = {
+ fields {
+ id = string
+ name = string
+ gcm_key = string
+ chrome_key = string
+ chrome_web_key = string
+ chrome_web_origin = string
+ chrome_web_gcm_sender_id = string
+ chrome_web_default_notification_icon = string
+ chrome_web_sub_domain = string
+ apns_env = string
+ apns_certificates = string
+ apns_p8 = string
+ apns_team_id = string
+ apns_key_id = string
+ apns_bundle_id = string
+ safari_apns_certificate = string
+ safari_site_origin = string
+ safari_push_id = string
+ safari_icon_16_16 = string
+ safari_icon_32_32 = string
+ safari_icon_64_64 = string
+ safari_icon_128_128 = string
+ safari_icon_256_256 = string
+ site_name = string
+ created_at = string
+ updated_at = string
+ players = int
+ messageable_players = int
+ basic_auth_key = string
+ additional_data_is_root_payload = string
+ }
+ }
+ }
+}
+
+sink {
+ Console {}
+ Assert {
+ rules {
+ row_rules = [
+ {
+ rule_type = MAX_ROW
+ rule_value = 5
+ },
+ {
+ rule_type = MIN_ROW
+ rule_value = 5
+ }
+ ],
+
+ field_rules = [
+ {
+ field_name = id
+ field_type = string
+ field_value = [
+ {
+ rule_type = NOT_NULL
+ }
+ ]
+ },
+ {
+ field_name = name
+ field_type = string
+ field_value = [
+ {
+ rule_type = NOT_NULL
+ }
+ ]
+ },
+ {
+ field_name = players
+ field_type = int
+ field_value = [
+ {
+ rule_type = NOT_NULL
+ }
+ ]
+ }
+ ]
+ }
+ }
+}
\ No newline at end of file
