This is an automated email from the ASF dual-hosted git repository. davsclaus pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/master by this push: new 2808dbf CAMEL-14092: Add camel-debezium-mongodb (#3305) 2808dbf is described below commit 2808dbf6c9e78953024ca66846b194582e5362e3 Author: Omar Al-Safi <omars...@gmail.com> AuthorDate: Thu Oct 31 19:15:35 2019 +0100 CAMEL-14092: Add camel-debezium-mongodb (#3305) Signed-off-by: Omar Al-Safi <omars...@gmail.com> --- apache-camel/pom.xml | 10 + apache-camel/src/main/descriptors/common-bin.xml | 2 + bom/camel-bom/pom.xml | 10 + components/camel-debezium-mongodb/pom.xml | 149 ++++++ .../src/main/docs/debezium-mongodb-component.adoc | 227 +++++++++ .../debezium/DebeziumMongodbComponent.java | 57 +++ .../debezium/DebeziumMongodbEndpoint.java | 49 ++ .../debezium/DebeziumMongodbComponentTest.java | 119 +++++ ...ConnectorEmbeddedDebeziumConfigurationTest.java | 89 ++++ .../src/test/resources/log4j2.properties | 30 ++ components/pom.xml | 1 + parent/pom.xml | 10 + .../karaf/features/src/main/resources/features.xml | 12 +- .../camel-debezium-mongodb-starter/pom.xml | 53 ++ .../DebeziumMongodbComponentAutoConfiguration.java | 129 +++++ .../DebeziumMongodbComponentConfiguration.java | 566 +++++++++++++++++++++ .../src/main/resources/META-INF/LICENSE.txt | 203 ++++++++ .../src/main/resources/META-INF/NOTICE.txt | 11 + .../src/main/resources/META-INF/spring.factories | 19 + .../src/main/resources/META-INF/spring.provides | 17 + platforms/spring-boot/components-starter/pom.xml | 1 + .../camel-spring-boot-dependencies/pom.xml | 10 + .../itest/karaf/CamelDebeziumMongodbTest.java | 35 ++ .../itest/springboot/CamelDebeziumMongodbTest.java | 46 ++ 24 files changed, 1854 insertions(+), 1 deletion(-) diff --git a/apache-camel/pom.xml b/apache-camel/pom.xml index aab587b..fd753ab 100644 --- a/apache-camel/pom.xml +++ b/apache-camel/pom.xml @@ -473,6 +473,11 @@ </dependency> <dependency> <groupId>org.apache.camel</groupId> + <artifactId>camel-debezium-mongodb</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.camel</groupId> <artifactId>camel-debezium-mysql</artifactId> <version>${project.version}</version> </dependency> @@ -2047,6 +2052,11 @@ </dependency> <dependency> <groupId>org.apache.camel</groupId> + <artifactId>camel-debezium-mongodb-starter</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.camel</groupId> <artifactId>camel-debezium-mysql-starter</artifactId> <version>${project.version}</version> </dependency> diff --git a/apache-camel/src/main/descriptors/common-bin.xml b/apache-camel/src/main/descriptors/common-bin.xml index 74b2a9c..aa37152 100644 --- a/apache-camel/src/main/descriptors/common-bin.xml +++ b/apache-camel/src/main/descriptors/common-bin.xml @@ -115,6 +115,7 @@ <include>org.apache.camel:camel-dataformat</include> <include>org.apache.camel:camel-dataset</include> <include>org.apache.camel:camel-debezium-common</include> + <include>org.apache.camel:camel-debezium-mongodb</include> <include>org.apache.camel:camel-debezium-mysql</include> <include>org.apache.camel:camel-debezium-postgres</include> <include>org.apache.camel:camel-digitalocean</include> @@ -469,6 +470,7 @@ <include>org.apache.camel:camel-dataformat-starter</include> <include>org.apache.camel:camel-dataset-starter</include> <include>org.apache.camel:camel-debezium-common-starter</include> + <include>org.apache.camel:camel-debezium-mongodb-starter</include> <include>org.apache.camel:camel-debezium-mysql-starter</include> <include>org.apache.camel:camel-debezium-postgres-starter</include> <include>org.apache.camel:camel-digitalocean-starter</include> diff --git a/bom/camel-bom/pom.xml b/bom/camel-bom/pom.xml index a2dc471..6bbf0e5 100644 --- a/bom/camel-bom/pom.xml +++ b/bom/camel-bom/pom.xml @@ -829,6 +829,16 @@ </dependency> <dependency> <groupId>org.apache.camel</groupId> + <artifactId>camel-debezium-mongodb</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.camel</groupId> + <artifactId>camel-debezium-mongodb-starter</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.camel</groupId> <artifactId>camel-debezium-mysql</artifactId> <version>${project.version}</version> </dependency> diff --git a/components/camel-debezium-mongodb/pom.xml b/components/camel-debezium-mongodb/pom.xml new file mode 100644 index 0000000..3786f43 --- /dev/null +++ b/components/camel-debezium-mongodb/pom.xml @@ -0,0 +1,149 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> +<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.camel</groupId> + <artifactId>components</artifactId> + <version>3.0.0-SNAPSHOT</version> + </parent> + + <artifactId>camel-debezium-mongodb</artifactId> + <packaging>jar</packaging> + <name>Camel :: Debezium :: MongoDB</name> + <description>Camel Debezium MongoDB support</description> + + <dependencies> + <!-- camel --> + <dependency> + <groupId>org.apache.camel</groupId> + <artifactId>camel-debezium-common</artifactId> + </dependency> + + <!-- debezium MongoDB Connector --> + <dependency> + <groupId>io.debezium</groupId> + <artifactId>debezium-connector-mongodb</artifactId> + <version>${debezium-version}</version> + </dependency> + + <!-- test --> + <dependency> + <groupId>org.apache.camel</groupId> + <artifactId>camel-test</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-api</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-core</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-slf4j-impl</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-1.2-api</artifactId> + <scope>test</scope> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.camel.maven</groupId> + <artifactId>camel-debezium-maven-plugin</artifactId> + <version>${project.version}</version> + <executions> + <execution> + <id>generate</id> + <goals> + <goal>generate-connector-config</goal> + </goals> + </execution> + </executions> + <dependencies> + <dependency> + <groupId>io.debezium</groupId> + <artifactId>debezium-connector-mongodb</artifactId> + <version>${debezium-version}</version> + <scope>runtime</scope> + </dependency> + </dependencies> + <configuration> + <connectorClassName>io.debezium.connector.mongodb.MongoDbConnector</connectorClassName> + <connectorConfigClassName>io.debezium.connector.mongodb.MongoDbConnectorConfig</connectorConfigClassName> + <fields> + <tombstones.on.delete>false</tombstones.on.delete> + </fields> + <requiredFields> + <field>mongodb.password</field> + <field>mongodb.name</field> + </requiredFields> + </configuration> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <executions> + <execution> + <id>add-generated-configurations-sources</id> + <goals> + <goal>add-source</goal> + </goals> + <configuration> + <sources> + <source>${project.build.directory}/generated-sources/connector-configurations</source> + </sources> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <!-- This used to copy EmbeddedDebeziumConfiguration source in order to generate the metadata via APT plugin--> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <executions> + <execution> + <id>src-dependencies</id> + <goals> + <goal>unpack-dependencies</goal> + </goals> + <configuration> + <classifier>sources</classifier> + <includeArtifactIds>camel-debezium-common</includeArtifactIds> + <includes>**/*EmbeddedDebeziumConfiguration.java</includes> + <outputDirectory>${project.build.directory}/generated-sources/connector-configurations</outputDirectory> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> +</project> diff --git a/components/camel-debezium-mongodb/src/main/docs/debezium-mongodb-component.adoc b/components/camel-debezium-mongodb/src/main/docs/debezium-mongodb-component.adoc new file mode 100644 index 0000000..6eeb4cf --- /dev/null +++ b/components/camel-debezium-mongodb/src/main/docs/debezium-mongodb-component.adoc @@ -0,0 +1,227 @@ +[[debezium-mongodb-component]] += Debezium MongoDB Connector Component + +*Since Camel 3.0* + +The Debezium MongoDB component is wrapper around https://debezium.io/[Debezium] using https://debezium.io/documentation/reference/0.9/operations/embedded.html[Debezium Embedded], which enables Change Data Capture from MongoDB database using Debezium without the need for Kafka or Kafka Connect. + +*Note:* The Debezium MongoDB connector uses MongoDB’s oplog to capture the changes, so the connector works only with MongoDB replica sets or with sharded clusters where each shard is a separate replica set, therefore you will need to have your MongoDB instance running either in replica set mode +or sharded clusters mode. + +*Note on handling failures:* Per https://debezium.io/documentation/reference/0.9/operations/embedded.html#_handling_failures[Debezium Embedded Engine] documentation, the engines is actively recording source offsets and periodically flushes these offsets to a persistent storage, so when the application is restarted or crashed, the engine will resume from the last recorded offset. +Thus, at normal operation, your downstream routes will receive each event exactly once, however in case of an application crash (not having a graceful shutdown), the application will resume from the last recorded offset, +which may result in receiving duplicate events immediately after the restart. Therefore, your downstream routes should be tolerant enough of such case and deduplicate events if needed. + +Maven users will need to add the following dependency to their `pom.xml` +for this component. + +[source,xml] +---- +<dependency> + <groupId>org.apache.camel</groupId> + <artifactId>camel-debezium-mongodb</artifactId> + <version>x.x.x</version> + <!-- use the same version as your Camel core version --> +</dependency> +---- + +== URI format + +[source,text] +--------------------------- +debezium-mongodb:name[?options] +--------------------------- + +== Options + + +// component options: START +The Debezium MongoDB Connector component supports 2 options, which are listed below. + + + +[width="100%",cols="2,5,^1,2",options="header"] +|=== +| Name | Description | Default | Type +| *configuration* (consumer) | Allow pre-configured Configurations to be set. | | MongoDbConnectorEmbeddedDebeziumConfiguration +| *basicPropertyBinding* (advanced) | Whether the component should use basic property binding (Camel 2.x) or the newer property binding with additional capabilities | false | boolean +|=== +// component options: END + + +// endpoint options: START +The Debezium MongoDB Connector endpoint is configured using URI syntax: + +---- +debezium-mongodb:name +---- + +with the following path and query parameters: + +=== Path Parameters (1 parameters): + + +[width="100%",cols="2,5,^1,2",options="header"] +|=== +| Name | Description | Default | Type +| *name* | *Required* Unique name for the connector. Attempting to register again with the same name will fail. | | String +|=== + + +=== Query Parameters (40 parameters): + + +[width="100%",cols="2,5,^1,2",options="header"] +|=== +| Name | Description | Default | Type +| *bridgeErrorHandler* (consumer) | Allows for bridging the consumer to the Camel routing Error Handler, which mean any exceptions occurred while the consumer is trying to pickup incoming messages, or the likes, will now be processed as a message and handled by the routing Error Handler. By default the consumer will use the org.apache.camel.spi.ExceptionHandler to deal with exceptions, that will be logged at WARN or ERROR level and ignored. | false | boolean +| *internalKeyConverter* (consumer) | The Converter class that should be used to serialize and deserialize key data for offsets. The default is JSON converter. | org.apache.kafka.connect.json.JsonConverter | String +| *internalValueConverter* (consumer) | The Converter class that should be used to serialize and deserialize value data for offsets. The default is JSON converter. | org.apache.kafka.connect.json.JsonConverter | String +| *offsetCommitPolicy* (consumer) | The name of the Java class of the commit policy. It defines when offsets commit has to be triggered based on the number of events processed and the time elapsed since the last commit. This class must implement the interface 'OffsetCommitPolicy'. The default is a periodic commit policy based upon time intervals. | io.debezium.embedded.spi.OffsetCommitPolicy.PeriodicCommitOffsetPolicy | String +| *offsetCommitTimeoutMs* (consumer) | Maximum number of milliseconds to wait for records to flush and partition offset data to be committed to offset storage before cancelling the process and restoring the offset data to be committed in a future attempt. The default is 5 seconds. | 5000 | long +| *offsetFlushIntervalMs* (consumer) | Interval at which to try committing offsets. The default is 1 minute. | 60000 | long +| *offsetStorage* (consumer) | The name of the Java class that is responsible for persistence of connector offsets. | org.apache.kafka.connect.storage.FileOffsetBackingStore | String +| *offsetStorageFileName* (consumer) | Path to file where offsets are to be stored. Required when offset.storage is set to the FileOffsetBackingStore | | String +| *offsetStoragePartitions* (consumer) | The number of partitions used when creating the offset storage topic. Required when offset.storage is set to the 'KafkaOffsetBackingStore'. | | int +| *offsetStorageReplication Factor* (consumer) | Replication factor used when creating the offset storage topic. Required when offset.storage is set to the KafkaOffsetBackingStore | | int +| *offsetStorageTopic* (consumer) | The name of the Kafka topic where offsets are to be stored. Required when offset.storage is set to the KafkaOffsetBackingStore. | | String +| *exceptionHandler* (consumer) | To let the consumer use a custom ExceptionHandler. Notice if the option bridgeErrorHandler is enabled then this option is not in use. By default the consumer will deal with exceptions, that will be logged at WARN or ERROR level and ignored. | | ExceptionHandler +| *exchangePattern* (consumer) | Sets the exchange pattern when the consumer creates an exchange. | | ExchangePattern +| *basicPropertyBinding* (advanced) | Whether the endpoint should use basic property binding (Camel 2.x) or the newer property binding with additional capabilities | false | boolean +| *synchronous* (advanced) | Sets whether synchronous processing should be strictly used, or Camel is allowed to use asynchronous processing (if supported). | false | boolean +| *collectionBlacklist* (mongodb) | Description is not available here, please check Debezium website for corresponding key 'collection.blacklist' description. | | String +| *collectionWhitelist* (mongodb) | The collections for which changes are to be captured | | String +| *connectBackoffInitialDelay Ms* (mongodb) | The initial delay when trying to reconnect to a primary after a connection cannot be made or when no primary is available. Defaults to 1 second (1000 ms). | 1000 | long +| *connectBackoffMaxDelayMs* (mongodb) | The maximum delay when trying to reconnect to a primary after a connection cannot be made or when no primary is available. Defaults to 120 second (120,000 ms). | 120000 | long +| *connectMaxAttempts* (mongodb) | Maximum number of failed connection attempts to a replica set primary before an exception occurs and task is aborted. Defaults to 16, which with the defaults for 'connect.backoff.initial.delay.ms' and 'connect.backoff.max.delay.ms' results in just over 20 minutes of attempts before failing. | 16 | int +| *databaseBlacklist* (mongodb) | The databases for which changes are to be excluded | | String +| *databaseHistoryFileFilename* (mongodb) | The path to the file that will be used to record the database history | | String +| *databaseWhitelist* (mongodb) | The databases for which changes are to be captured | | String +| *fieldBlacklist* (mongodb) | Description is not available here, please check Debezium website for corresponding key 'field.blacklist' description. | | String +| *fieldRenames* (mongodb) | Description is not available here, please check Debezium website for corresponding key 'field.renames' description. | | String +| *initialSyncMaxThreads* (mongodb) | Maximum number of threads used to perform an intial sync of the collections in a replica set. Defaults to 1. | 1 | int +| *maxBatchSize* (mongodb) | Maximum size of each batch of source records. Defaults to 2048. | 2048 | int +| *maxQueueSize* (mongodb) | Maximum size of the queue for change events read from the database log but not yet recorded or forwarded. Defaults to 8192, and should always be larger than the maximum batch size. | 8192 | int +| *mongodbHosts* (mongodb) | The hostname and port pairs (in the form 'host' or 'host:port') of the MongoDB server(s) in the replica set. | | String +| *mongodbMembersAutoDiscover* (mongodb) | Specifies whether the addresses in 'hosts' are seeds that should be used to discover all members of the cluster or replica set ('true'), or whether the address(es) in 'hosts' should be used as is ('false'). The default is 'true'. | true | boolean +| *mongodbName* (mongodb) | *Required* Unique name that identifies the MongoDB replica set or cluster and all recorded offsets, andthat is used as a prefix for all schemas and topics. Each distinct MongoDB installation should have a separate namespace and monitored by at most one Debezium connector. | | String +| *mongodbPassword* (mongodb) | *Required* Password to be used when connecting to MongoDB, if necessary. | | String +| *mongodbSslEnabled* (mongodb) | Should connector use SSL to connect to MongoDB instances | false | boolean +| *mongodbSslInvalidHostname Allowed* (mongodb) | Whether invalid host names are allowed when using SSL. If true the connection will not prevent man-in-the-middle attacks | false | boolean +| *mongodbUser* (mongodb) | Database user for connecting to MongoDB, if necessary. | | String +| *pollIntervalMs* (mongodb) | Frequency in milliseconds to wait for new change events to appear after receiving no events. Defaults to 500ms. | 500 | long +| *snapshotDelayMs* (mongodb) | The number of milliseconds to delay before a snapshot will begin. | 0 | long +| *snapshotFetchSize* (mongodb) | The maximum number of records that should be loaded into memory while performing a snapshot | | int +| *snapshotMode* (mongodb) | The criteria for running a snapshot upon startup of the connector. Options include: 'initial' (the default) to specify the connector should always perform an initial sync when required; 'never' to specify the connector should never perform an initial sync | initial | String +| *tombstonesOnDelete* (mongodb) | Whether delete operations should be represented by a delete event and a subsquenttombstone event (true) or only by a delete event (false). Emitting the tombstone event (the default behavior) allows Kafka to completely delete all events pertaining to the given key once the source record got deleted. | false | boolean +|=== +// endpoint options: END +// spring-boot-auto-configure options: START +== Spring Boot Auto-Configuration + +When using Spring Boot make sure to use the following Maven dependency to have support for auto configuration: + +[source,xml] +---- +<dependency> + <groupId>org.apache.camel</groupId> + <artifactId>camel-debezium-mongodb-starter</artifactId> + <version>x.x.x</version> + <!-- use the same version as your Camel core version --> +</dependency> +---- + + +The component supports 39 options, which are listed below. + + + +[width="100%",cols="2,5,^1,2",options="header"] +|=== +| Name | Description | Default | Type +| *camel.component.debezium-mongodb.basic-property-binding* | Whether the component should use basic property binding (Camel 2.x) or the newer property binding with additional capabilities | false | Boolean +| *camel.component.debezium-mongodb.configuration.collection-blacklist* | Description is not available here, please check Debezium website for corresponding key 'collection.blacklist' description. | | String +| *camel.component.debezium-mongodb.configuration.collection-whitelist* | The collections for which changes are to be captured | | String +| *camel.component.debezium-mongodb.configuration.connect-backoff-initial-delay-ms* | The initial delay when trying to reconnect to a primary after a connection cannot be made or when no primary is available. Defaults to 1 second (1000 ms). | 1000 | Long +| *camel.component.debezium-mongodb.configuration.connect-backoff-max-delay-ms* | The maximum delay when trying to reconnect to a primary after a connection cannot be made or when no primary is available. Defaults to 120 second (120,000 ms). | 120000 | Long +| *camel.component.debezium-mongodb.configuration.connect-max-attempts* | Maximum number of failed connection attempts to a replica set primary before an exception occurs and task is aborted. Defaults to 16, which with the defaults for 'connect.backoff.initial.delay.ms' and 'connect.backoff.max.delay.ms' results in just over 20 minutes of attempts before failing. | 16 | Integer +| *camel.component.debezium-mongodb.configuration.connector-class* | The name of the Java class for the connector | | Class +| *camel.component.debezium-mongodb.configuration.database-blacklist* | The databases for which changes are to be excluded | | String +| *camel.component.debezium-mongodb.configuration.database-history-file-filename* | The path to the file that will be used to record the database history | | String +| *camel.component.debezium-mongodb.configuration.database-whitelist* | The databases for which changes are to be captured | | String +| *camel.component.debezium-mongodb.configuration.field-blacklist* | Description is not available here, please check Debezium website for corresponding key 'field.blacklist' description. | | String +| *camel.component.debezium-mongodb.configuration.field-renames* | Description is not available here, please check Debezium website for corresponding key 'field.renames' description. | | String +| *camel.component.debezium-mongodb.configuration.initial-sync-max-threads* | Maximum number of threads used to perform an intial sync of the collections in a replica set. Defaults to 1. | 1 | Integer +| *camel.component.debezium-mongodb.configuration.internal-key-converter* | The Converter class that should be used to serialize and deserialize key data for offsets. The default is JSON converter. | org.apache.kafka.connect.json.JsonConverter | String +| *camel.component.debezium-mongodb.configuration.internal-value-converter* | The Converter class that should be used to serialize and deserialize value data for offsets. The default is JSON converter. | org.apache.kafka.connect.json.JsonConverter | String +| *camel.component.debezium-mongodb.configuration.max-batch-size* | Maximum size of each batch of source records. Defaults to 2048. | 2048 | Integer +| *camel.component.debezium-mongodb.configuration.max-queue-size* | Maximum size of the queue for change events read from the database log but not yet recorded or forwarded. Defaults to 8192, and should always be larger than the maximum batch size. | 8192 | Integer +| *camel.component.debezium-mongodb.configuration.mongodb-hosts* | The hostname and port pairs (in the form 'host' or 'host:port') of the MongoDB server(s) in the replica set. | | String +| *camel.component.debezium-mongodb.configuration.mongodb-members-auto-discover* | Specifies whether the addresses in 'hosts' are seeds that should be used to discover all members of the cluster or replica set ('true'), or whether the address(es) in 'hosts' should be used as is ('false'). The default is 'true'. | true | Boolean +| *camel.component.debezium-mongodb.configuration.mongodb-name* | Unique name that identifies the MongoDB replica set or cluster and all recorded offsets, andthat is used as a prefix for all schemas and topics. Each distinct MongoDB installation should have a separate namespace and monitored by at most one Debezium connector. | | String +| *camel.component.debezium-mongodb.configuration.mongodb-password* | Password to be used when connecting to MongoDB, if necessary. | | String +| *camel.component.debezium-mongodb.configuration.mongodb-ssl-enabled* | Should connector use SSL to connect to MongoDB instances | false | Boolean +| *camel.component.debezium-mongodb.configuration.mongodb-ssl-invalid-hostname-allowed* | Whether invalid host names are allowed when using SSL. If true the connection will not prevent man-in-the-middle attacks | false | Boolean +| *camel.component.debezium-mongodb.configuration.mongodb-user* | Database user for connecting to MongoDB, if necessary. | | String +| *camel.component.debezium-mongodb.configuration.name* | Unique name for the connector. Attempting to register again with the same name will fail. | | String +| *camel.component.debezium-mongodb.configuration.offset-commit-policy* | The name of the Java class of the commit policy. It defines when offsets commit has to be triggered based on the number of events processed and the time elapsed since the last commit. This class must implement the interface 'OffsetCommitPolicy'. The default is a periodic commit policy based upon time intervals. | io.debezium.embedded.spi.OffsetCommitPolicy.PeriodicCommitOffsetPolicy | String +| *camel.component.debezium-mongodb.configuration.offset-commit-timeout-ms* | Maximum number of milliseconds to wait for records to flush and partition offset data to be committed to offset storage before cancelling the process and restoring the offset data to be committed in a future attempt. The default is 5 seconds. | 5000 | Long +| *camel.component.debezium-mongodb.configuration.offset-flush-interval-ms* | Interval at which to try committing offsets. The default is 1 minute. | 60000 | Long +| *camel.component.debezium-mongodb.configuration.offset-storage* | The name of the Java class that is responsible for persistence of connector offsets. | org.apache.kafka.connect.storage.FileOffsetBackingStore | String +| *camel.component.debezium-mongodb.configuration.offset-storage-file-name* | Path to file where offsets are to be stored. Required when offset.storage is set to the FileOffsetBackingStore | | String +| *camel.component.debezium-mongodb.configuration.offset-storage-partitions* | The number of partitions used when creating the offset storage topic. Required when offset.storage is set to the 'KafkaOffsetBackingStore'. | | Integer +| *camel.component.debezium-mongodb.configuration.offset-storage-replication-factor* | Replication factor used when creating the offset storage topic. Required when offset.storage is set to the KafkaOffsetBackingStore | | Integer +| *camel.component.debezium-mongodb.configuration.offset-storage-topic* | The name of the Kafka topic where offsets are to be stored. Required when offset.storage is set to the KafkaOffsetBackingStore. | | String +| *camel.component.debezium-mongodb.configuration.poll-interval-ms* | Frequency in milliseconds to wait for new change events to appear after receiving no events. Defaults to 500ms. | 500 | Long +| *camel.component.debezium-mongodb.configuration.snapshot-delay-ms* | The number of milliseconds to delay before a snapshot will begin. | 0 | Long +| *camel.component.debezium-mongodb.configuration.snapshot-fetch-size* | The maximum number of records that should be loaded into memory while performing a snapshot | | Integer +| *camel.component.debezium-mongodb.configuration.snapshot-mode* | The criteria for running a snapshot upon startup of the connector. Options include: 'initial' (the default) to specify the connector should always perform an initial sync when required; 'never' to specify the connector should never perform an initial sync | initial | String +| *camel.component.debezium-mongodb.configuration.tombstones-on-delete* | Whether delete operations should be represented by a delete event and a subsquenttombstone event (true) or only by a delete event (false). Emitting the tombstone event (the default behavior) allows Kafka to completely delete all events pertaining to the given key once the source record got deleted. | false | Boolean +| *camel.component.debezium-mongodb.enabled* | Whether to enable auto configuration of the debezium-mongodb component. This is enabled by default. | | Boolean +|=== +// spring-boot-auto-configure options: END + +For more information about configuration: +https://debezium.io/documentation/reference/0.9/operations/embedded.html#engine-properties[https://debezium.io/documentation/reference/0.9/operations/embedded.html#engine-properties] +https://debezium.io/documentation/reference/0.9/connectors/mongodb.html#connector-properties[https://debezium.io/documentation/reference/0.9/connectors/mongodb.html#connector-properties] + +== Message headers + +=== Consumer headers + +The following headers are available when consuming change events from Debezium. +[width="100%",cols="2m,2m,1m,5",options="header"] +|=== +| Header constant | Header value | Type | Description +| DebeziumConstants.HEADER_IDENTIFIER | "CamelDebeziumIdentifier" | String | The identifier of the connector, normally is this format "{server-name}.{database-name}.{table-name}". +| DebeziumConstants.HEADER_KEY | "CamelDebeziumKey" | Struct | The key of the event, normally is the table Primary Key. +| DebeziumConstants.HEADER_SOURCE_METADATA | "CamelDebeziumSourceMetadata" | Map | The metadata about the source event, for example `table` name, database `name`, log position, etc, please refer to the Debezium documentation for more info. +| DebeziumConstants.HEADER_OPERATION | "CamelDebeziumOperation" | String | If presents, the type of event operation. Values for the connector are `c` for create (or insert), `u` for update, `d` for delete or `r` in case of a snapshot event. +| DebeziumConstants.HEADER_TIMESTAMP | "CamelDebeziumTimestamp" | Long | If presents, the time (using the system clock in the JVM) at which the connector processed the event. +| DebeziumConstants.HEADER_BEFORE | "CamelDebeziumBefore" | Struct | If presents, contains the state of the row before the event occurred. +|=== + +== Message body +The message body if is not `null` (in case of tombstones), it contains the state of the row after the event occurred as `String` JSON format and you can unmarchal using Camel JSON Data Format. + +== Samples + +=== Consuming events + +Here is a very simple route that you can use in order to listen to Debezium events from MongoDB connector. +[source,java] +---- +from("debezium-mongodb:dbz-test-1?offsetStorageFileName=/usr/offset-file-1.dat&mongodbHosts=rs0/localhost:27017&mongodbUser=debezium&mongodbPassword=dbz&mongodbName=dbserver1&databaseHistoryFileName=/usr/history-file-1.dat") + .unmarshal().json() + .log("Event received from Debezium : ${body}") + .log(" with this identifier ${headers.CamelDebeziumIdentifier}") + .log(" with these source metadata ${headers.CamelDebeziumSourceMetadata}") + .log(" the event occured upon this operation '${headers.CamelDebeziumSourceOperation}'") + .log(" on this database '${headers.CamelDebeziumSourceMetadata[db]}' and this table '${headers.CamelDebeziumSourceMetadata[table]}'") + .log(" with the key ${headers.CamelDebeziumKey}") + .log(" the previous value is ${headers.CamelDebeziumBefore}") +---- + +By default, the component will emit the events in the body and `CamelDebeziumBefore` header as String JSON format, this can be easily converted to JSON using Camel JSON Data Format e.g: `.unmarshal().json()` like the above example. + +*Important Note:* This component is a thin wrapper around Debezium Engine as mentioned, therefore before using this component in production, you need to understand how Debezium works and how configurations can reflect the expected behavior, especially in regards to https://debezium.io/documentation/reference/0.9/operations/embedded.html#_handling_failures[handling failures]. diff --git a/components/camel-debezium-mongodb/src/main/java/org/apache/camel/component/debezium/DebeziumMongodbComponent.java b/components/camel-debezium-mongodb/src/main/java/org/apache/camel/component/debezium/DebeziumMongodbComponent.java new file mode 100644 index 0000000..e291846 --- /dev/null +++ b/components/camel-debezium-mongodb/src/main/java/org/apache/camel/component/debezium/DebeziumMongodbComponent.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.debezium; + +import org.apache.camel.CamelContext; +import org.apache.camel.component.debezium.configuration.MongoDbConnectorEmbeddedDebeziumConfiguration; +import org.apache.camel.spi.annotations.Component; + +@Component("debezium-mongodb") +public final class DebeziumMongodbComponent extends DebeziumComponent<MongoDbConnectorEmbeddedDebeziumConfiguration> { + + private MongoDbConnectorEmbeddedDebeziumConfiguration configuration; + + public DebeziumMongodbComponent() { + } + + public DebeziumMongodbComponent(final CamelContext context) { + super(context); + } + + /** + * Allow pre-configured Configurations to be set. + * + * @return {@link MongoDbConnectorEmbeddedDebeziumConfiguration} + */ + @Override + public MongoDbConnectorEmbeddedDebeziumConfiguration getConfiguration() { + if (configuration == null) { + return new MongoDbConnectorEmbeddedDebeziumConfiguration(); + } + return configuration; + } + + @Override + public void setConfiguration(MongoDbConnectorEmbeddedDebeziumConfiguration configuration) { + this.configuration = configuration; + } + + @Override + protected DebeziumEndpoint initializeDebeziumEndpoint(String uri, MongoDbConnectorEmbeddedDebeziumConfiguration configuration) { + return new DebeziumMongodbEndpoint(uri, this, configuration); + } +} diff --git a/components/camel-debezium-mongodb/src/main/java/org/apache/camel/component/debezium/DebeziumMongodbEndpoint.java b/components/camel-debezium-mongodb/src/main/java/org/apache/camel/component/debezium/DebeziumMongodbEndpoint.java new file mode 100644 index 0000000..d62f235 --- /dev/null +++ b/components/camel-debezium-mongodb/src/main/java/org/apache/camel/component/debezium/DebeziumMongodbEndpoint.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.debezium; + +import org.apache.camel.component.debezium.configuration.MongoDbConnectorEmbeddedDebeziumConfiguration; +import org.apache.camel.spi.UriEndpoint; +import org.apache.camel.spi.UriParam; + +/** + * Represents a Debezium MongoDB endpoint which is used to capture changes in MongoDB database so that that applications can see those changes and respond to them. + */ +@UriEndpoint(firstVersion = "3.0.0", scheme = "debezium-mongodb", title = "Debezium MongoDB Connector", syntax = "debezium-mongodb:name", label = "database,nosql,mongodb", consumerOnly = true) +public final class DebeziumMongodbEndpoint extends DebeziumEndpoint<MongoDbConnectorEmbeddedDebeziumConfiguration> { + + @UriParam + private MongoDbConnectorEmbeddedDebeziumConfiguration configuration; + + public DebeziumMongodbEndpoint(final String uri, final DebeziumMongodbComponent component, final MongoDbConnectorEmbeddedDebeziumConfiguration configuration) { + super(uri, component); + this.configuration = configuration; + } + + public DebeziumMongodbEndpoint() { + } + + @Override + public MongoDbConnectorEmbeddedDebeziumConfiguration getConfiguration() { + return configuration; + } + + @Override + public void setConfiguration(final MongoDbConnectorEmbeddedDebeziumConfiguration configuration) { + this.configuration = configuration; + } +} diff --git a/components/camel-debezium-mongodb/src/test/java/org/apache/camel/component/debezium/DebeziumMongodbComponentTest.java b/components/camel-debezium-mongodb/src/test/java/org/apache/camel/component/debezium/DebeziumMongodbComponentTest.java new file mode 100644 index 0000000..bdc498b --- /dev/null +++ b/components/camel-debezium-mongodb/src/test/java/org/apache/camel/component/debezium/DebeziumMongodbComponentTest.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.debezium; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import org.apache.camel.component.debezium.configuration.MongoDbConnectorEmbeddedDebeziumConfiguration; +import org.apache.camel.impl.DefaultCamelContext; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +public class DebeziumMongodbComponentTest { + + @Test + public void testIfConnectorEndpointCreatedWithConfig() throws Exception { + final Map<String, Object> params = new HashMap<>(); + params.put("offsetStorageFileName", "/offset_test_file"); + params.put("mongodbHosts", "localhost"); + params.put("mongodbUser", "dbz"); + params.put("mongodbPassword", "pwd"); + params.put("mongodbName", "test"); + params.put("databaseHistoryFileFilename", "/db_history_file_test"); + + final String remaining = "test_name"; + final String uri = "debezium?name=test_name&offsetStorageFileName=/test&" + + "databaseHostName=localhost&databaseServerId=1234&databaseUser=dbz&databasePassword=pwd&" + + "databaseServerName=test&databaseHistoryFileName=/test"; + + final DebeziumComponent debeziumComponent = new DebeziumMongodbComponent(new DefaultCamelContext()); + final DebeziumEndpoint debeziumEndpoint = debeziumComponent.createEndpoint(uri, remaining, params); + + assertNotNull(debeziumEndpoint); + + // test for config + final MongoDbConnectorEmbeddedDebeziumConfiguration configuration = (MongoDbConnectorEmbeddedDebeziumConfiguration)debeziumEndpoint + .getConfiguration(); + assertEquals("test_name", configuration.getName()); + assertEquals("/offset_test_file", configuration.getOffsetStorageFileName()); + assertEquals("localhost", configuration.getMongodbHosts()); + assertEquals("dbz", configuration.getMongodbUser()); + assertEquals("pwd", configuration.getMongodbPassword()); + assertEquals("test", configuration.getMongodbName()); + assertEquals("/db_history_file_test", configuration.getDatabaseHistoryFileFilename()); + } + + @Test + public void testIfCreatesComponentWithExternalConfiguration() throws Exception { + final MongoDbConnectorEmbeddedDebeziumConfiguration configuration = new MongoDbConnectorEmbeddedDebeziumConfiguration(); + configuration.setName("test_config"); + configuration.setMongodbUser("test_db"); + configuration.setMongodbPassword("pwd"); + configuration.setOffsetStorageFileName("/offset/file"); + configuration.setMongodbName("test"); + + final String uri = "debezium:dummy"; + final DebeziumComponent debeziumComponent = new DebeziumMongodbComponent(new DefaultCamelContext()); + + // set configurations + debeziumComponent.setConfiguration(configuration); + + final DebeziumEndpoint debeziumEndpoint = debeziumComponent.createEndpoint(uri, null, + Collections.emptyMap()); + + assertNotNull(debeziumEndpoint); + + // assert configurations + final MongoDbConnectorEmbeddedDebeziumConfiguration actualConfigurations = (MongoDbConnectorEmbeddedDebeziumConfiguration)debeziumEndpoint + .getConfiguration(); + assertNotNull(actualConfigurations); + assertEquals(configuration.getName(), actualConfigurations.getName()); + assertEquals(configuration.getMongodbUser(), + actualConfigurations.getMongodbUser()); + assertEquals(configuration.getConnectorClass(), actualConfigurations.getConnectorClass()); + } + + @Test(expected = IllegalArgumentException.class) + public void testIfItHandlesNullExternalConfigurations() throws Exception { + final String remaining = ""; + final String uri = "debezium:"; + final DebeziumComponent debeziumComponent = new DebeziumMongodbComponent(new DefaultCamelContext()); + + // set configurations + debeziumComponent.setConfiguration(null); + + final DebeziumEndpoint debeziumEndpoint = debeziumComponent.createEndpoint(uri, remaining, + Collections.emptyMap()); + } + + @Test(expected = IllegalArgumentException.class) + public void testIfItHandlesNullExternalConfigurationsWithValidUri() throws Exception { + final String remaining = "dummy"; + final String uri = "debezium:dummy"; + final DebeziumComponent debeziumComponent = new DebeziumMongodbComponent(new DefaultCamelContext()); + + // set configurations + debeziumComponent.setConfiguration(null); + + final DebeziumEndpoint debeziumEndpoint = debeziumComponent.createEndpoint(uri, remaining, + Collections.emptyMap()); + } +} \ No newline at end of file diff --git a/components/camel-debezium-mongodb/src/test/java/org/apache/camel/component/debezium/configuration/MongodbConnectorEmbeddedDebeziumConfigurationTest.java b/components/camel-debezium-mongodb/src/test/java/org/apache/camel/component/debezium/configuration/MongodbConnectorEmbeddedDebeziumConfigurationTest.java new file mode 100644 index 0000000..394fd96 --- /dev/null +++ b/components/camel-debezium-mongodb/src/test/java/org/apache/camel/component/debezium/configuration/MongodbConnectorEmbeddedDebeziumConfigurationTest.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.debezium.configuration; + +import io.debezium.config.CommonConnectorConfig; +import io.debezium.config.Configuration; +import io.debezium.connector.mongodb.MongoDbConnector; +import io.debezium.connector.mongodb.MongoDbConnectorConfig; +import io.debezium.embedded.EmbeddedEngine; +import org.apache.camel.component.debezium.DebeziumConstants; +import org.junit.Test; + +import static org.junit.Assert.*; + +public class MongodbConnectorEmbeddedDebeziumConfigurationTest { + + @Test + public void testIfCreatesConfig() { + final MongoDbConnectorEmbeddedDebeziumConfiguration configuration = new MongoDbConnectorEmbeddedDebeziumConfiguration(); + configuration.setName("test_config"); + configuration.setMongodbUser("test_user"); + configuration.setMaxQueueSize(1212); + + final Configuration dbzConfigurations = configuration.createDebeziumConfiguration(); + + assertEquals("test_config", dbzConfigurations.getString(EmbeddedEngine.ENGINE_NAME)); + assertEquals("test_user", dbzConfigurations.getString(MongoDbConnectorConfig.USER)); + assertEquals(1212, dbzConfigurations.getInteger(CommonConnectorConfig.MAX_QUEUE_SIZE)); + assertEquals(MongoDbConnector.class.getName(), dbzConfigurations.getString(EmbeddedEngine.CONNECTOR_CLASS)); + assertEquals(DebeziumConstants.DEFAULT_OFFSET_STORAGE, + dbzConfigurations.getString(EmbeddedEngine.OFFSET_STORAGE)); + } + + @Test + public void testIfValidatesConfigurationCorrectly() { + final MongoDbConnectorEmbeddedDebeziumConfiguration configuration = new MongoDbConnectorEmbeddedDebeziumConfiguration(); + + configuration.setName("test_config"); + configuration.setMongodbUser("test_db"); + configuration.setMongodbName("test_server"); + configuration.setOffsetStorageFileName("/offset/file"); + configuration.setDatabaseHistoryFileFilename("/database_history/file"); + + assertFalse(configuration.validateConfiguration().isValid()); + + configuration.setMongodbHosts("localhost"); + configuration.setMongodbPassword("test_pwd"); + + assertTrue(configuration.validateConfiguration().isValid()); + } + + @Test + public void testValidateConfigurationsForAllRequiredFields() { + final MongoDbConnectorEmbeddedDebeziumConfiguration configuration = new MongoDbConnectorEmbeddedDebeziumConfiguration(); + configuration.setName("test_config"); + configuration.setMongodbUser("test_db"); + configuration.setMongodbHosts("localhost"); + configuration.setMongodbPassword("test_pwd"); + configuration.setMongodbName("test_server"); + configuration.setOffsetStorageFileName("/offset/file"); + configuration.setDatabaseHistoryFileFilename("/database_history/file"); + + final ConfigurationValidation validation = configuration.validateConfiguration(); + assertTrue(validation.isValid()); + + assertEquals("test_config", configuration.getName()); + assertEquals("test_db", configuration.getMongodbUser()); + assertEquals("localhost", configuration.getMongodbHosts()); + assertEquals("test_pwd", configuration.getMongodbPassword()); + assertEquals("test_server", configuration.getMongodbName()); + assertEquals("/offset/file", configuration.getOffsetStorageFileName()); + assertEquals("/database_history/file", configuration.getDatabaseHistoryFileFilename()); + } + +} \ No newline at end of file diff --git a/components/camel-debezium-mongodb/src/test/resources/log4j2.properties b/components/camel-debezium-mongodb/src/test/resources/log4j2.properties new file mode 100644 index 0000000..a3dd382 --- /dev/null +++ b/components/camel-debezium-mongodb/src/test/resources/log4j2.properties @@ -0,0 +1,30 @@ +## --------------------------------------------------------------------------- +## Licensed to the Apache Software Foundation (ASF) under one or more +## contributor license agreements. See the NOTICE file distributed with +## this work for additional information regarding copyright ownership. +## The ASF licenses this file to You under the Apache License, Version 2.0 +## (the "License"); you may not use this file except in compliance with +## the License. You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## --------------------------------------------------------------------------- + +appender.file.type = File +appender.file.name = file +appender.file.fileName = target/camel-debezium-test.log +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d [%-15.15t] %-5p %-30.30c{1} - %m%n + +appender.out.type = Console +appender.out.name = out +appender.out.layout.type = PatternLayout +appender.out.layout.pattern = %d [%-15.15t] %-5p %-30.30c{1} - %m%n + +rootLogger.level = INFO +rootLogger.appenderRef.file.ref = file \ No newline at end of file diff --git a/components/pom.xml b/components/pom.xml index 098d1b7..33affb5 100644 --- a/components/pom.xml +++ b/components/pom.xml @@ -160,6 +160,7 @@ <module>camel-debezium-common</module> <module>camel-debezium-mysql</module> <module>camel-debezium-postgres</module> + <module>camel-debezium-mongodb</module> <module>camel-ehcache</module> <module>camel-elasticsearch-rest</module> <module>camel-elsql</module> diff --git a/parent/pom.xml b/parent/pom.xml index db822d4..6b9c7b4 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -1189,6 +1189,11 @@ </dependency> <dependency> <groupId>org.apache.camel</groupId> + <artifactId>camel-debezium-mongodb</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.camel</groupId> <artifactId>camel-debezium-mysql</artifactId> <version>${project.version}</version> </dependency> @@ -2808,6 +2813,11 @@ </dependency> <dependency> <groupId>org.apache.camel</groupId> + <artifactId>camel-debezium-mongodb-starter</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.camel</groupId> <artifactId>camel-debezium-mysql-starter</artifactId> <version>${project.version}</version> </dependency> diff --git a/platforms/karaf/features/src/main/resources/features.xml b/platforms/karaf/features/src/main/resources/features.xml index ab27da9..46443da 100644 --- a/platforms/karaf/features/src/main/resources/features.xml +++ b/platforms/karaf/features/src/main/resources/features.xml @@ -908,6 +908,16 @@ <bundle>mvn:org.apache.camel/camel-debezium-common/${project.version}</bundle> <bundle>mvn:org.apache.camel/camel-debezium-postgres/${project.version}</bundle> </feature> + <feature name='camel-debezium-mongodb' version='${project.version}' start-level='50'> + <feature version="${project.version}">camel-core</feature> + <bundle dependency="true">wrap:mvn:io.debezium/debezium-core/${debezium-version}</bundle> + <bundle dependency="true">wrap:mvn:io.debezium/debezium-embedded/${debezium-version}</bundle> + <bundle dependency="true">wrap:mvn:io.debezium/debezium-connector-mongodb/${debezium-version}</bundle> + <bundle dependency="true">wrap:mvn:org.apache.kafka/connect-api/${kafka-version}</bundle> + <bundle dependency="true">wrap:mvn:org.apache.kafka/connect-json/${kafka-version}</bundle> + <bundle>mvn:org.apache.camel/camel-debezium-common/${project.version}</bundle> + <bundle>mvn:org.apache.camel/camel-debezium-mongodb/${project.version}</bundle> + </feature> <feature name='camel-ehcache' version='${project.version}' start-level='50'> <feature>scr</feature> <feature version="${project.version}">camel-core</feature> @@ -1216,7 +1226,7 @@ <!-- Third party dependencies --> <bundle dependency='true'>mvn:org.javassist/javassist/${javassist-version}</bundle> <bundle dependency='true'>wrap:mvn:io.perfmark/perfmark-api/${perfmark-version}</bundle> - <bundle dependency='true'>wrap:mvn:com.auth0/java-jwt/${grpc-java-jwt-version}</bundle> + <bundle dependency='true'>wrap:mvn:com.auth0/java-jwt/${grpc-java-jwt-version}</bundle> <bundle dependency='true'>wrap:mvn:com.google.auth/google-auth-library-credentials/${grpc-google-auth-library-version}</bundle> <bundle dependency='true'>wrap:mvn:com.google.auth/google-auth-library-oauth2-http/${grpc-google-auth-library-version}</bundle> <bundle dependency='true'>mvn:com.google.code.findbugs/jsr305/${google-findbugs-jsr305-version}</bundle> diff --git a/platforms/spring-boot/components-starter/camel-debezium-mongodb-starter/pom.xml b/platforms/spring-boot/components-starter/camel-debezium-mongodb-starter/pom.xml new file mode 100644 index 0000000..8876a83 --- /dev/null +++ b/platforms/spring-boot/components-starter/camel-debezium-mongodb-starter/pom.xml @@ -0,0 +1,53 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.camel</groupId> + <artifactId>components-starter</artifactId> + <version>3.0.0-SNAPSHOT</version> + </parent> + <artifactId>camel-debezium-mongodb-starter</artifactId> + <packaging>jar</packaging> + <name>Spring-Boot Starter :: Camel :: Debezium :: MongoDB</name> + <description>Spring-Boot Starter for Camel Debezium MongoDB support</description> + <dependencies> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter</artifactId> + <version>${spring-boot-version}</version> + </dependency> + <dependency> + <groupId>org.apache.camel</groupId> + <artifactId>camel-debezium-mongodb</artifactId> + <version>${project.version}</version> + </dependency> + <!--START OF GENERATED CODE--> + <dependency> + <groupId>org.apache.camel</groupId> + <artifactId>camel-core-starter</artifactId> + </dependency> + <dependency> + <groupId>org.apache.camel</groupId> + <artifactId>camel-spring-boot-starter</artifactId> + </dependency> + <!--END OF GENERATED CODE--> + </dependencies> +</project> diff --git a/platforms/spring-boot/components-starter/camel-debezium-mongodb-starter/src/main/java/org/apache/camel/component/debezium/springboot/DebeziumMongodbComponentAutoConfiguration.java b/platforms/spring-boot/components-starter/camel-debezium-mongodb-starter/src/main/java/org/apache/camel/component/debezium/springboot/DebeziumMongodbComponentAutoConfiguration.java new file mode 100644 index 0000000..aa46069 --- /dev/null +++ b/platforms/spring-boot/components-starter/camel-debezium-mongodb-starter/src/main/java/org/apache/camel/component/debezium/springboot/DebeziumMongodbComponentAutoConfiguration.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.debezium.springboot; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import javax.annotation.Generated; +import org.apache.camel.CamelContext; +import org.apache.camel.component.debezium.DebeziumMongodbComponent; +import org.apache.camel.spi.ComponentCustomizer; +import org.apache.camel.spi.HasId; +import org.apache.camel.spring.boot.CamelAutoConfiguration; +import org.apache.camel.spring.boot.ComponentConfigurationProperties; +import org.apache.camel.spring.boot.util.CamelPropertiesHelper; +import org.apache.camel.spring.boot.util.ConditionalOnCamelContextAndAutoConfigurationBeans; +import org.apache.camel.spring.boot.util.GroupCondition; +import org.apache.camel.spring.boot.util.HierarchicalPropertiesEvaluator; +import org.apache.camel.support.IntrospectionSupport; +import org.apache.camel.util.ObjectHelper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.AutoConfigureAfter; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.ApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Conditional; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Lazy; + +/** + * Generated by camel-package-maven-plugin - do not edit this file! + */ +@Generated("org.apache.camel.maven.packaging.SpringBootAutoConfigurationMojo") +@Configuration +@Conditional({ConditionalOnCamelContextAndAutoConfigurationBeans.class, + DebeziumMongodbComponentAutoConfiguration.GroupConditions.class}) +@AutoConfigureAfter(CamelAutoConfiguration.class) +@EnableConfigurationProperties({ComponentConfigurationProperties.class, + DebeziumMongodbComponentConfiguration.class}) +public class DebeziumMongodbComponentAutoConfiguration { + + private static final Logger LOGGER = LoggerFactory + .getLogger(DebeziumMongodbComponentAutoConfiguration.class); + @Autowired + private ApplicationContext applicationContext; + @Autowired + private CamelContext camelContext; + @Autowired + private DebeziumMongodbComponentConfiguration configuration; + @Autowired(required = false) + private List<ComponentCustomizer<DebeziumMongodbComponent>> customizers; + + static class GroupConditions extends GroupCondition { + public GroupConditions() { + super("camel.component", "camel.component.debezium-mongodb"); + } + } + + @Lazy + @Bean(name = "debezium-mongodb-component") + @ConditionalOnMissingBean(DebeziumMongodbComponent.class) + public DebeziumMongodbComponent configureDebeziumMongodbComponent() + throws Exception { + DebeziumMongodbComponent component = new DebeziumMongodbComponent(); + component.setCamelContext(camelContext); + Map<String, Object> parameters = new HashMap<>(); + IntrospectionSupport.getProperties(configuration, parameters, null, + false); + for (Map.Entry<String, Object> entry : parameters.entrySet()) { + Object value = entry.getValue(); + Class<?> paramClass = value.getClass(); + if (paramClass.getName().endsWith("NestedConfiguration")) { + Class nestedClass = null; + try { + nestedClass = (Class) paramClass.getDeclaredField( + "CAMEL_NESTED_CLASS").get(null); + HashMap<String, Object> nestedParameters = new HashMap<>(); + IntrospectionSupport.getProperties(value, nestedParameters, + null, false); + Object nestedProperty = nestedClass.newInstance(); + CamelPropertiesHelper.setCamelProperties(camelContext, + nestedProperty, nestedParameters, false); + entry.setValue(nestedProperty); + } catch (NoSuchFieldException e) { + } + } + } + CamelPropertiesHelper.setCamelProperties(camelContext, component, + parameters, false); + if (ObjectHelper.isNotEmpty(customizers)) { + for (ComponentCustomizer<DebeziumMongodbComponent> customizer : customizers) { + boolean useCustomizer = (customizer instanceof HasId) + ? HierarchicalPropertiesEvaluator.evaluate( + applicationContext.getEnvironment(), + "camel.component.customizer", + "camel.component.debezium-mongodb.customizer", + ((HasId) customizer).getId()) + : HierarchicalPropertiesEvaluator.evaluate( + applicationContext.getEnvironment(), + "camel.component.customizer", + "camel.component.debezium-mongodb.customizer"); + if (useCustomizer) { + LOGGER.debug("Configure component {}, with customizer {}", + component, customizer); + customizer.customize(component); + } + } + } + return component; + } +} \ No newline at end of file diff --git a/platforms/spring-boot/components-starter/camel-debezium-mongodb-starter/src/main/java/org/apache/camel/component/debezium/springboot/DebeziumMongodbComponentConfiguration.java b/platforms/spring-boot/components-starter/camel-debezium-mongodb-starter/src/main/java/org/apache/camel/component/debezium/springboot/DebeziumMongodbComponentConfiguration.java new file mode 100644 index 0000000..e029aa2 --- /dev/null +++ b/platforms/spring-boot/components-starter/camel-debezium-mongodb-starter/src/main/java/org/apache/camel/component/debezium/springboot/DebeziumMongodbComponentConfiguration.java @@ -0,0 +1,566 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.debezium.springboot; + +import javax.annotation.Generated; +import org.apache.camel.spring.boot.ComponentConfigurationPropertiesCommon; +import org.springframework.boot.context.properties.ConfigurationProperties; + +/** + * Represents a Debezium MongoDB endpoint which is used to capture changes in + * MongoDB database so that that applications can see those changes and respond + * to them. + * + * Generated by camel-package-maven-plugin - do not edit this file! + */ +@Generated("org.apache.camel.maven.packaging.SpringBootAutoConfigurationMojo") +@ConfigurationProperties(prefix = "camel.component.debezium-mongodb") +public class DebeziumMongodbComponentConfiguration + extends + ComponentConfigurationPropertiesCommon { + + /** + * Whether to enable auto configuration of the debezium-mongodb component. + * This is enabled by default. + */ + private Boolean enabled; + /** + * Allow pre-configured Configurations to be set. + */ + private MongoDbConnectorEmbeddedDebeziumConfigurationNestedConfiguration configuration; + /** + * Whether the component should use basic property binding (Camel 2.x) or + * the newer property binding with additional capabilities + */ + private Boolean basicPropertyBinding = false; + + public MongoDbConnectorEmbeddedDebeziumConfigurationNestedConfiguration getConfiguration() { + return configuration; + } + + public void setConfiguration( + MongoDbConnectorEmbeddedDebeziumConfigurationNestedConfiguration configuration) { + this.configuration = configuration; + } + + public Boolean getBasicPropertyBinding() { + return basicPropertyBinding; + } + + public void setBasicPropertyBinding(Boolean basicPropertyBinding) { + this.basicPropertyBinding = basicPropertyBinding; + } + + public static class MongoDbConnectorEmbeddedDebeziumConfigurationNestedConfiguration { + public static final Class CAMEL_NESTED_CLASS = org.apache.camel.component.debezium.configuration.MongoDbConnectorEmbeddedDebeziumConfiguration.class; + /** + * Maximum size of the queue for change events read from the database + * log but not yet recorded or forwarded. Defaults to 8192, and should + * always be larger than the maximum batch size. + */ + private Integer maxQueueSize = 8192; + /** + * Password to be used when connecting to MongoDB, if necessary. + */ + private String mongodbPassword; + /** + * Maximum number of threads used to perform an intial sync of the + * collections in a replica set. Defaults to 1. + */ + private Integer initialSyncMaxThreads = 1; + /** + * The number of milliseconds to delay before a snapshot will begin. + */ + private Long snapshotDelayMs = 0L; + /** + * Description is not available here, please check Debezium website for + * corresponding key 'collection.blacklist' description. + */ + private String collectionBlacklist; + /** + * The collections for which changes are to be captured + */ + private String collectionWhitelist; + /** + * Should connector use SSL to connect to MongoDB instances + */ + private Boolean mongodbSslEnabled = false; + /** + * Whether delete operations should be represented by a delete event and + * a subsquenttombstone event (true) or only by a delete event (false). + * Emitting the tombstone event (the default behavior) allows Kafka to + * completely delete all events pertaining to the given key once the + * source record got deleted. + */ + private Boolean tombstonesOnDelete = false; + /** + * Specifies whether the addresses in 'hosts' are seeds that should be + * used to discover all members of the cluster or replica set ('true'), + * or whether the address(es) in 'hosts' should be used as is ('false'). + * The default is 'true'. + */ + private Boolean mongodbMembersAutoDiscover = true; + /** + * Description is not available here, please check Debezium website for + * corresponding key 'field.renames' description. + */ + private String fieldRenames; + /** + * Frequency in milliseconds to wait for new change events to appear + * after receiving no events. Defaults to 500ms. + */ + private Long pollIntervalMs = 500L; + /** + * The databases for which changes are to be captured + */ + private String databaseWhitelist; + /** + * The hostname and port pairs (in the form 'host' or 'host:port') of + * the MongoDB server(s) in the replica set. + */ + private String mongodbHosts; + /** + * The initial delay when trying to reconnect to a primary after a + * connection cannot be made or when no primary is available. Defaults + * to 1 second (1000 ms). + */ + private Long connectBackoffInitialDelayMs = 1000L; + /** + * The maximum number of records that should be loaded into memory while + * performing a snapshot + */ + private Integer snapshotFetchSize; + /** + * The path to the file that will be used to record the database history + */ + private String databaseHistoryFileFilename; + /** + * Maximum number of failed connection attempts to a replica set primary + * before an exception occurs and task is aborted. Defaults to 16, which + * with the defaults for 'connect.backoff.initial.delay.ms' and + * 'connect.backoff.max.delay.ms' results in just over 20 minutes of + * attempts before failing. + */ + private Integer connectMaxAttempts = 16; + /** + * Database user for connecting to MongoDB, if necessary. + */ + private String mongodbUser; + /** + * Description is not available here, please check Debezium website for + * corresponding key 'field.blacklist' description. + */ + private String fieldBlacklist; + /** + * Unique name that identifies the MongoDB replica set or cluster and + * all recorded offsets, andthat is used as a prefix for all schemas and + * topics. Each distinct MongoDB installation should have a separate + * namespace and monitored by at most one Debezium connector. + */ + private String mongodbName; + /** + * The maximum delay when trying to reconnect to a primary after a + * connection cannot be made or when no primary is available. Defaults + * to 120 second (120,000 ms). + */ + private Long connectBackoffMaxDelayMs = 120000L; + /** + * Whether invalid host names are allowed when using SSL. If true the + * connection will not prevent man-in-the-middle attacks + */ + private Boolean mongodbSslInvalidHostnameAllowed = false; + /** + * The databases for which changes are to be excluded + */ + private String databaseBlacklist; + /** + * Maximum size of each batch of source records. Defaults to 2048. + */ + private Integer maxBatchSize = 2048; + /** + * The criteria for running a snapshot upon startup of the connector. + * Options include: 'initial' (the default) to specify the connector + * should always perform an initial sync when required; 'never' to + * specify the connector should never perform an initial sync + */ + private String snapshotMode = "initial"; + /** + * The name of the Java class for the connector + */ + private Class connectorClass; + /** + * Unique name for the connector. Attempting to register again with the + * same name will fail. + */ + private String name; + /** + * The name of the Java class that is responsible for persistence of + * connector offsets. + */ + private String offsetStorage = "org.apache.kafka.connect.storage.FileOffsetBackingStore"; + /** + * Path to file where offsets are to be stored. Required when + * offset.storage is set to the FileOffsetBackingStore + */ + private String offsetStorageFileName; + /** + * The name of the Kafka topic where offsets are to be stored. Required + * when offset.storage is set to the KafkaOffsetBackingStore. + */ + private String offsetStorageTopic; + /** + * Replication factor used when creating the offset storage topic. + * Required when offset.storage is set to the KafkaOffsetBackingStore + */ + private Integer offsetStorageReplicationFactor; + /** + * The name of the Java class of the commit policy. It defines when + * offsets commit has to be triggered based on the number of events + * processed and the time elapsed since the last commit. This class must + * implement the interface 'OffsetCommitPolicy'. The default is a + * periodic commit policy based upon time intervals. + */ + private String offsetCommitPolicy = "io.debezium.embedded.spi.OffsetCommitPolicy.PeriodicCommitOffsetPolicy"; + /** + * Interval at which to try committing offsets. The default is 1 minute. + */ + private Long offsetFlushIntervalMs = 60000L; + /** + * Maximum number of milliseconds to wait for records to flush and + * partition offset data to be committed to offset storage before + * cancelling the process and restoring the offset data to be committed + * in a future attempt. The default is 5 seconds. + */ + private Long offsetCommitTimeoutMs = 5000L; + /** + * The number of partitions used when creating the offset storage topic. + * Required when offset.storage is set to the 'KafkaOffsetBackingStore'. + */ + private Integer offsetStoragePartitions; + /** + * The Converter class that should be used to serialize and deserialize + * key data for offsets. The default is JSON converter. + */ + private String internalKeyConverter = "org.apache.kafka.connect.json.JsonConverter"; + /** + * The Converter class that should be used to serialize and deserialize + * value data for offsets. The default is JSON converter. + */ + private String internalValueConverter = "org.apache.kafka.connect.json.JsonConverter"; + + public Integer getMaxQueueSize() { + return maxQueueSize; + } + + public void setMaxQueueSize(Integer maxQueueSize) { + this.maxQueueSize = maxQueueSize; + } + + public String getMongodbPassword() { + return mongodbPassword; + } + + public void setMongodbPassword(String mongodbPassword) { + this.mongodbPassword = mongodbPassword; + } + + public Integer getInitialSyncMaxThreads() { + return initialSyncMaxThreads; + } + + public void setInitialSyncMaxThreads(Integer initialSyncMaxThreads) { + this.initialSyncMaxThreads = initialSyncMaxThreads; + } + + public Long getSnapshotDelayMs() { + return snapshotDelayMs; + } + + public void setSnapshotDelayMs(Long snapshotDelayMs) { + this.snapshotDelayMs = snapshotDelayMs; + } + + public String getCollectionBlacklist() { + return collectionBlacklist; + } + + public void setCollectionBlacklist(String collectionBlacklist) { + this.collectionBlacklist = collectionBlacklist; + } + + public String getCollectionWhitelist() { + return collectionWhitelist; + } + + public void setCollectionWhitelist(String collectionWhitelist) { + this.collectionWhitelist = collectionWhitelist; + } + + public Boolean getMongodbSslEnabled() { + return mongodbSslEnabled; + } + + public void setMongodbSslEnabled(Boolean mongodbSslEnabled) { + this.mongodbSslEnabled = mongodbSslEnabled; + } + + public Boolean getTombstonesOnDelete() { + return tombstonesOnDelete; + } + + public void setTombstonesOnDelete(Boolean tombstonesOnDelete) { + this.tombstonesOnDelete = tombstonesOnDelete; + } + + public Boolean getMongodbMembersAutoDiscover() { + return mongodbMembersAutoDiscover; + } + + public void setMongodbMembersAutoDiscover( + Boolean mongodbMembersAutoDiscover) { + this.mongodbMembersAutoDiscover = mongodbMembersAutoDiscover; + } + + public String getFieldRenames() { + return fieldRenames; + } + + public void setFieldRenames(String fieldRenames) { + this.fieldRenames = fieldRenames; + } + + public Long getPollIntervalMs() { + return pollIntervalMs; + } + + public void setPollIntervalMs(Long pollIntervalMs) { + this.pollIntervalMs = pollIntervalMs; + } + + public String getDatabaseWhitelist() { + return databaseWhitelist; + } + + public void setDatabaseWhitelist(String databaseWhitelist) { + this.databaseWhitelist = databaseWhitelist; + } + + public String getMongodbHosts() { + return mongodbHosts; + } + + public void setMongodbHosts(String mongodbHosts) { + this.mongodbHosts = mongodbHosts; + } + + public Long getConnectBackoffInitialDelayMs() { + return connectBackoffInitialDelayMs; + } + + public void setConnectBackoffInitialDelayMs( + Long connectBackoffInitialDelayMs) { + this.connectBackoffInitialDelayMs = connectBackoffInitialDelayMs; + } + + public Integer getSnapshotFetchSize() { + return snapshotFetchSize; + } + + public void setSnapshotFetchSize(Integer snapshotFetchSize) { + this.snapshotFetchSize = snapshotFetchSize; + } + + public String getDatabaseHistoryFileFilename() { + return databaseHistoryFileFilename; + } + + public void setDatabaseHistoryFileFilename( + String databaseHistoryFileFilename) { + this.databaseHistoryFileFilename = databaseHistoryFileFilename; + } + + public Integer getConnectMaxAttempts() { + return connectMaxAttempts; + } + + public void setConnectMaxAttempts(Integer connectMaxAttempts) { + this.connectMaxAttempts = connectMaxAttempts; + } + + public String getMongodbUser() { + return mongodbUser; + } + + public void setMongodbUser(String mongodbUser) { + this.mongodbUser = mongodbUser; + } + + public String getFieldBlacklist() { + return fieldBlacklist; + } + + public void setFieldBlacklist(String fieldBlacklist) { + this.fieldBlacklist = fieldBlacklist; + } + + public String getMongodbName() { + return mongodbName; + } + + public void setMongodbName(String mongodbName) { + this.mongodbName = mongodbName; + } + + public Long getConnectBackoffMaxDelayMs() { + return connectBackoffMaxDelayMs; + } + + public void setConnectBackoffMaxDelayMs(Long connectBackoffMaxDelayMs) { + this.connectBackoffMaxDelayMs = connectBackoffMaxDelayMs; + } + + public Boolean getMongodbSslInvalidHostnameAllowed() { + return mongodbSslInvalidHostnameAllowed; + } + + public void setMongodbSslInvalidHostnameAllowed( + Boolean mongodbSslInvalidHostnameAllowed) { + this.mongodbSslInvalidHostnameAllowed = mongodbSslInvalidHostnameAllowed; + } + + public String getDatabaseBlacklist() { + return databaseBlacklist; + } + + public void setDatabaseBlacklist(String databaseBlacklist) { + this.databaseBlacklist = databaseBlacklist; + } + + public Integer getMaxBatchSize() { + return maxBatchSize; + } + + public void setMaxBatchSize(Integer maxBatchSize) { + this.maxBatchSize = maxBatchSize; + } + + public String getSnapshotMode() { + return snapshotMode; + } + + public void setSnapshotMode(String snapshotMode) { + this.snapshotMode = snapshotMode; + } + + public Class getConnectorClass() { + return connectorClass; + } + + public void setConnectorClass(Class connectorClass) { + this.connectorClass = connectorClass; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getOffsetStorage() { + return offsetStorage; + } + + public void setOffsetStorage(String offsetStorage) { + this.offsetStorage = offsetStorage; + } + + public String getOffsetStorageFileName() { + return offsetStorageFileName; + } + + public void setOffsetStorageFileName(String offsetStorageFileName) { + this.offsetStorageFileName = offsetStorageFileName; + } + + public String getOffsetStorageTopic() { + return offsetStorageTopic; + } + + public void setOffsetStorageTopic(String offsetStorageTopic) { + this.offsetStorageTopic = offsetStorageTopic; + } + + public Integer getOffsetStorageReplicationFactor() { + return offsetStorageReplicationFactor; + } + + public void setOffsetStorageReplicationFactor( + Integer offsetStorageReplicationFactor) { + this.offsetStorageReplicationFactor = offsetStorageReplicationFactor; + } + + public String getOffsetCommitPolicy() { + return offsetCommitPolicy; + } + + public void setOffsetCommitPolicy(String offsetCommitPolicy) { + this.offsetCommitPolicy = offsetCommitPolicy; + } + + public Long getOffsetFlushIntervalMs() { + return offsetFlushIntervalMs; + } + + public void setOffsetFlushIntervalMs(Long offsetFlushIntervalMs) { + this.offsetFlushIntervalMs = offsetFlushIntervalMs; + } + + public Long getOffsetCommitTimeoutMs() { + return offsetCommitTimeoutMs; + } + + public void setOffsetCommitTimeoutMs(Long offsetCommitTimeoutMs) { + this.offsetCommitTimeoutMs = offsetCommitTimeoutMs; + } + + public Integer getOffsetStoragePartitions() { + return offsetStoragePartitions; + } + + public void setOffsetStoragePartitions(Integer offsetStoragePartitions) { + this.offsetStoragePartitions = offsetStoragePartitions; + } + + public String getInternalKeyConverter() { + return internalKeyConverter; + } + + public void setInternalKeyConverter(String internalKeyConverter) { + this.internalKeyConverter = internalKeyConverter; + } + + public String getInternalValueConverter() { + return internalValueConverter; + } + + public void setInternalValueConverter(String internalValueConverter) { + this.internalValueConverter = internalValueConverter; + } + } +} \ No newline at end of file diff --git a/platforms/spring-boot/components-starter/camel-debezium-mongodb-starter/src/main/resources/META-INF/LICENSE.txt b/platforms/spring-boot/components-starter/camel-debezium-mongodb-starter/src/main/resources/META-INF/LICENSE.txt new file mode 100644 index 0000000..6b0b127 --- /dev/null +++ b/platforms/spring-boot/components-starter/camel-debezium-mongodb-starter/src/main/resources/META-INF/LICENSE.txt @@ -0,0 +1,203 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/platforms/spring-boot/components-starter/camel-debezium-mongodb-starter/src/main/resources/META-INF/NOTICE.txt b/platforms/spring-boot/components-starter/camel-debezium-mongodb-starter/src/main/resources/META-INF/NOTICE.txt new file mode 100644 index 0000000..2e215bf --- /dev/null +++ b/platforms/spring-boot/components-starter/camel-debezium-mongodb-starter/src/main/resources/META-INF/NOTICE.txt @@ -0,0 +1,11 @@ + ========================================================================= + == NOTICE file corresponding to the section 4 d of == + == the Apache License, Version 2.0, == + == in this case for the Apache Camel distribution. == + ========================================================================= + + This product includes software developed by + The Apache Software Foundation (http://www.apache.org/). + + Please read the different LICENSE files present in the licenses directory of + this distribution. diff --git a/platforms/spring-boot/components-starter/camel-debezium-mongodb-starter/src/main/resources/META-INF/spring.factories b/platforms/spring-boot/components-starter/camel-debezium-mongodb-starter/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000..09fcee7 --- /dev/null +++ b/platforms/spring-boot/components-starter/camel-debezium-mongodb-starter/src/main/resources/META-INF/spring.factories @@ -0,0 +1,19 @@ +## --------------------------------------------------------------------------- +## Licensed to the Apache Software Foundation (ASF) under one or more +## contributor license agreements. See the NOTICE file distributed with +## this work for additional information regarding copyright ownership. +## The ASF licenses this file to You under the Apache License, Version 2.0 +## (the "License"); you may not use this file except in compliance with +## the License. You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## --------------------------------------------------------------------------- + +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ +org.apache.camel.component.debezium.springboot.DebeziumMongodbComponentAutoConfiguration diff --git a/platforms/spring-boot/components-starter/camel-debezium-mongodb-starter/src/main/resources/META-INF/spring.provides b/platforms/spring-boot/components-starter/camel-debezium-mongodb-starter/src/main/resources/META-INF/spring.provides new file mode 100644 index 0000000..e8b1c93 --- /dev/null +++ b/platforms/spring-boot/components-starter/camel-debezium-mongodb-starter/src/main/resources/META-INF/spring.provides @@ -0,0 +1,17 @@ +## --------------------------------------------------------------------------- +## Licensed to the Apache Software Foundation (ASF) under one or more +## contributor license agreements. See the NOTICE file distributed with +## this work for additional information regarding copyright ownership. +## The ASF licenses this file to You under the Apache License, Version 2.0 +## (the "License"); you may not use this file except in compliance with +## the License. You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## --------------------------------------------------------------------------- +provides: camel-debezium-mongodb diff --git a/platforms/spring-boot/components-starter/pom.xml b/platforms/spring-boot/components-starter/pom.xml index 0825962..15bd19d 100644 --- a/platforms/spring-boot/components-starter/pom.xml +++ b/platforms/spring-boot/components-starter/pom.xml @@ -169,6 +169,7 @@ <module>camel-dataformat-starter</module> <module>camel-dataset-starter</module> <module>camel-debezium-common-starter</module> + <module>camel-debezium-mongodb-starter</module> <module>camel-debezium-mysql-starter</module> <module>camel-debezium-postgres-starter</module> <module>camel-digitalocean-starter</module> diff --git a/platforms/spring-boot/spring-boot-dm/camel-spring-boot-dependencies/pom.xml b/platforms/spring-boot/spring-boot-dm/camel-spring-boot-dependencies/pom.xml index 441296c..3364576 100644 --- a/platforms/spring-boot/spring-boot-dm/camel-spring-boot-dependencies/pom.xml +++ b/platforms/spring-boot/spring-boot-dm/camel-spring-boot-dependencies/pom.xml @@ -1059,6 +1059,16 @@ </dependency> <dependency> <groupId>org.apache.camel</groupId> + <artifactId>camel-debezium-mongodb</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.camel</groupId> + <artifactId>camel-debezium-mongodb-starter</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.camel</groupId> <artifactId>camel-debezium-mysql</artifactId> <version>${project.version}</version> </dependency> diff --git a/tests/camel-itest-karaf/src/test/java/org/apache/camel/itest/karaf/CamelDebeziumMongodbTest.java b/tests/camel-itest-karaf/src/test/java/org/apache/camel/itest/karaf/CamelDebeziumMongodbTest.java new file mode 100644 index 0000000..852e32a --- /dev/null +++ b/tests/camel-itest-karaf/src/test/java/org/apache/camel/itest/karaf/CamelDebeziumMongodbTest.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.itest.karaf; + +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.ops4j.pax.exam.junit.PaxExam; + +@Ignore +@RunWith(PaxExam.class) +public class CamelDebeziumMongodbTest extends BaseKarafTest { + + public static final String COMPONENT = extractName(CamelDebeziumMongodbTest.class); + + @Test + public void test() throws Exception { + testComponent(COMPONENT); + } + +} diff --git a/tests/camel-itest-spring-boot/src/test/java/org/apache/camel/itest/springboot/CamelDebeziumMongodbTest.java b/tests/camel-itest-spring-boot/src/test/java/org/apache/camel/itest/springboot/CamelDebeziumMongodbTest.java new file mode 100644 index 0000000..5fa0a8e --- /dev/null +++ b/tests/camel-itest-spring-boot/src/test/java/org/apache/camel/itest/springboot/CamelDebeziumMongodbTest.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.itest.springboot; + +import org.apache.camel.itest.springboot.util.ArquillianPackager; +import org.jboss.arquillian.container.test.api.Deployment; +import org.jboss.arquillian.junit.Arquillian; +import org.jboss.shrinkwrap.api.Archive; +import org.junit.Test; +import org.junit.runner.RunWith; + + +@RunWith(Arquillian.class) +public class CamelDebeziumMongodbTest extends AbstractSpringBootTestSupport { + + @Deployment + public static Archive<?> createSpringBootPackage() throws Exception { + return ArquillianPackager.springBootPackage(createTestConfig()); + } + + public static ITestConfig createTestConfig() { + return new ITestConfigBuilder() + .module(inferModuleName(CamelDebeziumMongodbTest.class)) + .build(); + } + + @Test + public void componentTests() throws Exception { + this.runComponentTest(config); + this.runModuleUnitTestsIfEnabled(config); + } +}