[
https://issues.apache.org/jira/browse/BEAM-5107?focusedWorklogId=134252&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-134252
]
ASF GitHub Bot logged work on BEAM-5107:
----------------------------------------
Author: ASF GitHub Bot
Created on: 13/Aug/18 19:12
Start Date: 13/Aug/18 19:12
Worklog Time Spent: 10m
Work Description: timrobertson100 commented on a change in pull request
#6211: [BEAM-5107] Support ES-6.x for ElasticsearchIO
URL: https://github.com/apache/beam/pull/6211#discussion_r209725489
##########
File path:
sdks/java/io/elasticsearch-tests/elasticsearch-tests-6/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOIT.java
##########
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.elasticsearch;
+
+import static
org.apache.beam.sdk.io.elasticsearch.ElasticsearchIO.BoundedElasticsearchSource;
+import static
org.apache.beam.sdk.io.elasticsearch.ElasticsearchIO.ConnectionConfiguration;
+import static org.apache.beam.sdk.io.elasticsearch.ElasticsearchIO.Read;
+import static org.apache.beam.sdk.testing.SourceTestUtils.readFromSource;
+import static org.junit.Assert.assertEquals;
+
+import java.util.List;
+import org.apache.beam.sdk.io.BoundedSource;
+import
org.apache.beam.sdk.io.elasticsearch.ElasticsearchIOITCommon.ElasticsearchPipelineOptions;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.testing.SourceTestUtils;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.elasticsearch.client.RestClient;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+
+/**
+ * A test of {@link ElasticsearchIO} on an independent Elasticsearch v6.x
instance.
+ *
+ * <p>This test requires a running instance of Elasticsearch, and the test
dataset must exist in the
+ * database. See {@link ElasticsearchIOITCommon} for instructions to achieve
this.
+ *
+ * <p>You can run this test by doing the following from the beam parent module
directory with the
+ * correct server IP:
+ *
+ * <pre>
+ * ./gradlew integrationTest -p
sdks/java/io/elasticsearch-tests/elasticsearch-tests-6
+ * -DintegrationTestPipelineOptions='[
+ * "--elasticsearchServer=1.2.3.4",
+ * "--elasticsearchHttpPort=9200"]'
+ * --tests org.apache.beam.sdk.io.elasticsearch.ElasticsearchIOIT
+ * -DintegrationTestRunner=direct
+ * </pre>
+ *
+ * <p>It is likely that you will need to configure
<code>thread_pool.bulk.queue_size: 250</code> (or
+ * higher) in the backend Elasticsearch server for this test to run.
+ */
+public class ElasticsearchIOIT {
+ private static RestClient restClient;
+ private static ElasticsearchPipelineOptions options;
+ private static ConnectionConfiguration readConnectionConfiguration;
+ private static ConnectionConfiguration writeConnectionConfiguration;
+ private static ConnectionConfiguration updateConnectionConfiguration;
+ private static ElasticsearchIOTestCommon elasticsearchIOTestCommon;
+
+ @Rule public TestPipeline pipeline = TestPipeline.create();
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ PipelineOptionsFactory.register(ElasticsearchPipelineOptions.class);
+ options =
TestPipeline.testingPipelineOptions().as(ElasticsearchPipelineOptions.class);
+ readConnectionConfiguration =
+ ElasticsearchIOITCommon.getConnectionConfiguration(
+ options, ElasticsearchIOITCommon.IndexMode.READ);
+ writeConnectionConfiguration =
+ ElasticsearchIOITCommon.getConnectionConfiguration(
+ options, ElasticsearchIOITCommon.IndexMode.WRITE);
+ updateConnectionConfiguration =
+ ElasticsearchIOITCommon.getConnectionConfiguration(
+ options, ElasticsearchIOITCommon.IndexMode.WRITE_PARTIAL);
+ restClient = readConnectionConfiguration.createClient();
+ elasticsearchIOTestCommon =
+ new ElasticsearchIOTestCommon(readConnectionConfiguration, restClient,
true);
+ }
+
+ @AfterClass
+ public static void afterClass() throws Exception {
+ ElasticSearchIOTestUtils.deleteIndex(writeConnectionConfiguration,
restClient);
+ ElasticSearchIOTestUtils.deleteIndex(updateConnectionConfiguration,
restClient);
+ restClient.close();
+ }
+
+ @Test
+ public void testSplitsVolume() throws Exception {
+ Read read =
ElasticsearchIO.read().withConnectionConfiguration(readConnectionConfiguration);
+ BoundedElasticsearchSource initialSource =
+ new BoundedElasticsearchSource(read, null, null, null);
+ int desiredBundleSizeBytes = 10000;
+ List<? extends BoundedSource<String>> splits =
+ initialSource.split(desiredBundleSizeBytes, options);
+ SourceTestUtils.assertSourcesEqualReferenceSource(initialSource, splits,
options);
+ long indexSize =
BoundedElasticsearchSource.estimateIndexSize(readConnectionConfiguration);
+ float expectedNumSourcesFloat = (float) indexSize / desiredBundleSizeBytes;
+ int expectedNumSources = (int) Math.ceil(expectedNumSourcesFloat);
+ assertEquals(expectedNumSources, splits.size());
+ int nonEmptySplits = 0;
+ for (BoundedSource<String> subSource : splits) {
+ if (readFromSource(subSource, options).size() > 0) {
+ nonEmptySplits += 1;
+ }
+ }
+ assertEquals("Wrong number of empty splits", expectedNumSources,
nonEmptySplits);
+ }
Review comment:
This looks like a direct copy from ES5. Would it be possible to move to a
the common to avoid repetition?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
-------------------
Worklog Id: (was: 134252)
Time Spent: 3h 20m (was: 3h 10m)
> Support ES 6.x for ElasticsearchIO
> ----------------------------------
>
> Key: BEAM-5107
> URL: https://issues.apache.org/jira/browse/BEAM-5107
> Project: Beam
> Issue Type: Improvement
> Components: io-java-elasticsearch
> Reporter: Dat Tran
> Assignee: Etienne Chauchot
> Priority: Major
> Time Spent: 3h 20m
> Remaining Estimate: 0h
>
> Elasticsearch has released 6.3.2 but ElasticsearchIO only supports 2x-5.x.
> We should support ES 6.x for ElasticsearchIO.
> https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html
> https://github.com/apache/beam/blob/master/sdks/java/io/elasticsearch/src/main/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIO.java
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
