[ https://issues.apache.org/jira/browse/BEAM-9304?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Knut Olav Loite updated BEAM-9304: ---------------------------------- Attachment: SpannerRead.java > beam-sdks-java-io-google-cloud-platform imports conflicting versions for > BigTable and Spanner > --------------------------------------------------------------------------------------------- > > Key: BEAM-9304 > URL: https://issues.apache.org/jira/browse/BEAM-9304 > Project: Beam > Issue Type: Bug > Components: io-java-gcp > Affects Versions: 2.18.0 > Reporter: Knut Olav Loite > Priority: Minor > Attachments: SpannerRead.java, pom.xml > > > If I include `beam-sdks-java-io-google-cloud-platform` version 2.18.0 in a > project and try to use `SpannerIO`, the exception > `java.lang.NoClassDefFoundError: io/opencensus/trace/Tracestate`. This seems > to be caused by conflicting versions of `io.opencensus:opencensus-api` being > included by the BigTable client and the Spanner client. BigTable imports > version 0.15.0. Spanner depends on 0.18.0, but as they are at the same level > in the dependency tree and BigTable is defined first, version 0.15.0 is used. > > The workaround for this issue is to exclude the BigTable client in the > project pom in order to be able to use SpannerIO. > > An example pom and simple Java class are listed below. If the commented > exclusion of the BigTable client is removed, the example will run without > problems. The example will also run without problems on Beam version 2.17 > without the exclusion. > > --- POM File --- > <project xmlns="http://maven.apache.org/POM/4.0.0" > xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" > xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 > http://maven.apache.org/xsd/maven-4.0.0.xsd"> > <modelVersion>4.0.0</modelVersion> > <groupId>com.google.cloud</groupId> > <artifactId>beam-sdk-test</artifactId> > <version>0.0.1-SNAPSHOT</version> > <name>Beam SDK Test</name> > <properties> > <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> > <java.version>1.8</java.version> > <maven.compiler.source>1.8</maven.compiler.source> > <maven.compiler.target>1.8</maven.compiler.target> > <apache_beam.version>2.18.0</apache_beam.version> > </properties> > > <dependencies> > <dependency> > <groupId>org.apache.beam</groupId> > <artifactId>beam-sdks-java-io-google-cloud-platform</artifactId> > <version>${apache_beam.version}</version> > <!-- > <exclusions> > <exclusion> > <groupId>com.google.cloud.bigtable</groupId> > <artifactId>bigtable-client-core</artifactId> > </exclusion> > </exclusions> > --> > </dependency> > <dependency> > <groupId>org.apache.beam</groupId> > <artifactId>beam-runners-direct-java</artifactId> > <version>${apache_beam.version}</version> > </dependency> > </dependencies> > </project> > --- JAVA FILE --- > /* > * Copyright 2017 Google Inc. > * > * Licensed under the Apache License, Version 2.0 (the "License"); > * you may not use this file except in compliance with the License. > * You may obtain a copy of the License at > * > * http://www.apache.org/licenses/LICENSE-2.0 > * > * Unless required by applicable law or agreed to in writing, software > * distributed under the License is distributed on an "AS IS" BASIS, > * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. > * See the License for the specific language governing permissions and > * limitations under the License. > */ > package com.google.cloud.beamsdk.test; > import com.google.cloud.spanner.Struct; > import org.apache.beam.sdk.Pipeline; > import org.apache.beam.sdk.io.TextIO; > import org.apache.beam.sdk.io.gcp.spanner.SpannerIO; > import org.apache.beam.sdk.options.Description; > import org.apache.beam.sdk.options.PipelineOptions; > import org.apache.beam.sdk.options.PipelineOptionsFactory; > import org.apache.beam.sdk.options.Validation; > import org.apache.beam.sdk.transforms.Count; > import org.apache.beam.sdk.transforms.Sum; > import org.apache.beam.sdk.transforms.ToString; > import org.apache.beam.sdk.values.PCollection; > /* > This sample demonstrates how to read from a Spanner table. > ## Prerequisites > * Maven installed > * Set up GCP default credentials, one of the following: > - export GOOGLE_APPLICATION_CREDENTIALS=path/to/credentials.json > - gcloud auth application-default login > > [https://developers.google.com/identity/protocols/application-default-credentials] > * Create the Spanner table to read from, you'll need: > - Instance ID > - Database ID > - Any table, preferably populated > [https://cloud.google.com/spanner/docs/quickstart-console] > ## How to run > mvn clean > mvn compile > mvn exec:java \ > -Dexec.mainClass=com.example.dataflow.SpannerRead \ > -Dexec.args="--instanceId=my-instance-id \ > --databaseId=my-database-id \ > --table=my_table \ > --output=path/to/output_file" > */ > public class SpannerRead { > public interface Options extends PipelineOptions { > @Description("Spanner instance ID to query from") > @Validation.Required > String getInstanceId(); > void setInstanceId(String value); > @Description("Spanner database name to query from") > @Validation.Required > String getDatabaseId(); > void setDatabaseId(String value); > @Description("Spanner table name to query from") > @Validation.Required > String getTable(); > void setTable(String value); > @Description("Output filename for row count") > @Validation.Required > String getOutput(); > void setOutput(String value); > } > public static void main(String[] args) { > Options options = > PipelineOptionsFactory.fromArgs(args).withValidation().as(Options.class); > Pipeline p = Pipeline.create(options); > String instanceId = options.getInstanceId(); > String databaseId = options.getDatabaseId(); > PCollection<Struct> records = p.apply( > SpannerIO.read() > .withInstanceId(instanceId) > .withDatabaseId(databaseId) > .withQuery("SELECT * FROM " + options.getTable())); > PCollection<Long> rowCount = records > .apply(Count.globally()) > .apply(Sum.longsGlobally()); > // Write the row count to a file > rowCount > .apply(ToString.elements()) > .apply(TextIO.write().to(options.getOutput()).withoutSharding()); > p.run().waitUntilFinish(); > } > } -- This message was sent by Atlassian Jira (v8.3.4#803005)