[CARBONDATA-2369] updated the document about AVRO to carbon schema converter
updated the document about AVRO to carbon schema converter This closes #2296 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/443b717d Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/443b717d Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/443b717d Branch: refs/heads/spark-2.3 Commit: 443b717d20e0eedf8be75dae177d573e53c050fc Parents: d85fb72 Author: ajantha-bhat <ajanthab...@gmail.com> Authored: Thu May 10 21:03:28 2018 +0530 Committer: kumarvishal09 <kumarvishal1...@gmail.com> Committed: Fri May 11 15:47:17 2018 +0530 ---------------------------------------------------------------------- README.md | 1 + docs/sdk-writer-guide.md | 25 ++++++++++++++++++------- 2 files changed, 19 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/443b717d/README.md ---------------------------------------------------------------------- diff --git a/README.md b/README.md index 3f45917..4b4577e 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ CarbonData is built using Apache Maven, to [build CarbonData](https://github.com * [Cluster Installation and Deployment](https://github.com/apache/carbondata/blob/master/docs/installation-guide.md) * [Configuring Carbondata](https://github.com/apache/carbondata/blob/master/docs/configuration-parameters.md) * [Streaming Ingestion](https://github.com/apache/carbondata/blob/master/docs/streaming-guide.md) +* [SDK Writer Guide](https://github.com/apache/carbondata/blob/master/docs/sdk-writer-guide.md) * [CarbonData Pre-aggregate DataMap](https://github.com/apache/carbondata/blob/master/docs/datamap/preaggregate-datamap-guide.md) * [CarbonData Timeseries DataMap](https://github.com/apache/carbondata/blob/master/docs/datamap/timeseries-datamap-guide.md) * [FAQ](https://github.com/apache/carbondata/blob/master/docs/faq.md) http://git-wip-us.apache.org/repos/asf/carbondata/blob/443b717d/docs/sdk-writer-guide.md ---------------------------------------------------------------------- diff --git a/docs/sdk-writer-guide.md b/docs/sdk-writer-guide.md index 18b583a..9878b71 100644 --- a/docs/sdk-writer-guide.md +++ b/docs/sdk-writer-guide.md @@ -52,6 +52,7 @@ import java.io.IOException; import org.apache.carbondata.common.exceptions.sql.InvalidLoadOptionException; import org.apache.carbondata.core.metadata.datatype.DataTypes; +import org.apache.carbondata.sdk.file.AvroCarbonWriter; import org.apache.carbondata.sdk.file.CarbonWriter; import org.apache.carbondata.sdk.file.Field; @@ -75,25 +76,24 @@ public class TestSdkAvro { " \"type\" : \"record\"," + " \"name\" : \"Acme\"," + " \"fields\" : [" - + "{ \"name\" : \"name\", \"type\" : \"string\" }," + + "{ \"name\" : \"fname\", \"type\" : \"string\" }," + "{ \"name\" : \"age\", \"type\" : \"int\" }]" + "}"; - String json = "{\"name\":\"bob\", \"age\":10}"; + String json = "{\"fname\":\"bob\", \"age\":10}"; // conversion to GenericData.Record JsonAvroConverter converter = new JsonAvroConverter(); GenericData.Record record = converter.convertToGenericDataRecord( json.getBytes(CharEncoding.UTF_8), new org.apache.avro.Schema.Parser().parse(avroSchema)); - // for sdk schema - Field[] fields = new Field[2]; - fields[0] = new Field("name", DataTypes.STRING); - fields[1] = new Field("age", DataTypes.STRING); + // prepare carbon schema from avro schema + org.apache.carbondata.sdk.file.Schema carbonSchema = + AvroCarbonWriter.getCarbonSchemaFromAvroSchema(avroSchema); try { CarbonWriter writer = CarbonWriter.builder() - .withSchema(new org.apache.carbondata.sdk.file.Schema(fields)) + .withSchema(carbonSchema) .outputPath(path) .buildWriterForAvroInput(); @@ -345,4 +345,15 @@ public Schema(Field[] fields); * @return Schema */ public static Schema parseJson(String json); +``` + +### Class org.apache.carbondata.sdk.file.AvroCarbonWriter +``` +/** +* converts avro schema to carbon schema, required by carbonWriter +* +* @param avroSchemaString json formatted avro schema as string +* @return carbon sdk schema +*/ +public static org.apache.carbondata.sdk.file.Schema getCarbonSchemaFromAvroSchema(String avroSchemaString); ``` \ No newline at end of file