Repository: chukwa Updated Branches: refs/heads/master 83baac244 -> cac335a10
CHUKWA-734. Added GoraWriter. (Lewis John McGibbney via Eric Yang) Project: http://git-wip-us.apache.org/repos/asf/chukwa/repo Commit: http://git-wip-us.apache.org/repos/asf/chukwa/commit/cac335a1 Tree: http://git-wip-us.apache.org/repos/asf/chukwa/tree/cac335a1 Diff: http://git-wip-us.apache.org/repos/asf/chukwa/diff/cac335a1 Branch: refs/heads/master Commit: cac335a10c8f9b0ce09cc4dc5bead940e07f78bd Parents: 83baac2 Author: Eric Yang <[email protected]> Authored: Sat Dec 12 10:03:21 2015 -0800 Committer: Eric Yang <[email protected]> Committed: Sun Dec 13 11:41:33 2015 -0800 ---------------------------------------------------------------------- CHANGES.txt | 2 + pom.xml | 130 ++++ .../datacollection/writer/gora/ChukwaChunk.java | 682 +++++++++++++++++++ .../datacollection/writer/gora/GoraWriter.java | 122 ++++ .../writer/gora/package-info.java | 22 + src/main/resources/chukwachunk.json | 14 + src/main/resources/gora-accumulo-mapping.xml | 42 ++ src/main/resources/gora-cassandra-mapping.xml | 53 ++ src/main/resources/gora-hbase-mapping.xml | 56 ++ src/main/resources/gora-mongodb-mapping.xml | 34 + src/main/resources/gora-solr-mapping.xml | 32 + src/main/resources/gora.properties | 110 +++ 12 files changed, 1299 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/chukwa/blob/cac335a1/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index eacffbd..08bf171 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -4,6 +4,8 @@ Trunk (unreleased changes) NEW FEATURES + CHUKWA-734. Added GoraWriter. (Lewis John McGibbney via Eric Yang) + CHUKWA-787. Added full screen support, and quick link to Hadoop and HBase UI. (Eric Yang) CHUKWA-785. Added banner, pie chart, gauge chart to graph explorer. (Eric Yang) http://git-wip-us.apache.org/repos/asf/chukwa/blob/cac335a1/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 68ae8cf..caea1db 100644 --- a/pom.xml +++ b/pom.xml @@ -51,6 +51,7 @@ <hadoopVersion>2.6.0</hadoopVersion> <solrVersion>4.7.2</solrVersion> <!-- <JAVA_HOME>${java.home}</JAVA_HOME> --> + <gora.version>0.6</gora.version> </properties> <name>Apache Chukwa</name> @@ -327,6 +328,39 @@ <artifactId>parquet-avro</artifactId> <version>1.7.0</version> </dependency> + <!-- BEGINNING OF GORA DEPENDENCIES --> + <dependency> + <groupId>org.apache.gora</groupId> + <artifactId>gora-core</artifactId> + <version>${gora.version}</version> + </dependency> + <dependency> + <groupId>org.apache.gora</groupId> + <artifactId>gora-hbase</artifactId> + <version>${gora.version}</version> + </dependency> + <!--dependency> + <groupId>org.apache.gora</groupId> + <artifactId>gora-cassandra</artifactId> + <version>${gora.version}</version> + </dependency--> + <!--dependency> + <groupId>org.apache.gora</groupId> + <artifactId>gora-mongodb</artifactId> + <version>${gora.version}</version> + </dependency--> + <!--dependency> + <groupId>org.apache.gora</groupId> + <artifactId>gora-solr</artifactId> + <version>${gora.version}</version> + </dependency--> + <!--dependency> + <groupId>org.apache.gora</groupId> + <artifactId>gora-accumulo</artifactId> + <version>${gora.version}</version> + </dependency--> + + <!-- END OF GORA DEPENDENCIES --> </dependencies> <developers> @@ -891,6 +925,7 @@ <exclude>src/main/web/hicc/css/chartist.css.map</exclude> <exclude>src/main/web/hicc/js/chartist.min.js</exclude> <exclude>src/main/web/hicc/js/chartist.min.js.map</exclude> + <exclude>src/main/resources/chukwachunk.json</exclude> </excludes> </configuration> </plugin> @@ -902,6 +937,61 @@ <version>1.0</version> </extension> </extensions> + <pluginManagement> + <plugins> + <!--This plugin's configuration is used to store Eclipse m2e settings only. It has no influence on the Maven build itself.--> + <plugin> + <groupId>org.eclipse.m2e</groupId> + <artifactId>lifecycle-mapping</artifactId> + <version>1.0.0</version> + <configuration> + <lifecycleMappingMetadata> + <pluginExecutions> + <pluginExecution> + <pluginExecutionFilter> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-javadoc-plugin</artifactId> + <versionRange>[2.10.1,)</versionRange> + <goals> + <goal>javadoc</goal> + </goals> + </pluginExecutionFilter> + <action> + <ignore></ignore> + </action> + </pluginExecution> + <pluginExecution> + <pluginExecutionFilter> + <groupId>com.sun.jersey.contribs</groupId> + <artifactId>maven-wadl-plugin</artifactId> + <versionRange>[1.8,)</versionRange> + <goals> + <goal>generate</goal> + </goals> + </pluginExecutionFilter> + <action> + <ignore></ignore> + </action> + </pluginExecution> + <pluginExecution> + <pluginExecutionFilter> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-antrun-plugin</artifactId> + <versionRange>[1.6,)</versionRange> + <goals> + <goal>run</goal> + </goals> + </pluginExecutionFilter> + <action> + <ignore></ignore> + </action> + </pluginExecution> + </pluginExecutions> + </lifecycleMappingMetadata> + </configuration> + </plugin> + </plugins> + </pluginManagement> </build> <profiles> @@ -954,6 +1044,7 @@ </plugins> </build> </profile> + <profile> <id>hbase-1</id> <activation> @@ -1011,6 +1102,45 @@ </dependency> <dependency> <groupId>org.apache.hbase</groupId> + <artifactId>hbase-client</artifactId> + <version>${hbaseVersion}</version> + <exclusions> + <exclusion> + <groupId>org.mortbay.jetty</groupId> + <artifactId>jsp-2.1</artifactId> + </exclusion> + <exclusion> + <groupId>org.mortbay.jetty</groupId> + <artifactId>jsp-api-2.1</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-server</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-core</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-json</artifactId> + </exclusion> + <exclusion> + <groupId>tomcat</groupId> + <artifactId>jasper-compiler</artifactId> + </exclusion> + <exclusion> + <groupId>tomcat</groupId> + <artifactId>jasper-runtime</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdfs</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.hbase</groupId> <artifactId>hbase-server</artifactId> <version>${hbaseVersion}</version> <exclusions> http://git-wip-us.apache.org/repos/asf/chukwa/blob/cac335a1/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/gora/ChukwaChunk.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/gora/ChukwaChunk.java b/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/gora/ChukwaChunk.java new file mode 100644 index 0000000..61c61f0 --- /dev/null +++ b/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/gora/ChukwaChunk.java @@ -0,0 +1,682 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Autogenerated by Avro + * + * DO NOT EDIT DIRECTLY + */ +package org.apache.hadoop.chukwa.datacollection.writer.gora; +/** Chukwa Adaptors emit data in Chunks. A Chunk is a sequence of bytes, with some metadata. Several of these are set automatically by the Agent or Adaptors. Two of them require user intervention: cluster name and datatype. Cluster name is specified in conf/chukwa-agent-conf.xml, and is global to each Agent process. Datatype describes the expected format of the data collected by an Adaptor instance, and it is specified when that instance is started. */ +@SuppressWarnings("all") +public class ChukwaChunk extends org.apache.gora.persistency.impl.PersistentBase implements org.apache.avro.specific.SpecificRecord, org.apache.gora.persistency.Persistent { + public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"ChukwaChunk\",\"namespace\":\"org.apache.hadoop.chukwa.datacollection.writer.gora\",\"doc\":\"Chukwa Adaptors emit data in Chunks. A Chunk is a sequence of bytes, with some metadata. Several of these are set automatically by the Agent or Adaptors. Two of them require user intervention: cluster name and datatype. Cluster name is specified in conf/chukwa-agent-conf.xml, and is global to each Agent process. Datatype describes the expected format of the data collected by an Adaptor instance, and it is specified when that instance is started. \",\"fields\":[{\"name\":\"source\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"tags\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"datatype\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"sequenceID\",\"type\":[\"null\",\"long\"],\"default\":null},{\"name\":\"name\",\"type\ ":[\"null\",\"string\"],\"default\":null},{\"name\":\"data\",\"type\":[\"null\",\"bytes\"],\"default\":null}]}"); + + /** Enum containing all data bean's fields. */ + public static enum Field { + SOURCE(0, "source"), + TAGS(1, "tags"), + DATATYPE(2, "datatype"), + SEQUENCE_ID(3, "sequenceID"), + NAME(4, "name"), + DATA(5, "data"), + ; + /** + * Field's index. + */ + private int index; + + /** + * Field's name. + */ + private String name; + + /** + * Field's constructor + * @param index field's index. + * @param name field's name. + */ + Field(int index, String name) {this.index=index;this.name=name;} + + /** + * Gets field's index. + * @return int field's index. + */ + public int getIndex() {return index;} + + /** + * Gets field's name. + * @return String field's name. + */ + public String getName() {return name;} + + /** + * Gets field's attributes to string. + * @return String field's attributes to string. + */ + public String toString() {return name;} + }; + + public static final String[] _ALL_FIELDS = { + "source", + "tags", + "datatype", + "sequenceID", + "name", + "data", + }; + + /** + * Gets the total field count. + * @return int field count + */ + public int getFieldsCount() { + return ChukwaChunk._ALL_FIELDS.length; + } + + private java.lang.CharSequence source; + private java.lang.CharSequence tags; + private java.lang.CharSequence datatype; + private java.lang.Long sequenceID; + private java.lang.CharSequence name; + private java.nio.ByteBuffer data; + public org.apache.avro.Schema getSchema() { return SCHEMA$; } + // Used by DatumWriter. Applications should not call. + public java.lang.Object get(int field$) { + switch (field$) { + case 0: return source; + case 1: return tags; + case 2: return datatype; + case 3: return sequenceID; + case 4: return name; + case 5: return data; + default: throw new org.apache.avro.AvroRuntimeException("Bad index"); + } + } + + // Used by DatumReader. Applications should not call. + @SuppressWarnings(value="unchecked") + public void put(int field$, java.lang.Object value) { + switch (field$) { + case 0: source = (java.lang.CharSequence)(value); break; + case 1: tags = (java.lang.CharSequence)(value); break; + case 2: datatype = (java.lang.CharSequence)(value); break; + case 3: sequenceID = (java.lang.Long)(value); break; + case 4: name = (java.lang.CharSequence)(value); break; + case 5: data = (java.nio.ByteBuffer)(value); break; + default: throw new org.apache.avro.AvroRuntimeException("Bad index"); + } + } + + /** + * Gets the value of the 'source' field. + */ + public java.lang.CharSequence getSource() { + return source; + } + + /** + * Sets the value of the 'source' field. + * @param value the value to set. + */ + public void setSource(java.lang.CharSequence value) { + this.source = value; + setDirty(0); + } + + /** + * Checks the dirty status of the 'source' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isSourceDirty() { + return isDirty(0); + } + + /** + * Gets the value of the 'tags' field. + */ + public java.lang.CharSequence getTags() { + return tags; + } + + /** + * Sets the value of the 'tags' field. + * @param value the value to set. + */ + public void setTags(java.lang.CharSequence value) { + this.tags = value; + setDirty(1); + } + + /** + * Checks the dirty status of the 'tags' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isTagsDirty() { + return isDirty(1); + } + + /** + * Gets the value of the 'datatype' field. + */ + public java.lang.CharSequence getDatatype() { + return datatype; + } + + /** + * Sets the value of the 'datatype' field. + * @param value the value to set. + */ + public void setDatatype(java.lang.CharSequence value) { + this.datatype = value; + setDirty(2); + } + + /** + * Checks the dirty status of the 'datatype' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isDatatypeDirty() { + return isDirty(2); + } + + /** + * Gets the value of the 'sequenceID' field. + */ + public java.lang.Long getSequenceID() { + return sequenceID; + } + + /** + * Sets the value of the 'sequenceID' field. + * @param value the value to set. + */ + public void setSequenceID(java.lang.Long value) { + this.sequenceID = value; + setDirty(3); + } + + /** + * Checks the dirty status of the 'sequenceID' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isSequenceIDDirty() { + return isDirty(3); + } + + /** + * Gets the value of the 'name' field. + */ + public java.lang.CharSequence getName() { + return name; + } + + /** + * Sets the value of the 'name' field. + * @param value the value to set. + */ + public void setName(java.lang.CharSequence value) { + this.name = value; + setDirty(4); + } + + /** + * Checks the dirty status of the 'name' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isNameDirty() { + return isDirty(4); + } + + /** + * Gets the value of the 'data' field. + */ + public java.nio.ByteBuffer getData() { + return data; + } + + /** + * Sets the value of the 'data' field. + * @param value the value to set. + */ + public void setData(java.nio.ByteBuffer value) { + this.data = value; + setDirty(5); + } + + /** + * Checks the dirty status of the 'data' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isDataDirty() { + return isDirty(5); + } + + /** Creates a new ChukwaChunk RecordBuilder */ + public static org.apache.hadoop.chukwa.datacollection.writer.gora.ChukwaChunk.Builder newBuilder() { + return new org.apache.hadoop.chukwa.datacollection.writer.gora.ChukwaChunk.Builder(); + } + + /** Creates a new ChukwaChunk RecordBuilder by copying an existing Builder */ + public static org.apache.hadoop.chukwa.datacollection.writer.gora.ChukwaChunk.Builder newBuilder(org.apache.hadoop.chukwa.datacollection.writer.gora.ChukwaChunk.Builder other) { + return new org.apache.hadoop.chukwa.datacollection.writer.gora.ChukwaChunk.Builder(other); + } + + /** Creates a new ChukwaChunk RecordBuilder by copying an existing ChukwaChunk instance */ + public static org.apache.hadoop.chukwa.datacollection.writer.gora.ChukwaChunk.Builder newBuilder(org.apache.hadoop.chukwa.datacollection.writer.gora.ChukwaChunk other) { + return new org.apache.hadoop.chukwa.datacollection.writer.gora.ChukwaChunk.Builder(other); + } + + private static java.nio.ByteBuffer deepCopyToReadOnlyBuffer( + java.nio.ByteBuffer input) { + java.nio.ByteBuffer copy = java.nio.ByteBuffer.allocate(input.capacity()); + int position = input.position(); + input.reset(); + int mark = input.position(); + int limit = input.limit(); + input.rewind(); + input.limit(input.capacity()); + copy.put(input); + input.rewind(); + copy.rewind(); + input.position(mark); + input.mark(); + copy.position(mark); + copy.mark(); + input.position(position); + copy.position(position); + input.limit(limit); + copy.limit(limit); + return copy.asReadOnlyBuffer(); + } + + /** + * RecordBuilder for ChukwaChunk instances. + */ + public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase<ChukwaChunk> + implements org.apache.avro.data.RecordBuilder<ChukwaChunk> { + + private java.lang.CharSequence source; + private java.lang.CharSequence tags; + private java.lang.CharSequence datatype; + private java.lang.Long sequenceID; + private java.lang.CharSequence name; + private java.nio.ByteBuffer data; + + /** Creates a new Builder */ + private Builder() { + super(org.apache.hadoop.chukwa.datacollection.writer.gora.ChukwaChunk.SCHEMA$); + } + + /** Creates a Builder by copying an existing Builder */ + private Builder(org.apache.hadoop.chukwa.datacollection.writer.gora.ChukwaChunk.Builder other) { + super(other); + } + + /** Creates a Builder by copying an existing ChukwaChunk instance */ + private Builder(org.apache.hadoop.chukwa.datacollection.writer.gora.ChukwaChunk other) { + super(org.apache.hadoop.chukwa.datacollection.writer.gora.ChukwaChunk.SCHEMA$); + if (isValidValue(fields()[0], other.source)) { + this.source = (java.lang.CharSequence) data().deepCopy(fields()[0].schema(), other.source); + fieldSetFlags()[0] = true; + } + if (isValidValue(fields()[1], other.tags)) { + this.tags = (java.lang.CharSequence) data().deepCopy(fields()[1].schema(), other.tags); + fieldSetFlags()[1] = true; + } + if (isValidValue(fields()[2], other.datatype)) { + this.datatype = (java.lang.CharSequence) data().deepCopy(fields()[2].schema(), other.datatype); + fieldSetFlags()[2] = true; + } + if (isValidValue(fields()[3], other.sequenceID)) { + this.sequenceID = (java.lang.Long) data().deepCopy(fields()[3].schema(), other.sequenceID); + fieldSetFlags()[3] = true; + } + if (isValidValue(fields()[4], other.name)) { + this.name = (java.lang.CharSequence) data().deepCopy(fields()[4].schema(), other.name); + fieldSetFlags()[4] = true; + } + if (isValidValue(fields()[5], other.data)) { + this.data = (java.nio.ByteBuffer) data().deepCopy(fields()[5].schema(), other.data); + fieldSetFlags()[5] = true; + } + } + + /** Gets the value of the 'source' field */ + public java.lang.CharSequence getSource() { + return source; + } + + /** Sets the value of the 'source' field */ + public org.apache.hadoop.chukwa.datacollection.writer.gora.ChukwaChunk.Builder setSource(java.lang.CharSequence value) { + validate(fields()[0], value); + this.source = value; + fieldSetFlags()[0] = true; + return this; + } + + /** Checks whether the 'source' field has been set */ + public boolean hasSource() { + return fieldSetFlags()[0]; + } + + /** Clears the value of the 'source' field */ + public org.apache.hadoop.chukwa.datacollection.writer.gora.ChukwaChunk.Builder clearSource() { + source = null; + fieldSetFlags()[0] = false; + return this; + } + + /** Gets the value of the 'tags' field */ + public java.lang.CharSequence getTags() { + return tags; + } + + /** Sets the value of the 'tags' field */ + public org.apache.hadoop.chukwa.datacollection.writer.gora.ChukwaChunk.Builder setTags(java.lang.CharSequence value) { + validate(fields()[1], value); + this.tags = value; + fieldSetFlags()[1] = true; + return this; + } + + /** Checks whether the 'tags' field has been set */ + public boolean hasTags() { + return fieldSetFlags()[1]; + } + + /** Clears the value of the 'tags' field */ + public org.apache.hadoop.chukwa.datacollection.writer.gora.ChukwaChunk.Builder clearTags() { + tags = null; + fieldSetFlags()[1] = false; + return this; + } + + /** Gets the value of the 'datatype' field */ + public java.lang.CharSequence getDatatype() { + return datatype; + } + + /** Sets the value of the 'datatype' field */ + public org.apache.hadoop.chukwa.datacollection.writer.gora.ChukwaChunk.Builder setDatatype(java.lang.CharSequence value) { + validate(fields()[2], value); + this.datatype = value; + fieldSetFlags()[2] = true; + return this; + } + + /** Checks whether the 'datatype' field has been set */ + public boolean hasDatatype() { + return fieldSetFlags()[2]; + } + + /** Clears the value of the 'datatype' field */ + public org.apache.hadoop.chukwa.datacollection.writer.gora.ChukwaChunk.Builder clearDatatype() { + datatype = null; + fieldSetFlags()[2] = false; + return this; + } + + /** Gets the value of the 'sequenceID' field */ + public java.lang.Long getSequenceID() { + return sequenceID; + } + + /** Sets the value of the 'sequenceID' field */ + public org.apache.hadoop.chukwa.datacollection.writer.gora.ChukwaChunk.Builder setSequenceID(java.lang.Long value) { + validate(fields()[3], value); + this.sequenceID = value; + fieldSetFlags()[3] = true; + return this; + } + + /** Checks whether the 'sequenceID' field has been set */ + public boolean hasSequenceID() { + return fieldSetFlags()[3]; + } + + /** Clears the value of the 'sequenceID' field */ + public org.apache.hadoop.chukwa.datacollection.writer.gora.ChukwaChunk.Builder clearSequenceID() { + sequenceID = null; + fieldSetFlags()[3] = false; + return this; + } + + /** Gets the value of the 'name' field */ + public java.lang.CharSequence getName() { + return name; + } + + /** Sets the value of the 'name' field */ + public org.apache.hadoop.chukwa.datacollection.writer.gora.ChukwaChunk.Builder setName(java.lang.CharSequence value) { + validate(fields()[4], value); + this.name = value; + fieldSetFlags()[4] = true; + return this; + } + + /** Checks whether the 'name' field has been set */ + public boolean hasName() { + return fieldSetFlags()[4]; + } + + /** Clears the value of the 'name' field */ + public org.apache.hadoop.chukwa.datacollection.writer.gora.ChukwaChunk.Builder clearName() { + name = null; + fieldSetFlags()[4] = false; + return this; + } + + /** Gets the value of the 'data' field */ + public java.nio.ByteBuffer getData() { + return data; + } + + /** Sets the value of the 'data' field */ + public org.apache.hadoop.chukwa.datacollection.writer.gora.ChukwaChunk.Builder setData(java.nio.ByteBuffer value) { + validate(fields()[5], value); + this.data = value; + fieldSetFlags()[5] = true; + return this; + } + + /** Checks whether the 'data' field has been set */ + public boolean hasData() { + return fieldSetFlags()[5]; + } + + /** Clears the value of the 'data' field */ + public org.apache.hadoop.chukwa.datacollection.writer.gora.ChukwaChunk.Builder clearData() { + data = null; + fieldSetFlags()[5] = false; + return this; + } + + @Override + public ChukwaChunk build() { + try { + ChukwaChunk record = new ChukwaChunk(); + record.source = fieldSetFlags()[0] ? this.source : (java.lang.CharSequence) defaultValue(fields()[0]); + record.tags = fieldSetFlags()[1] ? this.tags : (java.lang.CharSequence) defaultValue(fields()[1]); + record.datatype = fieldSetFlags()[2] ? this.datatype : (java.lang.CharSequence) defaultValue(fields()[2]); + record.sequenceID = fieldSetFlags()[3] ? this.sequenceID : (java.lang.Long) defaultValue(fields()[3]); + record.name = fieldSetFlags()[4] ? this.name : (java.lang.CharSequence) defaultValue(fields()[4]); + record.data = fieldSetFlags()[5] ? this.data : (java.nio.ByteBuffer) defaultValue(fields()[5]); + return record; + } catch (Exception e) { + throw new org.apache.avro.AvroRuntimeException(e); + } + } + } + + public ChukwaChunk.Tombstone getTombstone(){ + return TOMBSTONE; + } + + public ChukwaChunk newInstance(){ + return newBuilder().build(); + } + + private static final Tombstone TOMBSTONE = new Tombstone(); + + public static final class Tombstone extends ChukwaChunk implements org.apache.gora.persistency.Tombstone { + + private Tombstone() { } + + /** + * Gets the value of the 'source' field. + */ + public java.lang.CharSequence getSource() { + throw new java.lang.UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'source' field. + * @param value the value to set. + */ + public void setSource(java.lang.CharSequence value) { + throw new java.lang.UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'source' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isSourceDirty() { + throw new java.lang.UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + /** + * Gets the value of the 'tags' field. + */ + public java.lang.CharSequence getTags() { + throw new java.lang.UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'tags' field. + * @param value the value to set. + */ + public void setTags(java.lang.CharSequence value) { + throw new java.lang.UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'tags' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isTagsDirty() { + throw new java.lang.UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + /** + * Gets the value of the 'datatype' field. + */ + public java.lang.CharSequence getDatatype() { + throw new java.lang.UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'datatype' field. + * @param value the value to set. + */ + public void setDatatype(java.lang.CharSequence value) { + throw new java.lang.UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'datatype' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isDatatypeDirty() { + throw new java.lang.UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + /** + * Gets the value of the 'sequenceID' field. + */ + public java.lang.Long getSequenceID() { + throw new java.lang.UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'sequenceID' field. + * @param value the value to set. + */ + public void setSequenceID(java.lang.Long value) { + throw new java.lang.UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'sequenceID' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isSequenceIDDirty() { + throw new java.lang.UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + /** + * Gets the value of the 'name' field. + */ + public java.lang.CharSequence getName() { + throw new java.lang.UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'name' field. + * @param value the value to set. + */ + public void setName(java.lang.CharSequence value) { + throw new java.lang.UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'name' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isNameDirty() { + throw new java.lang.UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + /** + * Gets the value of the 'data' field. + */ + public java.nio.ByteBuffer getData() { + throw new java.lang.UnsupportedOperationException("Get is not supported on tombstones"); + } + + /** + * Sets the value of the 'data' field. + * @param value the value to set. + */ + public void setData(java.nio.ByteBuffer value) { + throw new java.lang.UnsupportedOperationException("Set is not supported on tombstones"); + } + + /** + * Checks the dirty status of the 'data' field. A field is dirty if it represents a change that has not yet been written to the database. + * @param value the value to set. + */ + public boolean isDataDirty() { + throw new java.lang.UnsupportedOperationException("IsDirty is not supported on tombstones"); + } + + + } + +} + http://git-wip-us.apache.org/repos/asf/chukwa/blob/cac335a1/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/gora/GoraWriter.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/gora/GoraWriter.java b/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/gora/GoraWriter.java new file mode 100644 index 0000000..bdf66f2 --- /dev/null +++ b/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/gora/GoraWriter.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.chukwa.datacollection.writer.gora; + +import java.nio.ByteBuffer; +import java.util.List; + +import org.apache.gora.store.DataStore; +import org.apache.gora.store.DataStoreFactory; +import org.apache.gora.util.GoraException; +import org.apache.hadoop.chukwa.Chunk; +import org.apache.hadoop.chukwa.datacollection.agent.ChukwaAgent; +import org.apache.hadoop.chukwa.datacollection.writer.ChukwaWriter; +import org.apache.hadoop.chukwa.datacollection.writer.PipelineableWriter; +import org.apache.hadoop.chukwa.datacollection.writer.WriterException; +import org.apache.hadoop.chukwa.datacollection.writer.solr.SolrWriter; +import org.apache.hadoop.chukwa.util.ExceptionUtil; +import org.apache.hadoop.conf.Configuration; +import org.apache.log4j.Logger; + +/** + * This class leverages <a href="http://gora.apache.org";>Apache Gora</a> + * as a pipeline writer implementation for mapping Chukwa data chunks and + * metadata as {@link org.apache.hadoop.chukwa.datacollection.writer.gora.ChukwaChunk}'s. + * + */ +public class GoraWriter extends PipelineableWriter { + + private static Logger log = Logger.getLogger(SolrWriter.class); + + DataStore<String, ChukwaChunk> chunkStore; + + /** + * Default constructor for this class. + */ + public GoraWriter() throws WriterException { + log.debug("Initializing configuration for GoraWriter pipeline..."); + init(ChukwaAgent.getStaticConfiguration()); + } + + /** + * {@link org.apache.gora.store.DataStore} objects are created from a factory. It is necessary to + * provide the key and value class. The datastore class parameters is optional, + * and if not specified it will be read from the <code>gora.properties</code> file. + * @throws WriterException + * @throws GoraException + * @see org.apache.hadoop.chukwa.datacollection.writer.ChukwaWriter#init(org.apache.hadoop.conf.Configuration) + */ + @Override + public void init(Configuration c) throws WriterException { + try { + chunkStore = DataStoreFactory.getDataStore(String.class, ChukwaChunk.class, c); + } catch (GoraException e) { + log.error(ExceptionUtil.getStackTrace(e)); + e.printStackTrace(); + } + } + + /** + * <p> + * If the {@link org.apache.gora.store.DataStore} instance is not null, we + * execute a {@link org.apache.gora.store.DataStore#flush()}. This forces + * the write caches to be flushed. DataStore implementations may optimize + * their writing by deferring the actual put / delete operations until + * this moment. + * </p> + * <p>Otherwise, we utilize {@link org.apache.gora.store.DataStore#close()} + * which closes the DataStore. This should release any resources held by + * the implementation, so that the instance is ready for GC. All other + * DataStore methods cannot be used after this method was called. + * Subsequent calls of this method are ignored. + * </p> + * @see org.apache.hadoop.chukwa.datacollection.writer.ChukwaWriter#close() + */ + @Override + public void close() throws WriterException { + if (chunkStore != null) { + chunkStore.flush(); + } else { + chunkStore.close(); + } + log.debug("Gora datastore successfully closed."); + } + + @Override + public CommitStatus add(List<Chunk> chunks) throws WriterException { + CommitStatus cStatus = ChukwaWriter.COMMIT_OK; + for(Chunk chunk : chunks) { + try { + ChukwaChunk chukwaChunk = ChukwaChunk.newBuilder().build(); + chukwaChunk.setSource(chunk.getSource()); + chukwaChunk.setDatatype(chunk.getDataType()); + chukwaChunk.setSequenceID(chunk.getSeqID()); + chukwaChunk.setName(chunk.getStreamName()); + chukwaChunk.setTags(chunk.getTags()); + chukwaChunk.setData(ByteBuffer.wrap(chunk.getData())); + } catch (Exception e) { + log.error(ExceptionUtil.getStackTrace(e)); + throw new WriterException("Failed to store data to Solr Cloud."); + } + } + if (next != null) { + cStatus = next.add(chunks); //pass data through + } + return cStatus; + } +} http://git-wip-us.apache.org/repos/asf/chukwa/blob/cac335a1/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/gora/package-info.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/gora/package-info.java b/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/gora/package-info.java new file mode 100644 index 0000000..bffa7f4 --- /dev/null +++ b/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/gora/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Package dedicated to using <a href="http://gora.apache.org";>Apache Gora</a> + * as a pipeline writer implementation. + */ +package org.apache.hadoop.chukwa.datacollection.writer.gora; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/chukwa/blob/cac335a1/src/main/resources/chukwachunk.json ---------------------------------------------------------------------- diff --git a/src/main/resources/chukwachunk.json b/src/main/resources/chukwachunk.json new file mode 100644 index 0000000..9fc8841 --- /dev/null +++ b/src/main/resources/chukwachunk.json @@ -0,0 +1,14 @@ +{ + "type": "record", + "name": "ChukwaChunk", + "doc": "Chukwa Adaptors emit data in Chunks. A Chunk is a sequence of bytes, with some metadata. Several of these are set automatically by the Agent or Adaptors. Two of them require user intervention: cluster name and datatype. Cluster name is specified in conf/chukwa-agent-conf.xml, and is global to each Agent process. Datatype describes the expected format of the data collected by an Adaptor instance, and it is specified when that instance is started. ", + "namespace": "org.apache.hadoop.chukwa.datacollection.writer.gora", + "fields" : [ + {"name": "source", "type": ["null","string"], "default":null}, + {"name": "tags", "type": ["null","string"], "default":null}, + {"name": "datatype", "type": ["null","string"], "default":null}, + {"name": "sequenceID", "type": ["null","long"], "default":null}, + {"name": "name", "type": ["null","string"], "default":null}, + {"name": "data", "type": ["null","bytes"], "default":null} + ] +} http://git-wip-us.apache.org/repos/asf/chukwa/blob/cac335a1/src/main/resources/gora-accumulo-mapping.xml ---------------------------------------------------------------------- diff --git a/src/main/resources/gora-accumulo-mapping.xml b/src/main/resources/gora-accumulo-mapping.xml new file mode 100644 index 0000000..68b8146 --- /dev/null +++ b/src/main/resources/gora-accumulo-mapping.xml @@ -0,0 +1,42 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<!-- A comprehensive account of the entire gora-accumulo mapping + options can be found on the Gora website at: + http://gora.apache.org/current/gora-accumulo.html + Users are strongly advised to consult this documentation + prior to working with the default mapping configuration + provided below. +--> + +<gora-otd> + + <table name="chukwachunk"> + <family name="log" maxVersions="1"/> + </table> + <class table="chukwachunk" keyClass="java.lang.String" name="org.apache.hadoop.chukwa.datacollection.writer.gora.ChukwaChunk"> + <!-- log fields --> + <field name="source" family="log" qualifier="src"/> + <field name="tags" family="log" qualifier="tag"/> + <field name="datatype" family="log" qualifier="dt"/> + <field name="sequenceID" family="log" qualifier="sid"/> + <field name="name" family="log" qualifier="n"/> + <field name="data" family="log" qualifier="dat"/> + </class> + +</gora-otd> http://git-wip-us.apache.org/repos/asf/chukwa/blob/cac335a1/src/main/resources/gora-cassandra-mapping.xml ---------------------------------------------------------------------- diff --git a/src/main/resources/gora-cassandra-mapping.xml b/src/main/resources/gora-cassandra-mapping.xml new file mode 100644 index 0000000..119ede2 --- /dev/null +++ b/src/main/resources/gora-cassandra-mapping.xml @@ -0,0 +1,53 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<!-- + The value of 'host' attribute of keyspace tag should match exactly what is in + gora.properties file. Essentially this means that if you are using port number, you should + use it every where regardless of whether it is the default port or not. + At runtime Gora will otherwise try to connect to localhost + https://issues.apache.org/jira/browse/GORA-269 + + The value of 'ttl' (time to live) attribute of field tag should most likely always + be zero unless you want Cassandra to create Tombstones and delete portions of your + data once this period expires. Any positive value is read and bound to the number + of seconds after which the value for that field will disappear. + + The value used here for 'gc_grace_seconds' of '0' is ONLY VIABLE FOR SINGLE NODE + CLUSTER. you should update this value according to your cluster configuration. + https://wiki.apache.org/cassandra/StorageConfiguration + + More information on gora-cassandra configuration and mapping's can be found + at http://gora.apache.org/current/gora-cassandra.html +--> + +<gora-otd> + + <keyspace name="chukwachunk" cluster="Test Cluster" host="localhost:9160"> + <family name="log" gc_grace_seconds="0"/> + </keyspace> + <class keyClass="java.lang.String" name="org.apache.hadoop.chukwa.datacollection.writer.gora.ChukwaChunk" keyspace="chukwachunk"> + <!-- log fields --> + <field name="source" family="log" qualifier="src" ttl=0 /> + <field name="tags" family="log" qualifier="tag" ttl=0 /> + <field name="datatype" family="log" qualifier="dt" ttl=0 /> + <field name="sequenceID" family="log" qualifier="sid" ttl=0 /> + <field name="name" family="log" qualifier="n" ttl=0 /> + <field name="data" family="log" qualifier="dat" ttl=0 /> + </class> + +</gora-otd> http://git-wip-us.apache.org/repos/asf/chukwa/blob/cac335a1/src/main/resources/gora-hbase-mapping.xml ---------------------------------------------------------------------- diff --git a/src/main/resources/gora-hbase-mapping.xml b/src/main/resources/gora-hbase-mapping.xml new file mode 100644 index 0000000..670e832 --- /dev/null +++ b/src/main/resources/gora-hbase-mapping.xml @@ -0,0 +1,56 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<!-- +In addition to family 'name' and 'maxVersions' attributes, +individual table families can ve defined with several other +attributes such as + compression="$$$" - the compression option to use in HBase. + blockCache="$$$" - an LRU cache that contains three levels of block priority + to allow for scan-resistance and in-memory ColumnFamilies. + blockSize="$$$" - The blocksize can be configured for each ColumnFamily in + a table, and this defaults to 64k. + bloomFilter="$$$" - Bloom Filters can be enabled per-ColumnFamily. + maxVersions="$$$" - The maximum number of row versions to store is configured + per column family via HColumnDescriptor. + timeToLive="$$$" - ColumnFamilies can set a TTL length in seconds, and HBase + will automatically delete rows once the expiration time is + reached. + inMemory="$$$" - ColumnFamilies can optionally be defined as in-memory. + +You should consult the current Gora gora-hbase documentation +for further information on properties and mapping configurtion. +http://gora.apache.org/current/gora-hbase.html +--> + +<gora-otd> + + <table name="chukwachunk"> + <family name="log" maxVersions="1"/> + </table> + <class table="chukwachunk" keyClass="java.lang.String" name="org.apache.hadoop.chukwa.datacollection.writer.gora.ChukwaChunk"> + <!-- log fields --> + <field name="source" family="log" qualifier="src"/> + <field name="tags" family="log" qualifier="tag"/> + <field name="datatype" family="log" qualifier="dt"/> + <field name="sequenceID" family="log" qualifier="sid"/> + <field name="name" family="log" qualifier="n"/> + <field name="data" family="log" qualifier="dat"/> + </class> + +</gora-otd> http://git-wip-us.apache.org/repos/asf/chukwa/blob/cac335a1/src/main/resources/gora-mongodb-mapping.xml ---------------------------------------------------------------------- diff --git a/src/main/resources/gora-mongodb-mapping.xml b/src/main/resources/gora-mongodb-mapping.xml new file mode 100644 index 0000000..5aa5f06 --- /dev/null +++ b/src/main/resources/gora-mongodb-mapping.xml @@ -0,0 +1,34 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<!-- + More information on gora-mongodb configuration and mapping's can be found + at http://gora.apache.org/current/gora-mongodb.html +--> +<gora-otd> + + <class name="org.apache.hadoop.chukwa.datacollection.writer.gora.ChukwaChunk" keyClass="java.lang.String" document="chukwachunk"> + <!-- log fields --> + <field name="source" docfield="source" type="string"/> + <field name="tags" docfield="tags" type="string"/> + <field name="datatype" docfield="dtype" type="string"/> + <field name="sequenceID" docfield="seqID" type="double"/> + <field name="name" docfield="name" type="string"/> + <field name="data" docfield="data" type="binary"/> + </class> + +</gora-otd> http://git-wip-us.apache.org/repos/asf/chukwa/blob/cac335a1/src/main/resources/gora-solr-mapping.xml ---------------------------------------------------------------------- diff --git a/src/main/resources/gora-solr-mapping.xml b/src/main/resources/gora-solr-mapping.xml new file mode 100644 index 0000000..8796c29 --- /dev/null +++ b/src/main/resources/gora-solr-mapping.xml @@ -0,0 +1,32 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<!-- + More information on gora-solr configuration and mapping's can be found + at http://gora.apache.org/current/gora-solr.html +--> +<gora-otd> + <class name="org.apache.hadoop.chukwa.datacollection.writer.gora.ChukwaChunk" keyClass="java.lang.String" table="chukwachunk"> + <primarykey column="TBD"/> + <field name="source" column="source" /> + <field name="tags" column="tags" /> + <field name="datatype" column="datatype" /> + <field name="sequenceID" column="sequenceID" /> + <field name="name" column="name" /> + <field name="data" column="data" /> + </class> +</gora-otd> http://git-wip-us.apache.org/repos/asf/chukwa/blob/cac335a1/src/main/resources/gora.properties ---------------------------------------------------------------------- diff --git a/src/main/resources/gora.properties b/src/main/resources/gora.properties new file mode 100644 index 0000000..6c2c04e --- /dev/null +++ b/src/main/resources/gora.properties @@ -0,0 +1,110 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# MORE DOCUMENTATION ON ALL OF THESE PROPERTIES CAN BE FOUND AT +# http://gora.apache.org/current/ + +#gora.datastore.default=org.apache.gora.mock.store.MockDataStore +#gora.datastore.autocreateschema=true + +############################### +# Default SqlStore properties # +############################### + +#gora.sqlstore.jdbc.driver=org.hsqldb.jdbc.JDBCDriver +#gora.sqlstore.jdbc.url=jdbc:hsqldb:hsql://localhost/nutchtest +#gora.sqlstore.jdbc.user=sa +#gora.sqlstore.jdbc.password= + +################################ +# Default AvroStore properties # +################################ + +# gora.avrostore.codec.type=BINARY||JSON +# gora.avrostore.output.path=file:///tmp/gora.avrostore.test.output + +################################ +# DatafileAvroStore properties # +################################ +# DataFileAvroStore is file based store which uses Avro's +# DataFile{Writer,Reader}'s as a backend. This datastore supports +# mapreduce. + +# gora.datafileavrostore.###= + +######################### +# HBaseStore properties # +######################### +# HBase requires that the Configuration has a valid "hbase.zookeeper.quorum" +# property. It should be included within hbase-site.xml on the classpath. When +# this property is omitted, it expects Zookeeper to run on localhost:2181. + +# To greatly improve scan performance, increase the hbase-site Configuration +# property "hbase.client.scanner.caching". This sets the number of rows to grab +# per request. + +# HBase autoflushing. Enabling autoflush decreases write performance. +# Available since Gora 0.2. Defaults to disabled. +# hbase.client.autoflush.default=false + +# HBase client cache that improves the scan in HBase (default 0) +# gora.datastore.scanner.caching=1000 + +############################# +# CassandraStore properties # +############################# + +# gora.cassandrastore.servers=localhost:9160 + +####################### +# MemStore properties # +####################### +# This is a memory based {@link DataStore} implementation for tests. + +# gora.memstore.###= + +############################ +# AccumuloStore properties # +############################ +#gora.datastore.default=org.apache.gora.accumulo.store.AccumuloStore +#gora.accumulo.mapping.file=gora-accumulo-mapping.xml +#gora.datastore.accumulo.mock=true +#gora.datastore.accumulo.instance=a14 +#gora.datastore.accumulo.zookeepers=localhost +#gora.datastore.accumulo.user=root +#gora.datastore.accumulo.password=secret + +############################ +# SolrStore properties # +############################ +#gora.datastore.default=org.apache.gora.solr.store.SolrStore +#gora.solrstore.solr.url=http://localhost:9876/solr +#gora.solrstore.solr.config=solrconfig.xml +#gora.solrstore.solr.schema=gora-solr-schema.xml +#gora.solrstore.solr.batchSize=100 +#gora.solrstore.solr.solrjserver=http +#gora.solrstore.solr.commitWithin=1000 +#gora.solrstore.solr.resultsSize=100 + +############################ +# MongoDBStore properties # +############################ +#gora.datastore.default=org.apache.gora.mongodb.store.MongoStore +#gora.mongodb.override_hadoop_configuration=false +#gora.mongodb.mapping.file=/gora-mongodb-mapping.xml +#gora.mongodb.servers=localhost:27017 +#gora.mongodb.db=mytestdatabase +#gora.mongodb.login=login +#gora.mongodb.secret=secret
