Author: philz
Date: Tue Jan 26 18:51:54 2010
New Revision: 903362
URL: http://svn.apache.org/viewvc?rev=903362&view=rev
Log:
AVRO-368. Reserve avro.* in object container files, and
rename existing reserved words.
Added:
hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/TestDataFileMeta.java
Modified:
hadoop/avro/trunk/CHANGES.txt
hadoop/avro/trunk/doc/src/content/xdocs/spec.xml
hadoop/avro/trunk/lang/java/src/java/org/apache/avro/file/DataFileConstants.java
hadoop/avro/trunk/lang/java/src/java/org/apache/avro/file/DataFileWriter.java
hadoop/avro/trunk/lang/java/src/java/org/apache/avro/file/DeflateCodec.java
hadoop/avro/trunk/lang/java/src/java/org/apache/avro/file/NullCodec.java
hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/tool/TestDataFileTools.java
Modified: hadoop/avro/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/avro/trunk/CHANGES.txt?rev=903362&r1=903361&r2=903362&view=diff
==============================================================================
--- hadoop/avro/trunk/CHANGES.txt (original)
+++ hadoop/avro/trunk/CHANGES.txt Tue Jan 26 18:51:54 2010
@@ -74,6 +74,9 @@
AVRO-135. Add compression to data files. (philz)
+ AVRO-368. Reserve avro.* in object container files, and
+ rename existing reserved words. (philz)
+
IMPROVEMENTS
AVRO-157. Changes from code review comments for C++. (sbanacho)
Modified: hadoop/avro/trunk/doc/src/content/xdocs/spec.xml
URL:
http://svn.apache.org/viewvc/hadoop/avro/trunk/doc/src/content/xdocs/spec.xml?rev=903362&r1=903361&r2=903362&view=diff
==============================================================================
--- hadoop/avro/trunk/doc/src/content/xdocs/spec.xml (original)
+++ hadoop/avro/trunk/doc/src/content/xdocs/spec.xml Tue Jan 26 18:51:54 2010
@@ -619,11 +619,12 @@
<li>For each pair, a string key and bytes value.</li>
</ul>
- <p>The following file metadata properties are reserved:</p>
+ <p>All metadata properties that start with "avro." are reserved.
+ The following file metadata properties are currently used:</p>
<ul>
- <li><strong>schema</strong> contains the schema of objects
+ <li><strong>avro.schema</strong> contains the schema of objects
stored in the file, as JSON data (required).</li>
- <li><strong>codec</strong> the name of the compression codec
+ <li><strong>avro.codec</strong> the name of the compression codec
used to compress blocks, as a string. Implementations
are required to support the following codecs: "null" and "deflate".
If codec is absent, it is assumed to be "null". The codecs
Modified:
hadoop/avro/trunk/lang/java/src/java/org/apache/avro/file/DataFileConstants.java
URL:
http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/java/org/apache/avro/file/DataFileConstants.java?rev=903362&r1=903361&r2=903362&view=diff
==============================================================================
---
hadoop/avro/trunk/lang/java/src/java/org/apache/avro/file/DataFileConstants.java
(original)
+++
hadoop/avro/trunk/lang/java/src/java/org/apache/avro/file/DataFileConstants.java
Tue Jan 26 18:51:54 2010
@@ -30,8 +30,9 @@
public static final int SYNC_SIZE = 16;
public static final int DEFAULT_SYNC_INTERVAL = 1000*SYNC_SIZE;
- public static final String SCHEMA = "schema";
- public static final String COUNT = "count";
- public static final String CODEC = "codec";
+ public static final String SCHEMA = "avro.schema";
+ public static final String CODEC = "avro.codec";
public static final String NULL_CODEC = "null";
+ public static final String DEFLATE_CODEC = "deflate";
+
}
Modified:
hadoop/avro/trunk/lang/java/src/java/org/apache/avro/file/DataFileWriter.java
URL:
http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/java/org/apache/avro/file/DataFileWriter.java?rev=903362&r1=903361&r2=903362&view=diff
==============================================================================
---
hadoop/avro/trunk/lang/java/src/java/org/apache/avro/file/DataFileWriter.java
(original)
+++
hadoop/avro/trunk/lang/java/src/java/org/apache/avro/file/DataFileWriter.java
Tue Jan 26 18:51:54 2010
@@ -35,7 +35,6 @@
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.Map;
import org.apache.avro.AvroRuntimeException;
@@ -72,12 +71,6 @@
private boolean isOpen;
private Codec codec;
-
- private static final HashSet<String> RESERVED_META = new HashSet<String>();
- static {
- RESERVED_META.add("codec");
- RESERVED_META.add("schema");
- }
/** Construct a writer, not yet open. */
public DataFileWriter(DatumWriter<D> dout) {
@@ -98,7 +91,7 @@
public DataFileWriter<D> setCodec(CodecFactory c) {
assertNotOpen();
this.codec = c.createInstance();
- setMetaInternal("codec", codec.getName());
+ setMetaInternal(DataFileConstants.CODEC, codec.getName());
return this;
}
@@ -153,7 +146,7 @@
this.schema = reader.getSchema();
this.sync = reader.sync;
this.meta.putAll(reader.meta);
- byte[] codecBytes = this.meta.get("codec");
+ byte[] codecBytes = this.meta.get(DataFileConstants.CODEC);
if (codecBytes != null) {
String strCodec = new String(codecBytes, "UTF-8");
this.codec = CodecFactory.fromString(strCodec).createInstance();
@@ -208,12 +201,16 @@
/** Set a metadata property. */
public DataFileWriter<D> setMeta(String key, byte[] value) {
- if (RESERVED_META.contains(key)) {
+ if (isReserved(key)) {
throw new AvroRuntimeException("Cannot set reserved meta key: " + key);
}
return setMetaInternal(key, value);
}
+ private boolean isReserved(String key) {
+ return key.startsWith("avro.");
+ }
+
/** Set a metadata property. */
public DataFileWriter<D> setMeta(String key, String value) {
try {
Modified:
hadoop/avro/trunk/lang/java/src/java/org/apache/avro/file/DeflateCodec.java
URL:
http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/java/org/apache/avro/file/DeflateCodec.java?rev=903362&r1=903361&r2=903362&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/java/src/java/org/apache/avro/file/DeflateCodec.java
(original)
+++ hadoop/avro/trunk/lang/java/src/java/org/apache/avro/file/DeflateCodec.java
Tue Jan 26 18:51:54 2010
@@ -66,7 +66,7 @@
@Override
String getName() {
- return "deflate";
+ return DataFileConstants.DEFLATE_CODEC;
}
@Override
Modified:
hadoop/avro/trunk/lang/java/src/java/org/apache/avro/file/NullCodec.java
URL:
http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/java/org/apache/avro/file/NullCodec.java?rev=903362&r1=903361&r2=903362&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/java/src/java/org/apache/avro/file/NullCodec.java
(original)
+++ hadoop/avro/trunk/lang/java/src/java/org/apache/avro/file/NullCodec.java
Tue Jan 26 18:51:54 2010
@@ -42,7 +42,7 @@
@Override
String getName() {
- return "null";
+ return DataFileConstants.NULL_CODEC;
}
@Override
Added:
hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/TestDataFileMeta.java
URL:
http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/TestDataFileMeta.java?rev=903362&view=auto
==============================================================================
---
hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/TestDataFileMeta.java
(added)
+++
hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/TestDataFileMeta.java
Tue Jan 26 18:51:54 2010
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.avro;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+
+import org.apache.avro.Schema.Type;
+import org.apache.avro.file.DataFileStream;
+import org.apache.avro.file.DataFileWriter;
+import org.apache.avro.generic.GenericDatumReader;
+import org.apache.avro.generic.GenericDatumWriter;
+import org.junit.Test;
+
+public class TestDataFileMeta {
+ @Test(expected=AvroRuntimeException.class)
+ public void testUseReservedMeta() {
+ DataFileWriter<?> w = new DataFileWriter<Object>(new
GenericDatumWriter<Object>());
+ w.setMeta("avro.foo", "bar");
+ }
+
+ @Test()
+ public void testUseMeta() throws IOException {
+ DataFileWriter<?> w = new DataFileWriter<Object>(new
GenericDatumWriter<Object>());
+ File f = AvroTestUtil.tempFile("testDataFileMeta.avro");
+ w.setMeta("hello", "bar");
+ w.create(Schema.create(Type.NULL), f);
+ w.close();
+
+ DataFileStream<Void> r = new DataFileStream<Void>(new FileInputStream(f),
new GenericDatumReader<Void>());
+ assertEquals("bar", r.getMetaString("hello"));
+ }
+
+ @Test(expected=AvroRuntimeException.class)
+ public void testUseMetaAfterCreate() throws IOException {
+ DataFileWriter<?> w = new DataFileWriter<Object>(new
GenericDatumWriter<Object>());
+ w.create(Schema.create(Type.NULL), new ByteArrayOutputStream());
+ w.setMeta("foo", "bar");
+ }
+
+
+}
Modified:
hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/tool/TestDataFileTools.java
URL:
http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/tool/TestDataFileTools.java?rev=903362&r1=903361&r2=903362&view=diff
==============================================================================
---
hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/tool/TestDataFileTools.java
(original)
+++
hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/tool/TestDataFileTools.java
Tue Jan 26 18:51:54 2010
@@ -49,7 +49,8 @@
@BeforeClass
public static void writeSampleFile() throws IOException {
- sampleFile = AvroTestUtil.tempFile(TestDataFileTools.class + ".avro");
+ sampleFile = AvroTestUtil.tempFile(
+ TestDataFileTools.class.getName() + ".avro");
schema = Schema.create(Type.INT);
DataFileWriter<Object> writer
@@ -133,7 +134,7 @@
}
assertEquals(COUNT, i);
assertEquals(schema, fileReader.getSchema());
- assertEquals(expectedCodec, fileReader.getMetaString("codec"));
+ assertEquals(expectedCodec, fileReader.getMetaString("avro.codec"));
}
@Test
