This is an automated email from the ASF dual-hosted git repository.
mgrigorov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/avro.git
The following commit(s) were added to refs/heads/master by this push:
new 7a85764 AVRO-3407: Test for user metadata in the interop tests (#1552)
7a85764 is described below
commit 7a8576475f776109d060ffdf22e876b754ac3ab4
Author: Martin Grigorov <[email protected]>
AuthorDate: Sat Feb 19 09:40:22 2022 +0200
AVRO-3407: Test for user metadata in the interop tests (#1552)
* AVRO-3407: Test for user metadata in the interop tests
Signed-off-by: Martin Tzvetanov Grigorov <[email protected]>
* Add user_metadats interop tests in C# (#1554)
Co-authored-by: Zoltan Csizmadia <[email protected]>
Co-authored-by: Zoltan Csizmadia <[email protected]>
Co-authored-by: Zoltan Csizmadia <[email protected]>
---
lang/csharp/src/apache/test/Interop/InteropDataGenerator.cs | 1 +
lang/csharp/src/apache/test/Interop/InteropDataTests.cs | 5 +++++
.../avro/src/main/java/org/apache/avro/util/RandomData.java | 3 +--
.../src/test/java/org/apache/avro/DataFileInteropTest.java | 7 +++----
lang/perl/share/interop-data-generate | 3 +--
lang/perl/xt/interop.t | 13 +++----------
lang/py/avro/test/gen_interop_data.py | 1 +
lang/py/avro/test/test_datafile_interop.py | 7 ++++++-
lang/rust/examples/generate_interop_data.rs | 3 +--
lang/rust/examples/test_interop_data.rs | 13 ++-----------
10 files changed, 24 insertions(+), 32 deletions(-)
diff --git a/lang/csharp/src/apache/test/Interop/InteropDataGenerator.cs
b/lang/csharp/src/apache/test/Interop/InteropDataGenerator.cs
index 10c06f7..7aa10c0 100644
--- a/lang/csharp/src/apache/test/Interop/InteropDataGenerator.cs
+++ b/lang/csharp/src/apache/test/Interop/InteropDataGenerator.cs
@@ -86,6 +86,7 @@ namespace Avro.Test.Interop
var codec = Codec.CreateCodecFromString(codecName);
using (var dataFileWriter =
DataFileWriter<GenericRecord>.OpenWriter(datumWriter, outputPath, codec))
{
+ dataFileWriter.SetMeta("user_metadata", "someByteArray");
dataFileWriter.Append(record);
}
}
diff --git a/lang/csharp/src/apache/test/Interop/InteropDataTests.cs
b/lang/csharp/src/apache/test/Interop/InteropDataTests.cs
index 7215df4..4f66a7e 100644
--- a/lang/csharp/src/apache/test/Interop/InteropDataTests.cs
+++ b/lang/csharp/src/apache/test/Interop/InteropDataTests.cs
@@ -48,6 +48,11 @@ namespace Avro.Test.Interop
using(var reader =
DataFileReader<GenericRecord>.OpenReader(avroFile))
{
int i = 0;
+ string userMetadata =
reader.GetMetaString("user_metadata");
+ if (userMetadata != null)
+ {
+ Assert.AreEqual("someByteArray", userMetadata);
+ }
foreach (var record in reader.NextEntries)
{
i++;
diff --git a/lang/java/avro/src/main/java/org/apache/avro/util/RandomData.java
b/lang/java/avro/src/main/java/org/apache/avro/util/RandomData.java
index 12b8a7b..0f364b7 100644
--- a/lang/java/avro/src/main/java/org/apache/avro/util/RandomData.java
+++ b/lang/java/avro/src/main/java/org/apache/avro/util/RandomData.java
@@ -170,8 +170,7 @@ public class RandomData implements Iterable<Object> {
Schema sch = new Schema.Parser().parse(new File(args[0]));
try (DataFileWriter<Object> writer = new DataFileWriter<>(new
GenericDatumWriter<>())) {
writer.setCodec(CodecFactory.fromString(args.length >= 4 ? args[3] :
"null"));
- writer.setMeta("stringKey", "stringValue");
- writer.setMeta("bytesKey",
"bytesValue".getBytes(StandardCharsets.UTF_8));
+ writer.setMeta("user_metadata",
"someByteArray".getBytes(StandardCharsets.UTF_8));
writer.create(sch, new File(args[1]));
for (Object datum : new RandomData(sch, Integer.parseInt(args[2]))) {
diff --git
a/lang/java/ipc/src/test/java/org/apache/avro/DataFileInteropTest.java
b/lang/java/ipc/src/test/java/org/apache/avro/DataFileInteropTest.java
index 7828532..2ea1e5d 100644
--- a/lang/java/ipc/src/test/java/org/apache/avro/DataFileInteropTest.java
+++ b/lang/java/ipc/src/test/java/org/apache/avro/DataFileInteropTest.java
@@ -86,10 +86,9 @@ public class DataFileInteropTest {
try (DataFileReader<? extends Object> reader = (DataFileReader<? extends
Object>) DataFileReader.openReader(f,
provider.get())) {
- // Ignore avro.schema & avro.codec. Some SDKs do not support user
metadata.
- if (reader.getMetaKeys().size() > 2) {
- assertEquals("stringValue", reader.getMetaString("stringKey"));
- assertArrayEquals("bytesValue".getBytes(StandardCharsets.UTF_8),
reader.getMeta("bytesKey"));
+ byte[] user_metadata = reader.getMeta("user_metadata");
+ if (user_metadata != null) {
+ assertArrayEquals("someByteArray".getBytes(StandardCharsets.UTF_8),
user_metadata);
}
int i = 0;
diff --git a/lang/perl/share/interop-data-generate
b/lang/perl/share/interop-data-generate
index 5d2cd79..0144e71 100644
--- a/lang/perl/share/interop-data-generate
+++ b/lang/perl/share/interop-data-generate
@@ -55,8 +55,7 @@ my $datum = {
};
my $metadata = {
- stringKey => 'stringValue',
- bytesKey => 'bytesValue'
+ user_metadata => 'someByteArray'
};
while (my ($codec, $enabled) = each(%Avro::DataFile::ValidCodec)) {
diff --git a/lang/perl/xt/interop.t b/lang/perl/xt/interop.t
index d50ca6c..8bc2a84 100644
--- a/lang/perl/xt/interop.t
+++ b/lang/perl/xt/interop.t
@@ -24,11 +24,6 @@ use IO::File;
use_ok 'Avro::DataFile';
use_ok 'Avro::DataFileReader';
-my $expected_metadata = {
- stringKey => 'stringValue',
- bytesKey => 'bytesValue'
-};
-
for my $path (glob '../../build/interop/data/*.avro') {
my $fn = basename($path);
substr($fn, rindex $fn, '.') = '';
@@ -44,12 +39,10 @@ for my $path (glob '../../build/interop/data/*.avro') {
my $reader = Avro::DataFileReader->new(fh => $fh);
my $metadata = $reader->metadata;
- if (exists $metadata->{stringKey}) {
- is($metadata->{stringKey}, $expected_metadata->{stringKey}, "check
user metadata: stringKey ");
- }
- if (exists $metadata->{bytesKey}) {
- is($metadata->{bytesKey}, join('', $expected_metadata->{bytesKey}),
"check user metadata: bytesKey ");
+ if (exists $metadata->{user_metadata}) {
+ is($metadata->{user_metadata}, 'someByteArray', "check user metadata");
}
+
diag("Succeeded: ${path}");
}
diff --git a/lang/py/avro/test/gen_interop_data.py
b/lang/py/avro/test/gen_interop_data.py
index d1f3c0e..1993b01 100644
--- a/lang/py/avro/test/gen_interop_data.py
+++ b/lang/py/avro/test/gen_interop_data.py
@@ -54,6 +54,7 @@ DATUM = {
def gen_data(codec: str, datum_writer: avro.io.DatumWriter, interop_schema:
avro.schema.Schema) -> bytes:
with io.BytesIO() as file_, avro.datafile.DataFileWriter(file_,
datum_writer, interop_schema, codec=codec) as dfw:
+ dfw.set_meta("user_metadata", b"someByteArray")
dfw.append(DATUM)
dfw.flush()
return file_.getvalue()
diff --git a/lang/py/avro/test/test_datafile_interop.py
b/lang/py/avro/test/test_datafile_interop.py
index aeb6c21..d9e4c69 100644
--- a/lang/py/avro/test/test_datafile_interop.py
+++ b/lang/py/avro/test/test_datafile_interop.py
@@ -37,11 +37,16 @@ class TestDataFileInterop(unittest.TestCase):
for filename in _INTEROP_DATA_DIR.iterdir():
self.assertGreater(os.stat(filename).st_size, 0)
base_ext = filename.stem.split("_", 1)
- if len(base_ext) < 2 or base_ext[1] not in
avro.codecs.KNOWN_CODECS:
+ if len(base_ext) > 1 and base_ext[1] not in
avro.codecs.KNOWN_CODECS:
print(f"SKIPPING {filename} due to an unsupported codec\n")
continue
i = None
with self.subTest(filename=filename),
avro.datafile.DataFileReader(filename.open("rb"), avro.io.DatumReader()) as dfr:
+
+ user_metadata = dfr.get_meta("user_metadata")
+ if user_metadata is not None:
+ self.assertEqual(user_metadata, b"someByteArray")
+
for i, datum in enumerate(cast(avro.datafile.DataFileReader,
dfr), 1):
self.assertIsNotNone(datum)
self.assertIsNotNone(i)
diff --git a/lang/rust/examples/generate_interop_data.rs
b/lang/rust/examples/generate_interop_data.rs
index 3d91140..3bbbb3a 100644
--- a/lang/rust/examples/generate_interop_data.rs
+++ b/lang/rust/examples/generate_interop_data.rs
@@ -101,8 +101,7 @@ fn main() -> anyhow::Result<()> {
}
fn write_user_metadata(writer: &mut Writer<Vec<u8>>) -> anyhow::Result<()> {
- writer.add_user_metadata("stringKey".to_string(), "stringValue")?;
- writer.add_user_metadata("bytesKey".to_string(), b"bytesValue")?;
+ writer.add_user_metadata("user_metadata".to_string(), b"someByteArray")?;
Ok(())
}
diff --git a/lang/rust/examples/test_interop_data.rs
b/lang/rust/examples/test_interop_data.rs
index 85d59de..82d2aa1 100644
--- a/lang/rust/examples/test_interop_data.rs
+++ b/lang/rust/examples/test_interop_data.rs
@@ -19,7 +19,8 @@ use apache_avro::Reader;
use std::{collections::HashMap, ffi::OsStr, fs::File};
fn main() -> anyhow::Result<()> {
- let expected_user_metadata: HashMap<String, Vec<u8>> =
create_expected_user_metadata();
+ let mut expected_user_metadata: HashMap<String, Vec<u8>> = HashMap::new();
+ expected_user_metadata.insert("user_metadata".to_string(),
b"someByteArray".to_vec());
let data_dir = std::fs::read_dir("../../build/interop/data/")
.expect("Unable to list the interop data directory");
@@ -63,16 +64,6 @@ fn main() -> anyhow::Result<()> {
}
}
-fn create_expected_user_metadata() -> HashMap<String, Vec<u8>> {
- let mut user_metadata: HashMap<String, Vec<u8>> = HashMap::new();
- user_metadata.insert(
- "stringKey".to_string(),
- "stringValue".to_string().into_bytes(),
- );
- user_metadata.insert("bytesKey".to_string(), b"bytesValue".to_vec());
- user_metadata
-}
-
fn test_user_metadata(reader: &Reader<&File>, expected_user_metadata:
&HashMap<String, Vec<u8>>) {
let user_metadata = reader.user_metadata();
if !user_metadata.is_empty() {
