Repository: incubator-reef Updated Branches: refs/heads/master c43610fb1 -> 0850674bd
[REEF-620] Adding language property in Avro Configuration When the code in a language receives a configuration from another, it needs to know which language the configuration was created so that to determine which alias to choose. This PR added a language property in Avro Configuration to carry this information. It also modified the places where the AvroConfiguration object is created in both Java and C#. The evaluator.conf used in testing is a newly generated one from test runtime which contains language information. Test is updated to read language property from it after deserialization. JIRA: [REEF-620](https://issues.apache.org/jira/browse/REEF-620) Pull Request: This closes #395 Project: http://git-wip-us.apache.org/repos/asf/incubator-reef/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-reef/commit/0850674b Tree: http://git-wip-us.apache.org/repos/asf/incubator-reef/tree/0850674b Diff: http://git-wip-us.apache.org/repos/asf/incubator-reef/diff/0850674b Branch: refs/heads/master Commit: 0850674bd476c21de2c714fe378de7aaafb229a5 Parents: c43610f Author: Julia Wang <[email protected]> Authored: Thu Aug 20 16:41:46 2015 -0700 Committer: Markus Weimer <[email protected]> Committed: Fri Aug 21 15:09:05 2015 -0700 ---------------------------------------------------------------------- .../EvaluatorConfigurationsTests.cs | 7 +++---- .../ConfigFiles/evaluator.conf | Bin 3309 -> 2997 bytes .../Configuration/TestAvroConfiguration.cs | 2 +- .../AvroConfiguration.cs | 6 +++++- .../Formats/AvroConfigurationSerializer.cs | 5 ++++- .../tang/src/main/avro/configuration.avsc | 7 ++++--- .../formats/AvroConfigurationSerializer.java | 5 +++-- 7 files changed, 20 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/0850674b/lang/cs/Org.Apache.REEF.Evaluator.Tests/EvaluatorConfigurationsTests.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Evaluator.Tests/EvaluatorConfigurationsTests.cs b/lang/cs/Org.Apache.REEF.Evaluator.Tests/EvaluatorConfigurationsTests.cs index 8c3e010..0f3f2dd 100644 --- a/lang/cs/Org.Apache.REEF.Evaluator.Tests/EvaluatorConfigurationsTests.cs +++ b/lang/cs/Org.Apache.REEF.Evaluator.Tests/EvaluatorConfigurationsTests.cs @@ -17,8 +17,6 @@ * under the License. */ -using System; -using System.Globalization; using Microsoft.VisualStudio.TestTools.UnitTesting; using Org.Apache.REEF.Common.Runtime.Evaluator.Utils; using Org.Apache.REEF.Tang.Formats; @@ -46,9 +44,9 @@ namespace Org.Apache.REEF.Evaluator.Tests Logger.Log(Level.Info, "ApplicationId = " + aId); Logger.Log(Level.Info, "ErrorHandlerRID = " + rId); - Assert.IsTrue(eId.Equals("Node-1-1437686223482")); + Assert.IsTrue(eId.Equals("Node-1-1440108430564")); Assert.IsTrue(aId.Equals("REEF_LOCAL_RUNTIME")); - Assert.IsTrue(rId.Equals("socket://10.130.68.76:9267")); + Assert.IsTrue(rId.Equals("socket://10.130.68.76:9528")); var contextConfigString = evaluatorConfigurations.RootContextConfigurationString; var serviceConfigString = evaluatorConfigurations.RootServiceConfigurationString; @@ -66,6 +64,7 @@ namespace Org.Apache.REEF.Evaluator.Tests var avroConfiguration = serializer.AvroDeseriaizeFromFile("evaluator.conf"); Assert.IsNotNull(avroConfiguration); + Assert.AreEqual(avroConfiguration.language, AvroConfigurationSerializer.Java); foreach (var b in avroConfiguration.Bindings) { http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/0850674b/lang/cs/Org.Apache.REEF.Examples/ConfigFiles/evaluator.conf ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Examples/ConfigFiles/evaluator.conf b/lang/cs/Org.Apache.REEF.Examples/ConfigFiles/evaluator.conf index 1136a5a..a3578bd 100644 Binary files a/lang/cs/Org.Apache.REEF.Examples/ConfigFiles/evaluator.conf and b/lang/cs/Org.Apache.REEF.Examples/ConfigFiles/evaluator.conf differ http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/0850674b/lang/cs/Org.Apache.REEF.Tang.Tests/Configuration/TestAvroConfiguration.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Tang.Tests/Configuration/TestAvroConfiguration.cs b/lang/cs/Org.Apache.REEF.Tang.Tests/Configuration/TestAvroConfiguration.cs index d9392ab..e682342 100644 --- a/lang/cs/Org.Apache.REEF.Tang.Tests/Configuration/TestAvroConfiguration.cs +++ b/lang/cs/Org.Apache.REEF.Tang.Tests/Configuration/TestAvroConfiguration.cs @@ -62,7 +62,7 @@ namespace Org.Apache.REEF.Tang.Tests.Configuration e2.value = "b1=b2"; b.Add(e1); b.Add(e2); - var a = new AvroConfiguration(b); + var a = new AvroConfiguration(AvroConfigurationSerializer.Cs, b); return a; } } http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/0850674b/lang/cs/Org.Apache.REEF.Tang/Formats/AvroConfigurationDataContract/AvroConfiguration.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Tang/Formats/AvroConfigurationDataContract/AvroConfiguration.cs b/lang/cs/Org.Apache.REEF.Tang/Formats/AvroConfigurationDataContract/AvroConfiguration.cs index 9a4fa9d..98b2b43 100644 --- a/lang/cs/Org.Apache.REEF.Tang/Formats/AvroConfigurationDataContract/AvroConfiguration.cs +++ b/lang/cs/Org.Apache.REEF.Tang/Formats/AvroConfigurationDataContract/AvroConfiguration.cs @@ -27,12 +27,13 @@ namespace Org.Apache.REEF.Tang.Formats.AvroConfigurationDataContract [DataContract(Name = "AvroConfiguration", Namespace = "org.apache.reef.tang.formats.avro")] public class AvroConfiguration { - public AvroConfiguration(ISet<ConfigurationEntry> bindings) + public AvroConfiguration(string language, ISet<ConfigurationEntry> bindings) { // TODO: [REEF-276] AvroSerializer currently does not serialize HashSets // correctly, so using a List for now to get around the issue. // An ISet is still passed in to guarantee configuration uniqueness. this.Bindings = new List<ConfigurationEntry>(bindings); + this.language = language; } public AvroConfiguration() @@ -40,6 +41,9 @@ namespace Org.Apache.REEF.Tang.Formats.AvroConfigurationDataContract } [DataMember] + public string language { get; set; } + + [DataMember] public List<ConfigurationEntry> Bindings { get; set; } public static AvroConfiguration GetAvroConfigurationFromEmbeddedString(string jsonString) http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/0850674b/lang/cs/Org.Apache.REEF.Tang/Formats/AvroConfigurationSerializer.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Tang/Formats/AvroConfigurationSerializer.cs b/lang/cs/Org.Apache.REEF.Tang/Formats/AvroConfigurationSerializer.cs index 64331fc..569538b 100644 --- a/lang/cs/Org.Apache.REEF.Tang/Formats/AvroConfigurationSerializer.cs +++ b/lang/cs/Org.Apache.REEF.Tang/Formats/AvroConfigurationSerializer.cs @@ -53,6 +53,9 @@ namespace Org.Apache.REEF.Tang.Formats public class AvroConfigurationSerializer : IConfigurationSerializer { + public const string Java = "Java"; + public const string Cs = "Cs"; + private static readonly Logger LOGGER = Logger.GetLogger(typeof(AvroConfigurationResolver)); [Inject] @@ -252,7 +255,7 @@ namespace Org.Apache.REEF.Tang.Formats l.Add(new ConfigurationEntry(e.Key.GetFullName(), val)); } - return new AvroConfiguration(l); + return new AvroConfiguration(Cs, l); } private byte[] AvroSerialize(AvroConfiguration obj) http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/0850674b/lang/java/reef-tang/tang/src/main/avro/configuration.avsc ---------------------------------------------------------------------- diff --git a/lang/java/reef-tang/tang/src/main/avro/configuration.avsc b/lang/java/reef-tang/tang/src/main/avro/configuration.avsc index a942e46..888b257 100644 --- a/lang/java/reef-tang/tang/src/main/avro/configuration.avsc +++ b/lang/java/reef-tang/tang/src/main/avro/configuration.avsc @@ -22,8 +22,8 @@ "type":"record", "name":"ConfigurationEntry", "fields":[ - {"name":"key","type":"string"}, - {"name":"value","type":"string"} + {"name":"key","type":"string"}, + {"name":"value","type":"string"} ] }, { @@ -31,7 +31,8 @@ "type":"record", "name":"AvroConfiguration", "fields":[ - {"name":"Bindings","type":{"type":"array", "items":"ConfigurationEntry"}} + {"name":"language","type":"string"}, + {"name":"Bindings","type":{"type":"array", "items":"ConfigurationEntry"}} ] } ] http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/0850674b/lang/java/reef-tang/tang/src/main/java/org/apache/reef/tang/formats/AvroConfigurationSerializer.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-tang/tang/src/main/java/org/apache/reef/tang/formats/AvroConfigurationSerializer.java b/lang/java/reef-tang/tang/src/main/java/org/apache/reef/tang/formats/AvroConfigurationSerializer.java index 71a7e4d..229f76c 100644 --- a/lang/java/reef-tang/tang/src/main/java/org/apache/reef/tang/formats/AvroConfigurationSerializer.java +++ b/lang/java/reef-tang/tang/src/main/java/org/apache/reef/tang/formats/AvroConfigurationSerializer.java @@ -57,6 +57,8 @@ public final class AvroConfigurationSerializer implements ConfigurationSerialize * Copied from <code>org.apache.avro.io.JsonDecoder.CHARSET</code> */ private static final String JSON_CHARSET = "ISO-8859-1"; + public static final String JAVA = "Java"; + public static final String CS = "Cs"; @Inject public AvroConfigurationSerializer() { @@ -185,9 +187,8 @@ public final class AvroConfigurationSerializer implements ConfigurationSerialize if (configuration.getBoundLists() != null && !configuration.getBoundLists().isEmpty()) { throw new NotImplementedException("List serialization/deserialization is not supported"); } - - return AvroConfiguration.newBuilder().setBindings(configurationEntries).build(); + return AvroConfiguration.newBuilder().setLanguage(JAVA).setBindings(configurationEntries).build(); } @Override
