http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/sap/src/test/java/org/apache/zeppelin/sap/universe/UniverseUtilTest.java ---------------------------------------------------------------------- diff --git a/sap/src/test/java/org/apache/zeppelin/sap/universe/UniverseUtilTest.java b/sap/src/test/java/org/apache/zeppelin/sap/universe/UniverseUtilTest.java index af80c3a..81a027e 100644 --- a/sap/src/test/java/org/apache/zeppelin/sap/universe/UniverseUtilTest.java +++ b/sap/src/test/java/org/apache/zeppelin/sap/universe/UniverseUtilTest.java @@ -17,6 +17,12 @@ package org.apache.zeppelin.sap.universe; +import org.junit.Before; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; @@ -24,11 +30,6 @@ import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import java.util.HashMap; -import java.util.Map; -import org.junit.Before; -import org.junit.Test; - public class UniverseUtilTest { private UniverseClient universeClient; @@ -36,368 +37,335 @@ public class UniverseUtilTest { @Before public void beforeTest() throws UniverseException { - universeUtil = new UniverseUtil(); - UniverseInfo universeInfo = new UniverseInfo("1", "testUniverse", "uvx"); - Map<String, UniverseNodeInfo> testUniverseNodes = new HashMap<>(); - testUniverseNodes.put( - "[Dimension].[Test].[name1]", - new UniverseNodeInfo( - "name1id", - "name1", - "dimension", - "Dimension\\Test", - "Dimension|folder\\Test|folder\\name1|dimension")); - testUniverseNodes.put( - "[Dimension].[Test].[name2]", - new UniverseNodeInfo( - "name2id", - "name2", - "dimension", - "Filter\\Test", - "Dimension|folder\\Test|folder\\name2|dimension")); - testUniverseNodes.put( - "[Filter].[name3]", - new UniverseNodeInfo( - "name3id", "name3", "filter", "Filter", "Filter|folder\\name3|filter")); - testUniverseNodes.put( - "[Filter].[name4]", - new UniverseNodeInfo( - "name4id", "name4", "filter", "Filter", "Filter|folder\\name4|filter")); - testUniverseNodes.put( - "[Measure].[name5]", - new UniverseNodeInfo( - "name5id", "name5", "measure", "Measure", "Measure|folder\\name5|measure")); + universeUtil = new UniverseUtil(); + UniverseInfo universeInfo = new UniverseInfo("1", "testUniverse", "uvx"); + Map<String, UniverseNodeInfo> testUniverseNodes = new HashMap<>(); + testUniverseNodes.put("[Dimension].[Test].[name1]", + new UniverseNodeInfo("name1id", "name1", "dimension", "Dimension\\Test", + "Dimension|folder\\Test|folder\\name1|dimension")); + testUniverseNodes.put("[Dimension].[Test].[name2]", + new UniverseNodeInfo("name2id", "name2", "dimension", "Filter\\Test", + "Dimension|folder\\Test|folder\\name2|dimension")); + testUniverseNodes.put("[Filter].[name3]", + new UniverseNodeInfo("name3id", "name3", "filter", "Filter", + "Filter|folder\\name3|filter")); + testUniverseNodes.put("[Filter].[name4]", + new UniverseNodeInfo("name4id", "name4", "filter", "Filter", + "Filter|folder\\name4|filter")); + testUniverseNodes.put("[Measure].[name5]", + new UniverseNodeInfo("name5id", "name5", "measure", "Measure", + "Measure|folder\\name5|measure")); - universeClient = mock(UniverseClient.class); - when(universeClient.getUniverseInfo(anyString())).thenReturn(universeInfo); - when(universeClient.getUniverseNodesInfo(anyString(), anyString())) - .thenReturn(testUniverseNodes); + universeClient = mock(UniverseClient.class); + when(universeClient.getUniverseInfo(anyString())).thenReturn(universeInfo); + when(universeClient.getUniverseNodesInfo(anyString(), anyString())) + .thenReturn(testUniverseNodes); } @Test public void testForConvert() throws UniverseException { - String request = - "universe [testUniverse];\n" - + "select [Measure].[name5]\n" - + "where [Filter].[name3] and [Dimension].[Test].[name2] > 1;"; - UniverseQuery universeQuery = universeUtil.convertQuery(request, universeClient, null); - assertNotNull(universeQuery); - assertNotNull(universeQuery.getUniverseInfo()); - assertEquals( - "<resultObjects>\n" - + "<resultObject path=\"Measure|folder\\name5|measure\" id=\"name5id\"/>\n" - + "</resultObjects>", - universeQuery.getSelect()); - assertEquals( - "<and>\n" - + "<predefinedFilter path=\"Filter|folder\\name3|filter\" id=\"name3id\"/>\n" - + "\n<comparisonFilter path=\"Dimension|folder\\Test|folder\\name2|dimension\"" - + " operator=\"GreaterThan\" id=\"name2id\">\n" - + "<constantOperand>\n" - + "<value>\n" - + "<caption type=\"Numeric\">1</caption>\n" - + "</value>\n" - + "</constantOperand>\n" - + "</comparisonFilter>\n\n" - + "</and>\n", - universeQuery.getWhere()); - assertEquals("testUniverse", universeQuery.getUniverseInfo().getName()); + String request = "universe [testUniverse];\n" + + "select [Measure].[name5]\n" + + "where [Filter].[name3] and [Dimension].[Test].[name2] > 1;"; + UniverseQuery universeQuery = universeUtil.convertQuery(request, universeClient, null); + assertNotNull(universeQuery); + assertNotNull(universeQuery.getUniverseInfo()); + assertEquals("<resultObjects>\n" + + "<resultObject path=\"Measure|folder\\name5|measure\" id=\"name5id\"/>\n" + + "</resultObjects>", universeQuery.getSelect()); + assertEquals("<and>\n" + + "<predefinedFilter path=\"Filter|folder\\name3|filter\" id=\"name3id\"/>\n" + + "\n<comparisonFilter path=\"Dimension|folder\\Test|folder\\name2|dimension\"" + + " operator=\"GreaterThan\" id=\"name2id\">\n" + + "<constantOperand>\n" + + "<value>\n" + + "<caption type=\"Numeric\">1</caption>\n" + + "</value>\n" + + "</constantOperand>\n" + + "</comparisonFilter>\n\n" + + "</and>\n", universeQuery.getWhere()); + assertEquals("testUniverse", universeQuery.getUniverseInfo().getName()); } @Test public void testConvertConditions() throws UniverseException { - String request = - "universe [testUniverse];\n" - + "select [Measure].[name5]\n" - + "where [Filter].[name3] " - + "and [Dimension].[Test].[name2] >= 1 " - + "and [Dimension].[Test].[name2] < 20 " - + "and [Dimension].[Test].[name1] <> 'test' " - + "and [Dimension].[Test].[name1] is not null " - + "and [Measure].[name5] is null" - + "and [Dimension].[Test].[name1] in ('var1', 'v a r 2') " - + "and [Dimension].[Test].[name1] in ('var1','withoutspaces')" - + "and [Dimension].[Test].[name1] in ('one value')" - + "and [Dimension].[Test].[name2] in (1,3,4);"; + String request = "universe [testUniverse];\n" + + "select [Measure].[name5]\n" + + "where [Filter].[name3] " + + "and [Dimension].[Test].[name2] >= 1 " + + "and [Dimension].[Test].[name2] < 20 " + + "and [Dimension].[Test].[name1] <> 'test' " + + "and [Dimension].[Test].[name1] is not null " + + "and [Measure].[name5] is null" + + "and [Dimension].[Test].[name1] in ('var1', 'v a r 2') " + + "and [Dimension].[Test].[name1] in ('var1','withoutspaces')" + + "and [Dimension].[Test].[name1] in ('one value')" + + "and [Dimension].[Test].[name2] in (1,3,4);"; UniverseQuery universeQuery = universeUtil.convertQuery(request, universeClient, null); assertNotNull(universeQuery); - assertEquals( - "<and>\n" - + "<and>\n" - + "<and>\n" - + "<and>\n" - + "<and>\n" - + "<and>\n" - + "<and>\n" - + "<and>\n" - + "<and>\n" - + "<predefinedFilter path=\"Filter|folder\\name3|filter\" id=\"name3id\"/>\n\n" - + "<comparisonFilter path=\"Dimension|folder\\Test|folder\\name2|dimension\"" - + " operator=\"GreaterThanOrEqualTo\" id=\"name2id\">\n" - + "<constantOperand>\n" - + "<value>\n" - + "<caption type=\"Numeric\">1</caption>\n" - + "</value>\n" - + "</constantOperand>\n" - + "</comparisonFilter>\n\n" - + "</and>\n\n" - + "<comparisonFilter path=\"Dimension|folder\\Test|folder\\name2|dimension\"" - + " operator=\"LessThan\" id=\"name2id\">\n" - + "<constantOperand>\n" - + "<value>\n" - + "<caption type=\"Numeric\">20</caption>\n" - + "</value>\n" - + "</constantOperand>\n" - + "</comparisonFilter>\n\n" - + "</and>\n\n" - + "<comparisonFilter path=\"Dimension|folder\\Test|folder\\name1|dimension\"" - + " operator=\"NotEqualTo\" id=\"name1id\">\n" - + "<constantOperand>\n" - + "<value>\n" - + "<caption type=\"String\">test</caption>\n" - + "</value>\n" - + "</constantOperand>\n" - + "</comparisonFilter>\n\n" - + "</and>\n\n" - + "<comparisonFilter id=\"name1id\" path=\"Dimension|folder\\Test|folder\\name1|dimension\"" - + " operator=\"IsNotNull\"/>\n\n" - + "</and>\n\n" - + "<comparisonFilter id=\"name5id\" path=\"Measure|folder\\name5|measure\" operator=\"IsNull\"/>\n\n" - + "</and>\n\n" - + "<comparisonFilter path=\"Dimension|folder\\Test|folder\\name1|dimension\"" - + " operator=\"InList\" id=\"name1id\">\n" - + "<constantOperand>\n" - + "<value>\n" - + "<caption type=\"String\">var1</caption>\n" - + "</value>\n" - + "<value>\n" - + "<caption type=\"String\">v a r 2</caption>\n" - + "</value>\n" - + "</constantOperand>\n" - + "</comparisonFilter>\n\n" - + "</and>\n\n" - + "<comparisonFilter path=\"Dimension|folder\\Test|folder\\name1|dimension\"" - + " operator=\"InList\" id=\"name1id\">\n" - + "<constantOperand>\n" - + "<value>\n" - + "<caption type=\"String\">var1</caption>\n" - + "</value>\n" - + "<value>\n" - + "<caption type=\"String\">withoutspaces</caption>\n" - + "</value>\n" - + "</constantOperand>\n" - + "</comparisonFilter>\n\n" - + "</and>\n\n" - + "<comparisonFilter path=\"Dimension|folder\\Test|folder\\name1|dimension\"" - + " operator=\"InList\" id=\"name1id\">\n" - + "<constantOperand>\n" - + "<value>\n" - + "<caption type=\"String\">one value</caption>\n" - + "</value>\n" - + "</constantOperand>\n" - + "</comparisonFilter>\n\n" - + "</and>\n\n" - + "<comparisonFilter path=\"Dimension|folder\\Test|folder\\name2|dimension\"" - + " operator=\"InList\" id=\"name2id\">\n" - + "<constantOperand>\n" - + "<value>\n" - + "<caption type=\"Numeric\">1</caption>\n" - + "</value>\n" - + "<value>\n" - + "<caption type=\"Numeric\">3</caption>\n" - + "</value>\n" - + "<value>\n" - + "<caption type=\"Numeric\">4</caption>\n" - + "</value>\n" - + "</constantOperand>\n" - + "</comparisonFilter>\n\n" - + "</and>\n", + assertEquals("<and>\n" + + "<and>\n" + + "<and>\n" + + "<and>\n" + + "<and>\n" + + "<and>\n" + + "<and>\n" + + "<and>\n" + + "<and>\n" + + "<predefinedFilter path=\"Filter|folder\\name3|filter\" id=\"name3id\"/>\n\n" + + "<comparisonFilter path=\"Dimension|folder\\Test|folder\\name2|dimension\"" + + " operator=\"GreaterThanOrEqualTo\" id=\"name2id\">\n" + + "<constantOperand>\n" + + "<value>\n" + + "<caption type=\"Numeric\">1</caption>\n" + + "</value>\n" + + "</constantOperand>\n" + + "</comparisonFilter>\n\n" + + "</and>\n\n" + + "<comparisonFilter path=\"Dimension|folder\\Test|folder\\name2|dimension\"" + + " operator=\"LessThan\" id=\"name2id\">\n" + + "<constantOperand>\n" + + "<value>\n" + + "<caption type=\"Numeric\">20</caption>\n" + + "</value>\n" + + "</constantOperand>\n" + + "</comparisonFilter>\n\n" + + "</and>\n\n" + + "<comparisonFilter path=\"Dimension|folder\\Test|folder\\name1|dimension\"" + + " operator=\"NotEqualTo\" id=\"name1id\">\n" + + "<constantOperand>\n" + + "<value>\n" + + "<caption type=\"String\">test</caption>\n" + + "</value>\n" + + "</constantOperand>\n" + + "</comparisonFilter>\n\n" + + "</and>\n\n" + + "<comparisonFilter id=\"name1id\" path=\"Dimension|folder\\Test|folder\\name1|dimension\"" + + " operator=\"IsNotNull\"/>\n\n" + + "</and>\n\n" + + "<comparisonFilter id=\"name5id\" path=\"Measure|folder\\name5|measure\" operator=\"IsNull\"/>\n\n" + + "</and>\n\n" + + "<comparisonFilter path=\"Dimension|folder\\Test|folder\\name1|dimension\"" + + " operator=\"InList\" id=\"name1id\">\n" + + "<constantOperand>\n" + + "<value>\n" + + "<caption type=\"String\">var1</caption>\n" + + "</value>\n" + + "<value>\n" + + "<caption type=\"String\">v a r 2</caption>\n" + + "</value>\n" + + "</constantOperand>\n" + + "</comparisonFilter>\n\n" + + "</and>\n\n" + + "<comparisonFilter path=\"Dimension|folder\\Test|folder\\name1|dimension\"" + + " operator=\"InList\" id=\"name1id\">\n" + + "<constantOperand>\n" + + "<value>\n" + + "<caption type=\"String\">var1</caption>\n" + + "</value>\n" + + "<value>\n" + + "<caption type=\"String\">withoutspaces</caption>\n" + + "</value>\n" + + "</constantOperand>\n" + + "</comparisonFilter>\n\n" + + "</and>\n\n" + + "<comparisonFilter path=\"Dimension|folder\\Test|folder\\name1|dimension\"" + + " operator=\"InList\" id=\"name1id\">\n" + + "<constantOperand>\n" + + "<value>\n" + + "<caption type=\"String\">one value</caption>\n" + + "</value>\n" + + "</constantOperand>\n" + + "</comparisonFilter>\n\n" + + "</and>\n\n" + + "<comparisonFilter path=\"Dimension|folder\\Test|folder\\name2|dimension\"" + + " operator=\"InList\" id=\"name2id\">\n" + + "<constantOperand>\n" + + "<value>\n" + + "<caption type=\"Numeric\">1</caption>\n" + + "</value>\n" + + "<value>\n" + + "<caption type=\"Numeric\">3</caption>\n" + + "</value>\n" + + "<value>\n" + + "<caption type=\"Numeric\">4</caption>\n" + + "</value>\n" + + "</constantOperand>\n" + + "</comparisonFilter>\n\n" + + "</and>\n", universeQuery.getWhere()); } @Test(expected = UniverseException.class) public void testFailConvertWithoutUniverse() throws UniverseException { - String request = - "universe ;\n" - + "select [Measure].[name5]\n" - + "where [Filter].[name3] and [Dimension].[Test].[name2] > 1;"; + String request = "universe ;\n" + + "select [Measure].[name5]\n" + + "where [Filter].[name3] and [Dimension].[Test].[name2] > 1;"; universeUtil.convertQuery(request, universeClient, null); } @Test(expected = UniverseException.class) public void testFailConvertWithIncorrectSelect() throws UniverseException { - String request = "universe [testUniverse];\n" + "select [not].[exist];"; + String request = "universe [testUniverse];\n" + + "select [not].[exist];"; universeUtil.convertQuery(request, universeClient, null); } + @Test(expected = UniverseException.class) public void testFailConvertWithIncorrectCondition() throws UniverseException { - String request = - "universe [testUniverse];\n" + "select [Measure].[name5]\n" + "where [Filter].[name;"; + String request = "universe [testUniverse];\n" + + "select [Measure].[name5]\n" + + "where [Filter].[name;"; universeUtil.convertQuery(request, universeClient, null); } @Test public void testFiltersConditions() throws UniverseException { - String request1 = - "universe [testUniverse];\n" + "select [Measure].[name5]\n" + "where [Filter].[name3];"; - String request2 = - "universe [testUniverse];\n" - + "select [Measure].[name5]\n" - + "where [Measure].[name5] > 2 and [Filter].[name3];"; - String request3 = - "universe [testUniverse];\n" - + "select [Measure].[name5]\n" - + "where [Filter].[name3] or [Measure].[name5];"; - String request4 = - "universe [testUniverse];\n" - + "select [Measure].[name5]\n" - + "where [Filter].[name3] and [Measure].[name5] is null;"; + String request1 = "universe [testUniverse];\n" + + "select [Measure].[name5]\n" + + "where [Filter].[name3];"; + String request2 = "universe [testUniverse];\n" + + "select [Measure].[name5]\n" + + "where [Measure].[name5] > 2 and [Filter].[name3];"; + String request3 = "universe [testUniverse];\n" + + "select [Measure].[name5]\n" + + "where [Filter].[name3] or [Measure].[name5];"; + String request4 = "universe [testUniverse];\n" + + "select [Measure].[name5]\n" + + "where [Filter].[name3] and [Measure].[name5] is null;"; UniverseQuery universeQuery = universeUtil.convertQuery(request1, universeClient, null); - assertEquals( - "<predefinedFilter path=\"Filter|folder\\name3|filter\" id=\"name3id\"/>\n", + assertEquals("<predefinedFilter path=\"Filter|folder\\name3|filter\" id=\"name3id\"/>\n", universeQuery.getWhere()); universeQuery = universeUtil.convertQuery(request2, universeClient, null); - assertEquals( - "<and>\n" - + "<comparisonFilter path=\"Measure|folder\\name5|measure\" operator=\"GreaterThan\" id=\"name5id\">\n" - + "<constantOperand>\n" - + "<value>\n" - + "<caption type=\"Numeric\">2</caption>\n" - + "</value>\n" - + "</constantOperand>\n" - + "</comparisonFilter>\n\n" - + "<predefinedFilter path=\"Filter|folder\\name3|filter\" id=\"name3id\"/>\n\n" - + "</and>\n", + assertEquals("<and>\n" + + "<comparisonFilter path=\"Measure|folder\\name5|measure\" operator=\"GreaterThan\" id=\"name5id\">\n" + + "<constantOperand>\n" + + "<value>\n" + + "<caption type=\"Numeric\">2</caption>\n" + + "</value>\n" + + "</constantOperand>\n" + + "</comparisonFilter>\n\n" + + "<predefinedFilter path=\"Filter|folder\\name3|filter\" id=\"name3id\"/>\n\n" + + "</and>\n", universeQuery.getWhere()); universeQuery = universeUtil.convertQuery(request3, universeClient, null); - assertEquals( - "<or>\n" - + "<predefinedFilter path=\"Filter|folder\\name3|filter\" id=\"name3id\"/>\n\n" - + "<predefinedFilter path=\"Measure|folder\\name5|measure\" id=\"name5id\"/>\n\n" - + "</or>\n", + assertEquals("<or>\n" + + "<predefinedFilter path=\"Filter|folder\\name3|filter\" id=\"name3id\"/>\n\n" + + "<predefinedFilter path=\"Measure|folder\\name5|measure\" id=\"name5id\"/>\n\n" + + "</or>\n", universeQuery.getWhere()); universeQuery = universeUtil.convertQuery(request4, universeClient, null); - assertEquals( - "<and>\n" - + "<predefinedFilter path=\"Filter|folder\\name3|filter\" id=\"name3id\"/>\n\n" - + "<comparisonFilter id=\"name5id\" path=\"Measure|folder\\name5|measure\" operator=\"IsNull\"/>\n\n" - + "</and>\n", + assertEquals("<and>\n" + + "<predefinedFilter path=\"Filter|folder\\name3|filter\" id=\"name3id\"/>\n\n" + + "<comparisonFilter id=\"name5id\" path=\"Measure|folder\\name5|measure\" operator=\"IsNull\"/>\n\n" + + "</and>\n", universeQuery.getWhere()); } @Test public void testNestedConditions() throws UniverseException { - String request = - "universe [testUniverse];\n" - + "select [Dimension].[Test].[name2]\n" - + "where ([Measure].[name5] = 'text' or ([Dimension].[Test].[name1] in ('1','2', '3') and\n" - + "[Dimension].[Test].[name2] is not null)) and ([Filter].[name4] or [Measure].[name5] >=12)\n" - + "or [Dimension].[Test].[name2] not in (31, 65, 77);"; + String request = "universe [testUniverse];\n" + + "select [Dimension].[Test].[name2]\n" + + "where ([Measure].[name5] = 'text' or ([Dimension].[Test].[name1] in ('1','2', '3') and\n" + + "[Dimension].[Test].[name2] is not null)) and ([Filter].[name4] or [Measure].[name5] >=12)\n" + + "or [Dimension].[Test].[name2] not in (31, 65, 77);"; UniverseQuery universeQuery = universeUtil.convertQuery(request, universeClient, null); - assertEquals( - "<or>\n" - + "<and>\n" - + "<or>\n" - + "<comparisonFilter path=\"Measure|folder\\name5|measure\" operator=\"EqualTo\" id=\"name5id\">\n" - + "<constantOperand>\n" - + "<value>\n" - + "<caption type=\"String\">text</caption>\n" - + "</value>\n" - + "</constantOperand>\n" - + "</comparisonFilter>\n\n" - + "<and>\n" - + "<comparisonFilter path=\"Dimension|folder\\Test|folder\\name1|dimension\" operator=\"InList\" id=\"name1id\">\n" - + "<constantOperand>\n" - + "<value>\n" - + "<caption type=\"String\">1</caption>\n" - + "</value>\n" - + "<value>\n" - + "<caption type=\"String\">2</caption>\n" - + "</value>\n" - + "<value>\n" - + "<caption type=\"String\">3</caption>\n" - + "</value>\n" - + "</constantOperand>\n" - + "</comparisonFilter>\n\n" - + "<comparisonFilter id=\"name2id\" path=\"Dimension|folder\\Test|folder\\name2|dimension\" operator=\"IsNotNull\"/>\n\n" - + "</and>\n\n" - + "</or>\n\n" - + "<or>\n" - + "<predefinedFilter path=\"Filter|folder\\name4|filter\" id=\"name4id\"/>\n\n" - + "<comparisonFilter path=\"Measure|folder\\name5|measure\" operator=\"GreaterThanOrEqualTo\" id=\"name5id\">\n" - + "<constantOperand>\n" - + "<value>\n" - + "<caption type=\"Numeric\">12</caption>\n" - + "</value>\n" - + "</constantOperand>\n" - + "</comparisonFilter>\n\n" - + "</or>\n\n" - + "</and>\n\n" - + "<comparisonFilter path=\"Dimension|folder\\Test|folder\\name2|dimension\" operator=\"NotInList\" id=\"name2id\">\n" - + "<constantOperand>\n" - + "<value>\n" - + "<caption type=\"Numeric\">31</caption>\n" - + "</value>\n" - + "<value>\n" - + "<caption type=\"Numeric\">65</caption>\n" - + "</value>\n" - + "<value>\n" - + "<caption type=\"Numeric\">77</caption>\n" - + "</value>\n" - + "</constantOperand>\n" - + "</comparisonFilter>\n\n" - + "</or>\n", + assertEquals("<or>\n" + + "<and>\n" + + "<or>\n" + + "<comparisonFilter path=\"Measure|folder\\name5|measure\" operator=\"EqualTo\" id=\"name5id\">\n" + + "<constantOperand>\n" + + "<value>\n" + + "<caption type=\"String\">text</caption>\n" + + "</value>\n" + + "</constantOperand>\n" + + "</comparisonFilter>\n\n" + + "<and>\n" + + "<comparisonFilter path=\"Dimension|folder\\Test|folder\\name1|dimension\" operator=\"InList\" id=\"name1id\">\n" + + "<constantOperand>\n" + + "<value>\n" + + "<caption type=\"String\">1</caption>\n" + + "</value>\n" + + "<value>\n" + + "<caption type=\"String\">2</caption>\n" + + "</value>\n" + + "<value>\n" + + "<caption type=\"String\">3</caption>\n" + + "</value>\n" + + "</constantOperand>\n" + + "</comparisonFilter>\n\n" + + "<comparisonFilter id=\"name2id\" path=\"Dimension|folder\\Test|folder\\name2|dimension\" operator=\"IsNotNull\"/>\n\n" + + "</and>\n\n" + + "</or>\n\n" + + "<or>\n" + + "<predefinedFilter path=\"Filter|folder\\name4|filter\" id=\"name4id\"/>\n\n" + + "<comparisonFilter path=\"Measure|folder\\name5|measure\" operator=\"GreaterThanOrEqualTo\" id=\"name5id\">\n" + + "<constantOperand>\n" + + "<value>\n" + + "<caption type=\"Numeric\">12</caption>\n" + + "</value>\n" + + "</constantOperand>\n" + + "</comparisonFilter>\n\n" + + "</or>\n\n" + + "</and>\n\n" + + "<comparisonFilter path=\"Dimension|folder\\Test|folder\\name2|dimension\" operator=\"NotInList\" id=\"name2id\">\n" + + "<constantOperand>\n" + + "<value>\n" + + "<caption type=\"Numeric\">31</caption>\n" + + "</value>\n" + + "<value>\n" + + "<caption type=\"Numeric\">65</caption>\n" + + "</value>\n" + + "<value>\n" + + "<caption type=\"Numeric\">77</caption>\n" + + "</value>\n" + + "</constantOperand>\n" + + "</comparisonFilter>\n\n" + + "</or>\n", universeQuery.getWhere()); } @Test public void testWithoutConditions() throws UniverseException { - String request = - "universe [testUniverse];\n" - + "select [Dimension].[Test].[name2], [Measure].[name5],\n" - + "[Dimension].[Test].[name1] ;"; + String request = "universe [testUniverse];\n" + + "select [Dimension].[Test].[name2], [Measure].[name5],\n" + + "[Dimension].[Test].[name1] ;"; UniverseQuery universeQuery = universeUtil.convertQuery(request, universeClient, null); assertNull(universeQuery.getWhere()); - assertEquals( - "<resultObjects>\n" - + "<resultObject path=\"Dimension|folder\\Test|folder\\name2|dimension\" id=\"name2id\"/>\n" - + "<resultObject path=\"Measure|folder\\name5|measure\" id=\"name5id\"/>\n" - + "<resultObject path=\"Dimension|folder\\Test|folder\\name1|dimension\" id=\"name1id\"/>\n" - + "</resultObjects>", + assertEquals("<resultObjects>\n" + + "<resultObject path=\"Dimension|folder\\Test|folder\\name2|dimension\" id=\"name2id\"/>\n" + + "<resultObject path=\"Measure|folder\\name5|measure\" id=\"name5id\"/>\n" + + "<resultObject path=\"Dimension|folder\\Test|folder\\name1|dimension\" id=\"name1id\"/>\n" + + "</resultObjects>", universeQuery.getSelect()); } @Test public void testCaseSensitive() throws UniverseException { - String request = - "uniVersE [testUniverse];\n" - + "seLEct [Dimension].[Test].[name2], [Measure].[name5]\n" - + "whERE [Dimension].[Test].[name2] Is NULl Or [Measure].[name5] IN (1,2) aNd [Measure].[name5] is NOT nUll;"; + String request = "uniVersE [testUniverse];\n" + + "seLEct [Dimension].[Test].[name2], [Measure].[name5]\n" + + "whERE [Dimension].[Test].[name2] Is NULl Or [Measure].[name5] IN (1,2) aNd [Measure].[name5] is NOT nUll;"; UniverseQuery universeQuery = universeUtil.convertQuery(request, universeClient, null); - assertEquals( - "<resultObjects>\n" - + "<resultObject path=\"Dimension|folder\\Test|folder\\name2|dimension\" id=\"name2id\"/>\n" - + "<resultObject path=\"Measure|folder\\name5|measure\" id=\"name5id\"/>\n" - + "</resultObjects>", + assertEquals("<resultObjects>\n" + + "<resultObject path=\"Dimension|folder\\Test|folder\\name2|dimension\" id=\"name2id\"/>\n" + + "<resultObject path=\"Measure|folder\\name5|measure\" id=\"name5id\"/>\n" + + "</resultObjects>", universeQuery.getSelect()); - assertEquals( - "<or>\n" - + "<comparisonFilter id=\"name2id\" path=\"Dimension|folder\\Test|folder\\name2|dimension\" operator=\"IsNull\"/>\n\n" - + "<and>\n" - + "<comparisonFilter path=\"Measure|folder\\name5|measure\" operator=\"InList\" id=\"name5id\">\n" - + "<constantOperand>\n" - + "<value>\n" - + "<caption type=\"Numeric\">1</caption>\n" - + "</value>\n" - + "<value>\n" - + "<caption type=\"Numeric\">2</caption>\n" - + "</value>\n" - + "</constantOperand>\n" - + "</comparisonFilter>\n\n" - + "<comparisonFilter id=\"name5id\" path=\"Measure|folder\\name5|measure\" operator=\"IsNotNull\"/>\n\n" - + "</and>\n\n" - + "</or>\n", + assertEquals("<or>\n" + + "<comparisonFilter id=\"name2id\" path=\"Dimension|folder\\Test|folder\\name2|dimension\" operator=\"IsNull\"/>\n\n" + + "<and>\n" + + "<comparisonFilter path=\"Measure|folder\\name5|measure\" operator=\"InList\" id=\"name5id\">\n" + + "<constantOperand>\n" + "<value>\n" + "<caption type=\"Numeric\">1</caption>\n" + + "</value>\n" + + "<value>\n" + + "<caption type=\"Numeric\">2</caption>\n" + + "</value>\n" + + "</constantOperand>\n" + + "</comparisonFilter>\n\n" + + "<comparisonFilter id=\"name5id\" path=\"Measure|folder\\name5|measure\" operator=\"IsNotNull\"/>\n\n" + + "</and>\n\n" + + "</or>\n", universeQuery.getWhere()); } }
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/scalding/pom.xml ---------------------------------------------------------------------- diff --git a/scalding/pom.xml b/scalding/pom.xml index ec7fe40..2a9e456 100644 --- a/scalding/pom.xml +++ b/scalding/pom.xml @@ -184,6 +184,14 @@ </execution> </executions> </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-checkstyle-plugin</artifactId> + <configuration> + <skip>false</skip> + </configuration> + </plugin> </plugins> </build> http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/scalding/src/main/java/org/apache/zeppelin/scalding/ScaldingInterpreter.java ---------------------------------------------------------------------- diff --git a/scalding/src/main/java/org/apache/zeppelin/scalding/ScaldingInterpreter.java b/scalding/src/main/java/org/apache/zeppelin/scalding/ScaldingInterpreter.java index 0985c03..f46a1d7 100644 --- a/scalding/src/main/java/org/apache/zeppelin/scalding/ScaldingInterpreter.java +++ b/scalding/src/main/java/org/apache/zeppelin/scalding/ScaldingInterpreter.java @@ -17,7 +17,10 @@ package org.apache.zeppelin.scalding; -import com.twitter.scalding.ScaldingILoop; +import org.apache.hadoop.security.UserGroupInformation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.PrintStream; @@ -28,7 +31,11 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Properties; -import org.apache.hadoop.security.UserGroupInformation; + +import com.twitter.scalding.ScaldingILoop; + +import scala.Console; + import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterResult; @@ -36,11 +43,11 @@ import org.apache.zeppelin.interpreter.InterpreterResult.Code; import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; import org.apache.zeppelin.scheduler.Scheduler; import org.apache.zeppelin.scheduler.SchedulerFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import scala.Console; -/** Scalding interpreter for Zeppelin. Based off the Spark interpreter code. */ +/** + * Scalding interpreter for Zeppelin. Based off the Spark interpreter code. + * + */ public class ScaldingInterpreter extends Interpreter { Logger logger = LoggerFactory.getLogger(ScaldingInterpreter.class); @@ -63,7 +70,8 @@ public class ScaldingInterpreter extends Interpreter { @Override public void open() { numOpenInstances = numOpenInstances + 1; - String maxOpenInstancesStr = getProperty(MAX_OPEN_INSTANCES, MAX_OPEN_INSTANCES_DEFAULT); + String maxOpenInstancesStr = getProperty(MAX_OPEN_INSTANCES, + MAX_OPEN_INSTANCES_DEFAULT); int maxOpenInstances = 50; try { maxOpenInstances = Integer.valueOf(maxOpenInstancesStr); @@ -96,6 +104,7 @@ public class ScaldingInterpreter extends Interpreter { interpreter.intp().close(); } + @Override public InterpreterResult interpret(String cmd, InterpreterContext contextInterpreter) { String user = contextInterpreter.getAuthenticationInfo().getUser(); @@ -104,10 +113,10 @@ public class ScaldingInterpreter extends Interpreter { if (interpreter == null) { logger.error( "interpreter == null, open may not have been called because max.open.instances reached"); - return new InterpreterResult( - Code.ERROR, - "interpreter == null\n" - + "open may not have been called because max.open.instances reached"); + return new InterpreterResult(Code.ERROR, + "interpreter == null\n" + + "open may not have been called because max.open.instances reached" + ); } if (cmd == null || cmd.trim().length() == 0) { return new InterpreterResult(Code.SUCCESS); @@ -178,9 +187,9 @@ public class ScaldingInterpreter extends Interpreter { String nextLine = linesToRun[l + 1].trim(); boolean continuation = false; if (nextLine.isEmpty() - || nextLine.startsWith("//") // skip empty line or comment - || nextLine.startsWith("}") - || nextLine.startsWith("object")) { // include "} object" for Scala companion object + || nextLine.startsWith("//") // skip empty line or comment + || nextLine.startsWith("}") + || nextLine.startsWith("object")) { // include "} object" for Scala companion object continuation = true; } else if (!inComment && nextLine.startsWith("/*")) { inComment = true; @@ -189,9 +198,9 @@ public class ScaldingInterpreter extends Interpreter { inComment = false; continuation = true; } else if (nextLine.length() > 1 - && nextLine.charAt(0) == '.' - && nextLine.charAt(1) != '.' // ".." - && nextLine.charAt(1) != '/') { // "./" + && nextLine.charAt(0) == '.' + && nextLine.charAt(1) != '.' // ".." + && nextLine.charAt(1) != '/') { // "./" continuation = true; } else if (inComment) { continuation = true; @@ -257,13 +266,14 @@ public class ScaldingInterpreter extends Interpreter { @Override public Scheduler getScheduler() { - return SchedulerFactory.singleton() - .createOrGetFIFOScheduler(ScaldingInterpreter.class.getName() + this.hashCode()); + return SchedulerFactory.singleton().createOrGetFIFOScheduler( + ScaldingInterpreter.class.getName() + this.hashCode()); } @Override - public List<InterpreterCompletion> completion( - String buf, int cursor, InterpreterContext interpreterContext) { + public List<InterpreterCompletion> completion(String buf, int cursor, + InterpreterContext interpreterContext) { return NO_COMPLETION; } + } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/scalding/src/test/java/org/apache/zeppelin/scalding/ScaldingInterpreterTest.java ---------------------------------------------------------------------- diff --git a/scalding/src/test/java/org/apache/zeppelin/scalding/ScaldingInterpreterTest.java b/scalding/src/test/java/org/apache/zeppelin/scalding/ScaldingInterpreterTest.java index afb0bae..992c155 100644 --- a/scalding/src/test/java/org/apache/zeppelin/scalding/ScaldingInterpreterTest.java +++ b/scalding/src/test/java/org/apache/zeppelin/scalding/ScaldingInterpreterTest.java @@ -17,11 +17,6 @@ package org.apache.zeppelin.scalding; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.io.File; -import java.util.Properties; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResult.Code; @@ -32,7 +27,16 @@ import org.junit.FixMethodOrder; import org.junit.Test; import org.junit.runners.MethodSorters; -/** Tests for the Scalding interpreter for Zeppelin. */ +import java.io.File; +import java.util.Properties; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Tests for the Scalding interpreter for Zeppelin. + * + */ @FixMethodOrder(MethodSorters.NAME_ASCENDING) public class ScaldingInterpreterTest { public static ScaldingInterpreter repl; @@ -41,9 +45,8 @@ public class ScaldingInterpreterTest { @Before public void setUp() throws Exception { - tmpDir = - new File( - System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis()); + tmpDir = new File(System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + + System.currentTimeMillis()); System.setProperty("zeppelin.dep.localrepo", tmpDir.getAbsolutePath() + "/local-repo"); tmpDir.mkdirs(); @@ -56,12 +59,11 @@ public class ScaldingInterpreterTest { repl.open(); } - context = - InterpreterContext.builder() - .setNoteId("noteId") - .setParagraphId("paragraphId") - .setAuthenticationInfo(new AuthenticationInfo()) - .build(); + context = InterpreterContext.builder() + .setNoteId("noteId") + .setParagraphId("paragraphId") + .setAuthenticationInfo(new AuthenticationInfo()) + .build(); } @After @@ -86,45 +88,40 @@ public class ScaldingInterpreterTest { @Test public void testNextLineComments() { - assertEquals( - InterpreterResult.Code.SUCCESS, - repl.interpret("\"123\"\n/*comment here\n*/.toInt", context).code()); + assertEquals(InterpreterResult.Code.SUCCESS, + repl.interpret("\"123\"\n/*comment here\n*/.toInt", context).code()); } @Test public void testNextLineCompanionObject() { - String code = - "class Counter {\nvar value: Long = 0\n}\n // comment\n\n object Counter " - + "{\n def apply(x: Long) = new Counter()\n}"; + String code = "class Counter {\nvar value: Long = 0\n}\n // comment\n\n object Counter " + + "{\n def apply(x: Long) = new Counter()\n}"; assertEquals(InterpreterResult.Code.SUCCESS, repl.interpret(code, context).code()); } @Test public void testBasicIntp() { - assertEquals( - InterpreterResult.Code.SUCCESS, repl.interpret("val a = 1\nval b = 2", context).code()); + assertEquals(InterpreterResult.Code.SUCCESS, + repl.interpret("val a = 1\nval b = 2", context).code()); // when interpret incomplete expression InterpreterResult incomplete = repl.interpret("val a = \"\"\"", context); assertEquals(InterpreterResult.Code.INCOMPLETE, incomplete.code()); assertTrue(incomplete.message().get(0).getData().length() > 0); // expecting some error - // message + // message } @Test public void testBasicScalding() { - assertEquals( - InterpreterResult.Code.SUCCESS, - repl.interpret( - "case class Sale(state: String, name: String, sale: Int)\n" - + "val salesList = List(Sale(\"CA\", \"A\", 60), Sale(\"CA\", \"A\", 20), " - + "Sale(\"VA\", \"B\", 15))\n" - + "val salesPipe = TypedPipe.from(salesList)\n" - + "val results = salesPipe.map{x => (1, Set(x.state), x.sale)}.\n" - + " groupAll.sum.values.map{ case(count, set, sum) => (count, set.size, sum) }\n" - + "results.dump", - context) - .code()); + assertEquals(InterpreterResult.Code.SUCCESS, + repl.interpret("case class Sale(state: String, name: String, sale: Int)\n" + + "val salesList = List(Sale(\"CA\", \"A\", 60), Sale(\"CA\", \"A\", 20), " + + "Sale(\"VA\", \"B\", 15))\n" + + "val salesPipe = TypedPipe.from(salesList)\n" + + "val results = salesPipe.map{x => (1, Set(x.state), x.sale)}.\n" + + " groupAll.sum.values.map{ case(count, set, sum) => (count, set.size, sum) }\n" + + "results.dump", + context).code()); } @Test @@ -134,15 +131,14 @@ public class ScaldingInterpreterTest { @Test public void testEndWithComment() { - assertEquals( - InterpreterResult.Code.SUCCESS, repl.interpret("val c=1\n//comment", context).code()); + assertEquals(InterpreterResult.Code.SUCCESS, repl.interpret("val c=1\n//comment", + context).code()); } @Test public void testReferencingUndefinedVal() { - InterpreterResult result = - repl.interpret( - "def category(min: Int) = {" + " if (0 <= value) \"error\"" + "}", context); + InterpreterResult result = repl.interpret("def category(min: Int) = {" + + " if (0 <= value) \"error\"" + "}", context); assertEquals(Code.ERROR, result.code()); } } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/scio/src/test/java/org/apache/zeppelin/scio/ScioInterpreterTest.java ---------------------------------------------------------------------- diff --git a/scio/src/test/java/org/apache/zeppelin/scio/ScioInterpreterTest.java b/scio/src/test/java/org/apache/zeppelin/scio/ScioInterpreterTest.java index 63cc7ce..2e5c0d9 100644 --- a/scio/src/test/java/org/apache/zeppelin/scio/ScioInterpreterTest.java +++ b/scio/src/test/java/org/apache/zeppelin/scio/ScioInterpreterTest.java @@ -17,15 +17,21 @@ package org.apache.zeppelin.scio; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.util.LinkedList; -import java.util.Properties; +import org.apache.zeppelin.display.AngularObjectRegistry; +import org.apache.zeppelin.display.GUI; import org.apache.zeppelin.interpreter.*; +import org.apache.zeppelin.resource.LocalResourcePool; +import org.apache.zeppelin.user.AuthenticationInfo; import org.junit.Before; import org.junit.Test; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.Properties; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + public class ScioInterpreterTest { private static ScioInterpreter repl; private static InterpreterGroup intpGroup; @@ -34,7 +40,10 @@ public class ScioInterpreterTest { private final String newline = "\n"; private InterpreterContext getNewContext() { - return InterpreterContext.builder().setNoteId("noteId").setParagraphId("paragraphId").build(); + return InterpreterContext.builder() + .setNoteId("noteId") + .setParagraphId("paragraphId") + .build(); } @Before @@ -53,8 +62,7 @@ public class ScioInterpreterTest { @Test public void testBasicSuccess() { - assertEquals( - InterpreterResult.Code.SUCCESS, + assertEquals(InterpreterResult.Code.SUCCESS, repl.interpret("val a = 1" + newline + "val b = 2", context).code()); } @@ -74,36 +82,28 @@ public class ScioInterpreterTest { @Test public void testBasicPipeline() { - assertEquals( - InterpreterResult.Code.SUCCESS, - repl.interpret( - "val (sc, _) = ContextAndArgs(argz)" - + newline - + "sc.parallelize(1 to 10).closeAndCollect().toList", - context) - .code()); + assertEquals(InterpreterResult.Code.SUCCESS, + repl.interpret("val (sc, _) = ContextAndArgs(argz)" + newline + + "sc.parallelize(1 to 10).closeAndCollect().toList", context).code()); } @Test public void testBasicMultiStepPipeline() { final StringBuilder code = new StringBuilder(); - code.append("val (sc, _) = ContextAndArgs(argz)") - .append(newline) - .append("val numbers = sc.parallelize(1 to 10)") - .append(newline) - .append("val results = numbers.closeAndCollect().toList") - .append(newline) + code.append("val (sc, _) = ContextAndArgs(argz)").append(newline) + .append("val numbers = sc.parallelize(1 to 10)").append(newline) + .append("val results = numbers.closeAndCollect().toList").append(newline) .append("println(results)"); - assertEquals(InterpreterResult.Code.SUCCESS, repl.interpret(code.toString(), context).code()); + assertEquals(InterpreterResult.Code.SUCCESS, + repl.interpret(code.toString(), context).code()); } @Test public void testException() { - InterpreterResult exception = - repl.interpret( - "val (sc, _) = ContextAndArgs(argz)" + newline + "throw new Exception(\"test\")", - context); + InterpreterResult exception = repl.interpret("val (sc, _) = ContextAndArgs(argz)" + newline + + "throw new Exception(\"test\")", context); assertEquals(InterpreterResult.Code.ERROR, exception.code()); assertTrue(exception.message().get(0).getData().length() > 0); } + } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/shell/pom.xml ---------------------------------------------------------------------- diff --git a/shell/pom.xml b/shell/pom.xml index 6a7fda9..9f51dcc 100644 --- a/shell/pom.xml +++ b/shell/pom.xml @@ -88,6 +88,13 @@ <plugin> <artifactId>maven-resources-plugin</artifactId> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-checkstyle-plugin</artifactId> + <configuration> + <skip>false</skip> + </configuration> + </plugin> </plugins> </build> http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/shell/src/main/java/org/apache/zeppelin/shell/ShellInterpreter.java ---------------------------------------------------------------------- diff --git a/shell/src/main/java/org/apache/zeppelin/shell/ShellInterpreter.java b/shell/src/main/java/org/apache/zeppelin/shell/ShellInterpreter.java index 3072d5c..c686896 100644 --- a/shell/src/main/java/org/apache/zeppelin/shell/ShellInterpreter.java +++ b/shell/src/main/java/org/apache/zeppelin/shell/ShellInterpreter.java @@ -17,6 +17,15 @@ package org.apache.zeppelin.shell; +import org.apache.commons.exec.CommandLine; +import org.apache.commons.exec.DefaultExecutor; +import org.apache.commons.exec.ExecuteException; +import org.apache.commons.exec.ExecuteWatchdog; +import org.apache.commons.exec.PumpStreamHandler; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; @@ -24,12 +33,7 @@ import java.io.OutputStream; import java.util.List; import java.util.Properties; import java.util.concurrent.ConcurrentHashMap; -import org.apache.commons.exec.CommandLine; -import org.apache.commons.exec.DefaultExecutor; -import org.apache.commons.exec.ExecuteException; -import org.apache.commons.exec.ExecuteWatchdog; -import org.apache.commons.exec.PumpStreamHandler; -import org.apache.commons.lang3.StringUtils; + import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterResult; @@ -38,10 +42,10 @@ import org.apache.zeppelin.interpreter.KerberosInterpreter; import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; import org.apache.zeppelin.scheduler.Scheduler; import org.apache.zeppelin.scheduler.SchedulerFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -/** Shell interpreter for Zeppelin. */ +/** + * Shell interpreter for Zeppelin. + */ public class ShellInterpreter extends KerberosInterpreter { private static final Logger LOGGER = LoggerFactory.getLogger(ShellInterpreter.class); @@ -72,22 +76,21 @@ public class ShellInterpreter extends KerberosInterpreter { if (executor != null) { try { executor.getWatchdog().destroyProcess(); - } catch (Exception e) { + } catch (Exception e){ LOGGER.error("error destroying executor for paragraphId: " + executorKey, e); } } } } + @Override public InterpreterResult interpret(String originalCmd, InterpreterContext contextInterpreter) { - String cmd = - Boolean.parseBoolean(getProperty("zeppelin.shell.interpolation")) - ? interpolate(originalCmd, contextInterpreter.getResourcePool()) - : originalCmd; + String cmd = Boolean.parseBoolean(getProperty("zeppelin.shell.interpolation")) ? + interpolate(originalCmd, contextInterpreter.getResourcePool()) : originalCmd; LOGGER.debug("Run shell command '" + cmd + "'"); OutputStream outStream = new ByteArrayOutputStream(); - + CommandLine cmdLine = CommandLine.parse(shell); // the Windows CMD shell doesn't handle multiline statements, // they need to be delimited by '&&' instead @@ -99,22 +102,19 @@ public class ShellInterpreter extends KerberosInterpreter { try { DefaultExecutor executor = new DefaultExecutor(); - executor.setStreamHandler( - new PumpStreamHandler(contextInterpreter.out, contextInterpreter.out)); + executor.setStreamHandler(new PumpStreamHandler( + contextInterpreter.out, contextInterpreter.out)); - executor.setWatchdog( - new ExecuteWatchdog(Long.valueOf(getProperty(TIMEOUT_PROPERTY, defaultTimeoutProperty)))); + executor.setWatchdog(new ExecuteWatchdog( + Long.valueOf(getProperty(TIMEOUT_PROPERTY, defaultTimeoutProperty)))); executors.put(contextInterpreter.getParagraphId(), executor); if (Boolean.valueOf(getProperty(DIRECTORY_USER_HOME))) { executor.setWorkingDirectory(new File(System.getProperty("user.home"))); } int exitVal = executor.execute(cmdLine); - LOGGER.info( - "Paragraph " - + contextInterpreter.getParagraphId() - + " return with exit value: " - + exitVal); + LOGGER.info("Paragraph " + contextInterpreter.getParagraphId() + + " return with exit value: " + exitVal); return new InterpreterResult(Code.SUCCESS, outStream.toString()); } catch (ExecuteException e) { int exitValue = e.getExitValue(); @@ -124,11 +124,8 @@ public class ShellInterpreter extends KerberosInterpreter { if (exitValue == 143) { code = Code.INCOMPLETE; message += "Paragraph received a SIGTERM\n"; - LOGGER.info( - "The paragraph " - + contextInterpreter.getParagraphId() - + " stopped executing: " - + message); + LOGGER.info("The paragraph " + contextInterpreter.getParagraphId() + + " stopped executing: " + message); } message += "ExitValue: " + exitValue; return new InterpreterResult(code, message); @@ -146,7 +143,7 @@ public class ShellInterpreter extends KerberosInterpreter { if (executor != null) { try { executor.getWatchdog().destroyProcess(); - } catch (Exception e) { + } catch (Exception e){ LOGGER.error("error destroying executor for paragraphId: " + context.getParagraphId(), e); } } @@ -164,13 +161,13 @@ public class ShellInterpreter extends KerberosInterpreter { @Override public Scheduler getScheduler() { - return SchedulerFactory.singleton() - .createOrGetParallelScheduler(ShellInterpreter.class.getName() + this.hashCode(), 10); + return SchedulerFactory.singleton().createOrGetParallelScheduler( + ShellInterpreter.class.getName() + this.hashCode(), 10); } @Override - public List<InterpreterCompletion> completion( - String buf, int cursor, InterpreterContext interpreterContext) { + public List<InterpreterCompletion> completion(String buf, int cursor, + InterpreterContext interpreterContext) { return null; } @@ -189,11 +186,9 @@ public class ShellInterpreter extends KerberosInterpreter { Properties properties = getProperties(); CommandLine cmdLine = CommandLine.parse(shell); cmdLine.addArgument("-c", false); - String kinitCommand = - String.format( - "kinit -k -t %s %s", - properties.getProperty("zeppelin.shell.keytab.location"), - properties.getProperty("zeppelin.shell.principal")); + String kinitCommand = String.format("kinit -k -t %s %s", + properties.getProperty("zeppelin.shell.keytab.location"), + properties.getProperty("zeppelin.shell.principal")); cmdLine.addArgument(kinitCommand, false); DefaultExecutor executor = new DefaultExecutor(); try { @@ -206,10 +201,11 @@ public class ShellInterpreter extends KerberosInterpreter { @Override protected boolean isKerboseEnabled() { - if (!StringUtils.isAnyEmpty(getProperty("zeppelin.shell.auth.type")) - && getProperty("zeppelin.shell.auth.type").equalsIgnoreCase("kerberos")) { + if (!StringUtils.isAnyEmpty(getProperty("zeppelin.shell.auth.type")) && getProperty( + "zeppelin.shell.auth.type").equalsIgnoreCase("kerberos")) { return true; } return false; } + } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/shell/src/test/java/org/apache/zeppelin/shell/ShellInterpreterTest.java ---------------------------------------------------------------------- diff --git a/shell/src/test/java/org/apache/zeppelin/shell/ShellInterpreterTest.java b/shell/src/test/java/org/apache/zeppelin/shell/ShellInterpreterTest.java index f8b702f..5a8f4b4 100644 --- a/shell/src/test/java/org/apache/zeppelin/shell/ShellInterpreterTest.java +++ b/shell/src/test/java/org/apache/zeppelin/shell/ShellInterpreterTest.java @@ -20,13 +20,15 @@ package org.apache.zeppelin.shell; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + import java.util.Properties; + import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResult.Code; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; public class ShellInterpreterTest { @@ -45,7 +47,8 @@ public class ShellInterpreterTest { } @After - public void tearDown() throws Exception {} + public void tearDown() throws Exception { + } @Test public void test() { @@ -62,7 +65,7 @@ public class ShellInterpreterTest { } @Test - public void testInvalidCommand() { + public void testInvalidCommand(){ if (System.getProperty("os.name").startsWith("Windows")) { result = shell.interpret("invalid_command\ndir", context); } else { http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/spark/interpreter/src/main/java/org/apache/zeppelin/spark/AbstractSparkInterpreter.java ---------------------------------------------------------------------- diff --git a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/AbstractSparkInterpreter.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/AbstractSparkInterpreter.java index 7008a14..239a7fe 100644 --- a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/AbstractSparkInterpreter.java +++ b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/AbstractSparkInterpreter.java @@ -17,15 +17,17 @@ package org.apache.zeppelin.spark; -import java.util.Properties; import org.apache.spark.SparkContext; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.SQLContext; import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContext; + +import java.util.Properties; /** - * Abstract class for SparkInterpreter. For the purpose of co-exist of NewSparkInterpreter and - * OldSparkInterpreter + * Abstract class for SparkInterpreter. For the purpose of co-exist of NewSparkInterpreter + * and OldSparkInterpreter */ public abstract class AbstractSparkInterpreter extends Interpreter { http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/spark/interpreter/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java ---------------------------------------------------------------------- diff --git a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java index cce73bb..d76b09e 100644 --- a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java +++ b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java @@ -21,6 +21,8 @@ import java.io.ByteArrayOutputStream; import java.io.File; import java.io.PrintStream; import java.io.PrintWriter; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; import java.net.MalformedURLException; import java.net.URL; import java.net.URLClassLoader; @@ -28,13 +30,19 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Properties; + +import com.google.common.reflect.TypeToken; +import com.google.gson.Gson; + import org.apache.commons.lang.StringUtils; import org.apache.spark.repl.SparkILoop; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResult.Code; +import org.apache.zeppelin.interpreter.WrappedInterpreter; import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; import org.apache.zeppelin.scheduler.Scheduler; import org.apache.zeppelin.spark.dep.SparkDependencyContext; @@ -42,11 +50,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonatype.aether.resolution.ArtifactResolutionException; import org.sonatype.aether.resolution.DependencyResolutionException; + import scala.Console; import scala.None; import scala.Some; -import scala.collection.JavaConversions; import scala.collection.convert.WrapAsJava$; +import scala.collection.JavaConversions; import scala.tools.nsc.Settings; import scala.tools.nsc.interpreter.Completion.Candidates; import scala.tools.nsc.interpreter.Completion.ScalaCompleter; @@ -55,22 +64,24 @@ import scala.tools.nsc.interpreter.Results; import scala.tools.nsc.settings.MutableSettings.BooleanSetting; import scala.tools.nsc.settings.MutableSettings.PathSetting; + /** - * DepInterpreter downloads dependencies and pass them when SparkInterpreter initialized. It extends - * SparkInterpreter but does not create sparkcontext + * DepInterpreter downloads dependencies and pass them when SparkInterpreter initialized. + * It extends SparkInterpreter but does not create sparkcontext + * */ public class DepInterpreter extends Interpreter { /** - * intp - org.apache.spark.repl.SparkIMain (scala 2.10) intp - scala.tools.nsc.interpreter.IMain; - * (scala 2.11) + * intp - org.apache.spark.repl.SparkIMain (scala 2.10) + * intp - scala.tools.nsc.interpreter.IMain; (scala 2.11) */ private Object intp; - private ByteArrayOutputStream out; private SparkDependencyContext depc; - /** completer - org.apache.spark.repl.SparkJLineCompletion (scala 2.10) */ + /** + * completer - org.apache.spark.repl.SparkJLineCompletion (scala 2.10) + */ private Object completer; - private SparkILoop interpreter; static final Logger LOGGER = LoggerFactory.getLogger(DepInterpreter.class); @@ -82,7 +93,10 @@ public class DepInterpreter extends Interpreter { return depc; } - public static String getSystemDefault(String envName, String propertyName, String defaultValue) { + public static String getSystemDefault( + String envName, + String propertyName, + String defaultValue) { if (envName != null && !envName.isEmpty()) { String envValue = System.getenv().get(envName); @@ -113,6 +127,7 @@ public class DepInterpreter extends Interpreter { createIMain(); } + private void createIMain() { Settings settings = new Settings(); URL[] urls = getClassloaderUrls(); @@ -141,7 +156,8 @@ public class DepInterpreter extends Interpreter { settings.scala$tools$nsc$settings$ScalaSettings$_setter_$classpath_$eq(pathSettings); // set classloader for scala compiler - settings.explicitParentLoader_$eq(new Some<>(Thread.currentThread().getContextClassLoader())); + settings.explicitParentLoader_$eq(new Some<>(Thread.currentThread() + .getContextClassLoader())); BooleanSetting b = (BooleanSetting) settings.usejavacp(); b.v_$eq(true); @@ -152,6 +168,7 @@ public class DepInterpreter extends Interpreter { interpreter.createInterpreter(); + intp = Utils.invokeMethod(interpreter, "intp"); if (Utils.isScala2_10()) { @@ -159,16 +176,13 @@ public class DepInterpreter extends Interpreter { Utils.invokeMethod(intp, "initializeSynchronous"); } - depc = - new SparkDependencyContext( - getProperty("zeppelin.dep.localrepo"), - getProperty("zeppelin.dep.additionalRemoteRepository")); + depc = new SparkDependencyContext(getProperty("zeppelin.dep.localrepo"), + getProperty("zeppelin.dep.additionalRemoteRepository")); if (Utils.isScala2_10()) { - completer = - Utils.instantiateClass( - "org.apache.spark.repl.SparkJLineCompletion", - new Class[] {Utils.findClass("org.apache.spark.repl.SparkIMain")}, - new Object[] {intp}); + completer = Utils.instantiateClass( + "org.apache.spark.repl.SparkJLineCompletion", + new Class[]{Utils.findClass("org.apache.spark.repl.SparkIMain")}, + new Object[]{intp}); } interpret("@transient var _binder = new java.util.HashMap[String, Object]()"); Map<String, Object> binder; @@ -179,20 +193,23 @@ public class DepInterpreter extends Interpreter { } binder.put("depc", depc); - interpret( - "@transient val z = " - + "_binder.get(\"depc\")" - + ".asInstanceOf[org.apache.zeppelin.spark.dep.SparkDependencyContext]"); + interpret("@transient val z = " + + "_binder.get(\"depc\")" + + ".asInstanceOf[org.apache.zeppelin.spark.dep.SparkDependencyContext]"); + } private Results.Result interpret(String line) { - return (Results.Result) - Utils.invokeMethod(intp, "interpret", new Class[] {String.class}, new Object[] {line}); + return (Results.Result) Utils.invokeMethod( + intp, + "interpret", + new Class[] {String.class}, + new Object[] {line}); } public Object getValue(String name) { - Object ret = - Utils.invokeMethod(intp, "valueOfTerm", new Class[] {String.class}, new Object[] {name}); + Object ret = Utils.invokeMethod( + intp, "valueOfTerm", new Class[]{String.class}, new Object[]{name}); if (ret instanceof None) { return null; } else if (ret instanceof Some) { @@ -204,7 +221,8 @@ public class DepInterpreter extends Interpreter { public Object getLastObject() { IMain.Request r = (IMain.Request) Utils.invokeMethod(intp, "lastRequest"); - Object obj = r.lineRep().call("$result", JavaConversions.asScalaBuffer(new LinkedList<>())); + Object obj = r.lineRep().call("$result", + JavaConversions.asScalaBuffer(new LinkedList<>())); return obj; } @@ -219,11 +237,10 @@ public class DepInterpreter extends Interpreter { getInterpreterInTheSameSessionByClassName(SparkInterpreter.class, false); if (sparkInterpreter != null && sparkInterpreter.getDelegation().isSparkContextInitialized()) { - return new InterpreterResult( - Code.ERROR, - "Must be used before SparkInterpreter (%spark) initialized\n" - + "Hint: put this paragraph before any Spark code and " - + "restart Zeppelin/Interpreter"); + return new InterpreterResult(Code.ERROR, + "Must be used before SparkInterpreter (%spark) initialized\n" + + "Hint: put this paragraph before any Spark code and " + + "restart Zeppelin/Interpreter" ); } scala.tools.nsc.interpreter.Results.Result ret = interpret(st); @@ -231,8 +248,7 @@ public class DepInterpreter extends Interpreter { try { depc.fetch(); - } catch (MalformedURLException - | DependencyResolutionException + } catch (MalformedURLException | DependencyResolutionException | ArtifactResolutionException e) { LOGGER.error("Exception in DepInterpreter while interpret ", e); return new InterpreterResult(Code.ERROR, e.toString()); @@ -258,7 +274,9 @@ public class DepInterpreter extends Interpreter { } @Override - public void cancel(InterpreterContext context) {} + public void cancel(InterpreterContext context) { + } + @Override public FormType getFormType() { @@ -271,8 +289,8 @@ public class DepInterpreter extends Interpreter { } @Override - public List<InterpreterCompletion> completion( - String buf, int cursor, InterpreterContext interpreterContext) { + public List<InterpreterCompletion> completion(String buf, int cursor, + InterpreterContext interpreterContext) { if (Utils.isScala2_10()) { ScalaCompleter c = (ScalaCompleter) Utils.invokeMethod(completer, "completer"); Candidates ret = c.complete(buf, cursor); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/spark/interpreter/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java ---------------------------------------------------------------------- diff --git a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java index 9f43377..7589895 100644 --- a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java +++ b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java @@ -17,9 +17,6 @@ package org.apache.zeppelin.spark; -import java.io.IOException; -import java.util.Map; -import java.util.Properties; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaSparkContext; import org.apache.zeppelin.interpreter.BaseZeppelinContext; @@ -30,7 +27,13 @@ import org.apache.zeppelin.python.IPythonInterpreter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** PySparkInterpreter which use IPython underlying. */ +import java.io.IOException; +import java.util.Map; +import java.util.Properties; + +/** + * PySparkInterpreter which use IPython underlying. + */ public class IPySparkInterpreter extends IPythonInterpreter { private static final Logger LOGGER = LoggerFactory.getLogger(IPySparkInterpreter.class); @@ -47,19 +50,19 @@ public class IPySparkInterpreter extends IPythonInterpreter { getInterpreterInTheSameSessionByClassName(PySparkInterpreter.class, false); setProperty("zeppelin.python", pySparkInterpreter.getPythonExec()); sparkInterpreter = getInterpreterInTheSameSessionByClassName(SparkInterpreter.class); - setProperty( - "zeppelin.py4j.useAuth", sparkInterpreter.getSparkVersion().isSecretSocketSupported() + ""); + setProperty("zeppelin.py4j.useAuth", + sparkInterpreter.getSparkVersion().isSecretSocketSupported() + ""); SparkConf conf = sparkInterpreter.getSparkContext().getConf(); // only set PYTHONPATH in embedded, local or yarn-client mode. // yarn-cluster will setup PYTHONPATH automatically. - if (!conf.contains("spark.submit.deployMode") - || !conf.get("spark.submit.deployMode").equals("cluster")) { + if (!conf.contains("spark.submit.deployMode") || + !conf.get("spark.submit.deployMode").equals("cluster")) { setAdditionalPythonPath(PythonUtils.sparkPythonPath()); } setAddBulitinPy4j(false); setAdditionalPythonInitFile("python/zeppelin_ipyspark.py"); - setProperty( - "zeppelin.py4j.useAuth", sparkInterpreter.getSparkVersion().isSecretSocketSupported() + ""); + setProperty("zeppelin.py4j.useAuth", + sparkInterpreter.getSparkVersion().isSecretSocketSupported() + ""); super.open(); } @@ -84,7 +87,7 @@ public class IPySparkInterpreter extends IPythonInterpreter { InterpreterContext.set(context); String jobGroupId = Utils.buildJobGroupId(context); String jobDesc = Utils.buildJobDesc(context); - String setJobGroupStmt = "sc.setJobGroup('" + jobGroupId + "', '" + jobDesc + "')"; + String setJobGroupStmt = "sc.setJobGroup('" + jobGroupId + "', '" + jobDesc + "')"; InterpreterResult result = super.interpret(setJobGroupStmt, context); if (result.code().equals(InterpreterResult.Code.ERROR)) { return new InterpreterResult(InterpreterResult.Code.ERROR, "Fail to setJobGroup"); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/spark/interpreter/src/main/java/org/apache/zeppelin/spark/NewSparkInterpreter.java ---------------------------------------------------------------------- diff --git a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/NewSparkInterpreter.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/NewSparkInterpreter.java index 031fff4..23e6dad 100644 --- a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/NewSparkInterpreter.java +++ b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/NewSparkInterpreter.java @@ -18,13 +18,6 @@ package org.apache.zeppelin.spark; import com.google.common.collect.Lists; -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.spark.SparkConf; @@ -40,9 +33,17 @@ import org.apache.zeppelin.spark.dep.SparkDependencyContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + /** - * SparkInterpreter of Java implementation. It is just wrapper of Spark211Interpreter and - * Spark210Interpreter. + * SparkInterpreter of Java implementation. It is just wrapper of Spark211Interpreter + * and Spark210Interpreter. */ public class NewSparkInterpreter extends AbstractSparkInterpreter { @@ -63,11 +64,11 @@ public class NewSparkInterpreter extends AbstractSparkInterpreter { private static InterpreterHookRegistry hooks; + public NewSparkInterpreter(Properties properties) { super(properties); - this.enableSupportedVersionCheck = - java.lang.Boolean.parseBoolean( - properties.getProperty("zeppelin.spark.enableSupportedVersionCheck", "true")); + this.enableSupportedVersionCheck = java.lang.Boolean.parseBoolean( + properties.getProperty("zeppelin.spark.enableSupportedVersionCheck", "true")); innerInterpreterClassMap.put("2.10", "org.apache.zeppelin.spark.SparkScala210Interpreter"); innerInterpreterClassMap.put("2.11", "org.apache.zeppelin.spark.SparkScala211Interpreter"); } @@ -97,25 +98,19 @@ public class NewSparkInterpreter extends AbstractSparkInterpreter { String innerIntpClassName = innerInterpreterClassMap.get(scalaVersion); Class clazz = Class.forName(innerIntpClassName); - this.innerInterpreter = - (BaseSparkScalaInterpreter) - clazz - .getConstructor(SparkConf.class, List.class, Boolean.class) - .newInstance( - conf, - getDependencyFiles(), - Boolean.parseBoolean(getProperty("zeppelin.spark.printREPLOutput", "true"))); + this.innerInterpreter = (BaseSparkScalaInterpreter) + clazz.getConstructor(SparkConf.class, List.class, Boolean.class) + .newInstance(conf, getDependencyFiles(), + Boolean.parseBoolean(getProperty("zeppelin.spark.printREPLOutput", "true"))); this.innerInterpreter.open(); sc = this.innerInterpreter.sc(); jsc = JavaSparkContext.fromSparkContext(sc); sparkVersion = SparkVersion.fromVersionString(sc.version()); if (enableSupportedVersionCheck && sparkVersion.isUnsupportedVersion()) { - throw new Exception( - "This is not officially supported spark version: " - + sparkVersion - + "\nYou can set zeppelin.spark.enableSupportedVersionCheck to false if you really" - + " want to try this version of spark."); + throw new Exception("This is not officially supported spark version: " + sparkVersion + + "\nYou can set zeppelin.spark.enableSupportedVersionCheck to false if you really" + + " want to try this version of spark."); } sqlContext = this.innerInterpreter.sqlContext(); sparkSession = this.innerInterpreter.sparkSession(); @@ -128,11 +123,10 @@ public class NewSparkInterpreter extends AbstractSparkInterpreter { sparkShims = SparkShims.getInstance(sc.version(), getProperties()); sparkShims.setupSparkListener(sc.master(), sparkUrl, InterpreterContext.get()); - z = - new SparkZeppelinContext( - sc, sparkShims, hooks, Integer.parseInt(getProperty("zeppelin.spark.maxResult"))); - this.innerInterpreter.bind( - "z", z.getClass().getCanonicalName(), z, Lists.newArrayList("@transient")); + z = new SparkZeppelinContext(sc, sparkShims, hooks, + Integer.parseInt(getProperty("zeppelin.spark.maxResult"))); + this.innerInterpreter.bind("z", z.getClass().getCanonicalName(), z, + Lists.newArrayList("@transient")); } catch (Exception e) { LOGGER.error("Fail to open SparkInterpreter", ExceptionUtils.getStackTrace(e)); throw new InterpreterException("Fail to open SparkInterpreter", e); @@ -168,8 +162,9 @@ public class NewSparkInterpreter extends AbstractSparkInterpreter { } @Override - public List<InterpreterCompletion> completion( - String buf, int cursor, InterpreterContext interpreterContext) { + public List<InterpreterCompletion> completion(String buf, + int cursor, + InterpreterContext interpreterContext) { LOGGER.debug("buf: " + buf + ", cursor:" + cursor); return innerInterpreter.completion(buf, cursor, interpreterContext); } @@ -225,9 +220,8 @@ public class NewSparkInterpreter extends AbstractSparkInterpreter { private List<String> getDependencyFiles() throws InterpreterException { List<String> depFiles = new ArrayList<>(); // add jar from DepInterpreter - DepInterpreter depInterpreter = - getParentSparkInterpreter() - .getInterpreterInTheSameSessionByClassName(DepInterpreter.class, false); + DepInterpreter depInterpreter = getParentSparkInterpreter(). + getInterpreterInTheSameSessionByClassName(DepInterpreter.class, false); if (depInterpreter != null) { SparkDependencyContext depc = depInterpreter.getDependencyContext(); if (depc != null) { @@ -263,6 +257,6 @@ public class NewSparkInterpreter extends AbstractSparkInterpreter { @Override public boolean isUnsupportedSparkVersion() { - return enableSupportedVersionCheck && sparkVersion.isUnsupportedVersion(); + return enableSupportedVersionCheck && sparkVersion.isUnsupportedVersion(); } }
