Rename package/groupId to org.apache and apply rat plugin. This PR handles https://issues.apache.org/jira/browse/ZEPPELIN-12.
* groupId at pom.xml file is changed from com.nflabs.zeppelin to org.apache.zeppelin * package name is changed from com.nflabs.zeppelin to org.apache.zeppelin * apache-rat plugin is applied (license header is added to every file) and NOTICE is updated (https://www.apache.org/legal/src-headers.html) * removed sphinx doc. because of doc was out dated (it was for 0.3.0) Please, review the changes. Especially, check NOTICE file if there're something i missed. Author: Lee moon soo <[email protected]> Closes #13 from Leemoonsoo/rat and squashes the following commits: 892695f [Lee moon soo] hive interpreter module com.nflabs -> org.apache. Add license to the hive/pom.xml c9a07c9 [Lee moon soo] Use correct package name 06a802a [Lee moon soo] One file is missed while renaming it 9902997 [Lee moon soo] Add missing import 643530a [Lee moon soo] Exclude .log from rat fb15d0b [Lee moon soo] Exclude dependency-reduced-pom.xml from rat plugin 5faf7b1 [Lee moon soo] Apply rat plugin and com.nflabs -> org.apache 5edc77b [Lee moon soo] Update license of ScreenCaptureHtmlUnitDriver.java 1bfef1f [Lee moon soo] Update notice file d7172fe [Lee moon soo] Add source file license header 92eb87f [Lee moon soo] Remove old sphinx doc be06c43 [Lee moon soo] Remove unused erb 1ffca75 [Lee moon soo] Remove unused file Project: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/commit/669d408d Tree: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/tree/669d408d Diff: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/diff/669d408d Branch: refs/heads/master Commit: 669d408dc93226b3d3d1e9271f17eed09ad7d158 Parents: 7a60b33 Author: Lee moon soo <[email protected]> Authored: Mon Apr 6 12:45:40 2015 +0900 Committer: Lee moon soo <[email protected]> Committed: Mon Apr 6 13:05:04 2015 +0900 ---------------------------------------------------------------------- .travis.yml | 15 + NOTICE | 56 +- _tools/checkstyle.xml | 15 + bin/common.sh | 17 +- bin/interpreter.sh | 18 +- bin/zeppelin-daemon.sh | 2 +- bin/zeppelin.sh | 2 +- conf/log4j.properties | 17 + conf/zeppelin-env.sh.template | 16 + conf/zeppelin-site.xml.template | 2 +- hive/pom.xml | 21 +- .../apache/zeppelin/hive/HiveInterpreter.java | 8 +- .../zeppelin/hive/HiveInterpreterTest.java | 4 +- markdown/pom.xml | 23 +- .../com/nflabs/zeppelin/markdown/Markdown.java | 77 - .../org/apache/zeppelin/markdown/Markdown.java | 93 + .../nflabs/zeppelin/markdown/MarkdownTest.java | 32 - .../apache/zeppelin/markdown/MarkdownTest.java | 49 + pom.xml | 81 +- shell/pom.xml | 23 +- .../nflabs/zeppelin/shell/ShellInterpreter.java | 97 - .../apache/zeppelin/shell/ShellInterpreter.java | 113 + spark/pom.xml | 44 +- .../nflabs/zeppelin/spark/DepInterpreter.java | 273 - .../zeppelin/spark/PySparkInterpreter.java | 406 - .../nflabs/zeppelin/spark/SparkInterpreter.java | 718 -- .../zeppelin/spark/SparkSqlInterpreter.java | 339 - .../nflabs/zeppelin/spark/ZeppelinContext.java | 238 - .../com/nflabs/zeppelin/spark/dep/Booter.java | 53 - .../nflabs/zeppelin/spark/dep/Dependency.java | 73 - .../zeppelin/spark/dep/DependencyContext.java | 134 - .../zeppelin/spark/dep/DependencyResolver.java | 333 - .../nflabs/zeppelin/spark/dep/Repository.java | 37 - .../zeppelin/spark/dep/RepositoryListener.java | 87 - .../spark/dep/RepositorySystemFactory.java | 52 - .../zeppelin/spark/dep/TransferListener.java | 130 - .../apache/zeppelin/spark/DepInterpreter.java | 289 + .../zeppelin/spark/PySparkInterpreter.java | 422 + .../apache/zeppelin/spark/SparkInterpreter.java | 741 ++ .../zeppelin/spark/SparkSqlInterpreter.java | 362 + .../apache/zeppelin/spark/ZeppelinContext.java | 254 + .../org/apache/zeppelin/spark/dep/Booter.java | 70 + .../apache/zeppelin/spark/dep/Dependency.java | 90 + .../zeppelin/spark/dep/DependencyContext.java | 151 + .../zeppelin/spark/dep/DependencyResolver.java | 350 + .../apache/zeppelin/spark/dep/Repository.java | 54 + .../zeppelin/spark/dep/RepositoryListener.java | 121 + .../spark/dep/RepositorySystemFactory.java | 69 + .../zeppelin/spark/dep/TransferListener.java | 148 + .../main/resources/python/zeppelin_pyspark.py | 17 + .../zeppelin/spark/ReflectSparkILoop.scala | 54 - .../zeppelin/spark/ReflectSparkIMain.scala | 17 - .../com/nflabs/zeppelin/spark/SparkRepl.scala | 72 - .../zeppelin/spark/DepInterpreterTest.java | 73 - .../zeppelin/spark/SparkInterpreterTest.java | 118 - .../zeppelin/spark/SparkSqlInterpreterTest.java | 94 - .../spark/dep/DependencyResolverTest.java | 34 - .../zeppelin/spark/DepInterpreterTest.java | 91 + .../zeppelin/spark/SparkInterpreterTest.java | 135 + .../zeppelin/spark/SparkSqlInterpreterTest.java | 112 + .../spark/dep/DependencyResolverTest.java | 52 + testing/startSparkCluster.sh | 17 + testing/stopSparkCluster.sh | 19 +- zeppelin-distribution/README.md | 17 + zeppelin-distribution/build-infrastructure.md | 17 + zeppelin-distribution/pom.xml | 19 +- .../src/assemble/distribution.xml | 23 +- zeppelin-distribution/src/deb/control/control | 19 + zeppelin-distribution/src/deb/control/prerm | 20 +- zeppelin-distribution/src/deb/init.d/zeppelind | 18 + zeppelin-docs/Makefile | 174 - zeppelin-docs/README.md | 24 - zeppelin-docs/pom.xml | 87 - zeppelin-docs/src/main/sphinx/index.rst | 21 - zeppelin-docs/src/main/sphinx/installation.rst | 8 - .../src/main/sphinx/installation/install.rst | 101 - .../main/sphinx/templates/zeppelin/layout.html | 38 - .../templates/zeppelin/static/alert_info_32.png | Bin 1530 -> 0 bytes .../zeppelin/static/alert_warning_32.png | Bin 974 -> 0 bytes .../sphinx/templates/zeppelin/static/bullet.png | Bin 109 -> 0 bytes .../templates/zeppelin/static/zeppelin.css_t | 132 - .../main/sphinx/templates/zeppelin/theme.conf | 26 - zeppelin-interpreter/pom.xml | 21 +- .../java/com/nflabs/zeppelin/display/GUI.java | 68 - .../java/com/nflabs/zeppelin/display/Input.java | 458 - .../interpreter/ClassloaderInterpreter.java | 261 - .../zeppelin/interpreter/Interpreter.java | 267 - .../interpreter/InterpreterContext.java | 51 - .../interpreter/InterpreterException.java | 17 - .../zeppelin/interpreter/InterpreterGroup.java | 48 - .../interpreter/InterpreterProperty.java | 32 - .../interpreter/InterpreterPropertyBuilder.java | 20 - .../zeppelin/interpreter/InterpreterResult.java | 120 - .../zeppelin/interpreter/InterpreterUtils.java | 41 - .../interpreter/LazyOpenInterpreter.java | 131 - .../interpreter/WrappedInterpreter.java | 8 - .../interpreter/remote/ClientFactory.java | 63 - .../interpreter/remote/RemoteInterpreter.java | 330 - .../remote/RemoteInterpreterProcess.java | 192 - .../remote/RemoteInterpreterServer.java | 325 - .../remote/RemoteInterpreterUtils.java | 32 - .../thrift/RemoteInterpreterContext.java | 786 -- .../thrift/RemoteInterpreterResult.java | 786 -- .../thrift/RemoteInterpreterService.java | 8174 ------------------ .../zeppelin/scheduler/FIFOScheduler.java | 134 - .../java/com/nflabs/zeppelin/scheduler/Job.java | 246 - .../nflabs/zeppelin/scheduler/JobListener.java | 15 - .../zeppelin/scheduler/JobProgressPoller.java | 53 - .../zeppelin/scheduler/ParallelScheduler.java | 162 - .../zeppelin/scheduler/RemoteScheduler.java | 357 - .../nflabs/zeppelin/scheduler/Scheduler.java | 21 - .../zeppelin/scheduler/SchedulerFactory.java | 129 - .../zeppelin/scheduler/SchedulerListener.java | 13 - .../java/org/apache/zeppelin/display/GUI.java | 85 + .../java/org/apache/zeppelin/display/Input.java | 476 + .../interpreter/ClassloaderInterpreter.java | 278 + .../zeppelin/interpreter/Interpreter.java | 283 + .../interpreter/InterpreterContext.java | 68 + .../interpreter/InterpreterException.java | 34 + .../zeppelin/interpreter/InterpreterGroup.java | 65 + .../interpreter/InterpreterProperty.java | 49 + .../interpreter/InterpreterPropertyBuilder.java | 37 + .../zeppelin/interpreter/InterpreterResult.java | 137 + .../zeppelin/interpreter/InterpreterUtils.java | 36 + .../interpreter/LazyOpenInterpreter.java | 148 + .../interpreter/WrappedInterpreter.java | 25 + .../interpreter/remote/ClientFactory.java | 79 + .../interpreter/remote/RemoteInterpreter.java | 347 + .../remote/RemoteInterpreterProcess.java | 208 + .../remote/RemoteInterpreterServer.java | 342 + .../remote/RemoteInterpreterUtils.java | 49 + .../thrift/RemoteInterpreterContext.java | 786 ++ .../thrift/RemoteInterpreterResult.java | 786 ++ .../thrift/RemoteInterpreterService.java | 8174 ++++++++++++++++++ .../zeppelin/scheduler/FIFOScheduler.java | 151 + .../java/org/apache/zeppelin/scheduler/Job.java | 263 + .../apache/zeppelin/scheduler/JobListener.java | 32 + .../zeppelin/scheduler/JobProgressPoller.java | 70 + .../zeppelin/scheduler/ParallelScheduler.java | 179 + .../zeppelin/scheduler/RemoteScheduler.java | 373 + .../apache/zeppelin/scheduler/Scheduler.java | 38 + .../zeppelin/scheduler/SchedulerFactory.java | 145 + .../zeppelin/scheduler/SchedulerListener.java | 30 + .../main/thrift/RemoteInterpreterService.thrift | 20 +- .../com/nflabs/zeppelin/display/InputTest.java | 24 - .../remote/RemoteInterpreterProcessTest.java | 46 - .../remote/RemoteInterpreterServerTest.java | 57 - .../remote/RemoteInterpreterTest.java | 428 - .../remote/RemoteInterpreterUtilsTest.java | 16 - .../remote/mock/MockInterpreterA.java | 84 - .../remote/mock/MockInterpreterB.java | 101 - .../zeppelin/scheduler/FIFOSchedulerTest.java | 74 - .../scheduler/ParallelSchedulerTest.java | 49 - .../zeppelin/scheduler/RemoteSchedulerTest.java | 105 - .../nflabs/zeppelin/scheduler/SleepingJob.java | 51 - .../org/apache/zeppelin/display/InputTest.java | 39 + .../remote/RemoteInterpreterProcessTest.java | 63 + .../remote/RemoteInterpreterServerTest.java | 76 + .../remote/RemoteInterpreterTest.java | 446 + .../remote/RemoteInterpreterUtilsTest.java | 34 + .../remote/mock/MockInterpreterA.java | 101 + .../remote/mock/MockInterpreterB.java | 118 + .../zeppelin/scheduler/FIFOSchedulerTest.java | 94 + .../scheduler/ParallelSchedulerTest.java | 71 + .../zeppelin/scheduler/RemoteSchedulerTest.java | 124 + .../apache/zeppelin/scheduler/SleepingJob.java | 75 + .../src/test/resources/log4j.properties | 17 + zeppelin-server/pom.xml | 41 +- .../zeppelin/rest/InterpreterRestApi.java | 152 - .../nflabs/zeppelin/rest/NotebookResponse.java | 20 - .../nflabs/zeppelin/rest/NotebookRestApi.java | 98 - .../nflabs/zeppelin/rest/ZeppelinRestApi.java | 35 - .../InterpreterSettingListForNoteBind.java | 67 - .../message/NewInterpreterSettingRequest.java | 36 - .../UpdateInterpreterSettingRequest.java | 28 - .../zeppelin/server/AppScriptServlet.java | 76 - .../com/nflabs/zeppelin/server/CorsFilter.java | 54 - .../nflabs/zeppelin/server/JsonResponse.java | 126 - .../nflabs/zeppelin/server/ZeppelinServer.java | 311 - .../com/nflabs/zeppelin/socket/Message.java | 93 - .../nflabs/zeppelin/socket/NotebookServer.java | 483 -- .../socket/SslWebSocketServerFactory.java | 61 - .../zeppelin/rest/InterpreterRestApi.java | 169 + .../apache/zeppelin/rest/NotebookResponse.java | 37 + .../apache/zeppelin/rest/NotebookRestApi.java | 112 + .../apache/zeppelin/rest/ZeppelinRestApi.java | 52 + .../InterpreterSettingListForNoteBind.java | 84 + .../message/NewInterpreterSettingRequest.java | 53 + .../UpdateInterpreterSettingRequest.java | 45 + .../zeppelin/server/AppScriptServlet.java | 93 + .../org/apache/zeppelin/server/CorsFilter.java | 71 + .../apache/zeppelin/server/JsonResponse.java | 145 + .../apache/zeppelin/server/ZeppelinServer.java | 327 + .../org/apache/zeppelin/socket/Message.java | 110 + .../apache/zeppelin/socket/NotebookServer.java | 500 ++ .../socket/SslWebSocketServerFactory.java | 76 + .../java/com/nflabs/zeppelin/ZeppelinIT.java | 327 - .../zeppelin/rest/AbstractTestRestApi.java | 239 - .../zeppelin/rest/ZeppelinRestApiTest.java | 95 - .../ScreenCaptureHtmlUnitDriver.java | 3 +- .../java/org/apache/zeppelin/ZeppelinIT.java | 338 + .../zeppelin/rest/AbstractTestRestApi.java | 256 + .../zeppelin/rest/ZeppelinRestApiTest.java | 112 + .../src/test/resources/log4j.properties | 19 +- .../zeppelin/AbstractFunctionalSuite.scala | 65 - .../com/nflabs/zeppelin/WelcomePageSuite.scala | 20 - .../zeppelin/AbstractFunctionalSuite.scala | 82 + .../org/apache/zeppelin/WelcomePageSuite.scala | 37 + zeppelin-web/Gruntfile.js | 17 + zeppelin-web/app/404.html | 14 + zeppelin-web/app/WEB-INF/web.xml | 19 +- zeppelin-web/app/fonts/custom-font.svg | 18 +- zeppelin-web/app/index.html | 2 - .../scripts/ace/textarea/src/ace-bookmarklet.js | 15 +- zeppelin-web/app/scripts/app.js | 15 +- .../app/scripts/controllers/interpreter.js | 3 +- zeppelin-web/app/scripts/controllers/main.js | 3 +- zeppelin-web/app/scripts/controllers/nav.js | 3 +- .../app/scripts/controllers/notebook.js | 3 +- .../app/scripts/controllers/paragraph.js | 3 +- .../app/scripts/directives/dropdowninput.js | 13 + zeppelin-web/app/scripts/directives/ngdelete.js | 14 + zeppelin-web/app/scripts/directives/ngenter.js | 3 +- .../scripts/directives/popover-html-unsafe.js | 13 + .../app/scripts/directives/resizable.js | 13 + zeppelin-web/app/styles/custom-font.css | 14 + zeppelin-web/app/styles/interpreter.css | 14 + zeppelin-web/app/styles/looknfeel/default.css | 3 +- zeppelin-web/app/styles/looknfeel/report.css | 3 +- zeppelin-web/app/styles/looknfeel/simple.css | 3 +- zeppelin-web/app/styles/main.css | 3 +- zeppelin-web/app/styles/notebook.css | 3 +- zeppelin-web/app/styles/printMode.css | 3 +- zeppelin-web/app/styles/typography.css | 3 +- zeppelin-web/app/views/interpreter.html | 2 - zeppelin-web/app/views/main.html | 2 - zeppelin-web/app/views/modal-shortcut.html | 2 - zeppelin-web/app/views/notebooks.html | 2 - zeppelin-web/app/views/paragraph.html | 2 - .../app/views/popover-html-unsafe-popup.html | 16 +- zeppelin-web/bower | 16 + zeppelin-web/grunt | 16 + zeppelin-web/pom.xml | 62 +- zeppelin-zengine/pom.xml | 21 +- .../zeppelin/conf/ZeppelinConfiguration.java | 514 -- .../interpreter/InterpreterFactory.java | 596 -- .../interpreter/InterpreterInfoSaving.java | 12 - .../zeppelin/interpreter/InterpreterOption.java | 24 - .../interpreter/InterpreterSerializer.java | 39 - .../interpreter/InterpreterSetting.java | 91 - .../zeppelin/notebook/JobListenerFactory.java | 13 - .../java/com/nflabs/zeppelin/notebook/Note.java | 350 - .../notebook/NoteInterpreterLoader.java | 90 - .../com/nflabs/zeppelin/notebook/Notebook.java | 283 - .../com/nflabs/zeppelin/notebook/Paragraph.java | 221 - .../zeppelin/notebook/utility/IdHashes.java | 57 - .../java/com/nflabs/zeppelin/util/Util.java | 170 - .../zeppelin/conf/ZeppelinConfiguration.java | 531 ++ .../interpreter/InterpreterFactory.java | 613 ++ .../interpreter/InterpreterInfoSaving.java | 29 + .../zeppelin/interpreter/InterpreterOption.java | 41 + .../interpreter/InterpreterSerializer.java | 56 + .../interpreter/InterpreterSetting.java | 108 + .../zeppelin/notebook/JobListenerFactory.java | 30 + .../java/org/apache/zeppelin/notebook/Note.java | 367 + .../notebook/NoteInterpreterLoader.java | 107 + .../org/apache/zeppelin/notebook/Notebook.java | 299 + .../org/apache/zeppelin/notebook/Paragraph.java | 237 + .../zeppelin/notebook/utility/IdHashes.java | 74 + .../java/org/apache/zeppelin/util/Util.java | 187 + zeppelin-zengine/src/main/resources/exec.erb | 15 - zeppelin-zengine/src/main/resources/table.erb | 36 - .../interpreter/InterpreterFactoryTest.java | 112 - .../interpreter/mock/MockInterpreter1.java | 57 - .../interpreter/mock/MockInterpreter2.java | 57 - .../nflabs/zeppelin/notebook/NotebookTest.java | 173 - .../java/com/nflabs/zeppelin/util/UtilTest.java | 79 - .../com/nflabs/zeppelin/util/UtilsForTests.java | 104 - .../interpreter/InterpreterFactoryTest.java | 128 + .../interpreter/mock/MockInterpreter1.java | 74 + .../interpreter/mock/MockInterpreter2.java | 74 + .../apache/zeppelin/notebook/NotebookTest.java | 189 + .../java/org/apache/zeppelin/util/UtilTest.java | 100 + .../org/apache/zeppelin/util/UtilsForTests.java | 119 + 284 files changed, 25764 insertions(+), 24116 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/669d408d/.travis.yml ---------------------------------------------------------------------- diff --git a/.travis.yml b/.travis.yml index b44ee80..7eb5935 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + language: java jdk: - oraclejdk7 http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/669d408d/NOTICE ---------------------------------------------------------------------- diff --git a/NOTICE b/NOTICE index 7199f2d..c455bcb 100644 --- a/NOTICE +++ b/NOTICE @@ -1,6 +1,54 @@ -Zeppelin -Copyright 2013-2014 NFLabs, inc. +Apache Zeppelin (incubating) +Copyright 2015 The Apache Software Foundation -This product includes software developed by The Apache Software -Foundation (http://www.apache.org/). +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +Portions of this software were developed at NFLabs, Inc. (http://www.nflabs.com) +Copyright (c) 2010-2015 NFLabs Inc. + + + +This product contains webfonts from 'Font Awesome', which can be obtains at: + + * LICENSE: + * SIL OFL 1.1 (http://scripts.sil.org/OFL) + * HOMEPAGE: + * http://fortawesome.github.io/Font-Awesome/ + + + +This product contains a stylesheet from 'Font Awesome', which can be obtains at: + + * LICENSE: + * MIT License (http://opensource.org/licenses/mit-license.html) + * HOMEPAGE: + * http://fortawesome.github.io/Font-Awesome/ + + + +This product contains embeddable code editor 'Ace', which can be obtains at: + + * LICENSE: + * BSD (https://github.com/ajaxorg/ace/blob/master/LICENSE) + Copyright (c) 2010, Ajax.org B.V. + All rights reserved. + * HOMEPAGE: + * http://ace.c9.io/ + + + +This product contains 'Simple line icons', which can be obtains at: + + * LICENSE: + * MIT License (http://opensource.org/licenses/mit-license.html) + * HOMEPAGE: + * http://thesabbir.github.io/simple-line-icons/ + + +from https://code.google.com/p/selenium/issues/detail?id=1361 + + +The com.webautomation package contains software developed by Ivan Vasiliev +(https://groups.google.com/forum/#!msg/selenium-developers/PTR_j4xLVRM/k2yVq01Fa7oJ) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/669d408d/_tools/checkstyle.xml ---------------------------------------------------------------------- diff --git a/_tools/checkstyle.xml b/_tools/checkstyle.xml index 0157e84..618d74d 100644 --- a/_tools/checkstyle.xml +++ b/_tools/checkstyle.xml @@ -1,4 +1,19 @@ <?xml version="1.0" encoding="UTF-8"?> +<!-- +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> + <!DOCTYPE module PUBLIC "-//Puppy Crawl//DTD Check Configuration 1.3//EN" "http://www.puppycrawl.com/dtds/configuration_1_3.dtd";> http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/669d408d/bin/common.sh ---------------------------------------------------------------------- diff --git a/bin/common.sh b/bin/common.sh index ccd8a29..a825b76 100644 --- a/bin/common.sh +++ b/bin/common.sh @@ -1,16 +1,13 @@ #!/bin/bash # -# Copyright 2007 The Apache Software Foundation +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/669d408d/bin/interpreter.sh ---------------------------------------------------------------------- diff --git a/bin/interpreter.sh b/bin/interpreter.sh index 4aa3ca0..025df36 100755 --- a/bin/interpreter.sh +++ b/bin/interpreter.sh @@ -1,4 +1,20 @@ #!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# bin=$(dirname "${BASH_SOURCE-$0}") bin=$(cd "${bin}">/dev/null; pwd) @@ -39,7 +55,7 @@ addJarInDir "${INTERPRETER_DIR}" export CLASSPATH+=":${ZEPPELIN_CLASSPATH}" HOSTNAME=$(hostname) -ZEPPELIN_SERVER=com.nflabs.zeppelin.interpreter.remote.RemoteInterpreterServer +ZEPPELIN_SERVER=org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer INTERPRETER_ID=$(basename "${INTERPRETER_DIR}") ZEPPELIN_PID="${ZEPPELIN_PID_DIR}/zeppelin-interpreter-${INTERPRETER_ID}-${ZEPPELIN_IDENT_STRING}-${HOSTNAME}.pid" http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/669d408d/bin/zeppelin-daemon.sh ---------------------------------------------------------------------- diff --git a/bin/zeppelin-daemon.sh b/bin/zeppelin-daemon.sh index b5aaeff..56e6c88 100755 --- a/bin/zeppelin-daemon.sh +++ b/bin/zeppelin-daemon.sh @@ -51,7 +51,7 @@ ZEPPELIN_NAME="Zeppelin" ZEPPELIN_LOGFILE="${ZEPPELIN_LOG_DIR}/zeppelin-${ZEPPELIN_IDENT_STRING}-${HOSTNAME}.log" ZEPPELIN_OUTFILE="${ZEPPELIN_LOG_DIR}/zeppelin-${ZEPPELIN_IDENT_STRING}-${HOSTNAME}.out" ZEPPELIN_PID="${ZEPPELIN_PID_DIR}/zeppelin-${ZEPPELIN_IDENT_STRING}-${HOSTNAME}.pid" -ZEPPELIN_MAIN=com.nflabs.zeppelin.server.ZeppelinServer +ZEPPELIN_MAIN=org.apache.zeppelin.server.ZeppelinServer JAVA_OPTS+=" -Dzeppelin.log.file=${ZEPPELIN_LOGFILE}" if [[ "${ZEPPELIN_NICENESS}" = "" ]]; then http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/669d408d/bin/zeppelin.sh ---------------------------------------------------------------------- diff --git a/bin/zeppelin.sh b/bin/zeppelin.sh index 87dd7df..671673a 100755 --- a/bin/zeppelin.sh +++ b/bin/zeppelin.sh @@ -48,7 +48,7 @@ HOSTNAME=$(hostname) ZEPPELIN_LOGFILE="${ZEPPELIN_LOG_DIR}/zeppelin-${ZEPPELIN_IDENT_STRING}-${HOSTNAME}.log" LOG="${ZEPPELIN_LOG_DIR}/zeppelin-cli-${ZEPPELIN_IDENT_STRING}-${HOSTNAME}.out" -ZEPPELIN_SERVER=com.nflabs.zeppelin.server.ZeppelinServer +ZEPPELIN_SERVER=org.apache.zeppelin.server.ZeppelinServer JAVA_OPTS+=" -Dzeppelin.log.file=${ZEPPELIN_LOGFILE}" if [[ ! -d "${ZEPPELIN_LOG_DIR}" ]]; then http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/669d408d/conf/log4j.properties ---------------------------------------------------------------------- diff --git a/conf/log4j.properties b/conf/log4j.properties index 083940d..a7ef28b 100644 --- a/conf/log4j.properties +++ b/conf/log4j.properties @@ -1,3 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + log4j.rootLogger = INFO, dailyfile log4j.appender.stdout = org.apache.log4j.ConsoleAppender http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/669d408d/conf/zeppelin-env.sh.template ---------------------------------------------------------------------- diff --git a/conf/zeppelin-env.sh.template b/conf/zeppelin-env.sh.template index d6b6b00..62aa95a 100644 --- a/conf/zeppelin-env.sh.template +++ b/conf/zeppelin-env.sh.template @@ -1,4 +1,20 @@ #!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# # export JAVA_HOME= # export MASTER= # Spark master url. eg. spark://master_addr:7077. Leave empty if you want to use local mode http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/669d408d/conf/zeppelin-site.xml.template ---------------------------------------------------------------------- diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template index 7ce34f4..c203179 100644 --- a/conf/zeppelin-site.xml.template +++ b/conf/zeppelin-site.xml.template @@ -48,7 +48,7 @@ <property> <name>zeppelin.interpreters</name> - <value>com.nflabs.zeppelin.spark.SparkInterpreter,com.nflabs.zeppelin.spark.PySparkInterpreter,com.nflabs.zeppelin.spark.SparkSqlInterpreter,com.nflabs.zeppelin.spark.DepInterpreter,com.nflabs.zeppelin.markdown.Markdown,com.nflabs.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.hive.HiveInterpreter</value> + <value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.hive.HiveInterpreter</value> <description>Comma separated interpreter configurations. First interpreter become a default</description> </property> http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/669d408d/hive/pom.xml ---------------------------------------------------------------------- diff --git a/hive/pom.xml b/hive/pom.xml index fbc853d..8899979 100644 --- a/hive/pom.xml +++ b/hive/pom.xml @@ -1,10 +1,27 @@ <?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + <project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> <modelVersion>4.0.0</modelVersion> <parent> <artifactId>zeppelin</artifactId> - <groupId>com.nflabs.zeppelin</groupId> + <groupId>org.apache.zeppelin</groupId> <version>0.5.0-SNAPSHOT</version> </parent> @@ -21,7 +38,7 @@ </properties> <dependencies> <dependency> - <groupId>com.nflabs.zeppelin</groupId> + <groupId>org.apache.zeppelin</groupId> <artifactId>zeppelin-interpreter</artifactId> <version>${project.version}</version> <scope>provided</scope> http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/669d408d/hive/src/main/java/org/apache/zeppelin/hive/HiveInterpreter.java ---------------------------------------------------------------------- diff --git a/hive/src/main/java/org/apache/zeppelin/hive/HiveInterpreter.java b/hive/src/main/java/org/apache/zeppelin/hive/HiveInterpreter.java index 1fa9228..5e7fc7d 100644 --- a/hive/src/main/java/org/apache/zeppelin/hive/HiveInterpreter.java +++ b/hive/src/main/java/org/apache/zeppelin/hive/HiveInterpreter.java @@ -21,14 +21,14 @@ import java.sql.*; import java.util.List; import java.util.Properties; -import com.nflabs.zeppelin.interpreter.*; +import org.apache.zeppelin.interpreter.*; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.nflabs.zeppelin.interpreter.InterpreterResult.Code; -import com.nflabs.zeppelin.scheduler.Scheduler; -import com.nflabs.zeppelin.scheduler.SchedulerFactory; +import org.apache.zeppelin.interpreter.InterpreterResult.Code; +import org.apache.zeppelin.scheduler.Scheduler; +import org.apache.zeppelin.scheduler.SchedulerFactory; /** * Hive interpreter for Zeppelin. http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/669d408d/hive/src/test/java/org/apache/zeppelin/hive/HiveInterpreterTest.java ---------------------------------------------------------------------- diff --git a/hive/src/test/java/org/apache/zeppelin/hive/HiveInterpreterTest.java b/hive/src/test/java/org/apache/zeppelin/hive/HiveInterpreterTest.java index 4bae60b..41ab108 100644 --- a/hive/src/test/java/org/apache/zeppelin/hive/HiveInterpreterTest.java +++ b/hive/src/test/java/org/apache/zeppelin/hive/HiveInterpreterTest.java @@ -29,8 +29,8 @@ import java.util.Map; import java.util.Properties; import java.util.concurrent.Executor; -import com.nflabs.zeppelin.interpreter.InterpreterContext; -import com.nflabs.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterResult; import org.junit.After; import org.junit.Before; import org.junit.Test; http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/669d408d/markdown/pom.xml ---------------------------------------------------------------------- diff --git a/markdown/pom.xml b/markdown/pom.xml index 3b6cdef..8e6cc52 100644 --- a/markdown/pom.xml +++ b/markdown/pom.xml @@ -1,19 +1,36 @@ <?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + <project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> <modelVersion>4.0.0</modelVersion> <parent> <artifactId>zeppelin</artifactId> - <groupId>com.nflabs.zeppelin</groupId> + <groupId>org.apache.zeppelin</groupId> <version>0.5.0-SNAPSHOT</version> </parent> - <groupId>com.nflabs.zeppelin</groupId> + <groupId>org.apache.zeppelin</groupId> <artifactId>zeppelin-markdown</artifactId> <packaging>jar</packaging> <version>0.5.0-SNAPSHOT</version> <name>Zeppelin: Markdown interpreter</name> - <url>http://www.nflabs.com</url> + <url>http://zeppelin.incubator.apache.org</url> <dependencies> <dependency> http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/669d408d/markdown/src/main/java/com/nflabs/zeppelin/markdown/Markdown.java ---------------------------------------------------------------------- diff --git a/markdown/src/main/java/com/nflabs/zeppelin/markdown/Markdown.java b/markdown/src/main/java/com/nflabs/zeppelin/markdown/Markdown.java deleted file mode 100644 index aec8020..0000000 --- a/markdown/src/main/java/com/nflabs/zeppelin/markdown/Markdown.java +++ /dev/null @@ -1,77 +0,0 @@ -package com.nflabs.zeppelin.markdown; - -import java.io.IOException; -import java.util.List; -import java.util.Properties; - -import com.nflabs.zeppelin.interpreter.InterpreterUtils; -import org.markdown4j.Markdown4jProcessor; - -import com.nflabs.zeppelin.interpreter.Interpreter; -import com.nflabs.zeppelin.interpreter.InterpreterContext; -import com.nflabs.zeppelin.interpreter.InterpreterResult; -import com.nflabs.zeppelin.interpreter.InterpreterResult.Code; -import com.nflabs.zeppelin.scheduler.Scheduler; -import com.nflabs.zeppelin.scheduler.SchedulerFactory; - -/** - * Markdown interpreter for Zeppelin. - * - * @author Leemoonsoo - * @author anthonycorbacho - * - */ -public class Markdown extends Interpreter { - private Markdown4jProcessor md; - - static { - Interpreter.register("md", Markdown.class.getName()); - } - - public Markdown(Properties property) { - super(property); - } - - @Override - public void open() { - md = new Markdown4jProcessor(); - } - - @Override - public void close() {} - - @Override - public InterpreterResult interpret(String st, InterpreterContext interpreterContext) { - String html; - try { - html = md.process(st); - } catch (IOException | java.lang.RuntimeException e) { - return new InterpreterResult(Code.ERROR, InterpreterUtils.getMostRelevantMessage(e)); - } - return new InterpreterResult(Code.SUCCESS, "%html " + html); - } - - @Override - public void cancel(InterpreterContext context) {} - - @Override - public FormType getFormType() { - return FormType.SIMPLE; - } - - @Override - public int getProgress(InterpreterContext context) { - return 0; - } - - @Override - public Scheduler getScheduler() { - return SchedulerFactory.singleton().createOrGetParallelScheduler( - Markdown.class.getName() + this.hashCode(), 5); - } - - @Override - public List<String> completion(String buf, int cursor) { - return null; - } -} http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/669d408d/markdown/src/main/java/org/apache/zeppelin/markdown/Markdown.java ---------------------------------------------------------------------- diff --git a/markdown/src/main/java/org/apache/zeppelin/markdown/Markdown.java b/markdown/src/main/java/org/apache/zeppelin/markdown/Markdown.java new file mode 100644 index 0000000..88b3287 --- /dev/null +++ b/markdown/src/main/java/org/apache/zeppelin/markdown/Markdown.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.markdown; + +import java.io.IOException; +import java.util.List; +import java.util.Properties; + +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterResult.Code; +import org.apache.zeppelin.interpreter.InterpreterUtils; +import org.apache.zeppelin.scheduler.Scheduler; +import org.apache.zeppelin.scheduler.SchedulerFactory; +import org.markdown4j.Markdown4jProcessor; + +/** + * Markdown interpreter for Zeppelin. + * + * @author Leemoonsoo + * @author anthonycorbacho + * + */ +public class Markdown extends Interpreter { + private Markdown4jProcessor md; + + static { + Interpreter.register("md", Markdown.class.getName()); + } + + public Markdown(Properties property) { + super(property); + } + + @Override + public void open() { + md = new Markdown4jProcessor(); + } + + @Override + public void close() {} + + @Override + public InterpreterResult interpret(String st, InterpreterContext interpreterContext) { + String html; + try { + html = md.process(st); + } catch (IOException | java.lang.RuntimeException e) { + return new InterpreterResult(Code.ERROR, InterpreterUtils.getMostRelevantMessage(e)); + } + return new InterpreterResult(Code.SUCCESS, "%html " + html); + } + + @Override + public void cancel(InterpreterContext context) {} + + @Override + public FormType getFormType() { + return FormType.SIMPLE; + } + + @Override + public int getProgress(InterpreterContext context) { + return 0; + } + + @Override + public Scheduler getScheduler() { + return SchedulerFactory.singleton().createOrGetParallelScheduler( + Markdown.class.getName() + this.hashCode(), 5); + } + + @Override + public List<String> completion(String buf, int cursor) { + return null; + } +} http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/669d408d/markdown/src/test/java/com/nflabs/zeppelin/markdown/MarkdownTest.java ---------------------------------------------------------------------- diff --git a/markdown/src/test/java/com/nflabs/zeppelin/markdown/MarkdownTest.java b/markdown/src/test/java/com/nflabs/zeppelin/markdown/MarkdownTest.java deleted file mode 100644 index f62711c..0000000 --- a/markdown/src/test/java/com/nflabs/zeppelin/markdown/MarkdownTest.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.nflabs.zeppelin.markdown; - -import static org.junit.Assert.assertEquals; - -import java.util.Properties; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import com.nflabs.zeppelin.interpreter.InterpreterResult; - -public class MarkdownTest { - - @Before - public void setUp() throws Exception { - } - - @After - public void tearDown() throws Exception { - } - - @Test - public void test() { - Markdown md = new Markdown(new Properties()); - md.open(); - InterpreterResult result = md.interpret("This is ~~deleted~~ text", null); - assertEquals("<p>This is <s>deleted</s> text</p>\n", result.message()); - System.out.println(MarkdownTest.class.getName()); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/669d408d/markdown/src/test/java/org/apache/zeppelin/markdown/MarkdownTest.java ---------------------------------------------------------------------- diff --git a/markdown/src/test/java/org/apache/zeppelin/markdown/MarkdownTest.java b/markdown/src/test/java/org/apache/zeppelin/markdown/MarkdownTest.java new file mode 100644 index 0000000..86d6c59 --- /dev/null +++ b/markdown/src/test/java/org/apache/zeppelin/markdown/MarkdownTest.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.markdown; + +import static org.junit.Assert.assertEquals; + +import java.util.Properties; + +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.markdown.Markdown; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class MarkdownTest { + + @Before + public void setUp() throws Exception { + } + + @After + public void tearDown() throws Exception { + } + + @Test + public void test() { + Markdown md = new Markdown(new Properties()); + md.open(); + InterpreterResult result = md.interpret("This is ~~deleted~~ text", null); + assertEquals("<p>This is <s>deleted</s> text</p>\n", result.message()); + System.out.println(MarkdownTest.class.getName()); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/669d408d/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 7048a89..20a073e 100644 --- a/pom.xml +++ b/pom.xml @@ -1,4 +1,21 @@ <?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + <project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> @@ -15,7 +32,7 @@ <modelVersion>4.0.0</modelVersion> - <groupId>com.nflabs.zeppelin</groupId> + <groupId>org.apache.zeppelin</groupId> <artifactId>zeppelin</artifactId> <packaging>pom</packaging> <version>0.5.0-SNAPSHOT</version> @@ -38,9 +55,9 @@ </licenses> <scm> - <url>http://www.github.com/NFLabs/zeppelin</url> - <connection>scm:git:git:[email protected]:NFLabs/zeppelin.git</connection> - <developerConnection>scm:git:[email protected]:NFLabs/zeppelin.git</developerConnection> + <url>http://zeppelin.incubator.apache.org</url> + <connection>scm:git:git:[email protected]:apache/incubator-zeppelin.git</connection> + <developerConnection>scm:git:[email protected]:apache/incubator-zeppelin.git</developerConnection> </scm> <developers> @@ -853,6 +870,10 @@ <build> <plugins> <plugin> + <groupId>org.apache.rat</groupId> + <artifactId>apache-rat-plugin</artifactId> + </plugin> + <plugin> <artifactId>maven-compiler-plugin</artifactId> <version>3.1</version> <configuration> @@ -899,7 +920,7 @@ </goals> <configuration> <failOnViolation>true</failOnViolation> - <excludes>com/nflabs/zeppelin/interpreter/thrift/*</excludes> + <excludes>org/apache/zeppelin/interpreter/thrift/*</excludes> </configuration> </execution> </executions> @@ -1024,6 +1045,56 @@ <pluginManagement> <plugins> <plugin> + <groupId>org.apache.rat</groupId> + <artifactId>apache-rat-plugin</artifactId> + <version>0.11</version> + <configuration> + <excludes> + <exclude>**/.idea/</exclude> + <exclude>**/*.iml</exclude> + <exclude>.git/</exclude> + <exclude>.gitignore</exclude> + <exclude>.repository/</exclude> + <exclude>**/*.diff</exclude> + <exclude>**/*.patch</exclude> + <exclude>**/*.avsc</exclude> + <exclude>**/*.avro</exclude> + <exclude>**/*.log</exclude> + <exclude>**/docs/**</exclude> + <exclude>**/test/resources/**</exclude> + <exclude>**/.settings/*</exclude> + <exclude>**/.classpath</exclude> + <exclude>**/.project</exclude> + <exclude>**/target/**</exclude> + <exclude>**/derby.log</exclude> + <exclude>**/metastore_db/</exclude> + <exclude>**/logs/**</exclude> + <exclude>**/run/**</exclude> + <exclude>**/interpreter/**</exclude> + <exclude>**/local-repo/**</exclude> + <exclude>**/null/**</exclude> + <exclude>**/notebook/**</exclude> + <exclude>_tools/site/css/*</exclude> + <exclude>**/README.md</exclude> + <exclude>DEPLOY.md</exclude> + <exclude>CONTRIBUTING.md</exclude> + <exclude>STYLE.md</exclude> + <exclude>Roadmap.md</exclude> + <exclude>conf/interpreter.json</exclude> + </excludes> + </configuration> + + <executions> + <execution> + <id>verify.rat</id> + <phase>verify</phase> + <goals> + <goal>check</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-checkstyle-plugin</artifactId> <version>2.13</version> http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/669d408d/shell/pom.xml ---------------------------------------------------------------------- diff --git a/shell/pom.xml b/shell/pom.xml index d2ea754..7261332 100644 --- a/shell/pom.xml +++ b/shell/pom.xml @@ -1,19 +1,36 @@ <?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + <project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> <modelVersion>4.0.0</modelVersion> <parent> <artifactId>zeppelin</artifactId> - <groupId>com.nflabs.zeppelin</groupId> + <groupId>org.apache.zeppelin</groupId> <version>0.5.0-SNAPSHOT</version> </parent> - <groupId>com.nflabs.zeppelin</groupId> + <groupId>org.apache.zeppelin</groupId> <artifactId>zeppelin-shell</artifactId> <packaging>jar</packaging> <version>0.5.0-SNAPSHOT</version> <name>Zeppelin: Shell interpreter</name> - <url>http://www.nflabs.com</url> + <url>http://zeppelin.incubator.apache.org</url> <dependencies> <dependency> http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/669d408d/shell/src/main/java/com/nflabs/zeppelin/shell/ShellInterpreter.java ---------------------------------------------------------------------- diff --git a/shell/src/main/java/com/nflabs/zeppelin/shell/ShellInterpreter.java b/shell/src/main/java/com/nflabs/zeppelin/shell/ShellInterpreter.java deleted file mode 100644 index d1762f6..0000000 --- a/shell/src/main/java/com/nflabs/zeppelin/shell/ShellInterpreter.java +++ /dev/null @@ -1,97 +0,0 @@ -package com.nflabs.zeppelin.shell; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.util.List; -import java.util.Properties; - -import org.apache.commons.exec.CommandLine; -import org.apache.commons.exec.DefaultExecutor; -import org.apache.commons.exec.ExecuteException; -import org.apache.commons.exec.ExecuteWatchdog; -import org.apache.commons.exec.PumpStreamHandler; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.nflabs.zeppelin.interpreter.Interpreter; -import com.nflabs.zeppelin.interpreter.InterpreterContext; -import com.nflabs.zeppelin.interpreter.InterpreterResult; -import com.nflabs.zeppelin.interpreter.InterpreterResult.Code; -import com.nflabs.zeppelin.scheduler.Scheduler; -import com.nflabs.zeppelin.scheduler.SchedulerFactory; - -/** - * Shell interpreter for Zeppelin. - * - * @author Leemoonsoo - * @author anthonycorbacho - * - */ -public class ShellInterpreter extends Interpreter { - Logger logger = LoggerFactory.getLogger(ShellInterpreter.class); - int commandTimeOut = 600000; - - static { - Interpreter.register("sh", ShellInterpreter.class.getName()); - } - - public ShellInterpreter(Properties property) { - super(property); - } - - @Override - public void open() {} - - @Override - public void close() {} - - - @Override - public InterpreterResult interpret(String cmd, InterpreterContext contextInterpreter) { - logger.info("Run shell command '" + cmd + "'"); - long start = System.currentTimeMillis(); - CommandLine cmdLine = CommandLine.parse("bash"); - cmdLine.addArgument("-c", false); - cmdLine.addArgument(cmd, false); - DefaultExecutor executor = new DefaultExecutor(); - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - executor.setStreamHandler(new PumpStreamHandler(outputStream)); - - executor.setWatchdog(new ExecuteWatchdog(commandTimeOut)); - try { - int exitValue = executor.execute(cmdLine); - return new InterpreterResult(InterpreterResult.Code.SUCCESS, outputStream.toString()); - } catch (ExecuteException e) { - logger.error("Can not run " + cmd, e); - return new InterpreterResult(Code.ERROR, e.getMessage()); - } catch (IOException e) { - logger.error("Can not run " + cmd, e); - return new InterpreterResult(Code.ERROR, e.getMessage()); - } - } - - @Override - public void cancel(InterpreterContext context) {} - - @Override - public FormType getFormType() { - return FormType.SIMPLE; - } - - @Override - public int getProgress(InterpreterContext context) { - return 0; - } - - @Override - public Scheduler getScheduler() { - return SchedulerFactory.singleton().createOrGetFIFOScheduler( - ShellInterpreter.class.getName() + this.hashCode()); - } - - @Override - public List<String> completion(String buf, int cursor) { - return null; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/669d408d/shell/src/main/java/org/apache/zeppelin/shell/ShellInterpreter.java ---------------------------------------------------------------------- diff --git a/shell/src/main/java/org/apache/zeppelin/shell/ShellInterpreter.java b/shell/src/main/java/org/apache/zeppelin/shell/ShellInterpreter.java new file mode 100644 index 0000000..64a9485 --- /dev/null +++ b/shell/src/main/java/org/apache/zeppelin/shell/ShellInterpreter.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.shell; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.List; +import java.util.Properties; + +import org.apache.commons.exec.CommandLine; +import org.apache.commons.exec.DefaultExecutor; +import org.apache.commons.exec.ExecuteException; +import org.apache.commons.exec.ExecuteWatchdog; +import org.apache.commons.exec.PumpStreamHandler; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterResult.Code; +import org.apache.zeppelin.scheduler.Scheduler; +import org.apache.zeppelin.scheduler.SchedulerFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Shell interpreter for Zeppelin. + * + * @author Leemoonsoo + * @author anthonycorbacho + * + */ +public class ShellInterpreter extends Interpreter { + Logger logger = LoggerFactory.getLogger(ShellInterpreter.class); + int commandTimeOut = 600000; + + static { + Interpreter.register("sh", ShellInterpreter.class.getName()); + } + + public ShellInterpreter(Properties property) { + super(property); + } + + @Override + public void open() {} + + @Override + public void close() {} + + + @Override + public InterpreterResult interpret(String cmd, InterpreterContext contextInterpreter) { + logger.info("Run shell command '" + cmd + "'"); + long start = System.currentTimeMillis(); + CommandLine cmdLine = CommandLine.parse("bash"); + cmdLine.addArgument("-c", false); + cmdLine.addArgument(cmd, false); + DefaultExecutor executor = new DefaultExecutor(); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + executor.setStreamHandler(new PumpStreamHandler(outputStream)); + + executor.setWatchdog(new ExecuteWatchdog(commandTimeOut)); + try { + int exitValue = executor.execute(cmdLine); + return new InterpreterResult(InterpreterResult.Code.SUCCESS, outputStream.toString()); + } catch (ExecuteException e) { + logger.error("Can not run " + cmd, e); + return new InterpreterResult(Code.ERROR, e.getMessage()); + } catch (IOException e) { + logger.error("Can not run " + cmd, e); + return new InterpreterResult(Code.ERROR, e.getMessage()); + } + } + + @Override + public void cancel(InterpreterContext context) {} + + @Override + public FormType getFormType() { + return FormType.SIMPLE; + } + + @Override + public int getProgress(InterpreterContext context) { + return 0; + } + + @Override + public Scheduler getScheduler() { + return SchedulerFactory.singleton().createOrGetFIFOScheduler( + ShellInterpreter.class.getName() + this.hashCode()); + } + + @Override + public List<String> completion(String buf, int cursor) { + return null; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/669d408d/spark/pom.xml ---------------------------------------------------------------------- diff --git a/spark/pom.xml b/spark/pom.xml index 7741d2b..f3143c0 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -1,15 +1,31 @@ <?xml version="1.0" encoding="UTF-8"?> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> <modelVersion>4.0.0</modelVersion> <parent> <artifactId>zeppelin</artifactId> - <groupId>com.nflabs.zeppelin</groupId> + <groupId>org.apache.zeppelin</groupId> <version>0.5.0-SNAPSHOT</version> </parent> + <groupId>org.apache.zeppelin</groupId> <artifactId>zeppelin-spark</artifactId> <packaging>jar</packaging> <version>0.5.0-SNAPSHOT</version> @@ -276,6 +292,26 @@ <build> <plugins> <plugin> + <groupId>org.apache.rat</groupId> + <artifactId>apache-rat-plugin</artifactId> + <configuration> + <excludes> + <exclude>**/.idea/</exclude> + <exclude>**/*.iml</exclude> + <exclude>.gitignore</exclude> + <exclude>**/.settings/*</exclude> + <exclude>**/.classpath</exclude> + <exclude>**/.project</exclude> + <exclude>**/target/**</exclude> + <exclude>**/derby.log</exclude> + <exclude>**/metastore_db/</exclude> + <exclude>**/README.md</exclude> + <exclude>dependency-reduced-pom.xml</exclude> + </excludes> + </configuration> + </plugin> + + <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-deploy-plugin</artifactId> <version>2.7</version> http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/669d408d/spark/src/main/java/com/nflabs/zeppelin/spark/DepInterpreter.java ---------------------------------------------------------------------- diff --git a/spark/src/main/java/com/nflabs/zeppelin/spark/DepInterpreter.java b/spark/src/main/java/com/nflabs/zeppelin/spark/DepInterpreter.java deleted file mode 100644 index de09772..0000000 --- a/spark/src/main/java/com/nflabs/zeppelin/spark/DepInterpreter.java +++ /dev/null @@ -1,273 +0,0 @@ -package com.nflabs.zeppelin.spark; - -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.PrintStream; -import java.io.PrintWriter; -import java.net.MalformedURLException; -import java.net.URL; -import java.net.URLClassLoader; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Properties; - -import org.apache.spark.repl.SparkILoop; -import org.apache.spark.repl.SparkIMain; -import org.apache.spark.repl.SparkJLineCompletion; -import org.sonatype.aether.resolution.ArtifactResolutionException; -import org.sonatype.aether.resolution.DependencyResolutionException; - -import scala.Console; -import scala.None; -import scala.Some; -import scala.tools.nsc.Settings; -import scala.tools.nsc.interpreter.Completion.Candidates; -import scala.tools.nsc.interpreter.Completion.ScalaCompleter; -import scala.tools.nsc.settings.MutableSettings.BooleanSetting; -import scala.tools.nsc.settings.MutableSettings.PathSetting; - -import com.nflabs.zeppelin.interpreter.Interpreter; -import com.nflabs.zeppelin.interpreter.InterpreterContext; -import com.nflabs.zeppelin.interpreter.InterpreterGroup; -import com.nflabs.zeppelin.interpreter.InterpreterPropertyBuilder; -import com.nflabs.zeppelin.interpreter.InterpreterResult; -import com.nflabs.zeppelin.interpreter.InterpreterResult.Code; -import com.nflabs.zeppelin.interpreter.WrappedInterpreter; -import com.nflabs.zeppelin.scheduler.Scheduler; -import com.nflabs.zeppelin.spark.dep.DependencyContext; - - -/** - * DepInterpreter downloads dependencies and pass them when SparkInterpreter initialized. - * It extends SparkInterpreter but does not create sparkcontext - * - */ -public class DepInterpreter extends Interpreter { - - static { - Interpreter.register( - "dep", - "spark", - DepInterpreter.class.getName(), - new InterpreterPropertyBuilder() - .add("zeppelin.dep.localrepo", "local-repo", "local repository for dependency loader") - .build()); - - } - - private SparkIMain intp; - private ByteArrayOutputStream out; - private DependencyContext depc; - private SparkJLineCompletion completor; - private SparkILoop interpreter; - - public DepInterpreter(Properties property) { - super(property); - } - - public DependencyContext getDependencyContext() { - return depc; - } - - - @Override - public void close() { - if (intp != null) { - intp.close(); - } - } - - @Override - public void open() { - out = new ByteArrayOutputStream(); - createIMain(); - } - - - private void createIMain() { - Settings settings = new Settings(); - URL[] urls = getClassloaderUrls(); - - // set classpath for scala compiler - PathSetting pathSettings = settings.classpath(); - String classpath = ""; - List<File> paths = currentClassPath(); - for (File f : paths) { - if (classpath.length() > 0) { - classpath += File.pathSeparator; - } - classpath += f.getAbsolutePath(); - } - - if (urls != null) { - for (URL u : urls) { - if (classpath.length() > 0) { - classpath += File.pathSeparator; - } - classpath += u.getFile(); - } - } - - pathSettings.v_$eq(classpath); - settings.scala$tools$nsc$settings$ScalaSettings$_setter_$classpath_$eq(pathSettings); - - // set classloader for scala compiler - settings.explicitParentLoader_$eq(new Some<ClassLoader>(Thread.currentThread() - .getContextClassLoader())); - - BooleanSetting b = (BooleanSetting) settings.usejavacp(); - b.v_$eq(true); - settings.scala$tools$nsc$settings$StandardScalaSettings$_setter_$usejavacp_$eq(b); - - interpreter = new SparkILoop(null, new PrintWriter(out)); - interpreter.settings_$eq(settings); - - interpreter.createInterpreter(); - - - intp = interpreter.intp(); - intp.setContextClassLoader(); - intp.initializeSynchronous(); - - depc = new DependencyContext(getProperty("zeppelin.dep.localrepo")); - completor = new SparkJLineCompletion(intp); - - intp.interpret("@transient var _binder = new java.util.HashMap[String, Object]()"); - Map<String, Object> binder = (Map<String, Object>) getValue("_binder"); - binder.put("depc", depc); - - intp.interpret("@transient val z = " - + "_binder.get(\"depc\").asInstanceOf[com.nflabs.zeppelin.spark.dep.DependencyContext]"); - - } - - public Object getValue(String name) { - Object ret = intp.valueOfTerm(name); - if (ret instanceof None) { - return null; - } else if (ret instanceof Some) { - return ((Some) ret).get(); - } else { - return ret; - } - } - - @Override - public InterpreterResult interpret(String st, InterpreterContext context) { - PrintStream printStream = new PrintStream(out); - Console.setOut(printStream); - out.reset(); - - SparkInterpreter sparkInterpreter = getSparkInterpreter(); - - if (sparkInterpreter != null && sparkInterpreter.isSparkContextInitialized()) { - return new InterpreterResult(Code.ERROR, - "Must be used before SparkInterpreter (%spark) initialized"); - } - - scala.tools.nsc.interpreter.Results.Result ret = intp.interpret(st); - Code code = getResultCode(ret); - - try { - depc.fetch(); - } catch (MalformedURLException | DependencyResolutionException - | ArtifactResolutionException e) { - return new InterpreterResult(Code.ERROR, e.toString()); - } - - if (code == Code.INCOMPLETE) { - return new InterpreterResult(code, "Incomplete expression"); - } else if (code == Code.ERROR) { - return new InterpreterResult(code, out.toString()); - } else { - return new InterpreterResult(code, out.toString()); - } - } - - private Code getResultCode(scala.tools.nsc.interpreter.Results.Result r) { - if (r instanceof scala.tools.nsc.interpreter.Results.Success$) { - return Code.SUCCESS; - } else if (r instanceof scala.tools.nsc.interpreter.Results.Incomplete$) { - return Code.INCOMPLETE; - } else { - return Code.ERROR; - } - } - - @Override - public void cancel(InterpreterContext context) { - } - - - @Override - public FormType getFormType() { - return FormType.NATIVE; - } - - @Override - public int getProgress(InterpreterContext context) { - return 0; - } - - @Override - public List<String> completion(String buf, int cursor) { - ScalaCompleter c = completor.completer(); - Candidates ret = c.complete(buf, cursor); - return scala.collection.JavaConversions.asJavaList(ret.candidates()); - } - - private List<File> currentClassPath() { - List<File> paths = classPath(Thread.currentThread().getContextClassLoader()); - String[] cps = System.getProperty("java.class.path").split(File.pathSeparator); - if (cps != null) { - for (String cp : cps) { - paths.add(new File(cp)); - } - } - return paths; - } - - private List<File> classPath(ClassLoader cl) { - List<File> paths = new LinkedList<File>(); - if (cl == null) { - return paths; - } - - if (cl instanceof URLClassLoader) { - URLClassLoader ucl = (URLClassLoader) cl; - URL[] urls = ucl.getURLs(); - if (urls != null) { - for (URL url : urls) { - paths.add(new File(url.getFile())); - } - } - } - return paths; - } - - private SparkInterpreter getSparkInterpreter() { - InterpreterGroup intpGroup = getInterpreterGroup(); - if (intpGroup == null) { - return null; - } - synchronized (intpGroup) { - for (Interpreter intp : intpGroup){ - if (intp.getClassName().equals(SparkInterpreter.class.getName())) { - Interpreter p = intp; - while (p instanceof WrappedInterpreter) { - p = ((WrappedInterpreter) p).getInnerInterpreter(); - } - return (SparkInterpreter) p; - } - } - } - return null; - } - - @Override - public Scheduler getScheduler() { - return getSparkInterpreter().getScheduler(); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/669d408d/spark/src/main/java/com/nflabs/zeppelin/spark/PySparkInterpreter.java ---------------------------------------------------------------------- diff --git a/spark/src/main/java/com/nflabs/zeppelin/spark/PySparkInterpreter.java b/spark/src/main/java/com/nflabs/zeppelin/spark/PySparkInterpreter.java deleted file mode 100644 index f09667d..0000000 --- a/spark/src/main/java/com/nflabs/zeppelin/spark/PySparkInterpreter.java +++ /dev/null @@ -1,406 +0,0 @@ -package com.nflabs.zeppelin.spark; - -import java.io.BufferedWriter; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStreamWriter; -import java.io.PipedInputStream; -import java.io.PipedOutputStream; -import java.net.ServerSocket; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Properties; - -import org.apache.commons.compress.utils.IOUtils; -import org.apache.commons.exec.CommandLine; -import org.apache.commons.exec.DefaultExecutor; -import org.apache.commons.exec.ExecuteException; -import org.apache.commons.exec.ExecuteResultHandler; -import org.apache.commons.exec.ExecuteWatchdog; -import org.apache.commons.exec.PumpStreamHandler; -import org.apache.commons.exec.environment.EnvironmentUtils; -import org.apache.spark.SparkConf; -import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.sql.SQLContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import py4j.GatewayServer; - -import com.nflabs.zeppelin.interpreter.Interpreter; -import com.nflabs.zeppelin.interpreter.InterpreterContext; -import com.nflabs.zeppelin.interpreter.InterpreterException; -import com.nflabs.zeppelin.interpreter.InterpreterGroup; -import com.nflabs.zeppelin.interpreter.InterpreterPropertyBuilder; -import com.nflabs.zeppelin.interpreter.InterpreterResult; -import com.nflabs.zeppelin.interpreter.InterpreterResult.Code; -import com.nflabs.zeppelin.interpreter.LazyOpenInterpreter; -import com.nflabs.zeppelin.interpreter.WrappedInterpreter; - -/** - * - */ -public class PySparkInterpreter extends Interpreter implements ExecuteResultHandler { - Logger logger = LoggerFactory.getLogger(PySparkInterpreter.class); - private GatewayServer gatewayServer; - private DefaultExecutor executor; - private int port; - private ByteArrayOutputStream outputStream; - private ByteArrayOutputStream errStream; - private BufferedWriter ins; - private PipedInputStream in; - private ByteArrayOutputStream input; - private String scriptPath; - boolean pythonscriptRunning = false; - - static { - Interpreter.register( - "pyspark", - "spark", - PySparkInterpreter.class.getName(), - new InterpreterPropertyBuilder() - .add("spark.home", - SparkInterpreter.getSystemDefault("SPARK_HOME", "spark.home", ""), - "Spark home path. Should be provided for pyspark") - .add("zeppelin.pyspark.python", - SparkInterpreter.getSystemDefault("PYSPARK_PYTHON", null, "python"), - "Python command to run pyspark with").build()); - } - - public PySparkInterpreter(Properties property) { - super(property); - - scriptPath = System.getProperty("java.io.tmpdir") + "/zeppelin_pyspark.py"; - } - - private String getSparkHome() { - String sparkHome = getProperty("spark.home"); - if (sparkHome == null) { - throw new InterpreterException("spark.home is undefined"); - } else { - return sparkHome; - } - } - - - private void createPythonScript() { - ClassLoader classLoader = getClass().getClassLoader(); - File out = new File(scriptPath); - - if (out.exists() && out.isDirectory()) { - throw new InterpreterException("Can't create python script " + out.getAbsolutePath()); - } - - try { - FileOutputStream outStream = new FileOutputStream(out); - IOUtils.copy( - classLoader.getResourceAsStream("python/zeppelin_pyspark.py"), - outStream); - outStream.close(); - } catch (IOException e) { - throw new InterpreterException(e); - } - - logger.info("File {} created", scriptPath); - } - - @Override - public void open() { - // create python script - createPythonScript(); - - port = findRandomOpenPortOnAllLocalInterfaces(); - - gatewayServer = new GatewayServer(this, port); - gatewayServer.start(); - - // Run python shell - CommandLine cmd = CommandLine.parse(getProperty("zeppelin.pyspark.python")); - cmd.addArgument(scriptPath, false); - cmd.addArgument(Integer.toString(port), false); - executor = new DefaultExecutor(); - outputStream = new ByteArrayOutputStream(); - PipedOutputStream ps = new PipedOutputStream(); - in = null; - try { - in = new PipedInputStream(ps); - } catch (IOException e1) { - throw new InterpreterException(e1); - } - ins = new BufferedWriter(new OutputStreamWriter(ps)); - - input = new ByteArrayOutputStream(); - - PumpStreamHandler streamHandler = new PumpStreamHandler(outputStream, outputStream, in); - executor.setStreamHandler(streamHandler); - executor.setWatchdog(new ExecuteWatchdog(ExecuteWatchdog.INFINITE_TIMEOUT)); - - - try { - Map env = EnvironmentUtils.getProcEnvironment(); - - String pythonPath = (String) env.get("PYTHONPATH"); - if (pythonPath == null) { - pythonPath = ""; - } else { - pythonPath += ":"; - } - - pythonPath += getSparkHome() + "/python/lib/py4j-0.8.2.1-src.zip:" - + getSparkHome() + "/python"; - - env.put("PYTHONPATH", pythonPath); - - executor.execute(cmd, env, this); - pythonscriptRunning = true; - } catch (IOException e) { - throw new InterpreterException(e); - } - - - try { - input.write("import sys, getopt\n".getBytes()); - ins.flush(); - } catch (IOException e) { - throw new InterpreterException(e); - } - } - - private int findRandomOpenPortOnAllLocalInterfaces() { - int port; - try (ServerSocket socket = new ServerSocket(0);) { - port = socket.getLocalPort(); - socket.close(); - } catch (IOException e) { - throw new InterpreterException(e); - } - return port; - } - - @Override - public void close() { - executor.getWatchdog().destroyProcess(); - gatewayServer.shutdown(); - } - - PythonInterpretRequest pythonInterpretRequest = null; - - /** - * - */ - public class PythonInterpretRequest { - public String statements; - public String jobGroup; - - public PythonInterpretRequest(String statements, String jobGroup) { - this.statements = statements; - this.jobGroup = jobGroup; - } - - public String statements() { - return statements; - } - - public String jobGroup() { - return jobGroup; - } - } - - Integer statementSetNotifier = new Integer(0); - - public PythonInterpretRequest getStatements() { - synchronized (statementSetNotifier) { - while (pythonInterpretRequest == null) { - try { - statementSetNotifier.wait(1000); - } catch (InterruptedException e) { - } - } - PythonInterpretRequest req = pythonInterpretRequest; - pythonInterpretRequest = null; - return req; - } - } - - String statementOutput = null; - boolean statementError = false; - Integer statementFinishedNotifier = new Integer(0); - - public void setStatementsFinished(String out, boolean error) { - synchronized (statementFinishedNotifier) { - statementOutput = out; - statementError = error; - statementFinishedNotifier.notify(); - } - - } - - boolean pythonScriptInitialized = false; - Integer pythonScriptInitializeNotifier = new Integer(0); - - public void onPythonScriptInitialized() { - synchronized (pythonScriptInitializeNotifier) { - pythonScriptInitialized = true; - pythonScriptInitializeNotifier.notifyAll(); - } - } - - @Override - public InterpreterResult interpret(String st, InterpreterContext context) { - if (!pythonscriptRunning) { - return new InterpreterResult(Code.ERROR, "python process not running" - + outputStream.toString()); - } - - outputStream.reset(); - - synchronized (pythonScriptInitializeNotifier) { - long startTime = System.currentTimeMillis(); - while (pythonScriptInitialized == false - && pythonscriptRunning - && System.currentTimeMillis() - startTime < 10 * 1000) { - try { - pythonScriptInitializeNotifier.wait(1000); - } catch (InterruptedException e) { - } - } - } - - if (pythonscriptRunning == false) { - // python script failed to initialize and terminated - return new InterpreterResult(Code.ERROR, "failed to start pyspark" - + outputStream.toString()); - } - if (pythonScriptInitialized == false) { - // timeout. didn't get initialized message - return new InterpreterResult(Code.ERROR, "pyspark is not responding " - + outputStream.toString()); - } - - SparkInterpreter sparkInterpreter = getSparkInterpreter(); - if (!sparkInterpreter.getSparkContext().version().startsWith("1.2") && - !sparkInterpreter.getSparkContext().version().startsWith("1.3")) { - return new InterpreterResult(Code.ERROR, "pyspark " - + sparkInterpreter.getSparkContext().version() + " is not supported"); - } - String jobGroup = sparkInterpreter.getJobGroup(context); - ZeppelinContext z = sparkInterpreter.getZeppelinContext(); - z.setInterpreterContext(context); - z.setGui(context.getGui()); - pythonInterpretRequest = new PythonInterpretRequest(st, jobGroup); - statementOutput = null; - - synchronized (statementSetNotifier) { - statementSetNotifier.notify(); - } - - synchronized (statementFinishedNotifier) { - while (statementOutput == null) { - try { - statementFinishedNotifier.wait(1000); - } catch (InterruptedException e) { - } - } - } - - if (statementError) { - return new InterpreterResult(Code.ERROR, statementOutput); - } else { - return new InterpreterResult(Code.SUCCESS, statementOutput); - } - } - - @Override - public void cancel(InterpreterContext context) { - SparkInterpreter sparkInterpreter = getSparkInterpreter(); - sparkInterpreter.cancel(context); - } - - @Override - public FormType getFormType() { - return FormType.NATIVE; - } - - @Override - public int getProgress(InterpreterContext context) { - SparkInterpreter sparkInterpreter = getSparkInterpreter(); - return sparkInterpreter.getProgress(context); - } - - @Override - public List<String> completion(String buf, int cursor) { - // not supported - return new LinkedList<String>(); - } - - private SparkInterpreter getSparkInterpreter() { - InterpreterGroup intpGroup = getInterpreterGroup(); - synchronized (intpGroup) { - for (Interpreter intp : getInterpreterGroup()){ - if (intp.getClassName().equals(SparkInterpreter.class.getName())) { - Interpreter p = intp; - while (p instanceof WrappedInterpreter) { - if (p instanceof LazyOpenInterpreter) { - ((LazyOpenInterpreter) p).open(); - } - p = ((WrappedInterpreter) p).getInnerInterpreter(); - } - return (SparkInterpreter) p; - } - } - } - return null; - } - - public ZeppelinContext getZeppelinContext() { - SparkInterpreter sparkIntp = getSparkInterpreter(); - if (sparkIntp != null) { - return getSparkInterpreter().getZeppelinContext(); - } else { - return null; - } - } - - public JavaSparkContext getJavaSparkContext() { - SparkInterpreter intp = getSparkInterpreter(); - if (intp == null) { - return null; - } else { - return new JavaSparkContext(intp.getSparkContext()); - } - } - - public SparkConf getSparkConf() { - JavaSparkContext sc = getJavaSparkContext(); - if (sc == null) { - return null; - } else { - return getJavaSparkContext().getConf(); - } - } - - public SQLContext getSQLContext() { - SparkInterpreter intp = getSparkInterpreter(); - if (intp == null) { - return null; - } else { - return intp.getSQLContext(); - } - } - - - @Override - public void onProcessComplete(int exitValue) { - pythonscriptRunning = false; - logger.info("python process terminated. exit code " + exitValue); - } - - @Override - public void onProcessFailed(ExecuteException e) { - pythonscriptRunning = false; - logger.error("python process failed", e); - } -}
