Repository: incubator-zeppelin Updated Branches: refs/heads/master ecc2fc623 -> 2dc464cfd
[ZEPPELIN-647] - Native Windows support for startup scripts and configuration ### What is this PR for? This is to give Windows first-class support for running Zeppelin without the need for Cygwin or other hacks. ### What type of PR is it? Improvement ### Todos * [x] - Fix notebook dir path handling which right now assumes URI compatible string (see https://github.com/apache/incubator-zeppelin/blob/master/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java#L63) * [x] - Add documentation for configuring and running on Windows * [x] - Independent code review of the CMD scripts to ensure they're correct ### Is there a relevant Jira issue? ZEPPELIN-647 ### How should this be tested? * Pull this PR * Build * Override default ZEPPELIN_NOTEBOOK_DIR in zeppelin-env.cmd to be an absolute file URI such as file:///c:/notebook * Start with bin\zeppelin.cmd * If using any Hadoop system ensure you have winutils.exe in your HADOOP_HOME\bin, see (https://github.com/steveloughran/winutils) ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? Yes Author: Silvio Fiorito <[email protected]> Author: Silvio Fiorito <Silvio Fiorito> Closes #734 from granturing/windows-support and squashes the following commits: 8aadd45 [Silvio Fiorito] Fixes to handle spaces in paths properly, both for ZEPPELIN_HOME and CLASSPATH 73aaf4f [Silvio Fiorito] Default to the appropriate interpreter when running on Windows db28fe9 [Silvio Fiorito] Support for running unit tests on Windows using the appropriate interpreter script a1e3097 [Silvio Fiorito] Support for Windows CMD shell interpreter 82acdcf [Silvio Fiorito] Merge branch 'master' into windows-support 9e8b309 [Silvio Fiorito] Initital doc updates for running on Windows 03baf62 [Silvio Fiorito] Additional fix for embedded pyspark environment variables 2b9f01c [Silvio Fiorito] Fix for pyspark PYTHONPATH environment variable not being set properly due to delayed expansion c700808 [Silvio Fiorito] Check for Windows path before creating URI to prevent URISyntaxExecption d30e4b9 [Silvio Fiorito] And again fix indentations missed last time 5b49d3e [Silvio Fiorito] Cleaned up indentation 9e40482 [Silvio Fiorito] Initial support for Windows platform, startup scripts Project: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/commit/2dc464cf Tree: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/tree/2dc464cf Diff: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/diff/2dc464cf Branch: refs/heads/master Commit: 2dc464cfda43eddedd55432e70988cba771e7931 Parents: ecc2fc6 Author: Silvio Fiorito <[email protected]> Authored: Mon Mar 14 23:17:32 2016 -0400 Committer: Lee moon soo <[email protected]> Committed: Thu Mar 24 08:04:26 2016 -0700 ---------------------------------------------------------------------- .gitignore | 1 + bin/common.cmd | 112 +++++++++++++++ bin/functions.cmd | 38 ++++++ bin/interpreter.cmd | 136 +++++++++++++++++++ bin/zeppelin.cmd | 91 +++++++++++++ conf/zeppelin-env.cmd.template | 64 +++++++++ docs/install/install.md | 7 +- docs/interpreter/spark.md | 2 + .../apache/zeppelin/shell/ShellInterpreter.java | 21 +-- .../src/assemble/distribution.xml | 6 + .../remote/RemoteAngularObjectTest.java | 7 +- .../RemoteInterpreterOutputTestStream.java | 6 +- .../remote/RemoteInterpreterProcessTest.java | 8 +- .../remote/RemoteInterpreterTest.java | 13 +- .../resource/DistributedResourcePoolTest.java | 8 +- .../zeppelin/scheduler/RemoteSchedulerTest.java | 8 +- .../zeppelin/conf/ZeppelinConfiguration.java | 4 +- .../zeppelin/notebook/repo/VFSNotebookRepo.java | 8 +- 18 files changed, 515 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/2dc464cf/.gitignore ---------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore index 3f3d654..ad473ed 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ lens/lens-cli-hist.log # conf file conf/zeppelin-env.sh +conf/zeppelin-env.cmd conf/zeppelin-site.xml conf/keystore conf/truststore http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/2dc464cf/bin/common.cmd ---------------------------------------------------------------------- diff --git a/bin/common.cmd b/bin/common.cmd new file mode 100644 index 0000000..c84f077 --- /dev/null +++ b/bin/common.cmd @@ -0,0 +1,112 @@ +@echo off + +REM Licensed to the Apache Software Foundation (ASF) under one or more +REM contributor license agreements. See the NOTICE file distributed with +REM this work for additional information regarding copyright ownership. +REM The ASF licenses this file to You under the Apache License, Version 2.0 +REM (the "License"); you may not use this file except in compliance with +REM the License. You may obtain a copy of the License at +REM +REM http://www.apache.org/licenses/LICENSE-2.0 +REM +REM Unless required by applicable law or agreed to in writing, software +REM distributed under the License is distributed on an "AS IS" BASIS, +REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +REM See the License for the specific language governing permissions and +REM limitations under the License. + +if not defined ZEPPELIN_HOME ( + for %%d in ("%~dp0..") do ( + set ZEPPELIN_HOME=%%~fd + ) +) + +if not defined ZEPPELIN_CONF_DIR ( + set ZEPPELIN_CONF_DIR=%ZEPPELIN_HOME%\conf +) + +if not defined ZEPPELIN_LOG_DIR ( + set ZEPPELIN_LOG_DIR=%ZEPPELIN_HOME%\logs +) + +if not defined ZEPPELIN_NOTEBOOK_DIR ( + set ZEPPELIN_NOTEBOOK_DIR=%ZEPPELIN_HOME%\notebook +) + +if not defined ZEPPELIN_PID_DIR ( + set ZEPPELIN_PID_DIR=%ZEPPELIN_HOME%\run +) + +if not defined ZEPPELIN_WAR ( + if exist "%ZEPPELIN_HOME%\zeppelin-web\dist" ( + set ZEPPELIN_WAR=%ZEPPELIN_HOME%\zeppelin-web\dist + ) else ( + for %%d in ("%ZEPPELIN_HOME%\zeppelin-web*.war") do ( + set ZEPPELIN_WAR=%%d + ) + ) +) + +if not defined ZEPPELIN_INTERPRETER_DIR ( + set ZEPPELIN_INTERPRETER_DIR=%ZEPPELIN_HOME%\interpreter +) + +if exist "%ZEPPELIN_CONF_DIR%\zeppelin-env.cmd" ( + call "%ZEPPELIN_CONF_DIR%\zeppelin-env.cmd" +) + +if not defined ZEPPELIN_CLASSPATH ( + set ZEPPELIN_CLASSPATH="%ZEPPELIN_CONF_DIR%" +) else ( + set ZEPPELIN_CLASSPATH=%ZEPPELIN_CLASSPATH%;"%ZEPPELIN_CONF_DIR%" +) + +if not defined ZEPPELIN_ENCODING ( + set ZEPPELIN_ENCODING=UTF-8 +) + +if not defined ZEPPELIN_MEM ( + set ZEPPELIN_MEM=-Xms1024m -Xmx1024m -XX:MaxPermSize=512m +) + +if not defined ZEPPELIN_JAVA_OPTS ( + set ZEPPELIN_JAVA_OPTS=-Dfile.encoding=%ZEPPELIN_ENCODING% %ZEPPELIN_MEM% +) else ( + set ZEPPELIN_JAVA_OPTS=%ZEPPELIN_JAVA_OPTS% -Dfile.encoding=%ZEPPELIN_ENCODING% %ZEPPELIN_MEM% +) + +if not defined JAVA_OPTS ( + set JAVA_OPTS=%ZEPPELIN_JAVA_OPTS% +) else ( + set JAVA_OPTS=%JAVA_OPTS% %ZEPPELIN_JAVA_OPTS% +) + +if not defined ZEPPELIN_INTP_JAVA_OPTS ( + set ZEPPELIN_INTP_JAVA_OPTS=%ZEPPELIN_JAVA_OPTS% +) + +if not defined ZEPPELIN_INTP_MEM ( + set ZEPPELIN_INTP_MEM=%ZEPPELIN_MEM% +) + +set JAVA_INTP_OPTS=%ZEPPELIN_INTP_JAVA_OPTS% -Dfile.encoding=%ZEPPELIN_ENCODING% + +if not defined JAVA_HOME ( + set ZEPPELIN_RUNNER=java +) else ( + set ZEPPELIN_RUNNER=%JAVA_HOME%\bin\java +) + +if not defined ZEPPELIN_IDENT_STRING ( + set ZEPPELIN_IDENT_STRING=%USERNAME% +) + +if not defined DEBUG ( + set DEBUG=0 +) + +if not defined ZEPPELIN_INTERPRETER_REMOTE_RUNNER ( + set ZEPPELIN_INTERPRETER_REMOTE_RUNNER=bin\interpreter.cmd +) + +exit /b http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/2dc464cf/bin/functions.cmd ---------------------------------------------------------------------- diff --git a/bin/functions.cmd b/bin/functions.cmd new file mode 100644 index 0000000..2aa4f74 --- /dev/null +++ b/bin/functions.cmd @@ -0,0 +1,38 @@ +@echo off + +REM Licensed to the Apache Software Foundation (ASF) under one or more +REM contributor license agreements. See the NOTICE file distributed with +REM this work for additional information regarding copyright ownership. +REM The ASF licenses this file to You under the Apache License, Version 2.0 +REM (the "License"); you may not use this file except in compliance with +REM the License. You may obtain a copy of the License at +REM +REM http://www.apache.org/licenses/LICENSE-2.0 +REM +REM Unless required by applicable law or agreed to in writing, software +REM distributed under the License is distributed on an "AS IS" BASIS, +REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +REM See the License for the specific language governing permissions and +REM limitations under the License. + +if not "%1"=="" goto %1 + +exit /b + +:ADDEACHJARINDIR +for %%d in ("%~2\*.jar") do ( + set ZEPPELIN_CLASSPATH="%%d";!ZEPPELIN_CLASSPATH! +) +exit /b + +:ADDEACHJARINDIRRECURSIVE +for /r "%~2" %%d in (*.jar) do ( + set ZEPPELIN_CLASSPATH="%%d";!ZEPPELIN_CLASSPATH! +) +exit /b + +:ADDJARINDIR +if exist "%~2" ( + set ZEPPELIN_CLASSPATH="%~2\*";%ZEPPELIN_CLASSPATH% +) +exit /b http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/2dc464cf/bin/interpreter.cmd ---------------------------------------------------------------------- diff --git a/bin/interpreter.cmd b/bin/interpreter.cmd new file mode 100644 index 0000000..4a501f0 --- /dev/null +++ b/bin/interpreter.cmd @@ -0,0 +1,136 @@ +@echo off + +REM Licensed to the Apache Software Foundation (ASF) under one or more +REM contributor license agreements. See the NOTICE file distributed with +REM this work for additional information regarding copyright ownership. +REM The ASF licenses this file to You under the Apache License, Version 2.0 +REM (the "License"); you may not use this file except in compliance with +REM the License. You may obtain a copy of the License at +REM +REM http://www.apache.org/licenses/LICENSE-2.0 +REM +REM Unless required by applicable law or agreed to in writing, software +REM distributed under the License is distributed on an "AS IS" BASIS, +REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +REM See the License for the specific language governing permissions and +REM limitations under the License. + +setlocal enableextensions enabledelayedexpansion + +set bin=%~dp0 + +:loop +if "%~1"=="" goto cont +if /I "%~1"=="-h" goto usage +if /I "%~1"=="-d" ( + set INTERPRETER_DIR=%~2 + set INTERPRETER_ID=%~n2 +) +if /I "%~1"=="-p" set PORT=%~2 +if /I "%~1"=="-l" set LOCAL_INTERPRETER_REPO=%~2 +shift +goto loop +:cont + +if "%PORT%"=="" goto usage +if "%INTERPRETER_DIR%"=="" goto usage + +call "%bin%\common.cmd" + +if exist "%ZEPPELIN_HOME%\zeppelin-interpreter\target\classes" ( + set ZEPPELIN_CLASSPATH=%ZEPPELIN_CLASSPATH%;"%ZEPPELIN_HOME%\zeppelin-interpreter\target\classes" +) else ( + for %%d in ("%ZEPPELIN_HOME%\lib\zeppelin-interpreter*.jar") do ( + set ZEPPELIN_INTERPRETER_JAR=%%d + ) + set ZEPPELIN_CLASSPATH=%ZEPPELIN_CLASSPATH%;"!ZEPPELIN_INTERPRETER_JAR!" +) + +call "%bin%\functions.cmd" ADDJARINDIR "%ZEPPELIN_HOME%\zeppelin-interpreter\target\lib" +call "%bin%\functions.cmd" ADDJARINDIR "%INTERPRETER_DIR%" + +set HOSTNAME=%COMPUTERNAME% +set ZEPPELIN_SERVER=org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer + +set ZEPPELIN_LOGFILE=%ZEPPELIN_LOG_DIR%\zeppelin-interpreter-%INTERPRETER_ID%-%ZEPPELIN_IDENT_STRING%-%HOSTNAME%.log + +if not exist "%ZEPPELIN_LOG_DIR%" ( + echo Log dir doesn't exist, create %ZEPPELIN_LOG_DIR% + mkdir "%ZEPPELIN_LOG_DIR%" +) + +if /I "%INTERPRETER_ID%"=="spark" ( + if defined SPARK_HOME ( + set SPARK_SUBMIT=%SPARK_HOME%\bin\spark-submit.cmd + for %%d in ("%ZEPPELIN_HOME%\interpreter\spark\zeppelin-spark*.jar") do ( + set SPARK_APP_JAR=%%d + ) + set ZEPPELIN_CLASSPATH="!SPARK_APP_JAR!" + + for %%d in ("%SPARK_HOME%\python\lib\py4j-*-src.zip") do ( + set py4j=%%d + ) + + if not defined PYTHONPATH ( + set PYTHONPATH=!py4j!;%SPARK_HOME%\python + ) else ( + set PYTHONPATH=!py4j!;%SPARK_HOME%\python;%PYTHONPATH% + ) + ) else ( + if defined HADOOP_HOME if exist "%HADOOP_HOME%\bin\hadoop.cmd" ( + for /f "tokens=*" %%d in ('"%HADOOP_HOME%\bin\hadoop.cmd" classpath') do ( + set LOCAL_HADOOP_CLASSPATH=%%d + ) + set ZEPPELIN_CLASSPATH=!LOCAL_HADOOP_CLASSPATH!;%ZEPPELIN_CLASSPATH% + ) + + call "%bin%\functions.cmd" ADDJARINDIR "%INTERPRETER_DIR%\dep" + + for %%d in ("%ZEPPELIN_HOME%\interpreter\spark\pyspark\py4j-*-src.zip") do ( + set py4j=%%d + ) + + set PYSPARKPATH=%ZEPPELIN_HOME%\interpreter\spark\pyspark\pyspark.zip;!py4j! + + if not defined PYTHONPATH ( + set PYTHONPATH=!PYSPARKPATH! + ) else ( + set PYTHONPATH=%PYTHONPATH%;!PYSPARKPATH! + ) + + set PYSPARKPATH= + + if defined HADOOP_HOME if not defined HADOOP_CONF_DIR ( + if exist "%HADOOP_HOME%\etc\hadoop" ( + set HADOOP_CONF_DIR=%HADOOP_HOME%\etc\hadoop + ) + ) + + if exist "%HADOOP_CONF_DIR%" ( + set ZEPPELIN_CLASSPATH=%ZEPPELIN_CLASSPATH%;"%HADOOP_CONF_DIR%" + ) + ) +) + +call "%bin%\functions.cmd" ADDJARINDIR "%LOCAL_INTERPRETER_REPO%" + +if not defined ZEPPELIN_CLASSPATH_OVERRIDES ( + set CLASSPATH=%ZEPPELIN_CLASSPATH% +) else ( + set CLASSPATH=%ZEPPELIN_CLASSPATH_OVERRIDES%;%ZEPPELIN_CLASSPATH% +) + +if defined SPARK_SUBMIT ( + set JAVA_INTP_OPTS=%JAVA_INTP_OPTS% -Dzeppelin.log.file='%ZEPPELIN_LOGFILE%' + + "%SPARK_SUBMIT%" --class %ZEPPELIN_SERVER% --jars %CLASSPATH% --driver-java-options "!JAVA_INTP_OPTS!" %SPARK_SUBMIT_OPTIONS% "%SPARK_APP_JAR%" %PORT% +) else ( + set JAVA_INTP_OPTS=%JAVA_INTP_OPTS% -Dzeppelin.log.file="%ZEPPELIN_LOGFILE%" + + "%ZEPPELIN_RUNNER%" !JAVA_INTP_OPTS! %ZEPPELIN_INTP_MEM% -cp %ZEPPELIN_CLASSPATH_OVERRIDES%;%CLASSPATH% %ZEPPELIN_SERVER% %PORT% +) + +exit /b + +:usage +echo Usage: %~n0 -p ^<port^> -d ^<interpreter dir to load^> -l ^<local interpreter repo dir to load^> http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/2dc464cf/bin/zeppelin.cmd ---------------------------------------------------------------------- diff --git a/bin/zeppelin.cmd b/bin/zeppelin.cmd new file mode 100644 index 0000000..a2d5644 --- /dev/null +++ b/bin/zeppelin.cmd @@ -0,0 +1,91 @@ +@echo off + +REM Licensed to the Apache Software Foundation (ASF) under one or more +REM contributor license agreements. See the NOTICE file distributed with +REM this work for additional information regarding copyright ownership. +REM The ASF licenses this file to You under the Apache License, Version 2.0 +REM (the "License"); you may not use this file except in compliance with +REM the License. You may obtain a copy of the License at +REM +REM http://www.apache.org/licenses/LICENSE-2.0 +REM +REM Unless required by applicable law or agreed to in writing, software +REM distributed under the License is distributed on an "AS IS" BASIS, +REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +REM See the License for the specific language governing permissions and +REM limitations under the License. + +setlocal enableextensions enabledelayedexpansion + +set bin=%~dp0 + +if not "%1"=="--config" goto MAIN + +:SET_CONFIG +shift +set conf_dir=%~f1 +shift +if not exist "%conf_dir%" ( + echo ERROR: %conf_dir% is not a directory + echo Usage: %~n0 [--config ^<conf-dir^>] + exit /b 1 +) else ( + set ZEPPELIN_CONF_DIR=%conf_dir% +) + +:MAIN +call "%bin%\common.cmd" + +set HOSTNAME=%COMPUTERNAME% +set ZEPPELIN_LOGFILE=%ZEPPELIN_LOG_DIR%\zeppelin-%ZEPPELIN_IDENT_STRING%-%HOSTNAME%.log + +set ZEPPELIN_SERVER=org.apache.zeppelin.server.ZeppelinServer +set JAVA_OPTS=%JAVA_OPTS% -Dzeppelin.log.file="%ZEPPELIN_LOGFILE%" + +if exist "%ZEPPELIN_HOME%\zeppelin-interpreter\target\classes" ( + set ZEPPELIN_CLASSPATH=%ZEPPELIN_CLASSPATH%;"%ZEPPELIN_HOME%\zeppelin-interpreter\target\classes" +) + +if exist "%ZEPPELIN_HOME%\zeppelin-zengine\target\classes" ( + set ZEPPELIN_CLASSPATH=%ZEPPELIN_CLASSPATH%;"%ZEPPELIN_HOME%\zeppelin-zengine\target\classes" +) + +if exist "%ZEPPELIN_HOME%\zeppelin-server\target\classes" ( + set ZEPPELIN_CLASSPATH=%ZEPPELIN_CLASSPATH%;"%ZEPPELIN_HOME%\zeppelin-server\target\classes" +) + +call "%bin%\functions.cmd" ADDJARINDIR "%ZEPPELIN_HOME%" +call "%bin%\functions.cmd" ADDJARINDIR "%ZEPPELIN_HOME%\lib" +call "%bin%\functions.cmd" ADDJARINDIR "%ZEPPELIN_HOME%\zeppelin-interpreter\target\lib" +call "%bin%\functions.cmd" ADDJARINDIR "%ZEPPELIN_HOME%\zeppelin-zengine\target\lib" +call "%bin%\functions.cmd" ADDJARINDIR "%ZEPPELIN_HOME%\zeppelin-server\target\lib" +call "%bin%\functions.cmd" ADDJARINDIR "%ZEPPELIN_HOME%\zeppelin-web\target\lib" + +if not defined CLASSPATH ( + set CLASSPATH=%ZEPPELIN_CLASSPATH% +) else ( + set CLASSPATH=%CLASSPATH%;%ZEPPELIN_CLASSPATH% +) + +if not defined ZEPPELIN_CLASSPATH_OVERRIDES ( + set CLASSPATH=%ZEPPELIN_CLASSPATH% +) else ( + set CLASSPATH=%ZEPPELIN_CLASSPATH_OVERRIDES%;%ZEPPELIN_CLASSPATH% +) + +if not exist %ZEPPELIN_LOG_DIR% ( + echo Log dir doesn't exist, create %ZEPPELIN_LOG_DIR% + mkdir "%ZEPPELIN_LOG_DIR%" +) + +if not exist %ZEPPELIN_PID_DIR% ( + echo Pid dir doesn't exist, create %ZEPPELIN_PID_DIR% + mkdir "%ZEPPELIN_PID_DIR%" +) + +if not exist %ZEPPELIN_NOTEBOOK_DIR% ( + echo Notebook dir doesn't exist, create %ZEPPELIN_NOTEBOOK_DIR% + mkdir "%ZEPPELIN_NOTEBOOK_DIR%" +) + +"%ZEPPELIN_RUNNER%" %JAVA_OPTS% -cp %CLASSPATH% %ZEPPELIN_SERVER% "%*" http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/2dc464cf/conf/zeppelin-env.cmd.template ---------------------------------------------------------------------- diff --git a/conf/zeppelin-env.cmd.template b/conf/zeppelin-env.cmd.template new file mode 100644 index 0000000..06799b5 --- /dev/null +++ b/conf/zeppelin-env.cmd.template @@ -0,0 +1,64 @@ +@echo off + +REM Licensed to the Apache Software Foundation (ASF) under one or more +REM contributor license agreements. See the NOTICE file distributed with +REM this work for additional information regarding copyright ownership. +REM The ASF licenses this file to You under the Apache License, Version 2.0 +REM (the "License"); you may not use this file except in compliance with +REM the License. You may obtain a copy of the License at +REM +REM http://www.apache.org/licenses/LICENSE-2.0 +REM +REM Unless required by applicable law or agreed to in writing, software +REM distributed under the License is distributed on an "AS IS" BASIS, +REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +REM See the License for the specific language governing permissions and +REM limitations under the License. +REM + +REM set JAVA_HOME= +REM set MASTER= REM Spark master url. eg. spark://master_addr:7077. Leave empty if you want to use local mode. +REM set ZEPPELIN_JAVA_OPTS REM Additional jvm options. for example, set ZEPPELIN_JAVA_OPTS="-Dspark.executor.memory=8g -Dspark.cores.max=16" +REM set ZEPPELIN_MEM REM Zeppelin jvm mem options Default -Xmx1024m -XX:MaxPermSize=512m +REM set ZEPPELIN_INTP_MEM REM zeppelin interpreter process jvm mem options. Default = ZEPPELIN_MEM +REM set ZEPPELIN_INTP_JAVA_OPTS REM zeppelin interpreter process jvm options. Default = ZEPPELIN_JAVA_OPTS + +REM set ZEPPELIN_LOG_DIR REM Where log files are stored. PWD by default. +REM set ZEPPELIN_PID_DIR REM The pid files are stored. /tmp by default. +REM set ZEPPELIN_WAR_TEMPDIR REM The location of jetty temporary directory. +REM set ZEPPELIN_NOTEBOOK_DIR REM Where notebook saved +REM set ZEPPELIN_NOTEBOOK_HOMESCREEN REM Id of notebook to be displayed in homescreen. ex) 2A94M5J1Z +REM set ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE REM hide homescreen notebook from list when this value set to "true". default "false" +REM set ZEPPELIN_NOTEBOOK_S3_BUCKET REM Bucket where notebook saved +REM set ZEPPELIN_NOTEBOOK_S3_USER REM User in bucket where notebook saved. For example bucket/user/notebook/2A94M5J1Z/note.json +REM set ZEPPELIN_IDENT_STRING REM A string representing this instance of zeppelin. $USER by default. +REM set ZEPPELIN_NICENESS REM The scheduling priority for daemons. Defaults to 0. +REM set ZEPPELIN_INTERPRETER_LOCALREPO REM Local repository for interpreter's additional dependency loading + + +REM Spark interpreter configuration + +REM Use provided spark installation +REM defining SPARK_HOME makes Zeppelin run spark interpreter process using spark-submit +REM +REM set SPARK_HOME REM (required) When it is defined, load it instead of Zeppelin embedded Spark libraries +REM set SPARK_SUBMIT_OPTIONS REM (optional) extra options to pass to spark submit. eg) "--driver-memory 512M --executor-memory 1G". +REM set SPARK_APP_NAME REM (optional) The name of spark application. + +REM Use embedded spark binaries +REM without SPARK_HOME defined, Zeppelin still able to run spark interpreter process using embedded spark binaries. +REM however, it is not encouraged when you can define SPARK_HOME +REM +REM Options read in YARN client mode +REM set HADOOP_CONF_DIR REM yarn-site.xml is located in configuration directory in HADOOP_CONF_DIR. +REM Pyspark (supported with Spark 1.2.1 and above) +REM To configure pyspark, you need to set spark distribution's path to 'spark.home' property in Interpreter setting screen in Zeppelin GUI +REM set PYSPARK_PYTHON REM path to the python command. must be the same path on the driver(Zeppelin) and all workers. +REM set PYTHONPATH + +REM Spark interpreter options +REM +REM set ZEPPELIN_SPARK_USEHIVECONTEXT REM Use HiveContext instead of SQLContext if set true. true by default. +REM set ZEPPELIN_SPARK_CONCURRENTSQL REM Execute multiple SQL concurrently if set true. false by default. +REM set ZEPPELIN_SPARK_MAXRESULT REM Max number of SparkSQL result to display. 1000 by default. + http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/2dc464cf/docs/install/install.md ---------------------------------------------------------------------- diff --git a/docs/install/install.md b/docs/install/install.md index b86c5bb..884b4bf 100644 --- a/docs/install/install.md +++ b/docs/install/install.md @@ -45,7 +45,7 @@ If you don't have requirements prepared, please check instructions in [README.md <a name="zeppelin-configuration"> </a> ## Zeppelin Configuration -You can configure Zeppelin with both **environment variables** in `conf/zeppelin-env.sh` and **java properties** in `conf/zeppelin-site.xml`. If both are defined, then the **environment variables** will be used priorly. +You can configure Zeppelin with both **environment variables** in `conf/zeppelin-env.sh` (`conf\zeppelin-env.cmd` for Windows) and **Java properties** in `conf/zeppelin-site.xml`. If both are defined, then the **environment variables** will take priority. <table class="table-configuration"> <tr> @@ -278,4 +278,9 @@ chdir /usr/share/zeppelin exec bin/zeppelin-daemon.sh upstart ``` +#### Running on Windows + +``` +bin\zeppelin.cmd +``` http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/2dc464cf/docs/interpreter/spark.md ---------------------------------------------------------------------- diff --git a/docs/interpreter/spark.md b/docs/interpreter/spark.md index 21c3df5..5650af7 100644 --- a/docs/interpreter/spark.md +++ b/docs/interpreter/spark.md @@ -126,6 +126,8 @@ export HADOOP_CONF_DIR=/usr/lib/hadoop export SPARK_SUBMIT_OPTIONS="--packages com.databricks:spark-csv_2.10:1.2.0" ``` +For Windows, ensure you have `winutils.exe` in `%HADOOP_HOME%\bin`. For more details please see [Problems running Hadoop on Windows](https://wiki.apache.org/hadoop/WindowsProblems) + ### 2. Set master in Interpreter menu After start Zeppelin, go to **Interpreter** menu and edit **master** property in your Spark interpreter setting. The value may vary depending on your Spark cluster deployment type. http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/2dc464cf/shell/src/main/java/org/apache/zeppelin/shell/ShellInterpreter.java ---------------------------------------------------------------------- diff --git a/shell/src/main/java/org/apache/zeppelin/shell/ShellInterpreter.java b/shell/src/main/java/org/apache/zeppelin/shell/ShellInterpreter.java index 6133c32..e3c4d0c 100644 --- a/shell/src/main/java/org/apache/zeppelin/shell/ShellInterpreter.java +++ b/shell/src/main/java/org/apache/zeppelin/shell/ShellInterpreter.java @@ -19,17 +19,14 @@ package org.apache.zeppelin.shell; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Properties; - +import java.util.*; import org.apache.commons.exec.CommandLine; import org.apache.commons.exec.DefaultExecutor; import org.apache.commons.exec.ExecuteException; import org.apache.commons.exec.ExecuteWatchdog; import org.apache.commons.exec.Executor; import org.apache.commons.exec.PumpStreamHandler; +import org.apache.commons.lang3.StringUtils; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder; @@ -50,6 +47,10 @@ public class ShellInterpreter extends Interpreter { public static final String SHELL_COMMAND_TIMEOUT = "shell.command.timeout.millisecs"; public static final String DEFAULT_COMMAND_TIMEOUT = "600000"; int commandTimeOut; + private static final boolean isWindows = System + .getProperty("os.name") + .startsWith("Windows"); + final String shell = isWindows ? "cmd /c" : "bash -c"; static { Interpreter.register( @@ -83,11 +84,15 @@ public class ShellInterpreter extends Interpreter { @Override public InterpreterResult interpret(String cmd, InterpreterContext contextInterpreter) { logger.debug("Run shell command '" + cmd + "'"); - CommandLine cmdLine = CommandLine.parse("bash"); - cmdLine.addArgument("-c", false); + CommandLine cmdLine = CommandLine.parse(shell); + // the Windows CMD shell doesn't handle multiline statements, + // they need to be delimited by '&&' instead + if (isWindows) { + String[] lines = StringUtils.split(cmd, "\n"); + cmd = StringUtils.join(lines, " && "); + } cmdLine.addArgument(cmd, false); DefaultExecutor executor = new DefaultExecutor(); - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); ByteArrayOutputStream errorStream = new ByteArrayOutputStream(); executor.setStreamHandler(new PumpStreamHandler(contextInterpreter.out, errorStream)); executor.setWatchdog(new ExecuteWatchdog(commandTimeOut)); http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/2dc464cf/zeppelin-distribution/src/assemble/distribution.xml ---------------------------------------------------------------------- diff --git a/zeppelin-distribution/src/assemble/distribution.xml b/zeppelin-distribution/src/assemble/distribution.xml index 782aee9..4750a72 100644 --- a/zeppelin-distribution/src/assemble/distribution.xml +++ b/zeppelin-distribution/src/assemble/distribution.xml @@ -69,6 +69,12 @@ </fileSet> <fileSet> <directory>../conf</directory> + <excludes> + <exclude>interpreter.json</exclude> + <exclude>zeppelin-env.cmd</exclude> + <exclude>zeppelin-env.sh</exclude> + <exclude>zeppelin-site.xml</exclude> + </excludes> </fileSet> <fileSet> <directory>../interpreter</directory> http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/2dc464cf/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java index 84327dd..7ffa170 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java @@ -35,6 +35,11 @@ import org.junit.Before; import org.junit.Test; public class RemoteAngularObjectTest implements AngularObjectRegistryListener { + private static final String INTERPRETER_SCRIPT = + System.getProperty("os.name").startsWith("Windows") ? + "../bin/interpreter.cmd" : + "../bin/interpreter.sh"; + private InterpreterGroup intpGroup; private HashMap<String, String> env; private RemoteInterpreter intp; @@ -63,7 +68,7 @@ public class RemoteAngularObjectTest implements AngularObjectRegistryListener { p, "note", MockInterpreterAngular.class.getName(), - new File("../bin/interpreter.sh").getAbsolutePath(), + new File(INTERPRETER_SCRIPT).getAbsolutePath(), "fake", "fakeRepo", env, http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/2dc464cf/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java ---------------------------------------------------------------------- diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java index 3fbf5bc..4a473f3 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java @@ -38,6 +38,10 @@ import static org.junit.Assert.assertEquals; * Test for remote interpreter output stream */ public class RemoteInterpreterOutputTestStream implements RemoteInterpreterProcessListener { + private static final String INTERPRETER_SCRIPT = + System.getProperty("os.name").startsWith("Windows") ? + "../bin/interpreter.cmd" : + "../bin/interpreter.sh"; private InterpreterGroup intpGroup; private HashMap<String, String> env; @@ -61,7 +65,7 @@ public class RemoteInterpreterOutputTestStream implements RemoteInterpreterProce new Properties(), "note", MockInterpreterOutputStream.class.getName(), - new File("../bin/interpreter.sh").getAbsolutePath(), + new File(INTERPRETER_SCRIPT).getAbsolutePath(), "fake", "fakeRepo", env, http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/2dc464cf/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java index 7beaee1..52d3858 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java @@ -28,12 +28,16 @@ import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client; import org.junit.Test; public class RemoteInterpreterProcessTest { + private static final String INTERPRETER_SCRIPT = + System.getProperty("os.name").startsWith("Windows") ? + "../bin/interpreter.cmd" : + "../bin/interpreter.sh"; @Test public void testStartStop() { InterpreterGroup intpGroup = new InterpreterGroup(); RemoteInterpreterProcess rip = new RemoteInterpreterProcess( - "../bin/interpreter.sh", "nonexists", "fakeRepo", new HashMap<String, String>(), + INTERPRETER_SCRIPT, "nonexists", "fakeRepo", new HashMap<String, String>(), 10 * 1000, null); assertFalse(rip.isRunning()); assertEquals(0, rip.referenceCount()); @@ -50,7 +54,7 @@ public class RemoteInterpreterProcessTest { public void testClientFactory() throws Exception { InterpreterGroup intpGroup = new InterpreterGroup(); RemoteInterpreterProcess rip = new RemoteInterpreterProcess( - "../bin/interpreter.sh", "nonexists", "fakeRepo", new HashMap<String, String>(), + INTERPRETER_SCRIPT, "nonexists", "fakeRepo", new HashMap<String, String>(), mock(RemoteInterpreterEventPoller.class), 10 * 1000); rip.reference(intpGroup); assertEquals(0, rip.getNumActiveClient()); http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/2dc464cf/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java index f3b936b..b500906 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java @@ -53,6 +53,11 @@ import com.google.gson.reflect.TypeToken; public class RemoteInterpreterTest { + private static final String INTERPRETER_SCRIPT = + System.getProperty("os.name").startsWith("Windows") ? + "../bin/interpreter.cmd" : + "../bin/interpreter.sh"; + private InterpreterGroup intpGroup; private HashMap<String, String> env; @@ -78,7 +83,7 @@ public class RemoteInterpreterTest { p, noteId, MockInterpreterA.class.getName(), - new File("../bin/interpreter.sh").getAbsolutePath(), + new File(INTERPRETER_SCRIPT).getAbsolutePath(), "fake", "fakeRepo", env, @@ -95,7 +100,7 @@ public class RemoteInterpreterTest { p, noteId, MockInterpreterB.class.getName(), - new File("../bin/interpreter.sh").getAbsolutePath(), + new File(INTERPRETER_SCRIPT).getAbsolutePath(), "fake", "fakeRepo", env, @@ -193,7 +198,7 @@ public class RemoteInterpreterTest { p, "note", MockInterpreterA.class.getName(), - new File("../bin/interpreter.sh").getAbsolutePath(), + new File(INTERPRETER_SCRIPT).getAbsolutePath(), "fake", "fakeRepo", env, @@ -208,7 +213,7 @@ public class RemoteInterpreterTest { p, "note", MockInterpreterB.class.getName(), - new File("../bin/interpreter.sh").getAbsolutePath(), + new File(INTERPRETER_SCRIPT).getAbsolutePath(), "fake", "fakeRepo", env, http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/2dc464cf/zeppelin-interpreter/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java index e49b437..ae5e6f5 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java @@ -39,6 +39,10 @@ import static org.junit.Assert.assertTrue; * Unittest for DistributedResourcePool */ public class DistributedResourcePoolTest { + private static final String INTERPRETER_SCRIPT = + System.getProperty("os.name").startsWith("Windows") ? + "../bin/interpreter.cmd" : + "../bin/interpreter.sh"; private InterpreterGroup intpGroup1; private InterpreterGroup intpGroup2; private HashMap<String, String> env; @@ -60,7 +64,7 @@ public class DistributedResourcePoolTest { p, "note", MockInterpreterResourcePool.class.getName(), - new File("../bin/interpreter.sh").getAbsolutePath(), + new File(INTERPRETER_SCRIPT).getAbsolutePath(), "fake", "fakeRepo", env, @@ -77,7 +81,7 @@ public class DistributedResourcePoolTest { p, "note", MockInterpreterResourcePool.class.getName(), - new File("../bin/interpreter.sh").getAbsolutePath(), + new File(INTERPRETER_SCRIPT).getAbsolutePath(), "fake", "fakeRepo", env, http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/2dc464cf/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java index 9ce7a65..40dcef2 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java @@ -46,6 +46,10 @@ import org.junit.Test; public class RemoteSchedulerTest implements RemoteInterpreterProcessListener { + private static final String INTERPRETER_SCRIPT = + System.getProperty("os.name").startsWith("Windows") ? + "../bin/interpreter.cmd" : + "../bin/interpreter.sh"; private SchedulerFactory schedulerSvc; private static final int TICK_WAIT = 100; private static final int MAX_WAIT_CYCLES = 100; @@ -71,7 +75,7 @@ public class RemoteSchedulerTest implements RemoteInterpreterProcessListener { p, "note", MockInterpreterA.class.getName(), - new File("../bin/interpreter.sh").getAbsolutePath(), + new File(INTERPRETER_SCRIPT).getAbsolutePath(), "fake", "fakeRepo", env, @@ -159,7 +163,7 @@ public class RemoteSchedulerTest implements RemoteInterpreterProcessListener { p, "note", MockInterpreterA.class.getName(), - new File("../bin/interpreter.sh").getAbsolutePath(), + new File(INTERPRETER_SCRIPT).getAbsolutePath(), "fake", "fakeRepo", env, http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/2dc464cf/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index 25a9b12..3d51dfd 100755 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -481,7 +481,9 @@ public class ZeppelinConfiguration extends XMLConfiguration { ZEPPELIN_NOTEBOOK_AZURE_SHARE("zeppelin.notebook.azure.share", "zeppelin"), ZEPPELIN_NOTEBOOK_AZURE_USER("zeppelin.notebook.azure.user", "user"), ZEPPELIN_NOTEBOOK_STORAGE("zeppelin.notebook.storage", VFSNotebookRepo.class.getName()), - ZEPPELIN_INTERPRETER_REMOTE_RUNNER("zeppelin.interpreter.remoterunner", "bin/interpreter.sh"), + ZEPPELIN_INTERPRETER_REMOTE_RUNNER("zeppelin.interpreter.remoterunner", + System.getProperty("os.name") + .startsWith("Windows") ? "bin/interpreter.cmd" : "bin/interpreter.sh"), // Decide when new note is created, interpreter settings will be binded automatically or not. ZEPPELIN_NOTEBOOK_AUTO_INTERPRETER_BINDING("zeppelin.notebook.autoInterpreterBinding", true), ZEPPELIN_CONF_DIR("zeppelin.conf.dir", "conf"), http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/2dc464cf/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java index 6db4828..8c0cf2c 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java @@ -60,7 +60,11 @@ public class VFSNotebookRepo implements NotebookRepo { this.conf = conf; try { - filesystemRoot = new URI(conf.getNotebookDir()); + if (conf.isWindowsPath(conf.getNotebookDir())) { + filesystemRoot = new File(conf.getNotebookDir()).toURI(); + } else { + filesystemRoot = new URI(conf.getNotebookDir()); + } } catch (URISyntaxException e1) { throw new IOException(e1); } @@ -72,8 +76,6 @@ public class VFSNotebookRepo implements NotebookRepo { } catch (URISyntaxException e) { throw new IOException(e); } - } else { - this.filesystemRoot = filesystemRoot; } fsManager = VFS.getManager();
