http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/sql/functions.html ---------------------------------------------------------------------- diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/sql/functions.html b/site/docs/2.1.2/api/python/_modules/pyspark/sql/functions.html index 61efde4..3c07c92 100644 --- a/site/docs/2.1.2/api/python/_modules/pyspark/sql/functions.html +++ b/site/docs/2.1.2/api/python/_modules/pyspark/sql/functions.html @@ -86,8 +86,8 @@ <span class="n">sc</span> <span class="o">=</span> <span class="n">SparkContext</span><span class="o">.</span><span class="n">_active_spark_context</span> <span class="n">jc</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">sc</span><span class="o">.</span><span class="n">_jvm</span><span class="o">.</span><span class="n">functions</span><span class="p">,</span> <span class="n">name</span><span class="p">)(</span><span class="n">col</span><span class="o">.</span><span class="n">_jc</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">col</span><span class="p">,</span> <span class="n">Column</span><span class="p">)</span> <span class="k">else</span> <span class="n">col</span><span class="p">)</span> <span class="k">return</span> <span class="n">Column</span><span class="p">(</span><span class="n">jc</span><span class="p">)</span> - <span class="n">_</span><span class="o">.</span><span class="n">__name__</span> <span class="o">=</span> <span class="n">name</span> - <span class="n">_</span><span class="o">.</span><span class="n">__doc__</span> <span class="o">=</span> <span class="n">doc</span> + <span class="n">_</span><span class="o">.</span><span class="vm">__name__</span> <span class="o">=</span> <span class="n">name</span> + <span class="n">_</span><span class="o">.</span><span class="vm">__doc__</span> <span class="o">=</span> <span class="n">doc</span> <span class="k">return</span> <span class="n">_</span> @@ -99,8 +99,8 @@ <span class="n">jc</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">sc</span><span class="o">.</span><span class="n">_jvm</span><span class="o">.</span><span class="n">functions</span><span class="p">,</span> <span class="n">name</span><span class="p">)(</span><span class="n">col1</span><span class="o">.</span><span class="n">_jc</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">col1</span><span class="p">,</span> <span class="n">Column</span><span class="p">)</span> <span class="k">else</span> <span class="nb">float</span><span class="p">(</span><span class="n">col1</span><span class="p">),</span> <span class="n">col2</span><span class="o">.</span><span class="n">_jc</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">col2</span><span class="p">,</span> <span class="n">Column</span><span class="p">)</span> <span class="k">else</span> <span class="nb">float</span><span class="p">(</span><span class="n">col2</span><span class="p">))</span> <span class="k">return</span> <span class="n">Column</span><span class="p">(</span><span class="n">jc</span><span class="p">)</span> - <span class="n">_</span><span class="o">.</span><span class="n">__name__</span> <span class="o">=</span> <span class="n">name</span> - <span class="n">_</span><span class="o">.</span><span class="n">__doc__</span> <span class="o">=</span> <span class="n">doc</span> + <span class="n">_</span><span class="o">.</span><span class="vm">__name__</span> <span class="o">=</span> <span class="n">name</span> + <span class="n">_</span><span class="o">.</span><span class="vm">__doc__</span> <span class="o">=</span> <span class="n">doc</span> <span class="k">return</span> <span class="n">_</span> @@ -110,8 +110,8 @@ <span class="n">sc</span> <span class="o">=</span> <span class="n">SparkContext</span><span class="o">.</span><span class="n">_active_spark_context</span> <span class="n">jc</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">sc</span><span class="o">.</span><span class="n">_jvm</span><span class="o">.</span><span class="n">functions</span><span class="p">,</span> <span class="n">name</span><span class="p">)()</span> <span class="k">return</span> <span class="n">Column</span><span class="p">(</span><span class="n">jc</span><span class="p">)</span> - <span class="n">_</span><span class="o">.</span><span class="n">__name__</span> <span class="o">=</span> <span class="n">name</span> - <span class="n">_</span><span class="o">.</span><span class="n">__doc__</span> <span class="o">=</span> <span class="s1">'Window function: '</span> <span class="o">+</span> <span class="n">doc</span> + <span class="n">_</span><span class="o">.</span><span class="vm">__name__</span> <span class="o">=</span> <span class="n">name</span> + <span class="n">_</span><span class="o">.</span><span class="vm">__doc__</span> <span class="o">=</span> <span class="s1">'Window function: '</span> <span class="o">+</span> <span class="n">doc</span> <span class="k">return</span> <span class="n">_</span> @@ -1885,7 +1885,7 @@ <span class="n">jdt</span> <span class="o">=</span> <span class="n">spark</span><span class="o">.</span><span class="n">_jsparkSession</span><span class="o">.</span><span class="n">parseDataType</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">returnType</span><span class="o">.</span><span class="n">json</span><span class="p">())</span> <span class="k">if</span> <span class="n">name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> <span class="n">f</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">func</span> - <span class="n">name</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">__name__</span> <span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">f</span><span class="p">,</span> <span class="s1">'__name__'</span><span class="p">)</span> <span class="k">else</span> <span class="n">f</span><span class="o">.</span><span class="n">__class__</span><span class="o">.</span><span class="n">__name__</span> + <span class="n">name</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="vm">__name__</span> <span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">f</span><span class="p">,</span> <span class="s1">'__name__'</span><span class="p">)</span> <span class="k">else</span> <span class="n">f</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span> <span class="n">judf</span> <span class="o">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">_jvm</span><span class="o">.</span><span class="n">org</span><span class="o">.</span><span class="n">apache</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">sql</span><span class="o">.</span><span class="n">execution</span><span class="o">.</span><span class="n">python</span><span class="o">.</span><span class="n">UserDefinedPythonFunction</span><span class="p">(</span> <span class="n">name</span><span class="p">,</span> <span class="n">wrapped_func</span><span class="p">,</span> <span class="n">jdt</span><span class="p">)</span> <span class="k">return</span> <span class="n">judf</span> @@ -1929,7 +1929,7 @@ <span class="kn">import</span> <span class="nn">doctest</span> <span class="kn">from</span> <span class="nn">pyspark.sql</span> <span class="k">import</span> <span class="n">Row</span><span class="p">,</span> <span class="n">SparkSession</span> <span class="kn">import</span> <span class="nn">pyspark.sql.functions</span> - <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">sql</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span> + <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">sql</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span> <span class="n">spark</span> <span class="o">=</span> <span class="n">SparkSession</span><span class="o">.</span><span class="n">builder</span>\ <span class="o">.</span><span class="n">master</span><span class="p">(</span><span class="s2">"local[4]"</span><span class="p">)</span>\ <span class="o">.</span><span class="n">appName</span><span class="p">(</span><span class="s2">"sql.functions tests"</span><span class="p">)</span>\ @@ -1946,7 +1946,7 @@ <span class="n">exit</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span> -<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span> +<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span> <span class="n">_test</span><span class="p">()</span> </pre></div>
http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/sql/group.html ---------------------------------------------------------------------- diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/sql/group.html b/site/docs/2.1.2/api/python/_modules/pyspark/sql/group.html index f6687c9..c056446 100644 --- a/site/docs/2.1.2/api/python/_modules/pyspark/sql/group.html +++ b/site/docs/2.1.2/api/python/_modules/pyspark/sql/group.html @@ -74,21 +74,21 @@ <span class="k">def</span> <span class="nf">dfapi</span><span class="p">(</span><span class="n">f</span><span class="p">):</span> <span class="k">def</span> <span class="nf">_api</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> - <span class="n">name</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">__name__</span> + <span class="n">name</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="vm">__name__</span> <span class="n">jdf</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jgd</span><span class="p">,</span> <span class="n">name</span><span class="p">)()</span> <span class="k">return</span> <span class="n">DataFrame</span><span class="p">(</span><span class="n">jdf</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql_ctx</span><span class="p">)</span> - <span class="n">_api</span><span class="o">.</span><span class="n">__name__</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">__name__</span> - <span class="n">_api</span><span class="o">.</span><span class="n">__doc__</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">__doc__</span> + <span class="n">_api</span><span class="o">.</span><span class="vm">__name__</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="vm">__name__</span> + <span class="n">_api</span><span class="o">.</span><span class="vm">__doc__</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="vm">__doc__</span> <span class="k">return</span> <span class="n">_api</span> <span class="k">def</span> <span class="nf">df_varargs_api</span><span class="p">(</span><span class="n">f</span><span class="p">):</span> <span class="k">def</span> <span class="nf">_api</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">cols</span><span class="p">):</span> - <span class="n">name</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">__name__</span> + <span class="n">name</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="vm">__name__</span> <span class="n">jdf</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jgd</span><span class="p">,</span> <span class="n">name</span><span class="p">)(</span><span class="n">_to_seq</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sql_ctx</span><span class="o">.</span><span class="n">_sc</span><span class="p">,</span> <span class="n">cols</span><span class="p">))</span> <span class="k">return</span> <span class="n">DataFrame</span><span class="p">(</span><span class="n">jdf</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql_ctx</span><span class="p">)</span> - <span class="n">_api</span><span class="o">.</span><span class="n">__name__</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">__name__</span> - <span class="n">_api</span><span class="o">.</span><span class="n">__doc__</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">__doc__</span> + <span class="n">_api</span><span class="o">.</span><span class="vm">__name__</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="vm">__name__</span> + <span class="n">_api</span><span class="o">.</span><span class="vm">__doc__</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="vm">__doc__</span> <span class="k">return</span> <span class="n">_api</span> @@ -247,7 +247,7 @@ <span class="kn">import</span> <span class="nn">doctest</span> <span class="kn">from</span> <span class="nn">pyspark.sql</span> <span class="k">import</span> <span class="n">Row</span><span class="p">,</span> <span class="n">SparkSession</span> <span class="kn">import</span> <span class="nn">pyspark.sql.group</span> - <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">sql</span><span class="o">.</span><span class="n">group</span><span class="o">.</span><span class="n">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span> + <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">sql</span><span class="o">.</span><span class="n">group</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span> <span class="n">spark</span> <span class="o">=</span> <span class="n">SparkSession</span><span class="o">.</span><span class="n">builder</span>\ <span class="o">.</span><span class="n">master</span><span class="p">(</span><span class="s2">"local[4]"</span><span class="p">)</span>\ <span class="o">.</span><span class="n">appName</span><span class="p">(</span><span class="s2">"sql.group tests"</span><span class="p">)</span>\ @@ -273,7 +273,7 @@ <span class="n">exit</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span> -<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span> +<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span> <span class="n">_test</span><span class="p">()</span> </pre></div> http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/sql/readwriter.html ---------------------------------------------------------------------- diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/sql/readwriter.html b/site/docs/2.1.2/api/python/_modules/pyspark/sql/readwriter.html index bac533f..49f98d9 100644 --- a/site/docs/2.1.2/api/python/_modules/pyspark/sql/readwriter.html +++ b/site/docs/2.1.2/api/python/_modules/pyspark/sql/readwriter.html @@ -824,7 +824,7 @@ <span class="n">os</span><span class="o">.</span><span class="n">chdir</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="p">[</span><span class="s2">"SPARK_HOME"</span><span class="p">])</span> - <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">sql</span><span class="o">.</span><span class="n">readwriter</span><span class="o">.</span><span class="n">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span> + <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">sql</span><span class="o">.</span><span class="n">readwriter</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span> <span class="n">sc</span> <span class="o">=</span> <span class="n">SparkContext</span><span class="p">(</span><span class="s1">'local[4]'</span><span class="p">,</span> <span class="s1">'PythonTest'</span><span class="p">)</span> <span class="k">try</span><span class="p">:</span> <span class="n">spark</span> <span class="o">=</span> <span class="n">SparkSession</span><span class="o">.</span><span class="n">builder</span><span class="o">.</span><span class="n">enableHiveSupport</span><span class="p">()</span><span class="o">.</span><span class="n">getOrCreate</span><span class="p">()</span> @@ -844,7 +844,7 @@ <span class="n">exit</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span> -<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span> +<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span> <span class="n">_test</span><span class="p">()</span> </pre></div> http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/sql/session.html ---------------------------------------------------------------------- diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/sql/session.html b/site/docs/2.1.2/api/python/_modules/pyspark/sql/session.html index 919470d..c872244 100644 --- a/site/docs/2.1.2/api/python/_modules/pyspark/sql/session.html +++ b/site/docs/2.1.2/api/python/_modules/pyspark/sql/session.html @@ -276,7 +276,7 @@ <span class="sd"> registered temporary views and UDFs, but shared SparkContext and</span> <span class="sd"> table cache.</span> <span class="sd"> """</span> - <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">__class__</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_sc</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jsparkSession</span><span class="o">.</span><span class="n">newSession</span><span class="p">())</span></div> + <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_sc</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jsparkSession</span><span class="o">.</span><span class="n">newSession</span><span class="p">())</span></div> <span class="nd">@property</span> <span class="nd">@since</span><span class="p">(</span><span class="mf">2.0</span><span class="p">)</span> @@ -675,7 +675,7 @@ <span class="n">os</span><span class="o">.</span><span class="n">chdir</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="p">[</span><span class="s2">"SPARK_HOME"</span><span class="p">])</span> - <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">sql</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span> + <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">sql</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span> <span class="n">sc</span> <span class="o">=</span> <span class="n">SparkContext</span><span class="p">(</span><span class="s1">'local[4]'</span><span class="p">,</span> <span class="s1">'PythonTest'</span><span class="p">)</span> <span class="n">globs</span><span class="p">[</span><span class="s1">'sc'</span><span class="p">]</span> <span class="o">=</span> <span class="n">sc</span> <span class="n">globs</span><span class="p">[</span><span class="s1">'spark'</span><span class="p">]</span> <span class="o">=</span> <span class="n">SparkSession</span><span class="p">(</span><span class="n">sc</span><span class="p">)</span> @@ -691,7 +691,7 @@ <span class="k">if</span> <span class="n">failure_count</span><span class="p">:</span> <span class="n">exit</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span> -<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span> +<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span> <span class="n">_test</span><span class="p">()</span> </pre></div> http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/sql/streaming.html ---------------------------------------------------------------------- diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/sql/streaming.html b/site/docs/2.1.2/api/python/_modules/pyspark/sql/streaming.html index 7a8e78d..174b7dd 100644 --- a/site/docs/2.1.2/api/python/_modules/pyspark/sql/streaming.html +++ b/site/docs/2.1.2/api/python/_modules/pyspark/sql/streaming.html @@ -886,7 +886,7 @@ <span class="n">os</span><span class="o">.</span><span class="n">chdir</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="p">[</span><span class="s2">"SPARK_HOME"</span><span class="p">])</span> - <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">sql</span><span class="o">.</span><span class="n">streaming</span><span class="o">.</span><span class="n">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span> + <span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">sql</span><span class="o">.</span><span class="n">streaming</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span> <span class="k">try</span><span class="p">:</span> <span class="n">spark</span> <span class="o">=</span> <span class="n">SparkSession</span><span class="o">.</span><span class="n">builder</span><span class="o">.</span><span class="n">getOrCreate</span><span class="p">()</span> <span class="k">except</span> <span class="n">py4j</span><span class="o">.</span><span class="n">protocol</span><span class="o">.</span><span class="n">Py4JError</span><span class="p">:</span> @@ -911,7 +911,7 @@ <span class="n">exit</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span> -<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span> +<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span> <span class="n">_test</span><span class="p">()</span> </pre></div> http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/sql/types.html ---------------------------------------------------------------------- diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/sql/types.html b/site/docs/2.1.2/api/python/_modules/pyspark/sql/types.html index d3c861c..95b5e2b 100644 --- a/site/docs/2.1.2/api/python/_modules/pyspark/sql/types.html +++ b/site/docs/2.1.2/api/python/_modules/pyspark/sql/types.html @@ -92,20 +92,20 @@ <span class="sd">"""Base class for data types."""</span> <span class="k">def</span> <span class="nf">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> - <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">__class__</span><span class="o">.</span><span class="n">__name__</span> + <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span> <span class="k">def</span> <span class="nf">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> <span class="k">return</span> <span class="nb">hash</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="p">))</span> <span class="k">def</span> <span class="nf">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span> - <span class="k">return</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">__class__</span><span class="p">)</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">__dict__</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">__dict__</span> + <span class="k">return</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="p">)</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="vm">__dict__</span> <span class="k">def</span> <span class="nf">__ne__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span> - <span class="k">return</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">__eq__</span><span class="p">(</span><span class="n">other</span><span class="p">)</span> + <span class="k">return</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="fm">__eq__</span><span class="p">(</span><span class="n">other</span><span class="p">)</span> <span class="nd">@classmethod</span> -<div class="viewcode-block" id="DataType.typeName"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataType.typeName">[docs]</a> <span class="k">def</span> <span class="nf">typeName</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span> - <span class="k">return</span> <span class="n">cls</span><span class="o">.</span><span class="n">__name__</span><span class="p">[:</span><span class="o">-</span><span class="mi">4</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span></div> +<div class="viewcode-block" id="DataType.typeName"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataType.typeName">[docs]</a> <span class="k">def</span> <span class="nf">typeName</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span> + <span class="k">return</span> <span class="bp">cls</span><span class="o">.</span><span class="vm">__name__</span><span class="p">[:</span><span class="o">-</span><span class="mi">4</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span></div> <div class="viewcode-block" id="DataType.simpleString"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataType.simpleString">[docs]</a> <span class="k">def</span> <span class="nf">simpleString</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">typeName</span><span class="p">()</span></div> @@ -146,10 +146,10 @@ <span class="n">_instances</span> <span class="o">=</span> <span class="p">{}</span> - <span class="k">def</span> <span class="nf">__call__</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span> - <span class="k">if</span> <span class="n">cls</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">cls</span><span class="o">.</span><span class="n">_instances</span><span class="p">:</span> - <span class="n">cls</span><span class="o">.</span><span class="n">_instances</span><span class="p">[</span><span class="n">cls</span><span class="p">]</span> <span class="o">=</span> <span class="nb">super</span><span class="p">(</span><span class="n">DataTypeSingleton</span><span class="p">,</span> <span class="n">cls</span><span class="p">)</span><span class="o">.</span><span class="n">__call__</span><span class="p">()</span> - <span class="k">return</span> <span class="n">cls</span><span class="o">.</span><span class="n">_instances</span><span class="p">[</span><span class="n">cls</span><span class="p">]</span> + <span class="k">def</span> <span class="nf">__call__</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span> + <span class="k">if</span> <span class="bp">cls</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_instances</span><span class="p">:</span> + <span class="bp">cls</span><span class="o">.</span><span class="n">_instances</span><span class="p">[</span><span class="bp">cls</span><span class="p">]</span> <span class="o">=</span> <span class="nb">super</span><span class="p">(</span><span class="n">DataTypeSingleton</span><span class="p">,</span> <span class="bp">cls</span><span class="p">)</span><span class="o">.</span><span class="fm">__call__</span><span class="p">()</span> + <span class="k">return</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_instances</span><span class="p">[</span><span class="bp">cls</span><span class="p">]</span> <div class="viewcode-block" id="NullType"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.NullType">[docs]</a><span class="k">class</span> <span class="nc">NullType</span><span class="p">(</span><span class="n">DataType</span><span class="p">):</span> @@ -352,7 +352,7 @@ <span class="s2">"containsNull"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">containsNull</span><span class="p">}</span></div> <span class="nd">@classmethod</span> -<div class="viewcode-block" id="ArrayType.fromJson"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.ArrayType.fromJson">[docs]</a> <span class="k">def</span> <span class="nf">fromJson</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span> +<div class="viewcode-block" id="ArrayType.fromJson"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.ArrayType.fromJson">[docs]</a> <span class="k">def</span> <span class="nf">fromJson</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span> <span class="k">return</span> <span class="n">ArrayType</span><span class="p">(</span><span class="n">_parse_datatype_json_value</span><span class="p">(</span><span class="n">json</span><span class="p">[</span><span class="s2">"elementType"</span><span class="p">]),</span> <span class="n">json</span><span class="p">[</span><span class="s2">"containsNull"</span><span class="p">])</span></div> @@ -409,7 +409,7 @@ <span class="s2">"valueContainsNull"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">valueContainsNull</span><span class="p">}</span></div> <span class="nd">@classmethod</span> -<div class="viewcode-block" id="MapType.fromJson"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.MapType.fromJson">[docs]</a> <span class="k">def</span> <span class="nf">fromJson</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span> +<div class="viewcode-block" id="MapType.fromJson"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.MapType.fromJson">[docs]</a> <span class="k">def</span> <span class="nf">fromJson</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span> <span class="k">return</span> <span class="n">MapType</span><span class="p">(</span><span class="n">_parse_datatype_json_value</span><span class="p">(</span><span class="n">json</span><span class="p">[</span><span class="s2">"keyType"</span><span class="p">]),</span> <span class="n">_parse_datatype_json_value</span><span class="p">(</span><span class="n">json</span><span class="p">[</span><span class="s2">"valueType"</span><span class="p">]),</span> <span class="n">json</span><span class="p">[</span><span class="s2">"valueContainsNull"</span><span class="p">])</span></div> @@ -471,7 +471,7 @@ <span class="s2">"metadata"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="p">}</span></div> <span class="nd">@classmethod</span> -<div class="viewcode-block" id="StructField.fromJson"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.StructField.fromJson">[docs]</a> <span class="k">def</span> <span class="nf">fromJson</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span> +<div class="viewcode-block" id="StructField.fromJson"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.StructField.fromJson">[docs]</a> <span class="k">def</span> <span class="nf">fromJson</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span> <span class="k">return</span> <span class="n">StructField</span><span class="p">(</span><span class="n">json</span><span class="p">[</span><span class="s2">"name"</span><span class="p">],</span> <span class="n">_parse_datatype_json_value</span><span class="p">(</span><span class="n">json</span><span class="p">[</span><span class="s2">"type"</span><span class="p">]),</span> <span class="n">json</span><span class="p">[</span><span class="s2">"nullable"</span><span class="p">],</span> @@ -606,7 +606,7 @@ <span class="s2">"fields"</span><span class="p">:</span> <span class="p">[</span><span class="n">f</span><span class="o">.</span><span class="n">jsonValue</span><span class="p">()</span> <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="bp">self</span><span class="p">]}</span></div> <span class="nd">@classmethod</span> -<div class="viewcode-block" id="StructType.fromJson"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.StructType.fromJson">[docs]</a> <span class="k">def</span> <span class="nf">fromJson</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span> +<div class="viewcode-block" id="StructType.fromJson"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.StructType.fromJson">[docs]</a> <span class="k">def</span> <span class="nf">fromJson</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span> <span class="k">return</span> <span class="n">StructType</span><span class="p">([</span><span class="n">StructField</span><span class="o">.</span><span class="n">fromJson</span><span class="p">(</span><span class="n">f</span><span class="p">)</span> <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="n">json</span><span class="p">[</span><span class="s2">"fields"</span><span class="p">]])</span></div> <div class="viewcode-block" id="StructType.needConversion"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.StructType.needConversion">[docs]</a> <span class="k">def</span> <span class="nf">needConversion</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> @@ -623,7 +623,7 @@ <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="p">(</span><span class="nb">tuple</span><span class="p">,</span> <span class="nb">list</span><span class="p">)):</span> <span class="k">return</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">toInternal</span><span class="p">(</span><span class="n">v</span><span class="p">)</span> <span class="k">for</span> <span class="n">f</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">fields</span><span class="p">,</span> <span class="n">obj</span><span class="p">))</span> <span class="k">elif</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="s2">"__dict__"</span><span class="p">):</span> - <span class="n">d</span> <span class="o">=</span> <span class="n">obj</span><span class="o">.</span><span class="n">__dict__</span> + <span class="n">d</span> <span class="o">=</span> <span class="n">obj</span><span class="o">.</span><span class="vm">__dict__</span> <span class="k">return</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">toInternal</span><span class="p">(</span><span class="n">d</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">n</span><span class="p">))</span> <span class="k">for</span> <span class="n">n</span><span class="p">,</span> <span class="n">f</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">names</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">fields</span><span class="p">))</span> <span class="k">else</span><span class="p">:</span> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"Unexpected tuple </span><span class="si">%r</span><span class="s2"> with StructType"</span> <span class="o">%</span> <span class="n">obj</span><span class="p">)</span> @@ -635,7 +635,7 @@ <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="p">(</span><span class="nb">list</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">)):</span> <span class="k">return</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">obj</span><span class="p">)</span> <span class="k">elif</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="s2">"__dict__"</span><span class="p">):</span> - <span class="n">d</span> <span class="o">=</span> <span class="n">obj</span><span class="o">.</span><span class="n">__dict__</span> + <span class="n">d</span> <span class="o">=</span> <span class="n">obj</span><span class="o">.</span><span class="vm">__dict__</span> <span class="k">return</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">d</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">n</span><span class="p">)</span> <span class="k">for</span> <span class="n">n</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">names</span><span class="p">)</span> <span class="k">else</span><span class="p">:</span> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"Unexpected tuple </span><span class="si">%r</span><span class="s2"> with StructType"</span> <span class="o">%</span> <span class="n">obj</span><span class="p">)</span></div> @@ -660,25 +660,25 @@ <span class="sd"> """</span> <span class="nd">@classmethod</span> - <span class="k">def</span> <span class="nf">typeName</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span> - <span class="k">return</span> <span class="n">cls</span><span class="o">.</span><span class="n">__name__</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> + <span class="k">def</span> <span class="nf">typeName</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span> + <span class="k">return</span> <span class="bp">cls</span><span class="o">.</span><span class="vm">__name__</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="nd">@classmethod</span> - <span class="k">def</span> <span class="nf">sqlType</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span> + <span class="k">def</span> <span class="nf">sqlType</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span> <span class="sd">"""</span> <span class="sd"> Underlying SQL storage type for this UDT.</span> <span class="sd"> """</span> <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="s2">"UDT must implement sqlType()."</span><span class="p">)</span> <span class="nd">@classmethod</span> - <span class="k">def</span> <span class="nf">module</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span> + <span class="k">def</span> <span class="nf">module</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span> <span class="sd">"""</span> <span class="sd"> The Python module of the UDT.</span> <span class="sd"> """</span> <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="s2">"UDT must implement module()."</span><span class="p">)</span> <span class="nd">@classmethod</span> - <span class="k">def</span> <span class="nf">scalaUDT</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span> + <span class="k">def</span> <span class="nf">scalaUDT</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span> <span class="sd">"""</span> <span class="sd"> The class name of the paired Scala UDT (could be '', if there</span> <span class="sd"> is no corresponding one).</span> @@ -689,13 +689,13 @@ <span class="k">return</span> <span class="kc">True</span> <span class="nd">@classmethod</span> - <span class="k">def</span> <span class="nf">_cachedSqlType</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span> + <span class="k">def</span> <span class="nf">_cachedSqlType</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span> <span class="sd">"""</span> <span class="sd"> Cache the sqlType() into class, because it's heavy used in `toInternal`.</span> <span class="sd"> """</span> - <span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="s2">"_cached_sql_type"</span><span class="p">):</span> - <span class="n">cls</span><span class="o">.</span><span class="n">_cached_sql_type</span> <span class="o">=</span> <span class="n">cls</span><span class="o">.</span><span class="n">sqlType</span><span class="p">()</span> - <span class="k">return</span> <span class="n">cls</span><span class="o">.</span><span class="n">_cached_sql_type</span> + <span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="s2">"_cached_sql_type"</span><span class="p">):</span> + <span class="bp">cls</span><span class="o">.</span><span class="n">_cached_sql_type</span> <span class="o">=</span> <span class="bp">cls</span><span class="o">.</span><span class="n">sqlType</span><span class="p">()</span> + <span class="k">return</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_cached_sql_type</span> <span class="k">def</span> <span class="nf">toInternal</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">obj</span><span class="p">):</span> <span class="k">if</span> <span class="n">obj</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> @@ -730,7 +730,7 @@ <span class="n">schema</span> <span class="o">=</span> <span class="p">{</span> <span class="s2">"type"</span><span class="p">:</span> <span class="s2">"udt"</span><span class="p">,</span> <span class="s2">"class"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">scalaUDT</span><span class="p">(),</span> - <span class="s2">"pyClass"</span><span class="p">:</span> <span class="s2">"</span><span class="si">%s</span><span class="s2">.</span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">module</span><span class="p">(),</span> <span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__name__</span><span class="p">),</span> + <span class="s2">"pyClass"</span><span class="p">:</span> <span class="s2">"</span><span class="si">%s</span><span class="s2">.</span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">module</span><span class="p">(),</span> <span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="vm">__name__</span><span class="p">),</span> <span class="s2">"sqlType"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">sqlType</span><span class="p">()</span><span class="o">.</span><span class="n">jsonValue</span><span class="p">()</span> <span class="p">}</span> <span class="k">else</span><span class="p">:</span> @@ -738,14 +738,14 @@ <span class="n">b</span> <span class="o">=</span> <span class="n">ser</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">))</span> <span class="n">schema</span> <span class="o">=</span> <span class="p">{</span> <span class="s2">"type"</span><span class="p">:</span> <span class="s2">"udt"</span><span class="p">,</span> - <span class="s2">"pyClass"</span><span class="p">:</span> <span class="s2">"</span><span class="si">%s</span><span class="s2">.</span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">module</span><span class="p">(),</span> <span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__name__</span><span class="p">),</span> + <span class="s2">"pyClass"</span><span class="p">:</span> <span class="s2">"</span><span class="si">%s</span><span class="s2">.</span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">module</span><span class="p">(),</span> <span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="vm">__name__</span><span class="p">),</span> <span class="s2">"serializedClass"</span><span class="p">:</span> <span class="n">base64</span><span class="o">.</span><span class="n">b64encode</span><span class="p">(</span><span class="n">b</span><span class="p">)</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s1">'utf8'</span><span class="p">),</span> <span class="s2">"sqlType"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">sqlType</span><span class="p">()</span><span class="o">.</span><span class="n">jsonValue</span><span class="p">()</span> <span class="p">}</span> <span class="k">return</span> <span class="n">schema</span> <span class="nd">@classmethod</span> - <span class="k">def</span> <span class="nf">fromJson</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span> + <span class="k">def</span> <span class="nf">fromJson</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span> <span class="n">pyUDT</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">json</span><span class="p">[</span><span class="s2">"pyClass"</span><span class="p">])</span> <span class="c1"># convert unicode to str</span> <span class="n">split</span> <span class="o">=</span> <span class="n">pyUDT</span><span class="o">.</span><span class="n">rfind</span><span class="p">(</span><span class="s2">"."</span><span class="p">)</span> <span class="n">pyModule</span> <span class="o">=</span> <span class="n">pyUDT</span><span class="p">[:</span><span class="n">split</span><span class="p">]</span> @@ -1034,7 +1034,7 @@ <span class="n">items</span> <span class="o">=</span> <span class="nb">zip</span><span class="p">(</span><span class="n">names</span><span class="p">,</span> <span class="n">row</span><span class="p">)</span> <span class="k">elif</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">row</span><span class="p">,</span> <span class="s2">"__dict__"</span><span class="p">):</span> <span class="c1"># object</span> - <span class="n">items</span> <span class="o">=</span> <span class="nb">sorted</span><span class="p">(</span><span class="n">row</span><span class="o">.</span><span class="n">__dict__</span><span class="o">.</span><span class="n">items</span><span class="p">())</span> + <span class="n">items</span> <span class="o">=</span> <span class="nb">sorted</span><span class="p">(</span><span class="n">row</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">items</span><span class="p">())</span> <span class="k">else</span><span class="p">:</span> <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s2">"Can not infer schema for type: </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="nb">type</span><span class="p">(</span><span class="n">row</span><span class="p">))</span> @@ -1137,7 +1137,7 @@ <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span> <span class="n">d</span> <span class="o">=</span> <span class="n">obj</span> <span class="k">elif</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="s2">"__dict__"</span><span class="p">):</span> <span class="c1"># object</span> - <span class="n">d</span> <span class="o">=</span> <span class="n">obj</span><span class="o">.</span><span class="n">__dict__</span> + <span class="n">d</span> <span class="o">=</span> <span class="n">obj</span><span class="o">.</span><span class="vm">__dict__</span> <span class="k">else</span><span class="p">:</span> <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s2">"Unexpected obj type: </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="nb">type</span><span class="p">(</span><span class="n">obj</span><span class="p">))</span> @@ -1407,7 +1407,7 @@ <span class="k">for</span> <span class="n">v</span><span class="p">,</span> <span class="n">f</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="n">dataType</span><span class="o">.</span><span class="n">fields</span><span class="p">):</span> <span class="n">_verify_type</span><span class="p">(</span><span class="n">v</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="n">dataType</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="n">nullable</span><span class="p">)</span> <span class="k">elif</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="s2">"__dict__"</span><span class="p">):</span> - <span class="n">d</span> <span class="o">=</span> <span class="n">obj</span><span class="o">.</span><span class="n">__dict__</span> + <span class="n">d</span> <span class="o">=</span> <span class="n">obj</span><span class="o">.</span><span class="vm">__dict__</span> <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="n">dataType</span><span class="o">.</span><span class="n">fields</span><span class="p">:</span> <span class="n">_verify_type</span><span class="p">(</span><span class="n">d</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">name</span><span class="p">),</span> <span class="n">f</span><span class="o">.</span><span class="n">dataType</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="n">nullable</span><span class="p">)</span> <span class="k">else</span><span class="p">:</span> @@ -1472,14 +1472,14 @@ <span class="k">if</span> <span class="n">kwargs</span><span class="p">:</span> <span class="c1"># create row objects</span> <span class="n">names</span> <span class="o">=</span> <span class="nb">sorted</span><span class="p">(</span><span class="n">kwargs</span><span class="o">.</span><span class="n">keys</span><span class="p">())</span> - <span class="n">row</span> <span class="o">=</span> <span class="nb">tuple</span><span class="o">.</span><span class="n">__new__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="p">[</span><span class="n">kwargs</span><span class="p">[</span><span class="n">n</span><span class="p">]</span> <span class="k">for</span> <span class="n">n</span> <span class="ow">in</span> <span class="n">names</span><span class="p">])</span> + <span class="n">row</span> <span class="o">=</span> <span class="nb">tuple</span><span class="o">.</span><span class="fm">__new__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="p">[</span><span class="n">kwargs</span><span class="p">[</span><span class="n">n</span><span class="p">]</span> <span class="k">for</span> <span class="n">n</span> <span class="ow">in</span> <span class="n">names</span><span class="p">])</span> <span class="n">row</span><span class="o">.</span><span class="n">__fields__</span> <span class="o">=</span> <span class="n">names</span> <span class="n">row</span><span class="o">.</span><span class="n">__from_dict__</span> <span class="o">=</span> <span class="kc">True</span> <span class="k">return</span> <span class="n">row</span> <span class="k">else</span><span class="p">:</span> <span class="c1"># create row class or objects</span> - <span class="k">return</span> <span class="nb">tuple</span><span class="o">.</span><span class="n">__new__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">args</span><span class="p">)</span> + <span class="k">return</span> <span class="nb">tuple</span><span class="o">.</span><span class="fm">__new__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">args</span><span class="p">)</span> <div class="viewcode-block" id="Row.asDict"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.Row.asDict">[docs]</a> <span class="k">def</span> <span class="nf">asDict</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">recursive</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span> <span class="sd">"""</span> @@ -1516,7 +1516,7 @@ <span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s2">"__fields__"</span><span class="p">):</span> <span class="k">return</span> <span class="n">item</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">__fields__</span> <span class="k">else</span><span class="p">:</span> - <span class="k">return</span> <span class="nb">super</span><span class="p">(</span><span class="n">Row</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__contains__</span><span class="p">(</span><span class="n">item</span><span class="p">)</span> + <span class="k">return</span> <span class="nb">super</span><span class="p">(</span><span class="n">Row</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__contains__</span><span class="p">(</span><span class="n">item</span><span class="p">)</span> <span class="c1"># let object acts like class</span> <span class="k">def</span> <span class="nf">__call__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">):</span> @@ -1525,12 +1525,12 @@ <span class="k">def</span> <span class="nf">__getitem__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">item</span><span class="p">):</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="p">(</span><span class="nb">int</span><span class="p">,</span> <span class="nb">slice</span><span class="p">)):</span> - <span class="k">return</span> <span class="nb">super</span><span class="p">(</span><span class="n">Row</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__getitem__</span><span class="p">(</span><span class="n">item</span><span class="p">)</span> + <span class="k">return</span> <span class="nb">super</span><span class="p">(</span><span class="n">Row</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__getitem__</span><span class="p">(</span><span class="n">item</span><span class="p">)</span> <span class="k">try</span><span class="p">:</span> <span class="c1"># it will be slow when it has many fields,</span> <span class="c1"># but this will not be used in normal cases</span> <span class="n">idx</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">__fields__</span><span class="o">.</span><span class="n">index</span><span class="p">(</span><span class="n">item</span><span class="p">)</span> - <span class="k">return</span> <span class="nb">super</span><span class="p">(</span><span class="n">Row</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__getitem__</span><span class="p">(</span><span class="n">idx</span><span class="p">)</span> + <span class="k">return</span> <span class="nb">super</span><span class="p">(</span><span class="n">Row</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__getitem__</span><span class="p">(</span><span class="n">idx</span><span class="p">)</span> <span class="k">except</span> <span class="ne">IndexError</span><span class="p">:</span> <span class="k">raise</span> <span class="ne">KeyError</span><span class="p">(</span><span class="n">item</span><span class="p">)</span> <span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span> @@ -1552,7 +1552,7 @@ <span class="k">def</span> <span class="nf">__setattr__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span> <span class="k">if</span> <span class="n">key</span> <span class="o">!=</span> <span class="s1">'__fields__'</span> <span class="ow">and</span> <span class="n">key</span> <span class="o">!=</span> <span class="s2">"__from_dict__"</span><span class="p">:</span> <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="s2">"Row is read-only"</span><span class="p">)</span> - <span class="bp">self</span><span class="o">.</span><span class="n">__dict__</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="o">=</span> <span class="n">value</span> + <span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="o">=</span> <span class="n">value</span> <span class="k">def</span> <span class="nf">__reduce__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> <span class="sd">"""Returns a tuple so Python knows how to pickle Row."""</span> @@ -1610,7 +1610,7 @@ <span class="n">exit</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span> -<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span> +<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span> <span class="n">_test</span><span class="p">()</span> </pre></div> http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/sql/window.html ---------------------------------------------------------------------- diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/sql/window.html b/site/docs/2.1.2/api/python/_modules/pyspark/sql/window.html index b4012cc..1e9f4e5 100644 --- a/site/docs/2.1.2/api/python/_modules/pyspark/sql/window.html +++ b/site/docs/2.1.2/api/python/_modules/pyspark/sql/window.html @@ -281,7 +281,7 @@ <span class="n">exit</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span> -<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span> +<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span> <span class="n">_test</span><span class="p">()</span> </pre></div> http://git-wip-us.apache.org/repos/asf/spark-website/blob/6bbac496/site/docs/2.1.2/api/python/_modules/pyspark/streaming/context.html ---------------------------------------------------------------------- diff --git a/site/docs/2.1.2/api/python/_modules/pyspark/streaming/context.html b/site/docs/2.1.2/api/python/_modules/pyspark/streaming/context.html index acbebb9..ba0427c 100644 --- a/site/docs/2.1.2/api/python/_modules/pyspark/streaming/context.html +++ b/site/docs/2.1.2/api/python/_modules/pyspark/streaming/context.html @@ -119,7 +119,7 @@ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jvm</span><span class="o">.</span><span class="n">Duration</span><span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">seconds</span> <span class="o">*</span> <span class="mi">1000</span><span class="p">))</span> <span class="nd">@classmethod</span> - <span class="k">def</span> <span class="nf">_ensure_initialized</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span> + <span class="k">def</span> <span class="nf">_ensure_initialized</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span> <span class="n">SparkContext</span><span class="o">.</span><span class="n">_ensure_initialized</span><span class="p">()</span> <span class="n">gw</span> <span class="o">=</span> <span class="n">SparkContext</span><span class="o">.</span><span class="n">_gateway</span> @@ -129,7 +129,7 @@ <span class="c1"># start callback server</span> <span class="c1"># getattr will fallback to JVM, so we cannot test by hasattr()</span> - <span class="k">if</span> <span class="s2">"_callback_server"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">gw</span><span class="o">.</span><span class="n">__dict__</span> <span class="ow">or</span> <span class="n">gw</span><span class="o">.</span><span class="n">_callback_server</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> + <span class="k">if</span> <span class="s2">"_callback_server"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">gw</span><span class="o">.</span><span class="vm">__dict__</span> <span class="ow">or</span> <span class="n">gw</span><span class="o">.</span><span class="n">_callback_server</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> <span class="n">gw</span><span class="o">.</span><span class="n">callback_server_parameters</span><span class="o">.</span><span class="n">eager_load</span> <span class="o">=</span> <span class="kc">True</span> <span class="n">gw</span><span class="o">.</span><span class="n">callback_server_parameters</span><span class="o">.</span><span class="n">daemonize</span> <span class="o">=</span> <span class="kc">True</span> <span class="n">gw</span><span class="o">.</span><span class="n">callback_server_parameters</span><span class="o">.</span><span class="n">daemonize_connections</span> <span class="o">=</span> <span class="kc">True</span> @@ -146,11 +146,11 @@ <span class="c1"># register serializer for TransformFunction</span> <span class="c1"># it happens before creating SparkContext when loading from checkpointing</span> - <span class="n">cls</span><span class="o">.</span><span class="n">_transformerSerializer</span> <span class="o">=</span> <span class="n">TransformFunctionSerializer</span><span class="p">(</span> + <span class="bp">cls</span><span class="o">.</span><span class="n">_transformerSerializer</span> <span class="o">=</span> <span class="n">TransformFunctionSerializer</span><span class="p">(</span> <span class="n">SparkContext</span><span class="o">.</span><span class="n">_active_spark_context</span><span class="p">,</span> <span class="n">CloudPickleSerializer</span><span class="p">(),</span> <span class="n">gw</span><span class="p">)</span> <span class="nd">@classmethod</span> -<div class="viewcode-block" id="StreamingContext.getOrCreate"><a class="viewcode-back" href="../../../pyspark.streaming.html#pyspark.streaming.StreamingContext.getOrCreate">[docs]</a> <span class="k">def</span> <span class="nf">getOrCreate</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">checkpointPath</span><span class="p">,</span> <span class="n">setupFunc</span><span class="p">):</span> +<div class="viewcode-block" id="StreamingContext.getOrCreate"><a class="viewcode-back" href="../../../pyspark.streaming.html#pyspark.streaming.StreamingContext.getOrCreate">[docs]</a> <span class="k">def</span> <span class="nf">getOrCreate</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">checkpointPath</span><span class="p">,</span> <span class="n">setupFunc</span><span class="p">):</span> <span class="sd">"""</span> <span class="sd"> Either recreate a StreamingContext from checkpoint data or create a new StreamingContext.</span> <span class="sd"> If checkpoint data exists in the provided `checkpointPath`, then StreamingContext will be</span> @@ -160,7 +160,7 @@ <span class="sd"> @param checkpointPath: Checkpoint directory used in an earlier streaming program</span> <span class="sd"> @param setupFunc: Function to create a new context and setup DStreams</span> <span class="sd"> """</span> - <span class="n">cls</span><span class="o">.</span><span class="n">_ensure_initialized</span><span class="p">()</span> + <span class="bp">cls</span><span class="o">.</span><span class="n">_ensure_initialized</span><span class="p">()</span> <span class="n">gw</span> <span class="o">=</span> <span class="n">SparkContext</span><span class="o">.</span><span class="n">_gateway</span> <span class="c1"># Check whether valid checkpoint information exists in the given path</span> @@ -181,16 +181,16 @@ <span class="n">sc</span> <span class="o">=</span> <span class="n">SparkContext</span><span class="o">.</span><span class="n">_active_spark_context</span> <span class="c1"># update ctx in serializer</span> - <span class="n">cls</span><span class="o">.</span><span class="n">_transformerSerializer</span><span class="o">.</span><span class="n">ctx</span> <span class="o">=</span> <span class="n">sc</span> + <span class="bp">cls</span><span class="o">.</span><span class="n">_transformerSerializer</span><span class="o">.</span><span class="n">ctx</span> <span class="o">=</span> <span class="n">sc</span> <span class="k">return</span> <span class="n">StreamingContext</span><span class="p">(</span><span class="n">sc</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="n">jssc</span><span class="p">)</span></div> <span class="nd">@classmethod</span> -<div class="viewcode-block" id="StreamingContext.getActive"><a class="viewcode-back" href="../../../pyspark.streaming.html#pyspark.streaming.StreamingContext.getActive">[docs]</a> <span class="k">def</span> <span class="nf">getActive</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span> +<div class="viewcode-block" id="StreamingContext.getActive"><a class="viewcode-back" href="../../../pyspark.streaming.html#pyspark.streaming.StreamingContext.getActive">[docs]</a> <span class="k">def</span> <span class="nf">getActive</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span> <span class="sd">"""</span> <span class="sd"> Return either the currently active StreamingContext (i.e., if there is a context started</span> <span class="sd"> but not stopped) or None.</span> <span class="sd"> """</span> - <span class="n">activePythonContext</span> <span class="o">=</span> <span class="n">cls</span><span class="o">.</span><span class="n">_activeContext</span> + <span class="n">activePythonContext</span> <span class="o">=</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_activeContext</span> <span class="k">if</span> <span class="n">activePythonContext</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> <span class="c1"># Verify that the current running Java StreamingContext is active and is the same one</span> <span class="c1"># backing the supposedly active Python context</span> @@ -198,15 +198,15 @@ <span class="n">activeJvmContextOption</span> <span class="o">=</span> <span class="n">activePythonContext</span><span class="o">.</span><span class="n">_jvm</span><span class="o">.</span><span class="n">StreamingContext</span><span class="o">.</span><span class="n">getActive</span><span class="p">()</span> <span class="k">if</span> <span class="n">activeJvmContextOption</span><span class="o">.</span><span class="n">isEmpty</span><span class="p">():</span> - <span class="n">cls</span><span class="o">.</span><span class="n">_activeContext</span> <span class="o">=</span> <span class="kc">None</span> + <span class="bp">cls</span><span class="o">.</span><span class="n">_activeContext</span> <span class="o">=</span> <span class="kc">None</span> <span class="k">elif</span> <span class="n">activeJvmContextOption</span><span class="o">.</span><span class="n">get</span><span class="p">()</span><span class="o">.</span><span class="n">hashCode</span><span class="p">()</span> <span class="o">!=</span> <span class="n">activePythonContextJavaId</span><span class="p">:</span> - <span class="n">cls</span><span class="o">.</span><span class="n">_activeContext</span> <span class="o">=</span> <span class="kc">None</span> + <span class="bp">cls</span><span class="o">.</span><span class="n">_activeContext</span> <span class="o">=</span> <span class="kc">None</span> <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="s2">"JVM's active JavaStreamingContext is not the JavaStreamingContext "</span> <span class="s2">"backing the action Python StreamingContext. This is unexpected."</span><span class="p">)</span> - <span class="k">return</span> <span class="n">cls</span><span class="o">.</span><span class="n">_activeContext</span></div> + <span class="k">return</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_activeContext</span></div> <span class="nd">@classmethod</span> -<div class="viewcode-block" id="StreamingContext.getActiveOrCreate"><a class="viewcode-back" href="../../../pyspark.streaming.html#pyspark.streaming.StreamingContext.getActiveOrCreate">[docs]</a> <span class="k">def</span> <span class="nf">getActiveOrCreate</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">checkpointPath</span><span class="p">,</span> <span class="n">setupFunc</span><span class="p">):</span> +<div class="viewcode-block" id="StreamingContext.getActiveOrCreate"><a class="viewcode-back" href="../../../pyspark.streaming.html#pyspark.streaming.StreamingContext.getActiveOrCreate">[docs]</a> <span class="k">def</span> <span class="nf">getActiveOrCreate</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">checkpointPath</span><span class="p">,</span> <span class="n">setupFunc</span><span class="p">):</span> <span class="sd">"""</span> <span class="sd"> Either return the active StreamingContext (i.e. currently started but not stopped),</span> <span class="sd"> or recreate a StreamingContext from checkpoint data or create a new StreamingContext</span> @@ -222,11 +222,11 @@ <span class="k">if</span> <span class="n">setupFunc</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="s2">"setupFunc cannot be None"</span><span class="p">)</span> - <span class="n">activeContext</span> <span class="o">=</span> <span class="n">cls</span><span class="o">.</span><span class="n">getActive</span><span class="p">()</span> + <span class="n">activeContext</span> <span class="o">=</span> <span class="bp">cls</span><span class="o">.</span><span class="n">getActive</span><span class="p">()</span> <span class="k">if</span> <span class="n">activeContext</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> <span class="k">return</span> <span class="n">activeContext</span> <span class="k">elif</span> <span class="n">checkpointPath</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> - <span class="k">return</span> <span class="n">cls</span><span class="o">.</span><span class="n">getOrCreate</span><span class="p">(</span><span class="n">checkpointPath</span><span class="p">,</span> <span class="n">setupFunc</span><span class="p">)</span> + <span class="k">return</span> <span class="bp">cls</span><span class="o">.</span><span class="n">getOrCreate</span><span class="p">(</span><span class="n">checkpointPath</span><span class="p">,</span> <span class="n">setupFunc</span><span class="p">)</span> <span class="k">else</span><span class="p">:</span> <span class="k">return</span> <span class="n">setupFunc</span><span class="p">()</span></div> --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
