http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/hive_to_druid.html ---------------------------------------------------------------------- diff --git a/_modules/hive_to_druid.html b/_modules/hive_to_druid.html index cbd1586..d85bfb3 100644 --- a/_modules/hive_to_druid.html +++ b/_modules/hive_to_druid.html @@ -137,9 +137,24 @@ <div itemprop="articleBody"> <h1>Source code for hive_to_druid</h1><div class="highlight"><pre> -<span></span><span class="kn">import</span> <span class="nn">logging</span> - -<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span><span class="p">,</span> <span class="n">DruidHook</span><span class="p">,</span> <span class="n">HiveMetastoreHook</span> +<span></span><span class="c1"># -*- coding: utf-8 -*-</span> +<span class="c1">#</span> +<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span> +<span class="c1"># you may not use this file except in compliance with the License.</span> +<span class="c1"># You may obtain a copy of the License at</span> +<span class="c1">#</span> +<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span> +<span class="c1">#</span> +<span class="c1"># Unless required by applicable law or agreed to in writing, software</span> +<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span> +<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span> +<span class="c1"># See the License for the specific language governing permissions and</span> +<span class="c1"># limitations under the License.</span> + +<span class="kn">import</span> <span class="nn">logging</span> + +<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span><span class="p">,</span> <span class="n">HiveMetastoreHook</span> +<span class="kn">from</span> <span class="nn">airflow.hooks.druid_hook</span> <span class="kn">import</span> <span class="n">DruidHook</span> <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span> <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span> @@ -240,19 +255,20 @@ <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Inserting rows into Druid"</span><span class="p">)</span> <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"HDFS path: "</span> <span class="o">+</span> <span class="n">static_path</span><span class="p">)</span> - <span class="n">druid</span><span class="o">.</span><span class="n">load_from_hdfs</span><span class="p">(</span> - <span class="n">datasource</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">druid_datasource</span><span class="p">,</span> - <span class="n">intervals</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">intervals</span><span class="p">,</span> - <span class="n">static_path</span><span class="o">=</span><span class="n">static_path</span><span class="p">,</span> <span class="n">ts_dim</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">ts_dim</span><span class="p">,</span> - <span class="n">columns</span><span class="o">=</span><span class="n">columns</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">num_shards</span><span class="p">,</span> <span class="n">target_partition_size</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">target_partition_size</span><span class="p">,</span> - <span class="n">metric_spec</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">metric_spec</span><span class="p">,</span> <span class="n">hadoop_dependency_coordinates</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">hadoop_dependency_coordinates</span><span class="p">)</span> - <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Load seems to have succeeded!"</span><span class="p">)</span> - - <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span> - <span class="s2">"Cleaning up by dropping the temp "</span> - <span class="s2">"Hive table {}"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">hive_table</span><span class="p">))</span> - <span class="n">hql</span> <span class="o">=</span> <span class="s2">"DROP TABLE IF EXISTS {}"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">hive_table</span><span class="p">)</span> - <span class="n">hive</span><span class="o">.</span><span class="n">run_cli</span><span class="p">(</span><span class="n">hql</span><span class="p">)</span></div> + <span class="k">try</span><span class="p">:</span> + <span class="n">druid</span><span class="o">.</span><span class="n">load_from_hdfs</span><span class="p">(</span> + <span class="n">datasource</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">druid_datasource</span><span class="p">,</span> + <span class="n">intervals</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">intervals</span><span class="p">,</span> + <span class="n">static_path</span><span class="o">=</span><span class="n">static_path</span><span class="p">,</span> <span class="n">ts_dim</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">ts_dim</span><span class="p">,</span> + <span class="n">columns</span><span class="o">=</span><span class="n">columns</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">num_shards</span><span class="p">,</span> <span class="n">target_partition_size</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">target_partition_size</span><span class="p">,</span> + <span class="n">metric_spec</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">metric_spec</span><span class="p">,</span> <span class="n">hadoop_dependency_coordinates</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">hadoop_dependency_coordinates</span><span class="p">)</span> + <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Load seems to have succeeded!"</span><span class="p">)</span> + <span class="k">finally</span><span class="p">:</span> + <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span> + <span class="s2">"Cleaning up by dropping the temp "</span> + <span class="s2">"Hive table {}"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">hive_table</span><span class="p">))</span> + <span class="n">hql</span> <span class="o">=</span> <span class="s2">"DROP TABLE IF EXISTS {}"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">hive_table</span><span class="p">)</span> + <span class="n">hive</span><span class="o">.</span><span class="n">run_cli</span><span class="p">(</span><span class="n">hql</span><span class="p">)</span></div> </pre></div> </div>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/hive_to_mysql.html ---------------------------------------------------------------------- diff --git a/_modules/hive_to_mysql.html b/_modules/hive_to_mysql.html index 6a5bedd..867023b 100644 --- a/_modules/hive_to_mysql.html +++ b/_modules/hive_to_mysql.html @@ -137,9 +137,24 @@ <div itemprop="articleBody"> <h1>Source code for hive_to_mysql</h1><div class="highlight"><pre> -<span></span><span class="kn">import</span> <span class="nn">logging</span> - -<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">HiveServer2Hook</span><span class="p">,</span> <span class="n">MySqlHook</span> +<span></span><span class="c1"># -*- coding: utf-8 -*-</span> +<span class="c1">#</span> +<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span> +<span class="c1"># you may not use this file except in compliance with the License.</span> +<span class="c1"># You may obtain a copy of the License at</span> +<span class="c1">#</span> +<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span> +<span class="c1">#</span> +<span class="c1"># Unless required by applicable law or agreed to in writing, software</span> +<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span> +<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span> +<span class="c1"># See the License for the specific language governing permissions and</span> +<span class="c1"># limitations under the License.</span> + +<span class="kn">import</span> <span class="nn">logging</span> + +<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveServer2Hook</span> +<span class="kn">from</span> <span class="nn">airflow.hooks.mysql_hook</span> <span class="kn">import</span> <span class="n">MySqlHook</span> <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span> <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span> http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/hive_to_samba_operator.html ---------------------------------------------------------------------- diff --git a/_modules/hive_to_samba_operator.html b/_modules/hive_to_samba_operator.html index 3008244..a69680d 100644 --- a/_modules/hive_to_samba_operator.html +++ b/_modules/hive_to_samba_operator.html @@ -137,10 +137,25 @@ <div itemprop="articleBody"> <h1>Source code for hive_to_samba_operator</h1><div class="highlight"><pre> -<span></span><span class="kn">import</span> <span class="nn">logging</span> +<span></span><span class="c1"># -*- coding: utf-8 -*-</span> +<span class="c1">#</span> +<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span> +<span class="c1"># you may not use this file except in compliance with the License.</span> +<span class="c1"># You may obtain a copy of the License at</span> +<span class="c1">#</span> +<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span> +<span class="c1">#</span> +<span class="c1"># Unless required by applicable law or agreed to in writing, software</span> +<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span> +<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span> +<span class="c1"># See the License for the specific language governing permissions and</span> +<span class="c1"># limitations under the License.</span> + +<span class="kn">import</span> <span class="nn">logging</span> <span class="kn">import</span> <span class="nn">tempfile</span> -<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">HiveServer2Hook</span><span class="p">,</span> <span class="n">SambaHook</span> +<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveServer2Hook</span> +<span class="kn">from</span> <span class="nn">airflow.hooks.samba_hook</span> <span class="kn">import</span> <span class="n">SambaHook</span> <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span> <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span> http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/http_hook.html ---------------------------------------------------------------------- diff --git a/_modules/http_hook.html b/_modules/http_hook.html index 98ed5ad..b62b0d0 100644 --- a/_modules/http_hook.html +++ b/_modules/http_hook.html @@ -150,7 +150,7 @@ <span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span> <span class="c1"># See the License for the specific language governing permissions and</span> <span class="c1"># limitations under the License.</span> -<span class="c1">#</span> + <span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span> <span class="kn">import</span> <span class="nn">logging</span> http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/http_operator.html ---------------------------------------------------------------------- diff --git a/_modules/http_operator.html b/_modules/http_operator.html index 942ae0e..00496c9 100644 --- a/_modules/http_operator.html +++ b/_modules/http_operator.html @@ -137,10 +137,24 @@ <div itemprop="articleBody"> <h1>Source code for http_operator</h1><div class="highlight"><pre> -<span></span><span class="kn">import</span> <span class="nn">logging</span> +<span></span><span class="c1"># -*- coding: utf-8 -*-</span> +<span class="c1">#</span> +<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span> +<span class="c1"># you may not use this file except in compliance with the License.</span> +<span class="c1"># You may obtain a copy of the License at</span> +<span class="c1">#</span> +<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span> +<span class="c1">#</span> +<span class="c1"># Unless required by applicable law or agreed to in writing, software</span> +<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span> +<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span> +<span class="c1"># See the License for the specific language governing permissions and</span> +<span class="c1"># limitations under the License.</span> + +<span class="kn">import</span> <span class="nn">logging</span> <span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="kn">import</span> <span class="n">AirflowException</span> -<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">HttpHook</span> +<span class="kn">from</span> <span class="nn">airflow.hooks.http_hook</span> <span class="kn">import</span> <span class="n">HttpHook</span> <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span> <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span> http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/mssql_hook.html ---------------------------------------------------------------------- diff --git a/_modules/mssql_hook.html b/_modules/mssql_hook.html index b3b5503..9aff59d 100644 --- a/_modules/mssql_hook.html +++ b/_modules/mssql_hook.html @@ -137,7 +137,21 @@ <div itemprop="articleBody"> <h1>Source code for mssql_hook</h1><div class="highlight"><pre> -<span></span><span class="kn">import</span> <span class="nn">pymssql</span> +<span></span><span class="c1"># -*- coding: utf-8 -*-</span> +<span class="c1">#</span> +<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span> +<span class="c1"># you may not use this file except in compliance with the License.</span> +<span class="c1"># You may obtain a copy of the License at</span> +<span class="c1">#</span> +<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span> +<span class="c1">#</span> +<span class="c1"># Unless required by applicable law or agreed to in writing, software</span> +<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span> +<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span> +<span class="c1"># See the License for the specific language governing permissions and</span> +<span class="c1"># limitations under the License.</span> + +<span class="kn">import</span> <span class="nn">pymssql</span> <span class="kn">from</span> <span class="nn">airflow.hooks.dbapi_hook</span> <span class="kn">import</span> <span class="n">DbApiHook</span> @@ -157,10 +171,11 @@ <span class="sd"> """</span> <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">mssql_conn_id</span><span class="p">)</span> <span class="n">conn</span> <span class="o">=</span> <span class="n">pymssql</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span> - <span class="n">conn</span><span class="o">.</span><span class="n">host</span><span class="p">,</span> - <span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">,</span> - <span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">,</span> - <span class="n">conn</span><span class="o">.</span><span class="n">schema</span><span class="p">)</span> + <span class="n">server</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">host</span><span class="p">,</span> + <span class="n">user</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">,</span> + <span class="n">password</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">,</span> + <span class="n">database</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span> + <span class="n">port</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">port</span><span class="p">)</span> <span class="k">return</span> <span class="n">conn</span></div> <span class="k">def</span> <span class="nf">set_autocommit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">conn</span><span class="p">,</span> <span class="n">autocommit</span><span class="p">):</span> http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/mssql_operator.html ---------------------------------------------------------------------- diff --git a/_modules/mssql_operator.html b/_modules/mssql_operator.html index dcc2872..e933004 100644 --- a/_modules/mssql_operator.html +++ b/_modules/mssql_operator.html @@ -137,9 +137,23 @@ <div itemprop="articleBody"> <h1>Source code for mssql_operator</h1><div class="highlight"><pre> -<span></span><span class="kn">import</span> <span class="nn">logging</span> - -<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">MsSqlHook</span> +<span></span><span class="c1"># -*- coding: utf-8 -*-</span> +<span class="c1">#</span> +<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span> +<span class="c1"># you may not use this file except in compliance with the License.</span> +<span class="c1"># You may obtain a copy of the License at</span> +<span class="c1">#</span> +<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span> +<span class="c1">#</span> +<span class="c1"># Unless required by applicable law or agreed to in writing, software</span> +<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span> +<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span> +<span class="c1"># See the License for the specific language governing permissions and</span> +<span class="c1"># limitations under the License.</span> + +<span class="kn">import</span> <span class="nn">logging</span> + +<span class="kn">from</span> <span class="nn">airflow.hooks.mssql_hook</span> <span class="kn">import</span> <span class="n">MsSqlHook</span> <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span> <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span> http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/mssql_to_hive.html ---------------------------------------------------------------------- diff --git a/_modules/mssql_to_hive.html b/_modules/mssql_to_hive.html index 451dc7f..590dfb8 100644 --- a/_modules/mssql_to_hive.html +++ b/_modules/mssql_to_hive.html @@ -137,7 +137,21 @@ <div itemprop="articleBody"> <h1>Source code for mssql_to_hive</h1><div class="highlight"><pre> -<span></span><span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">chr</span> +<span></span><span class="c1"># -*- coding: utf-8 -*-</span> +<span class="c1">#</span> +<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span> +<span class="c1"># you may not use this file except in compliance with the License.</span> +<span class="c1"># You may obtain a copy of the License at</span> +<span class="c1">#</span> +<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span> +<span class="c1">#</span> +<span class="c1"># Unless required by applicable law or agreed to in writing, software</span> +<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span> +<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span> +<span class="c1"># See the License for the specific language governing permissions and</span> +<span class="c1"># limitations under the License.</span> + +<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">chr</span> <span class="kn">from</span> <span class="nn">collections</span> <span class="kn">import</span> <span class="n">OrderedDict</span> <span class="kn">import</span> <span class="nn">unicodecsv</span> <span class="kn">as</span> <span class="nn">csv</span> <span class="kn">import</span> <span class="nn">logging</span> @@ -145,7 +159,8 @@ <span class="kn">import</span> <span class="nn">pymssql</span> -<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span><span class="p">,</span> <span class="n">MsSqlHook</span> +<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span> +<span class="kn">from</span> <span class="nn">airflow.hooks.mssql_hook</span> <span class="kn">import</span> <span class="n">MsSqlHook</span> <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span> <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span> http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/mysql_hook.html ---------------------------------------------------------------------- diff --git a/_modules/mysql_hook.html b/_modules/mysql_hook.html index 23d3726..941737f 100644 --- a/_modules/mysql_hook.html +++ b/_modules/mysql_hook.html @@ -137,7 +137,21 @@ <div itemprop="articleBody"> <h1>Source code for mysql_hook</h1><div class="highlight"><pre> -<span></span><span class="kn">import</span> <span class="nn">MySQLdb</span> +<span></span><span class="c1"># -*- coding: utf-8 -*-</span> +<span class="c1">#</span> +<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span> +<span class="c1"># you may not use this file except in compliance with the License.</span> +<span class="c1"># You may obtain a copy of the License at</span> +<span class="c1">#</span> +<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span> +<span class="c1">#</span> +<span class="c1"># Unless required by applicable law or agreed to in writing, software</span> +<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span> +<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span> +<span class="c1"># See the License for the specific language governing permissions and</span> +<span class="c1"># limitations under the License.</span> + +<span class="kn">import</span> <span class="nn">MySQLdb</span> <span class="kn">import</span> <span class="nn">MySQLdb.cursors</span> <span class="kn">from</span> <span class="nn">airflow.hooks.dbapi_hook</span> <span class="kn">import</span> <span class="n">DbApiHook</span> http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/mysql_operator.html ---------------------------------------------------------------------- diff --git a/_modules/mysql_operator.html b/_modules/mysql_operator.html index 331f2b4..a1fc11a 100644 --- a/_modules/mysql_operator.html +++ b/_modules/mysql_operator.html @@ -137,9 +137,23 @@ <div itemprop="articleBody"> <h1>Source code for mysql_operator</h1><div class="highlight"><pre> -<span></span><span class="kn">import</span> <span class="nn">logging</span> - -<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">MySqlHook</span> +<span></span><span class="c1"># -*- coding: utf-8 -*-</span> +<span class="c1">#</span> +<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span> +<span class="c1"># you may not use this file except in compliance with the License.</span> +<span class="c1"># You may obtain a copy of the License at</span> +<span class="c1">#</span> +<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span> +<span class="c1">#</span> +<span class="c1"># Unless required by applicable law or agreed to in writing, software</span> +<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span> +<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span> +<span class="c1"># See the License for the specific language governing permissions and</span> +<span class="c1"># limitations under the License.</span> + +<span class="kn">import</span> <span class="nn">logging</span> + +<span class="kn">from</span> <span class="nn">airflow.hooks.mysql_hook</span> <span class="kn">import</span> <span class="n">MySqlHook</span> <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span> <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span> http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/mysql_to_hive.html ---------------------------------------------------------------------- diff --git a/_modules/mysql_to_hive.html b/_modules/mysql_to_hive.html index fa6b20f..cc8496b 100644 --- a/_modules/mysql_to_hive.html +++ b/_modules/mysql_to_hive.html @@ -137,14 +137,29 @@ <div itemprop="articleBody"> <h1>Source code for mysql_to_hive</h1><div class="highlight"><pre> -<span></span><span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">chr</span> +<span></span><span class="c1"># -*- coding: utf-8 -*-</span> +<span class="c1">#</span> +<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span> +<span class="c1"># you may not use this file except in compliance with the License.</span> +<span class="c1"># You may obtain a copy of the License at</span> +<span class="c1">#</span> +<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span> +<span class="c1">#</span> +<span class="c1"># Unless required by applicable law or agreed to in writing, software</span> +<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span> +<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span> +<span class="c1"># See the License for the specific language governing permissions and</span> +<span class="c1"># limitations under the License.</span> + +<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">chr</span> <span class="kn">from</span> <span class="nn">collections</span> <span class="kn">import</span> <span class="n">OrderedDict</span> <span class="kn">import</span> <span class="nn">unicodecsv</span> <span class="kn">as</span> <span class="nn">csv</span> <span class="kn">import</span> <span class="nn">logging</span> <span class="kn">from</span> <span class="nn">tempfile</span> <span class="kn">import</span> <span class="n">NamedTemporaryFile</span> <span class="kn">import</span> <span class="nn">MySQLdb</span> -<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span><span class="p">,</span> <span class="n">MySqlHook</span> +<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span> +<span class="kn">from</span> <span class="nn">airflow.hooks.mysql_hook</span> <span class="kn">import</span> <span class="n">MySqlHook</span> <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span> <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span> http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/postgres_hook.html ---------------------------------------------------------------------- diff --git a/_modules/postgres_hook.html b/_modules/postgres_hook.html index d8896ac..3ecfa69 100644 --- a/_modules/postgres_hook.html +++ b/_modules/postgres_hook.html @@ -137,7 +137,21 @@ <div itemprop="articleBody"> <h1>Source code for postgres_hook</h1><div class="highlight"><pre> -<span></span><span class="kn">import</span> <span class="nn">psycopg2</span> +<span></span><span class="c1"># -*- coding: utf-8 -*-</span> +<span class="c1">#</span> +<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span> +<span class="c1"># you may not use this file except in compliance with the License.</span> +<span class="c1"># You may obtain a copy of the License at</span> +<span class="c1">#</span> +<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span> +<span class="c1">#</span> +<span class="c1"># Unless required by applicable law or agreed to in writing, software</span> +<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span> +<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span> +<span class="c1"># See the License for the specific language governing permissions and</span> +<span class="c1"># limitations under the License.</span> + +<span class="kn">import</span> <span class="nn">psycopg2</span> <span class="kn">import</span> <span class="nn">psycopg2.extensions</span> <span class="kn">from</span> <span class="nn">airflow.hooks.dbapi_hook</span> <span class="kn">import</span> <span class="n">DbApiHook</span> http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/postgres_operator.html ---------------------------------------------------------------------- diff --git a/_modules/postgres_operator.html b/_modules/postgres_operator.html index 4eb0e00..5ade37d 100644 --- a/_modules/postgres_operator.html +++ b/_modules/postgres_operator.html @@ -137,9 +137,23 @@ <div itemprop="articleBody"> <h1>Source code for postgres_operator</h1><div class="highlight"><pre> -<span></span><span class="kn">import</span> <span class="nn">logging</span> - -<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">PostgresHook</span> +<span></span><span class="c1"># -*- coding: utf-8 -*-</span> +<span class="c1">#</span> +<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span> +<span class="c1"># you may not use this file except in compliance with the License.</span> +<span class="c1"># You may obtain a copy of the License at</span> +<span class="c1">#</span> +<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span> +<span class="c1">#</span> +<span class="c1"># Unless required by applicable law or agreed to in writing, software</span> +<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span> +<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span> +<span class="c1"># See the License for the specific language governing permissions and</span> +<span class="c1"># limitations under the License.</span> + +<span class="kn">import</span> <span class="nn">logging</span> + +<span class="kn">from</span> <span class="nn">airflow.hooks.postgres_hook</span> <span class="kn">import</span> <span class="n">PostgresHook</span> <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span> <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span> http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/presto_check_operator.html ---------------------------------------------------------------------- diff --git a/_modules/presto_check_operator.html b/_modules/presto_check_operator.html index 52dc0e6..e24b271 100644 --- a/_modules/presto_check_operator.html +++ b/_modules/presto_check_operator.html @@ -137,8 +137,22 @@ <div itemprop="articleBody"> <h1>Source code for presto_check_operator</h1><div class="highlight"><pre> -<span></span><span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">PrestoHook</span> -<span class="kn">from</span> <span class="nn">airflow.operators</span> <span class="kn">import</span> <span class="n">CheckOperator</span><span class="p">,</span> <span class="n">ValueCheckOperator</span><span class="p">,</span> <span class="n">IntervalCheckOperator</span> +<span></span><span class="c1"># -*- coding: utf-8 -*-</span> +<span class="c1">#</span> +<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span> +<span class="c1"># you may not use this file except in compliance with the License.</span> +<span class="c1"># You may obtain a copy of the License at</span> +<span class="c1">#</span> +<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span> +<span class="c1">#</span> +<span class="c1"># Unless required by applicable law or agreed to in writing, software</span> +<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span> +<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span> +<span class="c1"># See the License for the specific language governing permissions and</span> +<span class="c1"># limitations under the License.</span> + +<span class="kn">from</span> <span class="nn">airflow.hooks.presto_hook</span> <span class="kn">import</span> <span class="n">PrestoHook</span> +<span class="kn">from</span> <span class="nn">airflow.operators.check_operator</span> <span class="kn">import</span> <span class="n">CheckOperator</span><span class="p">,</span> <span class="n">ValueCheckOperator</span><span class="p">,</span> <span class="n">IntervalCheckOperator</span> <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span> http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/presto_hook.html ---------------------------------------------------------------------- diff --git a/_modules/presto_hook.html b/_modules/presto_hook.html index 73e4cc0..49b93a9 100644 --- a/_modules/presto_hook.html +++ b/_modules/presto_hook.html @@ -137,13 +137,28 @@ <div itemprop="articleBody"> <h1>Source code for presto_hook</h1><div class="highlight"><pre> -<span></span><span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span> +<span></span><span class="c1"># -*- coding: utf-8 -*-</span> +<span class="c1">#</span> +<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span> +<span class="c1"># you may not use this file except in compliance with the License.</span> +<span class="c1"># You may obtain a copy of the License at</span> +<span class="c1">#</span> +<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span> +<span class="c1">#</span> +<span class="c1"># Unless required by applicable law or agreed to in writing, software</span> +<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span> +<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span> +<span class="c1"># See the License for the specific language governing permissions and</span> +<span class="c1"># limitations under the License.</span> + +<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span> +<span class="kn">import</span> <span class="nn">logging</span> + <span class="kn">from</span> <span class="nn">pyhive</span> <span class="kn">import</span> <span class="n">presto</span> <span class="kn">from</span> <span class="nn">pyhive.exc</span> <span class="kn">import</span> <span class="n">DatabaseError</span> <span class="kn">from</span> <span class="nn">airflow.hooks.dbapi_hook</span> <span class="kn">import</span> <span class="n">DbApiHook</span> -<span class="kn">import</span> <span class="nn">logging</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s2">"pyhive"</span><span class="p">)</span><span class="o">.</span><span class="n">setLevel</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">INFO</span><span class="p">)</span> @@ -204,8 +219,7 @@ <span class="k">return</span> <span class="nb">super</span><span class="p">(</span><span class="n">PrestoHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">get_first</span><span class="p">(</span> <span class="bp">self</span><span class="o">.</span><span class="n">_strip_sql</span><span class="p">(</span><span class="n">hql</span><span class="p">),</span> <span class="n">parameters</span><span class="p">)</span> <span class="k">except</span> <span class="n">DatabaseError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span> - <span class="n">obj</span> <span class="o">=</span> <span class="nb">eval</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">))</span> - <span class="k">raise</span> <span class="n">PrestoException</span><span class="p">(</span><span class="n">obj</span><span class="p">[</span><span class="s1">'message'</span><span class="p">])</span></div> + <span class="k">raise</span> <span class="n">PrestoException</span><span class="p">(</span><span class="n">e</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="s1">'message'</span><span class="p">])</span></div> <div class="viewcode-block" id="PrestoHook.get_pandas_df"><a class="viewcode-back" href="../code.html#airflow.hooks.PrestoHook.get_pandas_df">[docs]</a> <span class="k">def</span> <span class="nf">get_pandas_df</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">hql</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span> <span class="sd">"""</span> @@ -217,8 +231,7 @@ <span class="n">cursor</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_strip_sql</span><span class="p">(</span><span class="n">hql</span><span class="p">),</span> <span class="n">parameters</span><span class="p">)</span> <span class="n">data</span> <span class="o">=</span> <span class="n">cursor</span><span class="o">.</span><span class="n">fetchall</span><span class="p">()</span> <span class="k">except</span> <span class="n">DatabaseError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span> - <span class="n">obj</span> <span class="o">=</span> <span class="nb">eval</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">))</span> - <span class="k">raise</span> <span class="n">PrestoException</span><span class="p">(</span><span class="n">obj</span><span class="p">[</span><span class="s1">'message'</span><span class="p">])</span> + <span class="k">raise</span> <span class="n">PrestoException</span><span class="p">(</span><span class="n">e</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="s1">'message'</span><span class="p">])</span> <span class="n">column_descriptions</span> <span class="o">=</span> <span class="n">cursor</span><span class="o">.</span><span class="n">description</span> <span class="k">if</span> <span class="n">data</span><span class="p">:</span> <span class="n">df</span> <span class="o">=</span> <span class="n">pandas</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">data</span><span class="p">)</span> http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/python_operator.html ---------------------------------------------------------------------- diff --git a/_modules/python_operator.html b/_modules/python_operator.html index 0f0de31..059ebb2 100644 --- a/_modules/python_operator.html +++ b/_modules/python_operator.html @@ -137,7 +137,21 @@ <div itemprop="articleBody"> <h1>Source code for python_operator</h1><div class="highlight"><pre> -<span></span><span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span> +<span></span><span class="c1"># -*- coding: utf-8 -*-</span> +<span class="c1">#</span> +<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span> +<span class="c1"># you may not use this file except in compliance with the License.</span> +<span class="c1"># You may obtain a copy of the License at</span> +<span class="c1">#</span> +<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span> +<span class="c1">#</span> +<span class="c1"># Unless required by applicable law or agreed to in writing, software</span> +<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span> +<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span> +<span class="c1"># See the License for the specific language governing permissions and</span> +<span class="c1"># limitations under the License.</span> + +<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span> <span class="kn">from</span> <span class="nn">datetime</span> <span class="kn">import</span> <span class="n">datetime</span> <span class="kn">import</span> <span class="nn">logging</span> http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/qubole_operator.html ---------------------------------------------------------------------- diff --git a/_modules/qubole_operator.html b/_modules/qubole_operator.html index 07ea626..fb33047 100644 --- a/_modules/qubole_operator.html +++ b/_modules/qubole_operator.html @@ -137,9 +137,23 @@ <div itemprop="articleBody"> <h1>Source code for qubole_operator</h1><div class="highlight"><pre> -<span></span><span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span> +<span></span><span class="c1"># -*- coding: utf-8 -*-</span> +<span class="c1">#</span> +<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span> +<span class="c1"># you may not use this file except in compliance with the License.</span> +<span class="c1"># You may obtain a copy of the License at</span> +<span class="c1">#</span> +<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span> +<span class="c1">#</span> +<span class="c1"># Unless required by applicable law or agreed to in writing, software</span> +<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span> +<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span> +<span class="c1"># See the License for the specific language governing permissions and</span> +<span class="c1"># limitations under the License.</span> + +<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span> <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span> -<span class="kn">from</span> <span class="nn">airflow.contrib.hooks</span> <span class="kn">import</span> <span class="n">QuboleHook</span> +<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.qubole_hook</span> <span class="kn">import</span> <span class="n">QuboleHook</span> <div class="viewcode-block" id="QuboleOperator"><a class="viewcode-back" href="../code.html#airflow.contrib.operators.QuboleOperator">[docs]</a><span class="k">class</span> <span class="nc">QuboleOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span> @@ -220,7 +234,7 @@ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'query'</span><span class="p">,</span> <span class="s1">'script_location'</span><span class="p">,</span> <span class="s1">'sub_command'</span><span class="p">,</span> <span class="s1">'script'</span><span class="p">,</span> <span class="s1">'files'</span><span class="p">,</span> <span class="s1">'archives'</span><span class="p">,</span> <span class="s1">'program'</span><span class="p">,</span> <span class="s1">'cmdline'</span><span class="p">,</span> <span class="s1">'sql'</span><span class="p">,</span> <span class="s1">'where_clause'</span><span class="p">,</span> <span class="s1">'extract_query'</span><span class="p">,</span> <span class="s1">'boundary_query'</span><span class="p">,</span> <span class="s1">'macros'</span><span class="p">,</span> <span class="s1">'tags'</span><span class="p">,</span> <span class="s1">'name'</span><span class="p">,</span> <span class="s1">'parameters'</span><span class="p">)</span> - <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.txt'</span><span class="p">)</span> + <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.txt'</span><span class="p">,)</span> <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#3064A1'</span> <span class="n">ui_fgcolor</span> <span class="o">=</span> <span class="s1">'#fff'</span> http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/s3_to_hive_operator.html ---------------------------------------------------------------------- diff --git a/_modules/s3_to_hive_operator.html b/_modules/s3_to_hive_operator.html index dc23eb5..ad7715e 100644 --- a/_modules/s3_to_hive_operator.html +++ b/_modules/s3_to_hive_operator.html @@ -137,13 +137,28 @@ <div itemprop="articleBody"> <h1>Source code for s3_to_hive_operator</h1><div class="highlight"><pre> -<span></span><span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">next</span> +<span></span><span class="c1"># -*- coding: utf-8 -*-</span> +<span class="c1">#</span> +<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span> +<span class="c1"># you may not use this file except in compliance with the License.</span> +<span class="c1"># You may obtain a copy of the License at</span> +<span class="c1">#</span> +<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span> +<span class="c1">#</span> +<span class="c1"># Unless required by applicable law or agreed to in writing, software</span> +<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span> +<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span> +<span class="c1"># See the License for the specific language governing permissions and</span> +<span class="c1"># limitations under the License.</span> + +<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">next</span> <span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">zip</span> <span class="kn">import</span> <span class="nn">logging</span> <span class="kn">from</span> <span class="nn">tempfile</span> <span class="kn">import</span> <span class="n">NamedTemporaryFile</span> <span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="kn">import</span> <span class="n">AirflowException</span> -<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span><span class="p">,</span> <span class="n">S3Hook</span> +<span class="kn">from</span> <span class="nn">airflow.hooks.S3_hook</span> <span class="kn">import</span> <span class="n">S3Hook</span> +<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span> <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span> <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span> http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/sensors.html ---------------------------------------------------------------------- diff --git a/_modules/sensors.html b/_modules/sensors.html index 19336b2..fab6313 100644 --- a/_modules/sensors.html +++ b/_modules/sensors.html @@ -137,7 +137,21 @@ <div itemprop="articleBody"> <h1>Source code for sensors</h1><div class="highlight"><pre> -<span></span><span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">print_function</span> +<span></span><span class="c1"># -*- coding: utf-8 -*-</span> +<span class="c1">#</span> +<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span> +<span class="c1"># you may not use this file except in compliance with the License.</span> +<span class="c1"># You may obtain a copy of the License at</span> +<span class="c1">#</span> +<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span> +<span class="c1">#</span> +<span class="c1"># Unless required by applicable law or agreed to in writing, software</span> +<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span> +<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span> +<span class="c1"># See the License for the specific language governing permissions and</span> +<span class="c1"># limitations under the License.</span> + +<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">print_function</span> <span class="kn">from</span> <span class="nn">future</span> <span class="kn">import</span> <span class="n">standard_library</span> <span class="n">standard_library</span><span class="o">.</span><span class="n">install_aliases</span><span class="p">()</span> <span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span> @@ -146,10 +160,11 @@ <span class="kn">from</span> <span class="nn">urllib.parse</span> <span class="kn">import</span> <span class="n">urlparse</span> <span class="kn">from</span> <span class="nn">time</span> <span class="kn">import</span> <span class="n">sleep</span> +<span class="kn">import</span> <span class="nn">airflow</span> <span class="kn">from</span> <span class="nn">airflow</span> <span class="kn">import</span> <span class="n">hooks</span><span class="p">,</span> <span class="n">settings</span> <span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="kn">import</span> <span class="n">AirflowException</span><span class="p">,</span> <span class="n">AirflowSensorTimeout</span><span class="p">,</span> <span class="n">AirflowSkipException</span> <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span><span class="p">,</span> <span class="n">TaskInstance</span><span class="p">,</span> <span class="n">Connection</span> <span class="k">as</span> <span class="n">DB</span> -<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">BaseHook</span> +<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="kn">import</span> <span class="n">BaseHook</span> <span class="kn">from</span> <span class="nn">airflow.utils.state</span> <span class="kn">import</span> <span class="n">State</span> <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span> @@ -193,12 +208,12 @@ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> <span class="n">started_at</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span> <span class="k">while</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">poke</span><span class="p">(</span><span class="n">context</span><span class="p">):</span> - <span class="n">sleep</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">poke_interval</span><span class="p">)</span> - <span class="k">if</span> <span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span> <span class="o">-</span> <span class="n">started_at</span><span class="p">)</span><span class="o">.</span><span class="n">seconds</span> <span class="o">></span> <span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">:</span> + <span class="k">if</span> <span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span> <span class="o">-</span> <span class="n">started_at</span><span class="p">)</span><span class="o">.</span><span class="n">total_seconds</span><span class="p">()</span> <span class="o">></span> <span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">:</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">soft_fail</span><span class="p">:</span> <span class="k">raise</span> <span class="n">AirflowSkipException</span><span class="p">(</span><span class="s1">'Snap. Time is OUT.'</span><span class="p">)</span> <span class="k">else</span><span class="p">:</span> <span class="k">raise</span> <span class="n">AirflowSensorTimeout</span><span class="p">(</span><span class="s1">'Snap. Time is OUT.'</span><span class="p">)</span> + <span class="n">sleep</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">poke_interval</span><span class="p">)</span> <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Success criteria met. Exiting."</span><span class="p">)</span> @@ -303,8 +318,14 @@ <span class="sd"> :type allowed_states: list</span> <span class="sd"> :param execution_delta: time difference with the previous execution to</span> <span class="sd"> look at, the default is the same execution_date as the current task.</span> -<span class="sd"> For yesterday, use [positive!] datetime.timedelta(days=1)</span> +<span class="sd"> For yesterday, use [positive!] datetime.timedelta(days=1). Either</span> +<span class="sd"> execution_delta or execution_date_fn can be passed to</span> +<span class="sd"> ExternalTaskSensor, but not both.</span> <span class="sd"> :type execution_delta: datetime.timedelta</span> +<span class="sd"> :param execution_date_fn: function that receives the current execution date</span> +<span class="sd"> and returns the desired execution date to query. Either execution_delta</span> +<span class="sd"> or execution_date_fn can be passed to ExternalTaskSensor, but not both.</span> +<span class="sd"> :type execution_date_fn: callable</span> <span class="sd"> """</span> <span class="nd">@apply_defaults</span> @@ -314,16 +335,25 @@ <span class="n">external_task_id</span><span class="p">,</span> <span class="n">allowed_states</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">execution_delta</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> + <span class="n">execution_date_fn</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span> <span class="nb">super</span><span class="p">(</span><span class="n">ExternalTaskSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> <span class="bp">self</span><span class="o">.</span><span class="n">allowed_states</span> <span class="o">=</span> <span class="n">allowed_states</span> <span class="ow">or</span> <span class="p">[</span><span class="n">State</span><span class="o">.</span><span class="n">SUCCESS</span><span class="p">]</span> + <span class="k">if</span> <span class="n">execution_delta</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span> <span class="ow">and</span> <span class="n">execution_date_fn</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span><span class="p">:</span> + <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span> + <span class="s1">'Only one of `execution_date` or `execution_date_fn` may'</span> + <span class="s1">'be provided to ExternalTaskSensor; not both.'</span><span class="p">)</span> + <span class="bp">self</span><span class="o">.</span><span class="n">execution_delta</span> <span class="o">=</span> <span class="n">execution_delta</span> + <span class="bp">self</span><span class="o">.</span><span class="n">execution_date_fn</span> <span class="o">=</span> <span class="n">execution_date_fn</span> <span class="bp">self</span><span class="o">.</span><span class="n">external_dag_id</span> <span class="o">=</span> <span class="n">external_dag_id</span> <span class="bp">self</span><span class="o">.</span><span class="n">external_task_id</span> <span class="o">=</span> <span class="n">external_task_id</span> <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_delta</span><span class="p">:</span> <span class="n">dttm</span> <span class="o">=</span> <span class="n">context</span><span class="p">[</span><span class="s1">'execution_date'</span><span class="p">]</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_delta</span> + <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date_fn</span><span class="p">:</span> + <span class="n">dttm</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date_fn</span><span class="p">(</span><span class="n">context</span><span class="p">[</span><span class="s1">'execution_date'</span><span class="p">])</span> <span class="k">else</span><span class="p">:</span> <span class="n">dttm</span> <span class="o">=</span> <span class="n">context</span><span class="p">[</span><span class="s1">'execution_date'</span><span class="p">]</span> @@ -346,15 +376,91 @@ <span class="k">return</span> <span class="n">count</span></div> +<span class="k">class</span> <span class="nc">NamedHivePartitionSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span> + <span class="sd">"""</span> +<span class="sd"> Waits for a set of partitions to show up in Hive.</span> + +<span class="sd"> :param partition_names: List of fully qualified names of the</span> +<span class="sd"> partitions to wait for. A fully qualified name is of the</span> +<span class="sd"> form schema.table/pk1=pv1/pk2=pv2, for example,</span> +<span class="sd"> default.users/ds=2016-01-01. This is passed as is to the metastore</span> +<span class="sd"> Thrift client "get_partitions_by_name" method. Note that</span> +<span class="sd"> you cannot use logical operators as in HivePartitionSensor.</span> +<span class="sd"> :type partition_names: list of strings</span> +<span class="sd"> :param metastore_conn_id: reference to the metastore thrift service</span> +<span class="sd"> connection id</span> +<span class="sd"> :type metastore_conn_id: str</span> +<span class="sd"> """</span> + + <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'partition_names'</span><span class="p">,</span> <span class="p">)</span> + + <span class="nd">@apply_defaults</span> + <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span> + <span class="bp">self</span><span class="p">,</span> + <span class="n">partition_names</span><span class="p">,</span> + <span class="n">metastore_conn_id</span><span class="o">=</span><span class="s1">'metastore_default'</span><span class="p">,</span> + <span class="n">poke_interval</span><span class="o">=</span><span class="mi">60</span><span class="o">*</span><span class="mi">3</span><span class="p">,</span> + <span class="o">*</span><span class="n">args</span><span class="p">,</span> + <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span> + <span class="nb">super</span><span class="p">(</span><span class="n">NamedHivePartitionSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span> + <span class="n">poke_interval</span><span class="o">=</span><span class="n">poke_interval</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> + + <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">partition_names</span><span class="p">,</span> <span class="nb">basestring</span><span class="p">):</span> + <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s1">'partition_names must be an array of strings'</span><span class="p">)</span> + + <span class="k">for</span> <span class="n">partition_name</span> <span class="ow">in</span> <span class="n">partition_names</span><span class="p">:</span> + <span class="bp">self</span><span class="o">.</span><span class="n">parse_partition_name</span><span class="p">(</span><span class="n">partition_name</span><span class="p">)</span> + + <span class="bp">self</span><span class="o">.</span><span class="n">metastore_conn_id</span> <span class="o">=</span> <span class="n">metastore_conn_id</span> + <span class="bp">self</span><span class="o">.</span><span class="n">partition_names</span> <span class="o">=</span> <span class="n">partition_names</span> + <span class="bp">self</span><span class="o">.</span><span class="n">next_poke_idx</span> <span class="o">=</span> <span class="mi">0</span> + + <span class="k">def</span> <span class="nf">parse_partition_name</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">partition</span><span class="p">):</span> + <span class="k">try</span><span class="p">:</span> + <span class="n">schema</span><span class="p">,</span> <span class="n">table_partition</span> <span class="o">=</span> <span class="n">partition</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'.'</span><span class="p">)</span> + <span class="n">table</span><span class="p">,</span> <span class="n">partition</span> <span class="o">=</span> <span class="n">table_partition</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'/'</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span> + <span class="k">return</span> <span class="n">schema</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">partition</span> + <span class="k">except</span> <span class="ne">ValueError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span> + <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">'Could not parse '</span> <span class="o">+</span> <span class="n">partition</span><span class="p">)</span> + + <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> + + <span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s1">'hook'</span><span class="p">):</span> + <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">airflow</span><span class="o">.</span><span class="n">hooks</span><span class="o">.</span><span class="n">hive_hooks</span><span class="o">.</span><span class="n">HiveMetastoreHook</span><span class="p">(</span> + <span class="n">metastore_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">metastore_conn_id</span><span class="p">)</span> + + <span class="k">def</span> <span class="nf">poke_partition</span><span class="p">(</span><span class="n">partition</span><span class="p">):</span> + + <span class="n">schema</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">partition</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">parse_partition_name</span><span class="p">(</span><span class="n">partition</span><span class="p">)</span> + + <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span> + <span class="s1">'Poking for {schema}.{table}/{partition}'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">())</span> + <span class="p">)</span> + <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">check_for_named_partition</span><span class="p">(</span> + <span class="n">schema</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">partition</span><span class="p">)</span> + + <span class="k">while</span> <span class="bp">self</span><span class="o">.</span><span class="n">next_poke_idx</span> <span class="o"><</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">partition_names</span><span class="p">):</span> + <span class="k">if</span> <span class="n">poke_partition</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">partition_names</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">next_poke_idx</span><span class="p">]):</span> + <span class="bp">self</span><span class="o">.</span><span class="n">next_poke_idx</span> <span class="o">+=</span> <span class="mi">1</span> + <span class="k">else</span><span class="p">:</span> + <span class="k">return</span> <span class="bp">False</span> + + <span class="k">return</span> <span class="bp">True</span> + + <div class="viewcode-block" id="HivePartitionSensor"><a class="viewcode-back" href="../code.html#airflow.operators.HivePartitionSensor">[docs]</a><span class="k">class</span> <span class="nc">HivePartitionSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span> <span class="sd">"""</span> -<span class="sd"> Waits for a partition to show up in Hive</span> +<span class="sd"> Waits for a partition to show up in Hive.</span> + +<span class="sd"> Note: Because @partition supports general logical operators, it</span> +<span class="sd"> can be inefficient. Consider using NamedHivePartitionSensor instead if</span> +<span class="sd"> you don't need the full flexibility of HivePartitionSensor.</span> <span class="sd"> :param table: The name of the table to wait for, supports the dot</span> <span class="sd"> notation (my_database.my_table)</span> <span class="sd"> :type table: string</span> <span class="sd"> :param partition: The partition clause to wait for. This is passed as</span> -<span class="sd"> is to the Metastore Thrift client "get_partitions_by_filter" method,</span> +<span class="sd"> is to the metastore Thrift client "get_partitions_by_filter" method,</span> <span class="sd"> and apparently supports SQL like notation as in `ds='2015-01-01'</span> <span class="sd"> AND type='value'` and > < sings as in "ds>=2015-01-01"</span> <span class="sd"> :type partition: string</span> @@ -388,7 +494,7 @@ <span class="s1">'Poking for table {self.schema}.{self.table}, '</span> <span class="s1">'partition {self.partition}'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span> <span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s1">'hook'</span><span class="p">):</span> - <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">HiveMetastoreHook</span><span class="p">(</span> + <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">airflow</span><span class="o">.</span><span class="n">hooks</span><span class="o">.</span><span class="n">hive_hooks</span><span class="o">.</span><span class="n">HiveMetastoreHook</span><span class="p">(</span> <span class="n">metastore_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">metastore_conn_id</span><span class="p">)</span> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">check_for_partition</span><span class="p">(</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">table</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">partition</span><span class="p">)</span></div> @@ -411,7 +517,8 @@ <span class="bp">self</span><span class="o">.</span><span class="n">hdfs_conn_id</span> <span class="o">=</span> <span class="n">hdfs_conn_id</span> <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> - <span class="n">sb</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">HDFSHook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">hdfs_conn_id</span><span class="p">)</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span> + <span class="kn">import</span> <span class="nn">airflow.hooks.hdfs_hook</span> + <span class="n">sb</span> <span class="o">=</span> <span class="n">airflow</span><span class="o">.</span><span class="n">hooks</span><span class="o">.</span><span class="n">hdfs_hook</span><span class="o">.</span><span class="n">HDFSHook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">hdfs_conn_id</span><span class="p">)</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s2">"snakebite"</span><span class="p">)</span><span class="o">.</span><span class="n">setLevel</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">WARNING</span><span class="p">)</span> <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span> <span class="s1">'Poking for file {self.filepath} '</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span> @@ -439,7 +546,7 @@ <span class="bp">self</span><span class="o">.</span><span class="n">webhdfs_conn_id</span> <span class="o">=</span> <span class="n">webhdfs_conn_id</span> <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> - <span class="n">c</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">WebHDFSHook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">webhdfs_conn_id</span><span class="p">)</span> + <span class="n">c</span> <span class="o">=</span> <span class="n">airflow</span><span class="o">.</span><span class="n">hooks</span><span class="o">.</span><span class="n">webhdfs_hook</span><span class="o">.</span><span class="n">WebHDFSHook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">webhdfs_conn_id</span><span class="p">)</span> <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span> <span class="s1">'Poking for file {self.filepath} '</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span> <span class="k">return</span> <span class="n">c</span><span class="o">.</span><span class="n">check_for_path</span><span class="p">(</span><span class="n">hdfs_path</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">filepath</span><span class="p">)</span></div> @@ -495,7 +602,8 @@ <span class="n">session</span><span class="o">.</span><span class="n">close</span><span class="p">()</span> <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> - <span class="n">hook</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">S3Hook</span><span class="p">(</span><span class="n">s3_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">s3_conn_id</span><span class="p">)</span> + <span class="kn">import</span> <span class="nn">airflow.hooks.S3_hook</span> + <span class="n">hook</span> <span class="o">=</span> <span class="n">airflow</span><span class="o">.</span><span class="n">hooks</span><span class="o">.</span><span class="n">S3_hook</span><span class="o">.</span><span class="n">S3Hook</span><span class="p">(</span><span class="n">s3_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">s3_conn_id</span><span class="p">)</span> <span class="n">full_url</span> <span class="o">=</span> <span class="s2">"s3://"</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">bucket_name</span> <span class="o">+</span> <span class="s2">"/"</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">bucket_key</span> <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Poking for key : {full_url}'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">wildcard_match</span><span class="p">:</span> @@ -547,7 +655,8 @@ <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Poking for prefix : {self.prefix}</span><span class="se">\n</span><span class="s1">'</span> <span class="s1">'in bucket s3://{self.bucket_name}'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span> - <span class="n">hook</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">S3Hook</span><span class="p">(</span><span class="n">s3_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">s3_conn_id</span><span class="p">)</span> + <span class="kn">import</span> <span class="nn">airflow.hooks.S3_hook</span> + <span class="n">hook</span> <span class="o">=</span> <span class="n">airflow</span><span class="o">.</span><span class="n">hooks</span><span class="o">.</span><span class="n">S3_hook</span><span class="o">.</span><span class="n">S3Hook</span><span class="p">(</span><span class="n">s3_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">s3_conn_id</span><span class="p">)</span> <span class="k">return</span> <span class="n">hook</span><span class="o">.</span><span class="n">check_for_prefix</span><span class="p">(</span> <span class="n">prefix</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">prefix</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delimiter</span><span class="p">,</span> @@ -639,7 +748,7 @@ <span class="bp">self</span><span class="o">.</span><span class="n">extra_options</span> <span class="o">=</span> <span class="n">extra_options</span> <span class="ow">or</span> <span class="p">{}</span> <span class="bp">self</span><span class="o">.</span><span class="n">response_check</span> <span class="o">=</span> <span class="n">response_check</span> - <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">HttpHook</span><span class="p">(</span><span class="n">method</span><span class="o">=</span><span class="s1">'GET'</span><span class="p">,</span> <span class="n">http_conn_id</span><span class="o">=</span><span class="n">http_conn_id</span><span class="p">)</span> + <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">http_hook</span><span class="o">.</span><span class="n">HttpHook</span><span class="p">(</span><span class="n">method</span><span class="o">=</span><span class="s1">'GET'</span><span class="p">,</span> <span class="n">http_conn_id</span><span class="o">=</span><span class="n">http_conn_id</span><span class="p">)</span> <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Poking: '</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">endpoint</span><span class="p">)</span> @@ -652,7 +761,7 @@ <span class="c1"># run content check on response</span> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">response_check</span><span class="p">(</span><span class="n">response</span><span class="p">)</span> <span class="k">except</span> <span class="n">AirflowException</span> <span class="k">as</span> <span class="n">ae</span><span class="p">:</span> - <span class="k">if</span> <span class="n">ae</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">"404"</span><span class="p">):</span> + <span class="k">if</span> <span class="nb">str</span><span class="p">(</span><span class="n">ae</span><span class="p">)</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">"404"</span><span class="p">):</span> <span class="k">return</span> <span class="bp">False</span> <span class="k">raise</span> <span class="n">ae</span>