Updated HiveServer2Hook.to_csv() to add fetch_size
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/6f7ea90a Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/6f7ea90a Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/6f7ea90a Branch: refs/heads/airbnb_rb1.7.1_4 Commit: 6f7ea90ae1f465d7ab1aa710ef05226a5226c862 Parents: 8bedf33 Author: Michael Musson <michael.mus...@airbnb.com> Authored: Thu May 19 16:31:27 2016 -0700 Committer: Dan Davydov <dan.davy...@airbnb.com> Committed: Mon May 23 17:00:54 2016 -0700 ---------------------------------------------------------------------- airflow/hooks/hive_hooks.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/6f7ea90a/airflow/hooks/hive_hooks.py ---------------------------------------------------------------------- diff --git a/airflow/hooks/hive_hooks.py b/airflow/hooks/hive_hooks.py index 6ea5ba7..d712e16 100644 --- a/airflow/hooks/hive_hooks.py +++ b/airflow/hooks/hive_hooks.py @@ -489,7 +489,8 @@ class HiveServer2Hook(BaseHook): schema='default', delimiter=',', lineterminator='\r\n', - output_header=True): + output_header=True, + fetch_size=1000): schema = schema or 'default' with self.get_conn() as conn: with conn.cursor() as cur: @@ -504,7 +505,7 @@ class HiveServer2Hook(BaseHook): for c in cur.description]) i = 0 while True: - rows = [row for row in cur.fetchmany() if row] + rows = [row for row in cur.fetchmany(fetch_size) if row] if not rows: break
