This is an automated email from the ASF dual-hosted git repository.
domino pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/madlib.git
commit 5ddec47242415b32f8b26849c73e70f3a4a20448
Author: Nikhil Kak <n...@pivotal.io>
AuthorDate: Tue Jan 14 11:12:33 2020 -0800
DL: Drop leftover tables
JIRA: MADLIB-1404
While calling the function `get_accessible_gpus_for_seg`, we create an
intermediate gpu table by calling the madlib function gpu_configuration.
This table was not getting dropped at the end.
This commit drops the intermediate gpu table created in
get_accessible_gpus_for_seg.
Co-authored-by: Ekta Khanna <ekha...@pivotal.io>
---
.../modules/deep_learning/input_data_preprocessor.py_in | 4 +++-
.../postgres/modules/deep_learning/madlib_keras_helper.py_in | 10 +++++-----
.../deep_learning/test/unit_tests/test_madlib_keras.py_in | 4 +++-
3 files changed, 11 insertions(+), 7 deletions(-)
diff --git
a/src/ports/postgres/modules/deep_learning/input_data_preprocessor.py_in
b/src/ports/postgres/modules/deep_learning/input_data_preprocessor.py_in
index 3a5f118..605d439 100644
--- a/src/ports/postgres/modules/deep_learning/input_data_preprocessor.py_in
+++ b/src/ports/postgres/modules/deep_learning/input_data_preprocessor.py_in
@@ -328,6 +328,8 @@ class InputDataPreprocessorDL(object):
all_segments = False
if self.distribution_rules == 'gpu_segments':
+ #TODO can we reuse the function `get_accessible_gpus_for_seg` from
+ # madlib_keras_helper
gpu_info_table = unique_string(desp='gpu_info')
plpy.execute("""
SELECT
{self.schema_madlib}.gpu_configuration('{gpu_info_table}')
@@ -339,6 +341,7 @@ class InputDataPreprocessorDL(object):
gpu_query_result = plpy.execute(gpu_query)[0]['gpu_config']
if not gpu_query_result:
plpy.error("{self.module_name}: No GPUs configured on
hosts.".format(self=self))
+ plpy.execute("DROP TABLE IF EXISTS {0}".format(gpu_info_table))
gpu_config_hostnames = "ARRAY{0}".format(gpu_query_result)
# find hosts with gpus
@@ -351,7 +354,6 @@ class InputDataPreprocessorDL(object):
AND hostname=ANY({gpu_config_hostnames})
""".format(**locals())
segment_ids_result = plpy.execute(get_segment_query)[0]
- plpy.execute("DROP TABLE IF EXISTS {0}".format(gpu_info_table))
self.gpu_config =
"ARRAY{0}".format(sorted(segment_ids_result['segment_ids']))
self.distribution_rules =
"ARRAY{0}".format(sorted(segment_ids_result['dbid']))
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_helper.py_in
b/src/ports/postgres/modules/deep_learning/madlib_keras_helper.py_in
index 6e006d5..5be078b 100644
--- a/src/ports/postgres/modules/deep_learning/madlib_keras_helper.py_in
+++ b/src/ports/postgres/modules/deep_learning/madlib_keras_helper.py_in
@@ -269,24 +269,24 @@ def create_summary_view(module_name, model_table,
mst_key):
""".format(**locals()))
return tmp_view_summary
-def get_accessible_gpus_for_seg(schema_madlib, segments_per_host, module_name):
+def get_accessible_gpus_for_seg(schema_madlib, segments_per_host, module_name):
if is_platform_pg():
gpus = GPUInfoFunctions.get_gpu_info_from_tensorflow()
if not gpus:
plpy.error("{0} error: No GPUs configured on
host.".format(module_name))
return [len(gpus)]
else:
- gpu_table_name = unique_string(desp = 'gpu_table')
+ gpu_info_table = unique_string(desp = 'gpu_info')
gpu_table_query = """
- SELECT {schema_madlib}.gpu_configuration('{gpu_table_name}')
+ SELECT {schema_madlib}.gpu_configuration('{gpu_info_table}')
""".format(**locals())
plpy.execute(gpu_table_query)
gpu_query = """
- SELECT hostname, count(*) AS count FROM {gpu_table_name} GROUP BY
hostname
+ SELECT hostname, count(*) AS count FROM {gpu_info_table} GROUP BY
hostname
""".format(**locals())
gpu_query_result = plpy.execute(gpu_query)
-
+ plpy.execute("DROP TABLE IF EXISTS {0}".format(gpu_info_table))
if not gpu_query_result:
plpy.error("{0} error: No GPUs configured on
hosts.".format(module_name))
diff --git
a/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras.py_in
b/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras.py_in
index a8bb629..3714de5 100644
---
a/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras.py_in
+++
b/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras.py_in
@@ -1336,6 +1336,7 @@ class MadlibKerasHelperTestCase(unittest.TestCase):
self.plpy_mock_execute.side_effect = \
[ [],
[ {'hostname': 'mdw0', 'count' : 1}],
+ [],
[ {'hostname': 'mdw0', 'segment_id' : 0},
{'hostname': 'mdw1', 'segment_id' : 1},
{'hostname': 'mdw2', 'segment_id' : 2}
@@ -1351,6 +1352,7 @@ class MadlibKerasHelperTestCase(unittest.TestCase):
self.plpy_mock_execute.side_effect = \
[[],
[ {'hostname': 'mdw0', 'count' : 1}],
+ [],
[ {'hostname': 'mdw0', 'segment_id' : 0},
{'hostname': 'mdw0', 'segment_id' : 1},
{'hostname': 'mdw1', 'segment_id' : 2},
@@ -1364,7 +1366,7 @@ class MadlibKerasHelperTestCase(unittest.TestCase):
self.subject.is_platform_pg = Mock(return_value = False)
- self.plpy_mock_execute.side_effect = [[],[]]
+ self.plpy_mock_execute.side_effect = [[],[],[]]
with self.assertRaises(plpy.PLPYException) as error:
self.subject.get_accessible_gpus_for_seg('schema_madlib', 2, 'foo')
self.assertIn('no gpus configured on hosts',
str(error.exception).lower())
