kaknikhil commented on a change in pull request #443: DL: Add training for
multiple models
URL: https://github.com/apache/madlib/pull/443#discussion_r325823937
##########
File path: src/ports/postgres/modules/deep_learning/madlib_keras.py_in
##########
@@ -591,43 +622,55 @@ def get_loss_metric_from_keras_eval(schema_madlib,
table, compile_params,
ARRAY{seg_ids},
ARRAY{images_per_seg},
{gpus_per_host},
- {segments_per_host}
+ {segments_per_host},
+ {is_final_iteration}
)) as loss_metric
from {table}
""".format(**locals()), ["bytea"])
res = plpy.execute(evaluate_query, [serialized_weights])
loss_metric = res[0]['loss_metric']
return loss_metric
+
def internal_keras_eval_transition(state, dependent_var, independent_var,
dependent_var_shape, independent_var_shape,
model_architecture, serialized_weights,
compile_params,
current_seg_id, seg_ids, images_per_seg,
- gpus_per_host, segments_per_host, **kwargs):
+ gpus_per_host, segments_per_host,
+ is_final_iteration, **kwargs):
SD = kwargs['SD']
device_name = get_device_name_and_set_cuda_env(gpus_per_host,
current_seg_id)
-
agg_loss, agg_metric, agg_image_count = state
- if not agg_image_count:
- set_keras_session(device_name, gpus_per_host, segments_per_host)
- model = model_from_json(model_architecture)
- compile_and_set_weights(model, compile_params, device_name,
- serialized_weights)
-
- SD['segment_model'] = model
+ # User called evaluate will always set is_final_iteration to true.
+ # If is_final_iteration is false, that means the fit already created a
session and a graph
+ # Otherwise, we may (last iteration of fit) or may not (user evaluate call)
+ # have a session.
+
+ if SD_NAMES.SESS in SD:
+ if SD_NAMES.SEGMENT_MODEL not in SD:
+ plpy.error("Session and model should exist in SD after the first
row"
+ "of the first iteration")
+ sess = SD[SD_NAMES.SESS]
+ segment_model = SD[SD_NAMES.SEGMENT_MODEL]
+ K.set_session(sess)
+ else:
+ sess = get_keras_session(device_name, gpus_per_host, segments_per_host)
+ SD[SD_NAMES.SESS] = sess
+ K.set_session(sess)
+ segment_model = init_model(model_architecture, compile_params)
+ SD[SD_NAMES.SEGMENT_MODEL] = segment_model
+ if not agg_image_count: # first row each iteration
Review comment:
Again creating a function/variable will help with readability and
maintainability
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
