kaknikhil commented on a change in pull request #443: DL: Add training for
multiple models
URL: https://github.com/apache/madlib/pull/443#discussion_r325822746
##########
File path: src/ports/postgres/modules/deep_learning/madlib_keras.py_in
##########
@@ -591,43 +622,55 @@ def get_loss_metric_from_keras_eval(schema_madlib,
table, compile_params,
ARRAY{seg_ids},
ARRAY{images_per_seg},
{gpus_per_host},
- {segments_per_host}
+ {segments_per_host},
+ {is_final_iteration}
)) as loss_metric
from {table}
""".format(**locals()), ["bytea"])
res = plpy.execute(evaluate_query, [serialized_weights])
loss_metric = res[0]['loss_metric']
return loss_metric
+
def internal_keras_eval_transition(state, dependent_var, independent_var,
dependent_var_shape, independent_var_shape,
model_architecture, serialized_weights,
compile_params,
current_seg_id, seg_ids, images_per_seg,
- gpus_per_host, segments_per_host, **kwargs):
+ gpus_per_host, segments_per_host,
+ is_final_iteration, **kwargs):
SD = kwargs['SD']
device_name = get_device_name_and_set_cuda_env(gpus_per_host,
current_seg_id)
-
agg_loss, agg_metric, agg_image_count = state
- if not agg_image_count:
- set_keras_session(device_name, gpus_per_host, segments_per_host)
- model = model_from_json(model_architecture)
- compile_and_set_weights(model, compile_params, device_name,
- serialized_weights)
-
- SD['segment_model'] = model
+ # User called evaluate will always set is_final_iteration to true.
+ # If is_final_iteration is false, that means the fit already created a
session and a graph
+ # Otherwise, we may (last iteration of fit) or may not (user evaluate call)
+ # have a session.
+
+ if SD_NAMES.SESS in SD:
Review comment:
lines 650-662 are the exact same as `fit_transition`. We can easily refactor
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
