[
https://issues.apache.org/jira/browse/MADLIB-1348?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16852301#comment-16852301
]
Frank McQuillan commented on MADLIB-1348:
-----------------------------------------
https://github.com/apache/madlib/pull/399
LGTM
Tested from a previous madlib run:
(1)
load arch table with arch but no weights
{code}
DROP TABLE IF EXISTS model_arch_library;
SELECT madlib.load_keras_model('model_arch_library', -- Output table,
$$
{"class_name": "Sequential", "keras_version": "2.1.6", "config":
[{"class_name": "Dense", "config": {"kernel_initializer": {"class_name":
"VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed":
null, "mode": "fan_avg"}}, "name": "dense_1", "kernel_constraint": null,
"bias_regularizer": null, "bias_constraint": null, "dtype": "float32",
"activation": "relu", "trainable": true, "kernel_regularizer": null,
"bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10,
"batch_input_shape": [null, 4], "use_bias": true, "activity_regularizer":
null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name":
"VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed":
null, "mode": "fan_avg"}}, "name": "dense_2", "kernel_constraint": null,
"bias_regularizer": null, "bias_constraint": null, "activation": "relu",
"trainable": true, "kernel_regularizer": null, "bias_initializer":
{"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true,
"activity_regularizer": null}}, {"class_name": "Dense", "config":
{"kernel_initializer": {"class_name": "VarianceScaling", "config":
{"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}},
"name": "dense_3", "kernel_constraint": null, "bias_regularizer": null,
"bias_constraint": null, "activation": "softmax", "trainable": true,
"kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros",
"config": {}}, "units": 3, "use_bias": true, "activity_regularizer": null}}],
"backend": "tensorflow"}
$$
::json, -- JSON blob
NULL, -- weights
'Sophie', -- name
'A pretty good model'
);
SELECT * FROM model_arch_library;
model_id | model_arch
| model_weights | name | description |
__internal_madlib_id_
_
----------+-----------------------------------------------------------------------------+---------------+--------+---------------------+---------------------------------
-------------
1 |
| | Sophie | A pretty good model |
__madlib_temp_72007016_155917654
2_57795859__
: {"class_name": "Sequential", "keras_version": "2.1.6",
: "config": [{"class_name": "Dense", "config": {"kernel_initializer":
: {"class_name": "VarianceScaling", "config": {"distribution":
"uniform",
: "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_1",
: "kernel_constraint": null, "bias_regularizer": null,
"bias_constraint":
: null, "dtype": "float32", "activation": "linear", "trainable": true,
: "kernel_regularizer": null, "bias_initializer": {"class_name":
"Zeros",
: "config": {}}, "units": 2, "batch_input_shape": [null, 3],
: "use_bias": true, "activity_regularizer": null}}, {"class_name":
"Dense",
: "config": {"kernel_initializer": {"class_name": "VarianceScaling",
: "config": {"distribution": "uniform", "scale": 1.0, "seed": null,
: "mode": "fan_avg"}}, "name": "new_dense", "kernel_constraint": null,
: "bias_regularizer": null, "bias_constraint": null, "activation":
"linear",
: "trainable": true, "kernel_regularizer": null,
: "bias_initializer": {"class_name": "Zeros", "config": {}},
: "units": 10, "use_bias": true, "activity_regularizer": null}}],
: "backend": "tensorflow"}
:
(1 row)
{code}
OK
(2)
add another arch to arch table but no weights
{code}
SELECT madlib.load_keras_model('model_arch_library', -- Output table,
$$
{"class_name": "Sequential", "keras_version": "2.1.6", "config":
[{"class_name": "Dense", "config": {"kernel_initializer": {"class_name":
"VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed":
null, "mode": "fan_avg"}}, "name": "dense_1", "kernel_constraint": null,
"bias_regularizer": null, "bias_constraint": null, "dtype": "float32",
"activation": "relu", "trainable": true, "kernel_regularizer": null,
"bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10,
"batch_input_shape": [null, 4], "use_bias": true, "activity_regularizer":
null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name":
"VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed":
null, "mode": "fan_avg"}}, "name": "dense_2", "kernel_constraint": null,
"bias_regularizer": null, "bias_constraint": null, "activation": "relu",
"trainable": true, "kernel_regularizer": null, "bias_initializer":
{"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true,
"activity_regularizer": null}}, {"class_name": "Dense", "config":
{"kernel_initializer": {"class_name": "VarianceScaling", "config":
{"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}},
"name": "dense_3", "kernel_constraint": null, "bias_regularizer": null,
"bias_constraint": null, "activation": "softmax", "trainable": true,
"kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros",
"config": {}}, "units": 3, "use_bias": true, "activity_regularizer": null}}],
"backend": "tensorflow"}
$$
::json, -- JSON blob
NULL, -- weights
'Maria', -- name
'Another pretty good model'
);
SELECT * FROM model_arch_library;
model_id | model_arch
| model_weights | name | description |
__internal_madl
ib_id__
----------+-----------------------------------------------------------------------------+---------------+--------+---------------------------+---------------------------
-------------------
1 |
| | Sophie | A pretty good model |
__madlib_temp_72007016_155
9176542_57795859__
: {"class_name": "Sequential", "keras_version": "2.1.6",
: "config": [{"class_name": "Dense", "config": {"kernel_initializer":
: {"class_name": "VarianceScaling", "config": {"distribution":
"uniform",
: "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_1",
: "kernel_constraint": null, "bias_regularizer": null,
"bias_constraint":
: null, "dtype": "float32", "activation": "linear", "trainable": true,
: "kernel_regularizer": null, "bias_initializer": {"class_name":
"Zeros",
: "config": {}}, "units": 2, "batch_input_shape": [null, 3],
: "use_bias": true, "activity_regularizer": null}}, {"class_name":
"Dense",
: "config": {"kernel_initializer": {"class_name": "VarianceScaling",
: "config": {"distribution": "uniform", "scale": 1.0, "seed": null,
: "mode": "fan_avg"}}, "name": "new_dense", "kernel_constraint": null,
: "bias_regularizer": null, "bias_constraint": null, "activation":
"linear",
: "trainable": true, "kernel_regularizer": null,
: "bias_initializer": {"class_name": "Zeros", "config": {}},
: "units": 10, "use_bias": true, "activity_regularizer": null}}],
: "backend": "tensorflow"}
:
2 |
| | Maria | Another pretty good model |
__madlib_temp_83687777_155
9177024_9294215__
: {"class_name": "Sequential", "keras_version": "2.1.6",
: "config": [{"class_name": "Dense", "config": {"kernel_initializer":
: {"class_name": "VarianceScaling", "config": {"distribution":
"uniform",
: "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_1",
: "kernel_constraint": null, "bias_regularizer": null,
"bias_constraint":
: null, "dtype": "float32", "activation": "linear", "trainable": true,
: "kernel_regularizer": null, "bias_initializer": {"class_name":
"Zeros",
: "config": {}}, "units": 2, "batch_input_shape": [null, 3],
: "use_bias": true, "activity_regularizer": null}}, {"class_name":
"Dense",
: "config": {"kernel_initializer": {"class_name": "VarianceScaling",
: "config": {"distribution": "uniform", "scale": 1.0, "seed": null,
: "mode": "fan_avg"}}, "name": "new_dense", "kernel_constraint": null,
: "bias_regularizer": null, "bias_constraint": null, "activation":
"linear",
: "trainable": true, "kernel_regularizer": null,
: "bias_initializer": {"class_name": "Zeros", "config": {}},
: "units": 10, "use_bias": true, "activity_regularizer": null}}],
: "backend": "tensorflow"}
(2 rows)
{code}
OK
(3)
train a model and copy over weights from madlib output table to model arch table
(a)
run 1
Initial training
{code}
DROP TABLE IF EXISTS iris_model, iris_model_summary;
SELECT madlib.madlib_keras_fit('iris_train_packed', -- source table
'iris_model', -- model output table
'model_arch_library', -- model arch table
1, -- model arch id
$$ loss='categorical_crossentropy',
optimizer='adam', metrics=['accuracy'] $$, -- compile_params
$$ batch_size=5, epochs=3 $$, -- fit_params
5, -- num_iterations
0, -- GPUs per host
NULL, -- validation dataset
1, -- metrics compute
frequency
'Sophie', -- name
'Simple MLP for iris dataset' -- description
);
INFO: Training set loss after iteration 1: 0.826811790466.
INFO: Training set loss after iteration 2: 0.721804022789.
INFO: Training set loss after iteration 3: 0.644849896431.
INFO: Training set loss after iteration 4: 0.585302889347.
INFO: Training set loss after iteration 5: 0.540865719318.
INFO: Training set metric after iteration 1: 0.666666686535.
INFO: Training set metric after iteration 2: 0.683333337307.
INFO: Training set metric after iteration 3: 0.683333337307.
INFO: Training set metric after iteration 4: 0.741666674614.
INFO: Training set metric after iteration 5: 0.741666674614.
SELECT * FROM iris_model_summary;
-[ RECORD 1
]-------------+--------------------------------------------------------------------------------------------
source_table | iris_train_packed
model | iris_model
dependent_varname | class_text
independent_varname | attributes
model_arch_table | model_arch_library
model_arch_id | 1
compile_params | loss='categorical_crossentropy', optimizer='adam',
metrics=['accuracy']
fit_params | batch_size=5, epochs=3
num_iterations | 5
validation_table |
metrics_compute_frequency | 1
name | Sophie
description | Simple MLP for iris dataset
model_type | madlib_keras
model_size | 0.7900390625
start_training_time | 2019-05-30 18:14:02.974886
end_training_time | 2019-05-30 18:14:13.137785
metrics_elapsed_time |
{3.07278394699097,4.35965490341187,5.76982879638672,7.13418292999268,10.162889957428}
madlib_version | 1.16-dev
num_classes | 3
class_values | {Iris-setosa,Iris-versicolor,Iris-virginica}
dependent_vartype | character varying
normalizing_const | 1
metrics_type | {accuracy}
training_metrics_final | 0.741666674614
training_loss_final | 0.540865719318
training_metrics |
{0.666666686534882,0.683333337306976,0.683333337306976,0.741666674613953,0.741666674613953}
training_loss |
{0.826811790466309,0.721804022789001,0.644849896430969,0.585302889347076,0.54086571931839}
validation_metrics_final |
validation_loss_final |
validation_metrics |
validation_loss |
metrics_iters | {1,2,3,4,5}
{code}
Now copy weights from madlib output table to model arch table:
{code}
UPDATE model_arch_library SET model_weights = model_data FROM iris_model WHERE
model_id = 1;
{code}
(b)
run 2
Re-train to use weights from previous run (kind of like warm start):
{code}
DROP TABLE IF EXISTS iris_model, iris_model_summary;
SELECT madlib.madlib_keras_fit('iris_train_packed', -- source table
'iris_model', -- model output table
'model_arch_library', -- model arch table
1, -- model arch id
$$ loss='categorical_crossentropy',
optimizer='adam', metrics=['accuracy'] $$, -- compile_params
$$ batch_size=5, epochs=3 $$, -- fit_params
5, -- num_iterations
0, -- GPUs per host
NULL, -- validation dataset
1, -- metrics compute
frequency
'Sophie', -- name
'Simple MLP for iris dataset' -- description
);
INFO: Training set loss after iteration 1: 0.503705084324.
INFO: Training set loss after iteration 2: 0.470966666937.
INFO: Training set loss after iteration 3: 0.445806771517.
INFO: Training set loss after iteration 4: 0.419732987881.
INFO: Training set loss after iteration 5: 0.396750748158.
INFO: Training set metric after iteration 1: 0.791666686535.
INFO: Training set metric after iteration 2: 0.833333313465.
INFO: Training set metric after iteration 3: 0.875.
INFO: Training set metric after iteration 4: 0.899999976158.
INFO: Training set metric after iteration 5: 0.916666686535.
SELECT * FROM iris_model_summary;
-[ RECORD 1
]-------------+------------------------------------------------------------------------------------------
source_table | iris_train_packed
model | iris_model
dependent_varname | class_text
independent_varname | attributes
model_arch_table | model_arch_library
model_arch_id | 1
compile_params | loss='categorical_crossentropy', optimizer='adam',
metrics=['accuracy']
fit_params | batch_size=5, epochs=3
num_iterations | 5
validation_table |
metrics_compute_frequency | 1
name | Sophie
description | Simple MLP for iris dataset
model_type | madlib_keras
model_size | 0.7900390625
start_training_time | 2019-05-30 18:23:08.888368
end_training_time | 2019-05-30 18:23:18.453456
metrics_elapsed_time |
{3.46771907806396,4.58241200447083,5.6557719707489,6.93159699440002,9.56508588790894}
madlib_version | 1.16-dev
num_classes | 3
class_values | {Iris-setosa,Iris-versicolor,Iris-virginica}
dependent_vartype | character varying
normalizing_const | 1
metrics_type | {accuracy}
training_metrics_final | 0.916666686535
training_loss_final | 0.396750748158
training_metrics |
{0.791666686534882,0.833333313465118,0.875,0.899999976158142,0.916666686534882}
training_loss |
{0.503705084323883,0.470966666936874,0.4458067715168,0.419732987880707,0.396750748157501}
validation_metrics_final |
validation_loss_final |
validation_metrics |
validation_loss |
metrics_iters | {1,2,3,4,5}
{code}
Now copy weights from madlib output table to model arch table:
{code}
UPDATE model_arch_library SET model_weights = model_data FROM iris_model WHERE
model_id = 1;
{code}
(c)
Run 3
Train again
{code}
DROP TABLE IF EXISTS iris_model, iris_model_summary;
SELECT madlib.madlib_keras_fit('iris_train_packed', -- source table
'iris_model', -- model output table
'model_arch_library', -- model arch table
1, -- model arch id
$$ loss='categorical_crossentropy',
optimizer='adam', metrics=['accuracy'] $$, -- compile_params
$$ batch_size=5, epochs=3 $$, -- fit_params
5, -- num_iterations
0, -- GPUs per host
NULL, -- validation dataset
1, -- metrics compute
frequency
'Sophie', -- name
'Simple MLP for iris dataset' -- description
);
INFO: Training set loss after iteration 1: 0.376342386007.
INFO: Training set loss after iteration 2: 0.356429725885.
INFO: Training set loss after iteration 3: 0.33902746439.
INFO: Training set loss after iteration 4: 0.323304623365.
INFO: Training set loss after iteration 5: 0.308936685324.
INFO: Training set metric after iteration 1: 0.949999988079.
INFO: Training set metric after iteration 2: 0.958333313465.
INFO: Training set metric after iteration 3: 0.949999988079.
INFO: Training set metric after iteration 4: 0.949999988079.
INFO: Training set metric after iteration 5: 0.958333313465.
SELECT * FROM iris_model_summary;
-[ RECORD 1
]-------------+--------------------------------------------------------------------------------------------
source_table | iris_train_packed
model | iris_model
dependent_varname | class_text
independent_varname | attributes
model_arch_table | model_arch_library
model_arch_id | 1
compile_params | loss='categorical_crossentropy', optimizer='adam',
metrics=['accuracy']
fit_params | batch_size=5, epochs=3
num_iterations | 5
validation_table |
metrics_compute_frequency | 1
name | Sophie
description | Simple MLP for iris dataset
model_type | madlib_keras
model_size | 0.7900390625
start_training_time | 2019-05-30 18:26:38.982926
end_training_time | 2019-05-30 18:26:48.017231
metrics_elapsed_time |
{3.31328082084656,5.12690782546997,6.50730085372925,7.78843688964844,9.03430604934692}
madlib_version | 1.16-dev
num_classes | 3
class_values | {Iris-setosa,Iris-versicolor,Iris-virginica}
dependent_vartype | character varying
normalizing_const | 1
metrics_type | {accuracy}
training_metrics_final | 0.958333313465
training_loss_final | 0.308936685324
training_metrics |
{0.949999988079071,0.958333313465118,0.949999988079071,0.949999988079071,0.958333313465118}
training_loss |
{0.376342386007309,0.356429725885391,0.339027464389801,0.323304623365402,0.308936685323715}
validation_metrics_final |
validation_loss_final |
validation_metrics |
validation_loss |
metrics_iters | {1,2,3,4,5}
{code}
Put them closer together to see if they pick up from where they left off:
(a)
INFO: Training set loss after iteration 1: 0.826811790466.
INFO: Training set loss after iteration 2: 0.721804022789.
INFO: Training set loss after iteration 3: 0.644849896431.
INFO: Training set loss after iteration 4: 0.585302889347.
INFO: Training set loss after iteration 5: 0.540865719318.
INFO: Training set metric after iteration 1: 0.666666686535.
INFO: Training set metric after iteration 2: 0.683333337307.
INFO: Training set metric after iteration 3: 0.683333337307.
INFO: Training set metric after iteration 4: 0.741666674614.
INFO: Training set metric after iteration 5: 0.741666674614.
(b)
INFO: Training set loss after iteration 1: 0.503705084324.
INFO: Training set loss after iteration 2: 0.470966666937.
INFO: Training set loss after iteration 3: 0.445806771517.
INFO: Training set loss after iteration 4: 0.419732987881.
INFO: Training set loss after iteration 5: 0.396750748158.
INFO: Training set metric after iteration 1: 0.791666686535.
INFO: Training set metric after iteration 2: 0.833333313465.
INFO: Training set metric after iteration 3: 0.875.
INFO: Training set metric after iteration 4: 0.899999976158.
INFO: Training set metric after iteration 5: 0.916666686535.
(c)
INFO: Training set loss after iteration 1: 0.376342386007.
INFO: Training set loss after iteration 2: 0.356429725885.
INFO: Training set loss after iteration 3: 0.33902746439.
INFO: Training set loss after iteration 4: 0.323304623365.
INFO: Training set loss after iteration 5: 0.308936685324.
INFO: Training set metric after iteration 1: 0.949999988079.
INFO: Training set metric after iteration 2: 0.958333313465.
INFO: Training set metric after iteration 3: 0.949999988079.
INFO: Training set metric after iteration 4: 0.949999988079.
INFO: Training set metric after iteration 5: 0.958333313465.
OK
(4)
model id does not exist
{code}
DROP TABLE IF EXISTS iris_model, iris_model_summary;
SELECT madlib.madlib_keras_fit('iris_train_packed', -- source table
'iris_model', -- model output table
'model_arch_library', -- model arch table
3, -- model arch id
$$ loss='categorical_crossentropy',
optimizer='adam', metrics=['accuracy'] $$, -- compile_params
$$ batch_size=5, epochs=3 $$, -- fit_params
5, -- num_iterations
0, -- GPUs per host
NULL, -- validation dataset
1, -- metrics compute
frequency
'Sophie', -- name
'Simple MLP for iris dataset' -- description
);
ERROR: plpy.Error: no model arch found in table model_arch_library with id 3
(plpython.c:5038)
CONTEXT: Traceback (most recent call last):
PL/Python function "madlib_keras_fit", line 21, in <module>
madlib_keras.fit(**globals())
PL/Python function "madlib_keras_fit", line 42, in wrapper
PL/Python function "madlib_keras_fit", line 105, in fit
PL/Python function "madlib_keras_fit"
{code}
OK
> Weight initialization/transfer learning madlib_keras_fit()
> ----------------------------------------------------------
>
> Key: MADLIB-1348
> URL: https://issues.apache.org/jira/browse/MADLIB-1348
> Project: Apache MADlib
> Issue Type: Improvement
> Components: Deep Learning
> Reporter: Nikhil
> Priority: Major
> Fix For: v1.16
>
>
> Context
> Many deep neural nets are not trained from scratch, but rather initialized
> from weights generated by training related data sets using the same model
> architecture (particularly true for CNN).
> Story
> As a data scientist,
> I want to start training a model based on weights that I have,
> so that I don't have to start from scratch.
> * e.g, use weights from one dataset (e.g., VGG-16 on Imagenet) as starting
> point to training VGG-16 model on my data.
> Details
> 1. add support for optional param to load weights
> 2. add “name” , “description” to model arch table
> Interface
> {code}
> load_keras_model(
> keras_model_arch_table,
> model_arch,
> model_weights, -- OPTIONAL
> name, -- OPTIONAL
> description -- OPTIONAL
> )
> {code}
> Acceptance
> 1. Take a trained model with a known accuracy and load into the model arch
> table (can be simple).
> 2. Use it as input to training with fit() on the same data set it was trained
> on. Since it has already converged, it should show the same accuracy on the
> 1st or 2nd iteration as before.
> 3. Test load from keras library [2]. Pick any model, get the weights and
> test load into model arch table. Test for 1 or 2 iterations on any dataset
> to check that it runs.
> Reference
> [1] VGG16 and other pre-trained weights for Imagenet are built into Keras
> https://keras.io/getting-started/faq/#how-can-i-use-pre-trained-models-in-keras
> [2] http://cs231n.github.io/transfer-learning/
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
