fmcquillan99 commented on pull request #511:
URL: https://github.com/apache/madlib/pull/511#issuecomment-677953577
testing custom metrics
(1)
training 1 model at a time
custom function definition
```
# import database connector psycopg2 and create connection cursor
import psycopg2 as p2
conn = p2.connect('postgresql://gpadmin@localhost:8000/madlib')
cur = conn.cursor()
# import Dill and define function
import dill
def my_loss_fn(y_true, y_pred):
import keras.backend as K
loss=K.mean(K.sum(K.square(y_true-y_pred)))
return loss
pb_my_loss=dill.dumps(my_loss_fn)
def mean_pred(y_true, y_pred):
import keras.backend as K
return K.mean(y_pred)
pb_mean_pred=dill.dumps(mean_pred)
def rmse(y_true, y_pred):
import keras.backend as K
return K.sqrt(K.mean(K.square(y_pred - y_true), axis=-1))
pb_rmse=dill.dumps(rmse)
# call load function
cur.execute("DROP TABLE IF EXISTS test_custom_function_table")
cur.execute("SELECT
madlib.load_custom_function('test_custom_function_table', %s,'my_loss_fn', 'my
loss function')", [p2.Binary(pb_my_loss)])
cur.execute("SELECT
madlib.load_custom_function('test_custom_function_table', %s,'mean_pred',
'mean of y_pred')", [p2.Binary(pb_mean_pred)])
cur.execute("SELECT
madlib.load_custom_function('test_custom_function_table', %s,'rmse', 'root
mean square error')", [p2.Binary(pb_rmse)])
conn.commit()
```
fit 1 model
```
DROP TABLE IF EXISTS iris_model_cl, iris_model_cl_summary;
SELECT madlib.madlib_keras_fit('iris_train_packed', -- source table
'iris_model_cl', -- model output
table
'model_arch_library', -- model arch table
1, -- model arch id
$$ loss=my_loss_fn, optimizer='adam',
metrics=['rmse'] $$, -- compile_params
$$ batch_size=5, epochs=3 $$, -- fit_params
10, -- num_iteration
NULL, -- use_gpus,
NULL, -- validation_table,
NULL, -- metrics_compute_frequency,
NULL, -- warm_start,
NULL, -- name
NULL, -- description
'test_custom_function_table' -- object table
);
```
produces
```
INFO:
Time for training in iteration 1: 4.30907201767 sec
CONTEXT: PL/Python function "madlib_keras_fit"
INFO:
Time for training in iteration 2: 0.0564551353455 sec
CONTEXT: PL/Python function "madlib_keras_fit"
INFO:
Time for training in iteration 3: 0.0538399219513 sec
CONTEXT: PL/Python function "madlib_keras_fit"
INFO:
Time for training in iteration 4: 0.056825876236 sec
CONTEXT: PL/Python function "madlib_keras_fit"
INFO:
Time for training in iteration 5: 0.052964925766 sec
CONTEXT: PL/Python function "madlib_keras_fit"
INFO:
Time for training in iteration 6: 0.0547108650208 sec
CONTEXT: PL/Python function "madlib_keras_fit"
INFO:
Time for training in iteration 7: 0.0535190105438 sec
CONTEXT: PL/Python function "madlib_keras_fit"
INFO:
Time for training in iteration 8: 0.050479888916 sec
CONTEXT: PL/Python function "madlib_keras_fit"
INFO:
Time for training in iteration 9: 0.0498030185699 sec
CONTEXT: PL/Python function "madlib_keras_fit"
INFO:
Time for training in iteration 10: 0.0547761917114 sec
DETAIL:
Time for evaluating training dataset in iteration 10: 0.160271167755 sec
Training set metric after iteration 10: 0.294620931149
Training set loss after iteration 10: 9.13136100769
CONTEXT: PL/Python function "madlib_keras_fit"
madlib_keras_fit
------------------
(1 row)
Time: 6067.860 ms
```
OK
(2)
train multiple models
```
DROP TABLE IF EXISTS mst_table, mst_table_summary;
SELECT madlib.load_model_selection_table('model_arch_library', -- model
architecture table
'mst_table', -- model
selection table output
ARRAY[1,2], -- model
ids from model architecture table
ARRAY[ --
compile params
$$loss=my_loss_fn,optimizer='Adam()',metrics=['rmse']$$,
$$loss=my_loss_fn,
optimizer='Adam(lr=0.01)',metrics=['accuracy']$$,
$$loss=my_loss_fn,optimizer='Adam(lr=0.001)',metrics=['accuracy']$$
],
ARRAY[ -- fit
params
$$batch_size=4,epochs=1$$,
$$batch_size=8,epochs=1$$
],
'test_custom_function_table'
);
SELECT * FROM mst_table ORDER BY mst_key;
```
produces
```
mst_key | model_id | compile_params
| fit_params
---------+----------+-----------------------------------------------------------------+-----------------------
1 | 1 | loss=my_loss_fn,optimizer='Adam()',metrics=['rmse']
| batch_size=4,epochs=1
2 | 1 | loss=my_loss_fn,optimizer='Adam()',metrics=['rmse']
| batch_size=8,epochs=1
3 | 1 |
loss=my_loss_fn,optimizer='Adam(lr=0.001)',metrics=['accuracy'] |
batch_size=4,epochs=1
4 | 1 |
loss=my_loss_fn,optimizer='Adam(lr=0.001)',metrics=['accuracy'] |
batch_size=8,epochs=1
5 | 1 | loss=my_loss_fn,
optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=4,epochs=1
6 | 1 | loss=my_loss_fn,
optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=8,epochs=1
7 | 2 | loss=my_loss_fn,optimizer='Adam()',metrics=['rmse']
| batch_size=4,epochs=1
8 | 2 | loss=my_loss_fn,optimizer='Adam()',metrics=['rmse']
| batch_size=8,epochs=1
9 | 2 |
loss=my_loss_fn,optimizer='Adam(lr=0.001)',metrics=['accuracy'] |
batch_size=4,epochs=1
10 | 2 |
loss=my_loss_fn,optimizer='Adam(lr=0.001)',metrics=['accuracy'] |
batch_size=8,epochs=1
11 | 2 | loss=my_loss_fn,
optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=4,epochs=1
12 | 2 | loss=my_loss_fn,
optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=8,epochs=1
(12 rows)
```
```
DROP TABLE IF EXISTS iris_multi_model, iris_multi_model_summary,
iris_multi_model_info;
SELECT madlib.madlib_keras_fit_multiple_model('iris_train_packed', --
source_table
'iris_multi_model', --
model_output_table
'mst_table', --
model_selection_table
3, --
num_iterations
FALSE -- use
gpus
);
```
produces
```
INFO:
Time for training in iteration 1: 13.0232429504 sec
CONTEXT: PL/Python function "madlib_keras_fit_multiple_model"
INFO:
Time for training in iteration 2: 9.58937096596 sec
CONTEXT: PL/Python function "madlib_keras_fit_multiple_model"
INFO:
Time for training in iteration 3: 9.76873111725 sec
DETAIL:
Training set after iteration 3:
mst_key=2: metric=0.409701615572, loss=16.9745140076
mst_key=11: metric=0.716666638851, loss=8.25237369537
mst_key=10: metric=0.583333313465, loss=20.2344932556
mst_key=4: metric=0.425000011921, loss=23.4776382446
mst_key=3: metric=0.675000011921, loss=13.1436872482
mst_key=12: metric=0.925000011921, loss=4.12444734573
mst_key=9: metric=0.433333337307, loss=19.3265666962
mst_key=1: metric=0.458484977484, loss=19.2592735291
mst_key=8: metric=0.457415133715, loss=21.9753704071
mst_key=7: metric=0.410767167807, loss=15.9223165512
mst_key=5: metric=0.708333313465, loss=9.34871482849
mst_key=6: metric=0.675000011921, loss=9.48431587219
CONTEXT: PL/Python function "madlib_keras_fit_multiple_model"
madlib_keras_fit_multiple_model
---------------------------------
(1 row)
Time: 37485.717 ms
```
OK
(3)
info table missing loss type
current info table lists the metrics type:
```
madlib=# \d iris_multi_model_info
Table "public.iris_multi_model_info"
Column | Type | Modifiers
--------------------------+--------------------+-----------
mst_key | integer | not null
model_id | integer |
compile_params | text |
fit_params | text |
model_type | text |
model_size | double precision |
metrics_elapsed_time | double precision[] |
metrics_type | text[] |
training_metrics_final | double precision |
training_loss_final | double precision |
training_metrics | double precision[] |
training_loss | double precision[] |
validation_metrics_final | double precision |
validation_loss_final | double precision |
validation_metrics | double precision[] |
validation_loss | double precision[] |
```
but we should also put the loss type in:
```
madlib=# \d iris_multi_model_info
Table "public.iris_multi_model_info"
Column | Type | Modifiers
--------------------------+--------------------+-----------
mst_key | integer | not null
model_id | integer |
compile_params | text |
fit_params | text |
model_type | text |
model_size | double precision |
metrics_elapsed_time | double precision[] |
metrics_type | text[] |
loss_type | text | <--new
training_metrics_final | double precision |
training_loss_final | double precision |
training_metrics | double precision[] |
training_loss | double precision[] |
validation_metrics_final | double precision |
validation_loss_final | double precision |
validation_metrics | double precision[] |
validation_loss | double precision[] |
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
