Github user edespino commented on a diff in the pull request: https://github.com/apache/incubator-madlib/pull/149#discussion_r127024441 --- Diff: src/ports/postgres/modules/convex/mlp.sql_in --- @@ -0,0 +1,736 @@ +/* ----------------------------------------------------------------------- *//** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * + * @file mlp.sql_in + * + * @brief SQL functions for multi-layer perceptron + * @date June 2012 + * + * + *//* ----------------------------------------------------------------------- */ + +m4_include(`SQLCommon.m4') + +/** +@addtogroup grp_mlp + +<div class="toc"><b>Contents</b><ul> +<li class="level1"><a href="#mlp_classification">Classification</a></li> +<li class="level1"><a href="#mlp_regression">Regression</a></li> +<li class="level1"><a href="#optimization_params">Optimization Parameters</a></li> +<li class="level1"><a href="#predict">Prediction Functions/a></li> +<li class="level1"><a href="#example">Examples</a></li> +<li class="level1"><a href="#background">Technical Background</a></li> +<li class="level1"><a href="#literature">Literature</a></li> +<li class="level1"><a href="#related">Related Topics</a></li> +</ul></div> + +Multi Layer Perceptron (MLP) is a model for regression and +classification + +Also called "vanilla neural networks", they consist of several +fully connected hidden layers with non-linear activation +functions. In the case of classification, the final layer of the +neural net has as many nodes as classes, and the output of the +neural net can be interpreted as the probability of a given input +feature belonging to a specific class. + + +@brief Solves classification and regression problems with several +fully connected layers and nonlinear activation functions. + +@anchor mlp_classification +@par Classification Training Function +The mlp classification training function has the following format: +<pre class="syntax"> +mlp_classification( + source_table, + output_table, + independent_varname, + dependent_varname, + hidden_layer_sizes, + optimizer_params, + activation + ) +</pre> +\b Arguments +<DL class="arglist"> + <DT>source_table</DT> + <DD>TEXT. Name of the table containing the training data.</DD> + + <DT>model_table</DT> + <DD>TEXT. Name of the output table containing the model. Details of the output + tables are provided below. + </DD> + + <DT>independent_varname</DT> + <DD>TEXT. Expression list to evaluate for the + independent variables. An intercept variable should not be included as part + of this expression. Please note that expression should be able to be cast + to DOUBLE PRECISION[]. + </DD> + + <DT>dependent_varname</DT> + <DD> TEXT. Name of the dependent variable column. For classification, supported types are: + text, varchar, character varying, char, character + integer, smallint, bigint, and boolean. </DD> + + <DT>hidden_layer_sizes</DT> + <DD>INTEGER[], default: ARRAY[]. + The number of neurons in each hidden layer. The length of this array will + determine the number of hidden layers. Empty for no hidden layers. + </DD> + + + <DT>optimizer_params (optional)</DT> + <DD>TEXT, default: NULL. + Parameters for optimization in a comma-separated string + of key-value pairs. See the description below for details. + </DD> + + <DT>activation</DT> + <DD>TEXT, default: 'sigmoid'. + Activation function. Currently three functions are supported: 'sigmoid' (default), + 'relu', and 'tanh'. The text can be any subset of the three + strings; for e.g., activation='s' will use the sigmoid activation. + </DD> +</DL> + +<b>Output tables</b> +<br> + The model table produced by mlp contains the following columns: + <table class="output"> + <tr> + <th>coeffs</th> + <td>FLOAT8[]. Flat array containing the weights of the neural net</td> + </tr> + <tr> + <th>n_iterations</th> + <td>INTEGER. Number of iterations completed by stochastic gradient descent + algorithm. The algorithm either converged in this number of iterations + or hit the maximum number specified in the optimization parameters. </td> + </tr> + <tr> + <th>loss</th> + <td>FLOAT8. The cross entropy over the training data. + See Technical Background section below for more details.</td> + </tr> + </table> + + +A summary table named \<model_table\>_summary is also created, which has the following columns: + <table class="output"> + <tr> + <th>source_table</th> + <td>The source table.</td> + </tr> + <tr> + <th>dependent_varname</th> + <td>The dependent variable.</td> + </tr> + <tr> + <th>independent_varname</th> + <td>The independent variables.</td> + </tr> + <tr> + <th>tolerance</th> + <td>The tolerance as given in optimizer_params.</td> + </tr> + <tr> + <th>step_size</th> + <td>The step size as given in optimizer_params.</td> + </tr> + <tr> + <th>n_iterations</th> + <td>The number of iterations run</td> + </tr> + <tr> + <th>n_tries</th> + <td>The number of tries as given in optimizer_params.</td> + </tr> + <tr> + <th>layer_sizes</th> + <td>The number of units in each layer including the input and output layer.</td> + </tr> + <tr> + <th>activation_function</th> + <td>The activation function.</td> + </tr> + <tr> + <th>is_classification</th> + <td>True if the model was trained for classification, False if it was trained + for regression</td> + </tr> + <tr> + <th>classes</th> + <td>The classes which were trained against (empty for regression)</td> + </tr> + + </table> + + +@anchor mlp_regression +@par Regression Training Function +The mlp regression training function has the following format: +<pre class="syntax"> +mlp_regression(source_table, + source_table, + output_table, + independent_varname, + dependent_varname, + hidden_layer_sizes, + optimizer_params, + activation + ) +</pre> + +\b Arguments + +Specifications for regression are largely the same as for classification. In the +model table, the loss will refer to mean square error instead of cross entropy. In the +summary table, there is classes column. The following +arguments have specifications which differ from mlp_classification: +<DL class="arglist"> +<DT>dependent_varname</DT> + <DD>TEXT. Name of the dependent variable column. + For regression supported types are any numeric type, or array + or numeric types (for multiple regression). + </DD> +</DL> + + +@anchor optimizer_params +@par Optimizer Parameters +Parameters in this section are supplied in the \e optimizer_params argument as a string +containing a comma-delimited list of name-value pairs. All of these named +parameters are optional, and their order does not matter. You must use the +format "<param_name> = <value>" to specify the value of a parameter, otherwise +the parameter is ignored. + + +<pre class="syntax"> + 'step_size = <value>, + n_iterations = <value>, + n_tries = <value>, + tolerance = <value>' +</pre> +\b Optimizer Parameters +<DL class="arglist"> + +<DT>step_size</dt> +<DD>Default: [0.001]. +Also known as the learning rate. A small value is usually desirable to +ensure convergence, while a large value provides more room for progress during +training. Since the best value depends on the condition number of the data, in +practice one often tunes this parameter. +</DD> + + +<DT>n_iterations</dt> +<DD>Default: [100]. The maximum number of iterations allowed. +</DD> +<DT>n_tries</dt> +<DD>Default: [1]. Number of times to retrain the network with randomly initialized +weights +</DD> + +<DT>tolerance</dt> +<DD>Default: 0.001. The criterion to end iterations. The training stops whenever +<the difference between the training models of two consecutive iterations is +<smaller than \e tolerance or the iteration number is larger than \e max_iter. +</DD> + +</DL> + +@anchor predict +@par Prediction Function +Used to generate predictions given a previously trained model on novel data. +The same syntax is used for classification, and regression. +<pre class="syntax"> +mlp_predict(model_table, + data_table, + id_col_name, + output_table, + pred_type) +</pre> + +\b Arguments +<DL class="arglist"> + <DT>model_table</DT> + <DD>TEXT. Model table produced by the training function.</DD> + + <DT>data_table</DT> + <DD>TEXT. Name of the table containing the data for prediction. This table is expected + to contain the same input features that were used during training. The table should + also contain id_col_name used for identifying each row.</DD> + + <DT>id_col_name</DT> + <DD>TEXT. The name of the id column in the input table.</DD> + + <DT>output_table</DT> + <DD>TEXT. Name of the table where output predictions are written. If this +table name is already in use, then an error is returned. Table contains:</DD> + <table class="output"> + <tr> + <th>id</th> + <td>Gives the 'id' for each prediction, corresponding to each row from the data_table.</td> + </tr> + <tr> + <th>estimated_<COL_NAME></th> + <td> + (For pred_type='response') The estimated class + for classification or value for regression, where + <COL_NAME> is the name of the column to be + predicted from training data + </td> + </tr> + <tr> + <th>prob_<CLASS></th> + <td> + (For pred_type='prob' for classification) The + probability of a given class <CLASS> as given by + softmax. There will be one column for each class + in the training data. + </td> + </tr> + + + <DT>pred_type</DT> + <DD>TEXT. + +the type of output requested: +'response' gives the actual prediction, +'prob' gives the probability of each class. +for regression, only type='response' is defined. +The name of the id column in the input table.</DD> +</DL> +</table> + +@anchor example +@par Examples +-# Create an input data set. +<pre class="example"> +CREATE TABLE iris_data( + id integer, + attributes numeric[], + class_text varchar, + class integer +); +INSERT INTO iris_data VALUES +(1,ARRAY[5.1,3.5,1.4,0.2],'Iris-setosa',1), +(2,ARRAY[4.9,3.0,1.4,0.2],'Iris-setosa',1), +(3,ARRAY[4.7,3.2,1.3,0.2],'Iris-setosa',1), +(4,ARRAY[4.6,3.1,1.5,0.2],'Iris-setosa',1), +(5,ARRAY[5.0,3.6,1.4,0.2],'Iris-setosa',1), +(6,ARRAY[5.4,3.9,1.7,0.4],'Iris-setosa',1), +(7,ARRAY[4.6,3.4,1.4,0.3],'Iris-setosa',1), +(8,ARRAY[5.0,3.4,1.5,0.2],'Iris-setosa',1), +(9,ARRAY[4.4,2.9,1.4,0.2],'Iris-setosa',1), +(10,ARRAY[4.9,3.1,1.5,0.1],'Iris-setosa',1), +(11,ARRAY[7.0,3.2,4.7,1.4],'Iris-versicolor',2), +(12,ARRAY[6.4,3.2,4.5,1.5],'Iris-versicolor',2), +(13,ARRAY[6.9,3.1,4.9,1.5],'Iris-versicolor',2), +(14,ARRAY[5.5,2.3,4.0,1.3],'Iris-versicolor',2), +(15,ARRAY[6.5,2.8,4.6,1.5],'Iris-versicolor',2), +(16,ARRAY[5.7,2.8,4.5,1.3],'Iris-versicolor',2), +(17,ARRAY[6.3,3.3,4.7,1.6],'Iris-versicolor',2), +(18,ARRAY[4.9,2.4,3.3,1.0],'Iris-versicolor',2), +(19,ARRAY[6.6,2.9,4.6,1.3],'Iris-versicolor',2), +(20,ARRAY[5.2,2.7,3.9,1.4],'Iris-versicolor',2); +</pre> +-# Generate a multilayer perceptron with a single hidden layer of 5 units. +Use the attributes column as the independent variables, and use the class +column as the classification. Set the tolerance to 0 so that 5000 +iterations will be run. Use a hyperbolic tangent activation function. +The model will be written to mlp_result. +<pre class="example"> +SELECT madlib.mlp_classification( + 'iris_data_2', -- Source table --- End diff -- Should the reference to `iris_data_2` (Source table) be `iris_data`?
--- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. ---