This is an automated email from the ASF dual-hosted git repository. wkcn pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push: new f4ab2d7 [MXNET-1291] solve pylint errors in examples with issue no.12205 (#13848) f4ab2d7 is described below commit f4ab2d759c8d13cd69faada2d0bc93a53dbe77b5 Author: Neil Chien <cchung1...@cs.ccu.edu.tw> AuthorDate: Thu Mar 7 11:13:02 2019 +0800 [MXNET-1291] solve pylint errors in examples with issue no.12205 (#13848) * [issue_12205 - PART1] * [issue_12205 - PART2] * Update example/cnn_chinese_text_classification/data_helpers.py Co-Authored-By: cchung100m <cchung1...@cs.ccu.edu.tw> * Update example/distributed_training/cifar10_dist.py Co-Authored-By: cchung100m <cchung1...@cs.ccu.edu.tw> * unify the style unify the style * unify the style unify the style * Update model.py * unify the style here unify the style here * Remove the preposition 'with' in comment Remove the preposition 'with' in comment 'Train module using Caffe operator in MXNet' * Unify the style of comments Unify the style of comments suggested by @sandeep-krishnamurthy * add CONTRIBUTORS.md * retrigger the CI --- CONTRIBUTORS.md | 2 +- example/bayesian-methods/bdk_demo.py | 1 - example/caffe/caffe_net.py | 4 +- example/caffe/train_model.py | 4 +- .../data_helpers.py | 3 +- .../cnn_chinese_text_classification/text_cnn.py | 9 +- example/cnn_text_classification/data_helpers.py | 8 +- example/cnn_text_classification/text_cnn.py | 162 +++++++++++++++------ example/ctc/captcha_generator.py | 45 ++---- example/ctc/ctc_metrics.py | 20 ++- example/ctc/hyperparams.py | 6 +- example/ctc/lstm_ocr_infer.py | 7 +- example/ctc/lstm_ocr_train.py | 4 +- example/ctc/multiproc_data.py | 36 ++--- example/ctc/ocr_iter.py | 17 ++- example/ctc/ocr_predict.py | 22 +-- example/deep-embedded-clustering/autoencoder.py | 29 ++-- example/deep-embedded-clustering/data.py | 8 +- example/deep-embedded-clustering/dec.py | 31 ++-- example/deep-embedded-clustering/model.py | 3 +- example/distributed_training/cifar10_dist.py | 78 +++++++--- 21 files changed, 288 insertions(+), 211 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 0a7eb42..584940b 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -214,8 +214,8 @@ List of Contributors * [Dang Trung Kien](https://github.com/kiendang) * [Zach Boldyga](https://github.com/zboldyga) * [Gordon Reid](https://github.com/gordon1992) - * [Ming Yang](http://ufoym.com) +* [Neo Chien](https://github.com/cchung100m) Label Bot --------- diff --git a/example/bayesian-methods/bdk_demo.py b/example/bayesian-methods/bdk_demo.py index 83a4319..bd883d2 100644 --- a/example/bayesian-methods/bdk_demo.py +++ b/example/bayesian-methods/bdk_demo.py @@ -162,7 +162,6 @@ def dev(gpu_id=None): return mx.gpu(gpu_id) if gpu_id else mx.cpu() - def run_mnist_SGD(num_training=50000, gpu_id=None): X, Y, X_test, Y_test = load_mnist(num_training) minibatch_size = 100 diff --git a/example/caffe/caffe_net.py b/example/caffe/caffe_net.py index 6796fca..803efda 100644 --- a/example/caffe/caffe_net.py +++ b/example/caffe/caffe_net.py @@ -14,6 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. + """Generate helper functions to load Caffe into MXNet""" import argparse import mxnet as mx @@ -86,8 +87,7 @@ def get_network_from_json_file(file_name): def parse_args(): - """Parse the arguments - """ + """Parse the arguments""" parser = argparse.ArgumentParser(description='train an image classifier on mnist') parser.add_argument('--network', type=str, default='lenet', help='the cnn to use (mlp | lenet | <path to network json file>') diff --git a/example/caffe/train_model.py b/example/caffe/train_model.py index 16b1867..d7dfd5d 100644 --- a/example/caffe/train_model.py +++ b/example/caffe/train_model.py @@ -14,14 +14,14 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -"""Train module with using Caffe operator in MXNet""" +"""Train module using Caffe operator in MXNet""" import os import logging import mxnet as mx def fit(args, network, data_loader, eval_metrics=None, batch_end_callback=None): - """Train the model with using Caffe operator in MXNet""" + """Train the model using Caffe operator in MXNet""" # kvstore kv = mx.kvstore.create(args.kv_store) diff --git a/example/cnn_chinese_text_classification/data_helpers.py b/example/cnn_chinese_text_classification/data_helpers.py index 49bb3d5..af3c62f 100644 --- a/example/cnn_chinese_text_classification/data_helpers.py +++ b/example/cnn_chinese_text_classification/data_helpers.py @@ -14,8 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - -"""Help functions to support for implementing CNN + Highway Network for Chinese Text Classification in MXNet""" +"""Helper functions to support for implementing CNN + Highway Network for Chinese Text Classification in MXNet""" import codecs import itertools diff --git a/example/cnn_chinese_text_classification/text_cnn.py b/example/cnn_chinese_text_classification/text_cnn.py index ce70681..f17ccb2 100644 --- a/example/cnn_chinese_text_classification/text_cnn.py +++ b/example/cnn_chinese_text_classification/text_cnn.py @@ -66,7 +66,6 @@ parser.add_argument('--save-period', type=int, default=10, def save_model(): """Save cnn model - Returns ---------- callback: A callback function that can be passed as epoch_end_callback to fit @@ -78,11 +77,9 @@ def save_model(): def highway(data): """Construct highway net - Parameters ---------- data: - Returns ---------- Highway Networks @@ -104,7 +101,6 @@ def highway(data): def data_iter(batch_size, num_embed, pre_trained_word2vec=False): """Construct data iter - Parameters ---------- batch_size: int @@ -166,7 +162,6 @@ def sym_gen(batch_size, sentences_size, num_embed, vocabulary_size, num_label=2, filter_list=None, num_filter=100, dropout=0.0, pre_trained_word2vec=False): """Generate network symbol - Parameters ---------- batch_size: int @@ -178,7 +173,7 @@ def sym_gen(batch_size, sentences_size, num_embed, vocabulary_size, num_filter: int dropout: int pre_trained_word2vec: boolean - identify the pre-trained layers or not + identify the pre-trained layers or not Returns ---------- sm: symbol @@ -236,7 +231,6 @@ def sym_gen(batch_size, sentences_size, num_embed, vocabulary_size, def train(symbol_data, train_iterator, valid_iterator, data_column_names, target_names): """Train cnn model - Parameters ---------- symbol_data: symbol @@ -291,7 +285,6 @@ class CustomInit(Initializer): """https://mxnet.incubator.apache.org/api/python/optimization.html#mxnet.initializer.register Create and register a custom initializer that Initialize the weight and bias with custom requirements - """ weightMethods = ["normal", "uniform", "orthogonal", "xavier"] biasMethods = ["costant"] diff --git a/example/cnn_text_classification/data_helpers.py b/example/cnn_text_classification/data_helpers.py index 093da7b..78dacb0 100644 --- a/example/cnn_text_classification/data_helpers.py +++ b/example/cnn_text_classification/data_helpers.py @@ -15,7 +15,9 @@ # specific language governing permissions and limitations # under the License. -"""Help functions to support for implementing CNN + Highway Network for Text Classification in MXNet""" +""" +Help functions to support for implementing CNN + Highway Network for Text Classification in MXNet +""" import itertools import os @@ -108,7 +110,9 @@ def build_input_data(sentences, labels, vocabulary): def build_input_data_with_word2vec(sentences, labels, word2vec_list): - """Map sentences and labels to vectors based on a pretrained word2vec""" + """ + Map sentences and labels to vectors based on a pretrained word2vec + """ x_vec = [] for sent in sentences: vec = [] diff --git a/example/cnn_text_classification/text_cnn.py b/example/cnn_text_classification/text_cnn.py index 9ad9443..79c3945 100644 --- a/example/cnn_text_classification/text_cnn.py +++ b/example/cnn_text_classification/text_cnn.py @@ -19,12 +19,13 @@ # -*- coding: utf-8 -*- -import sys +"""Implementing CNN + Highway Network for Text Classification in MXNet""" + import os -import mxnet as mx -import numpy as np -import argparse import logging +import argparse +import numpy as np +import mxnet as mx import data_helpers logging.basicConfig(level=logging.DEBUG) @@ -54,26 +55,55 @@ parser.add_argument('--disp-batches', type=int, default=50, parser.add_argument('--save-period', type=int, default=10, help='save checkpoint for every n epochs') + def save_model(): + """Save cnn model + + Returns + ---------- + callback: A callback function that can be passed as epoch_end_callback to fit + """ if not os.path.exists("checkpoint"): os.mkdir("checkpoint") return mx.callback.do_checkpoint("checkpoint/checkpoint", args.save_period) + def data_iter(batch_size, num_embed, pre_trained_word2vec=False): + """Construct data iter + + Parameters + ---------- + batch_size: int + num_embed: int + pre_trained_word2vec: boolean + identify the pre-trained layers or not + Returns + ---------- + train_set: DataIter + Train DataIter + valid: DataIter + Valid DataIter + sentences_size: int + array dimensions + embedded_size: int + array dimensions + vocab_size: int + array dimensions + """ print('Loading data...') if pre_trained_word2vec: word2vec = data_helpers.load_pretrained_word2vec('data/rt.vec') x, y = data_helpers.load_data_with_word2vec(word2vec) - # reshpae for convolution input + # reshape for convolution input x = np.reshape(x, (x.shape[0], 1, x.shape[1], x.shape[2])) - embed_size = x.shape[-1] - sentence_size = x.shape[2] - vocab_size = -1 + embedded_size = x.shape[-1] + sentences_size = x.shape[2] + vocabulary_size = -1 else: x, y, vocab, vocab_inv = data_helpers.load_data() - embed_size = num_embed - sentence_size = x.shape[1] - vocab_size = len(vocab) + embedded_size = num_embed + sentences_size = x.shape[1] + vocabulary_size = len(vocab) # randomly shuffle data np.random.seed(10) @@ -87,27 +117,53 @@ def data_iter(batch_size, num_embed, pre_trained_word2vec=False): print('Train/Valid split: %d/%d' % (len(y_train), len(y_dev))) print('train shape:', x_train.shape) print('valid shape:', x_dev.shape) - print('sentence max words', sentence_size) - print('embedding size', embed_size) - print('vocab size', vocab_size) + print('sentence max words', sentences_size) + print('embedding size', embedded_size) + print('vocab size', vocabulary_size) - train = mx.io.NDArrayIter( + train_set = mx.io.NDArrayIter( x_train, y_train, batch_size, shuffle=True) valid = mx.io.NDArrayIter( x_dev, y_dev, batch_size) - return (train, valid, sentence_size, embed_size, vocab_size) + return train_set, valid, sentences_size, embedded_size, vocabulary_size -def sym_gen(batch_size, sentence_size, num_embed, vocab_size, - num_label=2, filter_list=[3, 4, 5], num_filter=100, + +def sym_gen(batch_size, sentences_size, num_embed, vocabulary_size, + num_label=2, filter_list=None, num_filter=100, dropout=0.0, pre_trained_word2vec=False): + """Generate network symbol + + Parameters + ---------- + batch_size: int + sentences_size: int + num_embed: int + vocabulary_size: int + num_label: int + filter_list: list + num_filter: int + dropout: int + pre_trained_word2vec: boolean + identify the pre-trained layers or not + Returns + ---------- + sm: symbol + data: list of str + data names + softmax_label: list of str + label names + """ input_x = mx.sym.Variable('data') input_y = mx.sym.Variable('softmax_label') # embedding layer if not pre_trained_word2vec: - embed_layer = mx.sym.Embedding(data=input_x, input_dim=vocab_size, output_dim=num_embed, name='vocab_embed') - conv_input = mx.sym.Reshape(data=embed_layer, target_shape=(batch_size, 1, sentence_size, num_embed)) + embed_layer = mx.sym.Embedding(data=input_x, + input_dim=vocabulary_size, + output_dim=num_embed, + name='vocab_embed') + conv_input = mx.sym.Reshape(data=embed_layer, target_shape=(batch_size, 1, sentences_size, num_embed)) else: conv_input = input_x @@ -116,7 +172,7 @@ def sym_gen(batch_size, sentence_size, num_embed, vocab_size, for i, filter_size in enumerate(filter_list): convi = mx.sym.Convolution(data=conv_input, kernel=(filter_size, num_embed), num_filter=num_filter) relui = mx.sym.Activation(data=convi, act_type='relu') - pooli = mx.sym.Pooling(data=relui, pool_type='max', kernel=(sentence_size - filter_size + 1, 1), stride=(1,1)) + pooli = mx.sym.Pooling(data=relui, pool_type='max', kernel=(sentences_size - filter_size + 1, 1), stride=(1, 1)) pooled_outputs.append(pooli) # combine all pooled outputs @@ -141,20 +197,39 @@ def sym_gen(batch_size, sentence_size, num_embed, vocab_size, return sm, ('data',), ('softmax_label',) -def train(symbol, train_iter, valid_iter, data_names, label_names): - devs = mx.cpu() if args.gpus is None or args.gpus is '' else [ - mx.gpu(int(i)) for i in args.gpus.split(',')] - module = mx.mod.Module(symbol, data_names=data_names, label_names=label_names, context=devs) - module.fit(train_data = train_iter, - eval_data = valid_iter, - eval_metric = 'acc', - kvstore = args.kv_store, - optimizer = args.optimizer, - optimizer_params = { 'learning_rate': args.lr }, - initializer = mx.initializer.Uniform(0.1), - num_epoch = args.num_epochs, - batch_end_callback = mx.callback.Speedometer(args.batch_size, args.disp_batches), - epoch_end_callback = save_model()) + +def train(symbol_data, train_iterator, valid_iterator, data_column_names, target_names): + """Train cnn model + + Parameters + ---------- + symbol_data: symbol + train_iterator: DataIter + Train DataIter + valid_iterator: DataIter + Valid DataIter + data_column_names: list of str + Defaults to ('data') for a typical model used in image classification + target_names: list of str + Defaults to ('softmax_label') for a typical model used in image classification + """ + devs = mx.cpu() # default setting + if args.gpus is not None: + for i in args.gpus.split(','): + mx.gpu(int(i)) + devs = mx.gpu() + module = mx.mod.Module(symbol_data, data_names=data_column_names, label_names=target_names, context=devs) + module.fit(train_data=train_iterator, + eval_data=valid_iterator, + eval_metric='acc', + kvstore=args.kv_store, + optimizer=args.optimizer, + optimizer_params={'learning_rate': args.lr}, + initializer=mx.initializer.Uniform(0.1), + num_epoch=args.num_epochs, + batch_end_callback=mx.callback.Speedometer(args.batch_size, args.disp_batches), + epoch_end_callback=save_model()) + if __name__ == '__main__': # parse args @@ -162,14 +237,17 @@ if __name__ == '__main__': # data iter train_iter, valid_iter, sentence_size, embed_size, vocab_size = data_iter(args.batch_size, - args.num_embed, - args.pretrained_embedding) + args.num_embed, + args.pretrained_embedding) # network symbol symbol, data_names, label_names = sym_gen(args.batch_size, - sentence_size, - embed_size, - vocab_size, - num_label=2, filter_list=[3, 4, 5], num_filter=100, - dropout=args.dropout, pre_trained_word2vec=args.pretrained_embedding) + sentence_size, + embed_size, + vocab_size, + num_label=2, + filter_list=[3, 4, 5], + num_filter=100, + dropout=args.dropout, + pre_trained_word2vec=args.pretrained_embedding) # train cnn model train(symbol, train_iter, valid_iter, data_names, label_names) diff --git a/example/ctc/captcha_generator.py b/example/ctc/captcha_generator.py index 97fab40..8a3d726 100644 --- a/example/ctc/captcha_generator.py +++ b/example/ctc/captcha_generator.py @@ -15,22 +15,20 @@ # specific language governing permissions and limitations # under the License. """ Helper classes for multiprocess captcha image generation - This module also provides script for saving captcha images to file using CLI. """ from __future__ import print_function import random +import numpy as np from captcha.image import ImageCaptcha import cv2 from multiproc_data import MPData -import numpy as np class CaptchaGen(object): - """ - Generates a captcha image + """Generates a captcha image """ def __init__(self, h, w, font_paths): """ @@ -48,8 +46,7 @@ class CaptchaGen(object): self.w = w def image(self, captcha_str): - """ - Generate a greyscale captcha image representing number string + """Generate a greyscale captcha image representing number string Parameters ---------- @@ -71,8 +68,7 @@ class CaptchaGen(object): class DigitCaptcha(object): - """ - Provides shape() and get() interface for digit-captcha image generation + """Provides shape() and get() interface for digit-captcha image generation """ def __init__(self, font_paths, h, w, num_digit_min, num_digit_max): """ @@ -95,8 +91,7 @@ class DigitCaptcha(object): @property def shape(self): - """ - Returns shape of the image data generated + """Returns shape of the image data generated Returns ------- @@ -105,8 +100,7 @@ class DigitCaptcha(object): return self.captcha.h, self.captcha.w def get(self): - """ - Get an image from the queue + """Get an image from the queue Returns ------- @@ -117,9 +111,8 @@ class DigitCaptcha(object): @staticmethod def get_rand(num_digit_min, num_digit_max): - """ - Generates a character string of digits. Number of digits are - between self.num_digit_min and self.num_digit_max + """Generates a character string of digits. Number of digits are + between self.num_digit_min and self.num_digit_max Returns ------- str @@ -131,8 +124,7 @@ class DigitCaptcha(object): return buf def _gen_sample(self): - """ - Generate a random captcha image sample + """Generate a random captcha image sample Returns ------- (numpy.ndarray, str) @@ -143,13 +135,10 @@ class DigitCaptcha(object): class MPDigitCaptcha(DigitCaptcha): - """ - Handles multi-process captcha image generation + """Handles multi-process captcha image generation """ def __init__(self, font_paths, h, w, num_digit_min, num_digit_max, num_processes, max_queue_size): - """ - - Parameters + """Parameters ---------- font_paths: list of str List of path to ttf font files @@ -170,14 +159,11 @@ class MPDigitCaptcha(DigitCaptcha): self.mp_data = MPData(num_processes, max_queue_size, self._gen_sample) def start(self): - """ - Starts the processes - """ + """Starts the processes""" self.mp_data.start() def get(self): - """ - Get an image from the queue + """Get an image from the queue Returns ------- @@ -187,9 +173,7 @@ class MPDigitCaptcha(DigitCaptcha): return self.mp_data.get() def reset(self): - """ - Resets the generator by stopping all processes - """ + """Resets the generator by stopping all processes""" self.mp_data.reset() @@ -197,6 +181,7 @@ if __name__ == '__main__': import argparse def main(): + """Program entry point""" parser = argparse.ArgumentParser() parser.add_argument("font_path", help="Path to ttf font file") parser.add_argument("output", help="Output filename including extension (e.g. 'sample.jpg')") diff --git a/example/ctc/ctc_metrics.py b/example/ctc/ctc_metrics.py index 0db680a..5f51025 100644 --- a/example/ctc/ctc_metrics.py +++ b/example/ctc/ctc_metrics.py @@ -22,14 +22,17 @@ import numpy as np class CtcMetrics(object): + """Module for calculating the prediction accuracy during training. Two accuracy measures are implemented: + A simple accuracy measure that calculates number of correct predictions divided by total number of predictions + and a second accuracy measure based on sum of Longest Common Sequence(LCS) ratio of all predictions divided by total + number of predictions + """ def __init__(self, seq_len): self.seq_len = seq_len @staticmethod def ctc_label(p): - """ - Iterates through p, identifying non-zero and non-repeating values, and returns them in a list - Parameters + """Iterates through p, identifying non-zero and non-repeating values, and returns them in a list Parameters ---------- p: list of int @@ -42,7 +45,7 @@ class CtcMetrics(object): for i, _ in enumerate(p): c1 = p1[i] c2 = p1[i+1] - if c2 == 0 or c2 == c1: + if c2 in (0, c1): continue ret.append(c2) return ret @@ -65,12 +68,16 @@ class CtcMetrics(object): return 0 P = np.array(list(p)).reshape((1, len(p))) L = np.array(list(l)).reshape((len(l), 1)) - M = np.int32(P == L) + M = np.ndarray(shape=(len(P), len(L)), dtype=np.int32) for i in range(M.shape[0]): for j in range(M.shape[1]): up = 0 if i == 0 else M[i-1, j] left = 0 if j == 0 else M[i, j-1] - M[i, j] = max(up, left, M[i, j] if (i == 0 or j == 0) else M[i, j] + M[i-1, j-1]) + + if i == 0 or j == 0: + M[i, j] = max(up, left, M[i, j]) + else: + M[i, j] = M[i, j] + M[i - 1, j - 1] return M.max() def accuracy(self, label, pred): @@ -111,4 +118,3 @@ class CtcMetrics(object): total += 1.0 assert total == batch_size return hit / total - diff --git a/example/ctc/hyperparams.py b/example/ctc/hyperparams.py index cdcb874..ef24401 100644 --- a/example/ctc/hyperparams.py +++ b/example/ctc/hyperparams.py @@ -20,9 +20,7 @@ from __future__ import print_function class Hyperparams(object): - """ - Hyperparameters for LSTM network - """ + """Hyperparameters for LSTM network""" def __init__(self): # Training hyper parameters self._train_epoch_size = 30000 @@ -48,7 +46,7 @@ class Hyperparams(object): @property def batch_size(self): return self._batch_size - + @property def num_epoch(self): return self._num_epoch diff --git a/example/ctc/lstm_ocr_infer.py b/example/ctc/lstm_ocr_infer.py index 80de2c7..eec48ea 100644 --- a/example/ctc/lstm_ocr_infer.py +++ b/example/ctc/lstm_ocr_infer.py @@ -20,12 +20,12 @@ from __future__ import print_function import argparse +import numpy as np from ctc_metrics import CtcMetrics import cv2 from hyperparams import Hyperparams import lstm import mxnet as mx -import numpy as np from ocr_iter import SimpleBatch @@ -46,8 +46,7 @@ def lstm_init_states(batch_size): def load_module(prefix, epoch, data_names, data_shapes): - """ - Loads the model from checkpoint specified by prefix and epoch, binds it + """Loads the model from checkpoint specified by prefix and epoch, binds it to an executor, and sets its parameters and returns a mx.mod.Module """ sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch) @@ -64,6 +63,7 @@ def load_module(prefix, epoch, data_names, data_shapes): def main(): + """Program entry point""" parser = argparse.ArgumentParser() parser.add_argument("path", help="Path to the CAPTCHA image file") parser.add_argument("--prefix", help="Checkpoint prefix [Default 'ocr']", default='ocr') @@ -86,7 +86,6 @@ def main(): # Predictions are 1 to 10 for digits 0 to 9 respectively (prediction 0 means no-digit) prediction = [p - 1 for p in prediction] print("Digits:", prediction) - return if __name__ == '__main__': diff --git a/example/ctc/lstm_ocr_train.py b/example/ctc/lstm_ocr_train.py index 2c25f7e..49d9531 100644 --- a/example/ctc/lstm_ocr_train.py +++ b/example/ctc/lstm_ocr_train.py @@ -42,7 +42,7 @@ def get_fonts(path): def parse_args(): - # Parse command line arguments + """Parse command line arguments""" parser = argparse.ArgumentParser() parser.add_argument("font_path", help="Path to ttf font file or directory containing ttf files") parser.add_argument("--loss", help="'ctc' or 'warpctc' loss [Default 'ctc']", default='ctc') @@ -56,6 +56,7 @@ def parse_args(): def main(): + """Program entry point""" args = parse_args() if not any(args.loss == s for s in ['ctc', 'warpctc']): raise ValueError("Invalid loss '{}' (must be 'ctc' or 'warpctc')".format(args.loss)) @@ -122,4 +123,3 @@ def main(): if __name__ == '__main__': main() - diff --git a/example/ctc/multiproc_data.py b/example/ctc/multiproc_data.py index c5f8da5..313ab4e 100644 --- a/example/ctc/multiproc_data.py +++ b/example/ctc/multiproc_data.py @@ -15,6 +15,8 @@ # specific language governing permissions and limitations # under the License. +"""Contains a class for handling multi-process data generation""" + from __future__ import print_function from ctypes import c_bool import multiprocessing as mp @@ -25,12 +27,9 @@ except ImportError: from Queue import Full as QFullExcept from Queue import Empty as QEmptyExcept -import numpy as np - class MPData(object): - """ - Handles multi-process data generation. + """Handles multi-process data generation. Operation: - call start() to start the data generation @@ -38,9 +37,7 @@ class MPData(object): - call reset() to stop data generation """ def __init__(self, num_processes, max_queue_size, fn): - """ - - Parameters + """Parameters ---------- num_processes: int Number of processes to spawn @@ -56,22 +53,12 @@ class MPData(object): self.fn = fn def start(self): - """ - Starts the processes - Parameters - ---------- - fn: function - - """ - """ - Starts the processes - """ + """Starts the processes""" self._init_proc() @staticmethod def _proc_loop(proc_id, alive, queue, fn): - """ - Thread loop for generating data + """Thread loop for generating data Parameters ---------- @@ -102,9 +89,7 @@ class MPData(object): queue.close() def _init_proc(self): - """ - Start processes if not already started - """ + """Start processes if not already started""" if not self.proc: self.proc = [ mp.Process(target=self._proc_loop, args=(i, self.alive, self.queue, self.fn)) @@ -115,8 +100,7 @@ class MPData(object): p.start() def get(self): - """ - Get a datum from the queue + """Get a datum from the queue Returns ------- @@ -127,9 +111,7 @@ class MPData(object): return self.queue.get() def reset(self): - """ - Resets the generator by stopping all processes - """ + """Resets the generator by stopping all processes""" self.alive.value = False qsize = 0 try: diff --git a/example/ctc/ocr_iter.py b/example/ctc/ocr_iter.py index 1432e92..5a02f38 100644 --- a/example/ctc/ocr_iter.py +++ b/example/ctc/ocr_iter.py @@ -23,7 +23,11 @@ import mxnet as mx class SimpleBatch(object): - def __init__(self, data_names, data, label_names=list(), label=list()): + """Batch class for getting label data + Operation: + - call get_label() to start label data generation + """ + def __init__(self, data_names, data, label_names=None, label=None): self._data = data self._label = label self._data_names = data_names @@ -59,20 +63,17 @@ class SimpleBatch(object): def get_label(buf): ret = np.zeros(4) - for i in range(len(buf)): - ret[i] = 1 + int(buf[i]) + for i, element in enumerate(buf): + ret[i] = 1 + int(element) if len(buf) == 3: ret[3] = 0 return ret class OCRIter(mx.io.DataIter): - """ - Iterator class for generating captcha image data - """ + """Iterator class for generating captcha image data""" def __init__(self, count, batch_size, lstm_init_states, captcha, name): - """ - Parameters + """Parameters ---------- count: int Number of batches to produce for one epoch diff --git a/example/ctc/ocr_predict.py b/example/ctc/ocr_predict.py index 2cf1967..c5c691b 100644 --- a/example/ctc/ocr_predict.py +++ b/example/ctc/ocr_predict.py @@ -21,18 +21,12 @@ from __future__ import print_function import argparse import sys -import cv2 import numpy as np -import mxnet as mx -from collections import namedtuple -from ocr_iter import SimpleBatch -from captcha_generator import DigitCaptcha -from ctc_metrics import CtcMetrics -import lstm -from hyperparams import Hyperparams +import cv2 class lstm_ocr_model(object): + """LSTM network for predicting the Optical Character Recognition""" # Keep Zero index for blank. (CTC request it) CONST_CHAR = '0123456789' @@ -67,6 +61,16 @@ class lstm_ocr_model(object): all_shapes_dict) def forward_ocr(self, img_): + """Forward the image through the LSTM network model + + Parameters + ---------- + img_: int of array + + Returns + ---------- + label_list: string of list + """ img_ = cv2.resize(img_, (80, 30)) img_ = img_.transpose(1, 0) print(img_.shape) @@ -92,7 +96,7 @@ class lstm_ocr_model(object): for i in range(len(label_list)): c1 = label_list2[i] c2 = label_list2[i+1] - if c2 == 0 or c2 == c1: + if c2 in (0, c1): continue ret.append(c2) # change to ascii diff --git a/example/deep-embedded-clustering/autoencoder.py b/example/deep-embedded-clustering/autoencoder.py index 096f045..c756344 100644 --- a/example/deep-embedded-clustering/autoencoder.py +++ b/example/deep-embedded-clustering/autoencoder.py @@ -19,9 +19,8 @@ from __future__ import print_function import logging - -import mxnet as mx import numpy as np +import mxnet as mx import model from solver import Solver, Monitor @@ -95,18 +94,18 @@ class AutoEncoderModel(model.MXModel): else: x = mx.symbol.LinearRegressionOutput(data=x, label=data) - args = {'encoder_%d_weight'%istack: mx.nd.empty((num_hidden, num_input), self.xpu), - 'encoder_%d_bias'%istack: mx.nd.empty((num_hidden,), self.xpu), - 'decoder_%d_weight'%istack: mx.nd.empty((num_input, num_hidden), self.xpu), - 'decoder_%d_bias'%istack: mx.nd.empty((num_input,), self.xpu),} - args_grad = {'encoder_%d_weight'%istack: mx.nd.empty((num_hidden, num_input), self.xpu), - 'encoder_%d_bias'%istack: mx.nd.empty((num_hidden,), self.xpu), - 'decoder_%d_weight'%istack: mx.nd.empty((num_input, num_hidden), self.xpu), - 'decoder_%d_bias'%istack: mx.nd.empty((num_input,), self.xpu),} - args_mult = {'encoder_%d_weight'%istack: 1.0, - 'encoder_%d_bias'%istack: 2.0, - 'decoder_%d_weight'%istack: 1.0, - 'decoder_%d_bias'%istack: 2.0,} + args = {'encoder_%d_weight' % istack: mx.nd.empty((num_hidden, num_input), self.xpu), + 'encoder_%d_bias' % istack: mx.nd.empty((num_hidden,), self.xpu), + 'decoder_%d_weight' % istack: mx.nd.empty((num_input, num_hidden), self.xpu), + 'decoder_%d_bias' % istack: mx.nd.empty((num_input,), self.xpu), } + args_grad = {'encoder_%d_weight' % istack: mx.nd.empty((num_hidden, num_input), self.xpu), + 'encoder_%d_bias' % istack: mx.nd.empty((num_hidden,), self.xpu), + 'decoder_%d_weight' % istack: mx.nd.empty((num_input, num_hidden), self.xpu), + 'decoder_%d_bias' % istack: mx.nd.empty((num_input,), self.xpu), } + args_mult = {'encoder_%d_weight' % istack: 1.0, + 'encoder_%d_bias' % istack: 2.0, + 'decoder_%d_weight' % istack: 1.0, + 'decoder_%d_bias' % istack: 2.0, } auxs = {} if encoder_act == 'sigmoid' and sparseness_penalty: auxs['sparse_encoder_%d_moving_avg' % istack] = mx.nd.ones(num_hidden, self.xpu) * 0.5 @@ -203,4 +202,4 @@ class AutoEncoderModel(model.MXModel): last_batch_handle='pad') Y = list(model.extract_feature( self.loss, self.args, self.auxs, data_iter, X.shape[0], self.xpu).values())[0] - return np.mean(np.square(Y-X))/2.0 \ No newline at end of file + return np.mean(np.square(Y-X))/2.0 diff --git a/example/deep-embedded-clustering/data.py b/example/deep-embedded-clustering/data.py index 3649990..4ae3271 100644 --- a/example/deep-embedded-clustering/data.py +++ b/example/deep-embedded-clustering/data.py @@ -18,10 +18,8 @@ # pylint: disable=missing-docstring from __future__ import print_function -import os - -import mxnet as mx import numpy as np +import mxnet as mx def get_mnist(): @@ -35,7 +33,3 @@ def get_mnist(): X = X[p].reshape((X.shape[0], -1)).astype(np.float32)*5 Y = Y[p] return X, Y - - - - diff --git a/example/deep-embedded-clustering/dec.py b/example/deep-embedded-clustering/dec.py index d759470..8fb3891 100644 --- a/example/deep-embedded-clustering/dec.py +++ b/example/deep-embedded-clustering/dec.py @@ -17,27 +17,29 @@ # pylint: skip-file from __future__ import print_function -import sys + import os -import mxnet as mx +import logging import numpy as np -import data -from scipy.spatial.distance import cdist from sklearn.cluster import KMeans +from scipy.spatial.distance import cdist +import mxnet as mx +import data import model from autoencoder import AutoEncoderModel from solver import Solver, Monitor -import logging + def cluster_acc(Y_pred, Y): from sklearn.utils.linear_assignment_ import linear_assignment assert Y_pred.size == Y.size D = max(Y_pred.max(), Y.max())+1 - w = np.zeros((D,D), dtype=np.int64) + w = np.zeros((D, D), dtype=np.int64) for i in range(Y_pred.size): w[Y_pred[i], int(Y[i])] += 1 ind = linear_assignment(w.max() - w) - return sum([w[i,j] for i,j in ind])*1.0/Y_pred.size, w + return sum([w[i, j] for i, j in ind])*1.0/Y_pred.size, w + class DECModel(model.MXModel): class DECLoss(mx.operator.NumpyOp): @@ -81,12 +83,12 @@ class DECModel(model.MXModel): sep = X.shape[0]*9//10 X_train = X[:sep] X_val = X[sep:] - ae_model = AutoEncoderModel(self.xpu, [X.shape[1],500,500,2000,10], pt_dropout=0.2) + ae_model = AutoEncoderModel(self.xpu, [X.shape[1], 500, 500, 2000, 10], pt_dropout=0.2) if not os.path.exists(save_to+'_pt.arg'): ae_model.layerwise_pretrain(X_train, 256, 50000, 'sgd', l_rate=0.1, decay=0.0, - lr_scheduler=mx.lr_scheduler.FactorScheduler(20000,0.1)) + lr_scheduler=mx.lr_scheduler.FactorScheduler(20000, 0.1)) ae_model.finetune(X_train, 256, 100000, 'sgd', l_rate=0.1, decay=0.0, - lr_scheduler=mx.lr_scheduler.FactorScheduler(20000,0.1)) + lr_scheduler=mx.lr_scheduler.FactorScheduler(20000, 0.1)) ae_model.save(save_to+'_pt.arg') logging.log(logging.INFO, "Autoencoder Training error: %f"%ae_model.eval(X_train)) logging.log(logging.INFO, "Autoencoder Validation error: %f"%ae_model.eval(X_val)) @@ -98,9 +100,9 @@ class DECModel(model.MXModel): label = mx.sym.Variable('label') self.feature = self.ae_model.encoder self.loss = self.dec_op(data=self.ae_model.encoder, label=label, name='dec') - self.args.update({k:v for k,v in self.ae_model.args.items() if k in self.ae_model.encoder.list_arguments()}) + self.args.update({k: v for k, v in self.ae_model.args.items() if k in self.ae_model.encoder.list_arguments()}) self.args['dec_mu'] = mx.nd.empty((num_centers, self.ae_model.dims[-1]), ctx=self.xpu) - self.args_grad.update({k: mx.nd.empty(v.shape, ctx=self.xpu) for k,v in self.args.items()}) + self.args_grad.update({k: mx.nd.empty(v.shape, ctx=self.xpu) for k, v in self.args.items()}) self.args_mult.update({k: k.endswith('bias') and 2.0 or 1.0 for k in self.args}) self.num_centers = num_centers @@ -117,6 +119,7 @@ class DECModel(model.MXModel): kmeans.fit(z) args['dec_mu'][:] = kmeans.cluster_centers_ solver = Solver('sgd', momentum=0.9, wd=0.0, learning_rate=0.01) + def ce(label, pred): return np.sum(label*np.log(label/(pred+0.000001)))/label.shape[0] solver.set_metric(mx.metric.CustomMetric(ce)) @@ -125,6 +128,7 @@ class DECModel(model.MXModel): train_iter = mx.io.NDArrayIter({'data': X}, {'label': label_buff}, batch_size=batch_size, shuffle=False, last_batch_handle='roll_over') self.y_pred = np.zeros((X.shape[0])) + def refresh(i): if i%update_interval == 0: z = list(model.extract_feature(self.feature, args, None, test_iter, N, self.xpu).values())[0] @@ -155,6 +159,7 @@ class DECModel(model.MXModel): else: return -1 + def mnist_exp(xpu): X, Y = data.get_mnist() if not os.path.isdir('data'): @@ -167,7 +172,7 @@ def mnist_exp(xpu): logging.info(str(acc)) logging.info('Best Clustering ACC: %f at update_interval: %d'%(np.max(acc), 10*(2**np.argmax(acc)))) + if __name__ == '__main__': logging.basicConfig(level=logging.INFO) mnist_exp(mx.gpu(0)) - diff --git a/example/deep-embedded-clustering/model.py b/example/deep-embedded-clustering/model.py index b388c55..ac029c1 100644 --- a/example/deep-embedded-clustering/model.py +++ b/example/deep-embedded-clustering/model.py @@ -20,7 +20,6 @@ from __future__ import print_function import numpy as np import mxnet as mx - try: import cPickle as pickle except ImportError: @@ -54,7 +53,7 @@ def extract_feature(sym, args, auxs, data_iter, N, xpu=mx.cpu()): class MXModel(object): - def __init__(self, *args, xpu=mx.cpu(), **kwargs): + def __init__(self, xpu=mx.cpu(), *args, **kwargs): self.xpu = xpu self.loss = None self.args = {} diff --git a/example/distributed_training/cifar10_dist.py b/example/distributed_training/cifar10_dist.py index 506afbb..d3ba515 100644 --- a/example/distributed_training/cifar10_dist.py +++ b/example/distributed_training/cifar10_dist.py @@ -17,15 +17,17 @@ # specific language governing permissions and limitations # under the License. +"""cifar10_dist.py contains code that trains a ResNet18 network using distributed training""" + from __future__ import print_function -import random, sys +import sys +import random +import numpy as np import mxnet as mx from mxnet import autograd, gluon, kv, nd from mxnet.gluon.model_zoo import vision -import numpy as np - # Create a distributed key-value store store = kv.create('dist') @@ -45,11 +47,13 @@ batch_size = batch_size_per_gpu * gpus_per_machine # Create the context (a list of all GPUs to be used for training) ctx = [mx.gpu(i) for i in range(gpus_per_machine)] + # Convert to float 32 # Having channel as the first dimension makes computation more efficient. Hence the (2,0,1) transpose. # Dividing by 255 normalizes the input between 0 and 1 def transform(data, label): - return nd.transpose(data.astype(np.float32), (2,0,1))/255, label.astype(np.float32) + return nd.transpose(data.astype(np.float32), (2, 0, 1))/255, label.astype(np.float32) + class SplitSampler(gluon.data.sampler.Sampler): """ Split the dataset into `num_parts` parts and sample from the part with index `part_index` @@ -80,14 +84,14 @@ class SplitSampler(gluon.data.sampler.Sampler): def __len__(self): return self.part_len + # Load the training data -train_data = gluon.data.DataLoader(gluon.data.vision.CIFAR10(train=True, transform=transform), - batch_size, - sampler=SplitSampler(50000, store.num_workers, store.rank)) +train_data = gluon.data.DataLoader(gluon.data.vision.CIFAR10(train=True, transform=transform), batch_size, + sampler=SplitSampler(50000, store.num_workers, store.rank)) -# Load the test data +# Load the test data test_data = gluon.data.DataLoader(gluon.data.vision.CIFAR10(train=False, transform=transform), - batch_size, shuffle=False) + batch_size, shuffle=False) # Use ResNet from model zoo net = vision.resnet18_v1() @@ -98,12 +102,25 @@ net.collect_params().initialize(mx.init.Xavier(), ctx=ctx) # SoftmaxCrossEntropy is the most common choice of loss function for multiclass classification softmax_cross_entropy = gluon.loss.SoftmaxCrossEntropyLoss() -# Use Adam optimizer. Ask trainer to use the distributer kv store. +# Use Adam optimizer. Ask trainer to use the distributor kv store. trainer = gluon.Trainer(net.collect_params(), 'adam', {'learning_rate': .001}, kvstore=store) + # Evaluate accuracy of the given network using the given data -def evaluate_accuracy(data_iterator, net): +def evaluate_accuracy(data_iterator, network): + """ Measure the accuracy of ResNet + + Parameters + ---------- + data_iterator: Iter + examples of dataset + network: + ResNet + Returns + ---------- + tuple of array element + """ acc = mx.metric.Accuracy() # Iterate through data and label @@ -115,7 +132,7 @@ def evaluate_accuracy(data_iterator, net): # Get network's output which is a probability distribution # Apply argmax on the probability distribution to get network's classification. - output = net(data) + output = network(data) predictions = nd.argmax(output, axis=1) # Give network's prediction and the correct label to update the metric @@ -124,38 +141,54 @@ def evaluate_accuracy(data_iterator, net): # Return the accuracy return acc.get()[1] + # We'll use cross entropy loss since we are doing multiclass classification loss = gluon.loss.SoftmaxCrossEntropyLoss() + # Run one forward and backward pass on multiple GPUs -def forward_backward(net, data, label): +def forward_backward(network, data, label): # Ask autograd to remember the forward pass with autograd.record(): # Compute the loss on all GPUs - losses = [loss(net(X), Y) for X, Y in zip(data, label)] + losses = [loss(network(X), Y) for X, Y in zip(data, label)] # Run the backward pass (calculate gradients) on all GPUs for l in losses: l.backward() + # Train a batch using multiple GPUs -def train_batch(batch, ctx, net, trainer): +def train_batch(batch_list, context, network, gluon_trainer): + """ Training with multiple GPUs + Parameters + ---------- + batch_list: List + list of dataset + context: List + a list of all GPUs to be used for training + network: + ResNet + gluon_trainer: + rain module of gluon + """ # Split and load data into multiple GPUs - data = batch[0] - data = gluon.utils.split_and_load(data, ctx) + data = batch_list[0] + data = gluon.utils.split_and_load(data, context) # Split and load label into multiple GPUs - label = batch[1] - label = gluon.utils.split_and_load(label, ctx) + label = batch_list[1] + label = gluon.utils.split_and_load(label, context) # Run the forward and backward pass - forward_backward(net, data, label) + forward_backward(network, data, label) # Update the parameters - this_batch_size = batch[0].shape[0] - trainer.step(this_batch_size) + this_batch_size = batch_list[0].shape[0] + gluon_trainer.step(this_batch_size) + # Run as many epochs as required for epoch in range(epochs): @@ -173,4 +206,3 @@ for epoch in range(epochs): test_accuracy = evaluate_accuracy(test_data, net) print("Epoch %d: Test_acc %f" % (epoch, test_accuracy)) sys.stdout.flush() -