singa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zhaoj...@apache.org
Subject [1/4] incubator-singa git commit: SINGA-126 Python Binding for Interactive Training
Date Wed, 06 Apr 2016 08:38:11 GMT
Repository: incubator-singa
Updated Branches:
  refs/heads/master 8b7d1e09e -> c97b970dc


SINGA-126 Python Binding for Interactive Training

- revise Feed() function in dummy.cc
  . it now takes shape, data for input, aux_data for label
  . revised neuron_layer.h, driver.i

- add ImageInput() and LabelInput()
  . inherited from Dummy()
  . specific to image data and label data, respectively

- add compute_mean_image() method in train_cifar10.py
  . it loads 5 cifar10 datasets and computes the average image

- add Update() in model.py
  . it updates layer parameters


Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/5a8fc373
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/5a8fc373
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/5a8fc373

Branch: refs/heads/master
Commit: 5a8fc3732e00372493ffdb1422db56da7f6e4876
Parents: 8130b7e
Author: chonho <leech@comp.nus.edu.sg>
Authored: Sun Apr 3 03:06:28 2016 +0800
Committer: chonho <leech@comp.nus.edu.sg>
Committed: Tue Apr 5 23:22:51 2016 +0800

----------------------------------------------------------------------
 include/singa/neuralnet/neuron_layer.h |   2 +-
 src/neuralnet/neuron_layer/dummy.cc    |  10 +-
 tool/python/examples/train_cifar10.py  |  89 +++++++-----
 tool/python/examples/train_mnist.py    |  20 ++-
 tool/python/singa/driver.i             |   2 +-
 tool/python/singa/layer.py             | 207 +++++++---------------------
 tool/python/singa/model.py             |  16 +++
 tool/python/singa/utils/utility.py     |   5 -
 8 files changed, 146 insertions(+), 205 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5a8fc373/include/singa/neuralnet/neuron_layer.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/neuron_layer.h b/include/singa/neuralnet/neuron_layer.h
index 2d73854..4dab469 100644
--- a/include/singa/neuralnet/neuron_layer.h
+++ b/include/singa/neuralnet/neuron_layer.h
@@ -126,7 +126,7 @@ class DummyLayer: public NeuronLayer {
   void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
   void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
   void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
-  void Feed(int batchsize, vector<float>& data, int is_aux);
+  void Feed(int batchsize, vector<float>& data, vector<int>& aux_data);
   Layer* ToLayer() { return this;}
 
  private:

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5a8fc373/src/neuralnet/neuron_layer/dummy.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/dummy.cc b/src/neuralnet/neuron_layer/dummy.cc
index 001e053..9796407 100644
--- a/src/neuralnet/neuron_layer/dummy.cc
+++ b/src/neuralnet/neuron_layer/dummy.cc
@@ -78,22 +78,22 @@ void DummyLayer::ComputeGradient(int flag, const vector<Layer*>&
srclayers) {
     Copy(grad_, srclayers[0]->mutable_grad(this));
 }
 
-void DummyLayer::Feed(int batchsize, vector<float>& data, int is_aux){
+void DummyLayer::Feed(int batchsize, vector<float>& data, vector<int>&
aux_data){
 
     batchsize_ = batchsize;
     // input data
-    if (is_aux == 0) {
+    if (data.size() > 0) {
       int size = data.size();
       float* ptr = data_.mutable_cpu_data();
       for (int i = 0; i< size; i++) { 
           ptr[i] = data.at(i);
       }
     }
-    // label
-    else {
+    // auxiliary data, e.g., label
+    if (aux_data.size() > 0) {
       aux_data_.resize(batchsize_);
       for (int i = 0; i< batchsize_; i++) {
-          aux_data_[i] = static_cast<int>(data.at(i));
+          aux_data_[i] = static_cast<int>(aux_data.at(i));
       }
     }
     return;

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5a8fc373/tool/python/examples/train_cifar10.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/train_cifar10.py b/tool/python/examples/train_cifar10.py
index a757595..6c621a3 100755
--- a/tool/python/examples/train_cifar10.py
+++ b/tool/python/examples/train_cifar10.py
@@ -28,52 +28,73 @@ import os, sys
 import numpy as np
 
 current_path_ = os.path.dirname(__file__)
-singa_root_=os.path.abspath(os.path.join(current_path_,'../../..'))
+singa_root_ = os.path.abspath(os.path.join(current_path_,'../../..'))
 sys.path.append(os.path.join(singa_root_,'tool','python'))
 
 from singa.driver import Driver
 from singa.layer import *
 from singa.model import *
-from singa.utils.utility import swap32
 
-fname_mean_image = 'tool/python/examples/datasets/cifar10_mean_image'
-mean_image = np.fromfile(fname_mean_image)
+
+'''
+CIFAR10 dataset can be downloaded at [https://www.cs.toronto.edu/~kriz/cifar.html]
+- please specify dataset_dir 
+'''
+dataset_dir_ = singa_root_ + "/tool/python/examples/datasets/cifar-10-batches-py"
+mean_image = None
+
+def unpickle(file):
+    ''' This method loads dataset provided at CIFAR10 website
+        See [https://www.cs.toronto.edu/~kriz/cifar.html] for more details
+    '''
+    import cPickle
+    fo = open(file, 'rb')
+    dict = cPickle.load(fo)
+    fo.close()
+    return dict
+
+def compute_mean_image():
+    ''' This is a sample script to cmopute the average image
+        of all samples in 5 dataset of cifar10
+    '''
+    mean = None
+    nb_samples_total = 0
+    for did in range(1,6):
+        fname_train_data = dataset_dir_ + "/data_batch_{}".format(did)
+        cifar10 = unpickle(fname_train_data)
+        image = cifar10['data'].astype(dtype=np.uint8)
+        if did > 1:
+            image = np.vstack((image, image))
+    return np.average(image, axis=0)
 
 def load_dataset(did=1):
-    ''' CIFAR10 dataset
+    ''' CIFAR10 dataset includes
         5 binary dataset, each contains 10000 images
         1 row (1 image) includes 1 label & 3072 pixels
         3072 pixels are  3 channels of a 32x32 image
     '''
-    print '[Load CIFAR10 dataset]', did
-    dataset_dir_ = singa_root_ + "/examples/cifar10/cifar-10-batches-bin"
-    fname_train_data = dataset_dir_ + "/data_batch_{}.bin".format(did)
-    
-    nb_samples = 10000
-    nb_pixels = 3 * 32 * 32  
-    d = np.fromfile(fname_train_data, dtype=np.uint8)
-    d = d.reshape(nb_samples, nb_pixels + 1) # +1 for label
-    x = d[:, 1:] 
-    x = x - mean_image
-    print '   data x:', x.shape
-    y = d[:, 0]
-    y = y.reshape(nb_samples, 1) 
-    print '  label y:', y.shape
-    return x, y
-
-def get_labellist():
-    dataset_dir_ = singa_root_ + "/examples/cifar10/cifar-10-batches-bin"
-    fname_label_list = dataset_dir_ + "/batches.meta.txt"
-    label_list_ = np.genfromtxt(fname_label_list, dtype=str)
-    return label_list_
+    assert mean_image != None, 'mean_image is required'
+    print '[Load CIFAR10 dataset {}]'.format(did)
+    fname_train_data = dataset_dir_ + "/data_batch_{}".format(did)
+    cifar10 = unpickle(fname_train_data)
+    image = cifar10['data'].astype(dtype=np.uint8)
+    image = image - mean_image
+    print '  image x:', image.shape
+    label = np.asarray(cifar10['labels'], dtype=np.uint8)
+    label = label.reshape(label.size, 1) 
+    print '  label y:', label.shape
+    return image, label
 
 #-------------------------------------------------------------------
+mean_image = compute_mean_image()
+# mean_image = np.fromfile('tool/python/examples/datasets/cifar10_mean_image')
+
 print '[Layer registration/declaration]'
 d = Driver()
 d.Init(sys.argv)
 
-input = Dummy()
-label = Dummy()
+input = ImageInput(32, 32, 3) # image width, height, channel
+label = LabelInput()
 
 nn = []
 nn.append(input)
@@ -99,19 +120,23 @@ batchsize = 100
 disp_freq = 50
 train_step = 1000
 
+print '[Start training]'
 for dataset_id in range(train_step / batchsize):
 
     x, y = load_dataset(dataset_id%5+1)
 
-    print '[Start training]'
     for i in range(x.shape[0] / batchsize):
         xb, yb = x[i*batchsize:(i+1)*batchsize,:], y[i*batchsize:(i+1)*batchsize,:]
-        nn[0].Feed(xb, 3, 0)
-        label.Feed(yb, 1, 1)
+        nn[0].Feed(xb)
+        label.Feed(yb)
         for h in range(1, len(nn)):
             nn[h].ComputeFeature(nn[h-1])
         loss.ComputeFeature(nn[-1], label)
         if (i+1)%disp_freq == 0:
             print '  Step {:>3}: '.format(i+1 + dataset_id*(x.shape[0]/batchsize)),
             loss.display()
-        loss.ComputeGradient(i+1, sgd)
+
+        loss.ComputeGradient()
+        for h in range(len(nn)-1, 0, -1):
+            nn[h].ComputeGradient()
+            sgd.Update(i+1, nn[h])

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5a8fc373/tool/python/examples/train_mnist.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/train_mnist.py b/tool/python/examples/train_mnist.py
index 466bc58..b8e6217 100755
--- a/tool/python/examples/train_mnist.py
+++ b/tool/python/examples/train_mnist.py
@@ -34,7 +34,12 @@ sys.path.append(os.path.join(singa_root_,'tool','python'))
 from singa.driver import Driver
 from singa.layer import *
 from singa.model import *
-from singa.utils.utility import swap32
+
+def swap32(x):
+    return (((x << 24) & 0xFF000000) |
+            ((x <<  8) & 0x00FF0000) |
+            ((x >>  8) & 0x0000FF00) |
+            ((x >> 24) & 0x000000FF))
 
 def load_dataset():
     ''' MNIST dataset
@@ -66,8 +71,8 @@ print '[Layer registration/declaration]'
 d = Driver()
 d.Init(sys.argv)
 
-input = Dummy()
-label = Dummy()
+input = ImageInput(28, 28)
+label = LabelInput()
 
 nn = []
 nn.append(input)
@@ -97,11 +102,16 @@ print '[Start training]'
 for i in range(x.shape[0] / batchsize):
     xb, yb = x[i*batchsize:(i+1)*batchsize,:], y[i*batchsize:(i+1)*batchsize,:]
     nn[0].Feed(xb)
-    label.Feed(yb, is_label=1)
+    label.Feed(yb)
     for h in range(1, len(nn)):
         nn[h].ComputeFeature(nn[h-1])
     loss.ComputeFeature(nn[-1], label)
     if (i+1)%disp_freq == 0:
         print '  Step {:>3}: '.format(i+1),
         loss.display()
-    loss.ComputeGradient(i+1, sgd)
+
+    loss.ComputeGradient()
+    for h in range(len(nn)-1, 0, -1):
+        nn[h].ComputeGradient()
+        sgd.Update(i+1, nn[h])
+

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5a8fc373/tool/python/singa/driver.i
----------------------------------------------------------------------
diff --git a/tool/python/singa/driver.i b/tool/python/singa/driver.i
index 1650145..55cec02 100644
--- a/tool/python/singa/driver.i
+++ b/tool/python/singa/driver.i
@@ -79,7 +79,7 @@ namespace singa{
   class DummyLayer{
     public:
       void Setup(const std::string str, const std::vector<singa::Layer*>& srclayers);
-      void Feed(int batchsize, std::vector<float>& data, int is_aux);
+      void Feed(int batchsize, std::vector<float>& data, std::vector<int>&
aux_data);
       singa::Layer* ToLayer();
   };
 

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5a8fc373/tool/python/singa/layer.py
----------------------------------------------------------------------
diff --git a/tool/python/singa/layer.py b/tool/python/singa/layer.py
index 35da9aa..f786245 100644
--- a/tool/python/singa/layer.py
+++ b/tool/python/singa/layer.py
@@ -101,33 +101,21 @@ class Layer(object):
 
         self.singalayer.ComputeFeature(1, self.singaSrclayerVector)
 
-    def ComputeGradient(self, step, upd=None):
+    def ComputeGradient(self):
         ''' The method creates singa::Updater
             and calls ComputeGradient for gradient computation
             then updates the parameters.
-
-            step = (int)    // a training step
-            upd = (object)  // Updater object
         '''
-        # create singa::Updater
-        assert upd != None, 'required Updater (see model.py)'
-        if Layer.singaupdater == None:
-            Layer.singaupdater = SingaUpdater.CreateUpdater(
-                                          upd.proto.SerializeToString())
-
         # call ComputeGradient of Singa
         self.singalayer.ComputeGradient(1, self.singaSrclayerVector)
 
+    def UpdateParams(self, step, upd):
+        ''' The method updates parameter values
+        '''
         # update parameters
         singaParams = self.singalayer.GetParams()
         for par in singaParams:
-            Layer.singaupdater.Update(step, par, 1.0)
-
-        # recursively call ComputeGradient of srclayers
-        #(TODO) what if there are multiple source layers?
-        for sly in self.srclayers:
-            if sly.srclayers != None:
-                sly.ComputeGradient(step, upd)
+            upd.singaupdater.Update(step, par, 1.0)
 
     def GetParams(self):
         ''' The method gets parameter values
@@ -181,15 +169,13 @@ class Layer(object):
 
 class Dummy(object):
 
-    def __init__(self, shape=[], path='', dtype='', src=[], **kwargs):
+    def __init__(self, **kwargs):
         ''' Dummy layer is used for data layer to feed/fetch input data
-            shape = (list)   // [# of samples, # of channels, img h, img w]
-            path  = (string) // path to dataset
+            or label information
         '''
         self.is_datalayer = True
         self.srclayers = None
         self.singalayer = None
-        if 'is_label' in kwargs: self.is_label = kwargs['is_label']
 
         # create layer proto for Dummy layer
         kwargs = {'name':'dummy', 'type':kDummy}
@@ -206,155 +192,64 @@ class Dummy(object):
             self.singalayer.Setup(self.layer.SerializeToString(),
                                   layerVector(0))
 
-    def Feed(self, data, nb_channel=1, is_label=0):
+    def Feed(self, shape, data, aux_data):
         ''' Create and Setup singa::DummyLayer for input data
             Insert data using Feed()
         '''
-        batchsize, hdim = data.shape
+        batchsize = shape[0]
+        hdim = reduce(lambda x, y: x*y, shape[1:])
         datasize = batchsize * hdim
 
         # create and setup the dummy layer
         if self.singalayer == None:
-            imgsize = int(np.sqrt(hdim/nb_channel))
-            shapeVector = [batchsize, nb_channel, imgsize, imgsize]
-            self.setup(shapeVector)
-
-        data = data.astype(np.float)
-        dataVector = floatVector(datasize)
-        k = 0
-        for i in range(batchsize):
-            for j in range(hdim):
-                dataVector[k] = data[i, j]
-                k += 1
-        self.singalayer.Feed(batchsize, dataVector, is_label)
-
-    def FetchData(self, batchsize):
-        sidx = self.batch_index * batchsize
-        eidx = sidx + batchsize
-        batch = self.data[sidx:eidx, :]
-
-        self.Feed(batch, self.shape[1], self.is_label)
-
-        self.batch_index += 1
-        if eidx > self.data.shape[0]:
-            self.batch_index = 0
+            self.setup(shape)
+
+        if data != None:
+            data = data.astype(np.float)
+            dataVector = floatVector(datasize)
+            for i in range(batchsize):
+                for j in range(hdim):
+                    dataVector[i*hdim+j] = data[i, j]
+            labelVector = intVector(0)
+
+        if aux_data != None:
+            aux_data = aux_data.astype(np.int)
+            labelVector = intVector(datasize)
+            for i in range(batchsize):
+                labelVector[i] = aux_data[i, 0]
+            dataVector = floatVector(0)
+
+        self.singalayer.Feed(batchsize, dataVector, labelVector)
 
     def get_singalayer(self):
         return self.singalayer.ToLayer()
 
-class ImageData(Dummy):
-    ''' This class will be used for Rafiki, dlaas
+class ImageInput(Dummy):
+    ''' This class is used to feed image data
     '''
-    def __init__(self, shape=[], data_path='', data_type='byte', src=[],
-                 mean_path='', mean_type='float'):
-        ''' Dummy layer is used for data layer
-            shape = (list)   // [# of samples, # of channels, img h, img w]
-            data_path  = (string) // path to dataset
-            mean_path
-        '''
-        is_label = False
-        super(ImageData, self).__init__(shape, data_path, data_type, src,
-                                    is_label=is_label,
-                                    mean_path=mean_path,
-                                    mean_type=mean_type)
-
-        # if dataset path is not specified, skip
-        # otherwise, load dataset
-        if data_path == '' or mean_path == '':
-            return
-
-        self.shape = shape
-        self.data_path = data_path
-        self.mean_path = mean_path
-        self.src = None
-        self.batch_index = 0
-
-        nb_samples = shape[0]
-        nb_pixels = shape[1]
-        for i in range(len(shape)-2):
-            nb_pixels *= shape[i+2]
-
-        if data_type == 'byte':
-            d = np.fromfile(data_path, dtype=np.uint8)
-        elif data_type == 'int':
-            d = np.fromfile(data_path, dtype=np.int)
-        self.data = d.reshape(nb_samples, nb_pixels)
-
-        if mean_type == 'float':
-            d = np.fromfile(mean_path, dtype=np.float32)
-        self.mean = d.reshape(1, nb_pixels)
-
-    def Feed(self, data, nb_channel=1, is_label=0):
-        ''' Create and Setup singa::DummyLayer for input data
-            Insert data using Feed()
-            Need to minus the mean file
-        '''
-        batchsize, hdim = data.shape
-        datasize = batchsize * hdim
-
-        # create and setup the dummy layer
-        if self.singalayer == None:
-            imgsize = int(np.sqrt(hdim/nb_channel))
-            shapeVector = [batchsize, nb_channel, imgsize, imgsize]
-            self.setup(shapeVector)
-
-        # feed input data and minus mean
-        data = data.astype(np.float)
-        dataVector = floatVector(datasize)
-        k = 0
-        for i in range(batchsize):
-            for j in range(hdim):
-                dataVector[k] = data[i, j]-self.mean[0, j]
-                k += 1
-        self.singalayer.Feed(batchsize, dataVector, is_label)
-
-class LabelData(Dummy):
-    ''' This class will be used for Rafiki, dlaas
+    def __init__(self, width=None, height=None, nb_channel=1):
+        super(ImageInput, self).__init__()
+        self.width = width
+        self.height = height
+        self.nb_channel = nb_channel
+
+    def Feed(self, image_data):
+        batchsize = image_data.shape[0]
+        if self.width == None or self.height == None:
+            hdim = image_data.shape[1]
+            imgsize = int(np.sqrt(hdim/self.nb_channel))
+        shape = [batchsize, self.nb_channel, self.width, self.height]
+        Dummy.Feed(self, shape, image_data, None)
+
+class LabelInput(Dummy):
+    ''' This class is used to feed label data
     '''
-    def __init__(self, shape=[], label_path='', label_type='int', src=[]):
-        ''' Dummy layer is used for label data layer
-            shape = (list)   // [# of samples, # of channels, img h, img w]
-            data_path  = (string) // path to dataset
-            mean_path
-        '''
-        is_label = True
-        super(LabelData, self).__init__(shape, label_path, label_type, src,
-                                    is_label=is_label)
-
-        # if dataset path is not specified, skip
-        # otherwise, load dataset
-        if label_path == '':
-            return
-
-        self.shape = shape
-        self.label_path = label_path
-        self.src = None
-        self.batch_index = 0
-
-        nb_samples = shape[0]
-
-        if label_type == 'int':
-            d = np.fromfile(label_path, dtype=np.int)
-        self.data = d.reshape(nb_samples, 1)
-
-    def Feed(self, data, nb_chanel=1, is_label=1):
-        ''' Create and Setup singa::DummyLayer for input data
-            Insert data using Feed()
-            Need to minus the mean file
-        '''
-        batchsize = data.shape[0]
-
-        # create and setup the dummy layer
-        if self.singalayer == None:
-            shapeVector = [batchsize, 1]
-            self.setup(shapeVector)
-
-        data = data.astype(np.float)
-        dataVector = floatVector(batchsize)
-        for i in range(batchsize):
-            dataVector[i] = data[i, 0]
-        self.singalayer.Feed(batchsize, dataVector, 1)
+    def __init__(self):
+        super(LabelInput, self).__init__()
 
+    def Feed(self, label_data):
+        Dummy.Feed(self, label_data.shape, None, label_data)
+    
 
 class Data(Layer):
 

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5a8fc373/tool/python/singa/model.py
----------------------------------------------------------------------
diff --git a/tool/python/singa/model.py b/tool/python/singa/model.py
index b6664bd..4a6a688 100644
--- a/tool/python/singa/model.py
+++ b/tool/python/singa/model.py
@@ -32,6 +32,8 @@ from singa.utils.utility import *
 from singa.utils.message import *
 from google.protobuf import text_format
 
+from singa.driver import Updater as SingaUpdater
+
 class Model(object):
     ''' Configure model parameter
         - add(): add layer
@@ -487,7 +489,21 @@ class Updater(object):
             setval(upd.learning_rate, type=kLinear, linear_conf=cp.proto)
 
         self.proto = upd
+        self.singaupdater = None
 
+    def Update(self, step, layer):
+        ''' This method updates parameters of layer
+            step = (int)  // training step, i.e., param version
+        '''
+        if self.singaupdater == None:
+            self.singaupdater = SingaUpdater.CreateUpdater(
+                                  self.proto.SerializeToString())
+
+        # update parameters
+        singaParams = layer.singalayer.GetParams()
+        for par in singaParams:
+            self.singaupdater.Update(step, par, 1.0)
+    
 
 class SGD(Updater):
 

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5a8fc373/tool/python/singa/utils/utility.py
----------------------------------------------------------------------
diff --git a/tool/python/singa/utils/utility.py b/tool/python/singa/utils/utility.py
index bea784d..b88720c 100644
--- a/tool/python/singa/utils/utility.py
+++ b/tool/python/singa/utils/utility.py
@@ -84,8 +84,3 @@ def setval(proto, **kwargs):
                 else:
                     setattr(proto, key, val)
 
-def swap32(x):
-    return (((x << 24) & 0xFF000000) |
-            ((x <<  8) & 0x00FF0000) |
-            ((x >>  8) & 0x0000FF00) |
-            ((x >> 24) & 0x000000FF))


Mime
View raw message