singa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wang...@apache.org
Subject [4/4] incubator-singa git commit: SINGA-348 Support autograd MLP Example
Date Thu, 12 Apr 2018 09:16:07 GMT
SINGA-348 Support autograd MLP Example

rename some variables; and add more checks for Dummy operations.

move mlp.py into example/autograd; I can run mlp.py without any bugs.


Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/060e7dfe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/060e7dfe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/060e7dfe

Branch: refs/heads/master
Commit: 060e7dfe1cc847500e4beb115f3b24e923288b3e
Parents: f42d4d0
Author: Wang Wei <dcswaw@nus.edu.sg>
Authored: Wed Apr 11 21:09:45 2018 +0800
Committer: Wang Wei <dcswaw@nus.edu.sg>
Committed: Thu Apr 12 16:59:48 2018 +0800

----------------------------------------------------------------------
 examples/MLP.py          | 83 -------------------------------------------
 examples/autograd/mlp.py | 74 ++++++++++++++++++++++++++++++++++++++
 python/singa/autograd.py | 33 ++++++++++-------
 python/singa/tensor.py   | 58 +++++++++++++++++++-----------
 4 files changed, 131 insertions(+), 117 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/060e7dfe/examples/MLP.py
----------------------------------------------------------------------
diff --git a/examples/MLP.py b/examples/MLP.py
deleted file mode 100644
index b773efb..0000000
--- a/examples/MLP.py
+++ /dev/null
@@ -1,83 +0,0 @@
-from singa import tensor
-from singa import autograd
-from singa import optimizer
-import numpy as np
-
-
-if __name__ == '__main__':
-
-    # prepare training data in numpy array
-
-    # generate the boundary
-    f = lambda x: (5 * x + 1)
-    bd_x = np.linspace(-1., 1, 200)
-    bd_y = f(bd_x)
-    # generate the training data
-    x = np.random.uniform(-1, 1, 400)
-    y = f(x) + 2 * np.random.randn(len(x))
-    # convert training data to 2d space
-    label = np.asarray([5 * a + 1 > b for (a, b) in zip(x, y)])
-    data = np.array([[a, b] for (a, b) in zip(x, y)], dtype=np.float32)
-
-    def to_categorical(y, num_classes=None):
-        '''
-        Converts a class vector (integers) to binary class matrix.
-
-        Args
-            y: class vector to be converted into a matrix
-                (integers from 0 to num_classes).
-            num_classes: total number of classes.
-
-        Return
-            A binary matrix representation of the input.
-        '''
-        y = np.array(y, dtype='int')
-        input_shape = y.shape
-        if input_shape and input_shape[-1] == 1 and len(input_shape) > 1:
-            input_shape = tuple(input_shape[:-1])
-        y = y.ravel()
-        if not num_classes:
-            num_classes = np.max(y) + 1
-        n = y.shape[0]
-        categorical = np.zeros((n, num_classes))
-        categorical[np.arange(n), y] = 1
-        output_shape = input_shape + (num_classes,)
-        categorical = np.reshape(categorical, output_shape)
-        return categorical
-
-    label = to_categorical(label, 2).astype(np.float32)
-    print('train_data_shape:', data.shape)
-    print('train_label_shape:', label.shape)
-
-    inputs = tensor.Tensor(data=data, requires_grad=False)
-    target = tensor.Tensor(data=label, requires_grad=False)
-
-    w0 = tensor.Tensor(shape=(2, 3), requires_grad=True, stores_grad=True)
-    w0.gaussian(0.0, 0.1)
-    b0 = tensor.Tensor(shape=(1, 3), requires_grad=True, stores_grad=True)
-    b0.set_value(0.0)
-
-    w1 = tensor.Tensor(shape=(3, 2), requires_grad=True, stores_grad=True)
-    w1.gaussian(0.0, 0.1)
-    b1 = tensor.Tensor(shape=(1, 2), requires_grad=True, stores_grad=True)
-    b1.set_value(0.0)
-
-    sgd = optimizer.SGD(0.05)
-    # training process
-    for i in range(1001):
-        x = tensor.matmul(inputs, w0)
-        x = tensor.add_bias(x, b0)
-        x = tensor.relu(x)
-        x = tensor.matmul(x, w1)
-        x = tensor.add_bias(x, b1)
-        x = tensor.softmax(x)
-        loss = tensor.cross_entropy(x, target)
-        in_grads = autograd.backward(loss)
-
-        for param in in_grads:
-            sgd.apply(0, in_grads[param], param, '')
-
-        if (i % 100 == 0):
-            print('training loss = ', tensor.to_numpy(loss)[0])
-
-

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/060e7dfe/examples/autograd/mlp.py
----------------------------------------------------------------------
diff --git a/examples/autograd/mlp.py b/examples/autograd/mlp.py
new file mode 100644
index 0000000..7352c21
--- /dev/null
+++ b/examples/autograd/mlp.py
@@ -0,0 +1,74 @@
+from singa import tensor
+from singa.tensor import Tensor
+from singa import autograd
+from singa import optimizer
+import numpy as np
+
+
+if __name__ == '__main__':
+
+    # prepare training data in numpy array
+
+    # generate the boundary
+    f = lambda x: (5 * x + 1)
+    bd_x = np.linspace(-1., 1, 200)
+    bd_y = f(bd_x)
+    # generate the training data
+    x = np.random.uniform(-1, 1, 400)
+    y = f(x) + 2 * np.random.randn(len(x))
+    # convert training data to 2d space
+    label = np.asarray([5 * a + 1 > b for (a, b) in zip(x, y)])
+    data = np.array([[a, b] for (a, b) in zip(x, y)], dtype=np.float32)
+
+    def to_categorical(y, num_classes):
+        '''
+        Converts a class vector (integers) to binary class matrix.
+
+        Args
+            y: class vector to be converted into a matrix
+                (integers from 0 to num_classes).
+            num_classes: total number of classes.
+
+        Return
+            A binary matrix representation of the input.
+        '''
+        y = np.array(y, dtype='int')
+        n = y.shape[0]
+        categorical = np.zeros((n, num_classes))
+        categorical[np.arange(n), y] = 1
+        return categorical
+
+    label = to_categorical(label, 2).astype(np.float32)
+    print('train_data_shape:', data.shape)
+    print('train_label_shape:', label.shape)
+
+    inputs = Tensor(data=data)
+    target = Tensor(data=label)
+
+    w0 = Tensor(shape=(2, 3), requires_grad=True, stores_grad=True)
+    w0.gaussian(0.0, 0.1)
+    b0 = Tensor(shape=(1, 3), requires_grad=True, stores_grad=True)
+    b0.set_value(0.0)
+
+    w1 = Tensor(shape=(3, 2), requires_grad=True, stores_grad=True)
+    w1.gaussian(0.0, 0.1)
+    b1 = Tensor(shape=(1, 2), requires_grad=True, stores_grad=True)
+    b1.set_value(0.0)
+
+    sgd = optimizer.SGD(0.05)
+    # training process
+    for i in range(1001):
+        x = tensor.matmul(inputs, w0)
+        x = tensor.add_bias(x, b0)
+        x = tensor.relu(x)
+        x = tensor.matmul(x, w1)
+        x = tensor.add_bias(x, b1)
+        x = tensor.soft_max(x)
+        loss = tensor.cross_entropy(x, target)
+        in_grads = autograd.backward(loss)
+
+        for param in in_grads:
+            sgd.apply(0, in_grads[param], param, '')
+
+        if (i % 100 == 0):
+            print('training loss = ', tensor.to_numpy(loss)[0])

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/060e7dfe/python/singa/autograd.py
----------------------------------------------------------------------
diff --git a/python/singa/autograd.py b/python/singa/autograd.py
index 399fa19..175f8b2 100644
--- a/python/singa/autograd.py
+++ b/python/singa/autograd.py
@@ -1,6 +1,5 @@
 from collections import Counter, deque
-from singa import tensor
-
+from .tensor import Tensor, Dummy
 
 
 def infer_dependency(op):
@@ -23,7 +22,8 @@ def infer_dependency(op):
     while len(queue) > 0:
         cur_op = queue.pop()
         for src_op, _, _, _ in cur_op.src:
-            if src_op not in dependency_count:
+            if src_op not in dependency_count and \
+                    (not isinstance(src_op, Dummy)):
                 # dependency[src_op] = [Counter() for _ in src_op.y_id2idx]
                 dependency_count[src_op] = 0
                 queue.append(src_op)
@@ -53,7 +53,7 @@ def backward(y, dy=None):
     # by default the dy is a tensor with 1.0 for each sample;
     if dy is None:
         dy = float(1.0)
-    elif isinstance(dy, tensor.Tensor):
+    elif isinstance(dy, Tensor):
         dy = dy.data
     else:
         dy = float(dy)
@@ -62,17 +62,22 @@ def backward(y, dy=None):
     ready = deque([(y.creator, (dy,))])
     not_ready = {}  # mapping: op->[dy]
     gradients = {}  # mapping: x->dx if x.stores_grad
+    if y.stores_grad:
+        gradients[y] = dy
 
     while len(ready) > 0:
         op, dys = ready.pop()
-        #if not isinstance(op, tensor.Dummy):
+        if not op.requires_grad or isinstance(op, Dummy):
+            continue
+        # if not isinstance(op, tensor.Dummy):
         dxs = op._do_backward(*dys)
         # TODO src and dx must match
         assert len(op.src) == len(dxs), \
             'the number of src ops (=%d) and dx (=%d) not match' \
             % (len(op.src), len(dxs))
-        for (src_op, x_id, param, x_stores_grad), dx in zip(op.src, dxs):
-            # x_id is the python id of one input arg of op, denoted as x.
+        for (src_op, x_id, y, y_stores_grad), dx in zip(op.src, dxs):
+            # prefix x is w.r.t op; prefix y is w.r.t src_op.
+            # x_id is the python id of one input arg of src_op, denoted as x.
             # y_idx (below) is the index of x among the outputs of src_op.
             # not_ready[src_op][y_idx] records the intermediate gradient
             # of the y_idx'th output of src_op. 'intermediate gradient'
@@ -81,10 +86,11 @@ def backward(y, dy=None):
             # children operations. When src_op is ready, it means that
             # the gradient of all its outputs are available, i.e. all children
             # operations have been backwarded.
-            y_idx = src_op.y_ids[x_id]
+            # y is None if y.stores_grad is false; otherwise it is a Tensor
+            y_idx = src_op.y_id2idx[x_id]
             if src_op not in not_ready:
                 # src_op may have mulitple outputs
-                not_ready[src_op] = [None for _ in src_op.y_ids]
+                not_ready[src_op] = [None for _ in src_op.y_id2idx]
                 not_ready[src_op][y_idx] = dx
             else:
                 dxs = not_ready[src_op]
@@ -94,14 +100,15 @@ def backward(y, dy=None):
                     # add the gradient from another children operation that
                     # uses y_idx'th output of src_op as input arg
                     dxs[y_idx] += dx
-            if x_stores_grad:
+            if y_stores_grad:
                 # store the gradient for final return, e.g. if x is parameter
-                gradient = not_ready[src_op][y_idx]
-                gradients[param] = tensor.Tensor(device=gradient.device, data=gradient, requires_grad=False)
+                g = not_ready[src_op][y_idx]
+                gradients[y] = Tensor(device=g.device, data=g)
             dependency[src_op] -= 1
             if src_op.requires_grad is True:
                 if dependency[src_op] == 0:
-                    ready.append((src_op, not_ready[src_op]))
+                    if not isinstance(src_op, Dummy):
+                        ready.append((src_op, not_ready[src_op]))
                     del not_ready[src_op]
 
     return gradients

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/060e7dfe/python/singa/tensor.py
----------------------------------------------------------------------
diff --git a/python/singa/tensor.py b/python/singa/tensor.py
index f4801a4..70a9302 100644
--- a/python/singa/tensor.py
+++ b/python/singa/tensor.py
@@ -1130,32 +1130,42 @@ class Operation(object):
     a Xxxx instance and calls __call__ to do forward. The autograd engine
     is able to do backward propagation by calling the backward() of Xxxx
     automatically. Notice that the tensors are CTensor. NOT Python Tensor.
-    The arguments of forward includes both CTensor instances and other
-    types of data; The backward function ONLY supports CTensor args.
+    The arguments of forward() and backward() should only include CTensor args; 
     '''
 
     def __call__(self, *xs):
         return self._do_forward(*xs)
 
     def _do_forward(self, *xs):
+        '''
+        Do not call this function from user code. It is called by __call__().
+
+        Args:
+            xs, Tensor instance(s)
+
+        Returns:
+            Tensor instance(s)
+        '''
         # TODO add the pre hook
-        # filter out args that are not Tensor instances
-        tensor_xs = [x for x in xs if isinstance(x, Tensor)]
+        assert all([isinstance(x, Tensor) for x in xs]), \
+            'xs should include only Tensor instances'
 
         # need to do backward if any of its input arg needs gradient
-        self.requires_grad = any([x.requires_grad for x in tensor_xs])
-        # src records info of every input arg that needs gradient
-        # the backward() function computes grad only for those arg
+        self.requires_grad = any([x.requires_grad for x in xs])
 
         self.src = []
-        for x in tensor_xs:
+        for x in xs:
             if x.stores_grad:
+                # store the tensor whose gradient needs be returned in
+                # backward(), e.g. if x is parameter
                 self.src.append((x.creator, id(x), x, x.stores_grad))
             else:
+                # for intermediate tensors, they will be released soon;
+                # no need to store them --> use None
                 self.src.append((x.creator, id(x), None, x.stores_grad))
 
-        # use the CTensor (data) if the input arg is Tensor
-        xs = tuple(x.data if isinstance(x, Tensor) else x for x in xs)
+        # get the CTensor (data) if the input arg is Tensor
+        xs = tuple(x.data for x in xs)
         ys = self.forward(*xs)
         if not isinstance(ys, tuple):
             ys = (ys,)
@@ -1166,7 +1176,7 @@ class Operation(object):
                           requires_grad=self.requires_grad,
                           creator=self) for y in ys)
         # map from python id to output index
-        self.y_ids = {id(y): i for i, y in enumerate(ys)}
+        self.y_id2idx = {id(y): i for i, y in enumerate(ys)}
         # TODO add the post hook
         return ys
 
@@ -1180,7 +1190,7 @@ class Operation(object):
         '''Forward propagation.
 
         Args:
-            xs: input args consisting of CTensors and others.
+            xs: input args consisting of only CTensors.
 
         Returns:
             CTensor instance(s)
@@ -1191,7 +1201,7 @@ class Operation(object):
         ''' Backward propagation.
 
         Args:
-            dys: input args consisting of CTensors.
+            dys: input args consisting of only CTensors.
 
         Returns:
             CTensor instance(s)
@@ -1209,7 +1219,7 @@ class Dummy(Operation):
     def __init__(self, tensor, name=None):
         self.name = name
         self.src = []
-        self.y_ids = {id(tensor): 0}
+        self.y_id2idx = {id(tensor): 0}
         self.requires_grad = False
 
 
@@ -1268,7 +1278,8 @@ class Matmul(Operation):
         Returns:
             a tuple for (dx, dw)
         '''
-        return singa.Mult(dy, self.input[1].T()), singa.Mult(self.input[0].T(), dy)
+        return singa.Mult(dy, self.input[1].T()), \
+            singa.Mult(self.input[0].T(), dy)
 
 
 def matmul(x, w):
@@ -1279,10 +1290,11 @@ class AddBias(Operation):
     '''
     Add Bias to each row / column of the Tensor, depending on the parameter axis.
     '''
+
     def __init__(self, axis=0):
         '''
         To indicate the calculation axis, 0 for row, 1 for column.
-        
+
         Args:
             axis: 0 or 1, default is 0.
         '''
@@ -1327,6 +1339,7 @@ class SoftMax(Operation):
     Apply SoftMax for each row of the Tensor or each column of the Tensor
     according to the parameter axis.
     '''
+
     def __init__(self, axis=0):
         self.axis = axis
 
@@ -1372,7 +1385,7 @@ class SoftMax(Operation):
             return dx.T()
 
 
-def softmax(x, axis=0):
+def soft_max(x, axis=0):
     return SoftMax(axis)(x)[0]
 
 
@@ -1381,6 +1394,7 @@ class CrossEntropy(Operation):
     Calculte CrossEntropy loss for a batch of training data.
 
     '''
+
     def forward(self, x, t):
         '''
         Args:
@@ -1393,10 +1407,12 @@ class CrossEntropy(Operation):
         loss = CTensor((1,))
         loss_data = -singa.SumAsFloat(singa.__mul__(t, singa.Log(x)))
         loss.SetFloatValue(loss_data / x.shape()[0])
+        self.x = x
+        self.t = t
         self.input = (x, t)
         return loss
 
-    def backward(self, dy):
+    def backward(self, dy=1.0):
         '''
         Args:
             dy (float or CTensor): scalar, accumulate gradient from outside of current network,
usually
@@ -1406,8 +1422,8 @@ class CrossEntropy(Operation):
             dx (CTensor): data for the dL /dx, L is the loss, x is the output of current
network.
             note that this is true for dy = 1.0
         '''
-        dx = singa.__div__(self.input[1], self.input[0])
-        dx *= float(-1 / self.input[0].shape()[0])
+        dx = singa.__div__(self.t, self.x)
+        dx *= float(-1 / self.x.shape()[0])
         if isinstance(dy, float):
             # dtype of dy: float
             dx *= dy
@@ -1426,4 +1442,4 @@ def ctensor2numpy(x):
     Convert a singa_tensor to numpy_tensor.
     '''
     np_array = x.GetFloatValue(int(x.Size()))
-    return np_array.reshape(x.shape())
\ No newline at end of file
+    return np_array.reshape(x.shape())


Mime
View raw message