From commits-return-2396-archive-asf-public=cust-asf.ponee.io@singa.incubator.apache.org Wed Aug 22 16:55:03 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 9B788180679 for ; Wed, 22 Aug 2018 16:55:02 +0200 (CEST) Received: (qmail 69404 invoked by uid 500); 22 Aug 2018 14:55:01 -0000 Mailing-List: contact commits-help@singa.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@singa.incubator.apache.org Delivered-To: mailing list commits@singa.incubator.apache.org Received: (qmail 69395 invoked by uid 99); 22 Aug 2018 14:55:01 -0000 Received: from pnap-us-west-generic-nat.apache.org (HELO spamd2-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 22 Aug 2018 14:55:01 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd2-us-west.apache.org (ASF Mail Server at spamd2-us-west.apache.org) with ESMTP id 60B3A1A2773 for ; Wed, 22 Aug 2018 14:55:01 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd2-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: -11.701 X-Spam-Level: X-Spam-Status: No, score=-11.701 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, RCVD_IN_DNSWL_HI=-5, SPF_PASS=-0.001, USER_IN_DEF_SPF_WL=-7.5] autolearn=disabled Received: from mx1-lw-eu.apache.org ([10.40.0.8]) by localhost (spamd2-us-west.apache.org [10.40.0.9]) (amavisd-new, port 10024) with ESMTP id nge8KwRdLa42 for ; Wed, 22 Aug 2018 14:55:00 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-lw-eu.apache.org (ASF Mail Server at mx1-lw-eu.apache.org) with SMTP id 026F25F434 for ; Wed, 22 Aug 2018 14:54:58 +0000 (UTC) Received: (qmail 69133 invoked by uid 99); 22 Aug 2018 14:54:58 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 22 Aug 2018 14:54:58 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 9A924E0255; Wed, 22 Aug 2018 14:54:57 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: wangwei@apache.org To: commits@singa.incubator.apache.org Date: Wed, 22 Aug 2018 14:54:59 -0000 Message-Id: <1271cc1e2e404389bb2b726cc30dceb4@git.apache.org> In-Reply-To: <031dec41c514478aa33afd9a736bb7d6@git.apache.org> References: <031dec41c514478aa33afd9a736bb7d6@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [3/5] incubator-singa git commit: SINGA-388 Develop some RNN layers by calling tiny operations like matmul, addbias SINGA-388 Develop some RNN layers by calling tiny operations like matmul, addbias - develop Vanilla RNN by calling some smaller operations. - add some necessary operations which will be used in Vanilla RNN layer. - the developed RNN layer has passed test.(return correct number of gradients) Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/7df6a5db Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/7df6a5db Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/7df6a5db Branch: refs/heads/master Commit: 7df6a5db4b7ef30aca561889e46e2457b35b15c3 Parents: 770d6cd Author: xuewanqi Authored: Mon Aug 13 13:44:29 2018 +0000 Committer: xuewanqi Committed: Thu Aug 16 11:40:25 2018 +0000 ---------------------------------------------------------------------- python/singa/autograd.py | 74 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7df6a5db/python/singa/autograd.py ---------------------------------------------------------------------- diff --git a/python/singa/autograd.py b/python/singa/autograd.py old mode 100755 new mode 100644 index 56b5498..4c7959c --- a/python/singa/autograd.py +++ b/python/singa/autograd.py @@ -87,6 +87,7 @@ def backward(y, dy=None): a dictionary storing the gradient tensors of all tensors whose stores_grad is true (e.g. parameter tensors) ''' + assert isinstance(y, Tensor), 'wrong input type.' dependency = infer_dependency(y.creator) assert y.size() == 1, 'y must be a Tensor with a single value;'\ 'size of y is % d' % y.size() @@ -172,6 +173,7 @@ def backward(y, dy=None): del op # delete the operation to free all tensors from this op + class Operation(object): ''' An operation includes the forward and backward function of @@ -962,3 +964,75 @@ class AvgPool1d(Pooling2d): stride = kernel_size super(MaxPool2d, self).__init__( (1, kernel_size), (0, stride), (0, padding), False) + + +class Tanh(Operation): + + def forward(self, x): + out = singa.Tanh(x) + if training: + self.cache = (out,) + return out + + def backward(self, dy): + dx = singa.__mul__(self.cache[0], self.cache[0]) + dx = singa.MultFloat(dx, -1.0) + dx = singa.AddFloat(dx, 1.0) + dx = singa.__mul__(dy, dx) + return dx + + +def tanh(x): + return Tanh()(x)[0] + + +def add_all(*xs): + assert len(xs) > 2 + y=add(xs[0],xs[1]) + for x in xs[2:]: + y=add(y, x) + return + + +class Vanilla_RNN(Layer): + + def __init__(self, input_size, hidden_size, num_layers=1, nonlinearity='tanh', bias=True, batch_first=False, dropout=0, bidirectional=False): + self.nonlinearity=nonlinearity + + Wx_shape = (input_size, hidden_size) + self.Wx = Tensor(shape=Wx_shape, requires_grad=True, stores_grad=True) + self.Wx.gaussian(0.0, 1.0) + + Wh_shape = (hidden_size, hidden_size) + self.Wh = Tensor(shape=Wh_shape, requires_grad=True, stores_grad=True) + self.Wh.gaussian(0.0, 1.0) + + B_shape=(hidden_size,) + self.b = Tensor(shape=B_shape, requires_grad=True, stores_grad=True) + self.b.set_value(0.0) + + def __call__(self, h0, *xs): + batchsize=xs[0].shape[0] + self.out=[] + h = self.step_forward(xs[0], h0, self.Wx, self.Wh, self.b) + self.out.append(h) + for x in xs[1:]: + assert x.shape[0] == batchsize + h = self.step_forward(x, h, self.Wx, self.Wh, self.b) + self.out.append(h) + return self.out + + def step_forward(self, x, h, Wx, Wh, b): + y1=matmul(x, Wx) + y2=matmul(h, Wh) + y=add(y1,y2) + y=add_bias(y,b,axis=0) + if self.nonlinearity == 'tanh': + y=tanh(y) + elif self.nonlinearity == 'relu': + y=relu(y) + else: + raise ValueError + return y + +