From dev-return-5058-archive-asf-public=cust-asf.ponee.io@singa.apache.org Sat Apr 11 03:09:52 2020 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [207.244.88.153]) by mx-eu-01.ponee.io (Postfix) with SMTP id 07E6918057A for ; Sat, 11 Apr 2020 05:09:51 +0200 (CEST) Received: (qmail 10848 invoked by uid 500); 11 Apr 2020 03:09:51 -0000 Mailing-List: contact dev-help@singa.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@singa.apache.org Delivered-To: mailing list dev@singa.apache.org Received: (qmail 10837 invoked by uid 99); 11 Apr 2020 03:09:50 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 11 Apr 2020 03:09:50 +0000 From: GitBox To: dev@singa.apache.org Subject: [GitHub] [singa] dcslin commented on a change in pull request #662: CUDNN LSTM Message-ID: <158657459036.13398.13427114860962814093.gitbox@gitbox.apache.org> References: In-Reply-To: Date: Sat, 11 Apr 2020 03:09:50 -0000 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit dcslin commented on a change in pull request #662: CUDNN LSTM URL: https://github.com/apache/singa/pull/662#discussion_r407009673 ########## File path: python/singa/autograd.py ########## @@ -3239,95 +3239,176 @@ def __init__( bidirectional (bool): If True, becomes a bidirectional RNN. Default: False """ - self.nonlinearity = nonlinearity - - Wx_shape = (input_size, hidden_size) - self.Wx = [] - for i in range(4): - w = Tensor(shape=Wx_shape, requires_grad=True, stores_grad=True) - w.gaussian(0.0, 1.0) - self.Wx.append(w) - - Wh_shape = (hidden_size, hidden_size) - self.Wh = [] - for i in range(4): - w = Tensor(shape=Wh_shape, requires_grad=True, stores_grad=True) - w.gaussian(0.0, 1.0) - self.Wh.append(w) - - Bx_shape = (hidden_size,) - self.Bx = [] - for i in range(4): - b = Tensor(shape=Bx_shape, requires_grad=True, stores_grad=True) - b.set_value(0.0) - self.Bx.append(b) - - self.Bh = [] - for i in range(4): - b = Tensor(shape=Bx_shape, requires_grad=True, stores_grad=True) - b.set_value(0.0) - self.Bh.append(b) - - self.params = self.Wx + self.Wh + self.Bx + self.Bh + self.backend = backend + if backend == "singa": + self.nonlinearity = nonlinearity + + Wx_shape = (input_size, hidden_size) + self.Wx = [] + for i in range(4): + w = Tensor(shape=Wx_shape, requires_grad=True, stores_grad=True) + w.gaussian(0.0, 1.0) + self.Wx.append(w) + + Wh_shape = (hidden_size, hidden_size) + self.Wh = [] + for i in range(4): + w = Tensor(shape=Wh_shape, requires_grad=True, stores_grad=True) + w.gaussian(0.0, 1.0) + self.Wh.append(w) + + Bx_shape = (hidden_size,) + self.Bx = [] + for i in range(4): + b = Tensor(shape=Bx_shape, requires_grad=True, stores_grad=True) + b.set_value(0.0) + self.Bx.append(b) + + self.Bh = [] + for i in range(4): + b = Tensor(shape=Bx_shape, requires_grad=True, stores_grad=True) + b.set_value(0.0) + self.Bh.append(b) + + self.params = self.Wx + self.Wh + self.Bx + self.Bh + elif backend == "cudnn": + if not singa.USE_CUDA: + raise Exception("Could not use cudnn without cuda compiled.\n") + if not inputs: + raise Exception("Input is required for init cudnn LSTM.\n") + + CUDNN_LSTM_MODE = 2 + + cinputs = singa.VecTensor() + [cinputs.append(i.data) for i in inputs] + self.rnn_handle = singa.CudnnRNNHandle(cinputs, input_size, + hidden_size, CUDNN_LSTM_MODE) + else: + raise Exception("Unsupported backend %s.\n" % backend) + + def cpp_vec_tensor_to_py_tensor(self, cpp_vec_tensor): + py_tensors = list() + for cTensor in cpp_vec_tensor: + new_t = tensor.Tensor() + new_t.data = cTensor + new_t.shape = tuple(new_t.data.shape()) + new_t.device = new_t.data.device() + new_t.dtype = new_t.data.data_type() + py_tensors.append(new_t) + return py_tensors def __call__(self, xs, h0_c0): - # xs: a tuple or list of input tensors - # h0_c0: a tuple of (h0, c0) - h0, c0 = h0_c0 - if not isinstance(xs, list): - xs = list(xs) - inputs = xs + list((h0, c0)) - self.device_check(*inputs) - # self.device_check(inputs[0], *self.params) - self.device_check(inputs[0], *(self.Wx + self.Wh + self.Bx + self.Bh)) - batchsize = xs[0].shape[0] - out = [] - h, c = self.step_forward(xs[0], h0, c0, self.Wx, self.Wh, self.Bx, - self.Bh) - out.append(h) - for x in xs[1:]: - assert x.shape[0] == batchsize - h, c = self.step_forward(x, h, c, self.Wx, self.Wh, self.Bx, + if self.backend == "singa": Review comment: thank you for the explanation, this is changed to layer-operation model. ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: users@infra.apache.org With regards, Apache Git Services