singa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zhon...@apache.org
Subject incubator-singa git commit: SINGA-217 build python package with setup.py
Date Sat, 02 Jul 2016 13:08:58 GMT
Repository: incubator-singa
Updated Branches:
  refs/heads/dev c50255259 -> 37e6ad283


SINGA-217 build python package with setup.py

 - modify CMakefiles to build a standard python singa package
 - remove all hard code path
 - after cmake and make, can run "python unittest.py" under "/test/python/",
   currently the "build" path is hard code in unittest.py
 - user can install singa package by command "sudo pip install ."
   under "build/python" directory (better run in python virltualenv)
   make sure libsinga is installed (make install). After this, user can remove
   hard code path in the unittest.py.


Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/37e6ad28
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/37e6ad28
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/37e6ad28

Branch: refs/heads/dev
Commit: 37e6ad283ae8b095b1a675a8df3bdc40cfcbd544
Parents: c502552
Author: aaronwwf <dcswuw@gmail.com>
Authored: Sat Jul 2 17:15:23 2016 +0800
Committer: aaronwwf <dcswuw@gmail.com>
Committed: Sat Jul 2 20:46:44 2016 +0800

----------------------------------------------------------------------
 CMakeLists.txt                     | 28 +++++-------
 cmake/Protobuf.cmake               |  6 +--
 cmake/Utils.cmake                  |  6 ++-
 src/CMakeLists.txt                 | 16 ++++++-
 src/python/__init__.py             |  3 ++
 src/python/device.py               |  6 +--
 src/python/layer.py                |  8 ++--
 src/python/model.py                |  2 +-
 src/python/setup.py.in             | 76 ++++++++++++++++++++++++++++++++
 src/python/tensor.py               | 12 ++---
 test/python/example_test_tensor.py | 78 ++++++++++++++++-----------------
 test/python/test_tensor.py         | 14 +++---
 12 files changed, 164 insertions(+), 91 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/37e6ad28/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index a8be759..9eac5ee 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -6,13 +6,6 @@ SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
 LIST(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/Thirdparty)
 #message(STATUS "module path: ${CMAKE_MODULE_PATH}")
 
-OPTION(USE_CBLAS "Use CBlas libs" ON)
-OPTION(USE_CUDA "Use Cuda libs" ON)
-OPTION(USE_CUDNN "Use Cudnn libs" ON)
-OPTION(USE_OPENCV "Use opencv" OFF)
-OPTION(USE_LMDB "Use LMDB libs" OFF)
-OPTION(USE_PYTHON "Generate py wrappers" OFF)
-
 # Flags
 IF(UNIX OR APPLE)
   SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -Wall")
@@ -22,14 +15,16 @@ IF(CMAKE_BUILD_TYPE=Debug)
 ENDIF()
 #message(STATUS "${CMAKE_CXX_FLAGS}")
 SET(SINGA_INCLUDE_DIR
-    "${CMAKE_SOURCE_DIR}/include;${PROJECT_BINARY_DIR}")
-IF(USE_CUDA)
-    LIST(APPEND SINGA_INCLUDE_DIR "${CMAKE_SOURCE_DIR}/lib/cnmem/include")
-ELSEIF(USE_CUDA)
-    OPTION(USE_CUDNN "Use Cudnn libs" OFF)
-ENDIF()
+    "${CMAKE_SOURCE_DIR}/include;${CMAKE_SOURCE_DIR}/lib/cnmem/include;${PROJECT_BINARY_DIR}")
 INCLUDE_DIRECTORIES(${SINGA_INCLUDE_DIR})
 
+OPTION(USE_CBLAS "Use CBlas libs" ON)
+OPTION(USE_CUDA "Use Cuda libs" ON)
+OPTION(USE_CUDNN "Use Cudnn libs" ON)
+OPTION(USE_OPENCV "Use opencv" OFF)
+OPTION(USE_LMDB "Use LMDB libs" OFF)
+OPTION(USE_PYTHON "Generate py wrappers" ON)
+
 INCLUDE("cmake/Dependencies.cmake")
 INCLUDE("cmake/Utils.cmake")
 ADD_DEFINITIONS(-DUSE_CMAKE)
@@ -44,11 +39,8 @@ CONFIGURE_FILE (
 SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/lib)
 SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/bin)
 
-IF(USE_CUDA)
-    ADD_SUBDIRECTORY(lib/cnmem)
-    LIST(APPEND SINGA_LINKER_LIBS cnmem)
-ENDIF()
-
+ADD_SUBDIRECTORY(lib/cnmem)
+LIST(APPEND SINGA_LINKER_LIBS cnmem)
 ADD_SUBDIRECTORY(src)
 ADD_SUBDIRECTORY(test)
 ADD_SUBDIRECTORY(examples)

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/37e6ad28/cmake/Protobuf.cmake
----------------------------------------------------------------------
diff --git a/cmake/Protobuf.cmake b/cmake/Protobuf.cmake
index c72b2c0..9324749 100644
--- a/cmake/Protobuf.cmake
+++ b/cmake/Protobuf.cmake
@@ -12,12 +12,12 @@ function(PROTOBUF_GENERATE_PYTHON OUTPUT)
         get_filename_component(FIL_WE ${FIL} NAME_WE)
         get_filename_component(PATH ${FIL} PATH)
 
-        list(APPEND ${OUTPUT} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}_pb2.py")
+        list(APPEND ${OUTPUT} "${CMAKE_BINARY_DIR}/python/singa/proto/${FIL_WE}_pb2.py")
 
         add_custom_command(
-            OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}_pb2.py"
+            OUTPUT "${CMAKE_BINARY_DIR}/python/singa/proto/${FIL_WE}_pb2.py"
             COMMAND ${PROTOBUF_PROTOC_EXECUTABLE}
-            ARGS --python_out ${CMAKE_CURRENT_BINARY_DIR}
+            ARGS --python_out ${CMAKE_BINARY_DIR}/python/singa/proto
                  --proto_path ${PATH} ${ABS_FIL}
             DEPENDS ${ABS_FIL}
             COMMENT "Running Python protocol buffer compiler on ${FIL}" VERBATIM)

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/37e6ad28/cmake/Utils.cmake
----------------------------------------------------------------------
diff --git a/cmake/Utils.cmake b/cmake/Utils.cmake
index 7f9452e..c55cee3 100644
--- a/cmake/Utils.cmake
+++ b/cmake/Utils.cmake
@@ -3,11 +3,13 @@ macro(swig_generate_cxx pylist_variable)
     if(NOT EXISTS "${CMKAE_BINARY_DIR}/python")
         execute_process(
             COMMAND mkdir ${CMAKE_BINARY_DIR}/python
+            COMMAND mkdir ${CMAKE_BINARY_DIR}/python/singa
+            COMMAND mkdir ${CMAKE_BINARY_DIR}/python/singa/proto
             ERROR_QUIET)
     endif()
     execute_process(
-        COMMAND swig -c++ -python -I${CMAKE_SOURCE_DIR}/include
-        -outdir ${CMAKE_BINARY_DIR}/python/
+        COMMAND swig -c++ -python -I${CMAKE_SOURCE_DIR}/include 
+        -outdir ${CMAKE_BINARY_DIR}/python/singa
         ${ARGN})
 
     set(${pylist_variable} "${CMAKE_SOURCE_DIR}/src/python/swig/singa_wrap.cxx")

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/37e6ad28/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index af09799..54d19ec 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -81,11 +81,25 @@ LIST(APPEND SINGA_LINKER_LIBS singa_io)
 IF(USE_PYTHON)
     FILE(GLOB python_files python/swig/singa.i)
     swig_generate_cxx(python_srcs ${python_files})
+    FILE(COPY python/ DESTINATION ${CMAKE_BINARY_DIR}/python/singa FILES_MATCHING PATTERN
"swig" EXCLUDE PATTERN "*.py")
     SET(python_cxxs "${CMAKE_SOURCE_DIR}/src/core/tensor/tensor.cc;${CMAKE_SOURCE_DIR}/src/core/device/device.cc")
     ADD_LIBRARY(_singa SHARED ${python_srcs} ${python_cxxs})
     SET(WRAPPER_LINKER_LIBS "${SINGA_LINKER_LIBS};protobuf")
     TARGET_LINK_LIBRARIES(_singa ${WRAPPER_LINKER_LIBS})
     TARGET_INCLUDE_DIRECTORIES(_singa PRIVATE ${PYTHON_INCLUDE_DIRS})
+    #message(STATUS "PYTHON_INCLUDE_DIRS ${PYTHON_INCLUDE_DIRS}")
+    
     SET_TARGET_PROPERTIES(_singa
-        PROPERTIES PREFIX "")
+        PROPERTIES PREFIX ""
+        LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/python/singa
+        )
+
+    #SETUP
+    SET(SETUP_PY_IN "python/setup.py.in")
+    set(SETUP_PY    "${CMAKE_BINARY_DIR}/python/setup.py")
+    configure_file(${SETUP_PY_IN} ${SETUP_PY})
+    
+    #COPY protobuf files to python/singa/proto
+    FILE(COPY ${CMAKE_BINARY_DIR}/python/singa/__init__.py DESTINATION ${CMAKE_BINARY_DIR}/python/singa/proto
)
+
 ENDIF(USE_PYTHON)

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/37e6ad28/src/python/__init__.py
----------------------------------------------------------------------
diff --git a/src/python/__init__.py b/src/python/__init__.py
new file mode 100644
index 0000000..c32dc40
--- /dev/null
+++ b/src/python/__init__.py
@@ -0,0 +1,3 @@
+def main():
+    """Entry point for the application script"""
+    print("Welcome to SINGA!")
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/37e6ad28/src/python/device.py
----------------------------------------------------------------------
diff --git a/src/python/device.py b/src/python/device.py
index 92a92e7..0877ae5 100644
--- a/src/python/device.py
+++ b/src/python/device.py
@@ -29,11 +29,7 @@ import sys
 import os
 import numpy as np
 
-sys.path.append(os.path.join(os.path.dirname(__file__),
-                             '../../build/lib'))
-sys.path.append(os.path.join(os.path.dirname(__file__),
-                             '../../build/python'))
-import singa
+from . import singa
 
 
 class Device(object):

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/37e6ad28/src/python/layer.py
----------------------------------------------------------------------
diff --git a/src/python/layer.py b/src/python/layer.py
index b0bcfa9..a1ec556 100644
--- a/src/python/layer.py
+++ b/src/python/layer.py
@@ -24,12 +24,10 @@
 import sys
 import os
 import numpy as np
-import singa
+from . import singa
 
-sys.path.append(os.path.join(os.path.dirname(__file__), '../'))
-
-from core_pb2 import *
-from model_pb2 import *
+from .proto.core_pb2 import *
+from .proto.model_pb2 import *
 
 
 class Layer(object):

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/37e6ad28/src/python/model.py
----------------------------------------------------------------------
diff --git a/src/python/model.py b/src/python/model.py
index 6d9fe39..38d9950 100644
--- a/src/python/model.py
+++ b/src/python/model.py
@@ -17,5 +17,5 @@
 # */
 
 class Model(Object):
-
+    pass
 

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/37e6ad28/src/python/setup.py.in
----------------------------------------------------------------------
diff --git a/src/python/setup.py.in b/src/python/setup.py.in
new file mode 100644
index 0000000..9002c70
--- /dev/null
+++ b/src/python/setup.py.in
@@ -0,0 +1,76 @@
+# Always prefer setuptools over distutils
+from setuptools import setup, find_packages
+from codecs import open
+from os import path
+
+setup(
+    name='singa',
+
+    version='${PACKAGE_VERSION}',
+
+    description='A General Deep Learning System',
+
+    url='https://github.com/apache/incubator-singa',
+
+    author='NUS Database Group',
+    author_email='dbsystem@comp.nus.edu.sg',
+
+    license='Apache 2',
+
+    classifiers=[
+        #   3 - Alpha
+        #   4 - Beta
+        #   5 - Production/Stable
+        'Development Status :: 3 - Alpha',
+
+        'Intended Audience :: Developers',
+        'Topic :: Deep Learning System ',
+
+        'License :: Apache License',
+
+        # Specify the Python versions you support here. In particular, ensure
+        # that you indicate whether you support Python 2, Python 3 or both.
+        'Programming Language :: Python :: 2',
+        'Programming Language :: Python :: 2.6',
+        'Programming Language :: Python :: 2.7',
+        ],
+
+    keywords='deep learning singa apache',
+
+    packages= ['singa','singa.proto'],
+
+    # py_modules=["singa"],
+
+    # install_requires=['peppercorn'],
+
+    # List additional groups of dependencies here (e.g. development
+    # dependencies). You can install these using the following syntax,
+    # for example:
+    # $ pip install -e .[dev,test]
+    #extras_require={
+    #    'dev': ['check-manifest'],
+    #    'test': ['coverage'],
+    #},
+
+    # If there are data files included in your packages that need to be
+    # installed, specify them here.  If using Python 2.6 or less, then these
+    # have to be included in MANIFEST.in as well.
+    package_data={
+        'singa': ['_singa.so'],
+    },
+
+    # Although 'package_data' is the preferred approach, in some case you may
+    # need to place data files outside of your packages. See:
+    # http://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files #
noqa
+    # In this case, 'data_file' will be installed into '<sys.prefix>/my_data'
+    #data_files=[('my_data', ['data/data_file'])],
+
+    # To provide executable scripts, use entry points in preference to the
+    # "scripts" keyword. Entry points provide cross-platform support and allow
+    # pip to create the appropriate form of executable for the target platform.
+    entry_points={
+        'console_scripts': [
+            'singa=singa:main',
+        ],
+    },
+)
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/37e6ad28/src/python/tensor.py
----------------------------------------------------------------------
diff --git a/src/python/tensor.py b/src/python/tensor.py
index fc95d37..3b9fa52 100644
--- a/src/python/tensor.py
+++ b/src/python/tensor.py
@@ -29,15 +29,9 @@ import sys
 import os
 import numpy as np
 
-sys.path.append(os.path.join(os.path.dirname(__file__),
-                             '../../build/lib'))
-sys.path.append(os.path.join(os.path.dirname(__file__),
-                             '../../build/python'))
-import singa
-
-sys.path.append(os.path.join(os.path.dirname(__file__),
-                             '../../build/src'))
-from core_pb2 import *
+from . import singa
+
+from .proto.core_pb2 import *
 
 
 class Tensor(object):

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/37e6ad28/test/python/example_test_tensor.py
----------------------------------------------------------------------
diff --git a/test/python/example_test_tensor.py b/test/python/example_test_tensor.py
index 9b3d6f4..c5ce5f1 100644
--- a/test/python/example_test_tensor.py
+++ b/test/python/example_test_tensor.py
@@ -2,12 +2,12 @@ import sys, os
 import numpy as np
 
 sys.path.append(os.path.join(os.path.dirname(__file__),
-                             '../../src/python'))
-from tensor import *
-
+                             '../../build/python'))
 sys.path.append(os.path.join(os.path.dirname(__file__),
-                             '../../build/src'))
+                             '../../build/lib'))
+sys.path.append(os.path.join(os.path.dirname(__file__),'../../build/src'))
 from core_pb2 import *
+from singa.tensor import *
 
 
 #---------------------------------------------------------
@@ -37,25 +37,25 @@ print 'transpose', t.is_transpose()
 print 'nDim:', t.ndim()
 print 'size:', t.size()
 print 'memsize:', t.memsize()
-print 'data():', t.toarray()
+print 'data():', t.to_numpy()
 print
 
 print '----------------------------'
-print 't.data\n', t.toarray()
+print 't.data\n', t.to_numpy()
 q = t.copy()
 w = t.deepcopy()
-print 'q.data\n', q.toarray()
-print 'w.data\n', w.toarray()
+print 'q.data\n', q.to_numpy()
+print 'w.data\n', w.to_numpy()
 t += 1.23
-print 'q.data\n', q.toarray()
-print 'w.data\n', w.toarray()
+print 'q.data\n', q.to_numpy()
+print 'w.data\n', w.to_numpy()
 
 print '----------------------------'
 shape = (2, 3)
 t.reshape(shape)
 print 'shape = (3, 2)'
 print 'after reshape, t.shape():', t.shape()
-print 't.data(): \n', t.toarray()
+print 't.data(): \n', t.to_numpy()
 shape = (3, 2)
 t0 = reshape(t, shape)
 print 'shape = (2, 3)'
@@ -66,110 +66,110 @@ print
 
 print '----------------------------'
 t += 1.2345
-print 't += 1.234, i.e., t.__iadd__(1.2345): \n', t.toarray()
+print 't += 1.234, i.e., t.__iadd__(1.2345): \n', t.to_numpy()
 print
 
 t1 = t
 print 'copy\nt1 = t'
 print 't1.shape():', t1.shape()
-print 't1.data(): \n', t1.toarray()
+print 't1.data(): \n', t1.to_numpy()
 print
 
 r = t1.transpose()
-print 'r = t1.transpose()\nr.data() \n', r.toarray()
+print 'r = t1.transpose()\nr.data() \n', r.to_numpy()
 
 
 print '----------------------------'
 t2 = log(t1)
-print 't2 = log(t1): \n', t2.toarray()
+print 't2 = log(t1): \n', t2.to_numpy()
 print
 
 t1 += t2
-print 't1 += t2, i.e., t1.__iadd__(t2): \n', t1.toarray()
+print 't1 += t2, i.e., t1.__iadd__(t2): \n', t1.to_numpy()
 print
 
 t1 *= 2
-print 't1 *= 2, i.e., t1.__imul__(2): \n', t1.toarray()
+print 't1 *= 2, i.e., t1.__imul__(2): \n', t1.to_numpy()
 print
 
 print '----------------------------'
 tc = t2.clone()
-print 'clone\ntc = t2.clone()\ntc.data(): \n', tc.toarray()
+print 'clone\ntc = t2.clone()\ntc.data(): \n', tc.to_numpy()
 print
 
 print 'sum(tc) \n', sum(tc)
 print
 t3 = sum(tc,0)
-print 'sum(tc,0) \n', t3.toarray()
+print 'sum(tc,0) \n', t3.to_numpy()
 t3 = sum(tc,1)
-print 'sum(tc,1) \n', t3.toarray()
+print 'sum(tc,1) \n', t3.to_numpy()
 print
 
 t3 = average(tc,0)
-print 'average(tc,0) \n', t3.toarray()
+print 'average(tc,0) \n', t3.to_numpy()
 t3 = average(tc,1)
-print 'average(tc,1) \n', t3.toarray()
+print 'average(tc,1) \n', t3.to_numpy()
 print
 
 t3 = softmax(tc,0)
-print 'softmax(tc,0)\n', t3.toarray()
+print 'softmax(tc,0)\n', t3.to_numpy()
 t3 = softmax(tc,1)
-print 'softmax(tc,1)\n', t3.toarray()
+print 'softmax(tc,1)\n', t3.to_numpy()
 
 print '----------------------------'
-print 't1 \n', t1.toarray()
+print 't1 \n', t1.to_numpy()
 print
 
 n = t1 + t2
-print 't1 + t2: \n', n.toarray()
+print 't1 + t2: \n', n.to_numpy()
 print
 
 n = t1 * t2
-print 't1*t2: \n', n.toarray()
+print 't1*t2: \n', n.to_numpy()
 print
 
 n = t1 - 1.2
-print 't1 - 1.2 \n', n.toarray()
+print 't1 - 1.2 \n', n.to_numpy()
 print
 
 n = add(t1, t1)
-print 'add(t1, t1) \n', n.toarray()
+print 'add(t1, t1) \n', n.to_numpy()
 print
 
 n = add(t1, 3.4)
-print 'add(t1, 3.4) \n', n.toarray()
+print 'add(t1, 3.4) \n', n.to_numpy()
 print
 
 n = div(t1, 2.0)
-print 'div(t1, 2.0) \n', n.toarray()
+print 'div(t1, 2.0) \n', n.to_numpy()
 print
 
 print '----------------------------'
 shape = (2, 2)
 t4 = Tensor(shape)
 t4 += 3.45
-print 't4 += 3.45 \n', t4.toarray()
+print 't4 += 3.45 \n', t4.to_numpy()
 print
 
 n = t4 < 3.45
-print 't4 < 3.45 \n', n.toarray()
+print 't4 < 3.45 \n', n.to_numpy()
 print
 
 n = lt(t4, 3.45)
-print 'lt(t4, 3.45) \n', n.toarray()
+print 'lt(t4, 3.45) \n', n.to_numpy()
 print
 
 n = ge(t4, 3.45)
-print 'ge(t4, 3.45) \n', n.toarray()
+print 'ge(t4, 3.45) \n', n.to_numpy()
 print
 
 print '----------------------------'
-print t1.toarray()
-print tc.toarray()
+print t1.to_numpy()
+print tc.to_numpy()
 print
 copy_data_to_from(t1, tc, 2)
-print t1.toarray()
-print tc.toarray()
+print t1.to_numpy()
+print tc.to_numpy()
 
 #ttt = t1.singa_tensor < 5.2
 #ttt = lessthan(t1, 5.2)

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/37e6ad28/test/python/test_tensor.py
----------------------------------------------------------------------
diff --git a/test/python/test_tensor.py b/test/python/test_tensor.py
index d15e9ed..de9012e 100644
--- a/test/python/test_tensor.py
+++ b/test/python/test_tensor.py
@@ -27,14 +27,12 @@ import math
 import unittest
 import numpy as np
 
-sys.path.append(os.path.join(os.path.dirname(__file__),
-                             '../../src/python'))
-from tensor import *
-from device import *
-
-sys.path.append(os.path.join(os.path.dirname(__file__),
-                             '../../build/src'))
-from core_pb2 import *
+sys.path.append(os.path.join(os.path.dirname(__file__),'../../build/python'))
+
+from singa.tensor import *
+from singa.device import *
+
+from singa.proto.core_pb2 import *
 
 
 class TestTensorMethods(unittest.TestCase):


Mime
View raw message