Mailing-List: contact commits-help@singa.incubator.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@singa.incubator.apache.org
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: wangwei@apache.org
To: commits@singa.incubator.apache.org
Date: Thu, 20 Oct 2016 13:57:06 -0000
Message-Id: <561f986d22fc4e1f99467e6548d931bd@git.apache.org>
Subject: [1/2] incubator-singa git commit: SINGA-266 Add Rafiki python
 toolkits - add rafiki python package - firt version of rafiki agent module
archived-at: Thu, 20 Oct 2016 19:25:05 -0000

Repository: incubator-singa
Updated Branches:
  refs/heads/master 32b4368f0 -> 9eabb9563


SINGA-266 Add Rafiki python toolkits
  - add rafiki python package
  - firt version of rafiki agent module


Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/f536975c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/f536975c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/f536975c

Branch: refs/heads/master
Commit: f536975ced922cd7bd19303e2fbc616ae646e48a
Parents: 32b4368
Author: aaronwwf <dcswuw@gmail.com>
Authored: Thu Oct 20 09:05:59 2016 +0800
Committer: aaronwwf <dcswuw@gmail.com>
Committed: Thu Oct 20 09:05:59 2016 +0800

----------------------------------------------------------------------
 python/CMakeLists.txt     |   1 +
 python/rafiki/__init__.py |  19 ++++
 python/rafiki/agent.py    | 200 +++++++++++++++++++++++++++++++++++++++++
 python/setup.py.in        |   2 +-
 4 files changed, 221 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f536975c/python/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 7774af0..d8a0f2d 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -91,6 +91,7 @@ PROTOBUF_GENERATE_PYTHON(proto_pys ${proto_files})
 
 # generate cxx and wrap.py
 file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/python/singa/proto)
+file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/python/rafiki)
 file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/src/api)
 execute_process(
     COMMAND swig -c++ -python -I${CMAKE_SOURCE_DIR}/include

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f536975c/python/rafiki/__init__.py
----------------------------------------------------------------------
diff --git a/python/rafiki/__init__.py b/python/rafiki/__init__.py
new file mode 100644
index 0000000..c81c6ef
--- /dev/null
+++ b/python/rafiki/__init__.py
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# 
+
+

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f536975c/python/rafiki/agent.py
----------------------------------------------------------------------
diff --git a/python/rafiki/agent.py b/python/rafiki/agent.py
new file mode 100644
index 0000000..f8d4b51
--- /dev/null
+++ b/python/rafiki/agent.py
@@ -0,0 +1,200 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# =============================================================================
+from multiprocessing import Process, Queue
+from flask import Flask,request, send_from_directory, jsonify
+from flask_cors import CORS, cross_origin
+import os, traceback, sys
+import time
+from werkzeug.utils import secure_filename
+
+class MsgType:
+   def __init__(self, name):
+       self.name = name
+   def __str__(self):
+       return self.name
+   def __repr__(self):
+       return "<Msg: %s>" % self
+   def equal(self,target):
+       return str(self) == str(target)
+
+   def is_info(self):
+       return self.name.startswith('kInfo') 
+   def is_command(self):
+       return self.name.startswith('kCommand') 
+   def is_status(self):
+       return self.name.startswith('kStatus') 
+   def is_request(self):
+       return self.name.startswith('kRequest') 
+   def is_response(self):
+       return self.name.startswith('kResponse') 
+
+   @staticmethod
+   def parse(name):
+       return getattr(MsgType,str(name))
+   @staticmethod
+   def get_command(name):
+       if name=='stop':
+           return MsgType.kCommandStop
+       if name=='pause':
+           return MsgType.kCommandPause
+       if name=='resume':
+           return MsgType.kCommandResume
+       return MsgType.kCommand 
+
+types =  ['kInfo','kInfoMetric',
+           'kCommand','kCommandStop','kCommandPause','kCommandResume',
+           'kStatus','kStatusRunning','kStatusPaused','kStatusError',
+           'kRequest','kResponse']
+
+for t in types:
+    setattr(MsgType,t,MsgType(t))
+
+#####   NOTE the server currently only can handle request sequentially
+
+app = Flask(__name__)
+top_k_=5
+
+class Agent():
+
+    def __init__(self,port):
+        info_queue = Queue()
+        command_queue = Queue()
+        self.p = Process(target=start, args=(port, info_queue,command_queue))
+        self.p.start()
+        self.info_queue=info_queue
+        self.command_queue=command_queue
+        return
+
+    def pull(self):
+        if not self.command_queue.empty():
+            return self.command_queue.get()
+        return None,None 
+
+    def push(self,msg,value):
+        self.info_queue.put((msg,value))
+        return
+
+    def stop(self):
+        self.p.terminate()
+
+def start(port,info_queue,command_queue):
+    global info_queue_, command_queue_, data_
+    info_queue_=info_queue
+    command_queue_=command_queue
+    data_ = []
+    app.run(host='0.0.0.0', port=port)
+    return
+
+def getDataFromInfoQueue(need_return=False):
+    global info_queue_, data_
+    if not need_return:
+        while not info_queue_.empty():
+            info_queue_.top()
+            msg,d = info_queue_.get()
+            data_.append(d)
+    else:
+        while True: # loop until get answer
+            print 'test'
+            while not info_queue_.empty():
+                print 'test2'
+                msg,d = info_queue_.get()
+                if msg.is_info():
+                    print 'test3'
+                    data_.append(d)
+                else:
+                    return msg,d
+            print 'test4'
+            time.sleep(0.01)
+
+
+@app.route("/")
+@cross_origin()
+def index():
+    try:
+        req=send_from_directory(os.getcwd(),"index.html", mimetype='text/html')
+    except:
+        traceback.print_exc()
+        return "error"
+    return req
+
+# support two operations for user to monitor the training status
+@app.route('/getAllData')
+@cross_origin()
+def getAllData():
+    global data_
+    try:
+        getDataFromInfoQueue()
+    except:
+        traceback.print_exc()
+        return failure("Internal Error")
+    return success(data_)
+
+
+@app.route('/getTopKData')
+@cross_origin()
+def getTopKData():
+    global data_
+    try:
+        k = int(request.args.get("k", top_k_))
+    except:
+        traceback.print_exc()
+        return failure("k should be integer")
+    try:
+        getDataFromInfoQueue()
+    except:
+        traceback.print_exc()
+        return failure("Internal Error")
+    return success(data_[-k:])
+
+@app.route("/api", methods=['POST'])                                                                  
+@cross_origin()                                                                                           
+def api():
+    global info_queue_,command_queue_ 
+    try:
+        file = request.files['image']
+        filename = secure_filename(file.filename)
+        filepath=os.path.join(os.getcwd(), filename)
+        file.save(filepath)
+        command_queue_.put((MsgType.kRequest,filepath))
+        msg,response=getDataFromInfoQueue(True)
+        return response
+    except:                                                                            
+        traceback.print_exc()
+        return failure("Internal Error")
+
+@app.route("/command/<name>", methods=['GET','POST'])                                                                  
+@cross_origin()                                                                                           
+def command(name):
+    global info_queue_,command_queue_ 
+    try:
+        command=MsgType.get_command(name)
+        command_queue_.put((command,""))
+        msg,response=getDataFromInfoQueue(True)
+        return response
+    except:                                                                            
+        traceback.print_exc()
+        return failure("Internal Error")
+
+def success(data=""):
+    '''return success status in json format'''
+    res = dict(result="success", data=data)
+    return jsonify(res)
+def failure(message):
+    '''return failure status in json format'''
+    res = dict(result="message", message=message)
+    return jsonify(res)

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f536975c/python/setup.py.in
----------------------------------------------------------------------
diff --git a/python/setup.py.in b/python/setup.py.in
index 881cd30..95550d8 100644
--- a/python/setup.py.in
+++ b/python/setup.py.in
@@ -54,7 +54,7 @@ setup(
 
     keywords='deep learning singa apache',
 
-    packages= ['singa', 'singa.proto'],
+    packages= ['rafiki','singa', 'singa.proto'],
 
     #py_modules=["singa"],