aurora-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From s...@apache.org
Subject aurora git commit: Use the Thrift binary protocol in the Aurora client.
Date Thu, 06 Oct 2016 20:15:10 GMT
Repository: aurora
Updated Branches:
  refs/heads/master 8cd175ad2 -> 09b8e583f


Use the Thrift binary protocol in the Aurora client.

Now that Aurora 0.16 has been released, we can assume all schedulers
are supporting the binary protocol.

Switching the protocol comes with a slight performance boost: Given
the example job `devcluster/www-data/prod/hello` with 500 instances
we can reduce the average request time by about 40%.

Using the TJSONProtocol:

    $ time aurora job status devcluster > /dev/null
    real    0m0.906s
    user    0m0.759s
    sys     0m0.078s

Using the new TBinaryProtocol:

    $ time aurora job status devcluster > /dev/null
    real    0m0.524s
    user    0m0.410s
    sys     0m0.079s

Reviewed at https://reviews.apache.org/r/52610/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/09b8e583
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/09b8e583
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/09b8e583

Branch: refs/heads/master
Commit: 09b8e583fc5f6314de3099af9896380dc18f691d
Parents: 8cd175a
Author: Stephan Erb <serb@apache.org>
Authored: Thu Oct 6 22:14:28 2016 +0200
Committer: Stephan Erb <serb@apache.org>
Committed: Thu Oct 6 22:14:28 2016 +0200

----------------------------------------------------------------------
 RELEASE-NOTES.md                                |  1 +
 .../aurora/client/api/scheduler_client.py       |  4 ++--
 .../python/apache/aurora/common/transport.py    |  3 ++-
 .../apache/aurora/common/test_transport.py      | 20 ++++++++++----------
 4 files changed, 15 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/aurora/blob/09b8e583/RELEASE-NOTES.md
----------------------------------------------------------------------
diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md
index 6968bb5..f3dd8bb 100644
--- a/RELEASE-NOTES.md
+++ b/RELEASE-NOTES.md
@@ -8,6 +8,7 @@
   `minWaitInInstanceRunningMs` will no longer have to be chosen based on the worst observed
instance
   startup/warmup delay but rather as a desired health check duration.
 - A task's tier is now mapped to a label on the Mesos `TaskInfo` proto.
+- The Aurora client is now using the Thrift binary protocol to communicate with the scheduler.
 
 ### Deprecations and removals:
 

http://git-wip-us.apache.org/repos/asf/aurora/blob/09b8e583/src/main/python/apache/aurora/client/api/scheduler_client.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/api/scheduler_client.py b/src/main/python/apache/aurora/client/api/scheduler_client.py
index cbdb50a..9bbfece 100644
--- a/src/main/python/apache/aurora/client/api/scheduler_client.py
+++ b/src/main/python/apache/aurora/client/api/scheduler_client.py
@@ -19,7 +19,7 @@ import traceback
 
 import requests
 from pystachio import Default, Integer, String
-from thrift.protocol import TJSONProtocol
+from thrift.protocol import TBinaryProtocol
 from thrift.transport import TTransport
 from twitter.common import log
 from twitter.common.concurrent import Timeout, deadline
@@ -111,7 +111,7 @@ class SchedulerClient(object):
             _bypass_leader_redirect_session_factory,
             should_bypass=self._bypass_leader_redirect))
 
-    protocol = TJSONProtocol.TJSONProtocol(transport)
+    protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport)
     schedulerClient = AuroraAdmin.Client(protocol)
     for _ in range(self.THRIFT_RETRIES):
       try:

http://git-wip-us.apache.org/repos/asf/aurora/blob/09b8e583/src/main/python/apache/aurora/common/transport.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/common/transport.py b/src/main/python/apache/aurora/common/transport.py
index 2e38847..f44d13a 100644
--- a/src/main/python/apache/aurora/common/transport.py
+++ b/src/main/python/apache/aurora/common/transport.py
@@ -117,7 +117,8 @@ class TRequestsTransport(TTransportBase):
     data = self.__wbuf.getvalue()
     self.__wbuf = BytesIO()
 
-    self._session.headers['Content-Type'] = 'application/x-thrift'
+    self._session.headers['Accept'] = 'application/vnd.apache.thrift.binary'
+    self._session.headers['Content-Type'] = 'application/vnd.apache.thrift.binary'
     self._session.headers['Content-Length'] = str(len(data))
     self._session.headers['Host'] = self.__urlparse.hostname
 

http://git-wip-us.apache.org/repos/asf/aurora/blob/09b8e583/src/test/python/apache/aurora/common/test_transport.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/common/test_transport.py b/src/test/python/apache/aurora/common/test_transport.py
index 0a587f0..c3d35d2 100644
--- a/src/test/python/apache/aurora/common/test_transport.py
+++ b/src/test/python/apache/aurora/common/test_transport.py
@@ -19,7 +19,7 @@ import pytest
 import requests
 from mock import ANY, Mock, call, create_autospec
 from requests import exceptions as request_exceptions
-from thrift.protocol import TJSONProtocol
+from thrift.protocol import TBinaryProtocol
 from thrift.server import THttpServer
 from thrift.transport import TTransport
 
@@ -38,7 +38,7 @@ class ReadOnlySchedulerHandler(object):
 def test_request_transport_integration():
   handler = ReadOnlySchedulerHandler()
   processor = ReadOnlyScheduler.Processor(handler)
-  pfactory = TJSONProtocol.TJSONProtocolFactory()
+  pfactory = TBinaryProtocol.TBinaryProtocolAcceleratedFactory()
   server = THttpServer.THttpServer(processor, ('localhost', 0), pfactory)
   server_thread = Thread(target=server.serve)
   server_thread.start()
@@ -46,7 +46,7 @@ def test_request_transport_integration():
 
   try:
     transport = TRequestsTransport('http://localhost:%d' % server_port)
-    protocol = TJSONProtocol.TJSONProtocol(transport)
+    protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport)
     client = ReadOnlyScheduler.Client(protocol)
     response = client.getRoleSummary()
   finally:
@@ -64,7 +64,7 @@ def test_request_transport_timeout():
   session.headers = {}
   session.post = Mock(side_effect=request_exceptions.Timeout())
   transport = TRequestsTransport('http://localhost:12345', session_factory=lambda: session)
-  protocol = TJSONProtocol.TJSONProtocol(transport)
+  protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport)
   client = ReadOnlyScheduler.Client(protocol)
 
   with pytest.raises(TTransport.TTransportException) as execinfo:
@@ -84,7 +84,7 @@ def test_raise_for_status_causes_exception():
   session.post.return_value = response
 
   transport = TRequestsTransport('http://localhost:12345', session_factory=lambda: session)
-  protocol = TJSONProtocol.TJSONProtocol(transport)
+  protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport)
   client = ReadOnlyScheduler.Client(protocol)
 
   with pytest.raises(TTransport.TTransportException) as excinfo:
@@ -105,7 +105,7 @@ def test_raises_auth_error():
   session.post.return_value = response
 
   transport = TRequestsTransport('http://localhost:12345', session_factory=lambda: session)
-  protocol = TJSONProtocol.TJSONProtocol(transport)
+  protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport)
   client = ReadOnlyScheduler.Client(protocol)
 
   with pytest.raises(TRequestsTransport.AuthError):
@@ -119,7 +119,7 @@ def test_request_any_other_exception():
   session.headers = {}
   session.post = Mock(side_effect=request_exceptions.ConnectionError())
   transport = TRequestsTransport('http://localhost:12345', session_factory=lambda: session)
-  protocol = TJSONProtocol.TJSONProtocol(transport)
+  protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport)
   client = ReadOnlyScheduler.Client(protocol)
 
   with pytest.raises(TTransport.TTransportException):
@@ -161,7 +161,7 @@ def test_auth_type_valid():
 
   auth = requests.auth.AuthBase()
   transport = TRequestsTransport('http://localhost:1', auth=auth, session_factory=lambda:
session)
-  protocol = TJSONProtocol.TJSONProtocol(transport)
+  protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport)
   client = ReadOnlyScheduler.Client(protocol)
 
   with pytest.raises(TTransport.TTransportException):
@@ -181,7 +181,7 @@ def test_auth_type_invalid():
 def test_requests_transport_session_reuse():
   handler = ReadOnlySchedulerHandler()
   processor = ReadOnlyScheduler.Processor(handler)
-  pfactory = TJSONProtocol.TJSONProtocolFactory()
+  pfactory = TBinaryProtocol.TBinaryProtocolAcceleratedFactory()
   server = THttpServer.THttpServer(processor, ('localhost', 0), pfactory)
   server_thread = Thread(target=server.serve)
   server_thread.start()
@@ -189,7 +189,7 @@ def test_requests_transport_session_reuse():
 
   try:
     transport = TRequestsTransport('http://localhost:%d' % server_port)
-    protocol = TJSONProtocol.TJSONProtocol(transport)
+    protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport)
     client = ReadOnlyScheduler.Client(protocol)
     client.getRoleSummary()
     old_session = transport._session


Mime
View raw message