arrow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From u..@apache.org
Subject arrow git commit: ARROW-942: Support running integration tests with both Python 2.7 and 3.6
Date Thu, 05 Oct 2017 09:13:39 GMT
Repository: arrow
Updated Branches:
  refs/heads/master 7511cfdd5 -> f0873a99a


ARROW-942: Support running integration tests with both Python 2.7 and 3.6

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #1168 from wesm/ARROW-942 and squashes the following commits:

e4ef049 [Wes McKinney] Support running integration tests in both Python 2.7 and 3.6


Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/f0873a99
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/f0873a99
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/f0873a99

Branch: refs/heads/master
Commit: f0873a99aecdfda4b3d8ec04a41491af3442b827
Parents: 7511cfd
Author: Wes McKinney <wes.mckinney@twosigma.com>
Authored: Thu Oct 5 11:13:26 2017 +0200
Committer: Uwe L. Korn <uwelk@xhochy.com>
Committed: Thu Oct 5 11:13:26 2017 +0200

----------------------------------------------------------------------
 integration/integration_test.py | 43 +++++++++++++++++++++++++-----------
 1 file changed, 30 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/arrow/blob/f0873a99/integration/integration_test.py
----------------------------------------------------------------------
diff --git a/integration/integration_test.py b/integration/integration_test.py
index 4bb0a4b..59a1de5 100644
--- a/integration/integration_test.py
+++ b/integration/integration_test.py
@@ -17,6 +17,7 @@
 
 from collections import OrderedDict
 import argparse
+import binascii
 import glob
 import itertools
 import json
@@ -30,6 +31,7 @@ import uuid
 
 import numpy as np
 
+
 ARROW_HOME = os.path.abspath(__file__).rsplit("/", 2)[0]
 
 # Control for flakiness
@@ -63,11 +65,24 @@ def rands(nchars):
     return ''.join(np.random.choice(RANDS_CHARS, nchars))
 
 
-def str_from_bytes(x):
-    if six.PY2:
-        return x
-    else:
-        return x.decode('utf-8')
+if six.PY2:
+    def frombytes(o):
+        return o
+
+    def tobytes(o):
+        if isinstance(o, unicode):
+            return o.encode('utf8')
+        else:
+            return o
+else:
+    def tobytes(o):
+        if isinstance(o, str):
+            return o.encode('utf8')
+        else:
+            return o
+
+    def frombytes(o):
+        return o.decode('utf8')
 
 
 # from the merge_arrow_pr.py script
@@ -82,11 +97,11 @@ def run_cmd(cmd):
         print('Command failed: %s' % ' '.join(cmd))
         print('With output:')
         print('--------------')
-        print(str_from_bytes(e.output))
+        print(frombytes(e.output))
         print('--------------')
         raise e
 
-    return str_from_bytes(output)
+    return frombytes(output)
 
 # ----------------------------------------------------------------------
 # Data generation
@@ -353,6 +368,7 @@ class DecimalType(PrimitiveType):
 
 
 class DecimalColumn(PrimitiveColumn):
+
     def __init__(self, name, count, is_valid, values, bit_width):
         PrimitiveColumn.__init__(self, name, count, is_valid, values)
         self.bit_width = bit_width
@@ -416,7 +432,7 @@ class BinaryType(PrimitiveType):
                         .tostring())
                 values.append(draw)
             else:
-                values.append("")
+                values.append(b"")
 
         if name is None:
             name = self.name
@@ -439,9 +455,9 @@ class StringType(BinaryType):
 
         for i in range(size):
             if is_valid[i]:
-                values.append(rands(K))
+                values.append(tobytes(rands(K)))
             else:
-                values.append("")
+                values.append(b"")
 
         if name is None:
             name = self.name
@@ -462,7 +478,7 @@ class JsonSchema(object):
 class BinaryColumn(PrimitiveColumn):
 
     def _encode_value(self, x):
-        return ''.join('{:02x}'.format(c).upper() for c in x)
+        return frombytes(binascii.hexlify(x))
 
     def _get_buffers(self):
         offset = 0
@@ -473,7 +489,7 @@ class BinaryColumn(PrimitiveColumn):
             if self.is_valid[i]:
                 offset += len(v)
             else:
-                v = ""
+                v = b""
 
             offsets.append(offset)
             data.append(self._encode_value(v))
@@ -488,7 +504,8 @@ class BinaryColumn(PrimitiveColumn):
 class StringColumn(BinaryColumn):
 
     def _encode_value(self, x):
-        return x
+        return frombytes(x)
+
 
 class ListType(DataType):
 


Mime
View raw message