airflow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jlo...@apache.org
Subject incubator-airflow git commit: [AIRFLOW-179] Fix DbApiHook with non-ASCII chars
Date Fri, 04 Nov 2016 13:42:49 GMT
Repository: incubator-airflow
Updated Branches:
  refs/heads/master 9ae583eb1 -> 4fb86f920


[AIRFLOW-179] Fix DbApiHook with non-ASCII chars

String serialization fails when string contains non-ASCII characters

Closes #1553 from
johnbodley/dbapi_hook_serialization-remedy


Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/4fb86f92
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/4fb86f92
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/4fb86f92

Branch: refs/heads/master
Commit: 4fb86f920a6ea0b31efc4d5302065ab28c90a954
Parents: 9ae583e
Author: John Bodley <john.bodley@airbnb.com>
Authored: Fri Nov 4 09:41:44 2016 -0400
Committer: jlowin <jlowin@apache.org>
Committed: Fri Nov 4 09:41:46 2016 -0400

----------------------------------------------------------------------
 airflow/hooks/dbapi_hook.py    | 15 +++++++++++++--
 airflow/hooks/mysql_hook.py    | 15 +++++++++++++++
 airflow/hooks/postgres_hook.py | 13 ++++++++++++-
 3 files changed, 40 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/4fb86f92/airflow/hooks/dbapi_hook.py
----------------------------------------------------------------------
diff --git a/airflow/hooks/dbapi_hook.py b/airflow/hooks/dbapi_hook.py
index 9d6ddf0..939bae2 100644
--- a/airflow/hooks/dbapi_hook.py
+++ b/airflow/hooks/dbapi_hook.py
@@ -212,7 +212,7 @@ class DbApiHook(BaseHook):
             i += 1
             l = []
             for cell in row:
-                l.append(self._serialize_cell(cell))
+                l.append(self._serialize_cell(cell, conn))
             values = tuple(l)
             sql = "INSERT INTO {0} {1} VALUES ({2});".format(
                 table,
@@ -230,7 +230,18 @@ class DbApiHook(BaseHook):
             "Done loading. Loaded a total of {i} rows".format(**locals()))
 
     @staticmethod
-    def _serialize_cell(cell):
+    def _serialize_cell(cell, conn=None):
+        """
+        Returns the SQL literal of the cell as a string.
+
+        :param cell: The cell to insert into the table
+        :type cell: object
+        :param conn: The database connection
+        :type conn: connection object
+        :return: The serialized cell
+        :rtype: str
+        """
+
         if isinstance(cell, basestring):
             return "'" + str(cell).replace("'", "''") + "'"
         elif cell is None:

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/4fb86f92/airflow/hooks/mysql_hook.py
----------------------------------------------------------------------
diff --git a/airflow/hooks/mysql_hook.py b/airflow/hooks/mysql_hook.py
index 9c8224c..e4f9533 100644
--- a/airflow/hooks/mysql_hook.py
+++ b/airflow/hooks/mysql_hook.py
@@ -80,3 +80,18 @@ class MySqlHook(DbApiHook):
             INTO TABLE {table}
             """.format(**locals()))
         conn.commit()
+
+    @staticmethod
+    def _serialize_cell(cell, conn):
+        """
+        Returns the MySQL literal of the cell as a string.
+
+        :param cell: The cell to insert into the table
+        :type cell: object
+        :param conn: The database connection
+        :type conn: connection object
+        :return: The serialized cell
+        :rtype: str
+        """
+
+        return conn.literal(cell)

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/4fb86f92/airflow/hooks/postgres_hook.py
----------------------------------------------------------------------
diff --git a/airflow/hooks/postgres_hook.py b/airflow/hooks/postgres_hook.py
index ede1d22..d096d75 100644
--- a/airflow/hooks/postgres_hook.py
+++ b/airflow/hooks/postgres_hook.py
@@ -46,5 +46,16 @@ class PostgresHook(DbApiHook):
         return psycopg2_conn
 
     @staticmethod
-    def _serialize_cell(cell):
+    def _serialize_cell(cell, conn):
+        """
+        Returns the Postgres literal of the cell as a string.
+
+        :param cell: The cell to insert into the table
+        :type cell: object
+        :param conn: The database connection
+        :type conn: connection object
+        :return: The serialized cell
+        :rtype: str
+        """
+
         return psycopg2.extensions.adapt(cell).getquoted().decode('utf-8')


Mime
View raw message