airflow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From GitBox <...@apache.org>
Subject [GitHub] [airflow] potiuk commented on a change in pull request #6692: [AIRFLOW-6130] Make Cassandra to GCS operator pylint compatible
Date Sat, 30 Nov 2019 13:43:51 GMT
potiuk commented on a change in pull request #6692: [AIRFLOW-6130] Make Cassandra to GCS operator
pylint compatible
URL: https://github.com/apache/airflow/pull/6692#discussion_r352285982
 
 

 ##########
 File path: airflow/operators/cassandra_to_gcs.py
 ##########
 @@ -247,120 +248,140 @@ def convert_value(cls, name, value):
         elif isinstance(value, Time):
             return str(value).split('.')[0]
         elif isinstance(value, (list, SortedSet)):
-            return cls.convert_array_types(name, value)
+            return cls.convert_array_types(value)
         elif hasattr(value, '_fields'):
-            return cls.convert_user_type(name, value)
+            return cls.convert_user_type(value)
         elif isinstance(value, tuple):
-            return cls.convert_tuple_type(name, value)
+            return cls.convert_tuple_type(value)
         elif isinstance(value, OrderedMapSerializedKey):
-            return cls.convert_map_type(name, value)
+            return cls.convert_map_type(value)
         else:
-            raise AirflowException('unexpected value: ' + str(value))
+            raise AirflowException('Unexpected value: ' + str(value))
 
     @classmethod
-    def convert_array_types(cls, name, value):
-        return [cls.convert_value(name, nested_value) for nested_value in value]
+    def convert_array_types(cls, value: Union[list, SortedSet]) -> List:
+        """
+        Maps convert_value over array.
+        """
+        return [cls.convert_value(nested_value) for nested_value in value]
 
     @classmethod
-    def convert_user_type(cls, name, value):
+    def convert_user_type(cls, value):
         """
         Converts a user type to RECORD that contains n fields, where n is the
         number of attributes. Each element in the user type class will be converted to its
         corresponding data type in BQ.
         """
         names = value._fields
-        values = [cls.convert_value(name, getattr(value, name)) for name in names]
+        values = [cls.convert_value(getattr(value, name)) for name in names]
         return cls.generate_data_dict(names, values)
 
     @classmethod
-    def convert_tuple_type(cls, name, value):
+    def convert_tuple_type(cls, values: Tuple):
         """
         Converts a tuple to RECORD that contains n fields, each will be converted
         to its corresponding data type in bq and will be named 'field_<index>', where
         index is determined by the order of the tuple elements defined in cassandra.
         """
-        names = ['field_' + str(i) for i in range(len(value))]
-        values = [cls.convert_value(name, value) for name, value in zip(names, value)]
+        names = ['field_' + str(i) for i in range(len(values))]
         return cls.generate_data_dict(names, values)
 
     @classmethod
-    def convert_map_type(cls, name, value):
+    def convert_map_type(cls, value: OrderedMapSerializedKey) -> List[Dict]:
 
 Review comment:
   ```suggestion
       def convert_map_type(cls, value: OrderedMapSerializedKey) -> List[Dict[Any, Any]]:
   ```

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services

Mime
View raw message