airflow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject [airflow-site] 15/30: Add separate folder for 1.10.2
Date Wed, 27 Nov 2019 19:10:28 GMT
This is an automated email from the ASF dual-hosted git repository.

ash pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/airflow-site.git

commit 42910ec5d11bb8a26858920865f8bf23364d6280
Author: Kaxil Naik <kaxilnaik@gmail.com>
AuthorDate: Wed Jan 23 00:50:14 2019 +0000

    Add separate folder for 1.10.2
---
 1.10.2/.buildinfo                                  |     4 +
 1.10.2/_images/adhoc.png                           |   Bin 0 -> 182773 bytes
 1.10.2/_images/airflow.gif                         |   Bin 0 -> 622832 bytes
 1.10.2/_images/apache.jpg                          |   Bin 0 -> 43364 bytes
 1.10.2/_images/branch_bad.png                      |   Bin 0 -> 8825 bytes
 1.10.2/_images/branch_good.png                     |   Bin 0 -> 12035 bytes
 1.10.2/_images/chart.png                           |   Bin 0 -> 169382 bytes
 1.10.2/_images/chart_form.png                      |   Bin 0 -> 203224 bytes
 1.10.2/_images/code.png                            |   Bin 0 -> 285906 bytes
 1.10.2/_images/connection_create.png               |   Bin 0 -> 41547 bytes
 1.10.2/_images/connection_edit.png                 |   Bin 0 -> 53636 bytes
 1.10.2/_images/connections.png                     |   Bin 0 -> 48442 bytes
 1.10.2/_images/context.png                         |   Bin 0 -> 149588 bytes
 1.10.2/_images/dags.png                            |   Bin 0 -> 217069 bytes
 1.10.2/_images/duration.png                        |   Bin 0 -> 250846 bytes
 1.10.2/_images/gantt.png                           |   Bin 0 -> 121795 bytes
 1.10.2/_images/graph.png                           |   Bin 0 -> 152205 bytes
 1.10.2/_images/latest_only_with_trigger.png        |   Bin 0 -> 40034 bytes
 1.10.2/_images/pin_large.png                       |   Bin 0 -> 358276 bytes
 1.10.2/_images/subdag_after.png                    |   Bin 0 -> 30245 bytes
 1.10.2/_images/subdag_before.png                   |   Bin 0 -> 70382 bytes
 1.10.2/_images/subdag_zoom.png                     |   Bin 0 -> 150185 bytes
 1.10.2/_images/tree.png                            |   Bin 0 -> 128024 bytes
 1.10.2/_images/variable_hidden.png                 |   Bin 0 -> 154299 bytes
 .../contrib/executors/kubernetes_executor.html     |   897 ++
 .../airflow/contrib/executors/mesos_executor.html  |   549 +
 .../airflow/contrib/hooks/aws_athena_hook.html     |   369 +
 .../airflow/contrib/hooks/aws_dynamodb_hook.html   |   288 +
 .../airflow/contrib/hooks/aws_firehose_hook.html   |   275 +
 .../contrib/hooks/aws_glue_catalog_hook.html       |   337 +
 .../_modules/airflow/contrib/hooks/aws_hook.html   |   424 +
 .../airflow/contrib/hooks/aws_lambda_hook.html     |   287 +
 .../airflow/contrib/hooks/azure_cosmos_hook.html   |   506 +
 .../contrib/hooks/azure_data_lake_hook.html        |   360 +
 .../contrib/hooks/azure_fileshare_hook.html        |   431 +
 .../airflow/contrib/hooks/bigquery_hook.html       |  2184 +++
 .../airflow/contrib/hooks/cassandra_hook.html      |   416 +
 .../airflow/contrib/hooks/cloudant_hook.html       |   302 +
 .../airflow/contrib/hooks/databricks_hook.html     |   495 +
 .../airflow/contrib/hooks/datadog_hook.html        |   361 +
 .../airflow/contrib/hooks/datastore_hook.html      |   425 +
 .../contrib/hooks/discord_webhook_hook.html        |   359 +
 .../_modules/airflow/contrib/hooks/emr_hook.html   |   276 +
 1.10.2/_modules/airflow/contrib/hooks/fs_hook.html |   265 +
 .../_modules/airflow/contrib/hooks/ftp_hook.html   |   534 +
 .../airflow/contrib/hooks/gcp_api_base_hook.html   |   414 +
 .../airflow/contrib/hooks/gcp_compute_hook.html    |   554 +
 .../airflow/contrib/hooks/gcp_dataflow_hook.html   |   513 +
 .../airflow/contrib/hooks/gcp_dataproc_hook.html   |   464 +
 .../airflow/contrib/hooks/gcp_function_hook.html   |   417 +
 .../airflow/contrib/hooks/gcp_mlengine_hook.html   |   490 +
 .../airflow/contrib/hooks/gcp_pubsub_hook.html     |   504 +
 .../airflow/contrib/hooks/gcp_sql_hook.html        |  1216 ++
 .../airflow/contrib/hooks/gcp_transfer_hook.html   |   326 +
 .../_modules/airflow/contrib/hooks/gcs_hook.html   |   895 ++
 .../_modules/airflow/contrib/hooks/imap_hook.html  |   499 +
 .../_modules/airflow/contrib/hooks/jira_hook.html  |   306 +
 .../_modules/airflow/contrib/hooks/mongo_hook.html |   356 +
 .../airflow/contrib/hooks/openfaas_hook.html       |   318 +
 .../_modules/airflow/contrib/hooks/pinot_hook.html |   324 +
 .../airflow/contrib/hooks/qubole_hook.html         |   433 +
 .../_modules/airflow/contrib/hooks/redis_hook.html |   312 +
 .../airflow/contrib/hooks/redshift_hook.html       |   332 +
 .../airflow/contrib/hooks/sagemaker_hook.html      |   981 ++
 .../airflow/contrib/hooks/salesforce_hook.html     |   544 +
 .../_modules/airflow/contrib/hooks/sftp_hook.html  |   433 +
 .../airflow/contrib/hooks/slack_webhook_hook.html  |   348 +
 .../airflow/contrib/hooks/snowflake_hook.html      |   317 +
 .../airflow/contrib/hooks/spark_jdbc_hook.html     |   465 +
 .../airflow/contrib/hooks/spark_sql_hook.html      |   381 +
 .../airflow/contrib/hooks/spark_submit_hook.html   |   786 +
 .../_modules/airflow/contrib/hooks/sqoop_hook.html |   571 +
 .../_modules/airflow/contrib/hooks/ssh_hook.html   |   468 +
 .../airflow/contrib/hooks/vertica_hook.html        |   272 +
 .../_modules/airflow/contrib/hooks/wasb_hook.html  |   410 +
 .../_modules/airflow/contrib/hooks/winrm_hook.html |   356 +
 .../airflow/contrib/kubernetes/secret.html         |   260 +
 .../contrib/operators/adls_list_operator.html      |   289 +
 .../airflow/contrib/operators/adls_to_gcs.html     |   365 +
 .../contrib/operators/aws_athena_operator.html     |   317 +
 .../contrib/operators/awsbatch_operator.html       |   398 +
 .../contrib/operators/azure_cosmos_operator.html   |   288 +
 .../contrib/operators/bigquery_check_operator.html |   364 +
 .../contrib/operators/bigquery_get_data.html       |   335 +
 .../contrib/operators/bigquery_operator.html       |   856 ++
 .../operators/bigquery_table_delete_operator.html  |   285 +
 .../contrib/operators/bigquery_to_bigquery.html    |   314 +
 .../airflow/contrib/operators/bigquery_to_gcs.html |   324 +
 .../contrib/operators/cassandra_to_gcs.html        |   574 +
 .../contrib/operators/databricks_operator.html     |   699 +
 .../contrib/operators/dataflow_operator.html       |   593 +
 .../contrib/operators/dataproc_operator.html       |  1678 ++
 .../operators/datastore_export_operator.html       |   329 +
 .../operators/datastore_import_operator.html       |   317 +
 .../operators/discord_webhook_operator.html        |   317 +
 .../airflow/contrib/operators/druid_operator.html  |   283 +
 .../airflow/contrib/operators/ecs_operator.html    |   371 +
 .../contrib/operators/emr_add_steps_operator.html  |   282 +
 .../operators/emr_create_job_flow_operator.html    |   293 +
 .../operators/emr_terminate_job_flow_operator.html |   276 +
 .../airflow/contrib/operators/file_to_gcs.html     |   301 +
 .../airflow/contrib/operators/file_to_wasb.html    |   283 +
 .../contrib/operators/gcp_compute_operator.html    |   681 +
 .../contrib/operators/gcp_function_operator.html   |   539 +
 .../contrib/operators/gcp_sql_operator.html        |   988 ++
 .../contrib/operators/gcs_acl_operator.html        |   345 +
 .../contrib/operators/gcs_download_operator.html   |   310 +
 .../contrib/operators/gcs_list_operator.html       |   311 +
 .../airflow/contrib/operators/gcs_operator.html    |   341 +
 .../airflow/contrib/operators/gcs_to_bq.html       |   488 +
 .../airflow/contrib/operators/gcs_to_gcs.html      |   387 +
 .../operators/gcs_to_gcs_transfer_operator.html    |   346 +
 .../airflow/contrib/operators/gcs_to_s3.html       |   344 +
 .../contrib/operators/hipchat_operator.html        |   354 +
 .../contrib/operators/hive_to_dynamodb.html        |   328 +
 .../airflow/contrib/operators/jira_operator.html   |   313 +
 .../contrib/operators/kubernetes_pod_operator.html |   422 +
 .../contrib/operators/mlengine_operator.html       |   829 +
 .../airflow/contrib/operators/mongo_to_s3.html     |   347 +
 .../airflow/contrib/operators/mysql_to_gcs.html    |   508 +
 .../operators/postgres_to_gcs_operator.html        |   475 +
 .../airflow/contrib/operators/pubsub_operator.html |   652 +
 .../contrib/operators/qubole_check_operator.html   |   444 +
 .../airflow/contrib/operators/qubole_operator.html |   395 +
 .../contrib/operators/s3_list_operator.html        |   313 +
 .../contrib/operators/s3_to_gcs_operator.html      |   424 +
 .../operators/s3_to_gcs_transfer_operator.html     |   356 +
 .../contrib/operators/sagemaker_base_operator.html |   319 +
 .../sagemaker_endpoint_config_operator.html        |   285 +
 .../operators/sagemaker_endpoint_operator.html     |   369 +
 .../operators/sagemaker_model_operator.html        |   286 +
 .../operators/sagemaker_training_operator.html     |   317 +
 .../operators/sagemaker_transform_operator.html    |   343 +
 .../operators/sagemaker_tuning_operator.html       |   318 +
 .../airflow/contrib/operators/sftp_operator.html   |   399 +
 .../contrib/operators/slack_webhook_operator.html  |   311 +
 .../contrib/operators/snowflake_operator.html      |   286 +
 .../contrib/operators/spark_jdbc_operator.html     |   433 +
 .../contrib/operators/spark_sql_operator.html      |   330 +
 .../contrib/operators/spark_submit_operator.html   |   390 +
 .../airflow/contrib/operators/sqoop_operator.html  |   457 +
 .../airflow/contrib/operators/ssh_operator.html    |   392 +
 .../contrib/operators/vertica_operator.html        |   268 +
 .../airflow/contrib/operators/vertica_to_hive.html |   358 +
 .../airflow/contrib/operators/winrm_operator.html  |   329 +
 .../airflow/contrib/sensors/aws_athena_sensor.html |   298 +
 .../sensors/aws_glue_catalog_partition_sensor.html |   312 +
 .../sensors/aws_redshift_cluster_sensor.html       |   271 +
 .../contrib/sensors/azure_cosmos_sensor.html       |   286 +
 .../airflow/contrib/sensors/bash_sensor.html       |   314 +
 .../airflow/contrib/sensors/bigquery_sensor.html   |   289 +
 .../contrib/sensors/cassandra_record_sensor.html   |   280 +
 .../contrib/sensors/cassandra_table_sensor.html    |   275 +
 .../airflow/contrib/sensors/datadog_sensor.html    |   302 +
 .../airflow/contrib/sensors/emr_base_sensor.html   |   275 +
 .../contrib/sensors/emr_job_flow_sensor.html       |   274 +
 .../airflow/contrib/sensors/emr_step_sensor.html   |   277 +
 .../airflow/contrib/sensors/file_sensor.html       |   290 +
 .../airflow/contrib/sensors/ftp_sensor.html        |   319 +
 .../airflow/contrib/sensors/gcs_sensor.html        |   385 +
 .../airflow/contrib/sensors/hdfs_sensor.html       |   297 +
 .../airflow/contrib/sensors/jira_sensor.html       |   370 +
 .../airflow/contrib/sensors/pubsub_sensor.html     |   324 +
 .../airflow/contrib/sensors/python_sensor.html     |   300 +
 .../airflow/contrib/sensors/qubole_sensor.html     |   339 +
 .../airflow/contrib/sensors/redis_key_sensor.html  |   266 +
 .../contrib/sensors/sagemaker_base_sensor.html     |   293 +
 .../contrib/sensors/sagemaker_endpoint_sensor.html |   280 +
 .../contrib/sensors/sagemaker_training_sensor.html |   321 +
 .../sensors/sagemaker_transform_sensor.html        |   281 +
 .../contrib/sensors/sagemaker_tuning_sensor.html   |   281 +
 .../airflow/contrib/sensors/sftp_sensor.html       |   271 +
 .../airflow/contrib/sensors/wasb_sensor.html       |   318 +
 .../airflow/contrib/sensors/weekday_sensor.html    |   322 +
 .../airflow/executors/celery_executor.html         |   338 +
 .../_modules/airflow/executors/local_executor.html |   450 +
 .../airflow/executors/sequential_executor.html     |   276 +
 1.10.2/_modules/airflow/hooks/S3_hook.html         |   762 +
 1.10.2/_modules/airflow/hooks/dbapi_hook.html      |   522 +
 1.10.2/_modules/airflow/hooks/druid_hook.html      |   384 +
 1.10.2/_modules/airflow/hooks/hdfs_hook.html       |   320 +
 1.10.2/_modules/airflow/hooks/hive_hooks.html      |  1148 ++
 1.10.2/_modules/airflow/hooks/http_hook.html       |   422 +
 1.10.2/_modules/airflow/hooks/mssql_hook.html      |   271 +
 1.10.2/_modules/airflow/hooks/mysql_hook.html      |   363 +
 1.10.2/_modules/airflow/hooks/pig_hook.html        |   313 +
 1.10.2/_modules/airflow/hooks/postgres_hook.html   |   332 +
 1.10.2/_modules/airflow/hooks/presto_hook.html     |   359 +
 1.10.2/_modules/airflow/hooks/samba_hook.html      |   271 +
 1.10.2/_modules/airflow/hooks/slack_hook.html      |   281 +
 1.10.2/_modules/airflow/hooks/sqlite_hook.html     |   260 +
 1.10.2/_modules/airflow/hooks/zendesk_hook.html    |   331 +
 1.10.2/_modules/airflow/macros.html                |   312 +
 1.10.2/_modules/airflow/macros/hive.html           |   339 +
 1.10.2/_modules/airflow/models.html                |  5743 +++++++
 .../_modules/airflow/operators/bash_operator.html  |   357 +
 .../_modules/airflow/operators/check_operator.html |   481 +
 .../airflow/operators/dagrun_operator.html         |   317 +
 .../airflow/operators/druid_check_operator.html    |   310 +
 .../_modules/airflow/operators/dummy_operator.html |   257 +
 .../_modules/airflow/operators/email_operator.html |   297 +
 .../airflow/operators/generic_transfer.html        |   300 +
 .../_modules/airflow/operators/hive_operator.html  |   361 +
 .../airflow/operators/hive_stats_operator.html     |   408 +
 .../_modules/airflow/operators/hive_to_druid.html  |   465 +
 .../_modules/airflow/operators/hive_to_mysql.html  |   333 +
 .../airflow/operators/hive_to_samba_operator.html  |   286 +
 .../_modules/airflow/operators/http_operator.html  |   320 +
 .../airflow/operators/latest_only_operator.html    |   283 +
 .../_modules/airflow/operators/mssql_operator.html |   277 +
 .../_modules/airflow/operators/mssql_to_hive.html  |   358 +
 .../_modules/airflow/operators/mysql_operator.html |   280 +
 .../_modules/airflow/operators/mysql_to_hive.html  |   361 +
 .../_modules/airflow/operators/pig_operator.html   |   292 +
 .../airflow/operators/postgres_operator.html       |   281 +
 .../airflow/operators/presto_check_operator.html   |   346 +
 .../airflow/operators/presto_to_mysql.html         |   299 +
 .../airflow/operators/python_operator.html         |   620 +
 .../airflow/operators/redshift_to_s3_operator.html |   370 +
 .../operators/s3_file_transform_operator.html      |   386 +
 .../airflow/operators/s3_to_hive_operator.html     |   515 +
 .../airflow/operators/s3_to_redshift_operator.html |   327 +
 .../_modules/airflow/operators/slack_operator.html |   357 +
 .../airflow/operators/sqlite_operator.html         |   271 +
 .../airflow/operators/subdag_operator.html         |   322 +
 .../airflow/sensors/base_sensor_operator.html      |   350 +
 .../airflow/sensors/external_task_sensor.html      |   320 +
 1.10.2/_modules/airflow/sensors/hdfs_sensor.html   |   338 +
 .../airflow/sensors/hive_partition_sensor.html     |   294 +
 1.10.2/_modules/airflow/sensors/http_sensor.html   |   311 +
 .../sensors/metastore_partition_sensor.html        |   302 +
 .../sensors/named_hive_partition_sensor.html       |   323 +
 1.10.2/_modules/airflow/sensors/s3_key_sensor.html |   312 +
 .../_modules/airflow/sensors/s3_prefix_sensor.html |   301 +
 1.10.2/_modules/airflow/sensors/sql_sensor.html    |   277 +
 .../airflow/sensors/time_delta_sensor.html         |   265 +
 1.10.2/_modules/airflow/sensors/time_sensor.html   |   259 +
 .../_modules/airflow/sensors/web_hdfs_sensor.html  |   263 +
 1.10.2/_modules/index.html                         |   431 +
 1.10.2/_sources/api.rst.txt                        |   143 +
 1.10.2/_sources/changelog.rst.txt                  |    21 +
 1.10.2/_sources/cli.rst.txt                        |    28 +
 1.10.2/_sources/code.rst.txt                       |   483 +
 1.10.2/_sources/concepts.rst.txt                   |   901 ++
 1.10.2/_sources/faq.rst.txt                        |   201 +
 1.10.2/_sources/howto/check-health.rst.txt         |    47 +
 1.10.2/_sources/howto/executor/use-celery.rst.txt  |    71 +
 1.10.2/_sources/howto/executor/use-dask.rst.txt    |    50 +
 1.10.2/_sources/howto/executor/use-mesos.rst.txt   |    82 +
 1.10.2/_sources/howto/index.rst.txt                |    42 +
 1.10.2/_sources/howto/initialize-database.rst.txt  |    48 +
 1.10.2/_sources/howto/manage-connections.rst.txt   |   400 +
 1.10.2/_sources/howto/operator.rst.txt             |  1657 ++
 1.10.2/_sources/howto/run-with-systemd.rst.txt     |    32 +
 1.10.2/_sources/howto/run-with-upstart.rst.txt     |    37 +
 1.10.2/_sources/howto/secure-connections.rst.txt   |    50 +
 1.10.2/_sources/howto/set-config.rst.txt           |    66 +
 1.10.2/_sources/howto/use-test-config.rst.txt      |    34 +
 1.10.2/_sources/howto/write-logs.rst.txt           |   138 +
 1.10.2/_sources/index.rst.txt                      |    96 +
 1.10.2/_sources/installation.rst.txt               |   143 +
 1.10.2/_sources/integration.rst.txt                |  1419 ++
 1.10.2/_sources/kubernetes.rst.txt                 |   132 +
 1.10.2/_sources/license.rst.txt                    |   199 +
 1.10.2/_sources/lineage.rst.txt                    |   102 +
 1.10.2/_sources/metrics.rst.txt                    |    67 +
 1.10.2/_sources/plugins.rst.txt                    |   253 +
 1.10.2/_sources/profiling.rst.txt                  |    58 +
 1.10.2/_sources/project.rst.txt                    |    74 +
 1.10.2/_sources/scheduler.rst.txt                  |   186 +
 1.10.2/_sources/security.rst.txt                   |   439 +
 1.10.2/_sources/start.rst.txt                      |    71 +
 1.10.2/_sources/timezone.rst.txt                   |   163 +
 1.10.2/_sources/tutorial.rst.txt                   |   470 +
 1.10.2/_sources/ui.rst.txt                         |   119 +
 1.10.2/_static/ajax-loader.gif                     |   Bin 0 -> 673 bytes
 1.10.2/_static/basic.css                           |   676 +
 1.10.2/_static/comment-bright.png                  |   Bin 0 -> 756 bytes
 1.10.2/_static/comment-close.png                   |   Bin 0 -> 829 bytes
 1.10.2/_static/comment.png                         |   Bin 0 -> 641 bytes
 1.10.2/_static/css/badge_only.css                  |     1 +
 1.10.2/_static/css/theme.css                       |     6 +
 1.10.2/_static/doctools.js                         |   315 +
 1.10.2/_static/documentation_options.js            |    10 +
 1.10.2/_static/down-pressed.png                    |   Bin 0 -> 222 bytes
 1.10.2/_static/down.png                            |   Bin 0 -> 202 bytes
 1.10.2/_static/file.png                            |   Bin 0 -> 286 bytes
 1.10.2/_static/fonts/Inconsolata-Bold.ttf          |   Bin 0 -> 109948 bytes
 1.10.2/_static/fonts/Inconsolata-Regular.ttf       |   Bin 0 -> 96964 bytes
 1.10.2/_static/fonts/Inconsolata.ttf               |   Bin 0 -> 63184 bytes
 1.10.2/_static/fonts/Lato-Bold.ttf                 |   Bin 0 -> 656544 bytes
 1.10.2/_static/fonts/Lato-Regular.ttf              |   Bin 0 -> 656568 bytes
 1.10.2/_static/fonts/Lato/lato-bold.eot            |   Bin 0 -> 256056 bytes
 1.10.2/_static/fonts/Lato/lato-bold.ttf            |   Bin 0 -> 600856 bytes
 1.10.2/_static/fonts/Lato/lato-bold.woff           |   Bin 0 -> 309728 bytes
 1.10.2/_static/fonts/Lato/lato-bold.woff2          |   Bin 0 -> 184912 bytes
 1.10.2/_static/fonts/Lato/lato-bolditalic.eot      |   Bin 0 -> 266158 bytes
 1.10.2/_static/fonts/Lato/lato-bolditalic.ttf      |   Bin 0 -> 622572 bytes
 1.10.2/_static/fonts/Lato/lato-bolditalic.woff     |   Bin 0 -> 323344 bytes
 1.10.2/_static/fonts/Lato/lato-bolditalic.woff2    |   Bin 0 -> 193308 bytes
 1.10.2/_static/fonts/Lato/lato-italic.eot          |   Bin 0 -> 268604 bytes
 1.10.2/_static/fonts/Lato/lato-italic.ttf          |   Bin 0 -> 639388 bytes
 1.10.2/_static/fonts/Lato/lato-italic.woff         |   Bin 0 -> 328412 bytes
 1.10.2/_static/fonts/Lato/lato-italic.woff2        |   Bin 0 -> 195704 bytes
 1.10.2/_static/fonts/Lato/lato-regular.eot         |   Bin 0 -> 253461 bytes
 1.10.2/_static/fonts/Lato/lato-regular.ttf         |   Bin 0 -> 607720 bytes
 1.10.2/_static/fonts/Lato/lato-regular.woff        |   Bin 0 -> 309192 bytes
 1.10.2/_static/fonts/Lato/lato-regular.woff2       |   Bin 0 -> 182708 bytes
 1.10.2/_static/fonts/RobotoSlab-Bold.ttf           |   Bin 0 -> 170616 bytes
 1.10.2/_static/fonts/RobotoSlab-Regular.ttf        |   Bin 0 -> 169064 bytes
 .../fonts/RobotoSlab/roboto-slab-v7-bold.eot       |   Bin 0 -> 79520 bytes
 .../fonts/RobotoSlab/roboto-slab-v7-bold.ttf       |   Bin 0 -> 170616 bytes
 .../fonts/RobotoSlab/roboto-slab-v7-bold.woff      |   Bin 0 -> 87624 bytes
 .../fonts/RobotoSlab/roboto-slab-v7-bold.woff2     |   Bin 0 -> 67312 bytes
 .../fonts/RobotoSlab/roboto-slab-v7-regular.eot    |   Bin 0 -> 78331 bytes
 .../fonts/RobotoSlab/roboto-slab-v7-regular.ttf    |   Bin 0 -> 169064 bytes
 .../fonts/RobotoSlab/roboto-slab-v7-regular.woff   |   Bin 0 -> 86288 bytes
 .../fonts/RobotoSlab/roboto-slab-v7-regular.woff2  |   Bin 0 -> 66444 bytes
 1.10.2/_static/fonts/fontawesome-webfont.eot       |   Bin 0 -> 165742 bytes
 1.10.2/_static/fonts/fontawesome-webfont.svg       |  2671 ++++
 1.10.2/_static/fonts/fontawesome-webfont.ttf       |   Bin 0 -> 165548 bytes
 1.10.2/_static/fonts/fontawesome-webfont.woff      |   Bin 0 -> 98024 bytes
 1.10.2/_static/fonts/fontawesome-webfont.woff2     |   Bin 0 -> 77160 bytes
 1.10.2/_static/jquery-3.2.1.js                     | 10253 +++++++++++++
 1.10.2/_static/jquery.js                           |     4 +
 1.10.2/_static/js/modernizr.min.js                 |     4 +
 1.10.2/_static/js/theme.js                         |     3 +
 1.10.2/_static/language_data.js                    |   297 +
 1.10.2/_static/minus.png                           |   Bin 0 -> 90 bytes
 1.10.2/_static/plus.png                            |   Bin 0 -> 90 bytes
 1.10.2/_static/pygments.css                        |    69 +
 1.10.2/_static/searchtools.js                      |   481 +
 1.10.2/_static/underscore-1.3.1.js                 |   999 ++
 1.10.2/_static/underscore.js                       |    31 +
 1.10.2/_static/up-pressed.png                      |   Bin 0 -> 214 bytes
 1.10.2/_static/up.png                              |   Bin 0 -> 203 bytes
 1.10.2/_static/websupport.js                       |   808 +
 1.10.2/api.html                                    |   361 +
 1.10.2/changelog.html                              |  2587 ++++
 1.10.2/cli.html                                    |  1750 +++
 1.10.2/code.html                                   | 15156 +++++++++++++++++++
 1.10.2/concepts.html                               |  1035 ++
 1.10.2/faq.html                                    |   403 +
 1.10.2/genindex.html                               |  2230 +++
 1.10.2/howto/check-health.html                     |   282 +
 1.10.2/howto/executor/use-celery.html              |   293 +
 1.10.2/howto/executor/use-dask.html                |   275 +
 1.10.2/howto/executor/use-mesos.html               |   314 +
 1.10.2/howto/index.html                            |   297 +
 1.10.2/howto/initialize-database.html              |   280 +
 1.10.2/howto/manage-connections.html               |   597 +
 1.10.2/howto/operator.html                         |  2514 +++
 1.10.2/howto/run-with-systemd.html                 |   262 +
 1.10.2/howto/run-with-upstart.html                 |   264 +
 1.10.2/howto/secure-connections.html               |   278 +
 1.10.2/howto/set-config.html                       |   290 +
 1.10.2/howto/use-test-config.html                  |   263 +
 1.10.2/howto/write-logs.html                       |   376 +
 1.10.2/http-routingtable.html                      |   295 +
 1.10.2/index.html                                  |   937 ++
 1.10.2/installation.html                           |   425 +
 1.10.2/integration.html                            |  9075 +++++++++++
 1.10.2/kubernetes.html                             |   403 +
 1.10.2/license.html                                |   411 +
 1.10.2/lineage.html                                |   315 +
 1.10.2/metrics.html                                |   335 +
 1.10.2/objects.inv                                 |   Bin 0 -> 9678 bytes
 1.10.2/plugins.html                                |   457 +
 1.10.2/profiling.html                              |   272 +
 1.10.2/project.html                                |   288 +
 1.10.2/py-modindex.html                            |   252 +
 1.10.2/scheduler.html                              |   413 +
 1.10.2/search.html                                 |   238 +
 1.10.2/searchindex.js                              |     1 +
 1.10.2/security.html                               |   628 +
 1.10.2/start.html                                  |   283 +
 1.10.2/timezone.html                               |   365 +
 1.10.2/tutorial.html                               |   667 +
 1.10.2/ui.html                                     |   318 +
 379 files changed, 162695 insertions(+)

diff --git a/1.10.2/.buildinfo b/1.10.2/.buildinfo
new file mode 100644
index 0000000..2cd3f36
--- /dev/null
+++ b/1.10.2/.buildinfo
@@ -0,0 +1,4 @@
+# Sphinx build info version 1
+# This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
+config: 31d38953a908168368629ac053123a0f
+tags: 645f666f9bcd5a90fca523b33c5a78b7
diff --git a/1.10.2/_images/adhoc.png b/1.10.2/_images/adhoc.png
new file mode 100644
index 0000000..77ea780
Binary files /dev/null and b/1.10.2/_images/adhoc.png differ
diff --git a/1.10.2/_images/airflow.gif b/1.10.2/_images/airflow.gif
new file mode 100644
index 0000000..7646e37
Binary files /dev/null and b/1.10.2/_images/airflow.gif differ
diff --git a/1.10.2/_images/apache.jpg b/1.10.2/_images/apache.jpg
new file mode 100644
index 0000000..312251f
Binary files /dev/null and b/1.10.2/_images/apache.jpg differ
diff --git a/1.10.2/_images/branch_bad.png b/1.10.2/_images/branch_bad.png
new file mode 100644
index 0000000..586844f
Binary files /dev/null and b/1.10.2/_images/branch_bad.png differ
diff --git a/1.10.2/_images/branch_good.png b/1.10.2/_images/branch_good.png
new file mode 100644
index 0000000..fbd4650
Binary files /dev/null and b/1.10.2/_images/branch_good.png differ
diff --git a/1.10.2/_images/chart.png b/1.10.2/_images/chart.png
new file mode 100644
index 0000000..bfca26b
Binary files /dev/null and b/1.10.2/_images/chart.png differ
diff --git a/1.10.2/_images/chart_form.png b/1.10.2/_images/chart_form.png
new file mode 100644
index 0000000..f73daf5
Binary files /dev/null and b/1.10.2/_images/chart_form.png differ
diff --git a/1.10.2/_images/code.png b/1.10.2/_images/code.png
new file mode 100644
index 0000000..60e8f58
Binary files /dev/null and b/1.10.2/_images/code.png differ
diff --git a/1.10.2/_images/connection_create.png b/1.10.2/_images/connection_create.png
new file mode 100644
index 0000000..8a574d4
Binary files /dev/null and b/1.10.2/_images/connection_create.png differ
diff --git a/1.10.2/_images/connection_edit.png b/1.10.2/_images/connection_edit.png
new file mode 100644
index 0000000..c6d14da
Binary files /dev/null and b/1.10.2/_images/connection_edit.png differ
diff --git a/1.10.2/_images/connections.png b/1.10.2/_images/connections.png
new file mode 100644
index 0000000..3a28473
Binary files /dev/null and b/1.10.2/_images/connections.png differ
diff --git a/1.10.2/_images/context.png b/1.10.2/_images/context.png
new file mode 100644
index 0000000..bac718f
Binary files /dev/null and b/1.10.2/_images/context.png differ
diff --git a/1.10.2/_images/dags.png b/1.10.2/_images/dags.png
new file mode 100644
index 0000000..04c8213
Binary files /dev/null and b/1.10.2/_images/dags.png differ
diff --git a/1.10.2/_images/duration.png b/1.10.2/_images/duration.png
new file mode 100644
index 0000000..b59fc09
Binary files /dev/null and b/1.10.2/_images/duration.png differ
diff --git a/1.10.2/_images/gantt.png b/1.10.2/_images/gantt.png
new file mode 100644
index 0000000..d74e1e5
Binary files /dev/null and b/1.10.2/_images/gantt.png differ
diff --git a/1.10.2/_images/graph.png b/1.10.2/_images/graph.png
new file mode 100644
index 0000000..ea2b705
Binary files /dev/null and b/1.10.2/_images/graph.png differ
diff --git a/1.10.2/_images/latest_only_with_trigger.png b/1.10.2/_images/latest_only_with_trigger.png
new file mode 100644
index 0000000..629adfa
Binary files /dev/null and b/1.10.2/_images/latest_only_with_trigger.png differ
diff --git a/1.10.2/_images/pin_large.png b/1.10.2/_images/pin_large.png
new file mode 100644
index 0000000..986c88b
Binary files /dev/null and b/1.10.2/_images/pin_large.png differ
diff --git a/1.10.2/_images/subdag_after.png b/1.10.2/_images/subdag_after.png
new file mode 100644
index 0000000..166a6de
Binary files /dev/null and b/1.10.2/_images/subdag_after.png differ
diff --git a/1.10.2/_images/subdag_before.png b/1.10.2/_images/subdag_before.png
new file mode 100644
index 0000000..ebc3e58
Binary files /dev/null and b/1.10.2/_images/subdag_before.png differ
diff --git a/1.10.2/_images/subdag_zoom.png b/1.10.2/_images/subdag_zoom.png
new file mode 100644
index 0000000..08fcf5c
Binary files /dev/null and b/1.10.2/_images/subdag_zoom.png differ
diff --git a/1.10.2/_images/tree.png b/1.10.2/_images/tree.png
new file mode 100644
index 0000000..06750aa
Binary files /dev/null and b/1.10.2/_images/tree.png differ
diff --git a/1.10.2/_images/variable_hidden.png b/1.10.2/_images/variable_hidden.png
new file mode 100644
index 0000000..e081ca3
Binary files /dev/null and b/1.10.2/_images/variable_hidden.png differ
diff --git a/1.10.2/_modules/airflow/contrib/executors/kubernetes_executor.html b/1.10.2/_modules/airflow/contrib/executors/kubernetes_executor.html
new file mode 100644
index 0000000..69d7e74
--- /dev/null
+++ b/1.10.2/_modules/airflow/contrib/executors/kubernetes_executor.html
@@ -0,0 +1,897 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>airflow.contrib.executors.kubernetes_executor &mdash; Airflow Documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+    <link rel="index" title="Index" href="../../../../genindex.html" />
+    <link rel="search" title="Search" href="../../../../search.html" /> 
+
+  
+  <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../../index.html" class="icon icon-home"> Airflow
+          
+
+          
+          </a>
+
+          
+            
+            
+              <div class="version">
+                1.10.2
+              </div>
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../metrics.html">Metrics</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../../index.html">Airflow</a>
+        
+      </nav>
+
+
+      <div class="wy-nav-content">
+        
+        <div class="rst-content">
+        
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../../index.html">Module code</a> &raquo;</li>
+        
+      <li>airflow.contrib.executors.kubernetes_executor</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for airflow.contrib.executors.kubernetes_executor</h1><div class="highlight"><pre>
+<span></span><span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements.  See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership.  The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># &quot;License&quot;); you may not use this file except in compliance</span>
+<span class="c1"># with the License.  You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1">#   http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># &quot;AS IS&quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied.  See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">import</span> <span class="nn">base64</span>
+<span class="kn">import</span> <span class="nn">json</span>
+<span class="kn">import</span> <span class="nn">multiprocessing</span>
+<span class="kn">from</span> <span class="nn">queue</span> <span class="k">import</span> <span class="n">Queue</span>
+<span class="kn">from</span> <span class="nn">dateutil</span> <span class="k">import</span> <span class="n">parser</span>
+<span class="kn">from</span> <span class="nn">uuid</span> <span class="k">import</span> <span class="n">uuid4</span>
+<span class="kn">import</span> <span class="nn">kubernetes</span>
+<span class="kn">from</span> <span class="nn">kubernetes</span> <span class="k">import</span> <span class="n">watch</span><span class="p">,</span> <span class="n">client</span>
+<span class="kn">from</span> <span class="nn">kubernetes.client.rest</span> <span class="k">import</span> <span class="n">ApiException</span>
+<span class="kn">from</span> <span class="nn">airflow.configuration</span> <span class="k">import</span> <span class="n">conf</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.kubernetes.pod_launcher</span> <span class="k">import</span> <span class="n">PodLauncher</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.kubernetes.kube_client</span> <span class="k">import</span> <span class="n">get_kube_client</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.kubernetes.worker_configuration</span> <span class="k">import</span> <span class="n">WorkerConfiguration</span>
+<span class="kn">from</span> <span class="nn">airflow.executors.base_executor</span> <span class="k">import</span> <span class="n">BaseExecutor</span>
+<span class="kn">from</span> <span class="nn">airflow.executors</span> <span class="k">import</span> <span class="n">Executors</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">TaskInstance</span><span class="p">,</span> <span class="n">KubeResourceVersion</span><span class="p">,</span> <span class="n">KubeWorkerIdentifier</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.state</span> <span class="k">import</span> <span class="n">State</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.db</span> <span class="k">import</span> <span class="n">provide_session</span><span class="p">,</span> <span class="n">create_session</span>
+<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">configuration</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowConfigException</span><span class="p">,</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
+
+
+<span class="k">class</span> <span class="nc">KubernetesExecutorConfig</span><span class="p">:</span>
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">image</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">image_pull_policy</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">request_memory</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                 <span class="n">request_cpu</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">limit_memory</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">limit_cpu</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                 <span class="n">gcp_service_account_key</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">node_selectors</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">affinity</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                 <span class="n">annotations</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">volumes</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">volume_mounts</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">tolerations</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">image</span> <span class="o">=</span> <span class="n">image</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">image_pull_policy</span> <span class="o">=</span> <span class="n">image_pull_policy</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">request_memory</span> <span class="o">=</span> <span class="n">request_memory</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">request_cpu</span> <span class="o">=</span> <span class="n">request_cpu</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">limit_memory</span> <span class="o">=</span> <span class="n">limit_memory</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">limit_cpu</span> <span class="o">=</span> <span class="n">limit_cpu</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">gcp_service_account_key</span> <span class="o">=</span> <span class="n">gcp_service_account_key</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">node_selectors</span> <span class="o">=</span> <span class="n">node_selectors</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">affinity</span> <span class="o">=</span> <span class="n">affinity</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">annotations</span> <span class="o">=</span> <span class="n">annotations</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">volumes</span> <span class="o">=</span> <span class="n">volumes</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">volume_mounts</span> <span class="o">=</span> <span class="n">volume_mounts</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">tolerations</span> <span class="o">=</span> <span class="n">tolerations</span>
+
+    <span class="k">def</span> <span class="nf">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="k">return</span> <span class="s2">&quot;</span><span class="si">{}</span><span class="s2">(image=</span><span class="si">{}</span><span class="s2">, image_pull_policy=</span><span class="si">{}</span><span class="s2">, request_memory=</span><span class="si">{}</span><span class="s2">, request_cpu=</span><span class="si">{}</span><span class="s2">, &quot;</span> \
+               <span class="s2">&quot;limit_memory=</span><span class="si">{}</span><span class="s2">, limit_cpu=</span><span class="si">{}</span><span class="s2">, gcp_service_account_key=</span><span class="si">{}</span><span class="s2">, &quot;</span> \
+               <span class="s2">&quot;node_selectors=</span><span class="si">{}</span><span class="s2">, affinity=</span><span class="si">{}</span><span class="s2">, annotations=</span><span class="si">{}</span><span class="s2">, volumes=</span><span class="si">{}</span><span class="s2">, &quot;</span> \
+               <span class="s2">&quot;volume_mounts=</span><span class="si">{}</span><span class="s2">, tolerations=</span><span class="si">{}</span><span class="s2">)&quot;</span> \
+            <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">KubernetesExecutorConfig</span><span class="o">.</span><span class="vm">__name__</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">image</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">image_pull_policy</span><span class="p">,</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">request_memory</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">request_cpu</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">limit_memory</span><span class="p">,</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">limit_cpu</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">gcp_service_account_key</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">node_selectors</span><span class="p">,</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">affinity</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">annotations</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">volumes</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">volume_mounts</span><span class="p">,</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">tolerations</span><span class="p">)</span>
+
+    <span class="nd">@staticmethod</span>
+    <span class="k">def</span> <span class="nf">from_dict</span><span class="p">(</span><span class="n">obj</span><span class="p">):</span>
+        <span class="k">if</span> <span class="n">obj</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">KubernetesExecutorConfig</span><span class="p">()</span>
+
+        <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
+            <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span>
+                <span class="s1">&#39;Cannot convert a non-dictionary object into a KubernetesExecutorConfig&#39;</span><span class="p">)</span>
+
+        <span class="n">namespaced</span> <span class="o">=</span> <span class="n">obj</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">Executors</span><span class="o">.</span><span class="n">KubernetesExecutor</span><span class="p">,</span> <span class="p">{})</span>
+
+        <span class="k">return</span> <span class="n">KubernetesExecutorConfig</span><span class="p">(</span>
+            <span class="n">image</span><span class="o">=</span><span class="n">namespaced</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;image&#39;</span><span class="p">,</span> <span class="kc">None</span><span class="p">),</span>
+            <span class="n">image_pull_policy</span><span class="o">=</span><span class="n">namespaced</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;image_pull_policy&#39;</span><span class="p">,</span> <span class="kc">None</span><span class="p">),</span>
+            <span class="n">request_memory</span><span class="o">=</span><span class="n">namespaced</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;request_memory&#39;</span><span class="p">,</span> <span class="kc">None</span><span class="p">),</span>
+            <span class="n">request_cpu</span><span class="o">=</span><span class="n">namespaced</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;request_cpu&#39;</span><span class="p">,</span> <span class="kc">None</span><span class="p">),</span>
+            <span class="n">limit_memory</span><span class="o">=</span><span class="n">namespaced</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;limit_memory&#39;</span><span class="p">,</span> <span class="kc">None</span><span class="p">),</span>
+            <span class="n">limit_cpu</span><span class="o">=</span><span class="n">namespaced</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;limit_cpu&#39;</span><span class="p">,</span> <span class="kc">None</span><span class="p">),</span>
+            <span class="n">gcp_service_account_key</span><span class="o">=</span><span class="n">namespaced</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;gcp_service_account_key&#39;</span><span class="p">,</span> <span class="kc">None</span><span class="p">),</span>
+            <span class="n">node_selectors</span><span class="o">=</span><span class="n">namespaced</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;node_selectors&#39;</span><span class="p">,</span> <span class="kc">None</span><span class="p">),</span>
+            <span class="n">affinity</span><span class="o">=</span><span class="n">namespaced</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;affinity&#39;</span><span class="p">,</span> <span class="kc">None</span><span class="p">),</span>
+            <span class="n">annotations</span><span class="o">=</span><span class="n">namespaced</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;annotations&#39;</span><span class="p">,</span> <span class="p">{}),</span>
+            <span class="n">volumes</span><span class="o">=</span><span class="n">namespaced</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;volumes&#39;</span><span class="p">,</span> <span class="p">[]),</span>
+            <span class="n">volume_mounts</span><span class="o">=</span><span class="n">namespaced</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;volume_mounts&#39;</span><span class="p">,</span> <span class="p">[]),</span>
+            <span class="n">tolerations</span><span class="o">=</span><span class="n">namespaced</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;tolerations&#39;</span><span class="p">,</span> <span class="kc">None</span><span class="p">),</span>
+        <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">as_dict</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="k">return</span> <span class="p">{</span>
+            <span class="s1">&#39;image&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">image</span><span class="p">,</span>
+            <span class="s1">&#39;image_pull_policy&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">image_pull_policy</span><span class="p">,</span>
+            <span class="s1">&#39;request_memory&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">request_memory</span><span class="p">,</span>
+            <span class="s1">&#39;request_cpu&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">request_cpu</span><span class="p">,</span>
+            <span class="s1">&#39;limit_memory&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">limit_memory</span><span class="p">,</span>
+            <span class="s1">&#39;limit_cpu&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">limit_cpu</span><span class="p">,</span>
+            <span class="s1">&#39;gcp_service_account_key&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">gcp_service_account_key</span><span class="p">,</span>
+            <span class="s1">&#39;node_selectors&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">node_selectors</span><span class="p">,</span>
+            <span class="s1">&#39;affinity&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">affinity</span><span class="p">,</span>
+            <span class="s1">&#39;annotations&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">annotations</span><span class="p">,</span>
+            <span class="s1">&#39;volumes&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">volumes</span><span class="p">,</span>
+            <span class="s1">&#39;volume_mounts&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">volume_mounts</span><span class="p">,</span>
+            <span class="s1">&#39;tolerations&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">tolerations</span><span class="p">,</span>
+        <span class="p">}</span>
+
+
+<span class="k">class</span> <span class="nc">KubeConfig</span><span class="p">:</span>
+    <span class="n">core_section</span> <span class="o">=</span> <span class="s1">&#39;core&#39;</span>
+    <span class="n">kubernetes_section</span> <span class="o">=</span> <span class="s1">&#39;kubernetes&#39;</span>
+
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="n">configuration_dict</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">as_dict</span><span class="p">(</span><span class="n">display_sensitive</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">core_configuration</span> <span class="o">=</span> <span class="n">configuration_dict</span><span class="p">[</span><span class="s1">&#39;core&#39;</span><span class="p">]</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">kube_secrets</span> <span class="o">=</span> <span class="n">configuration_dict</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;kubernetes_secrets&#39;</span><span class="p">,</span> <span class="p">{})</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">airflow_home</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">core_section</span><span class="p">,</span> <span class="s1">&#39;airflow_home&#39;</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">dags_folder</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">core_section</span><span class="p">,</span> <span class="s1">&#39;dags_folder&#39;</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">parallelism</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">core_section</span><span class="p">,</span> <span class="s1">&#39;PARALLELISM&#39;</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">worker_container_repository</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">&#39;worker_container_repository&#39;</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">worker_container_tag</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">&#39;worker_container_tag&#39;</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">kube_image</span> <span class="o">=</span> <span class="s1">&#39;</span><span class="si">{}</span><span class="s1">:</span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">worker_container_repository</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">worker_container_tag</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">kube_image_pull_policy</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s2">&quot;worker_container_image_pull_policy&quot;</span>
+        <span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">kube_node_selectors</span> <span class="o">=</span> <span class="n">configuration_dict</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;kubernetes_node_selectors&#39;</span><span class="p">,</span> <span class="p">{})</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">delete_worker_pods</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">&#39;delete_worker_pods&#39;</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">worker_service_account_name</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">&#39;worker_service_account_name&#39;</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">image_pull_secrets</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">&#39;image_pull_secrets&#39;</span><span class="p">)</span>
+
+        <span class="c1"># NOTE: user can build the dags into the docker image directly,</span>
+        <span class="c1"># this will set to True if so</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">dags_in_image</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">&#39;dags_in_image&#39;</span><span class="p">)</span>
+
+        <span class="c1"># NOTE: `git_repo` and `git_branch` must be specified together as a pair</span>
+        <span class="c1"># The http URL of the git repository to clone from</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">git_repo</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">&#39;git_repo&#39;</span><span class="p">)</span>
+        <span class="c1"># The branch of the repository to be checked out</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">git_branch</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">&#39;git_branch&#39;</span><span class="p">)</span>
+        <span class="c1"># Optionally, the directory in the git repository containing the dags</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">git_subpath</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">&#39;git_subpath&#39;</span><span class="p">)</span>
+        <span class="c1"># Optionally, the root directory for git operations</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">git_sync_root</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">&#39;git_sync_root&#39;</span><span class="p">)</span>
+        <span class="c1"># Optionally, the name at which to publish the checked-out files under --root</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">git_sync_dest</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">&#39;git_sync_dest&#39;</span><span class="p">)</span>
+        <span class="c1"># Optionally, if git_dags_folder_mount_point is set the worker will use</span>
+        <span class="c1"># {git_dags_folder_mount_point}/{git_sync_dest}/{git_subpath} as dags_folder</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">git_dags_folder_mount_point</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span>
+                                                    <span class="s1">&#39;git_dags_folder_mount_point&#39;</span><span class="p">)</span>
+
+        <span class="c1"># Optionally a user may supply a `git_user` and `git_password` for private</span>
+        <span class="c1"># repositories</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">git_user</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">&#39;git_user&#39;</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">git_password</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">&#39;git_password&#39;</span><span class="p">)</span>
+
+        <span class="c1"># NOTE: The user may optionally use a volume claim to mount a PV containing</span>
+        <span class="c1"># DAGs directly</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">dags_volume_claim</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">&#39;dags_volume_claim&#39;</span><span class="p">)</span>
+
+        <span class="c1"># This prop may optionally be set for PV Claims and is used to write logs</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">logs_volume_claim</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">&#39;logs_volume_claim&#39;</span><span class="p">)</span>
+
+        <span class="c1"># This prop may optionally be set for PV Claims and is used to locate DAGs</span>
+        <span class="c1"># on a SubPath</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">dags_volume_subpath</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">&#39;dags_volume_subpath&#39;</span><span class="p">)</span>
+
+        <span class="c1"># This prop may optionally be set for PV Claims and is used to locate logs</span>
+        <span class="c1"># on a SubPath</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">logs_volume_subpath</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">&#39;logs_volume_subpath&#39;</span><span class="p">)</span>
+
+        <span class="c1"># Optionally, hostPath volume containing DAGs</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">dags_volume_host</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">&#39;dags_volume_host&#39;</span><span class="p">)</span>
+
+        <span class="c1"># Optionally, write logs to a hostPath Volume</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">logs_volume_host</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">&#39;logs_volume_host&#39;</span><span class="p">)</span>
+
+        <span class="c1"># This prop may optionally be set for PV Claims and is used to write logs</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">base_log_folder</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">core_section</span><span class="p">,</span> <span class="s1">&#39;base_log_folder&#39;</span><span class="p">)</span>
+
+        <span class="c1"># The Kubernetes Namespace in which the Scheduler and Webserver reside. Note</span>
+        <span class="c1"># that if your</span>
+        <span class="c1"># cluster has RBAC enabled, your scheduler may need service account permissions to</span>
+        <span class="c1"># create, watch, get, and delete pods in this namespace.</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">kube_namespace</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">&#39;namespace&#39;</span><span class="p">)</span>
+        <span class="c1"># The Kubernetes Namespace in which pods will be created by the executor. Note</span>
+        <span class="c1"># that if your</span>
+        <span class="c1"># cluster has RBAC enabled, your workers may need service account permissions to</span>
+        <span class="c1"># interact with cluster components.</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">executor_namespace</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">&#39;namespace&#39;</span><span class="p">)</span>
+        <span class="c1"># Task secrets managed by KubernetesExecutor.</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">gcp_service_account_keys</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span>
+                                                 <span class="s1">&#39;gcp_service_account_keys&#39;</span><span class="p">)</span>
+
+        <span class="c1"># If the user is using the git-sync container to clone their repository via git,</span>
+        <span class="c1"># allow them to specify repository, tag, and pod name for the init container.</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">git_sync_container_repository</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">&#39;git_sync_container_repository&#39;</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">git_sync_container_tag</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">&#39;git_sync_container_tag&#39;</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">git_sync_container</span> <span class="o">=</span> <span class="s1">&#39;</span><span class="si">{}</span><span class="s1">:</span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">git_sync_container_repository</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">git_sync_container_tag</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">git_sync_init_container_name</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">&#39;git_sync_init_container_name&#39;</span><span class="p">)</span>
+
+        <span class="c1"># The worker pod may optionally have a  valid Airflow config loaded via a</span>
+        <span class="c1"># configmap</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">airflow_configmap</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">&#39;airflow_configmap&#39;</span><span class="p">)</span>
+
+        <span class="n">affinity_json</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">&#39;affinity&#39;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="n">affinity_json</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">kube_affinity</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">affinity_json</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">kube_affinity</span> <span class="o">=</span> <span class="kc">None</span>
+
+        <span class="n">tolerations_json</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">kubernetes_section</span><span class="p">,</span> <span class="s1">&#39;tolerations&#39;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="n">tolerations_json</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">kube_tolerations</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">tolerations_json</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">kube_tolerations</span> <span class="o">=</span> <span class="kc">None</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">_validate</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="nf">_validate</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="c1"># TODO: use XOR for dags_volume_claim and git_dags_folder_mount_point</span>
+        <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">dags_volume_claim</span> \
+           <span class="ow">and</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">dags_volume_host</span> \
+           <span class="ow">and</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">dags_in_image</span> \
+           <span class="ow">and</span> <span class="p">(</span><span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">git_repo</span> <span class="ow">or</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">git_branch</span> <span class="ow">or</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">git_dags_folder_mount_point</span><span class="p">):</span>
+            <span class="k">raise</span> <span class="n">AirflowConfigException</span><span class="p">(</span>
+                <span class="s1">&#39;In kubernetes mode the following must be set in the `kubernetes` &#39;</span>
+                <span class="s1">&#39;config section: `dags_volume_claim` &#39;</span>
+                <span class="s1">&#39;or `dags_volume_host` &#39;</span>
+                <span class="s1">&#39;or `dags_in_image` &#39;</span>
+                <span class="s1">&#39;or `git_repo and git_branch and git_dags_folder_mount_point`&#39;</span><span class="p">)</span>
+
+
+<span class="k">class</span> <span class="nc">KubernetesJobWatcher</span><span class="p">(</span><span class="n">multiprocessing</span><span class="o">.</span><span class="n">Process</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">,</span> <span class="nb">object</span><span class="p">):</span>
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">namespace</span><span class="p">,</span> <span class="n">watcher_queue</span><span class="p">,</span> <span class="n">resource_version</span><span class="p">,</span> <span class="n">worker_uuid</span><span class="p">):</span>
+        <span class="n">multiprocessing</span><span class="o">.</span><span class="n">Process</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">namespace</span> <span class="o">=</span> <span class="n">namespace</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">worker_uuid</span> <span class="o">=</span> <span class="n">worker_uuid</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">watcher_queue</span> <span class="o">=</span> <span class="n">watcher_queue</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">resource_version</span> <span class="o">=</span> <span class="n">resource_version</span>
+
+    <span class="k">def</span> <span class="nf">run</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="n">kube_client</span> <span class="o">=</span> <span class="n">get_kube_client</span><span class="p">()</span>
+        <span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
+            <span class="k">try</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">resource_version</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_run</span><span class="p">(</span><span class="n">kube_client</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">resource_version</span><span class="p">,</span>
+                                                  <span class="bp">self</span><span class="o">.</span><span class="n">worker_uuid</span><span class="p">)</span>
+            <span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="s1">&#39;Unknown error in KubernetesJobWatcher. Failing&#39;</span><span class="p">)</span>
+                <span class="k">raise</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s1">&#39;Watch died gracefully, starting back up with: &#39;</span>
+                              <span class="s1">&#39;last resource_version: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">resource_version</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_run</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">kube_client</span><span class="p">,</span> <span class="n">resource_version</span><span class="p">,</span> <span class="n">worker_uuid</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+            <span class="s1">&#39;Event: and now my watch begins starting at resource_version: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span>
+            <span class="n">resource_version</span>
+        <span class="p">)</span>
+        <span class="n">watcher</span> <span class="o">=</span> <span class="n">watch</span><span class="o">.</span><span class="n">Watch</span><span class="p">()</span>
+
+        <span class="n">kwargs</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;label_selector&#39;</span><span class="p">:</span> <span class="s1">&#39;airflow-worker=</span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">worker_uuid</span><span class="p">)}</span>
+        <span class="k">if</span> <span class="n">resource_version</span><span class="p">:</span>
+            <span class="n">kwargs</span><span class="p">[</span><span class="s1">&#39;resource_version&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">resource_version</span>
+
+        <span class="n">last_resource_version</span> <span class="o">=</span> <span class="kc">None</span>
+        <span class="k">for</span> <span class="n">event</span> <span class="ow">in</span> <span class="n">watcher</span><span class="o">.</span><span class="n">stream</span><span class="p">(</span><span class="n">kube_client</span><span class="o">.</span><span class="n">list_namespaced_pod</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">namespace</span><span class="p">,</span>
+                                    <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+            <span class="n">task</span> <span class="o">=</span> <span class="n">event</span><span class="p">[</span><span class="s1">&#39;object&#39;</span><span class="p">]</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+                <span class="s1">&#39;Event: </span><span class="si">%s</span><span class="s1"> had an event of type </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span>
+                <span class="n">task</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="n">event</span><span class="p">[</span><span class="s1">&#39;type&#39;</span><span class="p">]</span>
+            <span class="p">)</span>
+            <span class="k">if</span> <span class="n">event</span><span class="p">[</span><span class="s1">&#39;type&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="s1">&#39;ERROR&#39;</span><span class="p">:</span>
+                <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">process_error</span><span class="p">(</span><span class="n">event</span><span class="p">)</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">process_status</span><span class="p">(</span>
+                <span class="n">task</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="n">task</span><span class="o">.</span><span class="n">status</span><span class="o">.</span><span class="n">phase</span><span class="p">,</span> <span class="n">task</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">labels</span><span class="p">,</span>
+                <span class="n">task</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">resource_version</span>
+            <span class="p">)</span>
+            <span class="n">last_resource_version</span> <span class="o">=</span> <span class="n">task</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">resource_version</span>
+
+        <span class="k">return</span> <span class="n">last_resource_version</span>
+
+    <span class="k">def</span> <span class="nf">process_error</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">event</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span>
+            <span class="s1">&#39;Encountered Error response from k8s list namespaced pod stream =&gt; </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span>
+            <span class="n">event</span>
+        <span class="p">)</span>
+        <span class="n">raw_object</span> <span class="o">=</span> <span class="n">event</span><span class="p">[</span><span class="s1">&#39;raw_object&#39;</span><span class="p">]</span>
+        <span class="k">if</span> <span class="n">raw_object</span><span class="p">[</span><span class="s1">&#39;code&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="mi">410</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+                <span class="s1">&#39;Kubernetes resource version is too old, must reset to 0 =&gt; </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span>
+                <span class="n">raw_object</span><span class="p">[</span><span class="s1">&#39;message&#39;</span><span class="p">]</span>
+            <span class="p">)</span>
+            <span class="c1"># Return resource version 0</span>
+            <span class="k">return</span> <span class="s1">&#39;0&#39;</span>
+        <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+            <span class="s1">&#39;Kubernetes failure for </span><span class="si">%s</span><span class="s1"> with code </span><span class="si">%s</span><span class="s1"> and message: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span>
+            <span class="n">raw_object</span><span class="p">[</span><span class="s1">&#39;reason&#39;</span><span class="p">],</span> <span class="n">raw_object</span><span class="p">[</span><span class="s1">&#39;code&#39;</span><span class="p">],</span> <span class="n">raw_object</span><span class="p">[</span><span class="s1">&#39;message&#39;</span><span class="p">]</span>
+        <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">process_status</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">pod_id</span><span class="p">,</span> <span class="n">status</span><span class="p">,</span> <span class="n">labels</span><span class="p">,</span> <span class="n">resource_version</span><span class="p">):</span>
+        <span class="k">if</span> <span class="n">status</span> <span class="o">==</span> <span class="s1">&#39;Pending&#39;</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Event: </span><span class="si">%s</span><span class="s1"> Pending&#39;</span><span class="p">,</span> <span class="n">pod_id</span><span class="p">)</span>
+        <span class="k">elif</span> <span class="n">status</span> <span class="o">==</span> <span class="s1">&#39;Failed&#39;</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Event: </span><span class="si">%s</span><span class="s1"> Failed&#39;</span><span class="p">,</span> <span class="n">pod_id</span><span class="p">)</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">watcher_queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="n">pod_id</span><span class="p">,</span> <span class="n">State</span><span class="o">.</span><span class="n">FAILED</span><span class="p">,</span> <span class="n">labels</span><span class="p">,</span> <span class="n">resource_version</span><span class="p">))</span>
+        <span class="k">elif</span> <span class="n">status</span> <span class="o">==</span> <span class="s1">&#39;Succeeded&#39;</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Event: </span><span class="si">%s</span><span class="s1"> Succeeded&#39;</span><span class="p">,</span> <span class="n">pod_id</span><span class="p">)</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">watcher_queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="n">pod_id</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="n">labels</span><span class="p">,</span> <span class="n">resource_version</span><span class="p">))</span>
+        <span class="k">elif</span> <span class="n">status</span> <span class="o">==</span> <span class="s1">&#39;Running&#39;</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Event: </span><span class="si">%s</span><span class="s1"> is Running&#39;</span><span class="p">,</span> <span class="n">pod_id</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
+                <span class="s1">&#39;Event: Invalid state: </span><span class="si">%s</span><span class="s1"> on pod: </span><span class="si">%s</span><span class="s1"> with labels: </span><span class="si">%s</span><span class="s1"> with &#39;</span>
+                <span class="s1">&#39;resource_version: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="n">status</span><span class="p">,</span> <span class="n">pod_id</span><span class="p">,</span> <span class="n">labels</span><span class="p">,</span> <span class="n">resource_version</span>
+            <span class="p">)</span>
+
+
+<span class="k">class</span> <span class="nc">AirflowKubernetesScheduler</span><span class="p">(</span><span class="n">LoggingMixin</span><span class="p">):</span>
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">kube_config</span><span class="p">,</span> <span class="n">task_queue</span><span class="p">,</span> <span class="n">result_queue</span><span class="p">,</span> <span class="n">kube_client</span><span class="p">,</span> <span class="n">worker_uuid</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;Creating Kubernetes executor&quot;</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">kube_config</span> <span class="o">=</span> <span class="n">kube_config</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span> <span class="o">=</span> <span class="n">task_queue</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span> <span class="o">=</span> <span class="n">result_queue</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">namespace</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">kube_config</span><span class="o">.</span><span class="n">kube_namespace</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;Kubernetes using namespace </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">namespace</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">kube_client</span> <span class="o">=</span> <span class="n">kube_client</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">launcher</span> <span class="o">=</span> <span class="n">PodLauncher</span><span class="p">(</span><span class="n">kube_client</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">kube_client</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">worker_configuration</span> <span class="o">=</span> <span class="n">WorkerConfiguration</span><span class="p">(</span><span class="n">kube_config</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">kube_config</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">watcher_queue</span> <span class="o">=</span> <span class="n">multiprocessing</span><span class="o">.</span><span class="n">Queue</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">worker_uuid</span> <span class="o">=</span> <span class="n">worker_uuid</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">kube_watcher</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_make_kube_watcher</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="nf">_make_kube_watcher</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="n">resource_version</span> <span class="o">=</span> <span class="n">KubeResourceVersion</span><span class="o">.</span><span class="n">get_current_resource_version</span><span class="p">()</span>
+        <span class="n">watcher</span> <span class="o">=</span> <span class="n">KubernetesJobWatcher</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">namespace</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">watcher_queue</span><span class="p">,</span>
+                                       <span class="n">resource_version</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">worker_uuid</span><span class="p">)</span>
+        <span class="n">watcher</span><span class="o">.</span><span class="n">start</span><span class="p">()</span>
+        <span class="k">return</span> <span class="n">watcher</span>
+
+    <span class="k">def</span> <span class="nf">_health_check_kube_watcher</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">kube_watcher</span><span class="o">.</span><span class="n">is_alive</span><span class="p">():</span>
+            <span class="k">pass</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span>
+                <span class="s1">&#39;Error while health checking kube watcher process. &#39;</span>
+                <span class="s1">&#39;Process died for unknown reasons&#39;</span><span class="p">)</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">kube_watcher</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_make_kube_watcher</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="nf">run_next</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">next_job</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+
+<span class="sd">        The run_next command will check the task_queue for any un-run jobs.</span>
+<span class="sd">        It will then create a unique job-id, launch that job in the cluster,</span>
+<span class="sd">        and store relevant info in the current_jobs map so we can track the job&#39;s</span>
+<span class="sd">        status</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Kubernetes job is </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">next_job</span><span class="p">))</span>
+        <span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,</span> <span class="n">kube_executor_config</span> <span class="o">=</span> <span class="n">next_job</span>
+        <span class="n">dag_id</span><span class="p">,</span> <span class="n">task_id</span><span class="p">,</span> <span class="n">execution_date</span><span class="p">,</span> <span class="n">try_number</span> <span class="o">=</span> <span class="n">key</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;Kubernetes running for command </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">command</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;Kubernetes launching image </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">kube_config</span><span class="o">.</span><span class="n">kube_image</span><span class="p">)</span>
+        <span class="n">pod</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">worker_configuration</span><span class="o">.</span><span class="n">make_pod</span><span class="p">(</span>
+            <span class="n">namespace</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">namespace</span><span class="p">,</span> <span class="n">worker_uuid</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">worker_uuid</span><span class="p">,</span>
+            <span class="n">pod_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_create_pod_id</span><span class="p">(</span><span class="n">dag_id</span><span class="p">,</span> <span class="n">task_id</span><span class="p">),</span>
+            <span class="n">dag_id</span><span class="o">=</span><span class="n">dag_id</span><span class="p">,</span> <span class="n">task_id</span><span class="o">=</span><span class="n">task_id</span><span class="p">,</span> <span class="n">try_number</span><span class="o">=</span><span class="n">try_number</span><span class="p">,</span>
+            <span class="n">execution_date</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_datetime_to_label_safe_datestring</span><span class="p">(</span><span class="n">execution_date</span><span class="p">),</span>
+            <span class="n">airflow_command</span><span class="o">=</span><span class="n">command</span><span class="p">,</span> <span class="n">kube_executor_config</span><span class="o">=</span><span class="n">kube_executor_config</span>
+        <span class="p">)</span>
+        <span class="c1"># the watcher will monitor pods, so we do not block.</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">launcher</span><span class="o">.</span><span class="n">run_pod_async</span><span class="p">(</span><span class="n">pod</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;Kubernetes Job created!&quot;</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">delete_pod</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">pod_id</span><span class="p">):</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">kube_config</span><span class="o">.</span><span class="n">delete_worker_pods</span><span class="p">:</span>
+            <span class="k">try</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">kube_client</span><span class="o">.</span><span class="n">delete_namespaced_pod</span><span class="p">(</span>
+                    <span class="n">pod_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">namespace</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="n">client</span><span class="o">.</span><span class="n">V1DeleteOptions</span><span class="p">())</span>
+            <span class="k">except</span> <span class="n">ApiException</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+                <span class="c1"># If the pod is already deleted</span>
+                <span class="k">if</span> <span class="n">e</span><span class="o">.</span><span class="n">status</span> <span class="o">!=</span> <span class="mi">404</span><span class="p">:</span>
+                    <span class="k">raise</span>
+
+    <span class="k">def</span> <span class="nf">sync</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        The sync function checks the status of all currently running kubernetes jobs.</span>
+<span class="sd">        If a job is completed, it&#39;s status is placed in the result queue to</span>
+<span class="sd">        be sent back to the scheduler.</span>
+
+<span class="sd">        :return:</span>
+
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_health_check_kube_watcher</span><span class="p">()</span>
+        <span class="k">while</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">watcher_queue</span><span class="o">.</span><span class="n">empty</span><span class="p">():</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">process_watcher_task</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="nf">process_watcher_task</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="n">pod_id</span><span class="p">,</span> <span class="n">state</span><span class="p">,</span> <span class="n">labels</span><span class="p">,</span> <span class="n">resource_version</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">watcher_queue</span><span class="o">.</span><span class="n">get</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+            <span class="s1">&#39;Attempting to finish pod; pod_id: </span><span class="si">%s</span><span class="s1">; state: </span><span class="si">%s</span><span class="s1">; labels: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span>
+            <span class="n">pod_id</span><span class="p">,</span> <span class="n">state</span><span class="p">,</span> <span class="n">labels</span>
+        <span class="p">)</span>
+        <span class="n">key</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_labels_to_key</span><span class="p">(</span><span class="n">labels</span><span class="o">=</span><span class="n">labels</span><span class="p">)</span>
+        <span class="k">if</span> <span class="n">key</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">&#39;finishing job </span><span class="si">%s</span><span class="s1"> - </span><span class="si">%s</span><span class="s1"> (</span><span class="si">%s</span><span class="s1">)&#39;</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">state</span><span class="p">,< [...]
+            <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="n">key</span><span class="p">,</span> <span class="n">state</span><span class="p">,</span> <span class="n">pod_id</span><span class="p">,</span> <span class="n">resource_version</span><span class="p">))</span>
+
+    <span class="nd">@staticmethod</span>
+    <span class="k">def</span> <span class="nf">_strip_unsafe_kubernetes_special_chars</span><span class="p">(</span><span class="n">string</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Kubernetes only supports lowercase alphanumeric characters and &quot;-&quot; and &quot;.&quot; in</span>
+<span class="sd">        the pod name</span>
+<span class="sd">        However, there are special rules about how &quot;-&quot; and &quot;.&quot; can be used so let&#39;s</span>
+<span class="sd">        only keep</span>
+<span class="sd">        alphanumeric chars  see here for detail:</span>
+<span class="sd">        https://kubernetes.io/docs/concepts/overview/working-with-objects/names/</span>
+
+<span class="sd">        :param string: The requested Pod name</span>
+<span class="sd">        :return: ``str`` Pod name stripped of any unsafe characters</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="s1">&#39;&#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">ch</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="k">for</span> <span class="n">ind</span><span class="p">,</span> <span class="n">ch</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">string</span><span class="p">)</span>  [...]
+
+    <span class="nd">@staticmethod</span>
+    <span class="k">def</span> <span class="nf">_make_safe_pod_id</span><span class="p">(</span><span class="n">safe_dag_id</span><span class="p">,</span> <span class="n">safe_task_id</span><span class="p">,</span> <span class="n">safe_uuid</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Kubernetes pod names must be &lt;= 253 chars and must pass the following regex for</span>
+<span class="sd">        validation</span>
+<span class="sd">        &quot;^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$&quot;</span>
+
+<span class="sd">        :param safe_dag_id: a dag_id with only alphanumeric characters</span>
+<span class="sd">        :param safe_task_id: a task_id with only alphanumeric characters</span>
+<span class="sd">        :param random_uuid: a uuid</span>
+<span class="sd">        :return: ``str`` valid Pod name of appropriate length</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">MAX_POD_ID_LEN</span> <span class="o">=</span> <span class="mi">253</span>
+
+        <span class="n">safe_key</span> <span class="o">=</span> <span class="n">safe_dag_id</span> <span class="o">+</span> <span class="n">safe_task_id</span>
+
+        <span class="n">safe_pod_id</span> <span class="o">=</span> <span class="n">safe_key</span><span class="p">[:</span><span class="n">MAX_POD_ID_LEN</span> <span class="o">-</span> <span class="nb">len</span><span class="p">(</span><span class="n">safe_uuid</span><span class="p">)</span> <span class="o">-</span> <span class="mi">1</span><span class="p">]</span> <span class="o">+</span> <span class="s2">&quot;-&quot;</span> <span class="o">+</span> <span class="n">safe_uuid</span>
+
+        <span class="k">return</span> <span class="n">safe_pod_id</span>
+
+    <span class="nd">@staticmethod</span>
+    <span class="k">def</span> <span class="nf">_create_pod_id</span><span class="p">(</span><span class="n">dag_id</span><span class="p">,</span> <span class="n">task_id</span><span class="p">):</span>
+        <span class="n">safe_dag_id</span> <span class="o">=</span> <span class="n">AirflowKubernetesScheduler</span><span class="o">.</span><span class="n">_strip_unsafe_kubernetes_special_chars</span><span class="p">(</span>
+            <span class="n">dag_id</span><span class="p">)</span>
+        <span class="n">safe_task_id</span> <span class="o">=</span> <span class="n">AirflowKubernetesScheduler</span><span class="o">.</span><span class="n">_strip_unsafe_kubernetes_special_chars</span><span class="p">(</span>
+            <span class="n">task_id</span><span class="p">)</span>
+        <span class="n">safe_uuid</span> <span class="o">=</span> <span class="n">AirflowKubernetesScheduler</span><span class="o">.</span><span class="n">_strip_unsafe_kubernetes_special_chars</span><span class="p">(</span>
+            <span class="n">uuid4</span><span class="p">()</span><span class="o">.</span><span class="n">hex</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">AirflowKubernetesScheduler</span><span class="o">.</span><span class="n">_make_safe_pod_id</span><span class="p">(</span><span class="n">safe_dag_id</span><span class="p">,</span> <span class="n">safe_task_id</span><span class="p">,</span>
+                                                            <span class="n">safe_uuid</span><span class="p">)</span>
+
+    <span class="nd">@staticmethod</span>
+    <span class="k">def</span> <span class="nf">_label_safe_datestring_to_datetime</span><span class="p">(</span><span class="n">string</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Kubernetes doesn&#39;t permit &quot;:&quot; in labels. ISO datetime format uses &quot;:&quot; but not</span>
+<span class="sd">        &quot;_&quot;, let&#39;s</span>
+<span class="sd">        replace &quot;:&quot; with &quot;_&quot;</span>
+
+<span class="sd">        :param string: string</span>
+<span class="sd">        :return: datetime.datetime object</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="n">parser</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">string</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;_plus_&#39;</span><span class="p">,</span> <span class="s1">&#39;+&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;_&quot;</span><sp [...]
+
+    <span class="nd">@staticmethod</span>
+    <span class="k">def</span> <span class="nf">_datetime_to_label_safe_datestring</span><span class="p">(</span><span class="n">datetime_obj</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Kubernetes doesn&#39;t like &quot;:&quot; in labels, since ISO datetime format uses &quot;:&quot; but</span>
+<span class="sd">        not &quot;_&quot; let&#39;s</span>
+<span class="sd">        replace &quot;:&quot; with &quot;_&quot;</span>
+<span class="sd">        :param datetime_obj: datetime.datetime object</span>
+<span class="sd">        :return: ISO-like string representing the datetime</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="n">datetime_obj</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;:&quot;</span><span class="p">,</span> <span class="s2">&quot;_&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;+&#39;</span><span class="p">,</span> [...]
+
+    <span class="k">def</span> <span class="nf">_labels_to_key</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">labels</span><span class="p">):</span>
+        <span class="n">try_num</span> <span class="o">=</span> <span class="mi">1</span>
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">try_num</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">labels</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;try_number&#39;</span><span class="p">,</span> <span class="s1">&#39;1&#39;</span><span class="p">))</span>
+        <span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s2">&quot;could not get try_number as an int: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">labels</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;try_number&#39;</span><span class="p">,</span> <span cl [...]
+
+        <span class="k">try</span><span class="p">:</span>
+            <span class="k">return</span> <span class="p">(</span>
+                <span class="n">labels</span><span class="p">[</span><span class="s1">&#39;dag_id&#39;</span><span class="p">],</span> <span class="n">labels</span><span class="p">[</span><span class="s1">&#39;task_id&#39;</span><span class="p">],</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">_label_safe_datestring_to_datetime</span><span class="p">(</span><span class="n">labels</span><span class="p">[</span><span class="s1">&#39;execution_date&#39;</span><span class="p">]),</span>
+                <span class="n">try_num</span><span class="p">,</span>
+            <span class="p">)</span>
+        <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
+                <span class="s1">&#39;Error while converting labels to key; labels: </span><span class="si">%s</span><span class="s1">; exception: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span>
+                <span class="n">labels</span><span class="p">,</span> <span class="n">e</span>
+            <span class="p">)</span>
+            <span class="k">return</span> <span class="kc">None</span>
+
+
+<div class="viewcode-block" id="KubernetesExecutor"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.executors.kubernetes_executor.KubernetesExecutor">[docs]</a><span class="k">class</span> <span class="nc">KubernetesExecutor</span><span class="p">(</span><span class="n">BaseExecutor</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span>
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">kube_config</span> <span class="o">=</span> <span class="n">KubeConfig</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span> <span class="o">=</span> <span class="kc">None</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span> <span class="o">=</span> <span class="kc">None</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">kube_scheduler</span> <span class="o">=</span> <span class="kc">None</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">kube_client</span> <span class="o">=</span> <span class="kc">None</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">worker_uuid</span> <span class="o">=</span> <span class="kc">None</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">KubernetesExecutor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">parallelism</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">kube_config</span><span class="o">.</span><span class="n">parallelism</span><span class="p">)</span>
+
+<div class="viewcode-block" id="KubernetesExecutor.clear_not_launched_queued_tasks"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.executors.kubernetes_executor.KubernetesExecutor.clear_not_launched_queued_tasks">[docs]</a>    <span class="nd">@provide_session</span>
+    <span class="k">def</span> <span class="nf">clear_not_launched_queued_tasks</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        If the airflow scheduler restarts with pending &quot;Queued&quot; tasks, the tasks may or</span>
+<span class="sd">        may not</span>
+<span class="sd">        have been launched Thus, on starting up the scheduler let&#39;s check every</span>
+<span class="sd">        &quot;Queued&quot; task to</span>
+<span class="sd">        see if it has been launched (ie: if there is a corresponding pod on kubernetes)</span>
+
+<span class="sd">        If it has been launched then do nothing, otherwise reset the state to &quot;None&quot; so</span>
+<span class="sd">        the task</span>
+<span class="sd">        will be rescheduled</span>
+
+<span class="sd">        This will not be necessary in a future version of airflow in which there is</span>
+<span class="sd">        proper support</span>
+<span class="sd">        for State.LAUNCHED</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">queued_tasks</span> <span class="o">=</span> <span class="n">session</span>\
+            <span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">TaskInstance</span><span class="p">)</span>\
+            <span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">TaskInstance</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">State</span><span class="o">.</span><span class="n">QUEUED</span><span class="p">)</span><span class="o">.</span><span class="n">all</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+            <span class="s1">&#39;When executor started up, found </span><span class="si">%s</span><span class="s1"> queued task instances&#39;</span><span class="p">,</span>
+            <span class="nb">len</span><span class="p">(</span><span class="n">queued_tasks</span><span class="p">)</span>
+        <span class="p">)</span>
+
+        <span class="k">for</span> <span class="n">task</span> <span class="ow">in</span> <span class="n">queued_tasks</span><span class="p">:</span>
+            <span class="n">dict_string</span> <span class="o">=</span> <span class="s2">&quot;dag_id=</span><span class="si">{}</span><span class="s2">,task_id=</span><span class="si">{}</span><span class="s2">,execution_date=</span><span class="si">{}</span><span class="s2">,airflow-worker=</span><span class="si">{}</span><span class="s2">&quot;</span> \
+                <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">task</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span> <span class="n">task</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span>
+                        <span class="n">AirflowKubernetesScheduler</span><span class="o">.</span><span class="n">_datetime_to_label_safe_datestring</span><span class="p">(</span>
+                            <span class="n">task</span><span class="o">.</span><span class="n">execution_date</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">worker_uuid</span><span class="p">)</span>
+            <span class="n">kwargs</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">label_selector</span><span class="o">=</span><span class="n">dict_string</span><span class="p">)</span>
+            <span class="n">pod_list</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">kube_client</span><span class="o">.</span><span class="n">list_namespaced_pod</span><span class="p">(</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">kube_config</span><span class="o">.</span><span class="n">kube_namespace</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+            <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">pod_list</span><span class="o">.</span><span class="n">items</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+                    <span class="s1">&#39;TaskInstance: </span><span class="si">%s</span><span class="s1"> found in queued state but was not launched, &#39;</span>
+                    <span class="s1">&#39;rescheduling&#39;</span><span class="p">,</span> <span class="n">task</span>
+                <span class="p">)</span>
+                <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">TaskInstance</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span>
+                    <span class="n">TaskInstance</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="n">task</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span>
+                    <span class="n">TaskInstance</span><span class="o">.</span><span class="n">task_id</span> <span class="o">==</span> <span class="n">task</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span>
+                    <span class="n">TaskInstance</span><span class="o">.</span><span class="n">execution_date</span> <span class="o">==</span> <span class="n">task</span><span class="o">.</span><span class="n">execution_date</span>
+                <span class="p">)</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="n">TaskInstance</span><span class="o">.</span><span class="n">state</span><span class="p">:</span> <span class="n">State</span><span class="o">.</span><span class="n">NONE</span><span class="p">})</span></div>
+
+    <span class="k">def</span> <span class="nf">_inject_secrets</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="k">def</span> <span class="nf">_create_or_update_secret</span><span class="p">(</span><span class="n">secret_name</span><span class="p">,</span> <span class="n">secret_path</span><span class="p">):</span>
+            <span class="k">try</span><span class="p">:</span>
+                <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">kube_client</span><span class="o">.</span><span class="n">create_namespaced_secret</span><span class="p">(</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">kube_config</span><span class="o">.</span><span class="n">executor_namespace</span><span class="p">,</span> <span class="n">kubernetes</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">V1Secret</span><span class="p">(</span>
+                        <span class="n">data</span><span class="o">=</span><span class="p">{</span>
+                            <span class="s1">&#39;key.json&#39;</span><span class="p">:</span> <span class="n">base64</span><span class="o">.</span><span class="n">b64encode</span><span class="p">(</span><span class="nb">open</span><span class="p">(</span><span class="n">secret_path</span><span class="p">,</span> <span class="s1">&#39;r&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">read</span><span class="p">())},</span>
+                        <span class="n">metadata</span><span class="o">=</span><span class="n">kubernetes</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">V1ObjectMeta</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="n">secret_name</span><span class="p">)))</span>
+            <span class="k">except</span> <span class="n">ApiException</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+                <span class="k">if</span> <span class="n">e</span><span class="o">.</span><span class="n">status</span> <span class="o">==</span> <span class="mi">409</span><span class="p">:</span>
+                    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">kube_client</span><span class="o">.</span><span class="n">replace_namespaced_secret</span><span class="p">(</span>
+                        <span class="n">secret_name</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">kube_config</span><span class="o">.</span><span class="n">executor_namespace</span><span class="p">,</span>
+                        <span class="n">kubernetes</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">V1Secret</span><span class="p">(</span>
+                            <span class="n">data</span><span class="o">=</span><span class="p">{</span><span class="s1">&#39;key.json&#39;</span><span class="p">:</span> <span class="n">base64</span><span class="o">.</span><span class="n">b64encode</span><span class="p">(</span>
+                                <span class="nb">open</span><span class="p">(</span><span class="n">secret_path</span><span class="p">,</span> <span class="s1">&#39;r&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">read</span><span class="p">())},</span>
+                            <span class="n">metadata</span><span class="o">=</span><span class="n">kubernetes</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">V1ObjectMeta</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="n">secret_name</span><span class="p">)))</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span>
+                    <span class="s1">&#39;Exception while trying to inject secret. &#39;</span>
+                    <span class="s1">&#39;Secret name: </span><span class="si">%s</span><span class="s1">, error details: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span>
+                    <span class="n">secret_name</span><span class="p">,</span> <span class="n">e</span>
+                <span class="p">)</span>
+                <span class="k">raise</span>
+
+        <span class="c1"># For each GCP service account key, inject it as a secret in executor</span>
+        <span class="c1"># namespace with the specific secret name configured in the airflow.cfg.</span>
+        <span class="c1"># We let exceptions to pass through to users.</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">kube_config</span><span class="o">.</span><span class="n">gcp_service_account_keys</span><span class="p">:</span>
+            <span class="n">name_path_pair_list</span> <span class="o">=</span> <span class="p">[</span>
+                <span class="p">{</span><span class="s1">&#39;name&#39;</span><span class="p">:</span> <span class="n">account_spec</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;=&#39;</span><span class="p">)[</span><span class="mi">0</span><span class="p">],</span>
+                 <span class="s1">&#39;path&#39;</span><span class="p">:</span> <span class="n">account_spec</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;=&#39;</span><span class="p">)[</span><span class="mi">1</span><span class="p">]}</span>
+                <span class="k">for</span> <span class="n">account_spec</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">kube_config</span><span class="o">.</span><span class="n">gcp_service_account_keys</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;,&#39;</span><span class="p">)]</span>
+            <span class="k">for</span> <span class="n">service_account</span> <span class="ow">in</span> <span class="n">name_path_pair_list</span><span class="p">:</span>
+                <span class="n">_create_or_update_secret</span><span class="p">(</span><span class="n">service_account</span><span class="p">[</span><span class="s1">&#39;name&#39;</span><span class="p">],</span> <span class="n">service_account</span><span class="p">[</span><span class="s1">&#39;path&#39;</span><span class="p">])</span>
+
+<div class="viewcode-block" id="KubernetesExecutor.start"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.executors.kubernetes_executor.KubernetesExecutor.start">[docs]</a>    <span class="k">def</span> <span class="nf">start</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Start Kubernetes executor&#39;</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">worker_uuid</span> <span class="o">=</span> <span class="n">KubeWorkerIdentifier</span><span class="o">.</span><span class="n">get_or_create_current_kube_worker_uuid</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">&#39;Start with worker_uuid: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">worker_uuid</span><span class="p">)</span>
+        <span class="c1"># always need to reset resource version since we don&#39;t know</span>
+        <span class="c1"># when we last started, note for behavior below</span>
+        <span class="c1"># https://github.com/kubernetes-client/python/blob/master/kubernetes/docs</span>
+        <span class="c1"># /CoreV1Api.md#list_namespaced_pod</span>
+        <span class="n">KubeResourceVersion</span><span class="o">.</span><span class="n">reset_resource_version</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span> <span class="o">=</span> <span class="n">Queue</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span> <span class="o">=</span> <span class="n">Queue</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">kube_client</span> <span class="o">=</span> <span class="n">get_kube_client</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">kube_scheduler</span> <span class="o">=</span> <span class="n">AirflowKubernetesScheduler</span><span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">kube_config</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="p">,</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">kube_client</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">worker_uuid</span>
+        <span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_inject_secrets</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">clear_not_launched_queued_tasks</span><span class="p">()</span></div>
+
+<div class="viewcode-block" id="KubernetesExecutor.execute_async"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.executors.kubernetes_executor.KubernetesExecutor.execute_async">[docs]</a>    <span class="k">def</span> <span class="nf">execute_async</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,</span> <span class="n">queue</span><spa [...]
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+            <span class="s1">&#39;Add task </span><span class="si">%s</span><span class="s1"> with command </span><span class="si">%s</span><span class="s1"> with executor_config </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span>
+            <span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,</span> <span class="n">executor_config</span>
+        <span class="p">)</span>
+        <span class="n">kube_executor_config</span> <span class="o">=</span> <span class="n">KubernetesExecutorConfig</span><span class="o">.</span><span class="n">from_dict</span><span class="p">(</span><span class="n">executor_config</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,</span> <span class="n">kube_executor_config</span><span class="p">))</span></div>
+
+<div class="viewcode-block" id="KubernetesExecutor.sync"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.executors.kubernetes_executor.KubernetesExecutor.sync">[docs]</a>    <span class="k">def</span> <span class="nf">sync</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">running</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">&#39;self.running: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">running</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">queued_tasks</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">&#39;self.queued: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">queued_tasks</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">kube_scheduler</span><span class="o">.</span><span class="n">sync</span><span class="p">()</span>
+
+        <span class="n">last_resource_version</span> <span class="o">=</span> <span class="kc">None</span>
+        <span class="k">while</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="o">.</span><span class="n">empty</span><span class="p">():</span>
+            <span class="n">results</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="o">.</span><span class="n">get</span><span class="p">()</span>
+            <span class="n">key</span><span class="p">,</span> <span class="n">state</span><span class="p">,</span> <span class="n">pod_id</span><span class="p">,</span> <span class="n">resource_version</span> <span class="o">=</span> <span class="n">results</span>
+            <span class="n">last_resource_version</span> <span class="o">=</span> <span class="n">resource_version</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Changing state of </span><span class="si">%s</span><span class="s1"> to </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="n">results</span><span class="p">,</span> <span class="n">state</span><span class="p">)</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">_change_state</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">state</span><span class="p">,</span> <span class="n">pod_id</span><span class="p">)</span>
+
+        <span class="n">KubeResourceVersion</span><span class="o">.</span><span class="n">checkpoint_resource_version</span><span class="p">(</span><span class="n">last_resource_version</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">empty</span><span class="p">():</span>
+            <span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,</span> <span class="n">kube_executor_config</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">get</span><span class="p">()</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">kube_scheduler</span><span class="o">.</span><span class="n">run_next</span><span class="p">((</span><span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,</span> <span class="n">kube_executor_config</span><span class="p">))</span></div>
+
+    <span class="k">def</span> <span class="nf">_change_state</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">state</span><span class="p">,</span> <span class="n">pod_id</span><span class="p">):</span>
+        <span class="k">if</span> <span class="n">state</span> <span class="o">!=</span> <span class="n">State</span><span class="o">.</span><span class="n">RUNNING</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">kube_scheduler</span><span class="o">.</span><span class="n">delete_pod</span><span class="p">(</span><span class="n">pod_id</span><span class="p">)</span>
+            <span class="k">try</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Deleted pod: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">key</span><span class="p">))</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">running</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="n">key</span><span class="p">)</span>
+            <span class="k">except</span> <span class="ne">KeyError</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">&#39;Could not find key: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">key</span><span class="p">))</span>
+                <span class="k">pass</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">event_buffer</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="o">=</span> <span class="n">state</span>
+        <span class="p">(</span><span class="n">dag_id</span><span class="p">,</span> <span class="n">task_id</span><span class="p">,</span> <span class="n">ex_time</span><span class="p">,</span> <span class="n">try_number</span><span class="p">)</span> <span class="o">=</span> <span class="n">key</span>
+        <span class="k">with</span> <span class="n">create_session</span><span class="p">()</span> <span class="k">as</span> <span class="n">session</span><span class="p">:</span>
+            <span class="n">item</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">TaskInstance</span><span class="p">)</span><span class="o">.</span><span class="n">filter_by</span><span class="p">(</span>
+                <span class="n">dag_id</span><span class="o">=</span><span class="n">dag_id</span><span class="p">,</span>
+                <span class="n">task_id</span><span class="o">=</span><span class="n">task_id</span><span class="p">,</span>
+                <span class="n">execution_date</span><span class="o">=</span><span class="n">ex_time</span>
+            <span class="p">)</span><span class="o">.</span><span class="n">one</span><span class="p">()</span>
+            <span class="k">if</span> <span class="n">state</span><span class="p">:</span>
+                <span class="n">item</span><span class="o">.</span><span class="n">state</span> <span class="o">=</span> <span class="n">state</span>
+                <span class="n">session</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
+
+<div class="viewcode-block" id="KubernetesExecutor.end"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.executors.kubernetes_executor.KubernetesExecutor.end">[docs]</a>    <span class="k">def</span> <span class="nf">end</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Shutting down Kubernetes executor&#39;</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">join</span><span class="p">()</span></div></div>
+</pre></div>
+
+           </div>
+           
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    
+    
+      <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+        <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+        <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+        <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+        <script type="text/javascript" src="../../../../_static/language_data.js"></script>
+    
+
+  
+
+  <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.Navigation.enable(true);
+      });
+  </script> 
+
+</body>
+</html>
\ No newline at end of file
diff --git a/1.10.2/_modules/airflow/contrib/executors/mesos_executor.html b/1.10.2/_modules/airflow/contrib/executors/mesos_executor.html
new file mode 100644
index 0000000..ec4460f
--- /dev/null
+++ b/1.10.2/_modules/airflow/contrib/executors/mesos_executor.html
@@ -0,0 +1,549 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>airflow.contrib.executors.mesos_executor &mdash; Airflow Documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+    <link rel="index" title="Index" href="../../../../genindex.html" />
+    <link rel="search" title="Search" href="../../../../search.html" /> 
+
+  
+  <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../../index.html" class="icon icon-home"> Airflow
+          
+
+          
+          </a>
+
+          
+            
+            
+              <div class="version">
+                1.10.2
+              </div>
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../metrics.html">Metrics</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../../index.html">Airflow</a>
+        
+      </nav>
+
+
+      <div class="wy-nav-content">
+        
+        <div class="rst-content">
+        
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../../index.html">Module code</a> &raquo;</li>
+        
+      <li>airflow.contrib.executors.mesos_executor</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for airflow.contrib.executors.mesos_executor</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements.  See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership.  The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># &quot;License&quot;); you may not use this file except in compliance</span>
+<span class="c1"># with the License.  You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1">#   http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># &quot;AS IS&quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied.  See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">future</span> <span class="k">import</span> <span class="n">standard_library</span>
+
+<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
+<span class="kn">from</span> <span class="nn">airflow.www.utils</span> <span class="k">import</span> <span class="n">LoginMixin</span>
+
+
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">str</span>
+<span class="kn">from</span> <span class="nn">queue</span> <span class="k">import</span> <span class="n">Queue</span>
+
+<span class="kn">import</span> <span class="nn">mesos.interface</span>
+<span class="kn">from</span> <span class="nn">mesos.interface</span> <span class="k">import</span> <span class="n">mesos_pb2</span>
+<span class="kn">import</span> <span class="nn">mesos.native</span>
+
+<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">configuration</span>
+<span class="kn">from</span> <span class="nn">airflow.executors.base_executor</span> <span class="k">import</span> <span class="n">BaseExecutor</span>
+<span class="kn">from</span> <span class="nn">airflow.settings</span> <span class="k">import</span> <span class="n">Session</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.state</span> <span class="k">import</span> <span class="n">State</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+
+<span class="n">standard_library</span><span class="o">.</span><span class="n">install_aliases</span><span class="p">()</span>
+<span class="n">DEFAULT_FRAMEWORK_NAME</span> <span class="o">=</span> <span class="s1">&#39;Airflow&#39;</span>
+<span class="n">FRAMEWORK_CONNID_PREFIX</span> <span class="o">=</span> <span class="s1">&#39;mesos_framework_&#39;</span>
+
+
+<span class="k">def</span> <span class="nf">get_framework_name</span><span class="p">():</span>
+    <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span class="s1">&#39;FRAMEWORK_NAME&#39;</span><span class="p">):</span>
+        <span class="k">return</span> <span class="n">DEFAULT_FRAMEWORK_NAME</span>
+    <span class="k">return</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span class="s1">&#39;FRAMEWORK_NAME&#39;</span><span class="p">)</span>
+
+
+<span class="c1"># AirflowMesosScheduler, implements Mesos Scheduler interface</span>
+<span class="c1"># To schedule airflow jobs on mesos</span>
+<span class="k">class</span> <span class="nc">AirflowMesosScheduler</span><span class="p">(</span><span class="n">mesos</span><span class="o">.</span><span class="n">interface</span><span class="o">.</span><span class="n">Scheduler</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Airflow Mesos scheduler implements mesos scheduler interface</span>
+<span class="sd">    to schedule airflow tasks on mesos.</span>
+<span class="sd">    Basically, it schedules a command like</span>
+<span class="sd">    &#39;airflow run &lt;dag_id&gt; &lt;task_instance_id&gt; &lt;start_date&gt; --local -p=&lt;pickle&gt;&#39;</span>
+<span class="sd">    to run on a mesos slave.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+                 <span class="n">task_queue</span><span class="p">,</span>
+                 <span class="n">result_queue</span><span class="p">,</span>
+                 <span class="n">task_cpu</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
+                 <span class="n">task_mem</span><span class="o">=</span><span class="mi">256</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span> <span class="o">=</span> <span class="n">task_queue</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span> <span class="o">=</span> <span class="n">result_queue</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">task_cpu</span> <span class="o">=</span> <span class="n">task_cpu</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">task_mem</span> <span class="o">=</span> <span class="n">task_mem</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">task_counter</span> <span class="o">=</span> <span class="mi">0</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">task_key_map</span> <span class="o">=</span> <span class="p">{}</span>
+        <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span class="s1">&#39;DOCKER_IMAGE_SLAVE&#39;</span><span class="p">):</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">mesos_slave_docker_image</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+                <span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span class="s1">&#39;DOCKER_IMAGE_SLAVE&#39;</span>
+            <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">registered</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">frameworkId</span><span class="p">,</span> <span class="n">masterInfo</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;AirflowScheduler registered to Mesos with framework ID </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span>
+                      <span class="n">frameworkId</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span class="s1">&#39;CHECKPOINT&#39;</span><span class="p">)</span> <span class="ow">and</span> \
+                <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span class="s1">&#39;FAILOVER_TIMEOUT&#39;</span><span class="p">):</span>
+            <span class="c1"># Import here to work around a circular import error</span>
+            <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">Connection</span>
+
+            <span class="c1"># Update the Framework ID in the database.</span>
+            <span class="n">session</span> <span class="o">=</span> <span class="n">Session</span><span class="p">()</span>
+            <span class="n">conn_id</span> <span class="o">=</span> <span class="n">FRAMEWORK_CONNID_PREFIX</span> <span class="o">+</span> <span class="n">get_framework_name</span><span class="p">()</span>
+            <span class="n">connection</span> <span class="o">=</span> <span class="n">Session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">Connection</span><span class="p">)</span><span class="o">.</span><span class="n">filter_by</span><span class="p">(</span><span class="n">conn_id</span><span class="o">=</span><span class="n">conn_id</span><span class="p">)</span><span class="o">.</span><span class="n">first</span><span class="p">()</span>
+            <span class="k">if</span> <span class="n">connection</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+                <span class="n">connection</span> <span class="o">=</span> <span class="n">Connection</span><span class="p">(</span><span class="n">conn_id</span><span class="o">=</span><span class="n">conn_id</span><span class="p">,</span> <span class="n">conn_type</span><span class="o">=</span><span class="s1">&#39;mesos_framework-id&#39;</span><span class="p">,</span>
+                                        <span class="n">extra</span><span class="o">=</span><span class="n">frameworkId</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">connection</span><span class="o">.</span><span class="n">extra</span> <span class="o">=</span> <span class="n">frameworkId</span><span class="o">.</span><span class="n">value</span>
+
+            <span class="n">session</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">connection</span><span class="p">)</span>
+            <span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span>
+            <span class="n">Session</span><span class="o">.</span><span class="n">remove</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="nf">reregistered</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">masterInfo</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;AirflowScheduler re-registered to mesos&quot;</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">disconnected</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;AirflowScheduler disconnected from mesos&quot;</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">offerRescinded</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">offerId</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;AirflowScheduler offer </span><span class="si">%s</span><span class="s2"> rescinded&quot;</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">offerId</span><span class="p">))</span>
+
+    <span class="k">def</span> <span class="nf">frameworkMessage</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">executorId</span><span class="p">,</span> <span class="n">slaveId</span><span class="p">,</span> <span class="n">message</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;AirflowScheduler received framework message </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">message</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">executorLost</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">executorId</span><span class="p">,</span> <span class="n">slaveId</span><span class="p">,</span> <span class="n">status</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">&quot;AirflowScheduler executor </span><span class="si">%s</span><span class="s2"> lost&quot;</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">executorId</span><span class="p">))</span>
+
+    <span class="k">def</span> <span class="nf">slaveLost</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">slaveId</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">&quot;AirflowScheduler slave </span><span class="si">%s</span><span class="s2"> lost&quot;</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">slaveId</span><span class="p">))</span>
+
+    <span class="k">def</span> <span class="nf">error</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">message</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;AirflowScheduler driver aborted </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">message</span><span class="p">)</span>
+        <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">&quot;AirflowScheduler driver aborted </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="n">message</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">resourceOffers</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">offers</span><span class="p">):</span>
+        <span class="k">for</span> <span class="n">offer</span> <span class="ow">in</span> <span class="n">offers</span><span class="p">:</span>
+            <span class="n">tasks</span> <span class="o">=</span> <span class="p">[]</span>
+            <span class="n">offerCpus</span> <span class="o">=</span> <span class="mi">0</span>
+            <span class="n">offerMem</span> <span class="o">=</span> <span class="mi">0</span>
+            <span class="k">for</span> <span class="n">resource</span> <span class="ow">in</span> <span class="n">offer</span><span class="o">.</span><span class="n">resources</span><span class="p">:</span>
+                <span class="k">if</span> <span class="n">resource</span><span class="o">.</span><span class="n">name</span> <span class="o">==</span> <span class="s2">&quot;cpus&quot;</span><span class="p">:</span>
+                    <span class="n">offerCpus</span> <span class="o">+=</span> <span class="n">resource</span><span class="o">.</span><span class="n">scalar</span><span class="o">.</span><span class="n">value</span>
+                <span class="k">elif</span> <span class="n">resource</span><span class="o">.</span><span class="n">name</span> <span class="o">==</span> <span class="s2">&quot;mem&quot;</span><span class="p">:</span>
+                    <span class="n">offerMem</span> <span class="o">+=</span> <span class="n">resource</span><span class="o">.</span><span class="n">scalar</span><span class="o">.</span><span class="n">value</span>
+
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Received offer </span><span class="si">%s</span><span class="s2"> with cpus: </span><span class="si">%s</span><span class="s2"> and mem: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span>
+                          <span class="n">offer</span><span class="o">.</span><span class="n">id</span><span class="o">.</span><span class="n">value</span><span class="p">,</span> <span class="n">offerCpus</span><span class="p">,</span> <span class="n">offerMem</span><span class="p">)</span>
+
+            <span class="n">remainingCpus</span> <span class="o">=</span> <span class="n">offerCpus</span>
+            <span class="n">remainingMem</span> <span class="o">=</span> <span class="n">offerMem</span>
+
+            <span class="k">while</span> <span class="p">(</span><span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">empty</span><span class="p">())</span> <span class="ow">and</span> \
+                    <span class="n">remainingCpus</span> <span class="o">&gt;=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_cpu</span> <span class="ow">and</span> \
+                    <span class="n">remainingMem</span> <span class="o">&gt;=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_mem</span><span class="p">:</span>
+                <span class="n">key</span><span class="p">,</span> <span class="n">cmd</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">get</span><span class="p">()</span>
+                <span class="n">tid</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_counter</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">task_counter</span> <span class="o">+=</span> <span class="mi">1</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">task_key_map</span><span class="p">[</span><span class="nb">str</span><span class="p">(</span><span class="n">tid</span><span class="p">)]</span> <span class="o">=</span> <span class="n">key</span>
+
+                <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Launching task </span><span class="si">%d</span><span class="s2"> using offer </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">tid</span><span class="p">,</span> <span class="n">offer</span><span class="o">.</span><span class="n">id</span><span  [...]
+
+                <span class="n">task</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">TaskInfo</span><span class="p">()</span>
+                <span class="n">task</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">tid</span><span class="p">)</span>
+                <span class="n">task</span><span class="o">.</span><span class="n">slave_id</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="n">offer</span><span class="o">.</span><span class="n">slave_id</span><span class="o">.</span><span class="n">value</span>
+                <span class="n">task</span><span class="o">.</span><span class="n">name</span> <span class="o">=</span> <span class="s2">&quot;AirflowTask </span><span class="si">%d</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="n">tid</span>
+
+                <span class="n">cpus</span> <span class="o">=</span> <span class="n">task</span><span class="o">.</span><span class="n">resources</span><span class="o">.</span><span class="n">add</span><span class="p">()</span>
+                <span class="n">cpus</span><span class="o">.</span><span class="n">name</span> <span class="o">=</span> <span class="s2">&quot;cpus&quot;</span>
+                <span class="n">cpus</span><span class="o">.</span><span class="n">type</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">Value</span><span class="o">.</span><span class="n">SCALAR</span>
+                <span class="n">cpus</span><span class="o">.</span><span class="n">scalar</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_cpu</span>
+
+                <span class="n">mem</span> <span class="o">=</span> <span class="n">task</span><span class="o">.</span><span class="n">resources</span><span class="o">.</span><span class="n">add</span><span class="p">()</span>
+                <span class="n">mem</span><span class="o">.</span><span class="n">name</span> <span class="o">=</span> <span class="s2">&quot;mem&quot;</span>
+                <span class="n">mem</span><span class="o">.</span><span class="n">type</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">Value</span><span class="o">.</span><span class="n">SCALAR</span>
+                <span class="n">mem</span><span class="o">.</span><span class="n">scalar</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_mem</span>
+
+                <span class="n">command</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">CommandInfo</span><span class="p">()</span>
+                <span class="n">command</span><span class="o">.</span><span class="n">shell</span> <span class="o">=</span> <span class="kc">True</span>
+                <span class="n">command</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="n">cmd</span>
+                <span class="n">task</span><span class="o">.</span><span class="n">command</span><span class="o">.</span><span class="n">MergeFrom</span><span class="p">(</span><span class="n">command</span><span class="p">)</span>
+
+                <span class="c1"># If docker image for airflow is specified in config then pull that</span>
+                <span class="c1"># image before running the above airflow command</span>
+                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mesos_slave_docker_image</span><span class="p">:</span>
+                    <span class="n">network</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">ContainerInfo</span><span class="o">.</span><span class="n">DockerInfo</span><span class="o">.</span><span class="n">Network</span><span class="o">.</span><span class="n">Value</span><span class="p">(</span><span class="s1">&#39;BRIDGE&#39;</span><span class="p">)</span>
+                    <span class="n">docker</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">ContainerInfo</span><span class="o">.</span><span class="n">DockerInfo</span><span class="p">(</span>
+                        <span class="n">image</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">mesos_slave_docker_image</span><span class="p">,</span>
+                        <span class="n">force_pull_image</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+                        <span class="n">network</span><span class="o">=</span><span class="n">network</span>
+                    <span class="p">)</span>
+                    <span class="n">container</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">ContainerInfo</span><span class="p">(</span>
+                        <span class="nb">type</span><span class="o">=</span><span class="n">mesos_pb2</span><span class="o">.</span><span class="n">ContainerInfo</span><span class="o">.</span><span class="n">DOCKER</span><span class="p">,</span>
+                        <span class="n">docker</span><span class="o">=</span><span class="n">docker</span>
+                    <span class="p">)</span>
+                    <span class="n">task</span><span class="o">.</span><span class="n">container</span><span class="o">.</span><span class="n">MergeFrom</span><span class="p">(</span><span class="n">container</span><span class="p">)</span>
+
+                <span class="n">tasks</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">task</span><span class="p">)</span>
+
+                <span class="n">remainingCpus</span> <span class="o">-=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_cpu</span>
+                <span class="n">remainingMem</span> <span class="o">-=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_mem</span>
+
+            <span class="n">driver</span><span class="o">.</span><span class="n">launchTasks</span><span class="p">(</span><span class="n">offer</span><span class="o">.</span><span class="n">id</span><span class="p">,</span> <span class="n">tasks</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">statusUpdate</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">update</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+            <span class="s2">&quot;Task </span><span class="si">%s</span><span class="s2"> is in state </span><span class="si">%s</span><span class="s2">, data </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span>
+            <span class="n">update</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">value</span><span class="p">,</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">TaskState</span><span class="o">.</span><span class="n">Name</span><span class="p">(</span><span class="n">update</span><span class="o">.</span><span class="n">state</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span [...]
+        <span class="p">)</span>
+
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">key</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_key_map</span><span class="p">[</span><span class="n">update</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">value</span><span class="p">]</span>
+        <span class="k">except</span> <span class="ne">KeyError</span><span class="p">:</span>
+            <span class="c1"># The map may not contain an item if the framework re-registered</span>
+            <span class="c1"># after a failover.</span>
+            <span class="c1"># Discard these tasks.</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">&quot;Unrecognised task key </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">update</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
+            <span class="k">return</span>
+
+        <span class="k">if</span> <span class="n">update</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">TASK_FINISHED</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="n">key</span><span class="p">,</span> <span class="n">State</span><span class="o">.</span><span class="n">SUCCESS</span><span class="p">))</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">task_done</span><span class="p">()</span>
+
+        <span class="k">if</span> <span class="n">update</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">TASK_LOST</span> <span class="ow">or</span> \
+           <span class="n">update</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">TASK_KILLED</span> <span class="ow">or</span> \
+           <span class="n">update</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">TASK_FAILED</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="n">key</span><span class="p">,</span> <span class="n">State</span><span class="o">.</span><span class="n">FAILED</span><span class="p">))</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">task_done</span><span class="p">()</span>
+
+
+<div class="viewcode-block" id="MesosExecutor"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.executors.mesos_executor.MesosExecutor">[docs]</a><span class="k">class</span> <span class="nc">MesosExecutor</span><span class="p">(</span><span class="n">BaseExecutor</span><span class="p">,</span> <span class="n">LoginMixin</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    MesosExecutor allows distributing the execution of task</span>
+<span class="sd">    instances to multiple mesos workers.</span>
+
+<span class="sd">    Apache Mesos is a distributed systems kernel which abstracts</span>
+<span class="sd">    CPU, memory, storage, and other compute resources away from</span>
+<span class="sd">    machines (physical or virtual), enabling fault-tolerant and</span>
+<span class="sd">    elastic distributed systems to easily be built and run effectively.</span>
+<span class="sd">    See http://mesos.apache.org/</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+<div class="viewcode-block" id="MesosExecutor.start"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.executors.mesos_executor.MesosExecutor.start">[docs]</a>    <span class="k">def</span> <span class="nf">start</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span> <span class="o">=</span> <span class="n">Queue</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span> <span class="o">=</span> <span class="n">Queue</span><span class="p">()</span>
+        <span class="n">framework</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">FrameworkInfo</span><span class="p">()</span>
+        <span class="n">framework</span><span class="o">.</span><span class="n">user</span> <span class="o">=</span> <span class="s1">&#39;&#39;</span>
+
+        <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span class="s1">&#39;MASTER&#39;</span><span class="p">):</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;Expecting mesos master URL for mesos executor&quot;</span><span class="p">)</span>
+            <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">&quot;mesos.master not provided for mesos executor&quot;</span><span class="p">)</span>
+
+        <span class="n">master</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span class="s1">&#39;MASTER&#39;</span><span class="p">)</span>
+
+        <span class="n">framework</span><span class="o">.</span><span class="n">name</span> <span class="o">=</span> <span class="n">get_framework_name</span><span class="p">()</span>
+
+        <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span class="s1">&#39;TASK_CPU&#39;</span><span class="p">):</span>
+            <span class="n">task_cpu</span> <span class="o">=</span> <span class="mi">1</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">task_cpu</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span class="s1">&#39;TASK_CPU&#39;</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span class="s1">&#39;TASK_MEMORY&#39;</span><span class="p">):</span>
+            <span class="n">task_memory</span> <span class="o">=</span> <span class="mi">256</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">task_memory</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span class="s1">&#39;TASK_MEMORY&#39;</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span class="s1">&#39;CHECKPOINT&#39;</span><span class="p">):</span>
+            <span class="n">framework</span><span class="o">.</span><span class="n">checkpoint</span> <span class="o">=</span> <span class="kc">True</span>
+
+            <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span class="s1">&#39;FAILOVER_TIMEOUT&#39;</span><span class="p">):</span>
+                <span class="c1"># Import here to work around a circular import error</span>
+                <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">Connection</span>
+
+                <span class="c1"># Query the database to get the ID of the Mesos Framework, if available.</span>
+                <span class="n">conn_id</span> <span class="o">=</span> <span class="n">FRAMEWORK_CONNID_PREFIX</span> <span class="o">+</span> <span class="n">framework</span><span class="o">.</span><span class="n">name</span>
+                <span class="n">session</span> <span class="o">=</span> <span class="n">Session</span><span class="p">()</span>
+                <span class="n">connection</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">Connection</span><span class="p">)</span><span class="o">.</span><span class="n">filter_by</span><span class="p">(</span><span class="n">conn_id</span><span class="o">=</span><span class="n">conn_id</span><span class="p">)</span><span class="o">.</span><span class="n">first</span><span class="p [...]
+                <span class="k">if</span> <span class="n">connection</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+                    <span class="c1"># Set the Framework ID to let the scheduler reconnect</span>
+                    <span class="c1"># with running tasks.</span>
+                    <span class="n">framework</span><span class="o">.</span><span class="n">id</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="n">connection</span><span class="o">.</span><span class="n">extra</span>
+
+                <span class="n">framework</span><span class="o">.</span><span class="n">failover_timeout</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span>
+                    <span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span class="s1">&#39;FAILOVER_TIMEOUT&#39;</span>
+                <span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">framework</span><span class="o">.</span><span class="n">checkpoint</span> <span class="o">=</span> <span class="kc">False</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+            <span class="s1">&#39;MesosFramework master : </span><span class="si">%s</span><span class="s1">, name : </span><span class="si">%s</span><span class="s1">, cpu : </span><span class="si">%s</span><span class="s1">, mem : </span><span class="si">%s</span><span class="s1">, checkpoint : </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span>
+            <span class="n">master</span><span class="p">,</span> <span class="n">framework</span><span class="o">.</span><span class="n">name</span><span class="p">,</span>
+            <span class="nb">str</span><span class="p">(</span><span class="n">task_cpu</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="n">task_memory</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="n">framework</span><span class="o">.</span><span class="n">checkpoint</span><span class="p">)</span>
+        <span class="p">)</span>
+
+        <span class="n">implicit_acknowledgements</span> <span class="o">=</span> <span class="mi">1</span>
+
+        <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span class="s1">&#39;AUTHENTICATE&#39;</span><span class="p">):</span>
+            <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span class="s1">&#39;DEFAULT_PRINCIPAL&#39;</span><span class="p">):</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;Expecting authentication principal in the environment&quot;</span><span class="p">)</span>
+                <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+                    <span class="s2">&quot;mesos.default_principal not provided in authenticated mode&quot;</span><span class="p">)</span>
+            <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span class="s1">&#39;DEFAULT_SECRET&#39;</span><span class="p">):</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;Expecting authentication secret in the environment&quot;</span><span class="p">)</span>
+                <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+                    <span class="s2">&quot;mesos.default_secret not provided in authenticated mode&quot;</span><span class="p">)</span>
+
+            <span class="n">credential</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">Credential</span><span class="p">()</span>
+            <span class="n">credential</span><span class="o">.</span><span class="n">principal</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span class="s1">&#39;DEFAULT_PRINCIPAL&#39;</span><span class="p">)</span>
+            <span class="n">credential</span><span class="o">.</span><span class="n">secret</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span class="s1">&#39;DEFAULT_SECRET&#39;</span><span class="p">)</span>
+
+            <span class="n">framework</span><span class="o">.</span><span class="n">principal</span> <span class="o">=</span> <span class="n">credential</span><span class="o">.</span><span class="n">principal</span>
+
+            <span class="n">driver</span> <span class="o">=</span> <span class="n">mesos</span><span class="o">.</span><span class="n">native</span><span class="o">.</span><span class="n">MesosSchedulerDriver</span><span class="p">(</span>
+                <span class="n">AirflowMesosScheduler</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="p">,</span>
+                                      <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="p">,</span>
+                                      <span class="n">task_cpu</span><span class="p">,</span>
+                                      <span class="n">task_memory</span><span class="p">),</span>
+                <span class="n">framework</span><span class="p">,</span>
+                <span class="n">master</span><span class="p">,</span>
+                <span class="n">implicit_acknowledgements</span><span class="p">,</span>
+                <span class="n">credential</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">framework</span><span class="o">.</span><span class="n">principal</span> <span class="o">=</span> <span class="s1">&#39;Airflow&#39;</span>
+            <span class="n">driver</span> <span class="o">=</span> <span class="n">mesos</span><span class="o">.</span><span class="n">native</span><span class="o">.</span><span class="n">MesosSchedulerDriver</span><span class="p">(</span>
+                <span class="n">AirflowMesosScheduler</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="p">,</span>
+                                      <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="p">,</span>
+                                      <span class="n">task_cpu</span><span class="p">,</span>
+                                      <span class="n">task_memory</span><span class="p">),</span>
+                <span class="n">framework</span><span class="p">,</span>
+                <span class="n">master</span><span class="p">,</span>
+                <span class="n">implicit_acknowledgements</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">mesos_driver</span> <span class="o">=</span> <span class="n">driver</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mesos_driver</span><span class="o">.</span><span class="n">start</span><span class="p">()</span></div>
+
+<div class="viewcode-block" id="MesosExecutor.execute_async"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.executors.mesos_executor.MesosExecutor.execute_async">[docs]</a>    <span class="k">def</span> <span class="nf">execute_async</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,</span> <span class="n">queue</span><span class="o">=</ [...]
+        <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">))</span></div>
+
+<div class="viewcode-block" id="MesosExecutor.sync"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.executors.mesos_executor.MesosExecutor.sync">[docs]</a>    <span class="k">def</span> <span class="nf">sync</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="k">while</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="o">.</span><span class="n">empty</span><span class="p">():</span>
+            <span class="n">results</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="o">.</span><span class="n">get</span><span class="p">()</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">change_state</span><span class="p">(</span><span class="o">*</span><span class="n">results</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="MesosExecutor.end"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.executors.mesos_executor.MesosExecutor.end">[docs]</a>    <span class="k">def</span> <span class="nf">end</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">join</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mesos_driver</span><span class="o">.</span><span class="n">stop</span><span class="p">()</span></div></div>
+</pre></div>
+
+           </div>
+           
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    
+    
+      <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+        <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+        <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+        <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+        <script type="text/javascript" src="../../../../_static/language_data.js"></script>
+    
+
+  
+
+  <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.Navigation.enable(true);
+      });
+  </script> 
+
+</body>
+</html>
\ No newline at end of file
diff --git a/1.10.2/_modules/airflow/contrib/hooks/aws_athena_hook.html b/1.10.2/_modules/airflow/contrib/hooks/aws_athena_hook.html
new file mode 100644
index 0000000..a310379
--- /dev/null
+++ b/1.10.2/_modules/airflow/contrib/hooks/aws_athena_hook.html
@@ -0,0 +1,369 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>airflow.contrib.hooks.aws_athena_hook &mdash; Airflow Documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+    <link rel="index" title="Index" href="../../../../genindex.html" />
+    <link rel="search" title="Search" href="../../../../search.html" /> 
+
+  
+  <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../../index.html" class="icon icon-home"> Airflow
+          
+
+          
+          </a>
+
+          
+            
+            
+              <div class="version">
+                1.10.2
+              </div>
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../metrics.html">Metrics</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../../index.html">Airflow</a>
+        
+      </nav>
+
+
+      <div class="wy-nav-content">
+        
+        <div class="rst-content">
+        
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../../index.html">Module code</a> &raquo;</li>
+        
+      <li>airflow.contrib.hooks.aws_athena_hook</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for airflow.contrib.hooks.aws_athena_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements.  See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership.  The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># &quot;License&quot;); you may not use this file except in compliance</span>
+<span class="c1"># with the License.  You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1">#   http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># &quot;AS IS&quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied.  See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">time</span> <span class="k">import</span> <span class="n">sleep</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.aws_hook</span> <span class="k">import</span> <span class="n">AwsHook</span>
+
+
+<div class="viewcode-block" id="AWSAthenaHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_athena_hook.AWSAthenaHook">[docs]</a><span class="k">class</span> <span class="nc">AWSAthenaHook</span><span class="p">(</span><span class="n">AwsHook</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Interact with AWS Athena to run, poll queries and return query results</span>
+
+<span class="sd">    :param aws_conn_id: aws connection to use.</span>
+<span class="sd">    :type aws_conn_id: str</span>
+<span class="sd">    :param sleep_time: Time to wait between two consecutive call to check query status on athena</span>
+<span class="sd">    :type sleep_time: int</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="n">INTERMEDIATE_STATES</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;QUEUED&#39;</span><span class="p">,</span> <span class="s1">&#39;RUNNING&#39;</span><span class="p">,)</span>
+    <span class="n">FAILURE_STATES</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;FAILED&#39;</span><span class="p">,</span> <span class="s1">&#39;CANCELLED&#39;</span><span class="p">,)</span>
+    <span class="n">SUCCESS_STATES</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;SUCCEEDED&#39;</span><span class="p">,)</span>
+
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">aws_conn_id</span><span class="o">=</span><span class="s1">&#39;aws_default&#39;</span><span class="p">,</span> <span class="n">sleep_time</span><span class="o">=</span><span class="mi">30</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n" [...]
+        <span class="nb">super</span><span class="p">(</span><span class="n">AWSAthenaHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">aws_conn_id</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">sleep_time</span> <span class="o">=</span> <span class="n">sleep_time</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="o">=</span> <span class="kc">None</span>
+
+<div class="viewcode-block" id="AWSAthenaHook.get_conn"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_athena_hook.AWSAthenaHook.get_conn">[docs]</a>    <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        check if aws conn exists already or create one and return it</span>
+
+<span class="sd">        :return: boto3 session</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_client_type</span><span class="p">(</span><span class="s1">&#39;athena&#39;</span><span class="p">)</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span></div>
+
+<div class="viewcode-block" id="AWSAthenaHook.run_query"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_athena_hook.AWSAthenaHook.run_query">[docs]</a>    <span class="k">def</span> <span class="nf">run_query</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">query</span><span class="p">,</span> <span class="n">query_context</span><span class="p">,</span> <span class="n">result_configuration</span><span class [...]
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Run Presto query on athena with provided config and return submitted query_execution_id</span>
+
+<span class="sd">        :param query: Presto query to run</span>
+<span class="sd">        :type query: str</span>
+<span class="sd">        :param query_context: Context in which query need to be run</span>
+<span class="sd">        :type query_context: dict</span>
+<span class="sd">        :param result_configuration: Dict with path to store results in and config related to encryption</span>
+<span class="sd">        :type result_configuration: dict</span>
+<span class="sd">        :param client_request_token: Unique token created by user to avoid multiple executions of same query</span>
+<span class="sd">        :type client_request_token: str</span>
+<span class="sd">        :return: str</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span><span class="o">.</span><span class="n">start_query_execution</span><span class="p">(</span><span class="n">QueryString</span><span class="o">=</span><span class="n">query</span><span class="p">,</span>
+                                                   <span class="n">ClientRequestToken</span><span class="o">=</span><span class="n">client_request_token</span><span class="p">,</span>
+                                                   <span class="n">QueryExecutionContext</span><span class="o">=</span><span class="n">query_context</span><span class="p">,</span>
+                                                   <span class="n">ResultConfiguration</span><span class="o">=</span><span class="n">result_configuration</span><span class="p">)</span>
+        <span class="n">query_execution_id</span> <span class="o">=</span> <span class="n">response</span><span class="p">[</span><span class="s1">&#39;QueryExecutionId&#39;</span><span class="p">]</span>
+        <span class="k">return</span> <span class="n">query_execution_id</span></div>
+
+<div class="viewcode-block" id="AWSAthenaHook.check_query_status"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_athena_hook.AWSAthenaHook.check_query_status">[docs]</a>    <span class="k">def</span> <span class="nf">check_query_status</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">query_execution_id</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Fetch the status of submitted athena query. Returns None or one of valid query states.</span>
+
+<span class="sd">        :param query_execution_id: Id of submitted athena query</span>
+<span class="sd">        :type query_execution_id: str</span>
+<span class="sd">        :return: str</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span><span class="o">.</span><span class="n">get_query_execution</span><span class="p">(</span><span class="n">QueryExecutionId</span><span class="o">=</span><span class="n">query_execution_id</span><span class="p">)</span>
+        <span class="n">state</span> <span class="o">=</span> <span class="kc">None</span>
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">state</span> <span class="o">=</span> <span class="n">response</span><span class="p">[</span><span class="s1">&#39;QueryExecution&#39;</span><span class="p">][</span><span class="s1">&#39;Status&#39;</span><span class="p">][</span><span class="s1">&#39;State&#39;</span><span class="p">]</span>
+        <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">ex</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s1">&#39;Exception while getting query state&#39;</span><span class="p">,</span> <span class="n">ex</span><span class="p">)</span>
+        <span class="k">finally</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">state</span></div>
+
+<div class="viewcode-block" id="AWSAthenaHook.get_query_results"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_athena_hook.AWSAthenaHook.get_query_results">[docs]</a>    <span class="k">def</span> <span class="nf">get_query_results</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">query_execution_id</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Fetch submitted athena query results. returns none if query is in intermediate state or</span>
+<span class="sd">        failed/cancelled state else dict of query output</span>
+
+<span class="sd">        :param query_execution_id: Id of submitted athena query</span>
+<span class="sd">        :type query_execution_id: str</span>
+<span class="sd">        :return: dict</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">query_state</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">check_query_status</span><span class="p">(</span><span class="n">query_execution_id</span><span class="p">)</span>
+        <span class="k">if</span> <span class="n">query_state</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s1">&#39;Invalid Query state&#39;</span><span class="p">)</span>
+            <span class="k">return</span> <span class="kc">None</span>
+        <span class="k">elif</span> <span class="n">query_state</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">INTERMEDIATE_STATES</span> <span class="ow">or</span> <span class="n">query_state</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">FAILURE_STATES</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s1">&#39;Query is in </span><span class="si">{state}</span><span class="s1"> state. Cannot fetch results&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">state</span><span class="o">=</span><span class="n">query_state</span><span class="p">))</span>
+            <span class="k">return</span> <span class="kc">None</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span><span class="o">.</span><span class="n">get_query_results</span><span class="p">(</span><span class="n">QueryExecutionId</span><span class="o">=</span><span class="n">query_execution_id</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="AWSAthenaHook.poll_query_status"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_athena_hook.AWSAthenaHook.poll_query_status">[docs]</a>    <span class="k">def</span> <span class="nf">poll_query_status</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">query_execution_id</span><span class="p">,</span> <span class="n">max_tries</span><span class="o">=</span><span class="kc">None< [...]
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Poll the status of submitted athena query until query state reaches final state.</span>
+<span class="sd">        Returns one of the final states</span>
+
+<span class="sd">        :param query_execution_id: Id of submitted athena query</span>
+<span class="sd">        :type query_execution_id: str</span>
+<span class="sd">        :param max_tries: Number of times to poll for query state before function exits</span>
+<span class="sd">        :type max_tries: int</span>
+<span class="sd">        :return: str</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">try_number</span> <span class="o">=</span> <span class="mi">1</span>
+        <span class="n">final_query_state</span> <span class="o">=</span> <span class="kc">None</span>  <span class="c1"># Query state when query reaches final state or max_tries reached</span>
+        <span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
+            <span class="n">query_state</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">check_query_status</span><span class="p">(</span><span class="n">query_execution_id</span><span class="p">)</span>
+            <span class="k">if</span> <span class="n">query_state</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Trial </span><span class="si">{try_number}</span><span class="s1">: Invalid query state. Retrying again&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+                    <span class="n">try_number</span><span class="o">=</span><span class="n">try_number</span><span class="p">))</span>
+            <span class="k">elif</span> <span class="n">query_state</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">INTERMEDIATE_STATES</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Trial </span><span class="si">{try_number}</span><span class="s1">: Query is still in an intermediate state - </span><span class="si">{state}</span><span class="s1">&#39;</span>
+                              <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">try_number</span><span class="o">=</span><span class="n">try_number</span><span class="p">,</span> <span class="n">state</span><span class="o">=</span><span class="n">query_state</span><span class="p">))</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Trial </span><span class="si">{try_number}</span><span class="s1">: Query execution completed. Final state is </span><span class="si">{state}</span><span class="s1">&#39;</span>
+                              <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">try_number</span><span class="o">=</span><span class="n">try_number</span><span class="p">,</span> <span class="n">state</span><span class="o">=</span><span class="n">query_state</span><span class="p">))</span>
+                <span class="n">final_query_state</span> <span class="o">=</span> <span class="n">query_state</span>
+                <span class="k">break</span>
+            <span class="k">if</span> <span class="n">max_tries</span> <span class="ow">and</span> <span class="n">try_number</span> <span class="o">&gt;=</span> <span class="n">max_tries</span><span class="p">:</span>  <span class="c1"># Break loop if max_tries reached</span>
+                <span class="n">final_query_state</span> <span class="o">=</span> <span class="n">query_state</span>
+                <span class="k">break</span>
+            <span class="n">try_number</span> <span class="o">+=</span> <span class="mi">1</span>
+            <span class="n">sleep</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sleep_time</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">final_query_state</span></div>
+
+<div class="viewcode-block" id="AWSAthenaHook.stop_query"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_athena_hook.AWSAthenaHook.stop_query">[docs]</a>    <span class="k">def</span> <span class="nf">stop_query</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">query_execution_id</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Cancel the submitted athena query</span>
+
+<span class="sd">        :param query_execution_id: Id of submitted athena query</span>
+<span class="sd">        :type query_execution_id: str</span>
+<span class="sd">        :return: dict</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span><span class="o">.</span><span class="n">stop_query_execution</span><span class="p">(</span><span class="n">QueryExecutionId</span><span class="o">=</span><span class="n">query_execution_id</span><span class="p">)</span></div></div>
+</pre></div>
+
+           </div>
+           
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    
+    
+      <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+        <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+        <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+        <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+        <script type="text/javascript" src="../../../../_static/language_data.js"></script>
+    
+
+  
+
+  <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.Navigation.enable(true);
+      });
+  </script> 
+
+</body>
+</html>
\ No newline at end of file
diff --git a/1.10.2/_modules/airflow/contrib/hooks/aws_dynamodb_hook.html b/1.10.2/_modules/airflow/contrib/hooks/aws_dynamodb_hook.html
new file mode 100644
index 0000000..3563f0d
--- /dev/null
+++ b/1.10.2/_modules/airflow/contrib/hooks/aws_dynamodb_hook.html
@@ -0,0 +1,288 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>airflow.contrib.hooks.aws_dynamodb_hook &mdash; Airflow Documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+    <link rel="index" title="Index" href="../../../../genindex.html" />
+    <link rel="search" title="Search" href="../../../../search.html" /> 
+
+  
+  <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../../index.html" class="icon icon-home"> Airflow
+          
+
+          
+          </a>
+
+          
+            
+            
+              <div class="version">
+                1.10.2
+              </div>
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../metrics.html">Metrics</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../../index.html">Airflow</a>
+        
+      </nav>
+
+
+      <div class="wy-nav-content">
+        
+        <div class="rst-content">
+        
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../../index.html">Module code</a> &raquo;</li>
+        
+      <li>airflow.contrib.hooks.aws_dynamodb_hook</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for airflow.contrib.hooks.aws_dynamodb_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements.  See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership.  The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># &quot;License&quot;); you may not use this file except in compliance</span>
+<span class="c1"># with the License.  You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1">#   http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># &quot;AS IS&quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied.  See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.aws_hook</span> <span class="k">import</span> <span class="n">AwsHook</span>
+
+
+<div class="viewcode-block" id="AwsDynamoDBHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_dynamodb_hook.AwsDynamoDBHook">[docs]</a><span class="k">class</span> <span class="nc">AwsDynamoDBHook</span><span class="p">(</span><span class="n">AwsHook</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Interact with AWS DynamoDB.</span>
+
+<span class="sd">    :param table_keys: partition key and sort key</span>
+<span class="sd">    :type table_keys: list</span>
+<span class="sd">    :param table_name: target DynamoDB table</span>
+<span class="sd">    :type table_name: str</span>
+<span class="sd">    :param region_name: aws region name (example: us-east-1)</span>
+<span class="sd">    :type region_name: str</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+                 <span class="n">table_keys</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                 <span class="n">table_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                 <span class="n">region_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                 <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">table_keys</span> <span class="o">=</span> <span class="n">table_keys</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">table_name</span> <span class="o">=</span> <span class="n">table_name</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">region_name</span> <span class="o">=</span> <span class="n">region_name</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">AwsDynamoDBHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_resource_type</span><span class="p">(</span><span class="s1">&#39;dynamodb&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">region_name</span><span class="p">)</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span>
+
+<div class="viewcode-block" id="AwsDynamoDBHook.write_batch_data"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_dynamodb_hook.AwsDynamoDBHook.write_batch_data">[docs]</a>    <span class="k">def</span> <span class="nf">write_batch_data</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">items</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Write batch items to dynamodb table with provisioned throughout capacity.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+
+        <span class="n">dynamodb_conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">table</span> <span class="o">=</span> <span class="n">dynamodb_conn</span><span class="o">.</span><span class="n">Table</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">table_name</span><span class="p">)</span>
+
+            <span class="k">with</span> <span class="n">table</span><span class="o">.</span><span class="n">batch_writer</span><span class="p">(</span><span class="n">overwrite_by_pkeys</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table_keys</span><span class="p">)</span> <span class="k">as</span> <span class="n">batch</span><span class="p">:</span>
+                <span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">items</span><span class="p">:</span>
+                    <span class="n">batch</span><span class="o">.</span><span class="n">put_item</span><span class="p">(</span><span class="n">Item</span><span class="o">=</span><span class="n">item</span><span class="p">)</span>
+            <span class="k">return</span> <span class="kc">True</span>
+        <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">general_error</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+                <span class="s1">&#39;Failed to insert items in dynamodb, error: </span><span class="si">{error}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+                    <span class="n">error</span><span class="o">=</span><span class="nb">str</span><span class="p">(</span><span class="n">general_error</span><span class="p">)</span>
+                <span class="p">)</span>
+            <span class="p">)</span></div></div>
+</pre></div>
+
+           </div>
+           
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    
+    
+      <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+        <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+        <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+        <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+        <script type="text/javascript" src="../../../../_static/language_data.js"></script>
+    
+
+  
+
+  <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.Navigation.enable(true);
+      });
+  </script> 
+
+</body>
+</html>
\ No newline at end of file
diff --git a/1.10.2/_modules/airflow/contrib/hooks/aws_firehose_hook.html b/1.10.2/_modules/airflow/contrib/hooks/aws_firehose_hook.html
new file mode 100644
index 0000000..feb126a
--- /dev/null
+++ b/1.10.2/_modules/airflow/contrib/hooks/aws_firehose_hook.html
@@ -0,0 +1,275 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>airflow.contrib.hooks.aws_firehose_hook &mdash; Airflow Documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+    <link rel="index" title="Index" href="../../../../genindex.html" />
+    <link rel="search" title="Search" href="../../../../search.html" /> 
+
+  
+  <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../../index.html" class="icon icon-home"> Airflow
+          
+
+          
+          </a>
+
+          
+            
+            
+              <div class="version">
+                1.10.2
+              </div>
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../metrics.html">Metrics</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../../index.html">Airflow</a>
+        
+      </nav>
+
+
+      <div class="wy-nav-content">
+        
+        <div class="rst-content">
+        
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../../index.html">Module code</a> &raquo;</li>
+        
+      <li>airflow.contrib.hooks.aws_firehose_hook</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for airflow.contrib.hooks.aws_firehose_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements.  See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership.  The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># &quot;License&quot;); you may not use this file except in compliance</span>
+<span class="c1"># with the License.  You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1">#   http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># &quot;AS IS&quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied.  See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.aws_hook</span> <span class="k">import</span> <span class="n">AwsHook</span>
+
+
+<div class="viewcode-block" id="AwsFirehoseHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_firehose_hook.AwsFirehoseHook">[docs]</a><span class="k">class</span> <span class="nc">AwsFirehoseHook</span><span class="p">(</span><span class="n">AwsHook</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Interact with AWS Kinesis Firehose.</span>
+<span class="sd">    :param delivery_stream: Name of the delivery stream</span>
+<span class="sd">    :type delivery_stream: str</span>
+<span class="sd">    :param region_name: AWS region name (example: us-east-1)</span>
+<span class="sd">    :type region_name: str</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">delivery_stream</span><span class="p">,</span> <span class="n">region_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">delivery_stream</span> <span class="o">=</span> <span class="n">delivery_stream</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">region_name</span> <span class="o">=</span> <span class="n">region_name</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">AwsFirehoseHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+<div class="viewcode-block" id="AwsFirehoseHook.get_conn"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_firehose_hook.AwsFirehoseHook.get_conn">[docs]</a>    <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Returns AwsHook connection object.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_client_type</span><span class="p">(</span><span class="s1">&#39;firehose&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">region_name</span><span class="p">)</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span></div>
+
+<div class="viewcode-block" id="AwsFirehoseHook.put_records"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_firehose_hook.AwsFirehoseHook.put_records">[docs]</a>    <span class="k">def</span> <span class="nf">put_records</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">records</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Write batch records to Kinesis Firehose</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+
+        <span class="n">firehose_conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+
+        <span class="n">response</span> <span class="o">=</span> <span class="n">firehose_conn</span><span class="o">.</span><span class="n">put_record_batch</span><span class="p">(</span>
+            <span class="n">DeliveryStreamName</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delivery_stream</span><span class="p">,</span>
+            <span class="n">Records</span><span class="o">=</span><span class="n">records</span>
+        <span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">response</span></div></div>
+</pre></div>
+
+           </div>
+           
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    
+    
+      <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+        <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+        <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+        <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+        <script type="text/javascript" src="../../../../_static/language_data.js"></script>
+    
+
+  
+
+  <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.Navigation.enable(true);
+      });
+  </script> 
+
+</body>
+</html>
\ No newline at end of file
diff --git a/1.10.2/_modules/airflow/contrib/hooks/aws_glue_catalog_hook.html b/1.10.2/_modules/airflow/contrib/hooks/aws_glue_catalog_hook.html
new file mode 100644
index 0000000..e2403e0
--- /dev/null
+++ b/1.10.2/_modules/airflow/contrib/hooks/aws_glue_catalog_hook.html
@@ -0,0 +1,337 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>airflow.contrib.hooks.aws_glue_catalog_hook &mdash; Airflow Documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+    <link rel="index" title="Index" href="../../../../genindex.html" />
+    <link rel="search" title="Search" href="../../../../search.html" /> 
+
+  
+  <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../../index.html" class="icon icon-home"> Airflow
+          
+
+          
+          </a>
+
+          
+            
+            
+              <div class="version">
+                1.10.2
+              </div>
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../metrics.html">Metrics</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../../index.html">Airflow</a>
+        
+      </nav>
+
+
+      <div class="wy-nav-content">
+        
+        <div class="rst-content">
+        
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../../index.html">Module code</a> &raquo;</li>
+        
+      <li>airflow.contrib.hooks.aws_glue_catalog_hook</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for airflow.contrib.hooks.aws_glue_catalog_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements.  See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership.  The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># &quot;License&quot;); you may not use this file except in compliance</span>
+<span class="c1"># with the License.  You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1">#   http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># &quot;AS IS&quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied.  See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.aws_hook</span> <span class="k">import</span> <span class="n">AwsHook</span>
+
+
+<div class="viewcode-block" id="AwsGlueCatalogHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_glue_catalog_hook.AwsGlueCatalogHook">[docs]</a><span class="k">class</span> <span class="nc">AwsGlueCatalogHook</span><span class="p">(</span><span class="n">AwsHook</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Interact with AWS Glue Catalog</span>
+
+<span class="sd">    :param aws_conn_id: ID of the Airflow connection where</span>
+<span class="sd">        credentials and extra configuration are stored</span>
+<span class="sd">    :type aws_conn_id: str</span>
+<span class="sd">    :param region_name: aws region name (example: us-east-1)</span>
+<span class="sd">    :type region_name: str</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+                 <span class="n">aws_conn_id</span><span class="o">=</span><span class="s1">&#39;aws_default&#39;</span><span class="p">,</span>
+                 <span class="n">region_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                 <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+                 <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">region_name</span> <span class="o">=</span> <span class="n">region_name</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">AwsGlueCatalogHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">aws_conn_id</span><span class="o">=</span><span class="n">aws_conn_id</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n" [...]
+
+<div class="viewcode-block" id="AwsGlueCatalogHook.get_conn"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_glue_catalog_hook.AwsGlueCatalogHook.get_conn">[docs]</a>    <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Returns glue connection object.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_client_type</span><span class="p">(</span><span class="s1">&#39;glue&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">region_name</span><span class="p">)</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span></div>
+
+<div class="viewcode-block" id="AwsGlueCatalogHook.get_partitions"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_glue_catalog_hook.AwsGlueCatalogHook.get_partitions">[docs]</a>    <span class="k">def</span> <span class="nf">get_partitions</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+                       <span class="n">database_name</span><span class="p">,</span>
+                       <span class="n">table_name</span><span class="p">,</span>
+                       <span class="n">expression</span><span class="o">=</span><span class="s1">&#39;&#39;</span><span class="p">,</span>
+                       <span class="n">page_size</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                       <span class="n">max_items</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Retrieves the partition values for a table.</span>
+
+<span class="sd">        :param database_name: The name of the catalog database where the partitions reside.</span>
+<span class="sd">        :type database_name: str</span>
+<span class="sd">        :param table_name: The name of the partitions&#39; table.</span>
+<span class="sd">        :type table_name: str</span>
+<span class="sd">        :param expression: An expression filtering the partitions to be returned.</span>
+<span class="sd">            Please see official AWS documentation for further information.</span>
+<span class="sd">            https://docs.aws.amazon.com/glue/latest/dg/aws-glue-api-catalog-partitions.html#aws-glue-api-catalog-partitions-GetPartitions</span>
+<span class="sd">        :type expression: str</span>
+<span class="sd">        :param page_size: pagination size</span>
+<span class="sd">        :type page_size: int</span>
+<span class="sd">        :param max_items: maximum items to return</span>
+<span class="sd">        :type max_items: int</span>
+<span class="sd">        :return: set of partition values where each value is a tuple since</span>
+<span class="sd">            a partition may be composed of multiple columns. For example:</span>
+<span class="sd">        {(&#39;2018-01-01&#39;,&#39;1&#39;), (&#39;2018-01-01&#39;,&#39;2&#39;)}</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">config</span> <span class="o">=</span> <span class="p">{</span>
+            <span class="s1">&#39;PageSize&#39;</span><span class="p">:</span> <span class="n">page_size</span><span class="p">,</span>
+            <span class="s1">&#39;MaxItems&#39;</span><span class="p">:</span> <span class="n">max_items</span><span class="p">,</span>
+        <span class="p">}</span>
+
+        <span class="n">paginator</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">get_paginator</span><span class="p">(</span><span class="s1">&#39;get_partitions&#39;</span><span class="p">)</span>
+        <span class="n">response</span> <span class="o">=</span> <span class="n">paginator</span><span class="o">.</span><span class="n">paginate</span><span class="p">(</span>
+            <span class="n">DatabaseName</span><span class="o">=</span><span class="n">database_name</span><span class="p">,</span>
+            <span class="n">TableName</span><span class="o">=</span><span class="n">table_name</span><span class="p">,</span>
+            <span class="n">Expression</span><span class="o">=</span><span class="n">expression</span><span class="p">,</span>
+            <span class="n">PaginationConfig</span><span class="o">=</span><span class="n">config</span>
+        <span class="p">)</span>
+
+        <span class="n">partitions</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
+        <span class="k">for</span> <span class="n">page</span> <span class="ow">in</span> <span class="n">response</span><span class="p">:</span>
+            <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="n">page</span><span class="p">[</span><span class="s1">&#39;Partitions&#39;</span><span class="p">]:</span>
+                <span class="n">partitions</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="nb">tuple</span><span class="p">(</span><span class="n">p</span><span class="p">[</span><span class="s1">&#39;Values&#39;</span><span class="p">]))</span>
+
+        <span class="k">return</span> <span class="n">partitions</span></div>
+
+<div class="viewcode-block" id="AwsGlueCatalogHook.check_for_partition"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_glue_catalog_hook.AwsGlueCatalogHook.check_for_partition">[docs]</a>    <span class="k">def</span> <span class="nf">check_for_partition</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">database_name</span><span class="p">,</span> <span class="n">table_name</span><span class="p">,</span> <sp [...]
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Checks whether a partition exists</span>
+
+<span class="sd">        :param database_name: Name of hive database (schema) @table belongs to</span>
+<span class="sd">        :type database_name: str</span>
+<span class="sd">        :param table_name: Name of hive table @partition belongs to</span>
+<span class="sd">        :type table_name: str</span>
+<span class="sd">        :expression: Expression that matches the partitions to check for</span>
+<span class="sd">            (eg `a = &#39;b&#39; AND c = &#39;d&#39;`)</span>
+<span class="sd">        :type expression: str</span>
+<span class="sd">        :rtype: bool</span>
+
+<span class="sd">        &gt;&gt;&gt; hook = AwsGlueCatalogHook()</span>
+<span class="sd">        &gt;&gt;&gt; t = &#39;static_babynames_partitioned&#39;</span>
+<span class="sd">        &gt;&gt;&gt; hook.check_for_partition(&#39;airflow&#39;, t, &quot;ds=&#39;2015-01-01&#39;&quot;)</span>
+<span class="sd">        True</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">partitions</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_partitions</span><span class="p">(</span><span class="n">database_name</span><span class="p">,</span> <span class="n">table_name</span><span class="p">,</span> <span class="n">expression</span><span class="p">,</span> <span class="n">max_items</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="n">partitions</span><span class="p">:</span>
+            <span class="k">return</span> <span class="kc">True</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="k">return</span> <span class="kc">False</span></div></div>
+</pre></div>
+
+           </div>
+           
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    
+    
+      <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+        <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+        <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+        <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+        <script type="text/javascript" src="../../../../_static/language_data.js"></script>
+    
+
+  
+
+  <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.Navigation.enable(true);
+      });
+  </script> 
+
+</body>
+</html>
\ No newline at end of file
diff --git a/1.10.2/_modules/airflow/contrib/hooks/aws_hook.html b/1.10.2/_modules/airflow/contrib/hooks/aws_hook.html
new file mode 100644
index 0000000..eb874a2
--- /dev/null
+++ b/1.10.2/_modules/airflow/contrib/hooks/aws_hook.html
@@ -0,0 +1,424 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>airflow.contrib.hooks.aws_hook &mdash; Airflow Documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+    <link rel="index" title="Index" href="../../../../genindex.html" />
+    <link rel="search" title="Search" href="../../../../search.html" /> 
+
+  
+  <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../../index.html" class="icon icon-home"> Airflow
+          
+
+          
+          </a>
+
+          
+            
+            
+              <div class="version">
+                1.10.2
+              </div>
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../metrics.html">Metrics</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../../index.html">Airflow</a>
+        
+      </nav>
+
+
+      <div class="wy-nav-content">
+        
+        <div class="rst-content">
+        
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../../index.html">Module code</a> &raquo;</li>
+        
+      <li>airflow.contrib.hooks.aws_hook</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for airflow.contrib.hooks.aws_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements.  See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership.  The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># &quot;License&quot;); you may not use this file except in compliance</span>
+<span class="c1"># with the License.  You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1">#   http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># &quot;AS IS&quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied.  See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">import</span> <span class="nn">boto3</span>
+<span class="kn">import</span> <span class="nn">configparser</span>
+<span class="kn">import</span> <span class="nn">logging</span>
+
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+
+
+<span class="k">def</span> <span class="nf">_parse_s3_config</span><span class="p">(</span><span class="n">config_file_name</span><span class="p">,</span> <span class="n">config_format</span><span class="o">=</span><span class="s1">&#39;boto&#39;</span><span class="p">,</span> <span class="n">profile</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Parses a config file for s3 credentials. Can currently</span>
+<span class="sd">    parse boto, s3cmd.conf and AWS SDK config formats</span>
+
+<span class="sd">    :param config_file_name: path to the config file</span>
+<span class="sd">    :type config_file_name: str</span>
+<span class="sd">    :param config_format: config type. One of &quot;boto&quot;, &quot;s3cmd&quot; or &quot;aws&quot;.</span>
+<span class="sd">        Defaults to &quot;boto&quot;</span>
+<span class="sd">    :type config_format: str</span>
+<span class="sd">    :param profile: profile name in AWS type config file</span>
+<span class="sd">    :type profile: str</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">config</span> <span class="o">=</span> <span class="n">configparser</span><span class="o">.</span><span class="n">ConfigParser</span><span class="p">()</span>
+    <span class="k">if</span> <span class="n">config</span><span class="o">.</span><span class="n">read</span><span class="p">(</span><span class="n">config_file_name</span><span class="p">):</span>  <span class="c1"># pragma: no cover</span>
+        <span class="n">sections</span> <span class="o">=</span> <span class="n">config</span><span class="o">.</span><span class="n">sections</span><span class="p">()</span>
+    <span class="k">else</span><span class="p">:</span>
+        <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">&quot;Couldn&#39;t read </span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">config_file_name</span><span class="p">))</span>
+    <span class="c1"># Setting option names depending on file format</span>
+    <span class="k">if</span> <span class="n">config_format</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="n">config_format</span> <span class="o">=</span> <span class="s1">&#39;boto&#39;</span>
+    <span class="n">conf_format</span> <span class="o">=</span> <span class="n">config_format</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
+    <span class="k">if</span> <span class="n">conf_format</span> <span class="o">==</span> <span class="s1">&#39;boto&#39;</span><span class="p">:</span>  <span class="c1"># pragma: no cover</span>
+        <span class="k">if</span> <span class="n">profile</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="s1">&#39;profile &#39;</span> <span class="o">+</span> <span class="n">profile</span> <span class="ow">in</span> <span class="n">sections</span><span class="p">:</span>
+            <span class="n">cred_section</span> <span class="o">=</span> <span class="s1">&#39;profile &#39;</span> <span class="o">+</span> <span class="n">profile</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">cred_section</span> <span class="o">=</span> <span class="s1">&#39;Credentials&#39;</span>
+    <span class="k">elif</span> <span class="n">conf_format</span> <span class="o">==</span> <span class="s1">&#39;aws&#39;</span> <span class="ow">and</span> <span class="n">profile</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="n">cred_section</span> <span class="o">=</span> <span class="n">profile</span>
+    <span class="k">else</span><span class="p">:</span>
+        <span class="n">cred_section</span> <span class="o">=</span> <span class="s1">&#39;default&#39;</span>
+    <span class="c1"># Option names</span>
+    <span class="k">if</span> <span class="n">conf_format</span> <span class="ow">in</span> <span class="p">(</span><span class="s1">&#39;boto&#39;</span><span class="p">,</span> <span class="s1">&#39;aws&#39;</span><span class="p">):</span>  <span class="c1"># pragma: no cover</span>
+        <span class="n">key_id_option</span> <span class="o">=</span> <span class="s1">&#39;aws_access_key_id&#39;</span>
+        <span class="n">secret_key_option</span> <span class="o">=</span> <span class="s1">&#39;aws_secret_access_key&#39;</span>
+        <span class="c1"># security_token_option = &#39;aws_security_token&#39;</span>
+    <span class="k">else</span><span class="p">:</span>
+        <span class="n">key_id_option</span> <span class="o">=</span> <span class="s1">&#39;access_key&#39;</span>
+        <span class="n">secret_key_option</span> <span class="o">=</span> <span class="s1">&#39;secret_key&#39;</span>
+    <span class="c1"># Actual Parsing</span>
+    <span class="k">if</span> <span class="n">cred_section</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">sections</span><span class="p">:</span>
+        <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">&quot;This config file format is not recognized&quot;</span><span class="p">)</span>
+    <span class="k">else</span><span class="p">:</span>
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">access_key</span> <span class="o">=</span> <span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">cred_section</span><span class="p">,</span> <span class="n">key_id_option</span><span class="p">)</span>
+            <span class="n">secret_key</span> <span class="o">=</span> <span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">cred_section</span><span class="p">,</span> <span class="n">secret_key_option</span><span class="p">)</span>
+        <span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span>
+            <span class="n">logging</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">&quot;Option Error in parsing s3 config file&quot;</span><span class="p">)</span>
+            <span class="k">raise</span>
+        <span class="k">return</span> <span class="n">access_key</span><span class="p">,</span> <span class="n">secret_key</span>
+
+
+<div class="viewcode-block" id="AwsHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_hook.AwsHook">[docs]</a><span class="k">class</span> <span class="nc">AwsHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Interact with AWS.</span>
+<span class="sd">    This class is a thin wrapper around the boto3 python library.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">aws_conn_id</span><span class="o">=</span><span class="s1">&#39;aws_default&#39;</span><span class="p">,</span> <span class="n">verify</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span> <span class="o">=</span> <span class="n">aws_conn_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">verify</span> <span class="o">=</span> <span class="n">verify</span>
+
+    <span class="k">def</span> <span class="nf">_get_credentials</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">region_name</span><span class="p">):</span>
+        <span class="n">aws_access_key_id</span> <span class="o">=</span> <span class="kc">None</span>
+        <span class="n">aws_secret_access_key</span> <span class="o">=</span> <span class="kc">None</span>
+        <span class="n">aws_session_token</span> <span class="o">=</span> <span class="kc">None</span>
+        <span class="n">endpoint_url</span> <span class="o">=</span> <span class="kc">None</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span><span class="p">:</span>
+            <span class="k">try</span><span class="p">:</span>
+                <span class="n">connection_object</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span><span class="p">)</span>
+                <span class="n">extra_config</span> <span class="o">=</span> <span class="n">connection_object</span><span class="o">.</span><span class="n">extra_dejson</span>
+                <span class="k">if</span> <span class="n">connection_object</span><span class="o">.</span><span class="n">login</span><span class="p">:</span>
+                    <span class="n">aws_access_key_id</span> <span class="o">=</span> <span class="n">connection_object</span><span class="o">.</span><span class="n">login</span>
+                    <span class="n">aws_secret_access_key</span> <span class="o">=</span> <span class="n">connection_object</span><span class="o">.</span><span class="n">password</span>
+
+                <span class="k">elif</span> <span class="s1">&#39;aws_secret_access_key&#39;</span> <span class="ow">in</span> <span class="n">extra_config</span><span class="p">:</span>
+                    <span class="n">aws_access_key_id</span> <span class="o">=</span> <span class="n">extra_config</span><span class="p">[</span>
+                        <span class="s1">&#39;aws_access_key_id&#39;</span><span class="p">]</span>
+                    <span class="n">aws_secret_access_key</span> <span class="o">=</span> <span class="n">extra_config</span><span class="p">[</span>
+                        <span class="s1">&#39;aws_secret_access_key&#39;</span><span class="p">]</span>
+
+                <span class="k">elif</span> <span class="s1">&#39;s3_config_file&#39;</span> <span class="ow">in</span> <span class="n">extra_config</span><span class="p">:</span>
+                    <span class="n">aws_access_key_id</span><span class="p">,</span> <span class="n">aws_secret_access_key</span> <span class="o">=</span> \
+                        <span class="n">_parse_s3_config</span><span class="p">(</span>
+                            <span class="n">extra_config</span><span class="p">[</span><span class="s1">&#39;s3_config_file&#39;</span><span class="p">],</span>
+                            <span class="n">extra_config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;s3_config_format&#39;</span><span class="p">),</span>
+                            <span class="n">extra_config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;profile&#39;</span><span class="p">))</span>
+
+                <span class="k">if</span> <span class="n">region_name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+                    <span class="n">region_name</span> <span class="o">=</span> <span class="n">extra_config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;region_name&#39;</span><span class="p">)</span>
+
+                <span class="n">role_arn</span> <span class="o">=</span> <span class="n">extra_config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;role_arn&#39;</span><span class="p">)</span>
+                <span class="n">external_id</span> <span class="o">=</span> <span class="n">extra_config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;external_id&#39;</span><span class="p">)</span>
+                <span class="n">aws_account_id</span> <span class="o">=</span> <span class="n">extra_config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;aws_account_id&#39;</span><span class="p">)</span>
+                <span class="n">aws_iam_role</span> <span class="o">=</span> <span class="n">extra_config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;aws_iam_role&#39;</span><span class="p">)</span>
+
+                <span class="k">if</span> <span class="n">role_arn</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">aws_account_id</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> \
+                        <span class="n">aws_iam_role</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+                    <span class="n">role_arn</span> <span class="o">=</span> <span class="s2">&quot;arn:aws:iam::</span><span class="si">{}</span><span class="s2">:role/</span><span class="si">{}</span><span class="s2">&quot;</span> \
+                        <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">aws_account_id</span><span class="p">,</span> <span class="n">aws_iam_role</span><span class="p">)</span>
+
+                <span class="k">if</span> <span class="n">role_arn</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+                    <span class="n">sts_session</span> <span class="o">=</span> <span class="n">boto3</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">Session</span><span class="p">(</span>
+                        <span class="n">aws_access_key_id</span><span class="o">=</span><span class="n">aws_access_key_id</span><span class="p">,</span>
+                        <span class="n">aws_secret_access_key</span><span class="o">=</span><span class="n">aws_secret_access_key</span><span class="p">,</span>
+                        <span class="n">region_name</span><span class="o">=</span><span class="n">region_name</span><span class="p">)</span>
+
+                    <span class="n">sts_client</span> <span class="o">=</span> <span class="n">sts_session</span><span class="o">.</span><span class="n">client</span><span class="p">(</span><span class="s1">&#39;sts&#39;</span><span class="p">)</span>
+
+                    <span class="k">if</span> <span class="n">external_id</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+                        <span class="n">sts_response</span> <span class="o">=</span> <span class="n">sts_client</span><span class="o">.</span><span class="n">assume_role</span><span class="p">(</span>
+                            <span class="n">RoleArn</span><span class="o">=</span><span class="n">role_arn</span><span class="p">,</span>
+                            <span class="n">RoleSessionName</span><span class="o">=</span><span class="s1">&#39;Airflow_&#39;</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span><span class="p">)</span>
+                    <span class="k">else</span><span class="p">:</span>
+                        <span class="n">sts_response</span> <span class="o">=</span> <span class="n">sts_client</span><span class="o">.</span><span class="n">assume_role</span><span class="p">(</span>
+                            <span class="n">RoleArn</span><span class="o">=</span><span class="n">role_arn</span><span class="p">,</span>
+                            <span class="n">RoleSessionName</span><span class="o">=</span><span class="s1">&#39;Airflow_&#39;</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span><span class="p">,</span>
+                            <span class="n">ExternalId</span><span class="o">=</span><span class="n">external_id</span><span class="p">)</span>
+
+                    <span class="n">credentials</span> <span class="o">=</span> <span class="n">sts_response</span><span class="p">[</span><span class="s1">&#39;Credentials&#39;</span><span class="p">]</span>
+                    <span class="n">aws_access_key_id</span> <span class="o">=</span> <span class="n">credentials</span><span class="p">[</span><span class="s1">&#39;AccessKeyId&#39;</span><span class="p">]</span>
+                    <span class="n">aws_secret_access_key</span> <span class="o">=</span> <span class="n">credentials</span><span class="p">[</span><span class="s1">&#39;SecretAccessKey&#39;</span><span class="p">]</span>
+                    <span class="n">aws_session_token</span> <span class="o">=</span> <span class="n">credentials</span><span class="p">[</span><span class="s1">&#39;SessionToken&#39;</span><span class="p">]</span>
+
+                <span class="n">endpoint_url</span> <span class="o">=</span> <span class="n">extra_config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;host&#39;</span><span class="p">)</span>
+
+            <span class="k">except</span> <span class="n">AirflowException</span><span class="p">:</span>
+                <span class="c1"># No connection found: fallback on boto3 credential strategy</span>
+                <span class="c1"># http://boto3.readthedocs.io/en/latest/guide/configuration.html</span>
+                <span class="k">pass</span>
+
+        <span class="k">return</span> <span class="n">boto3</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">Session</span><span class="p">(</span>
+            <span class="n">aws_access_key_id</span><span class="o">=</span><span class="n">aws_access_key_id</span><span class="p">,</span>
+            <span class="n">aws_secret_access_key</span><span class="o">=</span><span class="n">aws_secret_access_key</span><span class="p">,</span>
+            <span class="n">aws_session_token</span><span class="o">=</span><span class="n">aws_session_token</span><span class="p">,</span>
+            <span class="n">region_name</span><span class="o">=</span><span class="n">region_name</span><span class="p">),</span> <span class="n">endpoint_url</span>
+
+    <span class="k">def</span> <span class="nf">get_client_type</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">client_type</span><span class="p">,</span> <span class="n">region_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">config</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+        <span class="n">session</span><span class="p">,</span> <span class="n">endpoint_url</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_credentials</span><span class="p">(</span><span class="n">region_name</span><span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">session</span><span class="o">.</span><span class="n">client</span><span class="p">(</span><span class="n">client_type</span><span class="p">,</span> <span class="n">endpoint_url</span><span class="o">=</span><span class="n">endpoint_url</span><span class="p">,</span>
+                              <span class="n">config</span><span class="o">=</span><span class="n">config</span><span class="p">,</span> <span class="n">verify</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">verify</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">get_resource_type</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">resource_type</span><span class="p">,</span> <span class="n">region_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">config</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+        <span class="n">session</span><span class="p">,</span> <span class="n">endpoint_url</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_credentials</span><span class="p">(</span><span class="n">region_name</span><span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">session</span><span class="o">.</span><span class="n">resource</span><span class="p">(</span><span class="n">resource_type</span><span class="p">,</span> <span class="n">endpoint_url</span><span class="o">=</span><span class="n">endpoint_url</span><span class="p">,</span>
+                                <span class="n">config</span><span class="o">=</span><span class="n">config</span><span class="p">,</span> <span class="n">verify</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">verify</span><span class="p">)</span>
+
+<div class="viewcode-block" id="AwsHook.get_session"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_hook.AwsHook.get_session">[docs]</a>    <span class="k">def</span> <span class="nf">get_session</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">region_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;Get the underlying boto3.session.&quot;&quot;&quot;</span>
+        <span class="n">session</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_credentials</span><span class="p">(</span><span class="n">region_name</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">session</span></div>
+
+<div class="viewcode-block" id="AwsHook.get_credentials"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_hook.AwsHook.get_credentials">[docs]</a>    <span class="k">def</span> <span class="nf">get_credentials</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">region_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;Get the underlying `botocore.Credentials` object.</span>
+
+<span class="sd">        This contains the following authentication attributes: access_key, secret_key and token.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">session</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_credentials</span><span class="p">(</span><span class="n">region_name</span><span class="p">)</span>
+        <span class="c1"># Credentials are refreshable, so accessing your access key and</span>
+        <span class="c1"># secret key separately can lead to a race condition.</span>
+        <span class="c1"># See https://stackoverflow.com/a/36291428/8283373</span>
+        <span class="k">return</span> <span class="n">session</span><span class="o">.</span><span class="n">get_credentials</span><span class="p">()</span><span class="o">.</span><span class="n">get_frozen_credentials</span><span class="p">()</span></div>
+
+<div class="viewcode-block" id="AwsHook.expand_role"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_hook.AwsHook.expand_role">[docs]</a>    <span class="k">def</span> <span class="nf">expand_role</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">role</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        If the IAM role is a role name, get the Amazon Resource Name (ARN) for the role.</span>
+<span class="sd">        If IAM role is already an IAM role ARN, no change is made.</span>
+
+<span class="sd">        :param role: IAM role name or ARN</span>
+<span class="sd">        :return: IAM role ARN</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">if</span> <span class="s1">&#39;/&#39;</span> <span class="ow">in</span> <span class="n">role</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">role</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_client_type</span><span class="p">(</span><span class="s1">&#39;iam&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">get_role</span><span class="p">(</span><span class="n">RoleName</span><span class="o">=</span><span class="n">role</span><span class="p">)[</span><span class="s1">&#39;Role&#39;</span><span class="p">][</span><span class="s1">&#39;Arn& [...]
+</pre></div>
+
+           </div>
+           
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    
+    
+      <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+        <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+        <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+        <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+        <script type="text/javascript" src="../../../../_static/language_data.js"></script>
+    
+
+  
+
+  <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.Navigation.enable(true);
+      });
+  </script> 
+
+</body>
+</html>
\ No newline at end of file
diff --git a/1.10.2/_modules/airflow/contrib/hooks/aws_lambda_hook.html b/1.10.2/_modules/airflow/contrib/hooks/aws_lambda_hook.html
new file mode 100644
index 0000000..127ff53
--- /dev/null
+++ b/1.10.2/_modules/airflow/contrib/hooks/aws_lambda_hook.html
@@ -0,0 +1,287 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>airflow.contrib.hooks.aws_lambda_hook &mdash; Airflow Documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+    <link rel="index" title="Index" href="../../../../genindex.html" />
+    <link rel="search" title="Search" href="../../../../search.html" /> 
+
+  
+  <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../../index.html" class="icon icon-home"> Airflow
+          
+
+          
+          </a>
+
+          
+            
+            
+              <div class="version">
+                1.10.2
+              </div>
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../metrics.html">Metrics</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../../index.html">Airflow</a>
+        
+      </nav>
+
+
+      <div class="wy-nav-content">
+        
+        <div class="rst-content">
+        
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../../index.html">Module code</a> &raquo;</li>
+        
+      <li>airflow.contrib.hooks.aws_lambda_hook</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for airflow.contrib.hooks.aws_lambda_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements.  See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership.  The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># &quot;License&quot;); you may not use this file except in compliance</span>
+<span class="c1"># with the License.  You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1">#   http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># &quot;AS IS&quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied.  See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.aws_hook</span> <span class="k">import</span> <span class="n">AwsHook</span>
+
+
+<div class="viewcode-block" id="AwsLambdaHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_lambda_hook.AwsLambdaHook">[docs]</a><span class="k">class</span> <span class="nc">AwsLambdaHook</span><span class="p">(</span><span class="n">AwsHook</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Interact with AWS Lambda</span>
+
+<span class="sd">    :param function_name: AWS Lambda Function Name</span>
+<span class="sd">    :type function_name: str</span>
+<span class="sd">    :param region_name: AWS Region Name (example: us-west-2)</span>
+<span class="sd">    :type region_name: str</span>
+<span class="sd">    :param log_type: Tail Invocation Request</span>
+<span class="sd">    :type log_type: str</span>
+<span class="sd">    :param qualifier: AWS Lambda Function Version or Alias Name</span>
+<span class="sd">    :type qualifier: str</span>
+<span class="sd">    :param invocation_type: AWS Lambda Invocation Type (RequestResponse, Event etc)</span>
+<span class="sd">    :type invocation_type: str</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">function_name</span><span class="p">,</span> <span class="n">region_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                 <span class="n">log_type</span><span class="o">=</span><span class="s1">&#39;None&#39;</span><span class="p">,</span> <span class="n">qualifier</span><span class="o">=</span><span class="s1">&#39;$LATEST&#39;</span><span class="p">,</span>
+                 <span class="n">invocation_type</span><span class="o">=</span><span class="s1">&#39;RequestResponse&#39;</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">function_name</span> <span class="o">=</span> <span class="n">function_name</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">region_name</span> <span class="o">=</span> <span class="n">region_name</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log_type</span> <span class="o">=</span> <span class="n">log_type</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">invocation_type</span> <span class="o">=</span> <span class="n">invocation_type</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">qualifier</span> <span class="o">=</span> <span class="n">qualifier</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">AwsLambdaHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_client_type</span><span class="p">(</span><span class="s1">&#39;lambda&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">region_name</span><span class="p">)</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span>
+
+<div class="viewcode-block" id="AwsLambdaHook.invoke_lambda"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.aws_lambda_hook.AwsLambdaHook.invoke_lambda">[docs]</a>    <span class="k">def</span> <span class="nf">invoke_lambda</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">payload</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Invoke Lambda Function</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+
+        <span class="n">awslambda_conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+
+        <span class="n">response</span> <span class="o">=</span> <span class="n">awslambda_conn</span><span class="o">.</span><span class="n">invoke</span><span class="p">(</span>
+            <span class="n">FunctionName</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">function_name</span><span class="p">,</span>
+            <span class="n">InvocationType</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">invocation_type</span><span class="p">,</span>
+            <span class="n">LogType</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">log_type</span><span class="p">,</span>
+            <span class="n">Payload</span><span class="o">=</span><span class="n">payload</span><span class="p">,</span>
+            <span class="n">Qualifier</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">qualifier</span>
+        <span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">response</span></div></div>
+</pre></div>
+
+           </div>
+           
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    
+    
+      <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+        <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+        <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+        <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+        <script type="text/javascript" src="../../../../_static/language_data.js"></script>
+    
+
+  
+
+  <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.Navigation.enable(true);
+      });
+  </script> 
+
+</body>
+</html>
\ No newline at end of file
diff --git a/1.10.2/_modules/airflow/contrib/hooks/azure_cosmos_hook.html b/1.10.2/_modules/airflow/contrib/hooks/azure_cosmos_hook.html
new file mode 100644
index 0000000..72c04e5
--- /dev/null
+++ b/1.10.2/_modules/airflow/contrib/hooks/azure_cosmos_hook.html
@@ -0,0 +1,506 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>airflow.contrib.hooks.azure_cosmos_hook &mdash; Airflow Documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+    <link rel="index" title="Index" href="../../../../genindex.html" />
+    <link rel="search" title="Search" href="../../../../search.html" /> 
+
+  
+  <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../../index.html" class="icon icon-home"> Airflow
+          
+
+          
+          </a>
+
+          
+            
+            
+              <div class="version">
+                1.10.2
+              </div>
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../metrics.html">Metrics</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../../index.html">Airflow</a>
+        
+      </nav>
+
+
+      <div class="wy-nav-content">
+        
+        <div class="rst-content">
+        
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../../index.html">Module code</a> &raquo;</li>
+        
+      <li>airflow.contrib.hooks.azure_cosmos_hook</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for airflow.contrib.hooks.azure_cosmos_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements.  See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership.  The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># &quot;License&quot;); you may not use this file except in compliance</span>
+<span class="c1"># with the License.  You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1">#   http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># &quot;AS IS&quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied.  See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="kn">import</span> <span class="nn">azure.cosmos.cosmos_client</span> <span class="k">as</span> <span class="nn">cosmos_client</span>
+<span class="kn">from</span> <span class="nn">azure.cosmos.errors</span> <span class="k">import</span> <span class="n">HTTPFailure</span>
+<span class="kn">import</span> <span class="nn">uuid</span>
+
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowBadRequest</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+
+
+<div class="viewcode-block" id="AzureCosmosDBHook"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_cosmos_hook.AzureCosmosDBHook">[docs]</a><span class="k">class</span> <span class="nc">AzureCosmosDBHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Interacts with Azure CosmosDB.</span>
+
+<span class="sd">    login should be the endpoint uri, password should be the master key</span>
+<span class="sd">    optionally, you can use the following extras to default these values</span>
+<span class="sd">    {&quot;database_name&quot;: &quot;&lt;DATABASE_NAME&gt;&quot;, &quot;collection_name&quot;: &quot;COLLECTION_NAME&quot;}.</span>
+
+<span class="sd">    :param azure_cosmos_conn_id: Reference to the Azure CosmosDB connection.</span>
+<span class="sd">    :type azure_cosmos_conn_id: str</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">azure_cosmos_conn_id</span><span class="o">=</span><span class="s1">&#39;azure_cosmos_default&#39;</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span> <span class="o">=</span> <span class="n">azure_cosmos_conn_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">connection</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">extras</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">extra_dejson</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">endpoint_uri</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">login</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">master_key</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">password</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">default_database_name</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">extras</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;database_name&#39;</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">default_collection_name</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">extras</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;collection_name&#39;</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">cosmos_client</span> <span class="o">=</span> <span class="kc">None</span>
+
+<div class="viewcode-block" id="AzureCosmosDBHook.get_conn"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_cosmos_hook.AzureCosmosDBHook.get_conn">[docs]</a>    <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Return a cosmos db client.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cosmos_client</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">cosmos_client</span>
+
+        <span class="c1"># Initialize the Python Azure Cosmos DB client</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">cosmos_client</span> <span class="o">=</span> <span class="n">cosmos_client</span><span class="o">.</span><span class="n">CosmosClient</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">endpoint_uri</span><span class="p">,</span> <span class="p">{</span><span class="s1">&#39;masterKey&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</ [...]
+
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">cosmos_client</span></div>
+
+    <span class="k">def</span> <span class="nf">__get_database_name</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">database_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+        <span class="n">db_name</span> <span class="o">=</span> <span class="n">database_name</span>
+        <span class="k">if</span> <span class="n">db_name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">db_name</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">default_database_name</span>
+
+        <span class="k">if</span> <span class="n">db_name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">AirflowBadRequest</span><span class="p">(</span><span class="s2">&quot;Database name must be specified&quot;</span><span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">db_name</span>
+
+    <span class="k">def</span> <span class="nf">__get_collection_name</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">collection_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+        <span class="n">coll_name</span> <span class="o">=</span> <span class="n">collection_name</span>
+        <span class="k">if</span> <span class="n">coll_name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">coll_name</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">default_collection_name</span>
+
+        <span class="k">if</span> <span class="n">coll_name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">AirflowBadRequest</span><span class="p">(</span><span class="s2">&quot;Collection name must be specified&quot;</span><span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">coll_name</span>
+
+<div class="viewcode-block" id="AzureCosmosDBHook.does_collection_exist"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_cosmos_hook.AzureCosmosDBHook.does_collection_exist">[docs]</a>    <span class="k">def</span> <span class="nf">does_collection_exist</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">collection_name</span><span class="p">,</span> <span class="n">database_name</span><span class="o"> [...]
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Checks if a collection exists in CosmosDB.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">if</span> <span class="n">collection_name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">AirflowBadRequest</span><span class="p">(</span><span class="s2">&quot;Collection name cannot be None.&quot;</span><span class="p">)</span>
+
+        <span class="n">existing_container</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">QueryContainers</span><span class="p">(</span>
+            <span class="n">get_database_link</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">__get_database_name</span><span class="p">(</span><span class="n">database_name</span><span class="p">)),</span> <span class="p">{</span>
+                <span class="s2">&quot;query&quot;</span><span class="p">:</span> <span class="s2">&quot;SELECT * FROM r WHERE r.id=@id&quot;</span><span class="p">,</span>
+                <span class="s2">&quot;parameters&quot;</span><span class="p">:</span> <span class="p">[</span>
+                    <span class="p">{</span><span class="s2">&quot;name&quot;</span><span class="p">:</span> <span class="s2">&quot;@id&quot;</span><span class="p">,</span> <span class="s2">&quot;value&quot;</span><span class="p">:</span> <span class="n">collection_name</span><span class="p">}</span>
+                <span class="p">]</span>
+            <span class="p">}))</span>
+        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing_container</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+            <span class="k">return</span> <span class="kc">False</span>
+
+        <span class="k">return</span> <span class="kc">True</span></div>
+
+<div class="viewcode-block" id="AzureCosmosDBHook.create_collection"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_cosmos_hook.AzureCosmosDBHook.create_collection">[docs]</a>    <span class="k">def</span> <span class="nf">create_collection</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">collection_name</span><span class="p">,</span> <span class="n">database_name</span><span class="o">=</span><spa [...]
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Creates a new collection in the CosmosDB database.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">if</span> <span class="n">collection_name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">AirflowBadRequest</span><span class="p">(</span><span class="s2">&quot;Collection name cannot be None.&quot;</span><span class="p">)</span>
+
+        <span class="c1"># We need to check to see if this container already exists so we don&#39;t try</span>
+        <span class="c1"># to create it twice</span>
+        <span class="n">existing_container</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">QueryContainers</span><span class="p">(</span>
+            <span class="n">get_database_link</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">__get_database_name</span><span class="p">(</span><span class="n">database_name</span><span class="p">)),</span> <span class="p">{</span>
+                <span class="s2">&quot;query&quot;</span><span class="p">:</span> <span class="s2">&quot;SELECT * FROM r WHERE r.id=@id&quot;</span><span class="p">,</span>
+                <span class="s2">&quot;parameters&quot;</span><span class="p">:</span> <span class="p">[</span>
+                    <span class="p">{</span><span class="s2">&quot;name&quot;</span><span class="p">:</span> <span class="s2">&quot;@id&quot;</span><span class="p">,</span> <span class="s2">&quot;value&quot;</span><span class="p">:</span> <span class="n">collection_name</span><span class="p">}</span>
+                <span class="p">]</span>
+            <span class="p">}))</span>
+
+        <span class="c1"># Only create if we did not find it already existing</span>
+        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing_container</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">CreateContainer</span><span class="p">(</span>
+                <span class="n">get_database_link</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">__get_database_name</span><span class="p">(</span><span class="n">database_name</span><span class="p">)),</span>
+                <span class="p">{</span><span class="s2">&quot;id&quot;</span><span class="p">:</span> <span class="n">collection_name</span><span class="p">})</span></div>
+
+<div class="viewcode-block" id="AzureCosmosDBHook.does_database_exist"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_cosmos_hook.AzureCosmosDBHook.does_database_exist">[docs]</a>    <span class="k">def</span> <span class="nf">does_database_exist</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">database_name</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Checks if a database exists in CosmosDB.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">if</span> <span class="n">database_name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">AirflowBadRequest</span><span class="p">(</span><span class="s2">&quot;Database name cannot be None.&quot;</span><span class="p">)</span>
+
+        <span class="n">existing_database</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">QueryDatabases</span><span class="p">({</span>
+            <span class="s2">&quot;query&quot;</span><span class="p">:</span> <span class="s2">&quot;SELECT * FROM r WHERE r.id=@id&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;parameters&quot;</span><span class="p">:</span> <span class="p">[</span>
+                <span class="p">{</span><span class="s2">&quot;name&quot;</span><span class="p">:</span> <span class="s2">&quot;@id&quot;</span><span class="p">,</span> <span class="s2">&quot;value&quot;</span><span class="p">:</span> <span class="n">database_name</span><span class="p">}</span>
+            <span class="p">]</span>
+        <span class="p">}))</span>
+        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing_database</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+            <span class="k">return</span> <span class="kc">False</span>
+
+        <span class="k">return</span> <span class="kc">True</span></div>
+
+<div class="viewcode-block" id="AzureCosmosDBHook.create_database"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_cosmos_hook.AzureCosmosDBHook.create_database">[docs]</a>    <span class="k">def</span> <span class="nf">create_database</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">database_name</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Creates a new database in CosmosDB.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">if</span> <span class="n">database_name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">AirflowBadRequest</span><span class="p">(</span><span class="s2">&quot;Database name cannot be None.&quot;</span><span class="p">)</span>
+
+        <span class="c1"># We need to check to see if this database already exists so we don&#39;t try</span>
+        <span class="c1"># to create it twice</span>
+        <span class="n">existing_database</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">QueryDatabases</span><span class="p">({</span>
+            <span class="s2">&quot;query&quot;</span><span class="p">:</span> <span class="s2">&quot;SELECT * FROM r WHERE r.id=@id&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;parameters&quot;</span><span class="p">:</span> <span class="p">[</span>
+                <span class="p">{</span><span class="s2">&quot;name&quot;</span><span class="p">:</span> <span class="s2">&quot;@id&quot;</span><span class="p">,</span> <span class="s2">&quot;value&quot;</span><span class="p">:</span> <span class="n">database_name</span><span class="p">}</span>
+            <span class="p">]</span>
+        <span class="p">}))</span>
+
+        <span class="c1"># Only create if we did not find it already existing</span>
+        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing_database</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">CreateDatabase</span><span class="p">({</span><span class="s2">&quot;id&quot;</span><span class="p">:</span> <span class="n">database_name</span><span class="p">})</span></div>
+
+<div class="viewcode-block" id="AzureCosmosDBHook.delete_database"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_cosmos_hook.AzureCosmosDBHook.delete_database">[docs]</a>    <span class="k">def</span> <span class="nf">delete_database</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">database_name</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Deletes an existing database in CosmosDB.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">if</span> <span class="n">database_name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">AirflowBadRequest</span><span class="p">(</span><span class="s2">&quot;Database name cannot be None.&quot;</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">DeleteDatabase</span><span class="p">(</span><span class="n">get_database_link</span><span class="p">(</span><span class="n">database_name</span><span class="p">))</span></div>
+
+<div class="viewcode-block" id="AzureCosmosDBHook.delete_collection"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_cosmos_hook.AzureCosmosDBHook.delete_collection">[docs]</a>    <span class="k">def</span> <span class="nf">delete_collection</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">collection_name</span><span class="p">,</span> <span class="n">database_name</span><span class="o">=</span><spa [...]
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Deletes an existing collection in the CosmosDB database.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">if</span> <span class="n">collection_name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">AirflowBadRequest</span><span class="p">(</span><span class="s2">&quot;Collection name cannot be None.&quot;</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">DeleteContainer</span><span class="p">(</span>
+            <span class="n">get_collection_link</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">__get_database_name</span><span class="p">(</span><span class="n">database_name</span><span class="p">),</span> <span class="n">collection_name</span><span class="p">))</span></div>
+
+<div class="viewcode-block" id="AzureCosmosDBHook.upsert_document"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_cosmos_hook.AzureCosmosDBHook.upsert_document">[docs]</a>    <span class="k">def</span> <span class="nf">upsert_document</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">document</span><span class="p">,</span> <span class="n">database_name</span><span class="o">=</span><span class="kc"> [...]
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Inserts a new document (or updates an existing one) into an existing</span>
+<span class="sd">        collection in the CosmosDB database.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="c1"># Assign unique ID if one isn&#39;t provided</span>
+        <span class="k">if</span> <span class="n">document_id</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">document_id</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">uuid</span><span class="o">.</span><span class="n">uuid4</span><span class="p">())</span>
+
+        <span class="k">if</span> <span class="n">document</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">AirflowBadRequest</span><span class="p">(</span><span class="s2">&quot;You cannot insert a None document&quot;</span><span class="p">)</span>
+
+        <span class="c1"># Add document id if isn&#39;t found</span>
+        <span class="k">if</span> <span class="s1">&#39;id&#39;</span> <span class="ow">in</span> <span class="n">document</span><span class="p">:</span>
+            <span class="k">if</span> <span class="n">document</span><span class="p">[</span><span class="s1">&#39;id&#39;</span><span class="p">]</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+                <span class="n">document</span><span class="p">[</span><span class="s1">&#39;id&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">document_id</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">document</span><span class="p">[</span><span class="s1">&#39;id&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">document_id</span>
+
+        <span class="n">created_document</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">CreateItem</span><span class="p">(</span>
+            <span class="n">get_collection_link</span><span class="p">(</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">__get_database_name</span><span class="p">(</span><span class="n">database_name</span><span class="p">),</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">__get_collection_name</span><span class="p">(</span><span class="n">collection_name</span><span class="p">)),</span>
+            <span class="n">document</span><span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">created_document</span></div>
+
+<div class="viewcode-block" id="AzureCosmosDBHook.insert_documents"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_cosmos_hook.AzureCosmosDBHook.insert_documents">[docs]</a>    <span class="k">def</span> <span class="nf">insert_documents</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">documents</span><span class="p">,</span> <span class="n">database_name</span><span class="o">=</span><span class=" [...]
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Insert a list of new documents into an existing collection in the CosmosDB database.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">if</span> <span class="n">documents</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">AirflowBadRequest</span><span class="p">(</span><span class="s2">&quot;You cannot insert empty documents&quot;</span><span class="p">)</span>
+
+        <span class="n">created_documents</span> <span class="o">=</span> <span class="p">[]</span>
+        <span class="k">for</span> <span class="n">single_document</span> <span class="ow">in</span> <span class="n">documents</span><span class="p">:</span>
+            <span class="n">created_documents</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">CreateItem</span><span class="p">(</span>
+                    <span class="n">get_collection_link</span><span class="p">(</span>
+                        <span class="bp">self</span><span class="o">.</span><span class="n">__get_database_name</span><span class="p">(</span><span class="n">database_name</span><span class="p">),</span>
+                        <span class="bp">self</span><span class="o">.</span><span class="n">__get_collection_name</span><span class="p">(</span><span class="n">collection_name</span><span class="p">)),</span>
+                    <span class="n">single_document</span><span class="p">))</span>
+
+        <span class="k">return</span> <span class="n">created_documents</span></div>
+
+<div class="viewcode-block" id="AzureCosmosDBHook.delete_document"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_cosmos_hook.AzureCosmosDBHook.delete_document">[docs]</a>    <span class="k">def</span> <span class="nf">delete_document</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">document_id</span><span class="p">,</span> <span class="n">database_name</span><span class="o">=</span><span class="k [...]
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Delete an existing document out of a collection in the CosmosDB database.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">if</span> <span class="n">document_id</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">AirflowBadRequest</span><span class="p">(</span><span class="s2">&quot;Cannot delete a document without an id&quot;</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">DeleteItem</span><span class="p">(</span>
+            <span class="n">get_document_link</span><span class="p">(</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">__get_database_name</span><span class="p">(</span><span class="n">database_name</span><span class="p">),</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">__get_collection_name</span><span class="p">(</span><span class="n">collection_name</span><span class="p">),</span>
+                <span class="n">document_id</span><span class="p">))</span></div>
+
+<div class="viewcode-block" id="AzureCosmosDBHook.get_document"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_cosmos_hook.AzureCosmosDBHook.get_document">[docs]</a>    <span class="k">def</span> <span class="nf">get_document</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">document_id</span><span class="p">,</span> <span class="n">database_name</span><span class="o">=</span><span class="kc">None</ [...]
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Get a document from an existing collection in the CosmosDB database.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">if</span> <span class="n">document_id</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">AirflowBadRequest</span><span class="p">(</span><span class="s2">&quot;Cannot get a document without an id&quot;</span><span class="p">)</span>
+
+        <span class="k">try</span><span class="p">:</span>
+            <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">ReadItem</span><span class="p">(</span>
+                <span class="n">get_document_link</span><span class="p">(</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">__get_database_name</span><span class="p">(</span><span class="n">database_name</span><span class="p">),</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">__get_collection_name</span><span class="p">(</span><span class="n">collection_name</span><span class="p">),</span>
+                    <span class="n">document_id</span><span class="p">))</span>
+        <span class="k">except</span> <span class="n">HTTPFailure</span><span class="p">:</span>
+            <span class="k">return</span> <span class="kc">None</span></div>
+
+<div class="viewcode-block" id="AzureCosmosDBHook.get_documents"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_cosmos_hook.AzureCosmosDBHook.get_documents">[docs]</a>    <span class="k">def</span> <span class="nf">get_documents</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sql_string</span><span class="p">,</span> <span class="n">database_name</span><span class="o">=</span><span class="kc">None [...]
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Get a list of documents from an existing collection in the CosmosDB database via SQL query.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">if</span> <span class="n">sql_string</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">AirflowBadRequest</span><span class="p">(</span><span class="s2">&quot;SQL query string cannot be None&quot;</span><span class="p">)</span>
+
+        <span class="c1"># Query them in SQL</span>
+        <span class="n">query</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;query&#39;</span><span class="p">:</span> <span class="n">sql_string</span><span class="p">}</span>
+
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">result_iterable</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">QueryItems</span><span class="p">(</span>
+                <span class="n">get_collection_link</span><span class="p">(</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">__get_database_name</span><span class="p">(</span><span class="n">database_name</span><span class="p">),</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">__get_collection_name</span><span class="p">(</span><span class="n">collection_name</span><span class="p">)),</span>
+                <span class="n">query</span><span class="p">,</span>
+                <span class="n">partition_key</span><span class="p">)</span>
+
+            <span class="k">return</span> <span class="nb">list</span><span class="p">(</span><span class="n">result_iterable</span><span class="p">)</span>
+        <span class="k">except</span> <span class="n">HTTPFailure</span><span class="p">:</span>
+            <span class="k">return</span> <span class="kc">None</span></div></div>
+
+
+<span class="k">def</span> <span class="nf">get_database_link</span><span class="p">(</span><span class="n">database_id</span><span class="p">):</span>
+    <span class="k">return</span> <span class="s2">&quot;dbs/&quot;</span> <span class="o">+</span> <span class="n">database_id</span>
+
+
+<span class="k">def</span> <span class="nf">get_collection_link</span><span class="p">(</span><span class="n">database_id</span><span class="p">,</span> <span class="n">collection_id</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">get_database_link</span><span class="p">(</span><span class="n">database_id</span><span class="p">)</span> <span class="o">+</span> <span class="s2">&quot;/colls/&quot;</span> <span class="o">+</span> <span class="n">collection_id</span>
+
+
+<span class="k">def</span> <span class="nf">get_document_link</span><span class="p">(</span><span class="n">database_id</span><span class="p">,</span> <span class="n">collection_id</span><span class="p">,</span> <span class="n">document_id</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">get_collection_link</span><span class="p">(</span><span class="n">database_id</span><span class="p">,</span> <span class="n">collection_id</span><span class="p">)</span> <span class="o">+</span> <span class="s2">&quot;/docs/&quot;</span> <span class="o">+</span> <span class="n">document_id</span>
+</pre></div>
+
+           </div>
+           
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    
+    
+      <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+        <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+        <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+        <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+        <script type="text/javascript" src="../../../../_static/language_data.js"></script>
+    
+
+  
+
+  <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.Navigation.enable(true);
+      });
+  </script> 
+
+</body>
+</html>
\ No newline at end of file
diff --git a/1.10.2/_modules/airflow/contrib/hooks/azure_data_lake_hook.html b/1.10.2/_modules/airflow/contrib/hooks/azure_data_lake_hook.html
new file mode 100644
index 0000000..8308899
--- /dev/null
+++ b/1.10.2/_modules/airflow/contrib/hooks/azure_data_lake_hook.html
@@ -0,0 +1,360 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>airflow.contrib.hooks.azure_data_lake_hook &mdash; Airflow Documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+    <link rel="index" title="Index" href="../../../../genindex.html" />
+    <link rel="search" title="Search" href="../../../../search.html" /> 
+
+  
+  <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../../index.html" class="icon icon-home"> Airflow
+          
+
+          
+          </a>
+
+          
+            
+            
+              <div class="version">
+                1.10.2
+              </div>
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../metrics.html">Metrics</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../../index.html">Airflow</a>
+        
+      </nav>
+
+
+      <div class="wy-nav-content">
+        
+        <div class="rst-content">
+        
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../../index.html">Module code</a> &raquo;</li>
+        
+      <li>airflow.contrib.hooks.azure_data_lake_hook</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for airflow.contrib.hooks.azure_data_lake_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements.  See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership.  The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># &quot;License&quot;); you may not use this file except in compliance</span>
+<span class="c1"># with the License.  You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1">#   http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># &quot;AS IS&quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied.  See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="c1">#</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">azure.datalake.store</span> <span class="k">import</span> <span class="n">core</span><span class="p">,</span> <span class="n">lib</span><span class="p">,</span> <span class="n">multithread</span>
+
+
+<div class="viewcode-block" id="AzureDataLakeHook"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_data_lake_hook.AzureDataLakeHook">[docs]</a><span class="k">class</span> <span class="nc">AzureDataLakeHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Interacts with Azure Data Lake.</span>
+
+<span class="sd">    Client ID and client secret should be in user and password parameters.</span>
+<span class="sd">    Tenant and account name should be extra field as</span>
+<span class="sd">    {&quot;tenant&quot;: &quot;&lt;TENANT&gt;&quot;, &quot;account_name&quot;: &quot;ACCOUNT_NAME&quot;}.</span>
+
+<span class="sd">    :param azure_data_lake_conn_id: Reference to the Azure Data Lake connection.</span>
+<span class="sd">    :type azure_data_lake_conn_id: str</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">azure_data_lake_conn_id</span><span class="o">=</span><span class="s1">&#39;azure_data_lake_default&#39;</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span> <span class="o">=</span> <span class="n">azure_data_lake_conn_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">connection</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+
+<div class="viewcode-block" id="AzureDataLakeHook.get_conn"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_data_lake_hook.AzureDataLakeHook.get_conn">[docs]</a>    <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;Return a AzureDLFileSystem object.&quot;&quot;&quot;</span>
+        <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+        <span class="n">service_options</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">account_name</span> <span class="o">=</span> <span class="n">service_options</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;account_name&#39;</span><span class="p">)</span>
+
+        <span class="n">adlCreds</span> <span class="o">=</span> <span class="n">lib</span><span class="o">.</span><span class="n">auth</span><span class="p">(</span><span class="n">tenant_id</span><span class="o">=</span><span class="n">service_options</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;tenant&#39;</span><span class="p">),</span>
+                            <span class="n">client_secret</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">,</span>
+                            <span class="n">client_id</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">)</span>
+        <span class="n">adlsFileSystemClient</span> <span class="o">=</span> <span class="n">core</span><span class="o">.</span><span class="n">AzureDLFileSystem</span><span class="p">(</span><span class="n">adlCreds</span><span class="p">,</span>
+                                                      <span class="n">store_name</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">account_name</span><span class="p">)</span>
+        <span class="n">adlsFileSystemClient</span><span class="o">.</span><span class="n">connect</span><span class="p">()</span>
+        <span class="k">return</span> <span class="n">adlsFileSystemClient</span></div>
+
+<div class="viewcode-block" id="AzureDataLakeHook.check_for_file"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_data_lake_hook.AzureDataLakeHook.check_for_file">[docs]</a>    <span class="k">def</span> <span class="nf">check_for_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">file_path</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Check if a file exists on Azure Data Lake.</span>
+
+<span class="sd">        :param file_path: Path and name of the file.</span>
+<span class="sd">        :type file_path: str</span>
+<span class="sd">        :return: True if the file exists, False otherwise.</span>
+<span class="sd">        :rtype: bool</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">files</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">glob</span><span class="p">(</span><span class="n">file_path</span><span class="p">,</span> <span class="n">details</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">invalidate_cache</span><span class="o">=</span><span class="kc">True</span><span clas [...]
+            <span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="n">files</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span>
+        <span class="k">except</span> <span class="ne">FileNotFoundError</span><span class="p">:</span>
+            <span class="k">return</span> <span class="kc">False</span></div>
+
+<div class="viewcode-block" id="AzureDataLakeHook.upload_file"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_data_lake_hook.AzureDataLakeHook.upload_file">[docs]</a>    <span class="k">def</span> <span class="nf">upload_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">local_path</span><span class="p">,</span> <span class="n">remote_path</span><span class="p">,</span> <span class="n">nthreads< [...]
+                    <span class="n">buffersize</span><span class="o">=</span><span class="mi">4194304</span><span class="p">,</span> <span class="n">blocksize</span><span class="o">=</span><span class="mi">4194304</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Upload a file to Azure Data Lake.</span>
+
+<span class="sd">        :param local_path: local path. Can be single file, directory (in which case,</span>
+<span class="sd">            upload recursively) or glob pattern. Recursive glob patterns using `**`</span>
+<span class="sd">            are not supported.</span>
+<span class="sd">        :type local_path: str</span>
+<span class="sd">        :param remote_path: Remote path to upload to; if multiple files, this is the</span>
+<span class="sd">            dircetory root to write within.</span>
+<span class="sd">        :type remote_path: str</span>
+<span class="sd">        :param nthreads: Number of threads to use. If None, uses the number of cores.</span>
+<span class="sd">        :type nthreads: int</span>
+<span class="sd">        :param overwrite: Whether to forcibly overwrite existing files/directories.</span>
+<span class="sd">            If False and remote path is a directory, will quit regardless if any files</span>
+<span class="sd">            would be overwritten or not. If True, only matching filenames are actually</span>
+<span class="sd">            overwritten.</span>
+<span class="sd">        :type overwrite: bool</span>
+<span class="sd">        :param buffersize: int [2**22]</span>
+<span class="sd">            Number of bytes for internal buffer. This block cannot be bigger than</span>
+<span class="sd">            a chunk and cannot be smaller than a block.</span>
+<span class="sd">        :type buffersize: int</span>
+<span class="sd">        :param blocksize: int [2**22]</span>
+<span class="sd">            Number of bytes for a block. Within each chunk, we write a smaller</span>
+<span class="sd">            block for each API call. This block cannot be bigger than a chunk.</span>
+<span class="sd">        :type blocksize: int</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">multithread</span><span class="o">.</span><span class="n">ADLUploader</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="p">,</span>
+                                <span class="n">lpath</span><span class="o">=</span><span class="n">local_path</span><span class="p">,</span>
+                                <span class="n">rpath</span><span class="o">=</span><span class="n">remote_path</span><span class="p">,</span>
+                                <span class="n">nthreads</span><span class="o">=</span><span class="n">nthreads</span><span class="p">,</span>
+                                <span class="n">overwrite</span><span class="o">=</span><span class="n">overwrite</span><span class="p">,</span>
+                                <span class="n">buffersize</span><span class="o">=</span><span class="n">buffersize</span><span class="p">,</span>
+                                <span class="n">blocksize</span><span class="o">=</span><span class="n">blocksize</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="AzureDataLakeHook.download_file"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_data_lake_hook.AzureDataLakeHook.download_file">[docs]</a>    <span class="k">def</span> <span class="nf">download_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">local_path</span><span class="p">,</span> <span class="n">remote_path</span><span class="p">,</span> <span class="n">nth [...]
+                      <span class="n">buffersize</span><span class="o">=</span><span class="mi">4194304</span><span class="p">,</span> <span class="n">blocksize</span><span class="o">=</span><span class="mi">4194304</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Download a file from Azure Blob Storage.</span>
+
+<span class="sd">        :param local_path: local path. If downloading a single file, will write to this</span>
+<span class="sd">            specific file, unless it is an existing directory, in which case a file is</span>
+<span class="sd">            created within it. If downloading multiple files, this is the root</span>
+<span class="sd">            directory to write within. Will create directories as required.</span>
+<span class="sd">        :type local_path: str</span>
+<span class="sd">        :param remote_path: remote path/globstring to use to find remote files.</span>
+<span class="sd">            Recursive glob patterns using `**` are not supported.</span>
+<span class="sd">        :type remote_path: str</span>
+<span class="sd">        :param nthreads: Number of threads to use. If None, uses the number of cores.</span>
+<span class="sd">        :type nthreads: int</span>
+<span class="sd">        :param overwrite: Whether to forcibly overwrite existing files/directories.</span>
+<span class="sd">            If False and remote path is a directory, will quit regardless if any files</span>
+<span class="sd">            would be overwritten or not. If True, only matching filenames are actually</span>
+<span class="sd">            overwritten.</span>
+<span class="sd">        :type overwrite: bool</span>
+<span class="sd">        :param buffersize: int [2**22]</span>
+<span class="sd">            Number of bytes for internal buffer. This block cannot be bigger than</span>
+<span class="sd">            a chunk and cannot be smaller than a block.</span>
+<span class="sd">        :type buffersize: int</span>
+<span class="sd">        :param blocksize: int [2**22]</span>
+<span class="sd">            Number of bytes for a block. Within each chunk, we write a smaller</span>
+<span class="sd">            block for each API call. This block cannot be bigger than a chunk.</span>
+<span class="sd">        :type blocksize: int</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">multithread</span><span class="o">.</span><span class="n">ADLDownloader</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="p">,</span>
+                                  <span class="n">lpath</span><span class="o">=</span><span class="n">local_path</span><span class="p">,</span>
+                                  <span class="n">rpath</span><span class="o">=</span><span class="n">remote_path</span><span class="p">,</span>
+                                  <span class="n">nthreads</span><span class="o">=</span><span class="n">nthreads</span><span class="p">,</span>
+                                  <span class="n">overwrite</span><span class="o">=</span><span class="n">overwrite</span><span class="p">,</span>
+                                  <span class="n">buffersize</span><span class="o">=</span><span class="n">buffersize</span><span class="p">,</span>
+                                  <span class="n">blocksize</span><span class="o">=</span><span class="n">blocksize</span><span class="p">)</span></div></div>
+</pre></div>
+
+           </div>
+           
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    
+    
+      <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+        <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+        <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+        <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+        <script type="text/javascript" src="../../../../_static/language_data.js"></script>
+    
+
+  
+
+  <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.Navigation.enable(true);
+      });
+  </script> 
+
+</body>
+</html>
\ No newline at end of file
diff --git a/1.10.2/_modules/airflow/contrib/hooks/azure_fileshare_hook.html b/1.10.2/_modules/airflow/contrib/hooks/azure_fileshare_hook.html
new file mode 100644
index 0000000..a7696ab
--- /dev/null
+++ b/1.10.2/_modules/airflow/contrib/hooks/azure_fileshare_hook.html
@@ -0,0 +1,431 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>airflow.contrib.hooks.azure_fileshare_hook &mdash; Airflow Documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+    <link rel="index" title="Index" href="../../../../genindex.html" />
+    <link rel="search" title="Search" href="../../../../search.html" /> 
+
+  
+  <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../../index.html" class="icon icon-home"> Airflow
+          
+
+          
+          </a>
+
+          
+            
+            
+              <div class="version">
+                1.10.2
+              </div>
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../metrics.html">Metrics</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../../index.html">Airflow</a>
+        
+      </nav>
+
+
+      <div class="wy-nav-content">
+        
+        <div class="rst-content">
+        
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../../index.html">Module code</a> &raquo;</li>
+        
+      <li>airflow.contrib.hooks.azure_fileshare_hook</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for airflow.contrib.hooks.azure_fileshare_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements.  See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership.  The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># &quot;License&quot;); you may not use this file except in compliance</span>
+<span class="c1"># with the License.  You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1">#   http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># &quot;AS IS&quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied.  See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="c1">#</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">azure.storage.file</span> <span class="k">import</span> <span class="n">FileService</span>
+
+
+<div class="viewcode-block" id="AzureFileShareHook"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_fileshare_hook.AzureFileShareHook">[docs]</a><span class="k">class</span> <span class="nc">AzureFileShareHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Interacts with Azure FileShare Storage.</span>
+
+<span class="sd">    Additional options passed in the &#39;extra&#39; field of the connection will be</span>
+<span class="sd">    passed to the `FileService()` constructor.</span>
+
+<span class="sd">    :param wasb_conn_id: Reference to the wasb connection.</span>
+<span class="sd">    :type wasb_conn_id: str</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">wasb_conn_id</span><span class="o">=</span><span class="s1">&#39;wasb_default&#39;</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span> <span class="o">=</span> <span class="n">wasb_conn_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">connection</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+
+<div class="viewcode-block" id="AzureFileShareHook.get_conn"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_fileshare_hook.AzureFileShareHook.get_conn">[docs]</a>    <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;Return the FileService object.&quot;&quot;&quot;</span>
+        <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+        <span class="n">service_options</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span>
+        <span class="k">return</span> <span class="n">FileService</span><span class="p">(</span><span class="n">account_name</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">,</span>
+                           <span class="n">account_key</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">,</span> <span class="o">**</span><span class="n">service_options</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="AzureFileShareHook.check_for_directory"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_fileshare_hook.AzureFileShareHook.check_for_directory">[docs]</a>    <span class="k">def</span> <span class="nf">check_for_directory</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">share_name</span><span class="p">,</span> <span class="n">directory_name</span><span class="p">,</sp [...]
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Check if a directory exists on Azure File Share.</span>
+
+<span class="sd">        :param share_name: Name of the share.</span>
+<span class="sd">        :type share_name: str</span>
+<span class="sd">        :param directory_name: Name of the directory.</span>
+<span class="sd">        :type directory_name: str</span>
+<span class="sd">        :param kwargs: Optional keyword arguments that</span>
+<span class="sd">            `FileService.exists()` takes.</span>
+<span class="sd">        :type kwargs: object</span>
+<span class="sd">        :return: True if the file exists, False otherwise.</span>
+<span class="sd">        :rtype: bool</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">share_name</span><span class="p">,</span> <span class="n">directory_name</span><span class="p">,</span>
+                                      <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="AzureFileShareHook.check_for_file"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_fileshare_hook.AzureFileShareHook.check_for_file">[docs]</a>    <span class="k">def</span> <span class="nf">check_for_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">share_name</span><span class="p">,</span> <span class="n">directory_name</span><span class="p">,</span> <span class [...]
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Check if a file exists on Azure File Share.</span>
+
+<span class="sd">        :param share_name: Name of the share.</span>
+<span class="sd">        :type share_name: str</span>
+<span class="sd">        :param directory_name: Name of the directory.</span>
+<span class="sd">        :type directory_name: str</span>
+<span class="sd">        :param file_name: Name of the file.</span>
+<span class="sd">        :type file_name: str</span>
+<span class="sd">        :param kwargs: Optional keyword arguments that</span>
+<span class="sd">            `FileService.exists()` takes.</span>
+<span class="sd">        :type kwargs: object</span>
+<span class="sd">        :return: True if the file exists, False otherwise.</span>
+<span class="sd">        :rtype: bool</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">share_name</span><span class="p">,</span> <span class="n">directory_name</span><span class="p">,</span>
+                                      <span class="n">file_name</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="AzureFileShareHook.list_directories_and_files"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_fileshare_hook.AzureFileShareHook.list_directories_and_files">[docs]</a>    <span class="k">def</span> <span class="nf">list_directories_and_files</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">share_name</span><span class="p">,</span> <span class="n">directory_name</span> [...]
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Return the list of directories and files stored on a Azure File Share.</span>
+
+<span class="sd">        :param share_name: Name of the share.</span>
+<span class="sd">        :type share_name: str</span>
+<span class="sd">        :param directory_name: Name of the directory.</span>
+<span class="sd">        :type directory_name: str</span>
+<span class="sd">        :param kwargs: Optional keyword arguments that</span>
+<span class="sd">            `FileService.list_directories_and_files()` takes.</span>
+<span class="sd">        :type kwargs: object</span>
+<span class="sd">        :return: A list of files and directories</span>
+<span class="sd">        :rtype: list</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">list_directories_and_files</span><span class="p">(</span><span class="n">share_name</span><span class="p">,</span>
+                                                          <span class="n">directory_name</span><span class="p">,</span>
+                                                          <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="AzureFileShareHook.create_directory"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_fileshare_hook.AzureFileShareHook.create_directory">[docs]</a>    <span class="k">def</span> <span class="nf">create_directory</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">share_name</span><span class="p">,</span> <span class="n">directory_name</span><span class="p">,</span> <span [...]
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Create a new directory on a Azure File Share.</span>
+
+<span class="sd">        :param share_name: Name of the share.</span>
+<span class="sd">        :type share_name: str</span>
+<span class="sd">        :param directory_name: Name of the directory.</span>
+<span class="sd">        :type directory_name: str</span>
+<span class="sd">        :param kwargs: Optional keyword arguments that</span>
+<span class="sd">            `FileService.create_directory()` takes.</span>
+<span class="sd">        :type kwargs: object</span>
+<span class="sd">        :return: A list of files and directories</span>
+<span class="sd">        :rtype: list</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">create_directory</span><span class="p">(</span><span class="n">share_name</span><span class="p">,</span> <span class="n">directory_name</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="AzureFileShareHook.get_file"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_fileshare_hook.AzureFileShareHook.get_file">[docs]</a>    <span class="k">def</span> <span class="nf">get_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">file_path</span><span class="p">,</span> <span class="n">share_name</span><span class="p">,</span> <span class="n">directory_name</sp [...]
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Download a file from Azure File Share.</span>
+
+<span class="sd">        :param file_path: Where to store the file.</span>
+<span class="sd">        :type file_path: str</span>
+<span class="sd">        :param share_name: Name of the share.</span>
+<span class="sd">        :type share_name: str</span>
+<span class="sd">        :param directory_name: Name of the directory.</span>
+<span class="sd">        :type directory_name: str</span>
+<span class="sd">        :param file_name: Name of the file.</span>
+<span class="sd">        :type file_name: str</span>
+<span class="sd">        :param kwargs: Optional keyword arguments that</span>
+<span class="sd">            `FileService.get_file_to_path()` takes.</span>
+<span class="sd">        :type kwargs: object</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">get_file_to_path</span><span class="p">(</span><span class="n">share_name</span><span class="p">,</span> <span class="n">directory_name</span><span class="p">,</span>
+                                         <span class="n">file_name</span><span class="p">,</span> <span class="n">file_path</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="AzureFileShareHook.get_file_to_stream"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_fileshare_hook.AzureFileShareHook.get_file_to_stream">[docs]</a>    <span class="k">def</span> <span class="nf">get_file_to_stream</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">stream</span><span class="p">,</span> <span class="n">share_name</span><span class="p">,</span> <span c [...]
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Download a file from Azure File Share.</span>
+
+<span class="sd">        :param stream: A filehandle to store the file to.</span>
+<span class="sd">        :type stream: file-like object</span>
+<span class="sd">        :param share_name: Name of the share.</span>
+<span class="sd">        :type share_name: str</span>
+<span class="sd">        :param directory_name: Name of the directory.</span>
+<span class="sd">        :type directory_name: str</span>
+<span class="sd">        :param file_name: Name of the file.</span>
+<span class="sd">        :type file_name: str</span>
+<span class="sd">        :param kwargs: Optional keyword arguments that</span>
+<span class="sd">            `FileService.get_file_to_stream()` takes.</span>
+<span class="sd">        :type kwargs: object</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">get_file_to_stream</span><span class="p">(</span><span class="n">share_name</span><span class="p">,</span> <span class="n">directory_name</span><span class="p">,</span>
+                                           <span class="n">file_name</span><span class="p">,</span> <span class="n">stream</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="AzureFileShareHook.load_file"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_fileshare_hook.AzureFileShareHook.load_file">[docs]</a>    <span class="k">def</span> <span class="nf">load_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">file_path</span><span class="p">,</span> <span class="n">share_name</span><span class="p">,</span> <span class="n">directory_name< [...]
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Upload a file to Azure File Share.</span>
+
+<span class="sd">        :param file_path: Path to the file to load.</span>
+<span class="sd">        :type file_path: str</span>
+<span class="sd">        :param share_name: Name of the share.</span>
+<span class="sd">        :type share_name: str</span>
+<span class="sd">        :param directory_name: Name of the directory.</span>
+<span class="sd">        :type directory_name: str</span>
+<span class="sd">        :param file_name: Name of the file.</span>
+<span class="sd">        :type file_name: str</span>
+<span class="sd">        :param kwargs: Optional keyword arguments that</span>
+<span class="sd">            `FileService.create_file_from_path()` takes.</span>
+<span class="sd">        :type kwargs: object</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">create_file_from_path</span><span class="p">(</span><span class="n">share_name</span><span class="p">,</span> <span class="n">directory_name</span><span class="p">,</span>
+                                              <span class="n">file_name</span><span class="p">,</span> <span class="n">file_path</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="AzureFileShareHook.load_string"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_fileshare_hook.AzureFileShareHook.load_string">[docs]</a>    <span class="k">def</span> <span class="nf">load_string</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">string_data</span><span class="p">,</span> <span class="n">share_name</span><span class="p">,</span> <span class="n">directo [...]
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Upload a string to Azure File Share.</span>
+
+<span class="sd">        :param string_data: String to load.</span>
+<span class="sd">        :type string_data: str</span>
+<span class="sd">        :param share_name: Name of the share.</span>
+<span class="sd">        :type share_name: str</span>
+<span class="sd">        :param directory_name: Name of the directory.</span>
+<span class="sd">        :type directory_name: str</span>
+<span class="sd">        :param file_name: Name of the file.</span>
+<span class="sd">        :type file_name: str</span>
+<span class="sd">        :param kwargs: Optional keyword arguments that</span>
+<span class="sd">            `FileService.create_file_from_text()` takes.</span>
+<span class="sd">        :type kwargs: object</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">create_file_from_text</span><span class="p">(</span><span class="n">share_name</span><span class="p">,</span> <span class="n">directory_name</span><span class="p">,</span>
+                                              <span class="n">file_name</span><span class="p">,</span> <span class="n">string_data</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="AzureFileShareHook.load_stream"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.azure_fileshare_hook.AzureFileShareHook.load_stream">[docs]</a>    <span class="k">def</span> <span class="nf">load_stream</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">stream</span><span class="p">,</span> <span class="n">share_name</span><span class="p">,</span> <span class="n">directory_na [...]
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Upload a stream to Azure File Share.</span>
+
+<span class="sd">        :param stream: Opened file/stream to upload as the file content.</span>
+<span class="sd">        :type stream: file-like</span>
+<span class="sd">        :param share_name: Name of the share.</span>
+<span class="sd">        :type share_name: str</span>
+<span class="sd">        :param directory_name: Name of the directory.</span>
+<span class="sd">        :type directory_name: str</span>
+<span class="sd">        :param file_name: Name of the file.</span>
+<span class="sd">        :type file_name: str</span>
+<span class="sd">        :param count: Size of the stream in bytes</span>
+<span class="sd">        :type count: int</span>
+<span class="sd">        :param kwargs: Optional keyword arguments that</span>
+<span class="sd">            `FileService.create_file_from_stream()` takes.</span>
+<span class="sd">        :type kwargs: object</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">create_file_from_stream</span><span class="p">(</span><span class="n">share_name</span><span class="p">,</span> <span class="n">directory_name</span><span class="p">,</span>
+                                                <span class="n">file_name</span><span class="p">,</span> <span class="n">stream</span><span class="p">,</span> <span class="n">count</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></div></div>
+</pre></div>
+
+           </div>
+           
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    
+    
+      <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+        <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+        <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+        <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+        <script type="text/javascript" src="../../../../_static/language_data.js"></script>
+    
+
+  
+
+  <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.Navigation.enable(true);
+      });
+  </script> 
+
+</body>
+</html>
\ No newline at end of file
diff --git a/1.10.2/_modules/airflow/contrib/hooks/bigquery_hook.html b/1.10.2/_modules/airflow/contrib/hooks/bigquery_hook.html
new file mode 100644
index 0000000..162344f
--- /dev/null
+++ b/1.10.2/_modules/airflow/contrib/hooks/bigquery_hook.html
@@ -0,0 +1,2184 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>airflow.contrib.hooks.bigquery_hook &mdash; Airflow Documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+    <link rel="index" title="Index" href="../../../../genindex.html" />
+    <link rel="search" title="Search" href="../../../../search.html" /> 
+
+  
+  <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../../index.html" class="icon icon-home"> Airflow
+          
+
+          
+          </a>
+
+          
+            
+            
+              <div class="version">
+                1.10.2
+              </div>
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../metrics.html">Metrics</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../../index.html">Airflow</a>
+        
+      </nav>
+
+
+      <div class="wy-nav-content">
+        
+        <div class="rst-content">
+        
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../../index.html">Module code</a> &raquo;</li>
+        
+      <li>airflow.contrib.hooks.bigquery_hook</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for airflow.contrib.hooks.bigquery_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements.  See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership.  The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># &quot;License&quot;); you may not use this file except in compliance</span>
+<span class="c1"># with the License.  You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1">#   http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># &quot;AS IS&quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied.  See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="c1">#</span>
+<span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">This module contains a BigQuery Hook, as well as a very basic PEP 249</span>
+<span class="sd">implementation for BigQuery.</span>
+<span class="sd">&quot;&quot;&quot;</span>
+
+<span class="kn">import</span> <span class="nn">time</span>
+<span class="kn">import</span> <span class="nn">six</span>
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">range</span>
+<span class="kn">from</span> <span class="nn">copy</span> <span class="k">import</span> <span class="n">deepcopy</span>
+<span class="kn">from</span> <span class="nn">six</span> <span class="k">import</span> <span class="n">iteritems</span>
+
+<span class="kn">from</span> <span class="nn">past.builtins</span> <span class="k">import</span> <span class="n">basestring</span>
+
+<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcp_api_base_hook</span> <span class="k">import</span> <span class="n">GoogleCloudBaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.dbapi_hook</span> <span class="k">import</span> <span class="n">DbApiHook</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
+<span class="kn">from</span> <span class="nn">googleapiclient.discovery</span> <span class="k">import</span> <span class="n">build</span>
+<span class="kn">from</span> <span class="nn">googleapiclient.errors</span> <span class="k">import</span> <span class="n">HttpError</span>
+<span class="kn">from</span> <span class="nn">pandas_gbq.gbq</span> <span class="k">import</span> \
+    <span class="n">_check_google_client_version</span> <span class="k">as</span> <span class="n">gbq_check_google_client_version</span>
+<span class="kn">from</span> <span class="nn">pandas_gbq</span> <span class="k">import</span> <span class="n">read_gbq</span>
+<span class="kn">from</span> <span class="nn">pandas_gbq.gbq</span> <span class="k">import</span> \
+    <span class="n">_test_google_api_imports</span> <span class="k">as</span> <span class="n">gbq_test_google_api_imports</span>
+<span class="kn">from</span> <span class="nn">pandas_gbq.gbq</span> <span class="k">import</span> <span class="n">GbqConnector</span>
+
+
+<div class="viewcode-block" id="BigQueryHook"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.bigquery_hook.BigQueryHook">[docs]</a><span class="k">class</span> <span class="nc">BigQueryHook</span><span class="p">(</span><span class="n">GoogleCloudBaseHook</span><span class="p">,</span> <span class="n">DbApiHook</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Interact with BigQuery. This hook uses the Google Cloud Platform</span>
+<span class="sd">    connection.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">conn_name_attr</span> <span class="o">=</span> <span class="s1">&#39;bigquery_conn_id&#39;</span>
+
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+                 <span class="n">bigquery_conn_id</span><span class="o">=</span><span class="s1">&#39;bigquery_default&#39;</span><span class="p">,</span>
+                 <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                 <span class="n">use_legacy_sql</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
+                 <span class="n">location</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">BigQueryHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
+            <span class="n">gcp_conn_id</span><span class="o">=</span><span class="n">bigquery_conn_id</span><span class="p">,</span> <span class="n">delegate_to</span><span class="o">=</span><span class="n">delegate_to</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">use_legacy_sql</span> <span class="o">=</span> <span class="n">use_legacy_sql</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">location</span> <span class="o">=</span> <span class="n">location</span>
+
+<div class="viewcode-block" id="BigQueryHook.get_conn"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.bigquery_hook.BigQueryHook.get_conn">[docs]</a>    <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Returns a BigQuery PEP 249 connection object.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">service</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_service</span><span class="p">()</span>
+        <span class="n">project</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_field</span><span class="p">(</span><span class="s1">&#39;project&#39;</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">BigQueryConnection</span><span class="p">(</span>
+            <span class="n">service</span><span class="o">=</span><span class="n">service</span><span class="p">,</span>
+            <span class="n">project_id</span><span class="o">=</span><span class="n">project</span><span class="p">,</span>
+            <span class="n">use_legacy_sql</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">use_legacy_sql</span><span class="p">,</span>
+            <span class="n">location</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">location</span><span class="p">,</span>
+        <span class="p">)</span></div>
+
+<div class="viewcode-block" id="BigQueryHook.get_service"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.bigquery_hook.BigQueryHook.get_service">[docs]</a>    <span class="k">def</span> <span class="nf">get_service</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Returns a BigQuery service object.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">http_authorized</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_authorize</span><span class="p">()</span>
+        <span class="k">return</span> <span class="n">build</span><span class="p">(</span>
+            <span class="s1">&#39;bigquery&#39;</span><span class="p">,</span> <span class="s1">&#39;v2&#39;</span><span class="p">,</span> <span class="n">http</span><span class="o">=</span><span class="n">http_authorized</span><span class="p">,</span> <span class="n">cache_discovery</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="BigQueryHook.insert_rows"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.bigquery_hook.BigQueryHook.insert_rows">[docs]</a>    <span class="k">def</span> <span class="nf">insert_rows</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">rows</span><span class="p">,</span> <span class="n">target_fields</span><span class="o">=< [...]
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Insertion is currently unsupported. Theoretically, you could use</span>
+<span class="sd">        BigQuery&#39;s streaming API to insert rows into a table, but this hasn&#39;t</span>
+<span class="sd">        been implemented.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">()</span></div>
+
+<div class="viewcode-block" id="BigQueryHook.get_pandas_df"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.bigquery_hook.BigQueryHook.get_pandas_df">[docs]</a>    <span class="k">def</span> <span class="nf">get_pandas_df</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="kc">None</span><span class="p">, [...]
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Returns a Pandas DataFrame for the results produced by a BigQuery</span>
+<span class="sd">        query. The DbApiHook method must be overridden because Pandas</span>
+<span class="sd">        doesn&#39;t support PEP 249 connections, except for SQLite. See:</span>
+
+<span class="sd">        https://github.com/pydata/pandas/blob/master/pandas/io/sql.py#L447</span>
+<span class="sd">        https://github.com/pydata/pandas/issues/6900</span>
+
+<span class="sd">        :param sql: The BigQuery SQL to execute.</span>
+<span class="sd">        :type sql: string</span>
+<span class="sd">        :param parameters: The parameters to render the SQL query with (not</span>
+<span class="sd">            used, leave to override superclass method)</span>
+<span class="sd">        :type parameters: mapping or iterable</span>
+<span class="sd">        :param dialect: Dialect of BigQuery SQL – legacy SQL or standard SQL</span>
+<span class="sd">            defaults to use `self.use_legacy_sql` if not specified</span>
+<span class="sd">        :type dialect: string in {&#39;legacy&#39;, &#39;standard&#39;}</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">if</span> <span class="n">dialect</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">dialect</span> <span class="o">=</span> <span class="s1">&#39;legacy&#39;</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">use_legacy_sql</span> <span class="k">else</span> <span class="s1">&#39;standard&#39;</span>
+
+        <span class="k">return</span> <span class="n">read_gbq</span><span class="p">(</span><span class="n">sql</span><span class="p">,</span>
+                        <span class="n">project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_get_field</span><span class="p">(</span><span class="s1">&#39;project&#39;</span><span class="p">),</span>
+                        <span class="n">dialect</span><span class="o">=</span><span class="n">dialect</span><span class="p">,</span>
+                        <span class="n">verbose</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="BigQueryHook.table_exists"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.bigquery_hook.BigQueryHook.table_exists">[docs]</a>    <span class="k">def</span> <span class="nf">table_exists</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span cla [...]
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Checks for the existence of a table in Google BigQuery.</span>
+
+<span class="sd">        :param project_id: The Google cloud project in which to look for the</span>
+<span class="sd">            table. The connection supplied to the hook must provide access to</span>
+<span class="sd">            the specified project.</span>
+<span class="sd">        :type project_id: string</span>
+<span class="sd">        :param dataset_id: The name of the dataset in which to look for the</span>
+<span class="sd">            table.</span>
+<span class="sd">        :type dataset_id: string</span>
+<span class="sd">        :param table_id: The name of the table to check the existence of.</span>
+<span class="sd">        :type table_id: string</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">service</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_service</span><span class="p">()</span>
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+                <span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span> <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span>
+                <span class="n">tableId</span><span class="o">=</span><span class="n">table_id</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+            <span class="k">return</span> <span class="kc">True</span>
+        <span class="k">except</span> <span class="n">HttpError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+            <span class="k">if</span> <span class="n">e</span><span class="o">.</span><span class="n">resp</span><span class="p">[</span><span class="s1">&#39;status&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="s1">&#39;404&#39;</span><span class="p">:</span>
+                <span class="k">return</span> <span class="kc">False</span>
+            <span class="k">raise</span></div></div>
+
+
+<span class="k">class</span> <span class="nc">BigQueryPandasConnector</span><span class="p">(</span><span class="n">GbqConnector</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    This connector behaves identically to GbqConnector (from Pandas), except</span>
+<span class="sd">    that it allows the service to be injected, and disables a call to</span>
+<span class="sd">    self.get_credentials(). This allows Airflow to use BigQuery with Pandas</span>
+<span class="sd">    without forcing a three legged OAuth connection. Instead, we can inject</span>
+<span class="sd">    service account credentials into the binding.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+                 <span class="n">project_id</span><span class="p">,</span>
+                 <span class="n">service</span><span class="p">,</span>
+                 <span class="n">reauth</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+                 <span class="n">verbose</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+                 <span class="n">dialect</span><span class="o">=</span><span class="s1">&#39;legacy&#39;</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">BigQueryPandasConnector</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">project_id</span><span class="p">)</span>
+        <span class="n">gbq_check_google_client_version</span><span class="p">()</span>
+        <span class="n">gbq_test_google_api_imports</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span> <span class="o">=</span> <span class="n">project_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">reauth</span> <span class="o">=</span> <span class="n">reauth</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">service</span> <span class="o">=</span> <span class="n">service</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">=</span> <span class="n">verbose</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">dialect</span> <span class="o">=</span> <span class="n">dialect</span>
+
+
+<span class="k">class</span> <span class="nc">BigQueryConnection</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    BigQuery does not have a notion of a persistent connection. Thus, these</span>
+<span class="sd">    objects are small stateless factories for cursors, which do all the real</span>
+<span class="sd">    work.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_args</span> <span class="o">=</span> <span class="n">args</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_kwargs</span> <span class="o">=</span> <span class="n">kwargs</span>
+
+    <span class="k">def</span> <span class="nf">close</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot; BigQueryConnection does not have anything to close. &quot;&quot;&quot;</span>
+        <span class="k">pass</span>
+
+    <span class="k">def</span> <span class="nf">commit</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot; BigQueryConnection does not support transactions. &quot;&quot;&quot;</span>
+        <span class="k">pass</span>
+
+    <span class="k">def</span> <span class="nf">cursor</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot; Return a new :py:class:`Cursor` object using the connection. &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="n">BigQueryCursor</span><span class="p">(</span><span class="o">*</span><span class="bp">self</span><span class="o">.</span><span class="n">_args</span><span class="p">,</span> <span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">_kwargs</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">rollback</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span>
+            <span class="s2">&quot;BigQueryConnection does not have transactions&quot;</span><span class="p">)</span>
+
+
+<span class="k">class</span> <span class="nc">BigQueryBaseCursor</span><span class="p">(</span><span class="n">LoggingMixin</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    The BigQuery base cursor contains helper methods to execute queries against</span>
+<span class="sd">    BigQuery. The methods can be used directly by operators, in cases where a</span>
+<span class="sd">    PEP 249 cursor isn&#39;t needed.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+                 <span class="n">service</span><span class="p">,</span>
+                 <span class="n">project_id</span><span class="p">,</span>
+                 <span class="n">use_legacy_sql</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
+                 <span class="n">api_resource_configs</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                 <span class="n">location</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">service</span> <span class="o">=</span> <span class="n">service</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span> <span class="o">=</span> <span class="n">project_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">use_legacy_sql</span> <span class="o">=</span> <span class="n">use_legacy_sql</span>
+        <span class="k">if</span> <span class="n">api_resource_configs</span><span class="p">:</span>
+            <span class="n">_validate_value</span><span class="p">(</span><span class="s2">&quot;api_resource_configs&quot;</span><span class="p">,</span> <span class="n">api_resource_configs</span><span class="p">,</span> <span class="nb">dict</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">api_resource_configs</span> <span class="o">=</span> <span class="n">api_resource_configs</span> \
+            <span class="k">if</span> <span class="n">api_resource_configs</span> <span class="k">else</span> <span class="p">{}</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">running_job_id</span> <span class="o">=</span> <span class="kc">None</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">location</span> <span class="o">=</span> <span class="n">location</span>
+
+    <span class="k">def</span> <span class="nf">create_empty_table</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+                           <span class="n">project_id</span><span class="p">,</span>
+                           <span class="n">dataset_id</span><span class="p">,</span>
+                           <span class="n">table_id</span><span class="p">,</span>
+                           <span class="n">schema_fields</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                           <span class="n">time_partitioning</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                           <span class="n">labels</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                           <span class="n">view</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Creates a new, empty table in the dataset.</span>
+<span class="sd">        To create a view, which is defined by a SQL query, parse a dictionary to &#39;view&#39; kwarg</span>
+
+<span class="sd">        :param project_id: The project to create the table into.</span>
+<span class="sd">        :type project_id: str</span>
+<span class="sd">        :param dataset_id: The dataset to create the table into.</span>
+<span class="sd">        :type dataset_id: str</span>
+<span class="sd">        :param table_id: The Name of the table to be created.</span>
+<span class="sd">        :type table_id: str</span>
+<span class="sd">        :param schema_fields: If set, the schema field list as defined here:</span>
+<span class="sd">            https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.load.schema</span>
+<span class="sd">        :type schema_fields: list</span>
+<span class="sd">        :param labels: a dictionary containing labels for the table, passed to BigQuery</span>
+<span class="sd">        :type labels: dict</span>
+
+<span class="sd">        **Example**: ::</span>
+
+<span class="sd">            schema_fields=[{&quot;name&quot;: &quot;emp_name&quot;, &quot;type&quot;: &quot;STRING&quot;, &quot;mode&quot;: &quot;REQUIRED&quot;},</span>
+<span class="sd">                           {&quot;name&quot;: &quot;salary&quot;, &quot;type&quot;: &quot;INTEGER&quot;, &quot;mode&quot;: &quot;NULLABLE&quot;}]</span>
+
+<span class="sd">        :param time_partitioning: configure optional time partitioning fields i.e.</span>
+<span class="sd">            partition by field, type and expiration as per API specifications.</span>
+
+<span class="sd">            .. seealso::</span>
+<span class="sd">            https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#timePartitioning</span>
+<span class="sd">        :type time_partitioning: dict</span>
+<span class="sd">        :param view: [Optional] A dictionary containing definition for the view.</span>
+<span class="sd">            If set, it will create a view instead of a table:</span>
+<span class="sd">            https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#view</span>
+<span class="sd">        :type view: dict</span>
+
+<span class="sd">        **Example**: ::</span>
+
+<span class="sd">            view = {</span>
+<span class="sd">                &quot;query&quot;: &quot;SELECT * FROM `test-project-id.test_dataset_id.test_table_prefix*` LIMIT 1000&quot;,</span>
+<span class="sd">                &quot;useLegacySql&quot;: False</span>
+<span class="sd">            }</span>
+
+<span class="sd">        :return:</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+
+        <span class="n">project_id</span> <span class="o">=</span> <span class="n">project_id</span> <span class="k">if</span> <span class="n">project_id</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span>
+
+        <span class="n">table_resource</span> <span class="o">=</span> <span class="p">{</span>
+            <span class="s1">&#39;tableReference&#39;</span><span class="p">:</span> <span class="p">{</span>
+                <span class="s1">&#39;tableId&#39;</span><span class="p">:</span> <span class="n">table_id</span>
+            <span class="p">}</span>
+        <span class="p">}</span>
+
+        <span class="k">if</span> <span class="n">schema_fields</span><span class="p">:</span>
+            <span class="n">table_resource</span><span class="p">[</span><span class="s1">&#39;schema&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;fields&#39;</span><span class="p">:</span> <span class="n">schema_fields</span><span class="p">}</span>
+
+        <span class="k">if</span> <span class="n">time_partitioning</span><span class="p">:</span>
+            <span class="n">table_resource</span><span class="p">[</span><span class="s1">&#39;timePartitioning&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">time_partitioning</span>
+
+        <span class="k">if</span> <span class="n">labels</span><span class="p">:</span>
+            <span class="n">table_resource</span><span class="p">[</span><span class="s1">&#39;labels&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">labels</span>
+
+        <span class="k">if</span> <span class="n">view</span><span class="p">:</span>
+            <span class="n">table_resource</span><span class="p">[</span><span class="s1">&#39;view&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">view</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Creating Table </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span>
+                      <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">)</span>
+
+        <span class="k">try</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span><span class="o">.</span><span class="n">insert</span><span class="p">(</span>
+                <span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
+                <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span>
+                <span class="n">body</span><span class="o">=</span><span class="n">table_resource</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Table created successfully: </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span>
+                          <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">)</span>
+
+        <span class="k">except</span> <span class="n">HttpError</span> <span class="k">as</span> <span class="n">err</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+                <span class="s1">&#39;BigQuery job failed. Error was: </span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">err</span><span class="o">.</span><span class="n">content</span><span class="p">)</span>
+            <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">create_external_table</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+                              <span class="n">external_project_dataset_table</span><span class="p">,</span>
+                              <span class="n">schema_fields</span><span class="p">,</span>
+                              <span class="n">source_uris</span><span class="p">,</span>
+                              <span class="n">source_format</span><span class="o">=</span><span class="s1">&#39;CSV&#39;</span><span class="p">,</span>
+                              <span class="n">autodetect</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+                              <span class="n">compression</span><span class="o">=</span><span class="s1">&#39;NONE&#39;</span><span class="p">,</span>
+                              <span class="n">ignore_unknown_values</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+                              <span class="n">max_bad_records</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
+                              <span class="n">skip_leading_rows</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
+                              <span class="n">field_delimiter</span><span class="o">=</span><span class="s1">&#39;,&#39;</span><span class="p">,</span>
+                              <span class="n">quote_character</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                              <span class="n">allow_quoted_newlines</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+                              <span class="n">allow_jagged_rows</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+                              <span class="n">src_fmt_configs</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                              <span class="n">labels</span><span class="o">=</span><span class="kc">None</span>
+                              <span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Creates a new external table in the dataset with the data in Google</span>
+<span class="sd">        Cloud Storage. See here:</span>
+
+<span class="sd">        https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#resource</span>
+
+<span class="sd">        for more details about these parameters.</span>
+
+<span class="sd">        :param external_project_dataset_table:</span>
+<span class="sd">            The dotted (&lt;project&gt;.|&lt;project&gt;:)&lt;dataset&gt;.&lt;table&gt;($&lt;partition&gt;) BigQuery</span>
+<span class="sd">            table name to create external table.</span>
+<span class="sd">            If &lt;project&gt; is not included, project will be the</span>
+<span class="sd">            project defined in the connection json.</span>
+<span class="sd">        :type external_project_dataset_table: string</span>
+<span class="sd">        :param schema_fields: The schema field list as defined here:</span>
+<span class="sd">            https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#resource</span>
+<span class="sd">        :type schema_fields: list</span>
+<span class="sd">        :param source_uris: The source Google Cloud</span>
+<span class="sd">            Storage URI (e.g. gs://some-bucket/some-file.txt). A single wild</span>
+<span class="sd">            per-object name can be used.</span>
+<span class="sd">        :type source_uris: list</span>
+<span class="sd">        :param source_format: File format to export.</span>
+<span class="sd">        :type source_format: string</span>
+<span class="sd">        :param autodetect: Try to detect schema and format options automatically.</span>
+<span class="sd">            Any option specified explicitly will be honored.</span>
+<span class="sd">        :type autodetect: bool</span>
+<span class="sd">        :param compression: [Optional] The compression type of the data source.</span>
+<span class="sd">            Possible values include GZIP and NONE.</span>
+<span class="sd">            The default value is NONE.</span>
+<span class="sd">            This setting is ignored for Google Cloud Bigtable,</span>
+<span class="sd">                Google Cloud Datastore backups and Avro formats.</span>
+<span class="sd">        :type compression: string</span>
+<span class="sd">        :param ignore_unknown_values: [Optional] Indicates if BigQuery should allow</span>
+<span class="sd">            extra values that are not represented in the table schema.</span>
+<span class="sd">            If true, the extra values are ignored. If false, records with extra columns</span>
+<span class="sd">            are treated as bad records, and if there are too many bad records, an</span>
+<span class="sd">            invalid error is returned in the job result.</span>
+<span class="sd">        :type ignore_unknown_values: bool</span>
+<span class="sd">        :param max_bad_records: The maximum number of bad records that BigQuery can</span>
+<span class="sd">            ignore when running the job.</span>
+<span class="sd">        :type max_bad_records: int</span>
+<span class="sd">        :param skip_leading_rows: Number of rows to skip when loading from a CSV.</span>
+<span class="sd">        :type skip_leading_rows: int</span>
+<span class="sd">        :param field_delimiter: The delimiter to use when loading from a CSV.</span>
+<span class="sd">        :type field_delimiter: string</span>
+<span class="sd">        :param quote_character: The value that is used to quote data sections in a CSV</span>
+<span class="sd">            file.</span>
+<span class="sd">        :type quote_character: string</span>
+<span class="sd">        :param allow_quoted_newlines: Whether to allow quoted newlines (true) or not</span>
+<span class="sd">            (false).</span>
+<span class="sd">        :type allow_quoted_newlines: boolean</span>
+<span class="sd">        :param allow_jagged_rows: Accept rows that are missing trailing optional columns.</span>
+<span class="sd">            The missing values are treated as nulls. If false, records with missing</span>
+<span class="sd">            trailing columns are treated as bad records, and if there are too many bad</span>
+<span class="sd">            records, an invalid error is returned in the job result. Only applicable when</span>
+<span class="sd">            soure_format is CSV.</span>
+<span class="sd">        :type allow_jagged_rows: bool</span>
+<span class="sd">        :param src_fmt_configs: configure optional fields specific to the source format</span>
+<span class="sd">        :type src_fmt_configs: dict</span>
+<span class="sd">        :param labels: a dictionary containing labels for the table, passed to BigQuery</span>
+<span class="sd">        :type labels: dict</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+
+        <span class="k">if</span> <span class="n">src_fmt_configs</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">src_fmt_configs</span> <span class="o">=</span> <span class="p">{}</span>
+        <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">external_table_id</span> <span class="o">=</span> \
+            <span class="n">_split_tablename</span><span class="p">(</span><span class="n">table_input</span><span class="o">=</span><span class="n">external_project_dataset_table</span><span class="p">,</span>
+                             <span class="n">default_project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+                             <span class="n">var_name</span><span class="o">=</span><span class="s1">&#39;external_project_dataset_table&#39;</span><span class="p">)</span>
+
+        <span class="c1"># bigquery only allows certain source formats</span>
+        <span class="c1"># we check to make sure the passed source format is valid</span>
+        <span class="c1"># if it&#39;s not, we raise a ValueError</span>
+        <span class="c1"># Refer to this link for more details:</span>
+        <span class="c1">#   https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#externalDataConfiguration.sourceFormat</span>
+
+        <span class="n">source_format</span> <span class="o">=</span> <span class="n">source_format</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span>
+        <span class="n">allowed_formats</span> <span class="o">=</span> <span class="p">[</span>
+            <span class="s2">&quot;CSV&quot;</span><span class="p">,</span> <span class="s2">&quot;NEWLINE_DELIMITED_JSON&quot;</span><span class="p">,</span> <span class="s2">&quot;AVRO&quot;</span><span class="p">,</span> <span class="s2">&quot;GOOGLE_SHEETS&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;DATASTORE_BACKUP&quot;</span><span class="p">,</span> <span class="s2">&quot;PARQUET&quot;</span>
+        <span class="p">]</span>
+        <span class="k">if</span> <span class="n">source_format</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">allowed_formats</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;</span><span class="si">{0}</span><span class="s2"> is not a valid source format. &quot;</span>
+                             <span class="s2">&quot;Please use one of the following types: </span><span class="si">{1}</span><span class="s2">&quot;</span>
+                             <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">source_format</span><span class="p">,</span> <span class="n">allowed_formats</span><span class="p">))</span>
+
+        <span class="n">compression</span> <span class="o">=</span> <span class="n">compression</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span>
+        <span class="n">allowed_compressions</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;NONE&#39;</span><span class="p">,</span> <span class="s1">&#39;GZIP&#39;</span><span class="p">]</span>
+        <span class="k">if</span> <span class="n">compression</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">allowed_compressions</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;</span><span class="si">{0}</span><span class="s2"> is not a valid compression format. &quot;</span>
+                             <span class="s2">&quot;Please use one of the following types: </span><span class="si">{1}</span><span class="s2">&quot;</span>
+                             <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">compression</span><span class="p">,</span> <span class="n">allowed_compressions</span><span class="p">))</span>
+
+        <span class="n">table_resource</span> <span class="o">=</span> <span class="p">{</span>
+            <span class="s1">&#39;externalDataConfiguration&#39;</span><span class="p">:</span> <span class="p">{</span>
+                <span class="s1">&#39;autodetect&#39;</span><span class="p">:</span> <span class="n">autodetect</span><span class="p">,</span>
+                <span class="s1">&#39;sourceFormat&#39;</span><span class="p">:</span> <span class="n">source_format</span><span class="p">,</span>
+                <span class="s1">&#39;sourceUris&#39;</span><span class="p">:</span> <span class="n">source_uris</span><span class="p">,</span>
+                <span class="s1">&#39;compression&#39;</span><span class="p">:</span> <span class="n">compression</span><span class="p">,</span>
+                <span class="s1">&#39;ignoreUnknownValues&#39;</span><span class="p">:</span> <span class="n">ignore_unknown_values</span>
+            <span class="p">},</span>
+            <span class="s1">&#39;tableReference&#39;</span><span class="p">:</span> <span class="p">{</span>
+                <span class="s1">&#39;projectId&#39;</span><span class="p">:</span> <span class="n">project_id</span><span class="p">,</span>
+                <span class="s1">&#39;datasetId&#39;</span><span class="p">:</span> <span class="n">dataset_id</span><span class="p">,</span>
+                <span class="s1">&#39;tableId&#39;</span><span class="p">:</span> <span class="n">external_table_id</span><span class="p">,</span>
+            <span class="p">}</span>
+        <span class="p">}</span>
+
+        <span class="k">if</span> <span class="n">schema_fields</span><span class="p">:</span>
+            <span class="n">table_resource</span><span class="p">[</span><span class="s1">&#39;externalDataConfiguration&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">({</span>
+                <span class="s1">&#39;schema&#39;</span><span class="p">:</span> <span class="p">{</span>
+                    <span class="s1">&#39;fields&#39;</span><span class="p">:</span> <span class="n">schema_fields</span>
+                <span class="p">}</span>
+            <span class="p">})</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Creating external table: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="n">external_project_dataset_table</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="n">max_bad_records</span><span class="p">:</span>
+            <span class="n">table_resource</span><span class="p">[</span><span class="s1">&#39;externalDataConfiguration&#39;</span><span class="p">][</span><span class="s1">&#39;maxBadRecords&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">max_bad_records</span>
+
+        <span class="c1"># if following fields are not specified in src_fmt_configs,</span>
+        <span class="c1"># honor the top-level params for backward-compatibility</span>
+        <span class="k">if</span> <span class="s1">&#39;skipLeadingRows&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">src_fmt_configs</span><span class="p">:</span>
+            <span class="n">src_fmt_configs</span><span class="p">[</span><span class="s1">&#39;skipLeadingRows&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">skip_leading_rows</span>
+        <span class="k">if</span> <span class="s1">&#39;fieldDelimiter&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">src_fmt_configs</span><span class="p">:</span>
+            <span class="n">src_fmt_configs</span><span class="p">[</span><span class="s1">&#39;fieldDelimiter&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">field_delimiter</span>
+        <span class="k">if</span> <span class="s1">&#39;quote_character&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">src_fmt_configs</span><span class="p">:</span>
+            <span class="n">src_fmt_configs</span><span class="p">[</span><span class="s1">&#39;quote&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">quote_character</span>
+        <span class="k">if</span> <span class="s1">&#39;allowQuotedNewlines&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">src_fmt_configs</span><span class="p">:</span>
+            <span class="n">src_fmt_configs</span><span class="p">[</span><span class="s1">&#39;allowQuotedNewlines&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">allow_quoted_newlines</span>
+        <span class="k">if</span> <span class="s1">&#39;allowJaggedRows&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">src_fmt_configs</span><span class="p">:</span>
+            <span class="n">src_fmt_configs</span><span class="p">[</span><span class="s1">&#39;allowJaggedRows&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">allow_jagged_rows</span>
+
+        <span class="n">src_fmt_to_param_mapping</span> <span class="o">=</span> <span class="p">{</span>
+            <span class="s1">&#39;CSV&#39;</span><span class="p">:</span> <span class="s1">&#39;csvOptions&#39;</span><span class="p">,</span>
+            <span class="s1">&#39;GOOGLE_SHEETS&#39;</span><span class="p">:</span> <span class="s1">&#39;googleSheetsOptions&#39;</span>
+        <span class="p">}</span>
+
+        <span class="n">src_fmt_to_configs_mapping</span> <span class="o">=</span> <span class="p">{</span>
+            <span class="s1">&#39;csvOptions&#39;</span><span class="p">:</span> <span class="p">[</span>
+                <span class="s1">&#39;allowJaggedRows&#39;</span><span class="p">,</span> <span class="s1">&#39;allowQuotedNewlines&#39;</span><span class="p">,</span>
+                <span class="s1">&#39;fieldDelimiter&#39;</span><span class="p">,</span> <span class="s1">&#39;skipLeadingRows&#39;</span><span class="p">,</span>
+                <span class="s1">&#39;quote&#39;</span>
+            <span class="p">],</span>
+            <span class="s1">&#39;googleSheetsOptions&#39;</span><span class="p">:</span> <span class="p">[</span><span class="s1">&#39;skipLeadingRows&#39;</span><span class="p">]</span>
+        <span class="p">}</span>
+
+        <span class="k">if</span> <span class="n">source_format</span> <span class="ow">in</span> <span class="n">src_fmt_to_param_mapping</span><span class="o">.</span><span class="n">keys</span><span class="p">():</span>
+
+            <span class="n">valid_configs</span> <span class="o">=</span> <span class="n">src_fmt_to_configs_mapping</span><span class="p">[</span>
+                <span class="n">src_fmt_to_param_mapping</span><span class="p">[</span><span class="n">source_format</span><span class="p">]</span>
+            <span class="p">]</span>
+
+            <span class="n">src_fmt_configs</span> <span class="o">=</span> <span class="p">{</span>
+                <span class="n">k</span><span class="p">:</span> <span class="n">v</span>
+                <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">src_fmt_configs</span><span class="o">.</span><span class="n">items</span><span class="p">()</span> <span class="k">if</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">valid_configs</span>
+            <span class="p">}</span>
+
+            <span class="n">table_resource</span><span class="p">[</span><span class="s1">&#39;externalDataConfiguration&#39;</span><span class="p">][</span><span class="n">src_fmt_to_param_mapping</span><span class="p">[</span>
+                <span class="n">source_format</span><span class="p">]]</span> <span class="o">=</span> <span class="n">src_fmt_configs</span>
+
+        <span class="k">if</span> <span class="n">labels</span><span class="p">:</span>
+            <span class="n">table_resource</span><span class="p">[</span><span class="s1">&#39;labels&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">labels</span>
+
+        <span class="k">try</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span><span class="o">.</span><span class="n">insert</span><span class="p">(</span>
+                <span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
+                <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span>
+                <span class="n">body</span><span class="o">=</span><span class="n">table_resource</span>
+            <span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;External table created successfully: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span>
+                          <span class="n">external_project_dataset_table</span><span class="p">)</span>
+
+        <span class="k">except</span> <span class="n">HttpError</span> <span class="k">as</span> <span class="n">err</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span>
+                <span class="s1">&#39;BigQuery job failed. Error was: </span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">err</span><span class="o">.</span><span class="n">content</span><span class="p">)</span>
+            <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">run_query</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+                  <span class="n">bql</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                  <span class="n">sql</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                  <span class="n">destination_dataset_table</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                  <span class="n">write_disposition</span><span class="o">=</span><span class="s1">&#39;WRITE_EMPTY&#39;</span><span class="p">,</span>
+                  <span class="n">allow_large_results</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+                  <span class="n">flatten_results</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                  <span class="n">udf_config</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                  <span class="n">use_legacy_sql</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                  <span class="n">maximum_billing_tier</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                  <span class="n">maximum_bytes_billed</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                  <span class="n">create_disposition</span><span class="o">=</span><span class="s1">&#39;CREATE_IF_NEEDED&#39;</span><span class="p">,</span>
+                  <span class="n">query_params</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                  <span class="n">labels</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                  <span class="n">schema_update_options</span><span class="o">=</span><span class="p">(),</span>
+                  <span class="n">priority</span><span class="o">=</span><span class="s1">&#39;INTERACTIVE&#39;</span><span class="p">,</span>
+                  <span class="n">time_partitioning</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                  <span class="n">api_resource_configs</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                  <span class="n">cluster_fields</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                  <span class="n">location</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Executes a BigQuery SQL query. Optionally persists results in a BigQuery</span>
+<span class="sd">        table. See here:</span>
+
+<span class="sd">        https://cloud.google.com/bigquery/docs/reference/v2/jobs</span>
+
+<span class="sd">        For more details about these parameters.</span>
+
+<span class="sd">        :param bql: (Deprecated. Use `sql` parameter instead) The BigQuery SQL</span>
+<span class="sd">            to execute.</span>
+<span class="sd">        :type bql: string</span>
+<span class="sd">        :param sql: The BigQuery SQL to execute.</span>
+<span class="sd">        :type sql: string</span>
+<span class="sd">        :param destination_dataset_table: The dotted &lt;dataset&gt;.&lt;table&gt;</span>
+<span class="sd">            BigQuery table to save the query results.</span>
+<span class="sd">        :type destination_dataset_table: string</span>
+<span class="sd">        :param write_disposition: What to do if the table already exists in</span>
+<span class="sd">            BigQuery.</span>
+<span class="sd">        :type write_disposition: string</span>
+<span class="sd">        :param allow_large_results: Whether to allow large results.</span>
+<span class="sd">        :type allow_large_results: boolean</span>
+<span class="sd">        :param flatten_results: If true and query uses legacy SQL dialect, flattens</span>
+<span class="sd">            all nested and repeated fields in the query results. ``allowLargeResults``</span>
+<span class="sd">            must be true if this is set to false. For standard SQL queries, this</span>
+<span class="sd">            flag is ignored and results are never flattened.</span>
+<span class="sd">        :type flatten_results: boolean</span>
+<span class="sd">        :param udf_config: The User Defined Function configuration for the query.</span>
+<span class="sd">            See https://cloud.google.com/bigquery/user-defined-functions for details.</span>
+<span class="sd">        :type udf_config: list</span>
+<span class="sd">        :param use_legacy_sql: Whether to use legacy SQL (true) or standard SQL (false).</span>
+<span class="sd">            If `None`, defaults to `self.use_legacy_sql`.</span>
+<span class="sd">        :type use_legacy_sql: boolean</span>
+<span class="sd">        :param api_resource_configs: a dictionary that contain params</span>
+<span class="sd">            &#39;configuration&#39; applied for Google BigQuery Jobs API:</span>
+<span class="sd">            https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs</span>
+<span class="sd">            for example, {&#39;query&#39;: {&#39;useQueryCache&#39;: False}}. You could use it</span>
+<span class="sd">            if you need to provide some params that are not supported by the</span>
+<span class="sd">            BigQueryHook like args.</span>
+<span class="sd">        :type api_resource_configs: dict</span>
+<span class="sd">        :param maximum_billing_tier: Positive integer that serves as a</span>
+<span class="sd">            multiplier of the basic price.</span>
+<span class="sd">        :type maximum_billing_tier: integer</span>
+<span class="sd">        :param maximum_bytes_billed: Limits the bytes billed for this job.</span>
+<span class="sd">            Queries that will have bytes billed beyond this limit will fail</span>
+<span class="sd">            (without incurring a charge). If unspecified, this will be</span>
+<span class="sd">            set to your project default.</span>
+<span class="sd">        :type maximum_bytes_billed: float</span>
+<span class="sd">        :param create_disposition: Specifies whether the job is allowed to</span>
+<span class="sd">            create new tables.</span>
+<span class="sd">        :type create_disposition: string</span>
+<span class="sd">        :param query_params a dictionary containing query parameter types and</span>
+<span class="sd">            values, passed to BigQuery</span>
+<span class="sd">        :type query_params: dict</span>
+<span class="sd">        :param labels a dictionary containing labels for the job/query,</span>
+<span class="sd">            passed to BigQuery</span>
+<span class="sd">        :type labels: dict</span>
+<span class="sd">        :param schema_update_options: Allows the schema of the destination</span>
+<span class="sd">            table to be updated as a side effect of the query job.</span>
+<span class="sd">        :type schema_update_options: tuple</span>
+<span class="sd">        :param priority: Specifies a priority for the query.</span>
+<span class="sd">            Possible values include INTERACTIVE and BATCH.</span>
+<span class="sd">            The default value is INTERACTIVE.</span>
+<span class="sd">        :type priority: string</span>
+<span class="sd">        :param time_partitioning: configure optional time partitioning fields i.e.</span>
+<span class="sd">            partition by field, type and expiration as per API specifications.</span>
+<span class="sd">        :type time_partitioning: dict</span>
+<span class="sd">        :param cluster_fields: Request that the result of this query be stored sorted</span>
+<span class="sd">            by one or more columns. This is only available in combination with</span>
+<span class="sd">            time_partitioning. The order of columns given determines the sort order.</span>
+<span class="sd">        :type cluster_fields: list of str</span>
+<span class="sd">        :param location: The geographic location of the job. Required except for</span>
+<span class="sd">            US and EU. See details at</span>
+<span class="sd">            https://cloud.google.com/bigquery/docs/locations#specifying_your_location</span>
+<span class="sd">        :type location: str</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+
+        <span class="k">if</span> <span class="n">time_partitioning</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">time_partitioning</span> <span class="o">=</span> <span class="p">{}</span>
+
+        <span class="k">if</span> <span class="n">location</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">location</span> <span class="o">=</span> <span class="n">location</span>
+
+        <span class="k">if</span> <span class="ow">not</span> <span class="n">api_resource_configs</span><span class="p">:</span>
+            <span class="n">api_resource_configs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">api_resource_configs</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">_validate_value</span><span class="p">(</span><span class="s1">&#39;api_resource_configs&#39;</span><span class="p">,</span>
+                            <span class="n">api_resource_configs</span><span class="p">,</span> <span class="nb">dict</span><span class="p">)</span>
+        <span class="n">configuration</span> <span class="o">=</span> <span class="n">deepcopy</span><span class="p">(</span><span class="n">api_resource_configs</span><span class="p">)</span>
+        <span class="k">if</span> <span class="s1">&#39;query&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">configuration</span><span class="p">:</span>
+            <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;query&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
+
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">_validate_value</span><span class="p">(</span><span class="s2">&quot;api_resource_configs[&#39;query&#39;]&quot;</span><span class="p">,</span>
+                            <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;query&#39;</span><span class="p">],</span> <span class="nb">dict</span><span class="p">)</span>
+
+        <span class="n">sql</span> <span class="o">=</span> <span class="n">bql</span> <span class="k">if</span> <span class="n">sql</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">sql</span>
+
+        <span class="c1"># TODO remove `bql` in Airflow 2.0 - Jira: [AIRFLOW-2513]</span>
+        <span class="k">if</span> <span class="n">bql</span><span class="p">:</span>
+            <span class="kn">import</span> <span class="nn">warnings</span>
+            <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s1">&#39;Deprecated parameter `bql` used in &#39;</span>
+                          <span class="s1">&#39;`BigQueryBaseCursor.run_query` &#39;</span>
+                          <span class="s1">&#39;Use `sql` parameter instead to pass the sql to be &#39;</span>
+                          <span class="s1">&#39;executed. `bql` parameter is deprecated and &#39;</span>
+                          <span class="s1">&#39;will be removed in a future version of &#39;</span>
+                          <span class="s1">&#39;Airflow.&#39;</span><span class="p">,</span>
+                          <span class="n">category</span><span class="o">=</span><span class="ne">DeprecationWarning</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="n">sql</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;query&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;query&#39;</span><span class="p">,</span> <span class="kc">None</span><span class="p">):</span>
+            <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s1">&#39;`BigQueryBaseCursor.run_query` &#39;</span>
+                            <span class="s1">&#39;missing 1 required positional argument: `sql`&#39;</span><span class="p">)</span>
+
+        <span class="c1"># BigQuery also allows you to define how you want a table&#39;s schema to change</span>
+        <span class="c1"># as a side effect of a query job</span>
+        <span class="c1"># for more details:</span>
+        <span class="c1">#   https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.query.schemaUpdateOptions</span>
+
+        <span class="n">allowed_schema_update_options</span> <span class="o">=</span> <span class="p">[</span>
+            <span class="s1">&#39;ALLOW_FIELD_ADDITION&#39;</span><span class="p">,</span> <span class="s2">&quot;ALLOW_FIELD_RELAXATION&quot;</span>
+        <span class="p">]</span>
+
+        <span class="k">if</span> <span class="ow">not</span> <span class="nb">set</span><span class="p">(</span><span class="n">allowed_schema_update_options</span>
+                   <span class="p">)</span><span class="o">.</span><span class="n">issuperset</span><span class="p">(</span><span class="nb">set</span><span class="p">(</span><span class="n">schema_update_options</span><span class="p">)):</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;</span><span class="si">{0}</span><span class="s2"> contains invalid schema update options. &quot;</span>
+                             <span class="s2">&quot;Please only use one or more of the following &quot;</span>
+                             <span class="s2">&quot;options: </span><span class="si">{1}</span><span class="s2">&quot;</span>
+                             <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">schema_update_options</span><span class="p">,</span>
+                                     <span class="n">allowed_schema_update_options</span><span class="p">))</span>
+
+        <span class="k">if</span> <span class="n">schema_update_options</span><span class="p">:</span>
+            <span class="k">if</span> <span class="n">write_disposition</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">&quot;WRITE_APPEND&quot;</span><span class="p">,</span> <span class="s2">&quot;WRITE_TRUNCATE&quot;</span><span class="p">]:</span>
+                <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;schema_update_options is only &quot;</span>
+                                 <span class="s2">&quot;allowed if write_disposition is &quot;</span>
+                                 <span class="s2">&quot;&#39;WRITE_APPEND&#39; or &#39;WRITE_TRUNCATE&#39;.&quot;</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="n">destination_dataset_table</span><span class="p">:</span>
+            <span class="n">destination_project</span><span class="p">,</span> <span class="n">destination_dataset</span><span class="p">,</span> <span class="n">destination_table</span> <span class="o">=</span> \
+                <span class="n">_split_tablename</span><span class="p">(</span><span class="n">table_input</span><span class="o">=</span><span class="n">destination_dataset_table</span><span class="p">,</span>
+                                 <span class="n">default_project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">)</span>
+
+            <span class="n">destination_dataset_table</span> <span class="o">=</span> <span class="p">{</span>
+                <span class="s1">&#39;projectId&#39;</span><span class="p">:</span> <span class="n">destination_project</span><span class="p">,</span>
+                <span class="s1">&#39;datasetId&#39;</span><span class="p">:</span> <span class="n">destination_dataset</span><span class="p">,</span>
+                <span class="s1">&#39;tableId&#39;</span><span class="p">:</span> <span class="n">destination_table</span><span class="p">,</span>
+            <span class="p">}</span>
+
+        <span class="k">if</span> <span class="n">cluster_fields</span><span class="p">:</span>
+            <span class="n">cluster_fields</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;fields&#39;</span><span class="p">:</span> <span class="n">cluster_fields</span><span class="p">}</span>
+
+        <span class="n">query_param_list</span> <span class="o">=</span> <span class="p">[</span>
+            <span class="p">(</span><span class="n">sql</span><span class="p">,</span> <span class="s1">&#39;query&#39;</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="n">six</span><span class="o">.</span><span class="n">string_types</span><span class="p">),</span>
+            <span class="p">(</span><span class="n">priority</span><span class="p">,</span> <span class="s1">&#39;priority&#39;</span><span class="p">,</span> <span class="s1">&#39;INTERACTIVE&#39;</span><span class="p">,</span> <span class="n">six</span><span class="o">.</span><span class="n">string_types</span><span class="p">),</span>
+            <span class="p">(</span><span class="n">use_legacy_sql</span><span class="p">,</span> <span class="s1">&#39;useLegacySql&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">use_legacy_sql</span><span class="p">,</span> <span class="nb">bool</span><span class="p">),</span>
+            <span class="p">(</span><span class="n">query_params</span><span class="p">,</span> <span class="s1">&#39;queryParameters&#39;</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="nb">dict</span><span class="p">),</span>
+            <span class="p">(</span><span class="n">udf_config</span><span class="p">,</span> <span class="s1">&#39;userDefinedFunctionResources&#39;</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="nb">list</span><span class="p">),</span>
+            <span class="p">(</span><span class="n">maximum_billing_tier</span><span class="p">,</span> <span class="s1">&#39;maximumBillingTier&#39;</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="nb">int</span><span class="p">),</span>
+            <span class="p">(</span><span class="n">maximum_bytes_billed</span><span class="p">,</span> <span class="s1">&#39;maximumBytesBilled&#39;</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="nb">float</span><span class="p">),</span>
+            <span class="p">(</span><span class="n">time_partitioning</span><span class="p">,</span> <span class="s1">&#39;timePartitioning&#39;</span><span class="p">,</span> <span class="p">{},</span> <span class="nb">dict</span><span class="p">),</span>
+            <span class="p">(</span><span class="n">schema_update_options</span><span class="p">,</span> <span class="s1">&#39;schemaUpdateOptions&#39;</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">),</span>
+            <span class="p">(</span><span class="n">destination_dataset_table</span><span class="p">,</span> <span class="s1">&#39;destinationTable&#39;</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="nb">dict</span><span class="p">),</span>
+            <span class="p">(</span><span class="n">cluster_fields</span><span class="p">,</span> <span class="s1">&#39;clustering&#39;</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="nb">dict</span><span class="p">),</span>
+        <span class="p">]</span>
+
+        <span class="k">for</span> <span class="n">param_tuple</span> <span class="ow">in</span> <span class="n">query_param_list</span><span class="p">:</span>
+
+            <span class="n">param</span><span class="p">,</span> <span class="n">param_name</span><span class="p">,</span> <span class="n">param_default</span><span class="p">,</span> <span class="n">param_type</span> <span class="o">=</span> <span class="n">param_tuple</span>
+
+            <span class="k">if</span> <span class="n">param_name</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;query&#39;</span><span class="p">]</span> <span class="ow">and</span> <span class="n">param</span> <span class="ow">in</span> <span class="p">[</span><span class="kc">None</span><span class="p">,</span> <span class="p">{},</span> <span class="p">()]:</span>
+                <span class="k">if</span> <span class="n">param_name</span> <span class="o">==</span> <span class="s1">&#39;timePartitioning&#39;</span><span class="p">:</span>
+                    <span class="n">param_default</span> <span class="o">=</span> <span class="n">_cleanse_time_partitioning</span><span class="p">(</span>
+                        <span class="n">destination_dataset_table</span><span class="p">,</span> <span class="n">time_partitioning</span><span class="p">)</span>
+                <span class="n">param</span> <span class="o">=</span> <span class="n">param_default</span>
+
+            <span class="k">if</span> <span class="n">param</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="kc">None</span><span class="p">,</span> <span class="p">{},</span> <span class="p">()]:</span>
+                <span class="n">_api_resource_configs_duplication_check</span><span class="p">(</span>
+                    <span class="n">param_name</span><span class="p">,</span> <span class="n">param</span><span class="p">,</span> <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;query&#39;</span><span class="p">])</span>
+
+                <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;query&#39;</span><span class="p">][</span><span class="n">param_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">param</span>
+
+                <span class="c1"># check valid type of provided param,</span>
+                <span class="c1"># it last step because we can get param from 2 sources,</span>
+                <span class="c1"># and first of all need to find it</span>
+
+                <span class="n">_validate_value</span><span class="p">(</span><span class="n">param_name</span><span class="p">,</span> <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;query&#39;</span><span class="p">][</span><span class="n">param_name</span><span class="p">],</span>
+                                <span class="n">param_type</span><span class="p">)</span>
+
+                <span class="k">if</span> <span class="n">param_name</span> <span class="o">==</span> <span class="s1">&#39;schemaUpdateOptions&#39;</span> <span class="ow">and</span> <span class="n">param</span><span class="p">:</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Adding experimental &#39;schemaUpdateOptions&#39;: &quot;</span>
+                                  <span class="s2">&quot;</span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">schema_update_options</span><span class="p">))</span>
+
+                <span class="k">if</span> <span class="n">param_name</span> <span class="o">==</span> <span class="s1">&#39;destinationTable&#39;</span><span class="p">:</span>
+                    <span class="k">for</span> <span class="n">key</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">&#39;projectId&#39;</span><span class="p">,</span> <span class="s1">&#39;datasetId&#39;</span><span class="p">,</span> <span class="s1">&#39;tableId&#39;</span><span class="p">]:</span>
+                        <span class="k">if</span> <span class="n">key</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;query&#39;</span><span class="p">][</span><span class="s1">&#39;destinationTable&#39;</span><span class="p">]:</span>
+                            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                                <span class="s2">&quot;Not correct &#39;destinationTable&#39; in &quot;</span>
+                                <span class="s2">&quot;api_resource_configs. &#39;destinationTable&#39; &quot;</span>
+                                <span class="s2">&quot;must be a dict with {&#39;projectId&#39;:&#39;&#39;, &quot;</span>
+                                <span class="s2">&quot;&#39;datasetId&#39;:&#39;&#39;, &#39;tableId&#39;:&#39;&#39;}&quot;</span><span class="p">)</span>
+
+                    <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;query&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">({</span>
+                        <span class="s1">&#39;allowLargeResults&#39;</span><span class="p">:</span> <span class="n">allow_large_results</span><span class="p">,</span>
+                        <span class="s1">&#39;flattenResults&#39;</span><span class="p">:</span> <span class="n">flatten_results</span><span class="p">,</span>
+                        <span class="s1">&#39;writeDisposition&#39;</span><span class="p">:</span> <span class="n">write_disposition</span><span class="p">,</span>
+                        <span class="s1">&#39;createDisposition&#39;</span><span class="p">:</span> <span class="n">create_disposition</span><span class="p">,</span>
+                    <span class="p">})</span>
+
+        <span class="k">if</span> <span class="s1">&#39;useLegacySql&#39;</span> <span class="ow">in</span> <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;query&#39;</span><span class="p">]</span> <span class="ow">and</span> \
+                <span class="s1">&#39;queryParameters&#39;</span> <span class="ow">in</span> <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;query&#39;</span><span class="p">]:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Query parameters are not allowed &quot;</span>
+                             <span class="s2">&quot;when using legacy SQL&quot;</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="n">labels</span><span class="p">:</span>
+            <span class="n">_api_resource_configs_duplication_check</span><span class="p">(</span>
+                <span class="s1">&#39;labels&#39;</span><span class="p">,</span> <span class="n">labels</span><span class="p">,</span> <span class="n">configuration</span><span class="p">)</span>
+            <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;labels&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">labels</span>
+
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_with_configuration</span><span class="p">(</span><span class="n">configuration</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">run_extract</span><span class="p">(</span>  <span class="c1"># noqa</span>
+            <span class="bp">self</span><span class="p">,</span>
+            <span class="n">source_project_dataset_table</span><span class="p">,</span>
+            <span class="n">destination_cloud_storage_uris</span><span class="p">,</span>
+            <span class="n">compression</span><span class="o">=</span><span class="s1">&#39;NONE&#39;</span><span class="p">,</span>
+            <span class="n">export_format</span><span class="o">=</span><span class="s1">&#39;CSV&#39;</span><span class="p">,</span>
+            <span class="n">field_delimiter</span><span class="o">=</span><span class="s1">&#39;,&#39;</span><span class="p">,</span>
+            <span class="n">print_header</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
+            <span class="n">labels</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Executes a BigQuery extract command to copy data from BigQuery to</span>
+<span class="sd">        Google Cloud Storage. See here:</span>
+
+<span class="sd">        https://cloud.google.com/bigquery/docs/reference/v2/jobs</span>
+
+<span class="sd">        For more details about these parameters.</span>
+
+<span class="sd">        :param source_project_dataset_table: The dotted &lt;dataset&gt;.&lt;table&gt;</span>
+<span class="sd">            BigQuery table to use as the source data.</span>
+<span class="sd">        :type source_project_dataset_table: string</span>
+<span class="sd">        :param destination_cloud_storage_uris: The destination Google Cloud</span>
+<span class="sd">            Storage URI (e.g. gs://some-bucket/some-file.txt). Follows</span>
+<span class="sd">            convention defined here:</span>
+<span class="sd">            https://cloud.google.com/bigquery/exporting-data-from-bigquery#exportingmultiple</span>
+<span class="sd">        :type destination_cloud_storage_uris: list</span>
+<span class="sd">        :param compression: Type of compression to use.</span>
+<span class="sd">        :type compression: string</span>
+<span class="sd">        :param export_format: File format to export.</span>
+<span class="sd">        :type export_format: string</span>
+<span class="sd">        :param field_delimiter: The delimiter to use when extracting to a CSV.</span>
+<span class="sd">        :type field_delimiter: string</span>
+<span class="sd">        :param print_header: Whether to print a header for a CSV file extract.</span>
+<span class="sd">        :type print_header: boolean</span>
+<span class="sd">        :param labels: a dictionary containing labels for the job/query,</span>
+<span class="sd">            passed to BigQuery</span>
+<span class="sd">        :type labels: dict</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+
+        <span class="n">source_project</span><span class="p">,</span> <span class="n">source_dataset</span><span class="p">,</span> <span class="n">source_table</span> <span class="o">=</span> \
+            <span class="n">_split_tablename</span><span class="p">(</span><span class="n">table_input</span><span class="o">=</span><span class="n">source_project_dataset_table</span><span class="p">,</span>
+                             <span class="n">default_project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+                             <span class="n">var_name</span><span class="o">=</span><span class="s1">&#39;source_project_dataset_table&#39;</span><span class="p">)</span>
+
+        <span class="n">configuration</span> <span class="o">=</span> <span class="p">{</span>
+            <span class="s1">&#39;extract&#39;</span><span class="p">:</span> <span class="p">{</span>
+                <span class="s1">&#39;sourceTable&#39;</span><span class="p">:</span> <span class="p">{</span>
+                    <span class="s1">&#39;projectId&#39;</span><span class="p">:</span> <span class="n">source_project</span><span class="p">,</span>
+                    <span class="s1">&#39;datasetId&#39;</span><span class="p">:</span> <span class="n">source_dataset</span><span class="p">,</span>
+                    <span class="s1">&#39;tableId&#39;</span><span class="p">:</span> <span class="n">source_table</span><span class="p">,</span>
+                <span class="p">},</span>
+                <span class="s1">&#39;compression&#39;</span><span class="p">:</span> <span class="n">compression</span><span class="p">,</span>
+                <span class="s1">&#39;destinationUris&#39;</span><span class="p">:</span> <span class="n">destination_cloud_storage_uris</span><span class="p">,</span>
+                <span class="s1">&#39;destinationFormat&#39;</span><span class="p">:</span> <span class="n">export_format</span><span class="p">,</span>
+            <span class="p">}</span>
+        <span class="p">}</span>
+
+        <span class="k">if</span> <span class="n">labels</span><span class="p">:</span>
+            <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;labels&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">labels</span>
+
+        <span class="k">if</span> <span class="n">export_format</span> <span class="o">==</span> <span class="s1">&#39;CSV&#39;</span><span class="p">:</span>
+            <span class="c1"># Only set fieldDelimiter and printHeader fields if using CSV.</span>
+            <span class="c1"># Google does not like it if you set these fields for other export</span>
+            <span class="c1"># formats.</span>
+            <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;extract&#39;</span><span class="p">][</span><span class="s1">&#39;fieldDelimiter&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">field_delimiter</span>
+            <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;extract&#39;</span><span class="p">][</span><span class="s1">&#39;printHeader&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">print_header</span>
+
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_with_configuration</span><span class="p">(</span><span class="n">configuration</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">run_copy</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+                 <span class="n">source_project_dataset_tables</span><span class="p">,</span>
+                 <span class="n">destination_project_dataset_table</span><span class="p">,</span>
+                 <span class="n">write_disposition</span><span class="o">=</span><span class="s1">&#39;WRITE_EMPTY&#39;</span><span class="p">,</span>
+                 <span class="n">create_disposition</span><span class="o">=</span><span class="s1">&#39;CREATE_IF_NEEDED&#39;</span><span class="p">,</span>
+                 <span class="n">labels</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Executes a BigQuery copy command to copy data from one BigQuery table</span>
+<span class="sd">        to another. See here:</span>
+
+<span class="sd">        https://cloud.google.com/bigquery/docs/reference/v2/jobs#configuration.copy</span>
+
+<span class="sd">        For more details about these parameters.</span>
+
+<span class="sd">        :param source_project_dataset_tables: One or more dotted</span>
+<span class="sd">            (project:|project.)&lt;dataset&gt;.&lt;table&gt;</span>
+<span class="sd">            BigQuery tables to use as the source data. Use a list if there are</span>
+<span class="sd">            multiple source tables.</span>
+<span class="sd">            If &lt;project&gt; is not included, project will be the project defined</span>
+<span class="sd">            in the connection json.</span>
+<span class="sd">        :type source_project_dataset_tables: list|string</span>
+<span class="sd">        :param destination_project_dataset_table: The destination BigQuery</span>
+<span class="sd">            table. Format is: (project:|project.)&lt;dataset&gt;.&lt;table&gt;</span>
+<span class="sd">        :type destination_project_dataset_table: string</span>
+<span class="sd">        :param write_disposition: The write disposition if the table already exists.</span>
+<span class="sd">        :type write_disposition: string</span>
+<span class="sd">        :param create_disposition: The create disposition if the table doesn&#39;t exist.</span>
+<span class="sd">        :type create_disposition: string</span>
+<span class="sd">        :param labels a dictionary containing labels for the job/query,</span>
+<span class="sd">            passed to BigQuery</span>
+<span class="sd">        :type labels: dict</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">source_project_dataset_tables</span> <span class="o">=</span> <span class="p">([</span>
+            <span class="n">source_project_dataset_tables</span>
+        <span class="p">]</span> <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">source_project_dataset_tables</span><span class="p">,</span> <span class="nb">list</span><span class="p">)</span> <span class="k">else</span>
+            <span class="n">source_project_dataset_tables</span><span class="p">)</span>
+
+        <span class="n">source_project_dataset_tables_fixup</span> <span class="o">=</span> <span class="p">[]</span>
+        <span class="k">for</span> <span class="n">source_project_dataset_table</span> <span class="ow">in</span> <span class="n">source_project_dataset_tables</span><span class="p">:</span>
+            <span class="n">source_project</span><span class="p">,</span> <span class="n">source_dataset</span><span class="p">,</span> <span class="n">source_table</span> <span class="o">=</span> \
+                <span class="n">_split_tablename</span><span class="p">(</span><span class="n">table_input</span><span class="o">=</span><span class="n">source_project_dataset_table</span><span class="p">,</span>
+                                 <span class="n">default_project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+                                 <span class="n">var_name</span><span class="o">=</span><span class="s1">&#39;source_project_dataset_table&#39;</span><span class="p">)</span>
+            <span class="n">source_project_dataset_tables_fixup</span><span class="o">.</span><span class="n">append</span><span class="p">({</span>
+                <span class="s1">&#39;projectId&#39;</span><span class="p">:</span>
+                <span class="n">source_project</span><span class="p">,</span>
+                <span class="s1">&#39;datasetId&#39;</span><span class="p">:</span>
+                <span class="n">source_dataset</span><span class="p">,</span>
+                <span class="s1">&#39;tableId&#39;</span><span class="p">:</span>
+                <span class="n">source_table</span>
+            <span class="p">})</span>
+
+        <span class="n">destination_project</span><span class="p">,</span> <span class="n">destination_dataset</span><span class="p">,</span> <span class="n">destination_table</span> <span class="o">=</span> \
+            <span class="n">_split_tablename</span><span class="p">(</span><span class="n">table_input</span><span class="o">=</span><span class="n">destination_project_dataset_table</span><span class="p">,</span>
+                             <span class="n">default_project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">)</span>
+        <span class="n">configuration</span> <span class="o">=</span> <span class="p">{</span>
+            <span class="s1">&#39;copy&#39;</span><span class="p">:</span> <span class="p">{</span>
+                <span class="s1">&#39;createDisposition&#39;</span><span class="p">:</span> <span class="n">create_disposition</span><span class="p">,</span>
+                <span class="s1">&#39;writeDisposition&#39;</span><span class="p">:</span> <span class="n">write_disposition</span><span class="p">,</span>
+                <span class="s1">&#39;sourceTables&#39;</span><span class="p">:</span> <span class="n">source_project_dataset_tables_fixup</span><span class="p">,</span>
+                <span class="s1">&#39;destinationTable&#39;</span><span class="p">:</span> <span class="p">{</span>
+                    <span class="s1">&#39;projectId&#39;</span><span class="p">:</span> <span class="n">destination_project</span><span class="p">,</span>
+                    <span class="s1">&#39;datasetId&#39;</span><span class="p">:</span> <span class="n">destination_dataset</span><span class="p">,</span>
+                    <span class="s1">&#39;tableId&#39;</span><span class="p">:</span> <span class="n">destination_table</span>
+                <span class="p">}</span>
+            <span class="p">}</span>
+        <span class="p">}</span>
+
+        <span class="k">if</span> <span class="n">labels</span><span class="p">:</span>
+            <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;labels&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">labels</span>
+
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_with_configuration</span><span class="p">(</span><span class="n">configuration</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">run_load</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+                 <span class="n">destination_project_dataset_table</span><span class="p">,</span>
+                 <span class="n">source_uris</span><span class="p">,</span>
+                 <span class="n">schema_fields</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                 <span class="n">source_format</span><span class="o">=</span><span class="s1">&#39;CSV&#39;</span><span class="p">,</span>
+                 <span class="n">create_disposition</span><span class="o">=</span><span class="s1">&#39;CREATE_IF_NEEDED&#39;</span><span class="p">,</span>
+                 <span class="n">skip_leading_rows</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
+                 <span class="n">write_disposition</span><span class="o">=</span><span class="s1">&#39;WRITE_EMPTY&#39;</span><span class="p">,</span>
+                 <span class="n">field_delimiter</span><span class="o">=</span><span class="s1">&#39;,&#39;</span><span class="p">,</span>
+                 <span class="n">max_bad_records</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
+                 <span class="n">quote_character</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                 <span class="n">ignore_unknown_values</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+                 <span class="n">allow_quoted_newlines</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+                 <span class="n">allow_jagged_rows</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+                 <span class="n">schema_update_options</span><span class="o">=</span><span class="p">(),</span>
+                 <span class="n">src_fmt_configs</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                 <span class="n">time_partitioning</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                 <span class="n">cluster_fields</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                 <span class="n">autodetect</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Executes a BigQuery load command to load data from Google Cloud Storage</span>
+<span class="sd">        to BigQuery. See here:</span>
+
+<span class="sd">        https://cloud.google.com/bigquery/docs/reference/v2/jobs</span>
+
+<span class="sd">        For more details about these parameters.</span>
+
+<span class="sd">        :param destination_project_dataset_table:</span>
+<span class="sd">            The dotted (&lt;project&gt;.|&lt;project&gt;:)&lt;dataset&gt;.&lt;table&gt;($&lt;partition&gt;) BigQuery</span>
+<span class="sd">            table to load data into. If &lt;project&gt; is not included, project will be the</span>
+<span class="sd">            project defined in the connection json. If a partition is specified the</span>
+<span class="sd">            operator will automatically append the data, create a new partition or create</span>
+<span class="sd">            a new DAY partitioned table.</span>
+<span class="sd">        :type destination_project_dataset_table: string</span>
+<span class="sd">        :param schema_fields: The schema field list as defined here:</span>
+<span class="sd">            https://cloud.google.com/bigquery/docs/reference/v2/jobs#configuration.load</span>
+<span class="sd">            Required if autodetect=False; optional if autodetect=True.</span>
+<span class="sd">        :type schema_fields: list</span>
+<span class="sd">        :param autodetect: Attempt to autodetect the schema for CSV and JSON</span>
+<span class="sd">            source files.</span>
+<span class="sd">        :type autodetect: bool</span>
+<span class="sd">        :param source_uris: The source Google Cloud</span>
+<span class="sd">            Storage URI (e.g. gs://some-bucket/some-file.txt). A single wild</span>
+<span class="sd">            per-object name can be used.</span>
+<span class="sd">        :type source_uris: list</span>
+<span class="sd">        :param source_format: File format to export.</span>
+<span class="sd">        :type source_format: string</span>
+<span class="sd">        :param create_disposition: The create disposition if the table doesn&#39;t exist.</span>
+<span class="sd">        :type create_disposition: string</span>
+<span class="sd">        :param skip_leading_rows: Number of rows to skip when loading from a CSV.</span>
+<span class="sd">        :type skip_leading_rows: int</span>
+<span class="sd">        :param write_disposition: The write disposition if the table already exists.</span>
+<span class="sd">        :type write_disposition: string</span>
+<span class="sd">        :param field_delimiter: The delimiter to use when loading from a CSV.</span>
+<span class="sd">        :type field_delimiter: string</span>
+<span class="sd">        :param max_bad_records: The maximum number of bad records that BigQuery can</span>
+<span class="sd">            ignore when running the job.</span>
+<span class="sd">        :type max_bad_records: int</span>
+<span class="sd">        :param quote_character: The value that is used to quote data sections in a CSV</span>
+<span class="sd">            file.</span>
+<span class="sd">        :type quote_character: string</span>
+<span class="sd">        :param ignore_unknown_values: [Optional] Indicates if BigQuery should allow</span>
+<span class="sd">            extra values that are not represented in the table schema.</span>
+<span class="sd">            If true, the extra values are ignored. If false, records with extra columns</span>
+<span class="sd">            are treated as bad records, and if there are too many bad records, an</span>
+<span class="sd">            invalid error is returned in the job result.</span>
+<span class="sd">        :type ignore_unknown_values: bool</span>
+<span class="sd">        :param allow_quoted_newlines: Whether to allow quoted newlines (true) or not</span>
+<span class="sd">            (false).</span>
+<span class="sd">        :type allow_quoted_newlines: boolean</span>
+<span class="sd">        :param allow_jagged_rows: Accept rows that are missing trailing optional columns.</span>
+<span class="sd">            The missing values are treated as nulls. If false, records with missing</span>
+<span class="sd">            trailing columns are treated as bad records, and if there are too many bad</span>
+<span class="sd">            records, an invalid error is returned in the job result. Only applicable when</span>
+<span class="sd">            soure_format is CSV.</span>
+<span class="sd">        :type allow_jagged_rows: bool</span>
+<span class="sd">        :param schema_update_options: Allows the schema of the desitination</span>
+<span class="sd">            table to be updated as a side effect of the load job.</span>
+<span class="sd">        :type schema_update_options: tuple</span>
+<span class="sd">        :param src_fmt_configs: configure optional fields specific to the source format</span>
+<span class="sd">        :type src_fmt_configs: dict</span>
+<span class="sd">        :param time_partitioning: configure optional time partitioning fields i.e.</span>
+<span class="sd">            partition by field, type and  expiration as per API specifications.</span>
+<span class="sd">        :type time_partitioning: dict</span>
+<span class="sd">        :param cluster_fields: Request that the result of this load be stored sorted</span>
+<span class="sd">            by one or more columns. This is only available in combination with</span>
+<span class="sd">            time_partitioning. The order of columns given determines the sort order.</span>
+<span class="sd">        :type cluster_fields: list of str</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+
+        <span class="c1"># bigquery only allows certain source formats</span>
+        <span class="c1"># we check to make sure the passed source format is valid</span>
+        <span class="c1"># if it&#39;s not, we raise a ValueError</span>
+        <span class="c1"># Refer to this link for more details:</span>
+        <span class="c1">#   https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.query.tableDefinitions.(key).sourceFormat</span>
+
+        <span class="k">if</span> <span class="n">schema_fields</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">autodetect</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                <span class="s1">&#39;You must either pass a schema or autodetect=True.&#39;</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="n">src_fmt_configs</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">src_fmt_configs</span> <span class="o">=</span> <span class="p">{}</span>
+
+        <span class="n">source_format</span> <span class="o">=</span> <span class="n">source_format</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span>
+        <span class="n">allowed_formats</span> <span class="o">=</span> <span class="p">[</span>
+            <span class="s2">&quot;CSV&quot;</span><span class="p">,</span> <span class="s2">&quot;NEWLINE_DELIMITED_JSON&quot;</span><span class="p">,</span> <span class="s2">&quot;AVRO&quot;</span><span class="p">,</span> <span class="s2">&quot;GOOGLE_SHEETS&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;DATASTORE_BACKUP&quot;</span><span class="p">,</span> <span class="s2">&quot;PARQUET&quot;</span>
+        <span class="p">]</span>
+        <span class="k">if</span> <span class="n">source_format</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">allowed_formats</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;</span><span class="si">{0}</span><span class="s2"> is not a valid source format. &quot;</span>
+                             <span class="s2">&quot;Please use one of the following types: </span><span class="si">{1}</span><span class="s2">&quot;</span>
+                             <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">source_format</span><span class="p">,</span> <span class="n">allowed_formats</span><span class="p">))</span>
+
+        <span class="c1"># bigquery also allows you to define how you want a table&#39;s schema to change</span>
+        <span class="c1"># as a side effect of a load</span>
+        <span class="c1"># for more details:</span>
+        <span class="c1"># https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.load.schemaUpdateOptions</span>
+        <span class="n">allowed_schema_update_options</span> <span class="o">=</span> <span class="p">[</span>
+            <span class="s1">&#39;ALLOW_FIELD_ADDITION&#39;</span><span class="p">,</span> <span class="s2">&quot;ALLOW_FIELD_RELAXATION&quot;</span>
+        <span class="p">]</span>
+        <span class="k">if</span> <span class="ow">not</span> <span class="nb">set</span><span class="p">(</span><span class="n">allowed_schema_update_options</span><span class="p">)</span><span class="o">.</span><span class="n">issuperset</span><span class="p">(</span>
+                <span class="nb">set</span><span class="p">(</span><span class="n">schema_update_options</span><span class="p">)):</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                <span class="s2">&quot;</span><span class="si">{0}</span><span class="s2"> contains invalid schema update options.&quot;</span>
+                <span class="s2">&quot;Please only use one or more of the following options: </span><span class="si">{1}</span><span class="s2">&quot;</span>
+                <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">schema_update_options</span><span class="p">,</span> <span class="n">allowed_schema_update_options</span><span class="p">))</span>
+
+        <span class="n">destination_project</span><span class="p">,</span> <span class="n">destination_dataset</span><span class="p">,</span> <span class="n">destination_table</span> <span class="o">=</span> \
+            <span class="n">_split_tablename</span><span class="p">(</span><span class="n">table_input</span><span class="o">=</span><span class="n">destination_project_dataset_table</span><span class="p">,</span>
+                             <span class="n">default_project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+                             <span class="n">var_name</span><span class="o">=</span><span class="s1">&#39;destination_project_dataset_table&#39;</span><span class="p">)</span>
+
+        <span class="n">configuration</span> <span class="o">=</span> <span class="p">{</span>
+            <span class="s1">&#39;load&#39;</span><span class="p">:</span> <span class="p">{</span>
+                <span class="s1">&#39;autodetect&#39;</span><span class="p">:</span> <span class="n">autodetect</span><span class="p">,</span>
+                <span class="s1">&#39;createDisposition&#39;</span><span class="p">:</span> <span class="n">create_disposition</span><span class="p">,</span>
+                <span class="s1">&#39;destinationTable&#39;</span><span class="p">:</span> <span class="p">{</span>
+                    <span class="s1">&#39;projectId&#39;</span><span class="p">:</span> <span class="n">destination_project</span><span class="p">,</span>
+                    <span class="s1">&#39;datasetId&#39;</span><span class="p">:</span> <span class="n">destination_dataset</span><span class="p">,</span>
+                    <span class="s1">&#39;tableId&#39;</span><span class="p">:</span> <span class="n">destination_table</span><span class="p">,</span>
+                <span class="p">},</span>
+                <span class="s1">&#39;sourceFormat&#39;</span><span class="p">:</span> <span class="n">source_format</span><span class="p">,</span>
+                <span class="s1">&#39;sourceUris&#39;</span><span class="p">:</span> <span class="n">source_uris</span><span class="p">,</span>
+                <span class="s1">&#39;writeDisposition&#39;</span><span class="p">:</span> <span class="n">write_disposition</span><span class="p">,</span>
+                <span class="s1">&#39;ignoreUnknownValues&#39;</span><span class="p">:</span> <span class="n">ignore_unknown_values</span>
+            <span class="p">}</span>
+        <span class="p">}</span>
+
+        <span class="n">time_partitioning</span> <span class="o">=</span> <span class="n">_cleanse_time_partitioning</span><span class="p">(</span>
+            <span class="n">destination_project_dataset_table</span><span class="p">,</span>
+            <span class="n">time_partitioning</span>
+        <span class="p">)</span>
+        <span class="k">if</span> <span class="n">time_partitioning</span><span class="p">:</span>
+            <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;load&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">({</span>
+                <span class="s1">&#39;timePartitioning&#39;</span><span class="p">:</span> <span class="n">time_partitioning</span>
+            <span class="p">})</span>
+
+        <span class="k">if</span> <span class="n">cluster_fields</span><span class="p">:</span>
+            <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;load&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="s1">&#39;clustering&#39;</span><span class="p">:</span> <span class="p">{</span><span class="s1">&#39;fields&#39;</span><span class="p">:</span> <span class="n">cluster_fields</span><span class="p">}})</span>
+
+        <span class="k">if</span> <span class="n">schema_fields</span><span class="p">:</span>
+            <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;load&#39;</span><span class="p">][</span><span class="s1">&#39;schema&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;fields&#39;</span><span class="p">:</span> <span class="n">schema_fields</span><span class="p">}</span>
+
+        <span class="k">if</span> <span class="n">schema_update_options</span><span class="p">:</span>
+            <span class="k">if</span> <span class="n">write_disposition</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">&quot;WRITE_APPEND&quot;</span><span class="p">,</span> <span class="s2">&quot;WRITE_TRUNCATE&quot;</span><span class="p">]:</span>
+                <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;schema_update_options is only &quot;</span>
+                                 <span class="s2">&quot;allowed if write_disposition is &quot;</span>
+                                 <span class="s2">&quot;&#39;WRITE_APPEND&#39; or &#39;WRITE_TRUNCATE&#39;.&quot;</span><span class="p">)</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+                    <span class="s2">&quot;Adding experimental &quot;</span>
+                    <span class="s2">&quot;&#39;schemaUpdateOptions&#39;: </span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">schema_update_options</span><span class="p">))</span>
+                <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;load&#39;</span><span class="p">][</span>
+                    <span class="s1">&#39;schemaUpdateOptions&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">schema_update_options</span>
+
+        <span class="k">if</span> <span class="n">max_bad_records</span><span class="p">:</span>
+            <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;load&#39;</span><span class="p">][</span><span class="s1">&#39;maxBadRecords&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">max_bad_records</span>
+
+        <span class="c1"># if following fields are not specified in src_fmt_configs,</span>
+        <span class="c1"># honor the top-level params for backward-compatibility</span>
+        <span class="k">if</span> <span class="s1">&#39;skipLeadingRows&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">src_fmt_configs</span><span class="p">:</span>
+            <span class="n">src_fmt_configs</span><span class="p">[</span><span class="s1">&#39;skipLeadingRows&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">skip_leading_rows</span>
+        <span class="k">if</span> <span class="s1">&#39;fieldDelimiter&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">src_fmt_configs</span><span class="p">:</span>
+            <span class="n">src_fmt_configs</span><span class="p">[</span><span class="s1">&#39;fieldDelimiter&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">field_delimiter</span>
+        <span class="k">if</span> <span class="s1">&#39;ignoreUnknownValues&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">src_fmt_configs</span><span class="p">:</span>
+            <span class="n">src_fmt_configs</span><span class="p">[</span><span class="s1">&#39;ignoreUnknownValues&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">ignore_unknown_values</span>
+        <span class="k">if</span> <span class="n">quote_character</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">src_fmt_configs</span><span class="p">[</span><span class="s1">&#39;quote&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">quote_character</span>
+        <span class="k">if</span> <span class="n">allow_quoted_newlines</span><span class="p">:</span>
+            <span class="n">src_fmt_configs</span><span class="p">[</span><span class="s1">&#39;allowQuotedNewlines&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">allow_quoted_newlines</span>
+
+        <span class="n">src_fmt_to_configs_mapping</span> <span class="o">=</span> <span class="p">{</span>
+            <span class="s1">&#39;CSV&#39;</span><span class="p">:</span> <span class="p">[</span>
+                <span class="s1">&#39;allowJaggedRows&#39;</span><span class="p">,</span> <span class="s1">&#39;allowQuotedNewlines&#39;</span><span class="p">,</span> <span class="s1">&#39;autodetect&#39;</span><span class="p">,</span>
+                <span class="s1">&#39;fieldDelimiter&#39;</span><span class="p">,</span> <span class="s1">&#39;skipLeadingRows&#39;</span><span class="p">,</span> <span class="s1">&#39;ignoreUnknownValues&#39;</span><span class="p">,</span>
+                <span class="s1">&#39;nullMarker&#39;</span><span class="p">,</span> <span class="s1">&#39;quote&#39;</span>
+            <span class="p">],</span>
+            <span class="s1">&#39;DATASTORE_BACKUP&#39;</span><span class="p">:</span> <span class="p">[</span><span class="s1">&#39;projectionFields&#39;</span><span class="p">],</span>
+            <span class="s1">&#39;NEWLINE_DELIMITED_JSON&#39;</span><span class="p">:</span> <span class="p">[</span><span class="s1">&#39;autodetect&#39;</span><span class="p">,</span> <span class="s1">&#39;ignoreUnknownValues&#39;</span><span class="p">],</span>
+            <span class="s1">&#39;PARQUET&#39;</span><span class="p">:</span> <span class="p">[</span><span class="s1">&#39;autodetect&#39;</span><span class="p">,</span> <span class="s1">&#39;ignoreUnknownValues&#39;</span><span class="p">],</span>
+            <span class="s1">&#39;AVRO&#39;</span><span class="p">:</span> <span class="p">[],</span>
+        <span class="p">}</span>
+        <span class="n">valid_configs</span> <span class="o">=</span> <span class="n">src_fmt_to_configs_mapping</span><span class="p">[</span><span class="n">source_format</span><span class="p">]</span>
+        <span class="n">src_fmt_configs</span> <span class="o">=</span> <span class="p">{</span>
+            <span class="n">k</span><span class="p">:</span> <span class="n">v</span>
+            <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">src_fmt_configs</span><span class="o">.</span><span class="n">items</span><span class="p">()</span> <span class="k">if</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">valid_configs</span>
+        <span class="p">}</span>
+        <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;load&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">src_fmt_configs</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="n">allow_jagged_rows</span><span class="p">:</span>
+            <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;load&#39;</span><span class="p">][</span><span class="s1">&#39;allowJaggedRows&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">allow_jagged_rows</span>
+
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_with_configuration</span><span class="p">(</span><span class="n">configuration</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">run_with_configuration</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">configuration</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Executes a BigQuery SQL query. See here:</span>
+
+<span class="sd">        https://cloud.google.com/bigquery/docs/reference/v2/jobs</span>
+
+<span class="sd">        For more details about the configuration parameter.</span>
+
+<span class="sd">        :param configuration: The configuration parameter maps directly to</span>
+<span class="sd">            BigQuery&#39;s configuration field in the job object. See</span>
+<span class="sd">            https://cloud.google.com/bigquery/docs/reference/v2/jobs for</span>
+<span class="sd">            details.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">jobs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">jobs</span><span class="p">()</span>
+        <span class="n">job_data</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;configuration&#39;</span><span class="p">:</span> <span class="n">configuration</span><span class="p">}</span>
+
+        <span class="c1"># Send query and wait for reply.</span>
+        <span class="n">query_reply</span> <span class="o">=</span> <span class="n">jobs</span> \
+            <span class="o">.</span><span class="n">insert</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="n">job_data</span><span class="p">)</span> \
+            <span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">running_job_id</span> <span class="o">=</span> <span class="n">query_reply</span><span class="p">[</span><span class="s1">&#39;jobReference&#39;</span><span class="p">][</span><span class="s1">&#39;jobId&#39;</span><span class="p">]</span>
+
+        <span class="c1"># Wait for query to finish.</span>
+        <span class="n">keep_polling_job</span> <span class="o">=</span> <span class="kc">True</span>
+        <span class="k">while</span> <span class="n">keep_polling_job</span><span class="p">:</span>
+            <span class="k">try</span><span class="p">:</span>
+                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">location</span><span class="p">:</span>
+                    <span class="n">job</span> <span class="o">=</span> <span class="n">jobs</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+                        <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+                        <span class="n">jobId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">running_job_id</span><span class="p">,</span>
+                        <span class="n">location</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">location</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+                <span class="k">else</span><span class="p">:</span>
+                    <span class="n">job</span> <span class="o">=</span> <span class="n">jobs</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+                        <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+                        <span class="n">jobId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">running_job_id</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+                <span class="k">if</span> <span class="n">job</span><span class="p">[</span><span class="s1">&#39;status&#39;</span><span class="p">][</span><span class="s1">&#39;state&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="s1">&#39;DONE&#39;</span><span class="p">:</span>
+                    <span class="n">keep_polling_job</span> <span class="o">=</span> <span class="kc">False</span>
+                    <span class="c1"># Check if job had errors.</span>
+                    <span class="k">if</span> <span class="s1">&#39;errorResult&#39;</span> <span class="ow">in</span> <span class="n">job</span><span class="p">[</span><span class="s1">&#39;status&#39;</span><span class="p">]:</span>
+                        <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span>
+                            <span class="s1">&#39;BigQuery job failed. Final error was: </span><span class="si">{}</span><span class="s1">. The job was: </span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span>
+                            <span class="nb">format</span><span class="p">(</span><span class="n">job</span><span class="p">[</span><span class="s1">&#39;status&#39;</span><span class="p">][</span><span class="s1">&#39;errorResult&#39;</span><span class="p">],</span> <span class="n">job</span><span class="p">))</span>
+                <span class="k">else</span><span class="p">:</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Waiting for job to complete : </span><span class="si">%s</span><span class="s1">, </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span>
+                                  <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">running_job_id</span><span class="p">)</span>
+                    <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">5</span><span class="p">)</span>
+
+            <span class="k">except</span> <span class="n">HttpError</span> <span class="k">as</span> <span class="n">err</span><span class="p">:</span>
+                <span class="k">if</span> <span class="n">err</span><span class="o">.</span><span class="n">resp</span><span class="o">.</span><span class="n">status</span> <span class="ow">in</span> <span class="p">[</span><span class="mi">500</span><span class="p">,</span> <span class="mi">503</span><span class="p">]:</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+                        <span class="s1">&#39;</span><span class="si">%s</span><span class="s1">: Retryable error, waiting for job to complete: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span>
+                        <span class="n">err</span><span class="o">.</span><span class="n">resp</span><span class="o">.</span><span class="n">status</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">running_job_id</span><span class="p">)</span>
+                    <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">5</span><span class="p">)</span>
+                <span class="k">else</span><span class="p">:</span>
+                    <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span>
+                        <span class="s1">&#39;BigQuery job status check failed. Final error was: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span>
+                        <span class="n">err</span><span class="o">.</span><span class="n">resp</span><span class="o">.</span><span class="n">status</span><span class="p">)</span>
+
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">running_job_id</span>
+
+    <span class="k">def</span> <span class="nf">poll_job_complete</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">job_id</span><span class="p">):</span>
+        <span class="n">jobs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">jobs</span><span class="p">()</span>
+        <span class="k">try</span><span class="p">:</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">location</span><span class="p">:</span>
+                <span class="n">job</span> <span class="o">=</span> <span class="n">jobs</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+                               <span class="n">jobId</span><span class="o">=</span><span class="n">job_id</span><span class="p">,</span>
+                               <span class="n">location</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">location</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">job</span> <span class="o">=</span> <span class="n">jobs</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+                               <span class="n">jobId</span><span class="o">=</span><span class="n">job_id</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+            <span class="k">if</span> <span class="n">job</span><span class="p">[</span><span class="s1">&#39;status&#39;</span><span class="p">][</span><span class="s1">&#39;state&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="s1">&#39;DONE&#39;</span><span class="p">:</span>
+                <span class="k">return</span> <span class="kc">True</span>
+        <span class="k">except</span> <span class="n">HttpError</span> <span class="k">as</span> <span class="n">err</span><span class="p">:</span>
+            <span class="k">if</span> <span class="n">err</span><span class="o">.</span><span class="n">resp</span><span class="o">.</span><span class="n">status</span> <span class="ow">in</span> <span class="p">[</span><span class="mi">500</span><span class="p">,</span> <span class="mi">503</span><span class="p">]:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+                    <span class="s1">&#39;</span><span class="si">%s</span><span class="s1">: Retryable error while polling job with id </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span>
+                    <span class="n">err</span><span class="o">.</span><span class="n">resp</span><span class="o">.</span><span class="n">status</span><span class="p">,</span> <span class="n">job_id</span><span class="p">)</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span>
+                    <span class="s1">&#39;BigQuery job status check failed. Final error was: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span>
+                    <span class="n">err</span><span class="o">.</span><span class="n">resp</span><span class="o">.</span><span class="n">status</span><span class="p">)</span>
+        <span class="k">return</span> <span class="kc">False</span>
+
+    <span class="k">def</span> <span class="nf">cancel_query</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Cancel all started queries that have not yet completed</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">jobs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">jobs</span><span class="p">()</span>
+        <span class="k">if</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">running_job_id</span> <span class="ow">and</span>
+                <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">poll_job_complete</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">running_job_id</span><span class="p">)):</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Attempting to cancel job : </span><span class="si">%s</span><span class="s1">, </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+                          <span class="bp">self</span><span class="o">.</span><span class="n">running_job_id</span><span class="p">)</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">location</span><span class="p">:</span>
+                <span class="n">jobs</span><span class="o">.</span><span class="n">cancel</span><span class="p">(</span>
+                    <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+                    <span class="n">jobId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">running_job_id</span><span class="p">,</span>
+                    <span class="n">location</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">location</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">jobs</span><span class="o">.</span><span class="n">cancel</span><span class="p">(</span>
+                    <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+                    <span class="n">jobId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">running_job_id</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;No running BigQuery jobs to cancel.&#39;</span><span class="p">)</span>
+            <span class="k">return</span>
+
+        <span class="c1"># Wait for all the calls to cancel to finish</span>
+        <span class="n">max_polling_attempts</span> <span class="o">=</span> <span class="mi">12</span>
+        <span class="n">polling_attempts</span> <span class="o">=</span> <span class="mi">0</span>
+
+        <span class="n">job_complete</span> <span class="o">=</span> <span class="kc">False</span>
+        <span class="k">while</span> <span class="n">polling_attempts</span> <span class="o">&lt;</span> <span class="n">max_polling_attempts</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">job_complete</span><span class="p">:</span>
+            <span class="n">polling_attempts</span> <span class="o">=</span> <span class="n">polling_attempts</span> <span class="o">+</span> <span class="mi">1</span>
+            <span class="n">job_complete</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">poll_job_complete</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">running_job_id</span><span class="p">)</span>
+            <span class="k">if</span> <span class="n">job_complete</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Job successfully canceled: </span><span class="si">%s</span><span class="s1">, </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span>
+                              <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">running_job_id</span><span class="p">)</span>
+            <span class="k">elif</span> <span class="n">polling_attempts</span> <span class="o">==</span> <span class="n">max_polling_attempts</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+                    <span class="s2">&quot;Stopping polling due to timeout. Job with id </span><span class="si">%s</span><span class="s2"> &quot;</span>
+                    <span class="s2">&quot;has not completed cancel and may or may not finish.&quot;</span><span class="p">,</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">running_job_id</span><span class="p">)</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Waiting for canceled job with id </span><span class="si">%s</span><span class="s1"> to finish.&#39;</span><span class="p">,</span>
+                              <span class="bp">self</span><span class="o">.</span><span class="n">running_job_id</span><span class="p">)</span>
+                <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">5</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">get_schema</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Get the schema for a given datset.table.</span>
+<span class="sd">        see https://cloud.google.com/bigquery/docs/reference/v2/tables#resource</span>
+
+<span class="sd">        :param dataset_id: the dataset ID of the requested table</span>
+<span class="sd">        :param table_id: the table ID of the requested table</span>
+<span class="sd">        :return: a table schema</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">tables_resource</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span> \
+            <span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span> <span class="n">tableId</span><span class="o">=</span><span class="n">table_id</span><span class="p">)</span> \
+            <span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+        <span class="k">return</span> <span class="n">tables_resource</span><span class="p">[</span><span class="s1">&#39;schema&#39;</span><span class="p">]</span>
+
+    <span class="k">def</span> <span class="nf">get_tabledata</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">,</span>
+                      <span class="n">max_results</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">selected_fields</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">page_token</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                      <span class="n">start_index</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Get the data of a given dataset.table and optionally with selected columns.</span>
+<span class="sd">        see https://cloud.google.com/bigquery/docs/reference/v2/tabledata/list</span>
+
+<span class="sd">        :param dataset_id: the dataset ID of the requested table.</span>
+<span class="sd">        :param table_id: the table ID of the requested table.</span>
+<span class="sd">        :param max_results: the maximum results to return.</span>
+<span class="sd">        :param selected_fields: List of fields to return (comma-separated). If</span>
+<span class="sd">            unspecified, all fields are returned.</span>
+<span class="sd">        :param page_token: page token, returned from a previous call,</span>
+<span class="sd">            identifying the result set.</span>
+<span class="sd">        :param start_index: zero based index of the starting row to read.</span>
+<span class="sd">        :return: map containing the requested rows.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">optional_params</span> <span class="o">=</span> <span class="p">{}</span>
+        <span class="k">if</span> <span class="n">max_results</span><span class="p">:</span>
+            <span class="n">optional_params</span><span class="p">[</span><span class="s1">&#39;maxResults&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">max_results</span>
+        <span class="k">if</span> <span class="n">selected_fields</span><span class="p">:</span>
+            <span class="n">optional_params</span><span class="p">[</span><span class="s1">&#39;selectedFields&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">selected_fields</span>
+        <span class="k">if</span> <span class="n">page_token</span><span class="p">:</span>
+            <span class="n">optional_params</span><span class="p">[</span><span class="s1">&#39;pageToken&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">page_token</span>
+        <span class="k">if</span> <span class="n">start_index</span><span class="p">:</span>
+            <span class="n">optional_params</span><span class="p">[</span><span class="s1">&#39;startIndex&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">start_index</span>
+        <span class="k">return</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tabledata</span><span class="p">()</span><span class="o">.</span><span class="n">list</span><span class="p">(</span>
+            <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+            <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span>
+            <span class="n">tableId</span><span class="o">=</span><span class="n">table_id</span><span class="p">,</span>
+            <span class="o">**</span><span class="n">optional_params</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">())</span>
+
+    <span class="k">def</span> <span class="nf">run_table_delete</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">deletion_dataset_table</span><span class="p">,</span>
+                         <span class="n">ignore_if_missing</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Delete an existing table from the dataset;</span>
+<span class="sd">        If the table does not exist, return an error unless ignore_if_missing</span>
+<span class="sd">        is set to True.</span>
+
+<span class="sd">        :param deletion_dataset_table: A dotted</span>
+<span class="sd">        (&lt;project&gt;.|&lt;project&gt;:)&lt;dataset&gt;.&lt;table&gt; that indicates which table</span>
+<span class="sd">        will be deleted.</span>
+<span class="sd">        :type deletion_dataset_table: str</span>
+<span class="sd">        :param ignore_if_missing: if True, then return success even if the</span>
+<span class="sd">        requested table does not exist.</span>
+<span class="sd">        :type ignore_if_missing: boolean</span>
+<span class="sd">        :return:</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">deletion_project</span><span class="p">,</span> <span class="n">deletion_dataset</span><span class="p">,</span> <span class="n">deletion_table</span> <span class="o">=</span> \
+            <span class="n">_split_tablename</span><span class="p">(</span><span class="n">table_input</span><span class="o">=</span><span class="n">deletion_dataset_table</span><span class="p">,</span>
+                             <span class="n">default_project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">)</span>
+
+        <span class="k">try</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span> \
+                <span class="o">.</span><span class="n">delete</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="n">deletion_project</span><span class="p">,</span>
+                        <span class="n">datasetId</span><span class="o">=</span><span class="n">deletion_dataset</span><span class="p">,</span>
+                        <span class="n">tableId</span><span class="o">=</span><span class="n">deletion_table</span><span class="p">)</span> \
+                <span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Deleted table </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1">.&#39;</span><span class="p">,</span> <span class="n">deletion_project</span><span class="p">,</span>
+                          <span class="n">deletion_dataset</span><span class="p">,</span> <span class="n">deletion_table</span><span class="p">)</span>
+        <span class="k">except</span> <span class="n">HttpError</span><span class="p">:</span>
+            <span class="k">if</span> <span class="ow">not</span> <span class="n">ignore_if_missing</span><span class="p">:</span>
+                <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="s1">&#39;Table deletion failed. Table does not exist.&#39;</span><span class="p">)</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Table does not exist. Skipping.&#39;</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">run_table_upsert</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_resource</span><span class="p">,</span> <span class="n">project_id</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        creates a new, empty table in the dataset;</span>
+<span class="sd">        If the table already exists, update the existing table.</span>
+<span class="sd">        Since BigQuery does not natively allow table upserts, this is not an</span>
+<span class="sd">        atomic operation.</span>
+
+<span class="sd">        :param dataset_id: the dataset to upsert the table into.</span>
+<span class="sd">        :type dataset_id: str</span>
+<span class="sd">        :param table_resource: a table resource. see</span>
+<span class="sd">            https://cloud.google.com/bigquery/docs/reference/v2/tables#resource</span>
+<span class="sd">        :type table_resource: dict</span>
+<span class="sd">        :param project_id: the project to upsert the table into.  If None,</span>
+<span class="sd">        project will be self.project_id.</span>
+<span class="sd">        :return:</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="c1"># check to see if the table exists</span>
+        <span class="n">table_id</span> <span class="o">=</span> <span class="n">table_resource</span><span class="p">[</span><span class="s1">&#39;tableReference&#39;</span><span class="p">][</span><span class="s1">&#39;tableId&#39;</span><span class="p">]</span>
+        <span class="n">project_id</span> <span class="o">=</span> <span class="n">project_id</span> <span class="k">if</span> <span class="n">project_id</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span>
+        <span class="n">tables_list_resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span><span class="o">.</span><span class="n">list</span><span class="p">(</span>
+            <span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span> <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+        <span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
+            <span class="k">for</span> <span class="n">table</span> <span class="ow">in</span> <span class="n">tables_list_resp</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;tables&#39;</span><span class="p">,</span> <span class="p">[]):</span>
+                <span class="k">if</span> <span class="n">table</span><span class="p">[</span><span class="s1">&#39;tableReference&#39;</span><span class="p">][</span><span class="s1">&#39;tableId&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="n">table_id</span><span class="p">:</span>
+                    <span class="c1"># found the table, do update</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Table </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1"> exists, updating.&#39;</span><span class="p">,</span>
+                                  <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">)</span>
+                    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span><span class="o">.</span><span class="n">update</span><span class="p">(</span>
+                        <span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
+                        <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span>
+                        <span class="n">tableId</span><span class="o">=</span><span class="n">table_id</span><span class="p">,</span>
+                        <span class="n">body</span><span class="o">=</span><span class="n">table_resource</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+            <span class="c1"># If there is a next page, we need to check the next page.</span>
+            <span class="k">if</span> <span class="s1">&#39;nextPageToken&#39;</span> <span class="ow">in</span> <span class="n">tables_list_resp</span><span class="p">:</span>
+                <span class="n">tables_list_resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span>\
+                    <span class="o">.</span><span class="n">list</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
+                          <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span>
+                          <span class="n">pageToken</span><span class="o">=</span><span class="n">tables_list_resp</span><span class="p">[</span><span class="s1">&#39;nextPageToken&#39;</span><span class="p">])</span>\
+                    <span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+            <span class="c1"># If there is no next page, then the table doesn&#39;t exist.</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="c1"># do insert</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Table </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1"> does not exist. creating.&#39;</span><span class="p">,</span>
+                              <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">)</span>
+                <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span><span class="o">.</span><span class="n">insert</span><span class="p">(</span>
+                    <span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
+                    <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span>
+                    <span class="n">body</span><span class="o">=</span><span class="n">table_resource</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="nf">run_grant_dataset_view_access</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+                                      <span class="n">source_dataset</span><span class="p">,</span>
+                                      <span class="n">view_dataset</span><span class="p">,</span>
+                                      <span class="n">view_table</span><span class="p">,</span>
+                                      <span class="n">source_project</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                                      <span class="n">view_project</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Grant authorized view access of a dataset to a view table.</span>
+<span class="sd">        If this view has already been granted access to the dataset, do nothing.</span>
+<span class="sd">        This method is not atomic.  Running it may clobber a simultaneous update.</span>
+
+<span class="sd">        :param source_dataset: the source dataset</span>
+<span class="sd">        :type source_dataset: str</span>
+<span class="sd">        :param view_dataset: the dataset that the view is in</span>
+<span class="sd">        :type view_dataset: str</span>
+<span class="sd">        :param view_table: the table of the view</span>
+<span class="sd">        :type view_table: str</span>
+<span class="sd">        :param source_project: the project of the source dataset. If None,</span>
+<span class="sd">        self.project_id will be used.</span>
+<span class="sd">        :type source_project: str</span>
+<span class="sd">        :param view_project: the project that the view is in. If None,</span>
+<span class="sd">        self.project_id will be used.</span>
+<span class="sd">        :type view_project: str</span>
+<span class="sd">        :return: the datasets resource of the source dataset.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+
+        <span class="c1"># Apply default values to projects</span>
+        <span class="n">source_project</span> <span class="o">=</span> <span class="n">source_project</span> <span class="k">if</span> <span class="n">source_project</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span>
+        <span class="n">view_project</span> <span class="o">=</span> <span class="n">view_project</span> <span class="k">if</span> <span class="n">view_project</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span>
+
+        <span class="c1"># we don&#39;t want to clobber any existing accesses, so we have to get</span>
+        <span class="c1"># info on the dataset before we can add view access</span>
+        <span class="n">source_dataset_resource</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">datasets</span><span class="p">()</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+            <span class="n">projectId</span><span class="o">=</span><span class="n">source_project</span><span class="p">,</span> <span class="n">datasetId</span><span class="o">=</span><span class="n">source_dataset</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+        <span class="n">access</span> <span class="o">=</span> <span class="n">source_dataset_resource</span><span class="p">[</span>
+            <span class="s1">&#39;access&#39;</span><span class="p">]</span> <span class="k">if</span> <span class="s1">&#39;access&#39;</span> <span class="ow">in</span> <span class="n">source_dataset_resource</span> <span class="k">else</span> <span class="p">[]</span>
+        <span class="n">view_access</span> <span class="o">=</span> <span class="p">{</span>
+            <span class="s1">&#39;view&#39;</span><span class="p">:</span> <span class="p">{</span>
+                <span class="s1">&#39;projectId&#39;</span><span class="p">:</span> <span class="n">view_project</span><span class="p">,</span>
+                <span class="s1">&#39;datasetId&#39;</span><span class="p">:</span> <span class="n">view_dataset</span><span class="p">,</span>
+                <span class="s1">&#39;tableId&#39;</span><span class="p">:</span> <span class="n">view_table</span>
+            <span class="p">}</span>
+        <span class="p">}</span>
+        <span class="c1"># check to see if the view we want to add already exists.</span>
+        <span class="k">if</span> <span class="n">view_access</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">access</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+                <span class="s1">&#39;Granting table </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1"> authorized view access to </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1"> dataset.&#39;</span><span class="p">,</span>
+                <span class="n">view_project</span><span class="p">,</span> <span class="n">view_dataset</span><span class="p">,</span> <span class="n">view_table</span><span class="p">,</span> <span class="n">source_project</span><span class="p">,</span>
+                <span class="n">source_dataset</span><span class="p">)</span>
+            <span class="n">access</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">view_access</span><span class="p">)</span>
+            <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">datasets</span><span class="p">()</span><span class="o">.</span><span class="n">patch</span><span class="p">(</span>
+                <span class="n">projectId</span><span class="o">=</span><span class="n">source_project</span><span class="p">,</span>
+                <span class="n">datasetId</span><span class="o">=</span><span class="n">source_dataset</span><span class="p">,</span>
+                <span class="n">body</span><span class="o">=</span><span class="p">{</span>
+                    <span class="s1">&#39;access&#39;</span><span class="p">:</span> <span class="n">access</span>
+                <span class="p">})</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="c1"># if view is already in access, do nothing.</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+                <span class="s1">&#39;Table </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1"> already has authorized view access to </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1"> dataset.&#39;</span><span class="p">,</span>
+                <span class="n">view_project</span><span class="p">,</span> <span class="n">view_dataset</span><span class="p">,</span> <span class="n">view_table</span><span class="p">,</span> <span class="n">source_project</span><span class="p">,</span> <span class="n">source_dataset</span><span class="p">)</span>
+            <span class="k">return</span> <span class="n">source_dataset_resource</span>
+
+    <span class="k">def</span> <span class="nf">create_empty_dataset</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_id</span><span class="o">=</span><span class="s2">&quot;&quot;</span><span class="p">,</span> <span class="n">project_id</span><span class="o">=</span><span class="s2">&quot;&quot;</span><span class="p">,</span>
+                             <span class="n">dataset_reference</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Create a new empty dataset:</span>
+<span class="sd">        https://cloud.google.com/bigquery/docs/reference/rest/v2/datasets/insert</span>
+
+<span class="sd">        :param project_id: The name of the project where we want to create</span>
+<span class="sd">            an empty a dataset. Don&#39;t need to provide, if projectId in dataset_reference.</span>
+<span class="sd">        :type project_id: str</span>
+<span class="sd">        :param dataset_id: The id of dataset. Don&#39;t need to provide,</span>
+<span class="sd">            if datasetId in dataset_reference.</span>
+<span class="sd">        :type dataset_id: str</span>
+<span class="sd">        :param dataset_reference: Dataset reference that could be provided</span>
+<span class="sd">            with request body. More info:</span>
+<span class="sd">            https://cloud.google.com/bigquery/docs/reference/rest/v2/datasets#resource</span>
+<span class="sd">        :type dataset_reference: dict</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+
+        <span class="k">if</span> <span class="n">dataset_reference</span><span class="p">:</span>
+            <span class="n">_validate_value</span><span class="p">(</span><span class="s1">&#39;dataset_reference&#39;</span><span class="p">,</span> <span class="n">dataset_reference</span><span class="p">,</span> <span class="nb">dict</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">dataset_reference</span> <span class="o">=</span> <span class="p">{}</span>
+
+        <span class="k">if</span> <span class="s2">&quot;datasetReference&quot;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">dataset_reference</span><span class="p">:</span>
+            <span class="n">dataset_reference</span><span class="p">[</span><span class="s2">&quot;datasetReference&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
+
+        <span class="k">if</span> <span class="ow">not</span> <span class="n">dataset_reference</span><span class="p">[</span><span class="s2">&quot;datasetReference&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;datasetId&quot;</span><span class="p">)</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">dataset_id</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                <span class="s2">&quot;</span><span class="si">{}</span><span class="s2"> not provided datasetId. Impossible to create dataset&quot;</span><span class="p">)</span>
+
+        <span class="n">dataset_required_params</span> <span class="o">=</span> <span class="p">[(</span><span class="n">dataset_id</span><span class="p">,</span> <span class="s2">&quot;datasetId&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">),</span>
+                                   <span class="p">(</span><span class="n">project_id</span><span class="p">,</span> <span class="s2">&quot;projectId&quot;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">)]</span>
+        <span class="k">for</span> <span class="n">param_tuple</span> <span class="ow">in</span> <span class="n">dataset_required_params</span><span class="p">:</span>
+            <span class="n">param</span><span class="p">,</span> <span class="n">param_name</span><span class="p">,</span> <span class="n">param_default</span> <span class="o">=</span> <span class="n">param_tuple</span>
+            <span class="k">if</span> <span class="n">param_name</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">dataset_reference</span><span class="p">[</span><span class="s1">&#39;datasetReference&#39;</span><span class="p">]:</span>
+                <span class="k">if</span> <span class="n">param_default</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">param</span><span class="p">:</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;</span><span class="si">{}</span><span class="s2"> was not specified. Will be used default &quot;</span>
+                                  <span class="s2">&quot;value </span><span class="si">{}</span><span class="s2">.&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">param_name</span><span class="p">,</span>
+                                                     <span class="n">param_default</span><span class="p">))</span>
+                    <span class="n">param</span> <span class="o">=</span> <span class="n">param_default</span>
+                <span class="n">dataset_reference</span><span class="p">[</span><span class="s1">&#39;datasetReference&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">(</span>
+                    <span class="p">{</span><span class="n">param_name</span><span class="p">:</span> <span class="n">param</span><span class="p">})</span>
+            <span class="k">elif</span> <span class="n">param</span><span class="p">:</span>
+                <span class="n">_api_resource_configs_duplication_check</span><span class="p">(</span>
+                    <span class="n">param_name</span><span class="p">,</span> <span class="n">param</span><span class="p">,</span>
+                    <span class="n">dataset_reference</span><span class="p">[</span><span class="s1">&#39;datasetReference&#39;</span><span class="p">],</span> <span class="s1">&#39;dataset_reference&#39;</span><span class="p">)</span>
+
+        <span class="n">dataset_id</span> <span class="o">=</span> <span class="n">dataset_reference</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;datasetReference&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;datasetId&quot;</span><span class="p">)</span>
+        <span class="n">dataset_project_id</span> <span class="o">=</span> <span class="n">dataset_reference</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;datasetReference&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+            <span class="s2">&quot;projectId&quot;</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Creating Dataset: </span><span class="si">%s</span><span class="s1"> in project: </span><span class="si">%s</span><span class="s1"> &#39;</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span>
+                      <span class="n">dataset_project_id</span><span class="p">)</span>
+
+        <span class="k">try</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">datasets</span><span class="p">()</span><span class="o">.</span><span class="n">insert</span><span class="p">(</span>
+                <span class="n">projectId</span><span class="o">=</span><span class="n">dataset_project_id</span><span class="p">,</span>
+                <span class="n">body</span><span class="o">=</span><span class="n">dataset_reference</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Dataset created successfully: In project </span><span class="si">%s</span><span class="s1"> &#39;</span>
+                          <span class="s1">&#39;Dataset </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="n">dataset_project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">)</span>
+
+        <span class="k">except</span> <span class="n">HttpError</span> <span class="k">as</span> <span class="n">err</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+                <span class="s1">&#39;BigQuery job failed. Error was: </span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">err</span><span class="o">.</span><span class="n">content</span><span class="p">)</span>
+            <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">delete_dataset</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Delete a dataset of Big query in your project.</span>
+<span class="sd">        :param project_id: The name of the project where we have the dataset .</span>
+<span class="sd">        :type project_id: str</span>
+<span class="sd">        :param dataset_id: The dataset to be delete.</span>
+<span class="sd">        :type dataset_id: str</span>
+<span class="sd">        :return:</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">project_id</span> <span class="o">=</span> <span class="n">project_id</span> <span class="k">if</span> <span class="n">project_id</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Deleting from project: </span><span class="si">%s</span><span class="s1">  Dataset:</span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span>
+                      <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">)</span>
+
+        <span class="k">try</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">datasets</span><span class="p">()</span><span class="o">.</span><span class="n">delete</span><span class="p">(</span>
+                <span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
+                <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Dataset deleted successfully: In project </span><span class="si">%s</span><span class="s1"> &#39;</span>
+                          <span class="s1">&#39;Dataset </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">)</span>
+
+        <span class="k">except</span> <span class="n">HttpError</span> <span class="k">as</span> <span class="n">err</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+                <span class="s1">&#39;BigQuery job failed. Error was: </span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">err</span><span class="o">.</span><span class="n">content</span><span class="p">)</span>
+            <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">get_dataset</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">project_id</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Method returns dataset_resource if dataset exist</span>
+<span class="sd">        and raised 404 error if dataset does not exist</span>
+
+<span class="sd">        :param dataset_id: The BigQuery Dataset ID</span>
+<span class="sd">        :type dataset_id: str</span>
+<span class="sd">        :param project_id: The GCP Project ID</span>
+<span class="sd">        :type project_id: str</span>
+<span class="sd">        :return: dataset_resource</span>
+
+<span class="sd">            .. seealso::</span>
+<span class="sd">                For more information, see Dataset Resource content:</span>
+<span class="sd">                https://cloud.google.com/bigquery/docs/reference/rest/v2/datasets#resource</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+
+        <span class="k">if</span> <span class="ow">not</span> <span class="n">dataset_id</span> <span class="ow">or</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">dataset_id</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;dataset_id argument must be provided and has &quot;</span>
+                             <span class="s2">&quot;a type &#39;str&#39;. You provided: </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">dataset_id</span><span class="p">))</span>
+
+        <span class="n">dataset_project_id</span> <span class="o">=</span> <span class="n">project_id</span> <span class="k">if</span> <span class="n">project_id</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span>
+
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">dataset_resource</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">datasets</span><span class="p">()</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+                <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span> <span class="n">projectId</span><span class="o">=</span><span class="n">dataset_project_id</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Dataset Resource: </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">dataset_resource</span><span class="p">))</span>
+        <span class="k">except</span> <span class="n">HttpError</span> <span class="k">as</span> <span class="n">err</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+                <span class="s1">&#39;BigQuery job failed. Error was: </span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">err</span><span class="o">.</span><span class="n">content</span><span class="p">))</span>
+
+        <span class="k">return</span> <span class="n">dataset_resource</span>
+
+    <span class="k">def</span> <span class="nf">get_datasets_list</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project_id</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Method returns full list of BigQuery datasets in the current project</span>
+
+<span class="sd">        .. seealso::</span>
+<span class="sd">            For more information, see:</span>
+<span class="sd">            https://cloud.google.com/bigquery/docs/reference/rest/v2/datasets/list</span>
+
+<span class="sd">        :param project_id: Google Cloud Project for which you</span>
+<span class="sd">            try to get all datasets</span>
+<span class="sd">        :type project_id: str</span>
+<span class="sd">        :return: datasets_list</span>
+
+<span class="sd">            Example of returned datasets_list: ::</span>
+
+<span class="sd">                   {</span>
+<span class="sd">                      &quot;kind&quot;:&quot;bigquery#dataset&quot;,</span>
+<span class="sd">                      &quot;location&quot;:&quot;US&quot;,</span>
+<span class="sd">                      &quot;id&quot;:&quot;your-project:dataset_2_test&quot;,</span>
+<span class="sd">                      &quot;datasetReference&quot;:{</span>
+<span class="sd">                         &quot;projectId&quot;:&quot;your-project&quot;,</span>
+<span class="sd">                         &quot;datasetId&quot;:&quot;dataset_2_test&quot;</span>
+<span class="sd">                      }</span>
+<span class="sd">                   },</span>
+<span class="sd">                   {</span>
+<span class="sd">                      &quot;kind&quot;:&quot;bigquery#dataset&quot;,</span>
+<span class="sd">                      &quot;location&quot;:&quot;US&quot;,</span>
+<span class="sd">                      &quot;id&quot;:&quot;your-project:dataset_1_test&quot;,</span>
+<span class="sd">                      &quot;datasetReference&quot;:{</span>
+<span class="sd">                         &quot;projectId&quot;:&quot;your-project&quot;,</span>
+<span class="sd">                         &quot;datasetId&quot;:&quot;dataset_1_test&quot;</span>
+<span class="sd">                      }</span>
+<span class="sd">                   }</span>
+<span class="sd">                ]</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">dataset_project_id</span> <span class="o">=</span> <span class="n">project_id</span> <span class="k">if</span> <span class="n">project_id</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span>
+
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">datasets_list</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">datasets</span><span class="p">()</span><span class="o">.</span><span class="n">list</span><span class="p">(</span>
+                <span class="n">projectId</span><span class="o">=</span><span class="n">dataset_project_id</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()[</span><span class="s1">&#39;datasets&#39;</span><span class="p">]</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Datasets List: </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">datasets_list</span><span class="p">))</span>
+
+        <span class="k">except</span> <span class="n">HttpError</span> <span class="k">as</span> <span class="n">err</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+                <span class="s1">&#39;BigQuery job failed. Error was: </span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">err</span><span class="o">.</span><span class="n">content</span><span class="p">))</span>
+
+        <span class="k">return</span> <span class="n">datasets_list</span>
+
+    <span class="k">def</span> <span class="nf">insert_all</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">,</span>
+                   <span class="n">rows</span><span class="p">,</span> <span class="n">ignore_unknown_values</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+                   <span class="n">skip_invalid_rows</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">fail_on_error</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Method to stream data into BigQuery one record at a time without needing</span>
+<span class="sd">        to run a load job</span>
+
+<span class="sd">        .. seealso::</span>
+<span class="sd">            For more information, see:</span>
+<span class="sd">            https://cloud.google.com/bigquery/docs/reference/rest/v2/tabledata/insertAll</span>
+
+<span class="sd">        :param project_id: The name of the project where we have the table</span>
+<span class="sd">        :type project_id: str</span>
+<span class="sd">        :param dataset_id: The name of the dataset where we have the table</span>
+<span class="sd">        :type dataset_id: str</span>
+<span class="sd">        :param table_id: The name of the table</span>
+<span class="sd">        :type table_id: str</span>
+<span class="sd">        :param rows: the rows to insert</span>
+<span class="sd">        :type rows: list</span>
+
+<span class="sd">        **Example or rows**:</span>
+<span class="sd">            rows=[{&quot;json&quot;: {&quot;a_key&quot;: &quot;a_value_0&quot;}}, {&quot;json&quot;: {&quot;a_key&quot;: &quot;a_value_1&quot;}}]</span>
+
+<span class="sd">        :param ignore_unknown_values: [Optional] Accept rows that contain values</span>
+<span class="sd">            that do not match the schema. The unknown values are ignored.</span>
+<span class="sd">            The default value  is false, which treats unknown values as errors.</span>
+<span class="sd">        :type ignore_unknown_values: bool</span>
+<span class="sd">        :param skip_invalid_rows: [Optional] Insert all valid rows of a request,</span>
+<span class="sd">            even if invalid rows exist. The default value is false, which causes</span>
+<span class="sd">            the entire request to fail if any invalid rows exist.</span>
+<span class="sd">        :type skip_invalid_rows: bool</span>
+<span class="sd">        :param fail_on_error: [Optional] Force the task to fail if any errors occur.</span>
+<span class="sd">            The default value is false, which indicates the task should not fail</span>
+<span class="sd">            even if any insertion errors occur.</span>
+<span class="sd">        :type fail_on_error: bool</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+
+        <span class="n">dataset_project_id</span> <span class="o">=</span> <span class="n">project_id</span> <span class="k">if</span> <span class="n">project_id</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span>
+
+        <span class="n">body</span> <span class="o">=</span> <span class="p">{</span>
+            <span class="s2">&quot;rows&quot;</span><span class="p">:</span> <span class="n">rows</span><span class="p">,</span>
+            <span class="s2">&quot;ignoreUnknownValues&quot;</span><span class="p">:</span> <span class="n">ignore_unknown_values</span><span class="p">,</span>
+            <span class="s2">&quot;kind&quot;</span><span class="p">:</span> <span class="s2">&quot;bigquery#tableDataInsertAllRequest&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;skipInvalidRows&quot;</span><span class="p">:</span> <span class="n">skip_invalid_rows</span><span class="p">,</span>
+        <span class="p">}</span>
+
+        <span class="k">try</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Inserting </span><span class="si">{}</span><span class="s1"> row(s) into Table </span><span class="si">{}</span><span class="s1">:</span><span class="si">{}</span><span class="s1">.</span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span cla [...]
+                <span class="nb">len</span><span class="p">(</span><span class="n">rows</span><span class="p">),</span> <span class="n">dataset_project_id</span><span class="p">,</span>
+                <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">))</span>
+
+            <span class="n">resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tabledata</span><span class="p">()</span><span class="o">.</span><span class="n">insertAll</span><span class="p">(</span>
+                <span class="n">projectId</span><span class="o">=</span><span class="n">dataset_project_id</span><span class="p">,</span> <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span>
+                <span class="n">tableId</span><span class="o">=</span><span class="n">table_id</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="n">body</span>
+            <span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+
+            <span class="k">if</span> <span class="s1">&#39;insertErrors&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">resp</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;All row(s) inserted successfully: </span><span class="si">{}</span><span class="s1">:</span><span class="si">{}</span><span class="s1">.</span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+                    <span class="n">dataset_project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">))</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">error_msg</span> <span class="o">=</span> <span class="s1">&#39;</span><span class="si">{}</span><span class="s1"> insert error(s) occured: </span><span class="si">{}</span><span class="s1">:</span><span class="si">{}</span><span class="s1">.</span><span class="si">{}</span><span class="s1">. Details: </span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+                    <span class="nb">len</span><span class="p">(</span><span class="n">resp</span><span class="p">[</span><span class="s1">&#39;insertErrors&#39;</span><span class="p">]),</span>
+                    <span class="n">dataset_project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">,</span> <span class="n">resp</span><span class="p">[</span><span class="s1">&#39;insertErrors&#39;</span><span class="p">])</span>
+                <span class="k">if</span> <span class="n">fail_on_error</span><span class="p">:</span>
+                    <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+                        <span class="s1">&#39;BigQuery job failed. Error was: </span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error_msg</span><span class="p">)</span>
+                    <span class="p">)</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">error_msg</span><span class="p">)</span>
+        <span class="k">except</span> <span class="n">HttpError</span> <span class="k">as</span> <span class="n">err</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+                <span class="s1">&#39;BigQuery job failed. Error was: </span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">err</span><span class="o">.</span><span class="n">content</span><span class="p">)</span>
+            <span class="p">)</span>
+
+
+<span class="k">class</span> <span class="nc">BigQueryCursor</span><span class="p">(</span><span class="n">BigQueryBaseCursor</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    A very basic BigQuery PEP 249 cursor implementation. The PyHive PEP 249</span>
+<span class="sd">    implementation was used as a reference:</span>
+
+<span class="sd">    https://github.com/dropbox/PyHive/blob/master/pyhive/presto.py</span>
+<span class="sd">    https://github.com/dropbox/PyHive/blob/master/pyhive/common.py</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">service</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">use_legacy_sql</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">location</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">BigQueryCursor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
+            <span class="n">service</span><span class="o">=</span><span class="n">service</span><span class="p">,</span>
+            <span class="n">project_id</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
+            <span class="n">use_legacy_sql</span><span class="o">=</span><span class="n">use_legacy_sql</span><span class="p">,</span>
+            <span class="n">location</span><span class="o">=</span><span class="n">location</span><span class="p">,</span>
+        <span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">buffersize</span> <span class="o">=</span> <span class="kc">None</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">page_token</span> <span class="o">=</span> <span class="kc">None</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">job_id</span> <span class="o">=</span> <span class="kc">None</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">buffer</span> <span class="o">=</span> <span class="p">[]</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">all_pages_loaded</span> <span class="o">=</span> <span class="kc">False</span>
+
+    <span class="nd">@property</span>
+    <span class="k">def</span> <span class="nf">description</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot; The schema description method is not currently implemented. &quot;&quot;&quot;</span>
+        <span class="k">raise</span> <span class="ne">NotImplementedError</span>
+
+    <span class="k">def</span> <span class="nf">close</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot; By default, do nothing &quot;&quot;&quot;</span>
+        <span class="k">pass</span>
+
+    <span class="nd">@property</span>
+    <span class="k">def</span> <span class="nf">rowcount</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot; By default, return -1 to indicate that this is not supported. &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="o">-</span><span class="mi">1</span>
+
+    <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">operation</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Executes a BigQuery query, and returns the job ID.</span>
+
+<span class="sd">        :param operation: The query to execute.</span>
+<span class="sd">        :type operation: string</span>
+<span class="sd">        :param parameters: Parameters to substitute into the query.</span>
+<span class="sd">        :type parameters: dict</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">sql</span> <span class="o">=</span> <span class="n">_bind_parameters</span><span class="p">(</span><span class="n">operation</span><span class="p">,</span>
+                               <span class="n">parameters</span><span class="p">)</span> <span class="k">if</span> <span class="n">parameters</span> <span class="k">else</span> <span class="n">operation</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">job_id</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_query</span><span class="p">(</span><span class="n">sql</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">executemany</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">operation</span><span class="p">,</span> <span class="n">seq_of_parameters</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Execute a BigQuery query multiple times with different parameters.</span>
+
+<span class="sd">        :param operation: The query to execute.</span>
+<span class="sd">        :type operation: string</span>
+<span class="sd">        :param seq_of_parameters: List of dictionary parameters to substitute into the</span>
+<span class="sd">            query.</span>
+<span class="sd">        :type seq_of_parameters: list</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">for</span> <span class="n">parameters</span> <span class="ow">in</span> <span class="n">seq_of_parameters</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">operation</span><span class="p">,</span> <span class="n">parameters</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">fetchone</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot; Fetch the next row of a query result set. &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">next</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="nf">next</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Helper method for fetchone, which returns the next row from a buffer.</span>
+<span class="sd">        If the buffer is empty, attempts to paginate through the result set for</span>
+<span class="sd">        the next page, and load it into the buffer.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">job_id</span><span class="p">:</span>
+            <span class="k">return</span> <span class="kc">None</span>
+
+        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">buffer</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">all_pages_loaded</span><span class="p">:</span>
+                <span class="k">return</span> <span class="kc">None</span>
+
+            <span class="n">query_results</span> <span class="o">=</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">jobs</span><span class="p">()</span><span class="o">.</span><span class="n">getQueryResults</span><span class="p">(</span>
+                <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+                <span class="n">jobId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">job_id</span><span class="p">,</span>
+                <span class="n">pageToken</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">page_token</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">())</span>
+
+            <span class="k">if</span> <span class="s1">&#39;rows&#39;</span> <span class="ow">in</span> <span class="n">query_results</span> <span class="ow">and</span> <span class="n">query_results</span><span class="p">[</span><span class="s1">&#39;rows&#39;</span><span class="p">]:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">page_token</span> <span class="o">=</span> <span class="n">query_results</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;pageToken&#39;</span><span class="p">)</span>
+                <span class="n">fields</span> <span class="o">=</span> <span class="n">query_results</span><span class="p">[</span><span class="s1">&#39;schema&#39;</span><span class="p">][</span><span class="s1">&#39;fields&#39;</span><span class="p">]</span>
+                <span class="n">col_types</span> <span class="o">=</span> <span class="p">[</span><span class="n">field</span><span class="p">[</span><span class="s1">&#39;type&#39;</span><span class="p">]</span> <span class="k">for</span> <span class="n">field</span> <span class="ow">in</span> <span class="n">fields</span><span class="p">]</span>
+                <span class="n">rows</span> <span class="o">=</span> <span class="n">query_results</span><span class="p">[</span><span class="s1">&#39;rows&#39;</span><span class="p">]</span>
+
+                <span class="k">for</span> <span class="n">dict_row</span> <span class="ow">in</span> <span class="n">rows</span><span class="p">:</span>
+                    <span class="n">typed_row</span> <span class="o">=</span> <span class="p">([</span>
+                        <span class="n">_bq_cast</span><span class="p">(</span><span class="n">vs</span><span class="p">[</span><span class="s1">&#39;v&#39;</span><span class="p">],</span> <span class="n">col_types</span><span class="p">[</span><span class="n">idx</span><span class="p">])</span>
+                        <span class="k">for</span> <span class="n">idx</span><span class="p">,</span> <span class="n">vs</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">dict_row</span><span class="p">[</span><span class="s1">&#39;f&#39;</span><span class="p">])</span>
+                    <span class="p">])</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">buffer</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">typed_row</span><span class="p">)</span>
+
+                <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">page_token</span><span class="p">:</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">all_pages_loaded</span> <span class="o">=</span> <span class="kc">True</span>
+
+            <span class="k">else</span><span class="p">:</span>
+                <span class="c1"># Reset all state since we&#39;ve exhausted the results.</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">page_token</span> <span class="o">=</span> <span class="kc">None</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">job_id</span> <span class="o">=</span> <span class="kc">None</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">page_token</span> <span class="o">=</span> <span class="kc">None</span>
+                <span class="k">return</span> <span class="kc">None</span>
+
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">buffer</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">fetchmany</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">size</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Fetch the next set of rows of a query result, returning a sequence of sequences</span>
+<span class="sd">        (e.g. a list of tuples). An empty sequence is returned when no more rows are</span>
+<span class="sd">        available. The number of rows to fetch per call is specified by the parameter.</span>
+<span class="sd">        If it is not given, the cursor&#39;s arraysize determines the number of rows to be</span>
+<span class="sd">        fetched. The method should try to fetch as many rows as indicated by the size</span>
+<span class="sd">        parameter. If this is not possible due to the specified number of rows not being</span>
+<span class="sd">        available, fewer rows may be returned. An :py:class:`~pyhive.exc.Error`</span>
+<span class="sd">        (or subclass) exception is raised if the previous call to</span>
+<span class="sd">        :py:meth:`execute` did not produce any result set or no call was issued yet.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">if</span> <span class="n">size</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">size</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">arraysize</span>
+        <span class="n">result</span> <span class="o">=</span> <span class="p">[]</span>
+        <span class="k">for</span> <span class="n">_</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">size</span><span class="p">):</span>
+            <span class="n">one</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">fetchone</span><span class="p">()</span>
+            <span class="k">if</span> <span class="n">one</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+                <span class="k">break</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">result</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">one</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">result</span>
+
+    <span class="k">def</span> <span class="nf">fetchall</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Fetch all (remaining) rows of a query result, returning them as a sequence of</span>
+<span class="sd">        sequences (e.g. a list of tuples).</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">result</span> <span class="o">=</span> <span class="p">[]</span>
+        <span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
+            <span class="n">one</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">fetchone</span><span class="p">()</span>
+            <span class="k">if</span> <span class="n">one</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+                <span class="k">break</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">result</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">one</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">result</span>
+
+    <span class="k">def</span> <span class="nf">get_arraysize</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot; Specifies the number of rows to fetch at a time with .fetchmany() &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_buffersize</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">buffersize</span> <span class="k">else</span> <span class="mi">1</span>
+
+    <span class="k">def</span> <span class="nf">set_arraysize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">arraysize</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot; Specifies the number of rows to fetch at a time with .fetchmany() &quot;&quot;&quot;</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">buffersize</span> <span class="o">=</span> <span class="n">arraysize</span>
+
+    <span class="n">arraysize</span> <span class="o">=</span> <span class="nb">property</span><span class="p">(</span><span class="n">get_arraysize</span><span class="p">,</span> <span class="n">set_arraysize</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">setinputsizes</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sizes</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot; Does nothing by default &quot;&quot;&quot;</span>
+        <span class="k">pass</span>
+
+    <span class="k">def</span> <span class="nf">setoutputsize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">size</span><span class="p">,</span> <span class="n">column</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot; Does nothing by default &quot;&quot;&quot;</span>
+        <span class="k">pass</span>
+
+
+<span class="k">def</span> <span class="nf">_bind_parameters</span><span class="p">(</span><span class="n">operation</span><span class="p">,</span> <span class="n">parameters</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot; Helper method that binds parameters to a SQL query. &quot;&quot;&quot;</span>
+    <span class="c1"># inspired by MySQL Python Connector (conversion.py)</span>
+    <span class="n">string_parameters</span> <span class="o">=</span> <span class="p">{}</span>
+    <span class="k">for</span> <span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">value</span><span class="p">)</span> <span class="ow">in</span> <span class="n">iteritems</span><span class="p">(</span><span class="n">parameters</span><span class="p">):</span>
+        <span class="k">if</span> <span class="n">value</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">string_parameters</span><span class="p">[</span><span class="n">name</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;NULL&#39;</span>
+        <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="n">basestring</span><span class="p">):</span>
+            <span class="n">string_parameters</span><span class="p">[</span><span class="n">name</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;&#39;&quot;</span> <span class="o">+</span> <span class="n">_escape</span><span class="p">(</span><span class="n">value</span><span class="p">)</span> <span class="o">+</span> <span class="s2">&quot;&#39;&quot;</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">string_parameters</span><span class="p">[</span><span class="n">name</span><span class="p">]</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">value</span><span class="p">)</span>
+    <span class="k">return</span> <span class="n">operation</span> <span class="o">%</span> <span class="n">string_parameters</span>
+
+
+<span class="k">def</span> <span class="nf">_escape</span><span class="p">(</span><span class="n">s</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot; Helper method that escapes parameters to a SQL query. &quot;&quot;&quot;</span>
+    <span class="n">e</span> <span class="o">=</span> <span class="n">s</span>
+    <span class="n">e</span> <span class="o">=</span> <span class="n">e</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\\</span><span class="s1">&#39;</span><span class="p">,</span> <span class="s1">&#39;</span><span class="se">\\\\</span><span class="s1">&#39;</span><span class="p">)</span>
+    <span class="n">e</span> <span class="o">=</span> <span class="n">e</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">,</span> <span class="s1">&#39;</span><span class="se">\\</span><span class="s1">n&#39;</span><span class="p">)</span>
+    <span class="n">e</span> <span class="o">=</span> <span class="n">e</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\r</span><span class="s1">&#39;</span><span class="p">,</span> <span class="s1">&#39;</span><span class="se">\\</span><span class="s1">r&#39;</span><span class="p">)</span>
+    <span class="n">e</span> <span class="o">=</span> <span class="n">e</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;&#39;&quot;</span><span class="p">,</span> <span class="s2">&quot;</span><span class="se">\\</span><span class="s2">&#39;&quot;</span><span class="p">)</span>
+    <span class="n">e</span> <span class="o">=</span> <span class="n">e</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;&quot;&#39;</span><span class="p">,</span> <span class="s1">&#39;</span><span class="se">\\</span><span class="s1">&quot;&#39;</span><span class="p">)</span>
+    <span class="k">return</span> <span class="n">e</span>
+
+
+<span class="k">def</span> <span class="nf">_bq_cast</span><span class="p">(</span><span class="n">string_field</span><span class="p">,</span> <span class="n">bq_type</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Helper method that casts a BigQuery row to the appropriate data types.</span>
+<span class="sd">    This is useful because BigQuery returns all fields as strings.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">if</span> <span class="n">string_field</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="k">return</span> <span class="kc">None</span>
+    <span class="k">elif</span> <span class="n">bq_type</span> <span class="o">==</span> <span class="s1">&#39;INTEGER&#39;</span><span class="p">:</span>
+        <span class="k">return</span> <span class="nb">int</span><span class="p">(</span><span class="n">string_field</span><span class="p">)</span>
+    <span class="k">elif</span> <span class="n">bq_type</span> <span class="o">==</span> <span class="s1">&#39;FLOAT&#39;</span> <span class="ow">or</span> <span class="n">bq_type</span> <span class="o">==</span> <span class="s1">&#39;TIMESTAMP&#39;</span><span class="p">:</span>
+        <span class="k">return</span> <span class="nb">float</span><span class="p">(</span><span class="n">string_field</span><span class="p">)</span>
+    <span class="k">elif</span> <span class="n">bq_type</span> <span class="o">==</span> <span class="s1">&#39;BOOLEAN&#39;</span><span class="p">:</span>
+        <span class="k">if</span> <span class="n">string_field</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">&#39;true&#39;</span><span class="p">,</span> <span class="s1">&#39;false&#39;</span><span class="p">]:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;</span><span class="si">{}</span><span class="s2"> must have value &#39;true&#39; or &#39;false&#39;&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+                <span class="n">string_field</span><span class="p">))</span>
+        <span class="k">return</span> <span class="n">string_field</span> <span class="o">==</span> <span class="s1">&#39;true&#39;</span>
+    <span class="k">else</span><span class="p">:</span>
+        <span class="k">return</span> <span class="n">string_field</span>
+
+
+<span class="k">def</span> <span class="nf">_split_tablename</span><span class="p">(</span><span class="n">table_input</span><span class="p">,</span> <span class="n">default_project_id</span><span class="p">,</span> <span class="n">var_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+
+    <span class="k">if</span> <span class="s1">&#39;.&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">table_input</span><span class="p">:</span>
+        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+            <span class="s1">&#39;Expected target table name in the format of &#39;</span>
+            <span class="s1">&#39;&lt;dataset&gt;.&lt;table&gt;. Got: </span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">table_input</span><span class="p">))</span>
+
+    <span class="k">if</span> <span class="ow">not</span> <span class="n">default_project_id</span><span class="p">:</span>
+        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;INTERNAL: No default project is specified&quot;</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">var_print</span><span class="p">(</span><span class="n">var_name</span><span class="p">):</span>
+        <span class="k">if</span> <span class="n">var_name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">return</span> <span class="s2">&quot;&quot;</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="k">return</span> <span class="s2">&quot;Format exception for </span><span class="si">{var}</span><span class="s2">: &quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">var</span><span class="o">=</span><span class="n">var_name</span><span class="p">)</span>
+
+    <span class="k">if</span> <span class="n">table_input</span><span class="o">.</span><span class="n">count</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">)</span> <span class="o">+</span> <span class="n">table_input</span><span class="o">.</span><span class="n">count</span><span class="p">(</span><span class="s1">&#39;:&#39;</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">3</span><span class="p">:</span>
+        <span class="k">raise</span> <span class="ne">Exception</span><span class="p">((</span><span class="s1">&#39;</span><span class="si">{var}</span><span class="s1">Use either : or . to specify project &#39;</span>
+                         <span class="s1">&#39;got </span><span class="si">{input}</span><span class="s1">&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+                             <span class="n">var</span><span class="o">=</span><span class="n">var_print</span><span class="p">(</span><span class="n">var_name</span><span class="p">),</span> <span class="nb">input</span><span class="o">=</span><span class="n">table_input</span><span class="p">))</span>
+    <span class="n">cmpt</span> <span class="o">=</span> <span class="n">table_input</span><span class="o">.</span><span class="n">rsplit</span><span class="p">(</span><span class="s1">&#39;:&#39;</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
+    <span class="n">project_id</span> <span class="o">=</span> <span class="kc">None</span>
+    <span class="n">rest</span> <span class="o">=</span> <span class="n">table_input</span>
+    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">cmpt</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
+        <span class="n">project_id</span> <span class="o">=</span> <span class="kc">None</span>
+        <span class="n">rest</span> <span class="o">=</span> <span class="n">cmpt</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
+    <span class="k">elif</span> <span class="nb">len</span><span class="p">(</span><span class="n">cmpt</span><span class="p">)</span> <span class="o">==</span> <span class="mi">2</span> <span class="ow">and</span> <span class="n">cmpt</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">count</span><span class="p">(</span><span class="s1">&#39;:&#39;</span><span class="p">)</span> <span class="o">&lt;=</span> <span class= [...]
+        <span class="k">if</span> <span class="n">cmpt</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">count</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">)</span> <span class="o">!=</span> <span class="mi">2</span><span class="p">:</span>
+            <span class="n">project_id</span> <span class="o">=</span> <span class="n">cmpt</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
+            <span class="n">rest</span> <span class="o">=</span> <span class="n">cmpt</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
+    <span class="k">else</span><span class="p">:</span>
+        <span class="k">raise</span> <span class="ne">Exception</span><span class="p">((</span><span class="s1">&#39;</span><span class="si">{var}</span><span class="s1">Expect format of (&lt;project:)&lt;dataset&gt;.&lt;table&gt;, &#39;</span>
+                         <span class="s1">&#39;got </span><span class="si">{input}</span><span class="s1">&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+                             <span class="n">var</span><span class="o">=</span><span class="n">var_print</span><span class="p">(</span><span class="n">var_name</span><span class="p">),</span> <span class="nb">input</span><span class="o">=</span><span class="n">table_input</span><span class="p">))</span>
+
+    <span class="n">cmpt</span> <span class="o">=</span> <span class="n">rest</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">)</span>
+    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">cmpt</span><span class="p">)</span> <span class="o">==</span> <span class="mi">3</span><span class="p">:</span>
+        <span class="k">if</span> <span class="n">project_id</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                <span class="s2">&quot;</span><span class="si">{var}</span><span class="s2">Use either : or . to specify project&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+                    <span class="n">var</span><span class="o">=</span><span class="n">var_print</span><span class="p">(</span><span class="n">var_name</span><span class="p">)))</span>
+        <span class="n">project_id</span> <span class="o">=</span> <span class="n">cmpt</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
+        <span class="n">dataset_id</span> <span class="o">=</span> <span class="n">cmpt</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
+        <span class="n">table_id</span> <span class="o">=</span> <span class="n">cmpt</span><span class="p">[</span><span class="mi">2</span><span class="p">]</span>
+
+    <span class="k">elif</span> <span class="nb">len</span><span class="p">(</span><span class="n">cmpt</span><span class="p">)</span> <span class="o">==</span> <span class="mi">2</span><span class="p">:</span>
+        <span class="n">dataset_id</span> <span class="o">=</span> <span class="n">cmpt</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
+        <span class="n">table_id</span> <span class="o">=</span> <span class="n">cmpt</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
+    <span class="k">else</span><span class="p">:</span>
+        <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span>
+            <span class="p">(</span><span class="s1">&#39;</span><span class="si">{var}</span><span class="s1">Expect format of (&lt;project.|&lt;project:)&lt;dataset&gt;.&lt;table&gt;, &#39;</span>
+             <span class="s1">&#39;got </span><span class="si">{input}</span><span class="s1">&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">var</span><span class="o">=</span><span class="n">var_print</span><span class="p">(</span><span class="n">var_name</span><span class="p">),</span> <span class="nb">input</span><span class="o">=</span><span class="n">table_input</span><span class="p">))</span>
+
+    <span class="k">if</span> <span class="n">project_id</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="k">if</span> <span class="n">var_name</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">log</span> <span class="o">=</span> <span class="n">LoggingMixin</span><span class="p">()</span><span class="o">.</span><span class="n">log</span>
+            <span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Project not included in </span><span class="si">{var}</span><span class="s1">: </span><span class="si">{input}</span><span class="s1">; &#39;</span>
+                     <span class="s1">&#39;using project &quot;</span><span class="si">{project}</span><span class="s1">&quot;&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+                         <span class="n">var</span><span class="o">=</span><span class="n">var_name</span><span class="p">,</span>
+                         <span class="nb">input</span><span class="o">=</span><span class="n">table_input</span><span class="p">,</span>
+                         <span class="n">project</span><span class="o">=</span><span class="n">default_project_id</span><span class="p">))</span>
+        <span class="n">project_id</span> <span class="o">=</span> <span class="n">default_project_id</span>
+
+    <span class="k">return</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span>
+
+
+<span class="k">def</span> <span class="nf">_cleanse_time_partitioning</span><span class="p">(</span><span class="n">destination_dataset_table</span><span class="p">,</span> <span class="n">time_partitioning_in</span><span class="p">):</span>
+    <span class="c1"># if it is a partitioned table ($ is in the table name) add partition load option</span>
+
+    <span class="k">if</span> <span class="n">time_partitioning_in</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="n">time_partitioning_in</span> <span class="o">=</span> <span class="p">{}</span>
+
+    <span class="n">time_partitioning_out</span> <span class="o">=</span> <span class="p">{}</span>
+    <span class="k">if</span> <span class="n">destination_dataset_table</span> <span class="ow">and</span> <span class="s1">&#39;$&#39;</span> <span class="ow">in</span> <span class="n">destination_dataset_table</span><span class="p">:</span>
+        <span class="n">time_partitioning_out</span><span class="p">[</span><span class="s1">&#39;type&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;DAY&#39;</span>
+    <span class="n">time_partitioning_out</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">time_partitioning_in</span><span class="p">)</span>
+    <span class="k">return</span> <span class="n">time_partitioning_out</span>
+
+
+<span class="k">def</span> <span class="nf">_validate_value</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">value</span><span class="p">,</span> <span class="n">expected_type</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot; function to check expected type and raise</span>
+<span class="sd">    error if type is not correct &quot;&quot;&quot;</span>
+    <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="n">expected_type</span><span class="p">):</span>
+        <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s2">&quot;</span><span class="si">{}</span><span class="s2"> argument must have a type </span><span class="si">{}</span><span class="s2"> not </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+            <span class="n">key</span><span class="p">,</span> <span class="n">expected_type</span><span class="p">,</span> <span class="nb">type</span><span class="p">(</span><span class="n">value</span><span class="p">)))</span>
+
+
+<span class="k">def</span> <span class="nf">_api_resource_configs_duplication_check</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">value</span><span class="p">,</span> <span class="n">config_dict</span><span class="p">,</span>
+                                            <span class="n">config_dict_name</span><span class="o">=</span><span class="s1">&#39;api_resource_configs&#39;</span><span class="p">):</span>
+    <span class="k">if</span> <span class="n">key</span> <span class="ow">in</span> <span class="n">config_dict</span> <span class="ow">and</span> <span class="n">value</span> <span class="o">!=</span> <span class="n">config_dict</span><span class="p">[</span><span class="n">key</span><span class="p">]:</span>
+        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Values of </span><span class="si">{param_name}</span><span class="s2"> param are duplicated. &quot;</span>
+                         <span class="s2">&quot;</span><span class="si">{dict_name}</span><span class="s2"> contained </span><span class="si">{param_name}</span><span class="s2"> param &quot;</span>
+                         <span class="s2">&quot;in `query` config and </span><span class="si">{param_name}</span><span class="s2"> was also provided &quot;</span>
+                         <span class="s2">&quot;with arg to run_query() method. Please remove duplicates.&quot;</span>
+                         <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">param_name</span><span class="o">=</span><span class="n">key</span><span class="p">,</span> <span class="n">dict_name</span><span class="o">=</span><span class="n">config_dict_name</span><span class="p">))</span>
+</pre></div>
+
+           </div>
+           
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    
+    
+      <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+        <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+        <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+        <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+        <script type="text/javascript" src="../../../../_static/language_data.js"></script>
+    
+
+  
+
+  <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.Navigation.enable(true);
+      });
+  </script> 
+
+</body>
+</html>
\ No newline at end of file
diff --git a/1.10.2/_modules/airflow/contrib/hooks/cassandra_hook.html b/1.10.2/_modules/airflow/contrib/hooks/cassandra_hook.html
new file mode 100644
index 0000000..d51e64c
--- /dev/null
+++ b/1.10.2/_modules/airflow/contrib/hooks/cassandra_hook.html
@@ -0,0 +1,416 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>airflow.contrib.hooks.cassandra_hook &mdash; Airflow Documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+    <link rel="index" title="Index" href="../../../../genindex.html" />
+    <link rel="search" title="Search" href="../../../../search.html" /> 
+
+  
+  <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../../index.html" class="icon icon-home"> Airflow
+          
+
+          
+          </a>
+
+          
+            
+            
+              <div class="version">
+                1.10.2
+              </div>
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../metrics.html">Metrics</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../../index.html">Airflow</a>
+        
+      </nav>
+
+
+      <div class="wy-nav-content">
+        
+        <div class="rst-content">
+        
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../../index.html">Module code</a> &raquo;</li>
+        
+      <li>airflow.contrib.hooks.cassandra_hook</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for airflow.contrib.hooks.cassandra_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements.  See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership.  The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># &quot;License&quot;); you may not use this file except in compliance</span>
+<span class="c1"># with the License.  You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1">#   http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># &quot;AS IS&quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied.  See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">cassandra.cluster</span> <span class="k">import</span> <span class="n">Cluster</span>
+<span class="kn">from</span> <span class="nn">cassandra.policies</span> <span class="k">import</span> <span class="p">(</span><span class="n">RoundRobinPolicy</span><span class="p">,</span> <span class="n">DCAwareRoundRobinPolicy</span><span class="p">,</span>
+                                <span class="n">TokenAwarePolicy</span><span class="p">,</span> <span class="n">WhiteListRoundRobinPolicy</span><span class="p">)</span>
+<span class="kn">from</span> <span class="nn">cassandra.auth</span> <span class="k">import</span> <span class="n">PlainTextAuthProvider</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
+
+
+<div class="viewcode-block" id="CassandraHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.cassandra_hook.CassandraHook">[docs]</a><span class="k">class</span> <span class="nc">CassandraHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Hook used to interact with Cassandra</span>
+
+<span class="sd">    Contact points can be specified as a comma-separated string in the &#39;hosts&#39;</span>
+<span class="sd">    field of the connection.</span>
+
+<span class="sd">    Port can be specified in the port field of the connection.</span>
+
+<span class="sd">    If SSL is enabled in Cassandra, pass in a dict in the extra field as kwargs for</span>
+<span class="sd">    ``ssl.wrap_socket()``. For example:</span>
+<span class="sd">            {</span>
+<span class="sd">                &#39;ssl_options&#39; : {</span>
+<span class="sd">                    &#39;ca_certs&#39; : PATH_TO_CA_CERTS</span>
+<span class="sd">                }</span>
+<span class="sd">            }</span>
+
+<span class="sd">    Default load balancing policy is RoundRobinPolicy. To specify a different LB policy:</span>
+<span class="sd">        - DCAwareRoundRobinPolicy</span>
+<span class="sd">            {</span>
+<span class="sd">                &#39;load_balancing_policy&#39;: &#39;DCAwareRoundRobinPolicy&#39;,</span>
+<span class="sd">                &#39;load_balancing_policy_args&#39;: {</span>
+<span class="sd">                    &#39;local_dc&#39;: LOCAL_DC_NAME,                      // optional</span>
+<span class="sd">                    &#39;used_hosts_per_remote_dc&#39;: SOME_INT_VALUE,     // optional</span>
+<span class="sd">                }</span>
+<span class="sd">             }</span>
+<span class="sd">        - WhiteListRoundRobinPolicy</span>
+<span class="sd">            {</span>
+<span class="sd">                &#39;load_balancing_policy&#39;: &#39;WhiteListRoundRobinPolicy&#39;,</span>
+<span class="sd">                &#39;load_balancing_policy_args&#39;: {</span>
+<span class="sd">                    &#39;hosts&#39;: [&#39;HOST1&#39;, &#39;HOST2&#39;, &#39;HOST3&#39;]</span>
+<span class="sd">                }</span>
+<span class="sd">            }</span>
+<span class="sd">        - TokenAwarePolicy</span>
+<span class="sd">            {</span>
+<span class="sd">                &#39;load_balancing_policy&#39;: &#39;TokenAwarePolicy&#39;,</span>
+<span class="sd">                &#39;load_balancing_policy_args&#39;: {</span>
+<span class="sd">                    &#39;child_load_balancing_policy&#39;: CHILD_POLICY_NAME, // optional</span>
+<span class="sd">                    &#39;child_load_balancing_policy_args&#39;: { ... }       // optional</span>
+<span class="sd">                }</span>
+<span class="sd">            }</span>
+
+<span class="sd">    For details of the Cluster config, see cassandra.cluster.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cassandra_conn_id</span><span class="o">=</span><span class="s1">&#39;cassandra_default&#39;</span><span class="p">):</span>
+        <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="n">cassandra_conn_id</span><span class="p">)</span>
+
+        <span class="n">conn_config</span> <span class="o">=</span> <span class="p">{}</span>
+        <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">host</span><span class="p">:</span>
+            <span class="n">conn_config</span><span class="p">[</span><span class="s1">&#39;contact_points&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">host</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;,&#39;</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">port</span><span class="p">:</span>
+            <span class="n">conn_config</span><span class="p">[</span><span class="s1">&#39;port&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">port</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">:</span>
+            <span class="n">conn_config</span><span class="p">[</span><span class="s1">&#39;auth_provider&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">PlainTextAuthProvider</span><span class="p">(</span>
+                <span class="n">username</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">,</span> <span class="n">password</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">)</span>
+
+        <span class="n">policy_name</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;load_balancing_policy&#39;</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+        <span class="n">policy_args</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;load_balancing_policy_args&#39;</span><span class="p">,</span> <span class="p">{})</span>
+        <span class="n">lb_policy</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_lb_policy</span><span class="p">(</span><span class="n">policy_name</span><span class="p">,</span> <span class="n">policy_args</span><span class="p">)</span>
+        <span class="k">if</span> <span class="n">lb_policy</span><span class="p">:</span>
+            <span class="n">conn_config</span><span class="p">[</span><span class="s1">&#39;load_balancing_policy&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">lb_policy</span>
+
+        <span class="n">cql_version</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;cql_version&#39;</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+        <span class="k">if</span> <span class="n">cql_version</span><span class="p">:</span>
+            <span class="n">conn_config</span><span class="p">[</span><span class="s1">&#39;cql_version&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">cql_version</span>
+
+        <span class="n">ssl_options</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;ssl_options&#39;</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+        <span class="k">if</span> <span class="n">ssl_options</span><span class="p">:</span>
+            <span class="n">conn_config</span><span class="p">[</span><span class="s1">&#39;ssl_options&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">ssl_options</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">cluster</span> <span class="o">=</span> <span class="n">Cluster</span><span class="p">(</span><span class="o">**</span><span class="n">conn_config</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">keyspace</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">schema</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">session</span> <span class="o">=</span> <span class="kc">None</span>
+
+<div class="viewcode-block" id="CassandraHook.get_conn"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.cassandra_hook.CassandraHook.get_conn">[docs]</a>    <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Returns a cassandra Session object</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span> <span class="ow">and</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">is_shutdown</span><span class="p">:</span>
+            <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">session</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cluster</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">keyspace</span><span class="p">)</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span></div>
+
+    <span class="k">def</span> <span class="nf">get_cluster</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">cluster</span>
+
+<div class="viewcode-block" id="CassandraHook.shutdown_cluster"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.cassandra_hook.CassandraHook.shutdown_cluster">[docs]</a>    <span class="k">def</span> <span class="nf">shutdown_cluster</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Closes all sessions and connections associated with this Cluster.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">cluster</span><span class="o">.</span><span class="n">is_shutdown</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">cluster</span><span class="o">.</span><span class="n">shutdown</span><span class="p">()</span></div>
+
+    <span class="nd">@staticmethod</span>
+    <span class="k">def</span> <span class="nf">get_lb_policy</span><span class="p">(</span><span class="n">policy_name</span><span class="p">,</span> <span class="n">policy_args</span><span class="p">):</span>
+        <span class="n">policies</span> <span class="o">=</span> <span class="p">{</span>
+            <span class="s1">&#39;RoundRobinPolicy&#39;</span><span class="p">:</span> <span class="n">RoundRobinPolicy</span><span class="p">,</span>
+            <span class="s1">&#39;DCAwareRoundRobinPolicy&#39;</span><span class="p">:</span> <span class="n">DCAwareRoundRobinPolicy</span><span class="p">,</span>
+            <span class="s1">&#39;WhiteListRoundRobinPolicy&#39;</span><span class="p">:</span> <span class="n">WhiteListRoundRobinPolicy</span><span class="p">,</span>
+            <span class="s1">&#39;TokenAwarePolicy&#39;</span><span class="p">:</span> <span class="n">TokenAwarePolicy</span><span class="p">,</span>
+        <span class="p">}</span>
+
+        <span class="k">if</span> <span class="ow">not</span> <span class="n">policies</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">policy_name</span><span class="p">)</span> <span class="ow">or</span> <span class="n">policy_name</span> <span class="o">==</span> <span class="s1">&#39;RoundRobinPolicy&#39;</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">RoundRobinPolicy</span><span class="p">()</span>
+
+        <span class="k">if</span> <span class="n">policy_name</span> <span class="o">==</span> <span class="s1">&#39;DCAwareRoundRobinPolicy&#39;</span><span class="p">:</span>
+            <span class="n">local_dc</span> <span class="o">=</span> <span class="n">policy_args</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;local_dc&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span>
+            <span class="n">used_hosts_per_remote_dc</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">policy_args</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;used_hosts_per_remote_dc&#39;</span><span class="p">,</span> <span class="mi">0</span><span class="p">))</span>
+            <span class="k">return</span> <span class="n">DCAwareRoundRobinPolicy</span><span class="p">(</span><span class="n">local_dc</span><span class="p">,</span> <span class="n">used_hosts_per_remote_dc</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="n">policy_name</span> <span class="o">==</span> <span class="s1">&#39;WhiteListRoundRobinPolicy&#39;</span><span class="p">:</span>
+            <span class="n">hosts</span> <span class="o">=</span> <span class="n">policy_args</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;hosts&#39;</span><span class="p">)</span>
+            <span class="k">if</span> <span class="ow">not</span> <span class="n">hosts</span><span class="p">:</span>
+                <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="s1">&#39;Hosts must be specified for WhiteListRoundRobinPolicy&#39;</span><span class="p">)</span>
+            <span class="k">return</span> <span class="n">WhiteListRoundRobinPolicy</span><span class="p">(</span><span class="n">hosts</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="n">policy_name</span> <span class="o">==</span> <span class="s1">&#39;TokenAwarePolicy&#39;</span><span class="p">:</span>
+            <span class="n">allowed_child_policies</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;RoundRobinPolicy&#39;</span><span class="p">,</span>
+                                      <span class="s1">&#39;DCAwareRoundRobinPolicy&#39;</span><span class="p">,</span>
+                                      <span class="s1">&#39;WhiteListRoundRobinPolicy&#39;</span><span class="p">,)</span>
+            <span class="n">child_policy_name</span> <span class="o">=</span> <span class="n">policy_args</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;child_load_balancing_policy&#39;</span><span class="p">,</span>
+                                                <span class="s1">&#39;RoundRobinPolicy&#39;</span><span class="p">)</span>
+            <span class="n">child_policy_args</span> <span class="o">=</span> <span class="n">policy_args</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;child_load_balancing_policy_args&#39;</span><span class="p">,</span> <span class="p">{})</span>
+            <span class="k">if</span> <span class="n">child_policy_name</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">allowed_child_policies</span><span class="p">:</span>
+                <span class="k">return</span> <span class="n">TokenAwarePolicy</span><span class="p">(</span><span class="n">RoundRobinPolicy</span><span class="p">())</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">child_policy</span> <span class="o">=</span> <span class="n">CassandraHook</span><span class="o">.</span><span class="n">get_lb_policy</span><span class="p">(</span><span class="n">child_policy_name</span><span class="p">,</span>
+                                                           <span class="n">child_policy_args</span><span class="p">)</span>
+                <span class="k">return</span> <span class="n">TokenAwarePolicy</span><span class="p">(</span><span class="n">child_policy</span><span class="p">)</span>
+
+<div class="viewcode-block" id="CassandraHook.table_exists"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.cassandra_hook.CassandraHook.table_exists">[docs]</a>    <span class="k">def</span> <span class="nf">table_exists</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">table</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Checks if a table exists in Cassandra</span>
+
+<span class="sd">        :param table: Target Cassandra table.</span>
+<span class="sd">                      Use dot notation to target a specific keyspace.</span>
+<span class="sd">        :type table: string</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">keyspace</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">keyspace</span>
+        <span class="k">if</span> <span class="s1">&#39;.&#39;</span> <span class="ow">in</span> <span class="n">table</span><span class="p">:</span>
+            <span class="n">keyspace</span><span class="p">,</span> <span class="n">table</span> <span class="o">=</span> <span class="n">table</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
+        <span class="n">cluster_metadata</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">cluster</span><span class="o">.</span><span class="n">metadata</span>
+        <span class="k">return</span> <span class="p">(</span><span class="n">keyspace</span> <span class="ow">in</span> <span class="n">cluster_metadata</span><span class="o">.</span><span class="n">keyspaces</span> <span class="ow">and</span>
+                <span class="n">table</span> <span class="ow">in</span> <span class="n">cluster_metadata</span><span class="o">.</span><span class="n">keyspaces</span><span class="p">[</span><span class="n">keyspace</span><span class="p">]</span><span class="o">.</span><span class="n">tables</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="CassandraHook.record_exists"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.cassandra_hook.CassandraHook.record_exists">[docs]</a>    <span class="k">def</span> <span class="nf">record_exists</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">keys</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Checks if a record exists in Cassandra</span>
+
+<span class="sd">        :param table: Target Cassandra table.</span>
+<span class="sd">                      Use dot notation to target a specific keyspace.</span>
+<span class="sd">        :type table: string</span>
+<span class="sd">        :param keys: The keys and their values to check the existence.</span>
+<span class="sd">        :type keys: dict</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">keyspace</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">keyspace</span>
+        <span class="k">if</span> <span class="s1">&#39;.&#39;</span> <span class="ow">in</span> <span class="n">table</span><span class="p">:</span>
+            <span class="n">keyspace</span><span class="p">,</span> <span class="n">table</span> <span class="o">=</span> <span class="n">table</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
+        <span class="n">ks</span> <span class="o">=</span> <span class="s2">&quot; AND &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="s2">&quot;</span><span class="si">{}</span><span class="s2">=%(</span><span class="si">{}</span><span class="s2">)s&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">key</span><span class="p">)</spa [...]
+        <span class="n">cql</span> <span class="o">=</span> <span class="s2">&quot;SELECT * FROM </span><span class="si">{keyspace}</span><span class="s2">.</span><span class="si">{table}</span><span class="s2"> WHERE </span><span class="si">{keys}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+            <span class="n">keyspace</span><span class="o">=</span><span class="n">keyspace</span><span class="p">,</span> <span class="n">table</span><span class="o">=</span><span class="n">table</span><span class="p">,</span> <span class="n">keys</span><span class="o">=</span><span class="n">ks</span><span class="p">)</span>
+
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">rs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">cql</span><span class="p">,</span> <span class="n">keys</span><span class="p">)</span>
+            <span class="k">return</span> <span class="n">rs</span><span class="o">.</span><span class="n">one</span><span class="p">()</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+        <span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span>
+            <span class="k">return</span> <span class="kc">False</span></div></div>
+</pre></div>
+
+           </div>
+           
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    
+    
+      <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+        <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+        <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+        <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+        <script type="text/javascript" src="../../../../_static/language_data.js"></script>
+    
+
+  
+
+  <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.Navigation.enable(true);
+      });
+  </script> 
+
+</body>
+</html>
\ No newline at end of file
diff --git a/1.10.2/_modules/airflow/contrib/hooks/cloudant_hook.html b/1.10.2/_modules/airflow/contrib/hooks/cloudant_hook.html
new file mode 100644
index 0000000..b28e966
--- /dev/null
+++ b/1.10.2/_modules/airflow/contrib/hooks/cloudant_hook.html
@@ -0,0 +1,302 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>airflow.contrib.hooks.cloudant_hook &mdash; Airflow Documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+    <link rel="index" title="Index" href="../../../../genindex.html" />
+    <link rel="search" title="Search" href="../../../../search.html" /> 
+
+  
+  <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../../index.html" class="icon icon-home"> Airflow
+          
+
+          
+          </a>
+
+          
+            
+            
+              <div class="version">
+                1.10.2
+              </div>
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../metrics.html">Metrics</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../../index.html">Airflow</a>
+        
+      </nav>
+
+
+      <div class="wy-nav-content">
+        
+        <div class="rst-content">
+        
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../../index.html">Module code</a> &raquo;</li>
+        
+      <li>airflow.contrib.hooks.cloudant_hook</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for airflow.contrib.hooks.cloudant_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements.  See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership.  The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># &quot;License&quot;); you may not use this file except in compliance</span>
+<span class="c1"># with the License.  You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1">#   http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># &quot;AS IS&quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied.  See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">past.builtins</span> <span class="k">import</span> <span class="n">unicode</span>
+
+<span class="kn">import</span> <span class="nn">cloudant</span>
+
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
+
+
+<div class="viewcode-block" id="CloudantHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.cloudant_hook.CloudantHook">[docs]</a><span class="k">class</span> <span class="nc">CloudantHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;Interact with Cloudant.</span>
+
+<span class="sd">    This class is a thin wrapper around the cloudant python library. See the</span>
+<span class="sd">    documentation `here &lt;https://github.com/cloudant-labs/cloudant-python&gt;`_.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cloudant_conn_id</span><span class="o">=</span><span class="s1">&#39;cloudant_default&#39;</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">CloudantHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="s1">&#39;cloudant&#39;</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">cloudant_conn_id</span> <span class="o">=</span> <span class="n">cloudant_conn_id</span>
+
+    <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="k">def</span> <span class="nf">_str</span><span class="p">(</span><span class="n">s</span><span class="p">):</span>
+            <span class="c1"># cloudant-python doesn&#39;t support unicode.</span>
+            <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">s</span><span class="p">,</span> <span class="n">unicode</span><span class="p">):</span>
+                <span class="n">log</span> <span class="o">=</span> <span class="n">LoggingMixin</span><span class="p">()</span><span class="o">.</span><span class="n">log</span>
+                <span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
+                    <span class="s1">&#39;cloudant-python does not support unicode. Encoding </span><span class="si">%s</span><span class="s1"> as &#39;</span>
+                    <span class="s1">&#39;ascii using &quot;ignore&quot;.&#39;</span><span class="p">,</span> <span class="n">s</span>
+                <span class="p">)</span>
+                <span class="k">return</span> <span class="n">s</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">&#39;ascii&#39;</span><span class="p">,</span> <span class="s1">&#39;ignore&#39;</span><span class="p">)</span>
+
+            <span class="k">return</span> <span class="n">s</span>
+
+        <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cloudant_conn_id</span><span class="p">)</span>
+
+        <span class="k">for</span> <span class="n">conn_param</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">&#39;host&#39;</span><span class="p">,</span> <span class="s1">&#39;password&#39;</span><span class="p">,</span> <span class="s1">&#39;schema&#39;</span><span class="p">]:</span>
+            <span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">conn</span><span class="p">,</span> <span class="n">conn_param</span><span class="p">)</span> <span class="ow">or</span> <span class="ow">not</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">conn</span><span class="p">,</span> <span class="n">conn_param</span><span class="p">):</span>
+                <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+                    <span class="s1">&#39;missing connection parameter </span><span class="si">{0}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">conn_param</span><span class="p">)</span>
+                <span class="p">)</span>
+
+        <span class="c1"># In the connection form:</span>
+        <span class="c1"># - &#39;host&#39; is renamed to &#39;Account&#39;</span>
+        <span class="c1"># - &#39;login&#39; is renamed &#39;Username (or API Key)&#39;</span>
+        <span class="c1"># - &#39;schema&#39; is renamed to &#39;Database&#39;</span>
+        <span class="c1">#</span>
+        <span class="c1"># So, use the &#39;host&#39; attribute as the account name, and, if login is</span>
+        <span class="c1"># defined, use that as the username.</span>
+        <span class="n">account</span> <span class="o">=</span> <span class="n">cloudant</span><span class="o">.</span><span class="n">Account</span><span class="p">(</span><span class="n">_str</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">host</span><span class="p">))</span>
+
+        <span class="n">username</span> <span class="o">=</span> <span class="n">_str</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">login</span> <span class="ow">or</span> <span class="n">conn</span><span class="o">.</span><span class="n">host</span><span class="p">)</span>
+
+        <span class="n">account</span><span class="o">.</span><span class="n">login</span><span class="p">(</span>
+            <span class="n">username</span><span class="p">,</span>
+            <span class="n">_str</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">))</span><span class="o">.</span><span class="n">raise_for_status</span><span class="p">()</span>
+
+        <span class="k">return</span> <span class="n">account</span><span class="o">.</span><span class="n">database</span><span class="p">(</span><span class="n">_str</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">schema</span><span class="p">))</span>
+
+<div class="viewcode-block" id="CloudantHook.db"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.cloudant_hook.CloudantHook.db">[docs]</a>    <span class="k">def</span> <span class="nf">db</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;Returns the Database object for this hook.</span>
+
+<span class="sd">        See the documentation for cloudant-python here</span>
+<span class="sd">        https://github.com/cloudant-labs/cloudant-python.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span></div></div>
+</pre></div>
+
+           </div>
+           
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    
+    
+      <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+        <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+        <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+        <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+        <script type="text/javascript" src="../../../../_static/language_data.js"></script>
+    
+
+  
+
+  <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.Navigation.enable(true);
+      });
+  </script> 
+
+</body>
+</html>
\ No newline at end of file
diff --git a/1.10.2/_modules/airflow/contrib/hooks/databricks_hook.html b/1.10.2/_modules/airflow/contrib/hooks/databricks_hook.html
new file mode 100644
index 0000000..537515a
--- /dev/null
+++ b/1.10.2/_modules/airflow/contrib/hooks/databricks_hook.html
@@ -0,0 +1,495 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>airflow.contrib.hooks.databricks_hook &mdash; Airflow Documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+    <link rel="index" title="Index" href="../../../../genindex.html" />
+    <link rel="search" title="Search" href="../../../../search.html" /> 
+
+  
+  <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../../index.html" class="icon icon-home"> Airflow
+          
+
+          
+          </a>
+
+          
+            
+            
+              <div class="version">
+                1.10.2
+              </div>
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../metrics.html">Metrics</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../../index.html">Airflow</a>
+        
+      </nav>
+
+
+      <div class="wy-nav-content">
+        
+        <div class="rst-content">
+        
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../../index.html">Module code</a> &raquo;</li>
+        
+      <li>airflow.contrib.hooks.databricks_hook</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for airflow.contrib.hooks.databricks_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements.  See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership.  The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># &quot;License&quot;); you may not use this file except in compliance</span>
+<span class="c1"># with the License.  You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1">#   http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># &quot;AS IS&quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied.  See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="c1">#</span>
+<span class="kn">import</span> <span class="nn">requests</span>
+
+<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">__version__</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">requests</span> <span class="k">import</span> <span class="n">exceptions</span> <span class="k">as</span> <span class="n">requests_exceptions</span>
+<span class="kn">from</span> <span class="nn">requests.auth</span> <span class="k">import</span> <span class="n">AuthBase</span>
+<span class="kn">from</span> <span class="nn">time</span> <span class="k">import</span> <span class="n">sleep</span>
+
+<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
+
+<span class="k">try</span><span class="p">:</span>
+    <span class="kn">from</span> <span class="nn">urllib</span> <span class="k">import</span> <span class="n">parse</span> <span class="k">as</span> <span class="n">urlparse</span>
+<span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span>
+    <span class="kn">import</span> <span class="nn">urlparse</span>
+
+<span class="n">RESTART_CLUSTER_ENDPOINT</span> <span class="o">=</span> <span class="p">(</span><span class="s2">&quot;POST&quot;</span><span class="p">,</span> <span class="s2">&quot;api/2.0/clusters/restart&quot;</span><span class="p">)</span>
+<span class="n">START_CLUSTER_ENDPOINT</span> <span class="o">=</span> <span class="p">(</span><span class="s2">&quot;POST&quot;</span><span class="p">,</span> <span class="s2">&quot;api/2.0/clusters/start&quot;</span><span class="p">)</span>
+<span class="n">TERMINATE_CLUSTER_ENDPOINT</span> <span class="o">=</span> <span class="p">(</span><span class="s2">&quot;POST&quot;</span><span class="p">,</span> <span class="s2">&quot;api/2.0/clusters/delete&quot;</span><span class="p">)</span>
+
+<span class="n">RUN_NOW_ENDPOINT</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;POST&#39;</span><span class="p">,</span> <span class="s1">&#39;api/2.0/jobs/run-now&#39;</span><span class="p">)</span>
+<span class="n">SUBMIT_RUN_ENDPOINT</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;POST&#39;</span><span class="p">,</span> <span class="s1">&#39;api/2.0/jobs/runs/submit&#39;</span><span class="p">)</span>
+<span class="n">GET_RUN_ENDPOINT</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;GET&#39;</span><span class="p">,</span> <span class="s1">&#39;api/2.0/jobs/runs/get&#39;</span><span class="p">)</span>
+<span class="n">CANCEL_RUN_ENDPOINT</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;POST&#39;</span><span class="p">,</span> <span class="s1">&#39;api/2.0/jobs/runs/cancel&#39;</span><span class="p">)</span>
+<span class="n">USER_AGENT_HEADER</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;user-agent&#39;</span><span class="p">:</span> <span class="s1">&#39;airflow-</span><span class="si">{v}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">v</span><span class="o">=</span><span class="n">__version__</span><span class="p">)}</span>
+
+
+<div class="viewcode-block" id="DatabricksHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.databricks_hook.DatabricksHook">[docs]</a><span class="k">class</span> <span class="nc">DatabricksHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Interact with Databricks.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+            <span class="bp">self</span><span class="p">,</span>
+            <span class="n">databricks_conn_id</span><span class="o">=</span><span class="s1">&#39;databricks_default&#39;</span><span class="p">,</span>
+            <span class="n">timeout_seconds</span><span class="o">=</span><span class="mi">180</span><span class="p">,</span>
+            <span class="n">retry_limit</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span>
+            <span class="n">retry_delay</span><span class="o">=</span><span class="mf">1.0</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        :param databricks_conn_id: The name of the databricks connection to use.</span>
+<span class="sd">        :type databricks_conn_id: string</span>
+<span class="sd">        :param timeout_seconds: The amount of time in seconds the requests library</span>
+<span class="sd">            will wait before timing-out.</span>
+<span class="sd">        :type timeout_seconds: int</span>
+<span class="sd">        :param retry_limit: The number of times to retry the connection in case of</span>
+<span class="sd">            service outages.</span>
+<span class="sd">        :type retry_limit: int</span>
+<span class="sd">        :param retry_delay: The number of seconds to wait between retries (it</span>
+<span class="sd">            might be a floating point number).</span>
+<span class="sd">        :type retry_delay: float</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">databricks_conn_id</span> <span class="o">=</span> <span class="n">databricks_conn_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">databricks_conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="n">databricks_conn_id</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">timeout_seconds</span> <span class="o">=</span> <span class="n">timeout_seconds</span>
+        <span class="k">if</span> <span class="n">retry_limit</span> <span class="o">&lt;</span> <span class="mi">1</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;Retry limit must be greater than equal to 1&#39;</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">retry_limit</span> <span class="o">=</span> <span class="n">retry_limit</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">retry_delay</span> <span class="o">=</span> <span class="n">retry_delay</span>
+
+    <span class="nd">@staticmethod</span>
+    <span class="k">def</span> <span class="nf">_parse_host</span><span class="p">(</span><span class="n">host</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        The purpose of this function is to be robust to improper connections</span>
+<span class="sd">        settings provided by users, specifically in the host field.</span>
+
+
+<span class="sd">        For example -- when users supply ``https://xx.cloud.databricks.com`` as the</span>
+<span class="sd">        host, we must strip out the protocol to get the host.</span>
+<span class="sd">        &gt;&gt;&gt; h = DatabricksHook()</span>
+<span class="sd">        &gt;&gt;&gt; assert h._parse_host(&#39;https://xx.cloud.databricks.com&#39;) == \</span>
+<span class="sd">            &#39;xx.cloud.databricks.com&#39;</span>
+
+<span class="sd">        In the case where users supply the correct ``xx.cloud.databricks.com`` as the</span>
+<span class="sd">        host, this function is a no-op.</span>
+<span class="sd">        &gt;&gt;&gt; assert h._parse_host(&#39;xx.cloud.databricks.com&#39;) == &#39;xx.cloud.databricks.com&#39;</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">urlparse_host</span> <span class="o">=</span> <span class="n">urlparse</span><span class="o">.</span><span class="n">urlparse</span><span class="p">(</span><span class="n">host</span><span class="p">)</span><span class="o">.</span><span class="n">hostname</span>
+        <span class="k">if</span> <span class="n">urlparse_host</span><span class="p">:</span>
+            <span class="c1"># In this case, host = https://xx.cloud.databricks.com</span>
+            <span class="k">return</span> <span class="n">urlparse_host</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="c1"># In this case, host = xx.cloud.databricks.com</span>
+            <span class="k">return</span> <span class="n">host</span>
+
+    <span class="k">def</span> <span class="nf">_do_api_call</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">endpoint_info</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Utility function to perform an API call with retries</span>
+<span class="sd">        :param endpoint_info: Tuple of method and endpoint</span>
+<span class="sd">        :type endpoint_info: (string, string)</span>
+<span class="sd">        :param json: Parameters for this API call.</span>
+<span class="sd">        :type json: dict</span>
+<span class="sd">        :return: If the api call returns a OK status code,</span>
+<span class="sd">            this function returns the response in JSON. Otherwise,</span>
+<span class="sd">            we throw an AirflowException.</span>
+<span class="sd">        :rtype: dict</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">method</span><span class="p">,</span> <span class="n">endpoint</span> <span class="o">=</span> <span class="n">endpoint_info</span>
+        <span class="n">url</span> <span class="o">=</span> <span class="s1">&#39;https://</span><span class="si">{host}</span><span class="s1">/</span><span class="si">{endpoint}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+            <span class="n">host</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_parse_host</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">databricks_conn</span><span class="o">.</span><span class="n">host</span><span class="p">),</span>
+            <span class="n">endpoint</span><span class="o">=</span><span class="n">endpoint</span><span class="p">)</span>
+        <span class="k">if</span> <span class="s1">&#39;token&#39;</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">databricks_conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Using token auth.&#39;</span><span class="p">)</span>
+            <span class="n">auth</span> <span class="o">=</span> <span class="n">_TokenAuth</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">databricks_conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">[</span><span class="s1">&#39;token&#39;</span><span class="p">])</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Using basic auth.&#39;</span><span class="p">)</span>
+            <span class="n">auth</span> <span class="o">=</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">databricks_conn</span><span class="o">.</span><span class="n">login</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">databricks_conn</span><span class="o">.</span><span class="n">password</span><span class="p">)</span>
+        <span class="k">if</span> <span class="n">method</span> <span class="o">==</span> <span class="s1">&#39;GET&#39;</span><span class="p">:</span>
+            <span class="n">request_func</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">get</span>
+        <span class="k">elif</span> <span class="n">method</span> <span class="o">==</span> <span class="s1">&#39;POST&#39;</span><span class="p">:</span>
+            <span class="n">request_func</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">post</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">&#39;Unexpected HTTP Method: &#39;</span> <span class="o">+</span> <span class="n">method</span><span class="p">)</span>
+
+        <span class="n">attempt_num</span> <span class="o">=</span> <span class="mi">1</span>
+        <span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
+            <span class="k">try</span><span class="p">:</span>
+                <span class="n">response</span> <span class="o">=</span> <span class="n">request_func</span><span class="p">(</span>
+                    <span class="n">url</span><span class="p">,</span>
+                    <span class="n">json</span><span class="o">=</span><span class="n">json</span><span class="p">,</span>
+                    <span class="n">auth</span><span class="o">=</span><span class="n">auth</span><span class="p">,</span>
+                    <span class="n">headers</span><span class="o">=</span><span class="n">USER_AGENT_HEADER</span><span class="p">,</span>
+                    <span class="n">timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout_seconds</span><span class="p">)</span>
+                <span class="n">response</span><span class="o">.</span><span class="n">raise_for_status</span><span class="p">()</span>
+                <span class="k">return</span> <span class="n">response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
+            <span class="k">except</span> <span class="n">requests_exceptions</span><span class="o">.</span><span class="n">RequestException</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+                <span class="k">if</span> <span class="ow">not</span> <span class="n">_retryable_error</span><span class="p">(</span><span class="n">e</span><span class="p">):</span>
+                    <span class="c1"># In this case, the user probably made a mistake.</span>
+                    <span class="c1"># Don&#39;t retry.</span>
+                    <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">&#39;Response: </span><span class="si">{0}</span><span class="s1">, Status Code: </span><span class="si">{1}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+                        <span class="n">e</span><span class="o">.</span><span class="n">response</span><span class="o">.</span><span class="n">content</span><span class="p">,</span> <span class="n">e</span><span class="o">.</span><span class="n">response</span><span class="o">.</span><span class="n">status_code</span><span class="p">))</span>
+
+                <span class="bp">self</span><span class="o">.</span><span class="n">_log_request_error</span><span class="p">(</span><span class="n">attempt_num</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
+
+            <span class="k">if</span> <span class="n">attempt_num</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">retry_limit</span><span class="p">:</span>
+                <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">((</span><span class="s1">&#39;API requests to Databricks failed </span><span class="si">{}</span><span class="s1"> times. &#39;</span> <span class="o">+</span>
+                                        <span class="s1">&#39;Giving up.&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">retry_limit</span><span class="p">))</span>
+
+            <span class="n">attempt_num</span> <span class="o">+=</span> <span class="mi">1</span>
+            <span class="n">sleep</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">retry_delay</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_log_request_error</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">attempt_num</span><span class="p">,</span> <span class="n">error</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span>
+            <span class="s1">&#39;Attempt </span><span class="si">%s</span><span class="s1"> API Request to Databricks failed with reason: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span>
+            <span class="n">attempt_num</span><span class="p">,</span> <span class="n">error</span>
+        <span class="p">)</span>
+
+<div class="viewcode-block" id="DatabricksHook.run_now"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.databricks_hook.DatabricksHook.run_now">[docs]</a>    <span class="k">def</span> <span class="nf">run_now</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Utility function to call the ``api/2.0/jobs/run-now`` endpoint.</span>
+
+<span class="sd">        :param json: The data used in the body of the request to the ``run-now`` endpoint.</span>
+<span class="sd">        :type json: dict</span>
+<span class="sd">        :return: the run_id as a string</span>
+<span class="sd">        :rtype: string</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_do_api_call</span><span class="p">(</span><span class="n">RUN_NOW_ENDPOINT</span><span class="p">,</span> <span class="n">json</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">response</span><span class="p">[</span><span class="s1">&#39;run_id&#39;</span><span class="p">]</span></div>
+
+<div class="viewcode-block" id="DatabricksHook.submit_run"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.databricks_hook.DatabricksHook.submit_run">[docs]</a>    <span class="k">def</span> <span class="nf">submit_run</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Utility function to call the ``api/2.0/jobs/runs/submit`` endpoint.</span>
+
+<span class="sd">        :param json: The data used in the body of the request to the ``submit`` endpoint.</span>
+<span class="sd">        :type json: dict</span>
+<span class="sd">        :return: the run_id as a string</span>
+<span class="sd">        :rtype: string</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_do_api_call</span><span class="p">(</span><span class="n">SUBMIT_RUN_ENDPOINT</span><span class="p">,</span> <span class="n">json</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">response</span><span class="p">[</span><span class="s1">&#39;run_id&#39;</span><span class="p">]</span></div>
+
+    <span class="k">def</span> <span class="nf">get_run_page_url</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">run_id</span><span class="p">):</span>
+        <span class="n">json</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;run_id&#39;</span><span class="p">:</span> <span class="n">run_id</span><span class="p">}</span>
+        <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_do_api_call</span><span class="p">(</span><span class="n">GET_RUN_ENDPOINT</span><span class="p">,</span> <span class="n">json</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">response</span><span class="p">[</span><span class="s1">&#39;run_page_url&#39;</span><span class="p">]</span>
+
+    <span class="k">def</span> <span class="nf">get_run_state</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">run_id</span><span class="p">):</span>
+        <span class="n">json</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;run_id&#39;</span><span class="p">:</span> <span class="n">run_id</span><span class="p">}</span>
+        <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_do_api_call</span><span class="p">(</span><span class="n">GET_RUN_ENDPOINT</span><span class="p">,</span> <span class="n">json</span><span class="p">)</span>
+        <span class="n">state</span> <span class="o">=</span> <span class="n">response</span><span class="p">[</span><span class="s1">&#39;state&#39;</span><span class="p">]</span>
+        <span class="n">life_cycle_state</span> <span class="o">=</span> <span class="n">state</span><span class="p">[</span><span class="s1">&#39;life_cycle_state&#39;</span><span class="p">]</span>
+        <span class="c1"># result_state may not be in the state if not terminal</span>
+        <span class="n">result_state</span> <span class="o">=</span> <span class="n">state</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;result_state&#39;</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+        <span class="n">state_message</span> <span class="o">=</span> <span class="n">state</span><span class="p">[</span><span class="s1">&#39;state_message&#39;</span><span class="p">]</span>
+        <span class="k">return</span> <span class="n">RunState</span><span class="p">(</span><span class="n">life_cycle_state</span><span class="p">,</span> <span class="n">result_state</span><span class="p">,</span> <span class="n">state_message</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">cancel_run</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">run_id</span><span class="p">):</span>
+        <span class="n">json</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;run_id&#39;</span><span class="p">:</span> <span class="n">run_id</span><span class="p">}</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_do_api_call</span><span class="p">(</span><span class="n">CANCEL_RUN_ENDPOINT</span><span class="p">,</span> <span class="n">json</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">restart_cluster</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_do_api_call</span><span class="p">(</span><span class="n">RESTART_CLUSTER_ENDPOINT</span><span class="p">,</span> <span class="n">json</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">start_cluster</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_do_api_call</span><span class="p">(</span><span class="n">START_CLUSTER_ENDPOINT</span><span class="p">,</span> <span class="n">json</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">terminate_cluster</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_do_api_call</span><span class="p">(</span><span class="n">TERMINATE_CLUSTER_ENDPOINT</span><span class="p">,</span> <span class="n">json</span><span class="p">)</span></div>
+
+
+<span class="k">def</span> <span class="nf">_retryable_error</span><span class="p">(</span><span class="n">exception</span><span class="p">):</span>
+    <span class="k">return</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">exception</span><span class="p">,</span> <span class="n">requests_exceptions</span><span class="o">.</span><span class="n">ConnectionError</span><span class="p">)</span> \
+        <span class="ow">or</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">exception</span><span class="p">,</span> <span class="n">requests_exceptions</span><span class="o">.</span><span class="n">Timeout</span><span class="p">)</span> \
+        <span class="ow">or</span> <span class="n">exception</span><span class="o">.</span><span class="n">response</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">exception</span><span class="o">.</span><span class="n">response</span><span class="o">.</span><span class="n">status_code</span> <span class="o">&gt;=</span> <span class="mi">500</span>
+
+
+<span class="n">RUN_LIFE_CYCLE_STATES</span> <span class="o">=</span> <span class="p">[</span>
+    <span class="s1">&#39;PENDING&#39;</span><span class="p">,</span>
+    <span class="s1">&#39;RUNNING&#39;</span><span class="p">,</span>
+    <span class="s1">&#39;TERMINATING&#39;</span><span class="p">,</span>
+    <span class="s1">&#39;TERMINATED&#39;</span><span class="p">,</span>
+    <span class="s1">&#39;SKIPPED&#39;</span><span class="p">,</span>
+    <span class="s1">&#39;INTERNAL_ERROR&#39;</span>
+<span class="p">]</span>
+
+
+<span class="k">class</span> <span class="nc">RunState</span><span class="p">:</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Utility class for the run state concept of Databricks runs.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">life_cycle_state</span><span class="p">,</span> <span class="n">result_state</span><span class="p">,</span> <span class="n">state_message</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">life_cycle_state</span> <span class="o">=</span> <span class="n">life_cycle_state</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">result_state</span> <span class="o">=</span> <span class="n">result_state</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">state_message</span> <span class="o">=</span> <span class="n">state_message</span>
+
+    <span class="nd">@property</span>
+    <span class="k">def</span> <span class="nf">is_terminal</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">life_cycle_state</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">RUN_LIFE_CYCLE_STATES</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+                <span class="p">(</span><span class="s1">&#39;Unexpected life cycle state: </span><span class="si">{}</span><span class="s1">: If the state has &#39;</span>
+                 <span class="s1">&#39;been introduced recently, please check the Databricks user &#39;</span>
+                 <span class="s1">&#39;guide for troubleshooting information&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">life_cycle_state</span><span class="p">))</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">life_cycle_state</span> <span class="ow">in</span> <span class="p">(</span><span class="s1">&#39;TERMINATED&#39;</span><span class="p">,</span> <span class="s1">&#39;SKIPPED&#39;</span><span class="p">,</span> <span class="s1">&#39;INTERNAL_ERROR&#39;</span><span class="p">)</span>
+
+    <span class="nd">@property</span>
+    <span class="k">def</span> <span class="nf">is_successful</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">result_state</span> <span class="o">==</span> <span class="s1">&#39;SUCCESS&#39;</span>
+
+    <span class="k">def</span> <span class="nf">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">life_cycle_state</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">life_cycle_state</span> <span class="ow">and</span> \
+            <span class="bp">self</span><span class="o">.</span><span class="n">result_state</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">result_state</span> <span class="ow">and</span> \
+            <span class="bp">self</span><span class="o">.</span><span class="n">state_message</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">state_message</span>
+
+    <span class="k">def</span> <span class="nf">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">)</span>
+
+
+<span class="k">class</span> <span class="nc">_TokenAuth</span><span class="p">(</span><span class="n">AuthBase</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Helper class for requests Auth field. AuthBase requires you to implement the __call__</span>
+<span class="sd">    magic function.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">token</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">token</span> <span class="o">=</span> <span class="n">token</span>
+
+    <span class="k">def</span> <span class="nf">__call__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">r</span><span class="p">):</span>
+        <span class="n">r</span><span class="o">.</span><span class="n">headers</span><span class="p">[</span><span class="s1">&#39;Authorization&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;Bearer &#39;</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">token</span>
+        <span class="k">return</span> <span class="n">r</span>
+</pre></div>
+
+           </div>
+           
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    
+    
+      <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+        <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+        <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+        <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+        <script type="text/javascript" src="../../../../_static/language_data.js"></script>
+    
+
+  
+
+  <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.Navigation.enable(true);
+      });
+  </script> 
+
+</body>
+</html>
\ No newline at end of file
diff --git a/1.10.2/_modules/airflow/contrib/hooks/datadog_hook.html b/1.10.2/_modules/airflow/contrib/hooks/datadog_hook.html
new file mode 100644
index 0000000..a2b4683
--- /dev/null
+++ b/1.10.2/_modules/airflow/contrib/hooks/datadog_hook.html
@@ -0,0 +1,361 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>airflow.contrib.hooks.datadog_hook &mdash; Airflow Documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+    <link rel="index" title="Index" href="../../../../genindex.html" />
+    <link rel="search" title="Search" href="../../../../search.html" /> 
+
+  
+  <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../../index.html" class="icon icon-home"> Airflow
+          
+
+          
+          </a>
+
+          
+            
+            
+              <div class="version">
+                1.10.2
+              </div>
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../metrics.html">Metrics</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../../index.html">Airflow</a>
+        
+      </nav>
+
+
+      <div class="wy-nav-content">
+        
+        <div class="rst-content">
+        
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../../index.html">Module code</a> &raquo;</li>
+        
+      <li>airflow.contrib.hooks.datadog_hook</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for airflow.contrib.hooks.datadog_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements.  See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership.  The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># &quot;License&quot;); you may not use this file except in compliance</span>
+<span class="c1"># with the License.  You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1">#   http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># &quot;AS IS&quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied.  See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">import</span> <span class="nn">time</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">datadog</span> <span class="k">import</span> <span class="n">initialize</span><span class="p">,</span> <span class="n">api</span>
+
+<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
+
+
+<div class="viewcode-block" id="DatadogHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.datadog_hook.DatadogHook">[docs]</a><span class="k">class</span> <span class="nc">DatadogHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Uses datadog API to send metrics of practically anything measurable,</span>
+<span class="sd">    so it&#39;s possible to track # of db records inserted/deleted, records read</span>
+<span class="sd">    from file and many other useful metrics.</span>
+
+<span class="sd">    Depends on the datadog API, which has to be deployed on the same server where</span>
+<span class="sd">    Airflow runs.</span>
+
+<span class="sd">    :param datadog_conn_id: The connection to datadog, containing metadata for api keys.</span>
+<span class="sd">    :param datadog_conn_id: string</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">datadog_conn_id</span><span class="o">=</span><span class="s1">&#39;datadog_default&#39;</span><span class="p">):</span>
+        <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="n">datadog_conn_id</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">api_key</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;api_key&#39;</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">app_key</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;app_key&#39;</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">source_type_name</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;source_type_name&#39;</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+
+        <span class="c1"># If the host is populated, it will use that hostname instead.</span>
+        <span class="c1"># for all metric submissions.</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">host</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">host</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">api_key</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">&quot;api_key must be specified in the &quot;</span>
+                                   <span class="s2">&quot;Datadog connection details&quot;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">app_key</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">&quot;app_key must be specified in the &quot;</span>
+                                   <span class="s2">&quot;Datadog connection details&quot;</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Setting up api keys for Datadog&quot;</span><span class="p">)</span>
+        <span class="n">options</span> <span class="o">=</span> <span class="p">{</span>
+            <span class="s1">&#39;api_key&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">api_key</span><span class="p">,</span>
+            <span class="s1">&#39;app_key&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">app_key</span>
+        <span class="p">}</span>
+        <span class="n">initialize</span><span class="p">(</span><span class="o">**</span><span class="n">options</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">validate_response</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">response</span><span class="p">):</span>
+        <span class="k">if</span> <span class="n">response</span><span class="p">[</span><span class="s1">&#39;status&#39;</span><span class="p">]</span> <span class="o">!=</span> <span class="s1">&#39;ok&#39;</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;Datadog returned: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">response</span><span class="p">)</span>
+            <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">&quot;Error status received from Datadog&quot;</span><span class="p">)</span>
+
+<div class="viewcode-block" id="DatadogHook.send_metric"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.datadog_hook.DatadogHook.send_metric">[docs]</a>    <span class="k">def</span> <span class="nf">send_metric</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">metric_name</span><span class="p">,</span> <span class="n">datapoint</span><span class="p">,</span> <span class="n">tags</span><span class="o">=</span><s [...]
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Sends a single datapoint metric to DataDog</span>
+
+<span class="sd">        :param metric_name: The name of the metric</span>
+<span class="sd">        :type metric_name: string</span>
+<span class="sd">        :param datapoint: A single integer or float related to the metric</span>
+<span class="sd">        :type datapoint: integer or float</span>
+<span class="sd">        :param tags: A list of tags associated with the metric</span>
+<span class="sd">        :type tags: list</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">response</span> <span class="o">=</span> <span class="n">api</span><span class="o">.</span><span class="n">Metric</span><span class="o">.</span><span class="n">send</span><span class="p">(</span>
+            <span class="n">metric</span><span class="o">=</span><span class="n">metric_name</span><span class="p">,</span>
+            <span class="n">points</span><span class="o">=</span><span class="n">datapoint</span><span class="p">,</span>
+            <span class="n">host</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">host</span><span class="p">,</span>
+            <span class="n">tags</span><span class="o">=</span><span class="n">tags</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">validate_response</span><span class="p">(</span><span class="n">response</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">response</span></div>
+
+<div class="viewcode-block" id="DatadogHook.query_metric"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.datadog_hook.DatadogHook.query_metric">[docs]</a>    <span class="k">def</span> <span class="nf">query_metric</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+                     <span class="n">query</span><span class="p">,</span>
+                     <span class="n">from_seconds_ago</span><span class="p">,</span>
+                     <span class="n">to_seconds_ago</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Queries datadog for a specific metric, potentially with some</span>
+<span class="sd">        function applied to it and returns the results.</span>
+
+<span class="sd">        :param query: The datadog query to execute (see datadog docs)</span>
+<span class="sd">        :type query: string</span>
+<span class="sd">        :param from_seconds_ago: How many seconds ago to start querying for.</span>
+<span class="sd">        :type from_seconds_ago: int</span>
+<span class="sd">        :param to_seconds_ago: Up to how many seconds ago to query for.</span>
+<span class="sd">        :type to_seconds_ago: int</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">now</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">())</span>
+
+        <span class="n">response</span> <span class="o">=</span> <span class="n">api</span><span class="o">.</span><span class="n">Metric</span><span class="o">.</span><span class="n">query</span><span class="p">(</span>
+            <span class="n">start</span><span class="o">=</span><span class="n">now</span> <span class="o">-</span> <span class="n">from_seconds_ago</span><span class="p">,</span>
+            <span class="n">end</span><span class="o">=</span><span class="n">now</span> <span class="o">-</span> <span class="n">to_seconds_ago</span><span class="p">,</span>
+            <span class="n">query</span><span class="o">=</span><span class="n">query</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">validate_response</span><span class="p">(</span><span class="n">response</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">response</span></div>
+
+<div class="viewcode-block" id="DatadogHook.post_event"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.datadog_hook.DatadogHook.post_event">[docs]</a>    <span class="k">def</span> <span class="nf">post_event</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">title</span><span class="p">,</span> <span class="n">text</span><span class="p">,</span> <span class="n">tags</span><span class="o">=</span><span class="kc" [...]
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Posts an event to datadog (processing finished, potentially alerts, other issues)</span>
+<span class="sd">        Think about this as a means to maintain persistence of alerts, rather than</span>
+<span class="sd">        alerting itself.</span>
+
+<span class="sd">        :param title: The title of the event</span>
+<span class="sd">        :type title: string</span>
+<span class="sd">        :param text: The body of the event (more information)</span>
+<span class="sd">        :type text: string</span>
+<span class="sd">        :param tags: List of string tags to apply to the event</span>
+<span class="sd">        :type tags: list</span>
+<span class="sd">        :param alert_type: The alert type for the event, one of</span>
+<span class="sd">            [&quot;error&quot;, &quot;warning&quot;, &quot;info&quot;, &quot;success&quot;]</span>
+<span class="sd">        :type alert_type: string</span>
+<span class="sd">        :param aggregation_key: Key that can be used to aggregate this event in a stream</span>
+<span class="sd">        :type aggregation_key: string</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">response</span> <span class="o">=</span> <span class="n">api</span><span class="o">.</span><span class="n">Event</span><span class="o">.</span><span class="n">create</span><span class="p">(</span>
+            <span class="n">title</span><span class="o">=</span><span class="n">title</span><span class="p">,</span>
+            <span class="n">text</span><span class="o">=</span><span class="n">text</span><span class="p">,</span>
+            <span class="n">host</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">host</span><span class="p">,</span>
+            <span class="n">tags</span><span class="o">=</span><span class="n">tags</span><span class="p">,</span>
+            <span class="n">alert_type</span><span class="o">=</span><span class="n">alert_type</span><span class="p">,</span>
+            <span class="n">aggregation_key</span><span class="o">=</span><span class="n">aggregation_key</span><span class="p">,</span>
+            <span class="n">source_type_name</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">source_type_name</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">validate_response</span><span class="p">(</span><span class="n">response</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">response</span></div></div>
+</pre></div>
+
+           </div>
+           
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    
+    
+      <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
+        <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+        <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+        <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+        <script type="text/javascript" src="../../../../_static/language_data.js"></script>
+    
+
+  
+
+  <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.Navigation.enable(true);
+      });
+  </script> 
+
+</body>
+</html>
\ No newline at end of file
diff --git a/1.10.2/_modules/airflow/contrib/hooks/datastore_hook.html b/1.10.2/_modules/airflow/contrib/hooks/datastore_hook.html
new file mode 100644
index 0000000..3918dc5
--- /dev/null
+++ b/1.10.2/_modules/airflow/contrib/hooks/datastore_hook.html
@@ -0,0 +1,425 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>airflow.contrib.hooks.datastore_hook &mdash; Airflow Documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
+    <link rel="index" title="Index" href="../../../../genindex.html" />
+    <link rel="search" title="Search" href="../../../../search.html" /> 
+
+  
+  <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../../index.html" class="icon icon-home"> Airflow
+          
+
+          
+          </a>
+
+          
+            
+            
+              <div class="version">
+                1.10.2
+              </div>
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../metrics.html">Metrics</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../../index.html">Airflow</a>
+        
+      </nav>
+
+
+      <div class="wy-nav-content">
+        
+        <div class="rst-content">
+        
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../../index.html">Module code</a> &raquo;</li>
+        
+      <li>airflow.contrib.hooks.datastore_hook</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for airflow.contrib.hooks.datastore_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements.  See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership.  The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># &quot;License&quot;); you may not use this file except in compliance</span>
+<span class="c1"># with the License.  You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1">#   http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># &quot;AS IS&quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied.  See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+<span class="c1">#</span>
+
+<span class="kn">import</span> <span class="nn">time</span>
+<span class="kn">from</span> <span class="nn">googleapiclient.discovery</span> <span class="k">import</span> <span class="n">build</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcp_api_base_hook</span> <span class="k">import</span> <span class="n">GoogleCloudBaseHook</span>
+
+
+<div class="viewcode-block" id="DatastoreHook"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook">[docs]</a><span class="k">class</span> <span class="nc">DatastoreHook</span><span class="p">(</span><span class="n">GoogleCloudBaseHook</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Interact with Google Cloud Datastore. This hook uses the Google Cloud Platform</span>
+<span class="sd">    connection.</span>
+
+<span class="sd">    This object is not threads safe. If you want to make multiple requests</span>
+<span class="sd">    simultaneously, you will need to create a hook per thread.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+                 <span class="n">datastore_conn_id</span><span class="o">=</span><span class="s1">&#39;google_cloud_datastore_default&#39;</span><span class="p">,</span>
+                 <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">DatastoreHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">datastore_conn_id</span><span class="p">,</span> <span class="n">delegate_to</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">connection</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">admin_connection</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">(</span><span class="s1">&#39;v1beta1&#39;</span><span class="p">)</span>
+
+<div class="viewcode-block" id="DatastoreHook.get_conn"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook.get_conn">[docs]</a>    <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">version</span><span class="o">=</span><span class="s1">&#39;v1&#39;</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Returns a Google Cloud Datastore service object.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">http_authorized</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_authorize</span><span class="p">()</span>
+        <span class="k">return</span> <span class="n">build</span><span class="p">(</span>
+            <span class="s1">&#39;datastore&#39;</span><span class="p">,</span> <span class="n">version</span><span class="p">,</span> <span class="n">http</span><span class="o">=</span><span class="n">http_authorized</span><span class="p">,</span> <span class="n">cache_discovery</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="DatastoreHook.allocate_ids"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook.allocate_ids">[docs]</a>    <span class="k">def</span> <span class="nf">allocate_ids</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">partialKeys</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Allocate IDs for incomplete keys.</span>
+<span class="sd">        see https://cloud.google.com/datastore/docs/reference/rest/v1/projects/allocateIds</span>
+
+<span class="sd">        :param partialKeys: a list of partial keys</span>
+<span class="sd">        :return: a list of full keys.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">allocateIds</span><span class="p">(</span>
+            <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="p">{</span><span class="s1">&#39;keys&#39;</span><span class="p">:</span> <span class="n">partialKeys</span><span class="p">}</span>
+        <span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+        <span class="k">return</span> <span class="n">resp</span><span class="p">[</span><span class="s1">&#39;keys&#39;</span><span class="p">]</span></div>
+
+<div class="viewcode-block" id="DatastoreHook.begin_transaction"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook.begin_transaction">[docs]</a>    <span class="k">def</span> <span class="nf">begin_transaction</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Get a new transaction handle</span>
+
+<span class="sd">            .. seealso::</span>
+<span class="sd">                https://cloud.google.com/datastore/docs/reference/rest/v1/projects/beginTransaction</span>
+
+<span class="sd">        :return: a transaction handle</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">beginTransaction</span><span class="p">(</span>
+            <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="p">{})</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+        <span class="k">return</span> <span class="n">resp</span><span class="p">[</span><span class="s1">&#39;transaction&#39;</span><span class="p">]</span></div>
+
+<div class="viewcode-block" id="DatastoreHook.commit"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook.commit">[docs]</a>    <span class="k">def</span> <span class="nf">commit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">body</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Commit a transaction, optionally creating, deleting or modifying some entities.</span>
+
+<span class="sd">        .. seealso::</span>
+<span class="sd">            https://cloud.google.com/datastore/docs/reference/rest/v1/projects/commit</span>
+
+<span class="sd">        :param body: the body of the commit request</span>
+<span class="sd">        :return: the response body of the commit request</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">commit</span><span class="p">(</span>
+            <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="n">body</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+        <span class="k">return</span> <span class="n">resp</span></div>
+
+<div class="viewcode-block" id="DatastoreHook.lookup"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.hooks.datastore_hook.DatastoreHook.lookup">[docs]</a>    <span class="k">def</span> <span class="nf">lookup</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">keys</span><span class="p">,</span> <span class="n">read_consistency</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <sp [...]
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Lookup some entities by key</span>
+
+<span class="sd">        .. seealso::</span>
+<span class="sd">            https://cloud.google.com/datastore/docs/reference/rest/v1/projects/lookup</span>
+
+<span class="sd">