| :mod:`airflow.contrib.hooks.gcp_dataflow_hook` |
| ============================================== |
| |
| .. py:module:: airflow.contrib.hooks.gcp_dataflow_hook |
| |
| |
| Module Contents |
| --------------- |
| |
| .. data:: DEFAULT_DATAFLOW_LOCATION |
| :annotation: = us-central1 |
| |
| |
| |
| .. py:class:: _DataflowJob(dataflow, project_number, name, location, poll_sleep=10, job_id=None, num_retries=None) |
| |
| Bases: :class:`airflow.utils.log.logging_mixin.LoggingMixin` |
| |
| |
| .. method:: _get_job_id_from_name(self) |
| |
| |
| |
| |
| .. method:: _get_job(self) |
| |
| |
| |
| |
| .. method:: wait_for_done(self) |
| |
| |
| |
| |
| .. method:: get(self) |
| |
| |
| |
| |
| .. py:class:: _Dataflow(cmd) |
| |
| Bases: :class:`airflow.utils.log.logging_mixin.LoggingMixin` |
| |
| |
| .. method:: _line(self, fd) |
| |
| |
| |
| |
| .. staticmethod:: _extract_job(line) |
| |
| |
| |
| |
| .. method:: wait_for_done(self) |
| |
| |
| |
| |
| .. py:class:: DataFlowHook(gcp_conn_id='google_cloud_default', delegate_to=None, poll_sleep=10) |
| |
| Bases: :class:`airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook` |
| |
| |
| .. method:: get_conn(self) |
| |
| Returns a Google Cloud Dataflow service object. |
| |
| |
| |
| |
| .. method:: _start_dataflow(self, variables, name, command_prefix, label_formatter) |
| |
| |
| |
| |
| .. staticmethod:: _set_variables(variables) |
| |
| |
| |
| |
| .. method:: start_java_dataflow(self, job_name, variables, dataflow, job_class=None, append_job_name=True) |
| |
| |
| |
| |
| .. method:: start_template_dataflow(self, job_name, variables, parameters, dataflow_template, append_job_name=True) |
| |
| |
| |
| |
| .. method:: start_python_dataflow(self, job_name, variables, dataflow, py_options, append_job_name=True) |
| |
| |
| |
| |
| .. staticmethod:: _build_dataflow_job_name(job_name, append_job_name=True) |
| |
| |
| |
| |
| .. staticmethod:: _build_cmd(variables, label_formatter) |
| |
| |
| |
| |
| .. method:: _start_template_dataflow(self, name, variables, parameters, dataflow_template) |
| |
| |
| |
| |
