| |
| |
| :mod:`airflow.contrib.operators.cassandra_to_gcs` |
| ================================================= |
| |
| .. py:module:: airflow.contrib.operators.cassandra_to_gcs |
| |
| |
| |
| |
| |
| |
| |
| Module Contents |
| --------------- |
| |
| |
| |
| |
| |
| |
| .. py:class:: CassandraToGoogleCloudStorageOperator(cql, bucket, filename, schema_filename=None, approx_max_file_size_bytes=1900000000, cassandra_conn_id='cassandra_default', google_cloud_storage_conn_id='google_cloud_default', delegate_to=None, *args, **kwargs) |
| |
| Bases::class:`airflow.models.BaseOperator` |
| |
| |
| |
| Copy data from Cassandra to Google cloud storage in JSON format |
| |
| Note: Arrays of arrays are not supported. |
| |
| |
| |
| |
| |
| |
| .. attribute:: template_fields |
| :annotation: = ['cql', 'bucket', 'filename', 'schema_filename'] |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| .. attribute:: template_ext |
| :annotation: = ['.cql'] |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| .. attribute:: ui_color |
| :annotation: = #a0e08c |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| .. attribute:: CQL_TYPE_MAP |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| .. method:: execute(self, context) |
| |
| |
| |
| |
| |
| |
| |
| .. method:: _query_cassandra(self) |
| |
| |
| Queries cassandra and returns a cursor to the results. |
| |
| |
| |
| |
| |
| |
| |
| .. method:: _write_local_data_files(self, cursor) |
| |
| |
| Takes a cursor, and writes results to a local file. |
| |
| :return: A dictionary where keys are filenames to be used as object |
| names in GCS, and values are file handles to local files that |
| contain the data for the GCS objects. |
| |
| |
| |
| |
| |
| |
| |
| .. method:: _write_local_schema_file(self, cursor) |
| |
| |
| Takes a cursor, and writes the BigQuery schema for the results to a |
| local file system. |
| |
| :return: A dictionary where key is a filename to be used as an object |
| name in GCS, and values are file handles to local files that |
| contains the BigQuery schema fields in .json format. |
| |
| |
| |
| |
| |
| |
| |
| .. method:: _upload_to_gcs(self, files_to_upload) |
| |
| |
| |
| |
| |
| |
| |
| .. classmethod:: generate_data_dict(cls, names, values) |
| |
| |
| |
| |
| |
| |
| |
| .. classmethod:: convert_value(cls, name, value) |
| |
| |
| |
| |
| |
| |
| |
| .. classmethod:: convert_array_types(cls, name, value) |
| |
| |
| |
| |
| |
| |
| |
| .. classmethod:: convert_user_type(cls, name, value) |
| |
| |
| Converts a user type to RECORD that contains n fields, where n is the |
| number of attributes. Each element in the user type class will be converted to its |
| corresponding data type in BQ. |
| |
| |
| |
| |
| |
| |
| |
| .. classmethod:: convert_tuple_type(cls, name, value) |
| |
| |
| Converts a tuple to RECORD that contains n fields, each will be converted |
| to its corresponding data type in bq and will be named 'field_<index>', where |
| index is determined by the order of the tuple elements defined in cassandra. |
| |
| |
| |
| |
| |
| |
| |
| .. classmethod:: convert_map_type(cls, name, value) |
| |
| |
| Converts a map to a repeated RECORD that contains two fields: 'key' and 'value', |
| each will be converted to its corresponding data type in BQ. |
| |
| |
| |
| |
| |
| |
| |
| .. classmethod:: generate_schema_dict(cls, name, type) |
| |
| |
| |
| |
| |
| |
| |
| .. classmethod:: get_bq_fields(cls, name, type) |
| |
| |
| |
| |
| |
| |
| |
| .. classmethod:: is_simple_type(cls, type) |
| |
| |
| |
| |
| |
| |
| |
| .. classmethod:: is_array_type(cls, type) |
| |
| |
| |
| |
| |
| |
| |
| .. classmethod:: is_record_type(cls, type) |
| |
| |
| |
| |
| |
| |
| |
| .. classmethod:: get_bq_type(cls, type) |
| |
| |
| |
| |
| |
| |
| |
| .. classmethod:: get_bq_mode(cls, type) |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
