| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| |
| --- |
| package-name: apache-airflow-providers-amazon |
| name: Amazon |
| description: | |
| Amazon integration (including `Amazon Web Services (AWS) <https://aws.amazon.com/>`__). |
| |
| suspended: false |
| versions: |
| - 8.11.0 |
| - 8.10.0 |
| - 8.9.0 |
| - 8.8.0 |
| - 8.7.1 |
| - 8.7.0 |
| - 8.6.0 |
| - 8.5.1 |
| - 8.5.0 |
| - 8.4.0 |
| - 8.3.1 |
| - 8.3.0 |
| - 8.2.0 |
| - 8.1.0 |
| - 8.0.0 |
| - 7.4.1 |
| - 7.4.0 |
| - 7.3.0 |
| - 7.2.1 |
| - 7.2.0 |
| - 7.1.0 |
| - 7.0.0 |
| - 6.2.0 |
| - 6.1.0 |
| - 6.0.0 |
| - 5.1.0 |
| - 5.0.0 |
| - 4.1.0 |
| - 4.0.0 |
| - 3.4.0 |
| - 3.3.0 |
| - 3.2.0 |
| - 3.1.1 |
| - 3.0.0 |
| - 2.6.0 |
| - 2.5.0 |
| - 2.4.0 |
| - 2.3.0 |
| - 2.2.0 |
| - 2.1.0 |
| - 2.0.0 |
| - 1.4.0 |
| - 1.3.0 |
| - 1.2.0 |
| - 1.1.0 |
| - 1.0.0 |
| |
| dependencies: |
| - apache-airflow>=2.5.0 |
| - apache-airflow-providers-common-sql>=1.3.1 |
| - apache-airflow-providers-http |
| # We should update minimum version of boto3 and here regularly to avoid `pip` backtracking with the number |
| # of candidates to consider. We should also make sure that all the below related packages have also the |
| # same minimum version specified. Boto3 1.28.0 has been released on July 6 2023. We should also make sure we |
| # set it to the version that `aiobotocore` supports (see `aiobotocore` optional dependency at the end |
| # of this file). Currently we set aiobotocore as minimum 2.5.3 - as this is was the first version |
| # that supported boto3 1.28. NOTE!!! BOTOCORE VERSIONS ARE SHIFTED BY 3 MINOR VERSIONS |
| # NOTE!!! Make sure to update _MIN_BOTO3_VERSION in setup.py when you update it here |
| - boto3>=1.28.0 |
| # NOTE!!! BOTOCORE version is always shifted by 3 MINOR VERSIONS from boto3 |
| # See https://github.com/boto/boto3/issues/2702 |
| - botocore>=1.31.0 |
| - inflection>=0.5.1 |
| - watchtower~=3.0.1 |
| - jsonpath_ng>=1.5.3 |
| - redshift_connector>=2.0.888 |
| - sqlalchemy_redshift>=0.8.6 |
| - asgiref |
| |
| integrations: |
| - integration-name: Amazon Athena |
| external-doc-url: https://aws.amazon.com/athena/ |
| logo: /integration-logos/aws/Amazon-Athena_light-bg@4x.png |
| how-to-guide: |
| - /docs/apache-airflow-providers-amazon/operators/athena.rst |
| tags: [aws] |
| - integration-name: Amazon Chime |
| external-doc-url: https://aws.amazon.com/chime/ |
| logo: /integration-logos/aws/Amazon-Chime-light-bg.png |
| tags: [aws] |
| - integration-name: Amazon CloudFormation |
| external-doc-url: https://aws.amazon.com/cloudformation/ |
| logo: /integration-logos/aws/AWS-CloudFormation_light-bg@4x.png |
| how-to-guide: |
| - /docs/apache-airflow-providers-amazon/operators/cloudformation.rst |
| tags: [aws] |
| - integration-name: Amazon CloudWatch Logs |
| external-doc-url: https://aws.amazon.com/cloudwatch/ |
| logo: /integration-logos/aws/Amazon-CloudWatch_light-bg@4x.png |
| tags: [aws] |
| - integration-name: Amazon DataSync |
| external-doc-url: https://aws.amazon.com/datasync/ |
| how-to-guide: |
| - /docs/apache-airflow-providers-amazon/operators/datasync.rst |
| tags: [aws] |
| - integration-name: Amazon DynamoDB |
| external-doc-url: https://aws.amazon.com/dynamodb/ |
| logo: /integration-logos/aws/Amazon-DynamoDB_light-bg@4x.png |
| how-to-guide: |
| - /docs/apache-airflow-providers-amazon/operators/dynamodb.rst |
| tags: [aws] |
| - integration-name: Amazon EC2 |
| external-doc-url: https://aws.amazon.com/ec2/ |
| logo: /integration-logos/aws/Amazon-EC2_light-bg@4x.png |
| how-to-guide: |
| - /docs/apache-airflow-providers-amazon/operators/ec2.rst |
| tags: [aws] |
| - integration-name: Amazon Elastic Container Registry (ECR) |
| external-doc-url: https://aws.amazon.com/ecr/ |
| logo: /integration-logos/aws/Amazon-Elastic-Container-Registry_light-bg@4x.png |
| tags: [aws] |
| - integration-name: Amazon ECS |
| external-doc-url: https://aws.amazon.com/ecs/ |
| logo: /integration-logos/aws/Amazon-Elastic-Container-Service_light-bg@4x.png |
| tags: [aws] |
| - integration-name: Amazon Elastic Kubernetes Service (EKS) |
| external-doc-url: https://aws.amazon.com/eks/ |
| logo: /integration-logos/aws/Amazon-Elastic-Kubernetes-Service_light-bg@4x.png |
| how-to-guide: |
| - /docs/apache-airflow-providers-amazon/operators/eks.rst |
| tags: [aws] |
| - integration-name: Amazon ElastiCache |
| external-doc-url: https://aws.amazon.com/elasticache/redis// |
| logo: /integration-logos/aws/Amazon-ElastiCache_light-bg@4x.png |
| tags: [aws] |
| - integration-name: Amazon EMR |
| external-doc-url: https://aws.amazon.com/emr/ |
| how-to-guide: |
| - /docs/apache-airflow-providers-amazon/operators/emr/emr.rst |
| logo: /integration-logos/aws/Amazon-EMR_light-bg@4x.png |
| tags: [aws] |
| - integration-name: Amazon EMR on EKS |
| external-doc-url: https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/emr-eks.html |
| how-to-guide: |
| - /docs/apache-airflow-providers-amazon/operators/emr/emr_eks.rst |
| logo: /integration-logos/aws/Amazon-EMR_light-bg@4x.png |
| tags: [aws] |
| - integration-name: Amazon EMR Serverless |
| external-doc-url: https://docs.aws.amazon.com/emr/latest/EMR-Serverless-UserGuide/emr-serverless.html |
| how-to-guide: |
| - /docs/apache-airflow-providers-amazon/operators/emr/emr_serverless.rst |
| logo: /integration-logos/aws/Amazon-EMR_light-bg@4x.png |
| tags: [aws] |
| - integration-name: Amazon EventBridge |
| external-doc-url: https://docs.aws.amazon.com/eventbridge/latest/APIReference/Welcome.html |
| how-to-guide: |
| - /docs/apache-airflow-providers-amazon/operators/eventbridge.rst |
| logo: /integration-logos/aws/Amazon-EventBridge_64.png |
| tags: [aws] |
| - integration-name: Amazon Glacier |
| external-doc-url: https://aws.amazon.com/glacier/ |
| logo: /integration-logos/aws/Amazon-S3-Glacier_light-bg@4x.png |
| how-to-guide: |
| - /docs/apache-airflow-providers-amazon/operators/s3/glacier.rst |
| tags: [aws] |
| - integration-name: Amazon Kinesis Data Firehose |
| external-doc-url: https://aws.amazon.com/kinesis/data-firehose/ |
| logo: /integration-logos/aws/Amazon-Kinesis-Data-Firehose_light-bg@4x.png |
| tags: [aws] |
| - integration-name: Amazon RDS |
| external-doc-url: https://aws.amazon.com/rds/ |
| logo: /integration-logos/aws/Amazon-RDS_light-bg@4x.png |
| how-to-guide: |
| - /docs/apache-airflow-providers-amazon/operators/rds.rst |
| tags: [aws] |
| - integration-name: Amazon Redshift |
| external-doc-url: https://aws.amazon.com/redshift/ |
| logo: /integration-logos/aws/Amazon-Redshift_light-bg@4x.png |
| how-to-guide: |
| - /docs/apache-airflow-providers-amazon/operators/redshift/redshift_sql.rst |
| - /docs/apache-airflow-providers-amazon/operators/redshift/redshift_cluster.rst |
| tags: [aws] |
| - integration-name: Amazon Redshift Data |
| external-doc-url: https://aws.amazon.com/redshift/ |
| logo: /integration-logos/aws/Amazon-Redshift_light-bg@4x.png |
| how-to-guide: |
| - /docs/apache-airflow-providers-amazon/operators/redshift/redshift_data.rst |
| tags: [aws] |
| - integration-name: Amazon SageMaker |
| external-doc-url: https://aws.amazon.com/sagemaker/ |
| logo: /integration-logos/aws/Amazon-SageMaker_light-bg@4x.png |
| how-to-guide: |
| - /docs/apache-airflow-providers-amazon/operators/sagemaker.rst |
| tags: [aws] |
| - integration-name: Amazon SecretsManager |
| external-doc-url: https://aws.amazon.com/secrets-manager/ |
| logo: /integration-logos/aws/AWS-Secrets-Manager_light-bg@4x.png |
| tags: [aws] |
| - integration-name: Amazon Simple Email Service (SES) |
| external-doc-url: https://aws.amazon.com/ses/ |
| logo: /integration-logos/aws/Amazon-Simple-Email-Service-SES_light-bg@4x.png |
| how-to-guide: |
| - /docs/apache-airflow-providers-amazon/operators/ecs.rst |
| tags: [aws] |
| - integration-name: Amazon Simple Notification Service (SNS) |
| external-doc-url: https://aws.amazon.com/sns/ |
| logo: /integration-logos/aws/Amazon-Simple-Notification-Service-SNS_light-bg@4x.png |
| how-to-guide: |
| - /docs/apache-airflow-providers-amazon/operators/sns.rst |
| tags: [aws] |
| - integration-name: Amazon Simple Queue Service (SQS) |
| external-doc-url: https://aws.amazon.com/sqs/ |
| logo: /integration-logos/aws/Amazon-Simple-Queue-Service-SQS_light-bg@4x.png |
| how-to-guide: |
| - /docs/apache-airflow-providers-amazon/operators/sqs.rst |
| tags: [aws] |
| - integration-name: Amazon Simple Storage Service (S3) |
| external-doc-url: https://aws.amazon.com/s3/ |
| logo: /integration-logos/aws/Amazon-Simple-Storage-Service-S3_light-bg@4x.png |
| how-to-guide: |
| - /docs/apache-airflow-providers-amazon/operators/s3/s3.rst |
| tags: [aws] |
| - integration-name: Amazon Systems Manager (SSM) |
| external-doc-url: https://aws.amazon.com/systems-manager/ |
| logo: /integration-logos/aws/AWS-Systems-Manager_light-bg@4x.png |
| tags: [aws] |
| - integration-name: Amazon Web Services |
| external-doc-url: https://aws.amazon.com/ |
| logo: /integration-logos/aws/AWS-Cloud-alt_light-bg@4x.png |
| tags: [aws] |
| - integration-name: AWS Batch |
| external-doc-url: https://aws.amazon.com/batch/ |
| logo: /integration-logos/aws/AWS-Batch_light-bg@4x.png |
| how-to-guide: |
| - /docs/apache-airflow-providers-amazon/operators/batch.rst |
| tags: [aws] |
| - integration-name: AWS DataSync |
| external-doc-url: https://aws.amazon.com/datasync/ |
| logo: /integration-logos/aws/AWS-DataSync_light-bg@4x.png |
| tags: [aws] |
| - integration-name: AWS Glue |
| external-doc-url: https://aws.amazon.com/glue/ |
| logo: /integration-logos/aws/AWS-Glue_light-bg@4x.png |
| how-to-guide: |
| - /docs/apache-airflow-providers-amazon/operators/glue.rst |
| tags: [aws] |
| - integration-name: AWS Lambda |
| external-doc-url: https://aws.amazon.com/lambda/ |
| logo: /integration-logos/aws/AWS-Lambda_light-bg@4x.png |
| how-to-guide: |
| - /docs/apache-airflow-providers-amazon/operators/lambda.rst |
| tags: [aws] |
| - integration-name: AWS Step Functions |
| external-doc-url: https://aws.amazon.com/step-functions/ |
| logo: /integration-logos/aws/AWS-Step-Functions_light-bg@4x.png |
| how-to-guide: |
| - /docs/apache-airflow-providers-amazon/operators/step_functions.rst |
| tags: [aws] |
| - integration-name: AWS Database Migration Service |
| external-doc-url: https://aws.amazon.com/dms/ |
| logo: /integration-logos/aws/AWS-Database-Migration-Service_64@5x.png |
| how-to-guide: |
| - /docs/apache-airflow-providers-amazon/operators/dms.rst |
| tags: [aws] |
| - integration-name: Amazon QuickSight |
| external-doc-url: https://aws.amazon.com/quicksight/ |
| logo: /integration-logos/aws/Amazon-Quicksight_light-bg@4x.png |
| how-to-guide: |
| - /docs/apache-airflow-providers-amazon/operators/quicksight.rst |
| tags: [aws] |
| - integration-name: AWS Security Token Service (STS) |
| external-doc-url: https://docs.aws.amazon.com/STS/latest/APIReference/welcome.html |
| logo: /integration-logos/aws/AWS-STS_light-bg@4x.png |
| tags: [aws] |
| - integration-name: Amazon Appflow |
| external-doc-url: https://docs.aws.amazon.com/appflow/1.0/APIReference/Welcome.html |
| logo: /integration-logos/aws/Amazon_AppFlow_light.png |
| how-to-guide: |
| - /docs/apache-airflow-providers-amazon/operators/appflow.rst |
| tags: [aws] |
| - integration-name: AWS Glue DataBrew |
| external-doc-url: https://docs.aws.amazon.com/databrew/latest/dg/what-is.html |
| how-to-guide: |
| - /docs/apache-airflow-providers-amazon/operators/glue_databrew.rst |
| logo: /integration-logos/aws/AWS-Glue-DataBrew_64.png |
| tags: [aws] |
| |
| operators: |
| - integration-name: Amazon Athena |
| python-modules: |
| - airflow.providers.amazon.aws.operators.athena |
| - integration-name: Amazon Web Services |
| python-modules: |
| - airflow.providers.amazon.aws.operators.base_aws |
| - integration-name: AWS Batch |
| python-modules: |
| - airflow.providers.amazon.aws.operators.batch |
| - integration-name: Amazon CloudFormation |
| python-modules: |
| - airflow.providers.amazon.aws.operators.cloud_formation |
| - integration-name: Amazon DataSync |
| python-modules: |
| - airflow.providers.amazon.aws.operators.datasync |
| - integration-name: AWS Database Migration Service |
| python-modules: |
| - airflow.providers.amazon.aws.operators.dms |
| - integration-name: Amazon EC2 |
| python-modules: |
| - airflow.providers.amazon.aws.operators.ec2 |
| - integration-name: Amazon ECS |
| python-modules: |
| - airflow.providers.amazon.aws.operators.ecs |
| - integration-name: Amazon Elastic Kubernetes Service (EKS) |
| python-modules: |
| - airflow.providers.amazon.aws.operators.eks |
| - integration-name: Amazon EMR |
| python-modules: |
| - airflow.providers.amazon.aws.operators.emr |
| - integration-name: Amazon EMR on EKS |
| python-modules: |
| - airflow.providers.amazon.aws.operators.emr |
| - integration-name: Amazon EventBridge |
| python-modules: |
| - airflow.providers.amazon.aws.operators.eventbridge |
| - integration-name: Amazon Glacier |
| python-modules: |
| - airflow.providers.amazon.aws.operators.glacier |
| - integration-name: AWS Glue |
| python-modules: |
| - airflow.providers.amazon.aws.operators.glue |
| - airflow.providers.amazon.aws.operators.glue_crawler |
| - integration-name: AWS Lambda |
| python-modules: |
| - airflow.providers.amazon.aws.operators.lambda_function |
| - integration-name: Amazon Simple Storage Service (S3) |
| python-modules: |
| - airflow.providers.amazon.aws.operators.s3 |
| - integration-name: Amazon SageMaker |
| python-modules: |
| - airflow.providers.amazon.aws.operators.sagemaker |
| - integration-name: Amazon Simple Notification Service (SNS) |
| python-modules: |
| - airflow.providers.amazon.aws.operators.sns |
| - integration-name: Amazon Simple Queue Service (SQS) |
| python-modules: |
| - airflow.providers.amazon.aws.operators.sqs |
| - integration-name: AWS Step Functions |
| python-modules: |
| - airflow.providers.amazon.aws.operators.step_function |
| - integration-name: Amazon RDS |
| python-modules: |
| - airflow.providers.amazon.aws.operators.rds |
| - integration-name: Amazon Redshift |
| python-modules: |
| - airflow.providers.amazon.aws.operators.redshift_cluster |
| - airflow.providers.amazon.aws.operators.redshift_data |
| - integration-name: Amazon QuickSight |
| python-modules: |
| - airflow.providers.amazon.aws.operators.quicksight |
| - integration-name: Amazon Appflow |
| python-modules: |
| - airflow.providers.amazon.aws.operators.appflow |
| - integration-name: AWS Glue DataBrew |
| python-modules: |
| - airflow.providers.amazon.aws.operators.glue_databrew |
| |
| sensors: |
| - integration-name: Amazon Athena |
| python-modules: |
| - airflow.providers.amazon.aws.sensors.athena |
| - integration-name: Amazon Web Services |
| python-modules: |
| - airflow.providers.amazon.aws.sensors.base_aws |
| - integration-name: AWS Batch |
| python-modules: |
| - airflow.providers.amazon.aws.sensors.batch |
| - integration-name: Amazon CloudFormation |
| python-modules: |
| - airflow.providers.amazon.aws.sensors.cloud_formation |
| - integration-name: AWS Database Migration Service |
| python-modules: |
| - airflow.providers.amazon.aws.sensors.dms |
| - integration-name: Amazon DynamoDB |
| python-modules: |
| - airflow.providers.amazon.aws.sensors.dynamodb |
| - integration-name: Amazon EC2 |
| python-modules: |
| - airflow.providers.amazon.aws.sensors.ec2 |
| - integration-name: Amazon ECS |
| python-modules: |
| - airflow.providers.amazon.aws.sensors.ecs |
| - integration-name: Amazon Elastic Kubernetes Service (EKS) |
| python-modules: |
| - airflow.providers.amazon.aws.sensors.eks |
| - integration-name: Amazon EMR |
| python-modules: |
| - airflow.providers.amazon.aws.sensors.emr |
| - integration-name: Amazon EMR on EKS |
| python-modules: |
| - airflow.providers.amazon.aws.sensors.emr |
| - integration-name: Amazon Glacier |
| python-modules: |
| - airflow.providers.amazon.aws.sensors.glacier |
| - integration-name: AWS Glue |
| python-modules: |
| - airflow.providers.amazon.aws.sensors.glue |
| - airflow.providers.amazon.aws.sensors.glue_crawler |
| - airflow.providers.amazon.aws.sensors.glue_catalog_partition |
| - integration-name: AWS Lambda |
| python-modules: |
| - airflow.providers.amazon.aws.sensors.lambda_function |
| - integration-name: Amazon RDS |
| python-modules: |
| - airflow.providers.amazon.aws.sensors.rds |
| - integration-name: Amazon Redshift |
| python-modules: |
| - airflow.providers.amazon.aws.sensors.redshift_cluster |
| - integration-name: Amazon Simple Storage Service (S3) |
| python-modules: |
| - airflow.providers.amazon.aws.sensors.s3 |
| - integration-name: Amazon SageMaker |
| python-modules: |
| - airflow.providers.amazon.aws.sensors.sagemaker |
| - integration-name: Amazon Simple Queue Service (SQS) |
| python-modules: |
| - airflow.providers.amazon.aws.sensors.sqs |
| - integration-name: AWS Step Functions |
| python-modules: |
| - airflow.providers.amazon.aws.sensors.step_function |
| - integration-name: Amazon QuickSight |
| python-modules: |
| - airflow.providers.amazon.aws.sensors.quicksight |
| |
| filesystems: |
| - airflow.providers.amazon.aws.fs.s3 |
| |
| hooks: |
| - integration-name: Amazon Athena |
| python-modules: |
| - airflow.providers.amazon.aws.hooks.athena |
| - integration-name: Amazon Chime |
| python-modules: |
| - airflow.providers.amazon.aws.hooks.chime |
| - integration-name: Amazon DynamoDB |
| python-modules: |
| - airflow.providers.amazon.aws.hooks.dynamodb |
| - integration-name: Amazon Web Services |
| python-modules: |
| - airflow.providers.amazon.aws.hooks.base_aws |
| - integration-name: AWS Batch |
| python-modules: |
| - airflow.providers.amazon.aws.hooks.batch_client |
| - airflow.providers.amazon.aws.hooks.batch_waiters |
| - integration-name: Amazon CloudFormation |
| python-modules: |
| - airflow.providers.amazon.aws.hooks.cloud_formation |
| - integration-name: Amazon DataSync |
| python-modules: |
| - airflow.providers.amazon.aws.hooks.datasync |
| - integration-name: AWS Database Migration Service |
| python-modules: |
| - airflow.providers.amazon.aws.hooks.dms |
| - integration-name: Amazon EC2 |
| python-modules: |
| - airflow.providers.amazon.aws.hooks.ec2 |
| - integration-name: Amazon Elastic Container Registry (ECR) |
| python-modules: |
| - airflow.providers.amazon.aws.hooks.ecr |
| - integration-name: Amazon ECS |
| python-modules: |
| - airflow.providers.amazon.aws.hooks.ecs |
| - integration-name: Amazon ElastiCache |
| python-modules: |
| - airflow.providers.amazon.aws.hooks.elasticache_replication_group |
| - integration-name: Amazon Elastic Kubernetes Service (EKS) |
| python-modules: |
| - airflow.providers.amazon.aws.hooks.eks |
| - integration-name: Amazon EMR |
| python-modules: |
| - airflow.providers.amazon.aws.hooks.emr |
| - integration-name: Amazon EMR on EKS |
| python-modules: |
| - airflow.providers.amazon.aws.hooks.emr |
| - integration-name: Amazon EventBridge |
| python-modules: |
| - airflow.providers.amazon.aws.hooks.eventbridge |
| - integration-name: Amazon Glacier |
| python-modules: |
| - airflow.providers.amazon.aws.hooks.glacier |
| - integration-name: AWS Glue |
| python-modules: |
| - airflow.providers.amazon.aws.hooks.glue |
| - airflow.providers.amazon.aws.hooks.glue_crawler |
| - airflow.providers.amazon.aws.hooks.glue_catalog |
| - integration-name: Amazon Kinesis Data Firehose |
| python-modules: |
| - airflow.providers.amazon.aws.hooks.kinesis |
| - integration-name: AWS Lambda |
| python-modules: |
| - airflow.providers.amazon.aws.hooks.lambda_function |
| - integration-name: Amazon CloudWatch Logs |
| python-modules: |
| - airflow.providers.amazon.aws.hooks.logs |
| - integration-name: Amazon RDS |
| python-modules: |
| - airflow.providers.amazon.aws.hooks.rds |
| - integration-name: Amazon Redshift |
| python-modules: |
| - airflow.providers.amazon.aws.hooks.redshift_sql |
| - airflow.providers.amazon.aws.hooks.redshift_cluster |
| - airflow.providers.amazon.aws.hooks.redshift_data |
| - integration-name: Amazon Simple Storage Service (S3) |
| python-modules: |
| - airflow.providers.amazon.aws.hooks.s3 |
| - integration-name: Amazon SageMaker |
| python-modules: |
| - airflow.providers.amazon.aws.hooks.sagemaker |
| - integration-name: Amazon Simple Email Service (SES) |
| python-modules: |
| - airflow.providers.amazon.aws.hooks.ses |
| - integration-name: Amazon Systems Manager (SSM) |
| python-modules: |
| - airflow.providers.amazon.aws.hooks.ssm |
| - integration-name: Amazon SecretsManager |
| python-modules: |
| - airflow.providers.amazon.aws.hooks.secrets_manager |
| - integration-name: Amazon Simple Notification Service (SNS) |
| python-modules: |
| - airflow.providers.amazon.aws.hooks.sns |
| - integration-name: Amazon Simple Queue Service (SQS) |
| python-modules: |
| - airflow.providers.amazon.aws.hooks.sqs |
| - integration-name: AWS Step Functions |
| python-modules: |
| - airflow.providers.amazon.aws.hooks.step_function |
| - integration-name: Amazon QuickSight |
| python-modules: |
| - airflow.providers.amazon.aws.hooks.quicksight |
| - integration-name: AWS Security Token Service (STS) |
| python-modules: |
| - airflow.providers.amazon.aws.hooks.sts |
| - integration-name: Amazon Appflow |
| python-modules: |
| - airflow.providers.amazon.aws.hooks.appflow |
| - integration-name: AWS Glue DataBrew |
| python-modules: |
| - airflow.providers.amazon.aws.hooks.glue_databrew |
| |
| triggers: |
| - integration-name: Amazon Web Services |
| python-modules: |
| - airflow.providers.amazon.aws.triggers.base |
| - integration-name: Amazon Athena |
| python-modules: |
| - airflow.providers.amazon.aws.triggers.athena |
| - integration-name: AWS Batch |
| python-modules: |
| - airflow.providers.amazon.aws.triggers.batch |
| - integration-name: Amazon EC2 |
| python-modules: |
| - airflow.providers.amazon.aws.triggers.ec2 |
| - integration-name: AWS Lambda |
| python-modules: |
| - airflow.providers.amazon.aws.triggers.lambda_function |
| - integration-name: Amazon Redshift |
| python-modules: |
| - airflow.providers.amazon.aws.triggers.redshift_cluster |
| - integration-name: Amazon SageMaker |
| python-modules: |
| - airflow.providers.amazon.aws.triggers.sagemaker |
| - integration-name: AWS Glue |
| python-modules: |
| - airflow.providers.amazon.aws.triggers.glue |
| - airflow.providers.amazon.aws.triggers.glue_crawler |
| - integration-name: Amazon Simple Storage Service (S3) |
| python-modules: |
| - airflow.providers.amazon.aws.triggers.s3 |
| - integration-name: Amazon EMR |
| python-modules: |
| - airflow.providers.amazon.aws.triggers.emr |
| - integration-name: Amazon Elastic Kubernetes Service (EKS) |
| python-modules: |
| - airflow.providers.amazon.aws.triggers.eks |
| - integration-name: Amazon ECS |
| python-modules: |
| - airflow.providers.amazon.aws.triggers.ecs |
| - integration-name: Amazon RDS |
| python-modules: |
| - airflow.providers.amazon.aws.triggers.rds |
| - integration-name: AWS Step Functions |
| python-modules: |
| - airflow.providers.amazon.aws.triggers.step_function |
| - integration-name: Amazon Simple Queue Service (SQS) |
| python-modules: |
| - airflow.providers.amazon.aws.triggers.sqs |
| - integration-name: AWS Glue DataBrew |
| python-modules: |
| - airflow.providers.amazon.aws.triggers.glue_databrew |
| |
| transfers: |
| - source-integration-name: Amazon DynamoDB |
| target-integration-name: Amazon Simple Storage Service (S3) |
| how-to-guide: /docs/apache-airflow-providers-amazon/transfer/dynamodb_to_s3.rst |
| python-module: airflow.providers.amazon.aws.transfers.dynamodb_to_s3 |
| - source-integration-name: Google Cloud Storage (GCS) |
| target-integration-name: Amazon Simple Storage Service (S3) |
| how-to-guide: /docs/apache-airflow-providers-amazon/transfer/gcs_to_s3.rst |
| python-module: airflow.providers.amazon.aws.transfers.gcs_to_s3 |
| - source-integration-name: Amazon Glacier |
| target-integration-name: Google Cloud Storage (GCS) |
| how-to-guide: /docs/apache-airflow-providers-amazon/transfer/glacier_to_gcs.rst |
| python-module: airflow.providers.amazon.aws.transfers.glacier_to_gcs |
| - source-integration-name: Google |
| target-integration-name: Amazon Simple Storage Service (S3) |
| how-to-guide: /docs/apache-airflow-providers-amazon/transfer/google_api_to_s3.rst |
| python-module: airflow.providers.amazon.aws.transfers.google_api_to_s3 |
| - source-integration-name: Apache Hive |
| target-integration-name: Amazon DynamoDB |
| how-to-guide: /docs/apache-airflow-providers-amazon/transfer/hive_to_dynamodb.rst |
| python-module: airflow.providers.amazon.aws.transfers.hive_to_dynamodb |
| - source-integration-name: Hypertext Transfer Protocol (HTTP) |
| target-integration-name: Amazon Simple Storage Service (S3) |
| how-to-guide: /docs/apache-airflow-providers-amazon/transfer/http_to_s3.rst |
| python-module: airflow.providers.amazon.aws.transfers.http_to_s3 |
| - source-integration-name: Internet Message Access Protocol (IMAP) |
| target-integration-name: Amazon Simple Storage Service (S3) |
| how-to-guide: /docs/apache-airflow-providers-amazon/transfer/imap_attachment_to_s3.rst |
| python-module: airflow.providers.amazon.aws.transfers.imap_attachment_to_s3 |
| - source-integration-name: MongoDB |
| target-integration-name: Amazon Simple Storage Service (S3) |
| how-to-guide: /docs/apache-airflow-providers-amazon/transfer/mongo_to_s3.rst |
| python-module: airflow.providers.amazon.aws.transfers.mongo_to_s3 |
| - source-integration-name: Amazon Redshift |
| target-integration-name: Amazon Simple Storage Service (S3) |
| how-to-guide: /docs/apache-airflow-providers-amazon/transfer/redshift_to_s3.rst |
| python-module: airflow.providers.amazon.aws.transfers.redshift_to_s3 |
| - source-integration-name: Amazon Simple Storage Service (S3) |
| target-integration-name: Amazon Redshift |
| how-to-guide: /docs/apache-airflow-providers-amazon/transfer/s3_to_redshift.rst |
| python-module: airflow.providers.amazon.aws.transfers.s3_to_redshift |
| - source-integration-name: Amazon Simple Storage Service (S3) |
| target-integration-name: SSH File Transfer Protocol (SFTP) |
| how-to-guide: /docs/apache-airflow-providers-amazon/transfer/s3_to_sftp.rst |
| python-module: airflow.providers.amazon.aws.transfers.s3_to_sftp |
| - source-integration-name: SSH File Transfer Protocol (SFTP) |
| target-integration-name: Amazon Simple Storage Service (S3) |
| how-to-guide: /docs/apache-airflow-providers-amazon/transfer/sftp_to_s3.rst |
| python-module: airflow.providers.amazon.aws.transfers.sftp_to_s3 |
| - source-integration-name: Amazon Simple Storage Service (S3) |
| target-integration-name: File Transfer Protocol (FTP) |
| how-to-guide: /docs/apache-airflow-providers-amazon/transfer/s3_to_ftp.rst |
| python-module: airflow.providers.amazon.aws.transfers.s3_to_ftp |
| - source-integration-name: Exasol |
| target-integration-name: Amazon Simple Storage Service (S3) |
| python-module: airflow.providers.amazon.aws.transfers.exasol_to_s3 |
| - source-integration-name: File Transfer Protocol (FTP) |
| target-integration-name: Amazon Simple Storage Service (S3) |
| how-to-guide: /docs/apache-airflow-providers-amazon/transfer/ftp_to_s3.rst |
| python-module: airflow.providers.amazon.aws.transfers.ftp_to_s3 |
| - source-integration-name: Salesforce |
| target-integration-name: Amazon Simple Storage Service (S3) |
| how-to-guide: /docs/apache-airflow-providers-amazon/transfer/salesforce_to_s3.rst |
| python-module: airflow.providers.amazon.aws.transfers.salesforce_to_s3 |
| - source-integration-name: Local |
| target-integration-name: Amazon Simple Storage Service (S3) |
| how-to-guide: /docs/apache-airflow-providers-amazon/transfer/local_to_s3.rst |
| python-module: airflow.providers.amazon.aws.transfers.local_to_s3 |
| - source-integration-name: Common SQL |
| target-integration-name: Amazon Simple Storage Service (S3) |
| how-to-guide: /docs/apache-airflow-providers-amazon/transfer/sql_to_s3.rst |
| python-module: airflow.providers.amazon.aws.transfers.sql_to_s3 |
| - source-integration-name: Amazon Simple Storage Service (S3) |
| target-integration-name: Common SQL |
| how-to-guide: /docs/apache-airflow-providers-amazon/transfer/s3_to_sql.rst |
| python-module: airflow.providers.amazon.aws.transfers.s3_to_sql |
| - source-integration-name: Amazon Web Services |
| target-integration-name: Amazon Web Services |
| python-module: airflow.providers.amazon.aws.transfers.base |
| - source-integration-name: Microsoft Azure Blob Storage |
| target-integration-name: Amazon Simple Storage Service (S3) |
| how-to-guide: /docs/apache-airflow-providers-amazon/transfer/azure_blob_to_s3.rst |
| python-module: airflow.providers.amazon.aws.transfers.azure_blob_to_s3 |
| |
| extra-links: |
| - airflow.providers.amazon.aws.links.batch.BatchJobDefinitionLink |
| - airflow.providers.amazon.aws.links.batch.BatchJobDetailsLink |
| - airflow.providers.amazon.aws.links.batch.BatchJobQueueLink |
| - airflow.providers.amazon.aws.links.emr.EmrClusterLink |
| - airflow.providers.amazon.aws.links.emr.EmrLogsLink |
| - airflow.providers.amazon.aws.links.glue.GlueJobRunDetailsLink |
| - airflow.providers.amazon.aws.links.logs.CloudWatchEventsLink |
| |
| connection-types: |
| - hook-class-name: airflow.providers.amazon.aws.hooks.base_aws.AwsGenericHook |
| connection-type: aws |
| - hook-class-name: airflow.providers.amazon.aws.hooks.chime.ChimeWebhookHook |
| connection-type: chime |
| - hook-class-name: airflow.providers.amazon.aws.hooks.emr.EmrHook |
| connection-type: emr |
| - hook-class-name: airflow.providers.amazon.aws.hooks.redshift_sql.RedshiftSQLHook |
| connection-type: redshift |
| |
| notifications: |
| - airflow.providers.amazon.aws.notifications.chime.ChimeNotifier |
| - airflow.providers.amazon.aws.notifications.sns.SnsNotifier |
| - airflow.providers.amazon.aws.notifications.sqs.SqsNotifier |
| |
| secrets-backends: |
| - airflow.providers.amazon.aws.secrets.secrets_manager.SecretsManagerBackend |
| - airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend |
| |
| logging: |
| - airflow.providers.amazon.aws.log.s3_task_handler.S3TaskHandler |
| - airflow.providers.amazon.aws.log.cloudwatch_task_handler.CloudwatchTaskHandler |
| |
| additional-extras: |
| - name: pandas |
| dependencies: |
| - pandas>=0.17.1 |
| # There is conflict between boto3 and aiobotocore dependency botocore. |
| # TODO: We can remove it once boto3 and aiobotocore both have compatible botocore version or |
| # boto3 have native async support and we move away from aio aiobotocore |
| - name: aiobotocore |
| dependencies: |
| - aiobotocore[boto3]>=2.5.3 |
| - name: cncf.kubernetes |
| dependencies: |
| - apache-airflow-providers-cncf-kubernetes>=7.2.0 |
| - name: s3fs |
| dependencies: |
| - s3fs>=2023.9.2 |
| |
| config: |
| aws: |
| description: This section contains settings for Amazon Web Services (AWS) integration. |
| options: |
| session_factory: |
| description: | |
| Full import path to the class which implements a custom session factory for |
| ``boto3.session.Session``. For more details please have a look at |
| :ref:`howto/connection:aws:session-factory`. |
| default: ~ |
| example: my_company.aws.MyCustomSessionFactory |
| type: string |
| version_added: 3.1.1 |
| cloudwatch_task_handler_json_serializer: |
| description: | |
| By default, when logging non-string messages, all non-json objects are logged as `null`. |
| Except `datetime` objects which are ISO formatted. Users can optionally use a `repr` serializer or |
| provide their own JSON serializer for any non-JSON-serializable objects in the logged message. |
| |
| * `airflow.providers.amazon.aws.log.cloudwatch_task_handler.json_serialize` uses `repr` (be aware |
| there is the potential of logging sensitive data depending on the `repr` method of logged objects) |
| * `airflow.providers.amazon.aws.log.cloudwatch_task_handler.json_serialize_legacy` uses `null`. |
| |
| If a custom serializer is provided, it must adhere to `Callable[[Any], str | None]`, where `None` |
| serializes to `null` (e.g. `def my_serializer(o: Any) -> str | None`). Since this is on the logging |
| path and it's possible there's an exception being handled, special care should be taken to fail |
| gracefully without raising a new exception inside of your serializer. |
| type: string |
| version_added: 8.7.2 |
| example: airflow.providers.amazon.aws.log.cloudwatch_task_handler.json_serialize |
| default: airflow.providers.amazon.aws.log.cloudwatch_task_handler.json_serialize_legacy |
| aws_ecs_executor: |
| description: | |
| This section only applies if you are using the AwsEcsExecutor in |
| Airflow's ``[core]`` configuration. |
| For more information on any of these execution parameters, see the link below: |
| https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ecs/client/run_task.html |
| For boto3 credential management, see |
| https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html |
| options: |
| conn_id: |
| description: | |
| The Airflow connection (i.e. credentials) used by the ECS executor to make API calls to AWS ECS. |
| version_added: "8.10" |
| type: string |
| example: "aws_default" |
| default: "aws_default" |
| region_name: |
| description: | |
| The name of the AWS Region where Amazon ECS is configured. Required. |
| version_added: "8.10" |
| type: string |
| example: "us-east-1" |
| default: ~ |
| assign_public_ip: |
| description: | |
| Whether to assign a public IP address to the containers launched by the ECS executor. |
| For more info see url to Boto3 docs above. |
| version_added: "8.10" |
| type: boolean |
| example: "True" |
| default: "False" |
| cluster: |
| description: | |
| Name of the Amazon ECS Cluster. Required. |
| version_added: "8.10" |
| type: string |
| example: "ecs_executor_cluster" |
| default: ~ |
| container_name: |
| description: | |
| Name of the container that will be used to execute Airflow tasks via the ECS executor. |
| The container should be specified in the ECS Task Definition and will receive an airflow |
| CLI command as an additional parameter to its entrypoint. For more info see url to Boto3 |
| docs above. Required. |
| version_added: "8.10" |
| type: string |
| example: "ecs_executor_container" |
| default: ~ |
| launch_type: |
| description: | |
| Launch type can either be 'FARGATE' OR 'EC2'. For more info see url to |
| Boto3 docs above. |
| |
| If the launch type is EC2, the executor will attempt to place tasks on |
| empty EC2 instances. If there are no EC2 instances available, no task |
| is placed and this function will be called again in the next heart-beat. |
| |
| If the launch type is FARGATE, this will run the tasks on new AWS Fargate |
| instances. |
| version_added: "8.10" |
| type: string |
| example: "FARGATE" |
| default: "FARGATE" |
| platform_version: |
| description: | |
| The platform version the task uses. A platform version is only specified |
| for tasks hosted on Fargate. If one isn't specified, the LATEST platform |
| version is used. |
| version_added: "8.10" |
| type: string |
| example: "1.4.0" |
| default: "LATEST" |
| security_groups: |
| description: | |
| The comma-separated IDs of the security groups associated with the task. If you |
| don't specify a security group, the default security group for the VPC is used. |
| There's a limit of 5 security groups. For more info see url to Boto3 docs above. |
| version_added: "8.10" |
| type: string |
| example: "sg-XXXX,sg-YYYY" |
| default: ~ |
| subnets: |
| description: | |
| The comma-separated IDs of the subnets associated with the task or service. |
| There's a limit of 16 subnets. For more info see url to Boto3 docs above. |
| version_added: "8.10" |
| type: string |
| example: "subnet-XXXXXXXX,subnet-YYYYYYYY" |
| default: ~ |
| task_definition: |
| description: | |
| The family and revision (family:revision) or full ARN of the task definition |
| to run. If a revision isn't specified, the latest ACTIVE revision is used. |
| For more info see url to Boto3 docs above. |
| version_added: "8.10" |
| type: string |
| example: executor_task_definition:LATEST |
| default: ~ |
| max_run_task_attempts: |
| description: | |
| The maximum number of times the ECS Executor should attempt to run a task. |
| version_added: "8.10" |
| type: integer |
| example: "3" |
| default: "3" |
| run_task_kwargs: |
| description: | |
| A JSON string containing arguments to provide the ECS `run_task` API (see url above). |
| version_added: "8.10" |
| type: string |
| example: '{"tags": {"key": "schema", "value": "1.0"}}' |
| default: ~ |
| check_health_on_startup: |
| description: | |
| Whether or not to check the ECS Executor health on startup. |
| version_added: "8.11" |
| type: boolean |
| example: "True" |
| default: "True" |
| |
| executors: |
| - airflow.providers.amazon.aws.executors.ecs.ecs_executor.AwsEcsExecutor |
