Update metadata gen and add RewriteRule (#1181)
* Update metadata gen and add RewriteRule
* Copy generated docs index pages to S3
* Add cloudfront url
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 8343e0d..f05f44c 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -107,6 +107,30 @@
rm -rf ./docs-archive/*
echo "docs-archive directory files are deleted as they are no longer needed after the site is built in the above step"
/mnt/airflow-site/.github/scripts/print_df.sh
+
+ - name: Install AWS CLI v2
+ if: ${{ github.event_name == 'push' }}
+ run: |
+ curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o /tmp/awscliv2.zip
+ unzip -q /tmp/awscliv2.zip -d /tmp
+ rm /tmp/awscliv2.zip
+ sudo /tmp/aws/install --update
+ rm -rf /tmp/aws/
+
+ - name: Configure AWS credentials
+ if: ${{ github.event_name == 'push' }}
+ uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1
+ with:
+ aws-access-key-id: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }}
+ aws-secret-access-key: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }}
+ aws-region: us-east-2
+
+ - name: Copy docs index page to S3 docs
+ if: ${{ github.event_name == 'push' }}
+ run: |
+ aws s3 cp ./dist/docs/index.html s3://staging-docs-airflow-apache-org/docs/index.html
+ aws s3 cp ./dist/docs/index.xml s3://staging-docs-airflow-apache-org/docs/index.xml
+
- name: 🐅 Optimize artifacts
if: ${{ github.event_name == 'pull_request' }}
run: |
diff --git a/dump-docs-packages-metadata.py b/dump-docs-packages-metadata.py
index aee2b9a..6e19559 100755
--- a/dump-docs-packages-metadata.py
+++ b/dump-docs-packages-metadata.py
@@ -1,44 +1,10 @@
#!/usr/bin/env python
-
-import json
import sys
-import os
-from typing import List
-import semver
+import requests
+resp = requests.get("https://staging-docs-airflow-apache-org.s3.us-east-2.amazonaws.com/manifest/packages-metadata.json")
+if resp.status_code != 200:
+ raise RuntimeError(f"Failed to fetch metadata: {resp.status_code} - {resp.text}")
-ROOT_DIR = os.path.dirname(os.path.join(os.path.dirname(os.path.abspath(__file__)), os.path.pardir))
-APACHE_AIRFLOW_ARCHIVE = os.path.join(ROOT_DIR, "docs-archive")
-
-
-def get_all_versions(directory: str) -> List[str]:
- return sorted(
- (f for f in os.listdir(directory) if os.path.isdir(os.path.join(directory, f))),
- key=lambda d: semver.VersionInfo.parse(d),
- )
-
-
-def get_stable_version(directory: str):
- with open(os.path.join(directory, "stable.txt")) as f:
- return f.read().strip()
-
-
-def dump_docs_package_metadata():
- all_packages_infos = [
- {
- "package-name": package_name,
- "stable-version": get_stable_version(os.path.join(APACHE_AIRFLOW_ARCHIVE, package_name)),
- "all-versions": get_all_versions(os.path.join(APACHE_AIRFLOW_ARCHIVE, package_name)),
- }
- for package_name in os.listdir(APACHE_AIRFLOW_ARCHIVE)
- if (
- not package_name.startswith(".") and # Exclude .DS_Store/
- os.path.isfile(os.path.join(os.path.join(APACHE_AIRFLOW_ARCHIVE, package_name, 'stable.txt')))
- )
- ]
-
- json.dump(all_packages_infos, sys.stdout, indent=2)
-
-
-dump_docs_package_metadata()
+sys.stdout.write(resp.text)
diff --git a/landing-pages/site/static/.htaccess b/landing-pages/site/static/.htaccess
index a80ece1..379cdad 100644
--- a/landing-pages/site/static/.htaccess
+++ b/landing-pages/site/static/.htaccess
@@ -1,3 +1,8 @@
+RewriteEngine On
+
+RewriteCond %{REQUEST_URI} ^/docs [NC]
+RewriteRule (.*) https://d3a2du7x0n8ydr.cloudfront.net/$1 [P]
+
RedirectMatch Permanent ^/docs/(stable|1.10.10)/api(\.html)?$ "https://airflow.apache.org/docs/apache-airflow/$1/rest-api-ref"
RedirectMatch Permanent ^/docs/(stable|1.10.10)/(cli|macros)(\.html)?$ "https://airflow.apache.org/docs/apache-airflow/$1/$2-ref"
RedirectMatch Permanent ^/((_api|_images|_modules|_sources|_static|howto)/.*)$ "https://airflow.apache.org/docs/apache-airflow/stable/$1"
