diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c1402af8b4c41da05c599c449b2e0733969af4d1..c92811a9e848abc78de08eaba9b633b64435ff8e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -71,8 +71,13 @@ pages:
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-# Purge all of this site's URLs from the Cloudflare cache
-create-purge-json:
+# Create a list of URLs that need to be purged from the cache after GitLab Pages
+# deployment. Each real file/directory has multiple URLs because the router for
+# GitLab Pages fudges paths for better user experience. The fudging rules are:
+# - Directories can be accessed with or without a trailing slash
+# - Files can be accessed with or without a trailing slash
+# - Files with the '.html' extension can be accessed with or without .html
+create-purge-list:
stage: deploy
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
@@ -80,31 +85,28 @@ create-purge-json:
tags:
- docker
script:
+ # Add the index-less homepage
+ - echo "$CI_PAGES_URL" > purge-list.txt
+ # Add all the HTML files and HTML symlinks, with and without .html extension
- |-
- echo -en "{\n \"files\": [" > purge.json
- # Add the index-less homepage, with and without trailing slash
- echo -en "\n \"$CI_PAGES_URL\"" >> purge.json
- echo -en ",\n \"$CI_PAGES_URL/\"" >> purge.json
- # Add all the directories, non-HTML files and non-HTML symlinks; with and
- # without trailing slash
- for f in $(find public \( -type d -or -type f -or -type l \) -not -iname '*.html'); do
- echo -en ",\n \"$CI_PAGES_URL/${f#public/}\"" >> purge.json
- echo -en ",\n \"$CI_PAGES_URL/${f#public/}/\"" >> purge.json
+ for f in $(find public/* \( -type f -or -type l \) -iname '*.html'); do
+ f="${f#public/}"
+ echo "$CI_PAGES_URL/$f" >> purge-list.txt
+ echo "$CI_PAGES_URL/${f%.html}" >> purge-list.txt
done
- # Add all the HTML files and HTML symlinks, with and without trailing
- # slash. First with file extension, then without
- for f in $(find public \( -type f -or -type l \) -iname '*.html'); do
- echo -en ",\n \"$CI_PAGES_URL/${f#public/}\"" >> purge.json
- echo -en ",\n \"$CI_PAGES_URL/${f#public/}/\"" >> purge.json
- f="${f%.html}"
- echo -en ",\n \"$CI_PAGES_URL/${f#public/}\"" >> purge.json
- echo -en ",\n \"$CI_PAGES_URL/${f#public/}/\"" >> purge.json
+ # Add everything else
+ - |-
+ for f in $(find public/* \( -type d -or -type f -or -type l \) -not -iname '*.html'); do
+ echo "$CI_PAGES_URL/${f#public/}" >> purge-list.txt
done
- echo -e "\n ]\n}" >> purge.json
- - cat purge.json
+ # Remove any duplicate URLs
+ - sort -u -o purge-list.txt purge-list.txt
+ # Duplicate each line, adding a trailing slash to the duplicates
+ - sed -i 'p;s|$|/|' purge-list.txt
+ - cat purge-list.txt
artifacts:
paths:
- - purge.json
+ - purge-list.txt
trigger-cache-purge:
stage: .post
diff --git a/purge-cache.gitlab-ci.yml b/purge-cache.gitlab-ci.yml
index 0b4c616f97bb299e162ee611b5c6675a942eaa35..54620ab382428c87ce25974caf7338f4eaff4c3e 100644
--- a/purge-cache.gitlab-ci.yml
+++ b/purge-cache.gitlab-ci.yml
@@ -1,4 +1,4 @@
-# Purge the Cloudflare cache using the request body contained in purge.json
+# Purge the URLs contained in purge-list.txt from the Cloudflare cache
#
# We delay this job to give the pages:deploy job time to finish. If we don't
# delay, then the cache might refill with old pages before the new pages are
@@ -12,14 +12,37 @@ purge-cache:
start_in: 3 minutes
needs:
- pipeline: $PARENT_PIPELINE_ID
- job: create-purge-json
+ job: create-purge-list
+ before_script:
+ # Make sure the purge-list.txt file is readable, else exit
+ - test -r purge-list.txt || { echo "purge-list.txt not found" ; exit 1 ; }
+ - echo "Purge list has $(cat purge-list.txt | wc -l) URLs"
+ # Default to chunks of 30 URLs because Cloudflare only allows 30 URLs per
+ # purge request on free accounts
+ - echo "Chunk size of ${CF_PURGE_CACHE_CHUNK_SIZE:=30}"
script:
- - cat purge.json
- - >-
- wget -qO- "https://api.cloudflare.com/client/v4/zones/$CF_PURGE_CACHE_ZONE/purge_cache"
- --header "Content-Type: application/json"
- --header "Authorization: Bearer $CF_PURGE_CACHE_TOKEN"
- --post-file purge.json
+ # Split the purge list into chunks named 'purge-chunk-[aaa,aab,...]'
+ - split -l $CF_PURGE_CACHE_CHUNK_SIZE -a 3 purge-list.txt purge-chunk-
+ # Loop over the chunks, creating a purge request for each
+ - |-
+ for chunk in purge-chunk-* ; do
+ # Create the purge request body
+ echo -en "{\n \"files\": [" > purge.json
+ unset comma # This needs to be unset for the first line in each chunk
+ while read path; do
+ echo -en "$comma\n \"$path\"" >> purge.json
+ comma=','
+ done < $chunk
+ echo -e "\n ]\n}" >> purge.json
+ cat purge.json
+ # Make the API request to Cloudflare to purge the URLs from cache
+ wget -qO- "https://api.cloudflare.com/client/v4/zones/$CF_PURGE_CACHE_ZONE/purge_cache" \
+ --header "Content-Type: application/json" \
+ --header "Authorization: Bearer $CF_PURGE_CACHE_TOKEN" \
+ --post-file purge.json
+ # Rate limit ourselves to 1 request per second
+ sleep 1
+ done
# vi: set ts=2 sw=2 et ft=yaml: