diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c1402af8b4c41da05c599c449b2e0733969af4d1..c92811a9e848abc78de08eaba9b633b64435ff8e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -71,8 +71,13 @@ pages: rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH -# Purge all of this site's URLs from the Cloudflare cache -create-purge-json: +# Create a list of URLs that need to be purged from the cache after GitLab Pages +# deployment. Each real file/directory has multiple URLs because the router for +# GitLab Pages fudges paths for better user experience. The fudging rules are: +# - Directories can be accessed with or without a trailing slash +# - Files can be accessed with or without a trailing slash +# - Files with the '.html' extension can be accessed with or without .html +create-purge-list: stage: deploy rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH @@ -80,31 +85,28 @@ create-purge-json: tags: - docker script: + # Add the index-less homepage + - echo "$CI_PAGES_URL" > purge-list.txt + # Add all the HTML files and HTML symlinks, with and without .html extension - |- - echo -en "{\n \"files\": [" > purge.json - # Add the index-less homepage, with and without trailing slash - echo -en "\n \"$CI_PAGES_URL\"" >> purge.json - echo -en ",\n \"$CI_PAGES_URL/\"" >> purge.json - # Add all the directories, non-HTML files and non-HTML symlinks; with and - # without trailing slash - for f in $(find public \( -type d -or -type f -or -type l \) -not -iname '*.html'); do - echo -en ",\n \"$CI_PAGES_URL/${f#public/}\"" >> purge.json - echo -en ",\n \"$CI_PAGES_URL/${f#public/}/\"" >> purge.json + for f in $(find public/* \( -type f -or -type l \) -iname '*.html'); do + f="${f#public/}" + echo "$CI_PAGES_URL/$f" >> purge-list.txt + echo "$CI_PAGES_URL/${f%.html}" >> purge-list.txt done - # Add all the HTML files and HTML symlinks, with and without trailing - # slash. First with file extension, then without - for f in $(find public \( -type f -or -type l \) -iname '*.html'); do - echo -en ",\n \"$CI_PAGES_URL/${f#public/}\"" >> purge.json - echo -en ",\n \"$CI_PAGES_URL/${f#public/}/\"" >> purge.json - f="${f%.html}" - echo -en ",\n \"$CI_PAGES_URL/${f#public/}\"" >> purge.json - echo -en ",\n \"$CI_PAGES_URL/${f#public/}/\"" >> purge.json + # Add everything else + - |- + for f in $(find public/* \( -type d -or -type f -or -type l \) -not -iname '*.html'); do + echo "$CI_PAGES_URL/${f#public/}" >> purge-list.txt done - echo -e "\n ]\n}" >> purge.json - - cat purge.json + # Remove any duplicate URLs + - sort -u -o purge-list.txt purge-list.txt + # Duplicate each line, adding a trailing slash to the duplicates + - sed -i 'p;s|$|/|' purge-list.txt + - cat purge-list.txt artifacts: paths: - - purge.json + - purge-list.txt trigger-cache-purge: stage: .post diff --git a/purge-cache.gitlab-ci.yml b/purge-cache.gitlab-ci.yml index 0b4c616f97bb299e162ee611b5c6675a942eaa35..54620ab382428c87ce25974caf7338f4eaff4c3e 100644 --- a/purge-cache.gitlab-ci.yml +++ b/purge-cache.gitlab-ci.yml @@ -1,4 +1,4 @@ -# Purge the Cloudflare cache using the request body contained in purge.json +# Purge the URLs contained in purge-list.txt from the Cloudflare cache # # We delay this job to give the pages:deploy job time to finish. If we don't # delay, then the cache might refill with old pages before the new pages are @@ -12,14 +12,37 @@ purge-cache: start_in: 3 minutes needs: - pipeline: $PARENT_PIPELINE_ID - job: create-purge-json + job: create-purge-list + before_script: + # Make sure the purge-list.txt file is readable, else exit + - test -r purge-list.txt || { echo "purge-list.txt not found" ; exit 1 ; } + - echo "Purge list has $(cat purge-list.txt | wc -l) URLs" + # Default to chunks of 30 URLs because Cloudflare only allows 30 URLs per + # purge request on free accounts + - echo "Chunk size of ${CF_PURGE_CACHE_CHUNK_SIZE:=30}" script: - - cat purge.json - - >- - wget -qO- "https://api.cloudflare.com/client/v4/zones/$CF_PURGE_CACHE_ZONE/purge_cache" - --header "Content-Type: application/json" - --header "Authorization: Bearer $CF_PURGE_CACHE_TOKEN" - --post-file purge.json + # Split the purge list into chunks named 'purge-chunk-[aaa,aab,...]' + - split -l $CF_PURGE_CACHE_CHUNK_SIZE -a 3 purge-list.txt purge-chunk- + # Loop over the chunks, creating a purge request for each + - |- + for chunk in purge-chunk-* ; do + # Create the purge request body + echo -en "{\n \"files\": [" > purge.json + unset comma # This needs to be unset for the first line in each chunk + while read path; do + echo -en "$comma\n \"$path\"" >> purge.json + comma=',' + done < $chunk + echo -e "\n ]\n}" >> purge.json + cat purge.json + # Make the API request to Cloudflare to purge the URLs from cache + wget -qO- "https://api.cloudflare.com/client/v4/zones/$CF_PURGE_CACHE_ZONE/purge_cache" \ + --header "Content-Type: application/json" \ + --header "Authorization: Bearer $CF_PURGE_CACHE_TOKEN" \ + --post-file purge.json + # Rate limit ourselves to 1 request per second + sleep 1 + done # vi: set ts=2 sw=2 et ft=yaml: