.gitlab-ci.yml

@@ -38,9 +38,10 @@ setup:
     - echo $ROOT_RELATIVE_NAMESPACE
     - test "$ROOT_RELATIVE_NAMESPACE" == "${CI_PROJECT_NAMESPACE#$CI_PROJECT_ROOT_NAMESPACE}" || { echo "ROOT_RELATIVE_NAMESPACE is not set correctly" ; exit 1 ; }
   script:
-    - find src/ -type f -name '*.html'
+    - find src/ -type f \( -iname '*.html' -or -iname 'sitemap.xml' \)
       -exec sed -i
         -e "s|%%BRANCH%%|${CI_COMMIT_BRANCH}|g"
+        -e "s|%%BASE_URL%%|${CI_PAGES_URL}|g"
         -e "s|%%COMMIT_SHA%%|${CI_COMMIT_SHORT_SHA}|g"
         -e "s|%%COMMIT_TIMESTAMP%%|${CI_COMMIT_TIMESTAMP}|g"
         -e "s|%%JOB_ID%%|${CI_JOB_ID}|g"
@@ -71,8 +72,13 @@ pages:
   rules:
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
 
-# Purge all of this site's URLs from the Cloudflare cache
-create-purge-json:
+# Create a list of URLs that need to be purged from the cache after GitLab Pages
+# deployment. Each real file/directory has multiple URLs because the router for
+# GitLab Pages fudges paths for better user experience. The fudging rules are:
+#  - Directories can be accessed with or without a trailing slash
+#  - Files can be accessed with or without a trailing slash
+#  - Files with the '.html' extension can be accessed with or without .html
+create-purge-list:
   stage: deploy
   rules:
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
@@ -80,31 +86,27 @@ create-purge-json:
   tags:
     - docker
   script:
+    # Add the index-less homepage
+    - echo "$CI_PAGES_URL" > purge-list.txt
+    # Add all the HTML files and HTML symlinks, with and without .html extension
     - |-
-      echo -en "{\n  \"files\": [" > purge.json
-      # Add the index-less homepage, with and without trailing slash
-      echo -en "\n    \"$CI_PAGES_URL\"" >> purge.json
-      echo -en ",\n    \"$CI_PAGES_URL/\"" >> purge.json
-      # Add all the directories, non-HTML files and non-HTML symlinks; with and
-      # without trailing slash
-      for f in $(find public \( -type d -or -type f -or -type l \) -not -iname '*.html'); do
-        echo -en ",\n    \"$CI_PAGES_URL/${f#public/}\"" >> purge.json
-        echo -en ",\n    \"$CI_PAGES_URL/${f#public/}/\"" >> purge.json
+      for f in $(find public/* \( -type f -or -type l \) -iname '*.html'); do
+        f="${f#public/}"
+        echo "$CI_PAGES_URL/$f" >> purge-list.txt
+        echo "$CI_PAGES_URL/${f%.html}" >> purge-list.txt
       done
-      # Add all the HTML files and HTML symlinks, with and without trailing
-      # slash. First with file extension, then without
-      for f in $(find public \( -type f -or -type l \) -iname '*.html'); do
-        echo -en ",\n    \"$CI_PAGES_URL/${f#public/}\"" >> purge.json
-        echo -en ",\n    \"$CI_PAGES_URL/${f#public/}/\"" >> purge.json
-        f="${f%.html}"
-        echo -en ",\n    \"$CI_PAGES_URL/${f#public/}\"" >> purge.json
-        echo -en ",\n    \"$CI_PAGES_URL/${f#public/}/\"" >> purge.json
+    # Add everything else
+    - |-
+      for f in $(find public/* \( -type d -or -type f -or -type l \) -not -iname '*.html'); do
+        echo "$CI_PAGES_URL/${f#public/}" >> purge-list.txt
       done
-      echo -e "\n  ]\n}" >> purge.json
-    - cat purge.json
+    # Remove any duplicate URLs
+    - sort -u -o purge-list.txt purge-list.txt
+    # Duplicate each line, adding a trailing slash to the duplicates
+    - sed -i 'p;s|$|/|' purge-list.txt
   artifacts:
     paths:
-      - purge.json
+      - purge-list.txt
 
 trigger-cache-purge:
   stage: .post
@@ -114,6 +116,7 @@ trigger-cache-purge:
     include: purge-cache.gitlab-ci.yml
   variables:
     PARENT_PIPELINE_ID: $CI_PIPELINE_ID
+    CF_PURGE_CACHE_CHUNK_SIZE: 20
 
 
 # vi: set ts=2 sw=2 et ft=yaml:

purge-cache.gitlab-ci.yml

-# Purge the Cloudflare cache using the request body contained in purge.json
+# Purge the URLs contained in purge-list.txt from the Cloudflare cache
 #
 # We delay this job to give the pages:deploy job time to finish. If we don't
 # delay, then the cache might refill with old pages before the new pages are
@@ -12,14 +12,40 @@ purge-cache:
       start_in: 3 minutes
   needs:
     - pipeline: $PARENT_PIPELINE_ID
-      job: create-purge-json
+      job: create-purge-list
+  before_script:
+    # Make sure the purge-list.txt file is readable, else exit
+    - test -r purge-list.txt || { echo "purge-list.txt not found" ; exit 1 ; }
+    - echo "Purge list has $(cat purge-list.txt | wc -l) URLs"
+    # Default to chunks of 30 URLs because Cloudflare only allows 30 URLs per
+    # purge request on free accounts
+    - echo "Chunk size of ${CF_PURGE_CACHE_CHUNK_SIZE:=30}"
   script:
-    - cat purge.json
-    - >-
-      wget -qO- "https://api.cloudflare.com/client/v4/zones/$CF_PURGE_CACHE_ZONE/purge_cache"
-      --header "Content-Type: application/json"
-      --header "Authorization: Bearer $CF_PURGE_CACHE_TOKEN"
-      --post-file purge.json
+    # Split the purge list into chunks named 'purge-chunk-[aaa,aab,...]'
+    - split -l $CF_PURGE_CACHE_CHUNK_SIZE -a 3 purge-list.txt purge-chunk-
+    - chunks=$(find . -type f -name 'purge-chunk-*' | wc -l)
+    # Loop over the chunks, creating a purge request for each
+    - |-
+      counter=0
+      for chunk in purge-chunk-* ; do
+        echo -e "\nChunk $((++counter))/$chunks"
+        # Create the purge request body
+        echo -en "{\n  \"files\": [" > purge.json
+        unset comma # This needs to be unset for the first line in each chunk
+        while read path; do
+          echo -en "$comma\n    \"$path\"" >> purge.json
+          comma=','
+        done < $chunk
+        echo -e "\n  ]\n}" >> purge.json
+        cat purge.json
+        # Make the API request to Cloudflare to purge the URLs from cache
+        wget -qO- "https://api.cloudflare.com/client/v4/zones/$CF_PURGE_CACHE_ZONE/purge_cache" \
+          --header "Content-Type: application/json" \
+          --header "Authorization: Bearer $CF_PURGE_CACHE_TOKEN" \
+          --post-file purge.json
+        # Rate limit ourselves to 1 request per second
+        sleep 1
+      done
 
 
 # vi: set ts=2 sw=2 et ft=yaml:

src/cache-test.html

@@ -2,10 +2,11 @@
 <html lang="en">
 <head>
   <meta charset="utf-8">
-  <meta name="viewport" content="width=device-width, initial-scale=1">
-  <meta name="generator" content="GitLab Pages">
+  <base href="%%BASE_URL%%/">
   <title>Cache Test | %%PROJECT_TITLE%%</title>
+  <meta name="viewport" content="width=device-width, initial-scale=1">
   <link rel="shortcut icon" href="gitlab-favicon.png">
+  <link rel="canonical" href="cache-test.html">
   <link rel="preload" href="style.css" as="style">
   <link rel="stylesheet" href="style.css">
   <style>

src/index.html

@@ -2,10 +2,11 @@
 <html lang="en">
 <head>
   <meta charset="utf-8">
-  <meta name="viewport" content="width=device-width, initial-scale=1">
-  <meta name="generator" content="GitLab Pages">
+  <base href="%%BASE_URL%%/">
   <title>%%PROJECT_TITLE%%</title>
+  <meta name="viewport" content="width=device-width, initial-scale=1">
   <link rel="shortcut icon" href="gitlab-favicon.png">
+  <link rel="canonical" href="./">
   <link rel="preload" href="style.css" as="style">
   <link rel="stylesheet" href="style.css">
 </head>

src/sitemap.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <url>
+    <loc>%%BASE_URL%%/</loc>
+  </url>
+  <url>
+    <loc>%%BASE_URL%%/cache-test.html</loc>
+  </url>
+  <url>
+    <loc>%%BASE_URL%%/sub/</loc>
+  </url>
+</urlset>
+<!-- vi: set ts=2 sw=2 et ft=xml: -->

src/sub/index.html

@@ -2,18 +2,19 @@
 <html lang="en">
 <head>
   <meta charset="utf-8">
-  <meta name="viewport" content="width=device-width, initial-scale=1">
-  <meta name="generator" content="GitLab Pages">
+  <base href="%%BASE_URL%%/">
   <title>Subdirectory | %%PROJECT_TITLE%%</title>
-  <link rel="shortcut icon" href="../gitlab-favicon.png">
-  <link rel="preload" href="../style.css" as="style">
-  <link rel="stylesheet" href="../style.css">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <link rel="shortcut icon" href="gitlab-favicon.png">
+  <link rel="canonical" href="sub/">
+  <link rel="preload" href="style.css" as="style">
+  <link rel="stylesheet" href="style.css">
 </head>
 <body>
   <header>
-    <a href="../">Homepage</a>
-    <a href="../cache-test.html">Cache Test</a>
-    <a href="./">Subdirectory</a>
+    <a href="./">Homepage</a>
+    <a href="cache-test.html">Cache Test</a>
+    <a href="sub/">Subdirectory</a>
     <a href="%%PROJECT_URL%%">Repository</a>
   </header>
   <main>