Sitemap.xml and robots.txt

JackGilmore · JackGilmore · commit 516a10cba6ee · 2023-08-15T19:39:54.000Z
diff --git a/_data/site.js b/_data/site.js
@@ -2,4 +2,6 @@ module.exports = {
 
     // Outputs a unix timestamp that can be used to cache bust our files
     buildTime: Date.now(),
+    name: "ODS GovSpend",
+    url: "https://govspend.opendata.scot"
   };
diff --git a/_includes/layouts/base.njk b/_includes/layouts/base.njk
@@ -1,5 +1,4 @@
-{% set site_name = "ODS GovSpend" %}
-{% set page_title = title + " | " + site_name %}
+{% set page_title = title + " | " + site.name %}
 
 <!doctype html>
 <html lang="en" class="h-100">
@@ -10,8 +9,8 @@
 
     {# Metadata #}
     <meta property="og:title" content="{{page_title}}">
-    <meta property="og:site_name" content="{{site_name}}">
-    <meta property="og:url" content="https://govspend.opendata.scot{{page.url}}">
+    <meta property="og:site_name" content="{{ site.name }}">
+    <meta property="og:url" content="{{site.url}}/{{page.url}}">
     <meta property="og:description" content="{{description}}">
     <meta name="description" content="{{description}}">
 
@@ -20,7 +19,7 @@
     <script src="/assets/main.js?v={{ site.buildTime }}"></script>
 
     {# Analytics #}
-    <script defer data-domain="govspend.opendata.scot" src="https://plausible.io/js/script.js"></script>    
+    <script defer data-domain="govspend.opendata.scot" src="https://plausible.io/js/script.js"></script>
   </head>
 
 </head>
@@ -29,13 +28,13 @@
     <!-- Fixed navbar -->
     <nav class="navbar navbar-expand-md navbar-dark bg-dark">
       <div class="container-fluid">
-        <a class="navbar-brand" href="/">{{site_name}}</a>
+        <a class="navbar-brand" href="/">{{site.name}}</a>
         <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarCollapse" aria-controls="navbarCollapse" aria-expanded="false" aria-label="Toggle navigation">
           <span class="navbar-toggler-icon"></span>
         </button>
         <div class="collapse navbar-collapse" id="navbarCollapse">
           <ul class="navbar-nav me-auto mb-2 mb-md-0">
-            {# TODO: Add active class depending on what page is currently being visited #}            
+            {# TODO: Add active class depending on what page is currently being visited #}
             <li class="nav-item">
               <a class="nav-link active" href="/">Home</a>
             </li>
diff --git a/robots.txt b/robots.txt
@@ -0,0 +1,6 @@
+Sitemap: https://govspend.opendata.scot/sitemap.xml
+
+User-agent: *
+Allow: /
+User-agent: GPTBot
+Disallow: /
diff --git a/sitemap.xml.njk b/sitemap.xml.njk
@@ -0,0 +1,23 @@
+---
+permalink: /sitemap.xml
+addAllPagesToCollections: true
+---
+<?xml version="1.0" encoding="utf-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+    {% for page in collections.all %}
+        {%if page.url.startsWith("/spends/") == false%}
+            <url>
+                <loc>{{ site.url }}{{ page.url | url }}</loc>
+                <lastmod>{{ page.date.toISOString() }}</lastmod>
+            </url>
+            {%endif%}
+        {% endfor %}
+        {% for spendData in collections.spendsOver500 %}
+            <url>
+                <loc>{{ site.url }}/spends/{{ spendData.title | url }}/</loc>
+            </url>
+            <url>
+                <loc>{{ site.url }}/spends/{{ spendData.title | url }}.json</loc>
+            </url>
+        {% endfor %}
+    </urlset>
