add sitemap

AtlasOfLivingAustralia · Oct 4, 2023 · 730d0b2 · 730d0b2
1 parent aa3719e
commit 730d0b2
Show file tree

Hide file tree

Showing 10 changed files with 196 additions and 8 deletions.
diff --git a/gradle.properties b/gradle.properties
@@ -10,4 +10,4 @@ webdriverBinariesVersion=2.6
 chromeDriverVersion=2.45.0
 geckodriverVersion=0.24.0
 seleniumSafariDriverVersion=3.14.0
-alaSecurityLibsVersion=6.0.4
+alaSecurityLibsVersion=6.1.0
diff --git a/grails-app/conf/application.yml b/grails-app/conf/application.yml
@@ -178,7 +178,7 @@ security:
             - /css.*
             - /js.*
             - /static.*
-        appServerName: http://dev.ala.org.au:8080
+        appServerName: http://localhost:8080
         casServerUrlPrefix: https://auth-test.ala.org.au/cas
         loginUrl: https://auth-test.ala.org.au/cas/login
         logoutUrl: https://auth-test.ala.org.au/cas/logout
@@ -400,7 +400,9 @@ environments:
                     active: false
         dataSource:
             dbCreate: none
-            url: jdbc:mysql://localhost:3306/collectory?autoReconnect=true&connectTimeout=0&useUnicode=true&characterEncoding=UTF-8
+            url: jdbc:mysql://localhost:3306/collectory?autoReconnect=true&connectTimeout=0&useUnicode=true&characterEncoding=UTF-8&serverTimezone=Australia/Sydney
+            username: specieslist
+            password: "specieslist"
     test:
         grails:
             assets:
@@ -449,3 +451,7 @@ openapi:
         url: https://www.mozilla.org/en-US/MPL/1.1/
     version: '@info.app.version@'
     cachetimeoutms: 0
+
+sitemap:
+    dir: /data/collectory
+    enabled: true
diff --git a/grails-app/controllers/au/org/ala/collectory/AdminController.groovy b/grails-app/controllers/au/org/ala/collectory/AdminController.groovy
@@ -8,7 +8,7 @@ import org.springframework.core.io.support.PathMatchingResourcePatternResolver
 
 class AdminController {
 
-    def dataLoaderService, idGeneratorService, metadataService
+    def dataLoaderService, idGeneratorService, metadataService, sitemapService
 
     def index = {
         redirect(controller: 'manage')
@@ -299,4 +299,10 @@ class AdminController {
         }
         return g
     }
+
+    def buildSitemap() {
+        sitemapService.build()
+
+        render text: 'done'
+    }
 }
diff --git a/grails-app/controllers/au/org/ala/collectory/SitemapController.groovy b/grails-app/controllers/au/org/ala/collectory/SitemapController.groovy
@@ -0,0 +1,30 @@
+package au.org.ala.collectory
+
+class SitemapController {
+
+    def index(Integer idx) {
+        if (!grailsApplication.config.sitemap.enabled) {
+            response.status = 404
+            return
+        }
+
+        File index = new File(grailsApplication.config.sitemap.dir + '/sitemap.xml')
+        if (!index.exists()) {
+            response.status = 404
+            return
+        }
+
+        if (idx == null) {
+            // return sitemap index
+            response.outputStream << index.newInputStream()
+        } else {
+            // return sitemap urls
+            File part = new File(grailsApplication.config.sitemap.dir + '/sitemap' + idx + ".xml")
+            if (!part.exists()) {
+                response.status = 404
+                return
+            }
+            response.outputStream << part.newInputStream()
+        }
+    }
+}
diff --git a/grails-app/controllers/au/org/ala/collectory/UrlMappings.groovy b/grails-app/controllers/au/org/ala/collectory/UrlMappings.groovy
@@ -207,6 +207,8 @@ class UrlMappings {
         "/public/resources(.$format)"(controller: 'public', action: 'resources')
         "/public/condensed(.$format)"(controller: 'public', action: 'condensed')
 
+        "/sitemap($idx)?.xml"(controller: "sitemap", action: "index")
+
         "/"(controller: 'public', action: 'map')
 
         "/error"(view: '/error')

diff --git a/grails-app/i18n/messages.properties b/grails-app/i18n/messages.properties
@@ -1276,6 +1276,7 @@ manage.list.addtools.des11=View and edit all known collection and institution co
 manage.list.addtools.mpm=Provider maps
 manage.list.addtools.des12=View and edit the allocation of collection and institution codes to collections
 manage.list.addtools.eadaj=Export all data as JSON
+manage.list.addtools.buildsitemap=Build sitemap
 manage.list.addtools.des13=All tables exported verbatim as JSON
 manage.list.addtools.vae=Audit events
 manage.list.addtools.des14=All audit events

diff --git a/grails-app/init/collectory/Application.groovy b/grails-app/init/collectory/Application.groovy
@@ -4,10 +4,14 @@ import grails.boot.GrailsApp
 import grails.boot.config.GrailsAutoConfiguration
 
 import groovy.transform.CompileStatic
+import org.springframework.context.annotation.ComponentScan
+import org.springframework.scheduling.annotation.EnableScheduling
 
+@ComponentScan('au.org.ala.collectory')
 @CompileStatic
+@EnableScheduling
 class Application extends GrailsAutoConfiguration {
     static void main(String[] args) {
         GrailsApp.run(Application, args)
     }
-}
+}
diff --git a/grails-app/services/au/org/ala/collectory/SitemapService.groovy b/grails-app/services/au/org/ala/collectory/SitemapService.groovy
@@ -0,0 +1,127 @@
+/*
+ * Copyright (C) 2022 Atlas of Living Australia
+ * All Rights Reserved.
+ *
+ * The contents of this file are subject to the Mozilla Public
+ * License Version 1.1 (the "License"); you may not use this file
+ * except in compliance with the License. You may obtain a copy of
+ * the License at http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS
+ * IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * rights and limitations under the License.
+ */
+package au.org.ala.collectory
+
+import org.springframework.scheduling.annotation.Scheduled
+
+import java.text.SimpleDateFormat
+
+class SitemapService {
+
+    def grailsApplication
+
+
+    String URLSET_HEADER = "<?xml version='1.0' encoding='UTF-8'?><urlset xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd\" xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">"
+    String URLSET_FOOTER = "</urlset>"
+
+    int MAX_URLS = 50000 // maximum number of URLs in a sitemap file
+    int MAX_SIZE = 9*1024*1024 // use 9MB to keep the actual file size below 10MB (a gateway limit)
+
+    File currentFile
+    int fileCount = 0
+    int countUrls = 0
+
+    SimpleDateFormat simpleDateFormat = new SimpleDateFormat("YYYY-MM-dd")
+
+    FileWriter fw
+
+    // run daily, initial delay 1hr
+    @Scheduled(fixedDelay = 86400000L, initialDelay = 3600000L)
+    def build() throws Exception {
+        initWriter()
+        buildSitemap()
+        closeWriter()
+
+        buildSitemapIndex()
+    }
+
+    def buildSitemapIndex() {
+
+        // write parent sitemap file
+        fw = new FileWriter(grailsApplication.config.sitemap.dir + "/sitemap.xml")
+        fw.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?><sitemapindex xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">")
+
+        for (int i=0;i<fileCount;i++) {
+
+            // move the tmp file
+            File newFile = new File(grailsApplication.config.sitemap.dir + "/sitemap" + i + ".xml")
+            if (newFile.exists()) {
+                newFile.delete()
+            }
+            new File(grailsApplication.config.sitemap.dir + "/sitemap" + i + ".xml.tmp").renameTo(newFile)
+
+            // add an entry for this new file
+            fw.write("<sitemap><url>" + grailsApplication.config.grails.serverURL + "/sitemap" + i + ".xml" + "</url>")
+            fw.write("<lastmod>" + simpleDateFormat.format(new Date()) + "</lastmod></sitemap>")
+        }
+
+        fw.write("</sitemapindex>")
+        fw.flush()
+        fw.close()
+    }
+
+    def initWriter() {
+        currentFile = new File(grailsApplication.config.sitemap.dir + "/sitemap" + fileCount + ".xml.tmp")
+
+        fw = new FileWriter(currentFile)
+
+        fw.write(URLSET_HEADER)
+
+        countUrls = 0
+        fileCount++
+    }
+
+    def closeWriter() {
+        fw.write(URLSET_FOOTER)
+        fw.flush()
+        fw.close()
+    }
+
+    def writeUrl(Date lastUpdated, String changefreq, String encodedUrl) {
+        if (countUrls >= MAX_URLS || currentFile.size() >= MAX_SIZE) {
+            closeWriter()
+            initWriter()
+        }
+
+        fw.write("<url>")
+        fw.write("<loc>" + encodedUrl + "</loc>")
+        fw.write("<lastmod>" + simpleDateFormat.format(lastUpdated) + "</lastmod>")
+        fw.write("<changefreq>" + changefreq + "</changefreq>")
+        fw.write("</url>")
+
+        fw.flush()
+
+        countUrls++
+    }
+
+    def buildSitemap() throws Exception {
+
+        Collection.findAll().each {Collection it ->
+            writeUrl(it.lastUpdated, "weekly", grailsApplication.config.grails.serverURL + "/public/show/" + it.id)
+        }
+
+        Institution.findAll().each {Institution it ->
+            writeUrl(it.lastUpdated, "weekly", grailsApplication.config.grails.serverURL + "/public/show/" + it.id)
+        }
+
+        DataProvider.findAll().each {DataProvider it ->
+            writeUrl(it.lastUpdated, "weekly", grailsApplication.config.grails.serverURL + "/public/show/" + it.id)
+        }
+
+        DataResource.findAllByIsPrivate(false).each {DataResource it ->
+            writeUrl(it.lastUpdated, "weekly", grailsApplication.config.grails.serverURL + "/public/show/" + it.id)
+        }
+    }
+}
diff --git a/grails-app/views/manage/index.gsp b/grails-app/views/manage/index.gsp
@@ -7,10 +7,10 @@
         />
 	    <meta name="layout" content="${grailsApplication.config.skin.layout}" />
     </head>
-    
+
     <body>
       <div class="floating-content manage">
-    
+
         <div style="float:right;">
             <g:link class="mainLink" controller="public" action="map"><g:message code="manage.index.link" /></g:link>
         </div>
@@ -51,4 +51,4 @@
         </div>
 
     </body>
-</html>
+</html>
diff --git a/grails-app/views/manage/list.gsp b/grails-app/views/manage/list.gsp
@@ -237,6 +237,18 @@
 %{--                    </div>--}%
 
                   </div>
+
+                <div>
+                    <div class="homeCell btn btn-default">
+                        <g:link class="mainLink" controller="admin" action="buildSitemap"><g:message code="manage.list.addtools.buildsitemap" /></g:link>
+                    </div>
+
+                    %{--                    <div class="homeCell btn btn-default">--}%
+                    %{--                        <g:link class="mainLink" controller="auditLogEvent" action="list" params="[max:1000]"><g:message code="manage.list.addtools.vae" /></g:link>--}%
+                    %{--                        <p class="mainText hide"><g:message code="manage.list.addtools.des14" /></p>--}%
+                    %{--                    </div>--}%
+
+                </div>
                   <div>
                       <h4>Data sync with GBIF</h4>
                       <div class="homeCell btn btn-default">