Merge branch 'release/2.1.1'

application.properties

@@ -6,4 +6,4 @@ app.buildProfile=development
 app.grails.version=2.3.11
 app.name=volunteer-portal
 app.servlet.version=2.5
-app.version=2.1.0
+app.version=2.1.1

grails-app/conf/Config.groovy

@@ -129,15 +129,9 @@ expedition = [
 
 ]
 
-achievements = [
-        [ name: 'tenth_transcription', label:"10th transcription", description:'Submit ten transcription tasks for validation', icon: 'images/achievements/bronze_lens.png' ],
-        [ name: 'hundredth_transcription', label:"100th transcription", description:'Submit one hundred transcription tasks for validation', icon: 'images/achievements/silver_telescope.png' ],
-        [ name: 'fivehundredth_transcription', label:"500th transcription", description:'Submit five hundred transcription tasks for validation', icon: 'images/achievements/gold_microscope.png' ],
-        [ name: 'three_projects', label:"Three expeditions", description:'Transcribe tasks across three different expeditions', icon: 'images/achievements/bronze_net.png' ],
-        [ name: 'five_projects', label:"Five expeditions", description:'Transcribe tasks across five different expeditions', icon: 'images/achievements/silver_binoculars.png' ],
-        [ name: 'seven_projects', label:"Seven expeditions", description:'Transcribe tasks across seven different expeditions', icon: 'images/achievements/gold_telescope.png' ],
-
-]
+// TODO Remove this after ala-auth plugin is updated
+userDetails.url = "https://auth.ala.org.au/userdetails/userDetails/"
+userDetailsById.bulkPath = 'getUserDetailsFromIdList'
 
 volunteer.defaultProjectId = 6306
 viewedTask.timeout = 2 * 60 * 60 * 1000

grails-app/controllers/au/org/ala/volunteer/AjaxController.groovy

@@ -1,6 +1,7 @@
 package au.org.ala.volunteer
 
 import au.org.ala.volunteer.collectory.CollectoryProviderDto
+import com.google.common.base.Stopwatch
 import com.google.common.collect.ImmutableMap
 import com.google.common.collect.Sets
 import com.google.gson.Gson
@@ -14,6 +15,8 @@ import java.sql.ResultSet
 import org.grails.plugins.csv.CSVWriter
 import org.grails.plugins.csv.CSVWriterColumnsBuilder
 
+import static grails.async.Promises.*
+
 class AjaxController {
 
     def taskService
@@ -27,6 +30,7 @@ class AjaxController {
     def authService
     def settingsService
     def achievementService
+    def sessionFactory
 
     static responseFormats = ['json', 'xml']
 
@@ -90,46 +94,114 @@ class AjaxController {
             return;
         }
 
-        def report = []
-        def users = User.list()
 
-        def serviceResults = [:]
-        try {
-            serviceResults = authService.getUserDetailsById(users*.userId)
-        } catch (Exception e) {
-            log.warn("couldn't get user details from web service", e)
+        // Pre-create the writer and write the headings straight away to prevent a read timeout.
+        def writer
+        if (params.wt && params.wt == 'csv') {
+            def nodata = params.nodata ?: 'nodata'
+
+            response.addHeader("Content-type", "text/plain")
+
+            writer = new CSVHeadingsWriter((Writer) response.writer, {
+                'user_id' { it[0] }
+                'display_name' { it[1] }
+                'transcribed_count' { it[2] }
+                'validated_count' { it[3] }
+                'last_activity' { it[4] ?: nodata }
+                'projects_count' { it[5] }
+                'volunteer_since' { it[6] }
+            })
+            writer.writeHeadings()
+            response.flushBuffer()
+        }
+
+
+        def asyncCounts = Task.async.withStatelessSession {
+            def sw1 = new Stopwatch().start()
+            def vs = (Task.withCriteria {
+                projections {
+                    groupProperty('fullyValidatedBy')
+                    count('id')
+                }
+            }).collectEntries { [(it[0]): it[1]] }
+            def ts = (Task.withCriteria {
+                projections {
+                    groupProperty('fullyTranscribedBy')
+                    count('id')
+                }
+            }).collectEntries { [(it[0]): it[1]] }
+            sw1.stop()
+            log.debug("UserReport counts took ${sw1.toString()}")
+            [vs: vs, ts: ts]
         }
 
-        for (User user : users) {
-            def transcribedCount = Task.countByFullyTranscribedBy(user.userId)
-            def validatedCount = Task.countByFullyValidatedBy(user.userId)
-            def lastActivity = ViewedTask.executeQuery("select to_timestamp(max(vt.lastView)/1000) from ViewedTask vt where vt.userId = :userId", [userId: user.userId])[0]
+        def asyncLastActivities = ViewedTask.async.withStatelessSession {
+            def sw2 = new Stopwatch().start()
+            def lastActivities = ViewedTask.executeQuery("select vt.userId, to_timestamp(max(vt.lastView)/1000) from ViewedTask vt group by vt.userId").collectEntries { [(it[0]): it[1]] }
+            sw2.stop()
+            log.debug("UserReport viewedTasks took ${sw2.toString()}")
+            lastActivities
+        }
 
-            def projectCount = ViewedTask.executeQuery("select distinct t.project from Task t where t.fullyTranscribedBy = :userId", [userId:  user.userId]).size()
+        def asyncProjectCounts = Task.async.withStatelessSession {
+            def sw4 = new Stopwatch().start()
+            def projectCounts = Task.executeQuery("select t.fullyTranscribedBy, count(distinct t.project) from Task t group by t.fullyTranscribedBy ").collectEntries { [(it[0]): it[1]] }
+            sw4.stop()
+            log.debug("UserReport projectCounts took ${sw4.toString()}")
+            projectCounts
+        }
 
-            //def props = userService.detailsForUserId(user.userId)
-            def serviceResult = serviceResults?.users?.get(user.userId)
+        def sw3 = new Stopwatch().start()
+        def asyncUserDetails = User.async.list().then { users ->
+            def serviceResults = [:]
+            try {
+                serviceResults = authService.getUserDetailsById(users*.userId)
+            } catch (Exception e) {
+                log.warn("couldn't get user details from web service", e)
+            }
+            sw3.stop()
+            log.debug("UserReport user details took ${sw3.toString()}")
+
+            [users: users, results: serviceResults]
+        }
+
+        def asyncResults = waitAll(asyncCounts, asyncLastActivities, asyncProjectCounts, asyncUserDetails)
+
+        // transform raw results into map(id -> count)
+        def validateds = asyncResults[0].vs
+        def transcribeds = asyncResults[0].ts
+
+        def lastActivities = asyncResults[1]
+        def projectCounts = asyncResults[2]
+
+        def users = asyncResults[3].users
+        def serviceResults = asyncResults[3].results
+
+        def report = []
+
+        def sw5 = new Stopwatch().start()
+        for (User user: users) {
+            def id = user.userId
+            def transcribedCount = transcribeds[id] ?: 0
+            def validatedCount = validateds[id] ?: 0
+            def lastActivity = lastActivities[id]
+            def projectCount = projectCounts[id]?: 0
+
+            def serviceResult = serviceResults?.users?.get(id)
             report.add([serviceResult?.userName ?: user.email, serviceResult?.displayName ?: user.displayName, transcribedCount, validatedCount, lastActivity, projectCount, user.created])
         }
+        sw5.stop()
+        log.debug("UserReport generate report took ${sw5}")
 
+        sw5.reset().start()
         // Sort by the transcribed count
         report.sort({ row1, row2 -> row2[2] - row1[2]})
+        sw5.stop()
+        log.debug("UserReport sort took ${sw5.toString()}")
 
-        def nodata = params.nodata ?: 'nodata'
 
         if (params.wt && params.wt == 'csv') {
 
-            response.addHeader("Content-type", "text/plain")
-
-            def writer = new CSVWriter((Writer) response.writer,  {
-                'user_id' { it[0] }
-                'display_name' { it[1] }
-                'transcribed_count' { it[2] }
-                'validated_count' { it[3] }
-                'last_activity' { it[4] ?: nodata }
-                'projects_count' { it[5] }
-                'volunteer_since' { it[6] }
-            })
             for (def row : report) {
                 writer << row
             }

grails-app/controllers/au/org/ala/volunteer/PicklistController.groovy

@@ -21,7 +21,13 @@ class PicklistController {
     }
 
     def uploadCsvData = {
-        picklistService.replaceItems(Long.parseLong(params.picklistId), params.picklist, params.institutionCode)
+        picklistService.replaceItems(Long.parseLong(params.picklistId), params.picklist.toCsvReader(), params.institutionCode)
+        redirect(action: "manage")
+    }
+
+    def uploadCsvFile() {
+        def f = request.getFile('picklistFile')
+        picklistService.replaceItems(Long.parseLong(params.picklistId), f.inputStream.toCsvReader(['charset':'UTF-8']), params.institutionCode)
         redirect(action: "manage")
     }
 

grails-app/domain/au/org/ala/volunteer/PicklistItem.groovy

@@ -9,6 +9,8 @@ class PicklistItem implements Serializable {
 
     static mapping = {
       version false
+        picklist index: 'picklist_item_picklist_id_institution_code_idx'
+        institutionCode index: 'picklist_item_picklist_id_institution_code_idx'
     }
 
     static constraints = {

grails-app/services/au/org/ala/volunteer/PicklistService.groovy

@@ -1,5 +1,7 @@
 package au.org.ala.volunteer
 
+import au.com.bytecode.opencsv.CSVReader
+import org.grails.plugins.csv.CSVReaderUtils
 import org.hibernate.FlushMode
 
 class PicklistService {
@@ -33,7 +35,7 @@ class PicklistService {
         }
     }
 
-    def replaceItems(long picklistId, String csvdata, String institutionCode) {
+    def replaceItems(long picklistId, CSVReader csvdata, String institutionCode) {
         def picklist = Picklist.get(picklistId)
         // First delete the existing items...
         if (picklist) {
@@ -50,7 +52,7 @@ class PicklistService {
         int rowsProcessed = 0;
         try {
             sessionFactory.currentSession.setFlushMode(FlushMode.MANUAL)
-            csvdata.eachCsvLine { tokens ->
+            csvdata.eachLine { tokens ->
                 def value = tokens[0]
                 def m = pattern.matcher(value)
                 if (m.find()) {

grails-app/taglib/au/org/ala/volunteer/TranscribeTagLib.groovy

@@ -66,6 +66,8 @@ class TranscribeTagLib {
      * @attr labelClass
      * @attr valueClass
      * @attr field Optional, if the template already has the field object, no need to look up from it's name.
+     * @attr helpTargetPosition Optional, the target position for the qtip help pop up
+     * @attr helpTooltipPosition Optional, the tooltip position for the qtip help pop up
      */
     def renderFieldBootstrap = { attrs, body ->
 
@@ -75,6 +77,8 @@ class TranscribeTagLib {
         def valueClass = attrs.valueClass ?: "span12"
         def rowClass = attrs.rowClass ?: "row-fluid"
         def recordIdx = attrs.recordIdx ?: 0
+        def helpTargetPosition = attrs.helpTargetPosition
+        def helpTooltipPosition = attrs.helpTooltipPosition
 
         if (!task) {
             return
@@ -87,7 +91,7 @@ class TranscribeTagLib {
         }
 
         def mb = new MarkupBuilder(out)
-        renderFieldBootstrapImpl(mb, field, task, recordValues, recordIdx, labelClass, valueClass, attrs, rowClass)
+        renderFieldBootstrapImpl(mb, field, task, recordValues, recordIdx, labelClass, valueClass, attrs, rowClass, helpTargetPosition, helpTooltipPosition)
     }
 
     private String getFieldLabel(TemplateField field) {
@@ -98,7 +102,7 @@ class TranscribeTagLib {
         }
     }
 
-    private void renderFieldBootstrapImpl(MarkupBuilder mb, TemplateField field, Task task, recordValues, int recordIdx, String labelClass, String valueClass, Map attrs, String rowClass = "row-fluid") {
+    private void renderFieldBootstrapImpl(MarkupBuilder mb, TemplateField field, Task task, recordValues, int recordIdx, String labelClass, String valueClass, Map attrs, String rowClass = "row-fluid", String helpTargetPosition = null, String helpTooltipPosition = null) {
 
         if (!task || !field) {
             return
@@ -134,7 +138,7 @@ class TranscribeTagLib {
                         mkp.yieldUnescaped(widgetHtml)
                     }
                     div(class:'span2') {
-                        renderFieldHelp(mb, field)
+                        renderFieldHelp(mb, field, helpTargetPosition, helpTooltipPosition)
                     }
                 }
             }
@@ -518,13 +522,15 @@ class TranscribeTagLib {
 
     def fieldHelp = { attrs, body ->
         def field = attrs.field as TemplateField
-        renderFieldHelp(new MarkupBuilder(out), field)
+        def tooltipPosition = attrs.tooltipPosition
+        def targetPosition = attrs.targetPosition
+        renderFieldHelp(new MarkupBuilder(out), field, targetPosition, tooltipPosition)
     }
 
-    private renderFieldHelp(MarkupBuilder mb, TemplateField field) {
+    private renderFieldHelp(MarkupBuilder mb, TemplateField field, String targetPosition = null, String tooltipPosition = null) {
         if (field && field.helpText) {
             def helpText = markdownService.markdown(field.helpText)
-            mb.a(href:'#', class:'fieldHelp', title:helpText, tabindex: "-1") {
+            mb.a(href:'#', class:'fieldHelp', title:helpText, tabindex: "-1", targetPosition: targetPosition, tooltipPosition: tooltipPosition) {
                 span(class:'help-container') {
                     mkp.yieldUnescaped(' ')
                 }

grails-app/taglib/au/org/ala/volunteer/VolunteerTagLib.groovy

@@ -2,10 +2,14 @@ package au.org.ala.volunteer
 
 import grails.converters.JSON
 import grails.util.Environment
+import grails.util.Metadata
 import groovy.time.TimeCategory
 import au.org.ala.cas.util.AuthenticationCookieUtils
 import groovy.xml.MarkupBuilder
 
+import java.text.DateFormat
+import java.text.SimpleDateFormat
+
 class VolunteerTagLib {
 
     static namespace = 'cl'
@@ -19,7 +23,7 @@ class VolunteerTagLib {
     def authService
     def achievementService
 
-    static returnObjectForTags = ['emailForUserId', 'displayNameForUserId', 'achievementBadgeBase', 'newAchievements', 'achievementsEnabled']
+    static returnObjectForTags = ['emailForUserId', 'displayNameForUserId', 'achievementBadgeBase', 'newAchievements', 'achievementsEnabled', 'buildDate']
 
     def isLoggedIn = { attrs, body ->
 
@@ -722,4 +726,15 @@ class VolunteerTagLib {
     def achievementsEnabled = { attrs ->
         settingsService.getSetting(SettingDefinition.EnableMyNotebook) && settingsService.getSetting(SettingDefinition.EnableAchievementCalculations)
     }
+
+    def buildDate = { attrs ->
+        def bd = Metadata.current['build.date']
+        log.debug("Build Date type is ${bd?.class?.name}")
+        def df = new SimpleDateFormat('MMM d, yyyy')
+        if (bd) {
+            df.format(new SimpleDateFormat('EEE MMM dd HH:mm:ss zzz yyyy').parse(bd))
+        } else {
+            df.format(new Date())
+        }
+    }
 }

grails-app/views/admin/index.gsp

@@ -1,4 +1,4 @@
-<%@ page import="au.org.ala.volunteer.Project" %>
+<%@ page import="java.text.DateFormat; au.org.ala.volunteer.Project" %>
 <html>
     <head>
         <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
@@ -9,7 +9,7 @@
     <body>
 
         <cl:headerContent title="${message(code:'default.admin.label', default:'Admin')}">
-            <small class="muted">Version ${grailsApplication.metadata['app.version']}&nbsp;(built ${grailsApplication.metadata['app.buildDate']}&nbsp;${grailsApplication.metadata['app.buildProfile']}&nbsp;sha:&nbsp;<a href="https://github.com/AtlasOfLivingAustralia/volunteer-portal/commit/${grailsApplication.metadata['environment.TRAVIS_COMMIT']}">${grailsApplication.metadata['environment.TRAVIS_COMMIT']}</a>)</small>
+            <small class="muted">Version ${grailsApplication.metadata['app.version']}&nbsp;(built <cl:buildDate />&nbsp;${grails.util.Environment.current}&nbsp;sha:&nbsp;<a href="https://github.com/AtlasOfLivingAustralia/volunteer-portal/commit/${grailsApplication.metadata['environment.TRAVIS_COMMIT']}">${grailsApplication.metadata['environment.TRAVIS_COMMIT']}</a>)</small>
         </cl:headerContent>
 
 

grails-app/views/forum/postMessage.gsp

@@ -83,10 +83,10 @@
                 <h3>Conversation history:</h3>
                 <div style="height: 300px; overflow-y: scroll; border: 1px solid #a9a9a9" >
                     <g:each in="${topic.messages?.sort { it.date } }" var="reply">
-                        <div class="messageReply" author="<cl:userDetails id="${replyTo?.user?.userId}" displayName="true"/>" style="border: 1px solid #a9a9a9; margin: 3px; padding: 3px; background: white">
+                        <div class="messageReply" author="<cl:userDetails id="${reply?.user?.userId}" displayName="true"/>" style="border: 1px solid #a9a9a9; margin: 3px; padding: 3px; background: white">
                             <div style="background-color: #3a5c83; color: white">
                                 <img src="${resource(dir:'/images', file:'reply.png')}" style="vertical-align: bottom"/>
-                                On ${formatDate(date: reply.date, format: 'dd MMM yyyy')} at ${formatDate(date: reply.date, format: 'HH:mm:ss')} <strong><cl:userDetails id="${replyTo?.user?.userId}" displayName="true"/></strong> wrote:
+                                On ${formatDate(date: reply.date, format: 'dd MMM yyyy')} at ${formatDate(date: reply.date, format: 'HH:mm:ss')} <strong><cl:userDetails id="${reply?.user?.userId}" displayName="true"/></strong> wrote:
                             </div>
                             <markdown:renderHtml>${reply.text}</markdown:renderHtml>
                         </div>