Merge branch 'release/1.10.0'

OasisLMF · Sep 2, 2021 · fdff9e2 · fdff9e2
2 parents 0f6d8a2 + 4299128
commit fdff9e2
Show file tree

Hide file tree

Showing 73 changed files with 1,607 additions and 259 deletions.
diff --git a/BFE_RShiny/oasisui/DESCRIPTION b/BFE_RShiny/oasisui/DESCRIPTION
@@ -1,5 +1,5 @@
 Package: oasisui
-Version: 1.9.0
+Version: 1.10.0
 License: BSD_3_clause + file LICENSE
 Title: Oasis User Interface
 Description: This package provides an extensible implementation of a user
@@ -13,37 +13,39 @@ LazyData: true
 Depends: R (>= 3.5)
 NeedsCompilation: no
 Imports:
-  bsplus (== 0.1.1),
-  DT (== 0.16),
-  data.table (== 1.11.8),
-  dplyr (== 0.7.8),
-  geojsonio (== 0.6.0), 
-  geosphere (== 1.5.7),
-  ggplot2 (== 3.1.0),
-  htmltools (== 0.5.0),
-  htmlwidgets (== 1.5.2),
-  httr (== 1.4.0),
-  jsonlite (== 1.6),
-  leaflet (== 2.0.2),
+  arrow (== 4.0.0.1),
+  bsplus (== 0.1.2),
+  DT (== 0.18),
+  data.table (== 1.14.0),
+  dplyr (== 1.0.6),
+  geojsonio (== 0.9.4),
+  geosphere (== 1.5-10),
+  ggplot2 (== 3.3.3),
+  htmltools (== 0.5.1.1),
+  htmlwidgets (== 1.5.3),
+  httr (== 1.4.2),
+  jsonlite (== 1.7.2),
+  leaflet (== 2.0.4.1),
   leaflet.extras (== 1.0.0),
-  plotly (== 4.8.0),
-  readr (== 1.3.0),
+  plotly (== 4.9.3),
+  readr (== 1.4.0),
   revgeo (== 0.15),
-  R6 (== 2.3.0),
-  scales (== 1.0.0),
-  shiny (== 1.2.0),
-  shinycssloaders (== 0.2.0),
-  shinyjs (== 1.0.1.9006),
-  shinyWidgets (== 0.4.4),
-  tibble (== 1.4.2),
-  tidyr (== 0.8.2),
+  rgdal (== 1.5-23),
+  R6 (== 2.5.0),
+  scales (== 1.1.1),
+  shiny (== 1.6.0),
+  shinycssloaders (== 1.0.0),
+  shinyjs (== 1.1),
+  shinyWidgets (== 0.6.0),
+  tibble (== 3.1.2),
+  tidyr (== 1.1.3),
   utils,
-  V8 (== 1.5)
+  V8 (== 3.4.2)
 Suggests:
   knitr,
   testthat
 Encoding: UTF-8
 URL: https://oasislmf.github.io/, https://github.com/OasisLMF/OasisUI
 BugReports: https://github.com/OasisLMF/OasisUI/issues
-RoxygenNote: 6.1.1
+RoxygenNote: 7.1.1
 VignetteBuilder: knitr
diff --git a/BFE_RShiny/oasisui/NAMESPACE b/BFE_RShiny/oasisui/NAMESPACE
@@ -149,7 +149,9 @@ export(visualizationCBRUI)
 export(visualizationSBR)
 export(visualizationSBRUI)
 export(withModalSpinner)
+export(writeParquet)
 export(writefile)
+export(writefileJSON)
 import(shiny)
 importFrom(DT,DTOutput)
 importFrom(DT,dataTableProxy)
@@ -160,6 +162,8 @@ importFrom(DT,selectPage)
 importFrom(DT,selectRows)
 importFrom(DT,styleEqual)
 importFrom(R6,R6Class)
+importFrom(arrow,read_parquet)
+importFrom(arrow,write_parquet)
 importFrom(bsplus,bs_embed_tooltip)
 importFrom(data.table,fread)
 importFrom(data.table,fwrite)

diff --git a/BFE_RShiny/oasisui/R/ViewFilesInTable_module.R b/BFE_RShiny/oasisui/R/ViewFilesInTable_module.R
@@ -93,8 +93,12 @@ ViewFilesInTable <- function(input, output, session,
   result <- reactiveValues(
     #df to show in table
     tbl_filesListData_wButtons = NULL,
-    #View output file content
+    #View output file content csv
     currentFile = NULL,
+    #View output file content parquet
+    currentFileP = NULL,
+    # Display file category in file name
+    fileCategory = NULL,
     # Filepath of file to view
     currfilepath = NULL,
     #content of curr file
@@ -155,6 +159,15 @@ ViewFilesInTable <- function(input, output, session,
     }
   )
 
+  observeEvent(result$currentFile, {
+    if (!grepl(".json", result$currentFile)) {
+      show("FVEdownloadparquet")
+      show("FVEdownloadexcel")
+    } else {
+      show("FVEdownloadjson")
+    }
+  })
+
   # Download zip Files ----------------------------------------------------------
 
   # Download zip button
@@ -188,7 +201,8 @@ ViewFilesInTable <- function(input, output, session,
   output$FLdownloadexcel <- downloadHandler(
     filename = "file.csv",
     content = function(file) {
-      session$userData$data_hub$write_file(data = result$tbl_filesListData_wButton, dataset_identifier = filename, file_towrite = file)
+      session$userData$data_hub$write_file(data = result$tbl_filesListData_wButton, dataset_identifier = filename,
+                                           file_towrite = file)
     }
   )
 
@@ -267,7 +281,9 @@ ViewFilesInTable <- function(input, output, session,
       fluidRow(
         oasisuiButton(inputId = ns("abuttonview"), label = "Content", icon = icon("file")),
         hidden(oasisuiButton(inputId = ns("abuttonmap"), label = "Map", icon = icon("map"))),
-        downloadButton(ns("FVEdownloadexcel"), label = "Export"),
+        hidden(downloadButton(ns("FVEdownloadexcel"), label = "CSV")),
+        hidden(downloadButton(ns("FVEdownloadparquet"), label = "Parquet")),
+        hidden(downloadButton(ns("FVEdownloadjson"), label = "JSON")),
         style = "display: inline"),
 
       hidden(oasisuiPanel(
@@ -340,6 +356,21 @@ ViewFilesInTable <- function(input, output, session,
       session$userData$data_hub$write_file(data = result$tbl_fileData, dataset_identifier = result$currentFile, file_towrite = file)
     }
   )
+  # Export to .parquet
+  output$FVEdownloadparquet <- downloadHandler(
+    filename = function(){result$currentFileP},
+    content = function(file) {
+      session$userData$data_hub$write_parquet_file(data = result$tbl_fileData, dataset_identifier = result$currentFileP,
+                                              file_towrite = file)
+    }
+  )
+  # Export to .json
+  output$FVEdownloadjson <- downloadHandler(
+    filename = function(){result$currentFile},
+    content = function(file) {
+      session$userData$data_hub$write_file_json(data = result$tbl_fileData, dataset_identifier = result$currentFile, file_towrite = file)
+    }
+  )
 
   # Panel Map
   observeEvent(input$abuttonhidemapFVExposureSelected, {
@@ -358,14 +389,18 @@ ViewFilesInTable <- function(input, output, session,
 
     # Get dataframe
     result$currentFile <- result$tbl_filesListData_wButtons[idx, file_column] %>% as.character()
-    if (result$currentFile %in% c("location_file", "accounts_file", "reinsurance_info_file", "reinsurance_scope_file")) {
-      result$tbl_fileData <- session$userData$data_hub$get_pf_dataset_content(id = param(), dataset_identifier = result$currentFile)
+    result$fileCategory <- result$currentFile
+    result$currentFileP <- paste0(result$fileCategory, ".parquet")
+    if (result$fileCategory %in% c("location_file", "accounts_file", "reinsurance_info_file", "reinsurance_scope_file")) {
+      result$tbl_fileData <- session$userData$data_hub$get_pf_dataset_content(id = param(), dataset_identifier = result$fileCategory)
       if (!is.null(result$tbl_fileData)) {
-        filecolumns <- session$userData$data_hub$get_pf_dataset_header(id = param(), dataset_identifier = result$currentFile)
-        filerows <- session$userData$data_hub$get_pf_dataset_nrow(id = param(), dataset_identifier = result$currentFile)
-        result$currentFile <- paste0(result$currentFile, ".csv")
+        filecolumns <- session$userData$data_hub$get_pf_dataset_header(id = param(), dataset_identifier = result$fileCategory)
+        filerows <- session$userData$data_hub$get_pf_dataset_nrow(id = param(), dataset_identifier = result$fileCategory)
+        # result$currentFileP <- paste0(result$fileCategory, ".parquet")
+        result$currentFile <- paste0(result$fileCategory, ".csv")
         # Show buttons
         if ("latitude" %in% tolower(names(result$tbl_fileData)) && !is.null(result$tbl_fileData)) {
+          names(result$tbl_fileData) <- tolower(names(result$tbl_fileData))
           output$plainmap <- renderLeaflet({
             createPlainMap(result$tbl_fileData, session = session, paramID = param(), step = NULL)
           })
@@ -375,10 +410,10 @@ ViewFilesInTable <- function(input, output, session,
         }
       }
     } else {
-      result$tbl_fileData <- session$userData$data_hub$get_ana_dataset_content(id = param(), dataset_identifier = result$currentFile, type = folderpath)
+      result$tbl_fileData <- session$userData$data_hub$get_ana_dataset_content(id = param(), dataset_identifier = result$fileCategory, type = folderpath)
       if (!is.null(result$tbl_fileData)) {
-        filecolumns <- session$userData$data_hub$get_ana_dataset_header(id = param(), dataset_identifier = result$currentFile, type = folderpath)
-        filerows <- session$userData$data_hub$get_ana_dataset_nrow(id = param(), dataset_identifier = result$currentFile, type = folderpath)
+        filecolumns <- session$userData$data_hub$get_ana_dataset_header(id = param(), dataset_identifier = result$fileCategory, type = folderpath)
+        filerows <- session$userData$data_hub$get_ana_dataset_nrow(id = param(), dataset_identifier = result$fileCategory, type = folderpath)
       }
       # Show buttons
       if ("latitude" %in% tolower(names(result$tbl_fileData))) {
@@ -401,7 +436,7 @@ ViewFilesInTable <- function(input, output, session,
                  h5("File Name")
           ),
           column(10,
-                 p(result$currentFile, style = "margin-top: 10px;")
+                 p(result$fileCategory, style = "margin-top: 10px;")
           )
         ),
         fluidRow(

diff --git a/BFE_RShiny/oasisui/R/data_hub_helper_funcs.R b/BFE_RShiny/oasisui/R/data_hub_helper_funcs.R
@@ -59,36 +59,71 @@ read_file_from_tar <- function(tarfile, dataset_identifier, destdir = tempdir(),
   data
 }
 
-#' write file
+#' write CSV
 #'
-#' @rdname write_file
+#' @rdname writefile
 #'
-#' @description Writes ojbect in the correct format.
+#' @description Writes object in CSV format.
 #'
 #' @param data object to write.
 #' @param dataset_identifier name and relative path of file to write
 #' @param destdir path where to write file.
-#' @param file_towrite name of file where to wirte data.
+#' @param file_towrite name of file where to write data.
 #'
-#' @importFrom data.table fread
-#' @importFrom jsonlite read_json
-#' @importFrom utils tail
+#' @importFrom data.table fwrite
 #'
 #' @export
-
-writefile <- function(data, dataset_identifier = NULL, destdir = tempdir(), file_towrite = NULL){
+writefile <- function(data, dataset_identifier = NULL, destdir = tempdir(), file_towrite = NULL) {
   if (is.null(file_towrite)) {
     file_towrite <- file.path(destdir, dataset_identifier)
   }
-  extension <-  strsplit(dataset_identifier, split = "\\.") %>% unlist() %>% tail(n = 1)
-  if (extension == "json") {
-    write(toJSON(data, pretty = TRUE), file_towrite)
-  } else {
-    fwrite(data, file_towrite, row.names = FALSE, quote = TRUE)
+  fwrite(data, file_towrite, row.names = FALSE, quote = TRUE)
+  file_towrite
+}
+
+#' write parquet
+#'
+#' @rdname write_parquet
+#'
+#' @description Writes object in parquet format.
+#'
+#' @param data object to write.
+#' @param dataset_identifier name and relative path of file to write
+#' @param destdir path where to write file.
+#' @param file_towrite name of file where to write data.
+#'
+#' @importFrom arrow write_parquet
+#'
+#' @export
+writeParquet <- function(data, dataset_identifier = NULL, destdir = tempdir(), file_towrite = NULL) {
+  if (is.null(file_towrite)) {
+    file_towrite <- file.path(destdir, dataset_identifier)
   }
+  write_parquet(data, file_towrite)
   file_towrite
 }
 
+#' write file JSON
+#'
+#' @rdname writefileJSON
+#'
+#' @description Writes object in JSON format.
+#'
+#' @param data object to write.
+#' @param dataset_identifier name and relative path of file to write
+#' @param destdir path where to write file.
+#' @param file_towrite name of file where to write data.
+#'
+#' @importFrom jsonlite toJSON
+#'
+#' @export
+writefileJSON <- function(data, dataset_identifier = NULL, destdir = tempdir(), file_towrite = NULL) {
+  if (is.null(file_towrite)) {
+    file_towrite <- file.path(destdir, dataset_identifier)
+  }
+  write(toJSON(data, pretty = TRUE), file_towrite)
+  file_towrite
+}
 
 #' simplify_path
 #'

diff --git a/BFE_RShiny/oasisui/R/datahub.R b/BFE_RShiny/oasisui/R/datahub.R
@@ -125,6 +125,7 @@
 #' @importFrom tidyr separate
 #' @importFrom tidyr spread
 #' @importFrom tidyr gather
+#' @importFrom arrow read_parquet
 #'
 #' @export
 
@@ -241,7 +242,15 @@ DataHub <- R6Class(
     # dataset_identifier is location/account/reinsurance_info/reinsurance_source
     # If file does not exist, returns df details: Not Found
     get_pf_dataset_content = function(id, dataset_identifier, ...) {
-      dataset_content <- content(private$oasisapi$api_get_query(paste("portfolios", id, dataset_identifier, sep = "/"))$result)
+      # check if file has the parquet extension (i.e. is not csv). If yes, process differently then csv
+      if (!grepl("csv",
+                private$oasisapi$api_get_query(paste("portfolios", id, dataset_identifier, sep = "/"))$result[3]$headers$`content-type`)) {
+
+        dataset_content <- read_parquet(content(private$oasisapi$api_get_query(paste("portfolios", id, dataset_identifier, sep = "/"))$result))
+      } else {
+        dataset_content <- content(private$oasisapi$api_get_query(paste("portfolios", id, dataset_identifier, sep = "/"))$result)
+      }
+
       if (is.null(names(dataset_content))) {
         if (class(dataset_content) == "raw") {
           dataset_content <- as.data.frame(dataset_content)
@@ -478,6 +487,14 @@ DataHub <- R6Class(
       fs <- writefile(data, dataset_identifier, destdir = private$user_destdir, file_towrite)
       fs
     },
+    write_parquet_file = function(data, dataset_identifier, file_towrite = NULL, ...) {
+      fs <- writeParquet(data, dataset_identifier, destdir = private$user_destdir, file_towrite)
+      fs
+    },
+    write_file_json = function(data, dataset_identifier, file_towrite = NULL, ...) {
+      fs <- writefileJSON(data, dataset_identifier, destdir = private$user_destdir, file_towrite)
+      fs
+    },
     # > Helper methods ----
     get_analyses_tar = function(id, label, destdir = tempdir()) {
       dest <- tempfile(tmpdir = destdir, fileext = ".tar")

diff --git a/BFE_RShiny/oasisui/R/exposurevalidation_map_module.R b/BFE_RShiny/oasisui/R/exposurevalidation_map_module.R
@@ -44,7 +44,7 @@ exposurevalidationmapUI <- function(id) {
              hidden(DTOutput(ns("exposure_table")))
       )
     ),
-    fluidRow(column(2, hidden(downloadButton(ns("exp_tivs"), label = "Export to csv"))))
+    fluidRow(column(2, hidden(downloadButton(ns("exp_tivs"), label = "CSV"))))
   )
 }
 
@@ -192,7 +192,7 @@ exposurevalidationmap <- function(input,
         h4("Validated Exposure by Peril")),
       fluidRow(
         DTOutput(ns("dt_output_uploaded_lock_check")),
-        downloadButton(ns("exp_downloadexcel"), label = "Export to csv"),
+        downloadButton(ns("exp_downloadexcel"), label = "CSV"),
         style = "display: inline")
     )
   )

diff --git a/BFE_RShiny/oasisui/R/landingpage_module.R b/BFE_RShiny/oasisui/R/landingpage_module.R
@@ -27,7 +27,7 @@ landingPageUI <- function(id) {
         bs_embed_tooltip(title = landing_page_tooltips$abuttongotoana, placement = "right"),
       oasisuiTableButton(inputId = ns("abuttondelana"), label = "Delete Analysis") %>%
         bs_embed_tooltip(title = landing_page_tooltips$abuttondelana, placement = "right"),
-      downloadButton(ns("downloadexcel_ana"), label = "Export to csv") %>%
+      downloadButton(ns("downloadexcel_ana"), label = "CSV") %>%
         bs_embed_tooltip(title = landing_page_tooltips$downloadexcel_ana, placement = "right")
     )
   )

diff --git a/BFE_RShiny/oasisui/R/map.R b/BFE_RShiny/oasisui/R/map.R
@@ -35,8 +35,9 @@ createPlainMap <- function(df, session, paramID, step = NULL) {
     addMarkers(lng = ~longitude,
                lat = ~latitude,
                icon = icon_map,
-               clusterOptions = markerClusterOptions(maxClusterRadius = 50),
-               popup = ~popup) %>% # make map full screen
+               clusterOptions = markerClusterOptions(maxClusterRadius = 50)#,
+               # popup = popup
+               ) %>% # make map full screen
     addFullscreenControl(pseudoFullscreen = TRUE)
 }
 
@@ -112,7 +113,7 @@ build_marker_data <- function(data, session, paramID, step = NULL) {
 # Extract error messages
 .keys_errors_msg <- function(data, session, paramID) {
   keys_errors <- session$userData$data_hub$get_ana_errors_summary_content(id = paramID)
-  if (!is.null(keys_errors)) {
+  if (!is.null(keys_errors) && is.null(keys_errors$detail)) {
     message <- group_by(keys_errors, LocID) %>%
       summarize(message = paste(paste(PerilID, ":", Message), collapse = " / "))
     if (length(as.numeric(keys_errors$LocID)) > 0) {