From d1b3ef823c721c635190942a1b478052d91236ee Mon Sep 17 00:00:00 2001 From: sambles Date: Tue, 14 Nov 2023 16:14:20 +0000 Subject: [PATCH] Release 1.11.7 (#314) * fix for https://github.com/OasisLMF/OasisUI/issues/309 (#310) * added compose option for ui dev * tidy * Update Dockerfile.oasisui_development corrected context - addressing code review comments. * Feature/308 plus 307 improvements (#312) * #307 related improvements * addresses #308, fixes #307 * fix trivy call (#313) --------- Co-authored-by: Roland Schmid * Set UI to version 1.11.7 * test-release * fix release workflow * Update changelog --------- Co-authored-by: Darren Holdaway Co-authored-by: Roland Schmid Co-authored-by: awsbuild --- .github/workflows/build.yml | 2 +- .github/workflows/publish.yml | 2 +- .gitignore | 5 + BFE_RShiny/oasisui/DESCRIPTION | 2 +- .../R/exposurevalidation_summary_module.R | 7 +- BFE_RShiny/oasisui/R/output_config_module.R | 18 ++- .../oasisui/R/step2_chooseAnalysis_module.R | 14 +- CHANGELOG.rst | 7 + README.md | 37 ++++- compose/oasis-platform-ui-dev.yml | 141 ++++++++++++++++++ docker/Dockerfile.oasisui_development | 2 +- rstudio | 83 +++++++++++ 12 files changed, 299 insertions(+), 21 deletions(-) create mode 100644 .gitignore create mode 100644 compose/oasis-platform-ui-dev.yml create mode 100755 rstudio diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 46c110b3..c5531bdf 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -118,7 +118,7 @@ jobs: exit-code: '1' ignore-unfixed: true severity: ${{ env.SEVERITY }} - security-checks: 'vuln' + scanners: 'vuln' output: 'app_image.txt' - name: Store CVE report diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 30500c02..abe013f7 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -220,7 +220,7 @@ jobs: # --- Notify Slack --- # slack: - uses: OasisLMF/OasisLMF/.github/workflows/notify.yml@master + uses: OasisLMF/OasisLMF/.github/workflows/notify.yml@main secrets: inherit needs: release with: diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..64d279ed --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +.Rproj.user +.Rhistory +.RData +.Ruserdata +compose/OasisPiWind \ No newline at end of file diff --git a/BFE_RShiny/oasisui/DESCRIPTION b/BFE_RShiny/oasisui/DESCRIPTION index 1e0aae81..ecd4c48f 100644 --- a/BFE_RShiny/oasisui/DESCRIPTION +++ b/BFE_RShiny/oasisui/DESCRIPTION @@ -1,5 +1,5 @@ Package: oasisui -Version: 1.11.6 +Version: 1.11.7 License: BSD_3_clause + file LICENSE Title: Oasis User Interface Description: This package provides an extensible implementation of a user diff --git a/BFE_RShiny/oasisui/R/exposurevalidation_summary_module.R b/BFE_RShiny/oasisui/R/exposurevalidation_summary_module.R index 53096d99..215a9698 100644 --- a/BFE_RShiny/oasisui/R/exposurevalidation_summary_module.R +++ b/BFE_RShiny/oasisui/R/exposurevalidation_summary_module.R @@ -111,11 +111,16 @@ exposurevalidationsummary <- function(input, # note this will remain TRUE as the active tab when switching to step 3! # We should get the analysis first to check the status and only continue # if the status is > than NEW, otherwise subsequent queries won't work. + # Subsequent queries will also fail when creating a new analysis, in + # which case the status says "INPUTS_GENERATION_STARTED" at this point. + # There should be a second pass through here then when status changes + # to "READY", so this makes some sense. anaStatus <- session$userData$oasisapi$api_return_query_res( query_path = paste("analyses", analysisID(), sep = "/"), query_method = "GET" )[["status"]] - if (anaStatus == "NEW") { + + if (anaStatus %in% c("NEW", "INPUTS_GENERATION_STARTED")) { # chill } else { result$summary_tbl <- session$userData$data_hub$get_ana_validation_summary_content(analysisID()) diff --git a/BFE_RShiny/oasisui/R/output_config_module.R b/BFE_RShiny/oasisui/R/output_config_module.R index 7f210016..608e99ca 100644 --- a/BFE_RShiny/oasisui/R/output_config_module.R +++ b/BFE_RShiny/oasisui/R/output_config_module.R @@ -120,6 +120,7 @@ panelModelParams <- function(id) { align = "left", numericInput(ns("tinputnoofsample"), label = "Number of Samples:", value = 9), numericInput(ns("tinputthreshold"), label = "Loss Threshold:", value = 0), + checkboxInput(ns("tinputpostlossampli"), label = "Post Loss Amplification", FALSE), # 239: additional output cfg options (not part of modelsettings!) textInput(ns("tinputreturnperiods"), label = "Return Periods:", value = "") %>% bs_embed_tooltip(title = defineSingleAna_tooltips$tinputreturnperiods, placement = "right"), @@ -912,10 +913,19 @@ def_out_config <- function(input, # --> see "abuttonchoosetag") # - copy settings from setting templates (new functionality as of July 2022) if (is.null(ana_settings)) { - tbl_ana_settings <- session$userData$oasisapi$api_return_query_res( - query_path = paste("analyses", analysisID(), "settings", sep = "/"), + # check existence before calling on a freshly created analysis that lacks the settings and will result in a 404 + check_settings_exist <- session$userData$oasisapi$api_return_query_res( + query_path = paste("analyses", analysisID(), sep = "/"), query_method = "GET" - ) + )[["settings"]] + if (is.null(check_settings_exist)) { + tbl_ana_settings <- NULL + } else { + tbl_ana_settings <- session$userData$oasisapi$api_return_query_res( + query_path = paste("analyses", analysisID(), "settings", sep = "/"), + query_method = "GET" + ) + } } else { tbl_ana_settings <- ana_settings } @@ -964,6 +974,7 @@ def_out_config <- function(input, else x } updateNumericInput(session, "tinputthreshold", value = tbl_ana_settings$gul_threshold) + updateCheckboxInput(session, "tinputpostlossampli", value = tbl_ana_settings$pla) updateTextInput(session, "tinputreturnperiods", value = .null_to_empty(tbl_ana_settings$return_periods)) updateTextInput(session, "tinputeventids", value = .null_to_empty(tbl_ana_settings$event_ids)) updateTextInput(session, "tinputquantiles", value = .null_to_empty(tbl_ana_settings$quantiles)) @@ -1205,6 +1216,7 @@ def_out_config <- function(input, "ui_config_tag" = input$sintag, # potential new tag analysis_id "gul_threshold" = as.integer(input$tinputthreshold), + "pla" = input$tinputpostlossampli, # 239: insert additional output cfg options at the desired spot. # these are explicitly lists because a single element would be converted # differently to json otherwise diff --git a/BFE_RShiny/oasisui/R/step2_chooseAnalysis_module.R b/BFE_RShiny/oasisui/R/step2_chooseAnalysis_module.R index a29c2b18..d3be7a67 100644 --- a/BFE_RShiny/oasisui/R/step2_chooseAnalysis_module.R +++ b/BFE_RShiny/oasisui/R/step2_chooseAnalysis_module.R @@ -664,7 +664,6 @@ step2_chooseAnalysis <- function(input, output, session, # Create new Analysis -------------------------------------------------------- observeEvent(input$abuttonsubmit, { if (input$anaName != "") { - post_portfolios_create_analysis <- session$userData$oasisapi$api_body_query(query_path = paste("analyses", sep = "/"), query_body = list(name = input$anaName, portfolio = portfolioID(), @@ -684,15 +683,13 @@ step2_chooseAnalysis <- function(input, output, session, result$analysisID <- content(post_portfolios_create_analysis$result)$id result$analysisNAME <- content(post_portfolios_create_analysis$result)$name - logMessage(paste0("Calling api_post_analyses_generate_inputs with id", result$analysisID)) - - if (length(model_settings) > 0 && !is.null(model_settings$model_configurable) && - model_settings$model_configurable && !is.null(sub_modules$buildCustom$fullsettings())) { + hasCustom <- length(model_settings) > 0 && !is.null(model_settings$model_configurable) && model_settings$model_configurable + if (hasCustom && !is.null(sub_modules$buildCustom$fullsettings())) { post_analysis_settings <- session$userData$oasisapi$api_body_query( query_path = paste("analyses", result$analysisID, "settings", sep = "/"), query_body = sub_modules$buildCustom$fullsettings() ) - } else { + } else if (hasCustom && is.null(sub_modules$buildCustom$fullsettings())) { gul_summaries <- summary_template ana_settings_step_2 <- list(analysis_settings = c( @@ -707,11 +704,11 @@ step2_chooseAnalysis <- function(input, output, session, post_analysis_settings <- session$userData$oasisapi$api_body_query( query_path = paste("analyses", result$analysisID, "settings", sep = "/"), - query_body = ana_settings_step_2 + query_body = ana_settings_step_2 # should this be ana_settings_step_2[[1]]? ) } - if (post_portfolios_create_analysis$status == "Success" && post_analysis_settings$status == "Success" && !is.null(model_settings$model_configurable) && model_settings$model_configurable) { + if (hasCustom && post_portfolios_create_analysis$status == "Success" && post_analysis_settings$status == "Success") { fileids <- as.list(sub_modules$buildCustom$fileids()) patch_analyses <- session$userData$oasisapi$api_body_query(query_path = paste("analyses", result$analysisID, sep = "/"), query_body = list(complex_model_data_files = fileids), @@ -719,6 +716,7 @@ step2_chooseAnalysis <- function(input, output, session, # content(patch_analyses$result) } + logMessage(paste0("Calling api_post_analyses_generate_inputs with id", result$analysisID)) input_generation <- session$userData$oasisapi$api_post_query( query_path = paste("analyses", result$analysisID, "generate_inputs", sep = "/") ) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index cf76a6f7..8ecd33dc 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,13 @@ OasisUI Changelog ================== +`1.11.7`_ + --------- +* [#310](https://github.com/OasisLMF/OasisUI/pull/310) - fix for https://github.com/OasisLMF/OasisUI/issues/309 +* [#307, #308](https://github.com/OasisLMF/OasisUI/pull/312) - Feature/308 plus 307 improvements +* [#313](https://github.com/OasisLMF/OasisUI/pull/313) - Fixed Trivy CVE scanning +.. _`1.11.7`: https://github.com/OasisLMF/OasisUI/compare/1.11.6...1.11.7 + `1.11.6`_ --------- * [#292, #299, #277, #280, #282](https://github.com/OasisLMF/OasisUI/pull/305) - Feature/various bugfixes diff --git a/README.md b/README.md index dd0ea872..419bad0f 100644 --- a/README.md +++ b/README.md @@ -3,13 +3,17 @@ [![OasisUI Build](https://github.com/OasisLMF/OasisUI/actions/workflows/build.yml/badge.svg?branch=master&event=push)](https://github.com/OasisLMF/OasisUI/actions/workflows/build.yml) # Oasis UI + Web-based application client for managing exposure data and operating modelling workflows.There are three components: - * Shiny UI application (This repository) - * Python Django server providing services for interacting with exposure and output data - * PostgreSQL server database + +* Shiny UI application (This repository) +* Python Django server providing services for interacting with exposure and output data +* PostgreSQL server database ## Usage and documentation + For a detailed guide on using the OasisUI see + * [Oasis_UI_Guide.pdf](documentation/Oasis_UI_Guide.pdf?raw=true) * [Oasis UI - Youtube Walkthrough](https://www.youtube.com/watch?v=tHRetuhpQzA) @@ -17,13 +21,14 @@ For a detailed guide on using the OasisUI see To try out the OasisUI run the docker installation script `./install.sh` from [OasisEvaluation](https://github.com/OasisLMF/OasisEvaluation). For tutorials on running the Oasis Stack locally see: (The full stack is required for the UI to work) + * [Windows 10 - Installation Guide](https://www.youtube.com/watch?v=SxRt5E-Y5Sw) * [Linux - Installation Guide](https://www.youtube.com/watch?v=OFLTpGGEM10) - ## Testing and development The script `./run_rstudio.sh` deploys and runs a development version of OasisUI, it runs using an [RStudio Server](https://documentation.dnanexus.com/getting-started/developer-tutorials/web-app-let-tutorials/running-rstudio-server). + 1. run the script open the url [http://localhost:8787/](http://localhost:8787/) in a browser. 2. Load the project file ![Load Project](.img/dev_load_project.png?raw=true "Load RStudio Project") @@ -33,6 +38,28 @@ The script `./run_rstudio.sh` deploys and runs a development version of OasisUI, 4. In the R console enter `> oasisui::runOasisui()`, this will run the OasisUI in a new window ![Run App](.img/dev_run_oasisui.png?raw=true "Run Application") +## Running with Docker compose + +1. build the R Studio container + +```bash + ./rstudio build +``` + +2. bring up the stack + +```bash + ./rstudio start +``` + +open the url [http://localhost:8787/](http://localhost:8787/) in a browser. + +3. bring down the stack + +```bash + ./rstudio stop +``` + ## License -The code in this project is licensed under BSD 3-clause license. +The code in this project is licensed under BSD 3-clause license. diff --git a/compose/oasis-platform-ui-dev.yml b/compose/oasis-platform-ui-dev.yml new file mode 100644 index 00000000..f754a00f --- /dev/null +++ b/compose/oasis-platform-ui-dev.yml @@ -0,0 +1,141 @@ +version: "3.8" +volumes: + server-db-OasisData: + celery-db-OasisData: + filestore-OasisData: +services: + server: + restart: always + image: ${SERVER_IMG:-coreoasis/api_server}:${VERS_API:-latest} + ports: + - 8000:8000 + links: + - server-db + - celery-db + - rabbit + environment: + - OASIS_ADMIN_USER=admin + - OASIS_ADMIN_PASS=password + - OASIS_DEBUG=1 + - OASIS_RABBIT_HOST=rabbit + - OASIS_RABBIT_PORT=5672 + - OASIS_RABBIT_USER=rabbit + - OASIS_RABBIT_PASS=rabbit + - OASIS_SERVER_DB_HOST=server-db + - OASIS_SERVER_DB_PASS=oasis + - OASIS_SERVER_DB_USER=oasis + - OASIS_SERVER_DB_NAME=oasis + - OASIS_SERVER_DB_PORT=5432 + - OASIS_SERVER_DB_ENGINE=django.db.backends.postgresql_psycopg2 + - OASIS_CELERY_DB_ENGINE=db+postgresql+psycopg2 + - OASIS_CELERY_DB_HOST=celery-db + - OASIS_CELERY_DB_PASS=password + - OASIS_CELERY_DB_USER=celery + - OASIS_CELERY_DB_NAME=celery + - OASIS_CELERY_DB_PORT=5432 + - STARTUP_RUN_MIGRATIONS=true + volumes: + - filestore-OasisData:/shared-fs:rw + worker-monitor: + restart: always + image: ${SERVER_IMG:-coreoasis/api_server}:${VERS_API:-latest} + command: + [ + wait-for-server, + "server:8000", + celery, + -A, + src.server.oasisapi, + worker, + --loglevel=INFO, + ] + links: + - server-db + - celery-db + - rabbit + environment: + - OASIS_DEBUG=1 + - OASIS_RABBIT_HOST=rabbit + - OASIS_RABBIT_PORT=5672 + - OASIS_RABBIT_USER=rabbit + - OASIS_RABBIT_PASS=rabbit + - OASIS_SERVER_DB_HOST=server-db + - OASIS_SERVER_DB_PASS=oasis + - OASIS_SERVER_DB_USER=oasis + - OASIS_SERVER_DB_NAME=oasis + - OASIS_SERVER_DB_PORT=5432 + - OASIS_SERVER_DB_ENGINE=django.db.backends.postgresql_psycopg2 + - OASIS_CELERY_DB_ENGINE=db+postgresql+psycopg2 + - OASIS_CELERY_DB_HOST=celery-db + - OASIS_CELERY_DB_PASS=password + - OASIS_CELERY_DB_USER=celery + - OASIS_CELERY_DB_NAME=celery + - OASIS_CELERY_DB_PORT=5432 + volumes: + - filestore-OasisData:/shared-fs:rw + worker: + restart: always + image: ${WORKER_IMG:-coreoasis/model_worker}:${VERS_WORKER:-latest} + links: + - celery-db + - rabbit:myrabbit + environment: + - OASIS_MODEL_SUPPLIER_ID=OasisLMF + - OASIS_MODEL_ID=PiWind + - OASIS_MODEL_VERSION_ID=1 + - OASIS_RABBIT_HOST=rabbit + - OASIS_RABBIT_PORT=5672 + - OASIS_RABBIT_USER=rabbit + - OASIS_RABBIT_PASS=rabbit + - OASIS_SERVER_DB_ENGINE=django.db.backends.postgresql_psycopg2 + - OASIS_CELERY_DB_ENGINE=db+postgresql+psycopg2 + - OASIS_CELERY_DB_HOST=celery-db + - OASIS_CELERY_DB_PASS=password + - OASIS_CELERY_DB_USER=celery + - OASIS_CELERY_DB_NAME=celery + - OASIS_CELERY_DB_PORT=5432 + - OASIS_MODEL_DATA_DIRECTORY=/home/worker/model + volumes: + - ./OasisPiWind/:/home/worker/model + - filestore-OasisData:/shared-fs:rw + server-db: + restart: always + image: postgres + environment: + - POSTGRES_DB=oasis + - POSTGRES_USER=oasis + - POSTGRES_PASSWORD=oasis + volumes: + - server-db-OasisData:/var/lib/postgresql/data:rw + celery-db: + restart: always + image: postgres + environment: + - POSTGRES_DB=celery + - POSTGRES_USER=celery + - POSTGRES_PASSWORD=password + volumes: + - celery-db-OasisData:/var/lib/postgresql/data:rw + rabbit: + restart: always + image: rabbitmq:3.8.14-management + environment: + - RABBITMQ_DEFAULT_USER=rabbit + - RABBITMQ_DEFAULT_PASS=rabbit + ports: + - 5672:5672 + - 15672:15672 + rstudio: + image: oasisui_dev:latest + restart: always + environment: + - DISABLE_AUTH=true + - USERID=${UID:-1000} + - DISPLAY=${DISPLAY:-:0} + volumes: + - ${SHARED_DIR}:/home/rstudio/oasisui + - /tmp/.X11-unix:/tmp/.X11-unix:ro + ports: + - 8787:8787 + networks: + - default diff --git a/docker/Dockerfile.oasisui_development b/docker/Dockerfile.oasisui_development index ab1542b7..b261a7c5 100644 --- a/docker/Dockerfile.oasisui_development +++ b/docker/Dockerfile.oasisui_development @@ -29,7 +29,7 @@ RUN R -e 'devtools::install(pkg = "./BFE_RShiny/oasisui", dependencies = TRUE)' # Create .Rprofile and set default values RUN echo 'message("Run app with: oasisui::runOasisui()")' >> /home/rstudio/.Rprofile -ENV API_IP="127.0.0.1" +ENV API_IP="server" ENV API_PORT="8000" ENV API_VERSION="v1" ENV API_SHARE_FILEPATH="./downloads" diff --git a/rstudio b/rstudio new file mode 100755 index 00000000..56a28076 --- /dev/null +++ b/rstudio @@ -0,0 +1,83 @@ +#!/bin/bash + +SHARED_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +R_PROJECT_FILE="$SHARED_DIR/oasisui_dev.Rproj" +R_CONTAINER_NAME='rstudio_oasisui' +DOCKERFILE_PATH="$SHARED_DIR/docker/Dockerfile.oasisui_development" +IMAGE_NAME="oasisui_dev:latest" +COMPOSE_FILE="$SHARED_DIR/compose/oasis-platform-ui-dev.yml" + +export SHARED_DIR +export UID=${UID:-1000} +export DISPLAY=${DISPLAY:-:0} + +function build() { +echo "Creating a new R project file with default configurations..." +{ + echo "Version: 1.0" + echo "RestoreWorkspace: Default" + echo "SaveWorkspace: Default" + echo "AlwaysSaveHistory: Default" + echo "EnableCodeIndexing: Yes" + echo "UseSpacesForTab: Yes" + echo "NumSpacesForTab: 2" + echo "Encoding: UTF-8" + echo "RnwWeave: Sweave" + echo "LaTeX: pdfLaTeX" + echo "BuildType: Package" + echo "PackageUseDevtools: Yes" + echo "PackagePath: BFE_RShiny/oasisui" + echo "PackageInstallArgs: --no-multiarch --with-keep.source" +} > "$R_PROJECT_FILE" + + if [ -s "$R_PROJECT_FILE" ]; then + echo "R project file created successfully with content." + else + echo "R project file is empty or could not be created." + return 1 # Exit the function with an error status + fi + + echo "Building UI Docker image..." + docker build -f "$DOCKERFILE_PATH" -t "$IMAGE_NAME" . +} + + + + + +function stop() { + echo "Stopping and removing containers..." + docker-compose -f "$COMPOSE_FILE" down +} + +function start() { + echo "Starting services..." + docker-compose -f "$COMPOSE_FILE" up -d + echo "connect using: http://localhost:8787/" +} + +# Check and create the R project file if it doesn't exist. +if [ ! -f "$R_PROJECT_FILE" ]; then + cat >> "$R_PROJECT_FILE" <