Develop

.github/workflows/check-full.yaml

@@ -2,39 +2,41 @@ on:
   push:
     branches:
       - master
+      - testing
       - develop
   pull_request:
     branches:
       - master
+      - testing
       - develop
 
 name: R-CMD-check
 
 jobs:
   R-CMD-check:
+
+    # 0. Define the systems to run on:
     runs-on: ${{ matrix.config.os }}
 
     name: ${{ matrix.config.os }} (${{ matrix.config.r }})
 
-    # We include 'release' in order to always test the altest release, but mac and Windows binaries are not deployed to the drat repo:
     strategy:
       fail-fast: false
       matrix:
         config:
           - {os: macOS-latest,   r: 'release', pkgext: '.tgz'}
           - {os: macOS-latest,   r: '4.3',     pkgext: '.tgz'}
-          - {os: macOS-latest,   r: '4.3',     pkgext: '.tgz'}
           - {os: macOS-latest,   r: '4.2',     pkgext: '.tgz'}
           - {os: macOS-latest,   r: '4.1',     pkgext: '.tgz'}
-          - {os: macOS-latest,   r: '4.0',     pkgext: '.tgz'}
+          # - {os: macOS-latest,   r: '4.0',     pkgext: '.tgz'} # Failing in Install system dependencies on macOS
           # Disccarded due to the following missing packages on Mac for R 3.6: tinytest, Rcpp, xml2, units, stringi, data.table:
           #- {os: macOS-latest,   r: '3.6',     pkgext: '.tgz'}
           - {os: windows-latest, r: 'release', pkgext: '.zip'}
           - {os: windows-latest, r: '4.3',     pkgext: '.zip'}
           - {os: windows-latest, r: '4.2',     pkgext: '.zip'}
-          - {os: windows-latest, r: '4.1',     pkgext: '.zip'}
-          - {os: windows-latest, r: '4.0',     pkgext: '.zip',    rspm: "https://cloud.r-project.org"}
-          - {os: windows-latest, r: '3.6',     pkgext: '.zip',    rspm: "https://cloud.r-project.org"}
+          # - {os: windows-latest, r: '4.1',     pkgext: '.zip'} # Disabled due to stringi failure
+          # - {os: windows-latest, r: '4.0',     pkgext: '.zip',    rspm: "https://cloud.r-project.org"} # Disabled due to stringi failure
+          # - {os: windows-latest, r: '3.6',     pkgext: '.zip',    rspm: "https://cloud.r-project.org"} # Disabled due to stringi failure
           - {os: ubuntu-latest,  r: 'release', pkgext: '.tar.gz', rspm: "https://cloud.r-project.org"}
           - {os: ubuntu-20.04,   r: '4.3',     pkgext: '.tar.gz', rspm: "https://cloud.r-project.org"}
           - {os: ubuntu-20.04,   r: '4.2',     pkgext: '.tar.gz', rspm: "https://cloud.r-project.org"}
@@ -48,11 +50,20 @@ jobs:
       GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
 
     steps:
+
+      ###################
+      #### 1. Setup: ####
+      ###################
+
       - uses: actions/checkout@v3
 
-      - name: Do autorelease increment, tagging, and environment setup
-        if: github.event_name == 'push' && github.ref == 'refs/heads/master'
+      - name: Prepare for deployment at push
+        if: github.event_name == 'push'
         run: |
+          curl "https://raw.githubusercontent.com/StoXProject/unstableRepo/main/scripts/checktag-actions.sh" --output unstableRepo_checktag.sh
+          curl "https://raw.githubusercontent.com/StoXProject/unstableRepo/main/scripts/deploy-actions.sh" --output unstableRepo_deploy.sh
+          chmod +x ./unstableRepo_checktag.sh ./unstableRepo_deploy.sh
+          . unstableRepo_checktag.sh
           curl "https://raw.githubusercontent.com/StoXProject/testingRepo/main/scripts/checktag-actions.sh" --output testingRepo_checktag.sh
           curl "https://raw.githubusercontent.com/StoXProject/testingRepo/main/scripts/deploy-actions.sh" --output testingRepo_deploy.sh
           chmod +x ./testingRepo_checktag.sh ./testingRepo_deploy.sh
@@ -75,6 +86,13 @@ jobs:
 
       - uses: r-lib/actions/setup-pandoc@v2
 
+
+
+      #############################################################################
+      #### 2. R package installation (with system and R package dependencies): ####
+      #############################################################################
+
+      ## 2a. Cache R packages:
       - name: Query R package dependencies
         run: |
           install.packages('remotes')
@@ -90,34 +108,37 @@ jobs:
           key: ${{ env.cache-version }}-${{ runner.os }}-r-${{ matrix.config.r }}-${{ hashFiles('depends.Rds') }}
           restore-keys: ${{ env.cache-version }}-${{ runner.os }}-r-${{ matrix.config.r }}-
 
+
+      ## 2b. System dependencies:
+      - name: Install system dependencies on macOS
+        if: runner.os == 'macOS'
+        run: |
+          # gfortran may not exist in a period after a new R minor release:
+          gfortranFile='/usr/local/bin/gfortran'
+          if [ -f "$gfortranFile" ] ; then
+            rm "$gfortranFile"
+          fi
+          rm '/usr/local/bin/2to3'
+          rm '/usr/local/bin/2to3-3.11'
+          rm '/usr/local/bin/idle3.11'
+          rm '/usr/local/bin/idle3'
+          rm '/usr/local/bin/pydoc3'
+          rm '/usr/local/bin/python3'
+          rm '/usr/local/bin/python3-config'
+          rm '/usr/local/bin/pydoc3.11'
+          rm '/usr/local/bin/python3.11'
+          rm '/usr/local/bin/python3.11-config'
+          export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=false # This prevents Homebrew from re-installing R, which will lead to .Platform$pkgType = "source".
+          brew install gdal udunits
+        shell: bash
+
       - name: Install system dependencies on Linux
         if: runner.os == 'Linux'
         run: |
           while read -r cmd
           do
             eval sudo $cmd
-          done < <(Rscript -e 'writeLines(remotes::system_requirements("ubuntu", "18.04"))')
-
-      - name: Install R package dependencies on Linux
-        if: runner.os == 'Linux'
-        run: |
-          remotes::install_deps(dependencies = TRUE)
-        shell: Rscript {0}
-
-      - name: Install R package dependencies on Windows and macOS
-        if: runner.os != 'Linux'
-        run: |
-          # Show the repos:
-          getOption("repos")
-
-          # Make sure Rcpp is >= 1.0.10 due to deprecation of sprintf in c++:
-          install.packages("Rcpp")
-
-          # Install the rest of the dependencies:
-          options(install.packages.compile.from.source = "never") # Avoid installing sources which need compilation
-          # dependencies = TRUE installs "Depends", "Imports", "LinkingTo" and "Suggests", and the  "Depends", "Imports" and "LinkingTo" recursively from those dependencies
-          remotes::install_deps(dependencies = TRUE, type = "both")
-        shell: Rscript {0}
+          done < <(Rscript -e 'writeLines(remotes::system_requirements("ubuntu", "20.04"))')
 
       - name: Prepare for CRANdependencies on Linux
         if: runner.os == 'Linux'
@@ -126,24 +147,52 @@ jobs:
           sudo apt-get install -y libudunits2-dev
         shell: bash
 
+
+      ## 2c. R packages needed for this yaml script (not related to the Rstox package itself):
       - name: Install CRANdependencies
         run: |
-          remotes::install_cran("rcmdcheck")
-          remotes::install_cran("git2r")
-          remotes::install_cran("sessioninfo")
-          #remotes::install_cran("drat")
-          # Using the modfied drat that supports big sur:
-          #remotes::install_github(repo = "stoxproject/drat", dependencies = FALSE)
+          install.packages(c("rcmdcheck", "git2r", "sessioninfo", "knitr"))
+
+          # Using the latest drat that supports big sur (after inputs from us). 
+          # When the next drat appears on CRAN this can be changed to install from CRAN:
           remotes::install_github(repo = 'eddelbuettel/drat', dependencies = FALSE)
         shell: Rscript {0}
 
+
+      ## 2d. Install dependencies:
+      - name: Install R package dependencies
+        run: |
+          remotes::install_deps(dependencies = TRUE, repos = "https://cloud.r-project.org/")
+        shell: Rscript {0}
+
+
+      ## 2e. Session info:
       - name: Session info
         run: |
           options(width = 100)
           pkgs <- installed.packages()[, "Package"]
           sessioninfo::session_info(pkgs, include_base = TRUE)
         shell: Rscript {0}
 
+
+      ## 2f. Additional installs to secure specific versions to patch incompatibilities with older R versions:
+      - name: Additional installs to secure specific versions
+        run: |
+          pkgs <- installed.packages()[, "Package"]
+          s <-  sessioninfo::session_info(pkgs, include_base = TRUE)
+          onlyStringi <- subset(s$packages, package == "stringi")
+          if("stringi"  %in% as.data.frame(s$packages)$package && is.na(subset(s$packages, package == "stringi")$ondiskversion)) {
+          	remotes::install_github("gagolews/stringi")
+          }
+        shell: Rscript {0}
+
+
+
+
+      #####################################
+      #### 3. Check the Rstox package: ####
+      #####################################
+
       - name: Check
         env:
           _R_CHECK_CRAN_INCOMING_: false
@@ -157,15 +206,21 @@ jobs:
           name: ${{ runner.os }}-r${{ matrix.config.r }}-results
           path: check
 
-      - name: Build package source archive
-        if: runner.os == 'Linux' && matrix.config.r == 'release' && github.event_name == 'push' && github.ref == 'refs/heads/master'
+
+
+      #######################################################
+      #### 4. Build the package source and binary files: ####
+      #######################################################
+
+      - name: Build package source archive from branches testing and master
+        if: runner.os == 'Linux' && matrix.config.r == 'release' && github.event_name == 'push' && (github.ref_name == 'master' || github.ref_name == 'testing' || github.ref_name == 'develop')
         run: |
           pkgbuild::build(".", dest_path = ".", binary = FALSE)
           system(paste0("PKG_FREL=", Sys.getenv("PKG_FILE"), " >> ", Sys.getenv("GITHUB_ENV")))
         shell: Rscript {0}
 
-      - name: Build package binary archive
-        if: (runner.os == 'Windows' || runner.os == 'macOS') && github.event_name == 'push' && github.ref == 'refs/heads/master'
+      - name: Build package binary archive  from branches testing and master for Windows and macOS (this builds also on matrix.config.r = release, which can or can not be a duplicate)
+        if: (runner.os == 'Windows' || runner.os == 'macOS') && github.event_name == 'push' && (github.ref_name == 'master' || github.ref_name == 'testing' || github.ref_name == 'develop')
         run: |
           pkgbuild::build(".", dest_path = ".", binary = TRUE)
           print("PKG_FILE")
@@ -175,31 +230,65 @@ jobs:
           file.copy(Sys.getenv("PKG_FILE"), Sys.getenv("PKG_FREL"))
         shell: Rscript {0}
 
-      # We include 'release' in order to always test the latest release, but (mac and Windows) binaries are not deployed to the drat repo:
-      - name: Upload to DRAT repo
-        if: ( (runner.os == 'Windows' && matrix.config.r != 'release') || (runner.os == 'macOS' && matrix.config.r != 'release') || (runner.os == 'Linux' && matrix.config.r == 'release') ) && github.event_name == 'push' && github.ref == 'refs/heads/master'
+
+
+      ######################################
+      #### 5. Deploy to the drat repos: ####
+      ######################################
+
+      ## 5a. Upload to the official (DRAT) repo, but only from the master branch. Here we skip the matrix.config.r = release, as a new StoX release should be ready shortly after each R minor release and listed specifically in the matrix.config:
+      - name: Upload to the official (DRAT) repo
+        if: ( (runner.os == 'Windows' && matrix.config.r != 'release') || (runner.os == 'macOS' && matrix.config.r != 'release') || (runner.os == 'Linux' && matrix.config.r == 'release') ) && github.event_name == 'push' && github.ref_name == 'master'
         env:
           BUILD_NUMBER: ${{ github.sha  }}
           DRAT_DEPLOY_TOKEN: ${{ secrets.DRAT_DEPLOY_TOKEN }}
         run: . repo_deploy.sh
         shell: bash
 
-      # We include 'release' in order to always test the latest release, but (mac and Windows) binaries are not deployed to the drat repo:
-      - name: Upload to DRAT testingRepo
-        if: ( (runner.os == 'Windows' && matrix.config.r != 'release') || (runner.os == 'macOS' && matrix.config.r != 'release') || (runner.os == 'Linux' && matrix.config.r == 'release') ) && github.event_name == 'push' && github.ref == 'refs/heads/master'
+
+      ## 5b. Upload to the (DRAT) testingRepo, but only from the testing branch. Here we skip the matrix.config.r = release, as a new StoX release should be ready shortly after each R minor release and listed specifically in the matrix.config:
+      - name: Upload to the testing (DRAT) repo
+        if: ( (runner.os == 'Windows' && matrix.config.r != 'release') || (runner.os == 'macOS' && matrix.config.r != 'release') || (runner.os == 'Linux' && matrix.config.r == 'release') ) && github.event_name == 'push' && github.ref_name == 'testing'
         env:
           BUILD_NUMBER: ${{ github.sha  }}
           DRAT_DEPLOY_TOKEN: ${{ secrets.DRAT_DEPLOY_TOKEN }}
         run: . testingRepo_deploy.sh
         shell: bash
 
-      - name: Create release and upload files
-        if: ( (runner.os == 'Windows' && matrix.config.r != 'release') || (runner.os == 'macOS' && matrix.config.r != 'release') || (runner.os == 'Linux' && matrix.config.r == 'release') ) && github.event_name == 'push' && github.ref == 'refs/heads/master'
+
+      ## 5c. Upload to the (DRAT) unstableRepo, but only from the develop branch. Here we skip the matrix.config.r = release, as a new StoX release should be ready shortly after each R minor release and listed specifically in the matrix.config:
+      - name: Upload to the unstable (DRAT) repo
+        if: ( (runner.os == 'Windows' && matrix.config.r != 'release') || (runner.os == 'macOS' && matrix.config.r != 'release') || (runner.os == 'Linux' && matrix.config.r == 'release') ) && github.event_name == 'push' && github.ref_name == 'develop'
+        env:
+          BUILD_NUMBER: ${{ github.sha  }}
+          DRAT_DEPLOY_TOKEN: ${{ secrets.DRAT_DEPLOY_TOKEN }}
+        run: . unstableRepo_deploy.sh
+        shell: bash
+
+
+
+      ###############################
+      #### 6. Release on GitHub: ####
+      ###############################
+
+      - name: Create release on GitHub and upload files for testing branch
+        if: ( (runner.os == 'Windows' && matrix.config.r != 'release') || (runner.os == 'macOS' && matrix.config.r != 'release') || (runner.os == 'Linux' && matrix.config.r == 'release') ) && github.event_name == 'push' && github.ref_name == 'testing'
         uses: ncipollo/release-action@v1
         with:
           allowUpdates: true
-          commit: master
+          commit: testing
           tag: ${{ env.FINAL_TAG }}
           name: ${{ env.FINAL_TAG }}
           artifacts: ${{ env.PKG_REPO }}/${{ env.PKG_FREL }}
           token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Create release on GitHub and upload files for master branch
+        if: ( (runner.os == 'Windows' && matrix.config.r != 'release') || (runner.os == 'macOS' && matrix.config.r != 'release') || (runner.os == 'Linux' && matrix.config.r == 'release') ) && github.event_name == 'push' && github.ref_name == 'master'
+        uses: ncipollo/release-action@v1
+        with:
+          allowUpdates: true
+          commit: master
+          tag: ${{ env.FINAL_TAG }}
+          name: ${{ env.FINAL_TAG }}
+          artifacts: ${{ env.PKG_REPO }}/${{ env.PKG_FREL }}
+          token: ${{ secrets.GITHUB_TOKEN }}

DESCRIPTION

@@ -1,6 +1,6 @@
 Package: RstoxData
-Version: 1.10.2-9001
-Date: 2023-10-13
+Version: 1.10.2-9003
+Date: 2023-11-23
 Title: Tools to Read and Manipulate Fisheries Data
 Authors@R: c(
   person(given = "Edvin", 
@@ -36,13 +36,17 @@ BugReports: https://github.com/StoXProject/RstoxData/issues
 License: LGPL-3
 LazyData: true
 Encoding: UTF-8
+VignetteBuilder: knitr
 Imports: 
     data.table (>= 1.12.0),
     Rcpp (>= 1.0.0),
     stringi (>= 1.4.0),
     units (>= 0.7),
     xml2 (>= 1.2.0)
 Suggests: 
+    httr,
+    knitr,
+    rmarkdown,
     tinytest (>= 1.2.0)
 LinkingTo: 
     Rcpp

NAMESPACE

@@ -11,6 +11,7 @@ export(CopyBiotic)
 export(CopyICESAcoustic)
 export(CopyICESBiotic)
 export(CopyICESDatras)
+export(CopyICESDatsusc)
 export(CopyLanding)
 export(CopyStoxAcoustic)
 export(CopyStoxBiotic)
@@ -21,16 +22,19 @@ export(FilterBiotic)
 export(FilterICESAcoustic)
 export(FilterICESBiotic)
 export(FilterICESDatras)
+export(FilterICESDatsusc)
 export(FilterLanding)
 export(FilterStoxAcoustic)
 export(FilterStoxBiotic)
 export(FilterStoxLanding)
 export(ICESAcoustic)
 export(ICESBiotic)
 export(ICESDatras)
+export(ICESDatsusc)
 export(MergeAcoustic)
 export(MergeStoxAcoustic)
 export(MergeStoxBiotic)
+export(PrepareNmdBioticTable)
 export(ReadAcoustic)
 export(ReadBiotic)
 export(ReadLanding)
@@ -44,6 +48,7 @@ export(TranslateBiotic)
 export(TranslateICESAcoustic)
 export(TranslateICESBiotic)
 export(TranslateICESDatras)
+export(TranslateICESDatsusc)
 export(TranslateLanding)
 export(TranslateStoxAcoustic)
 export(TranslateStoxBiotic)

NEWS.md

@@ -1,3 +1,12 @@
+# RstoxData v1.10.2-9002  (2023-11-06)
+* Added parameter format class "folderPath".
+* Added branch "testing" which will deploy to the testingRepo.
+* Structured the check-full.yaml.
+* Fixed issue with parsing some valid variants of the xml landings format (landingerv2)
+* Fixed issue with parsing some valid variants of the InterCatch Exchange format
+* Added functions for extracting some standard tables from NMD biotic
+* Added vignette for guiding data users at IMR to locate and parse biological samples.
+
 # RstoxData v1.10.2-9001  (2023-10-13)
 * Moved functions for setting precision to RstoxFramework, which actually assumed an sp object. 
 * Changed definition of CatchFractionNumber and SampleNumber to numeric instead of integer, as this was the result from runProcess() used in runModel().