Merge pull request #383 from StoXProject/develop

Develop

DESCRIPTION

@@ -1,6 +1,6 @@
 Package: RstoxData
-Version: 2.0.1-9001
-Date: 2024-08-31
+Version: 2.0.1-9002
+Date: 2024-09-17
 Title: Tools to Read and Manipulate Fisheries Data
 Authors@R: c(
   person(given = "Edvin", 

NEWS.md

@@ -1,3 +1,14 @@
+# RstoxData v2.0.1-9002  (2024-09-16)
+* Added documentation of PreySpeciesCategory and PreySample in StoxBiotic.
+* Fixed bug where PreyCatchfractionWeight = 0 was set to NA if PreyCatchFractionWeightResolution was missing. Now returning 0.
+* Fixed inaccuracies in the documentation of the StoxBiotic format.
+* Added warning when there are missing values in keys in StoxBiotic.
+
+
+# RstoxData v2.0.1-9001  (2024-09-01)
+* Added tables PreySpeciesCategory and PreySample in StoxBiotic, and prepared for adding PreyIndividual. 
+* Fixed bug where reports could be run even though the Baseline model had been rerun."
+
 # RstoxData v2.0.0  (2024-07-09)
 * Final release for StoX 4.0.0.
 

R/StoxBiotic.R

@@ -16,6 +16,12 @@ StoxBiotic <- function(BioticData) {
 	# Extract the StoxBiotic data and rbind across files:
 	StoxBioticData <- GeneralSamplingHierarchy2StoxBiotic(GeneralSamplingHierarchy, NumberOfCores = 1L)
 
+	# Warn if there are keys with missing values:
+	warningMissingKeys(
+		StoxData = StoxBioticData, 
+		stoxDataFormat = "Biotic"
+	)
+
 	# Remove rows of duplicated keys:
 	#StoxBioticData <- removeRowsOfDuplicatedKeysFromStoxBioticData(StoxBioticData)
 	StoxBioticData <- removeRowsOfDuplicatedKeys(

R/Utilities.R

@@ -434,6 +434,25 @@ removeRowsOfDuplicatedKeys <- function(StoxData, stoxDataFormat = c("Biotic", "A
 
 
 
+# Function to remove rows with duplicated keys in StoxBioticData:
+#' @importFrom data.table .I
+warningMissingKeys <- function(StoxData, stoxDataFormat = c("Biotic", "Acoustic")) {
+
+	stoxDataFormat <- match_arg_informative(stoxDataFormat)
+	StoxKeys <- getRstoxDataDefinitions(paste0("Stox", stoxDataFormat, "Keys"))
+
+	presentKeys <- lapply(StoxData, function(x) intersect(names(x), StoxKeys))
+	atMaxNumberOfKeys <- which.max(lengths(presentKeys))
+	hasmissingKeys <- StoxData[[atMaxNumberOfKeys]][, any(unlist(lapply(.SD, is.na))), .SDcols = presentKeys[[atMaxNumberOfKeys]]]
+
+	# Warn if any keys have missing values:
+	if(any(hasmissingKeys)) {
+		warning("The Stox", stoxDataFormat, "Data has missing keys! Please translate fields in the Read", stoxDataFormat, "process to avoid this. ")
+	}
+}
+
+
+
 AddToStoxData <- function(
 	StoxData, 
 	RawData, 
@@ -989,6 +1008,7 @@ deparse_onestring <- function(...) {
 #' 
 applyFunctionArgumentHierarchy <- function(functionArgumentHierarchy, functionArguments, return.only.names = TRUE) {
 
+
 	# Support an expression at the top level:
 	if(inherits(functionArgumentHierarchy, "expression")) {
 		functionArgumentHierarchy <- eval(functionArgumentHierarchy)
@@ -1031,6 +1051,7 @@ applyFunctionArgumentHierarchy <- function(functionArgumentHierarchy, functionAr
 		}
 	}
 
+
 	# Return only the names of the arguments to show:
 	if(return.only.names) {
 		toShow <- names(toShow)[toShow]

R/data.R

@@ -123,7 +123,7 @@ NULL
 #' \bold{SpeciesCategory level}:
 #' \tabular{lllllll}{
 #' \bold{Variable} \tab \bold{Description} \tab \bold{Unit} \tab \bold{Data type} \tab \bold{Example} \tab \bold{NMDBiotic} \tab \bold{ICESBiotic} \cr
-#' SpeciesCategoryKey \tab Key of the SpeciesCategory level \tab None \tab Character \tab "126417" \tab \code{commonname/catchcategory/aphia/scientificname} (\code{noname/species/aphia/group} for NMDBiotic 1.1 and 1.4) \tab \code{SpeciesCode} \cr
+#' SpeciesCategoryKey \tab Key of the SpeciesCategory level \tab None \tab Character \tab "sild'G03/161722.G03/126417/Clupea harengus" \tab \code{commonname/catchcategory/aphia/scientificname} (\code{noname/species/aphia/group} for NMDBiotic 1.1 and 1.4) \tab \code{SpeciesCode} \cr
 #' SpeciesCategory \tab The species category \tab None \tab Character \tab "Herring" \tab Same as SpeciesCategoryKey. Can be translated. \tab Same as SpeciesCategoryKey. Can be translated. \cr
 #' }
 #' 
@@ -132,9 +132,9 @@ NULL
 #' \tabular{lllllll}{
 #' \bold{Variable} \tab \bold{Description} \tab \bold{Unit} \tab \bold{Data type} \tab \bold{Example} \tab \bold{NMDBiotic} \tab \bold{ICESBiotic} \cr
 #' SampleKey \tab Key of the Sample level \tab None \tab Character \tab "1" \tab \code{catchsampleid} (\code{samplenumber} for NMDBiotic 1.1 and 1.4) \tab \code{SpeciesCategory} \cr          
-#' Sample \tab Unique Sample identifier \tab None \tab Character \tab "2021105-1-2-126417-1" \tab \code{CruiseKey-StationKey-HaulKey-SpeciesCategoryKey-SampleKey} \tab \code{CruiseKey-StationKey-HaulKey-SpeciesCategoryKey-SampleKey} \cr             
-#' CatchFractionWeight \tab Total weight of the catch SpeciesCategory and sub category (fractions such as juveniles and adults) \tab kg \tab Numeric \tab 49.9 \tab \code{catchweight} (missing (\code{NA}) if \code{catchproducttype != 1}) (\code{weight} (missing (\code{NA}) if \code{producttype != 1}) for NMDBiotic 1.1 and 1.4) \tab \code{SpeciesCategoryWeight} (taking \code{WeightUnit} into account) \cr
-#' CatchFractionNumber \tab Total number of individuals of the catch SpeciesCategory and sub category (fractions such as juveniles and adults) \tab individuals \tab Integer \tab 295 \tab \code{catchcount} (\code{count} for NMDBiotic 1.1 and 1.4) \tab \code{SpeciesCategoryNumber} \cr 
+#' Sample \tab Unique Sample identifier \tab None \tab Character \tab "2021105-1-2-sild'G03/161722.G03/126417/Clupea harengus-1" \tab \code{CruiseKey-StationKey-HaulKey-SpeciesCategoryKey-SampleKey} \tab \code{CruiseKey-StationKey-HaulKey-SpeciesCategoryKey-SampleKey} \cr             
+#' CatchFractionWeight \tab Total weight of the catch for the SpeciesCategory and sub category (e.g., fractions such as juveniles and adults) \tab kg \tab Numeric \tab 49.9 \tab \code{catchweight} (missing (\code{NA}) if \code{catchproducttype != 1}) (\code{weight} (missing (\code{NA}) if \code{producttype != 1}) for NMDBiotic 1.1 and 1.4) \tab \code{SpeciesCategoryWeight} (taking \code{WeightUnit} into account) \cr
+#' CatchFractionNumber \tab Total number of individuals of the catch for the SpeciesCategory and sub category (e.g., fractions such as juveniles and adults) \tab individuals \tab Integer \tab 295 \tab \code{catchcount} (\code{count} for NMDBiotic 1.1 and 1.4) \tab \code{SpeciesCategoryNumber} \cr 
 #' SampleWeight \tab Total weight of the sample for individual measurements \tab kg \tab Numeric \tab 4.6 \tab \code{lengthsampleweight} (missing (\code{NA}) if \code{sampleproducttype != 1}) \tab \code{SubsampleWeight} (taking \code{WeightUnit} into account) \cr       
 #' SampleNumber \tab Size of the sample for individual measurements \tab individuals \tab Integer \tab 100 \tab \code{lengthsamplecount} \tab \code{SubsampledNumber} \cr
 #' }
@@ -144,7 +144,7 @@ NULL
 #' \tabular{lllllll}{
 #' \bold{Variable} \tab \bold{Description} \tab \bold{Unit} \tab \bold{Data type} \tab \bold{Example} \tab \bold{NMDBiotic} \tab \bold{ICESBiotic} \cr
 #' IndividualKey \tab Key of the Individual level \tab None \tab Character \tab "2" \tab \code{specimenid} (\code{specimenno} for NMDBiotic 1.1 and 1.4) \tab \code{FishID} \cr        
-#' Individual \tab Unique Individual identifier \tab None \tab Character \tab "2021105-1-2-126417-1-2" \tab \code{CruiseKey-StationKey-HaulKey-SpeciesCategoryKey-SampleKey-IndividualKey} \tab \code{CruiseKey-StationKey-HaulKey-SpeciesCategoryKey-SampleKey-IndividualKey} \cr           
+#' Individual \tab Unique Individual identifier \tab None \tab Character \tab "2021105-1-2-sild'G03/161722.G03/126417/Clupea harengus-1-2" \tab \code{CruiseKey-StationKey-HaulKey-SpeciesCategoryKey-SampleKey-IndividualKey} \tab \code{CruiseKey-StationKey-HaulKey-SpeciesCategoryKey-SampleKey-IndividualKey} \cr           
 #' IndividualRoundWeight \tab Round weight (the whole fish) fo the individual \tab g \tab Numeric \tab 123 \tab \code{individualweight * 1000} (missing (\code{NA}) if \code{individualproducttype != 1}) (\code{weight.individual} (missing (\code{NA}) if \code{producttype.individual != 1}) for NMDBiotic 1.1 and 1.4) \tab \code{IndividualWeight} (taking \code{WeightUnit.Biology} into account) \cr
 #' IndividualTotalLength \tab Total length (from snout to end of fin), given as the lower end of the interval of width given by LengthResolution \tab cm \tab Numeric \tab 14.5 \tab \code{length * 100}  (missing (\code{NA}) if \code{lengthmeasurement != "E"}) \tab \code{LengthClass.Biology} (taking \code{LengthCode.Biology} into account) \cr
 #' LengthResolution \tab Resolution of IndividualTotalLength \tab cm \tab Numeric \tab 0.5 \tab \code{lengthresolution} (\code{lengthunit} for NMDBiotic 1.1 and 1.4) converted to cm \tab \code{LengthCode.Biology} converted to cm \cr     
@@ -153,6 +153,26 @@ NULL
 #' IndividualSex \tab sex of an individual \tab F is female, M is male \tab Character \tab "F" \tab "F" if \code{sex = 1}, "M" if \code{sex = 2} \tab \code{IndividualSex} \cr
 #' }
 #' 
+#' 
+#' \bold{PreySpeciesCategory level}:
+#' \tabular{lllllll}{
+#' \bold{Variable} \tab \bold{Description} \tab \bold{Unit} \tab \bold{Data type} \tab \bold{Example} \tab \bold{NMDBiotic} \tab \bold{ICESBiotic} \cr
+#' PreySpeciesCategoryKey \tab Key of the PreySpeciesCategory level \tab None \tab Character \tab "85258" \tab \code{preycategory} (missing (NA) for NMDBiotic 1.1 and 1.4) \tab Missing (NA) \cr
+#' PreySpeciesCategory \tab The species category \tab None \tab Character \tab "Calanoida" \tab Same as PreySpeciesCategoryKey. Can be translated. \tab Missing (NA) \cr
+#' }
+#' 
+#' 
+#' \bold{PreySample level}:
+#' \tabular{lllllll}{
+#' \bold{Variable} \tab \bold{Description} \tab \bold{Unit} \tab \bold{Data type} \tab \bold{Example} \tab \bold{NMDBiotic} \tab \bold{ICESBiotic} \cr
+#' PreySampleKey \tab Key of the PreySample level \tab None \tab Character \tab "1" \tab \code{preysampleid} (missing (NA) for NMDBiotic 1.1 and 1.4) \tab Missing (NA) \cr          
+#' PreySample \tab Unique PreySample identifier \tab None \tab Character \tab "2021105-1-2-sild'G03/161722.G03/126417/Clupea harengus-1-2-85258-1" \tab \code{CruiseKey-StationKey-HaulKey-SpeciesCategoryKey-SampleKey-IndividualKey-PreySpeciesCategoryKey-PreySampleKey} (missing (NA) for NMDBiotic 1.1 and 1.4) \tab Missing (NA) \cr
+#' PreyCatchFractionWeight \tab Total weight of the catch for the PreySpeciesCategory and sub category (e.g., fractions for different digestion) \tab mg \tab Numeric \tab 12.3 \tab \code{totalweight} (taking \code{LengthCode.Biology} into account) (missing (NA) for NMDBiotic 1.1 and 1.4) \tab Missing (NA) \cr
+#' PreyCatchFractionWeightResolution \tab Resolution of PreyCatchFractionWeight \tab mg \tab Numeric \tab 0.001 \tab \code{weightresolution} converted to mg (missing (NA) for NMDBiotic 1.1 and 1.4) \tab Missing (NA) \cr
+#' PreyCatchFractionNumber \tab Total number of individual prey of the PreySpeciesCategory and sub category (e.g., fractions for different digestion) \tab individuals \tab Integer \tab 4 \tab \code{totalcount} (missing (NA) for NMDBiotic 1.1 and 1.4) \tab Missing (NA) \cr     
+#' }
+#' 
+#' 
 #' @name StoxBioticFormat
 #' 
 NULL

data-raw/processBioticData.R

@@ -152,9 +152,7 @@ convertWeightRes_NMDBiotic <- function(x, scale) {
 	# Change this to use the NMD-R-packages in the future!!!!!!!!!!
 
 	# This is a hard coded copy of the reference data https://referenceeditor.hi.no/apps/referenceeditor/v2/tables/weightresolution:
-	# Multiply by 100 to give these in cm: 
 	z <- c(1, 1e-3, 1e-6, 1e-9) * scale
-	# The letters here are deprecated, but used in historical data:
 	names(z) <- as.character(1:4)
 	return(z[x])
 }
@@ -257,9 +255,9 @@ getIndividualTotalLength_NMDBiotic1 <- function(length, lengthmeasurement, speci
 }
 
 
-getPreyCatchFractionWeight_NMDBiotic3 <- function(totalweight, PreyCatchFractionWeightResolution){
-	# Multiply by 1e6 since the weight of prey is given in mg in StoxBiotic (while kg in NMDBiotic), and divide by the resolution, which is also given in mg. Thus, if the reolution is g we divide by 1000, which seems reasonable:
-	totalweight * 1e6 / PreyCatchFractionWeightResolution
+getPreyCatchFractionWeight_NMDBiotic3 <- function(totalweight){
+	# Multiply by 1e6 since the weight of prey is given in mg in StoxBiotic (while kg in NMDBiotic):
+	totalweight * 1e6
 
 }
 

data-raw/stox-biotic-final-phase.csv

@@ -58,8 +58,8 @@ HaulKey,PreySpeciesCategory,HaulKey,HaulKey,HaulKey,HaulKey,HaulKey,Y,character
 SpeciesCategoryKey,PreySpeciesCategory,SpeciesCategoryKey,SpeciesCategoryKey,SpeciesCategoryKey,SpeciesCategoryKey,SpeciesCategoryKey,Y,character
 SampleKey,PreySpeciesCategory,SampleKey,SampleKey,SampleKey,SampleKey,SampleKey,Y,character
 IndividualKey,PreySpeciesCategory,IndividualKey,IndividualKey,IndividualKey,IndividualKey,IndividualKey,Y,character
-PreySpeciesCategoryKey,PreySpeciesCategory,PreySpeciesCategoryKey,PreySpeciesCategoryKey,PreySpeciesCategoryKey,PreySpeciesCategoryKey,PreySpeciesCategoryKey,Y,character
-PreySpeciesCategory,PreySpeciesCategory,PreySpeciesCategoryKey,PreySpeciesCategoryKey,PreySpeciesCategoryKey,PreySpeciesCategoryKey,PreySpeciesCategoryKey,,character
+PreySpeciesCategoryKey,PreySpeciesCategory,PreySpeciesCategoryKey,PreySpeciesCategoryKey,NA,NA,NA,Y,character
+PreySpeciesCategory,PreySpeciesCategory,PreySpeciesCategoryKey,PreySpeciesCategoryKey,NA,NA,NA,Y,character
 CruiseKey,PreySample,CruiseKey,CruiseKey,CruiseKey,CruiseKey,CruiseKey,Y,character
 StationKey,PreySample,StationKey,StationKey,StationKey,StationKey,StationKey,Y,character
 HaulKey,PreySample,HaulKey,HaulKey,HaulKey,HaulKey,HaulKey,Y,character
@@ -70,9 +70,9 @@ PreySpeciesCategoryKey,PreySample,PreySpeciesCategoryKey,PreySpeciesCategoryKey,
 PreySampleKey,PreySample,PreySampleKey,PreySampleKey,NA,NA,NA,Y,character
 PreySample,PreySample,"paste(CruiseKey, StationKey, HaulKey, SpeciesCategoryKey, SampleKey, IndividualKey, PreySpeciesCategoryKey, PreySampleKey, sep = '-')","paste(CruiseKey, StationKey, HaulKey, SpeciesCategoryKey, SampleKey, IndividualKey, PreySpeciesCategoryKey, PreySampleKey, sep = '-')",NA,NA,NA,,character
 # The scale = 1e6 corresponds to the unit of the PreySampleWeight, which is defined as mg here:
-PreyCatchFractionNumber,PreySample,totalcount,totalcount,NA,NA,NA,,numeric
+PreyCatchFractionWeight,PreySample,"getPreyCatchFractionWeight(totalweight)","getPreyCatchFractionWeight(totalweight)",NA,NA,NA,,numeric
 PreyCatchFractionWeightResolution,PreySample,"convertWeightRes(weightresolution, scale = 1e6)","convertWeightRes(weightresolution, scale = 1e6)",NA,NA,NA,,character
-PreyCatchFractionWeight,PreySample,"getPreyCatchFractionWeight(totalweight, PreyCatchFractionWeightResolution)","getPreyCatchFractionWeight(totalweight, PreyCatchFractionWeightResolution)",NA,NA,NA,,numeric
+PreyCatchFractionNumber,PreySample,totalcount,totalcount,NA,NA,NA,,numeric
 #CruiseKey,PreyIndividual,CruiseKey,CruiseKey,CruiseKey,CruiseKey,CruiseKey,Y,character
 #StationKey,PreyIndividual,StationKey,StationKey,StationKey,StationKey,StationKey,Y,character
 #HaulKey,PreyIndividual,HaulKey,HaulKey,HaulKey,HaulKey,HaulKey,Y,character