Merge branch 'release/0.7' into main

Author: pkiraly

README.md

@@ -93,10 +93,207 @@ while (iterator.hasNext()) {
 String metrics = output.toString();
 ```
 
+## Defining schema with a configuration file
+
+It is possible to define the schema with a YAML or JSON 
+configuration file.
+
+```
+Schema schema = ConfigurationReader
+  .readYaml("path/to/some/configuration.yaml")
+  .asSchema();
+```
+
+A YAML example
+```yaml
+format: json
+fields:
+  - name: edm:ProvidedCHO/@about
+    path:  $.['providedCHOs'][0]['about']
+    categories:
+      - MANDATORY
+  - name: Proxy/dc:title
+    path: $.['proxies'][?(@['europeanaProxy'] == false)]['dcTitle']
+    categories:
+      - DESCRIPTIVENESS
+      - SEARCHABILITY
+      - IDENTIFICATION
+      - MULTILINGUALITY
+      - CUSTOM
+  - name: Proxy/dcterms:alternative
+    path: $.['proxies'][?(@['europeanaProxy'] == false)]['dctermsAlternative']
+    categories:
+      - DESCRIPTIVENESS
+      - SEARCHABILITY
+      - IDENTIFICATION
+      - MULTILINGUALITY
+groups:
+  - fields:
+      - Proxy/dc:title
+      - Proxy/dc:description
+    categories:
+      - MANDATORY
+```
+
+The same in JSON:
+```json
+{
+  "format": "json",
+  "fields": [
+    {
+      "name": "edm:ProvidedCHO/@about",
+      "path":  "$.['providedCHOs'][0]['about']",
+      "categories": ["MANDATORY"]
+    },
+    {
+      "name": "Proxy/dc:title",
+      "path": "$.['proxies'][?(@['europeanaProxy'] == false)]['dcTitle']",
+      "categories": [
+        "DESCRIPTIVENESS",
+        "SEARCHABILITY",
+        "IDENTIFICATION",
+        "MULTILINGUALITY"
+      ]
+    },
+    {
+      "name": "Proxy/dcterms:alternative",
+      "path": "$.['proxies'][?(@['europeanaProxy'] == false)]['dctermsAlternative']",
+      "categories": [
+        "DESCRIPTIVENESS",
+        "SEARCHABILITY",
+        "IDENTIFICATION",
+        "MULTILINGUALITY"
+      ]
+    }
+  ],
+  "groups": [
+    {
+      "fields": [
+        "Proxy/dc:title",
+        "Proxy/dc:description"
+      ],
+      "categories": [
+        "MANDATORY"
+      ]
+    }
+  ]
+}
+```
+
+
+Optionaly you can set the "canonical list" of categories. It provides
+two additional functionalities 
+* if a field contains a category which is not listed in the list, 
+that will be excluded (with a warning in the log)
+* the order of the categories in the output follows the order set 
+in the configuration.
+
+Here is an example (in YAML):
+
+```yaml
+format: json
+...
+categories:
+  - MANDATORY
+  - DESCRIPTIVENESS
+  - SEARCHABILITY
+  - IDENTIFICATION
+  - CUSTOM
+  - MULTILINGUALITY
+
+```
+## Constraints
+
+One can add constraints to the fields. There are content rules, which
+the tool will check. In this version the tool mimin SHACL constraints.
+
+### Cardinality
+ * `minCount <number>` - specifies the minimum number of field occurence (API: `setMinCount()` or `withMinCount()`)
+ * `maxCount <number>` - specifies the maximum number of field occurence (API: `setMaxCount()` or `withMaxCount()`)
+
+### Value Range
+
+ * `minExclusive <number>` - The minimum exclusive value ([field value] > limit, API: `setMinExclusive(Double)` or `withMinExclusive(Double)`)
+ * `minInclusive <number>` - The minimum inclusive value ([field value] >= limit, API: `setMinInclusive(Double)` or `withMinExclusive(Double)`)
+ * `maxExclusive <number>` - The maximum exclusive value ([field value] < limit, API: `setMaxExclusive(Double)` or `withMaxExclusive(Double)`)
+ * `maxInclusive <number>` - The maximum inclusive value ([field value] <= limit, API: `setMaxInclusive(Double)` or `withMaxInclusive(Double)`)
+
+### String constraints
+
+ * `minLength <number>` - The minimum string length of each field value (API: `setMinLength(Integer)` or `withMinLength(Integer)`)
+ * `maxLength <number>` - The maximum string length of each field value (API: `setMinLength(Integer)` or `withMaxLength(Integer)`)
+ * `pattern <regular expression>` - A regular expression that each field value matches to satisfy the condition (API: `setPattern(String)` or `withPattern(String)`)
+
+### Property pair
+
+ * `equals <field label>` - The set of all values of a field is equal to the set of all values of another field 
+ (API: `setEquals(String)` or `withEquals(String)`)
+ * `disjoint <field label>` - The set of values of a field is disjoint (not equal) with the set of all values of another field 
+ (API: `setDisjoint(String)` or `withDisjoint(String)`)
+ * `lessThan <field label>` - Each values of a field is smaller than each values of another field
+  (API: `setLessThan(String)` or `withLessThan(String)`)
+ * `lessThanOrEquals <field label>` - Each values of a field is smaller than or equals to each values of another field
+  (API: `setLessThanOrEquals(String)` or `withLessThanOrEquals(String)`)
+
+### Set rules
+
+Via API 
+```java
+Schema schema = new BaseSchema()
+  .setFormat(Format.CSV)
+  .addField(
+    new JsonBranch("title", "title")
+      .setRule(
+        new Rule()
+          .withDisjoint("description")
+      )
+  )
+  .addField(
+    new JsonBranch("url", "url")
+      .setRule(
+        new Rule()
+          .withMinCount(1)
+          .withMaxCount(1)
+          .withPattern("^https?://.*$")
+      )
+  )
+  ;
+```
+
+Via configuration file (a YAML example):
+
+```yaml
+format: csv
+fields:
+  - name: title
+    categories: [MANDATORY]
+    rules:
+      disjoint: description
+  - name: url
+    categories: [MANDATORY]
+    extractable: true
+    rules:
+      minCount: 1
+      maxCount: 1
+      pattern: ^https?://.*$
+```
+
+In both cases we defined two fields. `title` has one constraints: it should not be equal to 
+the value of `description` field (which is masked out from the example). Note: if this hypothetical
+`description` field is not available the API drops an error message into the log. `url` should have
+one and only one instance, and its value should start with "http://" or "https://".
+
+As you can see there are two types of setters in the API: setSomething and withSomething. The
+difference is that setSomething returs with void, but withSomething returns with the Rule object,
+so you can use it in a chain such as `new Rule().withMinCount(1).withMaxCount(3)` 
+(while `new Rule().setMinCount(1).setMaxCount(3)` doesn't work).
+
+## More info
+
 For the usage and implementation of the API see https://github.com/pkiraly/europeana-qa-api.
 
 Java doc for the actual development version of the API: https://pkiraly.github.io/metadata-qa-api.
 
 [![Build Status](https://travis-ci.org/pkiraly/metadata-qa-api.svg?branch=master)](https://travis-ci.org/pkiraly/metadata-qa-api)
-[![Coverage Status](https://coveralls.io/repos/github/pkiraly/metadata-qa-api/badge.svg?branch=master)](https://coveralls.io/github/pkiraly/metadata-qa-api?branch=master)
+[![Coverage Status](https://coveralls.io/repos/github/pkiraly/metadata-qa-api/badge.svg?branch=master)](https://coveralls.io/github/pkiraly/metadata-qa-api?branch=develop)
 [![javadoc](https://javadoc.io/badge2/de.gwdg.metadataqa/metadata-qa-api/javadoc.svg)](https://javadoc.io/doc/de.gwdg.metadataqa/metadata-qa-api)

pom.xml

@@ -5,7 +5,7 @@
   <groupId>de.gwdg.metadataqa</groupId>
   <artifactId>metadata-qa-api</artifactId>
   <packaging>jar</packaging>
-  <version>0.7-SNAPSHOT</version>
+  <version>0.7</version>
   <name>Metadata Quality Assurance Framework API</name>
   <description>
     A metadata quality assurance framework. It checks some metrics of 

src/main/java/de/gwdg/metadataqa/api/calculator/TfIdfCalculator.java

@@ -11,7 +11,7 @@
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.Serializable;
-import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -103,7 +103,7 @@ private String getSolrResponse(String recordId) {
       IOUtils.copy(method.getResponseBodyAsStream(), baos);
       byte[] responseBody = baos.toByteArray();
 
-      jsonString = new String(responseBody, Charset.forName("UTF-8"));
+      jsonString = new String(responseBody, StandardCharsets.UTF_8);
     } catch (HttpException e) {
       LOGGER.severe("Fatal protocol violation: " + e.getMessage());
     } catch (IOException e) {

src/main/java/de/gwdg/metadataqa/api/calculator/UniquenessCalculator.java

@@ -1,10 +1,13 @@
 package de.gwdg.metadataqa.api.calculator;
 
 import de.gwdg.metadataqa.api.model.pathcache.PathCache;
-import de.gwdg.metadataqa.api.uniqueness.*;
 import de.gwdg.metadataqa.api.counter.FieldCounter;
 import de.gwdg.metadataqa.api.interfaces.Calculator;
 import de.gwdg.metadataqa.api.schema.Schema;
+import de.gwdg.metadataqa.api.uniqueness.SolrClient;
+import de.gwdg.metadataqa.api.uniqueness.UniquenessExtractor;
+import de.gwdg.metadataqa.api.uniqueness.UniquenessField;
+import de.gwdg.metadataqa.api.uniqueness.UniquenessFieldCalculator;
 import de.gwdg.metadataqa.api.util.CompressionLevel;
 import org.apache.commons.lang3.StringUtils;
 
@@ -13,7 +16,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.LinkedHashMap;
-import java.util.logging.Logger;
 
 /**
  *
@@ -23,16 +25,13 @@ public class UniquenessCalculator implements Calculator, Serializable {
 
   public static final String CALCULATOR_NAME = "uniqueness";
 
-  private static final Logger LOGGER = Logger.getLogger(
-      UniquenessCalculator.class.getCanonicalName()
-  );
   public static final String SUFFIX = "_txt";
   public static final int SUFFIX_LENGTH = SUFFIX.length();
 
   private UniquenessExtractor extractor;
   private List<UniquenessField> solrFields;
 
-  private SolrClient solrClient;
+  private final SolrClient solrClient;
 
   private FieldCounter<Double> resultMap;
 
@@ -108,13 +107,13 @@ public String getTotals() {
   }
 
   @Override
-  public Map<String, ? extends Object> getResultMap() {
+  public Map<String, ?> getResultMap() {
     return resultMap.getMap();
   }
 
   @Override
-  public Map<String, Map<String, ? extends Object>> getLabelledResultMap() {
-    Map<String, Map<String, ? extends Object>> labelledResultMap = new LinkedHashMap<>();
+  public Map<String, Map<String, ?>> getLabelledResultMap() {
+    Map<String, Map<String, ?>> labelledResultMap = new LinkedHashMap<>();
     labelledResultMap.put(getCalculatorName(), resultMap.getMap());
     return labelledResultMap;
   }

src/main/java/de/gwdg/metadataqa/api/configuration/ConfigurationReader.java

@@ -4,15 +4,19 @@
 import org.yaml.snakeyaml.Yaml;
 import org.yaml.snakeyaml.constructor.Constructor;
 
-import java.io.*;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.FileInputStream;
 
 public class ConfigurationReader {
 
   public static Configuration readJson(String fileName) throws FileNotFoundException {
     ObjectMapper objectMapper = new ObjectMapper();
 
     File file = new File(fileName);
-    Configuration config = null;
+    Configuration config;
     try {
       config = objectMapper.readValue(file, Configuration.class);
     } catch (IOException e) {
@@ -24,8 +28,6 @@ public static Configuration readJson(String fileName) throws FileNotFoundExcepti
   public static Configuration readYaml(String fileName) throws FileNotFoundException {
     Yaml yaml = new Yaml(new Constructor(Configuration.class));
     InputStream inputStream = new FileInputStream(new File(fileName));
-    Configuration config = (Configuration) yaml.load(inputStream);
-    return config;
+    return yaml.load(inputStream);
   }
-
 }

src/main/java/de/gwdg/metadataqa/api/configuration/Rule.java

@@ -18,8 +18,8 @@ public class Rule {
   private Integer maxInclusive;
   private Integer minLength;
   private Integer maxLength;
-  private Integer lessThan;
-  private Integer lessThanOrEquals;
+  private String lessThan;
+  private String lessThanOrEquals;
   private String hasValue;
 
   public String getPattern() {
@@ -204,28 +204,28 @@ public Rule withMaxLength(int maxLength) {
     return this;
   }
 
-  public Integer getLessThan() {
+  public String getLessThan() {
     return lessThan;
   }
 
-  public void setLessThan(int lessThan) {
+  public void setLessThan(String lessThan) {
     this.lessThan = lessThan;
   }
 
-  public Rule withLessThan(int lessThan) {
+  public Rule withLessThan(String lessThan) {
     setLessThan(lessThan);
     return this;
   }
 
-  public Integer getLessThanOrEquals() {
+  public String getLessThanOrEquals() {
     return lessThanOrEquals;
   }
 
-  public void setLessThanOrEquals(int lessThanOrEquals) {
+  public void setLessThanOrEquals(String lessThanOrEquals) {
     this.lessThanOrEquals = lessThanOrEquals;
   }
 
-  public Rule withLessThanOrEquals(int lessThanOrEquals) {
+  public Rule withLessThanOrEquals(String lessThanOrEquals) {
     setLessThanOrEquals(lessThanOrEquals);
     return this;
   }