First pass at extending provenance report history

#152

Author: antony-wilson

data_management/prov.py

@@ -114,8 +114,11 @@ def _add_author_agents(authors, doc, entity, reg_uri_prefix, vocab_namespaces):
             entity,
             author_agent,
             None,
-            {PROV_ROLE: QualifiedName(
-                vocab_namespaces[DCTERMS_VOCAB_PREFIX], 'creator')},
+            {
+                PROV_ROLE: QualifiedName(
+                    vocab_namespaces[DCTERMS_VOCAB_PREFIX], 'creator'
+                )
+            },
         )
 
 
@@ -148,7 +151,7 @@ def _add_code_repo_release(
             (
                 (
                     PROV_TYPE,
-                    QualifiedName(vocab_namespaces[DCMITYPE_VOCAB_PREFIX], 'Software')
+                    QualifiedName(vocab_namespaces[DCMITYPE_VOCAB_PREFIX], 'Software'),
                 ),
                 *_generate_object_meta(code_repo, vocab_namespaces),
                 (
@@ -352,7 +355,6 @@ def _add_linked_files(
     cr_activity,
     doc,
     dp_entity,
-    dp_id,
     input_objects,
     object_components,
     reg_uri_prefix,
@@ -364,24 +366,29 @@ def _add_linked_files(
     @param cr_activity: a prov.activity representing the code run
     @param doc: a ProvDocument that the entities will belong to
     @param dp_entity: a prov.entity representing the data_product
-    @param dp_id: the data_product id
     @param input_objects: boolean, 'True' if the object_components represent input
                 objects
     @param object_components: a list of object_components from the ObjectComponent table
     @param reg_uri_prefix: a str containing the name of the prefix
     @param vocab_namespaces: a dict containing the Namespaces for the vocab
 
+    @return a list of data products that were added
+
     """
     for component in object_components:
         obj = component.object
         data_products = obj.data_products.all()
 
         for data_product in data_products:
-            if not input_objects and data_product.id == dp_id:
-                # we have already added the original data product
+            file_id = f'{reg_uri_prefix}:api/data_product/{data_product.id}'
+
+            entity = doc.get_record(file_id)
+            # check to see if we have already created an entity for this data product
+            if len(entity) > 0:
+                # The prov documentation says a ProvRecord is returned, but actually a
+                # list of ProvRecord is returned
                 continue
 
-            file_id = f'{reg_uri_prefix}:api/data_product/{data_product.id}'
             file_entity = doc.entity(
                 file_id,
                 (
@@ -421,6 +428,8 @@ def _add_linked_files(
                 # add the link to the code run
                 doc.wasGeneratedBy(file_entity, cr_activity)
 
+    return data_products
+
 
 def _add_model_config(cr_activity, doc, model_config, reg_uri_prefix, vocab_namespaces):
     """
@@ -458,6 +467,48 @@ def _add_model_config(cr_activity, doc, model_config, reg_uri_prefix, vocab_name
     )
 
 
+def _add_prime_data_product(doc, data_product, reg_uri_prefix, vocab_namespaces):
+    """
+    Add the prime data product for this level of the provenance report.
+
+    @param doc: a ProvDocument that the entities will belong to
+    @param data_product: The DataProduct to generate the PROV document for
+    @param reg_uri_prefix: a str containing the name of the prefix
+    @param vocab_namespaces: a dict containing the Namespaces for the vocab
+
+    @return the data product entity
+
+    """
+    data_product_id = f'{reg_uri_prefix}:api/data_product/{data_product.id}'
+    entity = doc.get_record(data_product_id)
+    # check to see if we have already created an entity for this data product
+    if len(entity) > 0:
+        # The prov documentation says a ProvRecord is returned, but actually a
+        # list of ProvRecord is returned
+        return entity[0]
+
+    # add the data product
+    dp_entity = doc.entity(
+        f'{reg_uri_prefix}:api/data_product/{data_product.id}',
+        (
+            (PROV_TYPE, QualifiedName(vocab_namespaces[DCAT_VOCAB_PREFIX], 'Dataset')),
+            *_generate_object_meta(data_product.object, vocab_namespaces),
+        ),
+    )
+
+    _add_author_agents(
+        data_product.object.authors.all(),
+        doc,
+        dp_entity,
+        reg_uri_prefix,
+        vocab_namespaces,
+    )
+
+    _add_external_object(doc, data_product, dp_entity, reg_uri_prefix, vocab_namespaces)
+
+    return dp_entity
+
+
 def _add_submission_script(
     cr_activity, doc, submission_script, reg_uri_prefix, vocab_namespaces
 ):
@@ -476,9 +527,10 @@ def _add_submission_script(
         (
             (
                 PROV_TYPE,
-                QualifiedName(vocab_namespaces[DCMITYPE_VOCAB_PREFIX], 'Software')
+                QualifiedName(vocab_namespaces[DCMITYPE_VOCAB_PREFIX], 'Software'),
             ),
-            *_generate_object_meta(submission_script, vocab_namespaces),),
+            *_generate_object_meta(submission_script, vocab_namespaces),
+        ),
     )
 
     _add_author_agents(
@@ -501,69 +553,26 @@ def _add_submission_script(
     )
 
 
-def get_whole_object_component(components):
-    for component in components:
-        if component.whole_object:
-            return component
-
-
-def generate_prov_document(data_product, request):
+def _generate_prov_document(doc, data_product, reg_uri_prefix, vocab_namespaces):
     """
-    Generate a PROV document for a DataProduct detailing all the input and outputs and
-    how they were generated.
+    Add the next level to the provenance doc.
 
-    This uses the W3C PROV ontology (https://www.w3.org/TR/prov-o/).
+    This takes a data product and finds generates it provenance. A list of input files
+    are returned so they can be used to create the next level of the provenane if
+    needed.
 
-    :param data_product: The DataProduct to generate the PROV document for
+    @param doc: a ProvDocument that the entities will belong to
+    @param data_product: The DataProduct to generate the PROV document for
+    @param reg_uri_prefix: a str containing the name of the prefix
+    @param vocab_namespaces: a dict containing the Namespaces for the vocab
 
-    :return: A PROV-O document
+    @return a list of files that were used as input files, may be empty
 
     """
-    url = request.build_absolute_uri('/')
-    cenral_registry_url = settings.CENTRAL_REGISTRY_URL
-    if not cenral_registry_url.endswith('/'):
-        cenral_registry_url = f'{cenral_registry_url}/'
-
-    doc = prov.model.ProvDocument()
-
-    if url == cenral_registry_url:
-        # we are using the main registry
-        reg_uri_prefix = 'reg'
-        doc.add_namespace(reg_uri_prefix, cenral_registry_url)
-    else:
-        # we are using a local registry
-        reg_uri_prefix = 'lreg'
-        doc.add_namespace(reg_uri_prefix, url)
-
-    # the vocab namespace is always the main registry
-    doc.add_namespace(FAIR_VOCAB_PREFIX, f'{cenral_registry_url}vocab/#')
-    # we need to tell SONAR to ignore 'http' in the vocab URLs
-    doc.add_namespace(DCAT_VOCAB_PREFIX, 'http://www.w3.org/ns/dcat#')  # NOSONAR
-    doc.add_namespace(DCMITYPE_VOCAB_PREFIX, 'http://purl.org/dc/dcmitype/')  # NOSONAR
-    doc.add_namespace(DCTERMS_VOCAB_PREFIX, 'http://purl.org/dc/terms/')  # NOSONAR
-    doc.add_namespace(FOAF_VOCAB_PREFIX, 'http://xmlns.com/foaf/spec/#')  # NOSONAR
-
-    vocab_namespaces = {}
-    for namespace in doc.get_registered_namespaces():
-        vocab_namespaces[namespace.prefix] = namespace
-
-    # add the data product
-    dp_entity = doc.entity(
-        f'{reg_uri_prefix}:api/data_product/{data_product.id}',
-        (
-            (PROV_TYPE, QualifiedName(vocab_namespaces[DCAT_VOCAB_PREFIX], 'Dataset')),
-            *_generate_object_meta(data_product.object, vocab_namespaces),
-        ),
-    )
-
-    _add_author_agents(
-        data_product.object.authors.all(),
-        doc,
-        dp_entity,
-        reg_uri_prefix,
-        vocab_namespaces,
+    # add the the root data product
+    dp_entity = _add_prime_data_product(
+        doc, data_product, reg_uri_prefix, vocab_namespaces
     )
-    _add_external_object(doc, data_product, dp_entity, reg_uri_prefix, vocab_namespaces)
 
     # add the activity, i.e. the code run
     components = data_product.object.components.all()
@@ -572,11 +581,12 @@ def generate_prov_document(data_product, request):
         code_run = whole_object.outputs_of.all()[0]
     except IndexError:
         # there is no code run so we cannot add any more provenance data
-        return doc
+        return []
 
     # add the code run, this is the central activity
     cr_activity = _add_code_run(
-        dp_entity, doc, code_run, reg_uri_prefix, vocab_namespaces)
+        dp_entity, doc, code_run, reg_uri_prefix, vocab_namespaces
+    )
 
     # add the code repo release
     if code_run.code_repo is not None:
@@ -596,11 +606,10 @@ def generate_prov_document(data_product, request):
     )
 
     # add input files
-    _add_linked_files(
+    input_files = _add_linked_files(
         cr_activity,
         doc,
         dp_entity,
-        None,
         True,
         code_run.inputs.all(),
         reg_uri_prefix,
@@ -612,19 +621,89 @@ def generate_prov_document(data_product, request):
         cr_activity,
         doc,
         dp_entity,
-        data_product.id,
         False,
         code_run.outputs.all(),
         reg_uri_prefix,
         vocab_namespaces,
     )
 
+    return input_files
+
+
+def get_whole_object_component(components):
+    for component in components:
+        if component.whole_object:
+            return component
+
+
+def generate_prov_document(data_product, depth, request):
+    """
+    Generate a PROV document for a DataProduct detailing all the input and outputs and
+    how they were generated.
+
+    This uses the W3C PROV ontology (https://www.w3.org/TR/prov-o/).
+
+    :param data_product: The DataProduct to generate the PROV document for
+    :param depth: The depth for the document. How many levels of code runs to include.
+    :param request: A request object
+
+    :return: A PROV-O document
+
+    """
+    url = request.build_absolute_uri('/')
+    cenral_registry_url = settings.CENTRAL_REGISTRY_URL
+    if not cenral_registry_url.endswith('/'):
+        cenral_registry_url = f'{cenral_registry_url}/'
+
+    doc = prov.model.ProvDocument()
+
+    if url == cenral_registry_url:
+        # we are using the main registry
+        reg_uri_prefix = 'reg'
+        doc.add_namespace(reg_uri_prefix, cenral_registry_url)
+    else:
+        # we are using a local registry
+        reg_uri_prefix = 'lreg'
+        doc.add_namespace(reg_uri_prefix, url)
+
+    # the vocab namespace is always the main registry
+    doc.add_namespace(FAIR_VOCAB_PREFIX, f'{cenral_registry_url}vocab/#')
+    # we need to tell SONAR to ignore 'http' in the vocab URLs
+    doc.add_namespace(DCAT_VOCAB_PREFIX, 'http://www.w3.org/ns/dcat#')  # NOSONAR
+    doc.add_namespace(DCMITYPE_VOCAB_PREFIX, 'http://purl.org/dc/dcmitype/')  # NOSONAR
+    doc.add_namespace(DCTERMS_VOCAB_PREFIX, 'http://purl.org/dc/terms/')  # NOSONAR
+    doc.add_namespace(FOAF_VOCAB_PREFIX, 'http://xmlns.com/foaf/spec/#')  # NOSONAR
+
+    vocab_namespaces = {}
+    for namespace in doc.get_registered_namespaces():
+        vocab_namespaces[namespace.prefix] = namespace
+
+    # get the initial set of input files
+    input_files = _generate_prov_document(
+        doc, data_product, reg_uri_prefix, vocab_namespaces
+    )
+
+    if depth == 1:
+        return doc
+
+    # add extra layers to the report if requested by the user
+    while depth > 1:
+        next_level_input_files = []
+
+        for input_file in input_files:
+            next_input_files = _generate_prov_document(
+                doc, input_file, reg_uri_prefix, vocab_namespaces
+            )
+            next_level_input_files.extend(next_input_files)
+
+        # reset the input files for the next level
+        input_files = next_level_input_files
+        depth = depth - 1
+
     return doc
 
 
-def serialize_prov_document(
-    doc, format_, aspect_ratio, dpi=None, show_attributes=True
-):
+def serialize_prov_document(doc, format_, aspect_ratio, dpi=None, show_attributes=True):
     """
     Serialise a PROV document as either a JPEG or SVG image or an XML or PROV-N report.
 

data_management/rest/views.py

@@ -123,6 +123,8 @@ class ProvReportView(views.APIView):
 
     `dpi` (optional): A float used to define the dpi for the `JPEG` and `SVG` images
 
+    `depth` (optional): An integer used to determine how many code runs to include,
+    the default is 1
     """
     try:
         Dot(prog='dot').create()
@@ -137,7 +139,6 @@ class ProvReportView(views.APIView):
 
     def get(self, request, pk):
         data_product = get_object_or_404(models.DataProduct, pk=pk)
-        doc = generate_prov_document(data_product, request)
 
         show_attributes = request.query_params.get('attributes', True)
         if show_attributes == "False":
@@ -150,12 +151,23 @@ def get(self, request, pk):
         except ValueError:
             aspect_ratio = default_aspect_ratio
 
+        default_depth = 1
+        depth = request.query_params.get('depth', default_depth)
+        try:
+            depth = int(depth)
+        except ValueError:
+            depth = default_depth
+        if depth < 1:
+            depth = 1
+
         dpi = request.query_params.get('dpi', None)
         try:
             dpi = float(dpi)
         except (TypeError, ValueError):
             dpi = None
 
+        doc = generate_prov_document(data_product, depth, request)
+
         value = serialize_prov_document(
             doc,
             request.accepted_renderer.format,