Feature/260 json_file_handlers (#273)

Add JSON endpoints for settings

* Create JOSN to File handlers
* Add openapi setting schemas
* Working - json to file
* Add JSON `settings` endpoints to models and analyses + validation
* Fix filewrite issue
* Adjust unittest + add dependencies
* generic model settings
* Fix external schema ref's in swagger
* Fix missing link re-names
* Add unittests for new endpoints, show all validation errors
* Keep default model_settings file for the moment
* Rename ModelResource -> ModelSettings

model_resource.json

@@ -1,69 +1,13 @@
 {
     "model_settings":[
-        {"event_set":{
-	      "name": "Event Set",
-	      "desc": "Either Probablistic or Historic",
-              "type":"dictionary",
-               "default": "P",
-
-                      "values":{
-                        "P": "Proabilistic",
-                        "H": "Historic"
-                    }
-            }
-         },
-             {
-                "event_occurrence_id":{
-		    "name": "Occurrence Set",
-		    "desc": "Tooltip for Occurrence selection",
-                    "type":"dictionary",
-                    "default": "1",
-                    "values":{
-                       "1":"Long Term",
-                       "2":"Near Term WSST",
-                       "3":"Historic"
-                    }
-                }
-
-            },
-            {
-                "peril_wind":{
-	            "name": "Wind Peril",
-		    "desc": "Run model with Wind Peril",
-                    "type":"boolean",
-                    "default": true
-
-                 }
-            },
-            {
-                 "peril_surge":{
-                    "name": "Surge Peril",
-		    "desc": "Run model with Surge Peril",
-                    "type":"boolean",
-                    "default": false
-
-                  }
-            },                                                                                                                                                                           
-            {
-               "leakage_factor": {
-                   "name": "Leakage Factor",
-		   "desc": "Tooltip for Leakage option",
-                   "type": "float",
-                   "default": 0.5,
-                   "min": 0.0,
-                   "max":1.0
-                }
-            }
+        {}
     ],
     "lookup_settings":[
         {
-        "PerilCodes":{
-            "type":"dictionary",
-            "values":{
-                "WW1": "Windstorm with storm surge",
-                "WW2": "Windstorm w/o storm surge"
+            "PerilCodes":{
+                "type":"dictionary",
+                "values":{"AA1": "All perils"}
             }
-         }
         }
     ]
 }

requirements.in

@@ -2,6 +2,7 @@ oasislmf
 celery
 pymysql
 jsonpickle
+jsonschema
 requests
 pyopenssl
 fasteners

requirements.txt

@@ -53,6 +53,7 @@ itypes==1.1.0             # via coreapi
 jinja2-time==0.2.0        # via cookiecutter
 jinja2==2.10.1            # via cookiecutter, coreschema, jinja2-time
 jsonpickle==1.2
+jsonschema==2.6.0
 kombu==4.6.4              # via celery
 markdown==3.1.1
 markupsafe==1.1.1         # via jinja2

src/server/oasisapi/analyses/models.py

@@ -87,6 +87,9 @@ def get_absolute_copy_url(self, request=None):
     def get_absolute_settings_file_url(self, request=None):
         return reverse('analysis-settings-file', kwargs={'version': 'v1', 'pk': self.pk}, request=request)
 
+    def get_absolute_settings_url(self, request=None):
+        return reverse('analysis-settings', kwargs={'version': 'v1', 'pk': self.pk}, request=request)
+
     def get_absolute_input_file_url(self, request=None):
         return reverse('analysis-input-file', kwargs={'version': 'v1', 'pk': self.pk}, request=request)
 

src/server/oasisapi/analyses/serializers.py

@@ -7,6 +7,7 @@
 class AnalysisSerializer(serializers.ModelSerializer):
     input_file = serializers.SerializerMethodField()
     settings_file = serializers.SerializerMethodField()
+    settings = serializers.SerializerMethodField()
     lookup_errors_file = serializers.SerializerMethodField()
     lookup_success_file = serializers.SerializerMethodField()
     lookup_validation_file = serializers.SerializerMethodField()
@@ -30,6 +31,7 @@ class Meta:
             'complex_model_data_files',
             'input_file',
             'settings_file',
+            'settings',
             'lookup_errors_file',
             'lookup_success_file',
             'lookup_validation_file',
@@ -49,6 +51,11 @@ def get_settings_file(self, instance):
         request = self.context.get('request')
         return instance.get_absolute_settings_file_url(request=request) if instance.settings_file else None
 
+    @swagger_serializer_method(serializer_or_field=serializers.URLField)
+    def get_settings(self, instance):
+        request = self.context.get('request')
+        return instance.get_absolute_settings_url(request=request) if instance.settings_file else None
+
     @swagger_serializer_method(serializer_or_field=serializers.URLField)
     def get_lookup_errors_file(self, instance):
         request = self.context.get('request')

src/server/oasisapi/analyses/tests/test_analysis_api.py

@@ -126,6 +126,7 @@ def test_cleaned_name_portfolio_and_model_are_present___object_is_created(self,
                     'portfolio': portfolio.pk,
                     'model': model.pk,
                     'settings_file': response.request.application_url + analysis.get_absolute_settings_file_url(),
+                    'settings': response.request.application_url + analysis.get_absolute_settings_url(),
                     'input_file': response.request.application_url + analysis.get_absolute_input_file_url(),
                     'lookup_errors_file': response.request.application_url + analysis.get_absolute_lookup_errors_file_url(),
                     'lookup_success_file': response.request.application_url + analysis.get_absolute_lookup_success_file_url(),
@@ -181,6 +182,7 @@ def test_complex_model_file_present___object_is_created(self, name):
                     'portfolio': portfolio.pk,
                     'model': model.pk,
                     'settings_file': None,
+                    'settings': None,
                     'input_file': None,
                     'lookup_errors_file': None,
                     'lookup_success_file': None,
@@ -680,6 +682,142 @@ def test_output_file_is_cleared(self):
                 self.assertIsNone(Analysis.objects.get(pk=response.json['id']).output_file)
 
 
+class AnalysisSettingsJson(WebTestMixin, TestCase):
+    def test_user_is_not_authenticated___response_is_forbidden(self):
+        analysis = fake_analysis()
+
+        response = self.app.get(analysis.get_absolute_settings_url(), expect_errors=True)
+        self.assertIn(response.status_code, [401,403])
+
+    def test_settings_json_is_not_present___get_response_is_404(self):
+        user = fake_user()
+        analysis = fake_analysis()
+
+        response = self.app.get(
+            analysis.get_absolute_settings_url(),
+            headers={
+                'Authorization': 'Bearer {}'.format(AccessToken.for_user(user))
+            },
+            expect_errors=True,
+        )
+
+        self.assertEqual(404, response.status_code)
+
+    def test_settings_json_is_not_present___delete_response_is_404(self):
+        user = fake_user()
+        analysis = fake_analysis()
+
+        response = self.app.delete(
+            analysis.get_absolute_settings_url(),
+            headers={
+                'Authorization': 'Bearer {}'.format(AccessToken.for_user(user))
+            },
+            expect_errors=True,
+        )
+
+        self.assertEqual(404, response.status_code)
+
+    def test_settings_json_is_not_valid___response_is_400(self):
+        with TemporaryDirectory() as d:
+            with override_settings(MEDIA_ROOT=d):
+                user = fake_user()
+                analysis = fake_analysis()
+                json_data = {
+                    "analysis_settings": {
+                        "analysis_tag": "test_analysis",
+                        "module_supplier_id": "OasisIM",
+                        "model_version_id": "1",
+                        "number_of_samples": 0,
+                        "gul_threshold": 0,
+                        "model_settings": {
+                          "use_random_number_file": True,
+                          "event_occurrence_file_id": 1
+                        },
+                        "gul_output": True,
+                        "gul_summaries": [
+                          {
+                            "id": 1,
+                            "summarycalc": True,
+                            "eltcalc": True,
+                            "aalcalc": "Not-A-Boolean",
+                            "pltcalc": True,
+                            "lec_output":False
+                          }
+                        ],
+                        "il_output": False
+                      }
+                }
+
+                response = self.app.post(
+                    analysis.get_absolute_settings_url(),
+                    headers={
+                        'Authorization': 'Bearer {}'.format(AccessToken.for_user(user))
+                    },
+                    params=json.dumps(json_data),
+                    content_type='application/json',
+                    expect_errors=True,
+                )
+
+                validation_error =  {
+                    'number_of_samples': '0 is less than the minimum of 1',
+                    'gul_summaries-0-aalcalc': "'Not-A-Boolean' is not of type 'boolean'",
+                    'required': "'source_tag' is a required property"
+                }
+                self.assertEqual(400, response.status_code)
+                self.assertEqual(json.loads(response.body), validation_error)
+
+
+    def test_settings_json_is_uploaded___can_be_retrieved(self):
+        with TemporaryDirectory() as d:
+            with override_settings(MEDIA_ROOT=d):
+                user = fake_user()
+                analysis = fake_analysis()
+                json_data = {
+                    "analysis_settings": {
+                        "source_tag": "test_source",
+                        "analysis_tag": "test_analysis",
+                        "module_supplier_id": "OasisIM",
+                        "model_version_id": "1",
+                        "number_of_samples": 10,
+                        "gul_threshold": 0,
+                        "model_settings": {
+                          "use_random_number_file": True,
+                          "event_occurrence_file_id": 1
+                        },
+                        "gul_output": True,
+                        "gul_summaries": [
+                          {
+                            "id": 1,
+                            "summarycalc": True,
+                            "eltcalc": True,
+                            "aalcalc": True,
+                            "pltcalc": True,
+                            "lec_output":False
+                          }
+                        ],
+                        "il_output": False
+                      }
+                }
+
+                self.app.post(
+                    analysis.get_absolute_settings_url(),
+                    headers={
+                        'Authorization': 'Bearer {}'.format(AccessToken.for_user(user))
+                    },
+                    params=json.dumps(json_data),
+                    content_type='application/json'
+                )
+
+                response = self.app.get(
+                    analysis.get_absolute_settings_url(),
+                    headers={
+                        'Authorization': 'Bearer {}'.format(AccessToken.for_user(user))
+                    },
+                )
+                self.assertEqual(json.loads(response.body), json_data['analysis_settings'])
+                self.assertEqual(response.content_type, 'application/json')
+
+
 class AnalysisSettingsFile(WebTestMixin, TestCase):
     def test_user_is_not_authenticated___response_is_forbidden(self):
         analysis = fake_analysis()

src/server/oasisapi/analyses/viewsets.py

@@ -10,15 +10,16 @@
 from drf_yasg.utils import swagger_auto_schema
 from django_filters import rest_framework as filters
 
-from ..analysis_models.models import AnalysisModel
-from ..filters import TimeStampedFilter, CsvMultipleChoiceFilter, CsvModelMultipleChoiceFilter
-from ..files.views import handle_related_file
-from ..files.serializers import RelatedFileSerializer
 from .models import Analysis
 from .serializers import AnalysisSerializer, AnalysisCopySerializer
-from ..schemas import FILE_RESPONSE
 
+from ..analysis_models.models import AnalysisModel
 from ..data_files.serializers import DataFileSerializer
+from ..filters import TimeStampedFilter, CsvMultipleChoiceFilter, CsvModelMultipleChoiceFilter
+from ..files.views import handle_related_file, handle_json_data
+from ..files.serializers import RelatedFileSerializer
+from ..schemas.custom_swagger import FILE_RESPONSE
+from ..schemas.serializers import AnalysisSettingsSerializer 
 
 
 class AnalysisFilter(TimeStampedFilter):
@@ -364,3 +365,29 @@ def data_files(self, request, pk=None, version=None):
 
         df_serializer = DataFileSerializer(df, many=True, context=context)
         return Response(df_serializer.data)
+
+
+class AnalysisSettingsView(viewsets.ModelViewSet):
+    """
+    list:
+    Return the settings of an Analysis object.
+    """
+    queryset = Analysis.objects.all()
+    serializer_class = AnalysisSerializer
+    filter_class = AnalysisFilter
+
+    @swagger_auto_schema(methods=['get'], responses={200: AnalysisSettingsSerializer})
+    @swagger_auto_schema(methods=['post'], request_body=AnalysisSettingsSerializer, responses={201: RelatedFileSerializer})
+    @action(methods=['get', 'post', 'delete'], detail=True)
+    def analysis_settings(self, request, pk=None, version=None):
+        """
+        get:
+        Gets the analyses `settings` contents
+
+        post:
+        Sets the analyses `settings` contents
+
+        delete:
+        Disassociates the portfolios `settings_file` contents
+        """
+        return handle_json_data(self.get_object(), 'settings_file', request, AnalysisSettingsSerializer)

src/server/oasisapi/analysis_models/models.py

@@ -30,3 +30,5 @@ def queue_name(self):
 
     def get_absolute_resources_file_url(self, request=None):
         return reverse('analysis-model-resource-file', kwargs={'version': 'v1', 'pk': self.pk}, request=request)
+    def get_absolute_settings_url(self, request=None):
+        return reverse('model-settings', kwargs={'version': 'v1', 'pk': self.pk}, request=request)

src/server/oasisapi/analysis_models/serializers.py

@@ -1,11 +1,12 @@
 from drf_yasg.utils import swagger_serializer_method
+from django.core.files import File
 from rest_framework import serializers
 
 from .models import AnalysisModel
 
-
 class AnalysisModelSerializer(serializers.ModelSerializer):
     resource_file = serializers.SerializerMethodField()
+    settings = serializers.SerializerMethodField()
 
     class Meta:
         model = AnalysisModel
@@ -18,6 +19,7 @@ class Meta:
             'modified',
             'data_files',
             'resource_file',
+            'settings',
         )
 
     def create(self, validated_data):
@@ -30,3 +32,7 @@ def create(self, validated_data):
     def get_resource_file(self, instance):
         request = self.context.get('request')
         return instance.get_absolute_resources_file_url(request=request)
+    @swagger_serializer_method(serializer_or_field=serializers.URLField)
+    def get_settings(self, instance):
+        request = self.context.get('request')
+        return instance.get_absolute_settings_url(request=request)