fix tests, remove treatments

adaurat · adaurat · commit b12fb3dd1871 · 2025-02-25T09:35:20.000+01:00
diff --git a/qmra/risk_assessment/tests/test_risk_assessment_form.py b/qmra/risk_assessment/tests/test_risk_assessment_form.py
@@ -2,6 +2,7 @@
 from assertpy import assert_that
 
 from qmra.risk_assessment.forms import RiskAssessmentForm, InflowForm, InflowFormSet, TreatmentForm, TreatmentFormSet
+from qmra.risk_assessment.models import DefaultTreatments
 
 
 class TestRiskAssessmentForm(TestCase):
@@ -89,6 +90,8 @@ def test_that_negative_are_allowed(self):
             protozoa_max=-1
         )
         given_form = TreatmentForm(data=data)
+        # ugly hack to work around dynamic choices...
+        given_form.fields["name"].choices = DefaultTreatments.choices()
         given_form.full_clean()
         print(given_form.errors)
         assert_that(len(given_form.errors)).is_equal_to(0)
@@ -108,6 +111,8 @@ def test_that_min_needs_to_be_less_than_max(self):
             mx = prfx + "_max"
             data = {**default_data, mn: 2, mx: 1}
             given_form = TreatmentForm(data=data)
+            # ugly hack to work around dynamic choices...
+            given_form.fields["name"].choices = DefaultTreatments.choices()
             given_form.full_clean()
             assert_that(len(given_form.errors)).is_equal_to(1)
             assert_that(given_form.errors).contains_key(mn)
diff --git a/qmra/risk_assessment/views.py b/qmra/risk_assessment/views.py
@@ -232,8 +232,9 @@ def create_exposure(request):
     return HttpResponse(status=404)
 
 
-@login_required(login_url="/login")
 def list_exposures(request):
+    if not request.user.is_authenticated:
+        return JsonResponse({})
     return JsonResponse({e["name"]: e for e in UserExposure.objects.filter(user=request.user).values().all()})
 
 
@@ -255,13 +256,15 @@ def create_source(request):
     return HttpResponse(status=404)
 
 
-@login_required(login_url="/login")
 def list_sources(request):
+    if not request.user.is_authenticated:
+        return JsonResponse({})
     return JsonResponse({s["name"]: s for s in UserSource.objects.filter(user=request.user).values().all()})
 
 
-@login_required(login_url="/login")
 def list_inflows(request):
+    if not request.user.is_authenticated:
+        return JsonResponse({})
     return JsonResponse({s["name"]: [
         dict(pathogen_name="Rotavirus", source_name=s["name"],
              min=s["rotavirus_min"], max=s["rotavirus_max"],
@@ -296,7 +299,8 @@ def create_treatment(request):
     return HttpResponse(status=404)
 
 
-@login_required(login_url="/login")
 def list_treatments(request):
+    if not request.user.is_authenticated:
+        return JsonResponse({})
     return JsonResponse({t["name"]: {**t, "treatment_name": t["name"]}
                          for t in UserTreatment.objects.filter(user=request.user).values().all()})
diff --git a/qmra/static/data/default-treatments.json b/qmra/static/data/default-treatments.json
diff --git a/raw_public_data/qry_treatmentRemovals.csv b/raw_public_data/qry_treatmentRemovals.csv
@@ -70,9 +70,6 @@
 "Filtration","Reverse osmosis","Bacteria",5.44,6,"NSF/ANSI 419  validation","http://info.nsf.org/Certified/pdwe/Listings.asp"
 "Filtration","Reverse osmosis","Viruses",5.44,6,"NSF/ANSI 419  validation","http://info.nsf.org/Certified/pdwe/Listings.asp"
 "Filtration","Reverse osmosis","Protozoa",5.75,6.32,"NSF/ANSI 419  validation","http://info.nsf.org/Certified/pdwe/Listings.asp"
-"Filtration","Partial reverse osmosis","Bacteria",0.3,0.3,"NSF/ANSI 419  validation","http://info.nsf.org/Certified/pdwe/Listings.asp"
-"Filtration","Partial reverse osmosis","Viruses",0.3,0.3,"NSF/ANSI 419  validation","http://info.nsf.org/Certified/pdwe/Listings.asp"
-"Filtration","Partial reverse osmosis","Protozoa",0.3,0.3,"NSF/ANSI 419  validation","http://info.nsf.org/Certified/pdwe/Listings.asp"
 "Primary disinfection","UV disinfection 40 mJ/cm2, drinking","Bacteria",4.6,6,"Hijnen et al. (2006)","https://doi.org/10.1016/j.watres.2005.10.030"
 "Primary disinfection","UV disinfection 40 mJ/cm2, drinking","Viruses",4.1,5.9,"Hijnen et al. (2006)","https://doi.org/10.1016/j.watres.2005.10.030"
 "Primary disinfection","UV disinfection 40 mJ/cm2, drinking","Protozoa",2.5,3,"Hijnen et al. (2006)","https://doi.org/10.1016/j.watres.2005.10.030"
diff --git a/raw_public_data/tbl_logRemoval.csv b/raw_public_data/tbl_logRemoval.csv
@@ -2,24 +2,13 @@ treatment_id,"min","max","reference","pathogen_group","mean","alpha","beta","dis
 1,0.2,2,40,"Bacteria",NA,NA,NA,"uniform",NA
 1,0.1,3.4,40,"Viruses",NA,NA,NA,"uniform",NA
 1,1,2,40,"Protozoa",NA,NA,NA,"uniform",NA
-2,0.6,2.6,40,"Protozoa",NA,NA,NA,"uniform",NA
 3,2,2.8,40,"Protozoa",NA,NA,NA,"uniform",NA
-4,1,4,40,"Bacteria",NA,NA,NA,"uniform",NA
-4,2,4,40,"Viruses",NA,NA,NA,"uniform",NA
-4,0,2,40,"Protozoa",NA,NA,NA,"uniform",NA
-5,0.2,4.4,40,"Bacteria",NA,NA,NA,"uniform",NA
-5,0,3.5,40,"Viruses",NA,NA,NA,"uniform",NA
-5,0.4,3.3,40,"Protozoa",NA,NA,NA,"uniform",NA
-7,0.2,2.3,40,"Bacteria",NA,NA,NA,"uniform",NA
-7,1,1.7,40,"Viruses",NA,NA,NA,"uniform",NA
-7,3,6.7,40,"Protozoa",NA,NA,NA,"uniform",NA
 8,2,6,40,"Bacteria",NA,NA,NA,"uniform",NA
 8,0.25,4,40,"Viruses",NA,NA,NA,"uniform",NA
 8,0.3,5,40,"Protozoa",NA,NA,NA,"uniform",NA
 9,2,6,40,"Bacteria",NA,NA,NA,"uniform",NA
 9,2.1,8.3,40,"Viruses",NA,NA,NA,"uniform",NA
 9,1,2,40,"Protozoa",NA,NA,NA,"uniform",NA
-10,0.2,2.3,40,"Bacteria",NA,NA,NA,"uniform",NA
 11,0.7,2.2,40,"Bacteria",NA,NA,NA,"uniform",NA
 11,1.4,2.3,40,"Protozoa",NA,NA,NA,"uniform",NA
 12,2,2,40,"Bacteria",NA,NA,NA,"uniform",NA
@@ -43,9 +32,6 @@ treatment_id,"min","max","reference","pathogen_group","mean","alpha","beta","dis
 18,0,1,8,"Bacteria",NA,NA,NA,"uniform",NA
 18,0.5,3,8,"Viruses",NA,NA,NA,"uniform",NA
 18,1.5,2.5,8,"Protozoa",NA,NA,NA,"uniform",NA
-19,3.5,6,8,"Bacteria",NA,NA,NA,"uniform",NA
-19,2.5,6,8,"Viruses",NA,NA,NA,"uniform",NA
-19,6,6,8,"Protozoa",NA,NA,NA,"uniform",NA
 20,2,6,8,"Bacteria",NA,NA,NA,"uniform",NA
 20,1,3,8,"Viruses",NA,NA,NA,"uniform",NA
 20,0,1.5,8,"Protozoa",NA,NA,NA,"uniform",NA
@@ -72,7 +58,4 @@ treatment_id,"min","max","reference","pathogen_group","mean","alpha","beta","dis
 21,5.75,6.32,47,"Protozoa",NA,NA,NA,"uniform","identical to Nanofiltration"
 29,4.6,6,50,"Bacteria",NA,NA,NA,"uniform","based on Streptococcus faecalis: kmin = 0.312, kmax = 0.88, MICmax_min = 4.6, MICmax_max = 6,  dose = 40"
 29,4.1,5.9,50,"Viruses",NA,NA,NA,"uniform","based on Rotavirus (MC vs PC): kmin = 0.102, kmax = 0.1514, MICmax_min = 4.1, MICmax_max = 5.9,  dose = 40"
-29,2.5,3,50,"Protozoa",NA,NA,NA,"uniform","based on Giardia lambia: kmin = 0.122, kmax = 0.243, MICmax_min = 2.5, MICmax_max = 3,  dose = 40"
-30,0.3,0.3,NA,"Bacteria",NA,NA,NA,"uniform",NA
-30,0.3,0.3,NA,"Viruses",NA,NA,NA,"uniform",NA
-30,0.3,0.3,NA,"Protozoa",NA,NA,NA,"uniform",NA
+29,2.5,3,50,"Protozoa",NA,NA,NA,"uniform","based on Giardia lambia: kmin = 0.122, kmax = 0.243, MICmax_min = 2.5, MICmax_max = 3,  dose = 40"
diff --git a/raw_public_data/tbl_treatment.csv b/raw_public_data/tbl_treatment.csv
@@ -1,19 +1,12 @@
 id,name,group,description
 29,"UV disinfection 40 mJ/cm2, drinking","Primary disinfection","UV-light is mostly effective at 254 nm where it affects DNA or RNA thus preventing reproduction of the organism (inactivation). Log reduction for drinking water UV is based on closed UV-reactors wich have been validated according to appropriate standards (e.g. USEPA or DVGW). Effectiveness of disinfection depends on delivered fluence (dose in mJ/cm2), which varies with lamp intensity, exposure time (flow rate) and UV-absorption by the water (organics). Excessive turbidity and certain dissolved species inhibit this process; hence, turbidity should be kept below 1 NTU to support effective disinfection."
 1,"Conventional clarification","Coagulation, flocculation and sedimentation","Consists of coagulant and/or flocculant aid (e.g. polymer) dosing, rapid mixing, slow mixing and sedimentation. Log removal depends on process optimisation. Rapid changes in source water quality such as turbidity increase due to monsoon rainfall or algeal blooms may decrease treatment effect and require adjustment of process settings."
-2,"Dissolved air flotation","Coagulation, flocculation and sedimentation","Compressed air is injected in the water such that air bubbles bring suspended solids to the water surface where they are skimmed off. Coagulant may be dosed.  Log removal depends on process optimisation. Rapid changes in source water quality such as turbidity increase due to monsoon rainfall or algeal blooms may decrease treatment effect and require adjustment of process settings."
 3,High-rate clarification,"Coagulation, flocculation and sedimentation","Consists of coagulant and/or flocculant aid (e.g. polymer) dosing, mixing and enhanced sedimentation by flock blankets, lamellae- or tube settlers. Log removal depends on process optimisation. Rapid changes in source water quality such as turbidity increase due to monsoon rainfall or algeal blooms may decrease treatment effect and require adjustment of process settings."
-4,"Lime softening","Coagulation, flocculation and sedimentation","Lime is dosed to the water to reduce hardness. When flocks are formed, they can entrap pathogens. Note that the technical design of the softening process affects the log reduction. e.g. pellet-softening has no effect on pathogens."
-5,"Granular high-rate filtration","Filtration","Water is filtered through a fixed bed of granular media (e.g. sand) generally operatied down flow with rates of 5 to 20 m/h and contact times of 4 to 15 minutes. They are regularly backwashed to remove built up solids in the filter. Log removal depends on filter media and coagulation pretreatment;consistent low filtered water turbidity of ? 0.3 NTU (none to exceed 1 NTU)
- are associated higher log removal of pathogens"
-7,"Precoat filtration","Filtration","Consist of a fine filter (e.g. candle filter or drum filter) that is precoated by dosing fine granular material (often diatomaceous earth). This material forms a fine filter cake that removes solids from the water. The log removal can only be achieved if the filter cake is present and depends on precoat media grade and filtratin 
-rate."
 8,"Slow sand filtration","Filtration","Water is filtered through a fixed bed sand operatied down flow with rates of 0.1 to 1 m/h and contact times of 3 to 6 hours. The filter is not backwashed. In weeks to months a 'schmutzdecke' will develop on the filter which enhances log removal. Grain size, flow rate and temperature also affect log removal. Consistent low filtered water turbidity of ? 0.3 NTU (none to exceed 1 NTU) are associated higher log removal of pathogens
 
 associated with 1 - 2 log reduction of viruses and 2.5 - 3 log reduction of Cryptosporidiuma"
 9,"Bank filtration","Pretreatment","Water is abstracted through wells located close to surface water, thus the bank serves as a natural filter. Log removal depends on travel distance and time, soil type (grain size),
  and geochemicl conditions (oxygen level, pH)"
-10,"Roughing filters","Pretreatment","Water is filtered through a fixed bed of coarse granular media (e.g. rocks 5-20 mm) operated at high rates. They are not backwashed. Log removal depends on filter media and coagulation pretreatment."
 11,"Storage reservoirs","Pretreatment","Water is protected from human recontamination in reservoirs, however wildlife and waterfoul may introduce pathogens. Log reduction occurs due to sedimentation, UV radiation from sunlight and die-off in time,  depending on construction (mixing) and temperature. Reporded reduction based on residence time > 40 days (bacteria), 160 days (protozoa)"
 12,"Chlorination, wastewater","Primary disinfection","Log inactivation depends on free chlorine concentration and contact time (CT); not effective against Cryptosporidium oocysts, reported protozoan log reduction is mostly for Giardia. Turbidity and chlorine-demanding solutes inhibit this process; hence, effect in wastewater is limited since free chlorine will rapidly decay. 
 
@@ -30,7 +23,6 @@ Cryptosporidium varies widely"
 17,"Secondary treatment","Pretreatment","Secondary treatment consists of an activated sludge process to break down organics in the wastewater and a settling stage to separate the biologiscal sludge from the water."
 18,"Dual media filtration","Filtration","Water is filtered through a fixed bed consisting of two layers of granular media (e.g. antracite and sand) generally operatied down flow with rates of 5 to 20 m/h and contact times of 4 to 15 minutes. They are regularly backwashed to remove built up solids in the filter. Log removal depends on filter media and coagulation pretreatment;consistent low filtered water turbidity of ? 0.3 NTU (none to exceed 1 NTU)
  are associated higher log removal of pathogens"
-19,"Membrane filtration","Filtration","A membrane is a thin sheet with small openings that removes solids and depending on membrane type, solutes from the water when this is led through the membrane."
 20,"Chlorination, drinking water","Primary disinfection","Log inactivation depends on free chlorine concentration and contact time (CT); not effective against Cryptosporidium oocysts, reported log reduction is mostly for Giardia. Turbidity and chlorine-demanding solutes inhibit this process; hence, turbidity should be kept below 1 NTU to support
  effective disinfection. Where this is not practical, turbidities should be kept below 5 NTU with higher chlorine doses or contact times. In addition to initial disinfection, the benefits of maintaining free chlorine residuals throughout distribution systems at or above 0.2 mg/l should be considered"
 21,"Reverse osmosis","Filtration","A reverse osmosis membrane is a thin sheet with small openings that removes solids and most soluble molecules, including salts (< 0,004 �m depending on selected membrane) from the water when this is led through the membrane. It can take the form of spiral wound membranes, hollow fibers or sheets. Actual log reduction depends on the selected membrane and is determined by challenge testing."
@@ -40,5 +32,4 @@ Cryptosporidium varies widely"
 25,"UV disinfection, wastewater","Primary disinfection","UV-light is mostly effective at 254 nm where it affects DNA or RNA thus preventing reproduction of the organism (inactivation). Effectiveness of disinfection depends on delivered fluence (dose in mJ/cm2), which varies with lamp intensity, exposure time (flow rate) and UV-absorption by the water (organics). Wastewater UV-reactors are generally open-channel reactors in which UV lamps are placed. Excessive turbidity and certain dissolved species inhibit this process; hence the effect in wastewater highly depends on the water quality an is generally lower than in drinking water at the same dose."
 26,"Microfiltration","Filtration","A microfiltration membrane is a thin sheet with small openings that removes solids (0.1-10 �m depending on selected membrane) from the water when this is led through the membrane. It can take the form of capilary tubes, hollow fibers or sheet membranes. Actual log reduction depends on the selected membrane and is determined by challenge testing."
 27,"Ultrafiltration","Filtration","An ultrafiltration membrane is a thin sheet with small openings that removes solids (0.005-0,2 �m depending on selected membrane) from the water when this is led through the membrane. It can take the form of capilary tubes, hollow fibers, spiral wound or sheet membranes. Actual log reduction depends on the selected membrane and is determined by challenge testing."
-28,"Nanofiltration","Filtration","An nanofiltration membrane is a thin sheet with small openings that removes solids and larger soluble molecules (0.001-0,03 �m depending on selected membrane) from the water when this is led through the membrane. It can take the form of spiral wound or hollow fiber membranes. Actual log reduction depends on the selected membrane and is determined by challenge testing."
-30,"Partial reverse osmosis","Filtration","Treatment of 50% of the volume stream with revserse osmosis"
+28,"Nanofiltration","Filtration","An nanofiltration membrane is a thin sheet with small openings that removes solids and larger soluble molecules (0.001-0,03 �m depending on selected membrane) from the water when this is led through the membrane. It can take the form of spiral wound or hollow fiber membranes. Actual log reduction depends on the selected membrane and is determined by challenge testing."
