#3532 separated health check signals into 2 and displaying them on he… (

#941)

* #3532 separated health check signals into 2 and displaying them on health check pages.

* #3532 separated health check signals into 2 and displaying them on health check pages.

annotation/signals/clinvar_annotation_health_check.py

@@ -3,11 +3,12 @@
 from django.dispatch import receiver
 
 from annotation.models import ClinVarVersion
-from library.health_check import health_check_signal, HealthCheckRequest, HealthCheckAge
+from library.health_check import HealthCheckRequest, HealthCheckAge, \
+    health_check_overall_stats_signal
 from snpdb.models import GenomeBuild
 
 
-@receiver(signal=health_check_signal)
+@receiver(signal=health_check_overall_stats_signal)
 def ontology_health_check(sender, health_request: HealthCheckRequest, **kwargs):
     checks = []
     for genome_build in [GenomeBuild.grch37(), GenomeBuild.grch38()]:

classification/models/discordance_models.py

@@ -563,7 +563,7 @@ def preview_icon(cls) -> str:
 
     @property
     def preview(self) -> 'PreviewData':
-        title = f"Discordance ID {self.discordance_report.pk}"
+        title = f"DR_ {self.discordance_report.pk}"
 
         extras = [
             PreviewKeyValue(key="Status", value=self.get_triage_status_display()),

classification/signals/classification_health_checks.py

@@ -8,7 +8,8 @@
 from flags.models import FlagType
 from flags.models.flag_health_check import flag_chanced_since
 from library.health_check import health_check_signal, \
-    HealthCheckRequest, HealthCheckTotalAmount, HealthCheckRecentActivity, HealthCheckStat
+    HealthCheckRequest, HealthCheckTotalAmount, HealthCheckRecentActivity, HealthCheckStat, \
+    health_check_overall_stats_signal
 
 """
 Reports information about classifications to the Slack health report
@@ -19,15 +20,6 @@
 def allele_info_health_check(sender, health_request: HealthCheckRequest, **kwargs):
     output: List[HealthCheckStat] = []
 
-    for status in [ImportedAlleleInfoStatus.PROCESSING, ImportedAlleleInfoStatus.MATCHED_IMPORTED_BUILD]:
-        not_complete = ImportedAlleleInfo.objects.filter(status=status).count()
-        if not_complete:
-            output.append(HealthCheckTotalAmount(
-                emoji=":hourglass_flowing_sand:",
-                amount=not_complete,
-                name=f"Imported Allele Matching in status of \"{status.label}\""
-            ))
-
     last_failures = ImportedAlleleInfo.objects.filter(
         latest_validation__created__gte=health_request.since,
         latest_validation__created__lte=health_request.now,
@@ -52,7 +44,23 @@ def allele_info_health_check(sender, health_request: HealthCheckRequest, **kwarg
     return output
 
 
-@receiver(signal=health_check_signal)
+@receiver(signal=health_check_overall_stats_signal)
+def allele_info_overall_stats(sender, **kwargs):
+    output: List[HealthCheckStat] = []
+
+    for status in [ImportedAlleleInfoStatus.PROCESSING, ImportedAlleleInfoStatus.MATCHED_IMPORTED_BUILD]:
+        not_complete = ImportedAlleleInfo.objects.filter(status=status).count()
+        if not_complete:
+            output.append(HealthCheckTotalAmount(
+                emoji=":hourglass_flowing_sand:",
+                amount=not_complete,
+                name=f"Imported Allele Matching in status of \"{status.label}\""
+            ))
+
+    return output
+
+
+@receiver(signal=health_check_overall_stats_signal)
 def classifications_health_check_count(sender, health_request: HealthCheckRequest, **kwargs):
     total_classification_qs = Classification.objects.filter(lab__external=False, withdrawn=False, created__lte=health_request.now).exclude(lab__name__icontains='legacy')
     total = total_classification_qs.count()
@@ -74,12 +82,15 @@ def classification_health_check_activity(sender, health_request: HealthCheckRequ
     classifications_of_interest = Classification.dashboard_report_classifications_of_interest(since=health_request.since)
     new_classification_count = Classification.dashboard_report_new_classifications(since=health_request.since)
 
+    classification_new = Classification.objects.filter(created__gte=health_request.since)
+
     return [
         HealthCheckRecentActivity(
             emoji=":blue_book:",
             amount=new_classification_count,
             name="Classifications",
-            sub_type="Created"
+            sub_type="Created",
+            preview=[classification.preview for classification in classification_new],
         ),
         HealthCheckRecentActivity(
             emoji=":blue_book:",

classification/templatetags/classification_tags.py

@@ -575,19 +575,19 @@ def imported_allele_info(imported_allele_info: ImportedAlleleInfo, on_allele_pag
 
 
 @register.simple_tag
-def user_view_events(user: User):
+def user_view_events(user: User, days: int = 1):
     if isinstance(user, str):
         try:
             user = User.objects.get(username=user)
         except User.DoesNotExist:
             return {}
     now = localtime()
-    since = now - timedelta(days=1)
+    since = now - timedelta(days=days)
     health_request = HealthCheckRequest(since=since, now=now)
     view_events = ViewEvent.objects.filter(user=user, created__gte=health_request.since,
-                                           created__lt=health_request.now).order_by('created')
+                                           created__lt=health_request.now).order_by('-created')
     view_event_data = []
-    for event in view_events:
+    for event in view_events[:20]:
         view_event_data.append({
             'created': event.created,
             'view_name': event.view_name,

eventlog/signals/active_users_health_check.py

@@ -1,5 +1,4 @@
 from collections import Counter
-from random import randint
 
 from django.contrib.auth.models import User
 from django.db.models import Q
@@ -17,27 +16,51 @@ def active_users_health_check(sender, health_request: HealthCheckRequest, **kwar
     event_log_activity = Event.objects.filter(date__gte=health_request.since, date__lt=health_request.now).values_list('user', flat=True).distinct()
     view_event_activity = ViewEvent.objects.filter(created__gte=health_request.since, created__lt=health_request.now).values_list('user', flat=True).distinct()
 
-    active_users_queryset = User.objects.filter(Q(pk__in=event_log_activity) | Q(pk__in=view_event_activity)).exclude(groups=bot_group()).order_by('username')
-    active_users = active_users_queryset.values_list('username', flat=True)
+    superusers_queryset = User.objects.filter(
+        Q(is_superuser=True) | Q(groups__name='bot'),
+        Q(pk__in=event_log_activity) | Q(pk__in=view_event_activity)
+    ).distinct().order_by('username')
 
-    count = active_users.count()
-    emoji: str
-    if count == 0:
-        emoji = ":ghost:"
-    else:
-        emojis = [":nerd_face:", ":thinking_face:", ":face_with_monocle:", ":face_with_cowboy_hat:"]
-        emoji = emojis[randint(0, len(emojis) - 1)]
+    non_admin_queryset = User.objects.filter(Q(pk__in=event_log_activity) | Q(pk__in=view_event_activity)).exclude(is_superuser=True).exclude(groups=bot_group()).order_by('username')
 
-    user_previews = [UserPreview(user).preview for user in active_users_queryset]
+    superusers = superusers_queryset.values_list('username', flat=True)
+    non_admin_users = non_admin_queryset.values_list('username', flat=True)
 
-    return HealthCheckRecentActivity(
-        emoji=emoji,
-        name="Active Users",
-        amount=count,
-        extra=", ".join(list(active_users)),
-        preview=user_previews,
-        stand_alone=True  # always give active users its own line
-    )
+    superusers_count = superusers.count()
+    non_admin_count = non_admin_users.count()
+
+    superusers_emoji = ":crown:" if superusers_count > 0 else ":ghost:"
+    non_admin_emoji = ":nerd_face:" if non_admin_count > 0 else ":ghost:"
+
+    superusers_previews = [UserPreview(user).preview for user in superusers_queryset]
+    non_admin_previews = [UserPreview(user).preview for user in non_admin_queryset]
+
+    health_checks = []
+
+    if superusers_count > 0:
+        health_checks.append(
+            HealthCheckRecentActivity(
+                emoji=superusers_emoji,
+                name="Active Admin Users",
+                amount=superusers_count,
+                extra=", ".join(list(superusers)),
+                preview=superusers_previews,
+                stand_alone=True  # always give superusers their own line
+            )
+        )
+
+    if non_admin_count > 0:
+        health_checks.append(
+            HealthCheckRecentActivity(
+                emoji=non_admin_emoji,
+                name="Active Users",
+                amount=non_admin_count,
+                extra=", ".join(list(non_admin_users)),
+                preview=non_admin_previews,
+                stand_alone=True
+            )
+        )
+    return health_checks
 
 
 @receiver(signal=health_check_signal)
@@ -51,7 +74,7 @@ def email_health_check(sender, health_request: HealthCheckRequest, **kwargs):
         email_subj = ", ".join(
             f"{subject} x *{count}*" if count > 1 else subject for subject, count in subject_counts.items()
         )
-        
+
         return HealthCheckRecentActivity(
             emoji=":email:",
             name="Emails Sent",

library/health_check.py

@@ -72,7 +72,7 @@ def is_zero(self):
         # if is_zero (and not stand_alone) then the record can be summarised
         return self.amount == 0 or self.amount is None or self.amount == ""
 
-    def __str__(self):
+    def as_markdown(self):
         amount_str = self.amount
         if self.amount:
             amount_str = f"*{amount_str}*"
@@ -83,6 +83,17 @@ def __str__(self):
             result = f"{result} : {self.extra}"
         return result
 
+    def as_html(self):
+        amount_str = self.amount
+        if self.amount:
+            amount_str = f"<b>{amount_str}</b>"
+        result = f"{amount_str} : {self.name}"
+        if self.sub_type:
+            result = f"{result} {self.sub_type}"
+        if self.extra:
+            result = f"{result} : {self.extra}"
+        return result
+
     @classmethod
     def sort_order(cls):
         return 1
@@ -179,7 +190,7 @@ class HealthCheckTotalAmount(HealthCheckStat):
     amount: int
     extra: Optional[str] = None
 
-    def __str__(self):
+    def as_markdown(self):
         amount_str = self.amount
         if self.amount:
             amount_str = f"*{amount_str:,}*"
@@ -188,6 +199,15 @@ def __str__(self):
             result = f"{result} - {self.extra}"
         return result
 
+    def as_html(self):
+        amount_str = self.amount
+        if self.amount:
+            amount_str = f"<b>{amount_str:,}</b>"
+        result = f"{amount_str} : {self.name}"
+        if self.extra:
+            result = f"{result} - {self.extra}"
+        return result
+
     @classmethod
     def sort_order(cls):
         return 2
@@ -201,10 +221,13 @@ class HealthCheckCapacity(HealthCheckStat):
     available: str
     warning: bool = False
 
-    def __str__(self):
+    def as_markdown(self):
         emoji = ":floppy_disk:" if not self.warning else ":fire:"
         return f"{emoji} {self.name} ({self.used} used, {self.available} available)"
 
+    def as_html(self):
+        return f"{self.name} ({self.used} used, {self.available} available)"
+
     @classmethod
     def sort_order(cls):
         return 3
@@ -238,12 +261,17 @@ def age_in_days(self):
             return age.days
         return HealthCheckAge._NEVER_RUN_AGE
 
-    def __str__(self):
+    def as_markdown(self):
         if not self.last_performed_tz:
             return f":dizzy_face: Never Run : {self.name}"
         emoji = HealthCheckAge._face_for_age(self.age_in_days, self.warning_age)
         return f"{emoji} {self.age_in_days} days old : {self.name}"
 
+    def as_html(self):
+        if not self.last_performed_tz:
+            return f"<b> Never Run </b> : {self.name}"
+        return f"<b> {self.age_in_days} </b> days old : {self.name}"
+
     _MULTIPLIER_TO_FACE = {
         0: ":simple_smile:",
         1: ":neutral_face:",
@@ -281,19 +309,8 @@ def to_lines(cls, items: List['HealthCheckAge'], health_request: HealthCheckRequ
         return lines
 
 
-@dataclass
-class HealthCheckCustom(HealthCheckStat):
-    text: str
-
-    def __str__(self):
-        return self.text
-
-    @classmethod
-    def sort_order(cls):
-        return 5
-
-
 health_check_signal = django.dispatch.Signal()
+health_check_overall_stats_signal = django.dispatch.Signal()
 
 
 def populate_health_check(notification: NotificationBuilder, since: Optional[datetime] = None):
@@ -311,20 +328,34 @@ def populate_health_check(notification: NotificationBuilder, since: Optional[dat
         if isinstance(result, Exception):
             notification.add_markdown(f"Exception generating health check by {caller}: {result}")
         else:
-            results.append(result)
+            results.extend(result if isinstance(result, list) else [result])
+
+    for caller, result in health_check_overall_stats_signal.send_robust(sender=None, health_request=health_request):
+        if isinstance(result, Exception):
+            notification.add_markdown(f"Exception generating health check by {caller}: {result}")
+        else:
+            results.extend(result if isinstance(result, list) else [result])
+
     checks: List[HealthCheckStat] = flatten_nested_lists(results)
+    checks.sort(key=lambda hc: hc.sort_order())
 
-    checks = sorted(checks, key=lambda hc: hc.sort_order())
-    grouped_checks = [(key, list(values)) for key, values in itertools.groupby(checks, type)]
-    grouped_checks = sorted(grouped_checks, key=lambda gc: gc[0].sort_order())
     recent_lines = []
     overall_lines = []
-    for check_type, checks_typed in grouped_checks:
-        section_lines = check_type.to_lines(checks_typed, health_request=health_request)
-        if check_type.is_recent_activity():
-            recent_lines.extend(section_lines)
+    for check_type, checks_typed in itertools.groupby(checks, key=lambda hc: type(hc)):
+        checks_typed_list = list(checks_typed)
+
+        if issubclass(check_type, HealthCheckAge):
+            age_lines = HealthCheckAge.to_lines(checks_typed_list, health_request)
+            if check_type.is_recent_activity():
+                recent_lines.extend(age_lines)
+            else:
+                overall_lines.extend(age_lines)
         else:
-            overall_lines.extend(section_lines)
+            section_lines = [check.as_markdown() for check in checks_typed_list]
+            if check_type.is_recent_activity():
+                recent_lines.extend(section_lines)
+            else:
+                overall_lines.extend(section_lines)
 
     if recent_lines:
         notification.add_markdown("\n".join(recent_lines), indented=True)

ontology/signals/ontology_health_check.py

@@ -3,11 +3,12 @@
 
 from django.dispatch import receiver
 
-from library.health_check import health_check_signal, HealthCheckRequest, HealthCheckAge
+from library.health_check import HealthCheckRequest, HealthCheckAge, \
+    health_check_overall_stats_signal
 from ontology.models import OntologyImport
 
 
-@receiver(signal=health_check_signal)
+@receiver(signal=health_check_overall_stats_signal)
 def ontology_health_check(sender, health_request: HealthCheckRequest, **kwargs):
     checks = []
     warning_age = timedelta(days=60)

snpdb/signals/disk_usage_health_check.py

@@ -1,10 +1,11 @@
 from django.dispatch import receiver
 
-from library.health_check import health_check_signal, HealthCheckRequest, HealthCheckCapacity
+from library.health_check import HealthCheckRequest, HealthCheckCapacity, \
+    health_check_overall_stats_signal
 from variantgrid.tasks.server_monitoring_tasks import get_disk_usage_objects
 
 
-@receiver(signal=health_check_signal)
+@receiver(signal=health_check_overall_stats_signal)
 def disk_usage_health_check(sender, health_request: HealthCheckRequest, **kwargs):
     checks = []
     for disk_usage in get_disk_usage_objects():