[IMP] runbot: add common and unique qualifiers on build error

This commit adds a `common_qualifiers` field on build error. Its purpose
is mainly to find similary qualified errors and similary qualified error
contents. This field is computed by finding qualifiers in common in all
the qualifiers of the error contents linked to the error.

A new `unique_qualifiers` is also added for the same kind of puprpose.
This field is computed by finding non contradictory qualifiers of the
linked error contents.

The fields can be used in 4 tabs added on the build error form.

runbot/models/build_error.py

@@ -103,6 +103,13 @@ class BuildError(models.Model):
     tags_min_version_id = fields.Many2one('runbot.version', 'Tags Min version', help="Minimal version where the test tags will be applied.")
     tags_max_version_id = fields.Many2one('runbot.version', 'Tags Max version', help="Maximal version where the test tags will be applied.")
 
+    common_qualifiers = JsonDictField('Common Qualifiers', compute='_compute_common_qualifiers', store=True, help="Minimal qualifiers in common needed to link error content.")
+    similar_ids = fields.One2many('runbot.build.error', compute='_compute_similar_ids', string="Similar Errors", help="Similar Errors based on common qualifiers")
+    similar_content_ids = fields.One2many('runbot.build.error.content', compute='_compute_similar_content_ids', string="Similar Error Contents", help="Similar Error contents based on common qualifiers")
+    unique_qualifiers = JsonDictField('Non conflicting Qualifiers', compute='_compute_unique_qualifiers', store=True, help="Non conflicting qualifiers in common needed to link error content.")
+    analogous_ids = fields.One2many('runbot.build.error', compute='_compute_analogous_ids', string="Analogous Errors", help="Analogous Errors based on unique qualifiers")
+    analogous_content_ids= fields.One2many('runbot.build.error.content', compute='_compute_analogous_content_ids', string="Analogous Error Contents", help="Analogous Error contents based on unique qualifiers")
+
     # Build error related data
     build_error_link_ids = fields.Many2many('runbot.build.error.link', compute=_compute_related_error_content_ids('build_error_link_ids'), search=_search_related_error_content_ids('build_error_link_ids'))
     unique_build_error_link_ids = fields.Many2many('runbot.build.error.link', compute='_compute_unique_build_error_link_ids')
@@ -149,6 +156,84 @@ def _compute_random(self):
         for record in self:
             record.random = any(error.random for error in record.error_content_ids)
 
+    @api.depends('error_content_ids.qualifiers')
+    def _compute_common_qualifiers(self):
+        for record in self:
+            qualifiers = defaultdict(set)
+            key_count = defaultdict(int)
+            for content in record.error_content_ids:
+                for key, value in content.qualifiers.dict.items():
+                    qualifiers[key].add(value)
+                    key_count[key] += 1
+            record.common_qualifiers = {k: v.pop() for k, v in qualifiers.items() if len(v) == 1 and key_count[k] == len(record.error_content_ids)}
+
+    @api.depends('error_content_ids.qualifiers')
+    def _compute_unique_qualifiers(self):
+        for record in self:
+            qualifiers = defaultdict(set)
+            key_count = defaultdict(int)
+            for content in record.error_content_ids:
+                for key, value in content.qualifiers.dict.items():
+                    qualifiers[key].add(value)
+                    key_count[key] += 1
+            record.unique_qualifiers = {k: v.pop() for k, v in qualifiers.items() if len(v) == 1}
+
+    @api.depends('common_qualifiers')
+    def _compute_similar_ids(self):
+        for record in self:
+            if record.common_qualifiers:
+                query = SQL(
+                    r"""SELECT id FROM runbot_build_error WHERE id != %s AND common_qualifiers @> %s""",
+                    record.id,
+                    json.dumps(record.common_qualifiers.dict),
+                )
+                self.env.cr.execute(query)
+                record.similar_ids = self.env['runbot.build.error'].browse([rec[0] for rec in self.env.cr.fetchall()])
+            else:
+                record.similar_ids = False
+
+    @api.depends('common_qualifiers')
+    def _compute_similar_content_ids(self):
+        for record in self:
+            if record.common_qualifiers:
+                query = SQL(
+                    r"""SELECT id FROM runbot_build_error_content WHERE error_id != %s AND qualifiers @> %s""",
+                    record.id,
+                    json.dumps(record.common_qualifiers.dict),
+                )
+                self.env.cr.execute(query)
+                record.similar_content_ids = self.env['runbot.build.error.content'].browse([rec[0] for rec in self.env.cr.fetchall()])
+            else:
+                record.similar_content_ids = False
+
+    @api.depends('common_qualifiers')
+    def _compute_analogous_ids(self):
+        for record in self:
+            if record.common_qualifiers:
+                query = SQL(
+                    r"""SELECT id FROM runbot_build_error WHERE id != %s AND unique_qualifiers @> %s""",
+                    record.id,
+                    json.dumps(record.unique_qualifiers.dict),
+                )
+                self.env.cr.execute(query)
+                record.analogous_ids = self.env['runbot.build.error'].browse([rec[0] for rec in self.env.cr.fetchall()])
+            else:
+                record.analogous_ids = False
+
+    @api.depends('common_qualifiers')
+    def _compute_analogous_content_ids(self):
+        for record in self:
+            if record.common_qualifiers:
+                query = SQL(
+                    r"""SELECT id FROM runbot_build_error_content WHERE error_id != %s AND qualifiers @> %s""",
+                    record.id,
+                    json.dumps(record.unique_qualifiers.dict),
+                )
+                self.env.cr.execute(query)
+                record.analogous_content_ids = self.env['runbot.build.error.content'].browse([rec[0] for rec in self.env.cr.fetchall()])
+            else:
+                record.analogous_content_ids = False
+
 
     @api.constrains('test_tags')
     def _check_test_tags(self):
@@ -202,6 +287,7 @@ def _merge(self, others):
                 if not error.team_id:
                     error.team_id = previous_error.team_id
             previous_error.error_content_ids.write({'error_id': self})
+            previous_error.common_qualifiers = dict()
             if not previous_error.test_tags:
                 previous_error.message_post(body=Markup('Error merged into %s') % error._get_form_link())
                 previous_error.active = False
@@ -245,6 +331,50 @@ def action_view_errors(self):
             'target': 'current',
         }
 
+    def action_view_similary_qualified(self):
+        return {
+            'type': 'ir.actions.act_window',
+            'views': [(False, 'list'), (False, 'form')],
+            'res_model': 'runbot.build.error',
+            'domain': [('id', 'in', [self.id] + self.similar_ids.ids)],
+            'context': {'active_test': False},
+            'target': 'current',
+            'name': 'Similary Qualified Errors'
+        }
+
+    def action_view_similary_qualified_contents(self):
+        return {
+            'type': 'ir.actions.act_window',
+            'views': [(False, 'list'), (False, 'form')],
+            'res_model': 'runbot.build.error.content',
+            'domain': [('id', 'in', self.similar_content_ids.ids)],
+            'context': {'active_test': False},
+            'target': 'current',
+            'name': 'Similary Qualified Contents'
+        }
+
+    def action_view_analogous_qualified(self):
+        return {
+            'type': 'ir.actions.act_window',
+            'views': [(False, 'list'), (False, 'form')],
+            'res_model': 'runbot.build.error',
+            'domain': [('id', 'in', [self.id] + self.analogous_ids.ids)],
+            'context': {'active_test': False},
+            'target': 'current',
+            'name': 'Similary Qualified Errors'
+        }
+
+    def action_view_analogous_qualified_contents(self):
+        return {
+            'type': 'ir.actions.act_window',
+            'views': [(False, 'list'), (False, 'form')],
+            'res_model': 'runbot.build.error.content',
+            'domain': [('id', 'in', self.analogous_content_ids.ids)],
+            'context': {'active_test': False},
+            'target': 'current',
+            'name': 'Similary Qualified Contents'
+        }
+
     def action_assign(self):
         teams = None
         repos = None
@@ -731,7 +861,8 @@ def _qualify(self, build_error_content):
         result = False
         if content and self.regex:
             result = re.search(self.regex, content, flags=re.MULTILINE)
-        return result.groupdict() if result else {}
+        # filtering empty values to allow non mandatory named groups
+        return {k:v for k,v in result.groupdict().items() if v} if result else {}
 
 
 class QualifyErrorTest(models.Model):

runbot/tests/test_build_error.py

@@ -529,6 +529,61 @@ def test_build_error_test_tags_fixing_pr(self):
         self.assertEqual([], self.BuildError._disabling_tags(build_fixing))
         self.assertEqual(['-bar'], self.BuildError._disabling_tags(build_random))
 
+    def test_build_error_qualifers(self):
+        error_contents = self.BuildErrorContent.create(
+            [
+                {
+                    "content": "Tour foobar_tour failed at step click_here in mode admin",
+                },
+                                {
+                    "content": "Tour foobar_tour failed at step click_here in mode demo",
+                },
+                {
+                    "content": "Tour foobar_tour failed -> click_here",
+                },
+                {
+                    "content": "Tour foobar_tour failed",
+                },
+            ]
+        )
+        self.assertEqual(len(error_contents), 4)
+        self.assertEqual(len(self.BuildError.search([('error_content_ids', 'in', error_contents.ids)])), 4)
+
+        self.env['runbot.error.qualify.regex'].create({
+            "regex": r"Tour (?P<tour_name>\w+) failed( at step (?P<tour_step>\w+) in mode (?P<tour_mode>\w+))?"
+        })
+        error_contents._qualify()
+
+        expected_common_qualifiers = {'tour_name': 'foobar_tour', 'tour_step': 'click_here'}
+        self.assertEqual(error_contents[0].qualifiers.dict, {**expected_common_qualifiers, 'tour_mode': 'admin'})
+        self.assertEqual(error_contents[1].qualifiers.dict, {**expected_common_qualifiers, 'tour_mode': 'demo'})
+        self.assertEqual(error_contents[2].qualifiers.dict, {'tour_name': 'foobar_tour'})
+
+        self.env['runbot.error.qualify.regex'].create({
+            "regex": r"Tour (?P<tour_name>\w+) failed -> (?P<tour_step>\w+)"
+        })
+
+        error_contents._qualify()
+        self.assertEqual(error_contents[0].qualifiers.dict, {**expected_common_qualifiers, 'tour_mode': 'admin'})
+        self.assertEqual(error_contents[1].qualifiers.dict, {**expected_common_qualifiers, 'tour_mode': 'demo'})
+        self.assertEqual(error_contents[2].qualifiers.dict, expected_common_qualifiers)
+
+        # now let's say that we merge admin and demo errors
+        main_error = error_contents[0].error_id
+        self.assertEqual(len(main_error.error_content_ids), 1)
+        main_error._merge(error_contents[1].error_id)
+        self.assertEqual(len(main_error.error_content_ids), 2)
+
+        self.assertEqual(main_error.common_qualifiers.dict, expected_common_qualifiers)
+        self.env.flush_all()
+        self.assertEqual(len(main_error.similar_ids), 1)
+        self.assertEqual(main_error.similar_ids[0], error_contents[2].error_id)
+
+        # let's merge all errors and verify the qualifiers
+        main_error._merge(error_contents.error_id)
+        self.assertEqual(main_error.common_qualifiers.dict, {'tour_name': 'foobar_tour'})
+        self.assertEqual(main_error.unique_qualifiers.dict, {'tour_name': 'foobar_tour', 'tour_step': 'click_here'})
+
     def test_build_error_team_wildcards(self):
         website_team = self.RunbotTeam.create({
             'name': 'website_test',