Skip to content

Commit c25cc96

Browse files
marcelklehrmarcelklehr
authored
Merge pull request #1318 from nextcloud/fix/duplicate-face-detections
fix(FaceDetectionMapper): Prevent inserting duplicate face detections
2 parents 47ae1c6 + 9288f35 commit c25cc96

File tree

4 files changed

+171
-0
lines changed

4 files changed

+171
-0
lines changed

‎appinfo/info.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@ The app does not send any sensitive data to cloud providers or similar services.
101101
<repair-steps>
102102
<post-migration>
103103
<step>OCA\Recognize\Migration\InstallDeps</step>
104+
<step>OCA\Recognize\Migration\RemoveDuplicateFaceDetections</step>
104105
</post-migration>
105106
<install>
106107
<step>OCA\Recognize\Migration\InstallDeps</step>

‎lib/Db/FaceDetectionMapper.php‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
use OCP\AppFramework\Db\DoesNotExistException;
1212
use OCP\AppFramework\Db\Entity;
1313
use OCP\AppFramework\Db\QBMapper;
14+
use OCP\DB\Exception;
1415
use OCP\DB\QueryBuilder\IQueryBuilder;
1516
use OCP\IConfig;
1617
use OCP\IDBConnection;
@@ -42,6 +43,35 @@ public function find(int $id): FaceDetection {
4243
return $this->findEntity($qb);
4344
}
4445

46+
/**
47+
* @throws \OCP\DB\Exception
48+
*/
49+
public function insert(Entity $entity): FaceDetection {
50+
$qb = $this->db->getQueryBuilder();
51+
$qb->select(FaceDetection::$columns)
52+
->from('recognize_face_detections')
53+
->where($qb->expr()->eq('file_id', $qb->createPositionalParameter($entity->getFileId(), IQueryBuilder::PARAM_INT)))
54+
->andWhere($qb->expr()->eq('user_id', $qb->createPositionalParameter($entity->getUserId(), IQueryBuilder::PARAM_STR)))
55+
->andWhere($qb->expr()->eq('x', $qb->createPositionalParameter($entity->getX(), IQueryBuilder::PARAM_INT)))
56+
->andWhere($qb->expr()->eq('y', $qb->createPositionalParameter($entity->getY(), IQueryBuilder::PARAM_INT)))
57+
->andWhere($qb->expr()->eq('height', $qb->createPositionalParameter($entity->getHeight(), IQueryBuilder::PARAM_INT)))
58+
->andWhere($qb->expr()->eq('width', $qb->createPositionalParameter($entity->getWidth(), IQueryBuilder::PARAM_INT)));
59+
$duplicates = $this->findEntities($qb);
60+
61+
if (empty($duplicates)) {
62+
return parent::insert($entity);
63+
}
64+
65+
return $duplicates[0];
66+
}
67+
68+
/**
69+
* @throws Exception
70+
*/
71+
public function insertWithoutDeduplication(Entity $entity): FaceDetection {
72+
return parent::insert($entity);
73+
}
74+
4575
/**
4676
* @throws \OCP\DB\Exception
4777
*/

‎lib/Migration/RemoveDuplicateFaceDetections.php‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
/**
5+
* @copyright Copyright (c) 2025, Marcel Klehr <[email protected]>
6+
*
7+
* @author Marcel Klehr <[email protected]>
8+
*
9+
* @license GNU AGPL version 3 or any later version
10+
*
11+
* This program is free software: you can redistribute it and/or modify
12+
* it under the terms of the GNU Affero General Public License as
13+
* published by the Free Software Foundation, either version 3 of the
14+
* License, or (at your option) any later version.
15+
*
16+
* This program is distributed in the hope that it will be useful,
17+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
18+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19+
* GNU Affero General Public License for more details.
20+
*
21+
* You should have received a copy of the GNU Affero General Public License
22+
* along with this program. If not, see <http://www.gnu.org/licenses/>.
23+
*
24+
*/
25+
namespace OCA\Recognize\Migration;
26+
27+
use OCP\IDBConnection;
28+
use OCP\Migration\IOutput;
29+
use OCP\Migration\IRepairStep;
30+
use Psr\Log\LoggerInterface;
31+
32+
final class RemoveDuplicateFaceDetections implements IRepairStep {
33+
34+
public function __construct(
35+
private IDBConnection $db,
36+
private LoggerInterface $logger,
37+
) {
38+
}
39+
40+
public function getName(): string {
41+
return 'Remove duplicate face detections';
42+
}
43+
44+
public function run(IOutput $output): void {
45+
try {
46+
$subQuery = $this->db->getQueryBuilder();
47+
$subQuery->select($subQuery->func()->min('id'))
48+
->from('recognize_face_detections')
49+
->groupBy('file_id', 'user_id', 'x', 'y', 'height', 'width');
50+
51+
$qb = $this->db->getQueryBuilder();
52+
$qb->delete('recognize_face_detections')
53+
->where($qb->expr()->notIn('id', $qb->createFunction('(' . $subQuery->getSQL() .')')));
54+
55+
$qb->executeStatement();
56+
} catch (\Throwable $e) {
57+
$output->warning('Failed to automatically remove duplicate face detections for recognize.');
58+
$this->logger->error('Failed to automatically remove duplicate face detections', ['exception' => $e]);
59+
}
60+
}
61+
}

‎tests/RemoveDuplicateFaceDetectionsTest.php‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
<?php
2+
3+
use OCA\Recognize\Db\FaceDetectionMapper;
4+
use OCA\Recognize\Migration\RemoveDuplicateFaceDetections;
5+
use OCP\IDBConnection;
6+
use OCP\Server;
7+
use Test\TestCase;
8+
9+
/**
10+
* @group DB
11+
*/
12+
class RemoveDuplicateFaceDetectionsTest extends TestCase {
13+
private IDBConnection $db;
14+
private FaceDetectionMapper $faceDetectionMapper;
15+
16+
public function setUp(): void {
17+
parent::setUp();
18+
$this->db = Server::get(IDBConnection::class);
19+
$this->faceDetectionMapper = Server::get(FaceDetectionMapper::class);
20+
21+
// Clear
22+
$qb = $this->db->getQueryBuilder();
23+
$qb->delete('recognize_face_detections')->executeStatement();
24+
25+
// Generate 11 face detections per file (1000 files per user; 100 users)
26+
// = 1.100.000 face detections out of which 900.000 are superfluous duplicates to be removed
27+
// After the repair step there should be 200.000 left
28+
for ($k = 0; $k < 100; $k++) {
29+
for ($j = 0; $j < 1000; $j++) {
30+
$user = 'user' . $k;
31+
$x = rand(0, 100) / 100;
32+
$y = rand(0, 100) / 100;
33+
$height = rand(0, 100) / 100;
34+
$width = rand(0, 100) / 100;
35+
for ($i = 0; $i < 10; $i++) {
36+
$face = new \OCA\Recognize\Db\FaceDetection();
37+
$face->setUserId($user);
38+
$face->setX($x);
39+
$face->setY($y);
40+
$face->setHeight($height);
41+
$face->setWidth($width);
42+
$face->setFileId($j);
43+
$face->setThreshold(0.5);
44+
$face->setVector([1, 2, 3, 4, 5, 6, 7, 8, 9, 0]);
45+
$this->faceDetectionMapper->insertWithoutDeduplication($face);
46+
}
47+
$face2 = new \OCA\Recognize\Db\FaceDetection();
48+
$face2->setUserId($user);
49+
$face2->setX(rand(0, 100) / 100);
50+
$face2->setY(rand(0, 100) / 100);
51+
$face2->setHeight(rand(0, 100) / 100);
52+
$face2->setWidth(rand(0, 100) / 100);
53+
$face2->setFileId($k * $j);
54+
$face2->setThreshold(0.5);
55+
$face2->setVector([1, 2, 3, 4, 5, 6, 7, 8, 9, 0]);
56+
$this->faceDetectionMapper->insertWithoutDeduplication($face2);
57+
}
58+
}
59+
}
60+
61+
public function testRepairStep() : void {
62+
// Prepare
63+
$repairStep = Server::get(RemoveDuplicateFaceDetections::class);
64+
$output = $this->createMock(\OCP\Migration\IOutput::class);
65+
66+
// Check
67+
$qb = $this->db->getQueryBuilder();
68+
$count = $qb->select($qb->func()->count('*'))->from('recognize_face_detections')->executeQuery()->fetchOne();
69+
$this->assertEquals(1100000, (int)$count);
70+
71+
// Run
72+
$repairStep->run($output);
73+
74+
// Assert
75+
$qb = $this->db->getQueryBuilder();
76+
$count = $qb->select($qb->func()->count('*'))->from('recognize_face_detections')->executeQuery()->fetchOne();
77+
$this->assertEquals(200000, (int)$count);
78+
}
79+
}

0 commit comments

Comments
 (0)