Skip to content

Commit 26a476b

Browse files
flamsensflamsens
authored
Merge pull request #65 from GhentCDH/perf/elastic_indexing
Perf/elastic indexing
2 parents 54dae79 + 204cb54 commit 26a476b

14 files changed

+76
-47
lines changed

‎app/migrations/060-data-cleanup.sql

+5-1
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,8 @@
22

33
update charter_actor
44
set role_nl = 'Beneficiaris'
5-
where role_nl = 'Bénéficiaire';
5+
where role_nl = 'Bénéficiaire';
6+
7+
update charter_type
8+
set type_nl = 'charter'
9+
where type_nl = 'charte';

‎app/src/Command/IndexElasticsearchCommand.php

+18-17
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ protected function configure()
4242
protected function execute(InputInterface $input, OutputInterface $output): int
4343
{
4444
$io = new SymfonyStyle($input, $output);
45+
$chunkSize = 200;
4546

4647
$count = 0;
4748
$maxItems = $input->getArgument('maxItems');
@@ -73,22 +74,22 @@ protected function execute(InputInterface $input, OutputInterface $output): int
7374
$progressBar->start();
7475

7576
/*** @var $repository RepositoryInterface */
77+
Charter::setLocale('en');
7678
foreach( $repositories as $repository_name ) {
7779
$repository = $this->container->get($repository_name);
78-
$repository->indexQuery()->chunk(100,
79-
function($res) use ($service, &$count, $progressBar, $maxItems) {
80+
$repository->indexQuery()->chunk($chunkSize,
81+
function($traditions) use ($service, &$count, $progressBar, $maxItems, $chunkSize) {
8082
if ( $maxItems && $count >= $maxItems ) {
8183
return false;
8284
}
8385

84-
/** @var Charter $charter */
85-
foreach ($res as $charter) {
86-
$res = new ElasticTraditionResource($charter->translate('en'));
87-
$service->add($res);
88-
$count++;
89-
}
86+
// index traditions
87+
$traditionResources = ElasticTraditionResource::collection($traditions);
88+
$count += $traditionResources->count();
89+
$service->addMultiple($traditionResources);
9090

91-
$progressBar->advance(100);
91+
// update progress bar
92+
$progressBar->advance($chunkSize);
9293
});
9394
}
9495

@@ -112,19 +113,19 @@ function($res) use ($service, &$count, $progressBar, $maxItems) {
112113
$progressBar->start();
113114

114115
$repository->indexQuery()->chunk(100,
115-
function($res) use ($service, &$count, $progressBar, $maxItems) {
116+
function($charters) use ($service, &$count, $progressBar, $maxItems, $chunkSize): bool {
116117
if ( $maxItems && $count >= $maxItems ) {
117118
return false;
118119
}
119120

120-
/** @var Charter $charter */
121-
foreach ($res as $charter) {
122-
$res = new ElasticCharterResource($charter->translate('en'));
123-
$service->add($res);
124-
$count++;
125-
}
121+
// index charters
122+
$charterResources = ElasticCharterResource::collection($charters);
123+
$count += $charterResources->count();
124+
$service->addMultiple($charterResources);
126125

127-
$progressBar->advance(100);
126+
// update progress bar
127+
$progressBar->advance($chunkSize);
128+
return true;
128129
});
129130

130131
$service->switchToNewIndex($indexName);

‎app/src/Controller/BaseController.php

-2
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,7 @@
22
namespace App\Controller;
33

44
use App\Service\ElasticSearch\Base\SearchServiceInterface;
5-
use Illuminate\Container\Container;
65
use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
7-
use Symfony\Component\DependencyInjection\ContainerInterface;
86
use Symfony\Component\HttpFoundation\JsonResponse;
97
use Symfony\Component\HttpFoundation\Request;
108
use Symfony\Component\HttpFoundation\Response;

‎app/src/Model/ActorName.php

+1-1
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ class ActorName extends AbstractModel
2727
'actor_standardized_name_published' => 'boolean'
2828
];
2929
protected $with = [
30-
'standardizedName'
30+
// 'standardizedName'
3131
];
3232

3333
/**

‎app/src/Model/Charter.php

+20-4
Original file line numberDiff line numberDiff line change
@@ -51,10 +51,11 @@ class Charter extends AbstractModel
5151
'edition_indication_published' => 'boolean'
5252
];
5353
protected $with = [
54-
'place', 'edition_indication', 'edition', 'authenticity',
55-
'nature', 'language', 'type', 'udt', 'actors', 'actors.pivot.role', 'codexes',
56-
'edition_indications', 'secondary_literature_indications',
57-
'copies', 'datations', 'originals', 'vidimuses'
54+
// 'place',
55+
// 'edition_indication', 'edition', 'authenticity',
56+
// 'nature', 'language', 'type', 'udt', 'actors', 'actors.pivot.role', 'codexes',
57+
// 'edition_indications', 'secondary_literature_indications',
58+
// 'copies', 'datations', 'originals', 'vidimuses'
5859
];
5960

6061
/**
@@ -140,6 +141,21 @@ public function actors(): BelongsToMany
140141
->withPivot('charter_actor_role_id');
141142
}
142143

144+
public function issuers(): BelongsToMany
145+
{
146+
return $this->actors()->using(Charter_Actor::class)->wherePivot('charter_actor_role_id', 2);
147+
}
148+
149+
public function beneficiaries(): BelongsToMany
150+
{
151+
return $this->actors()->using(Charter_Actor::class)->wherePivot('charter_actor_role_id', 3);
152+
}
153+
154+
public function disposants(): BelongsToMany
155+
{
156+
return $this->actors()->using(Charter_Actor::class)->wherePivot('charter_actor_role_id', 1);
157+
}
158+
143159
/**
144160
* @return BelongsToMany|Collection|Codex[]
145161
* @throws ReflectionException

‎app/src/Model/TraitLocalizedAttributes.php

+4
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,10 @@ public function translate(string $locale) {
3131
return $this;
3232
}
3333

34+
public static function setLocale(string $locale) {
35+
static::$locale = $locale;
36+
}
37+
3438
/**
3539
* Check if attributed is localized
3640
* @param string $key

‎app/src/Repository/CharterRepository.php

+9-1
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,14 @@
1111
class CharterRepository extends AbstractRepository
1212
{
1313
// auto load relations
14-
protected $relations = [];
14+
protected $relations = [
15+
'place', 'place.localisation', 'place.diocese', 'place.principality', 'place.placeLocalisation',
16+
'edition_indication', 'edition', 'authenticity',
17+
'nature', 'language', 'type', 'udt', 'actors', 'actors.pivot.role', 'codexes',
18+
'edition_indications', 'secondary_literature_indications',
19+
'copies',
20+
'originals', 'vidimuses',
21+
'datations', 'datations.time'
22+
];
1523
protected $model = Charter::class;
1624
}

‎app/src/Repository/CodexRepository.php

+5-1
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,10 @@
1010
class CodexRepository extends AbstractRepository
1111
{
1212
// auto load relations
13-
protected $relations = [];
13+
protected $relations = [
14+
'charters',
15+
'charters.datations',
16+
'charters.actors',
17+
];
1418
protected $model = Codex::class;
1519
}

‎app/src/Repository/CopyRepository.php

+5-1
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,10 @@
1111
class CopyRepository extends AbstractRepository
1212
{
1313
// auto load relations
14-
protected $relations = [];
14+
protected $relations = [
15+
'charter',
16+
'charter.datations',
17+
'charter.actors',
18+
];
1519
protected $model = Copy::class;
1620
}

‎app/src/Repository/OriginalRepository.php

+5-1
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,10 @@
1111
class OriginalRepository extends AbstractRepository
1212
{
1313
// auto load relations
14-
protected $relations = [];
14+
protected $relations = [
15+
'charter',
16+
'charter.datations',
17+
'charter.actors',
18+
];
1519
protected $model = Original::class;
1620
}

‎app/src/Resource/ElasticSearch/ElasticBaseResource.php

+1-1
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ public function attributesToArray(bool $hideForeignKeys = false)
4444
return [];
4545
}
4646

47-
$ret = parent::toArray(null);
47+
$ret = $this->resource->attributesToArray();
4848

4949
// add id_name if resource has name property
5050
$ret['id_name'] = $this->when(is_string($this->resource->getAttribute('name')), $this->resource->getId().'_'.$this->resource->name);

‎app/src/Resource/ElasticSearch/ElasticTraditionCharterResource.php

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ public function toArray($request = null): ?array
1616
$actor = $this->resource->actors;
1717
$ret['id'] = $this->resource->getId();
1818
$ret['datations'] = $this->resource->datations;
19-
$ret['actors'] = ElasticActorResource::collection ($actor);
19+
$ret['actors'] = ElasticActorResource::collection($actor);
2020

2121
return $ret;
2222
}

‎app/src/Resource/ElasticSearch/ElasticTraditionResource.php

+2-2
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ public function toArray($request=null)
2929
{
3030
$resource = $this->resource;
3131

32-
$ret = $this->attributesToArray(true);
32+
$ret = $this->attributesToArray();
3333

3434
// shared relations
3535
$ret['urls'] = ElasticBaseResource::collection($resource->urls);
@@ -53,7 +53,7 @@ public function toArray($request=null)
5353
// $ret['charters'] = ElasticTraditionCharterResource::collection($this->charters);
5454
// }
5555

56-
if ($ret['type'] == 'original' or 'copy') {
56+
if ($ret['type'] == 'original' || $ret['type'] == 'copy') {
5757
$ret['charters'] = ElasticTraditionCharterResource::collection([$this->charter]);
5858
}
5959

‎app/src/Service/ElasticSearch/Base/AbstractIndexService.php

-14
Original file line numberDiff line numberDiff line change
@@ -68,20 +68,6 @@ public function switchToNewIndex(string $newIndexName): void
6868

6969
public function addMultiple(ResourceCollection $resources): void
7070
{
71-
/*
72-
$json_array = $resources->toJson();
73-
74-
$bulk_documents = [];
75-
while (count($elastics) > 0) {
76-
$bulk_contents = array_splice($elastics, 0, 500);
77-
foreach ($bulk_contents as $bc) {
78-
$bulk_documents[] = new Document($resource->getId(), $resource->getJson());
79-
}
80-
$this->getIndex()->addDocuments($bulk_documents);
81-
$bulk_documents = [];
82-
}
83-
$this->getIndex()->refresh();
84-
*/
8571
$documents = [];
8672
foreach( $resources as $resource ) {
8773
$documents[] = new Document($resource->getId(), $resource->toJson());

0 commit comments

Comments
 (0)