v1.190.0-rc1

.github/dependabot.yml

@@ -1,12 +1,50 @@
 version: 2
 updates:
-  - package-ecosystem: "npm"
-    directory: "/"
+  - package-ecosystem: npm
+    directory: /
     schedule:
-      interval: "daily"
+      interval: daily
     ignore:
       - dependency-name: "@types/*"
+      - dependency-name: "@sentry/node"
+        versions: [">= 7.114.0"]
     open-pull-requests-limit: 5
     labels:
-      - "dependencies"
-    rebase-strategy: 'disabled'
+      - dependencies
+    rebase-strategy: disabled
+    groups:
+      dev-minor-dependencies:
+        applies-to: version-updates
+        update-types: [minor, patch]
+        patterns:
+          - "*"
+        exclude-patterns:
+          - "@babel*"
+          - "@storybook*"
+          - "@sentry*"
+          - "@dnd-kit"
+          - socket.io
+      babel:
+        applies-to: version-updates
+        patterns:
+          - "@babel*"
+      storybook:
+        applies-to: version-updates
+        patterns:
+          - "@storybook*"
+      sentry:
+        applies-to: version-updates
+        patterns:
+          - "@sentry*"
+      dnd-kit:
+        applies-to: version-updates
+        patterns:
+          - "@dnd-kit*"
+      socket.io:
+        applies-to: version-updates
+        patterns:
+          - socket.io*
+      eslint:
+        applies-to: version-updates
+        patterns:
+          - socket.io*

app/api/migrations/migrations/169-reindex_persist_filename_with_fullText_object/index.ts

@@ -0,0 +1,17 @@
+import { Db } from 'mongodb';
+
+export default {
+  delta: 169,
+
+  name: 'reindex_persist_filename_with_fullText_object',
+
+  description:
+    "We're now indexing document.filename within the fullText object on elasticsearch, this is usefull because on search/v2 endpoint we need to return which filename each text snippet belongs to.",
+
+  reindex: true,
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  async up(db: Db) {
+    process.stdout.write(`${this.name}...\r\n`);
+  },
+};

app/api/migrations/migrations/169-reindex_persist_filename_with_fullText_object/specs/169-reindex_persist_filename_with_fullText_object.spec.ts

@@ -0,0 +1,37 @@
+import { Db } from 'mongodb';
+
+import testingDB from 'api/utils/testing_db';
+import migration from '../index';
+import { Fixture } from '../types';
+import { fixtures } from './fixtures';
+
+let db: Db | null;
+
+const initTest = async (fixture: Fixture) => {
+  await testingDB.setupFixturesAndContext(fixture);
+  db = testingDB.mongodb!;
+  await migration.up(db);
+};
+
+beforeAll(async () => {
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  jest.spyOn(process.stdout, 'write').mockImplementation((str: string | Uint8Array) => true);
+});
+
+afterAll(async () => {
+  await testingDB.tearDown();
+});
+
+describe('migration test', () => {
+  beforeAll(async () => {
+    await initTest(fixtures);
+  });
+
+  it('should have a delta number', () => {
+    expect(migration.delta).toBe(169);
+  });
+
+  it('should check if a reindex is needed', async () => {
+    expect(migration.reindex).toBe(true);
+  });
+});

app/api/migrations/migrations/169-reindex_persist_filename_with_fullText_object/specs/fixtures.ts

@@ -0,0 +1,13 @@
+import { ObjectId } from 'mongodb';
+import { Fixture } from '../types';
+
+const fixtures: Fixture = {
+  entities: [
+    {
+      _id: new ObjectId(),
+      title: 'test_doc',
+    },
+  ],
+};
+
+export { fixtures };

app/api/migrations/migrations/169-reindex_persist_filename_with_fullText_object/types.ts

@@ -0,0 +1,13 @@
+import { ObjectId } from 'mongodb';
+
+interface Entity {
+  _id: ObjectId;
+  title: string;
+  [k: string]: unknown | undefined;
+}
+
+interface Fixture {
+  entities: Entity[];
+}
+
+export type { Entity, Fixture };

app/api/search.v2/buildQuery.ts

@@ -53,7 +53,9 @@ const fullTextSearch = (
             type: 'fullText',
             score_mode: 'max',
             inner_hits: {
-              _source: false,
+              _source: {
+                excludes: ['fullText*'],
+              },
               ...snippetsHighlight(query, [{ 'fullText_*': {} }]),
             },
             query: {

app/api/search.v2/searchResponse.ts

@@ -12,10 +12,10 @@ function getSnippetsForNonFullText(hit: ElasticHit<EntitySchema>) {
 }
 
 function extractFullTextSnippets(hit: ElasticHit<EntitySchema>) {
-  const fullTextSnippets: { text: string; page: number }[] = [];
+  const fullTextSnippets: { text: string; page: number; filename: string }[] = [];
 
   if (hit.inner_hits && hit.inner_hits.fullText.hits.hits[0]?.highlight) {
-    const { highlight } = hit.inner_hits.fullText.hits.hits[0];
+    const { highlight, _source } = hit.inner_hits.fullText.hits.hits[0];
     const regex = /\[{2}(\d+)]{2}/g;
 
     Object.values<string[]>(highlight).forEach(snippets => {
@@ -24,6 +24,7 @@ function extractFullTextSnippets(hit: ElasticHit<EntitySchema>) {
         fullTextSnippets.push({
           text: snippet.replace(regex, ''),
           page: matches ? Number(matches[1]) : 0,
+          filename: _source.filename,
         });
       });
     });

app/api/search.v2/specs/snippetsSearch.spec.ts

@@ -42,8 +42,8 @@ describe('searchSnippets', () => {
           count: 2,
           metadata: [],
           fullText: [
-            { page: 2, text: matches },
-            { page: 4, text: matches },
+            { page: 2, text: matches, filename: 'entity1SharedId.pdf' },
+            { page: 4, text: matches, filename: 'entity1SharedId.pdf' },
           ],
         },
       }),
@@ -97,7 +97,13 @@ describe('searchSnippets', () => {
         snippets: {
           count: 1,
           metadata: [],
-          fullText: [{ page: 2, text: expect.stringContaining('<b>searched:term</b>') }],
+          fullText: [
+            {
+              page: 2,
+              text: expect.stringContaining('<b>searched:term</b>'),
+              filename: 'entity4SharedId.pdf',
+            },
+          ],
         },
       }),
     ];

app/api/search/elasticTypes.ts

@@ -26,7 +26,7 @@ export interface ElasticHit<T> {
   fields?: any;
   highlight?: any;
   // eslint-disable-next-line camelcase
-  inner_hits?: { fullText: { hits: { hits: [{ highlight: {} }] } } };
+  inner_hits?: { fullText: { hits: { hits: [{ highlight: {}; _source: Record<string, any> }] } } };
   // eslint-disable-next-line camelcase
   matched_queries?: string[];
   sort?: string[];

app/api/search/entitiesIndex.js

@@ -54,6 +54,7 @@ function setFullTextSettings(defaultDocument, id, body, doc) {
   }
   const fullTextObject = {
     [`fullText_${language}`]: fullText,
+    filename: defaultDocument.filename,
     fullText: { name: 'fullText', parent: id },
   };
   body.push(fullTextObject);

app/api/services/informationextraction/specs/InformationExtraction.spec.ts

@@ -759,12 +759,12 @@ describe('InformationExtraction', () => {
           status: 'processing',
           state: {
             labeled: true,
+            match: null,
             withValue: true,
             withSuggestion: true,
-            match: false,
             hasContext: true,
             processing: true,
-            obsolete: false,
+            obsolete: true,
             error: false,
           },
         })
@@ -1011,9 +1011,9 @@ describe('InformationExtraction', () => {
           error: 'Issue calculation suggestion',
           state: {
             labeled: true,
+            match: null,
             withValue: true,
             withSuggestion: false,
-            match: false,
             hasContext: false,
             processing: false,
             obsolete: false,

app/api/services/informationextraction/specs/ixextractors.spec.ts

@@ -93,15 +93,15 @@ const fixtures: DBFixture = {
     fixtureFactory.ixExtractor('fungusKindExtractor', 'kind', ['fungusTemplate']),
   ],
   ixsuggestions: [
-    fixtureFactory.ixSuggestion(
+    fixtureFactory.ixSuggestion_deprecated(
       'sh1_en',
       'existingExtractor',
       'shared1',
       'animalTemplate',
       'F3',
       'kind'
     ),
-    fixtureFactory.ixSuggestion(
+    fixtureFactory.ixSuggestion_deprecated(
       'sh1_es',
       'existingExtractor',
       'shared1',
@@ -110,23 +110,23 @@ const fixtures: DBFixture = {
       'kind',
       { language: 'es' }
     ),
-    fixtureFactory.ixSuggestion(
+    fixtureFactory.ixSuggestion_deprecated(
       'sh3_en',
       'existingExtractor',
       'shared3',
       'plantTemplate',
       'F5',
       'kind'
     ),
-    fixtureFactory.ixSuggestion(
+    fixtureFactory.ixSuggestion_deprecated(
       'sh4_en',
       'fungusKindExtractor',
       'shared4',
       'fungusTemplate',
       'F7',
       'kind'
     ),
-    fixtureFactory.ixSuggestion(
+    fixtureFactory.ixSuggestion_deprecated(
       'sh4_es',
       'fungusKindExtractor',
       'shared4',

app/api/suggestions/pipelineStages.ts

@@ -14,46 +14,23 @@ export const baseQueryFragment = (extractorId: ObjectId, ignoreProcessing = true
 };
 
 export const filterFragments = {
-  labeled: {
-    _fragment: { 'state.labeled': true },
-    match: { 'state.labeled': true, 'state.match': true },
-    mismatch: { 'state.labeled': true, 'state.match': false },
-  },
-  nonLabeled: {
-    _fragment: { 'state.labeled': false },
-    withSuggestion: { 'state.labeled': false, 'state.withSuggestion': true },
-    noSuggestion: { 'state.labeled': false, 'state.withSuggestion': false },
-    noContext: { 'state.labeled': false, 'state.hasContext': false },
-    obsolete: { 'state.labeled': false, 'state.obsolete': true },
-    others: { 'state.labeled': false, 'state.error': true },
-  },
+  labeled: { 'state.labeled': true },
+  nonLabeled: { 'state.labeled': false },
+  match: { 'state.match': true },
+  mismatch: { 'state.match': false },
+  obsolete: { 'state.obsolete': true },
+  error: { 'state.error': true },
 };
 
 export const translateCustomFilter = (customFilter: SuggestionCustomFilter) => {
   const orFilters = [];
-  if (customFilter.labeled.match) {
-    orFilters.push(filterFragments.labeled.match);
-  }
-  if (customFilter.labeled.mismatch) {
-    orFilters.push(filterFragments.labeled.mismatch);
-  }
-
-  if (customFilter.nonLabeled.withSuggestion) {
-    orFilters.push(filterFragments.nonLabeled.withSuggestion);
-  }
+  if (customFilter.labeled) orFilters.push(filterFragments.labeled);
+  if (customFilter.nonLabeled) orFilters.push(filterFragments.nonLabeled);
+  if (customFilter.match) orFilters.push(filterFragments.match);
+  if (customFilter.mismatch) orFilters.push(filterFragments.mismatch);
+  if (customFilter.obsolete) orFilters.push(filterFragments.obsolete);
+  if (customFilter.error) orFilters.push(filterFragments.error);
 
-  if (customFilter.nonLabeled.noSuggestion) {
-    orFilters.push(filterFragments.nonLabeled.noSuggestion);
-  }
-  if (customFilter.nonLabeled.noContext) {
-    orFilters.push(filterFragments.nonLabeled.noContext);
-  }
-  if (customFilter.nonLabeled.obsolete) {
-    orFilters.push(filterFragments.nonLabeled.obsolete);
-  }
-  if (customFilter.nonLabeled.others) {
-    orFilters.push(filterFragments.nonLabeled.others);
-  }
   return orFilters;
 };