diff --git a/app/api/files.v2/FilesHealthCheck.ts b/app/api/files.v2/FilesHealthCheck.ts new file mode 100644 index 0000000000..ee9962053f --- /dev/null +++ b/app/api/files.v2/FilesHealthCheck.ts @@ -0,0 +1,68 @@ +import { FilesDataSource } from './contracts/FilesDataSource'; +import { FileStorage } from './contracts/FileStorage'; +import { URLAttachment } from './model/URLAttachment'; + +function filterFilesInStorage(files: string[]) { + return files.filter(file => !file.endsWith('activity.log')); +} + +export class FilesHealthCheck { + // eslint-disable-next-line class-methods-use-this + private onMissingInDBCB: (filename: string) => void = () => {}; + + // eslint-disable-next-line class-methods-use-this + private onMissingInStorageCB: (fileDTO: { _id: string; filename: string }) => void = () => {}; + + private fileStorage: FileStorage; + + private filesDS: FilesDataSource; + + constructor(fileStorage: FileStorage, filesDS: FilesDataSource) { + this.fileStorage = fileStorage; + this.filesDS = filesDS; + } + + async execute() { + const allFilesInDb = await this.filesDS.getAll().all(); + const allFilesInStorage = await this.fileStorage.list(); + const filteredFilesInStorage = new Set(filterFilesInStorage(allFilesInStorage)); + let missingInStorage = 0; + const missingInStorageList: string[] = []; + const missingInDbList: string[] = []; + const countInStorage = filteredFilesInStorage.size; + let countInDb = 0; + + allFilesInDb.forEach(file => { + countInDb += 1; + const existsInStorage = filteredFilesInStorage.delete(this.fileStorage.getPath(file)); + + if (!existsInStorage && !(file instanceof URLAttachment)) { + missingInStorage += 1; + missingInStorageList.push(this.fileStorage.getPath(file)); + this.onMissingInStorageCB({ _id: file.id, filename: file.filename }); + } + }); + + filteredFilesInStorage.forEach(file => { + missingInDbList.push(file); + this.onMissingInDBCB(file); + }); + + return { + missingInStorageList, + missingInStorage, + missingInDbList, + missingInDb: filteredFilesInStorage.size, + countInDb, + countInStorage, + }; + } + + onMissingInDB(cb: (filename: string) => void) { + this.onMissingInDBCB = cb; + } + + onMissingInStorage(cb: (fileDTO: { _id: string; filename: string }) => void) { + this.onMissingInStorageCB = cb; + } +} diff --git a/app/api/files.v2/contracts/FileStorage.ts b/app/api/files.v2/contracts/FileStorage.ts new file mode 100644 index 0000000000..5025b46d60 --- /dev/null +++ b/app/api/files.v2/contracts/FileStorage.ts @@ -0,0 +1,6 @@ +import { UwaziFile } from '../model/UwaziFile'; + +export interface FileStorage { + list(): Promise; + getPath(file: UwaziFile): string; +} diff --git a/app/api/files.v2/contracts/FilesDataSource.ts b/app/api/files.v2/contracts/FilesDataSource.ts index 8b90179cb3..50ff40c378 100644 --- a/app/api/files.v2/contracts/FilesDataSource.ts +++ b/app/api/files.v2/contracts/FilesDataSource.ts @@ -1,3 +1,7 @@ +import { ResultSet } from 'api/common.v2/contracts/ResultSet'; +import { UwaziFile } from '../model/UwaziFile'; + export interface FilesDataSource { filesExistForEntities(files: { entity: string; _id: string }[]): Promise; + getAll(): ResultSet; } diff --git a/app/api/files.v2/database/FilesMappers.ts b/app/api/files.v2/database/FilesMappers.ts new file mode 100644 index 0000000000..289091bed4 --- /dev/null +++ b/app/api/files.v2/database/FilesMappers.ts @@ -0,0 +1,53 @@ +// import { OptionalId } from 'mongodb'; + +import { FileDBOType } from './schemas/filesTypes'; +import { UwaziFile } from '../model/UwaziFile'; +import { Document } from '../model/Document'; +import { URLAttachment } from '../model/URLAttachment'; +import { Attachment } from '../model/Attachment'; +import { CustomUpload } from '../model/CustomUpload'; + +export const FileMappers = { + // toDBO(file: UwaziFile): OptionalId { + // return { + // filename: file.filename, + // entity: file.entity, + // type: 'document', + // totalPages: file.totalPages, + // }; + // }, + + toModel(fileDBO: FileDBOType): UwaziFile { + if (fileDBO.type === 'attachment' && fileDBO.url) { + return new URLAttachment( + fileDBO._id.toString(), + fileDBO.entity, + fileDBO.totalPages, + fileDBO.url + ); + } + if (fileDBO.type === 'attachment') { + return new Attachment( + fileDBO._id.toString(), + fileDBO.entity, + fileDBO.totalPages, + fileDBO.filename + ); + } + + if (fileDBO.type === 'custom') { + return new CustomUpload( + fileDBO._id.toString(), + fileDBO.entity, + fileDBO.totalPages, + fileDBO.filename + ); + } + return new Document( + fileDBO._id.toString(), + fileDBO.entity, + fileDBO.totalPages, + fileDBO.filename + ); + }, +}; diff --git a/app/api/files.v2/database/MongoFilesDataSource.ts b/app/api/files.v2/database/MongoFilesDataSource.ts index 3dc91d10fc..d19a2889b2 100644 --- a/app/api/files.v2/database/MongoFilesDataSource.ts +++ b/app/api/files.v2/database/MongoFilesDataSource.ts @@ -1,9 +1,19 @@ import { MongoDataSource } from 'api/common.v2/database/MongoDataSource'; +import { MongoResultSet } from 'api/common.v2/database/MongoResultSet'; import { ObjectId } from 'mongodb'; import { FilesDataSource } from '../contracts/FilesDataSource'; +import { FileMappers } from './FilesMappers'; import { FileDBOType } from './schemas/filesTypes'; +import { UwaziFile } from '../model/UwaziFile'; export class MongoFilesDataSource extends MongoDataSource implements FilesDataSource { + getAll() { + return new MongoResultSet( + this.getCollection().find({}, { projection: { fullText: 0 } }), + FileMappers.toModel + ); + } + protected collectionName = 'files'; async filesExistForEntities(files: { entity: string; _id: string }[]) { diff --git a/app/api/files.v2/database/schemas/filesTypes.ts b/app/api/files.v2/database/schemas/filesTypes.ts index e507afb003..defbad6299 100644 --- a/app/api/files.v2/database/schemas/filesTypes.ts +++ b/app/api/files.v2/database/schemas/filesTypes.ts @@ -3,10 +3,12 @@ import { ObjectId } from 'mongodb'; interface BaseFileDBOType { _id: ObjectId; entity: string; + filename: string; + url: string; } interface DocumentFileDBOType extends BaseFileDBOType { - type: 'document'; + type: 'document' | 'attachment' | 'custom'; totalPages: number; } diff --git a/app/api/files.v2/database/specs/MongoFilesDataSource.spec.ts b/app/api/files.v2/database/specs/MongoFilesDataSource.spec.ts index 3704efae91..6ed03c11c0 100644 --- a/app/api/files.v2/database/specs/MongoFilesDataSource.spec.ts +++ b/app/api/files.v2/database/specs/MongoFilesDataSource.spec.ts @@ -8,9 +8,9 @@ const factory = getFixturesFactory(); const fixtures = { files: [ - factory.file('file1', 'entity1', 'document', 'file1.pdf'), - factory.file('file2', 'entity2', 'document', 'file2.pdf'), - factory.file('file3', 'entity3', 'document', 'file3.pdf'), + factory.fileDeprecated('file1', 'entity1', 'document', 'file1.pdf'), + factory.fileDeprecated('file2', 'entity2', 'document', 'file2.pdf'), + factory.fileDeprecated('file3', 'entity3', 'document', 'file3.pdf'), ], }; diff --git a/app/api/files.v2/infrastructure/S3FileStorage.ts b/app/api/files.v2/infrastructure/S3FileStorage.ts new file mode 100644 index 0000000000..2e890040e0 --- /dev/null +++ b/app/api/files.v2/infrastructure/S3FileStorage.ts @@ -0,0 +1,57 @@ +import { _Object, ListObjectsV2Command, S3Client } from '@aws-sdk/client-s3'; +import { config } from 'api/config'; +import { Tenant } from 'api/tenants/tenantContext'; +import path from 'path'; +import { FileStorage } from '../contracts/FileStorage'; +import { Attachment } from '../model/Attachment'; +import { UwaziFile } from '../model/UwaziFile'; +import { URLAttachment } from '../model/URLAttachment'; +import { CustomUpload } from '../model/CustomUpload'; + +export class S3FileStorage implements FileStorage { + private s3Client: S3Client; + + private tenant: Tenant; + + constructor(s3Client: S3Client, tenant: Tenant) { + this.s3Client = s3Client; + this.tenant = tenant; + } + + getPath(file: UwaziFile): string { + if (file instanceof Attachment) { + return path.join(this.tenant.attachments, file.filename); + } + if (file instanceof CustomUpload) { + return path.join(this.tenant.customUploads, file.filename); + } + if (file instanceof URLAttachment) { + return 'not implemented'; + } + return path.join(this.tenant.uploadedDocuments, file.filename); + } + + async list(): Promise { + const objects: _Object[] = []; + const requestNext = async (token?: string) => { + const response = await this.s3Client.send( + new ListObjectsV2Command({ + Bucket: config.s3.bucket, + Prefix: this.tenant.name, + ContinuationToken: token, + MaxKeys: config.s3.batchSize, + }) + ); + objects.push(...(response.Contents || [])); + return response.NextContinuationToken; + }; + + let continuationToken = await requestNext(); + while (continuationToken) { + // eslint-disable-next-line no-await-in-loop + continuationToken = await requestNext(continuationToken); + } + + return objects.map(c => c.Key!); + } +} diff --git a/app/api/files.v2/infrastructure/specs/S3FileStorage.spec.ts b/app/api/files.v2/infrastructure/specs/S3FileStorage.spec.ts new file mode 100644 index 0000000000..375a2e1011 --- /dev/null +++ b/app/api/files.v2/infrastructure/specs/S3FileStorage.spec.ts @@ -0,0 +1,125 @@ +import { + CreateBucketCommand, + DeleteBucketCommand, + DeleteObjectCommand, + ListObjectsCommand, + PutObjectCommand, + S3Client, +} from '@aws-sdk/client-s3'; +import { config } from 'api/config'; +import { Attachment } from 'api/files.v2/model/Attachment'; +import { Document } from 'api/files.v2/model/Document'; +import { Tenant } from 'api/tenants/tenantContext'; +import { S3FileStorage } from '../S3FileStorage'; + +describe('S3FileStorage', () => { + let s3Client: S3Client; + let s3fileStorage: S3FileStorage; + let tenant: Tenant; + + beforeEach(async () => { + config.s3 = { + endpoint: 'http://127.0.0.1:9000', + bucket: 'uwazi-development', + credentials: { + accessKeyId: 'minioadmin', + secretAccessKey: 'minioadmin', + }, + batchSize: 1, + }; + + s3Client = new S3Client({ + apiVersion: 'latest', + region: 'region', + forcePathStyle: true, // needed for minio + ...config.s3, + }); + await s3Client.send(new CreateBucketCommand({ Bucket: 'uwazi-development' })); + + tenant = { + name: 'test-tenant', + dbName: 'test-tenant', + indexName: 'test-tenant', + uploadedDocuments: 'test-tenant/documents', + attachments: 'test-tenant/attachments', + customUploads: 'test-tenant/customUploads', + activityLogs: 'test-tenant/log', + }; + + s3fileStorage = new S3FileStorage(s3Client, tenant); + }); + + afterEach(async () => { + const allBucketKeys = ( + ( + await s3Client.send( + new ListObjectsCommand({ + Bucket: 'uwazi-development', + }) + ) + ).Contents || [] + ).map(content => content.Key); + + await Promise.all( + allBucketKeys.map(async key => + s3Client.send( + new DeleteObjectCommand({ + Bucket: 'uwazi-development', + Key: key, + }) + ) + ) + ); + await s3Client.send(new DeleteBucketCommand({ Bucket: 'uwazi-development' })); + s3Client.destroy(); + }); + + describe('list', () => { + it('should list all s3 keys', async () => { + await s3Client.send( + new PutObjectCommand({ + Bucket: 'uwazi-development', + Key: 'test-tenant/documents/document1', + Body: 'body', + }) + ); + + await s3Client.send( + new PutObjectCommand({ + Bucket: 'uwazi-development', + Key: 'test-tenant/documents/document2', + Body: 'body', + }) + ); + + const listedFiles = await s3fileStorage.list(); + + expect(listedFiles.sort()).toEqual( + ['test-tenant/documents/document1', 'test-tenant/documents/document2'].sort() + ); + }); + }); + + describe('getPath', () => { + it.each([ + { + file: new Document('id', 'entity', 1, 'document'), + expected: 'test-tenant/documents/document', + }, + { + file: new Attachment('id', 'entity', 1, 'attachment'), + expected: 'test-tenant/attachments/attachment', + }, + // { + // file: new URLAttachment('id', 'filename', 'entity', 1, 'url'), + // expected: 'test-tenant/?????/filename', + // }, + ])( + 'should use dinamic paths based on tenant ($file.filename -> $expected)', + async ({ file, expected }) => { + const key = s3fileStorage.getPath(file); + expect(key).toBe(expected); + } + ); + }); +}); diff --git a/app/api/files.v2/model/Attachment.ts b/app/api/files.v2/model/Attachment.ts new file mode 100644 index 0000000000..a06a4bf2bd --- /dev/null +++ b/app/api/files.v2/model/Attachment.ts @@ -0,0 +1,10 @@ +import { BaseFile } from './BaseFile'; + +export class Attachment extends BaseFile { + readonly filename: string; + + constructor(id: string, entity: string, totalPages: number, filename: string) { + super(id, entity, totalPages); + this.filename = filename; + } +} diff --git a/app/api/files.v2/model/BaseFile.ts b/app/api/files.v2/model/BaseFile.ts new file mode 100644 index 0000000000..5b11e40ce4 --- /dev/null +++ b/app/api/files.v2/model/BaseFile.ts @@ -0,0 +1,13 @@ +export class BaseFile { + readonly id: string; + + readonly entity: string; + + readonly totalPages: number; + + constructor(id: string, entity: string, totalPages: number) { + this.id = id; + this.entity = entity; + this.totalPages = totalPages; + } +} diff --git a/app/api/files.v2/model/CustomUpload.ts b/app/api/files.v2/model/CustomUpload.ts new file mode 100644 index 0000000000..45e167387d --- /dev/null +++ b/app/api/files.v2/model/CustomUpload.ts @@ -0,0 +1,10 @@ +import { BaseFile } from './BaseFile'; + +export class CustomUpload extends BaseFile { + readonly filename: string; + + constructor(id: string, entity: string, totalPages: number, filename: string) { + super(id, entity, totalPages); + this.filename = filename; + } +} diff --git a/app/api/files.v2/model/Document.ts b/app/api/files.v2/model/Document.ts new file mode 100644 index 0000000000..b1e571047b --- /dev/null +++ b/app/api/files.v2/model/Document.ts @@ -0,0 +1,10 @@ +import { BaseFile } from './BaseFile'; + +export class Document extends BaseFile { + readonly filename: string; + + constructor(id: string, entity: string, totalPages: number, filename: string) { + super(id, entity, totalPages); + this.filename = filename; + } +} diff --git a/app/api/files.v2/model/URLAttachment.ts b/app/api/files.v2/model/URLAttachment.ts new file mode 100644 index 0000000000..36fdf036a1 --- /dev/null +++ b/app/api/files.v2/model/URLAttachment.ts @@ -0,0 +1,10 @@ +import { BaseFile } from './BaseFile'; + +export class URLAttachment extends BaseFile { + readonly url: string; + + constructor(id: string, entity: string, totalPages: number, url: string) { + super(id, entity, totalPages); + this.url = url; + } +} diff --git a/app/api/files.v2/model/UwaziFile.ts b/app/api/files.v2/model/UwaziFile.ts new file mode 100644 index 0000000000..aa094d1133 --- /dev/null +++ b/app/api/files.v2/model/UwaziFile.ts @@ -0,0 +1,6 @@ +import { Attachment } from './Attachment'; +import { CustomUpload } from './CustomUpload'; +import { Document } from './Document'; +import { URLAttachment } from './URLAttachment'; + +export type UwaziFile = Document | Attachment | URLAttachment | CustomUpload; diff --git a/app/api/files.v2/specs/FilesHealthCheck.spec.ts b/app/api/files.v2/specs/FilesHealthCheck.spec.ts new file mode 100644 index 0000000000..6f88de9500 --- /dev/null +++ b/app/api/files.v2/specs/FilesHealthCheck.spec.ts @@ -0,0 +1,152 @@ +import { testingEnvironment } from 'api/utils/testingEnvironment'; +import { getFixturesFactory } from 'api/utils/fixturesFactory'; +import { DefaultTransactionManager } from 'api/common.v2/database/data_source_defaults'; +import { FilesHealthCheck } from '../FilesHealthCheck'; +import { FileStorage } from '../contracts/FileStorage'; +import { DefaultFilesDataSource } from '../database/data_source_defaults'; +import { UwaziFile } from '../model/UwaziFile'; +import { URLAttachment } from '../model/URLAttachment'; +import { CustomUpload } from '../model/CustomUpload'; + +const factory = getFixturesFactory(); + +beforeEach(async () => { + await testingEnvironment.setUp({ files: [] }); +}); + +afterAll(async () => { + await testingEnvironment.tearDown(); +}); + +let testStorageFiles: string[] = []; +class TestFileStorage implements FileStorage { + // eslint-disable-next-line class-methods-use-this + getPath(file: UwaziFile): string { + if (file instanceof URLAttachment) { + return ''; + } + if (file instanceof CustomUpload) { + return `custom_uploads/${file.filename}`; + } + return `document/${file.filename}`; + } + + // eslint-disable-next-line class-methods-use-this + async list(): Promise { + return testStorageFiles; + } +} + +describe('FilesHealthCheck', () => { + let filesHealthCheck: FilesHealthCheck; + + beforeEach(() => { + filesHealthCheck = new FilesHealthCheck( + new TestFileStorage(), + DefaultFilesDataSource(DefaultTransactionManager()) + ); + }); + + it('should report full count in storage and in db', async () => { + testStorageFiles = ['document/file1', 'document/file3']; + await testingEnvironment.setUp({ + files: [factory.document('file1'), factory.document('file2'), factory.document('file4')], + }); + + const summary = await filesHealthCheck.execute(); + + expect(summary).toMatchObject({ + countInDb: 3, + countInStorage: 2, + }); + }); + + it('should report missing in storage files', async () => { + testStorageFiles = ['document/file1', 'document/file3', 'custom_uploads/custom1']; + await testingEnvironment.setUp({ + files: [ + factory.document('file1'), + factory.document('file2'), + factory.document('file3'), + factory.document('file4'), + factory.custom_upload('custom1'), + ], + }); + + const summary = await filesHealthCheck.execute(); + + expect(summary).toMatchObject({ + missingInStorageList: ['document/file2', 'document/file4'], + missingInStorage: 2, + }); + }); + + it('should report missing in DB files', async () => { + testStorageFiles = ['document/file1', 'document/file2', 'document/file3', 'document/file4']; + await testingEnvironment.setUp({ + files: [factory.document('file2'), factory.document('file3')], + }); + + const summary = await filesHealthCheck.execute(); + + expect(summary).toMatchObject({ + missingInDbList: ['document/file1', 'document/file4'], + missingInDb: 2, + }); + }); + + it('should ignore activity.log files', async () => { + testStorageFiles = ['log/1-activity.log', 'log/2-activity.log', 'document/file1']; + await testingEnvironment.setUp({ files: [] }); + + const summary = await filesHealthCheck.execute(); + + expect(summary).toMatchObject({ + missingInDbList: ['document/file1'], + missingInDb: 1, + }); + }); + + it('should ignore external attachemnts (have url)', async () => { + testStorageFiles = ['document/file1']; + await testingEnvironment.setUp({ files: [factory.attachment('url_file', { url: 'url' })] }); + + const summary = await filesHealthCheck.execute(); + + expect(summary).toMatchObject({ + missingInDb: 1, + missingInStorage: 0, + }); + }); + + it('should be able to subscribe to an "event" for each file missing in db', async () => { + testStorageFiles = ['document/file1', 'document/file2']; + await testingEnvironment.setUp({ files: [] }); + + const events: string[] = []; + filesHealthCheck.onMissingInDB(file => { + events.push(file); + }); + + await filesHealthCheck.execute(); + expect(events).toEqual(['document/file1', 'document/file2']); + }); + + it('should be able to subscribe to an "event" for each file missing in storage', async () => { + testStorageFiles = []; + await testingEnvironment.setUp({ + files: [factory.document('file1'), factory.document('file2')], + }); + + const events: { _id: string; filename: string }[] = []; + filesHealthCheck.onMissingInStorage(file => { + events.push(file); + }); + await filesHealthCheck.execute(); + + expect(events).toEqual([ + { _id: factory.idString('file1'), filename: 'file1' }, + { _id: factory.idString('file2'), filename: 'file2' }, + ]); + }); +}); diff --git a/app/api/files/specs/ocrRoutes.spec.ts b/app/api/files/specs/ocrRoutes.spec.ts index ca7ed79ffc..cd0757f77a 100644 --- a/app/api/files/specs/ocrRoutes.spec.ts +++ b/app/api/files/specs/ocrRoutes.spec.ts @@ -33,7 +33,7 @@ const attachmentFile = 'spn.pdf'; const FIXTURES: DBFixture = { entities: [fixturesFactory.entity('parentEntity')], files: [ - fixturesFactory.file( + fixturesFactory.fileDeprecated( 'fileToProcess', 'parentEntity', 'document', @@ -41,7 +41,12 @@ const FIXTURES: DBFixture = { 'eng', 'fileNameToProcess.pdf' ), - fixturesFactory.file('unrelatedAttachment', 'parentEntity', 'attachment', attachmentFile), + fixturesFactory.fileDeprecated( + 'unrelatedAttachment', + 'parentEntity', + 'attachment', + attachmentFile + ), ], users: [fixturesFactory.user('collab'), fixturesFactory.user('admin', UserRole.ADMIN)], connections: [{ entity: 'parentEntity', file: fixturesFactory.id('fileToProcess').toString() }], diff --git a/app/api/files/specs/v2_support.spec.ts b/app/api/files/specs/v2_support.spec.ts index d85394279e..2e28a2f7be 100644 --- a/app/api/files/specs/v2_support.spec.ts +++ b/app/api/files/specs/v2_support.spec.ts @@ -46,9 +46,9 @@ const fixtures: DBFixture = { ]), ], files: [ - factory.file('file1', 'entity1', 'document', 'file1.pdf'), - factory.file('file2', 'entity2', 'document', 'file2.pdf'), - factory.file('file3', 'entity3', 'document', 'file3.pdf'), + factory.fileDeprecated('file1', 'entity1', 'document', 'file1.pdf'), + factory.fileDeprecated('file2', 'entity2', 'document', 'file2.pdf'), + factory.fileDeprecated('file3', 'entity3', 'document', 'file3.pdf'), ], relationships: [ { diff --git a/app/api/odm/specs/EntitiesUpdateLogHelper.spec.ts b/app/api/odm/specs/EntitiesUpdateLogHelper.spec.ts index 19834fb710..0cfd6e04f1 100644 --- a/app/api/odm/specs/EntitiesUpdateLogHelper.spec.ts +++ b/app/api/odm/specs/EntitiesUpdateLogHelper.spec.ts @@ -20,12 +20,12 @@ const fixtures = { fixtureFactory.entity('entity2', 'template1'), ], files: [ - fixtureFactory.file('document1', 'entity1', 'document', 'document1_filename'), - fixtureFactory.file('attachment1', 'entity1', 'attachment', 'attachment1_filename'), - fixtureFactory.file('thumbnail1', 'entity1', 'thumbnail', 'thumbnail1_filename'), - fixtureFactory.file('custom1', undefined, 'custom', 'custom1_filename'), - fixtureFactory.file('document2', 'entity2', 'document', 'document2_filename'), - fixtureFactory.file('attachment2', 'entity2', 'attachment', 'attachment2_filename'), + fixtureFactory.fileDeprecated('document1', 'entity1', 'document', 'document1_filename'), + fixtureFactory.fileDeprecated('attachment1', 'entity1', 'attachment', 'attachment1_filename'), + fixtureFactory.fileDeprecated('thumbnail1', 'entity1', 'thumbnail', 'thumbnail1_filename'), + fixtureFactory.fileDeprecated('custom1', undefined, 'custom', 'custom1_filename'), + fixtureFactory.fileDeprecated('document2', 'entity2', 'document', 'document2_filename'), + fixtureFactory.fileDeprecated('attachment2', 'entity2', 'attachment', 'attachment2_filename'), ], }; diff --git a/app/api/relationships.v2/routes/specs/routes.spec.ts b/app/api/relationships.v2/routes/specs/routes.spec.ts index 921f7f1e12..6ef3d7f660 100644 --- a/app/api/relationships.v2/routes/specs/routes.spec.ts +++ b/app/api/relationships.v2/routes/specs/routes.spec.ts @@ -49,7 +49,7 @@ const fixtures: DBFixture = { }, }, ], - files: [factory.file('file1', 'entity1', 'document', 'file1.pdf')], + files: [factory.fileDeprecated('file1', 'entity1', 'document', 'file1.pdf')], users: [adminUser], }; diff --git a/app/api/relationships.v2/services/specs/CreateRelationshipService.spec.ts b/app/api/relationships.v2/services/specs/CreateRelationshipService.spec.ts index ed4a66739e..09ee6743b7 100644 --- a/app/api/relationships.v2/services/specs/CreateRelationshipService.spec.ts +++ b/app/api/relationships.v2/services/specs/CreateRelationshipService.spec.ts @@ -89,9 +89,9 @@ const fixtures: DBFixture = { }, ], files: [ - factory.file('file1', 'entity1', 'document', 'file1.pdf'), - factory.file('file2', 'entity2', 'document', 'file2.pdf'), - factory.file('file3', 'entity3', 'document', 'file3.pdf'), + factory.fileDeprecated('file1', 'entity1', 'document', 'file1.pdf'), + factory.fileDeprecated('file2', 'entity2', 'document', 'file2.pdf'), + factory.fileDeprecated('file3', 'entity3', 'document', 'file3.pdf'), ], settings: [ { diff --git a/app/api/relationships.v2/services/specs/DenormalizationService.spec.ts b/app/api/relationships.v2/services/specs/DenormalizationService.spec.ts index 00c1da14fd..5338f44305 100644 --- a/app/api/relationships.v2/services/specs/DenormalizationService.spec.ts +++ b/app/api/relationships.v2/services/specs/DenormalizationService.spec.ts @@ -152,7 +152,7 @@ const fixtures: DBFixture = { ...entityInLanguages(['hu', 'es'], 'entity9', 'template7'), ...entityInLanguages(['hu', 'es'], 'entity10', 'template1'), ], - files: [factory.file('file4', 'entity4', 'document', 'file4.pdf', 'hu')], + files: [factory.fileDeprecated('file4', 'entity4', 'document', 'file4.pdf', 'hu')], templates: [ factory.template('formerHubsTemplate', [ { diff --git a/app/api/services/informationextraction/specs/fixtures.ts b/app/api/services/informationextraction/specs/fixtures.ts index b98f1db615..466e8fa5b0 100644 --- a/app/api/services/informationextraction/specs/fixtures.ts +++ b/app/api/services/informationextraction/specs/fixtures.ts @@ -139,7 +139,7 @@ const fixtures: DBFixture = { }), ], files: [ - factory.file('F1', 'A1', 'document', fixturesPdfNameA, 'other', '', [ + factory.fileDeprecated('F1', 'A1', 'document', fixturesPdfNameA, 'other', '', [ { name: 'property1', selection: { @@ -155,7 +155,7 @@ const fixtures: DBFixture = { }, }, ]), - factory.file('F2', 'A2', 'document', fixturesPdfNameB, 'eng', '', [ + factory.fileDeprecated('F2', 'A2', 'document', fixturesPdfNameB, 'eng', '', [ { name: 'text', selection: { @@ -164,7 +164,7 @@ const fixtures: DBFixture = { }, }, ]), - factory.file('F3', 'A3', 'document', fixturesPdfNameC, 'eng', '', [ + factory.fileDeprecated('F3', 'A3', 'document', fixturesPdfNameC, 'eng', '', [ { name: 'property1', selection: { @@ -173,7 +173,7 @@ const fixtures: DBFixture = { }, }, ]), - factory.file('F4', 'A1', 'document', fixturesPdfNameD, 'eng', '', [ + factory.fileDeprecated('F4', 'A1', 'document', fixturesPdfNameD, 'eng', '', [ { name: 'property2', selection: { @@ -182,17 +182,17 @@ const fixtures: DBFixture = { }, }, ]), - factory.file('F5', 'A5', 'document', fixturesPdfNameE, 'spa'), - factory.file('F6', 'A6', 'document', fixturesPdfNameF, 'eng'), - factory.file('F15', 'A15', 'document', fixturesPdfNameA, 'eng'), - factory.file('F16', 'A16', 'document', fixturesPdfNameC, 'eng'), - factory.file('F17', 'A17', 'document', fixturesPdfNameG, 'eng'), - factory.file('F18', 'A18', 'document', fixturesPdfNameH, 'eng'), - factory.file('F19', 'A19', 'document', fixturesPdfNameI, 'eng'), - factory.file('F20', 'A20', 'document', ficturesPdfNameJ, 'eng'), - factory.file('F21', 'A21', 'document', fixturesPdfNameK, 'eng'), - factory.file('F22', 'A22', 'document', fixturesPdfNameL, 'eng'), - factory.file('F23', 'A23', 'document', fixturesPdfNameM, 'eng'), + factory.fileDeprecated('F5', 'A5', 'document', fixturesPdfNameE, 'spa'), + factory.fileDeprecated('F6', 'A6', 'document', fixturesPdfNameF, 'eng'), + factory.fileDeprecated('F15', 'A15', 'document', fixturesPdfNameA, 'eng'), + factory.fileDeprecated('F16', 'A16', 'document', fixturesPdfNameC, 'eng'), + factory.fileDeprecated('F17', 'A17', 'document', fixturesPdfNameG, 'eng'), + factory.fileDeprecated('F18', 'A18', 'document', fixturesPdfNameH, 'eng'), + factory.fileDeprecated('F19', 'A19', 'document', fixturesPdfNameI, 'eng'), + factory.fileDeprecated('F20', 'A20', 'document', ficturesPdfNameJ, 'eng'), + factory.fileDeprecated('F21', 'A21', 'document', fixturesPdfNameK, 'eng'), + factory.fileDeprecated('F22', 'A22', 'document', fixturesPdfNameL, 'eng'), + factory.fileDeprecated('F23', 'A23', 'document', fixturesPdfNameM, 'eng'), ], segmentations: [ { diff --git a/app/api/services/informationextraction/specs/ixextractors.spec.ts b/app/api/services/informationextraction/specs/ixextractors.spec.ts index f4bcff15c5..5ed882491b 100644 --- a/app/api/services/informationextraction/specs/ixextractors.spec.ts +++ b/app/api/services/informationextraction/specs/ixextractors.spec.ts @@ -63,7 +63,7 @@ const fixtures: DBFixture = { fixtureFactory.entity('shared4', 'fungusTemplate', {}, { language: 'en' }), ], files: [ - fixtureFactory.file('F1', 'shared2', 'document', 'documentB.pdf', 'eng', '', [ + fixtureFactory.fileDeprecated('F1', 'shared2', 'document', 'documentB.pdf', 'eng', '', [ { name: 'age', selection: { @@ -72,7 +72,7 @@ const fixtures: DBFixture = { }, }, ]), - fixtureFactory.file('F2', 'shared2', 'document', 'documentC.pdf', 'spa', '', [ + fixtureFactory.fileDeprecated('F2', 'shared2', 'document', 'documentC.pdf', 'spa', '', [ { name: 'age', selection: { @@ -81,12 +81,12 @@ const fixtures: DBFixture = { }, }, ]), - fixtureFactory.file('F3', 'shared1', 'document', 'documentA.pdf', 'eng'), - fixtureFactory.file('F4', 'shared1', 'document', 'documentD.pdf', 'spa'), - fixtureFactory.file('F5', 'shared3', 'document', 'documentE.pdf', 'eng'), - fixtureFactory.file('F6', 'shared3', 'document', 'documentF.pdf', 'spa'), - fixtureFactory.file('F7', 'shared4', 'document', 'documentG.pdf', 'eng'), - fixtureFactory.file('F8', 'shared4', 'document', 'documentH.pdf', 'spa'), + fixtureFactory.fileDeprecated('F3', 'shared1', 'document', 'documentA.pdf', 'eng'), + fixtureFactory.fileDeprecated('F4', 'shared1', 'document', 'documentD.pdf', 'spa'), + fixtureFactory.fileDeprecated('F5', 'shared3', 'document', 'documentE.pdf', 'eng'), + fixtureFactory.fileDeprecated('F6', 'shared3', 'document', 'documentF.pdf', 'spa'), + fixtureFactory.fileDeprecated('F7', 'shared4', 'document', 'documentG.pdf', 'eng'), + fixtureFactory.fileDeprecated('F8', 'shared4', 'document', 'documentH.pdf', 'spa'), ], ixextractors: [ fixtureFactory.ixExtractor('existingExtractor', 'kind', ['animalTemplate', 'plantTemplate']), diff --git a/app/api/services/ocr/specs/fixtures/fixtures.ts b/app/api/services/ocr/specs/fixtures/fixtures.ts index 251b716fa1..e8e3080a4a 100644 --- a/app/api/services/ocr/specs/fixtures/fixtures.ts +++ b/app/api/services/ocr/specs/fixtures/fixtures.ts @@ -9,7 +9,7 @@ const fixtures = { fixturesFactory.entity('parentForExistingRecord'), ], files: [ - fixturesFactory.file( + fixturesFactory.fileDeprecated( 'unrelatedAttachment', 'parentEntity', 'attachment', @@ -17,7 +17,7 @@ const fixtures = { 'eng' ), { - ...fixturesFactory.file( + ...fixturesFactory.fileDeprecated( 'sourceFile', 'parentEntity', 'document', @@ -27,7 +27,7 @@ const fixtures = { originalname: 'sourceFileOriginalName.pdf', }, { - ...fixturesFactory.file( + ...fixturesFactory.fileDeprecated( 'erroringSourceFile', 'parentEntity', 'document', @@ -35,42 +35,42 @@ const fixtures = { 'notALanguage' ), }, - fixturesFactory.file( + fixturesFactory.fileDeprecated( 'resultForExistingRecord', 'parentForExistingRecord', 'document', 'ocr_existingFileName.pdf', 'eng' ), - fixturesFactory.file( + fixturesFactory.fileDeprecated( 'sourceForExistingRecord', 'parentForExistingRecord', 'attachment', 'existingFileName.pdf', 'eng' ), - fixturesFactory.file( + fixturesFactory.fileDeprecated( 'resultToDelete', 'parentForExistingRecord', 'document', 'ocr_toDeleteFileName.pdf', 'eng' ), - fixturesFactory.file( + fixturesFactory.fileDeprecated( 'sourceToDelete', 'parentForExistingRecord', 'attachment', 'toDeleteFileName.pdf', 'eng' ), - fixturesFactory.file( + fixturesFactory.fileDeprecated( 'resultToDelete2', 'parentForExistingRecord', 'document', 'ocr_toDeleteFileName2.pdf', 'eng' ), - fixturesFactory.file( + fixturesFactory.fileDeprecated( 'sourceToDelete2', 'parentForExistingRecord', 'attachment', diff --git a/app/api/services/pdfsegmentation/specs/fixtures.ts b/app/api/services/pdfsegmentation/specs/fixtures.ts index 492345ebf1..cc4d13dec2 100644 --- a/app/api/services/pdfsegmentation/specs/fixtures.ts +++ b/app/api/services/pdfsegmentation/specs/fixtures.ts @@ -31,19 +31,19 @@ const fixturesPdfNameB = 'documentB.pdf'; const fixturesOneFile: DBFixture = { entities: [factory.entity('A1', 'templateToSegmentA')], settings, - files: [factory.file('F1', 'A1', 'document', fixturesPdfNameA)], + files: [factory.fileDeprecated('F1', 'A1', 'document', fixturesPdfNameA)], }; const fixturesOtherFile: DBFixture = { entities: [factory.entity('A2', 'templateToSegmentB')], settings: otherSettings, - files: [factory.file('F2', 'A2', 'document', fixturesPdfNameB)], + files: [factory.fileDeprecated('F2', 'A2', 'document', fixturesPdfNameB)], }; const fixturesMissingPdf: DBFixture = { entities: [factory.entity('A1', 'templateToSegmentA')], settings, - files: [factory.file('F1', 'A1', 'document', 'missing.pdf')], + files: [factory.fileDeprecated('F1', 'A1', 'document', 'missing.pdf')], }; const fixturesFiveFiles: DBFixture = { @@ -56,11 +56,11 @@ const fixturesFiveFiles: DBFixture = { factory.entity('A5', 'templateToSegmentA'), ], files: [ - factory.file('F1', 'A1', 'document', fixturesPdfNameA), - factory.file('F2', 'A2', 'document', fixturesPdfNameA), - factory.file('F3', 'A3', 'document', fixturesPdfNameA), - factory.file('F4', 'A4', 'document', fixturesPdfNameA), - factory.file('F5', 'A5', 'document', fixturesPdfNameA), + factory.fileDeprecated('F1', 'A1', 'document', fixturesPdfNameA), + factory.fileDeprecated('F2', 'A2', 'document', fixturesPdfNameA), + factory.fileDeprecated('F3', 'A3', 'document', fixturesPdfNameA), + factory.fileDeprecated('F4', 'A4', 'document', fixturesPdfNameA), + factory.fileDeprecated('F5', 'A5', 'document', fixturesPdfNameA), ], }; @@ -70,7 +70,7 @@ const fixturesOneHundredFiles: DBFixture = { factory.entity(`A${x.toString()}`, 'templateToSegmentA') ), files: [...Array(100).keys()].map(x => - factory.file(`F${x.toString()}`, `A${x.toString()}`, 'document', fixturesPdfNameA) + factory.fileDeprecated(`F${x.toString()}`, `A${x.toString()}`, 'document', fixturesPdfNameA) ), }; diff --git a/app/api/suggestions/specs/eventListeners.spec.ts b/app/api/suggestions/specs/eventListeners.spec.ts index 8eecda80a8..0c9d5e7820 100644 --- a/app/api/suggestions/specs/eventListeners.spec.ts +++ b/app/api/suggestions/specs/eventListeners.spec.ts @@ -58,8 +58,8 @@ const fixtures: DBFixture = { ), ], files: [ - fixturesFactory.file('entfile', 'entity for new file', 'document', 'entfile.pdf'), - fixturesFactory.file( + fixturesFactory.fileDeprecated('entfile', 'entity for new file', 'document', 'entfile.pdf'), + fixturesFactory.fileDeprecated( 'entfile2', 'entity with template not in config', 'document', @@ -514,7 +514,7 @@ describe(`On ${FileCreatedEvent.name}`, () => { const saveSpy = jest.spyOn(Suggestions, 'saveMultiple'); - const fileInfo = fixturesFactory.file( + const fileInfo = fixturesFactory.fileDeprecated( 'new file', 'entity for new file', 'document', @@ -533,7 +533,7 @@ describe(`On ${FileCreatedEvent.name}`, () => { it('should create blank suggestions, if the new file is an entity document', async () => { const saveSpy = jest.spyOn(Suggestions, 'saveMultiple'); - const fileInfo = fixturesFactory.file( + const fileInfo = fixturesFactory.fileDeprecated( 'new file', 'entity for new file', 'document', @@ -633,7 +633,7 @@ describe(`On ${FileCreatedEvent.name}`, () => { it('should not fail on not configured templates', async () => { const saveSpy = jest.spyOn(Suggestions, 'saveMultiple'); - const fileInfo = fixturesFactory.file( + const fileInfo = fixturesFactory.fileDeprecated( 'new file', 'entity with template not in config', 'document', diff --git a/app/api/suggestions/specs/fixtures.ts b/app/api/suggestions/specs/fixtures.ts index 5b201df0ab..cb68ca189a 100644 --- a/app/api/suggestions/specs/fixtures.ts +++ b/app/api/suggestions/specs/fixtures.ts @@ -716,7 +716,7 @@ const fixtures: DBFixture = { }, ], files: [ - factory.file('F1', 'shared1', 'document', 'documentRedRobin.pdf', 'eng', '', [ + factory.fileDeprecated('F1', 'shared1', 'document', 'documentRedRobin.pdf', 'eng', '', [ { name: 'age', selection: { @@ -725,7 +725,7 @@ const fixtures: DBFixture = { }, }, ]), - factory.file('F2', 'shared2', 'document', 'documentB.pdf', 'eng', '', [ + factory.fileDeprecated('F2', 'shared2', 'document', 'documentB.pdf', 'eng', '', [ { name: 'super_powers', selection: { @@ -741,7 +741,7 @@ const fixtures: DBFixture = { }, }, ]), - factory.file('F3', 'shared2', 'document', 'documentC.pdf', 'spa', '', [ + factory.fileDeprecated('F3', 'shared2', 'document', 'documentC.pdf', 'spa', '', [ { name: 'super_powers', selection: { @@ -750,7 +750,7 @@ const fixtures: DBFixture = { }, }, ]), - factory.file('Fshared5', 'shared5', 'document', 'documentPoisonIvy.pdf', 'eng', '', [ + factory.fileDeprecated('Fshared5', 'shared5', 'document', 'documentPoisonIvy.pdf', 'eng', '', [ { name: 'enemy', selection: { @@ -759,7 +759,7 @@ const fixtures: DBFixture = { }, }, ]), - factory.file('F4', 'shared6', 'document', 'documentD.pdf', 'eng', '', [ + factory.fileDeprecated('F4', 'shared6', 'document', 'documentD.pdf', 'eng', '', [ { name: 'enemy', selection: { @@ -768,7 +768,7 @@ const fixtures: DBFixture = { }, }, ]), - factory.file('F5', 'shared7', 'document', 'documentRiddler.pdf', 'eng', '', [ + factory.fileDeprecated('F5', 'shared7', 'document', 'documentRiddler.pdf', 'eng', '', [ { name: 'first_encountered', selection: { @@ -785,8 +785,8 @@ const fixtures: DBFixture = { }, }, ]), - factory.file('F6', 'shared8', 'document', 'documentRiddler.pdf', 'eng', '', []), - factory.file('F7', 'shared3', 'document', 'documentAlfred.pdf', 'eng', '', [ + factory.fileDeprecated('F6', 'shared8', 'document', 'documentRiddler.pdf', 'eng', '', []), + factory.fileDeprecated('F7', 'shared3', 'document', 'documentAlfred.pdf', 'eng', '', [ { name: 'super_powers', selection: { @@ -803,7 +803,7 @@ const fixtures: DBFixture = { }, }, ]), - factory.file( + factory.fileDeprecated( 'fileForentityWithSelects', 'entityWithSelects', 'document', @@ -811,7 +811,7 @@ const fixtures: DBFixture = { 'eng', 'documentWithSelects.pdf' ), - factory.file( + factory.fileDeprecated( 'fileForentityWithSelects2', 'entityWithSelects2', 'document', @@ -819,7 +819,7 @@ const fixtures: DBFixture = { 'eng', 'documentWithSelects2.pdf' ), - factory.file( + factory.fileDeprecated( 'fileForentityWithSelects3', 'entityWithSelects3', 'document', @@ -963,13 +963,25 @@ const stateFilterFixtures: DBFixture = { }), ], files: [ - factory.file('label-match-file-en', 'labeled-match', 'document', 'lmfen.pdf', 'en', undefined, [ - factory.fileExtractedMetadata('testprop', 'test-labeled-match'), - ]), - factory.file('label-match-file-es', 'labeled-match', 'document', 'lmfes.pdf', 'es', undefined, [ - factory.fileExtractedMetadata('testprop', 'test-labeled-match'), - ]), - factory.file( + factory.fileDeprecated( + 'label-match-file-en', + 'labeled-match', + 'document', + 'lmfen.pdf', + 'en', + undefined, + [factory.fileExtractedMetadata('testprop', 'test-labeled-match')] + ), + factory.fileDeprecated( + 'label-match-file-es', + 'labeled-match', + 'document', + 'lmfes.pdf', + 'es', + undefined, + [factory.fileExtractedMetadata('testprop', 'test-labeled-match')] + ), + factory.fileDeprecated( 'label-mismatch-file-en', 'labeled-mismatch', 'document', @@ -978,7 +990,7 @@ const stateFilterFixtures: DBFixture = { undefined, [factory.fileExtractedMetadata('testprop', 'test-labeled-mismatch')] ), - factory.file( + factory.fileDeprecated( 'label-mismatch-file-es', 'labeled-mismatch', 'document', @@ -987,7 +999,7 @@ const stateFilterFixtures: DBFixture = { undefined, [factory.fileExtractedMetadata('testprop', 'test-labeled-mismatch')] ), - factory.file( + factory.fileDeprecated( 'unlabeled-no-suggestion-file-en', 'unlabeled-no-suggestion', 'document', @@ -995,7 +1007,7 @@ const stateFilterFixtures: DBFixture = { 'en', undefined ), - factory.file( + factory.fileDeprecated( 'unlabeled-no-suggestion-file-es', 'unlabeled-no-suggestion', 'document', @@ -1003,7 +1015,7 @@ const stateFilterFixtures: DBFixture = { 'es', undefined ), - factory.file( + factory.fileDeprecated( 'unlabeled-no-context-file-en', 'unlabeled-no-context', 'document', @@ -1011,7 +1023,7 @@ const stateFilterFixtures: DBFixture = { 'en', undefined ), - factory.file( + factory.fileDeprecated( 'unlabeled-no-context-file-es', 'unlabeled-no-context', 'document', @@ -1019,7 +1031,7 @@ const stateFilterFixtures: DBFixture = { 'es', undefined ), - factory.file( + factory.fileDeprecated( 'unlabeled-obsolete-file-en', 'unlabeled-obsolete', 'document', @@ -1027,7 +1039,7 @@ const stateFilterFixtures: DBFixture = { 'en', undefined ), - factory.file( + factory.fileDeprecated( 'unlabeled-obsolete-file-es', 'unlabeled-obsolete', 'document', @@ -1035,7 +1047,7 @@ const stateFilterFixtures: DBFixture = { 'es', undefined ), - factory.file( + factory.fileDeprecated( 'unlabeled-others-file-en', 'unlabeled-others', 'document', @@ -1043,7 +1055,7 @@ const stateFilterFixtures: DBFixture = { 'en', undefined ), - factory.file( + factory.fileDeprecated( 'unlabeled-others-file-es', 'unlabeled-others', 'document', @@ -1051,7 +1063,7 @@ const stateFilterFixtures: DBFixture = { 'es', undefined ), - factory.file( + factory.fileDeprecated( 'unlabeled-processing-file-en', 'unlabeled-processing', 'document', @@ -1059,7 +1071,7 @@ const stateFilterFixtures: DBFixture = { 'en', undefined ), - factory.file( + factory.fileDeprecated( 'unlabeled-processing-file-es', 'unlabeled-processing', 'document', @@ -1067,7 +1079,7 @@ const stateFilterFixtures: DBFixture = { 'es', undefined ), - factory.file( + factory.fileDeprecated( 'unlabeled-error-file-en', 'unlabeled-error', 'document', @@ -1075,7 +1087,7 @@ const stateFilterFixtures: DBFixture = { 'en', undefined ), - factory.file( + factory.fileDeprecated( 'unlabeled-error-file-es', 'unlabeled-error', 'document', @@ -1363,7 +1375,7 @@ const selectAcceptanceFixtureBase: DBFixture = { }, ], files: [ - factory.file( + factory.fileDeprecated( 'fileForentityWithSelects', 'entityWithSelects', 'document', @@ -1712,7 +1724,7 @@ const relationshipAcceptanceFixtureBase: DBFixture = { }, ], files: [ - factory.file( + factory.fileDeprecated( 'fileForEntityWithRelationships', 'entityWithRelationships_sId', 'document', diff --git a/app/api/utils/fixturesFactory.ts b/app/api/utils/fixturesFactory.ts index 8bd83fcc83..e6c2c3e4d3 100644 --- a/app/api/utils/fixturesFactory.ts +++ b/app/api/utils/fixturesFactory.ts @@ -161,11 +161,36 @@ function getFixturesFactory() { }, }), - file: ( + attachment(id: string, extra: Partial = {}): WithId { + return this.file(id, { ...extra, type: 'attachment' }); + }, + + document(id: string, extra: Partial = {}): WithId { + return this.file(id, { ...extra, type: 'document' }); + }, + + custom_upload(id: string, extra: Partial = {}): WithId { + return this.file(id, { ...extra, type: 'custom' }); + }, + + file: (id: string, extra: Partial = {}): WithId => ({ + filename: id, + originalname: id, + ...extra, + _id: idMapper(`${id}`), + }), + + /** + * @deprecated too many parameters and dificult to read/use + * convention should be id and then a partial object with the + * desired extra params or id, something else important, extra params + * no more than 3 params + */ + fileDeprecated: ( id: string, - entity: string | undefined, - type: 'custom' | 'document' | 'thumbnail' | 'attachment' | undefined, - filename: string, + entity?: string | undefined, + type?: 'custom' | 'document' | 'thumbnail' | 'attachment' | undefined, + filename?: string | undefined, language: string = 'en', originalname: string | undefined = undefined, extractedMetadata: ExtractedMetadataSchema[] = [] @@ -174,7 +199,7 @@ function getFixturesFactory() { entity, language, type, - filename, + filename: filename || id, originalname: originalname || filename, extractedMetadata, }), diff --git a/package.json b/package.json index 8eaf9f790e..a3bb154064 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "uwazi", - "version": "1.184.0-rc2", + "version": "1.184.0-rc3", "description": "Uwazi is a free, open-source solution for organising, analysing and publishing your documents.", "keywords": [ "react" @@ -31,6 +31,7 @@ "dev-worker": "node --no-experimental-fetch ./scripts/run.js ../app/worker.ts", "dev-queue": "node --no-experimental-fetch ./scripts/run.js ../app/queueWorker.ts", "files-healthcheck": "USE_CWD=true node --no-experimental-fetch ./scripts/run.js ../scripts/filesHealthCheck.ts", + "new-files-healthcheck": "tsx ./scripts/scripts.v2/filesHealthCheck.ts", "generateAutomaticTranslationConfig": "node --no-experimental-fetch ./scripts/run.js ../scripts/scripts.v2/generateAutomaticTranslationConfig.ts", "check-translations": "node --no-experimental-fetch ./scripts/checkTranslations.mjs", "update-translations-db": "node --no-experimental-fetch scripts/run.js ../scripts/updateTranslationsDB.js", @@ -185,7 +186,7 @@ "mongoose": "8.1.2", "multer": "^1.4.5-lts.1", "node-uuid": "^1.4.7", - "nodemailer": "^6.9.15", + "nodemailer": "^6.9.14", "nprogress": "^0.2.0", "otplib": "^11.0.1", "passport": "^0.7.0", @@ -201,6 +202,8 @@ "react-color": "^2.19.3", "react-datepicker": "7.3.0", "react-device-detect": "^2.2.3", + "react-dnd": "^16.0.1", + "react-dnd-html5-backend": "^16.0.1", "react-dnd-html5-backend-old": "yarn:react-dnd-html5-backend@^15.1.2", "react-dnd-old": "yarn:react-dnd@2.6.0", "react-dom": "^18.3.1", @@ -236,7 +239,7 @@ "socket.io": "4.7.5", "socket.io-client": "4.7.5", "socket.io-parser": "4.2.4", - "stopword": "^3.1.1", + "stopword": "^3.0.1", "superagent": "10.1.0", "svg-captcha": "^1.4.0", "tiny-cookie": "^2.5.1", @@ -251,13 +254,13 @@ }, "devDependencies": { "@4tw/cypress-drag-drop": "^2.2.5", - "@babel/cli": "7.25.6", + "@babel/cli": "7.24.8", "@babel/core": "7.25.2", "@babel/eslint-parser": "7.25.1", "@babel/helper-call-delegate": "^7.12.13", "@babel/helper-get-function-arity": "^7.16.7", "@babel/helper-string-parser": "^7.24.8", - "@babel/parser": "^7.25.6", + "@babel/parser": "^7.25.4", "@babel/plugin-proposal-class-properties": "^7.18.6", "@babel/plugin-proposal-nullish-coalescing-operator": "^7.18.6", "@babel/plugin-proposal-object-rest-spread": "^7.20.7", @@ -270,7 +273,7 @@ "@babel/preset-react": "^7.24.7", "@babel/preset-typescript": "^7.24.7", "@babel/register": "^7.24.6", - "@babel/traverse": "^7.25.6", + "@babel/traverse": "^7.25.4", "@cfaester/enzyme-adapter-react-18": "^0.8.0", "@chromatic-com/storybook": "^1.6.1", "@cypress/react18": "^2.0.1", @@ -350,8 +353,8 @@ "eslint-config-airbnb": "19.0.4", "eslint-plugin-cypress": "^3.5.0", "eslint-plugin-import": "v2.29.1", - "eslint-plugin-jasmine": "4.2.1", - "eslint-plugin-jest": "v28.8.2", + "eslint-plugin-jasmine": "4.2.0", + "eslint-plugin-jest": "v28.8.0", "eslint-plugin-jsx-a11y": "6.7.1", "eslint-plugin-node": "^11.1.0", "eslint-plugin-prettier": "5.1.3", @@ -389,11 +392,12 @@ "terser-webpack-plugin": "^5.3.10", "ts-node": "^10.9.2", "tsconfig-paths": "^4.2.0", + "tsx": "^4.19.1", "wait-for-expect": "^3.0.2", "webpack": "^5.94.0", "webpack-bundle-analyzer": "^4.10.2", "webpack-cli": "5.1.4", - "webpack-dev-middleware": "7.4.2", + "webpack-dev-middleware": "7.3.0", "webpack-hot-middleware": "^2.26.1", "worker-loader": "^3.0.8" }, diff --git a/scripts/filesHealthCheck.ts b/scripts/filesHealthCheck.ts index 25adc78376..6c468371e4 100644 --- a/scripts/filesHealthCheck.ts +++ b/scripts/filesHealthCheck.ts @@ -47,8 +47,10 @@ async function handleTenant(tenantName: string) { const filteredFilesInStorage = new Set(filterFilesInStorage(allFilesInStorage)); let missingInStorage = 0; const countInStorage = filteredFilesInStorage.size; + let count = 0; allFilesInDb.forEach(file => { + count += 1; const existsInStorage = filteredFilesInStorage.delete( storage.getPath(file.filename, file.type) ); diff --git a/scripts/scripts.v2/filesHealthCheck.ts b/scripts/scripts.v2/filesHealthCheck.ts new file mode 100644 index 0000000000..13a9ce4e20 --- /dev/null +++ b/scripts/scripts.v2/filesHealthCheck.ts @@ -0,0 +1,112 @@ +import { S3Client } from '@aws-sdk/client-s3'; +import { DefaultTransactionManager } from 'api/common.v2/database/data_source_defaults'; +import { config } from 'api/config'; +import { DefaultFilesDataSource } from 'api/files.v2/database/data_source_defaults'; +import { FilesHealthCheck } from 'api/files.v2/FilesHealthCheck'; +import { S3FileStorage } from 'api/files.v2/infrastructure/S3FileStorage'; +import { DB } from 'api/odm'; +import { tenants } from 'api/tenants'; + +const { tenant, allTenants } = require('yargs') + .option('tenant', { + alias: 't', + type: 'string', + describe: 'Tenant to check', + default: undefined, + }) + .option('allTenants', { + alias: 'a', + type: 'boolean', + describe: 'Tenant to check', + default: false, + }).argv; + +let dbAuth = {}; + +if (process.env.DBUSER) { + dbAuth = { + auth: { authSource: 'admin' }, + user: process.env.DBUSER, + pass: process.env.DBPASS, + }; +} + +const LINE_PREFIX = process.env.LINE_PREFIX || '%> '; + +function print(content: any, error?: 'error') { + process[error ? 'stderr' : 'stdout'].write(`${LINE_PREFIX}${JSON.stringify(content)}\n`); +} + +type FileRecord = { type: string; filename: string; url?: string }; + +function filterFilesInStorage(files: string[]) { + return files.filter(file => !file.endsWith('activity.log')); +} + +async function handleTenant(tenantName: string) { + await tenants.run(async () => { + const s3Client = new S3Client({ + apiVersion: 'latest', + region: 'region', + ...config.s3, + }); + + const transactionManager = DefaultTransactionManager(); + const filesHealthCheck = new FilesHealthCheck( + new S3FileStorage(s3Client, tenants.current()), + DefaultFilesDataSource(transactionManager) + ); + + filesHealthCheck.onMissingInDB(file => { + print( + { + logType: 'missingInDb', + tenant: tenantName, + file, + }, + 'error' + ); + }); + + filesHealthCheck.onMissingInStorage(file => { + print( + { + logType: 'missingInStorage', + tenant: tenantName, + file, + }, + 'error' + ); + }); + + const result = await filesHealthCheck.execute(); + + print({ + logType: 'summary', + tenant: tenantName, + storage: tenants.current().featureFlags?.s3Storage ? 's3' : 'local', + missingInStorage: result.missingInStorage, + missingInDb: result.missingInDb, + countInDb: result.countInDb, + countInStorage: result.countInStorage, + }); + }, tenantName); +} + +async function run() { + await DB.connect(config.DBHOST, dbAuth); + await tenants.setupTenants(); + + if (!allTenants) { + return await handleTenant(tenant); + } + + await Object.keys(tenants.tenants).reduce(async (prev, tenantName) => { + await prev; + await handleTenant(tenantName); + }, Promise.resolve()); +} + +run().finally(() => { + process.exit(); +}); diff --git a/yarn.lock b/yarn.lock index 9b844e73c6..006215c336 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1935,116 +1935,236 @@ resolved "https://registry.yarnpkg.com/@esbuild/aix-ppc64/-/aix-ppc64-0.20.2.tgz#a70f4ac11c6a1dfc18b8bbb13284155d933b9537" integrity sha512-D+EBOJHXdNZcLJRBkhENNG8Wji2kgc9AZ9KiPr1JuZjsNtyHzrsfLRrY0tk2H2aoFu6RANO1y1iPPUCDYWkb5g== +"@esbuild/aix-ppc64@0.23.1": + version "0.23.1" + resolved "https://registry.yarnpkg.com/@esbuild/aix-ppc64/-/aix-ppc64-0.23.1.tgz#51299374de171dbd80bb7d838e1cfce9af36f353" + integrity sha512-6VhYk1diRqrhBAqpJEdjASR/+WVRtfjpqKuNw11cLiaWpAT/Uu+nokB+UJnevzy/P9C/ty6AOe0dwueMrGh/iQ== + "@esbuild/android-arm64@0.20.2": version "0.20.2" resolved "https://registry.yarnpkg.com/@esbuild/android-arm64/-/android-arm64-0.20.2.tgz#db1c9202a5bc92ea04c7b6840f1bbe09ebf9e6b9" integrity sha512-mRzjLacRtl/tWU0SvD8lUEwb61yP9cqQo6noDZP/O8VkwafSYwZ4yWy24kan8jE/IMERpYncRt2dw438LP3Xmg== +"@esbuild/android-arm64@0.23.1": + version "0.23.1" + resolved "https://registry.yarnpkg.com/@esbuild/android-arm64/-/android-arm64-0.23.1.tgz#58565291a1fe548638adb9c584237449e5e14018" + integrity sha512-xw50ipykXcLstLeWH7WRdQuysJqejuAGPd30vd1i5zSyKK3WE+ijzHmLKxdiCMtH1pHz78rOg0BKSYOSB/2Khw== + "@esbuild/android-arm@0.20.2": version "0.20.2" resolved "https://registry.yarnpkg.com/@esbuild/android-arm/-/android-arm-0.20.2.tgz#3b488c49aee9d491c2c8f98a909b785870d6e995" integrity sha512-t98Ra6pw2VaDhqNWO2Oph2LXbz/EJcnLmKLGBJwEwXX/JAN83Fym1rU8l0JUWK6HkIbWONCSSatf4sf2NBRx/w== +"@esbuild/android-arm@0.23.1": + version "0.23.1" + resolved "https://registry.yarnpkg.com/@esbuild/android-arm/-/android-arm-0.23.1.tgz#5eb8c652d4c82a2421e3395b808e6d9c42c862ee" + integrity sha512-uz6/tEy2IFm9RYOyvKl88zdzZfwEfKZmnX9Cj1BHjeSGNuGLuMD1kR8y5bteYmwqKm1tj8m4cb/aKEorr6fHWQ== + "@esbuild/android-x64@0.20.2": version "0.20.2" resolved "https://registry.yarnpkg.com/@esbuild/android-x64/-/android-x64-0.20.2.tgz#3b1628029e5576249d2b2d766696e50768449f98" integrity sha512-btzExgV+/lMGDDa194CcUQm53ncxzeBrWJcncOBxuC6ndBkKxnHdFJn86mCIgTELsooUmwUm9FkhSp5HYu00Rg== +"@esbuild/android-x64@0.23.1": + version "0.23.1" + resolved "https://registry.yarnpkg.com/@esbuild/android-x64/-/android-x64-0.23.1.tgz#ae19d665d2f06f0f48a6ac9a224b3f672e65d517" + integrity sha512-nlN9B69St9BwUoB+jkyU090bru8L0NA3yFvAd7k8dNsVH8bi9a8cUAUSEcEEgTp2z3dbEDGJGfP6VUnkQnlReg== + "@esbuild/darwin-arm64@0.20.2": version "0.20.2" resolved "https://registry.yarnpkg.com/@esbuild/darwin-arm64/-/darwin-arm64-0.20.2.tgz#6e8517a045ddd86ae30c6608c8475ebc0c4000bb" integrity sha512-4J6IRT+10J3aJH3l1yzEg9y3wkTDgDk7TSDFX+wKFiWjqWp/iCfLIYzGyasx9l0SAFPT1HwSCR+0w/h1ES/MjA== +"@esbuild/darwin-arm64@0.23.1": + version "0.23.1" + resolved "https://registry.yarnpkg.com/@esbuild/darwin-arm64/-/darwin-arm64-0.23.1.tgz#05b17f91a87e557b468a9c75e9d85ab10c121b16" + integrity sha512-YsS2e3Wtgnw7Wq53XXBLcV6JhRsEq8hkfg91ESVadIrzr9wO6jJDMZnCQbHm1Guc5t/CdDiFSSfWP58FNuvT3Q== + "@esbuild/darwin-x64@0.20.2": version "0.20.2" resolved "https://registry.yarnpkg.com/@esbuild/darwin-x64/-/darwin-x64-0.20.2.tgz#90ed098e1f9dd8a9381695b207e1cff45540a0d0" integrity sha512-tBcXp9KNphnNH0dfhv8KYkZhjc+H3XBkF5DKtswJblV7KlT9EI2+jeA8DgBjp908WEuYll6pF+UStUCfEpdysA== +"@esbuild/darwin-x64@0.23.1": + version "0.23.1" + resolved "https://registry.yarnpkg.com/@esbuild/darwin-x64/-/darwin-x64-0.23.1.tgz#c58353b982f4e04f0d022284b8ba2733f5ff0931" + integrity sha512-aClqdgTDVPSEGgoCS8QDG37Gu8yc9lTHNAQlsztQ6ENetKEO//b8y31MMu2ZaPbn4kVsIABzVLXYLhCGekGDqw== + "@esbuild/freebsd-arm64@0.20.2": version "0.20.2" resolved "https://registry.yarnpkg.com/@esbuild/freebsd-arm64/-/freebsd-arm64-0.20.2.tgz#d71502d1ee89a1130327e890364666c760a2a911" integrity sha512-d3qI41G4SuLiCGCFGUrKsSeTXyWG6yem1KcGZVS+3FYlYhtNoNgYrWcvkOoaqMhwXSMrZRl69ArHsGJ9mYdbbw== +"@esbuild/freebsd-arm64@0.23.1": + version "0.23.1" + resolved "https://registry.yarnpkg.com/@esbuild/freebsd-arm64/-/freebsd-arm64-0.23.1.tgz#f9220dc65f80f03635e1ef96cfad5da1f446f3bc" + integrity sha512-h1k6yS8/pN/NHlMl5+v4XPfikhJulk4G+tKGFIOwURBSFzE8bixw1ebjluLOjfwtLqY0kewfjLSrO6tN2MgIhA== + "@esbuild/freebsd-x64@0.20.2": version "0.20.2" resolved "https://registry.yarnpkg.com/@esbuild/freebsd-x64/-/freebsd-x64-0.20.2.tgz#aa5ea58d9c1dd9af688b8b6f63ef0d3d60cea53c" integrity sha512-d+DipyvHRuqEeM5zDivKV1KuXn9WeRX6vqSqIDgwIfPQtwMP4jaDsQsDncjTDDsExT4lR/91OLjRo8bmC1e+Cw== +"@esbuild/freebsd-x64@0.23.1": + version "0.23.1" + resolved "https://registry.yarnpkg.com/@esbuild/freebsd-x64/-/freebsd-x64-0.23.1.tgz#69bd8511fa013b59f0226d1609ac43f7ce489730" + integrity sha512-lK1eJeyk1ZX8UklqFd/3A60UuZ/6UVfGT2LuGo3Wp4/z7eRTRYY+0xOu2kpClP+vMTi9wKOfXi2vjUpO1Ro76g== + "@esbuild/linux-arm64@0.20.2": version "0.20.2" resolved "https://registry.yarnpkg.com/@esbuild/linux-arm64/-/linux-arm64-0.20.2.tgz#055b63725df678379b0f6db9d0fa85463755b2e5" integrity sha512-9pb6rBjGvTFNira2FLIWqDk/uaf42sSyLE8j1rnUpuzsODBq7FvpwHYZxQ/It/8b+QOS1RYfqgGFNLRI+qlq2A== +"@esbuild/linux-arm64@0.23.1": + version "0.23.1" + resolved "https://registry.yarnpkg.com/@esbuild/linux-arm64/-/linux-arm64-0.23.1.tgz#8050af6d51ddb388c75653ef9871f5ccd8f12383" + integrity sha512-/93bf2yxencYDnItMYV/v116zff6UyTjo4EtEQjUBeGiVpMmffDNUyD9UN2zV+V3LRV3/on4xdZ26NKzn6754g== + "@esbuild/linux-arm@0.20.2": version "0.20.2" resolved "https://registry.yarnpkg.com/@esbuild/linux-arm/-/linux-arm-0.20.2.tgz#76b3b98cb1f87936fbc37f073efabad49dcd889c" integrity sha512-VhLPeR8HTMPccbuWWcEUD1Az68TqaTYyj6nfE4QByZIQEQVWBB8vup8PpR7y1QHL3CpcF6xd5WVBU/+SBEvGTg== +"@esbuild/linux-arm@0.23.1": + version "0.23.1" + resolved "https://registry.yarnpkg.com/@esbuild/linux-arm/-/linux-arm-0.23.1.tgz#ecaabd1c23b701070484990db9a82f382f99e771" + integrity sha512-CXXkzgn+dXAPs3WBwE+Kvnrf4WECwBdfjfeYHpMeVxWE0EceB6vhWGShs6wi0IYEqMSIzdOF1XjQ/Mkm5d7ZdQ== + "@esbuild/linux-ia32@0.20.2": version "0.20.2" resolved "https://registry.yarnpkg.com/@esbuild/linux-ia32/-/linux-ia32-0.20.2.tgz#c0e5e787c285264e5dfc7a79f04b8b4eefdad7fa" integrity sha512-o10utieEkNPFDZFQm9CoP7Tvb33UutoJqg3qKf1PWVeeJhJw0Q347PxMvBgVVFgouYLGIhFYG0UGdBumROyiig== +"@esbuild/linux-ia32@0.23.1": + version "0.23.1" + resolved "https://registry.yarnpkg.com/@esbuild/linux-ia32/-/linux-ia32-0.23.1.tgz#3ed2273214178109741c09bd0687098a0243b333" + integrity sha512-VTN4EuOHwXEkXzX5nTvVY4s7E/Krz7COC8xkftbbKRYAl96vPiUssGkeMELQMOnLOJ8k3BY1+ZY52tttZnHcXQ== + "@esbuild/linux-loong64@0.20.2": version "0.20.2" resolved "https://registry.yarnpkg.com/@esbuild/linux-loong64/-/linux-loong64-0.20.2.tgz#a6184e62bd7cdc63e0c0448b83801001653219c5" integrity sha512-PR7sp6R/UC4CFVomVINKJ80pMFlfDfMQMYynX7t1tNTeivQ6XdX5r2XovMmha/VjR1YN/HgHWsVcTRIMkymrgQ== +"@esbuild/linux-loong64@0.23.1": + version "0.23.1" + resolved "https://registry.yarnpkg.com/@esbuild/linux-loong64/-/linux-loong64-0.23.1.tgz#a0fdf440b5485c81b0fbb316b08933d217f5d3ac" + integrity sha512-Vx09LzEoBa5zDnieH8LSMRToj7ir/Jeq0Gu6qJ/1GcBq9GkfoEAoXvLiW1U9J1qE/Y/Oyaq33w5p2ZWrNNHNEw== + "@esbuild/linux-mips64el@0.20.2": version "0.20.2" resolved "https://registry.yarnpkg.com/@esbuild/linux-mips64el/-/linux-mips64el-0.20.2.tgz#d08e39ce86f45ef8fc88549d29c62b8acf5649aa" integrity sha512-4BlTqeutE/KnOiTG5Y6Sb/Hw6hsBOZapOVF6njAESHInhlQAghVVZL1ZpIctBOoTFbQyGW+LsVYZ8lSSB3wkjA== +"@esbuild/linux-mips64el@0.23.1": + version "0.23.1" + resolved "https://registry.yarnpkg.com/@esbuild/linux-mips64el/-/linux-mips64el-0.23.1.tgz#e11a2806346db8375b18f5e104c5a9d4e81807f6" + integrity sha512-nrFzzMQ7W4WRLNUOU5dlWAqa6yVeI0P78WKGUo7lg2HShq/yx+UYkeNSE0SSfSure0SqgnsxPvmAUu/vu0E+3Q== + "@esbuild/linux-ppc64@0.20.2": version "0.20.2" resolved "https://registry.yarnpkg.com/@esbuild/linux-ppc64/-/linux-ppc64-0.20.2.tgz#8d252f0b7756ffd6d1cbde5ea67ff8fd20437f20" integrity sha512-rD3KsaDprDcfajSKdn25ooz5J5/fWBylaaXkuotBDGnMnDP1Uv5DLAN/45qfnf3JDYyJv/ytGHQaziHUdyzaAg== +"@esbuild/linux-ppc64@0.23.1": + version "0.23.1" + resolved "https://registry.yarnpkg.com/@esbuild/linux-ppc64/-/linux-ppc64-0.23.1.tgz#06a2744c5eaf562b1a90937855b4d6cf7c75ec96" + integrity sha512-dKN8fgVqd0vUIjxuJI6P/9SSSe/mB9rvA98CSH2sJnlZ/OCZWO1DJvxj8jvKTfYUdGfcq2dDxoKaC6bHuTlgcw== + "@esbuild/linux-riscv64@0.20.2": version "0.20.2" resolved "https://registry.yarnpkg.com/@esbuild/linux-riscv64/-/linux-riscv64-0.20.2.tgz#19f6dcdb14409dae607f66ca1181dd4e9db81300" integrity sha512-snwmBKacKmwTMmhLlz/3aH1Q9T8v45bKYGE3j26TsaOVtjIag4wLfWSiZykXzXuE1kbCE+zJRmwp+ZbIHinnVg== +"@esbuild/linux-riscv64@0.23.1": + version "0.23.1" + resolved "https://registry.yarnpkg.com/@esbuild/linux-riscv64/-/linux-riscv64-0.23.1.tgz#65b46a2892fc0d1af4ba342af3fe0fa4a8fe08e7" + integrity sha512-5AV4Pzp80fhHL83JM6LoA6pTQVWgB1HovMBsLQ9OZWLDqVY8MVobBXNSmAJi//Csh6tcY7e7Lny2Hg1tElMjIA== + "@esbuild/linux-s390x@0.20.2": version "0.20.2" resolved "https://registry.yarnpkg.com/@esbuild/linux-s390x/-/linux-s390x-0.20.2.tgz#3c830c90f1a5d7dd1473d5595ea4ebb920988685" integrity sha512-wcWISOobRWNm3cezm5HOZcYz1sKoHLd8VL1dl309DiixxVFoFe/o8HnwuIwn6sXre88Nwj+VwZUvJf4AFxkyrQ== +"@esbuild/linux-s390x@0.23.1": + version "0.23.1" + resolved "https://registry.yarnpkg.com/@esbuild/linux-s390x/-/linux-s390x-0.23.1.tgz#e71ea18c70c3f604e241d16e4e5ab193a9785d6f" + integrity sha512-9ygs73tuFCe6f6m/Tb+9LtYxWR4c9yg7zjt2cYkjDbDpV/xVn+68cQxMXCjUpYwEkze2RcU/rMnfIXNRFmSoDw== + "@esbuild/linux-x64@0.20.2": version "0.20.2" resolved "https://registry.yarnpkg.com/@esbuild/linux-x64/-/linux-x64-0.20.2.tgz#86eca35203afc0d9de0694c64ec0ab0a378f6fff" integrity sha512-1MdwI6OOTsfQfek8sLwgyjOXAu+wKhLEoaOLTjbijk6E2WONYpH9ZU2mNtR+lZ2B4uwr+usqGuVfFT9tMtGvGw== +"@esbuild/linux-x64@0.23.1": + version "0.23.1" + resolved "https://registry.yarnpkg.com/@esbuild/linux-x64/-/linux-x64-0.23.1.tgz#d47f97391e80690d4dfe811a2e7d6927ad9eed24" + integrity sha512-EV6+ovTsEXCPAp58g2dD68LxoP/wK5pRvgy0J/HxPGB009omFPv3Yet0HiaqvrIrgPTBuC6wCH1LTOY91EO5hQ== + "@esbuild/netbsd-x64@0.20.2": version "0.20.2" resolved "https://registry.yarnpkg.com/@esbuild/netbsd-x64/-/netbsd-x64-0.20.2.tgz#e771c8eb0e0f6e1877ffd4220036b98aed5915e6" integrity sha512-K8/DhBxcVQkzYc43yJXDSyjlFeHQJBiowJ0uVL6Tor3jGQfSGHNNJcWxNbOI8v5k82prYqzPuwkzHt3J1T1iZQ== +"@esbuild/netbsd-x64@0.23.1": + version "0.23.1" + resolved "https://registry.yarnpkg.com/@esbuild/netbsd-x64/-/netbsd-x64-0.23.1.tgz#44e743c9778d57a8ace4b72f3c6b839a3b74a653" + integrity sha512-aevEkCNu7KlPRpYLjwmdcuNz6bDFiE7Z8XC4CPqExjTvrHugh28QzUXVOZtiYghciKUacNktqxdpymplil1beA== + +"@esbuild/openbsd-arm64@0.23.1": + version "0.23.1" + resolved "https://registry.yarnpkg.com/@esbuild/openbsd-arm64/-/openbsd-arm64-0.23.1.tgz#05c5a1faf67b9881834758c69f3e51b7dee015d7" + integrity sha512-3x37szhLexNA4bXhLrCC/LImN/YtWis6WXr1VESlfVtVeoFJBRINPJ3f0a/6LV8zpikqoUg4hyXw0sFBt5Cr+Q== + "@esbuild/openbsd-x64@0.20.2": version "0.20.2" resolved "https://registry.yarnpkg.com/@esbuild/openbsd-x64/-/openbsd-x64-0.20.2.tgz#9a795ae4b4e37e674f0f4d716f3e226dd7c39baf" integrity sha512-eMpKlV0SThJmmJgiVyN9jTPJ2VBPquf6Kt/nAoo6DgHAoN57K15ZghiHaMvqjCye/uU4X5u3YSMgVBI1h3vKrQ== +"@esbuild/openbsd-x64@0.23.1": + version "0.23.1" + resolved "https://registry.yarnpkg.com/@esbuild/openbsd-x64/-/openbsd-x64-0.23.1.tgz#2e58ae511bacf67d19f9f2dcd9e8c5a93f00c273" + integrity sha512-aY2gMmKmPhxfU+0EdnN+XNtGbjfQgwZj43k8G3fyrDM/UdZww6xrWxmDkuz2eCZchqVeABjV5BpildOrUbBTqA== + "@esbuild/sunos-x64@0.20.2": version "0.20.2" resolved "https://registry.yarnpkg.com/@esbuild/sunos-x64/-/sunos-x64-0.20.2.tgz#7df23b61a497b8ac189def6e25a95673caedb03f" integrity sha512-2UyFtRC6cXLyejf/YEld4Hajo7UHILetzE1vsRcGL3earZEW77JxrFjH4Ez2qaTiEfMgAXxfAZCm1fvM/G/o8w== +"@esbuild/sunos-x64@0.23.1": + version "0.23.1" + resolved "https://registry.yarnpkg.com/@esbuild/sunos-x64/-/sunos-x64-0.23.1.tgz#adb022b959d18d3389ac70769cef5a03d3abd403" + integrity sha512-RBRT2gqEl0IKQABT4XTj78tpk9v7ehp+mazn2HbUeZl1YMdaGAQqhapjGTCe7uw7y0frDi4gS0uHzhvpFuI1sA== + "@esbuild/win32-arm64@0.20.2": version "0.20.2" resolved "https://registry.yarnpkg.com/@esbuild/win32-arm64/-/win32-arm64-0.20.2.tgz#f1ae5abf9ca052ae11c1bc806fb4c0f519bacf90" integrity sha512-GRibxoawM9ZCnDxnP3usoUDO9vUkpAxIIZ6GQI+IlVmr5kP3zUq+l17xELTHMWTWzjxa2guPNyrpq1GWmPvcGQ== +"@esbuild/win32-arm64@0.23.1": + version "0.23.1" + resolved "https://registry.yarnpkg.com/@esbuild/win32-arm64/-/win32-arm64-0.23.1.tgz#84906f50c212b72ec360f48461d43202f4c8b9a2" + integrity sha512-4O+gPR5rEBe2FpKOVyiJ7wNDPA8nGzDuJ6gN4okSA1gEOYZ67N8JPk58tkWtdtPeLz7lBnY6I5L3jdsr3S+A6A== + "@esbuild/win32-ia32@0.20.2": version "0.20.2" resolved "https://registry.yarnpkg.com/@esbuild/win32-ia32/-/win32-ia32-0.20.2.tgz#241fe62c34d8e8461cd708277813e1d0ba55ce23" integrity sha512-HfLOfn9YWmkSKRQqovpnITazdtquEW8/SoHW7pWpuEeguaZI4QnCRW6b+oZTztdBnZOS2hqJ6im/D5cPzBTTlQ== +"@esbuild/win32-ia32@0.23.1": + version "0.23.1" + resolved "https://registry.yarnpkg.com/@esbuild/win32-ia32/-/win32-ia32-0.23.1.tgz#5e3eacc515820ff729e90d0cb463183128e82fac" + integrity sha512-BcaL0Vn6QwCwre3Y717nVHZbAa4UBEigzFm6VdsVdT/MbZ38xoj1X9HPkZhbmaBGUD1W8vxAfffbDe8bA6AKnQ== + "@esbuild/win32-x64@0.20.2": version "0.20.2" resolved "https://registry.yarnpkg.com/@esbuild/win32-x64/-/win32-x64-0.20.2.tgz#9c907b21e30a52db959ba4f80bb01a0cc403d5cc" integrity sha512-N49X4lJX27+l9jbLKSqZ6bKNjzQvHaT8IIFUy+YIqmXQdjYCToGWwOItDrfby14c78aDd5NHQl29xingXfCdLQ== +"@esbuild/win32-x64@0.23.1": + version "0.23.1" + resolved "https://registry.yarnpkg.com/@esbuild/win32-x64/-/win32-x64-0.23.1.tgz#81fd50d11e2c32b2d6241470e3185b70c7b30699" + integrity sha512-BHpFFeslkWrXWyUPnbKm+xYYVYruCinGcftSBaa8zoF9hZO4BcSCFUvHVTtzpIY6YzUnYtuEhZ+C9iEXjxnasg== + "@eslint-community/eslint-utils@^4.2.0", "@eslint-community/eslint-utils@^4.4.0": version "4.4.0" resolved "https://registry.yarnpkg.com/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz#a23514e8fb9af1269d5f7788aa556798d61c6b59" @@ -9029,6 +9149,36 @@ esbuild-register@^3.5.0: "@esbuild/win32-ia32" "0.20.2" "@esbuild/win32-x64" "0.20.2" +esbuild@~0.23.0: + version "0.23.1" + resolved "https://registry.yarnpkg.com/esbuild/-/esbuild-0.23.1.tgz#40fdc3f9265ec0beae6f59824ade1bd3d3d2dab8" + integrity sha512-VVNz/9Sa0bs5SELtn3f7qhJCDPCF5oMEl5cO9/SSinpE9hbPVvxbd572HH5AKiP7WD8INO53GgfDDhRjkylHEg== + optionalDependencies: + "@esbuild/aix-ppc64" "0.23.1" + "@esbuild/android-arm" "0.23.1" + "@esbuild/android-arm64" "0.23.1" + "@esbuild/android-x64" "0.23.1" + "@esbuild/darwin-arm64" "0.23.1" + "@esbuild/darwin-x64" "0.23.1" + "@esbuild/freebsd-arm64" "0.23.1" + "@esbuild/freebsd-x64" "0.23.1" + "@esbuild/linux-arm" "0.23.1" + "@esbuild/linux-arm64" "0.23.1" + "@esbuild/linux-ia32" "0.23.1" + "@esbuild/linux-loong64" "0.23.1" + "@esbuild/linux-mips64el" "0.23.1" + "@esbuild/linux-ppc64" "0.23.1" + "@esbuild/linux-riscv64" "0.23.1" + "@esbuild/linux-s390x" "0.23.1" + "@esbuild/linux-x64" "0.23.1" + "@esbuild/netbsd-x64" "0.23.1" + "@esbuild/openbsd-arm64" "0.23.1" + "@esbuild/openbsd-x64" "0.23.1" + "@esbuild/sunos-x64" "0.23.1" + "@esbuild/win32-arm64" "0.23.1" + "@esbuild/win32-ia32" "0.23.1" + "@esbuild/win32-x64" "0.23.1" + escalade@^3.1.1, escalade@^3.1.2: version "3.1.2" resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.2.tgz#54076e9ab29ea5bf3d8f1ed62acffbb88272df27" @@ -10145,6 +10295,11 @@ fsevents@^2.3.2, fsevents@~2.3.2: resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a" integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA== +fsevents@~2.3.3: + version "2.3.3" + resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.3.tgz#cac6407785d03675a2a5e1a5305c697b347d90d6" + integrity sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw== + function-bind@^1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.2.tgz#2c02d864d97f3ea6c8830c464cbd11ab6eab7a1c" @@ -10256,6 +10411,13 @@ get-symbol-description@^1.0.2: es-errors "^1.3.0" get-intrinsic "^1.2.4" +get-tsconfig@^4.7.5: + version "4.8.1" + resolved "https://registry.yarnpkg.com/get-tsconfig/-/get-tsconfig-4.8.1.tgz#8995eb391ae6e1638d251118c7b56de7eb425471" + integrity sha512-k9PN+cFBmaLWtVz29SkUoqU5O0slLuHJXt/2P+tMVFT+phsSGXGkp9t3rQIqdz0e+06EHNGs3oM6ZX1s2zHxRg== + dependencies: + resolve-pkg-maps "^1.0.0" + getos@^3.2.1: version "3.2.1" resolved "https://registry.yarnpkg.com/getos/-/getos-3.2.1.tgz#0134d1f4e00eb46144c5a9c0ac4dc087cbb27dc5" @@ -15643,6 +15805,11 @@ resolve-from@^5.0.0: resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-5.0.0.tgz#c35225843df8f776df21c57557bc087e9dfdfc69" integrity sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw== +resolve-pkg-maps@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz#616b3dc2c57056b5588c31cdf4b3d64db133720f" + integrity sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw== + resolve.exports@^2.0.0: version "2.0.2" resolved "https://registry.yarnpkg.com/resolve.exports/-/resolve.exports-2.0.2.tgz#f8c934b8e6a13f539e38b7098e2e36134f01e800" @@ -16422,16 +16589,7 @@ string-length@^4.0.1: char-regex "^1.0.2" strip-ansi "^6.0.0" -"string-width-cjs@npm:string-width@^4.2.0": - version "4.2.3" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" - integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== - dependencies: - emoji-regex "^8.0.0" - is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.1" - -"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: +"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -16526,14 +16684,7 @@ string_decoder@~1.1.1: dependencies: safe-buffer "~5.1.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1": - version "6.0.1" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" - integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== - dependencies: - ansi-regex "^5.0.1" - -strip-ansi@^6.0.0, strip-ansi@^6.0.1: +"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -17149,6 +17300,16 @@ tsutils@^3.21.0: dependencies: tslib "^1.8.1" +tsx@^4.19.1: + version "4.19.1" + resolved "https://registry.yarnpkg.com/tsx/-/tsx-4.19.1.tgz#b7bffdf4b565813e4dea14b90872af279cd0090b" + integrity sha512-0flMz1lh74BR4wOvBjuh9olbnwqCPc35OOlfyzHba0Dc+QNUeWX/Gq2YTbnwcWPO3BMd8fkzRVrHcsR+a7z7rA== + dependencies: + esbuild "~0.23.0" + get-tsconfig "^4.7.5" + optionalDependencies: + fsevents "~2.3.3" + tty-browserify@^0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/tty-browserify/-/tty-browserify-0.0.1.tgz#3f05251ee17904dfd0677546670db9651682b811" @@ -17982,7 +18143,7 @@ world-countries@5.0.0: resolved "https://registry.yarnpkg.com/world-countries/-/world-countries-5.0.0.tgz#6f75ebcce3d5224d84e9117eaf0d75a7726b6501" integrity sha512-wAfOT9Y5i/xnxNOdKJKXdOCw9Q3yQLahBUeuRol+s+o20F6h2a4tLEbJ1lBCYwEQ30Sf9Meqeipk1gib3YwF5w== -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== @@ -18000,15 +18161,6 @@ wrap-ansi@^6.2.0: string-width "^4.1.0" strip-ansi "^6.0.0" -wrap-ansi@^7.0.0: - version "7.0.0" - resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" - integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== - dependencies: - ansi-styles "^4.0.0" - string-width "^4.1.0" - strip-ansi "^6.0.0" - wrap-ansi@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214"