From cb8f45cb316c6015a75b65875e5cc45141298e5a Mon Sep 17 00:00:00 2001 From: Joan Gallego Girona Date: Mon, 16 Sep 2024 13:12:02 +0200 Subject: [PATCH 1/2] Files health check tweaks (#7229) * Files health check tweaks - ignore all files inside /log/ not only activity log, this includes error.log debug.log, index.html - ignore all files inside /segmentation/ since there is no correspondence in the db for them - use the tenant prefix proerly on s3 calls instead of "tenant-name" use "tenant-name/" to ensure similar names do not appear wrongly reported * files health check ignores all index.html files * updated yarn.lock --- app/api/files.v2/FilesHealthCheck.ts | 5 +- .../files.v2/infrastructure/S3FileStorage.ts | 2 +- .../specs/S3FileStorage.spec.ts | 31 ++++++++ .../files.v2/specs/FilesHealthCheck.spec.ts | 45 ++++++++++- yarn.lock | 74 +++++++++++-------- 5 files changed, 121 insertions(+), 36 deletions(-) diff --git a/app/api/files.v2/FilesHealthCheck.ts b/app/api/files.v2/FilesHealthCheck.ts index ee9962053f..599f27d47e 100644 --- a/app/api/files.v2/FilesHealthCheck.ts +++ b/app/api/files.v2/FilesHealthCheck.ts @@ -3,7 +3,10 @@ import { FileStorage } from './contracts/FileStorage'; import { URLAttachment } from './model/URLAttachment'; function filterFilesInStorage(files: string[]) { - return files.filter(file => !file.endsWith('activity.log')); + return files.filter( + file => + !file.includes('/log/') && !file.includes('/segmentation/') && !file.includes('index.html') + ); } export class FilesHealthCheck { diff --git a/app/api/files.v2/infrastructure/S3FileStorage.ts b/app/api/files.v2/infrastructure/S3FileStorage.ts index 2e890040e0..6f719e2c34 100644 --- a/app/api/files.v2/infrastructure/S3FileStorage.ts +++ b/app/api/files.v2/infrastructure/S3FileStorage.ts @@ -37,7 +37,7 @@ export class S3FileStorage implements FileStorage { const response = await this.s3Client.send( new ListObjectsV2Command({ Bucket: config.s3.bucket, - Prefix: this.tenant.name, + Prefix: `${this.tenant.name}/`, ContinuationToken: token, MaxKeys: config.s3.batchSize, }) diff --git a/app/api/files.v2/infrastructure/specs/S3FileStorage.spec.ts b/app/api/files.v2/infrastructure/specs/S3FileStorage.spec.ts index 375a2e1011..9957cb8ace 100644 --- a/app/api/files.v2/infrastructure/specs/S3FileStorage.spec.ts +++ b/app/api/files.v2/infrastructure/specs/S3FileStorage.spec.ts @@ -75,6 +75,37 @@ describe('S3FileStorage', () => { }); describe('list', () => { + it('should list all s3 keys', async () => { + await s3Client.send( + new PutObjectCommand({ + Bucket: 'uwazi-development', + Key: 'test-tenant/documents/document1', + Body: 'body', + }) + ); + + await s3Client.send( + new PutObjectCommand({ + Bucket: 'uwazi-development', + Key: 'test-tenant/documents/document2', + Body: 'body', + }) + ); + + await s3Client.send( + new PutObjectCommand({ + Bucket: 'uwazi-development', + Key: 'test-tenant-2/documents/document3', + Body: 'body', + }) + ); + + const listedFiles = await s3fileStorage.list(); + + expect(listedFiles.sort()).toEqual( + ['test-tenant/documents/document1', 'test-tenant/documents/document2'].sort() + ); + }); it('should list all s3 keys', async () => { await s3Client.send( new PutObjectCommand({ diff --git a/app/api/files.v2/specs/FilesHealthCheck.spec.ts b/app/api/files.v2/specs/FilesHealthCheck.spec.ts index 6f88de9500..88a2ff1864 100644 --- a/app/api/files.v2/specs/FilesHealthCheck.spec.ts +++ b/app/api/files.v2/specs/FilesHealthCheck.spec.ts @@ -95,14 +95,53 @@ describe('FilesHealthCheck', () => { }); }); - it('should ignore activity.log files', async () => { - testStorageFiles = ['log/1-activity.log', 'log/2-activity.log', 'document/file1']; + it('should ignore all /log files', async () => { + testStorageFiles = [ + '/log/1-activity.log', + '/log/log.log', + '/log/error.log', + '/log/debug.log', + '/document/file1', + ]; await testingEnvironment.setUp({ files: [] }); const summary = await filesHealthCheck.execute(); expect(summary).toMatchObject({ - missingInDbList: ['document/file1'], + missingInDbList: ['/document/file1'], + missingInDb: 1, + }); + }); + + it('should ignore all /segmentation files', async () => { + testStorageFiles = [ + '/segmentation/1-activity.log', + '/documents/segmentation/1-activity.log', + '/document/file1', + ]; + await testingEnvironment.setUp({ files: [] }); + + const summary = await filesHealthCheck.execute(); + + expect(summary).toMatchObject({ + missingInDbList: ['/document/file1'], + missingInDb: 1, + }); + }); + + it('should ignore all index.html files', async () => { + testStorageFiles = [ + '/documents/index.html', + '/index.html', + '/segmentation/index.html', + '/document/file1', + ]; + await testingEnvironment.setUp({ files: [] }); + + const summary = await filesHealthCheck.execute(); + + expect(summary).toMatchObject({ + missingInDbList: ['/document/file1'], missingInDb: 1, }); }); diff --git a/yarn.lock b/yarn.lock index 642166b4f7..c1ef1a43cb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -623,10 +623,10 @@ "@smithy/types" "^3.3.0" tslib "^2.6.2" -"@babel/cli@7.25.6": - version "7.25.6" - resolved "https://registry.yarnpkg.com/@babel/cli/-/cli-7.25.6.tgz#bc35561adc78ade43ac9c09a690768493ab9ed95" - integrity sha512-Z+Doemr4VtvSD2SNHTrkiFZ1LX+JI6tyRXAAOb4N9khIuPyoEPmTPJarPm8ljJV1D6bnMQjyHMWTT9NeKbQuXA== +"@babel/cli@7.24.8": + version "7.24.8" + resolved "https://registry.yarnpkg.com/@babel/cli/-/cli-7.24.8.tgz#79eaa55a69c77cafbea3e87537fd1df5a5a2edf8" + integrity sha512-isdp+G6DpRyKc+3Gqxy2rjzgF7Zj9K0mzLNnxz+E/fgeag8qT3vVulX4gY9dGO1q0y+0lUv6V3a+uhUzMzrwXg== dependencies: "@jridgewell/trace-mapping" "^0.3.25" commander "^6.2.0" @@ -637,7 +637,7 @@ slash "^2.0.0" optionalDependencies: "@nicolo-ribaudo/chokidar-2" "2.1.8-no-fsevents.3" - chokidar "^3.6.0" + chokidar "^3.4.0" "@babel/code-frame@^7.0.0", "@babel/code-frame@^7.10.4", "@babel/code-frame@^7.12.13", "@babel/code-frame@^7.16.7", "@babel/code-frame@^7.24.7": version "7.24.7" @@ -895,7 +895,7 @@ js-tokens "^4.0.0" picocolors "^1.0.0" -"@babel/parser@^7.1.0", "@babel/parser@^7.14.7", "@babel/parser@^7.20.7", "@babel/parser@^7.23.0", "@babel/parser@^7.23.9", "@babel/parser@^7.24.4", "@babel/parser@^7.25.0", "@babel/parser@^7.25.6": +"@babel/parser@^7.1.0", "@babel/parser@^7.14.7", "@babel/parser@^7.20.7", "@babel/parser@^7.23.0", "@babel/parser@^7.23.9", "@babel/parser@^7.24.4", "@babel/parser@^7.25.0", "@babel/parser@^7.25.4", "@babel/parser@^7.25.6": version "7.25.6" resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.25.6.tgz#85660c5ef388cbbf6e3d2a694ee97a38f18afe2f" integrity sha512-trGdfBdbD0l1ZPmcJ83eNxB9rbEax4ALFTF7fN386TMYbeCQbyme5cOEXQhbGXKebwGaB/J52w1mrklMcbgy6Q== @@ -1758,7 +1758,7 @@ "@babel/parser" "^7.25.0" "@babel/types" "^7.25.0" -"@babel/traverse@^7.18.9", "@babel/traverse@^7.24.1", "@babel/traverse@^7.24.7", "@babel/traverse@^7.24.8", "@babel/traverse@^7.25.0", "@babel/traverse@^7.25.1", "@babel/traverse@^7.25.2", "@babel/traverse@^7.25.3", "@babel/traverse@^7.25.4", "@babel/traverse@^7.25.6": +"@babel/traverse@^7.18.9", "@babel/traverse@^7.24.1", "@babel/traverse@^7.24.7", "@babel/traverse@^7.24.8", "@babel/traverse@^7.25.0", "@babel/traverse@^7.25.1", "@babel/traverse@^7.25.2", "@babel/traverse@^7.25.3", "@babel/traverse@^7.25.4": version "7.25.6" resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.25.6.tgz#04fad980e444f182ecf1520504941940a90fea41" integrity sha512-9Vrcx5ZW6UwK5tvqsj0nGpp/XzqthkT0dqIc9g1AdtygFToNtTF67XzYS//dm+SAK9cp3B9R4ZO/46p63SCjlQ== @@ -7114,7 +7114,7 @@ child-process-promise@^2.2.1: node-version "^1.0.0" promise-polyfill "^6.0.1" -"chokidar@>=3.0.0 <4.0.0", chokidar@^3.5.2, chokidar@^3.5.3, chokidar@^3.6.0: +"chokidar@>=3.0.0 <4.0.0", chokidar@^3.4.0, chokidar@^3.5.2, chokidar@^3.5.3: version "3.6.0" resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.6.0.tgz#197c6cc669ef2a8dc5e7b4d97ee4e092c3eb0d5b" integrity sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw== @@ -7974,10 +7974,10 @@ cypress-real-events@^1.13.0: resolved "https://registry.yarnpkg.com/cypress-real-events/-/cypress-real-events-1.13.0.tgz#6b7cd32dcac172db1493608f97a2576c7d0bd5af" integrity sha512-LoejtK+dyZ1jaT8wGT5oASTPfsNV8/ClRp99ruN60oPj8cBJYod80iJDyNwfPAu4GCxTXOhhAv9FO65Hpwt6Hg== -cypress@13.14.1: - version "13.14.1" - resolved "https://registry.yarnpkg.com/cypress/-/cypress-13.14.1.tgz#05875bbbf6333e858a92aed33ba67d7ddf8370d7" - integrity sha512-Wo+byPmjps66hACEH5udhXINEiN3qS3jWNGRzJOjrRJF3D0+YrcP2LVB1T7oYaVQM/S+eanqEvBWYc8cf7Vcbg== +cypress@13.13.2: + version "13.13.2" + resolved "https://registry.yarnpkg.com/cypress/-/cypress-13.13.2.tgz#c71f8d92056c430b1b879e5313f6de25ccce0eda" + integrity sha512-PvJQU33933NvS1StfzEb8/mu2kMy4dABwCF+yd5Bi7Qly1HOVf+Bufrygee/tlmty/6j5lX+KIi8j9Q3JUMbhA== dependencies: "@cypress/request" "^3.0.1" "@cypress/xvfb" "^1.2.4" @@ -9307,15 +9307,15 @@ eslint-plugin-import@v2.29.1: semver "^6.3.1" tsconfig-paths "^3.15.0" -eslint-plugin-jasmine@4.2.1: - version "4.2.1" - resolved "https://registry.yarnpkg.com/eslint-plugin-jasmine/-/eslint-plugin-jasmine-4.2.1.tgz#80925dc6b24ee263eb834bedb09ff4abf3740b3d" - integrity sha512-Vwecc66rjMgz2e9UtGScsUdo6D+SbfgPA4Kf0zdAl4+5IQMRL0mXd8973MaZuYYF89XpRjQEGl5TNmg2Bv+KcQ== +eslint-plugin-jasmine@4.2.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/eslint-plugin-jasmine/-/eslint-plugin-jasmine-4.2.0.tgz#ed0fe988b6e3b123905a7bf68d77239649fd018c" + integrity sha512-zSCsnP4gMqBSt8jApExP0ja43nAI1fpAD5kY+knrIJylBxC/LEth25PkqcKJqW32GjesjsiA1SSSR3Z5qIranA== -eslint-plugin-jest@v28.8.2: - version "28.8.2" - resolved "https://registry.yarnpkg.com/eslint-plugin-jest/-/eslint-plugin-jest-28.8.2.tgz#7f8307179c5cf8d51101b3aa002be168daadecbc" - integrity sha512-mC3OyklHmS5i7wYU1rGId9EnxRI8TVlnFG56AE+8U9iRy6zwaNygZR+DsdZuCL0gRG0wVeyzq+uWcPt6yJrrMA== +eslint-plugin-jest@v28.8.0: + version "28.8.0" + resolved "https://registry.yarnpkg.com/eslint-plugin-jest/-/eslint-plugin-jest-28.8.0.tgz#54f597b5a3295ad04ec946baa245ad02b9b2bca0" + integrity sha512-Tubj1hooFxCl52G4qQu0edzV/+EZzPUeN8p2NnW5uu4fbDs+Yo7+qDVDc4/oG3FbCqEBmu/OC3LSsyiU22oghw== dependencies: "@typescript-eslint/utils" "^6.0.0 || ^7.0.0 || ^8.0.0" @@ -13599,7 +13599,7 @@ node-version@^1.0.0: version "1.2.0" resolved "https://registry.npmjs.org/node-version/-/node-version-1.2.0.tgz" -nodemailer@^6.9.15: +nodemailer@^6.9.14: version "6.9.15" resolved "https://registry.yarnpkg.com/nodemailer/-/nodemailer-6.9.15.tgz#57b79dc522be27e0e47ac16cc860aa0673e62e04" integrity sha512-AHf04ySLC6CIfuRtRiEYtGEXgRfa6INgWGluDhnxTZhHSKvrBu7lc1VVchQ0d8nPc4cFaZoPq8vkyNoZr0TpGQ== @@ -14758,10 +14758,10 @@ prettier-linter-helpers@^1.0.0: dependencies: fast-diff "^1.1.2" -prettier@3.3.3, prettier@^3.1.1: - version "3.3.3" - resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.3.3.tgz#30c54fe0be0d8d12e6ae61dbb10109ea00d53105" - integrity sha512-i2tDNA0O5IrMO757lfrdQZCc2jPNDVntV0m/+4whiDfWaTKfMNgR7Qz0NAeGz/nRqF4m5/6CLzbP4/liHt12Ew== +prettier@3.2.5: + version "3.2.5" + resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.2.5.tgz#e52bc3090586e824964a8813b09aba6233b28368" + integrity sha512-3/GWa9aOC0YeD7LUfvOG2NiDyhOWRvt1k+rcKhOuYnMY24iiCphgneUfJDyFXd6rZCAnuLBv6UeAULtrhT/F4A== prettier@^1.16.4: version "1.19.1" @@ -14773,6 +14773,11 @@ prettier@^2.6.2: resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.8.8.tgz#e8c5d7e98a4305ffe3de2e1fc4aca1a71c28b1da" integrity sha512-tdN8qQGvNjw4CHbY+XXk0JgCXn9QiF21a55rBe5LJAU+kDyC4WQn4+awm2Xfk2lQMk5fKup9XgzTZtGkjBdP9Q== +prettier@^3.1.1: + version "3.3.3" + resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.3.3.tgz#30c54fe0be0d8d12e6ae61dbb10109ea00d53105" + integrity sha512-i2tDNA0O5IrMO757lfrdQZCc2jPNDVntV0m/+4whiDfWaTKfMNgR7Qz0NAeGz/nRqF4m5/6CLzbP4/liHt12Ew== + pretty-bytes@^5.6.0: version "5.6.0" resolved "https://registry.yarnpkg.com/pretty-bytes/-/pretty-bytes-5.6.0.tgz#356256f643804773c82f64723fe78c92c62beaeb" @@ -15122,6 +15127,13 @@ react-device-detect@^2.2.3: dependencies: dnd-core "15.1.2" +react-dnd-html5-backend@^16.0.1: + version "16.0.1" + resolved "https://registry.yarnpkg.com/react-dnd-html5-backend/-/react-dnd-html5-backend-16.0.1.tgz#87faef15845d512a23b3c08d29ecfd34871688b6" + integrity sha512-Wu3dw5aDJmOGw8WjH1I1/yTH+vlXEL4vmjk5p+MHxP8HuHJS1lAGeIdG/hze1AvNeXWo/JgULV87LyQOr+r5jw== + dependencies: + dnd-core "^16.0.1" + "react-dnd-old@yarn:react-dnd@2.6.0": version "2.6.0" resolved "https://registry.yarnpkg.com/react-dnd/-/react-dnd-2.6.0.tgz#7fa25676cf827d58a891293e3c1ab59da002545a" @@ -15141,7 +15153,7 @@ react-dnd-test-backend@15.1.1: dependencies: dnd-core "15.1.1" -react-dnd@*: +react-dnd@*, react-dnd@^16.0.1: version "16.0.1" resolved "https://registry.yarnpkg.com/react-dnd/-/react-dnd-16.0.1.tgz#2442a3ec67892c60d40a1559eef45498ba26fa37" integrity sha512-QeoM/i73HHu2XF9aKksIUuamHPDvRglEwdHL4jsp784BgUuWcg6mzfxT0QDdQz8Wj0qyRKx2eMg8iZtWvU4E2Q== @@ -16530,7 +16542,7 @@ stdin-discarder@^0.2.1: resolved "https://registry.yarnpkg.com/stdin-discarder/-/stdin-discarder-0.2.2.tgz#390037f44c4ae1a1ae535c5fe38dc3aba8d997be" integrity sha512-UhDfHmA92YAlNnCfhmq0VeNL5bDbiZGg7sZ2IvPsXubGkiNa9EC+tUTsjBRsYUAz87btI6/1wf4XoVvQ3uRnmQ== -stopword@^3.1.1: +stopword@^3.0.1: version "3.1.1" resolved "https://registry.yarnpkg.com/stopword/-/stopword-3.1.1.tgz#ce3cf748cafd962904902dd050f6f16ca42d3ec0" integrity sha512-TzJdIuzqJNo6IaFvrF3fYqu08uJ/0VMsdABl6d6+dt6daD7QeHJnMt9sPqhVIxEmNaaeE8+eandVPJv9RhAL5Q== @@ -17918,10 +17930,10 @@ webpack-cli@5.1.4: rechoir "^0.8.0" webpack-merge "^5.7.3" -webpack-dev-middleware@7.4.2: - version "7.4.2" - resolved "https://registry.yarnpkg.com/webpack-dev-middleware/-/webpack-dev-middleware-7.4.2.tgz#40e265a3d3d26795585cff8207630d3a8ff05877" - integrity sha512-xOO8n6eggxnwYpy1NlzUKpvrjfJTvae5/D6WOK0S2LSo7vjmo5gCM1DbLUmFqrMTJP+W/0YZNctm7jasWvLuBA== +webpack-dev-middleware@7.3.0: + version "7.3.0" + resolved "https://registry.yarnpkg.com/webpack-dev-middleware/-/webpack-dev-middleware-7.3.0.tgz#5975ea41271083dc5678886b99d4c058382fb311" + integrity sha512-xD2qnNew+F6KwOGZR7kWdbIou/ud7cVqLEXeK1q0nHcNsX/u7ul/fSdlOTX4ntSL5FNFy7ZJJXbf0piF591JYw== dependencies: colorette "^2.0.10" memfs "^4.6.0" From 6485b6c58228f70a1f37af2ef016b5d60403a198 Mon Sep 17 00:00:00 2001 From: Daneryl Date: Mon, 16 Sep 2024 13:12:08 +0200 Subject: [PATCH 2/2] Bump version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 27433db65a..e5cc0f9553 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "uwazi", - "version": "1.184.0", + "version": "1.184.1", "description": "Uwazi is a free, open-source solution for organising, analysing and publishing your documents.", "keywords": [ "react"