From b37db6da4155b5922498ac056d38f9364304b9f9 Mon Sep 17 00:00:00 2001 From: Jacob Bowdoin <7559478+jacob-8@users.noreply.github.com> Date: Fri, 25 Oct 2024 09:56:46 +0800 Subject: [PATCH] migration (#486) * update vite and vitest packages * switch orama to main route * fix table header being over-displayed by ear icons and image icons * improve entry modal * remove algolia * add supabase notes on how to seed local db from production * move generated types; add dialects table * merge Supabase types via script * connect photos and videos to senses * connect dialect_ids * connect speaker_ids to audio and videos * speakers * use speakers view * create hourly refreshing materialized entries view * keep photos, videos, speakers, dialects (many-many items) out of entries_view * add in batching ability * cached-data-store * use rpc for entries view * handle entry store errors better ... and many other tasks needed for migrating entries from Firebase to Supabase --- .firebaserc | 2 +- .github/workflows/lighthouse-audit.yml | 2 +- .github/workflows/lint.yml | 4 +- .vscode/settings.json | 7 +- README.md | 18 +- package.json | 20 +- packages/functions/package.json | 2 +- .../{tsconfig.json => tsconfig.json.bak} | 0 packages/scripts/.gitignore | 3 +- packages/scripts/config-firebase.ts | 12 +- packages/scripts/config-supabase.ts | 68 +- packages/scripts/import/post-request.ts | 4 + .../scripts/migrate-to-supabase/.gitignore | 2 - ...-current-senses-have-entry-placeholders.ts | 72 + packages/scripts/migrate-to-supabase/auth.ts | 12 +- .../convert-entries.test.ts | 74 +- .../migrate-to-supabase/convert-entries.ts | 208 +- .../convert-speakers.test.ts | 15 + .../migrate-to-supabase/convert-speakers.ts | 58 + .../create-rest-of-dictionary-placeholders.ts | 47 + .../migrate-to-supabase/fetch-entries.ts | 31 - .../migrate-to-supabase/get-user-id.ts | 20 + .../scripts/migrate-to-supabase/log-once.ts | 2 +- .../migrate-entries.test.ts | 65 + .../migrate-to-supabase/migrate-entries.ts | 184 + packages/scripts/migrate-to-supabase/notes.md | 67 +- .../operations/constants.ts | 2 + .../operations/operations.test.ts | 213 + .../operations/operations.ts | 266 + .../operations/test-timestamp.ts | 1 + .../migrate-to-supabase/operations/utils.ts | 6 + .../scripts/migrate-to-supabase/reset-db.ts | 17 + .../migrate-to-supabase/run-migration.ts | 198 + .../save-content-update.ts | 216 + .../save-firestore-data.ts | 56 + .../migrate-to-supabase/to-sql-string.ts | 15 + .../utils/remove-seconds-underscore.ts | 37 + .../migrate-to-supabase/write-users-insert.ts | 88 +- packages/scripts/package.json | 17 +- packages/scripts/record-logs.ts | 11 + packages/scripts/vitest.config.migration.ts | 15 + packages/scripts/vitest.config.ts | 7 +- packages/site/.env | 3 - packages/site/package.json | 21 +- packages/site/src/app.d.ts | 20 +- .../site/src/db-tests/cached-data.test.ts | 441 + packages/site/src/db-tests/clients.ts | 4 +- ...update.test.ts => content-update.test.bak} | 433 +- packages/site/src/db-tests/users.test.ts | 25 +- packages/site/src/docs/CONTRIBUTING.md | 7 +- packages/site/src/docs/Supabase.md | 42 +- .../src/docs/data/new-entry-field-creation.md | 4 +- packages/site/src/docs/misc/entry-fields.md | 6 +- .../site/src/docs/misc/import-dictionary.md | 2 +- packages/site/src/docs/misc/power-search.md | 23 - .../src/lib/components/SeoMetaTags.svelte | 4 +- .../src/lib/components/audio/EditAudio.svelte | 107 +- .../components/audio/EditAudio.variants.ts | 52 +- ....svelte => UploadProgressBarStatus.svelte} | 4 +- ...ts => UploadProgressBarStatus.variants.ts} | 2 +- .../src/lib/components/audio/upload-audio.ts | 7 +- .../src/lib/components/entry/EditField.svelte | 187 +- .../components/entry/EditFieldModal.svelte | 27 +- .../lib/components/entry/EntryDialect.svelte | 80 +- .../components/entry/EntryPartOfSpeech.svelte | 41 +- .../entry/EntrySemanticDomains.svelte | 6 +- .../src/lib/components/image/Image.svelte | 2 +- .../keyboards/ipa/IpaKeyboard.svelte | 59 +- .../lib/components/media/AddSpeaker.svelte | 26 +- .../lib/components/media/SelectSpeaker.svelte | 92 +- .../components/search/ClearRefinements.svelte | 21 - .../src/lib/components/search/Hits.svelte | 28 - .../lib/components/search/InfiniteHits.svelte | 38 - .../components/search/InstantSearch.svelte | 61 - .../lib/components/search/Pagination.svelte | 47 - .../components/search/RefinementList.svelte | 137 - .../lib/components/search/SearchBox.svelte | 58 - .../src/lib/components/search/SortBy.svelte | 40 - .../src/lib/components/search/Stats.svelte | 24 - .../components/search/ToggleRefinement.svelte | 46 - .../components/ui/array/MultiSelect.svelte | 2 +- .../src/lib/components/video/AddVideo.svelte | 43 +- .../components/video/PasteVideoLink.svelte | 23 +- .../src/lib/components/video/PlayVideo.svelte | 32 +- .../lib/components/video/UploadVideo.svelte | 115 - .../components/video/VideoThirdParty.svelte | 12 +- .../video/parse-hosted-video-url.ts | 79 + .../components/video/parseVideoData.test.ts | 46 - .../lib/components/video/parseVideoData.ts | 21 - .../src/lib/components/video/upload-video.ts | 57 + packages/site/src/lib/dbOperations.ts | 42 +- ...iods_and_comma_separate_parts_of_speech.ts | 32 +- .../helpers/entry/get_local_orthagraphies.ts | 52 +- packages/site/src/lib/helpers/entry/new.ts | 39 - packages/site/src/lib/helpers/entry/update.ts | 36 - .../site/src/lib/helpers/get-post-requests.ts | 10 +- packages/site/src/lib/helpers/glosses.test.ts | 33 +- packages/site/src/lib/helpers/glosses.ts | 14 +- packages/site/src/lib/helpers/media.ts | 88 +- packages/site/src/lib/helpers/scrollPoint.ts | 21 - packages/site/src/lib/helpers/share.ts | 32 +- packages/site/src/lib/helpers/time.ts | 18 +- packages/site/src/lib/mocks/db.ts | 59 +- packages/site/src/lib/mocks/entries.ts | 188 +- packages/site/src/lib/mocks/layout.ts | 19 +- packages/site/src/lib/mocks/seed/postgres.ts | 46 +- packages/site/src/lib/mocks/seed/tables.ts | 7 +- .../src/lib/mocks/seed/to-sql-string.test.sql | 8 +- .../site/src/lib/mocks/seed/to-sql-string.ts | 85 +- .../lib/mocks/seed/write-seed-and-reset-db.ts | 16 +- .../search/augment-entry-for-search.test.ts | 21 +- .../lib/search/augment-entry-for-search.ts | 149 +- packages/site/src/lib/search/index.ts | 15 +- packages/site/src/lib/search/orama.worker.ts | 118 +- packages/site/src/lib/stores/algolia.ts | 3 - packages/site/src/lib/supabase/admin.ts | 8 +- packages/site/src/lib/supabase/cached-data.ts | 153 + .../site/src/lib/supabase/change/sense.ts | 35 - .../site/src/lib/supabase/change/sentence.ts | 35 - .../site/src/lib/supabase/database.types.ts | 111 +- packages/site/src/lib/supabase/index.ts | 5 +- packages/site/src/lib/supabase/operations.ts | 479 ++ .../site/src/lib/supabase/tsconfig.json.bak | 14 - .../convert_and_expand_entry.test.ts | 28 - .../transformers/convert_and_expand_entry.ts | 22 - .../convert_entry_to_current_shape.test.ts | 264 - .../convert_entry_to_current_shape.ts | 104 - .../convert_photo_file_to_current_shape.ts | 51 - .../convert_sound_file_to_current_shape.ts | 58 - .../convert_video_file_to_current_shape.ts | 26 - .../src/lib/transformers/expand_entry.test.ts | 162 - .../site/src/lib/transformers/expand_entry.ts | 94 - packages/site/src/routes/+layout.ts | 84 +- packages/site/src/routes/Banner.svelte | 6 +- .../src/routes/[dictionaryId]/+layout.svelte | 8 +- .../site/src/routes/[dictionaryId]/+layout.ts | 81 +- .../site/src/routes/[dictionaryId]/+page.ts | 11 +- .../src/routes/[dictionaryId]/SideMenu.svelte | 37 +- .../[dictionaryId]/entries-local/+page.svelte | 107 - .../entries-local/EntriesGallery.svelte | 26 - .../entries-local/EntryFilters.svelte | 172 - .../entries-local/FilterList.variants.ts | 75 - .../entries-local/Pagination.svelte | 34 - .../[dictionaryId]/entries-local/View.svelte | 93 - .../[dictionaryId]/entries/+layout.svelte | 104 - .../[dictionaryId]/entries/+page.svelte | 111 + .../{entries-local => entries}/+page.ts | 1 + .../[dictionaryId]/entries}/AddEntry.svelte | 18 +- .../[dictionaryId]/entries/Audio.variants.ts | 79 - .../ClearFilters.svelte | 10 +- .../entries/EntriesGallery.svelte | 23 + .../EntriesPrint.svelte | 27 +- .../entries/EntryFilters.svelte | 292 +- .../FilterList.svelte | 14 +- .../entries/FilterList.variants.ts | 75 + .../[dictionaryId]/entries/Pagination.svelte | 34 + .../entries}/PaginationButtons.svelte | 6 +- .../entries}/PaginationButtons.variants.ts | 0 .../SearchInput.svelte | 10 +- .../SwitchView.svelte | 4 +- .../ToggleFacet.svelte | 0 .../routes/[dictionaryId]/entries/View.svelte | 72 + .../entries/[redirectId]/+page.ts | 10 +- .../entries/{ => components}/Audio.svelte | 10 +- .../entries/components/Audio.variants.ts | 79 + .../entries/{ => components}/Video.svelte | 6 +- .../entries/gallery/+page.svelte | 99 - .../entries/gallery/GalleryEntry.svelte | 49 +- .../entries/gallery/GalleryEntry.variants.ts | 234 +- .../[dictionaryId]/entries/list/+page.svelte | 65 - .../[dictionaryId]/entries/list/List.svelte | 16 - .../entries/list/List.variants.ts | 126 +- .../entries/list/ListEntry.svelte | 91 +- .../entries/list/ListEntry.variants.ts | 254 +- .../[dictionaryId]/entries/print/+page.svelte | 156 - .../entries/print/PrintEntry.svelte | 121 +- .../entries/print/PrintEntry.variants.ts | 162 +- .../entries/print/PrintFieldCheckboxes.svelte | 49 +- .../entries/print/QrCode.svelte | 37 +- .../[dictionaryId]/entries/print/mock-data.ts | 748 +- .../entries/print/printFields.ts | 4 +- .../[dictionaryId]/entries/print/qrcodegen.ts | 1101 ++- .../[dictionaryId]/entries/table/+page.svelte | 62 - .../[dictionaryId]/entries/table/Cell.svelte | 208 +- .../table/ColumnAdjustSlideover.svelte | 44 +- .../entries/table/EntriesTable.svelte | 15 +- .../entries/table/EntriesTable.variants.ts | 40 +- .../entries/table/cells/CheckboxCell.svelte | 42 +- .../entries/table/cells/PhotoCell.svelte | 11 +- .../table/cells/SelectSpeakerCell.svelte | 55 +- .../table/cells/SelectSpeakerCell.variants.ts | 118 +- .../entries/table/cells/Textbox.svelte | 27 +- .../entries/table/setUpColumns.ts | 50 +- .../entry/[entryId]/+page.svelte | 72 +- .../[dictionaryId]/entry/[entryId]/+page.ts | 59 +- .../entry/[entryId]/EntryDisplay.svelte | 151 +- .../entry/[entryId]/EntryDisplay.variants.ts | 408 +- .../entry/[entryId]/EntryMedia.svelte | 45 +- .../entry/[entryId]/EntryMedia.variants.ts | 34 +- .../entry/[entryId]/EntrySentence.svelte | 59 + .../entry/[entryId]/GeoTaggingModal.svelte | 13 +- .../[entryId]/GeoTaggingModal.variants.ts | 114 +- .../entry/[entryId]/Sense.svelte | 90 - .../entry/[entryId]/Sense.variants.ts | 43 - .../entry/[entryId]/SupaSense.svelte | 176 +- .../entry/[entryId]/SupaSense.variants.ts | 136 +- .../entry/[entryId]/_page.variants.ts | 118 +- .../entry/[entryId]/seo_description.test.ts | 99 +- .../entry/[entryId]/seo_description.ts | 44 +- .../routes/[dictionaryId]/export/+page.bak | 144 + .../routes/[dictionaryId]/export/+page.svelte | 127 +- .../export/DownloadMedia.svelte | 4 +- .../assignFormattedEntryValuesForCsv.test.ts | 72 +- .../assignFormattedEntryValuesForCsv.ts | 72 +- .../export/assignHeadersForCsv.test.ts | 196 +- .../export/assignHeadersForCsv.ts | 98 +- .../[dictionaryId]/export/fetchSpeakers.ts | 22 +- .../export/friendlyName.test.ts | 46 +- .../[dictionaryId]/export/friendlyName.ts | 30 +- .../export/prepareEntriesForCsv.test.ts | 224 +- .../export/prepareEntriesForCsv.ts | 123 +- .../invite/[inviteId]/+page.svelte | 6 +- .../src/routes/[dictionaryId]/load-entries.ts | 100 - .../api/db/build-search-indexes/+server.ts | 84 + .../routes/api/db/content-update/+server.ts | 466 +- .../src/routes/api/db/content-update/_call.ts | 7 + .../email/new_user/save-user-to-supabase.ts | 21 +- .../src/routes/create-dictionary/+page.ts | 9 +- packages/site/src/routes/og/+server.ts | 19 +- .../site/src/routes/og/LoadOgImage.svelte | 35 +- .../search-by-algolia-light-background.svg | 1 - packages/site/tsconfig.json | 12 +- packages/site/vite.config.ts | 21 +- packages/site/vitest.config.db.ts | 3 +- packages/site/vitest.config.ts | 1 + packages/types/entry.algolia.interface.ts | 25 - packages/types/entry.interface.ts | 181 +- packages/types/index.ts | 13 +- packages/types/package.json | 20 +- packages/types/print-entry.interface.ts | 8 +- packages/types/speaker.interface.ts | 16 +- packages/types/supabase/augments.types.ts | 175 + packages/types/supabase/combined.types.ts | 2199 +++++ .../supabase/content-import.interface.ts | 103 + .../supabase/content-update.interface.ts | 196 +- packages/types/supabase/entry.interface.ts | 18 + .../lib => types}/supabase/generated.types.ts | 416 +- packages/types/supabase/merge-types.test.ts | 206 + packages/types/tsconfig.json | 23 + packages/types/user.interface.ts | 29 +- packages/types/vitest.config.ts | 8 + pnpm-lock.yaml | 7247 +++++------------ pnpm-workspace.yaml | 7 +- supabase/clear-db-data.sql | 1 - supabase/config.toml | 134 +- supabase/ideas/construct-view.sql | 281 + supabase/ideas/entry-relationships.sql | 14 + supabase/{notes.md => ideas/misc.md} | 0 supabase/ideas/sentence-updates.sql | 116 - ...055922_senses-and-entry-updates-tables.sql | 30 +- ...115031759_fb-email-to-supabase-user-id.sql | 4 +- ...onaries-entries-sentences-texts-tables.sql | 10 +- .../20240222001122_media-tables.sql | 5 + ...2557_include-sentences-in-entries-view.sql | 11 +- .../migrations/20240225012557_updates.sql | 3 +- ...pl-variant-dialects-ei-field-plus-meta.sql | 383 + ...5_notes-multistring-ei-field-plus-meta.sql | 4 - .../20240825052204_connect-audio-video.sql | 9 - .../20241024024631_faster_entries_view.sql | 135 + .../20241024024888_entries_view_indexes.sql | 24 + vitest.workspace.ts | 1 + 271 files changed, 15270 insertions(+), 13713 deletions(-) rename packages/functions/{tsconfig.json => tsconfig.json.bak} (100%) delete mode 100644 packages/scripts/migrate-to-supabase/.gitignore create mode 100644 packages/scripts/migrate-to-supabase/all-current-senses-have-entry-placeholders.ts create mode 100644 packages/scripts/migrate-to-supabase/convert-speakers.test.ts create mode 100644 packages/scripts/migrate-to-supabase/convert-speakers.ts create mode 100644 packages/scripts/migrate-to-supabase/create-rest-of-dictionary-placeholders.ts delete mode 100644 packages/scripts/migrate-to-supabase/fetch-entries.ts create mode 100644 packages/scripts/migrate-to-supabase/get-user-id.ts create mode 100644 packages/scripts/migrate-to-supabase/migrate-entries.test.ts create mode 100644 packages/scripts/migrate-to-supabase/migrate-entries.ts create mode 100644 packages/scripts/migrate-to-supabase/operations/constants.ts create mode 100644 packages/scripts/migrate-to-supabase/operations/operations.test.ts create mode 100644 packages/scripts/migrate-to-supabase/operations/operations.ts create mode 100644 packages/scripts/migrate-to-supabase/operations/test-timestamp.ts create mode 100644 packages/scripts/migrate-to-supabase/operations/utils.ts create mode 100644 packages/scripts/migrate-to-supabase/reset-db.ts create mode 100644 packages/scripts/migrate-to-supabase/run-migration.ts create mode 100644 packages/scripts/migrate-to-supabase/save-content-update.ts create mode 100644 packages/scripts/migrate-to-supabase/save-firestore-data.ts create mode 100644 packages/scripts/migrate-to-supabase/to-sql-string.ts create mode 100644 packages/scripts/migrate-to-supabase/utils/remove-seconds-underscore.ts create mode 100644 packages/scripts/record-logs.ts create mode 100644 packages/scripts/vitest.config.migration.ts create mode 100644 packages/site/src/db-tests/cached-data.test.ts rename packages/site/src/db-tests/{content-update.test.ts => content-update.test.bak} (61%) delete mode 100644 packages/site/src/docs/misc/power-search.md rename packages/site/src/lib/components/audio/{UploadAudioStatus.svelte => UploadProgressBarStatus.svelte} (92%) rename packages/site/src/lib/components/audio/{UploadAudioStatus.variants.ts => UploadProgressBarStatus.variants.ts} (89%) delete mode 100644 packages/site/src/lib/components/search/ClearRefinements.svelte delete mode 100644 packages/site/src/lib/components/search/Hits.svelte delete mode 100644 packages/site/src/lib/components/search/InfiniteHits.svelte delete mode 100644 packages/site/src/lib/components/search/InstantSearch.svelte delete mode 100644 packages/site/src/lib/components/search/Pagination.svelte delete mode 100644 packages/site/src/lib/components/search/RefinementList.svelte delete mode 100644 packages/site/src/lib/components/search/SearchBox.svelte delete mode 100644 packages/site/src/lib/components/search/SortBy.svelte delete mode 100644 packages/site/src/lib/components/search/Stats.svelte delete mode 100644 packages/site/src/lib/components/search/ToggleRefinement.svelte delete mode 100644 packages/site/src/lib/components/video/UploadVideo.svelte create mode 100644 packages/site/src/lib/components/video/parse-hosted-video-url.ts delete mode 100644 packages/site/src/lib/components/video/parseVideoData.test.ts delete mode 100644 packages/site/src/lib/components/video/parseVideoData.ts create mode 100644 packages/site/src/lib/components/video/upload-video.ts delete mode 100644 packages/site/src/lib/helpers/entry/new.ts delete mode 100644 packages/site/src/lib/helpers/entry/update.ts delete mode 100644 packages/site/src/lib/helpers/scrollPoint.ts delete mode 100644 packages/site/src/lib/stores/algolia.ts create mode 100644 packages/site/src/lib/supabase/cached-data.ts delete mode 100644 packages/site/src/lib/supabase/change/sense.ts delete mode 100644 packages/site/src/lib/supabase/change/sentence.ts create mode 100644 packages/site/src/lib/supabase/operations.ts delete mode 100644 packages/site/src/lib/supabase/tsconfig.json.bak delete mode 100644 packages/site/src/lib/transformers/convert_and_expand_entry.test.ts delete mode 100644 packages/site/src/lib/transformers/convert_and_expand_entry.ts delete mode 100644 packages/site/src/lib/transformers/convert_entry_to_current_shape.test.ts delete mode 100644 packages/site/src/lib/transformers/convert_entry_to_current_shape.ts delete mode 100644 packages/site/src/lib/transformers/convert_photo_file_to_current_shape.ts delete mode 100644 packages/site/src/lib/transformers/convert_sound_file_to_current_shape.ts delete mode 100644 packages/site/src/lib/transformers/convert_video_file_to_current_shape.ts delete mode 100644 packages/site/src/lib/transformers/expand_entry.test.ts delete mode 100644 packages/site/src/lib/transformers/expand_entry.ts delete mode 100644 packages/site/src/routes/[dictionaryId]/entries-local/+page.svelte delete mode 100644 packages/site/src/routes/[dictionaryId]/entries-local/EntriesGallery.svelte delete mode 100644 packages/site/src/routes/[dictionaryId]/entries-local/EntryFilters.svelte delete mode 100644 packages/site/src/routes/[dictionaryId]/entries-local/FilterList.variants.ts delete mode 100644 packages/site/src/routes/[dictionaryId]/entries-local/Pagination.svelte delete mode 100644 packages/site/src/routes/[dictionaryId]/entries-local/View.svelte delete mode 100644 packages/site/src/routes/[dictionaryId]/entries/+layout.svelte create mode 100644 packages/site/src/routes/[dictionaryId]/entries/+page.svelte rename packages/site/src/routes/[dictionaryId]/{entries-local => entries}/+page.ts (93%) rename packages/site/src/{lib/components/search => routes/[dictionaryId]/entries}/AddEntry.svelte (61%) delete mode 100644 packages/site/src/routes/[dictionaryId]/entries/Audio.variants.ts rename packages/site/src/routes/[dictionaryId]/{entries-local => entries}/ClearFilters.svelte (59%) create mode 100644 packages/site/src/routes/[dictionaryId]/entries/EntriesGallery.svelte rename packages/site/src/routes/[dictionaryId]/{entries-local => entries}/EntriesPrint.svelte (84%) rename packages/site/src/routes/[dictionaryId]/{entries-local => entries}/FilterList.svelte (84%) create mode 100644 packages/site/src/routes/[dictionaryId]/entries/FilterList.variants.ts create mode 100644 packages/site/src/routes/[dictionaryId]/entries/Pagination.svelte rename packages/site/src/{lib/components/search => routes/[dictionaryId]/entries}/PaginationButtons.svelte (86%) rename packages/site/src/{lib/components/search => routes/[dictionaryId]/entries}/PaginationButtons.variants.ts (100%) rename packages/site/src/routes/[dictionaryId]/{entries-local => entries}/SearchInput.svelte (83%) rename packages/site/src/routes/[dictionaryId]/{entries-local => entries}/SwitchView.svelte (94%) rename packages/site/src/routes/[dictionaryId]/{entries-local => entries}/ToggleFacet.svelte (100%) create mode 100644 packages/site/src/routes/[dictionaryId]/entries/View.svelte rename packages/site/src/routes/[dictionaryId]/entries/{ => components}/Audio.svelte (89%) create mode 100644 packages/site/src/routes/[dictionaryId]/entries/components/Audio.variants.ts rename packages/site/src/routes/[dictionaryId]/entries/{ => components}/Video.svelte (80%) delete mode 100644 packages/site/src/routes/[dictionaryId]/entries/gallery/+page.svelte delete mode 100644 packages/site/src/routes/[dictionaryId]/entries/list/+page.svelte delete mode 100644 packages/site/src/routes/[dictionaryId]/entries/list/List.svelte delete mode 100644 packages/site/src/routes/[dictionaryId]/entries/print/+page.svelte delete mode 100644 packages/site/src/routes/[dictionaryId]/entries/table/+page.svelte create mode 100644 packages/site/src/routes/[dictionaryId]/entry/[entryId]/EntrySentence.svelte delete mode 100644 packages/site/src/routes/[dictionaryId]/entry/[entryId]/Sense.svelte delete mode 100644 packages/site/src/routes/[dictionaryId]/entry/[entryId]/Sense.variants.ts create mode 100644 packages/site/src/routes/[dictionaryId]/export/+page.bak delete mode 100644 packages/site/src/routes/[dictionaryId]/load-entries.ts create mode 100644 packages/site/src/routes/api/db/build-search-indexes/+server.ts create mode 100644 packages/site/src/routes/api/db/content-update/_call.ts delete mode 100644 packages/site/static/images/search-by-algolia-light-background.svg delete mode 100644 packages/types/entry.algolia.interface.ts create mode 100644 packages/types/supabase/augments.types.ts create mode 100644 packages/types/supabase/combined.types.ts create mode 100644 packages/types/supabase/content-import.interface.ts create mode 100644 packages/types/supabase/entry.interface.ts rename packages/{site/src/lib => types}/supabase/generated.types.ts (77%) create mode 100644 packages/types/supabase/merge-types.test.ts create mode 100644 packages/types/tsconfig.json create mode 100644 packages/types/vitest.config.ts delete mode 100644 supabase/clear-db-data.sql create mode 100644 supabase/ideas/construct-view.sql create mode 100644 supabase/ideas/entry-relationships.sql rename supabase/{notes.md => ideas/misc.md} (100%) delete mode 100644 supabase/ideas/sentence-updates.sql create mode 100644 supabase/migrations/20240824035215_fb-sb-migration-needs-connect-audio-video--pl-variant-dialects-ei-field-plus-meta.sql delete mode 100644 supabase/migrations/20240824035215_notes-multistring-ei-field-plus-meta.sql delete mode 100644 supabase/migrations/20240825052204_connect-audio-video.sql create mode 100644 supabase/migrations/20241024024631_faster_entries_view.sql create mode 100644 supabase/migrations/20241024024888_entries_view_indexes.sql diff --git a/.firebaserc b/.firebaserc index eb80fd5ff..a1215de8a 100644 --- a/.firebaserc +++ b/.firebaserc @@ -1,6 +1,6 @@ { "projects": { "default": "talking-dictionaries-dev", - "production": "talking-dictionaries-alpha" + "production": "talking-dictionaries-alpha", } } diff --git a/.github/workflows/lighthouse-audit.yml b/.github/workflows/lighthouse-audit.yml index fa9ba6b82..30a1fdb6c 100644 --- a/.github/workflows/lighthouse-audit.yml +++ b/.github/workflows/lighthouse-audit.yml @@ -21,7 +21,7 @@ jobs: with: urls: | ${{ github.event.deployment_status.target_url }} - ${{ github.event.deployment_status.target_url }}/achi/entries/list + ${{ github.event.deployment_status.target_url }}/achi/entries ${{ github.event.deployment_status.target_url }}/jaRhn6MAZim4Blvr1iEv/entry/yt9ja7ymh9xgba5i # configPath: './.github/lighthouserc.json' # https://github.com/GoogleChrome/lighthouse-ci/blob/main/docs/getting-started.md https://github.com/GoogleChrome/lighthouse-ci/blob/main/docs/configuration.md#assert uploadArtifacts: true diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index e6a433825..7011ba684 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -11,11 +11,11 @@ jobs: - uses: pnpm/action-setup@v2 with: - version: 8.6.0 + version: 9.6.0 - uses: actions/setup-node@v3 with: - node-version: 18 + node-version: 20 cache: pnpm - run: pnpm install diff --git a/.vscode/settings.json b/.vscode/settings.json index cfcb791c5..ba9865865 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -33,9 +33,9 @@ }, // Silence stylistic rules in IDE, but still auto fix them "eslint.rules.customizations": [ - { "rule": "@stylistic/*", "severity": "off" }, - { "rule": "style*", "severity": "off" }, - { "rule": "*indent", "severity": "off" }, + { "rule": "style/*", "severity": "off" }, + { "rule": "format/*", "severity": "off" }, + { "rule": "*-indent", "severity": "off" }, { "rule": "*-spacing", "severity": "off" }, { "rule": "*-spaces", "severity": "off" }, { "rule": "*-order", "severity": "off" }, @@ -47,6 +47,7 @@ { "rule": "import/order", "severity": "off" }, { "rule": "sort-imports", "severity": "off" }, { "rule": "ts/no-empty-function", "severity": "off" }, + { "rule": "svelte/indent", "severity": "off" }, ], "eslint.validate": [ "javascript", diff --git a/README.md b/README.md index deb4ffcd7..ef189d5be 100644 --- a/README.md +++ b/README.md @@ -7,22 +7,12 @@ A mobile-first community focused dictionary-building web app built by [Living To [](https://svelte.dev/) [](https://kit.svelte.dev/) [](https://unocss.dev/integrations/svelte-scoped) -[](https://firebase.google.com/) -[](https://firebase.google.com/) +[](https://firebase.google.com/) [](https://vercel.com/) -[](https://www.algolia.com/) - -Firebase is used for: - -- Authentication -- Cloud Firestore -- Storage - -These functions are being transitioned to Supabase with the exception of Storage, images will remain in GCP. Supabase may support this directly in the future. +[](https://www.orama.com/) +[](https://cloud.google.com/storage) ## Contributing -- Choose an already approved task from the [Development Roadmap](https://github.com/jwrunner/Living-Dictionaries/projects/1) or [create an issue](https://github.com/jwrunner/Living-Dictionaries/issues) to propose a new feature (please await discussion before creating a pull request). +- Choose an already approved task from the [Development Roadmap](https://github.com/jwrunner/Living-Dictionaries/projects/1) or [create an issue](https://github.com/jwrunner/Living-Dictionaries/issues) to propose a new feature (please await discussion before creating a pull request). - Read [CONTRIBUTING](https://livingdictionaries.app/kitbook/docs/CONTRIBUTING) and then move on to all the other pages under "Docs" in the Kitbook to understand how to set up a dev environment, repo conventions, commit conventions, and more. - - diff --git a/package.json b/package.json index 44f50437e..3fc10c07e 100644 --- a/package.json +++ b/package.json @@ -3,6 +3,8 @@ "type": "module", "scripts": { "dev": "pnpm --filter=site dev", + "prod": "pnpm --filter=site prod", + "mixed": "pnpm --filter=site mixed", "dev:open": "pnpm --filter=site dev --open", "build": "pnpm --filter=site build", "preview": "pnpm --filter=site preview", @@ -19,22 +21,22 @@ "lint:inspect": "npx @eslint/config-inspector", "lint:inspect-future": "eslint --inspect-config", "reset-db": "pnpm --filter=site reset-db", - "generate-types": "supabase gen types typescript --local --schema public > packages/site/src/lib/supabase/generated.types.ts", + "generate-types": "supabase gen types typescript --local --schema public > packages/types/supabase/generated.types.ts", "check-packages": "pnpm update --interactive --recursive --latest" }, "devDependencies": { - "@antfu/eslint-config": "^2.12.1", - "@typescript-eslint/eslint-plugin": "6.21.0", - "eslint": "^8.56.0", - "eslint-plugin-svelte": "^2.33.2", + "@antfu/eslint-config": "^2.27.3", + "@typescript-eslint/eslint-plugin": "8.3.0", + "eslint": "^9.9.1", + "eslint-plugin-svelte": "^2.43.0", "eslint-plugin-svelte-stylistic": "^0.0.4", - "lint-staged": "^13.2.2", - "simple-git-hooks": "^2.11.0", - "start-server-and-test": "^2.0.3", + "lint-staged": "^15.2.9", + "simple-git-hooks": "^2.11.1", + "start-server-and-test": "^2.0.5", "supabase": "^1.145.4", "svelte": "^4.2.12", "typescript": "~5.1.6", - "vitest": "^1.4.0" + "vitest": "^2.1.3" }, "simple-git-hooks": { "pre-commit": "pnpm lint-staged" diff --git a/packages/functions/package.json b/packages/functions/package.json index 1cc22153f..86131acad 100644 --- a/packages/functions/package.json +++ b/packages/functions/package.json @@ -30,6 +30,6 @@ "@types/node": "^18.11.18", "node-fetch": "^2.6.7", "typescript": "^5.1.6", - "vitest": "^1.4.0" + "vitest": "^2.0.5" } } diff --git a/packages/functions/tsconfig.json b/packages/functions/tsconfig.json.bak similarity index 100% rename from packages/functions/tsconfig.json rename to packages/functions/tsconfig.json.bak diff --git a/packages/scripts/.gitignore b/packages/scripts/.gitignore index 7f2090e84..032816f4e 100644 --- a/packages/scripts/.gitignore +++ b/packages/scripts/.gitignore @@ -2,4 +2,5 @@ logs service-account* .env sheets-viewer-SA.json -.env.supabase \ No newline at end of file +.env.supabase +.env.production.supabase \ No newline at end of file diff --git a/packages/scripts/config-firebase.ts b/packages/scripts/config-firebase.ts index b84985f6e..f66131373 100644 --- a/packages/scripts/config-firebase.ts +++ b/packages/scripts/config-firebase.ts @@ -1,4 +1,3 @@ -import fs from 'node:fs' import { program } from 'commander' import { cert, initializeApp } from 'firebase-admin/app' import { FieldValue, getFirestore } from 'firebase-admin/firestore' @@ -7,8 +6,7 @@ import { getAuth } from 'firebase-admin/auth' // import serviceAccountDev from './service-account-dev.json'; // import serviceAccountProd from './service-account-prod.json'; import { serviceAccountDev, serviceAccountProd } from './service-accounts' - -/// LOGGER/// +import './record-logs' program // .version('0.0.1') @@ -34,11 +32,5 @@ export const db = getFirestore() export const timestamp = FieldValue.serverTimestamp() export const storage = getStorage() export const auth = getAuth() -const logFile = fs.createWriteStream(`./logs/${Date.now()}.txt`, { flags: 'w' }) // 'a' to append, 'w' to truncate the file every time the process starts. -console.log = function (data: any) { - logFile.write(`${JSON.stringify(data)}\n`) - process.stdout.write(`${JSON.stringify(data)}\n`) -} -/// END-LOGGER/// -console.log(`Running on ${environment}`) +console.log(`Firebase running on ${environment}`) diff --git a/packages/scripts/config-supabase.ts b/packages/scripts/config-supabase.ts index 3e36fd64d..066714030 100644 --- a/packages/scripts/config-supabase.ts +++ b/packages/scripts/config-supabase.ts @@ -1,31 +1,63 @@ import PG from 'pg' import { createClient } from '@supabase/supabase-js' -import type { Database } from '@living-dictionaries/site/src/lib/supabase/database.types' +import type { Database } from '@living-dictionaries/types' import * as dotenv from 'dotenv' +import './record-logs' -dotenv.config({ path: '.env.supabase' }) +// TODO: change to .env.development and .env.production +dotenv.config({ path: '.env.supabase' }) // local project variables +// dotenv.config({ path: '.env.production.supabase' }) // production project variables -export const supabase = createClient(process.env.PUBLIC_SUPABASE_API_URL, process.env.SUPABASE_SERVICE_ROLE_KEY) +export const admin_supabase = createClient(process.env.PUBLIC_SUPABASE_API_URL, process.env.SUPABASE_SERVICE_ROLE_KEY) +export const anon_supabase = createClient(process.env.PUBLIC_SUPABASE_API_URL, process.env.PUBLIC_SUPABASE_ANON_KEY) +export const jacob_ld_user_id = 'de2d3715-6337-45a3-a81a-d82c3210b2a7' -export async function executeQuery(query: string) { - const client = new PG.Client({ +class DB { + private pool: PG.Pool + + private config: PG.PoolConfig = { user: 'postgres', host: '127.0.0.1', - // host: 'db.actkqboqpzniojhgtqzw.supabase.co', database: 'postgres', password: 'postgres', - // password: '**', port: 54322, - // port: 5432, - }) - try { - await client.connect() - await client.query(query) - } catch (error) { - console.error('Error in connection/executing query:', error) - } finally { - await client.end().catch((error) => { - console.error('Error ending client connection:', error) - }) + + // user: 'postgres.actkqboqpzniojhgtqzw', + // host: 'aws-0-us-west-1.pooler.supabase.com', + // database: 'postgres', + // password: '**', + // port: 6543, + + max: 10, + idleTimeoutMillis: 30000, + connectionTimeoutMillis: 5000, + allowExitOnIdle: false, + } + + async get_db_connection(): Promise { + if (!this.pool) { + this.pool = new PG.Pool(this.config) + const client = await this.pool.connect() + console.info(`----> √ Postgres DB connection established! <----`) + return client + } + return this.pool.connect() + } + + async execute_query(query: string): Promise { + const client = await this.get_db_connection() + try { + await client.query(query) + } catch (error) { + console.error('Error executing query:', error) + throw new Error(error) + } finally { + client.release() + } } } + +export const postgres = new DB() + +const environment = 'dev' +console.log(`Supabase running on ${environment}`) diff --git a/packages/scripts/import/post-request.ts b/packages/scripts/import/post-request.ts index 16ca684d3..35e9ddf1b 100644 --- a/packages/scripts/import/post-request.ts +++ b/packages/scripts/import/post-request.ts @@ -16,6 +16,10 @@ export async function post_request, ExpectedRespon fetch?: typeof fetch headers?: RequestInit['headers'] }): Promise> { + console.info(data) + // for running through data without db involved + // return { data: { speaker_id: data?.speaker_id, dialect_id: data?.dialect_id }, error: null } + const fetch_to_use = options?.fetch || fetch const response = await fetch_to_use(route, { diff --git a/packages/scripts/migrate-to-supabase/.gitignore b/packages/scripts/migrate-to-supabase/.gitignore deleted file mode 100644 index c0bf54b75..000000000 --- a/packages/scripts/migrate-to-supabase/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -entries_full.json -entries.json \ No newline at end of file diff --git a/packages/scripts/migrate-to-supabase/all-current-senses-have-entry-placeholders.ts b/packages/scripts/migrate-to-supabase/all-current-senses-have-entry-placeholders.ts new file mode 100644 index 000000000..9b380c726 --- /dev/null +++ b/packages/scripts/migrate-to-supabase/all-current-senses-have-entry-placeholders.ts @@ -0,0 +1,72 @@ +import fs from 'node:fs' +import Chain from 'stream-chain' +import Parser from 'stream-json' +import StreamArray from 'stream-json/streamers/StreamArray' +import { admin_supabase, jacob_ld_user_id } from '../config-supabase' +import { remove_seconds_underscore } from './utils/remove-seconds-underscore' +import { convert_entry } from './convert-entries' + +export async function ensure_all_current_senses_have_entry_placeholders() { + // load + const { data: senses } = await admin_supabase.from('senses') + .select('id, entry_id') + const { data: entries } = await admin_supabase.from('entries') + .select('id') + + const entries_needing_added = new Set() + for (const sense of senses) { + if (!entries.find(entry => entry.id === sense.entry_id)) { + console.log(`need placeholder entry for sense ${sense.id}, entry ${sense.entry_id}`) + entries_needing_added.add(sense.entry_id) + } + } + console.log({ entries_to_add: entries_needing_added.size }) + + const pipeline = Chain.chain([ + fs.createReadStream('./migrate-to-supabase/firestore-data/firestore-entries.json'), + Parser.parser(), + StreamArray.streamArray(), + ]) + + try { + for await (const { value: fb_entry } of pipeline) { + if (entries_needing_added.has(fb_entry.id)) { + const corrected_fb_entry = remove_seconds_underscore(fb_entry) + const [, supa_data] = convert_entry(JSON.parse(JSON.stringify(corrected_fb_entry))) + const { entry } = supa_data + + const { data: dictionary } = await admin_supabase.from('dictionaries').select().eq('id', entry.dictionary_id).single() + if (!dictionary) { + console.log({ creating_dict: entry.dictionary_id }) + const { error } = await admin_supabase.from('dictionaries').insert({ + created_by: jacob_ld_user_id, + updated_by: jacob_ld_user_id, + id: entry.dictionary_id, + name: 'CHANGE', + }) + if (error) { + console.info({ entry }) + throw new Error(error.message) + } + } + + const insert = { + id: entry.id, + dictionary_id: entry.dictionary_id, + lexeme: {}, + created_by: entry.created_by, + created_at: entry.created_at, + updated_by: entry.updated_by, + updated_at: entry.updated_at, + } + console.log({ insert }) + await admin_supabase.from('entries').insert(insert) + } + } + console.log('finished') + } catch (err) { + console.error(err) + pipeline.destroy() + pipeline.input.destroy() + } +} diff --git a/packages/scripts/migrate-to-supabase/auth.ts b/packages/scripts/migrate-to-supabase/auth.ts index 5ec6b2ef2..25d94519e 100644 --- a/packages/scripts/migrate-to-supabase/auth.ts +++ b/packages/scripts/migrate-to-supabase/auth.ts @@ -1,11 +1,9 @@ import type { UserRecord } from 'firebase-admin/auth' import { auth } from '../config-firebase' -import { executeQuery } from '../config-supabase' +import { postgres } from '../config-supabase' import { write_users_insert } from './write-users-insert' -migrate_users() - -async function migrate_users() { +export async function migrate_users() { const users = await get_users() console.log({ total_users: users.length }) for (const user of users) @@ -14,12 +12,12 @@ async function migrate_users() { console.log(user.toJSON()) const sql = write_users_insert(users) console.log(sql) - await executeQuery(sql) + await postgres.execute_query(sql) } const BATCH_SIZE = 1000 -async function get_users(): Promise { +export async function get_users(): Promise { try { const listUsersResult = await auth.listUsers() const { users, pageToken } = listUsersResult @@ -29,7 +27,7 @@ async function get_users(): Promise { if (pageToken) { const listUsersResult = await auth.listUsers(BATCH_SIZE, pageToken) const { users: nextUsers } = listUsersResult - // const nextUsers = await get_users(pageToken) + // const nextUsers = await get_users(pageToken) // had issue return [...users, ...nextUsers] } diff --git a/packages/scripts/migrate-to-supabase/convert-entries.test.ts b/packages/scripts/migrate-to-supabase/convert-entries.test.ts index cfda26c1a..3a50a90a1 100644 --- a/packages/scripts/migrate-to-supabase/convert-entries.test.ts +++ b/packages/scripts/migrate-to-supabase/convert-entries.test.ts @@ -2,10 +2,28 @@ import fs from 'node:fs' import { chain } from 'stream-chain' import { parser } from 'stream-json' import { streamArray } from 'stream-json/streamers/StreamArray' -import type { ActualDatabaseEntry } from '@living-dictionaries/types' import { convert_entry } from './convert-entries' +import entries_to_test from './entries_to_test.json' +import { remove_seconds_underscore } from './utils/remove-seconds-underscore' -// Snapshotting 1-255 and specific entries moving beyond that point +let id_count = 0 +function randomUUID() { + id_count++ + return `use-crypto-uuid-in-real-thing_${id_count}` +} + +test(convert_entry, () => { + const converted_entries = entries_to_test.map((entry) => { + const [processed_fb_entry_remains, supa_data] = convert_entry(JSON.parse(JSON.stringify(entry)), randomUUID) + if (Object.keys(processed_fb_entry_remains).length !== 0) + throw new Error('Entry not fully converted') + return { entry, supa_data } + }, + ) + expect(converted_entries).toMatchFileSnapshot('convert-entries-to-test.snap.json') +}) + +// Snapshotting 1-228 and specific entries moving beyond that point // 229 - has write-in semantic domains (sd array) // 231 - has no ab for audio and sf.ts is an object with seconds and nanoseconds // 235 - rare xe for vernacular example sentence @@ -47,17 +65,16 @@ import { convert_entry } from './convert-entries' // 266408 - deleted vfs date const to_snapshot = [229, 231, 235, 252, 253, 254, 255, 1228, 1718, 1759, 4577, 4609, 4945, 5377, 5394, 8005, 14072, 15715, 16141, 23958, 29994, 36138, 39845, 39858, 47304, 47829, 85363, 128736, 166042, 167017, 172023, 200582, 248444, 251721, 253088, 266408] -// pnpm t -- --ui convert-entries -// eslint-disable-next-line test/no-disabled-tests -- only run locally and not in CI because data does not exist in repo -test.skip(convert_entry, { timeout: 16000 }, async () => { - // const count = 266408 +// pnpm -F scripts test:migration convert-entries -- --ui +test(convert_entry, { timeout: 26000 }, async () => { + // const count = 300 const count = 278631 // total entries - const success: any[] = [] + const success: { entry: any, supa_data: any }[] = [] const todo: any[] = [] - const result: Promise = new Promise((resolve, reject) => { + const result = new Promise<{ entry: any, supa_data: any }[]>((resolve, reject) => { const pipeline = chain([ - fs.createReadStream('./packages/scripts/migrate-to-supabase/entries_full.json'), + fs.createReadStream('./migrate-to-supabase/entries_full.json'), parser(), streamArray(), ]) @@ -105,40 +122,7 @@ test.skip(convert_entry, { timeout: 16000 }, async () => { const specific_entries = converted_entries.filter((_, index) => to_snapshot.includes(index + 1)) expect(specific_entries).toMatchFileSnapshot('convert-entries.specific.snap.json') -}) -function remove_seconds_underscore(entry: Partial & Record) { - // @ts-expect-error - if (entry.updatedAt?._seconds) { - // @ts-expect-error - entry.updatedAt = { - // @ts-expect-error - seconds: entry.updatedAt._seconds, - } - } - // @ts-expect-error - if (entry.createdAt?._seconds) { - // @ts-expect-error - entry.createdAt = { - // @ts-expect-error - seconds: entry.createdAt._seconds, - } - } - // @ts-expect-error - if (entry.ua?._seconds) { - // @ts-expect-error - entry.ua = { - // @ts-expect-error - seconds: entry.ua._seconds, - } - } - // @ts-expect-error - if (entry.ca?._seconds) { - // @ts-expect-error - entry.ca = { - // @ts-expect-error - seconds: entry.ca._seconds, - } - } - return entry -} + // const entries_to_test = [...first_chunk, ...specific_entries].map(({ entry }) => entry) + // fs.writeFileSync('entries_to_test.json', JSON.stringify(entries_to_test, null, 2)) +}) diff --git a/packages/scripts/migrate-to-supabase/convert-entries.ts b/packages/scripts/migrate-to-supabase/convert-entries.ts index 20d3eafa5..296ed6e64 100644 --- a/packages/scripts/migrate-to-supabase/convert-entries.ts +++ b/packages/scripts/migrate-to-supabase/convert-entries.ts @@ -1,39 +1,47 @@ -// import { randomUUID } from 'node:crypto' -import type { ActualDatabaseEntry } from '@living-dictionaries/types' -import type { TablesInsert } from '../../site/src/lib/supabase/generated.types' -// import { log_once } from './log-once' - -let id_count = 0 -function randomUUID() { - id_count++ - return `use-crypto-uuid-in-real-thing_${id_count}` +import { randomUUID } from 'node:crypto' +import type { TablesInsert } from '@living-dictionaries/types' +import type { ActualDatabaseEntry } from '@living-dictionaries/types/entry.interface' +import type { ActualDatabaseVideo } from '@living-dictionaries/types/video.interface' +import { jacob_ld_user_id } from '../config-supabase' +import { get_supabase_user_id_from_firebase_uid } from './get-user-id' + +interface DataForSupabase { + entry: TablesInsert<'entries'> + senses: TablesInsert<'senses'>[] + sentences: TablesInsert<'sentences'>[] + senses_in_sentences: TablesInsert<'senses_in_sentences'>[] + audios: TablesInsert<'audio'>[] + audio_speakers: TablesInsert<'audio_speakers'>[] + photos: TablesInsert<'photos'>[] + sense_photos: TablesInsert<'sense_photos'>[] + videos: TablesInsert<'videos'>[] + video_speakers: TablesInsert<'video_speakers'>[] + sense_videos: TablesInsert<'sense_videos'>[] + dialects: string[] + new_speaker_name?: string + prior_import_id: string | null } -const admin_uid_if_no_owner = 'de2d3715-6337-45a3-a81a-d82c3210b2a7' // jacob@livingtongues.org -const old_talking_dictionaries = '00000000-0000-0000-0000-000000000000' // TODO - decide what user to use or create one for attribution - -export function convert_entry(_entry: Partial & Record) { - // if (_entry.deletedVfs) { - // console.log(`deletedVfs in ${_entry.id} in ${_entry.dictionary_id}, ${_entry.deletedVfs[0].youtubeId}`) - // } - // if (_entry.xv && _entry.xs?.vn) - // console.log(`both xv ${_entry.xv} and xs.vn ${_entry.xs.vn} for ${_entry.id} in ${_entry.dictionary_id}`) - // if (_entry.lo && _entry.lo1 && _entry.lo !== _entry.lo1) - // console.log(`lost lo: ${_entry.lo} in favor of lo1: ${_entry.lo1} for ${_entry.id} in ${_entry.dictionary_id}`) - - // if (_entry.sf && _entry.sfs?.length) { - // if (!_entry.sfs[0].sp.includes(_entry.sf.sp)) - // console.log(`${_entry.id} in ${_entry.dictionary_id} has speaker ${_entry.sf.sp} in sf and ${_entry.sfs[0].sp.join(', ')} sfs`) - // } - // if (_entry.source) - // console.log(`source ${_entry.source} in ${_entry.id} in ${_entry.dictionary_id}`) - // return [{}, {}] +export function convert_entry(_entry: ActualDatabaseEntry & Record, uuid: () => string = randomUUID): [any, DataForSupabase] { + const dict_entry_id = `${_entry.dictionary_id}:${_entry.id}` + if (_entry.deletedVfs) + console.log(`deletedVfs in ${dict_entry_id} - youtubeId: ${_entry.deletedVfs[0].youtubeId}`) + + if (_entry.xv && _entry.xs?.vn) + console.log(`both xv ${_entry.xv} and xs.vn ${_entry.xs.vn} for ${dict_entry_id}`) + + if (_entry.lo && _entry.lo1 && _entry.lo !== _entry.lo1) + console.log(`lost lo: ${_entry.lo} in favor of lo1: ${_entry.lo1} for ${dict_entry_id}`) + + if (_entry.sf && _entry.sfs?.length && !_entry.sfs[0].sp.includes(_entry.sf.sp)) + console.log(`different speakers in ${dict_entry_id} - ${_entry.sf.sp} in sf and ${_entry.sfs[0].sp.join(', ')} sfs`) try { - const entry: Partial> = { - id: _entry.id, + const entry: TablesInsert<'entries'> = { + id: _entry.id!, dictionary_id: _entry.dictionary_id, - } + } as TablesInsert<'entries'> + let prior_import_id = null if (typeof _entry.updatedAt?.seconds === 'number') { entry.updated_at = seconds_to_timestamp_string(_entry.updatedAt.seconds) @@ -44,11 +52,11 @@ export function convert_entry(_entry: Partial & Record & Record[] = [] + const dialects = new Set() const first_sense_from_base: TablesInsert<'senses'> = { entry_id: _entry.id, - created_by: entry.created_by, + created_by: entry.created_by || entry.updated_by, + created_at: entry.created_at || entry.updated_at, updated_by: entry.updated_by || entry.created_by, - id: randomUUID(), + updated_at: entry.updated_at || entry.created_at, + id: uuid(), } + if (!_entry.lx) + console.log(`no lexeme for ${dict_entry_id}`) entry.lexeme = { default: _entry.lx || '' } delete _entry.lx @@ -143,16 +154,7 @@ export function convert_entry(_entry: Partial & Record & Record & Record dialects.add(d)) delete _entry[key] continue } @@ -346,7 +348,7 @@ export function convert_entry(_entry: Partial & Record = { id: sentence_id, @@ -371,10 +373,12 @@ export function convert_entry(_entry: Partial & Record[] = [] let new_speaker_name: string = null + if (_entry.sf && !_entry.sf.path) + delete _entry.sf if (_entry.sf?.path || _entry.sfs?.[0].path) { - const audio_id = randomUUID() - const sf = _entry.sf?.path ? _entry.sf : _entry.sfs[0] as unknown as ActualDatabaseEntry['sf'] - const { ab, path, ts, cr, sp, speakerName, source } = sf + const audio_id = uuid() + const sf = _entry.sf?.path ? _entry.sf : _entry.sfs![0] as unknown as ActualDatabaseEntry['sf'] + const { ab, path, ts, cr, sp, speakerName, source } = sf! if (typeof speakerName === 'string') { if (speakerName.trim()) new_speaker_name = speakerName.trim() @@ -383,11 +387,13 @@ export function convert_entry(_entry: Partial & Record = { id: audio_id, + dictionary_id: entry.dictionary_id, entry_id: _entry.id, - created_by: ab || entry.created_by, - updated_by: ab || entry.created_by, + created_by, + updated_by: created_by, storage_path: path, } delete sf.ab @@ -395,12 +401,13 @@ export function convert_entry(_entry: Partial & Record & Record & Record & Record[] = [] if (_entry.pf) { - const photo_id = randomUUID() + const photo_id = uuid() const { ab, path, ts, sc, cr, gcs, source, uploadedAt, uploadedBy } = _entry.pf if (uploadedAt) console.info({ uploadedAt }) if (!ab && !entry.created_by) console.info(`No ab for ${_entry.id} pf`) + + const created_by = get_supabase_user_id_from_firebase_uid(ab) || get_supabase_user_id_from_firebase_uid(uploadedBy) || entry.created_by const photo: TablesInsert<'photos'> = { id: photo_id, - created_by: ab || uploadedBy || entry.created_by, - updated_by: ab || uploadedBy || entry.created_by, + dictionary_id: entry.dictionary_id, + created_by, + updated_by: created_by, storage_path: path, serving_url: remove_newline_from_end(gcs), } @@ -470,6 +484,7 @@ export function convert_entry(_entry: Partial & Record & Record & Record & Record = { id: video_id, - created_by: ab || entry.created_by, - updated_by: ab || entry.created_by, + dictionary_id: entry.dictionary_id, + created_by, + updated_by: created_by, } delete vf.ab if (path) { @@ -536,27 +557,32 @@ export function convert_entry(_entry: Partial & Record & Record & Record { @@ -599,7 +627,7 @@ export function convert_entry(_entry: Partial & Record { + const converted_speakers: Record> = {} + for (const fb_speaker of firebase_speakers) { + const { id, speaker } = convert_speaker(JSON.parse(JSON.stringify(fb_speaker))) + const jacob_test_speaker_id = '2PELJgjxMHXEOcuZfv9MtGyiXdE3' + if (id === jacob_test_speaker_id) + continue + converted_speakers[id] = speaker + } + expect(converted_speakers).toMatchFileSnapshot('converted-speakers.snap.json') +}) diff --git a/packages/scripts/migrate-to-supabase/convert-speakers.ts b/packages/scripts/migrate-to-supabase/convert-speakers.ts new file mode 100644 index 000000000..d588e5d24 --- /dev/null +++ b/packages/scripts/migrate-to-supabase/convert-speakers.ts @@ -0,0 +1,58 @@ +import type { TablesUpdate } from '@living-dictionaries/types' +import type { ISpeaker } from '@living-dictionaries/types/speaker.interface' +import { jacob_ld_user_id } from '../config-supabase' +import { seconds_to_timestamp_string } from './convert-entries' +import { get_supabase_user_id_from_firebase_uid } from './get-user-id' + +export function convert_speaker(fb_speaker: ISpeaker): { id: string, speaker: TablesUpdate<'speakers'> } { + const { + displayName, + uid, + decade, + birthplace, + contributingTo, + gender, + id, + createdAt, + createdBy, + updatedAt, + updatedBy, + } = fb_speaker + + if (!contributingTo || contributingTo.length === 0) { + throw new Error(`Speaker ${displayName} has no contributingTo`) + } + + if (contributingTo.length > 1) { + console.log(`Speaker ${displayName} has multiple contributingTo`) + } + + if (!createdBy) + console.log(`Speaker ${displayName} has no createdBy`) + + const created_by = get_supabase_user_id_from_firebase_uid(createdBy) || jacob_ld_user_id + + const decade_number = (typeof decade === 'string' && decade.trim() !== '') + ? Number.parseInt(decade) + : (typeof decade === 'number') + ? decade + : null + + const speaker: TablesUpdate<'speakers'> = { + dictionary_id: contributingTo[0], + birthplace, + name: displayName, + decade: decade_number, + gender: gender ? gender.trim() as 'm' | 'f' | 'o' : null, + created_by, + updated_by: get_supabase_user_id_from_firebase_uid(updatedBy) || created_by, + user_id: get_supabase_user_id_from_firebase_uid(uid) || null, + } + + if (createdAt?.seconds) { + speaker.created_at = seconds_to_timestamp_string(createdAt.seconds) + speaker.updated_at = updatedAt ? seconds_to_timestamp_string(updatedAt.seconds) : speaker.created_at + } + + return { id, speaker } +} diff --git a/packages/scripts/migrate-to-supabase/create-rest-of-dictionary-placeholders.ts b/packages/scripts/migrate-to-supabase/create-rest-of-dictionary-placeholders.ts new file mode 100644 index 000000000..79ad35e5c --- /dev/null +++ b/packages/scripts/migrate-to-supabase/create-rest-of-dictionary-placeholders.ts @@ -0,0 +1,47 @@ +import fs from 'node:fs' +import Chain from 'stream-chain' +import Parser from 'stream-json' +import StreamArray from 'stream-json/streamers/StreamArray' +import { admin_supabase, jacob_ld_user_id } from '../config-supabase' + +export async function create_rest_of_dictionary_placeholders() { + const { data: dictionaries } = await admin_supabase.from('dictionaries') + .select('id') + + const dictionary_ids = new Set(dictionaries.map(dict => dict.id)) + const dictionary_ids_needing_added = new Set() + + const pipeline = Chain.chain([ + fs.createReadStream('./migrate-to-supabase/firestore-data/firestore-entries.json'), + Parser.parser(), + StreamArray.streamArray(), + ]) + + try { + for await (const { value: fb_entry } of pipeline) { + if (!dictionary_ids.has(fb_entry.dictionary_id)) { + dictionary_ids_needing_added.add(fb_entry.dictionary_id) + } + } + console.log('finished') + } catch (err) { + console.error(err) + pipeline.destroy() + pipeline.input.destroy() + } + + const inserts = Array.from(dictionary_ids_needing_added).map(dictionary_id => ({ + created_by: jacob_ld_user_id, + updated_by: jacob_ld_user_id, + id: dictionary_id, + name: 'CHANGE', + })) + + console.log({ inserts }) + + const { error } = await admin_supabase.from('dictionaries').insert(inserts) + if (error) { + console.info({ error }) + throw new Error(error.message) + } +} diff --git a/packages/scripts/migrate-to-supabase/fetch-entries.ts b/packages/scripts/migrate-to-supabase/fetch-entries.ts deleted file mode 100644 index 78258c2de..000000000 --- a/packages/scripts/migrate-to-supabase/fetch-entries.ts +++ /dev/null @@ -1,31 +0,0 @@ -import fs from 'node:fs' -import path, { dirname } from 'node:path' -import { fileURLToPath } from 'node:url' -import type { ActualDatabaseEntry } from '@living-dictionaries/types' -import { db } from '../config-firebase' - -write_entries() - -async function write_entries() { - const entries: ActualDatabaseEntry[] = [] - - const dict_snapshot = await db.collection('dictionaries').get() - - for (const { id: dictionary_id } of dict_snapshot.docs) { - console.log(dictionary_id) - // const allow = /^[a].*/ - // if (!allow.test(dictionary_id.toLowerCase())) continue - - const snapshot = await db.collection(`dictionaries/${dictionary_id}/words`).get() - - for (const snap of snapshot.docs) { - const entry = { id: snap.id, dictionary_id, ...(snap.data() as ActualDatabaseEntry) } - entries.push(entry) - } - } - - console.log(`Done fetching ${entries.length} entries from ${dict_snapshot.docs.length} dictionaries.`) - - const __dirname = dirname(fileURLToPath(import.meta.url)) - fs.writeFileSync(path.resolve(__dirname, 'entries.json'), JSON.stringify(entries, null, 2)) -} diff --git a/packages/scripts/migrate-to-supabase/get-user-id.ts b/packages/scripts/migrate-to-supabase/get-user-id.ts new file mode 100644 index 000000000..7e68e04ca --- /dev/null +++ b/packages/scripts/migrate-to-supabase/get-user-id.ts @@ -0,0 +1,20 @@ +import { log_once } from './log-once' + +let firebase_uid_to_supabase_user_ids: Record | null = null + +export async function load_fb_to_sb_user_ids() { + if (!firebase_uid_to_supabase_user_ids) { + // eslint-disable-next-line require-atomic-updates + firebase_uid_to_supabase_user_ids = (await import('./firestore-data/fb-sb-user-ids.json')).default + } +} + +export function get_supabase_user_id_from_firebase_uid(firebase_uid: string): string | null { + if (!firebase_uid) return null + + const supabase_user_id = firebase_uid_to_supabase_user_ids![firebase_uid] + if (!supabase_user_id) { + log_once(`No Supabase user found for Firebase UID: ${firebase_uid}`) + } + return supabase_user_id +} diff --git a/packages/scripts/migrate-to-supabase/log-once.ts b/packages/scripts/migrate-to-supabase/log-once.ts index c8bc91410..7ebcfcf91 100644 --- a/packages/scripts/migrate-to-supabase/log-once.ts +++ b/packages/scripts/migrate-to-supabase/log-once.ts @@ -3,6 +3,6 @@ const logged = new Set() export function log_once(msg: string) { if (logged.has(msg)) return - console.info(msg) + console.log(msg) logged.add(msg) } diff --git a/packages/scripts/migrate-to-supabase/migrate-entries.test.ts b/packages/scripts/migrate-to-supabase/migrate-entries.test.ts new file mode 100644 index 000000000..a25e374af --- /dev/null +++ b/packages/scripts/migrate-to-supabase/migrate-entries.test.ts @@ -0,0 +1,65 @@ +import type { ISpeaker } from '@living-dictionaries/types/speaker.interface' +import { anon_supabase } from '../config-supabase' +import { migrate_entries, migrate_speakers } from './migrate-entries' +import entries_to_test_264 from './entries_to_test.json' +import firebase_speakers from './speakers.json' +import { reset_db } from './reset-db' + +vi.mock('node:crypto', () => { + const uuid_template = '11111111-1111-1111-1111-111111111111' + let current_uuid_index = 0 + + function incremental_consistent_uuid() { + return uuid_template.slice(0, -5) + (current_uuid_index++).toString().padStart(5, '0') + } + + return { + randomUUID: incremental_consistent_uuid, + } +}) + +vi.mock('./test-timestamp', () => { + return { + test_timestamp: new Date('2024-03-08T00:44:04.600392+00:00').toISOString(), + } +}) + +describe(migrate_entries, () => { + beforeEach(reset_db) + + test.todo('works on all use cases', { timeout: 60000 }, async () => { + const speakers = await migrate_speakers(firebase_speakers as ISpeaker[]) + await migrate_entries(entries_to_test_264, speakers) + const { data: entry_view } = await anon_supabase.from('entries_view').select() + expect(entry_view).toMatchFileSnapshot('view-after-migrating-entries.json') + }) + + test('write in speaker names are added but not duplicated and assigned', async () => { + const speakerName = 'Write-in Speaker Name not in db' + await migrate_entries([{ + id: 'custom-id-1', + dictionary_id: 'create-me', + lx: 'hi', + sf: { + path: 'foo.mp3', + speakerName, + }, + }, { + id: 'custom-id-2', + dictionary_id: 'create-me', + lx: 'hello', + sf: { + path: 'food.mp3', + speakerName, + }, + }], {}) + + const { data: entry_view } = await anon_supabase.from('entries_view').select() + expect(entry_view[0].audios[0].speaker_ids[0]).toMatchInlineSnapshot(`"11111111-1111-1111-1111-111111100005"`) + expect(entry_view[0].audios[0].speaker_ids[0]).toEqual(entry_view[1].audios[0].speaker_ids[0]) + + const { data: speakers } = await anon_supabase.from('speakers_view').select() + expect(speakers).toHaveLength(1) + expect(speakers[0].name).toEqual(speakerName) + }) +}) diff --git a/packages/scripts/migrate-to-supabase/migrate-entries.ts b/packages/scripts/migrate-to-supabase/migrate-entries.ts new file mode 100644 index 000000000..ef0822b31 --- /dev/null +++ b/packages/scripts/migrate-to-supabase/migrate-entries.ts @@ -0,0 +1,184 @@ +import { randomUUID } from 'node:crypto' +import fs from 'node:fs' +import path, { dirname } from 'node:path' +import { fileURLToPath } from 'node:url' +import type { TablesUpdate } from '@living-dictionaries/types' +import type { ISpeaker } from '@living-dictionaries/types/speaker.interface' +import { convert_entry } from './convert-entries' +import { assign_dialect, assign_speaker, insert_dialect, insert_entry, insert_photo, insert_sense, insert_sentence, insert_video, upsert_audio, upsert_speaker } from './operations/operations' +import { convert_speaker } from './convert-speakers' +import { log_once } from './log-once' + +const import_id = 'fb_sb_migration' +const FOLDER = 'firestore-data' +const __dirname = dirname(fileURLToPath(import.meta.url)) + +export type AllSpeakerData = Record }> + +export async function migrate_speakers() { + const converted_speakers: AllSpeakerData = {} + + const firebase_speakers = (await import('./firestore-data/firestore-speakers.json')).default as ISpeaker[] + + for (const fb_speaker of firebase_speakers) { + const { id: firebase_id, speaker } = convert_speaker(JSON.parse(JSON.stringify(fb_speaker))) + const jacob_test_speaker_id = '2PELJgjxMHXEOcuZfv9MtGyiXdE3' + if (firebase_id === jacob_test_speaker_id) + continue + + const supabase_speaker_id = randomUUID() + const { error } = await upsert_speaker({ dictionary_id: speaker.dictionary_id, speaker, speaker_id: supabase_speaker_id, import_id }) + if (error) + throw new Error(error.message) + + converted_speakers[firebase_id] = { supabase_id: supabase_speaker_id, speaker } + } + + fs.writeFileSync(path.resolve(__dirname, FOLDER, 'fb-to-sb-speakers-mapping.json'), JSON.stringify(converted_speakers, null, 2)) +} + +export async function load_speakers() { + const speakers = (await import('./firestore-data/fb-to-sb-speakers-mapping.json')).default as AllSpeakerData + return speakers +} + +export function migrate_entries(entries_to_test: any[], speakers: AllSpeakerData) { + const dictionary_dialects: Record> = {} + const dictionary_new_speakers: Record> = {} + for (const fb_entry of entries_to_test) { + migrate_entry(fb_entry, speakers, dictionary_dialects, dictionary_new_speakers) + } +} + +export function migrate_entry(fb_entry: any, speakers: AllSpeakerData, dictionary_dialects: Record>, dictionary_new_speakers: Record>) { + const [processed_fb_entry_remains, supa_data] = convert_entry(JSON.parse(JSON.stringify(fb_entry))) + if (Object.keys(processed_fb_entry_remains).length > 0) { + console.log({ fb_entry, processed_fb_entry_remains, supa_data }) + throw new Error('processed_fb_entry_remains not empty') + } + + const { entry, audio_speakers, audios, dialects, photos, sense_photos, sense_videos, senses, senses_in_sentences, sentences, videos, video_speakers, new_speaker_name, prior_import_id } = supa_data + const { id: entry_id, dictionary_id } = entry + + let sql_statements = '' + + const sql = insert_entry({ + dictionary_id, + entry, + entry_id, + import_id: prior_import_id || import_id, + }) + sql_statements += `\n${sql}` + + for (const audio of audios) { + const sql = upsert_audio({ dictionary_id, entry_id, audio, audio_id: audio.id, import_id }) + sql_statements += `\n${sql}` + + if (new_speaker_name) { + let new_speaker_id = dictionary_new_speakers[dictionary_id]?.[new_speaker_name] + + if (!new_speaker_id) { + new_speaker_id = randomUUID() + + const sql = upsert_speaker({ dictionary_id, speaker_id: new_speaker_id, speaker: { name: new_speaker_name, created_at: entry.created_at, created_by: entry.created_by }, import_id }) + sql_statements += `\n${sql}` + + if (!dictionary_new_speakers[dictionary_id]) + dictionary_new_speakers[dictionary_id] = { [new_speaker_name]: new_speaker_id } + else + dictionary_new_speakers[dictionary_id] = { ...dictionary_new_speakers[dictionary_id], [new_speaker_name]: new_speaker_id } + } + + const sql = assign_speaker({ + dictionary_id, + speaker_id: new_speaker_id, + media_id: audio.id, + media: 'audio', + import_id, + user_id: entry.created_by, + timestamp: entry.created_at, + }) + sql_statements += `\n${sql}` + } + } + + for (const audio_speaker of audio_speakers) { + if (speakers[audio_speaker.speaker_id]) { + const sql = assign_speaker({ + dictionary_id, + speaker_id: speakers[audio_speaker.speaker_id].supabase_id, + media_id: audio_speaker.audio_id, + media: 'audio', + import_id, + user_id: audio_speaker.created_by, + timestamp: audio_speaker.created_at, + }) + sql_statements += `\n${sql}` + } else { + log_once(`speaker ${audio_speaker.speaker_id} in ${dictionary_id}:${entry_id} not found`) + } + } + + for (const dialect of dialects) { + let dialect_id = dictionary_dialects[dictionary_id]?.[dialect] + + if (!dialect_id) { + dialect_id = randomUUID() + + const sql = insert_dialect({ dictionary_id, name: dialect, import_id, user_id: entry.created_by, timestamp: entry.created_at, dialect_id }) + sql_statements += `\n${sql}` + + if (!dictionary_dialects[dictionary_id]) + dictionary_dialects[dictionary_id] = { [dialect]: dialect_id } + else + dictionary_dialects[dictionary_id] = { ...dictionary_dialects[dictionary_id], [dialect]: dialect_id } + } + + const sql = assign_dialect({ dictionary_id, dialect_id, entry_id, import_id, user_id: entry.created_by, timestamp: entry.created_at }) + sql_statements += `\n${sql}` + } + + for (const sense of senses) { + const sql = insert_sense({ dictionary_id, entry_id, sense, sense_id: sense.id, import_id }) + sql_statements += `\n${sql}` + } + + for (const sentence of sentences) { + const { sense_id } = senses_in_sentences.find(s => s.sentence_id === sentence.id) + if (!sense_id) + throw new Error('sense_id not found') + const sql = insert_sentence({ dictionary_id, sense_id, sentence, sentence_id: sentence.id, import_id }) + sql_statements += `\n${sql}` + } + + for (const photo of photos) { + const { sense_id } = sense_photos.find(s => s.photo_id === photo.id) + if (!sense_id) + throw new Error('sense_id not found') + const sql = insert_photo({ dictionary_id, sense_id, photo, photo_id: photo.id, import_id }) + sql_statements += `\n${sql}` + } + + for (const video of videos) { + const { sense_id } = sense_videos.find(s => s.video_id === video.id) + if (!sense_id) + throw new Error('sense_id not found') + const sql = insert_video({ dictionary_id, sense_id, video, video_id: video.id, import_id }) + sql_statements += `\n${sql}` + } + + for (const video_speaker of video_speakers) { + const sql = assign_speaker({ + dictionary_id, + speaker_id: speakers[video_speaker.speaker_id].supabase_id, + media_id: video_speaker.video_id, + media: 'video', + import_id, + user_id: video_speaker.created_by, + timestamp: video_speaker.created_at, + }) + sql_statements += `\n${sql}` + } + + return sql_statements +} diff --git a/packages/scripts/migrate-to-supabase/notes.md b/packages/scripts/migrate-to-supabase/notes.md index 0547b9839..fdc62a7a2 100644 --- a/packages/scripts/migrate-to-supabase/notes.md +++ b/packages/scripts/migrate-to-supabase/notes.md @@ -1,32 +1,47 @@ # Migrate Entries and Speakers from Firestore to Supabase - -## TODO -- Move off Algolia fully onto local Orama + using Orama Cloud for (100K max*3 indexes) for large, public dictionaries + Onondaga - - solve bugs - - flip usage to using the new search but make Algolia still available as a second option -- On a new migration branch - - Make sure all items from "clean-up" below are being actively logged again as they are run into - - update migration script to migrate speakers across as they are needed, when one is found, save into speakers table, then create a map of firestore speaker id to supabase speaker id, in future times when this Firestore speaker id comes up, check the map first to see if speaker already exists - - figure out how different user ids (creater of speaker) between Firestore and Firebase will be handled and document - - run tests on migration - - visual inspection of the results locally - should work similar to current prod - - update saving functions - - make all types of edits -- Run migration process below - -## Migration Process -- post notice on logged-in users a week ahead of time -- send email notice a week ahead of time -- Lock down Firestore dictionary words and speakers using security rules (tell admins not to edit anything) -- Make Supabase backup -- Migrate data -- Test viewing -- Merge new saving methods code (this will be a natural unblock) -- Test editing entries -- Remove notice -- Email letting everyone know editing is available again +- Push migration +- Merge PR to unblock +- Test editing entries on live site +- Remove banner +- build Orama indexes in Cloudflare R2 for dictionaries over 1000 entries (try dictionaries from phone and test load speeds) + - run size check query on prod to get a few dictionaries +- Use cached Orama indexes from Cloudflare R2 if it exists ## Clean-up +- optimize searching for entries in the dictionary layout page (and error handling) +- check how the view is if an audio file does not have a speaker +- Delayed email letting everyone know editing is available again +- migrate dictionaries and setup materialized view with entry counts +- cached_data_store should set store value from cached items instead of waiting until all load is done. +- get semantic domains working in filters ( currently just filters out entries without a semantic domain) +- get exports working again +- create indexes using help from index_advisor https://supabase.com/docs/guides/database/extensions/index_advisor +- Orama: replaceState in createQueryParamStore? look into improving the history to change for view and page changes but not for the others +- look at print, table, gallery, and list page files history to make sure there are no missed improvements - check github history too +- drop content_updates' table column +- drop entry_updates +- ensure all auth users are brought over +- clean up old history data in content_updates +- make alternate writing systems of the sentence translations as different bcp keys (same as for glosses) +- get failed tests working again +- bring back in variants and tests that relied on old forms of test data +- look at deletedEntries to see if they should be saved somewhere +- rework about content loading in dictionary layout page +- test new db triggers, especially when deleting relationships +- change cb/ub ids on old senses from firebase ids to user_ids and then connect relationships +- look at how the quotes came through in arrays with strings (like sources in https://livingdictionaries.app/canichana/entry/ehRKCec3J4flzJhscXiQ) https://www.postgresql.org/message-id/CAEs%3D6DnhWbh5QfbZAZy6BdoGdLJ28MCoU1T6pAiZhw6Ze4gMUw%40mail.gmail.com +- search enzi in http://localhost:3041/sora/entries?q=%7B%22query%22%3A%22enz%22%2C%22view%22%3A%22gallery%22%7D will show no results and then pagination goes full bore +- remove `pnpm mixed` +- remove `pnpm -F scripts test:migration` +- Remove algolia keys from vercel +- cleaner format for content-updates and refactor current ones + +## Notes +- 1st manual backup was before any action +- 11:37 at 50, 13:54 at 100 = 2 min 17 seconds for 50 entries, 387000/50 = 7740 chunks, 7740 * 2:17 = 17492 minutes = 12 days (1440 minutes in a day), 18:30 at 200 +- didn't add 331 megabytes of content_updates to db just yet, rather save those sql queries aside for now to avoid upgrading to the $25/month +- `pnpm -F scripts run-migration` +- look through local orthographies - especially in the table view, `/birhor/entries?q=%7B"query"%3A"horomsi"%2C"view"%3A"table"%7D` ### No lexeme no lx for 0svukh699MsB4svuCDdO in ho diff --git a/packages/scripts/migrate-to-supabase/operations/constants.ts b/packages/scripts/migrate-to-supabase/operations/constants.ts new file mode 100644 index 000000000..ead0acf2e --- /dev/null +++ b/packages/scripts/migrate-to-supabase/operations/constants.ts @@ -0,0 +1,2 @@ +export const dictionary_id = 'import_dictionary' +export const content_update_endpoint = 'http://localhost:3041/api/db/content-update' diff --git a/packages/scripts/migrate-to-supabase/operations/operations.test.ts b/packages/scripts/migrate-to-supabase/operations/operations.test.ts new file mode 100644 index 000000000..603ae50de --- /dev/null +++ b/packages/scripts/migrate-to-supabase/operations/operations.test.ts @@ -0,0 +1,213 @@ +import type { TablesUpdate } from '@living-dictionaries/types' +import { anon_supabase, jacob_ld_user_id } from '../../config-supabase' +import { reset_db } from '../reset-db' +import { assign_dialect, assign_speaker, insert_dialect, insert_entry, insert_photo, insert_sense, insert_sentence, insert_video, upsert_audio, upsert_speaker } from './operations' +import { dictionary_id } from './constants' +import { test_timestamp } from './test-timestamp' + +vi.mock('node:crypto', () => { + const uuid_template = '11111111-1111-1111-1111-111111111111' + let current_uuid_index = 0 + + function incremental_consistent_uuid() { + return uuid_template.slice(0, -2) + (current_uuid_index++).toString().padStart(2, '0') + } + + return { + randomUUID: incremental_consistent_uuid, + } +}) + +const import_id = '1' + +async function seed_entry_and_sense() { + const { data } = await insert_entry({ dictionary_id, entry: { lexeme: { default: 'hi' } }, entry_id: '1', import_id }) + const { data: sense_data } = await insert_sense({ dictionary_id, entry_id: data.entry_id, sense: { glosses: { en: 'hello' } }, import_id }) + return { entry_id: data.entry_id, sense_id: sense_data.sense_id } +} + +describe('entries and senses', () => { + beforeAll(reset_db) + + describe(insert_entry, () => { + test('adds entry, adds sense, and deletes sense', async () => { + const lexeme = { default: 'hi' } + const { data } = await insert_entry({ dictionary_id, entry: { lexeme }, import_id, entry_id: '1' }) + expect(data?.import_id).toEqual('1') + const { data: entry_view } = await anon_supabase.from('entries_view').select().eq('id', data.entry_id).single() + expect(entry_view.dictionary_id).toEqual(dictionary_id) + expect(entry_view.dictionary_id).toEqual(dictionary_id) + expect(entry_view.main.lexeme).toEqual(lexeme) + expect(entry_view.senses).toBeNull() + const glosses = { en: 'hello' } + const { data: sense_save } = await insert_sense({ dictionary_id, entry_id: data.entry_id, sense: { + glosses, + }, import_id: '1' }) + const { data: entry_view2 } = await anon_supabase.from('entries_view').select().eq('id', data.entry_id).single() + expect(entry_view2.senses[0].glosses).toEqual(glosses) + + await insert_sense({ dictionary_id, entry_id: data.entry_id, sense: { deleted: 'true' }, sense_id: sense_save.sense_id, import_id }) + const { data: entry_view3 } = await anon_supabase.from('entries_view').select().eq('id', data.entry_id).single() + expect(entry_view3.senses).toBeNull() + }) + }) +}) + +describe(insert_dialect, () => { + beforeAll(reset_db) + + test('adds to dialects table, edits dialect, and connects to entry', async () => { + const name = 'Eastern' + const { data } = await insert_dialect({ dictionary_id, name, import_id, user_id: jacob_ld_user_id, timestamp: test_timestamp }) + expect(data?.import_id).toEqual('1') + const { data: dialect } = await anon_supabase.from('dialects').select('*').eq('id', data.dialect_id).single() + expect(dialect.name.default).toEqual(name) + expect(dialect.dictionary_id).toEqual(dictionary_id) + + const edited_name = 'Western' + await insert_dialect({ dictionary_id, name: edited_name, dialect_id: data.dialect_id, import_id, user_id: jacob_ld_user_id, timestamp: test_timestamp }) + const { data: edited_dialect } = await anon_supabase.from('dialects').select('name').eq('id', data.dialect_id).single() + expect(edited_dialect.name.default).toEqual(edited_name) + + const { entry_id } = await seed_entry_and_sense() + await assign_dialect({ dictionary_id, dialect_id: data.dialect_id, entry_id, import_id, user_id: jacob_ld_user_id, timestamp: test_timestamp }) + const { data: entry_view } = await anon_supabase.from('entries_view').select().eq('id', entry_id).single() + expect(entry_view.dialect_ids).toEqual([data.dialect_id]) + }) +}) + +describe(upsert_audio, () => { + beforeAll(reset_db) + + test('adds audio and displays properly in view', async () => { + const { entry_id } = await seed_entry_and_sense() + + const audio: TablesUpdate<'audio'> = { + created_at: '2019-08-27T05:06:40.796Z', + // created_by: 'Wr77x8C4e0PI3TMqOnJnJ7VmlLF3', + entry_id, + id: 'use-crypto-uuid-in-real-thing_2', + source: 'javier domingo', + storage_path: 'audio/dict_80CcDQ4DRyiYSPIWZ9Hy/0DyO0JQrRUVXPvVNLEyN_1566882399481.mpeg', + updated_at: '2019-08-27T05:06:40.796Z', + // updated_by: 'Wr77x8C4e0PI3TMqOnJnJ7VmlLF3', + } + await upsert_audio({ dictionary_id, entry_id, audio, import_id: '1' }) + + const { data: entry_view } = await anon_supabase.from('entries_view').select().eq('id', entry_id).single() + expect(entry_view.audios[0].source).toEqual(audio.source) + expect(entry_view.audios[0].storage_path).toEqual(audio.storage_path) + }) +}) + +describe(insert_sentence, () => { + beforeAll(reset_db) + + test('adds sentence and links to sense', async () => { + const { entry_id, sense_id } = await seed_entry_and_sense() + const { data } = await insert_sentence({ dictionary_id, sense_id, sentence: { text: { default: 'hello, this is my sentence' } }, import_id: '1' }) + + const { data: entry_view } = await anon_supabase.from('entries_view').select().eq('id', entry_id).single() + expect(entry_view.senses[0].sentence_ids).toEqual([data.sentence_id]) + }) +}) + +describe(insert_photo, () => { + beforeAll(reset_db) + + test('adds photo and links to sense', async () => { + const { entry_id, sense_id } = await seed_entry_and_sense() + const storage_path = 'bee/images/baz.jpeg' + const { data } = await insert_photo({ dictionary_id, photo: { serving_url: 'foo', source: 'Bob', storage_path }, sense_id }) + + const { data: entry_view } = await anon_supabase.from('entries_view').select().eq('id', entry_id).single() + expect(entry_view.senses[0].photo_ids).toEqual([data.photo_id]) + const { data: photo } = await anon_supabase.from('photos').select().eq('id', data.photo_id).single() + expect(photo.storage_path).toEqual(storage_path) + }) +}) + +describe(insert_video, () => { + beforeAll(reset_db) + + test('adds video and links to sense', async () => { + const { entry_id, sense_id } = await seed_entry_and_sense() + const { data } = await insert_video({ dictionary_id, video: { source: 'Bob', storage_path: 'baz.wbm' }, sense_id }) + + const { data: entry_view } = await anon_supabase.from('entries_view').select().eq('id', entry_id).single() + expect(entry_view.senses[0].video_ids).toEqual([data.video_id]) + }) +}) + +describe(upsert_speaker, () => { + beforeAll(reset_db) + + test('adds speaker to audio and to video', async () => { + const { entry_id, sense_id } = await seed_entry_and_sense() + const { data: speaker_change } = await upsert_speaker({ dictionary_id, speaker: { name: 'Bob', created_at: test_timestamp, created_by: jacob_ld_user_id }, import_id }) + + const { data: audio_change } = await upsert_audio({ dictionary_id, entry_id, audio: { storage_path: 'foo.mp3' } }) + await assign_speaker({ dictionary_id, speaker_id: speaker_change.speaker_id, media_id: audio_change.audio_id, media: 'audio', import_id, user_id: jacob_ld_user_id, timestamp: test_timestamp }) + + const { data: video_change } = await insert_video({ dictionary_id, video: { source: 'Bob Family', storage_path: 'baz.wbm' }, sense_id }) + await assign_speaker({ dictionary_id, speaker_id: speaker_change.speaker_id, media_id: video_change.video_id, media: 'video', import_id, user_id: jacob_ld_user_id, timestamp: test_timestamp }) + + const { data: entry_view } = await anon_supabase.from('entries_view').select().eq('id', entry_id).single() + expect(entry_view.audios[0].speaker_ids).toEqual([speaker_change.speaker_id]) + const { data: videos_view } = await anon_supabase.from('videos_view').select().eq('id', video_change.video_id).single() + expect(videos_view.speaker_ids).toEqual([speaker_change.speaker_id]) + }) +}) + +describe('entries have their updated_at timestamp updated whenever nested properties change', () => { + beforeEach(reset_db) + + describe('values which are props', () => { + test('audio', async () => { + const { entry_id } = await seed_entry_and_sense() + const { data: audio_change } = await upsert_audio({ dictionary_id, entry_id, audio: { storage_path: 'foo.mp3' } }) + const { data: entry_view } = await anon_supabase.from('entries_view').select().eq('id', entry_id).single() + expect(entry_view.updated_at).not.toEqual(entry_view.created_at) + expect(entry_view.updated_at).toEqual(audio_change.timestamp) + }) + + test('sense', async () => { + const { entry_id, sense_id } = await seed_entry_and_sense() + const { data: sense_change } = await insert_sense({ dictionary_id, entry_id, sense_id, sense: { glosses: { en: 'hi, again' } }, import_id }) + const { data: entry_view } = await anon_supabase.from('entries_view').select().eq('id', entry_id).single() + expect(entry_view.updated_at).toEqual(sense_change.timestamp) + }) + + test('dialect ids', async () => { + const { entry_id } = await seed_entry_and_sense() + const { data: dialect_addition } = await insert_dialect({ dictionary_id, name: 'Eastern', import_id, user_id: jacob_ld_user_id, timestamp: test_timestamp }) + const { data: dialect_assign } = await assign_dialect({ dictionary_id, dialect_id: dialect_addition.dialect_id, entry_id, import_id, user_id: jacob_ld_user_id, timestamp: test_timestamp }) + const { data: entry_view } = await anon_supabase.from('entries_view').select().eq('id', entry_id).single() + console.log({ entry_view, dialect_addition, dialect_assign }) + expect(entry_view.updated_at).toEqual(dialect_assign.timestamp) + }) + }) + + describe('id-related fields', () => { + test('sentence', async () => { + const { entry_id, sense_id } = await seed_entry_and_sense() + const { data: sentence_change } = await insert_sentence({ dictionary_id, sense_id, sentence: { text: { default: 'hi, this is my sentence' } } }) + const { data: entry_view } = await anon_supabase.from('entries_view').select().eq('id', entry_id).single() + expect(entry_view.updated_at).toEqual(sentence_change.timestamp) + }) + + test('photo', async () => { + const { entry_id, sense_id } = await seed_entry_and_sense() + const { data: photo_change } = await insert_photo({ dictionary_id, photo: { serving_url: 'foo', source: 'Bob', storage_path: 'bee/images/baz.jpeg' }, sense_id }) + const { data: entry_view } = await anon_supabase.from('entries_view').select().eq('id', entry_id).single() + expect(entry_view.updated_at).toEqual(photo_change.timestamp) + }) + + test('video', async () => { + const { entry_id, sense_id } = await seed_entry_and_sense() + const { data: video_change } = await insert_video({ dictionary_id, video: { source: 'Bob', storage_path: 'baz.wbm' }, sense_id }) + const { data: entry_view } = await anon_supabase.from('entries_view').select().eq('id', entry_id).single() + expect(entry_view.updated_at).toEqual(video_change.timestamp) + }) + }) +}) diff --git a/packages/scripts/migrate-to-supabase/operations/operations.ts b/packages/scripts/migrate-to-supabase/operations/operations.ts new file mode 100644 index 000000000..b54640852 --- /dev/null +++ b/packages/scripts/migrate-to-supabase/operations/operations.ts @@ -0,0 +1,266 @@ +import { randomUUID } from 'node:crypto' +import type { TablesInsert, TablesUpdate } from '@living-dictionaries/types' +import { prepare_sql } from '../save-content-update' + +export function insert_entry({ + dictionary_id, + entry, + entry_id, + import_id, +}: { + dictionary_id: string + entry: Omit, 'dictionary_id' | 'id'> + entry_id: string + import_id: string +}) { + return prepare_sql({ + update_id: randomUUID(), + auth_token: null, + dictionary_id, + entry_id, + type: 'insert_entry', + data: entry, + import_id, + }) +} + +export function insert_sense({ + dictionary_id, + entry_id, + sense, + sense_id, + import_id, +}: { + dictionary_id: string + entry_id: string + sense: Omit, 'dictionary_id' | 'id' | 'entry_id'> + sense_id: string + import_id: string +}) { + return prepare_sql({ + update_id: randomUUID(), + auth_token: null, + dictionary_id, + entry_id, + sense_id, + type: 'insert_sense', + data: sense, + import_id, + }) +} + +export function insert_dialect({ + dictionary_id, + name, + dialect_id, + import_id, + user_id, + timestamp, +}: { + dictionary_id: string + name: string + dialect_id: string + import_id: string + user_id: string + timestamp: string +}) { + return prepare_sql({ + update_id: randomUUID(), + auth_token: null, + dictionary_id, + dialect_id, + type: 'insert_dialect', + data: { + name: { + default: name, + }, + created_by: user_id, + created_at: timestamp, + }, + import_id, + }) +} + +export function assign_dialect({ + dictionary_id, + dialect_id, + entry_id, + import_id, + user_id, + timestamp, +}: { + dictionary_id: string + dialect_id: string + entry_id: string + import_id: string + user_id: string + timestamp: string +}) { + return prepare_sql({ + update_id: randomUUID(), + auth_token: null, + dictionary_id, + dialect_id, + entry_id, + type: 'assign_dialect', + data: { + created_by: user_id, + created_at: timestamp, + }, + import_id, + }) +} + +export function upsert_speaker({ + dictionary_id, + speaker, + speaker_id, + import_id, +}: { + dictionary_id: string + speaker: Omit, 'updated_by' | 'dictionary_id' | 'id'> + speaker_id: string + import_id: string +}) { + return prepare_sql({ + update_id: randomUUID(), + auth_token: null, + dictionary_id, + speaker_id, + type: 'upsert_speaker', + data: speaker, + import_id, + }) +} + +export function assign_speaker({ + dictionary_id, + speaker_id, + media_id, + media, + import_id, + user_id, + timestamp, +}: { + dictionary_id: string + speaker_id: string + media_id: string + media: 'audio' | 'video' + import_id: string + user_id: string + timestamp: string +}) { + return prepare_sql({ + update_id: randomUUID(), + auth_token: null, + data: { + created_by: user_id, + created_at: timestamp, + }, + dictionary_id, + speaker_id, + ...(media === 'audio' ? { audio_id: media_id } : { video_id: media_id }), + type: 'assign_speaker', + import_id, + }) +} + +export function upsert_audio({ + dictionary_id, + audio, + entry_id, + audio_id, + import_id, +}: { + dictionary_id: string + audio: Omit, 'updated_by' | 'dictionary_id' | 'id'> + entry_id: string + audio_id: string + import_id: string +}) { + return prepare_sql({ + update_id: randomUUID(), + auth_token: null, + dictionary_id, + entry_id, + audio_id, + type: 'upsert_audio', + data: audio, + import_id, + }) +} + +export function insert_sentence({ + dictionary_id, + sense_id, + sentence, + sentence_id, + import_id, +}: { + dictionary_id: string + sense_id: string + sentence: TablesInsert<'sentences'> + sentence_id: string + import_id: string +}) { + return prepare_sql({ + update_id: randomUUID(), + auth_token: null, + dictionary_id, + sentence_id, + sense_id, + type: 'insert_sentence', + data: sentence, + import_id, + }) +} + +export function insert_photo({ + dictionary_id, + photo, + sense_id, + photo_id, + import_id, +}: { + dictionary_id: string + photo: Omit, 'updated_by' | 'dictionary_id' | 'id'> + sense_id: string + photo_id: string + import_id: string +}) { + return prepare_sql({ + update_id: randomUUID(), + auth_token: null, + dictionary_id, + sense_id, + photo_id, + type: 'insert_photo', + data: photo, + import_id, + }) +} + +export function insert_video({ + dictionary_id, + video, + sense_id, + video_id, + import_id, +}: { + dictionary_id: string + video: TablesUpdate<'videos'> + sense_id: string + video_id: string + import_id: string +}) { + return prepare_sql({ + update_id: randomUUID(), + auth_token: null, + dictionary_id, + sense_id, + video_id, + type: 'insert_video', + data: video, + import_id, + }) +} diff --git a/packages/scripts/migrate-to-supabase/operations/test-timestamp.ts b/packages/scripts/migrate-to-supabase/operations/test-timestamp.ts new file mode 100644 index 000000000..7c1332ac1 --- /dev/null +++ b/packages/scripts/migrate-to-supabase/operations/test-timestamp.ts @@ -0,0 +1 @@ +export const test_timestamp: string = null diff --git a/packages/scripts/migrate-to-supabase/operations/utils.ts b/packages/scripts/migrate-to-supabase/operations/utils.ts new file mode 100644 index 000000000..9ee2c343f --- /dev/null +++ b/packages/scripts/migrate-to-supabase/operations/utils.ts @@ -0,0 +1,6 @@ +const uuid_template = '11111111-1111-1111-1111-111111111111' +let current_uuid_index = 0 + +export function incremental_consistent_uuid() { + return uuid_template.slice(0, -2) + (current_uuid_index++).toString().padStart(2, '0') +} diff --git a/packages/scripts/migrate-to-supabase/reset-db.ts b/packages/scripts/migrate-to-supabase/reset-db.ts new file mode 100644 index 000000000..7b7bf484a --- /dev/null +++ b/packages/scripts/migrate-to-supabase/reset-db.ts @@ -0,0 +1,17 @@ +// import { readFileSync } from 'node:fs' +import { jacob_ld_user_id, postgres } from '../config-supabase' + +export async function reset_db() { + console.info('reseting db from seed sql') + + await postgres.execute_query(`truncate table auth.users cascade;`) + await postgres.execute_query('truncate table senses cascade;') + + // const seedFilePath = '../../supabase/seed.sql' + // const seed_sql = readFileSync(seedFilePath, 'utf8') + // await postgres.execute_query(seed_sql) + + const add_user_sql = `INSERT INTO auth.users ("aud", "email", "id", "instance_id", "role") VALUES +('authenticated', 'jacob@livingtongues.org', '${jacob_ld_user_id}', '00000000-0000-0000-0000-000000000000', 'authenticated');` + await postgres.execute_query(add_user_sql) +} diff --git a/packages/scripts/migrate-to-supabase/run-migration.ts b/packages/scripts/migrate-to-supabase/run-migration.ts new file mode 100644 index 000000000..1e1dac4e6 --- /dev/null +++ b/packages/scripts/migrate-to-supabase/run-migration.ts @@ -0,0 +1,198 @@ +import fs, { writeFileSync } from 'node:fs' +import path from 'node:path' +import { fileURLToPath } from 'node:url' +import { access } from 'node:fs/promises' +import type { GoogleAuthUserMetaData, IUser } from '@living-dictionaries/types' +import Chain from 'stream-chain' +import Parser from 'stream-json' +import StreamArray from 'stream-json/streamers/StreamArray' +import type { UserRecord } from 'firebase-admin/auth' +import { admin_supabase, postgres } from '../config-supabase' +import type { AllSpeakerData } from './migrate-entries' +import { load_speakers, migrate_entry } from './migrate-entries' +import { remove_seconds_underscore } from './utils/remove-seconds-underscore' +import { load_fb_to_sb_user_ids } from './get-user-id' +import { write_users_insert } from './write-users-insert' + +const FOLDER = 'firestore-data' + +const __dirname = path.dirname(fileURLToPath(import.meta.url)) + +function local_filepath(filename: string): string { + return path.join(__dirname, FOLDER, filename) +} + +async function file_exists(filename: string): Promise { + try { + await access(local_filepath(filename), fs.constants.F_OK) + return true + } catch { + return false + } +} + +// run_migration({ start_index: 2233, batch_size: 47767 }) + +async function run_migration({ start_index, batch_size }: { start_index: number, batch_size: number }) { + console.log({ start_index, batch_size }) + + // if local + // await seed_local_db_with_production_data() + + // const entries_downloaded = await file_exists('firestore-entries.json') + // if (!entries_downloaded) + // await write_entries() + // const speakers_downloaded = await file_exists('firestore-speakers.json') + // if (!speakers_downloaded) + // await write_speakers() + // const users_downloaded = await file_exists('firestore-users.json') + // if (!users_downloaded) + // await write_users() + + // if (start_index === 0) { + // await write_fb_sb_mappings() // user mappings + // } + + await load_fb_to_sb_user_ids() + + // if (start_index === 0) { + // await migrate_speakers() // speaker mappings + // } + + const fb_to_sb_speakers = await load_speakers() + await migrate_all_entries({ fb_to_sb_speakers, start_index, batch_size }) +} + +async function migrate_all_entries({ fb_to_sb_speakers, start_index, batch_size }: { fb_to_sb_speakers: AllSpeakerData, start_index: number, batch_size: number }) { + const { data: dialects } = await admin_supabase.from('dialects').select('id, name, dictionary_id') + const dictionary_dialects: Record> = dialects.reduce((acc: Record>, { id, name, dictionary_id }) => { + if (!acc[dictionary_id]) { + acc[dictionary_id] = {} + } + acc[dictionary_id][name.default] = id + return acc + }, {}) + + const { data: speakers_added_to_sb_by_name } = await admin_supabase.from('speakers').select('id, name, dictionary_id') + const dictionary_new_speakers: Record> = speakers_added_to_sb_by_name.reduce((acc: Record>, { id, name, dictionary_id }) => { + if (!acc[dictionary_id]) { + acc[dictionary_id] = {} + } + acc[dictionary_id][name] = id + return acc + }, {}) + + const pipeline = Chain.chain([ + fs.createReadStream('./migrate-to-supabase/firestore-data/firestore-entries.json'), + Parser.parser(), + StreamArray.streamArray(), + ]) + + const end_index = start_index + batch_size + let index = 0 + let current_dictionary_entry_id = '' + let sql_query = 'BEGIN;' // Start a transaction + + try { + for await (const { value: fb_entry } of pipeline) { + if (index >= start_index && index < end_index) { + current_dictionary_entry_id = `${fb_entry.dictionary_id}/${fb_entry.id}` + const seconds_corrected_entry = remove_seconds_underscore(fb_entry) + console.info(index) + const sql_statements = migrate_entry(seconds_corrected_entry, fb_to_sb_speakers, dictionary_dialects, dictionary_new_speakers) + sql_query += `${sql_statements}\n` + + if (index % 500 === 0) + console.log(`import reached ${index}`) + } + index++ + } + } catch (err) { + console.log(`error at index ${index}: _ROOT_/${current_dictionary_entry_id}, ${err}`) + console.error(err) + } finally { + pipeline.destroy() + pipeline.input.destroy() + } + + sql_query += '\nCOMMIT;' // End the transaction + + try { + writeFileSync(`./logs/${Date.now()}_${start_index}-${start_index + batch_size}-query.sql`, sql_query) + console.log('executing sql query') + await postgres.execute_query(sql_query) + console.log('finished') + } catch (err) { + console.error(err) + await postgres.execute_query('ROLLBACK;') // Rollback the transaction in case of error + } +} + +// async function seed_local_db_with_production_data() { +// console.log('Seeding local db with production data') +// await postgres.execute_query(`truncate table auth.users cascade;`) +// await postgres.execute_query('truncate table entry_updates cascade;') +// await postgres.execute_query(readFileSync('../../supabase/seeds/backup-after-2232-imported-ready-for-local.sql', 'utf8')) +// } + +async function write_fb_sb_mappings() { + console.log('writing user mappings') + const firebase_uid_to_supabase_user_id: Record = {} + const { data: sb_users_1 } = await admin_supabase.from('user_emails') + .select('id, email') + .order('id', { ascending: true }) + .range(0, 999) + const { data: sb_users_2 } = await admin_supabase.from('user_emails') + .select('id, email') + .order('id', { ascending: true }) + .range(1000, 1999) + + const sb_users = [...sb_users_1, ...sb_users_2] + + const supabase_users_not_in_firebase = new Set(sb_users.map(sb_user => sb_user.email)) + const unmatched_firebase = [] + + const firebase_users = (await import('./firestore-data/firestore-users.json')).default + + for (const fb_user of firebase_users) { + const matching_sb_user = sb_users.find(sb_user => sb_user.email === fb_user.email) + + if (matching_sb_user) { + firebase_uid_to_supabase_user_id[fb_user.uid] = matching_sb_user.id + supabase_users_not_in_firebase.delete(matching_sb_user.email) + } else { + const sql = write_users_insert([fb_user as UserRecord]) + console.log(sql) + // await execute_query(sql) + + // const new_sb_user_id = await save_user_to_supabase(fb_user as UserRecord) + // firebase_uid_to_supabase_user_id[fb_user.uid] = new_sb_user_id + } + } + + console.log({ unmatched_firebase: unmatched_firebase.length, sb_users: sb_users.length, firebase_users: firebase_users.length, supabase_users_not_in_firebase: supabase_users_not_in_firebase.size }) + + fs.writeFileSync(path.resolve(__dirname, FOLDER, 'fb-sb-user-ids.json'), JSON.stringify(firebase_uid_to_supabase_user_id, null, 2)) +} + +async function save_user_to_supabase(user: UserRecord): Promise { + const { data, error } = await admin_supabase.auth.admin.createUser({ + email: user.email, + email_confirm: user.emailVerified, + app_metadata: { fb_uid: user.uid }, + user_metadata: get_firebase_user_meta_data(user), + }) + if (error) + throw new Error(`Error creating user: ${user.email}`) + console.info({ created: data.user.email }) + return data?.user?.id +} + +function get_firebase_user_meta_data({ displayName, photoURL }: IUser) { + const metadata: GoogleAuthUserMetaData = {} + if (displayName) + metadata.full_name = displayName + if (photoURL) + metadata.avatar_url = photoURL + return metadata +} diff --git a/packages/scripts/migrate-to-supabase/save-content-update.ts b/packages/scripts/migrate-to-supabase/save-content-update.ts new file mode 100644 index 000000000..e470f4e10 --- /dev/null +++ b/packages/scripts/migrate-to-supabase/save-content-update.ts @@ -0,0 +1,216 @@ +import fs from 'node:fs' +import type { ContentImportBody } from '@living-dictionaries/types/supabase/content-import.interface' +import { jacob_ld_user_id } from '../config-supabase' +import { sql_file_string } from './to-sql-string' + +const content_update_file = fs.createWriteStream(`./logs/${Date.now()}_content-updates.json`, { flags: 'w' }) // 'a' to append, 'w' to truncate the file every time the process starts. +content_update_file.write('[\n') + +const yesterday = new Date(Date.now() - 24 * 60 * 60 * 1000) +let milliseconds_to_add = 0 + +function millisecond_incrementing_timestamp(): string { + milliseconds_to_add += 1 + return new Date(yesterday.getTime() + milliseconds_to_add).toISOString() +} + +export function prepare_sql(body: ContentImportBody) { + console.info(body) + + let sql_statements = '' + + const { update_id, dictionary_id, import_id, type, data } = body + + const created_at = data?.created_at || millisecond_incrementing_timestamp() + // @ts-expect-error + const updated_at = data?.updated_at || created_at + + // @ts-expect-error + const created_by = data?.created_by || jacob_ld_user_id + // @ts-expect-error + const updated_by = data?.updated_by || created_by + + const c_u_meta = { + created_by, + updated_by, + created_at, + updated_at, + } + + const c_meta = { + created_by, + created_at, + } + + if (type === 'insert_entry') { + const sql = sql_file_string('entries', { + ...data, + ...c_u_meta, + dictionary_id, + id: body.entry_id, + }, 'UPSERT') + sql_statements += `\n${sql}` + } + + if (type === 'insert_sense') { + const sql = sql_file_string('senses', { + ...data, + ...c_u_meta, + entry_id: body.entry_id, + id: body.sense_id, + }) + sql_statements += `\n${sql}` + } + + if (type === 'insert_dialect') { + const sql = sql_file_string('dialects', { + ...data, + ...c_u_meta, + dictionary_id, + id: body.dialect_id, + }) + sql_statements += `\n${sql}` + } + + if (type === 'assign_dialect') { + const sql = sql_file_string('entry_dialects', { + ...c_meta, + dialect_id: body.dialect_id, + entry_id: body.entry_id, + }) + sql_statements += `\n${sql}` + } + + if (type === 'upsert_speaker') { + const sql = sql_file_string('speakers', { + ...data, + ...c_u_meta, + dictionary_id, + id: body.speaker_id, + }) + sql_statements += `\n${sql}` + } + + if (type === 'assign_speaker' && body.audio_id) { + const sql = sql_file_string('audio_speakers', { + ...c_meta, + speaker_id: body.speaker_id, + audio_id: body.audio_id, + }) + sql_statements += `\n${sql}` + } + + if (type === 'assign_speaker' && body.video_id) { + const sql = sql_file_string('video_speakers', { + ...c_meta, + speaker_id: body.speaker_id, + video_id: body.video_id, + }) + sql_statements += `\n${sql}` + } + + if (type === 'insert_sentence') { + const sql = sql_file_string('sentences', { + ...data, + ...c_u_meta, + dictionary_id, + id: body.sentence_id, + }) + sql_statements += `\n${sql}` + + const sql2 = sql_file_string('senses_in_sentences', { + ...c_meta, + sentence_id: body.sentence_id, + sense_id: body.sense_id, + }) + sql_statements += `\n${sql2}` + } + + if (type === 'upsert_audio') { + const sql = sql_file_string('audio', { + ...data, + ...c_u_meta, + dictionary_id, + id: body.audio_id, + entry_id: body.entry_id, + }) + sql_statements += `\n${sql}` + } + + if (type === 'insert_photo') { + const sql = sql_file_string('photos', { + ...data, + ...c_u_meta, + dictionary_id, + id: body.photo_id, + }) + sql_statements += `\n${sql}` + + const sql2 = sql_file_string('sense_photos', { + ...c_meta, + photo_id: body.photo_id, + sense_id: body.sense_id, + }) + sql_statements += `\n${sql2}` + } + + if (type === 'insert_video') { + const sql = sql_file_string('videos', { + ...data, + ...c_u_meta, + dictionary_id, + id: body.video_id, + }) + sql_statements += `\n${sql}` + + const sql2 = sql_file_string('sense_videos', { + ...c_meta, + video_id: body.video_id, + sense_id: body.sense_id, + }) + sql_statements += `\n${sql2}` + } + + const data_without_ids = { ...data } + delete data_without_ids.id + delete data_without_ids.dictionary_id + delete data_without_ids.entry_id + delete data_without_ids.sense_id + delete data_without_ids.created_at + delete data_without_ids.created_by + delete data_without_ids.updated_at + delete data_without_ids.updated_by + + const content_update = { + id: update_id, + user_id: updated_by, + dictionary_id, + timestamp: updated_at, + import_id, + type, + data: Object.keys(data_without_ids)?.length ? data_without_ids : null, + // @ts-expect-error - avoiding verbosity but requires manual type checking + ...(body.audio_id && { audio_id: body.audio_id }), + // @ts-expect-error + ...(body.dialect_id && { dialect_id: body.dialect_id }), + // @ts-expect-error + ...(body.entry_id && { entry_id: body.entry_id }), + // @ts-expect-error + ...(body.photo_id && { photo_id: body.photo_id }), + // @ts-expect-error + ...(body.sense_id && { sense_id: body.sense_id }), + // @ts-expect-error + ...(body.sentence_id && { sentence_id: body.sentence_id }), + // @ts-expect-error + ...(body.speaker_id && { speaker_id: body.speaker_id }), + // @ts-expect-error + ...(body.text_id && { text_id: body.text_id }), + // @ts-expect-error + ...(body.video_id && { video_id: body.video_id }), + // This is the properly typed version but much more verbose as requires one for each change type + // ...(type === 'insert_sense' && { sense_id: body.sense_id, entry_id: body.entry_id }), + } + content_update_file.write(`${JSON.stringify(content_update)},\n`) + + return sql_statements +} diff --git a/packages/scripts/migrate-to-supabase/save-firestore-data.ts b/packages/scripts/migrate-to-supabase/save-firestore-data.ts new file mode 100644 index 000000000..959e3c04d --- /dev/null +++ b/packages/scripts/migrate-to-supabase/save-firestore-data.ts @@ -0,0 +1,56 @@ +import fs from 'node:fs' +import path, { dirname } from 'node:path' +import { fileURLToPath } from 'node:url' +import type { ActualDatabaseEntry } from '@living-dictionaries/types/entry.interface' +import type { ISpeaker } from '@living-dictionaries/types/speaker.interface' +import { db } from '../config-firebase' +import { get_users } from './auth' + +const FOLDER = 'firestore-data' +const __dirname = dirname(fileURLToPath(import.meta.url)) + +export async function write_entries() { + const entries: ActualDatabaseEntry[] = [] + + const dict_snapshot = await db.collection('dictionaries').get() + + for (const { id: dictionary_id } of dict_snapshot.docs) { + console.log(dictionary_id) + // const allow = /^[a].*/ + // if (!allow.test(dictionary_id.toLowerCase())) continue + + const snapshot = await db.collection(`dictionaries/${dictionary_id}/words`).get() + + for (const snap of snapshot.docs) { + const entry = { id: snap.id, dictionary_id, ...(snap.data() as ActualDatabaseEntry) } + entries.push(entry) + } + } + + console.log(`Done fetching ${entries.length} entries from ${dict_snapshot.docs.length} dictionaries.`) + + fs.writeFileSync(path.resolve(__dirname, FOLDER, 'firestore-entries.json'), JSON.stringify(entries, null, 2)) +} + +export async function write_speakers() { + const speakers: ISpeaker[] = [] + + const speaker_snapshots = await db.collection('speakers').get() + + for (const speaker_snap of speaker_snapshots.docs) { + const speaker = { id: speaker_snap.id, ...(speaker_snap.data() as ISpeaker) } + speakers.push(speaker) + } + + console.log(`Done fetching ${speakers.length} speakers.`) + + fs.writeFileSync(path.resolve(__dirname, FOLDER, 'firestore-speakers.json'), JSON.stringify(speakers, null, 2)) +} + +export async function write_users() { + const users = await get_users() + + console.log(`Done fetching ${users.length} users.`) + + fs.writeFileSync(path.resolve(__dirname, FOLDER, 'firestore-users.json'), JSON.stringify(users, null, 2)) +} diff --git a/packages/scripts/migrate-to-supabase/to-sql-string.ts b/packages/scripts/migrate-to-supabase/to-sql-string.ts new file mode 100644 index 000000000..bb88fc72c --- /dev/null +++ b/packages/scripts/migrate-to-supabase/to-sql-string.ts @@ -0,0 +1,15 @@ +import type { Database, TablesInsert } from '@living-dictionaries/types' +import { convert_to_sql_string } from '../../site/src/lib/mocks/seed/to-sql-string' + +export function sql_file_string(table_name: Table, row: TablesInsert
, operation: 'INSERT' | 'UPSERT' = 'INSERT') { + const column_names = Object.keys(row).sort() + const column_names_string = `"${column_names.join('", "')}"` + + const values = column_names.map(column => convert_to_sql_string(row[column])) + const values_string = `(${values.join(', ')})` + if (operation === 'INSERT') { + return `INSERT INTO ${table_name} (${column_names_string}) VALUES\n${values_string};` + } else if (operation === 'UPSERT') { + return `INSERT INTO ${table_name} (${column_names_string}) VALUES\n${values_string}\nON CONFLICT (id) DO UPDATE SET ${column_names.map(column => `"${column}" = EXCLUDED."${column}"`).join(', ')};` + } +} diff --git a/packages/scripts/migrate-to-supabase/utils/remove-seconds-underscore.ts b/packages/scripts/migrate-to-supabase/utils/remove-seconds-underscore.ts new file mode 100644 index 000000000..d2af613d7 --- /dev/null +++ b/packages/scripts/migrate-to-supabase/utils/remove-seconds-underscore.ts @@ -0,0 +1,37 @@ +import type { ActualDatabaseEntry } from '@living-dictionaries/types/entry.interface' + +export function remove_seconds_underscore(entry: ActualDatabaseEntry & Record) { + // @ts-expect-error + if (entry.updatedAt?._seconds) { + // @ts-expect-error + entry.updatedAt = { + // @ts-expect-error + seconds: entry.updatedAt._seconds, + } + } + // @ts-expect-error + if (entry.createdAt?._seconds) { + // @ts-expect-error + entry.createdAt = { + // @ts-expect-error + seconds: entry.createdAt._seconds, + } + } + // @ts-expect-error + if (entry.ua?._seconds) { + // @ts-expect-error + entry.ua = { + // @ts-expect-error + seconds: entry.ua._seconds, + } + } + // @ts-expect-error + if (entry.ca?._seconds) { + // @ts-expect-error + entry.ca = { + // @ts-expect-error + seconds: entry.ca._seconds, + } + } + return entry +} diff --git a/packages/scripts/migrate-to-supabase/write-users-insert.ts b/packages/scripts/migrate-to-supabase/write-users-insert.ts index 4b3f5df2d..74e503136 100644 --- a/packages/scripts/migrate-to-supabase/write-users-insert.ts +++ b/packages/scripts/migrate-to-supabase/write-users-insert.ts @@ -1,9 +1,10 @@ -import { UserRecord } from 'firebase-admin/auth'; +import type { UserRecord } from 'firebase-admin/auth' + const POSTGRES_NOW = 'NOW()' const POSTGRESS_UUID_GENERATE_V4 = 'uuid_generate_v4()' export function write_users_insert(users: UserRecord[]) { - const user_rows = users.map(user => { + const user_rows = users.map((user) => { return { instance_id: '00000000-0000-0000-0000-000000000000', id: POSTGRESS_UUID_GENERATE_V4, @@ -26,7 +27,7 @@ function write_sql_insert(table_name: string, rows: object[]) { const column_names = Object.keys(rows[0]).sort() const column_names_string = `"${column_names.join('", "')}"` - const values_string = rows.map(row => { + const values_string = rows.map((row) => { // @ts-expect-error const values = column_names.map(column => convert_to_sql_string(row[column])) return `(${values.join(', ')})` @@ -37,7 +38,7 @@ function write_sql_insert(table_name: string, rows: object[]) { function convert_to_sql_string(value: string | number | object) { if (value === POSTGRES_NOW || value === POSTGRESS_UUID_GENERATE_V4) - return value; + return value if (typeof value === 'boolean') return `${value}` @@ -56,75 +57,60 @@ function convert_to_sql_string(value: string | number | object) { if (import.meta.vitest) { const users: UserRecord[] = [ - { 'uid': '024bvoAhcSaAiBfZ8Um2KQaQRc92', - 'email': 'robert@me.org', - 'emailVerified': true, - 'disabled': false, - 'metadata': { - 'lastSignInTime': 'Thu, 08 Jun 2023 04:42:14 GMT', - 'creationTime': 'Thu, 08 Jun 2023 04:42:14 GMT', - 'lastRefreshTime': 'Thu, 08 Jun 2023 04:42:14 GMT', - }, - 'tokensValidAfterTime': 'Thu, 08 Jun 2023 04:42:14 GMT', - 'providerData': [{ 'email': 'robert@me.org', 'providerId': 'password' }], - }, - {'uid':'0FCdmOc6qlWuKxFkKPi7VeC2mp52', - 'email':'bob@gmail.com', - 'emailVerified':false, - 'displayName':'Bob D\'Smith', - 'photoURL':'https://lh3.googleusercontent.com/a-/AOh14Gg-GMlUaNPYaSYvzMEjyHW9Q5PAngePLc26LsI4=s96-c', - 'disabled':false, - 'metadata':{ - 'lastSignInTime':'Fri, 12 Mar 2021 21:04:29 GMT', - 'creationTime':'Tue, 29 Dec 2020 00:08:26 GMT', - 'lastRefreshTime':null - }, - 'tokensValidAfterTime':'Thu, 18 Mar 2021 08:26:40 GMT', - 'providerData':[{'displayName':'Bob Smith','email':'bob@gmail.com','photoURL':'https://lh3.googleusercontent.com/a-/AOh14Gg-GMlUaNPYaSYvzMEjyHW9Q5PAngePLc26LsI4=s96-c','providerId':'google.com'}]} - ] as UserRecord[]; + { uid: '024bvoAhcSaAiBfZ8Um2KQaQRc92', email: 'robert@me.org', emailVerified: true, disabled: false, metadata: { + lastSignInTime: 'Thu, 08 Jun 2023 04:42:14 GMT', + creationTime: 'Thu, 08 Jun 2023 04:42:14 GMT', + lastRefreshTime: 'Thu, 08 Jun 2023 04:42:14 GMT', + }, tokensValidAfterTime: 'Thu, 08 Jun 2023 04:42:14 GMT', providerData: [{ email: 'robert@me.org', providerId: 'password' }] }, + { uid: '0FCdmOc6qlWuKxFkKPi7VeC2mp52', email: 'bob@gmail.com', emailVerified: false, displayName: 'Bob D\'Smith', photoURL: 'https://lh3.googleusercontent.com/a-/AOh14Gg-GMlUaNPYaSYvzMEjyHW9Q5PAngePLc26LsI4=s96-c', disabled: false, metadata: { + lastSignInTime: 'Fri, 12 Mar 2021 21:04:29 GMT', + creationTime: 'Tue, 29 Dec 2020 00:08:26 GMT', + lastRefreshTime: null, + }, tokensValidAfterTime: 'Thu, 18 Mar 2021 08:26:40 GMT', providerData: [{ displayName: 'Bob Smith', email: 'bob@gmail.com', photoURL: 'https://lh3.googleusercontent.com/a-/AOh14Gg-GMlUaNPYaSYvzMEjyHW9Q5PAngePLc26LsI4=s96-c', providerId: 'google.com' }] }, + ] as UserRecord[] test(write_users_insert, () => { - expect(write_users_insert(users)).toMatchFileSnapshot('./users-js.sql'); - }); + expect(write_users_insert(users)).toMatchFileSnapshot('./users-js.sql') + }) } function convert_utc_string_to_timestamp(dateString: string) { - const date = new Date(dateString); - const isoString = date.toISOString(); - return isoString.replace('T', ' ').replace('Z', '1+00'); // if decimal are all zeros as they are in coming from Firebase, postgres will reject the timestamp so we must remove it or add a final 1 + const date = new Date(dateString) + const isoString = date.toISOString() + return isoString.replace('T', ' ').replace('Z', '1+00') // if decimal are all zeros as they are in coming from Firebase, postgres will reject the timestamp so we must remove it or add a final 1 } if (import.meta.vitest) { test('times', () => { - expect(convert_utc_string_to_timestamp('Fri, 12 Mar 2021 21:04:29 GMT')).toEqual('2021-03-12 21:04:29.0001+00'); - expect(convert_utc_string_to_timestamp('Thu, 08 Jun 2023 04:42:14 GMT')).toEqual('2023-06-08 04:42:14.0001+00'); - }); + expect(convert_utc_string_to_timestamp('Fri, 12 Mar 2021 21:04:29 GMT')).toEqual('2021-03-12 21:04:29.0001+00') + expect(convert_utc_string_to_timestamp('Thu, 08 Jun 2023 04:42:14 GMT')).toEqual('2023-06-08 04:42:14.0001+00') + }) } -function get_firebase_app_meta_data({providerData, uid}: UserRecord) { - const providers = providerData.map(({providerId}) => { +function get_firebase_app_meta_data({ providerData, uid }: UserRecord) { + const providers = providerData.map(({ providerId }) => { if (providerId === 'password') - return 'email'; + return 'email' if (providerId === 'google.com') - return 'google'; - return 'email'; + return 'google' + return 'email' }) - return `{"provider": "${providers[0]}","providers":["${providers.join('","')}"], "fb_uid": "${uid}"}`; + return `{"provider": "${providers[0]}","providers":["${providers.join('","')}"], "fb_uid": "${uid}"}` } -function get_firebase_user_meta_data({displayName, photoURL}: UserRecord) { - const escapedDisplayName = displayName ? escape_apostrophes(displayName) : ''; +function get_firebase_user_meta_data({ displayName, photoURL }: UserRecord) { + const escapedDisplayName = displayName ? escape_apostrophes(displayName) : '' if (!escapedDisplayName && !photoURL) - return '{}'; + return '{}' if (!escapedDisplayName) - return `{"avatar_url":"${photoURL}"}`; + return `{"avatar_url":"${photoURL}"}` if (!photoURL) - return `{"full_name": "${escapedDisplayName}"}`; - return `{"full_name": "${escapedDisplayName}","avatar_url":"${photoURL}"}`; + return `{"full_name": "${escapedDisplayName}"}` + return `{"full_name": "${escapedDisplayName}","avatar_url":"${photoURL}"}` } function escape_apostrophes(str: string): string { - return str.replace(/'/g, '\'\''); + return str.replace(/'/g, '\'\'') } diff --git a/packages/scripts/package.json b/packages/scripts/package.json index d63d83fd1..22ed27e61 100644 --- a/packages/scripts/package.json +++ b/packages/scripts/package.json @@ -14,7 +14,7 @@ }, "main": "index.ts", "scripts": { - "fetch-entries": "tsx migrate-to-supabase/fetch-entries.ts -e prod", + "run-migration": "tsx migrate-to-supabase/run-migration.ts -e prod", "migrate-users": "tsx migrate-to-supabase/auth.ts", "countAllEntries": "tsx countAllEntries.ts", "getEmails": "tsx refactor/get-email.ts -e prod", @@ -24,18 +24,15 @@ "importDictionary:dev:dry": "tsx import/import.ts --id tseltal", "importDictionary:dev:live": "tsx import/import.ts --id tseltal --live", "importDictionary:prod:live": "tsx import/import.ts --id tseltal -e prod --live", - "addDictionariesToIndex:dev": "tsx algolia/addDictionariesToIndex.ts dev", - "addDictionariesToIndex:prod": "tsx algolia/addDictionariesToIndex.js prod", - "updateIndex": "tsx algolia/updateIndex.ts -e prod", - "test": "vitest" + "test": "vitest", + "test:migration": "vitest --config ./vitest.config.migration.ts" }, "devDependencies": { - "@living-dictionaries/functions": "workspace:^0.0.1", "@living-dictionaries/site": "workspace:^0.0.1", - "@living-dictionaries/types": "^1.0.0", + "@living-dictionaries/types": "workspace:^1.0.0", "@supabase/supabase-js": "^2.38.4", "@types/node": "^18.11.18", - "@types/pg": "^8.10.9", + "@types/pg": "^8.11.8", "@types/stream-chain": "^2.1.0", "@types/stream-json": "^1.7.7", "algoliasearch": "^4.11.0", @@ -50,8 +47,8 @@ "pg": "^8.11.3", "stream-chain": "^2.2.5", "stream-json": "^1.8.0", - "tsx": "^4.7.1", + "tsx": "^4.19.0", "typescript": "~5.1.6", - "vitest": "^1.4.0" + "vitest": "^2.1.3" } } diff --git a/packages/scripts/record-logs.ts b/packages/scripts/record-logs.ts new file mode 100644 index 000000000..42ca148f1 --- /dev/null +++ b/packages/scripts/record-logs.ts @@ -0,0 +1,11 @@ +import fs from 'node:fs' + +const logFile = fs.createWriteStream(`./logs/${Date.now()}.txt`, { flags: 'w' }) // 'a' to append, 'w' to truncate the file every time the process starts. +console.log = function (data: any) { + logFile.write(`${JSON.stringify(data)}\n`) + process.stdout.write(`${JSON.stringify(data)}\n`) +} +const postFile = fs.createWriteStream(`./logs/${Date.now()}_post_requests.txt`, { flags: 'w' }) // 'a' to append, 'w' to truncate the file every time the process starts. +console.info = function (data: any) { + postFile.write(`${JSON.stringify(data)}\n`) +} diff --git a/packages/scripts/vitest.config.migration.ts b/packages/scripts/vitest.config.migration.ts new file mode 100644 index 000000000..bfd63ffa6 --- /dev/null +++ b/packages/scripts/vitest.config.migration.ts @@ -0,0 +1,15 @@ +import { defineConfig } from 'vitest/config' + +// run separately from other unit tests because it requires local Supabase running +export default defineConfig({ + test: { + name: 'scripts:migration', + globals: true, + poolOptions: { + threads: { + singleThread: true, + }, + }, + include: ['migrate-to-supabase/**/*.test.ts'], + }, +}) diff --git a/packages/scripts/vitest.config.ts b/packages/scripts/vitest.config.ts index df66a8db5..686dcd3b5 100644 --- a/packages/scripts/vitest.config.ts +++ b/packages/scripts/vitest.config.ts @@ -1,9 +1,10 @@ -import { defineProject } from 'vitest/config'; +import { defaultExclude, defineProject } from 'vitest/config' export default defineProject({ test: { name: 'scripts:unit', globals: true, - includeSource: ['./migrate-to-supabase/**/*.ts', './import/**/*.ts', './refactor/**/*.ts', './spreadsheet_helpers/**/*.ts'], + includeSource: ['./import/**/*.ts', './refactor/**/*.ts', './spreadsheet_helpers/**/*.ts'], + exclude: [...defaultExclude, 'migrate-to-supabase/**'], }, -}); +}) diff --git a/packages/site/.env b/packages/site/.env index 01af5389e..2b7942039 100644 --- a/packages/site/.env +++ b/packages/site/.env @@ -1,8 +1,5 @@ # See https://vitejs.dev/config/#envdir if wanting to store these in monorepo root -PUBLIC_ALGOLIA_APPLICATION_ID=XCVBAYSYXD -PUBLIC_ALGOLIA_SEARCH_ONLY_API_KEY=e6d98efb32d3dc2435dce7b97ea87c3e - # URL Restricted PUBLIC_mapboxAccessToken=pk.eyJ1IjoidGFsa2luZ2RpY3Rpb25hcmllcyIsImEiOiJja3BkYW84NjcwYXA2Mm90NjEzemdrZmxjIn0.W9YL4gEpnfIvHhZ_XaFa1g diff --git a/packages/site/package.json b/packages/site/package.json index 239fd22be..51a689711 100644 --- a/packages/site/package.json +++ b/packages/site/package.json @@ -7,6 +7,7 @@ "scripts": { "dev": "vite dev", "prod": "vite dev --mode production", + "mixed": "vite dev --mode mixed", "build": "vite build", "preview": "vite preview", "sync": "svelte-kit sync", @@ -24,10 +25,11 @@ "local-sveltefirets": "link:../../../sveltefirets" }, "devDependencies": { - "@iconify/json": "^2.2.192", + "@iconify/json": "^2.2.242", "@julr/unocss-preset-forms": "^0.0.5", "@living-dictionaries/types": "workspace:^1.0.0", - "@orama/orama": "^2.0.11", + "@orama/orama": "2.0.23", + "@orama/plugin-data-persistence": "2.0.23", "@playwright/test": "1.41.2", "@resvg/resvg-js": "^2.2.0", "@sentry/browser": "^6.18.2", @@ -45,12 +47,11 @@ "@types/geojson": "^7946.0.10", "@types/mapbox-gl": "^2.7.6", "@types/mapbox__mapbox-gl-geocoder": "^4.7.3", - "@types/pg": "^8.10.9", + "@types/pg": "^8.11.8", "@types/recordrtc": "^5.6.10", "@unocss/preset-icons": "^0.58.5", "@unocss/svelte-scoped": "^0.58.5", - "@vitest/ui": "^1.4.0", - "algoliasearch": "^4.11.0", + "@vitest/ui": "^2.1.3", "ckeditor5-build-classic-with-alignment-underline-smallcaps": "^1.0.0", "comlink": "^4.4.1", "csvtojson": "^2.0.10", @@ -59,26 +60,24 @@ "file-saver": "^2.0.5", "firebase": "^10.9.0", "firebase-admin": "^12.0.0", - "instantsearch.js": "^4.33.2", + "idb-keyval": "^6.2.1", "jszip": "^3.7.1", "kitbook": "1.0.0-beta.31", "logrocket": "^2.1.2", - "lz-string": "^1.4.4", "pg": "^8.11.3", "recordrtc": "^5.6.2", "satori": "^0.0.44", "satori-html": "^0.3.0", "svelte": "^4.2.12", "svelte-check": "^3.6.8", - "svelte-pieces": "2.0.0-next.14", + "svelte-pieces": "2.0.0-next.16", "sveltefirets": "0.0.42", "topojson-client": "^3.1.0", "tslib": "^2.6.1", - "type-fest": "^4.13.0", "typescript": "^5.1.6", "unocss": "^0.58.5", - "vite": "^5.1.4", - "vitest": "^1.4.0", + "vite": "^5.4.2", + "vitest": "^2.1.3", "wavesurfer.js": "^5.2.0", "web-vitals": "^2.1.2", "xss": "^1.0.14" diff --git a/packages/site/src/app.d.ts b/packages/site/src/app.d.ts index df3d939a6..0fdfbba15 100644 --- a/packages/site/src/app.d.ts +++ b/packages/site/src/app.d.ts @@ -2,11 +2,8 @@ // import type { BaseUser } from '$lib/supabase/user' // import type { AuthResponse } from '@supabase/supabase-js' import type { Readable } from 'svelte/store' -import type { collectionStore } from 'sveltefirets' -import type { ISpeaker } from '@living-dictionaries/types' -import type { DbOperations } from '$lib/dbOperations' - -// import type { Supabase } from '$lib/supabase/database.types' +import type { LayoutData as DictionaryLayoutData } from './routes/[dictionaryId]/$types' +import type { Supabase } from '$lib/supabase/database.types' declare global { namespace App { @@ -18,12 +15,19 @@ declare global { t: import('$lib/i18n/types.ts').TranslateFunction user: Readable admin: Readable - // supabase: Supabase // authResponse: AuthResponse // From dictionary layout so all optional - dbOperations?: DbOperations - speakers?: Awaited>> + supabase?: Supabase + dictionary?: DictionaryLayoutData['dictionary'] + dbOperations?: DictionaryLayoutData['dbOperations'] + url_from_storage_path?: DictionaryLayoutData['url_from_storage_path'] + entries?: DictionaryLayoutData['entries'] + speakers?: DictionaryLayoutData['speakers'] + dialects?: DictionaryLayoutData['dialects'] + photos?: DictionaryLayoutData['photos'] + videos?: DictionaryLayoutData['videos'] + sentences?: DictionaryLayoutData['sentences'] } interface PageState { entry_id?: string diff --git a/packages/site/src/db-tests/cached-data.test.ts b/packages/site/src/db-tests/cached-data.test.ts new file mode 100644 index 000000000..7ad4a96a5 --- /dev/null +++ b/packages/site/src/db-tests/cached-data.test.ts @@ -0,0 +1,441 @@ +import { get as getStore } from 'svelte/store' +import { get as get_idb } from 'idb-keyval' +import type { TablesInsert, TablesUpdate } from '@living-dictionaries/types' +import { anon_supabase } from './clients' +import { cached_data_store } from '$lib/supabase/cached-data' +import { postgres } from '$lib/mocks/seed/postgres' +import { sql_file_string } from '$lib/mocks/seed/to-sql-string' +import { seeded_dictionary_id as dictionary_id, seed_dictionaries, seeded_user_id_1, users } from '$lib/mocks/seed/tables' + +const log = false + +vi.mock('$app/environment', () => { + return { + browser: true, + } +}) + +let cached_data: any[] = [] + +vi.mock('idb-keyval', () => { + return { + set: vi.fn((key: string, value: any[]) => { + cached_data = value + }), + get: vi.fn((_key: string) => { + return cached_data?.length ? cached_data : undefined + }), + } +}) + +function incremental_consistent_uuid(index: number) { + return '22222222-2222-2222-2222-222222222222'.slice(0, -6) + (index).toString().padStart(6, '0') +} + +const refresh_materialized_view_sql = 'REFRESH MATERIALIZED VIEW materialized_entries_view' +const reset_db_sql = ` + truncate table auth.users cascade; + ${sql_file_string('auth.users', users)} + ${sql_file_string('dictionaries', seed_dictionaries)} + ${refresh_materialized_view_sql}` + +const startTimestamp = new Date('1980-01-01T00:00:00Z').getTime() + +function seed_with_entries({ count, offset }: { count: number, offset: number }) { + const seed_entries: TablesInsert<'entries'>[] = Array.from({ length: count }, (_, index) => { + const offset_index = index + offset + const entryTimestamp = new Date(startTimestamp + offset_index * 1000).toISOString() + return { + id: incremental_consistent_uuid(offset_index), + lexeme: { + default: `${offset_index} lexeme`, + }, + dictionary_id, + created_by: seeded_user_id_1, + updated_by: seeded_user_id_1, + created_at: entryTimestamp, + updated_at: entryTimestamp, + } + }) + + return `${sql_file_string('entries', seed_entries)}` +} + +describe(cached_data_store, () => { + beforeEach(async () => { + cached_data = [] + await postgres.execute_query(reset_db_sql) + }) + + test('gets materialized_entries_view in batches and then gets rest from entries_view', async () => { + const materialized_count = 1500 + const recent_count = 20 + await postgres.execute_query(seed_with_entries({ offset: 0, count: materialized_count })) + await postgres.execute_query(refresh_materialized_view_sql) + await postgres.execute_query(seed_with_entries({ offset: materialized_count, count: recent_count })) + + const cache_key = `entries_${dictionary_id}` + + const store = cached_data_store({ + dictionary_id, + materialized_view: 'materialized_entries_view', + table: 'entries_view', + supabase: anon_supabase, + log, + }) + + expect(getStore(store.error)).toBe(null) + expect(getStore(store.loading)).toBeTruthy() + + await new Promise((r) => { + const unsub = store.loading.subscribe((loading) => { + if (!loading) { + r('loaded') + unsub() + } + }) + }) + const $store = getStore(store) + expect($store).toHaveLength(materialized_count + recent_count) + + const cached = get_idb(cache_key) + expect(cached).toEqual($store) + }) + + test('first gets cached items, then gets more recent from db starting with materialized_entries_view and then from entries_view', async () => { + const cache_key = `entries_${dictionary_id}` + const cached_count = 30 + const materialized_count = 1200 + const recent_count = 1100 + + await postgres.execute_query(seed_with_entries({ offset: 0, count: cached_count })) + + const store_to_set_up_cache = cached_data_store({ + dictionary_id, + table: 'entries_view', + supabase: anon_supabase, + log, + }) + await new Promise((r) => { + const unsub = store_to_set_up_cache.loading.subscribe((loading) => { + if (!loading) { + r('loaded') + unsub() + } + }) + }) + const cached = get_idb(cache_key) + expect(cached).toHaveLength(cached_count) + + await postgres.execute_query(seed_with_entries({ offset: cached_count, count: materialized_count })) + await postgres.execute_query(refresh_materialized_view_sql) + await postgres.execute_query(seed_with_entries({ offset: cached_count + materialized_count, count: recent_count })) + + const store = cached_data_store({ + dictionary_id, + materialized_view: 'materialized_entries_view', + table: 'entries_view', + supabase: anon_supabase, + log, + }) + + await new Promise((r) => { + const unsub = store.loading.subscribe((loading) => { + if (!loading) { + r('loaded') + unsub() + } + }) + }) + const $store = getStore(store) + expect($store).toHaveLength(cached_count + materialized_count + recent_count) + + const cached_2 = get_idb(cache_key) + expect(cached_2).toEqual($store) + }) + + test('logs errors properly', async () => { + const store = cached_data_store({ + dictionary_id, + // @ts-expect-error + table: 'not_real', + supabase: anon_supabase, + log, + }) + + await new Promise((r) => { + const unsub = store.loading.subscribe((loading) => { + if (!loading) { + r('loaded') + unsub() + } + }) + }) + expect(getStore(store.error)).toEqual(`relation "public.not_real" does not exist`) + }) + + test('updates an item already cached if changes are made to it', async () => { + const cached_count = 3 + const recent_count = 3 + await postgres.execute_query(seed_with_entries({ offset: 0, count: cached_count })) + + const store = cached_data_store({ + dictionary_id, + table: 'entries_view', + supabase: anon_supabase, + log, + }) + await new Promise((r) => { + const unsub = store.loading.subscribe((loading) => { + if (!loading) { + r('loaded') + unsub() + } + }) + }) + + await postgres.execute_query(seed_with_entries({ offset: cached_count, count: recent_count })) + + const id_to_update = incremental_consistent_uuid(1) + const phonetic = '[fu]' + const updated_entry: TablesUpdate<'entries'> = { + id: id_to_update, + phonetic, + updated_at: new Date(startTimestamp + 10000 * 1000).toISOString(), + } + await postgres.execute_query(sql_file_string('entries', [updated_entry], 'UPDATE')) + await store.refresh() + const $store = getStore(store) + const total_entries = cached_count + recent_count + expect($store).toHaveLength(total_entries) + const updated_index = $store.findIndex(entry => entry.id === id_to_update) + expect(updated_index).toEqual(total_entries - 1) + expect($store[updated_index].main.phonetic).toEqual(phonetic) + }) + + test('does not load down deleted items if there is nothing cached yet', async () => { + const initial_count = 2 + await postgres.execute_query(seed_with_entries({ offset: 0, count: initial_count })) + const id_to_update = incremental_consistent_uuid(1) + const timestamp = new Date(startTimestamp + 10000 * 1000).toISOString() + const updated_entry: TablesUpdate<'entries'> = { + id: id_to_update, + updated_at: timestamp, + deleted: timestamp, + } + await postgres.execute_query(sql_file_string('entries', [updated_entry], 'UPDATE')) + const store = cached_data_store({ + dictionary_id, + table: 'entries_view', + supabase: anon_supabase, + log, + }) + await new Promise((r) => { + const unsub = store.loading.subscribe((loading) => { + if (!loading) { + r('loaded') + unsub() + } + }) + }) + expect(getStore(store)).toHaveLength(initial_count - 1) + }) + + test('removes deleted items from the cache when they are deleted at a later date (without materialized view)', async () => { + const initial_count = 3 + await postgres.execute_query(seed_with_entries({ offset: 0, count: initial_count })) + + const store = cached_data_store({ + dictionary_id, + table: 'entries_view', + supabase: anon_supabase, + log, + }) + await new Promise((r) => { + const unsub = store.loading.subscribe((loading) => { + if (!loading) { + r('loaded') + unsub() + } + }) + }) + expect(getStore(store)).toHaveLength(initial_count) + + const id_to_update = incremental_consistent_uuid(1) + const timestamp = new Date(startTimestamp + 10000 * 1000).toISOString() + const updated_entry: TablesUpdate<'entries'> = { + id: id_to_update, + updated_at: timestamp, + deleted: timestamp, + } + await postgres.execute_query(sql_file_string('entries', [updated_entry], 'UPDATE')) + await store.refresh() + + expect(getStore(store)).toHaveLength(initial_count - 1) + }) + + test('removes deleted items from the cache when they are deleted at a later date (materialized view)', async () => { + const initial_count = 1 + await postgres.execute_query(seed_with_entries({ offset: 0, count: initial_count })) + await postgres.execute_query(refresh_materialized_view_sql) + + const store = cached_data_store({ + dictionary_id, + materialized_view: 'materialized_entries_view', + table: 'entries_view', + supabase: anon_supabase, + log, + }) + await new Promise((r) => { + const unsub = store.loading.subscribe((loading) => { + if (!loading) { + r('loaded') + unsub() + } + }) + }) + expect(getStore(store)).toHaveLength(initial_count) + + const id_to_update = incremental_consistent_uuid(0) + const timestamp = new Date(startTimestamp + 10000 * 1000).toISOString() + const updated_entry: TablesUpdate<'entries'> = { + id: id_to_update, + updated_at: timestamp, + deleted: timestamp, + } + await postgres.execute_query(sql_file_string('entries', [updated_entry], 'UPDATE')) + + const store2 = cached_data_store({ + dictionary_id, + materialized_view: 'materialized_entries_view', + table: 'entries_view', + supabase: anon_supabase, + log, + }) + await new Promise((r) => { + const unsub = store2.loading.subscribe((loading) => { + if (!loading) { + r('loaded') + unsub() + } + }) + }) + expect(getStore(store2)).toHaveLength(initial_count - 1) + + const store3 = cached_data_store({ + dictionary_id, + materialized_view: 'materialized_entries_view', + table: 'entries_view', + supabase: anon_supabase, + log, + }) + await new Promise((r) => { + const unsub = store3.loading.subscribe((loading) => { + if (!loading) { + r('loaded') + unsub() + } + }) + }) + expect(getStore(store3)).toHaveLength(initial_count - 1) + }) + + test('updated store emits new arrivals after store refresh', async () => { + const initial_count = 3 + await postgres.execute_query(seed_with_entries({ offset: 0, count: initial_count })) + + const store = cached_data_store({ + dictionary_id, + table: 'entries_view', + supabase: anon_supabase, + log, + }) + await new Promise((r) => { + const unsub = store.loading.subscribe((loading) => { + if (!loading) { + r('loaded') + unsub() + } + }) + }) + + const updated_items = [] + const unsub = store.updated_item.subscribe((item) => { + if (item) { + updated_items.push(item) + unsub() + } + }) + + const id_to_update = incremental_consistent_uuid(1) + const timestamp = new Date(startTimestamp + 10000 * 1000).toISOString() + const updated_entry: TablesUpdate<'entries'> = { + id: id_to_update, + updated_at: timestamp, + lexeme: { default: 'a change!' }, + } + await postgres.execute_query(sql_file_string('entries', [updated_entry], 'UPDATE')) + await store.refresh() + expect(updated_items).toHaveLength(1) + }) + + test('speakers_view', async () => { + const speakers: TablesInsert<'speakers'>[] = [{ + id: incremental_consistent_uuid(0), + dictionary_id, + name: 'Bob', + created_by: seeded_user_id_1, + updated_by: seeded_user_id_1, + created_at: new Date(startTimestamp).toISOString(), + updated_at: new Date(startTimestamp).toISOString(), + }] + await postgres.execute_query(`${sql_file_string('speakers', speakers)}`) + + const store = cached_data_store({ + dictionary_id, + table: 'speakers_view', + supabase: anon_supabase, + log, + }) + + await new Promise((r) => { + const unsub = store.loading.subscribe((loading) => { + if (!loading) { + r('loaded') + unsub() + } + }) + }) + + const $store = getStore(store) + expect($store).toMatchInlineSnapshot(` + [ + { + "birthplace": null, + "created_at": "1980-01-01T00:00:00+00:00", + "decade": null, + "deleted": null, + "dictionary_id": "dictionary1", + "gender": null, + "id": "22222222-2222-2222-2222-222222000000", + "name": "Bob", + "updated_at": "1980-01-01T00:00:00+00:00", + }, + ] + `) + + const { data: speakers_view } = await anon_supabase.from('speakers_view') + .select() + .limit(1000) + .order('updated_at', { ascending: true }) + .gt('updated_at', '1971-01-01T00:00:00Z') + expect(speakers_view).toEqual($store) + }) +}) + +// const { data: materialized_entries } = await anon_supabase.from('materialized_entries_view') +// .select() +// .limit(1000) +// .order('updated_at', { ascending: true }) +// .gt('updated_at', '1971-01-01T00:00:00Z') +// expect(materialized_entries).toMatchFileSnapshot('materialized_entries.snaps.json') diff --git a/packages/site/src/db-tests/clients.ts b/packages/site/src/db-tests/clients.ts index 8e0c43364..e92e9242d 100644 --- a/packages/site/src/db-tests/clients.ts +++ b/packages/site/src/db-tests/clients.ts @@ -1,5 +1,5 @@ -import type { Database } from '$lib/supabase/generated.types'; -import { createClient } from '@supabase/supabase-js'; +import type { Database } from '@living-dictionaries/types' +import { createClient } from '@supabase/supabase-js' // local keys from .env.development - ok to commit const PUBLIC_SUPABASE_API_URL = 'http://127.0.0.1:54321' diff --git a/packages/site/src/db-tests/content-update.test.ts b/packages/site/src/db-tests/content-update.test.bak similarity index 61% rename from packages/site/src/db-tests/content-update.test.ts rename to packages/site/src/db-tests/content-update.test.bak index 775ed4753..affae4233 100644 --- a/packages/site/src/db-tests/content-update.test.ts +++ b/packages/site/src/db-tests/content-update.test.bak @@ -1,10 +1,12 @@ +import type { ContentUpdateRequestBody } from '@living-dictionaries/types' import { admin_supabase, anon_supabase, uuid_template } from './clients' -import type { ContentUpdateRequestBody, ContentUpdateResponseBody } from '$api/db/content-update/+server' +import type { ContentUpdateResponseBody } from '$api/db/content-update/+server' import { post_request } from '$lib/helpers/get-post-requests' import { first_entry_id, seeded_dictionary_id, seeded_user_id_1, seeded_user_id_2 } from '$lib/mocks/seed/tables' import { reset_db } from '$lib/mocks/seed/write-seed-and-reset-db' const content_update_endpoint = 'http://localhost:3041/api/db/content-update' +const test_timestamp = new Date('2024-03-08T00:44:04.600392+00:00').toISOString() beforeAll(async () => { await reset_db() @@ -22,78 +24,68 @@ const first_entry_third_sense_id = incremental_consistent_uuid() describe('sense operations', () => { test('add sense with noun class to first entry', async () => { const { error } = await post_request(content_update_endpoint, { - id: incremental_consistent_uuid(), + update_id: incremental_consistent_uuid(), auth_token: null, - user_id_from_local: seeded_user_id_1, + import_meta: { + user_id: seeded_user_id_1, + timestamp: test_timestamp, + }, dictionary_id: seeded_dictionary_id, entry_id: first_entry_id, sense_id: first_entry_first_sense_id, - table: 'senses', - change: { - sense: { - noun_class: { - new: '2', - }, - }, + type: 'insert_sense', + data: { + noun_class: '2', }, - timestamp: new Date('2024-03-08T00:44:04.600392+00:00').toISOString(), }) expect(error?.message).toBeFalsy() const { data } = await anon_supabase.from('entries_view').select() - expect(data).toMatchInlineSnapshot(` - [ - { - "id": "entry1", - "senses": [ - { - "id": "11111111-1111-1111-1111-111111111100", - "noun_class": "2", - }, - ], - }, - ] + const [first_entry] = data + expect({ id: first_entry.id, senses: first_entry.senses }).toMatchInlineSnapshot(` + { + "id": "entry1", + "senses": [ + { + "id": "11111111-1111-1111-1111-111111111100", + "noun_class": "2", + }, + ], + } `) }) describe('different user add parts of speech to first sense in first entry', () => { - test('noun class remains (upsert)', async () => { - const { error } = await post_request(content_update_endpoint, { - id: incremental_consistent_uuid(), - auth_token: null, - user_id_from_local: seeded_user_id_2, - dictionary_id: seeded_dictionary_id, - entry_id: first_entry_id, - sense_id: first_entry_first_sense_id, - table: 'senses', - change: { - sense: { - parts_of_speech: { - new: ['n', 'v'], - }, - }, - }, - timestamp: new Date('2024-03-08T00:44:04.600392+00:00').toISOString(), - }) - expect(error?.message).toBeFalsy() - const { data } = await anon_supabase.from('entries_view').select() - expect(data).toMatchInlineSnapshot(` - [ - { - "id": "entry1", - "senses": [ - { - "id": "11111111-1111-1111-1111-111111111100", - "noun_class": "2", - "parts_of_speech": [ - "n", - "v", - ], - }, - ], - }, - ] - `) - }) + // test.fails('noun class remains (upsert)', async () => { + // const { error } = await post_request(content_update_endpoint, { + // update_id: incremental_consistent_uuid(), + // auth_token: null, + // import_meta: { + // user_id: seeded_user_id_2, + // timestamp: test_timestamp, + // }, + // dictionary_id: seeded_dictionary_id, + // entry_id: first_entry_id, + // sense_id: first_entry_first_sense_id, + // type: 'insert_sense', + // data: { + // parts_of_speech: ['n', 'v'], + // }, + // }) + // expect(error?.message).toBeFalsy() + // const { data } = await anon_supabase.from('entries_view').select() + // expect(data[0].senses).toMatchInlineSnapshot(` + // [ + // { + // "id": "11111111-1111-1111-1111-111111111100", + // "noun_class": "2", + // "parts_of_speech": [ + // "n", + // "v", + // ], + // }, + // ] + // `) + // }) test('updated_by is set to the second user but created_by is left alone', async () => { const { data } = await admin_supabase.from('senses').select().eq('id', first_entry_first_sense_id).single() @@ -104,101 +96,85 @@ describe('sense operations', () => { test('adds glosses field to second sense in first entry', async () => { const { error } = await post_request(content_update_endpoint, { - id: incremental_consistent_uuid(), + update_id: incremental_consistent_uuid(), auth_token: null, - user_id_from_local: seeded_user_id_1, + import_meta: { + user_id: seeded_user_id_1, + timestamp: test_timestamp, + }, dictionary_id: seeded_dictionary_id, entry_id: first_entry_id, sense_id: first_entry_second_sense_id, - table: 'senses', - change: { - sense: { - glosses: { - new: { - en: 'Hi', - es: 'Hola', - }, - }, + type: 'insert_sense', + data: { + glosses: { + en: 'Hi', + es: 'Hola', }, }, - timestamp: new Date('2024-03-08T00:44:04.600392+00:00').toISOString(), }) expect(error?.message).toBeFalsy() const { data } = await anon_supabase.from('entries_view').select() - expect(data).toMatchInlineSnapshot(` + expect(data[0].senses).toMatchInlineSnapshot(` [ { - "id": "entry1", - "senses": [ - { - "id": "11111111-1111-1111-1111-111111111100", - "noun_class": "2", - "parts_of_speech": [ - "n", - "v", - ], - }, - { - "glosses": { - "en": "Hi", - "es": "Hola", - }, - "id": "11111111-1111-1111-1111-111111111101", - }, + "id": "11111111-1111-1111-1111-111111111100", + "noun_class": "2", + "parts_of_speech": [ + "n", + "v", ], }, + { + "glosses": { + "en": "Hi", + "es": "Hola", + }, + "id": "11111111-1111-1111-1111-111111111101", + }, ] `) }) test('add a third sense to first entry with a glosses field', async () => { const { error } = await post_request(content_update_endpoint, { - id: incremental_consistent_uuid(), + update_id: incremental_consistent_uuid(), auth_token: null, user_id_from_local: seeded_user_id_1, dictionary_id: seeded_dictionary_id, entry_id: first_entry_id, sense_id: first_entry_third_sense_id, - table: 'senses', - change: { - sense: { - semantic_domains: { - new: ['1', '2'], - }, - }, + type: 'insert_sense', + data: { + semantic_domains: ['1', '2'], }, - timestamp: new Date('2024-03-08T00:44:04.600392+00:00').toISOString(), + test_timestamp, }) expect(error?.message).toBeFalsy() const { data } = await anon_supabase.from('entries_view').select() - expect(data).toMatchInlineSnapshot(` + expect(data[0].senses).toMatchInlineSnapshot(` [ { - "id": "entry1", - "senses": [ - { - "id": "11111111-1111-1111-1111-111111111100", - "noun_class": "2", - "parts_of_speech": [ - "n", - "v", - ], - }, - { - "glosses": { - "en": "Hi", - "es": "Hola", - }, - "id": "11111111-1111-1111-1111-111111111101", - }, - { - "id": "11111111-1111-1111-1111-111111111102", - "semantic_domains": [ - "1", - "2", - ], - }, + "id": "11111111-1111-1111-1111-111111111100", + "noun_class": "2", + "parts_of_speech": [ + "n", + "v", + ], + }, + { + "glosses": { + "en": "Hi", + "es": "Hola", + }, + "id": "11111111-1111-1111-1111-111111111101", + }, + { + "id": "11111111-1111-1111-1111-111111111102", + "semantic_domains": [ + "1", + "2", ], }, ] @@ -207,44 +183,37 @@ describe('sense operations', () => { test('delete the third sense from the first entry', async () => { const { error } = await post_request(content_update_endpoint, { - id: incremental_consistent_uuid(), + update_id: incremental_consistent_uuid(), auth_token: null, user_id_from_local: seeded_user_id_1, dictionary_id: seeded_dictionary_id, entry_id: first_entry_id, sense_id: first_entry_third_sense_id, - table: 'senses', - change: { - sense: { - deleted: true, - }, + type: 'insert_sense', + data: { + deleted: 'true', }, - timestamp: new Date('2024-03-08T00:44:04.600392+00:00').toISOString(), + test_timestamp, }) expect(error?.message).toBeFalsy() const { data } = await anon_supabase.from('entries_view').select() - expect(data).toMatchInlineSnapshot(` + expect(data[0].senses).toMatchInlineSnapshot(` [ { - "id": "entry1", - "senses": [ - { - "id": "11111111-1111-1111-1111-111111111100", - "noun_class": "2", - "parts_of_speech": [ - "n", - "v", - ], - }, - { - "glosses": { - "en": "Hi", - "es": "Hola", - }, - "id": "11111111-1111-1111-1111-111111111101", - }, + "id": "11111111-1111-1111-1111-111111111100", + "noun_class": "2", + "parts_of_speech": [ + "n", + "v", ], }, + { + "glosses": { + "en": "Hi", + "es": "Hola", + }, + "id": "11111111-1111-1111-1111-111111111101", + }, ] `) }) @@ -258,24 +227,19 @@ describe('sense sentence operations', () => { test('post to endpoint', async () => { const { data, error } = await post_request(content_update_endpoint, { - id: change_id, + update_id: change_id, auth_token: null, user_id_from_local: seeded_user_id_1, dictionary_id: seeded_dictionary_id, - entry_id: first_entry_id, sentence_id: first_sentence_id, sense_id: first_entry_first_sense_id, - table: 'sentences', - change: { - sentence: { - text: { - new: { - lo1: 'abcd efgh ijkl', - }, - }, + type: 'insert_sentence', + data: { + text: { + lo1: 'abcd efgh ijkl', }, }, - timestamp: new Date('2024-03-08T00:44:04.600392+00:00').toISOString(), + test_timestamp, }) expect(error?.message).toBeFalsy() @@ -283,23 +247,23 @@ describe('sense sentence operations', () => { { "audio_id": null, "change": { - "sentence": { + "data": { "text": { - "new": { - "lo1": "abcd efgh ijkl", - }, + "lo1": "abcd efgh ijkl", }, }, + "type": "add_sentence", }, + "dialect_id": null, "dictionary_id": "dictionary1", - "entry_id": "entry1", + "entry_id": null, "id": "11111111-1111-1111-1111-111111111104", "import_id": null, "photo_id": null, "sense_id": "11111111-1111-1111-1111-111111111100", "sentence_id": "11111111-1111-1111-1111-111111111103", "speaker_id": null, - "table": "sentences", + "table": null, "text_id": null, "timestamp": "2024-03-08T00:44:04.6+00:00", "user_id": "12345678-abcd-efab-cdef-123456789012", @@ -316,23 +280,23 @@ describe('sense sentence operations', () => { { "audio_id": null, "change": { - "sentence": { + "data": { "text": { - "new": { - "lo1": "abcd efgh ijkl", - }, + "lo1": "abcd efgh ijkl", }, }, + "type": "add_sentence", }, + "dialect_id": null, "dictionary_id": "dictionary1", - "entry_id": "entry1", + "entry_id": null, "id": "11111111-1111-1111-1111-111111111104", "import_id": null, "photo_id": null, "sense_id": "11111111-1111-1111-1111-111111111100", "sentence_id": "11111111-1111-1111-1111-111111111103", "speaker_id": null, - "table": "sentences", + "table": null, "text_id": null, "timestamp": "2024-03-08T00:44:04.6+00:00", "user_id": "12345678-abcd-efab-cdef-123456789012", @@ -355,38 +319,6 @@ describe('sense sentence operations', () => { }, ] `) - expect(data).toMatchInlineSnapshot(` - [ - { - "id": "entry1", - "senses": [ - { - "id": "11111111-1111-1111-1111-111111111100", - "noun_class": "2", - "parts_of_speech": [ - "n", - "v", - ], - "sentences": [ - { - "id": "11111111-1111-1111-1111-111111111103", - "text": { - "lo1": "abcd efgh ijkl", - }, - }, - ], - }, - { - "glosses": { - "en": "Hi", - "es": "Hola", - }, - "id": "11111111-1111-1111-1111-111111111101", - }, - ], - }, - ] - `) }) }) @@ -395,24 +327,18 @@ describe('sense sentence operations', () => { test('post to endpoint', async () => { const { data, error } = await post_request(content_update_endpoint, { - id: change_id, + update_id: change_id, auth_token: null, user_id_from_local: seeded_user_id_1, dictionary_id: seeded_dictionary_id, - entry_id: first_entry_id, sentence_id: first_sentence_id, - sense_id: first_entry_first_sense_id, - table: 'sentences', - change: { - sentence: { - translation: { - new: { - en: 'I am hungry', - }, - }, + type: 'update_sentence', + data: { + translation: { + en: 'I am hungry', }, }, - timestamp: new Date('2024-03-09T00:44:04.600392+00:00').toISOString(), + test_timestamp: new Date('2024-03-09T00:44:04.600392+00:00').toISOString(), }) expect(error?.message).toBeFalsy() @@ -420,23 +346,23 @@ describe('sense sentence operations', () => { { "audio_id": null, "change": { - "sentence": { + "data": { "translation": { - "new": { - "en": "I am hungry", - }, + "en": "I am hungry", }, }, + "type": "update_sentence", }, + "dialect_id": null, "dictionary_id": "dictionary1", - "entry_id": "entry1", + "entry_id": null, "id": "11111111-1111-1111-1111-111111111105", "import_id": null, "photo_id": null, - "sense_id": "11111111-1111-1111-1111-111111111100", + "sense_id": null, "sentence_id": "11111111-1111-1111-1111-111111111103", "speaker_id": null, - "table": "sentences", + "table": null, "text_id": null, "timestamp": "2024-03-09T00:44:04.6+00:00", "user_id": "12345678-abcd-efab-cdef-123456789012", @@ -473,26 +399,18 @@ describe('sense sentence operations', () => { test('update sentence text updates just the text and leaves translation alone', async () => { const change_id = incremental_consistent_uuid() await post_request(content_update_endpoint, { - id: change_id, + update_id: change_id, auth_token: null, user_id_from_local: seeded_user_id_1, dictionary_id: seeded_dictionary_id, sentence_id: first_sentence_id, - sense_id: first_entry_first_sense_id, - table: 'sentences', - change: { - sentence: { - text: { - new: { - lo1: 'abcd efgh', - }, - old: { - lo1: 'abcd efgh ijkl', - }, - }, + type: 'update_sentence', + data: { + text: { + lo1: 'abcd efgh', }, }, - timestamp: new Date('2024-03-09T00:44:04.600392+00:00').toISOString(), + test_timestamp: new Date('2024-03-09T00:44:04.600392+00:00').toISOString(), }) const { data: { senses }, error } = await anon_supabase.from('entries_view').select().eq('id', first_entry_id).single() @@ -516,37 +434,31 @@ describe('sense sentence operations', () => { const { data: { senses: old_senses } } = await anon_supabase.from('entries_view').select().eq('id', first_entry_id).single() const change_id = incremental_consistent_uuid() const { data, error } = await post_request(content_update_endpoint, { - id: change_id, + update_id: change_id, auth_token: null, user_id_from_local: seeded_user_id_1, dictionary_id: seeded_dictionary_id, sentence_id: first_sentence_id, - sense_id: first_entry_first_sense_id, - table: 'sentences', - change: { - sentence: { - translation: { - new: { - ...old_senses[0].sentences[0].translation, - es: 'Estoy hambriento', - }, - }, + type: 'update_sentence', + data: { + translation: { + ...old_senses[0].sentences[0].translation, + es: 'Estoy hambriento', }, }, - timestamp: new Date('2024-03-09T00:44:04.600392+00:00').toISOString(), + test_timestamp: new Date('2024-03-09T00:44:04.600392+00:00').toISOString(), }) expect(error?.message).toBeFalsy() expect(data.change).toMatchInlineSnapshot(` { - "sentence": { + "data": { "translation": { - "new": { - "en": "I am hungry", - "es": "Estoy hambriento", - }, + "en": "I am hungry", + "es": "Estoy hambriento", }, }, + "type": "update_sentence", } `) }) @@ -571,19 +483,14 @@ describe('sense sentence operations', () => { test('remove sentence from sense', async () => { const { error } = await post_request(content_update_endpoint, { - id: incremental_consistent_uuid(), + update_id: incremental_consistent_uuid(), auth_token: null, user_id_from_local: seeded_user_id_1, dictionary_id: seeded_dictionary_id, sentence_id: first_sentence_id, sense_id: first_entry_first_sense_id, - table: 'sentences', - change: { - sentence: { - removed_from_sense: true, - }, - }, - timestamp: new Date('2024-03-08T00:44:04.600392+00:00').toISOString(), + type: 'remove_sentence', + test_timestamp, }) expect(error?.message).toBeFalsy() diff --git a/packages/site/src/db-tests/users.test.ts b/packages/site/src/db-tests/users.test.ts index 2c56f5ac3..4a1c398dd 100644 --- a/packages/site/src/db-tests/users.test.ts +++ b/packages/site/src/db-tests/users.test.ts @@ -1,20 +1,19 @@ -import { seed_user_email_1, seeded_user_id_1 } from '$lib/mocks/seed/tables'; -import { reset_db } from '$lib/mocks/seed/write-seed-and-reset-db'; -import { admin_supabase, anon_supabase } from './clients'; - -beforeAll(async () => { - await reset_db() -}) +import { admin_supabase, anon_supabase } from './clients' +import { seed_user_email_1, seeded_user_id_1 } from '$lib/mocks/seed/tables' +import { reset_db } from '$lib/mocks/seed/write-seed-and-reset-db' describe('users table access', () => { + beforeEach(async () => { + await reset_db() + }) + test('admin can check user emails', async () => { const { data } = await admin_supabase.from('user_emails').select().eq('id', seeded_user_id_1).single() - expect(data.email).toEqual(seed_user_email_1); - }); + expect(data.email).toEqual(seed_user_email_1) + }) test('normal users cannot check user emails', async () => { const { data } = await anon_supabase.from('user_emails').select().eq('id', seeded_user_id_1).single() - expect(data).toBeNull(); - }); -}); - + expect(data).toBeNull() + }) +}) diff --git a/packages/site/src/docs/CONTRIBUTING.md b/packages/site/src/docs/CONTRIBUTING.md index 87f27ae53..91d2c63a3 100644 --- a/packages/site/src/docs/CONTRIBUTING.md +++ b/packages/site/src/docs/CONTRIBUTING.md @@ -3,15 +3,12 @@ ## Set Up Dev Environment - Install the recommended VSCode extensions. -- Install [pnpm](https://pnpm.io/) globaly using `npm install -g pnpm` if you don't have it yet. +- Install [pnpm](https://pnpm.io/) globaly using `npm install -g pnpm` if you don't have it yet. - Then install dependencies with `pnpm i` and run `pnpm dev`. Each command will start up a Vite dev server which will give you a link to open on localhost. Changes will hot reload almost instantly (consider using auto-save if you want to). - -Note that you will need to ask for our dev Mapbox and Firebase API keys or bring your own by adding `PUBLIC_mapboxAccessToken=...` and -`PUBLIC_FIREBASE_CONFIG=...` to a `.env.local` file on the root level. +...notes outdated and incomplete \*_Note that on localhost you will not see the live (prod) site's data, but rather the data from the dev database, which allows us to develop and make changes freely without worrying about deleting or corrupting important data._ - ## Git methodology We follow [Github flow](https://guides.github.com/introduction/flow/). Changes are made on feature branches and then pull requests are submitted to the main branch. The main branch code is automatically deployed to production. Be sure to read through the GitHub flow article as it gives important information on how often to commit. diff --git a/packages/site/src/docs/Supabase.md b/packages/site/src/docs/Supabase.md index ff385f96b..15430d033 100644 --- a/packages/site/src/docs/Supabase.md +++ b/packages/site/src/docs/Supabase.md @@ -23,7 +23,12 @@ Once you have run `supabase start` you can open the Studio URL to explore your l ## Generate Types -- `supabase gen types typescript --local --schema public > packages/site/src/lib/supabase/generated.types.ts` +Local: +- `pnpm generate-types` to run `supabase gen types typescript --local --schema public > packages/types/supabase/generated.types.ts` +- save the file to have lint fix auto-run +- `pnpm t merge-types` to merge the generated types with the manually written types + +Deployed (we don't use this): - `supabase gen types typescript --project-id=actkqboqpzniojhgtqzw --schema public > packages/site/src/lib/supabase/generated.types.ts` ## Tests @@ -43,3 +48,38 @@ You can check current prod migrations at https://supabase.com/dashboard/project/ ## Misc - `supabase status` check status and get local urls + +## Use data from a `pg_dump` backup locally + +These four commands are run daily to backup the production database: +- `supabase db dump --db-url "$supabase_db_url" -f roles.sql --role-only` +- `supabase db dump --db-url "$supabase_db_url" -f schema.sql` +- `supabase db dump --db-url "$supabase_db_url" -f data-copy.sql --data-only --use-copy` +- `supabase db dump --db-url "$supabase_db_url" -f data-insert.sql --data-only` + +To make the local DB match the current production download just download the data as the schema already matches production (or is a step ahead): +- Get the DB url by pasting the password into here: postgresql://postgres.actkqboqpzniojhgtqzw:[YOUR-PASSWORD]@aws-0-us-west-1.pooler.supabase.com:6543/postgres +- Run `supabase db dump --db-url "your-db-url" -f supabase/seed.sql --data-only` but using the db url above +- Remove the `is_anonymous` column in auth.users and find-replace `, false),` for `),` (and don't forget the last row with a semi-colon) because `is_anonymous` doesn't exist in local db. Remove the audit_log block if using a reset sql script instead of fully resetting the db. +- Run `supabase db reset` to build the db with the production data + +## Other `pg_dump` notes that did not pan out but may be useful in other situations + +- Read how to migrate a project: https://supabase.com/docs/guides/platform/migrating-and-upgrading-projects#migrate-your-project +- The dump file produced by pg_dump does not contain the statistics used by the optimizer to make query planning decisions. Therefore, it is wise to run ANALYZE after restoring from a dump file to ensure good performance. (source: https://www.postgresql.org/docs/8.0/app-pgdump.html) `psql -d 'postgres://supabase_admin:postgres@127.0.0.1:54322/postgres' -c 'ANALYZE;'` +- comment out `COPY "auth"."flow_state"...` block +- reset db with no migrations + +psql \ + --single-transaction \ + --file roles.sql \ + --file schema.sql \ + --command 'SET session_replication_role = replica' \ + --file data.sql \ + --dbname "postgresql://postgres:postgres@127.0.0.1:54322/postgres" + +Other options + --variable ON_ERROR_STOP=1 \ - not using because of syntax errors + +- `psql postgresql://postgres:postgres@127.0.0.1:54322/postgres -f live-db-dump.sql` +- `psql -d database -f data.sql` to restore the data from a dump file obtained from Supabase's automatic backup. -d database: Specifies the name of the database to connect to. diff --git a/packages/site/src/docs/data/new-entry-field-creation.md b/packages/site/src/docs/data/new-entry-field-creation.md index 5e760c220..61ad7eb4e 100644 --- a/packages/site/src/docs/data/new-entry-field-creation.md +++ b/packages/site/src/docs/data/new-entry-field-creation.md @@ -14,6 +14,6 @@ These are the main steps to follow if you need to add a new entry field: 6. Add the new field in `packages/site/src/routes/[dictionaryId]/export/_formatEntries.ts` in the `headers` object and in the `itemsFormatted` array. -7. Consider whether the new field needs added to the dictionary import spreadsheet template. If so add to the latest spreadsheet template. Then update the example import csv (`packages/scripts/import/data/example/example.csv`) then in `packages/scripts` run `pnpm test` to run tests while you add the needed code in `packages/scripts/import/convertJsonRowToEntryFormat.ts` to import the new field. You are finished when the test results include the new field and nothing is broken. +7. Consider whether the new field needs added to the dictionary import spreadsheet template. If so add to the latest spreadsheet template. Then update the example import csv (`packages/scripts/import/data/example/example.csv`) then in `packages/scripts` run `pnpm test` to run tests while you add the needed code in `packages/scripts/import/convertJsonRowToEntryFormat.ts` to import the new field. You are finished when the test results include the new field and nothing is broken. -8. Ask if a new filter is needed for this new field. If so, add to `packages/scripts/algolia/prepareDataForIndex.ts` and deploy the updated cloud functions that use that function (add, update, and delete). Then update Algolia to be aware of the new field and add a search filter to the front end if appropriate (e.g. add a ToggleRefinement component for `hasPluralForm` in `packages\site\src\routes\[dictionaryId]\entries\_EntryFilters.svelte`) +8. Ask if a new filter is needed for this new field. If so, add to Orama code and add a search filter to the front end if appropriate (e.g. add a ToggleRefinement component for `hasPluralForm` in `packages\site\src\routes\[dictionaryId]\entries\_EntryFilters.svelte`) diff --git a/packages/site/src/docs/misc/entry-fields.md b/packages/site/src/docs/misc/entry-fields.md index db599192c..fd0b888a2 100644 --- a/packages/site/src/docs/misc/entry-fields.md +++ b/packages/site/src/docs/misc/entry-fields.md @@ -4,12 +4,12 @@ ### Glosses containing scientific names -In these cases, when scientific names are italicized by convention we need to create a specific field for scientific names, such as many entries in [Birhor's Plants domain](https://livingdictionaries.app/birhor/entries/list?entries_prod%5BrefinementList%5D%5Bsdn%5D%5B0%5D=1.4) +In these cases, when scientific names are italicized by convention we need to create a specific field for scientific names, such as many entries in [Birhor's Plants domain](https://livingdictionaries.app/birhor/entries) ## Interlinearization - Consider researching more about [Leipzig glossing](https://ctan.org/pkg/leipzig?lang=en) - Need to be able to support small-caps, used and [Unicode Lookup](https://unicode.emnudge.dev/) to find characters. -- Can be done via rich-text as in `3sg.pres` or via specific small capitals unicode letters like done in [Toggle SmallCaps](https://svelte.dev/repl/231af5758d6b484dbbee7827b0b95540?version=3.46.4) to come up with "3sg.ᴘʀᴇꜱ" these can actually be typed/pasted into this text box that text doesn't support smallCaps in the rich-text sort of manner. The benefit of doing this via unicode is that we can export dictionaries with the smallCaps intact, as well as people can copy paste into and out of the site and keep the small caps intact. This is a topic to research more on as small caps q is not contained in common fonts and may need a font added. https://en.wikipedia.org/wiki/Small_caps and https://yaytext.com/small-caps/#preview_small-caps are good resources. See also https://developer.mozilla.org/en-US/docs/Web/CSS/font-variant-caps +- Can be done via rich-text as in `3sg.pres` or via specific small capitals unicode letters like done in [Toggle SmallCaps](https://svelte.dev/repl/231af5758d6b484dbbee7827b0b95540?version=3.46.4) to come up with "3sg.ᴘʀᴇꜱ" these can actually be typed/pasted into this text box that text doesn't support smallCaps in the rich-text sort of manner. The benefit of doing this via unicode is that we can export dictionaries with the smallCaps intact, as well as people can copy paste into and out of the site and keep the small caps intact. This is a topic to research more on as small caps q is not contained in common fonts and may need a font added. https://en.wikipedia.org/wiki/Small_caps and https://yaytext.com/small-caps/#preview_small-caps are good resources. See also https://developer.mozilla.org/en-US/docs/Web/CSS/font-variant-caps ?? What about other rich-text features? @@ -24,4 +24,4 @@ Is there a need for rich-text and which ones? If so use [QuillJs](https://quillj ### Scientific Name -- for Latin names that are normally italicized - do we need to support italics as optional here using *italics* or not - if so we can do them using something like https://svelte.dev/examples/textarea-inputs \ No newline at end of file +- for Latin names that are normally italicized - do we need to support italics as optional here using *italics* or not - if so we can do them using something like https://svelte.dev/examples/textarea-inputs diff --git a/packages/site/src/docs/misc/import-dictionary.md b/packages/site/src/docs/misc/import-dictionary.md index 7027f075b..10be69bdb 100644 --- a/packages/site/src/docs/misc/import-dictionary.md +++ b/packages/site/src/docs/misc/import-dictionary.md @@ -12,7 +12,7 @@ - Run `pnpm importDictionary -- --id example --dry` if you want to test things are working - Start by running `pnpm importDictionary -- --id kalinago --dry` but use your appropriate dictionary id instead. This will emulate (dry run) an import to dev environment to let you inspect the log and see if there are any missing files - Check the console log or the outputted log file found in `packages/scripts/logs` - it will be the newest one as they are saved by datestamp. -- Rerun your script with the `--dry` option: `pnpm importDictionary -- --id kalinago` and inspect the dev-imported dictionary on localhost (or any deployed dev url) to make sure all is good - Note that because we don't have Algolia indexing the dev database currently, I added some code that let's dev show the 10 most recent entries from Firestore, but that's all you'll get for now until we make further changes - and only 10 from the last 60 minutes (see `Hits.svelte`). +- Rerun your script with the `--dry` option: `pnpm importDictionary -- --id kalinago` and inspect the dev-imported dictionary on localhost (or any deployed dev url) to make sure all is good. - Before moving on to prod, delete all media that has been imported to dev DB. - If all looks good then run `pnpm importDictionary -- --id kalinago --environment prod` to push the data live - Look through the imported dictionary and then tell Anna to look at it and make it public if everyone is happy with it. diff --git a/packages/site/src/docs/misc/power-search.md b/packages/site/src/docs/misc/power-search.md deleted file mode 100644 index 5997b956b..000000000 --- a/packages/site/src/docs/misc/power-search.md +++ /dev/null @@ -1,23 +0,0 @@ -# Power Search - -There are a few features in the pipeline and a few pain points that would all be solved by a better search system: -- API -- No advanced search ability with Algolia, for example, search just the start or end of lexemes, search for minimal pairs, use wildcards, etc... (phoneme analysis features, etc...) -- Sort by any field -- Offline search (and usage), server-side search - -Three possible solutions would be: - -## 1 -- Index all dictionaries up-front (and when adding features like a "hasAudio" flag) -- Update the current indexing function that listens to document writes to download the proper csv for that dictionary, update the row by entry id and then save the csv back to the cloud -- Con: Race conditions, possible to get around this by writing each value to a KV store and then updating the overall CSV from there - sounds over-engineered - -## 2 -- Run a persistent server (App Engine or other?) with firebase client SDK (firebase-admin doesn't cache reads) running that is subscribed to each dictionary's words collection and on changes, writes over the cloud csv for that dictionary. -- Con: this sounds fraught with the chance that Firebase on that server will crash as the client SDK wasn't built to be subscribed to 150K documents at once nor do I trust it's cache to be 100% reliable at that scale. - -## 3 -- Load collection of entries from Firestore, but perform searching in worker thread to avoid jank we previously experienced using this method. -- If the collection is to heavy, then create a parallel collection in Firestore that's lighter weight w/o data not needed for search - this will nix the ability to do offline edits so perhaps not so good - perhaps another solution if entries are too heavy is to pull out some data from the main entry body into sub documents if just needed for the entry view as a manager sees it, not for first load. -- Con: must wait for all entries to come down from Firestore before allowing search (just on the first time through as they will be cached after that), this can be mitigated by having a dictionary landing page which loads entries in the background. Another possible con is costs, for every 5,000 new browsers searching entries from a dictionary averaging 2,000 entries in size, that will cost us $6. $6 per 5,000 new dictionary visits (not return visitors) isn't so bad. Also because of the free tier we get the first 25 visits/day for free, adding up to 750 free new visitors a month. (Based on $0.06/100,000 reads = $6 per 10 million reads) \ No newline at end of file diff --git a/packages/site/src/lib/components/SeoMetaTags.svelte b/packages/site/src/lib/components/SeoMetaTags.svelte index cc841fc10..9f5065146 100644 --- a/packages/site/src/lib/components/SeoMetaTags.svelte +++ b/packages/site/src/lib/components/SeoMetaTags.svelte @@ -1,10 +1,8 @@ - {entry.lexeme} + {entry.main.lexeme.default} - {#if sound_file?.speakerName} -
- {$page.data.t('entry_field.speaker')}: - {sound_file.speakerName} -
- - {:else} - - {#if sound_file} -
- -
- {:else if speakerId} - {#if file || audioBlob} - {#if file} - - {:else} - - {/if} -
- {#if file || audioBlob} - {@const upload_status = dbOperations.addAudio({ file: file || audioBlob, entryId: entry.id, speakerId })} - {#await import('$lib/components/audio/UploadAudioStatus.svelte') then { default: UploadAudioStatus }} - - {/await} - {/if} + + {#if sound_file} +
+ +
+ {:else if speaker_id} + {#if file || audioBlob} + {#if file} + {:else} -
-
- -
- {#if !readyToRecord} - - {/if} -
+ {/if} +
+ {#if file || audioBlob} + {@const upload_status = dbOperations.addAudio({ file: file || audioBlob, entry_id: entry.id, speaker_id })} + {#await import('$lib/components/audio/UploadProgressBarStatus.svelte') then { default: UploadProgressBarStatus }} + + {/await} + {/if} + {:else} +
+
+ +
+ {#if !readyToRecord} + + {/if} +
{/if} - - {/if} + {/if} +