-
-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* update vite and vitest packages * switch orama to main route * fix table header being over-displayed by ear icons and image icons * improve entry modal * remove algolia * add supabase notes on how to seed local db from production * move generated types; add dialects table * merge Supabase types via script * connect photos and videos to senses * connect dialect_ids * connect speaker_ids to audio and videos * speakers * use speakers view * create hourly refreshing materialized entries view * keep photos, videos, speakers, dialects (many-many items) out of entries_view * add in batching ability * cached-data-store * use rpc for entries view * handle entry store errors better ... and many other tasks needed for migrating entries from Firebase to Supabase
- Loading branch information
Showing
271 changed files
with
15,270 additions
and
13,713 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,6 +1,6 @@ | ||
| { | ||
| "projects": { | ||
| "default": "talking-dictionaries-dev", | ||
| "production": "talking-dictionaries-alpha" | ||
| "production": "talking-dictionaries-alpha", | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2,4 +2,5 @@ logs | |
| service-account* | ||
| .env | ||
| sheets-viewer-SA.json | ||
| .env.supabase | ||
| .env.supabase | ||
| .env.production.supabase | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,31 +1,63 @@ | ||
| import PG from 'pg' | ||
| import { createClient } from '@supabase/supabase-js' | ||
| import type { Database } from '@living-dictionaries/site/src/lib/supabase/database.types' | ||
| import type { Database } from '@living-dictionaries/types' | ||
| import * as dotenv from 'dotenv' | ||
| import './record-logs' | ||
|
|
||
| dotenv.config({ path: '.env.supabase' }) | ||
| // TODO: change to .env.development and .env.production | ||
| dotenv.config({ path: '.env.supabase' }) // local project variables | ||
| // dotenv.config({ path: '.env.production.supabase' }) // production project variables | ||
|
|
||
| export const supabase = createClient<Database>(process.env.PUBLIC_SUPABASE_API_URL, process.env.SUPABASE_SERVICE_ROLE_KEY) | ||
| export const admin_supabase = createClient<Database>(process.env.PUBLIC_SUPABASE_API_URL, process.env.SUPABASE_SERVICE_ROLE_KEY) | ||
| export const anon_supabase = createClient<Database>(process.env.PUBLIC_SUPABASE_API_URL, process.env.PUBLIC_SUPABASE_ANON_KEY) | ||
| export const jacob_ld_user_id = 'de2d3715-6337-45a3-a81a-d82c3210b2a7' | ||
|
|
||
| export async function executeQuery(query: string) { | ||
| const client = new PG.Client({ | ||
| class DB { | ||
| private pool: PG.Pool | ||
|
|
||
| private config: PG.PoolConfig = { | ||
| user: 'postgres', | ||
| host: '127.0.0.1', | ||
| // host: 'db.actkqboqpzniojhgtqzw.supabase.co', | ||
| database: 'postgres', | ||
| password: 'postgres', | ||
| // password: '**', | ||
| port: 54322, | ||
| // port: 5432, | ||
| }) | ||
| try { | ||
| await client.connect() | ||
| await client.query(query) | ||
| } catch (error) { | ||
| console.error('Error in connection/executing query:', error) | ||
| } finally { | ||
| await client.end().catch((error) => { | ||
| console.error('Error ending client connection:', error) | ||
| }) | ||
|
|
||
| // user: 'postgres.actkqboqpzniojhgtqzw', | ||
| // host: 'aws-0-us-west-1.pooler.supabase.com', | ||
| // database: 'postgres', | ||
| // password: '**', | ||
| // port: 6543, | ||
|
|
||
| max: 10, | ||
| idleTimeoutMillis: 30000, | ||
| connectionTimeoutMillis: 5000, | ||
| allowExitOnIdle: false, | ||
| } | ||
|
|
||
| async get_db_connection(): Promise<PG.PoolClient> { | ||
| if (!this.pool) { | ||
| this.pool = new PG.Pool(this.config) | ||
| const client = await this.pool.connect() | ||
| console.info(`----> √ Postgres DB connection established! <----`) | ||
| return client | ||
| } | ||
| return this.pool.connect() | ||
| } | ||
|
|
||
| async execute_query(query: string): Promise<void> { | ||
| const client = await this.get_db_connection() | ||
| try { | ||
| await client.query(query) | ||
| } catch (error) { | ||
| console.error('Error executing query:', error) | ||
| throw new Error(error) | ||
| } finally { | ||
| client.release() | ||
| } | ||
| } | ||
| } | ||
|
|
||
| export const postgres = new DB() | ||
|
|
||
| const environment = 'dev' | ||
| console.log(`Supabase running on ${environment}`) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
72 changes: 72 additions & 0 deletions
72
packages/scripts/migrate-to-supabase/all-current-senses-have-entry-placeholders.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,72 @@ | ||
| import fs from 'node:fs' | ||
| import Chain from 'stream-chain' | ||
| import Parser from 'stream-json' | ||
| import StreamArray from 'stream-json/streamers/StreamArray' | ||
| import { admin_supabase, jacob_ld_user_id } from '../config-supabase' | ||
| import { remove_seconds_underscore } from './utils/remove-seconds-underscore' | ||
| import { convert_entry } from './convert-entries' | ||
|
|
||
| export async function ensure_all_current_senses_have_entry_placeholders() { | ||
| // load | ||
| const { data: senses } = await admin_supabase.from('senses') | ||
| .select('id, entry_id') | ||
| const { data: entries } = await admin_supabase.from('entries') | ||
| .select('id') | ||
|
|
||
| const entries_needing_added = new Set<string>() | ||
| for (const sense of senses) { | ||
| if (!entries.find(entry => entry.id === sense.entry_id)) { | ||
| console.log(`need placeholder entry for sense ${sense.id}, entry ${sense.entry_id}`) | ||
| entries_needing_added.add(sense.entry_id) | ||
| } | ||
| } | ||
| console.log({ entries_to_add: entries_needing_added.size }) | ||
|
|
||
| const pipeline = Chain.chain([ | ||
| fs.createReadStream('./migrate-to-supabase/firestore-data/firestore-entries.json'), | ||
| Parser.parser(), | ||
| StreamArray.streamArray(), | ||
| ]) | ||
|
|
||
| try { | ||
| for await (const { value: fb_entry } of pipeline) { | ||
| if (entries_needing_added.has(fb_entry.id)) { | ||
| const corrected_fb_entry = remove_seconds_underscore(fb_entry) | ||
| const [, supa_data] = convert_entry(JSON.parse(JSON.stringify(corrected_fb_entry))) | ||
| const { entry } = supa_data | ||
|
|
||
| const { data: dictionary } = await admin_supabase.from('dictionaries').select().eq('id', entry.dictionary_id).single() | ||
| if (!dictionary) { | ||
| console.log({ creating_dict: entry.dictionary_id }) | ||
| const { error } = await admin_supabase.from('dictionaries').insert({ | ||
| created_by: jacob_ld_user_id, | ||
| updated_by: jacob_ld_user_id, | ||
| id: entry.dictionary_id, | ||
| name: 'CHANGE', | ||
| }) | ||
| if (error) { | ||
| console.info({ entry }) | ||
| throw new Error(error.message) | ||
| } | ||
| } | ||
|
|
||
| const insert = { | ||
| id: entry.id, | ||
| dictionary_id: entry.dictionary_id, | ||
| lexeme: {}, | ||
| created_by: entry.created_by, | ||
| created_at: entry.created_at, | ||
| updated_by: entry.updated_by, | ||
| updated_at: entry.updated_at, | ||
| } | ||
| console.log({ insert }) | ||
| await admin_supabase.from('entries').insert(insert) | ||
| } | ||
| } | ||
| console.log('finished') | ||
| } catch (err) { | ||
| console.error(err) | ||
| pipeline.destroy() | ||
| pipeline.input.destroy() | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.