Use incredible introspection query thanks to graphile-engine

Author: typeofweb

src/__tests__/test.ts

@@ -5,6 +5,7 @@ import {
   schemaToTableObj,
   tableObjToTSCode,
   generateTSCodeForAllSchema as generateTSCodeForAllSchemas,
+  getPostgresVersion,
 } from '../generator';
 import Path from 'path';
 import { compileTypeScriptCode } from './tsCompiler';
@@ -121,6 +122,10 @@ describe('integration tests', () => {
   afterEach(drop);
   afterAll(() => pgp.end());
 
+  it('reads postgres version', async () => {
+    expect(await getPostgresVersion()).toBeGreaterThanOrEqual(120000);
+  });
+
   it('correctly reads schema for table user', async () => {
     const result = await getTablesSchemas();
 

src/generator/index.ts

@@ -1,34 +1,321 @@
-import { db, sql } from './db';
-import { ColumnSchema, TableName, TableSchema } from './types';
-import Path from 'path';
+import { db } from './db';
+import { TableSchema, ColumnType } from './types';
 import { Table } from '..';
 import Prettier from 'prettier';
-import { defaults } from 'lodash';
+import { defaults, flatMap } from 'lodash';
+import { makeIntrospectionQuery } from './introspectionQuery';
+import { xByY, xByYAndZ, parseTags } from './utils';
 
 // @ts-ignore
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
 class NotSupportedError extends Error {
   message = 'NotSupportedError';
 }
 
-function getTableNames(): Promise<TableName[]> {
-  return db.manyOrNone(sql(Path.join(__dirname, 'select_table_name.sql')));
-}
+// Ref: https://github.com/graphile/postgraphile/tree/master/src/postgres/introspection/object
+
+type PgNamespace = {
+  kind: 'namespace';
+  id: string;
+  name: string;
+  description?: string;
+  comment?: string;
+  tags: Record<string, string | boolean | (string | boolean)[]>;
+};
+
+type PgProc = {
+  kind: 'procedure';
+  id: string;
+  name: string;
+  description?: string;
+  comment?: string;
+  namespaceId: string;
+  namespaceName: string;
+  isStrict: boolean;
+  returnsSet: boolean;
+  isStable: boolean;
+  returnTypeId: string;
+  argTypeIds: Array<string>;
+  argNames: Array<string>;
+  argModes: Array<'i' | 'o' | 'b' | 'v' | 't'>;
+  inputArgsCount: number;
+  argDefaultsNum: number;
+  namespace: PgNamespace;
+  tags: Record<string, string | boolean | (string | boolean)[]>;
+  cost: number;
+  aclExecutable: boolean;
+  language: string;
+};
+
+type PgClass = {
+  kind: 'class';
+  id: string;
+  name: string;
+  description?: string;
+  comment?: string;
+  classKind: string;
+  namespaceId: string;
+  namespaceName: string;
+  typeId: string;
+  isSelectable: boolean;
+  isInsertable: boolean;
+  isUpdatable: boolean;
+  isDeletable: boolean;
+  isExtensionConfigurationTable: boolean;
+  namespace: PgNamespace;
+  type: PgType;
+  tags: Record<string, string | boolean | (string | boolean)[]>;
+  attributes: Array<PgAttribute>;
+  constraints: Array<PgConstraint>;
+  foreignConstraints: Array<PgConstraint>;
+  primaryKeyConstraint?: PgConstraint;
+  aclSelectable: boolean;
+  aclInsertable: boolean;
+  aclUpdatable: boolean;
+  aclDeletable: boolean;
+  canUseAsterisk: boolean;
+};
+
+type PgType = {
+  kind: 'type';
+  id: string;
+  name: ColumnType;
+  description?: string;
+  comment?: string;
+  namespaceId: string;
+  namespaceName: string;
+  type: string;
+  category: string;
+  domainIsNotNull: boolean;
+  arrayItemTypeId?: string;
+  arrayItemType?: PgType;
+  arrayType?: PgType;
+  typeLength?: number;
+  isPgArray: boolean;
+  classId?: string;
+  class?: PgClass;
+  domainBaseTypeId?: string;
+  domainBaseType?: PgType;
+  domainTypeModifier?: number;
+  tags: Record<string, string | boolean | (string | boolean)[]>;
+};
+
+type PgAttribute = {
+  kind: 'attribute';
+  classId: string;
+  num: number;
+  name: string;
+  description?: string;
+  comment?: string;
+  typeId: string;
+  typeModifier: number;
+  isNotNull: boolean;
+  hasDefault: boolean;
+  identity: '' | 'a' | 'd';
+  class: PgClass;
+  type: PgType;
+  namespace: PgNamespace;
+  tags: Record<string, string | boolean | (string | boolean)[]>;
+  aclSelectable: boolean;
+  aclInsertable: boolean;
+  aclUpdatable: boolean;
+  isIndexed?: boolean;
+  isUnique?: boolean;
+  columnLevelSelectGrant: boolean;
+};
+
+type PgConstraint = {
+  kind: 'constraint';
+  id: string;
+  name: string;
+  type: string;
+  classId: string;
+  class: PgClass;
+  foreignClassId?: string;
+  foreignClass?: PgClass;
+  description?: string;
+  comment?: string;
+  keyAttributeNums: Array<number>;
+  keyAttributes: Array<PgAttribute>;
+  foreignKeyAttributeNums: Array<number>;
+  foreignKeyAttributes: Array<PgAttribute>;
+  namespace: PgNamespace;
+  isIndexed?: boolean;
+  tags: Record<string, string | boolean | (string | boolean)[]>;
+};
+
+type PgExtension = {
+  kind: 'extension';
+  id: string;
+  name: string;
+  namespaceId: string;
+  namespaceName: string;
+  relocatable: boolean;
+  version: string;
+  configurationClassIds?: Array<string>;
+  description?: string;
+  comment?: string;
+  tags: Record<string, string | boolean | (string | boolean)[]>;
+};
 
-function getColumnSchemas(tableName: string): Promise<ColumnSchema[]> {
-  return db.manyOrNone(sql(Path.join(__dirname, 'select_table_schema.sql')), [tableName]);
+type PgIndex = {
+  kind: 'index';
+  id: string;
+  name: string;
+  namespaceName: string;
+  classId: string;
+  numberOfAttributes: number;
+  indexType: string;
+  isUnique: boolean;
+  isPrimary: boolean;
+  /*
+  Though these exist, we don't want to officially
+  support them yet.
+  isImmediate: boolean,
+  isReplicaIdentity: boolean,
+  isValid: boolean,
+  */
+  isPartial: boolean;
+  attributeNums: Array<number>;
+  attributePropertiesAsc?: Array<boolean>;
+  attributePropertiesNullsFirst?: Array<boolean>;
+  description?: string;
+  comment?: string;
+  tags: Record<string, string | boolean | (string | boolean)[]>;
+};
+
+type PgEntity =
+  | PgNamespace
+  | PgProc
+  | PgClass
+  | PgType
+  | PgAttribute
+  | PgConstraint
+  | PgExtension
+  | PgIndex;
+
+type PgIntrospectionOriginalResultsByKind = {
+  __pgVersion: number;
+  attribute: PgAttribute[];
+  class: PgClass[];
+  constraint: PgConstraint[];
+  extension: PgExtension[];
+  index: PgIndex[];
+  namespace: PgNamespace[];
+  procedure: PgProc[];
+  type: PgType[];
+};
+
+type PgIntrospectionResultsByKind = PgIntrospectionOriginalResultsByKind & {
+  attributeByClassIdAndNum: {
+    [classId: string]: { [num: string]: PgAttribute };
+  };
+  classById: { [classId: string]: PgClass };
+  extensionById: { [extId: string]: PgExtension };
+  namespaceById: { [namespaceId: string]: PgNamespace };
+  typeById: { [typeId: string]: PgType };
+};
+
+export async function getPostgresVersion(): Promise<number> {
+  const versionResult = await db.one('show server_version_num;');
+  return Number.parseInt(versionResult.server_version_num, 10);
 }
 
-export async function getTablesSchemas(): Promise<Array<TableSchema>> {
-  const tableNames = await getTableNames();
+export async function runIntrospectionQuery(): Promise<PgIntrospectionResultsByKind> {
+  const version = await getPostgresVersion();
+  const sql = makeIntrospectionQuery(version);
+  const kinds = [
+    'namespace',
+    'class',
+    'attribute',
+    'type',
+    'constraint',
+    'procedure',
+    'extension',
+    'index',
+  ] as const;
+  type Kinds = PgEntity['kind'];
+  type Row = {
+    object: PgEntity;
+  };
+  const rows: Row[] = await db.many(sql, [['public'], false]);
+
+  const originalResultsByType: PgIntrospectionOriginalResultsByKind = {
+    __pgVersion: version,
+    namespace: [],
+    class: [],
+    attribute: [],
+    type: [],
+    constraint: [],
+    procedure: [],
+    extension: [],
+    index: [],
+  };
+
+  for (const { object } of rows) {
+    originalResultsByType[object.kind].push(object as any);
+  }
+
+  // Parse tags from comments
+  kinds.forEach((kind) => {
+    originalResultsByType[kind].forEach((object: PgIntrospectionResultsByKind[Kinds][number]) => {
+      // Keep a copy of the raw comment
+      object.comment = object.description;
+      // https://www.graphile.org/postgraphile/smart-comments/
+      if (object.description) {
+        const parsed = parseTags(object.description);
+        object.tags = parsed.tags;
+        object.description = parsed.text;
+      } else {
+        object.tags = {};
+      }
+    });
+  });
 
-  const result = await Promise.all(
-    tableNames.map(async (i) => {
-      return { tableName: i.table_name, schema: await getColumnSchemas(i.table_name) };
-    }),
+  const extensionConfigurationClassIds = flatMap(
+    originalResultsByType.extension,
+    (e) => e.configurationClassIds,
   );
+  originalResultsByType.class.forEach((pgClass) => {
+    pgClass.isExtensionConfigurationTable = extensionConfigurationClassIds.includes(pgClass.id);
+  });
+
+  kinds.forEach((k) => {
+    originalResultsByType[k].forEach(Object.freeze);
+  });
+
+  const result = originalResultsByType as PgIntrospectionResultsByKind;
+  result.namespaceById = xByY(originalResultsByType.namespace, 'id');
+  result.classById = xByY(originalResultsByType.class, 'id');
+  result.typeById = xByY(originalResultsByType.type, 'id');
+  result.attributeByClassIdAndNum = xByYAndZ(originalResultsByType.attribute, 'classId', 'num');
+  result.extensionById = xByY(originalResultsByType.extension, 'id');
 
-  return result;
+  return Object.freeze(result);
+}
+
+export async function introspectSchemas() {
+  const intro = await runIntrospectionQuery();
+  const namespaceIds = intro.namespace.map((n) => n.id);
+  const classes = intro.class.filter((c) => namespaceIds.includes(c.namespaceId));
+  return { classes, intro };
+}
+
+export async function getTablesSchemas(): Promise<Array<TableSchema>> {
+  const { classes, intro } = await introspectSchemas();
+
+  return classes.map((klass) => {
+    return {
+      tableName: klass.name,
+      schema: Object.values(intro.attributeByClassIdAndNum[klass.id]).map((attribute) => {
+        return {
+          column_name: attribute.name,
+          udt_name: intro.typeById[attribute.typeId].name,
+          is_nullable: attribute.isNotNull ? 'NO' : 'YES',
+        };
+      }),
+    };
+  });
 }
 
 export function schemaToTableObj(schema: TableSchema): Table {