From 6b45bf7ac9a89e18f406b03eb736b07bb26befed Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sat, 17 May 2025 00:09:11 -0400 Subject: [PATCH 01/29] db WIP --- packages/postgres-test/README.md | 64 +++++++++ .../__tests__/postgres-test.grants.test.ts | 58 ++++++++ .../__tests__/postgres-test.template.test.ts | 73 ++++++++++ .../__tests__/postgres-test.test.ts | 43 ++++++ packages/postgres-test/jest.config.js | 18 +++ packages/postgres-test/package.json | 35 +++++ packages/postgres-test/sql/roles.sql | 48 +++++++ packages/postgres-test/sql/test.sql | 36 +++++ packages/postgres-test/src/admin.ts | 96 +++++++++++++ packages/postgres-test/src/connection.ts | 11 ++ packages/postgres-test/src/db.ts | 107 +++++++++++++++ packages/postgres-test/src/index.ts | 4 + packages/postgres-test/src/types.ts | 7 + packages/postgres-test/src/utils.ts | 126 +++++++++++++++++ packages/postgres-test/src/wrapper.ts | 129 ++++++++++++++++++ packages/postgres-test/test-utils/index.ts | 9 ++ packages/postgres-test/tsconfig.esm.json | 9 ++ packages/postgres-test/tsconfig.json | 9 ++ 18 files changed, 882 insertions(+) create mode 100644 packages/postgres-test/README.md create mode 100644 packages/postgres-test/__tests__/postgres-test.grants.test.ts create mode 100644 packages/postgres-test/__tests__/postgres-test.template.test.ts create mode 100644 packages/postgres-test/__tests__/postgres-test.test.ts create mode 100644 packages/postgres-test/jest.config.js create mode 100644 packages/postgres-test/package.json create mode 100644 packages/postgres-test/sql/roles.sql create mode 100644 packages/postgres-test/sql/test.sql create mode 100644 packages/postgres-test/src/admin.ts create mode 100644 packages/postgres-test/src/connection.ts create mode 100644 packages/postgres-test/src/db.ts create mode 100644 packages/postgres-test/src/index.ts create mode 100644 packages/postgres-test/src/types.ts create mode 100644 packages/postgres-test/src/utils.ts create mode 100644 packages/postgres-test/src/wrapper.ts create mode 100644 packages/postgres-test/test-utils/index.ts create mode 100644 packages/postgres-test/tsconfig.esm.json create mode 100644 packages/postgres-test/tsconfig.json diff --git a/packages/postgres-test/README.md b/packages/postgres-test/README.md new file mode 100644 index 0000000000..e2f7d9b24c --- /dev/null +++ b/packages/postgres-test/README.md @@ -0,0 +1,64 @@ +# postgres-test + +

+
+ PostgreSQL Testing in TypeScript +

+ +## install + +```sh +npm install postgres-test +``` +## Table of contents + +- [postgres-test](#postgres-test) + - [Install](#install) + - [Table of contents](#table-of-contents) +- [Developing](#developing) +- [Credits](#credits) + +## Developing + +When first cloning the repo: + +```sh +yarn +# build the prod packages. When devs would like to navigate to the source code, this will only navigate from references to their definitions (.d.ts files) between packages. +yarn build +``` + +Or if you want to make your dev process smoother, you can run: + +```sh +yarn +# build the dev packages with .map files, this enables navigation from references to their source code between packages. +yarn build:dev +``` + +## Interchain JavaScript Stack + +A unified toolkit for building applications and smart contracts in the Interchain ecosystem βš›οΈ + +| Category | Tools | Description | +|----------------------|------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------| +| **Chain Information** | [**Chain Registry**](https://github.com/hyperweb-io/chain-registry), [**Utils**](https://www.npmjs.com/package/@chain-registry/utils), [**Client**](https://www.npmjs.com/package/@chain-registry/client) | Everything from token symbols, logos, and IBC denominations for all assets you want to support in your application. | +| **Wallet Connectors**| [**Interchain Kit**](https://github.com/hyperweb-io/interchain-kit)beta, [**Cosmos Kit**](https://github.com/hyperweb.io/cosmos-kit) | Experience the convenience of connecting with a variety of web3 wallets through a single, streamlined interface. | +| **Signing Clients** | [**InterchainJS**](https://github.com/hyperweb-io/interchainjs)beta, [**CosmJS**](https://github.com/cosmos/cosmjs) | A single, universal signing interface for any network | +| **SDK Clients** | [**Telescope**](https://github.com/hyperweb.io/telescope) | Your Frontend Companion for Building with TypeScript with Cosmos SDK Modules. | +| **Starter Kits** | [**Create Interchain App**](https://github.com/hyperweb-io/create-interchain-app)beta, [**Create Cosmos App**](https://github.com/hyperweb.io/create-cosmos-app) | Set up a modern Interchain app by running one command. | +| **UI Kits** | [**Interchain UI**](https://github.com/hyperweb.io/interchain-ui) | The Interchain Design System, empowering developers with a flexible, easy-to-use UI kit. | +| **Testing Frameworks** | [**Starship**](https://github.com/hyperweb.io/starship) | Unified Testing and Development for the Interchain. | +| **TypeScript Smart Contracts** | [**Create Hyperweb App**](https://github.com/hyperweb-io/create-hyperweb-app) | Build and deploy full-stack blockchain applications with TypeScript | +| **CosmWasm Contracts** | [**CosmWasm TS Codegen**](https://github.com/CosmWasm/ts-codegen) | Convert your CosmWasm smart contracts into dev-friendly TypeScript classes. | + +## Credits + +πŸ›  Built by Hyperweb (formerly Cosmology) β€”Β if you like our tools, please checkout and contribute to [our github βš›οΈ](https://github.com/hyperweb-io) + + +## Disclaimer + +AS DESCRIBED IN THE LICENSES, THE SOFTWARE IS PROVIDED β€œAS IS”, AT YOUR OWN RISK, AND WITHOUT WARRANTIES OF ANY KIND. + +No developer or entity involved in creating this software will be liable for any claims or damages whatsoever associated with your use, inability to use, or your interaction with other users of the code, including any direct, indirect, incidental, special, exemplary, punitive or consequential damages, or loss of profits, cryptocurrencies, tokens, or anything else of value. diff --git a/packages/postgres-test/__tests__/postgres-test.grants.test.ts b/packages/postgres-test/__tests__/postgres-test.grants.test.ts new file mode 100644 index 0000000000..afc085033b --- /dev/null +++ b/packages/postgres-test/__tests__/postgres-test.grants.test.ts @@ -0,0 +1,58 @@ + +import { getEnvOptions } from '@launchql/types'; + +import { + getConnection, + closeConnection, + dropdb, + createdb, + PgConfig, +} from '../src'; +import { runSQLFile } from '../test-utils'; +import { randomUUID } from 'crypto'; +import { PgWrapper } from '../src/wrapper'; + +const TEST_DB_BASE = `postgres_test_${randomUUID()}`; + +function setupBaseDB(config: PgConfig): void { + createdb(config); + runSQLFile('test.sql', config.database); + runSQLFile('roles.sql', config.database); + dropdb(config); +} + +const opts = getEnvOptions({ + pg: { + database: TEST_DB_BASE + } +}); + +const config: PgConfig = { + user: opts.pg.user, + port: opts.pg.port, + password: opts.pg.password, + host: opts.pg.host, + database: TEST_DB_BASE +}; + +beforeAll(() => { + setupBaseDB(config); +}); + +describe('Postgres Test Framework', () => { + let db: PgWrapper; + + afterEach(() => { + if (db) closeConnection(db); + }); + + it('creates a test DB with hot mode (FAST_TEST)', () => { + db = getConnection({ hot: true, extensions: ['uuid-ossp'] }); + expect(db).toBeDefined(); + }); + + it('creates a test DB from scratch (default)', () => { + db = getConnection({}); + expect(db).toBeDefined(); + }); +}); diff --git a/packages/postgres-test/__tests__/postgres-test.template.test.ts b/packages/postgres-test/__tests__/postgres-test.template.test.ts new file mode 100644 index 0000000000..2e8075c248 --- /dev/null +++ b/packages/postgres-test/__tests__/postgres-test.template.test.ts @@ -0,0 +1,73 @@ +import { getEnvOptions } from '@launchql/types'; +import path from 'path'; +import fs from 'fs'; + +import { + getConnection, + closeConnection, + dropdb, + PgConfig, + run, + createTemplateFromBase, + cleanupTemplateDatabase +} from '../src'; +import { PgWrapper } from '../src/wrapper'; + +const TEMPLATE_NAME = 'test_template'; +const TEST_DB_BASE = 'postgres_test_db_template'; + +function runSQLFile(file: string, database: string): void { + const filePath = path.resolve(__dirname, '../sql', file); + if (!fs.existsSync(filePath)) { + throw new Error(`Missing SQL file: ${filePath}`); + } + run(`psql -f ${filePath} ${database}`); +} + +function setupTemplateDB(config: PgConfig, template: string): void { + try { + dropdb(config); + } catch {} + run(`createdb ${config.database}`); + runSQLFile('test.sql', config.database); + runSQLFile('roles.sql', config.database); + cleanupTemplateDatabase(config, template); + createTemplateFromBase(config, config.database, template); +} + +const opts = getEnvOptions({ + pg: { + database: TEST_DB_BASE + } +}); + +const config: PgConfig = { + user: opts.pg.user, + port: opts.pg.port, + password: opts.pg.password, + host: opts.pg.host, + database: TEST_DB_BASE +}; + +beforeAll(() => { + setupTemplateDB({ + user: opts.pg.user, + port: opts.pg.port, + password: opts.pg.password, + host: opts.pg.host, + database: TEST_DB_BASE + }, TEMPLATE_NAME); +}); + +describe('Template Database Test', () => { + let db: PgWrapper; + + afterEach(() => { + if (db) closeConnection(db); + }); + + it('creates a test DB from a template', () => { + db = getConnection({ template: TEMPLATE_NAME }); + expect(db).toBeDefined(); + }); +}); diff --git a/packages/postgres-test/__tests__/postgres-test.test.ts b/packages/postgres-test/__tests__/postgres-test.test.ts new file mode 100644 index 0000000000..6c431f90a0 --- /dev/null +++ b/packages/postgres-test/__tests__/postgres-test.test.ts @@ -0,0 +1,43 @@ +import { getEnvOptions } from '@launchql/types'; + +import { + getConnection, + closeConnection, + PgConfig, +} from '../src'; +import { randomUUID } from 'crypto'; +import { PgWrapper } from '../src/wrapper'; + +const TEST_DB_BASE = `postgres_test_${randomUUID()}`; + +const opts = getEnvOptions({ + pg: { + database: TEST_DB_BASE + } +}); + +const config: PgConfig = { + user: opts.pg.user, + port: opts.pg.port, + password: opts.pg.password, + host: opts.pg.host, + database: TEST_DB_BASE +}; + +describe('Postgres Test Framework', () => { + let db: PgWrapper; + + afterEach(() => { + if (db) closeConnection(db); + }); + + it('creates a test DB with hot mode (FAST_TEST)', () => { + db = getConnection({ hot: true, extensions: ['uuid-ossp'] }); + expect(db).toBeDefined(); + }); + + it('creates a test DB from scratch (default)', () => { + db = getConnection({}); + expect(db).toBeDefined(); + }); +}); diff --git a/packages/postgres-test/jest.config.js b/packages/postgres-test/jest.config.js new file mode 100644 index 0000000000..0aa3aaa499 --- /dev/null +++ b/packages/postgres-test/jest.config.js @@ -0,0 +1,18 @@ +/** @type {import('ts-jest').JestConfigWithTsJest} */ +module.exports = { + preset: "ts-jest", + testEnvironment: "node", + transform: { + "^.+\\.tsx?$": [ + "ts-jest", + { + babelConfig: false, + tsconfig: "tsconfig.json", + }, + ], + }, + transformIgnorePatterns: [`/node_modules/*`], + testRegex: "(/__tests__/.*|(\\.|/)(test|spec))\\.(jsx?|tsx?)$", + moduleFileExtensions: ["ts", "tsx", "js", "jsx", "json", "node"], + modulePathIgnorePatterns: ["dist/*"] +}; diff --git a/packages/postgres-test/package.json b/packages/postgres-test/package.json new file mode 100644 index 0000000000..adc0a30b0c --- /dev/null +++ b/packages/postgres-test/package.json @@ -0,0 +1,35 @@ +{ + "name": "postgres-test", + "version": "0.0.1", + "author": "Dan Lynch ", + "description": "PostgreSQL Testing in TypeScript", + "main": "index.js", + "module": "esm/index.js", + "types": "index.d.ts", + "homepage": "https://github.com/launchql/launchql", + "license": "SEE LICENSE IN LICENSE", + "publishConfig": { + "access": "public", + "directory": "dist" + }, + "repository": { + "type": "git", + "url": "https://github.com/launchql/launchql" + }, + "bugs": { + "url": "https://github.com/launchql/launchql/issues" + }, + "scripts": { + "copy": "copyfiles -f ../../LICENSE README.md package.json dist", + "clean": "rimraf dist/**", + "prepare": "npm run build", + "build": "npm run clean; tsc; tsc -p tsconfig.esm.json; npm run copy", + "build:dev": "npm run clean; tsc --declarationMap; tsc -p tsconfig.esm.json; npm run copy", + "lint": "eslint . --fix", + "test": "jest", + "test:watch": "jest --watch" + }, + "dependencies": { + "@launchql/types": "^2.0.4" + } +} \ No newline at end of file diff --git a/packages/postgres-test/sql/roles.sql b/packages/postgres-test/sql/roles.sql new file mode 100644 index 0000000000..1983433bc3 --- /dev/null +++ b/packages/postgres-test/sql/roles.sql @@ -0,0 +1,48 @@ +BEGIN; +DO $do$ +BEGIN + IF NOT EXISTS ( + SELECT + FROM + pg_catalog.pg_roles + WHERE + rolname = 'administrator') THEN + CREATE ROLE administrator; +END IF; + IF NOT EXISTS ( + SELECT + FROM + pg_catalog.pg_roles + WHERE + rolname = 'anonymous') THEN + CREATE ROLE anonymous; +END IF; + IF NOT EXISTS ( + SELECT + FROM + pg_catalog.pg_roles + WHERE + rolname = 'authenticated') THEN + CREATE ROLE authenticated; +END IF; +END +$do$; +ALTER USER administrator WITH NOCREATEDB; +ALTER USER administrator WITH NOCREATEROLE; +ALTER USER administrator WITH NOLOGIN; +ALTER USER administrator WITH NOREPLICATION; +ALTER USER administrator WITH BYPASSRLS; +ALTER USER anonymous WITH NOCREATEDB; +ALTER USER anonymous WITH NOCREATEROLE; +ALTER USER anonymous WITH NOLOGIN; +ALTER USER anonymous WITH NOREPLICATION; +ALTER USER anonymous WITH NOBYPASSRLS; +ALTER USER authenticated WITH NOCREATEDB; +ALTER USER authenticated WITH NOCREATEROLE; +ALTER USER authenticated WITH NOLOGIN; +ALTER USER authenticated WITH NOREPLICATION; +ALTER USER authenticated WITH NOBYPASSRLS; +GRANT anonymous TO administrator; +GRANT authenticated TO administrator; +COMMIT; + diff --git a/packages/postgres-test/sql/test.sql b/packages/postgres-test/sql/test.sql new file mode 100644 index 0000000000..ea7f70dd7a --- /dev/null +++ b/packages/postgres-test/sql/test.sql @@ -0,0 +1,36 @@ +-- https://en.wikipedia.org/wiki/Role-based_access_control +BEGIN; +CREATE EXTENSION IF NOT EXISTS citext; +DROP SCHEMA IF EXISTS app_public CASCADE; +CREATE SCHEMA app_public; +CREATE TABLE app_public.users ( + id serial PRIMARY KEY, + username citext, + UNIQUE (username), + CHECK (length(username) < 127) +); +CREATE TABLE app_public.roles ( + id serial PRIMARY KEY, + org_id bigint NOT NULL REFERENCES app_public.users (id) +); +CREATE TABLE app_public.user_settings ( + user_id bigint NOT NULL PRIMARY KEY REFERENCES app_public.users (id), + setting1 text, + UNIQUE (user_id) +); +CREATE TABLE app_public.permissions ( + id serial PRIMARY KEY, + name citext +); +CREATE TABLE app_public.permission_assignment ( + perm_id bigint NOT NULL REFERENCES app_public.permissions (id), + role_id bigint NOT NULL REFERENCES app_public.roles (id), + PRIMARY KEY (perm_id, role_id) +); +CREATE TABLE app_public.subject_assignment ( + subj_id bigint NOT NULL REFERENCES app_public.users (id), + role_id bigint NOT NULL REFERENCES app_public.roles (id), + PRIMARY KEY (subj_id, role_id) +); +COMMIT; + diff --git a/packages/postgres-test/src/admin.ts b/packages/postgres-test/src/admin.ts new file mode 100644 index 0000000000..9915d841dd --- /dev/null +++ b/packages/postgres-test/src/admin.ts @@ -0,0 +1,96 @@ +import { execSync } from 'child_process'; +import { PgConfig } from './types'; + +export class DbAdmin { + constructor( + private config: PgConfig, + private verbose: boolean = false + ) {} + + private getEnv(): Record { + return { + PGHOST: this.config.host, + PGPORT: String(this.config.port), + PGUSER: this.config.user, + PGPASSWORD: this.config.password, + }; + } + + private run(command: string): void { + execSync(command, { + stdio: this.verbose ? 'inherit' : 'pipe', + env: { + ...process.env, + ...this.getEnv(), + }, + }); + } + + private safeDropDb(name: string): void { + try { + this.run(`dropdb "${name}"`); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + if (!message.includes('does not exist')) { + console.warn(`⚠️ Could not drop database ${name}: ${message}`); + } + } + } + + drop(dbName?: string): void { + this.safeDropDb(dbName ?? this.config.database); + } + + dropTemplate(dbName?: string): void { + const db = dbName ?? this.config.database; + this.run(`psql -c "UPDATE pg_database SET datistemplate='false' WHERE datname='${db}';"`); + this.drop(db); + } + + create(dbName?: string): void { + const db = dbName ?? this.config.database; + this.run(`createdb -U ${this.config.user} -h ${this.config.host} -p ${this.config.port} "${db}"`); + } + + createFromTemplate(template: string, dbName?: string): void { + const db = dbName ?? this.config.database; + this.run(`createdb -U ${this.config.user} -h ${this.config.host} -p ${this.config.port} -e "${db}" -T "${template}"`); + } + + installExtensions(extensions: string[] | string, dbName?: string): void { + const db = dbName ?? this.config.database; + const extList = typeof extensions === 'string' ? extensions.split(',') : extensions; + + for (const extension of extList) { + this.run(`psql --dbname "${db}" -c 'CREATE EXTENSION IF NOT EXISTS "${extension}" CASCADE;'`); + } + } + + connectionString(dbName?: string): string { + const { user, password, host, port } = this.config; + const db = dbName ?? this.config.database; + return `postgres://${user}:${password}@${host}:${port}/${db}`; + } + + createTemplateFromBase(base: string, template: string): void { + this.run(`createdb -T "${base}" "${template}"`); + this.run(`psql -c "UPDATE pg_database SET datistemplate = true WHERE datname = '${template}';"`); + } + + cleanupTemplate(template: string): void { + try { + this.run(`psql -c "UPDATE pg_database SET datistemplate = false WHERE datname = '${template}'"`); + } catch {} + this.safeDropDb(template); + } + + createRole(role: string, password: string, dbName?: string): void { + const db = dbName ?? this.config.database; + this.run(`psql -d "${db}" -c "CREATE ROLE ${role} WITH LOGIN PASSWORD '${password}';"`); + } + + grantConnect(role: string, dbName?: string): void { + const db = dbName ?? this.config.database; + this.run(`psql -d "${db}" -c "GRANT CONNECT ON DATABASE ${db} TO ${role};"`); + } +} diff --git a/packages/postgres-test/src/connection.ts b/packages/postgres-test/src/connection.ts new file mode 100644 index 0000000000..f99fabfb32 --- /dev/null +++ b/packages/postgres-test/src/connection.ts @@ -0,0 +1,11 @@ +import { PgConfig } from './types'; +import { PgWrapper } from './wrapper'; + +export function connect(config: PgConfig): PgWrapper { + const db = new PgWrapper(config); + return db; +} + +export function close(client: PgWrapper): void { + client.close(); +} diff --git a/packages/postgres-test/src/db.ts b/packages/postgres-test/src/db.ts new file mode 100644 index 0000000000..a05a8cd039 --- /dev/null +++ b/packages/postgres-test/src/db.ts @@ -0,0 +1,107 @@ +import { execSync } from 'child_process'; +import { PgConfig } from './types'; + +export interface TemplatedbConfig extends PgConfig { + template: string; +} + +const getPgEnv = (config: PgConfig) => { + return { + PGHOST: config.host, + PGPORT: String(config.port), + PGUSER: config.user, + PGPASSWORD: config.password, + } +} + +export function run(command: string, env: Record = {}) { + execSync(command, { + stdio: 'inherit', + env: { + ...process.env, + ...env, + }, + }); +} + +function safeDropDb(config: PgConfig, name: string): void { + try { + run(`dropdb ${name}`, getPgEnv(config)); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + if (!message.includes('does not exist')) { + console.warn(`⚠️ Could not drop database ${name}: ${message}`); + } + } +} + +export function dropdb(config: PgConfig): void { + safeDropDb(config, config.database); +} + +export function droptemplatedb(config: PgConfig): void { + run( + `psql -c "UPDATE pg_database SET datistemplate='false' WHERE datname='${config.database}';"`, + getPgEnv(config) + ); + dropdb(config); +} + +export function createdb(config: PgConfig): void { + run(`createdb -U ${config.user} -h ${config.host} -p ${config.port} ${config.database}`, { + PGPASSWORD: config.password, + }); +} + +export function templatedb(config: TemplatedbConfig): void { + run( + `createdb -U ${config.user} -h ${config.host} -p ${config.port} -e ${config.database} -T ${config.template}`, + { + PGPASSWORD: config.password, + } + ); +} + +export function installExt( + config: PgConfig, + extensions: string[] | string +): void { + const extList = typeof extensions === 'string' ? extensions.split(',') : extensions; + + for (const extension of extList) { + run( + `psql --dbname "${config.database}" -c 'CREATE EXTENSION IF NOT EXISTS "${extension}" CASCADE;'`, + getPgEnv(config) + ); + } +} + +export function connectionString(config: PgConfig): string { + return `postgres://${config.user}:${config.password}@${config.host}:${config.port}/${config.database}`; +} + +export function createTemplateFromBase(config: PgConfig, base: string, template: string): void { + run(`createdb -T ${base} ${template}`, getPgEnv(config)); + run(`psql -c "UPDATE pg_database SET datistemplate = true WHERE datname = '${template}';"`, getPgEnv(config)); +} + +export function cleanupTemplateDatabase(config: PgConfig, template: string): void { + try { + run(`psql -c "UPDATE pg_database SET datistemplate = false WHERE datname = '${template}'"`, getPgEnv(config)); + } catch {} + safeDropDb(config, template); +} + +export function createRole(config: PgConfig, role: string, password: string): void { + run( + `psql -d ${config.database} -c "CREATE ROLE ${role} WITH LOGIN PASSWORD '${password}';"`, + getPgEnv(config) + ); +} + +export function grantConnect(config: PgConfig, role: string): void { + run( + `psql -d ${config.database} -c "GRANT CONNECT ON DATABASE ${config.database} TO ${role};"`, + getPgEnv(config) + ); +} diff --git a/packages/postgres-test/src/index.ts b/packages/postgres-test/src/index.ts new file mode 100644 index 0000000000..b6d2732bd5 --- /dev/null +++ b/packages/postgres-test/src/index.ts @@ -0,0 +1,4 @@ +export * from './connection'; +export * from './db'; +export * from './types'; +export * from './utils'; \ No newline at end of file diff --git a/packages/postgres-test/src/types.ts b/packages/postgres-test/src/types.ts new file mode 100644 index 0000000000..fffe795126 --- /dev/null +++ b/packages/postgres-test/src/types.ts @@ -0,0 +1,7 @@ +export interface PgConfig { + database: string; + host: string; + password: string; + port: number; + user: string; +} \ No newline at end of file diff --git a/packages/postgres-test/src/utils.ts b/packages/postgres-test/src/utils.ts new file mode 100644 index 0000000000..f802364046 --- /dev/null +++ b/packages/postgres-test/src/utils.ts @@ -0,0 +1,126 @@ +import { Client, Pool } from 'pg'; +import { createdb, dropdb, templatedb, installExt, grantConnect } from './db'; +import { connect, close } from './connection'; +import { PgConfig } from './types'; +import { getEnvOptions } from '@launchql/types'; +import { randomUUID } from 'crypto'; +import { PgWrapper } from './wrapper'; + +export interface TestOptions { + hot?: boolean; + template?: string; + prefix?: string; + extensions?: string[]; +} + +export function getOpts(configOpts: TestOptions = {}): TestOptions { + return { + template: configOpts.template, + prefix: configOpts.prefix || 'testing-db', + extensions: configOpts.extensions || [], + }; +} + +export function getConnection(configOpts: TestOptions, database?: string): PgWrapper { + const envOpts = getEnvOptions(); + const opts = getOpts(configOpts); + const dbName = database || `${opts.prefix}-${Date.now()}`; + + const config: PgConfig = { + database: dbName, + user: envOpts.pg.user, + password: envOpts.pg.password, + port: envOpts.pg.port, + host: envOpts.pg.host + }; + + if (process.env.TEST_DB) { + config.database = process.env.TEST_DB; + } else if (opts.hot) { + createdb(config); + installExt(config, opts.extensions); + } else if (opts.template) { + templatedb({ ...config, template: opts.template }); + } else { + createdb(config); + installExt(config, opts.extensions); + } + + return connect(config); +} + +export function closeConnection(db: PgWrapper): void { + db.kill(); +} + +export function connectTest(database: string, user: string, password: string): PgWrapper { + const envOpts = getEnvOptions(); + const config: PgConfig = { + port: envOpts.pg.port, + host: envOpts.pg.host, + database, + user, + password + }; + return connect(config); +} + +export async function createUserRole(db: PgWrapper, user: string, password: string): Promise { + await db.query(` + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_roles WHERE rolname = '${user}' + ) THEN + CREATE ROLE ${user} LOGIN PASSWORD '${password}'; + GRANT anonymous TO ${user}; + GRANT authenticated TO ${user}; + END IF; + END $$; + `); +} + +export function closeConnections({ db, conn }: { db: PgWrapper; conn: PgWrapper }): void { + conn.close(); + closeConnection(db); +} + +export const getConnections = async () => { + const opts = getEnvOptions({ + pg: { + database: `db-${randomUUID()}` + } + }); + + const config: PgConfig = { + user: opts.pg.user, + port: opts.pg.port, + password: opts.pg.password, + host: opts.pg.host, + database: opts.pg.database + }; + + const db = await getConnection({ + + }); + + await createUserRole(db, 'app_user', 'app_password'); + await grantConnect(config, 'app_user'); + + const conn = await connectTest(opts.pg.database, 'app_user', 'app_password'); + conn.setContext({ + role: 'anonymous' + }); + + const teardown = async () => { + await closeConnections({ db, conn }); + }; + + return { db, conn, teardown }; +}; + + +// export async function grantConnect(db: any, user: string): Promise { +// await db.query(`GRANT CONNECT ON DATABASE "${db.database}" TO ${user};`); +// } + diff --git a/packages/postgres-test/src/wrapper.ts b/packages/postgres-test/src/wrapper.ts new file mode 100644 index 0000000000..ab5d4b69c0 --- /dev/null +++ b/packages/postgres-test/src/wrapper.ts @@ -0,0 +1,129 @@ +import { Client, QueryResult } from 'pg'; +import { PgConfig } from './types'; +import { dropdb } from './db'; + +export class PgWrapper { + public config: PgConfig; + private client: Client; + private ctxStmts: string = ''; + private _ended: boolean = false; + private _dropped: boolean = false; + + constructor(config: PgConfig) { + this.config = config; + this.client = new Client({ + host: this.config.host, + port: this.config.port, + database: this.config.database, + user: this.config.user, + password: this.config.password + }); + this.client.connect(); + } + + close(): void { + if (!this._ended) { + this._ended = true; + this.client.end(); + } + } + + kill(): void { + this.close(); + if (!this._dropped && !process.env.TEST_DB) { + this.drop(); + } + } + + drop(): void { + if (!this._dropped) { + this._dropped = true; + dropdb(this.config); + } + } + + async begin(): Promise { + await this.client.query('BEGIN;'); + } + + async savepoint(name: string = 'lqlsavepoint'): Promise { + await this.client.query(`SAVEPOINT "${name}";`); + } + + async rollback(name: string = 'lqlsavepoint'): Promise { + await this.client.query(`ROLLBACK TO SAVEPOINT "${name}";`); + } + + async commit(): Promise { + await this.client.query('COMMIT;'); + } + + async beforeEach(): Promise { + await this.begin(); + await this.savepoint(); + } + + async afterEach(): Promise { + await this.rollback(); + await this.commit(); + } + + setContext(ctx: Record): void { + this.ctxStmts = Object.entries(ctx) + .map(([key, val]) => + val === null + ? `SELECT set_config('${key}', NULL, true);` + : `SELECT set_config('${key}', '${val}', true);` + ) + .join('\n'); + } + + private async runCtxQuery(query: string, values?: any[]): Promise> { + if (this.ctxStmts) { + await this.client.query(this.ctxStmts); + } + const result = await this.client.query(query, values); + return result; + } + + async any(query: string, values?: any[]): Promise { + const result = await this.runCtxQuery(query, values); + return result.rows; + } + + async one(query: string, values?: any[]): Promise { + const rows = await this.any(query, values); + if (rows.length !== 1) { + throw new Error('Expected exactly one result'); + } + return rows[0]; + } + + async oneOrNone(query: string, values?: any[]): Promise { + const rows = await this.any(query, values); + return rows[0] || null; + } + + async many(query: string, values?: any[]): Promise { + const rows = await this.any(query, values); + if (rows.length === 0) throw new Error('Expected many rows, got none'); + return rows; + } + + async manyOrNone(query: string, values?: any[]): Promise { + return this.any(query, values); + } + + async none(query: string, values?: any[]): Promise { + await this.runCtxQuery(query, values); + } + + async result(query: string, values?: any[]): Promise { + return this.runCtxQuery(query, values); + } + + async query(query: string, values?: any[]): Promise> { + return this.client.query(query, values); + } + +} diff --git a/packages/postgres-test/test-utils/index.ts b/packages/postgres-test/test-utils/index.ts new file mode 100644 index 0000000000..b4447cec71 --- /dev/null +++ b/packages/postgres-test/test-utils/index.ts @@ -0,0 +1,9 @@ +import path from 'path'; +import fs from 'fs'; +import { run } from '../src'; + +export function runSQLFile(file: string, database: string): void { + const filePath = path.resolve(__dirname, '../sql', file); + if (!fs.existsSync(filePath)) throw new Error(`Missing SQL file: ${file}`); + run(`psql -f ${filePath} ${database}`); +} diff --git a/packages/postgres-test/tsconfig.esm.json b/packages/postgres-test/tsconfig.esm.json new file mode 100644 index 0000000000..800d7506d3 --- /dev/null +++ b/packages/postgres-test/tsconfig.esm.json @@ -0,0 +1,9 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "dist/esm", + "module": "es2022", + "rootDir": "src/", + "declaration": false + } +} diff --git a/packages/postgres-test/tsconfig.json b/packages/postgres-test/tsconfig.json new file mode 100644 index 0000000000..1a9d5696cb --- /dev/null +++ b/packages/postgres-test/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src/" + }, + "include": ["src/**/*.ts"], + "exclude": ["dist", "node_modules", "**/*.spec.*", "**/*.test.*"] +} From d72088f040b620ad436b6453f5e42fe8014744b0 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sat, 17 May 2025 00:09:55 -0400 Subject: [PATCH 02/29] test --- .github/workflows/run-tests.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 8f97410644..39a79a0d5d 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -75,6 +75,15 @@ jobs: PGUSER: postgres PGPASSWORD: password + - name: postgres-test + run: | + cd ./packages/postgres-test + yarn test + env: + PGHOST: pg_db + PGUSER: postgres + PGPASSWORD: password + - name: launchql/orm run: cd ./packages/orm && yarn test From 6924297f9e7eca9774f45fa00bebb4f70332d34c Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sat, 17 May 2025 00:55:17 -0700 Subject: [PATCH 03/29] cleanup --- .../__tests__/postgres-test.grants.test.ts | 42 +++--- .../__tests__/postgres-test.template.test.ts | 62 +++----- .../__tests__/postgres-test.test.ts | 22 +-- packages/postgres-test/package.json | 3 +- packages/postgres-test/src/admin.ts | 11 +- .../src/{wrapper.ts => client.ts} | 22 +-- packages/postgres-test/src/connection.ts | 35 ++++- packages/postgres-test/src/db.ts | 107 ------------- packages/postgres-test/src/index.ts | 2 - packages/postgres-test/src/manager.ts | 140 ++++++++++++++++++ packages/postgres-test/src/types.ts | 7 - packages/postgres-test/src/utils.ts | 65 ++++---- packages/postgres-test/test-utils/index.ts | 7 - packages/server-utils/src/pg.ts | 4 +- packages/types/src/env.ts | 29 +++- packages/types/src/launchql.ts | 17 +-- 16 files changed, 297 insertions(+), 278 deletions(-) rename packages/postgres-test/src/{wrapper.ts => client.ts} (89%) delete mode 100644 packages/postgres-test/src/db.ts create mode 100644 packages/postgres-test/src/manager.ts delete mode 100644 packages/postgres-test/src/types.ts diff --git a/packages/postgres-test/__tests__/postgres-test.grants.test.ts b/packages/postgres-test/__tests__/postgres-test.grants.test.ts index afc085033b..d79f3f30c7 100644 --- a/packages/postgres-test/__tests__/postgres-test.grants.test.ts +++ b/packages/postgres-test/__tests__/postgres-test.grants.test.ts @@ -1,46 +1,42 @@ - -import { getEnvOptions } from '@launchql/types'; +import { getPgEnvOptions, PgConfig } from '@launchql/types'; import { getConnection, closeConnection, - dropdb, - createdb, - PgConfig, + Connection, } from '../src'; -import { runSQLFile } from '../test-utils'; import { randomUUID } from 'crypto'; -import { PgWrapper } from '../src/wrapper'; +import { PgTestClient } from '../src/client'; +import { DbAdmin } from '../src/admin'; +import { resolve } from 'path'; + +const sql = (file: string) => resolve(__dirname, '../sql', file); const TEST_DB_BASE = `postgres_test_${randomUUID()}`; function setupBaseDB(config: PgConfig): void { - createdb(config); - runSQLFile('test.sql', config.database); - runSQLFile('roles.sql', config.database); - dropdb(config); + const admin = new DbAdmin(config); + admin.create(config.database) + admin.loadSql(sql('test.sql'), config.database); + admin.loadSql(sql('roles.sql'), config.database); + admin.drop(config.database); } -const opts = getEnvOptions({ - pg: { +const config = getPgEnvOptions({ database: TEST_DB_BASE - } }); -const config: PgConfig = { - user: opts.pg.user, - port: opts.pg.port, - password: opts.pg.password, - host: opts.pg.host, - database: TEST_DB_BASE -}; - beforeAll(() => { setupBaseDB(config); }); +afterAll(() => { + Connection.getManager().closeAll(); +}); + + describe('Postgres Test Framework', () => { - let db: PgWrapper; + let db: PgTestClient; afterEach(() => { if (db) closeConnection(db); diff --git a/packages/postgres-test/__tests__/postgres-test.template.test.ts b/packages/postgres-test/__tests__/postgres-test.template.test.ts index 2e8075c248..4f317e741e 100644 --- a/packages/postgres-test/__tests__/postgres-test.template.test.ts +++ b/packages/postgres-test/__tests__/postgres-test.template.test.ts @@ -1,66 +1,46 @@ -import { getEnvOptions } from '@launchql/types'; +import { getPgEnvOptions, PgConfig } from '@launchql/types'; import path from 'path'; -import fs from 'fs'; import { getConnection, closeConnection, - dropdb, - PgConfig, - run, - createTemplateFromBase, - cleanupTemplateDatabase + Connection } from '../src'; -import { PgWrapper } from '../src/wrapper'; +import { PgTestClient } from '../src/client'; +import { DbAdmin } from '../src/admin'; + +const sql = (file: string) => path.resolve(__dirname, '../sql', file); const TEMPLATE_NAME = 'test_template'; const TEST_DB_BASE = 'postgres_test_db_template'; -function runSQLFile(file: string, database: string): void { - const filePath = path.resolve(__dirname, '../sql', file); - if (!fs.existsSync(filePath)) { - throw new Error(`Missing SQL file: ${filePath}`); - } - run(`psql -f ${filePath} ${database}`); -} - function setupTemplateDB(config: PgConfig, template: string): void { + const admin = new DbAdmin(config); try { - dropdb(config); + admin.drop(config.database); } catch {} - run(`createdb ${config.database}`); - runSQLFile('test.sql', config.database); - runSQLFile('roles.sql', config.database); - cleanupTemplateDatabase(config, template); - createTemplateFromBase(config, config.database, template); + admin.create(config.database); + admin.loadSql(sql('test.sql'), config.database); + admin.loadSql(sql('roles.sql'), config.database); + admin.cleanupTemplate(template); + admin.createTemplateFromBase(config.database, template); + admin.drop(config.database); } -const opts = getEnvOptions({ - pg: { +const config = getPgEnvOptions({ database: TEST_DB_BASE - } }); -const config: PgConfig = { - user: opts.pg.user, - port: opts.pg.port, - password: opts.pg.password, - host: opts.pg.host, - database: TEST_DB_BASE -}; - beforeAll(() => { - setupTemplateDB({ - user: opts.pg.user, - port: opts.pg.port, - password: opts.pg.password, - host: opts.pg.host, - database: TEST_DB_BASE - }, TEMPLATE_NAME); + setupTemplateDB(config, TEMPLATE_NAME); +}); + +afterAll(() => { + Connection.getManager().closeAll(); }); describe('Template Database Test', () => { - let db: PgWrapper; + let db: PgTestClient; afterEach(() => { if (db) closeConnection(db); diff --git a/packages/postgres-test/__tests__/postgres-test.test.ts b/packages/postgres-test/__tests__/postgres-test.test.ts index 6c431f90a0..62eab4ff55 100644 --- a/packages/postgres-test/__tests__/postgres-test.test.ts +++ b/packages/postgres-test/__tests__/postgres-test.test.ts @@ -1,31 +1,25 @@ -import { getEnvOptions } from '@launchql/types'; +import { getPgEnvOptions, PgConfig } from '@launchql/types'; import { getConnection, closeConnection, - PgConfig, + Connection } from '../src'; import { randomUUID } from 'crypto'; -import { PgWrapper } from '../src/wrapper'; +import { PgTestClient } from '../src/client'; const TEST_DB_BASE = `postgres_test_${randomUUID()}`; -const opts = getEnvOptions({ - pg: { +const config = getPgEnvOptions({ database: TEST_DB_BASE - } }); -const config: PgConfig = { - user: opts.pg.user, - port: opts.pg.port, - password: opts.pg.password, - host: opts.pg.host, - database: TEST_DB_BASE -}; +afterAll(() => { + Connection.getManager().closeAll(); +}); describe('Postgres Test Framework', () => { - let db: PgWrapper; + let db: PgTestClient; afterEach(() => { if (db) closeConnection(db); diff --git a/packages/postgres-test/package.json b/packages/postgres-test/package.json index adc0a30b0c..4302d6c141 100644 --- a/packages/postgres-test/package.json +++ b/packages/postgres-test/package.json @@ -30,6 +30,7 @@ "test:watch": "jest --watch" }, "dependencies": { - "@launchql/types": "^2.0.4" + "@launchql/types": "^2.0.4", + "chalk": "^4.1.0" } } \ No newline at end of file diff --git a/packages/postgres-test/src/admin.ts b/packages/postgres-test/src/admin.ts index 9915d841dd..f62457a83a 100644 --- a/packages/postgres-test/src/admin.ts +++ b/packages/postgres-test/src/admin.ts @@ -1,5 +1,6 @@ import { execSync } from 'child_process'; -import { PgConfig } from './types'; +import { PgConfig } from '@launchql/types'; +import { existsSync } from 'fs'; export class DbAdmin { constructor( @@ -93,4 +94,12 @@ export class DbAdmin { const db = dbName ?? this.config.database; this.run(`psql -d "${db}" -c "GRANT CONNECT ON DATABASE ${db} TO ${role};"`); } + + loadSql(file: string, dbName: string): void { + if (!existsSync(file)) { + throw new Error(`Missing SQL file: ${file}`); + } + this.run(`psql -f ${file} ${dbName}`); + } + } diff --git a/packages/postgres-test/src/wrapper.ts b/packages/postgres-test/src/client.ts similarity index 89% rename from packages/postgres-test/src/wrapper.ts rename to packages/postgres-test/src/client.ts index ab5d4b69c0..e39bb6d878 100644 --- a/packages/postgres-test/src/wrapper.ts +++ b/packages/postgres-test/src/client.ts @@ -1,13 +1,11 @@ import { Client, QueryResult } from 'pg'; -import { PgConfig } from './types'; -import { dropdb } from './db'; +import { PgConfig } from '@launchql/types'; -export class PgWrapper { +export class PgTestClient { public config: PgConfig; private client: Client; private ctxStmts: string = ''; private _ended: boolean = false; - private _dropped: boolean = false; constructor(config: PgConfig) { this.config = config; @@ -28,20 +26,6 @@ export class PgWrapper { } } - kill(): void { - this.close(); - if (!this._dropped && !process.env.TEST_DB) { - this.drop(); - } - } - - drop(): void { - if (!this._dropped) { - this._dropped = true; - dropdb(this.config); - } - } - async begin(): Promise { await this.client.query('BEGIN;'); } @@ -125,5 +109,5 @@ export class PgWrapper { async query(query: string, values?: any[]): Promise> { return this.client.query(query, values); } - + } diff --git a/packages/postgres-test/src/connection.ts b/packages/postgres-test/src/connection.ts index f99fabfb32..07838ddd31 100644 --- a/packages/postgres-test/src/connection.ts +++ b/packages/postgres-test/src/connection.ts @@ -1,11 +1,34 @@ -import { PgConfig } from './types'; -import { PgWrapper } from './wrapper'; +import { PgTestClient } from './client'; +import { PgTestConnector } from './manager'; +import { randomUUID } from 'crypto'; +import { getPgEnvOptions, PgConfig } from '@launchql/types'; -export function connect(config: PgConfig): PgWrapper { - const db = new PgWrapper(config); - return db; +export function connect(config: PgConfig): PgTestClient { + const manager = PgTestConnector.getInstance(); + return manager.getClient(config); } -export function close(client: PgWrapper): void { +export function close(client: PgTestClient): void { client.close(); } + +const manager = PgTestConnector.getInstance(); + +export const Connection = { + connect(config: Partial): PgTestClient { + const creds = getPgEnvOptions(config); + return manager.getClient(creds); + }, + + close(client: PgTestClient): void { + client.close(); + }, + + closeAll(): Promise { + return manager.closeAll(); + }, + + getManager(): PgTestConnector { + return manager; + } +}; diff --git a/packages/postgres-test/src/db.ts b/packages/postgres-test/src/db.ts deleted file mode 100644 index a05a8cd039..0000000000 --- a/packages/postgres-test/src/db.ts +++ /dev/null @@ -1,107 +0,0 @@ -import { execSync } from 'child_process'; -import { PgConfig } from './types'; - -export interface TemplatedbConfig extends PgConfig { - template: string; -} - -const getPgEnv = (config: PgConfig) => { - return { - PGHOST: config.host, - PGPORT: String(config.port), - PGUSER: config.user, - PGPASSWORD: config.password, - } -} - -export function run(command: string, env: Record = {}) { - execSync(command, { - stdio: 'inherit', - env: { - ...process.env, - ...env, - }, - }); -} - -function safeDropDb(config: PgConfig, name: string): void { - try { - run(`dropdb ${name}`, getPgEnv(config)); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - if (!message.includes('does not exist')) { - console.warn(`⚠️ Could not drop database ${name}: ${message}`); - } - } -} - -export function dropdb(config: PgConfig): void { - safeDropDb(config, config.database); -} - -export function droptemplatedb(config: PgConfig): void { - run( - `psql -c "UPDATE pg_database SET datistemplate='false' WHERE datname='${config.database}';"`, - getPgEnv(config) - ); - dropdb(config); -} - -export function createdb(config: PgConfig): void { - run(`createdb -U ${config.user} -h ${config.host} -p ${config.port} ${config.database}`, { - PGPASSWORD: config.password, - }); -} - -export function templatedb(config: TemplatedbConfig): void { - run( - `createdb -U ${config.user} -h ${config.host} -p ${config.port} -e ${config.database} -T ${config.template}`, - { - PGPASSWORD: config.password, - } - ); -} - -export function installExt( - config: PgConfig, - extensions: string[] | string -): void { - const extList = typeof extensions === 'string' ? extensions.split(',') : extensions; - - for (const extension of extList) { - run( - `psql --dbname "${config.database}" -c 'CREATE EXTENSION IF NOT EXISTS "${extension}" CASCADE;'`, - getPgEnv(config) - ); - } -} - -export function connectionString(config: PgConfig): string { - return `postgres://${config.user}:${config.password}@${config.host}:${config.port}/${config.database}`; -} - -export function createTemplateFromBase(config: PgConfig, base: string, template: string): void { - run(`createdb -T ${base} ${template}`, getPgEnv(config)); - run(`psql -c "UPDATE pg_database SET datistemplate = true WHERE datname = '${template}';"`, getPgEnv(config)); -} - -export function cleanupTemplateDatabase(config: PgConfig, template: string): void { - try { - run(`psql -c "UPDATE pg_database SET datistemplate = false WHERE datname = '${template}'"`, getPgEnv(config)); - } catch {} - safeDropDb(config, template); -} - -export function createRole(config: PgConfig, role: string, password: string): void { - run( - `psql -d ${config.database} -c "CREATE ROLE ${role} WITH LOGIN PASSWORD '${password}';"`, - getPgEnv(config) - ); -} - -export function grantConnect(config: PgConfig, role: string): void { - run( - `psql -d ${config.database} -c "GRANT CONNECT ON DATABASE ${config.database} TO ${role};"`, - getPgEnv(config) - ); -} diff --git a/packages/postgres-test/src/index.ts b/packages/postgres-test/src/index.ts index b6d2732bd5..313a36d930 100644 --- a/packages/postgres-test/src/index.ts +++ b/packages/postgres-test/src/index.ts @@ -1,4 +1,2 @@ export * from './connection'; -export * from './db'; -export * from './types'; export * from './utils'; \ No newline at end of file diff --git a/packages/postgres-test/src/manager.ts b/packages/postgres-test/src/manager.ts new file mode 100644 index 0000000000..59e8b681bd --- /dev/null +++ b/packages/postgres-test/src/manager.ts @@ -0,0 +1,140 @@ +import { Pool } from 'pg'; +import chalk from 'chalk'; +import { DbAdmin } from './admin'; +import { PgConfig } from '@launchql/types'; +import { PgTestClient } from './client'; + +const SYS_EVENTS = ['SIGTERM']; + +const end = (pool: Pool) => { + try { + if ((pool as any).ended || (pool as any).ending) { + console.warn(chalk.yellow('⚠️ pg pool already ended or ending')); + return; + } + pool.end(); + } catch (err) { + console.error(chalk.red('❌ pg pool termination error:'), err); + } +}; + +export class PgTestConnector { + private static instance: PgTestConnector; + + private readonly clients = new Set(); + private readonly pgPools = new Map(); + private readonly seenDbConfigs = new Map(); + + private verbose = false; + + private constructor(verbose = false) { + this.verbose = verbose; + + SYS_EVENTS.forEach((event) => { + process.on(event, () => { + this.log(chalk.magenta(`⏹ Received ${event}, closing all connections...`)); + this.closeAll(); + }); + }); + } + + static getInstance(verbose = false): PgTestConnector { + if (!PgTestConnector.instance) { + PgTestConnector.instance = new PgTestConnector(verbose); + } + return PgTestConnector.instance; + } + + private log(...args: any[]) { + if (this.verbose) console.log(...args); + } + + private poolKey(config: PgConfig): string { + return `${config.user}@${config.host}:${config.port}/${config.database}`; + } + + private dbKey(config: PgConfig): string { + return `${config.host}:${config.port}/${config.database}`; + } + + getAdmin(config: PgConfig): DbAdmin { + return new DbAdmin(config, this.verbose); + } + + getPool(config: PgConfig): Pool { + const key = this.poolKey(config); + if (!this.pgPools.has(key)) { + const pool = new Pool(config); + this.pgPools.set(key, pool); + this.log(chalk.blue(`πŸ“˜ Created new pg pool: ${chalk.white(key)}`)); + } + return this.pgPools.get(key)!; + } + + getClient(config: PgConfig): PgTestClient { + const client = new PgTestClient(config); + this.clients.add(client); + + const key = this.dbKey(config); + this.seenDbConfigs.set(key, config); + + this.log(chalk.green(`πŸ”Œ New PgTestClient connected to ${config.database}`)); + return client; + } + + async closeAll(): Promise { + this.log(chalk.cyan('\n🧹 Closing all PgTestClients...')); + await Promise.all( + Array.from(this.clients).map(async (client) => { + try { + await client.close(); + this.log(chalk.green(`βœ… Closed client for ${client.config.database}`)); + } catch (err) { + console.warn(chalk.red(`❌ Error closing PgTestClient for ${client.config.database}:`), err); + } + }) + ); + this.clients.clear(); + + this.log(chalk.cyan('\n🧯 Disposing pg pools...')); + for (const [key, pool] of this.pgPools.entries()) { + this.log(chalk.gray(`🧯 Disposing pg pool [${key}]`)); + end(pool); + } + this.pgPools.clear(); + + this.log(chalk.cyan('\nπŸ—‘οΈ Dropping seen databases...')); + await Promise.all( + Array.from(this.seenDbConfigs.values()).map(async (config) => { + try { + const admin = new DbAdmin(config, this.verbose); + admin.drop(); + this.log(chalk.yellow(`🧨 Dropped database: ${chalk.white(config.database)}`)); + } catch (err) { + console.warn(chalk.red(`❌ Failed to drop database ${config.database}:`), err); + } + }) + ); + this.seenDbConfigs.clear(); + + this.log(chalk.green('\nβœ… All PgTestClients closed, pools disposed, databases dropped.')); + } + + close(): void { + this.closeAll(); + } + + drop(config: PgConfig): void { + const key = this.dbKey(config); + const admin = new DbAdmin(config, this.verbose); + admin.drop(); + this.log(chalk.red(`🧨 Dropped database: ${chalk.white(config.database)}`)); + this.seenDbConfigs.delete(key); + } + + kill(client: PgTestClient): void { + client.close(); + this.drop(client.config); + } + +} diff --git a/packages/postgres-test/src/types.ts b/packages/postgres-test/src/types.ts deleted file mode 100644 index fffe795126..0000000000 --- a/packages/postgres-test/src/types.ts +++ /dev/null @@ -1,7 +0,0 @@ -export interface PgConfig { - database: string; - host: string; - password: string; - port: number; - user: string; -} \ No newline at end of file diff --git a/packages/postgres-test/src/utils.ts b/packages/postgres-test/src/utils.ts index f802364046..756a3b7ec6 100644 --- a/packages/postgres-test/src/utils.ts +++ b/packages/postgres-test/src/utils.ts @@ -1,10 +1,11 @@ import { Client, Pool } from 'pg'; -import { createdb, dropdb, templatedb, installExt, grantConnect } from './db'; +// import { createdb, dropdb, templatedb, installExt, grantConnect } from './db'; import { connect, close } from './connection'; -import { PgConfig } from './types'; +import { getPgEnvOptions, PgConfig } from '@launchql/types'; import { getEnvOptions } from '@launchql/types'; import { randomUUID } from 'crypto'; -import { PgWrapper } from './wrapper'; +import { PgTestClient } from './client'; +import { DbAdmin } from './admin'; export interface TestOptions { hot?: boolean; @@ -21,39 +22,37 @@ export function getOpts(configOpts: TestOptions = {}): TestOptions { }; } -export function getConnection(configOpts: TestOptions, database?: string): PgWrapper { - const envOpts = getEnvOptions(); +export function getConnection(configOpts: TestOptions, database?: string): PgTestClient { const opts = getOpts(configOpts); const dbName = database || `${opts.prefix}-${Date.now()}`; - const config: PgConfig = { - database: dbName, - user: envOpts.pg.user, - password: envOpts.pg.password, - port: envOpts.pg.port, - host: envOpts.pg.host - }; + const config = getPgEnvOptions({ + database: dbName + }); + + const admin = new DbAdmin(config); if (process.env.TEST_DB) { config.database = process.env.TEST_DB; } else if (opts.hot) { - createdb(config); - installExt(config, opts.extensions); + admin.create(config.database); + admin.installExtensions(opts.extensions); } else if (opts.template) { - templatedb({ ...config, template: opts.template }); + admin.createFromTemplate(opts.template, config.database); + // admin.createFromTemplate(config.database, opts.template); } else { - createdb(config); - installExt(config, opts.extensions); + admin.create(config.database); + admin.installExtensions(opts.extensions); } return connect(config); } -export function closeConnection(db: PgWrapper): void { - db.kill(); +export function closeConnection(db: PgTestClient): void { + // db.kill(); } -export function connectTest(database: string, user: string, password: string): PgWrapper { +export function connectTest(database: string, user: string, password: string): PgTestClient { const envOpts = getEnvOptions(); const config: PgConfig = { port: envOpts.pg.port, @@ -65,7 +64,7 @@ export function connectTest(database: string, user: string, password: string): P return connect(config); } -export async function createUserRole(db: PgWrapper, user: string, password: string): Promise { +export async function createUserRole(db: PgTestClient, user: string, password: string): Promise { await db.query(` DO $$ BEGIN @@ -80,34 +79,28 @@ export async function createUserRole(db: PgWrapper, user: string, password: stri `); } -export function closeConnections({ db, conn }: { db: PgWrapper; conn: PgWrapper }): void { +export function closeConnections({ db, conn }: { db: PgTestClient; conn: PgTestClient }): void { conn.close(); closeConnection(db); } export const getConnections = async () => { - const opts = getEnvOptions({ - pg: { - database: `db-${randomUUID()}` - } - }); - const config: PgConfig = { - user: opts.pg.user, - port: opts.pg.port, - password: opts.pg.password, - host: opts.pg.host, - database: opts.pg.database - }; + const config = getPgEnvOptions({ + database: `db-${randomUUID()}` + }) + + const admin = new DbAdmin(config); const db = await getConnection({ }); await createUserRole(db, 'app_user', 'app_password'); - await grantConnect(config, 'app_user'); + admin.grantConnect('app_user', config.database); + // await grantConnect(config, 'app_user'); - const conn = await connectTest(opts.pg.database, 'app_user', 'app_password'); + const conn = await connectTest(config.database, 'app_user', 'app_password'); conn.setContext({ role: 'anonymous' }); diff --git a/packages/postgres-test/test-utils/index.ts b/packages/postgres-test/test-utils/index.ts index b4447cec71..edf10e5f31 100644 --- a/packages/postgres-test/test-utils/index.ts +++ b/packages/postgres-test/test-utils/index.ts @@ -1,9 +1,2 @@ import path from 'path'; import fs from 'fs'; -import { run } from '../src'; - -export function runSQLFile(file: string, database: string): void { - const filePath = path.resolve(__dirname, '../sql', file); - if (!fs.existsSync(filePath)) throw new Error(`Missing SQL file: ${file}`); - run(`psql -f ${filePath} ${database}`); -} diff --git a/packages/server-utils/src/pg.ts b/packages/server-utils/src/pg.ts index 3094a43e27..04888bf800 100644 --- a/packages/server-utils/src/pg.ts +++ b/packages/server-utils/src/pg.ts @@ -1,7 +1,7 @@ import pg from 'pg'; import { pgCache } from './lru'; -import { PostgresOptions } from '@launchql/types'; +import { PgConfig } from '@launchql/types'; export const getDbString = ( user: string, @@ -18,7 +18,7 @@ export const getRootPgPool = ({ host, port, database, -}: PostgresOptions): pg.Pool => { +}: PgConfig): pg.Pool => { if (pgCache.has(database)) { const cached = pgCache.get(database); if (cached) return cached; diff --git a/packages/types/src/env.ts b/packages/types/src/env.ts index 62611c1d7f..0c68086172 100644 --- a/packages/types/src/env.ts +++ b/packages/types/src/env.ts @@ -1,4 +1,5 @@ -import { getMergedOptions, LaunchQLOptions } from './launchql'; +import deepmerge from 'deepmerge'; +import { getMergedOptions, LaunchQLOptions, PgConfig } from './launchql'; const parseEnvNumber = (val?: string): number | undefined => { const num = Number(val); @@ -19,8 +20,14 @@ export const getEnvOptions = (overrides: LaunchQLOptions = {}): LaunchQLOptions }); }; +export const getPgEnvOptions = (overrides: Partial = {}): PgConfig => { + const envOpts = getPgEnvVars(); + const options = deepmerge(envOpts, overrides); + // if you need to sanitize... + return options; +}; -export const getEnvVars = (): LaunchQLOptions => { +const getEnvVars = (): LaunchQLOptions => { const { PORT, SERVER_HOST, @@ -74,3 +81,21 @@ export const getEnvVars = (): LaunchQLOptions => { } }; }; + +const getPgEnvVars = (): PgConfig => { + const { + PGHOST, + PGPORT, + PGUSER, + PGPASSWORD, + PGDATABASE + } = process.env; + + return { + ...(PGHOST && { host: PGHOST }), + ...(PGPORT && { port: parseEnvNumber(PGPORT) }), + ...(PGUSER && { user: PGUSER }), + ...(PGPASSWORD && { password: PGPASSWORD }), + ...(PGDATABASE && { database: PGDATABASE }), + }; +}; diff --git a/packages/types/src/launchql.ts b/packages/types/src/launchql.ts index f1c61369ac..ea1ba996d1 100644 --- a/packages/types/src/launchql.ts +++ b/packages/types/src/launchql.ts @@ -43,17 +43,16 @@ declare module 'express-serve-static-core' { } } - -export interface PostgresOptions { - host?: string; - port?: number; - user?: string; - password?: string; - database?: string; +export interface PgConfig { + host: string; + port: number; + user: string; + password: string; + database: string; } export interface LaunchQLOptions { - pg?: PostgresOptions; + pg?: Partial; graphile?: { isPublic?: boolean; schema?: string | string[]; @@ -120,8 +119,6 @@ export const launchqlDefaults: LaunchQLOptions = { } }; - - export const getMergedOptions = (options: LaunchQLOptions): LaunchQLOptions => { options = deepmerge(launchqlDefaults, options ?? {}); // if you need to sanitize... From 6bdc657b3aa47e99c358448302d48f24ecaa9e86 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sat, 17 May 2025 01:23:43 -0700 Subject: [PATCH 04/29] user role --- .../__tests__/postgres-test.grants.test.ts | 3 +- .../__tests__/postgres-test.template.test.ts | 3 +- .../__tests__/postgres-test.test.ts | 3 +- packages/postgres-test/src/admin.ts | 24 +++++++-- packages/postgres-test/src/utils.ts | 52 ++++++------------- 5 files changed, 41 insertions(+), 44 deletions(-) diff --git a/packages/postgres-test/__tests__/postgres-test.grants.test.ts b/packages/postgres-test/__tests__/postgres-test.grants.test.ts index d79f3f30c7..f9e5e515a6 100644 --- a/packages/postgres-test/__tests__/postgres-test.grants.test.ts +++ b/packages/postgres-test/__tests__/postgres-test.grants.test.ts @@ -2,7 +2,6 @@ import { getPgEnvOptions, PgConfig } from '@launchql/types'; import { getConnection, - closeConnection, Connection, } from '../src'; import { randomUUID } from 'crypto'; @@ -39,7 +38,7 @@ describe('Postgres Test Framework', () => { let db: PgTestClient; afterEach(() => { - if (db) closeConnection(db); + // if (db) closeConnection(db); }); it('creates a test DB with hot mode (FAST_TEST)', () => { diff --git a/packages/postgres-test/__tests__/postgres-test.template.test.ts b/packages/postgres-test/__tests__/postgres-test.template.test.ts index 4f317e741e..28ad6fbd24 100644 --- a/packages/postgres-test/__tests__/postgres-test.template.test.ts +++ b/packages/postgres-test/__tests__/postgres-test.template.test.ts @@ -3,7 +3,6 @@ import path from 'path'; import { getConnection, - closeConnection, Connection } from '../src'; import { PgTestClient } from '../src/client'; @@ -43,7 +42,7 @@ describe('Template Database Test', () => { let db: PgTestClient; afterEach(() => { - if (db) closeConnection(db); + // if (db) closeConnection(db); }); it('creates a test DB from a template', () => { diff --git a/packages/postgres-test/__tests__/postgres-test.test.ts b/packages/postgres-test/__tests__/postgres-test.test.ts index 62eab4ff55..039c4d38f9 100644 --- a/packages/postgres-test/__tests__/postgres-test.test.ts +++ b/packages/postgres-test/__tests__/postgres-test.test.ts @@ -2,7 +2,6 @@ import { getPgEnvOptions, PgConfig } from '@launchql/types'; import { getConnection, - closeConnection, Connection } from '../src'; import { randomUUID } from 'crypto'; @@ -22,7 +21,7 @@ describe('Postgres Test Framework', () => { let db: PgTestClient; afterEach(() => { - if (db) closeConnection(db); + // if (db) closeConnection(db); }); it('creates a test DB with hot mode (FAST_TEST)', () => { diff --git a/packages/postgres-test/src/admin.ts b/packages/postgres-test/src/admin.ts index f62457a83a..5be5238a33 100644 --- a/packages/postgres-test/src/admin.ts +++ b/packages/postgres-test/src/admin.ts @@ -6,7 +6,7 @@ export class DbAdmin { constructor( private config: PgConfig, private verbose: boolean = false - ) {} + ) { } private getEnv(): Record { return { @@ -81,7 +81,7 @@ export class DbAdmin { cleanupTemplate(template: string): void { try { this.run(`psql -c "UPDATE pg_database SET datistemplate = false WHERE datname = '${template}'"`); - } catch {} + } catch { } this.safeDropDb(template); } @@ -95,11 +95,29 @@ export class DbAdmin { this.run(`psql -d "${db}" -c "GRANT CONNECT ON DATABASE ${db} TO ${role};"`); } + createUserRole(user: string, password: string, dbName?: string): void { + const db = dbName ?? this.config.database; + + const sql = ` + DO $$ + BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = '${user}') THEN + CREATE ROLE ${user} LOGIN PASSWORD '${password}'; + GRANT anonymous TO ${user}; + GRANT authenticated TO ${user}; + END IF; + END $$; + `.trim(); + + this.run(`psql -d "${db}" -c "${sql.replace(/\n/g, ' ')}"`); + } + + loadSql(file: string, dbName: string): void { if (!existsSync(file)) { throw new Error(`Missing SQL file: ${file}`); } this.run(`psql -f ${file} ${dbName}`); } - + } diff --git a/packages/postgres-test/src/utils.ts b/packages/postgres-test/src/utils.ts index 756a3b7ec6..3c21842443 100644 --- a/packages/postgres-test/src/utils.ts +++ b/packages/postgres-test/src/utils.ts @@ -39,7 +39,6 @@ export function getConnection(configOpts: TestOptions, database?: string): PgTes admin.installExtensions(opts.extensions); } else if (opts.template) { admin.createFromTemplate(opts.template, config.database); - // admin.createFromTemplate(config.database, opts.template); } else { admin.create(config.database); admin.installExtensions(opts.extensions); @@ -48,63 +47,46 @@ export function getConnection(configOpts: TestOptions, database?: string): PgTes return connect(config); } -export function closeConnection(db: PgTestClient): void { - // db.kill(); -} - -export function connectTest(database: string, user: string, password: string): PgTestClient { - const envOpts = getEnvOptions(); - const config: PgConfig = { - port: envOpts.pg.port, - host: envOpts.pg.host, +export function getTestConnection(database: string, user: string, password: string): PgTestClient { + const config = getPgEnvOptions({ database, user, password - }; + }) + console.log(config); + console.log(config); + console.log(config); return connect(config); } -export async function createUserRole(db: PgTestClient, user: string, password: string): Promise { - await db.query(` - DO $$ - BEGIN - IF NOT EXISTS ( - SELECT 1 FROM pg_roles WHERE rolname = '${user}' - ) THEN - CREATE ROLE ${user} LOGIN PASSWORD '${password}'; - GRANT anonymous TO ${user}; - GRANT authenticated TO ${user}; - END IF; - END $$; - `); -} - export function closeConnections({ db, conn }: { db: PgTestClient; conn: PgTestClient }): void { conn.close(); - closeConnection(db); + // closeConnection(db); } export const getConnections = async () => { const config = getPgEnvOptions({ database: `db-${randomUUID()}` - }) + }); - const admin = new DbAdmin(config); + const app_user = 'app_user'; + const app_password = 'app_password'; - const db = await getConnection({ + const admin = new DbAdmin(config); - }); + const db = await getConnection({ }); - await createUserRole(db, 'app_user', 'app_password'); - admin.grantConnect('app_user', config.database); - // await grantConnect(config, 'app_user'); + admin.createUserRole(app_user, app_password, config.database); + admin.grantConnect(app_user, config.database); - const conn = await connectTest(config.database, 'app_user', 'app_password'); + const conn = await getTestConnection(config.database, app_user, app_password); conn.setContext({ role: 'anonymous' }); + // const res = await conn.query(`SELECT 1`); + const teardown = async () => { await closeConnections({ db, conn }); }; From 8b4ab1c99dc0395c704e153035229c0f939a0aa8 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sat, 17 May 2025 02:20:20 -0700 Subject: [PATCH 05/29] wip --- .../postgres-test.connections.test.ts | 31 +++++++++ .../__tests__/postgres-test.records.test.ts | 66 +++++++++++++++++++ packages/postgres-test/src/admin.ts | 40 +++++++---- packages/postgres-test/src/connect.ts | 43 ++++++++++++ packages/postgres-test/src/index.ts | 2 +- .../src/{connection.ts => legacy-connect.ts} | 0 packages/postgres-test/src/manager.ts | 4 +- packages/postgres-test/src/stream.ts | 61 +++++++++++++++++ packages/postgres-test/src/utils.ts | 15 +---- 9 files changed, 234 insertions(+), 28 deletions(-) create mode 100644 packages/postgres-test/__tests__/postgres-test.connections.test.ts create mode 100644 packages/postgres-test/__tests__/postgres-test.records.test.ts create mode 100644 packages/postgres-test/src/connect.ts rename packages/postgres-test/src/{connection.ts => legacy-connect.ts} (100%) create mode 100644 packages/postgres-test/src/stream.ts diff --git a/packages/postgres-test/__tests__/postgres-test.connections.test.ts b/packages/postgres-test/__tests__/postgres-test.connections.test.ts new file mode 100644 index 0000000000..f745f80d85 --- /dev/null +++ b/packages/postgres-test/__tests__/postgres-test.connections.test.ts @@ -0,0 +1,31 @@ +import { getPgEnvOptions } from '@launchql/types'; +import { Connection, getConnections } from '../src'; +import { PgTestClient } from '../src/client'; + +// let client: PgTestClient; +let conn: PgTestClient; +let db: PgTestClient; +let teardown: any; +beforeAll(async () => { +// client = Connection.connect(getPgEnvOptions()) + ({ conn, db, teardown } = await getConnections()) +}); + +afterAll(async () => { + await Connection.getManager().closeAll(); + // await teardown(); +}); + +describe('Postgres Test Framework', () => { + let db: PgTestClient; + + afterEach(() => { + if (db) db.afterEach() + }); + + it('creates a test DB', async () => { + const result = await conn.query('SELECT 1'); + console.log(result); + + }); +}); diff --git a/packages/postgres-test/__tests__/postgres-test.records.test.ts b/packages/postgres-test/__tests__/postgres-test.records.test.ts new file mode 100644 index 0000000000..a9895b5956 --- /dev/null +++ b/packages/postgres-test/__tests__/postgres-test.records.test.ts @@ -0,0 +1,66 @@ +import { getConnections } from '../src/connect'; +import { PgTestClient } from '../src/client'; + +let conn: PgTestClient; +let db: PgTestClient; +let teardown: () => Promise; + +const setupSchemaSQL = ` + CREATE TABLE users ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL + ); + + CREATE TABLE posts ( + id SERIAL PRIMARY KEY, + user_id INT NOT NULL REFERENCES users(id), + content TEXT NOT NULL + ); +`; + +const seedDataSQL = ` + INSERT INTO users (name) VALUES ('Alice'), ('Bob'); + INSERT INTO posts (user_id, content) VALUES + (1, 'Hello world!'), + (2, 'Graphile is cool!'); +`; + +beforeAll(async () => { + ({ conn, db, teardown } = await getConnections()); + // create schema + seed *once* + await db.query(setupSchemaSQL); + await db.query(seedDataSQL); +}); + +afterAll(async () => { + await teardown(); +}); + +describe('Postgres Test Framework', () => { + beforeEach(async () => { + await db.beforeEach(); // BEGIN + SAVEPOINT + }); + + afterEach(async () => { + await db.afterEach(); // ROLLBACK TO SAVEPOINT + COMMIT + }); + + it('should have 2 users initially', async () => { + const { rows } = await db.query('SELECT COUNT(*) FROM users'); + expect(rows[0].count).toBe('2'); + }); + + it('inserts a user but rollback leaves baseline intact', async () => { + await db.query(`INSERT INTO users (name) VALUES ('Carol')`); + let res = await db.query('SELECT COUNT(*) FROM users'); + expect(res.rows[0].count).toBe('3'); // inside this tx + + // after rollback (next test) we’ll still see 2 + }); + + it('still sees 2 users after previous insert test', async () => { + const { rows } = await db.query('SELECT COUNT(*) FROM users'); + expect(rows[0].count).toBe('2'); + }); + +}); diff --git a/packages/postgres-test/src/admin.ts b/packages/postgres-test/src/admin.ts index 5be5238a33..aa6881d9eb 100644 --- a/packages/postgres-test/src/admin.ts +++ b/packages/postgres-test/src/admin.ts @@ -1,6 +1,7 @@ import { execSync } from 'child_process'; import { PgConfig } from '@launchql/types'; import { existsSync } from 'fs'; +import { streamSql as stream } from './stream'; export class DbAdmin { constructor( @@ -42,10 +43,9 @@ export class DbAdmin { this.safeDropDb(dbName ?? this.config.database); } - dropTemplate(dbName?: string): void { - const db = dbName ?? this.config.database; - this.run(`psql -c "UPDATE pg_database SET datistemplate='false' WHERE datname='${db}';"`); - this.drop(db); + dropTemplate(dbName: string): void { + this.run(`psql -c "UPDATE pg_database SET datistemplate='false' WHERE datname='${dbName}';"`); + this.drop(dbName); } create(dbName?: string): void { @@ -85,19 +85,25 @@ export class DbAdmin { this.safeDropDb(template); } - createRole(role: string, password: string, dbName?: string): void { + async createRole(role: string, password: string, dbName?: string): Promise { const db = dbName ?? this.config.database; - this.run(`psql -d "${db}" -c "CREATE ROLE ${role} WITH LOGIN PASSWORD '${password}';"`); + const sql = `CREATE ROLE ${role} WITH LOGIN PASSWORD '${password}';`; + await this.streamSql(sql, db); } - - grantConnect(role: string, dbName?: string): void { + + async grantRole(role: string, user: string, dbName?: string): Promise { const db = dbName ?? this.config.database; - this.run(`psql -d "${db}" -c "GRANT CONNECT ON DATABASE ${db} TO ${role};"`); + const sql = `GRANT ${role} TO ${user};`; + await this.streamSql(sql, db); } - - createUserRole(user: string, password: string, dbName?: string): void { + + async grantConnect(role: string, dbName?: string): Promise { const db = dbName ?? this.config.database; - + const sql = `GRANT CONNECT ON DATABASE "${db}" TO ${role};`; + await this.streamSql(sql, db); + } + + async createUserRole(user: string, password: string, dbName: string): Promise { const sql = ` DO $$ BEGIN @@ -109,10 +115,9 @@ export class DbAdmin { END $$; `.trim(); - this.run(`psql -d "${db}" -c "${sql.replace(/\n/g, ' ')}"`); + this.streamSql(sql, dbName); } - loadSql(file: string, dbName: string): void { if (!existsSync(file)) { throw new Error(`Missing SQL file: ${file}`); @@ -120,4 +125,11 @@ export class DbAdmin { this.run(`psql -f ${file} ${dbName}`); } + async streamSql(sql: string, dbName: string): Promise { + await stream({ + ...this.config, + database: dbName + }, sql); + } + } diff --git a/packages/postgres-test/src/connect.ts b/packages/postgres-test/src/connect.ts new file mode 100644 index 0000000000..705a366312 --- /dev/null +++ b/packages/postgres-test/src/connect.ts @@ -0,0 +1,43 @@ +import { randomUUID } from 'crypto'; +import { getPgEnvOptions, PgConfig } from '@launchql/types'; +import { PgTestConnector } from './manager'; +import { DbAdmin } from './admin'; +import { getTestConnection } from './utils'; // adjust as needed + +let manager: PgTestConnector; +export const getConnections = async () => { + const config: PgConfig = getPgEnvOptions({ + database: `db-${randomUUID()}` + }); + + const app_user = 'app_user'; + const app_password = 'app_password'; + + const admin = new DbAdmin(config); + + // Create the test database + admin.create(config.database); + + // Main admin client (optional unless needed elsewhere) + manager = PgTestConnector.getInstance(); + const db = manager.getClient(config); + + // Set up test role + admin.createUserRole(app_user, app_password, config.database); + admin.grantConnect(app_user, config.database); + + // App user connection + const conn = manager.getClient({ + ...config, + user: app_user, + password: app_password + }) +// const conn = await getTestConnection(config.database, app_user, app_password); + conn.setContext({ role: 'anonymous' }); + + const teardown = async () => { + await manager.closeAll(); + }; + + return { db, conn, teardown }; +}; diff --git a/packages/postgres-test/src/index.ts b/packages/postgres-test/src/index.ts index 313a36d930..687d076f53 100644 --- a/packages/postgres-test/src/index.ts +++ b/packages/postgres-test/src/index.ts @@ -1,2 +1,2 @@ -export * from './connection'; +export * from './legacy-connect'; export * from './utils'; \ No newline at end of file diff --git a/packages/postgres-test/src/connection.ts b/packages/postgres-test/src/legacy-connect.ts similarity index 100% rename from packages/postgres-test/src/connection.ts rename to packages/postgres-test/src/legacy-connect.ts diff --git a/packages/postgres-test/src/manager.ts b/packages/postgres-test/src/manager.ts index 59e8b681bd..66128bb6ae 100644 --- a/packages/postgres-test/src/manager.ts +++ b/packages/postgres-test/src/manager.ts @@ -107,7 +107,9 @@ export class PgTestConnector { await Promise.all( Array.from(this.seenDbConfigs.values()).map(async (config) => { try { - const admin = new DbAdmin(config, this.verbose); + // somehow an "admin" db had app_user creds? + const admin = new DbAdmin({...config, user: 'postgres', password: 'password'}, this.verbose); + // console.log(config); admin.drop(); this.log(chalk.yellow(`🧨 Dropped database: ${chalk.white(config.database)}`)); } catch (err) { diff --git a/packages/postgres-test/src/stream.ts b/packages/postgres-test/src/stream.ts new file mode 100644 index 0000000000..a9cb88d1d1 --- /dev/null +++ b/packages/postgres-test/src/stream.ts @@ -0,0 +1,61 @@ +import { spawn } from 'child_process'; +import { Readable } from 'stream'; +import { env } from 'process'; +import { PgConfig } from '@launchql/types'; + +function setArgs(config: PgConfig): string[] { + const args = [ + '-U', config.user, + '-h', config.host, + '-d', config.database + ]; + if (config.port) { + args.push('-p', String(config.port)); + } + return args; +} + +// Converts a string to a readable stream (replaces streamify-string) +function stringToStream(text: string): Readable { + const stream = new Readable({ + read() { + this.push(text); + this.push(null); + } + }); + return stream; +} + +export async function streamSql(config: PgConfig, sql: string): Promise { + const args = setArgs(config); + + return new Promise((resolve, reject) => { + const sqlStream = stringToStream(sql); + + // TODO set env vars! + const proc = spawn('psql', args, { + env: { + ...env, + PGUSER: config.user, + PGHOST: config.host, + PGDATABASE: config.database, + PGPASSWORD: config.password, + // PGPORT: config.port + } + }); + + sqlStream.pipe(proc.stdin); + + proc.on('close', (code) => { + resolve(); + }); + + proc.on('error', (error) => { + reject(error); + }); + + proc.stderr.on('data', (data: Buffer) => { + reject(new Error(data.toString())); + }); + }); +} diff --git a/packages/postgres-test/src/utils.ts b/packages/postgres-test/src/utils.ts index 3c21842443..cd93e5511a 100644 --- a/packages/postgres-test/src/utils.ts +++ b/packages/postgres-test/src/utils.ts @@ -1,6 +1,6 @@ import { Client, Pool } from 'pg'; // import { createdb, dropdb, templatedb, installExt, grantConnect } from './db'; -import { connect, close } from './connection'; +import { connect, close } from './legacy-connect'; import { getPgEnvOptions, PgConfig } from '@launchql/types'; import { getEnvOptions } from '@launchql/types'; import { randomUUID } from 'crypto'; @@ -53,9 +53,6 @@ export function getTestConnection(database: string, user: string, password: stri user, password }) - console.log(config); - console.log(config); - console.log(config); return connect(config); } @@ -75,6 +72,8 @@ export const getConnections = async () => { const admin = new DbAdmin(config); + admin.create(config.database); + const db = await getConnection({ }); admin.createUserRole(app_user, app_password, config.database); @@ -85,17 +84,9 @@ export const getConnections = async () => { role: 'anonymous' }); - // const res = await conn.query(`SELECT 1`); - const teardown = async () => { await closeConnections({ db, conn }); }; return { db, conn, teardown }; }; - - -// export async function grantConnect(db: any, user: string): Promise { -// await db.query(`GRANT CONNECT ON DATABASE "${db.database}" TO ${user};`); -// } - From 913f5597d6b254709faf2a4400ed4585c564e2df Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sat, 17 May 2025 02:24:56 -0700 Subject: [PATCH 06/29] cleanup --- .../postgres-test.connections.test.ts | 8 ++-- packages/postgres-test/src/connect.ts | 1 - packages/postgres-test/src/utils.ts | 46 +------------------ 3 files changed, 4 insertions(+), 51 deletions(-) diff --git a/packages/postgres-test/__tests__/postgres-test.connections.test.ts b/packages/postgres-test/__tests__/postgres-test.connections.test.ts index f745f80d85..7e73d83caf 100644 --- a/packages/postgres-test/__tests__/postgres-test.connections.test.ts +++ b/packages/postgres-test/__tests__/postgres-test.connections.test.ts @@ -1,19 +1,17 @@ import { getPgEnvOptions } from '@launchql/types'; -import { Connection, getConnections } from '../src'; import { PgTestClient } from '../src/client'; +import { getConnections } from '../src/connect'; // let client: PgTestClient; let conn: PgTestClient; let db: PgTestClient; let teardown: any; beforeAll(async () => { -// client = Connection.connect(getPgEnvOptions()) ({ conn, db, teardown } = await getConnections()) }); afterAll(async () => { - await Connection.getManager().closeAll(); - // await teardown(); + await teardown(); }); describe('Postgres Test Framework', () => { @@ -25,7 +23,7 @@ describe('Postgres Test Framework', () => { it('creates a test DB', async () => { const result = await conn.query('SELECT 1'); - console.log(result); + // console.log(result); }); }); diff --git a/packages/postgres-test/src/connect.ts b/packages/postgres-test/src/connect.ts index 705a366312..2fd63a4df3 100644 --- a/packages/postgres-test/src/connect.ts +++ b/packages/postgres-test/src/connect.ts @@ -2,7 +2,6 @@ import { randomUUID } from 'crypto'; import { getPgEnvOptions, PgConfig } from '@launchql/types'; import { PgTestConnector } from './manager'; import { DbAdmin } from './admin'; -import { getTestConnection } from './utils'; // adjust as needed let manager: PgTestConnector; export const getConnections = async () => { diff --git a/packages/postgres-test/src/utils.ts b/packages/postgres-test/src/utils.ts index cd93e5511a..19f8a0eb54 100644 --- a/packages/postgres-test/src/utils.ts +++ b/packages/postgres-test/src/utils.ts @@ -45,48 +45,4 @@ export function getConnection(configOpts: TestOptions, database?: string): PgTes } return connect(config); -} - -export function getTestConnection(database: string, user: string, password: string): PgTestClient { - const config = getPgEnvOptions({ - database, - user, - password - }) - return connect(config); -} - -export function closeConnections({ db, conn }: { db: PgTestClient; conn: PgTestClient }): void { - conn.close(); - // closeConnection(db); -} - -export const getConnections = async () => { - - const config = getPgEnvOptions({ - database: `db-${randomUUID()}` - }); - - const app_user = 'app_user'; - const app_password = 'app_password'; - - const admin = new DbAdmin(config); - - admin.create(config.database); - - const db = await getConnection({ }); - - admin.createUserRole(app_user, app_password, config.database); - admin.grantConnect(app_user, config.database); - - const conn = await getTestConnection(config.database, app_user, app_password); - conn.setContext({ - role: 'anonymous' - }); - - const teardown = async () => { - await closeConnections({ db, conn }); - }; - - return { db, conn, teardown }; -}; +} \ No newline at end of file From 4663a7e405d8636d9af4e4fc73e6c84bc3048770 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sat, 17 May 2025 02:38:05 -0700 Subject: [PATCH 07/29] test-client --- .../postgres-test.connections.test.ts | 78 +++++++++++++++---- .../__tests__/postgres-test.grants.test.ts | 2 +- .../__tests__/postgres-test.records.test.ts | 2 +- .../__tests__/postgres-test.template.test.ts | 2 +- .../__tests__/postgres-test.test.ts | 2 +- packages/postgres-test/src/legacy-connect.ts | 2 +- packages/postgres-test/src/manager.ts | 2 +- .../src/{client.ts => test-client.ts} | 0 packages/postgres-test/src/utils.ts | 2 +- 9 files changed, 72 insertions(+), 20 deletions(-) rename packages/postgres-test/src/{client.ts => test-client.ts} (100%) diff --git a/packages/postgres-test/__tests__/postgres-test.connections.test.ts b/packages/postgres-test/__tests__/postgres-test.connections.test.ts index 7e73d83caf..c5cdc20b33 100644 --- a/packages/postgres-test/__tests__/postgres-test.connections.test.ts +++ b/packages/postgres-test/__tests__/postgres-test.connections.test.ts @@ -1,29 +1,81 @@ -import { getPgEnvOptions } from '@launchql/types'; -import { PgTestClient } from '../src/client'; import { getConnections } from '../src/connect'; +import { PgTestClient } from '../src/test-client'; -// let client: PgTestClient; let conn: PgTestClient; let db: PgTestClient; -let teardown: any; +let teardown: () => Promise; + beforeAll(async () => { - ({ conn, db, teardown } = await getConnections()) + ({ conn, db, teardown } = await getConnections()); + + // Setup schema + seed ONCE globally + await db.query(` + CREATE TABLE users ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL + ); + CREATE TABLE posts ( + id SERIAL PRIMARY KEY, + user_id INT NOT NULL REFERENCES users(id), + content TEXT NOT NULL + ); + `); + + await db.query(` + INSERT INTO users (name) VALUES ('Alice'), ('Bob'); + INSERT INTO posts (user_id, content) VALUES + (1, 'Hello world!'), + (2, 'Graphile is cool!'); + `); }); afterAll(async () => { - await teardown(); + await teardown(); }); -describe('Postgres Test Framework', () => { - let db: PgTestClient; +describe('anonymous', () => { + beforeEach(async () => { + await db.beforeEach(); // this starts tx + savepoint + }); + + afterEach(async () => { + await db.afterEach(); // this rolls back and commits + }); + + it('inserts a user but rollback leaves baseline intact', async () => { + await db.query(`INSERT INTO users (name) VALUES ('Carol')`); + const res = await db.query('SELECT COUNT(*) FROM users'); + expect(res.rows[0].count).toBe('3'); + }); + + it('should still have 2 users after rollback', async () => { + const res = await db.query('SELECT COUNT(*) FROM users'); + expect(res.rows[0].count).toBe('2'); + }); - afterEach(() => { - if (db) db.afterEach() + it('runs under anonymous context', async () => { + const result = await conn.query('SELECT current_setting(\'role\', true) AS role'); + console.log(JSON.stringify({result}, null, 2)) + console.error(JSON.stringify({result}, null, 2)) + // expect(result.rows[0].role).toBe('anonymous'); }); +}); - it('creates a test DB', async () => { - const result = await conn.query('SELECT 1'); - // console.log(result); +describe('authenticated', () => { + beforeEach(async () => { + conn.setContext({ + role: 'authenticated' + }); + await conn.beforeEach(); // required for rollback later + }); + + afterEach(async () => { + await conn.afterEach(); // now safe to rollback + }); + it('runs under authenticated context', async () => { + const result = await conn.query('SELECT current_setting(\'role\', true) AS role'); + // expect(result.rows[0].role).toBe('authenticated'); + console.error('why no JWT') }); }); diff --git a/packages/postgres-test/__tests__/postgres-test.grants.test.ts b/packages/postgres-test/__tests__/postgres-test.grants.test.ts index f9e5e515a6..ad14090e31 100644 --- a/packages/postgres-test/__tests__/postgres-test.grants.test.ts +++ b/packages/postgres-test/__tests__/postgres-test.grants.test.ts @@ -5,7 +5,7 @@ import { Connection, } from '../src'; import { randomUUID } from 'crypto'; -import { PgTestClient } from '../src/client'; +import { PgTestClient } from '../src/test-client'; import { DbAdmin } from '../src/admin'; import { resolve } from 'path'; diff --git a/packages/postgres-test/__tests__/postgres-test.records.test.ts b/packages/postgres-test/__tests__/postgres-test.records.test.ts index a9895b5956..4a0057a1f2 100644 --- a/packages/postgres-test/__tests__/postgres-test.records.test.ts +++ b/packages/postgres-test/__tests__/postgres-test.records.test.ts @@ -1,5 +1,5 @@ import { getConnections } from '../src/connect'; -import { PgTestClient } from '../src/client'; +import { PgTestClient } from '../src/test-client'; let conn: PgTestClient; let db: PgTestClient; diff --git a/packages/postgres-test/__tests__/postgres-test.template.test.ts b/packages/postgres-test/__tests__/postgres-test.template.test.ts index 28ad6fbd24..2d99683dd9 100644 --- a/packages/postgres-test/__tests__/postgres-test.template.test.ts +++ b/packages/postgres-test/__tests__/postgres-test.template.test.ts @@ -5,7 +5,7 @@ import { getConnection, Connection } from '../src'; -import { PgTestClient } from '../src/client'; +import { PgTestClient } from '../src/test-client'; import { DbAdmin } from '../src/admin'; const sql = (file: string) => path.resolve(__dirname, '../sql', file); diff --git a/packages/postgres-test/__tests__/postgres-test.test.ts b/packages/postgres-test/__tests__/postgres-test.test.ts index 039c4d38f9..d138dcb756 100644 --- a/packages/postgres-test/__tests__/postgres-test.test.ts +++ b/packages/postgres-test/__tests__/postgres-test.test.ts @@ -5,7 +5,7 @@ import { Connection } from '../src'; import { randomUUID } from 'crypto'; -import { PgTestClient } from '../src/client'; +import { PgTestClient } from '../src/test-client'; const TEST_DB_BASE = `postgres_test_${randomUUID()}`; diff --git a/packages/postgres-test/src/legacy-connect.ts b/packages/postgres-test/src/legacy-connect.ts index 07838ddd31..16433575f1 100644 --- a/packages/postgres-test/src/legacy-connect.ts +++ b/packages/postgres-test/src/legacy-connect.ts @@ -1,4 +1,4 @@ -import { PgTestClient } from './client'; +import { PgTestClient } from './test-client'; import { PgTestConnector } from './manager'; import { randomUUID } from 'crypto'; import { getPgEnvOptions, PgConfig } from '@launchql/types'; diff --git a/packages/postgres-test/src/manager.ts b/packages/postgres-test/src/manager.ts index 66128bb6ae..633b70faab 100644 --- a/packages/postgres-test/src/manager.ts +++ b/packages/postgres-test/src/manager.ts @@ -2,7 +2,7 @@ import { Pool } from 'pg'; import chalk from 'chalk'; import { DbAdmin } from './admin'; import { PgConfig } from '@launchql/types'; -import { PgTestClient } from './client'; +import { PgTestClient } from './test-client'; const SYS_EVENTS = ['SIGTERM']; diff --git a/packages/postgres-test/src/client.ts b/packages/postgres-test/src/test-client.ts similarity index 100% rename from packages/postgres-test/src/client.ts rename to packages/postgres-test/src/test-client.ts diff --git a/packages/postgres-test/src/utils.ts b/packages/postgres-test/src/utils.ts index 19f8a0eb54..24b04243d9 100644 --- a/packages/postgres-test/src/utils.ts +++ b/packages/postgres-test/src/utils.ts @@ -4,7 +4,7 @@ import { connect, close } from './legacy-connect'; import { getPgEnvOptions, PgConfig } from '@launchql/types'; import { getEnvOptions } from '@launchql/types'; import { randomUUID } from 'crypto'; -import { PgTestClient } from './client'; +import { PgTestClient } from './test-client'; import { DbAdmin } from './admin'; export interface TestOptions { From 2cc6c509d7c2f7adf28e2ea473bacdcfff8a1978 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sat, 17 May 2025 02:43:19 -0700 Subject: [PATCH 08/29] kill() --- packages/cli/src/commands/kill.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/packages/cli/src/commands/kill.ts b/packages/cli/src/commands/kill.ts index c4664fdf40..f7b1098dd5 100644 --- a/packages/cli/src/commands/kill.ts +++ b/packages/cli/src/commands/kill.ts @@ -1,5 +1,5 @@ import { CLIOptions, Inquirerer, OptionValue } from 'inquirerer'; -import { getEnvOptions } from '@launchql/types'; +import { getPgEnvOptions } from '@launchql/types'; import { getRootPgPool } from '@launchql/server-utils'; import chalk from 'chalk'; @@ -8,13 +8,10 @@ export default async ( prompter: Inquirerer, _options: CLIOptions ) => { - const options = getEnvOptions(); - const { pg } = options; - const db = await getRootPgPool({ - ...pg, + const db = await getRootPgPool(getPgEnvOptions({ database: 'postgres' - }); + })); const databasesResult = await db.query(` SELECT datname FROM pg_catalog.pg_database From 27b6dded3cfb4f03004fc6c64ea66e867b89ae57 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sat, 17 May 2025 02:48:42 -0700 Subject: [PATCH 09/29] pg-spec --- .github/workflows/run-tests.yaml | 4 ++-- packages/{postgres-test => pg-spec}/README.md | 6 +++--- .../__tests__/postgres-test.connections.test.ts | 0 .../__tests__/postgres-test.grants.test.ts | 0 .../__tests__/postgres-test.records.test.ts | 0 .../__tests__/postgres-test.template.test.ts | 0 .../__tests__/postgres-test.test.ts | 0 packages/{postgres-test => pg-spec}/jest.config.js | 0 packages/{postgres-test => pg-spec}/package.json | 2 +- packages/{postgres-test => pg-spec}/sql/roles.sql | 0 packages/{postgres-test => pg-spec}/sql/test.sql | 0 packages/{postgres-test => pg-spec}/src/admin.ts | 0 packages/{postgres-test => pg-spec}/src/connect.ts | 0 packages/{postgres-test => pg-spec}/src/index.ts | 0 packages/{postgres-test => pg-spec}/src/legacy-connect.ts | 0 packages/{postgres-test => pg-spec}/src/manager.ts | 0 packages/{postgres-test => pg-spec}/src/stream.ts | 0 packages/{postgres-test => pg-spec}/src/test-client.ts | 0 packages/{postgres-test => pg-spec}/src/utils.ts | 0 packages/{postgres-test => pg-spec}/test-utils/index.ts | 0 packages/{postgres-test => pg-spec}/tsconfig.esm.json | 0 packages/{postgres-test => pg-spec}/tsconfig.json | 0 22 files changed, 6 insertions(+), 6 deletions(-) rename packages/{postgres-test => pg-spec}/README.md (98%) rename packages/{postgres-test => pg-spec}/__tests__/postgres-test.connections.test.ts (100%) rename packages/{postgres-test => pg-spec}/__tests__/postgres-test.grants.test.ts (100%) rename packages/{postgres-test => pg-spec}/__tests__/postgres-test.records.test.ts (100%) rename packages/{postgres-test => pg-spec}/__tests__/postgres-test.template.test.ts (100%) rename packages/{postgres-test => pg-spec}/__tests__/postgres-test.test.ts (100%) rename packages/{postgres-test => pg-spec}/jest.config.js (100%) rename packages/{postgres-test => pg-spec}/package.json (97%) rename packages/{postgres-test => pg-spec}/sql/roles.sql (100%) rename packages/{postgres-test => pg-spec}/sql/test.sql (100%) rename packages/{postgres-test => pg-spec}/src/admin.ts (100%) rename packages/{postgres-test => pg-spec}/src/connect.ts (100%) rename packages/{postgres-test => pg-spec}/src/index.ts (100%) rename packages/{postgres-test => pg-spec}/src/legacy-connect.ts (100%) rename packages/{postgres-test => pg-spec}/src/manager.ts (100%) rename packages/{postgres-test => pg-spec}/src/stream.ts (100%) rename packages/{postgres-test => pg-spec}/src/test-client.ts (100%) rename packages/{postgres-test => pg-spec}/src/utils.ts (100%) rename packages/{postgres-test => pg-spec}/test-utils/index.ts (100%) rename packages/{postgres-test => pg-spec}/tsconfig.esm.json (100%) rename packages/{postgres-test => pg-spec}/tsconfig.json (100%) diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 39a79a0d5d..808d5c75c9 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -75,9 +75,9 @@ jobs: PGUSER: postgres PGPASSWORD: password - - name: postgres-test + - name: pg-spec run: | - cd ./packages/postgres-test + cd ./packages/pg-spec yarn test env: PGHOST: pg_db diff --git a/packages/postgres-test/README.md b/packages/pg-spec/README.md similarity index 98% rename from packages/postgres-test/README.md rename to packages/pg-spec/README.md index e2f7d9b24c..d2749874fe 100644 --- a/packages/postgres-test/README.md +++ b/packages/pg-spec/README.md @@ -1,4 +1,4 @@ -# postgres-test +# pg-spec


@@ -8,11 +8,11 @@ ## install ```sh -npm install postgres-test +npm install pg-spec ``` ## Table of contents -- [postgres-test](#postgres-test) +- [pg-spec](#pg-spec) - [Install](#install) - [Table of contents](#table-of-contents) - [Developing](#developing) diff --git a/packages/postgres-test/__tests__/postgres-test.connections.test.ts b/packages/pg-spec/__tests__/postgres-test.connections.test.ts similarity index 100% rename from packages/postgres-test/__tests__/postgres-test.connections.test.ts rename to packages/pg-spec/__tests__/postgres-test.connections.test.ts diff --git a/packages/postgres-test/__tests__/postgres-test.grants.test.ts b/packages/pg-spec/__tests__/postgres-test.grants.test.ts similarity index 100% rename from packages/postgres-test/__tests__/postgres-test.grants.test.ts rename to packages/pg-spec/__tests__/postgres-test.grants.test.ts diff --git a/packages/postgres-test/__tests__/postgres-test.records.test.ts b/packages/pg-spec/__tests__/postgres-test.records.test.ts similarity index 100% rename from packages/postgres-test/__tests__/postgres-test.records.test.ts rename to packages/pg-spec/__tests__/postgres-test.records.test.ts diff --git a/packages/postgres-test/__tests__/postgres-test.template.test.ts b/packages/pg-spec/__tests__/postgres-test.template.test.ts similarity index 100% rename from packages/postgres-test/__tests__/postgres-test.template.test.ts rename to packages/pg-spec/__tests__/postgres-test.template.test.ts diff --git a/packages/postgres-test/__tests__/postgres-test.test.ts b/packages/pg-spec/__tests__/postgres-test.test.ts similarity index 100% rename from packages/postgres-test/__tests__/postgres-test.test.ts rename to packages/pg-spec/__tests__/postgres-test.test.ts diff --git a/packages/postgres-test/jest.config.js b/packages/pg-spec/jest.config.js similarity index 100% rename from packages/postgres-test/jest.config.js rename to packages/pg-spec/jest.config.js diff --git a/packages/postgres-test/package.json b/packages/pg-spec/package.json similarity index 97% rename from packages/postgres-test/package.json rename to packages/pg-spec/package.json index 4302d6c141..f7e18487b1 100644 --- a/packages/postgres-test/package.json +++ b/packages/pg-spec/package.json @@ -1,5 +1,5 @@ { - "name": "postgres-test", + "name": "pg-spec", "version": "0.0.1", "author": "Dan Lynch ", "description": "PostgreSQL Testing in TypeScript", diff --git a/packages/postgres-test/sql/roles.sql b/packages/pg-spec/sql/roles.sql similarity index 100% rename from packages/postgres-test/sql/roles.sql rename to packages/pg-spec/sql/roles.sql diff --git a/packages/postgres-test/sql/test.sql b/packages/pg-spec/sql/test.sql similarity index 100% rename from packages/postgres-test/sql/test.sql rename to packages/pg-spec/sql/test.sql diff --git a/packages/postgres-test/src/admin.ts b/packages/pg-spec/src/admin.ts similarity index 100% rename from packages/postgres-test/src/admin.ts rename to packages/pg-spec/src/admin.ts diff --git a/packages/postgres-test/src/connect.ts b/packages/pg-spec/src/connect.ts similarity index 100% rename from packages/postgres-test/src/connect.ts rename to packages/pg-spec/src/connect.ts diff --git a/packages/postgres-test/src/index.ts b/packages/pg-spec/src/index.ts similarity index 100% rename from packages/postgres-test/src/index.ts rename to packages/pg-spec/src/index.ts diff --git a/packages/postgres-test/src/legacy-connect.ts b/packages/pg-spec/src/legacy-connect.ts similarity index 100% rename from packages/postgres-test/src/legacy-connect.ts rename to packages/pg-spec/src/legacy-connect.ts diff --git a/packages/postgres-test/src/manager.ts b/packages/pg-spec/src/manager.ts similarity index 100% rename from packages/postgres-test/src/manager.ts rename to packages/pg-spec/src/manager.ts diff --git a/packages/postgres-test/src/stream.ts b/packages/pg-spec/src/stream.ts similarity index 100% rename from packages/postgres-test/src/stream.ts rename to packages/pg-spec/src/stream.ts diff --git a/packages/postgres-test/src/test-client.ts b/packages/pg-spec/src/test-client.ts similarity index 100% rename from packages/postgres-test/src/test-client.ts rename to packages/pg-spec/src/test-client.ts diff --git a/packages/postgres-test/src/utils.ts b/packages/pg-spec/src/utils.ts similarity index 100% rename from packages/postgres-test/src/utils.ts rename to packages/pg-spec/src/utils.ts diff --git a/packages/postgres-test/test-utils/index.ts b/packages/pg-spec/test-utils/index.ts similarity index 100% rename from packages/postgres-test/test-utils/index.ts rename to packages/pg-spec/test-utils/index.ts diff --git a/packages/postgres-test/tsconfig.esm.json b/packages/pg-spec/tsconfig.esm.json similarity index 100% rename from packages/postgres-test/tsconfig.esm.json rename to packages/pg-spec/tsconfig.esm.json diff --git a/packages/postgres-test/tsconfig.json b/packages/pg-spec/tsconfig.json similarity index 100% rename from packages/postgres-test/tsconfig.json rename to packages/pg-spec/tsconfig.json From d4d5a04543e9a6d2d151ee13dc5ac3f309887f2d Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sat, 17 May 2025 02:51:14 -0700 Subject: [PATCH 10/29] tests --- .github/workflows/run-tests.yaml | 4 ++-- packages/{pg-spec => pgsql-test}/README.md | 6 +++--- .../__tests__/postgres-test.connections.test.ts | 0 .../__tests__/postgres-test.grants.test.ts | 0 .../__tests__/postgres-test.records.test.ts | 0 .../__tests__/postgres-test.template.test.ts | 0 .../{pg-spec => pgsql-test}/__tests__/postgres-test.test.ts | 0 packages/{pg-spec => pgsql-test}/jest.config.js | 0 packages/{pg-spec => pgsql-test}/package.json | 2 +- packages/{pg-spec => pgsql-test}/sql/roles.sql | 0 packages/{pg-spec => pgsql-test}/sql/test.sql | 0 packages/{pg-spec => pgsql-test}/src/admin.ts | 0 packages/{pg-spec => pgsql-test}/src/connect.ts | 0 packages/{pg-spec => pgsql-test}/src/index.ts | 0 packages/{pg-spec => pgsql-test}/src/legacy-connect.ts | 0 packages/{pg-spec => pgsql-test}/src/manager.ts | 0 packages/{pg-spec => pgsql-test}/src/stream.ts | 0 packages/{pg-spec => pgsql-test}/src/test-client.ts | 0 packages/{pg-spec => pgsql-test}/src/utils.ts | 0 packages/{pg-spec => pgsql-test}/test-utils/index.ts | 0 packages/{pg-spec => pgsql-test}/tsconfig.esm.json | 0 packages/{pg-spec => pgsql-test}/tsconfig.json | 0 22 files changed, 6 insertions(+), 6 deletions(-) rename packages/{pg-spec => pgsql-test}/README.md (98%) rename packages/{pg-spec => pgsql-test}/__tests__/postgres-test.connections.test.ts (100%) rename packages/{pg-spec => pgsql-test}/__tests__/postgres-test.grants.test.ts (100%) rename packages/{pg-spec => pgsql-test}/__tests__/postgres-test.records.test.ts (100%) rename packages/{pg-spec => pgsql-test}/__tests__/postgres-test.template.test.ts (100%) rename packages/{pg-spec => pgsql-test}/__tests__/postgres-test.test.ts (100%) rename packages/{pg-spec => pgsql-test}/jest.config.js (100%) rename packages/{pg-spec => pgsql-test}/package.json (97%) rename packages/{pg-spec => pgsql-test}/sql/roles.sql (100%) rename packages/{pg-spec => pgsql-test}/sql/test.sql (100%) rename packages/{pg-spec => pgsql-test}/src/admin.ts (100%) rename packages/{pg-spec => pgsql-test}/src/connect.ts (100%) rename packages/{pg-spec => pgsql-test}/src/index.ts (100%) rename packages/{pg-spec => pgsql-test}/src/legacy-connect.ts (100%) rename packages/{pg-spec => pgsql-test}/src/manager.ts (100%) rename packages/{pg-spec => pgsql-test}/src/stream.ts (100%) rename packages/{pg-spec => pgsql-test}/src/test-client.ts (100%) rename packages/{pg-spec => pgsql-test}/src/utils.ts (100%) rename packages/{pg-spec => pgsql-test}/test-utils/index.ts (100%) rename packages/{pg-spec => pgsql-test}/tsconfig.esm.json (100%) rename packages/{pg-spec => pgsql-test}/tsconfig.json (100%) diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 808d5c75c9..0e7e456513 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -75,9 +75,9 @@ jobs: PGUSER: postgres PGPASSWORD: password - - name: pg-spec + - name: pgsql-test run: | - cd ./packages/pg-spec + cd ./packages/pgsql-test yarn test env: PGHOST: pg_db diff --git a/packages/pg-spec/README.md b/packages/pgsql-test/README.md similarity index 98% rename from packages/pg-spec/README.md rename to packages/pgsql-test/README.md index d2749874fe..e0a62c8f38 100644 --- a/packages/pg-spec/README.md +++ b/packages/pgsql-test/README.md @@ -1,4 +1,4 @@ -# pg-spec +# pgsql-test


@@ -8,11 +8,11 @@ ## install ```sh -npm install pg-spec +npm install pgsql-test ``` ## Table of contents -- [pg-spec](#pg-spec) +- [pgsql-test](#pgsql-test) - [Install](#install) - [Table of contents](#table-of-contents) - [Developing](#developing) diff --git a/packages/pg-spec/__tests__/postgres-test.connections.test.ts b/packages/pgsql-test/__tests__/postgres-test.connections.test.ts similarity index 100% rename from packages/pg-spec/__tests__/postgres-test.connections.test.ts rename to packages/pgsql-test/__tests__/postgres-test.connections.test.ts diff --git a/packages/pg-spec/__tests__/postgres-test.grants.test.ts b/packages/pgsql-test/__tests__/postgres-test.grants.test.ts similarity index 100% rename from packages/pg-spec/__tests__/postgres-test.grants.test.ts rename to packages/pgsql-test/__tests__/postgres-test.grants.test.ts diff --git a/packages/pg-spec/__tests__/postgres-test.records.test.ts b/packages/pgsql-test/__tests__/postgres-test.records.test.ts similarity index 100% rename from packages/pg-spec/__tests__/postgres-test.records.test.ts rename to packages/pgsql-test/__tests__/postgres-test.records.test.ts diff --git a/packages/pg-spec/__tests__/postgres-test.template.test.ts b/packages/pgsql-test/__tests__/postgres-test.template.test.ts similarity index 100% rename from packages/pg-spec/__tests__/postgres-test.template.test.ts rename to packages/pgsql-test/__tests__/postgres-test.template.test.ts diff --git a/packages/pg-spec/__tests__/postgres-test.test.ts b/packages/pgsql-test/__tests__/postgres-test.test.ts similarity index 100% rename from packages/pg-spec/__tests__/postgres-test.test.ts rename to packages/pgsql-test/__tests__/postgres-test.test.ts diff --git a/packages/pg-spec/jest.config.js b/packages/pgsql-test/jest.config.js similarity index 100% rename from packages/pg-spec/jest.config.js rename to packages/pgsql-test/jest.config.js diff --git a/packages/pg-spec/package.json b/packages/pgsql-test/package.json similarity index 97% rename from packages/pg-spec/package.json rename to packages/pgsql-test/package.json index f7e18487b1..04120f5af6 100644 --- a/packages/pg-spec/package.json +++ b/packages/pgsql-test/package.json @@ -1,5 +1,5 @@ { - "name": "pg-spec", + "name": "pgsql-test", "version": "0.0.1", "author": "Dan Lynch ", "description": "PostgreSQL Testing in TypeScript", diff --git a/packages/pg-spec/sql/roles.sql b/packages/pgsql-test/sql/roles.sql similarity index 100% rename from packages/pg-spec/sql/roles.sql rename to packages/pgsql-test/sql/roles.sql diff --git a/packages/pg-spec/sql/test.sql b/packages/pgsql-test/sql/test.sql similarity index 100% rename from packages/pg-spec/sql/test.sql rename to packages/pgsql-test/sql/test.sql diff --git a/packages/pg-spec/src/admin.ts b/packages/pgsql-test/src/admin.ts similarity index 100% rename from packages/pg-spec/src/admin.ts rename to packages/pgsql-test/src/admin.ts diff --git a/packages/pg-spec/src/connect.ts b/packages/pgsql-test/src/connect.ts similarity index 100% rename from packages/pg-spec/src/connect.ts rename to packages/pgsql-test/src/connect.ts diff --git a/packages/pg-spec/src/index.ts b/packages/pgsql-test/src/index.ts similarity index 100% rename from packages/pg-spec/src/index.ts rename to packages/pgsql-test/src/index.ts diff --git a/packages/pg-spec/src/legacy-connect.ts b/packages/pgsql-test/src/legacy-connect.ts similarity index 100% rename from packages/pg-spec/src/legacy-connect.ts rename to packages/pgsql-test/src/legacy-connect.ts diff --git a/packages/pg-spec/src/manager.ts b/packages/pgsql-test/src/manager.ts similarity index 100% rename from packages/pg-spec/src/manager.ts rename to packages/pgsql-test/src/manager.ts diff --git a/packages/pg-spec/src/stream.ts b/packages/pgsql-test/src/stream.ts similarity index 100% rename from packages/pg-spec/src/stream.ts rename to packages/pgsql-test/src/stream.ts diff --git a/packages/pg-spec/src/test-client.ts b/packages/pgsql-test/src/test-client.ts similarity index 100% rename from packages/pg-spec/src/test-client.ts rename to packages/pgsql-test/src/test-client.ts diff --git a/packages/pg-spec/src/utils.ts b/packages/pgsql-test/src/utils.ts similarity index 100% rename from packages/pg-spec/src/utils.ts rename to packages/pgsql-test/src/utils.ts diff --git a/packages/pg-spec/test-utils/index.ts b/packages/pgsql-test/test-utils/index.ts similarity index 100% rename from packages/pg-spec/test-utils/index.ts rename to packages/pgsql-test/test-utils/index.ts diff --git a/packages/pg-spec/tsconfig.esm.json b/packages/pgsql-test/tsconfig.esm.json similarity index 100% rename from packages/pg-spec/tsconfig.esm.json rename to packages/pgsql-test/tsconfig.esm.json diff --git a/packages/pg-spec/tsconfig.json b/packages/pgsql-test/tsconfig.json similarity index 100% rename from packages/pg-spec/tsconfig.json rename to packages/pgsql-test/tsconfig.json From 398aad05e824464d838f61870fef8c344e0383d9 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sat, 17 May 2025 02:59:24 -0700 Subject: [PATCH 11/29] readme --- packages/pgsql-test/README.md | 150 +++++++++++++++++++++++++--------- 1 file changed, 113 insertions(+), 37 deletions(-) diff --git a/packages/pgsql-test/README.md b/packages/pgsql-test/README.md index e0a62c8f38..466b012f15 100644 --- a/packages/pgsql-test/README.md +++ b/packages/pgsql-test/README.md @@ -1,61 +1,137 @@ # pgsql-test -

-
- PostgreSQL Testing in TypeScript +

+

-## install +

+ + + + + + + + + +

+ +## Install ```sh npm install pgsql-test ``` -## Table of contents +--- + +## Features + +* πŸ§ͺ Auto-generated test databases with `UUID` suffix +* πŸ”„ Per-test isolation using transactions and savepoints +* πŸ›‘οΈ Role-based context for RLS testing +* 🧹 Easy teardown and cleanup +* 🧰 Designed for `Jest`, `Mocha`, or any async test runner + +--- + +## How to Use + +`pgsql-test` provides an isolated PostgreSQL testing environment with per-test transaction rollback, ideal for integration tests involving SQL, roles, or GraphQL (e.g., with PostGraphile). + +### Basic Example + +```ts +import { getConnections } from 'pgsql-test'; +import { PgTestClient } from 'pgsql-test/client'; + +let conn: PgTestClient; +let db: PgTestClient; +let teardown: () => Promise; + +beforeAll(async () => { + ({ conn, db, teardown } = await getConnections()); + + await db.query(` + CREATE TABLE users ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL + ); + CREATE TABLE posts ( + id SERIAL PRIMARY KEY, + user_id INT NOT NULL REFERENCES users(id), + content TEXT NOT NULL + ); + `); + + await db.query(` + INSERT INTO users (name) VALUES ('Alice'), ('Bob'); + INSERT INTO posts (user_id, content) VALUES + (1, 'Hello world!'), + (2, 'Graphile is cool!'); + `); +}); + +afterAll(async () => { + await teardown(); +}); + +beforeEach(async () => { + await db.beforeEach(); // Starts transaction + SAVEPOINT +}); + +afterEach(async () => { + await db.afterEach(); // Rolls back to SAVEPOINT +}); + +test('user count starts at 2', async () => { + const res = await db.query('SELECT COUNT(*) FROM users'); + expect(res.rows[0].count).toBe('2'); +}); +``` -- [pgsql-test](#pgsql-test) - - [Install](#install) - - [Table of contents](#table-of-contents) -- [Developing](#developing) -- [Credits](#credits) +--- -## Developing +## Role-Based Contexts -When first cloning the repo: +You can simulate different PostgreSQL roles for RLS and permission testing. -```sh -yarn -# build the prod packages. When devs would like to navigate to the source code, this will only navigate from references to their definitions (.d.ts files) between packages. -yarn build -``` +```ts +describe('authenticated role', () => { + beforeEach(async () => { + conn.setContext({ role: 'authenticated' }); + await conn.beforeEach(); + }); -Or if you want to make your dev process smoother, you can run: + afterEach(async () => { + await conn.afterEach(); + }); -```sh -yarn -# build the dev packages with .map files, this enables navigation from references to their source code between packages. -yarn build:dev + it('runs as authenticated', async () => { + const result = await conn.query(`SELECT current_setting('role', true) AS role`); + expect(result.rows[0].role).toBe('authenticated'); + }); +}); ``` -## Interchain JavaScript Stack +--- -A unified toolkit for building applications and smart contracts in the Interchain ecosystem βš›οΈ +## Environment Overrides -| Category | Tools | Description | -|----------------------|------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------| -| **Chain Information** | [**Chain Registry**](https://github.com/hyperweb-io/chain-registry), [**Utils**](https://www.npmjs.com/package/@chain-registry/utils), [**Client**](https://www.npmjs.com/package/@chain-registry/client) | Everything from token symbols, logos, and IBC denominations for all assets you want to support in your application. | -| **Wallet Connectors**| [**Interchain Kit**](https://github.com/hyperweb-io/interchain-kit)beta, [**Cosmos Kit**](https://github.com/hyperweb.io/cosmos-kit) | Experience the convenience of connecting with a variety of web3 wallets through a single, streamlined interface. | -| **Signing Clients** | [**InterchainJS**](https://github.com/hyperweb-io/interchainjs)beta, [**CosmJS**](https://github.com/cosmos/cosmjs) | A single, universal signing interface for any network | -| **SDK Clients** | [**Telescope**](https://github.com/hyperweb.io/telescope) | Your Frontend Companion for Building with TypeScript with Cosmos SDK Modules. | -| **Starter Kits** | [**Create Interchain App**](https://github.com/hyperweb-io/create-interchain-app)beta, [**Create Cosmos App**](https://github.com/hyperweb.io/create-cosmos-app) | Set up a modern Interchain app by running one command. | -| **UI Kits** | [**Interchain UI**](https://github.com/hyperweb.io/interchain-ui) | The Interchain Design System, empowering developers with a flexible, easy-to-use UI kit. | -| **Testing Frameworks** | [**Starship**](https://github.com/hyperweb.io/starship) | Unified Testing and Development for the Interchain. | -| **TypeScript Smart Contracts** | [**Create Hyperweb App**](https://github.com/hyperweb-io/create-hyperweb-app) | Build and deploy full-stack blockchain applications with TypeScript | -| **CosmWasm Contracts** | [**CosmWasm TS Codegen**](https://github.com/CosmWasm/ts-codegen) | Convert your CosmWasm smart contracts into dev-friendly TypeScript classes. | +`pgsql-test` respects the following env vars for DB connectivity: -## Credits +* `PGHOST` +* `PGPORT` +* `PGUSER` +* `PGPASSWORD` -πŸ›  Built by Hyperweb (formerly Cosmology) β€”Β if you like our tools, please checkout and contribute to [our github βš›οΈ](https://github.com/hyperweb-io) +Override them in your test runner or CI config: +```yaml +env: + PGHOST: localhost + PGPORT: 5432 + PGUSER: postgres + PGPASSWORD: password +``` ## Disclaimer From c32da3369f9a9c24033d61d0b56d803268e41ee3 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sat, 17 May 2025 03:01:30 -0700 Subject: [PATCH 12/29] fix --- packages/cli/src/commands/export.ts | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/packages/cli/src/commands/export.ts b/packages/cli/src/commands/export.ts index 922dd7a081..cb18a80c59 100644 --- a/packages/cli/src/commands/export.ts +++ b/packages/cli/src/commands/export.ts @@ -1,6 +1,6 @@ import { CLIOptions, Inquirerer, OptionValue } from 'inquirerer'; import { LaunchQLProject, exportMigrations } from '@launchql/migrate'; -import { getEnvOptions } from '@launchql/types'; +import { getEnvOptions, getPgEnvOptions } from '@launchql/types'; import chalk from 'chalk'; import { resolve } from 'path'; import { execSync } from 'child_process'; @@ -20,12 +20,10 @@ export default async ( project.ensureWorkspace(); const options = getEnvOptions(); - const { pg } = options; - const db = await getRootPgPool({ - ...pg, + const db = await getRootPgPool(getPgEnvOptions({ database: 'postgres' - }); + })); const databasesResult = await db.query(` SELECT datname FROM pg_catalog.pg_database @@ -45,10 +43,9 @@ export default async ( ])); const dbname = databases.filter(d=>d.selected).map(d=>d.value)[0]; - const selectedDb = await getRootPgPool({ - ...pg, + const selectedDb = await getRootPgPool(getPgEnvOptions({ database: dbname - }); + })); const dbsResult = await selectedDb.query(` SELECT id, name FROM collections_public.database; From 4e71d5c22d86edcd4a2c451fadc1a75440ff6c4b Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sat, 17 May 2025 11:21:11 -0700 Subject: [PATCH 13/29] cleaner opts --- packages/cli/src/commands/export.ts | 8 +- packages/cli/src/commands/kill.ts | 4 +- packages/explorer/src/server.ts | 44 +-- packages/migrate/src/export-migrations.ts | 324 +++++++++++----------- packages/server-utils/src/pg.ts | 16 +- packages/server/src/server.ts | 2 +- 6 files changed, 201 insertions(+), 197 deletions(-) diff --git a/packages/cli/src/commands/export.ts b/packages/cli/src/commands/export.ts index cb18a80c59..b7a06c81c8 100644 --- a/packages/cli/src/commands/export.ts +++ b/packages/cli/src/commands/export.ts @@ -21,9 +21,9 @@ export default async ( const options = getEnvOptions(); - const db = await getRootPgPool(getPgEnvOptions({ + const db = await getRootPgPool({ database: 'postgres' - })); + }); const databasesResult = await db.query(` SELECT datname FROM pg_catalog.pg_database @@ -43,9 +43,9 @@ export default async ( ])); const dbname = databases.filter(d=>d.selected).map(d=>d.value)[0]; - const selectedDb = await getRootPgPool(getPgEnvOptions({ + const selectedDb = await getRootPgPool({ database: dbname - })); + }); const dbsResult = await selectedDb.query(` SELECT id, name FROM collections_public.database; diff --git a/packages/cli/src/commands/kill.ts b/packages/cli/src/commands/kill.ts index f7b1098dd5..6b610d1ae1 100644 --- a/packages/cli/src/commands/kill.ts +++ b/packages/cli/src/commands/kill.ts @@ -9,9 +9,9 @@ export default async ( _options: CLIOptions ) => { - const db = await getRootPgPool(getPgEnvOptions({ + const db = await getRootPgPool({ database: 'postgres' - })); + }); const databasesResult = await db.query(` SELECT datname FROM pg_catalog.pg_database diff --git a/packages/explorer/src/server.ts b/packages/explorer/src/server.ts index f0f43857db..7bfdd56dd3 100644 --- a/packages/explorer/src/server.ts +++ b/packages/explorer/src/server.ts @@ -12,7 +12,7 @@ import { } from '@launchql/server-utils'; import { printSchemas, printDatabases } from './render'; import { getGraphileSettings } from './settings'; -import { LaunchQLOptions } from '@launchql/types'; +import { getPgEnvOptions, LaunchQLOptions } from '@launchql/types'; import { getMergedOptions } from '@launchql/types'; export const LaunchQLExplorer = (rawOpts: LaunchQLOptions = {}): Express => { @@ -39,10 +39,12 @@ export const LaunchQLExplorer = (rawOpts: LaunchQLOptions = {}): Express => { graphiqlRoute: '/graphiql' }; - const pgPool = getRootPgPool({ - ...opts.pg, - database: dbname - }); + const pgPool = getRootPgPool( + getPgEnvOptions({ + ...opts.pg, + database: dbname + })); + const handler = postgraphile(pgPool, schemaname, settings); const obj = { @@ -67,10 +69,12 @@ export const LaunchQLExplorer = (rawOpts: LaunchQLOptions = {}): Express => { if (req.urlDomains?.subdomains.length === 1) { const [dbName] = req.urlDomains.subdomains; try { - const pgPool = getRootPgPool({ - ...opts.pg, - database: dbName - }); + const pgPool = getRootPgPool( + getPgEnvOptions({ + ...opts.pg, + database: dbName + })); + const results = await pgPool.query(` SELECT s.nspname AS table_schema FROM pg_catalog.pg_namespace s @@ -103,11 +107,15 @@ export const LaunchQLExplorer = (rawOpts: LaunchQLOptions = {}): Express => { if (req.urlDomains?.subdomains.length === 2) { const [, dbName] = req.urlDomains.subdomains; try { - const pgPool = getRootPgPool({ - ...opts.pg, - database: dbName - }); + + const pgPool = getRootPgPool( + getPgEnvOptions({ + ...opts.pg, + database: dbName + })); + await pgPool.query('SELECT 1;'); + } catch (e: any) { if (e.message?.match(/does not exist/)) { res.status(404).send('DB Not found'); @@ -150,10 +158,12 @@ export const LaunchQLExplorer = (rawOpts: LaunchQLOptions = {}): Express => { app.use(async (req: Request, res: Response, next: NextFunction) => { if (req.urlDomains?.subdomains.length === 0) { try { - const rootPgPool = getRootPgPool({ - ...opts.pg, - database: opts.pg.user // is this to get postgres? - }); + const rootPgPool = getRootPgPool( + getPgEnvOptions({ + ...opts.pg, + database: opts.pg.user // is this to get postgres? + })); + const results = await rootPgPool.query(` SELECT * FROM pg_catalog.pg_database WHERE datistemplate = FALSE AND datname != 'postgres' AND datname !~ '^pg_' diff --git a/packages/migrate/src/export-migrations.ts b/packages/migrate/src/export-migrations.ts index ed85710326..c8458b1aeb 100644 --- a/packages/migrate/src/export-migrations.ts +++ b/packages/migrate/src/export-migrations.ts @@ -5,154 +5,154 @@ import Case from 'case'; import { exportMeta } from './export-meta'; import { getRootPgPool } from '@launchql/server-utils'; -import { LaunchQLOptions } from '@launchql/types'; +import { getPgEnvOptions, LaunchQLOptions } from '@launchql/types'; import { SqitchRow, writeSqitchFiles, writeSqitchPlan } from './sqitch'; import { LaunchQLProject } from './class/launchql'; interface ExportMigrationsToDiskOptions { - project: LaunchQLProject; - options: LaunchQLOptions; - database: string; - databaseId: string; - author: string; - outdir: string; - schema_names: string[]; - extensionName?: string; - metaExtensionName: string; + project: LaunchQLProject; + options: LaunchQLOptions; + database: string; + databaseId: string; + author: string; + outdir: string; + schema_names: string[]; + extensionName?: string; + metaExtensionName: string; } interface ExportOptions { - project: LaunchQLProject; - options: LaunchQLOptions; - dbInfo: { - dbname: string; - database_ids: string[]; - }; - author: string; - outdir: string; - schema_names: string[]; - extensionName?: string; - metaExtensionName: string; + project: LaunchQLProject; + options: LaunchQLOptions; + dbInfo: { + dbname: string; + database_ids: string[]; + }; + author: string; + outdir: string; + schema_names: string[]; + extensionName?: string; + metaExtensionName: string; } const exportMigrationsToDisk = async ({ - project, - options, - database, - databaseId, - author, - outdir, - schema_names, - extensionName, - metaExtensionName + project, + options, + database, + databaseId, + author, + outdir, + schema_names, + extensionName, + metaExtensionName }: ExportMigrationsToDiskOptions): Promise => { - outdir = outdir + '/'; + outdir = outdir + '/'; - const pgPool = getRootPgPool({ - ...options.pg, - database - }); + const pgPool = getRootPgPool({ + ...options.pg, + database + }); - const db = await pgPool.query( - `select * from collections_public.database where id=$1`, - [databaseId] - ); + const db = await pgPool.query( + `select * from collections_public.database where id=$1`, + [databaseId] + ); - const schemas = await pgPool.query( - `select * from collections_public.schema where database_id=$1`, - [databaseId] - ); + const schemas = await pgPool.query( + `select * from collections_public.schema where database_id=$1`, + [databaseId] + ); - if (!db?.rows?.length) { - console.log('NO DATABASES.'); - return; - } + if (!db?.rows?.length) { + console.log('NO DATABASES.'); + return; + } - if (!schemas?.rows?.length) { - console.log('NO SCHEMAS.'); - return; - } + if (!schemas?.rows?.length) { + console.log('NO SCHEMAS.'); + return; + } + + const name = extensionName || db.rows[0].name; - const name = extensionName || db.rows[0].name; + const { replace, replacer } = makeReplacer({ + schemas: schemas.rows.filter((schema: any) => + schema_names.includes(schema.schema_name) + ), + name + }); + + const results = await pgPool.query( + `select * from db_migrate.sql_actions order by id` + ); - const { replace, replacer } = makeReplacer({ - schemas: schemas.rows.filter((schema: any) => - schema_names.includes(schema.schema_name) - ), - name + const opts = { + name, + replace, + replacer, + outdir, + author + }; + + if (results?.rows?.length > 0) { + await preparePackage({ + project, + author, + outdir, + name, + extensions: [ + 'plpgsql', + 'uuid-ossp', + 'citext', + 'pgcrypto', + 'btree_gist', + 'postgis', + 'hstore', + 'db_meta', + 'launchql-inflection', + 'launchql-uuid', + 'launchql-utils', + 'launchql-ext-jobs', + 'launchql-jwt-claims', + 'launchql-stamps', + 'launchql-base32', + 'launchql-totp', + 'launchql-ext-types', + 'launchql-ext-default-roles' + ] + }); + + writeSqitchPlan(results.rows, opts); + writeSqitchFiles(results.rows, opts); + + let meta = await exportMeta({ + opts: options, + dbname: database, + database_id: databaseId + }); + + meta = replacer(meta); + + await preparePackage({ + project, + author, + outdir, + extensions: ['plpgsql', 'db_meta', 'db_meta_modules'], + name: metaExtensionName + }); + + const metaReplacer = makeReplacer({ + schemas: schemas.rows.filter((schema: any) => + schema_names.includes(schema.schema_name) + ), + name: metaExtensionName }); - const results = await pgPool.query( - `select * from db_migrate.sql_actions order by id` - ); - - const opts = { - name, - replace, - replacer, - outdir, - author - }; - - if (results?.rows?.length > 0) { - await preparePackage({ - project, - author, - outdir, - name, - extensions: [ - 'plpgsql', - 'uuid-ossp', - 'citext', - 'pgcrypto', - 'btree_gist', - 'postgis', - 'hstore', - 'db_meta', - 'launchql-inflection', - 'launchql-uuid', - 'launchql-utils', - 'launchql-ext-jobs', - 'launchql-jwt-claims', - 'launchql-stamps', - 'launchql-base32', - 'launchql-totp', - 'launchql-ext-types', - 'launchql-ext-default-roles' - ] - }); - - writeSqitchPlan(results.rows, opts); - writeSqitchFiles(results.rows, opts); - - let meta = await exportMeta({ - opts: options, - dbname: database, - database_id: databaseId - }); - - meta = replacer(meta); - - await preparePackage({ - project, - author, - outdir, - extensions: ['plpgsql', 'db_meta', 'db_meta_modules'], - name: metaExtensionName - }); - - const metaReplacer = makeReplacer({ - schemas: schemas.rows.filter((schema: any) => - schema_names.includes(schema.schema_name) - ), - name: metaExtensionName - }); - - const metaPackage: SqitchRow[] = [ - { - deps: [], - deploy: 'migrate/meta', - content: `SET session_replication_role TO replica; + const metaPackage: SqitchRow[] = [ + { + deps: [], + deploy: 'migrate/meta', + content: `SET session_replication_role TO replica; -- using replica in case we are deploying triggers to collections_public -- unaccent, postgis affected and require grants @@ -178,43 +178,43 @@ UPDATE meta_public.sites SET session_replication_role TO DEFAULT; ` - } - ]; + } + ]; - opts.replacer = metaReplacer.replacer; - opts.name = metaExtensionName; + opts.replacer = metaReplacer.replacer; + opts.name = metaExtensionName; - writeSqitchPlan(metaPackage, opts); - writeSqitchFiles(metaPackage, opts); - } + writeSqitchPlan(metaPackage, opts); + writeSqitchFiles(metaPackage, opts); + } - pgPool.end(); + pgPool.end(); }; export const exportMigrations = async ({ - project, - options, - dbInfo, - author, - outdir, - schema_names, - extensionName, - metaExtensionName + project, + options, + dbInfo, + author, + outdir, + schema_names, + extensionName, + metaExtensionName }: ExportOptions): Promise => { - for (let v = 0; v < dbInfo.database_ids.length; v++) { - const databaseId = dbInfo.database_ids[v]; - await exportMigrationsToDisk({ - project, - options, - extensionName, - metaExtensionName, - database: dbInfo.dbname, - databaseId, - schema_names, - author, - outdir - }); - } + for (let v = 0; v < dbInfo.database_ids.length; v++) { + const databaseId = dbInfo.database_ids[v]; + await exportMigrationsToDisk({ + project, + options, + extensionName, + metaExtensionName, + database: dbInfo.dbname, + databaseId, + schema_names, + author, + outdir + }); + } }; @@ -259,10 +259,10 @@ const preparePackage = async ({ const plan = glob(path.join(sqitchDir, 'sqitch.plan')); if (!plan.length) { project.initModule({ - name, - description: name, - author, - extensions, + name, + description: name, + author, + extensions, }); } else { rmSync(path.resolve(sqitchDir, 'deploy'), { recursive: true, force: true }); diff --git a/packages/server-utils/src/pg.ts b/packages/server-utils/src/pg.ts index 04888bf800..1987756a68 100644 --- a/packages/server-utils/src/pg.ts +++ b/packages/server-utils/src/pg.ts @@ -1,7 +1,7 @@ import pg from 'pg'; import { pgCache } from './lru'; -import { PgConfig } from '@launchql/types'; +import { getPgEnvOptions, PgConfig } from '@launchql/types'; export const getDbString = ( user: string, @@ -12,21 +12,15 @@ export const getDbString = ( ): string => `postgres://${user}:${password}@${host}:${port}/${database}`; -export const getRootPgPool = ({ - user, - password, - host, - port, - database, -}: PgConfig): pg.Pool => { +export const getRootPgPool = (pgConfig: Partial): pg.Pool => { + const config = getPgEnvOptions(pgConfig); + const { user, password, host, port, database, } = config; if (pgCache.has(database)) { const cached = pgCache.get(database); if (cached) return cached; } - const connectionString = getDbString(user, password, host, port, database); const pgPool = new pg.Pool({ connectionString }); - pgCache.set(database, pgPool); return pgPool; -}; +}; \ No newline at end of file diff --git a/packages/server/src/server.ts b/packages/server/src/server.ts index 20a938628a..d5c5e7c4de 100644 --- a/packages/server/src/server.ts +++ b/packages/server/src/server.ts @@ -16,7 +16,7 @@ import { flush, flushService } from './middleware/flush'; import requestIp from 'request-ip'; import { Pool, PoolClient } from 'pg'; -import { LaunchQLOptions, getMergedOptions } from '@launchql/types'; +import { LaunchQLOptions, getMergedOptions, getPgEnvOptions } from '@launchql/types'; export const LaunchQLServer = (rawOpts: LaunchQLOptions = {}) => { const app = new Server(getMergedOptions(rawOpts)); From 6803d00ad8ee427ddbf50926e24922eef44629b9 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sat, 17 May 2025 11:21:59 -0700 Subject: [PATCH 14/29] imports --- packages/cli/src/commands/export.ts | 3 +-- packages/cli/src/commands/kill.ts | 1 - packages/migrate/src/export-migrations.ts | 2 +- packages/server/src/server.ts | 2 +- 4 files changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/cli/src/commands/export.ts b/packages/cli/src/commands/export.ts index b7a06c81c8..0db5f84ea5 100644 --- a/packages/cli/src/commands/export.ts +++ b/packages/cli/src/commands/export.ts @@ -1,7 +1,6 @@ import { CLIOptions, Inquirerer, OptionValue } from 'inquirerer'; import { LaunchQLProject, exportMigrations } from '@launchql/migrate'; -import { getEnvOptions, getPgEnvOptions } from '@launchql/types'; -import chalk from 'chalk'; +import { getEnvOptions } from '@launchql/types'; import { resolve } from 'path'; import { execSync } from 'child_process'; import { getRootPgPool } from '@launchql/server-utils'; diff --git a/packages/cli/src/commands/kill.ts b/packages/cli/src/commands/kill.ts index 6b610d1ae1..da9a0ab3e2 100644 --- a/packages/cli/src/commands/kill.ts +++ b/packages/cli/src/commands/kill.ts @@ -1,5 +1,4 @@ import { CLIOptions, Inquirerer, OptionValue } from 'inquirerer'; -import { getPgEnvOptions } from '@launchql/types'; import { getRootPgPool } from '@launchql/server-utils'; import chalk from 'chalk'; diff --git a/packages/migrate/src/export-migrations.ts b/packages/migrate/src/export-migrations.ts index c8458b1aeb..7238340c96 100644 --- a/packages/migrate/src/export-migrations.ts +++ b/packages/migrate/src/export-migrations.ts @@ -5,7 +5,7 @@ import Case from 'case'; import { exportMeta } from './export-meta'; import { getRootPgPool } from '@launchql/server-utils'; -import { getPgEnvOptions, LaunchQLOptions } from '@launchql/types'; +import { LaunchQLOptions } from '@launchql/types'; import { SqitchRow, writeSqitchFiles, writeSqitchPlan } from './sqitch'; import { LaunchQLProject } from './class/launchql'; diff --git a/packages/server/src/server.ts b/packages/server/src/server.ts index d5c5e7c4de..20a938628a 100644 --- a/packages/server/src/server.ts +++ b/packages/server/src/server.ts @@ -16,7 +16,7 @@ import { flush, flushService } from './middleware/flush'; import requestIp from 'request-ip'; import { Pool, PoolClient } from 'pg'; -import { LaunchQLOptions, getMergedOptions, getPgEnvOptions } from '@launchql/types'; +import { LaunchQLOptions, getMergedOptions } from '@launchql/types'; export const LaunchQLServer = (rawOpts: LaunchQLOptions = {}) => { const app = new Server(getMergedOptions(rawOpts)); From 9b2844127db327b488fadf3f4386628350f04f70 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sat, 17 May 2025 11:51:23 -0700 Subject: [PATCH 15/29] jwt --- packages/pgsql-test/README.md | 3 +- .../postgres-test.connections.test.ts | 56 +++++-------------- packages/pgsql-test/src/test-client.ts | 34 +++++------ 3 files changed, 30 insertions(+), 63 deletions(-) diff --git a/packages/pgsql-test/README.md b/packages/pgsql-test/README.md index 466b012f15..7cd8961991 100644 --- a/packages/pgsql-test/README.md +++ b/packages/pgsql-test/README.md @@ -40,8 +40,7 @@ npm install pgsql-test ### Basic Example ```ts -import { getConnections } from 'pgsql-test'; -import { PgTestClient } from 'pgsql-test/client'; +import { PgTestClient, getConnections } from 'pgsql-test'; let conn: PgTestClient; let db: PgTestClient; diff --git a/packages/pgsql-test/__tests__/postgres-test.connections.test.ts b/packages/pgsql-test/__tests__/postgres-test.connections.test.ts index c5cdc20b33..7eb0c61ae5 100644 --- a/packages/pgsql-test/__tests__/postgres-test.connections.test.ts +++ b/packages/pgsql-test/__tests__/postgres-test.connections.test.ts @@ -7,26 +7,6 @@ let teardown: () => Promise; beforeAll(async () => { ({ conn, db, teardown } = await getConnections()); - - // Setup schema + seed ONCE globally - await db.query(` - CREATE TABLE users ( - id SERIAL PRIMARY KEY, - name TEXT NOT NULL - ); - CREATE TABLE posts ( - id SERIAL PRIMARY KEY, - user_id INT NOT NULL REFERENCES users(id), - content TEXT NOT NULL - ); - `); - - await db.query(` - INSERT INTO users (name) VALUES ('Alice'), ('Bob'); - INSERT INTO posts (user_id, content) VALUES - (1, 'Hello world!'), - (2, 'Graphile is cool!'); - `); }); afterAll(async () => { @@ -35,47 +15,39 @@ afterAll(async () => { describe('anonymous', () => { beforeEach(async () => { - await db.beforeEach(); // this starts tx + savepoint + await conn.beforeEach(); }); afterEach(async () => { - await db.afterEach(); // this rolls back and commits - }); - - it('inserts a user but rollback leaves baseline intact', async () => { - await db.query(`INSERT INTO users (name) VALUES ('Carol')`); - const res = await db.query('SELECT COUNT(*) FROM users'); - expect(res.rows[0].count).toBe('3'); - }); - - it('should still have 2 users after rollback', async () => { - const res = await db.query('SELECT COUNT(*) FROM users'); - expect(res.rows[0].count).toBe('2'); + await conn.afterEach(); }); it('runs under anonymous context', async () => { const result = await conn.query('SELECT current_setting(\'role\', true) AS role'); - console.log(JSON.stringify({result}, null, 2)) - console.error(JSON.stringify({result}, null, 2)) - // expect(result.rows[0].role).toBe('anonymous'); + expect(result.rows[0].role).toBe('anonymous'); }); }); describe('authenticated', () => { beforeEach(async () => { conn.setContext({ - role: 'authenticated' + role: 'authenticated', + 'jwt.claims.user_agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36', + 'jwt.claims.ip_address': '127.0.0.1', + 'jwt.claims.database_id': 'jwt.database_id', + 'jwt.claims.user_id': 'jwt.user_id' }); - await conn.beforeEach(); // required for rollback later + await conn.beforeEach(); }); afterEach(async () => { - await conn.afterEach(); // now safe to rollback + await conn.afterEach(); }); it('runs under authenticated context', async () => { - const result = await conn.query('SELECT current_setting(\'role\', true) AS role'); - // expect(result.rows[0].role).toBe('authenticated'); - console.error('why no JWT') + const role = await conn.query('SELECT current_setting(\'role\', true) AS role'); + expect(role.rows[0].role).toBe('authenticated'); + const ip = await conn.query('SELECT current_setting(\'jwt.claims.ip_address\', true) AS role'); + expect(ip.rows[0].role).toBe('127.0.0.1'); }); }); diff --git a/packages/pgsql-test/src/test-client.ts b/packages/pgsql-test/src/test-client.ts index e39bb6d878..c665b7c279 100644 --- a/packages/pgsql-test/src/test-client.ts +++ b/packages/pgsql-test/src/test-client.ts @@ -54,24 +54,16 @@ export class PgTestClient { setContext(ctx: Record): void { this.ctxStmts = Object.entries(ctx) - .map(([key, val]) => - val === null - ? `SELECT set_config('${key}', NULL, true);` - : `SELECT set_config('${key}', '${val}', true);` - ) - .join('\n'); - } - - private async runCtxQuery(query: string, values?: any[]): Promise> { - if (this.ctxStmts) { - await this.client.query(this.ctxStmts); - } - const result = await this.client.query(query, values); - return result; + .map(([key, val]) => + val === null + ? `SELECT set_config('${key}', NULL, true);` + : `SELECT set_config('${key}', '${val}', true);` + ) + .join('\n'); } async any(query: string, values?: any[]): Promise { - const result = await this.runCtxQuery(query, values); + const result = await this.query(query, values); return result.rows; } @@ -99,15 +91,19 @@ export class PgTestClient { } async none(query: string, values?: any[]): Promise { - await this.runCtxQuery(query, values); + await this.query(query, values); } async result(query: string, values?: any[]): Promise { - return this.runCtxQuery(query, values); + return this.query(query, values); } async query(query: string, values?: any[]): Promise> { - return this.client.query(query, values); - } + if (this.ctxStmts) { + await this.client.query(this.ctxStmts); + } + const result = await this.client.query(query, values); + return result; + } } From b0e24e143ac9f11240ed1a91ac72198191199a09 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sat, 17 May 2025 13:10:49 -0700 Subject: [PATCH 16/29] seeding templates --- .../postgres-test.connections.test.ts | 19 +++-- .../__tests__/postgres-test.grants.test.ts | 66 ++++++++--------- .../__tests__/postgres-test.records.test.ts | 24 +++---- .../__tests__/postgres-test.template.test.ts | 72 ++++++++++--------- packages/pgsql-test/package.json | 3 +- packages/pgsql-test/src/connect.ts | 65 +++++++++++++---- 6 files changed, 144 insertions(+), 105 deletions(-) diff --git a/packages/pgsql-test/__tests__/postgres-test.connections.test.ts b/packages/pgsql-test/__tests__/postgres-test.connections.test.ts index 7eb0c61ae5..90d5195cd2 100644 --- a/packages/pgsql-test/__tests__/postgres-test.connections.test.ts +++ b/packages/pgsql-test/__tests__/postgres-test.connections.test.ts @@ -1,12 +1,11 @@ import { getConnections } from '../src/connect'; import { PgTestClient } from '../src/test-client'; -let conn: PgTestClient; let db: PgTestClient; let teardown: () => Promise; beforeAll(async () => { - ({ conn, db, teardown } = await getConnections()); + ({ db, teardown } = await getConnections()); }); afterAll(async () => { @@ -15,39 +14,39 @@ afterAll(async () => { describe('anonymous', () => { beforeEach(async () => { - await conn.beforeEach(); + await db.beforeEach(); }); afterEach(async () => { - await conn.afterEach(); + await db.afterEach(); }); it('runs under anonymous context', async () => { - const result = await conn.query('SELECT current_setting(\'role\', true) AS role'); + const result = await db.query('SELECT current_setting(\'role\', true) AS role'); expect(result.rows[0].role).toBe('anonymous'); }); }); describe('authenticated', () => { beforeEach(async () => { - conn.setContext({ + db.setContext({ role: 'authenticated', 'jwt.claims.user_agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36', 'jwt.claims.ip_address': '127.0.0.1', 'jwt.claims.database_id': 'jwt.database_id', 'jwt.claims.user_id': 'jwt.user_id' }); - await conn.beforeEach(); + await db.beforeEach(); }); afterEach(async () => { - await conn.afterEach(); + await db.afterEach(); }); it('runs under authenticated context', async () => { - const role = await conn.query('SELECT current_setting(\'role\', true) AS role'); + const role = await db.query('SELECT current_setting(\'role\', true) AS role'); expect(role.rows[0].role).toBe('authenticated'); - const ip = await conn.query('SELECT current_setting(\'jwt.claims.ip_address\', true) AS role'); + const ip = await db.query('SELECT current_setting(\'jwt.claims.ip_address\', true) AS role'); expect(ip.rows[0].role).toBe('127.0.0.1'); }); }); diff --git a/packages/pgsql-test/__tests__/postgres-test.grants.test.ts b/packages/pgsql-test/__tests__/postgres-test.grants.test.ts index ad14090e31..fdc240adc7 100644 --- a/packages/pgsql-test/__tests__/postgres-test.grants.test.ts +++ b/packages/pgsql-test/__tests__/postgres-test.grants.test.ts @@ -1,53 +1,49 @@ -import { getPgEnvOptions, PgConfig } from '@launchql/types'; - -import { - getConnection, - Connection, -} from '../src'; import { randomUUID } from 'crypto'; +import { resolve } from 'path'; import { PgTestClient } from '../src/test-client'; +import { PgConfig } from '@launchql/types'; import { DbAdmin } from '../src/admin'; -import { resolve } from 'path'; +import { getConnections } from '../src/connect'; +import { PgTestConnector } from '../src/manager'; -const sql = (file: string) => resolve(__dirname, '../sql', file); +let db: PgTestClient; +let pg: PgTestClient; +let teardown: () => Promise; -const TEST_DB_BASE = `postgres_test_${randomUUID()}`; +const TEST_DB_NAME = `postgres_test_${randomUUID()}`; +const sqlPath = (file: string) => resolve(__dirname, '../sql', file); -function setupBaseDB(config: PgConfig): void { +/** + * Optionally load seed SQL into the base template before forking connections. + */ +function setupBaseDatabase(config: PgConfig) { const admin = new DbAdmin(config); - admin.create(config.database) - admin.loadSql(sql('test.sql'), config.database); - admin.loadSql(sql('roles.sql'), config.database); - admin.drop(config.database); + admin.loadSql(sqlPath('roles.sql'), config.database); + admin.loadSql(sqlPath('test.sql'), config.database); } -const config = getPgEnvOptions({ - database: TEST_DB_BASE -}); +beforeAll(async () => { + // Create test DB and clients + ({ db, pg, teardown } = await getConnections({ + database: TEST_DB_NAME + })); -beforeAll(() => { - setupBaseDB(config); + // Optionally preload schema/data before tests + setupBaseDatabase(pg.config); }); -afterAll(() => { - Connection.getManager().closeAll(); +afterAll(async () => { + await teardown(); // closes manager and drops DB if needed }); - -describe('Postgres Test Framework', () => { - let db: PgTestClient; - - afterEach(() => { - // if (db) closeConnection(db); - }); - - it('creates a test DB with hot mode (FAST_TEST)', () => { - db = getConnection({ hot: true, extensions: ['uuid-ossp'] }); - expect(db).toBeDefined(); +describe('Database Setup', () => { + it('has valid connection (pg)', async () => { + const result = await pg.query('SELECT 1 AS ok'); + expect(result.rows[0].ok).toBe(1); }); - it('creates a test DB from scratch (default)', () => { - db = getConnection({}); - expect(db).toBeDefined(); + it('has valid connection (db)', async () => { + const result = await db.query('SELECT 2 AS ok'); + expect(result.rows[0].ok).toBe(2); }); }); diff --git a/packages/pgsql-test/__tests__/postgres-test.records.test.ts b/packages/pgsql-test/__tests__/postgres-test.records.test.ts index 4a0057a1f2..83d23c01f9 100644 --- a/packages/pgsql-test/__tests__/postgres-test.records.test.ts +++ b/packages/pgsql-test/__tests__/postgres-test.records.test.ts @@ -1,8 +1,7 @@ import { getConnections } from '../src/connect'; import { PgTestClient } from '../src/test-client'; -let conn: PgTestClient; -let db: PgTestClient; +let pg: PgTestClient; let teardown: () => Promise; const setupSchemaSQL = ` @@ -26,10 +25,10 @@ const seedDataSQL = ` `; beforeAll(async () => { - ({ conn, db, teardown } = await getConnections()); + ({ pg, teardown } = await getConnections()); // create schema + seed *once* - await db.query(setupSchemaSQL); - await db.query(seedDataSQL); + await pg.query(setupSchemaSQL); + await pg.query(seedDataSQL); }); afterAll(async () => { @@ -38,28 +37,27 @@ afterAll(async () => { describe('Postgres Test Framework', () => { beforeEach(async () => { - await db.beforeEach(); // BEGIN + SAVEPOINT + await pg.beforeEach(); // BEGIN + SAVEPOINT }); afterEach(async () => { - await db.afterEach(); // ROLLBACK TO SAVEPOINT + COMMIT + await pg.afterEach(); // ROLLBACK TO SAVEPOINT + COMMIT }); it('should have 2 users initially', async () => { - const { rows } = await db.query('SELECT COUNT(*) FROM users'); + const { rows } = await pg.query('SELECT COUNT(*) FROM users'); expect(rows[0].count).toBe('2'); }); it('inserts a user but rollback leaves baseline intact', async () => { - await db.query(`INSERT INTO users (name) VALUES ('Carol')`); - let res = await db.query('SELECT COUNT(*) FROM users'); + await pg.query(`INSERT INTO users (name) VALUES ('Carol')`); + let res = await pg.query('SELECT COUNT(*) FROM users'); expect(res.rows[0].count).toBe('3'); // inside this tx - - // after rollback (next test) we’ll still see 2 + // after rollback... the next test, we’ll still see 2 }); it('still sees 2 users after previous insert test', async () => { - const { rows } = await db.query('SELECT COUNT(*) FROM users'); + const { rows } = await pg.query('SELECT COUNT(*) FROM users'); expect(rows[0].count).toBe('2'); }); diff --git a/packages/pgsql-test/__tests__/postgres-test.template.test.ts b/packages/pgsql-test/__tests__/postgres-test.template.test.ts index 2d99683dd9..bb7bcd6bf9 100644 --- a/packages/pgsql-test/__tests__/postgres-test.template.test.ts +++ b/packages/pgsql-test/__tests__/postgres-test.template.test.ts @@ -1,52 +1,60 @@ -import { getPgEnvOptions, PgConfig } from '@launchql/types'; import path from 'path'; +import { randomUUID } from 'crypto'; +import { getPgEnvOptions, PgConfig } from '@launchql/types'; -import { - getConnection, - Connection -} from '../src'; import { PgTestClient } from '../src/test-client'; import { DbAdmin } from '../src/admin'; +import { getConnections } from '../src/connect'; const sql = (file: string) => path.resolve(__dirname, '../sql', file); +let db: PgTestClient; +let pg: PgTestClient; +let teardown: () => Promise; + const TEMPLATE_NAME = 'test_template'; -const TEST_DB_BASE = 'postgres_test_db_template'; +const SEED_DB_TO_CREATE_TEMPLATE = `postgres_test_${randomUUID()}`; + +/** + * Load SQL and create a reusable template database. + */ +function setupTemplateDatabase(): void { + const templateConfig = getPgEnvOptions({ + database: SEED_DB_TO_CREATE_TEMPLATE + }); -function setupTemplateDB(config: PgConfig, template: string): void { - const admin = new DbAdmin(config); + const admin = new DbAdmin(templateConfig); try { - admin.drop(config.database); - } catch {} - admin.create(config.database); - admin.loadSql(sql('test.sql'), config.database); - admin.loadSql(sql('roles.sql'), config.database); - admin.cleanupTemplate(template); - admin.createTemplateFromBase(config.database, template); - admin.drop(config.database); + admin.drop(SEED_DB_TO_CREATE_TEMPLATE); + } catch { } + admin.create(SEED_DB_TO_CREATE_TEMPLATE); + + // Load schema/data into base DB + admin.loadSql(sql('test.sql'), SEED_DB_TO_CREATE_TEMPLATE); + admin.loadSql(sql('roles.sql'), SEED_DB_TO_CREATE_TEMPLATE); + admin.cleanupTemplate(TEMPLATE_NAME); + admin.createTemplateFromBase(SEED_DB_TO_CREATE_TEMPLATE, TEMPLATE_NAME); + admin.drop(SEED_DB_TO_CREATE_TEMPLATE); } -const config = getPgEnvOptions({ - database: TEST_DB_BASE -}); +beforeAll(async () => { + // Step 1: Spin up a base DB client to set up the template + setupTemplateDatabase(); + + // // Step 2: Spin up a new DB from that template for test use + ({ db, pg, teardown } = await getConnections({}, { + template: TEMPLATE_NAME + })); -beforeAll(() => { - setupTemplateDB(config, TEMPLATE_NAME); }); -afterAll(() => { - Connection.getManager().closeAll(); +afterAll(async () => { + await teardown(); // Cleans up all connections via PgTestConnector }); describe('Template Database Test', () => { - let db: PgTestClient; - - afterEach(() => { - // if (db) closeConnection(db); - }); - - it('creates a test DB from a template', () => { - db = getConnection({ template: TEMPLATE_NAME }); - expect(db).toBeDefined(); + it('uses a database created from the template', async () => { + const res = await db.query('SELECT 1 AS ok'); + expect(res.rows[0].ok).toBe(1); }); }); diff --git a/packages/pgsql-test/package.json b/packages/pgsql-test/package.json index 04120f5af6..516322a836 100644 --- a/packages/pgsql-test/package.json +++ b/packages/pgsql-test/package.json @@ -31,6 +31,7 @@ }, "dependencies": { "@launchql/types": "^2.0.4", - "chalk": "^4.1.0" + "chalk": "^4.1.0", + "deepmerge": "^4.3.1" } } \ No newline at end of file diff --git a/packages/pgsql-test/src/connect.ts b/packages/pgsql-test/src/connect.ts index 2fd63a4df3..e68c0ff268 100644 --- a/packages/pgsql-test/src/connect.ts +++ b/packages/pgsql-test/src/connect.ts @@ -2,41 +2,78 @@ import { randomUUID } from 'crypto'; import { getPgEnvOptions, PgConfig } from '@launchql/types'; import { PgTestConnector } from './manager'; import { DbAdmin } from './admin'; +import deepmerge from 'deepmerge'; let manager: PgTestConnector; -export const getConnections = async () => { + +export interface TestConnectionOptions { + hot?: boolean; + template?: string; + prefix?: string; + extensions?: string[]; + connection?: { + user?: string; + password?: string; + role?: string; + } +} + +const defaultTestConnOpts: Partial = { + prefix: 'db-', + extensions: [], + connection: { + user: 'app_user', + password: 'app_password', + role: 'anonymous' + } + +} + +export const getConnections = async ( + _pgConfig: Partial = {}, + _opts: TestConnectionOptions = {} +) => { + const connOpts = deepmerge(defaultTestConnOpts, _opts); const config: PgConfig = getPgEnvOptions({ - database: `db-${randomUUID()}` + database: `${connOpts.prefix}${randomUUID()}`, + ..._pgConfig }); - const app_user = 'app_user'; - const app_password = 'app_password'; - const admin = new DbAdmin(config); // Create the test database - admin.create(config.database); + if (process.env.TEST_DB) { + config.database = process.env.TEST_DB; + } else if (connOpts.hot) { + admin.create(config.database); + admin.installExtensions(connOpts.extensions); + } else if (connOpts.template) { + admin.createFromTemplate(connOpts.template, config.database); + } else { + admin.create(config.database); + admin.installExtensions(connOpts.extensions); + } // Main admin client (optional unless needed elsewhere) manager = PgTestConnector.getInstance(); - const db = manager.getClient(config); + const pg = manager.getClient(config); // Set up test role - admin.createUserRole(app_user, app_password, config.database); - admin.grantConnect(app_user, config.database); + admin.createUserRole(connOpts.connection.user, connOpts.connection.password, config.database); + admin.grantConnect(connOpts.connection.user, config.database); // App user connection - const conn = manager.getClient({ + const db = manager.getClient({ ...config, - user: app_user, - password: app_password + user: connOpts.connection.user, + password: connOpts.connection.password }) // const conn = await getTestConnection(config.database, app_user, app_password); - conn.setContext({ role: 'anonymous' }); + db.setContext({ role: 'anonymous' }); const teardown = async () => { await manager.closeAll(); }; - return { db, conn, teardown }; + return { pg, db, teardown, manager }; }; From 0ff84d9bf894a8f1c2c8d72afcf1968aec5f1fe6 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sat, 17 May 2025 13:43:47 -0700 Subject: [PATCH 17/29] template vs. rollbacks --- .../__tests__/postgres-test.rollbacks.test.ts | 75 ++++++++++++++++++ .../__tests__/postgres-test.template.test.ts | 77 +++++++++++++------ packages/pgsql-test/src/admin.ts | 45 ++++++++++- 3 files changed, 173 insertions(+), 24 deletions(-) create mode 100644 packages/pgsql-test/__tests__/postgres-test.rollbacks.test.ts diff --git a/packages/pgsql-test/__tests__/postgres-test.rollbacks.test.ts b/packages/pgsql-test/__tests__/postgres-test.rollbacks.test.ts new file mode 100644 index 0000000000..8e3d8bbfe7 --- /dev/null +++ b/packages/pgsql-test/__tests__/postgres-test.rollbacks.test.ts @@ -0,0 +1,75 @@ +import path from 'path'; +import { getPgEnvOptions } from '@launchql/types'; + +import { PgTestClient } from '../src/test-client'; +import { DbAdmin } from '../src/admin'; +import { getConnections } from '../src/connect'; + +const sql = (file: string) => path.resolve(__dirname, '../sql', file); + +let pg: PgTestClient; +let teardown: () => Promise; + +const usedDbNames: string[] = []; +const testResults: { name: string; time: number }[] = []; + +let start: number; + +beforeAll(async () => { + ({ pg, teardown } = await getConnections()); + + const admin = new DbAdmin(pg.config); + admin.loadSql(sql('test.sql'), pg.config.database); + admin.loadSql(sql('roles.sql'), pg.config.database); + + usedDbNames.push(pg.config.database); +}); + +beforeEach(async () => { + await pg.beforeEach(); + start = Date.now(); +}); + +afterEach(async () => { + const elapsed = Date.now() - start; + const name = expect.getState().currentTestName ?? 'unknown'; + testResults.push({ name, time: elapsed }); + + await pg.afterEach(); +}); + +afterAll(async () => { + await teardown(); + + const avg = + testResults.reduce((sum, r) => sum + r.time, 0) / testResults.length; + + const summaryLines = [ + `πŸ§ͺ Rollback DB Benchmark`, + `πŸ“‚ DB Used: ${usedDbNames[0]}`, + `⏱️ Test Timings:`, + ...testResults.map(({ name, time }) => ` β€’ ${name}: ${time}ms`), + `🏁 Average Test Time: ${avg.toFixed(2)}ms` + ]; + + console.log('\n' + summaryLines.join('\n') + '\n'); +}); + +describe('Rollback DB Benchmark', () => { + it('inserts Alice', async () => { + await pg.query(`INSERT INTO app_public.users (username) VALUES ('alice')`); + const res = await pg.query(`SELECT COUNT(*) FROM app_public.users`); + expect(res.rows[0].count).toBe('1'); + }); + + it('starts clean without Alice', async () => { + const res = await pg.query(`SELECT * FROM app_public.users WHERE username = 'alice'`); + expect(res.rows).toHaveLength(0); + }); + + it('inserts Bob, settings should be empty', async () => { + await pg.query(`INSERT INTO app_public.users (username) VALUES ('bob')`); + const settings = await pg.query(`SELECT * FROM app_public.user_settings`); + expect(settings.rows).toHaveLength(0); + }); +}); diff --git a/packages/pgsql-test/__tests__/postgres-test.template.test.ts b/packages/pgsql-test/__tests__/postgres-test.template.test.ts index bb7bcd6bf9..adab2e0904 100644 --- a/packages/pgsql-test/__tests__/postgres-test.template.test.ts +++ b/packages/pgsql-test/__tests__/postgres-test.template.test.ts @@ -1,6 +1,6 @@ import path from 'path'; import { randomUUID } from 'crypto'; -import { getPgEnvOptions, PgConfig } from '@launchql/types'; +import { getPgEnvOptions } from '@launchql/types'; import { PgTestClient } from '../src/test-client'; import { DbAdmin } from '../src/admin'; @@ -15,46 +15,77 @@ let teardown: () => Promise; const TEMPLATE_NAME = 'test_template'; const SEED_DB_TO_CREATE_TEMPLATE = `postgres_test_${randomUUID()}`; -/** - * Load SQL and create a reusable template database. - */ +const usedDbNames: string[] = []; +const testResults: { name: string; time: number }[] = []; + +let start: number; + function setupTemplateDatabase(): void { const templateConfig = getPgEnvOptions({ database: SEED_DB_TO_CREATE_TEMPLATE }); const admin = new DbAdmin(templateConfig); - try { - admin.drop(SEED_DB_TO_CREATE_TEMPLATE); - } catch { } - admin.create(SEED_DB_TO_CREATE_TEMPLATE); - - // Load schema/data into base DB - admin.loadSql(sql('test.sql'), SEED_DB_TO_CREATE_TEMPLATE); - admin.loadSql(sql('roles.sql'), SEED_DB_TO_CREATE_TEMPLATE); admin.cleanupTemplate(TEMPLATE_NAME); - admin.createTemplateFromBase(SEED_DB_TO_CREATE_TEMPLATE, TEMPLATE_NAME); - admin.drop(SEED_DB_TO_CREATE_TEMPLATE); + admin.createSeededTemplate(TEMPLATE_NAME, { + seed: (db, dbName) => { + db.loadSql(sql('test.sql'), dbName); + db.loadSql(sql('roles.sql'), dbName); + } + }); } -beforeAll(async () => { - // Step 1: Spin up a base DB client to set up the template +beforeAll(() => { setupTemplateDatabase(); +}); - // // Step 2: Spin up a new DB from that template for test use +beforeEach(async () => { ({ db, pg, teardown } = await getConnections({}, { template: TEMPLATE_NAME })); + usedDbNames.push(db?.config?.database ?? '(unknown)'); + start = Date.now(); +}); +afterEach(async () => { + await teardown(); + const elapsed = Date.now() - start; + const name = expect.getState().currentTestName ?? 'unknown'; + testResults.push({ name, time: elapsed }); }); -afterAll(async () => { - await teardown(); // Cleans up all connections via PgTestConnector +afterAll(() => { + const uniqueNames = new Set(usedDbNames); + const avg = testResults.reduce((sum, r) => sum + r.time, 0) / testResults.length; + + const summaryLines = [ + `πŸ§ͺ Template DB Benchmark`, + `πŸ“¦ Total DBs Created: ${usedDbNames.length}`, + `πŸ“‚ Template Used: ${TEMPLATE_NAME}`, + `βœ… Unique DBs: ${uniqueNames.size === usedDbNames.length}`, + `⏱️ Test Timings:`, + ...testResults.map(({ name, time }) => ` β€’ ${name}: ${time}ms`), + `🏁 Average Test Time: ${avg.toFixed(2)}ms` + ]; + + console.log('\n' + summaryLines.join('\n') + '\n'); }); -describe('Template Database Test', () => { - it('uses a database created from the template', async () => { - const res = await db.query('SELECT 1 AS ok'); - expect(res.rows[0].ok).toBe(1); +describe('Template DB Benchmark', () => { + it('inserts Alice', async () => { + await pg.query(`INSERT INTO app_public.users (username) VALUES ('alice')`); + const res = await pg.query(`SELECT COUNT(*) FROM app_public.users`); + expect(res.rows[0].count).toBe('1'); + }); + + it('starts clean without Alice', async () => { + const res = await pg.query(`SELECT * FROM app_public.users WHERE username = 'alice'`); + expect(res.rows).toHaveLength(0); + }); + + it('inserts Bob, settings should be empty', async () => { + await pg.query(`INSERT INTO app_public.users (username) VALUES ('bob')`); + const settings = await pg.query(`SELECT * FROM app_public.user_settings`); + expect(settings.rows).toHaveLength(0); }); }); diff --git a/packages/pgsql-test/src/admin.ts b/packages/pgsql-test/src/admin.ts index aa6881d9eb..583d6cc91e 100644 --- a/packages/pgsql-test/src/admin.ts +++ b/packages/pgsql-test/src/admin.ts @@ -2,6 +2,40 @@ import { execSync } from 'child_process'; import { PgConfig } from '@launchql/types'; import { existsSync } from 'fs'; import { streamSql as stream } from './stream'; +import { randomUUID } from 'crypto'; + +export interface SeedAdapter { + seed(db: DbAdmin, dbName: string): Promise | void; +} + +export function sqlFileSeedAdapter(files: string[]): SeedAdapter { + return { + seed(db, dbName) { + for (const file of files) { + db.loadSql(file, dbName); + } + } + }; +} + +export function programmaticSeedAdapter( + fn: (db: DbAdmin, dbName: string) => Promise +): SeedAdapter { + return { + seed: fn + }; +} + +export function compositeSeedAdapter(adapters: SeedAdapter[]): SeedAdapter { + return { + async seed(db, dbName) { + for (const adapter of adapters) { + await adapter.seed(db, dbName); + } + } + }; +} + export class DbAdmin { constructor( @@ -130,6 +164,15 @@ export class DbAdmin { ...this.config, database: dbName }, sql); - } + } + async createSeededTemplate(templateName: string, adapter: SeedAdapter): Promise { + const seedDb = this.config.database; + this.create(seedDb); + await adapter.seed(this, seedDb); + this.cleanupTemplate(templateName); + this.createTemplateFromBase(seedDb, templateName); + this.drop(seedDb); + } + } From 192435005bb10507fcdea86c7eafea3d9afa3b1f Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sat, 17 May 2025 14:33:17 -0700 Subject: [PATCH 18/29] benchmarking various db seeding strategies during testing --- packages/migrate/src/deploy.ts | 2 +- .../postgres-test.deploy-fast.test.ts | 88 ++++++++++++++++++ .../postgres-test.deploy-sqitch.test.ts | 89 +++++++++++++++++++ .../__tests__/postgres-test.rollbacks.test.ts | 8 +- .../__tests__/postgres-test.template.test.ts | 6 +- packages/pgsql-test/package.json | 1 + packages/pgsql-test/src/connect.ts | 54 ++++++++--- 7 files changed, 232 insertions(+), 16 deletions(-) create mode 100644 packages/pgsql-test/__tests__/postgres-test.deploy-fast.test.ts create mode 100644 packages/pgsql-test/__tests__/postgres-test.deploy-sqitch.test.ts diff --git a/packages/migrate/src/deploy.ts b/packages/migrate/src/deploy.ts index bd67041a75..1ab13c4f9b 100644 --- a/packages/migrate/src/deploy.ts +++ b/packages/migrate/src/deploy.ts @@ -91,6 +91,6 @@ export const deploy = async ( } console.log(chalk.green(`\nβœ… Deployment complete for ${chalk.bold(name)}.\n`)); - await pgPool.end(); + // await pgPool.end(); return extensions; }; diff --git a/packages/pgsql-test/__tests__/postgres-test.deploy-fast.test.ts b/packages/pgsql-test/__tests__/postgres-test.deploy-fast.test.ts new file mode 100644 index 0000000000..92a4377098 --- /dev/null +++ b/packages/pgsql-test/__tests__/postgres-test.deploy-fast.test.ts @@ -0,0 +1,88 @@ +import path, { resolve } from 'path'; + +import { PgTestClient } from '../src/test-client'; +import { DbAdmin } from '../src/admin'; +import { getConnections } from '../src/connect'; +import { getRootPgPool } from '@launchql/server-utils'; + +const sql = (file: string) => path.resolve(__dirname, '../sql', file); + +let pg: PgTestClient; +let teardown: () => Promise; + +const usedDbNames: string[] = []; +const testResults: { name: string; time: number }[] = []; + +let start: number; +let totalStart: number; + +beforeAll(async () => { + totalStart = Date.now(); + + ({ pg, teardown } = await getConnections({}, { + cwd: resolve(__dirname + '/../../../__fixtures__/sqitch/simple/packages/my-third') + })); + + const admin = new DbAdmin(pg.config); + admin.loadSql(sql('test.sql'), pg.config.database); + admin.loadSql(sql('roles.sql'), pg.config.database); + + usedDbNames.push(pg.config.database); +}); + +beforeEach(async () => { + await pg.beforeEach(); + start = Date.now(); +}); + +afterEach(async () => { + const elapsed = Date.now() - start; + const name = expect.getState().currentTestName ?? 'unknown'; + testResults.push({ name, time: elapsed }); + + await pg.afterEach(); +}); + +afterAll(async () => { + + // clear this DB first! so teardown() doesn't choke...a + const pgPool = getRootPgPool({ ...pg.config, database: pg.config.database }); + await pgPool.end(); + + await teardown(); + + const avg = + testResults.reduce((sum, r) => sum + r.time, 0) / testResults.length; + + const totalTime = Date.now() - totalStart; + + const summaryLines = [ + `πŸ§ͺ LaunchQL DeployFast DB Benchmark`, + `πŸ“‚ DB Used: ${usedDbNames[0]}`, + `⏱️ Test Timings:`, + ...testResults.map(({ name, time }) => ` β€’ ${name}: ${time}ms`), + `🏁 Average Test Time: ${avg.toFixed(2)}ms`, + `πŸ•’ Total Test Time: ${totalTime}ms` + ]; + + console.log('\n' + summaryLines.join('\n') + '\n'); +}); + +describe('LaunchQL DeployFast DB Benchmark', () => { + it('inserts Alice', async () => { + await pg.query(`INSERT INTO app_public.users (username) VALUES ('alice')`); + const res = await pg.query(`SELECT COUNT(*) FROM app_public.users`); + expect(res.rows[0].count).toBe('1'); + }); + + it('starts clean without Alice', async () => { + const res = await pg.query(`SELECT * FROM app_public.users WHERE username = 'alice'`); + expect(res.rows).toHaveLength(0); + }); + + it('inserts Bob, settings should be empty', async () => { + await pg.query(`INSERT INTO app_public.users (username) VALUES ('bob')`); + const settings = await pg.query(`SELECT * FROM app_public.user_settings`); + expect(settings.rows).toHaveLength(0); + }); +}); diff --git a/packages/pgsql-test/__tests__/postgres-test.deploy-sqitch.test.ts b/packages/pgsql-test/__tests__/postgres-test.deploy-sqitch.test.ts new file mode 100644 index 0000000000..6512ed16bf --- /dev/null +++ b/packages/pgsql-test/__tests__/postgres-test.deploy-sqitch.test.ts @@ -0,0 +1,89 @@ +import path, { resolve } from 'path'; + +import { PgTestClient } from '../src/test-client'; +import { DbAdmin } from '../src/admin'; +import { getConnections } from '../src/connect'; +import { getRootPgPool } from '@launchql/server-utils'; + +const sql = (file: string) => path.resolve(__dirname, '../sql', file); + +let pg: PgTestClient; +let teardown: () => Promise; + +const usedDbNames: string[] = []; +const testResults: { name: string; time: number }[] = []; + +let start: number; +let totalStart: number; + +beforeAll(async () => { + totalStart = Date.now(); + + ({ pg, teardown } = await getConnections({}, { + deployFast: false, + cwd: resolve(__dirname + '/../../../__fixtures__/sqitch/simple/packages/my-third') + })); + + const admin = new DbAdmin(pg.config); + admin.loadSql(sql('test.sql'), pg.config.database); + admin.loadSql(sql('roles.sql'), pg.config.database); + + usedDbNames.push(pg.config.database); +}); + +beforeEach(async () => { + await pg.beforeEach(); + start = Date.now(); +}); + +afterEach(async () => { + const elapsed = Date.now() - start; + const name = expect.getState().currentTestName ?? 'unknown'; + testResults.push({ name, time: elapsed }); + + await pg.afterEach(); +}); + +afterAll(async () => { + + // clear this DB first! so teardown() doesn't choke...a + const pgPool = getRootPgPool({ ...pg.config, database: pg.config.database }); + await pgPool.end(); + + await teardown(); + + const avg = + testResults.reduce((sum, r) => sum + r.time, 0) / testResults.length; + + const totalTime = Date.now() - totalStart; + + const summaryLines = [ + `πŸ§ͺ LaunchQL Deploy (sqtich) DB Benchmark`, + `πŸ“‚ DB Used: ${usedDbNames[0]}`, + `⏱️ Test Timings:`, + ...testResults.map(({ name, time }) => ` β€’ ${name}: ${time}ms`), + `🏁 Average Test Time: ${avg.toFixed(2)}ms`, + `πŸ•’ Total Test Time: ${totalTime}ms` + ]; + + console.log('\n' + summaryLines.join('\n') + '\n'); +}); + +describe('LaunchQL Deploy (sqitch) DB Benchmark', () => { + it('inserts Alice', async () => { + await pg.query(`INSERT INTO app_public.users (username) VALUES ('alice')`); + const res = await pg.query(`SELECT COUNT(*) FROM app_public.users`); + expect(res.rows[0].count).toBe('1'); + }); + + it('starts clean without Alice', async () => { + const res = await pg.query(`SELECT * FROM app_public.users WHERE username = 'alice'`); + expect(res.rows).toHaveLength(0); + }); + + it('inserts Bob, settings should be empty', async () => { + await pg.query(`INSERT INTO app_public.users (username) VALUES ('bob')`); + const settings = await pg.query(`SELECT * FROM app_public.user_settings`); + expect(settings.rows).toHaveLength(0); + }); +}); diff --git a/packages/pgsql-test/__tests__/postgres-test.rollbacks.test.ts b/packages/pgsql-test/__tests__/postgres-test.rollbacks.test.ts index 8e3d8bbfe7..b42f8088b3 100644 --- a/packages/pgsql-test/__tests__/postgres-test.rollbacks.test.ts +++ b/packages/pgsql-test/__tests__/postgres-test.rollbacks.test.ts @@ -1,5 +1,4 @@ import path from 'path'; -import { getPgEnvOptions } from '@launchql/types'; import { PgTestClient } from '../src/test-client'; import { DbAdmin } from '../src/admin'; @@ -14,8 +13,11 @@ const usedDbNames: string[] = []; const testResults: { name: string; time: number }[] = []; let start: number; +let totalStart: number; beforeAll(async () => { + totalStart = Date.now(); + ({ pg, teardown } = await getConnections()); const admin = new DbAdmin(pg.config); @@ -41,6 +43,7 @@ afterEach(async () => { afterAll(async () => { await teardown(); + const totalTime = Date.now() - totalStart; const avg = testResults.reduce((sum, r) => sum + r.time, 0) / testResults.length; @@ -49,7 +52,8 @@ afterAll(async () => { `πŸ“‚ DB Used: ${usedDbNames[0]}`, `⏱️ Test Timings:`, ...testResults.map(({ name, time }) => ` β€’ ${name}: ${time}ms`), - `🏁 Average Test Time: ${avg.toFixed(2)}ms` + `🏁 Average Test Time: ${avg.toFixed(2)}ms`, + `πŸ•’ Total Test Time: ${totalTime}ms` ]; console.log('\n' + summaryLines.join('\n') + '\n'); diff --git a/packages/pgsql-test/__tests__/postgres-test.template.test.ts b/packages/pgsql-test/__tests__/postgres-test.template.test.ts index adab2e0904..e508e01d56 100644 --- a/packages/pgsql-test/__tests__/postgres-test.template.test.ts +++ b/packages/pgsql-test/__tests__/postgres-test.template.test.ts @@ -19,6 +19,7 @@ const usedDbNames: string[] = []; const testResults: { name: string; time: number }[] = []; let start: number; +let totalStart: number; function setupTemplateDatabase(): void { const templateConfig = getPgEnvOptions({ @@ -36,6 +37,7 @@ function setupTemplateDatabase(): void { } beforeAll(() => { + totalStart = Date.now(); setupTemplateDatabase(); }); @@ -55,6 +57,7 @@ afterEach(async () => { }); afterAll(() => { + const totalTime = Date.now() - totalStart; const uniqueNames = new Set(usedDbNames); const avg = testResults.reduce((sum, r) => sum + r.time, 0) / testResults.length; @@ -65,7 +68,8 @@ afterAll(() => { `βœ… Unique DBs: ${uniqueNames.size === usedDbNames.length}`, `⏱️ Test Timings:`, ...testResults.map(({ name, time }) => ` β€’ ${name}: ${time}ms`), - `🏁 Average Test Time: ${avg.toFixed(2)}ms` + `🏁 Average Test Time: ${avg.toFixed(2)}ms`, + `πŸ•’ Total Test Time: ${totalTime}ms` ]; console.log('\n' + summaryLines.join('\n') + '\n'); diff --git a/packages/pgsql-test/package.json b/packages/pgsql-test/package.json index 516322a836..d078c91c5c 100644 --- a/packages/pgsql-test/package.json +++ b/packages/pgsql-test/package.json @@ -31,6 +31,7 @@ }, "dependencies": { "@launchql/types": "^2.0.4", + "@launchql/migrate": "^2.0.11", "chalk": "^4.1.0", "deepmerge": "^4.3.1" } diff --git a/packages/pgsql-test/src/connect.ts b/packages/pgsql-test/src/connect.ts index e68c0ff268..219506e28d 100644 --- a/packages/pgsql-test/src/connect.ts +++ b/packages/pgsql-test/src/connect.ts @@ -1,7 +1,8 @@ -import { randomUUID } from 'crypto'; -import { getPgEnvOptions, PgConfig } from '@launchql/types'; -import { PgTestConnector } from './manager'; import { DbAdmin } from './admin'; +import { getEnvOptions, getPgEnvOptions, PgConfig } from '@launchql/types'; +import { deploy, deployFast, LaunchQLProject } from '@launchql/migrate'; +import { PgTestConnector } from './manager'; +import { randomUUID } from 'crypto'; import deepmerge from 'deepmerge'; let manager: PgTestConnector; @@ -11,6 +12,8 @@ export interface TestConnectionOptions { template?: string; prefix?: string; extensions?: string[]; + cwd?: string; + deployFast?: boolean; connection?: { user?: string; password?: string; @@ -21,6 +24,8 @@ export interface TestConnectionOptions { const defaultTestConnOpts: Partial = { prefix: 'db-', extensions: [], + cwd: process.cwd(), + deployFast: true, connection: { user: 'app_user', password: 'app_password', @@ -41,19 +46,44 @@ export const getConnections = async ( const admin = new DbAdmin(config); - // Create the test database - if (process.env.TEST_DB) { - config.database = process.env.TEST_DB; - } else if (connOpts.hot) { + const proj = new LaunchQLProject(connOpts.cwd); + if (proj.isInModule()) { admin.create(config.database); admin.installExtensions(connOpts.extensions); - } else if (connOpts.template) { - admin.createFromTemplate(connOpts.template, config.database); + const opts = getEnvOptions({ + pg: config + }) + if (connOpts.deployFast) { + await deployFast({ + opts, + name: proj.getModuleName(), + database: config.database, + dir: proj.modulePath, + usePlan: true, + verbose: false + }) + } else { + await deploy(opts, proj.getModuleName(), config.database, proj.modulePath); + } } else { - admin.create(config.database); - admin.installExtensions(connOpts.extensions); + + // Create the test database + if (process.env.TEST_DB) { + config.database = process.env.TEST_DB; + } else if (connOpts.hot) { + admin.create(config.database); + admin.installExtensions(connOpts.extensions); + } else if (connOpts.template) { + admin.createFromTemplate(connOpts.template, config.database); + } else { + admin.create(config.database); + admin.installExtensions(connOpts.extensions); + } + } + + // Main admin client (optional unless needed elsewhere) manager = PgTestConnector.getInstance(); const pg = manager.getClient(config); @@ -68,7 +98,7 @@ export const getConnections = async ( user: connOpts.connection.user, password: connOpts.connection.password }) -// const conn = await getTestConnection(config.database, app_user, app_password); + // const conn = await getTestConnection(config.database, app_user, app_password); db.setContext({ role: 'anonymous' }); const teardown = async () => { From 55f5faa9c14edda53da18c0b4a67cc851c01d477 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sat, 17 May 2025 14:43:36 -0700 Subject: [PATCH 19/29] config --- packages/pgsql-test/src/admin.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/pgsql-test/src/admin.ts b/packages/pgsql-test/src/admin.ts index 583d6cc91e..47e2ba0d0b 100644 --- a/packages/pgsql-test/src/admin.ts +++ b/packages/pgsql-test/src/admin.ts @@ -1,5 +1,5 @@ import { execSync } from 'child_process'; -import { PgConfig } from '@launchql/types'; +import { getPgEnvOptions, PgConfig } from '@launchql/types'; import { existsSync } from 'fs'; import { streamSql as stream } from './stream'; import { randomUUID } from 'crypto'; @@ -41,7 +41,9 @@ export class DbAdmin { constructor( private config: PgConfig, private verbose: boolean = false - ) { } + ) { + this.config = getPgEnvOptions(config); + } private getEnv(): Record { return { From 0f22d1cbf671f9ce8c5d7c9737f2d45e3492bc57 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sat, 17 May 2025 14:45:55 -0700 Subject: [PATCH 20/29] defaults --- packages/types/src/env.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/types/src/env.ts b/packages/types/src/env.ts index 0c68086172..732a3891cd 100644 --- a/packages/types/src/env.ts +++ b/packages/types/src/env.ts @@ -1,5 +1,5 @@ import deepmerge from 'deepmerge'; -import { getMergedOptions, LaunchQLOptions, PgConfig } from './launchql'; +import { getMergedOptions, launchqlDefaults, LaunchQLOptions, PgConfig } from './launchql'; const parseEnvNumber = (val?: string): number | undefined => { const num = Number(val); @@ -22,7 +22,8 @@ export const getEnvOptions = (overrides: LaunchQLOptions = {}): LaunchQLOptions export const getPgEnvOptions = (overrides: Partial = {}): PgConfig => { const envOpts = getPgEnvVars(); - const options = deepmerge(envOpts, overrides); + const defaults = deepmerge(launchqlDefaults, envOpts); + const options = deepmerge(defaults, overrides); // if you need to sanitize... return options; }; From d9a8145abb5a8931dde41bb03072d0ae4ede36ae Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sat, 17 May 2025 14:51:54 -0700 Subject: [PATCH 21/29] clean pkgs --- packages/graphile-settings/package.json | 1 - packages/server-utils/package.json | 1 - packages/server-utils/src/env.ts | 8 -------- packages/server/package.json | 1 - 4 files changed, 11 deletions(-) delete mode 100644 packages/server-utils/src/env.ts diff --git a/packages/graphile-settings/package.json b/packages/graphile-settings/package.json index fd9deb6bb5..81ecf81540 100644 --- a/packages/graphile-settings/package.json +++ b/packages/graphile-settings/package.json @@ -38,7 +38,6 @@ "@pyramation/postgis": "^0.1.1", "@pyramation/postgraphile-plugin-fulltext-filter": "^2.0.0", "cors": "^2.8.5", - "envalid": "^8.0.0", "express": "^5.1.0", "graphile-build": "^4.14.1", "graphile-i18n": "^0.0.3", diff --git a/packages/server-utils/package.json b/packages/server-utils/package.json index 4adf56bef5..53cd509ff5 100644 --- a/packages/server-utils/package.json +++ b/packages/server-utils/package.json @@ -32,7 +32,6 @@ "dependencies": { "@launchql/types": "^2.0.4", "cors": "^2.8.5", - "envalid": "^8.0.0", "express": "^5.1.0", "lru-cache": "^11.1.0", "pg": "^8.15.6", diff --git a/packages/server-utils/src/env.ts b/packages/server-utils/src/env.ts deleted file mode 100644 index 308e614c32..0000000000 --- a/packages/server-utils/src/env.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { cleanEnv, str, port } from 'envalid'; - -export const env = cleanEnv(process.env, { - PGUSER: str({ default: 'postgres' }), - PGHOST: str({ default: 'localhost' }), - PGPASSWORD: str({ default: 'password' }), - PGPORT: port({ default: 5432 }) -}); diff --git a/packages/server/package.json b/packages/server/package.json index 39e5b89ea9..8b5e5b963d 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -44,7 +44,6 @@ "@pyramation/postgis": "^0.1.1", "@pyramation/postgraphile-plugin-fulltext-filter": "^2.0.0", "cors": "^2.8.5", - "envalid": "^8.0.0", "express": "^5.1.0", "graphile-build": "^4.14.1", "graphile-i18n": "^0.0.3", From d3c3d4679ee7b6933494193f055b8978844cf9d5 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sat, 17 May 2025 14:53:44 -0700 Subject: [PATCH 22/29] remove hot option --- packages/pgsql-test/src/connect.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/pgsql-test/src/connect.ts b/packages/pgsql-test/src/connect.ts index 219506e28d..e714d9b386 100644 --- a/packages/pgsql-test/src/connect.ts +++ b/packages/pgsql-test/src/connect.ts @@ -8,7 +8,6 @@ import deepmerge from 'deepmerge'; let manager: PgTestConnector; export interface TestConnectionOptions { - hot?: boolean; template?: string; prefix?: string; extensions?: string[]; @@ -70,9 +69,6 @@ export const getConnections = async ( // Create the test database if (process.env.TEST_DB) { config.database = process.env.TEST_DB; - } else if (connOpts.hot) { - admin.create(config.database); - admin.installExtensions(connOpts.extensions); } else if (connOpts.template) { admin.createFromTemplate(connOpts.template, config.database); } else { From fe9afe630950329ef702bd9fb83e55b48f2035e1 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sat, 17 May 2025 15:06:34 -0700 Subject: [PATCH 23/29] pg --- packages/types/src/env.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/types/src/env.ts b/packages/types/src/env.ts index 732a3891cd..fb9475b4a6 100644 --- a/packages/types/src/env.ts +++ b/packages/types/src/env.ts @@ -22,7 +22,7 @@ export const getEnvOptions = (overrides: LaunchQLOptions = {}): LaunchQLOptions export const getPgEnvOptions = (overrides: Partial = {}): PgConfig => { const envOpts = getPgEnvVars(); - const defaults = deepmerge(launchqlDefaults, envOpts); + const defaults = deepmerge(launchqlDefaults.pg, envOpts); const options = deepmerge(defaults, overrides); // if you need to sanitize... return options; From f74b8252c94ee7b174e8325810a65e75736ea74f Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sat, 17 May 2025 15:39:09 -0700 Subject: [PATCH 24/29] spawn env --- packages/migrate/src/deploy.ts | 4 +- packages/migrate/src/stream-sql.ts | 12 +- packages/pgsql-test/package.json | 1 + packages/pgsql-test/src/connect.ts | 2 + packages/pgsql-test/src/stream.ts | 11 +- packages/server-utils/src/lru.ts | 172 +++++++++++++++++++--------- packages/types/src/env.ts | 175 ++++++++++++++++------------- 7 files changed, 228 insertions(+), 149 deletions(-) diff --git a/packages/migrate/src/deploy.ts b/packages/migrate/src/deploy.ts index 1ab13c4f9b..e551a63e04 100644 --- a/packages/migrate/src/deploy.ts +++ b/packages/migrate/src/deploy.ts @@ -1,7 +1,7 @@ import { resolve } from 'path'; import chalk from 'chalk'; -import { LaunchQLOptions } from '@launchql/types'; +import { getSpawnEnvWithPg, LaunchQLOptions } from '@launchql/types'; import { getRootPgPool } from '@launchql/server-utils'; import { LaunchQLProject } from './class/launchql'; import { spawn } from 'child_process'; @@ -53,7 +53,7 @@ export const deploy = async ( const child = spawn('sqitch', ['deploy', `db:pg:${database}`], { cwd: modulePath, - env: { ...process.env } + env: getSpawnEnvWithPg(opts.pg) }); const exitCode: number = await new Promise((resolve, reject) => { diff --git a/packages/migrate/src/stream-sql.ts b/packages/migrate/src/stream-sql.ts index 10bcd0fe86..c95cc06ca8 100644 --- a/packages/migrate/src/stream-sql.ts +++ b/packages/migrate/src/stream-sql.ts @@ -1,6 +1,6 @@ +import { getSpawnEnvWithPg } from '@launchql/types'; import { spawn } from 'child_process'; import { Readable } from 'stream'; -import { env } from 'process'; interface PgConfig { user: string; @@ -39,16 +39,8 @@ export async function streamSql(config: PgConfig, sql: string): Promise { return new Promise((resolve, reject) => { const sqlStream = stringToStream(sql); - // TODO set env vars! const proc = spawn('psql', args, { - env: { - ...env, - PGUSER: config.user, - PGHOST: config.host, - PGDATABASE: config.database, - PGPASSWORD: config.password, - // PGPORT: config.port - } + env: getSpawnEnvWithPg(config) }); sqlStream.pipe(proc.stdin); diff --git a/packages/pgsql-test/package.json b/packages/pgsql-test/package.json index d078c91c5c..6dadd2abb0 100644 --- a/packages/pgsql-test/package.json +++ b/packages/pgsql-test/package.json @@ -31,6 +31,7 @@ }, "dependencies": { "@launchql/types": "^2.0.4", + "@launchql/server-utils": "^2.0.4", "@launchql/migrate": "^2.0.11", "chalk": "^4.1.0", "deepmerge": "^4.3.1" diff --git a/packages/pgsql-test/src/connect.ts b/packages/pgsql-test/src/connect.ts index e714d9b386..6605812ac4 100644 --- a/packages/pgsql-test/src/connect.ts +++ b/packages/pgsql-test/src/connect.ts @@ -4,6 +4,7 @@ import { deploy, deployFast, LaunchQLProject } from '@launchql/migrate'; import { PgTestConnector } from './manager'; import { randomUUID } from 'crypto'; import deepmerge from 'deepmerge'; +import { teardownPgPools } from '@launchql/server-utils'; let manager: PgTestConnector; @@ -98,6 +99,7 @@ export const getConnections = async ( db.setContext({ role: 'anonymous' }); const teardown = async () => { + await teardownPgPools(); await manager.closeAll(); }; diff --git a/packages/pgsql-test/src/stream.ts b/packages/pgsql-test/src/stream.ts index a9cb88d1d1..be9617790d 100644 --- a/packages/pgsql-test/src/stream.ts +++ b/packages/pgsql-test/src/stream.ts @@ -1,7 +1,7 @@ import { spawn } from 'child_process'; import { Readable } from 'stream'; import { env } from 'process'; -import { PgConfig } from '@launchql/types'; +import { getSpawnEnvWithPg, PgConfig } from '@launchql/types'; function setArgs(config: PgConfig): string[] { const args = [ @@ -34,14 +34,7 @@ export async function streamSql(config: PgConfig, sql: string): Promise { // TODO set env vars! const proc = spawn('psql', args, { - env: { - ...env, - PGUSER: config.user, - PGHOST: config.host, - PGDATABASE: config.database, - PGPASSWORD: config.password, - // PGPORT: config.port - } + env: getSpawnEnvWithPg(config) }); sqlStream.pipe(proc.stdin); diff --git a/packages/server-utils/src/lru.ts b/packages/server-utils/src/lru.ts index 1262085e5a..afe34b9958 100644 --- a/packages/server-utils/src/lru.ts +++ b/packages/server-utils/src/lru.ts @@ -1,5 +1,4 @@ import { LRUCache } from 'lru-cache'; - import pg from 'pg'; import { HttpRequestHandler } from 'postgraphile'; @@ -10,19 +9,7 @@ const ONE_YEAR = ONE_DAY * 366; // Kubernetes sends only SIGTERM on pod shutdown const SYS_EVENTS = ['SIGTERM']; -const end = (pool: any) => { - try { - if (pool.ended || pool.ending) { - console.error( - 'Avoid calling end() β€” pool is already ended or ending.' - ); - return; - } - pool.end(); - } catch (e) { - process.stderr.write(String(e)); - } -}; +type PgPoolKey = string; export interface GraphileCache { pgPool: pg.Pool; @@ -30,62 +17,145 @@ export interface GraphileCache { handler: HttpRequestHandler; } +// +// --- Service Cache --- +// +export const svcCache = new LRUCache({ + max: 25, + ttl: ONE_YEAR, + updateAgeOnGet: true, + dispose: (svc, key) => { + console.log(`πŸ—‘οΈ Disposing service[${key}]`); + } +}); + +// // --- Graphile Cache --- +// export const graphileCache = new LRUCache({ max: 15, - dispose: (obj: GraphileCache, key: string) => { - console.log(`disposing PostGraphile[${key}]`); - }, - updateAgeOnGet: true, ttl: ONE_YEAR, + updateAgeOnGet: true, + dispose: (obj, key) => { + console.log(`πŸ—‘οΈ Disposing PostGraphile[${key}]`); + } }); -// --- Postgres Pool Cache --- -export const pgCache = new LRUCache({ - max: 10, - dispose: (pgPool: pg.Pool, key: string) => { - console.log(`disposing pg ${key}`); - graphileCache.forEach((obj: GraphileCache, k: string) => { - if (obj.pgPoolKey === key) { - graphileCache.delete(k); +// +// --- PgPoolCacheManager --- +// +export class PgPoolCacheManager { + private cleanupTasks: Promise[] = []; + + private readonly pgCache = new LRUCache({ + max: 10, + ttl: ONE_YEAR, + updateAgeOnGet: true, + dispose: (pool, key, reason) => { + console.log(`🧹 Disposing pg pool [${key}] (${reason})`); + this.cleanGraphileDependencies(key); + this.deferDispose(pool, key); + }, + }); + + constructor(private readonly graphileCache: LRUCache) {} + + get(key: PgPoolKey): pg.Pool | undefined { + return this.pgCache.get(key); + } + + has(key: PgPoolKey): pg.Pool | boolean { + return this.pgCache.has(key); + } + + set(key: PgPoolKey, pool: pg.Pool): void { + this.pgCache.set(key, pool); + } + + delete(key: PgPoolKey): void { + const pool = this.pgCache.get(key); + const existed = this.pgCache.delete(key); + if (!existed && pool) { + this.cleanGraphileDependencies(key); + this.deferDispose(pool, key); + } + } + + clear(): void { + const entries = [...this.pgCache.entries()]; + this.pgCache.clear(); + for (const [key, pool] of entries) { + this.cleanGraphileDependencies(key); + this.deferDispose(pool, key); + } + } + + async waitForDisposals(): Promise { + await Promise.allSettled(this.cleanupTasks); + } + + private cleanGraphileDependencies(pgPoolKey: string): void { + this.graphileCache.forEach((entry, k) => { + if (entry.pgPoolKey === pgPoolKey) { + console.log(`🧽 Removing graphileCache[${k}] due to pgPool[${pgPoolKey}]`); + this.graphileCache.delete(k); } }); - end(pgPool); - }, - updateAgeOnGet: true, - ttl: ONE_YEAR, -}); + } -// --- Generic Service Cache --- -export const svcCache = new LRUCache({ - max: 25, - dispose: (svc: any, key: any) => { - console.log(`disposing service[${key}]`); - }, - updateAgeOnGet: true, - ttl: ONE_YEAR, -}); + private deferDispose(pool: pg.Pool, key: string): void { + setImmediate(() => { + const task = (async () => { + try { + if (!pool.ended && !pool.ending) { + await pool.end(); + console.log(`βœ… pg.Pool ${key} ended.`); + } + } catch (err) { + console.error(`❌ Error ending pg.Pool ${key}:`, err); + } + })(); + this.cleanupTasks.push(task); + }); + } +} + +// +// --- Instantiate the pgCache manager --- +// +export const pgCache = new PgPoolCacheManager(graphileCache); +// // --- Graceful Shutdown --- -const once = any>(fn: T, context?: any) => { +// +const once = any>(fn: T): T => { + let called = false; let result: ReturnType; - return function (...args: Parameters) { - if (fn) { - // @ts-ignore - result = fn.apply(context || this, args); - fn = null!; + return ((...args: Parameters) => { + if (!called) { + called = true; + result = fn(...args); } return result; - }; + }) as T; }; -const close = once(() => { - console.log('closing server utils...'); +const close = once(async () => { + console.log('πŸ›‘ Closing all server caches...'); + svcCache.clear(); graphileCache.clear(); pgCache.clear(); - svcCache.clear(); + await pgCache.waitForDisposals(); + console.log('βœ… All caches disposed.'); }); SYS_EVENTS.forEach((event) => { - process.on(event, close); + process.on(event, () => { + console.log(`πŸ“¦ Received ${event}`); + close(); + }); }); + +export const teardownPgPools = async () => { + await close(); +} \ No newline at end of file diff --git a/packages/types/src/env.ts b/packages/types/src/env.ts index fb9475b4a6..ba58e15091 100644 --- a/packages/types/src/env.ts +++ b/packages/types/src/env.ts @@ -2,101 +2,122 @@ import deepmerge from 'deepmerge'; import { getMergedOptions, launchqlDefaults, LaunchQLOptions, PgConfig } from './launchql'; const parseEnvNumber = (val?: string): number | undefined => { - const num = Number(val); - return !isNaN(num) ? num : undefined; + const num = Number(val); + return !isNaN(num) ? num : undefined; }; const parseEnvBoolean = (val?: string): boolean | undefined => { - if (val === undefined) return undefined; - return ['true', '1', 'yes'].includes(val.toLowerCase()); + if (val === undefined) return undefined; + return ['true', '1', 'yes'].includes(val.toLowerCase()); }; export const getEnvOptions = (overrides: LaunchQLOptions = {}): LaunchQLOptions => { - const envOpts = getEnvVars(); - return getMergedOptions - ({ - ...envOpts, - ...overrides, - }); + const envOpts = getEnvVars(); + return getMergedOptions + ({ + ...envOpts, + ...overrides, + }); }; export const getPgEnvOptions = (overrides: Partial = {}): PgConfig => { - const envOpts = getPgEnvVars(); - const defaults = deepmerge(launchqlDefaults.pg, envOpts); - const options = deepmerge(defaults, overrides); - // if you need to sanitize... - return options; + const envOpts = getPgEnvVars(); + const defaults = deepmerge(launchqlDefaults.pg, envOpts); + const options = deepmerge(defaults, overrides); + // if you need to sanitize... + return options; }; const getEnvVars = (): LaunchQLOptions => { - const { - PORT, - SERVER_HOST, - SERVER_TRUST_PROXY, - SERVER_ORIGIN, - SERVER_STRICT_AUTH, + const { + PORT, + SERVER_HOST, + SERVER_TRUST_PROXY, + SERVER_ORIGIN, + SERVER_STRICT_AUTH, - PGHOST, - PGPORT, - PGUSER, - PGPASSWORD, - PGDATABASE, + PGHOST, + PGPORT, + PGUSER, + PGPASSWORD, + PGDATABASE, - FEATURES_SIMPLE_INFLECTION, - FEATURES_OPPOSITE_BASE_NAMES, - FEATURES_POSTGIS, + FEATURES_SIMPLE_INFLECTION, + FEATURES_OPPOSITE_BASE_NAMES, + FEATURES_POSTGIS, - BUCKET_NAME, - AWS_REGION, - AWS_ACCESS_KEY, - AWS_SECRET_KEY, - MINIO_ENDPOINT, - } = process.env; + BUCKET_NAME, + AWS_REGION, + AWS_ACCESS_KEY, + AWS_SECRET_KEY, + MINIO_ENDPOINT, + } = process.env; - return { - server: { - ...(PORT && { port: parseEnvNumber(PORT) }), - ...(SERVER_HOST && { host: SERVER_HOST }), - ...(SERVER_TRUST_PROXY && { trustProxy: parseEnvBoolean(SERVER_TRUST_PROXY) }), - ...(SERVER_ORIGIN && { origin: SERVER_ORIGIN }), - ...(SERVER_STRICT_AUTH && { strictAuth: parseEnvBoolean(SERVER_STRICT_AUTH) }), - }, - pg: { - ...(PGHOST && { host: PGHOST }), - ...(PGPORT && { port: parseEnvNumber(PGPORT) }), - ...(PGUSER && { user: PGUSER }), - ...(PGPASSWORD && { password: PGPASSWORD }), - ...(PGDATABASE && { database: PGDATABASE }), - }, - features: { - ...(FEATURES_SIMPLE_INFLECTION && { simpleInflection: parseEnvBoolean(FEATURES_SIMPLE_INFLECTION) }), - ...(FEATURES_OPPOSITE_BASE_NAMES && { oppositeBaseNames: parseEnvBoolean(FEATURES_OPPOSITE_BASE_NAMES) }), - ...(FEATURES_POSTGIS && { postgis: parseEnvBoolean(FEATURES_POSTGIS) }), - }, - cdn: { - ...(BUCKET_NAME && { bucketName: BUCKET_NAME }), - ...(AWS_REGION && { awsRegion: AWS_REGION }), - ...(AWS_ACCESS_KEY && { awsAccessKey: AWS_ACCESS_KEY }), - ...(AWS_SECRET_KEY && { awsSecretKey: AWS_SECRET_KEY }), - ...(MINIO_ENDPOINT && { minioEndpoint: MINIO_ENDPOINT }), - } - }; + return { + server: { + ...(PORT && { port: parseEnvNumber(PORT) }), + ...(SERVER_HOST && { host: SERVER_HOST }), + ...(SERVER_TRUST_PROXY && { trustProxy: parseEnvBoolean(SERVER_TRUST_PROXY) }), + ...(SERVER_ORIGIN && { origin: SERVER_ORIGIN }), + ...(SERVER_STRICT_AUTH && { strictAuth: parseEnvBoolean(SERVER_STRICT_AUTH) }), + }, + pg: { + ...(PGHOST && { host: PGHOST }), + ...(PGPORT && { port: parseEnvNumber(PGPORT) }), + ...(PGUSER && { user: PGUSER }), + ...(PGPASSWORD && { password: PGPASSWORD }), + ...(PGDATABASE && { database: PGDATABASE }), + }, + features: { + ...(FEATURES_SIMPLE_INFLECTION && { simpleInflection: parseEnvBoolean(FEATURES_SIMPLE_INFLECTION) }), + ...(FEATURES_OPPOSITE_BASE_NAMES && { oppositeBaseNames: parseEnvBoolean(FEATURES_OPPOSITE_BASE_NAMES) }), + ...(FEATURES_POSTGIS && { postgis: parseEnvBoolean(FEATURES_POSTGIS) }), + }, + cdn: { + ...(BUCKET_NAME && { bucketName: BUCKET_NAME }), + ...(AWS_REGION && { awsRegion: AWS_REGION }), + ...(AWS_ACCESS_KEY && { awsAccessKey: AWS_ACCESS_KEY }), + ...(AWS_SECRET_KEY && { awsSecretKey: AWS_SECRET_KEY }), + ...(MINIO_ENDPOINT && { minioEndpoint: MINIO_ENDPOINT }), + } + }; }; const getPgEnvVars = (): PgConfig => { - const { - PGHOST, - PGPORT, - PGUSER, - PGPASSWORD, - PGDATABASE - } = process.env; + const { + PGHOST, + PGPORT, + PGUSER, + PGPASSWORD, + PGDATABASE + } = process.env; - return { - ...(PGHOST && { host: PGHOST }), - ...(PGPORT && { port: parseEnvNumber(PGPORT) }), - ...(PGUSER && { user: PGUSER }), - ...(PGPASSWORD && { password: PGPASSWORD }), - ...(PGDATABASE && { database: PGDATABASE }), - }; + return { + ...(PGHOST && { host: PGHOST }), + ...(PGPORT && { port: parseEnvNumber(PGPORT) }), + ...(PGUSER && { user: PGUSER }), + ...(PGPASSWORD && { password: PGPASSWORD }), + ...(PGDATABASE && { database: PGDATABASE }), + }; }; + +export function toPgEnvVars(config: Partial): Record { + const opts = deepmerge(launchqlDefaults.pg, config); + return { + ...(opts.host && { PGHOST: opts.host }), + ...(opts.port && { PGPORT: String(opts.port) }), + ...(opts.user && { PGUSER: opts.user }), + ...(opts.password && { PGPASSWORD: opts.password }), + ...(opts.database && { PGDATABASE: opts.database }), + }; +}; + +export function getSpawnEnvWithPg( + config: Partial, + baseEnv: NodeJS.ProcessEnv = process.env +): NodeJS.ProcessEnv { + return { + ...baseEnv, + ...toPgEnvVars(config) + }; +} \ No newline at end of file From 236a7d0d68710a6370b63b692ae6b66a5517b968 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sat, 17 May 2025 17:06:37 -0700 Subject: [PATCH 25/29] grantUser and teardownPgPools verbose --- packages/cli/src/commands.ts | 3 +++ packages/pgsql-test/src/connect.ts | 16 +++++++++------- packages/pgsql-test/src/stream.ts | 1 - packages/server-utils/src/lru.ts | 10 +++++----- 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/packages/cli/src/commands.ts b/packages/cli/src/commands.ts index 7c13c54abf..556f8477c8 100644 --- a/packages/cli/src/commands.ts +++ b/packages/cli/src/commands.ts @@ -16,6 +16,7 @@ import plan from './commands/plan'; import _export from './commands/export'; import _package from './commands/package'; import kill from './commands/kill'; +import { teardownPgPools } from '@launchql/server-utils'; // Command map const commandMap: Record = { @@ -82,5 +83,7 @@ export const commands = async (argv: Partial, prompter: Inquirerer, await commandFn(newArgv, prompter, options); prompter.close(); + + await teardownPgPools(); return argv; }; diff --git a/packages/pgsql-test/src/connect.ts b/packages/pgsql-test/src/connect.ts index 6605812ac4..c023e5c49d 100644 --- a/packages/pgsql-test/src/connect.ts +++ b/packages/pgsql-test/src/connect.ts @@ -46,9 +46,17 @@ export const getConnections = async ( const admin = new DbAdmin(config); + const grantUser = () => { + // Set up test role + admin.createUserRole(connOpts.connection.user, connOpts.connection.password, config.database); + admin.grantConnect(connOpts.connection.user, config.database); + + }; + const proj = new LaunchQLProject(connOpts.cwd); if (proj.isInModule()) { admin.create(config.database); + grantUser(); admin.installExtensions(connOpts.extensions); const opts = getEnvOptions({ pg: config @@ -74,21 +82,15 @@ export const getConnections = async ( admin.createFromTemplate(connOpts.template, config.database); } else { admin.create(config.database); + grantUser(); admin.installExtensions(connOpts.extensions); } } - - // Main admin client (optional unless needed elsewhere) manager = PgTestConnector.getInstance(); const pg = manager.getClient(config); - - // Set up test role - admin.createUserRole(connOpts.connection.user, connOpts.connection.password, config.database); - admin.grantConnect(connOpts.connection.user, config.database); - // App user connection const db = manager.getClient({ ...config, diff --git a/packages/pgsql-test/src/stream.ts b/packages/pgsql-test/src/stream.ts index be9617790d..ef29dd0cc8 100644 --- a/packages/pgsql-test/src/stream.ts +++ b/packages/pgsql-test/src/stream.ts @@ -32,7 +32,6 @@ export async function streamSql(config: PgConfig, sql: string): Promise { return new Promise((resolve, reject) => { const sqlStream = stringToStream(sql); - // TODO set env vars! const proc = spawn('psql', args, { env: getSpawnEnvWithPg(config) }); diff --git a/packages/server-utils/src/lru.ts b/packages/server-utils/src/lru.ts index afe34b9958..50fe46508b 100644 --- a/packages/server-utils/src/lru.ts +++ b/packages/server-utils/src/lru.ts @@ -140,13 +140,13 @@ const once = any>(fn: T): T => { }) as T; }; -const close = once(async () => { - console.log('πŸ›‘ Closing all server caches...'); +const close = once(async (verbose: boolean = false) => { + if (verbose) console.log('πŸ›‘ Closing all server caches...'); svcCache.clear(); graphileCache.clear(); pgCache.clear(); await pgCache.waitForDisposals(); - console.log('βœ… All caches disposed.'); + if (verbose) console.log('βœ… All caches disposed.'); }); SYS_EVENTS.forEach((event) => { @@ -156,6 +156,6 @@ SYS_EVENTS.forEach((event) => { }); }); -export const teardownPgPools = async () => { - await close(); +export const teardownPgPools = async (verbose: boolean = false) => { + await close(verbose); } \ No newline at end of file From 6db19154349b65e66846c6c57d9ac4cefe3f3a5a Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sat, 17 May 2025 17:29:28 -0700 Subject: [PATCH 26/29] cleanup db connections --- .../postgres-test.deploy-fast.test.ts | 6 - packages/server-utils/src/lru.ts | 134 +++++++++++------- 2 files changed, 86 insertions(+), 54 deletions(-) diff --git a/packages/pgsql-test/__tests__/postgres-test.deploy-fast.test.ts b/packages/pgsql-test/__tests__/postgres-test.deploy-fast.test.ts index 92a4377098..234c7fb649 100644 --- a/packages/pgsql-test/__tests__/postgres-test.deploy-fast.test.ts +++ b/packages/pgsql-test/__tests__/postgres-test.deploy-fast.test.ts @@ -3,7 +3,6 @@ import path, { resolve } from 'path'; import { PgTestClient } from '../src/test-client'; import { DbAdmin } from '../src/admin'; import { getConnections } from '../src/connect'; -import { getRootPgPool } from '@launchql/server-utils'; const sql = (file: string) => path.resolve(__dirname, '../sql', file); @@ -44,11 +43,6 @@ afterEach(async () => { }); afterAll(async () => { - - // clear this DB first! so teardown() doesn't choke...a - const pgPool = getRootPgPool({ ...pg.config, database: pg.config.database }); - await pgPool.end(); - await teardown(); const avg = diff --git a/packages/server-utils/src/lru.ts b/packages/server-utils/src/lru.ts index 50fe46508b..285fa14707 100644 --- a/packages/server-utils/src/lru.ts +++ b/packages/server-utils/src/lru.ts @@ -17,6 +17,38 @@ export interface GraphileCache { handler: HttpRequestHandler; } +// Wrapper for pg.Pool to track disposal state +class ManagedPgPool { + public isDisposed = false; + private disposePromise: Promise | null = null; + + constructor(public readonly pool: pg.Pool, public readonly key: string) {} + + async dispose(): Promise { + if (this.isDisposed) { + return this.disposePromise; + } + + this.isDisposed = true; + this.disposePromise = (async () => { + try { + if (!this.pool.ended) { + await this.pool.end(); + console.log(`βœ… pg.Pool ${this.key} ended.`); + } else { + console.log(`β˜‘οΈ pg.Pool ${this.key} ALREADY ended.`); + } + } catch (err) { + console.error(`❌ Error ending pg.Pool ${this.key}:`, err); + // Re-throw to ensure Promise.allSettled captures the error + throw err; + } + })(); + + return this.disposePromise; + } +} + // // --- Service Cache --- // @@ -46,52 +78,71 @@ export const graphileCache = new LRUCache({ // export class PgPoolCacheManager { private cleanupTasks: Promise[] = []; + private closed = false; - private readonly pgCache = new LRUCache({ + private readonly pgCache = new LRUCache({ max: 10, ttl: ONE_YEAR, updateAgeOnGet: true, - dispose: (pool, key, reason) => { + dispose: (managedPool, key, reason) => { console.log(`🧹 Disposing pg pool [${key}] (${reason})`); this.cleanGraphileDependencies(key); - this.deferDispose(pool, key); + this.disposePool(managedPool); }, }); constructor(private readonly graphileCache: LRUCache) {} get(key: PgPoolKey): pg.Pool | undefined { - return this.pgCache.get(key); + const managedPool = this.pgCache.get(key); + return managedPool?.pool; } - has(key: PgPoolKey): pg.Pool | boolean { + has(key: PgPoolKey): boolean { return this.pgCache.has(key); } set(key: PgPoolKey, pool: pg.Pool): void { - this.pgCache.set(key, pool); + if (this.closed) { + throw new Error('Cannot add to cache after it has been closed'); + } + this.pgCache.set(key, new ManagedPgPool(pool, key)); } delete(key: PgPoolKey): void { - const pool = this.pgCache.get(key); + const managedPool = this.pgCache.get(key); const existed = this.pgCache.delete(key); - if (!existed && pool) { + if (!existed && managedPool) { this.cleanGraphileDependencies(key); - this.deferDispose(pool, key); + this.disposePool(managedPool); } } clear(): void { const entries = [...this.pgCache.entries()]; this.pgCache.clear(); - for (const [key, pool] of entries) { + for (const [key, managedPool] of entries) { this.cleanGraphileDependencies(key); - this.deferDispose(pool, key); + this.disposePool(managedPool); } } + async close(): Promise { + if (this.closed) return; + + this.closed = true; + this.clear(); + await this.waitForDisposals(); + } + async waitForDisposals(): Promise { - await Promise.allSettled(this.cleanupTasks); + if (this.cleanupTasks.length === 0) return; + + const tasks = [...this.cleanupTasks]; + // Clear the array before awaiting to avoid potential race conditions + this.cleanupTasks = []; + + await Promise.allSettled(tasks); } private cleanGraphileDependencies(pgPoolKey: string): void { @@ -103,20 +154,11 @@ export class PgPoolCacheManager { }); } - private deferDispose(pool: pg.Pool, key: string): void { - setImmediate(() => { - const task = (async () => { - try { - if (!pool.ended && !pool.ending) { - await pool.end(); - console.log(`βœ… pg.Pool ${key} ended.`); - } - } catch (err) { - console.error(`❌ Error ending pg.Pool ${key}:`, err); - } - })(); - this.cleanupTasks.push(task); - }); + private disposePool(managedPool: ManagedPgPool): void { + if (managedPool.isDisposed) return; + + const task = managedPool.dispose(); + this.cleanupTasks.push(task); } } @@ -128,34 +170,30 @@ export const pgCache = new PgPoolCacheManager(graphileCache); // // --- Graceful Shutdown --- // -const once = any>(fn: T): T => { - let called = false; - let result: ReturnType; - return ((...args: Parameters) => { - if (!called) { - called = true; - result = fn(...args); - } - return result; - }) as T; +const closePromise: { promise: Promise | null } = { promise: null }; + +export const close = async (verbose: boolean = false): Promise => { + if (closePromise.promise) return closePromise.promise; + + closePromise.promise = (async () => { + if (verbose) console.log('πŸ›‘ Closing all server caches...'); + svcCache.clear(); + graphileCache.clear(); + await pgCache.close(); + if (verbose) console.log('βœ… All caches disposed.'); + })(); + + return closePromise.promise; }; -const close = once(async (verbose: boolean = false) => { - if (verbose) console.log('πŸ›‘ Closing all server caches...'); - svcCache.clear(); - graphileCache.clear(); - pgCache.clear(); - await pgCache.waitForDisposals(); - if (verbose) console.log('βœ… All caches disposed.'); -}); - SYS_EVENTS.forEach((event) => { process.on(event, () => { console.log(`πŸ“¦ Received ${event}`); + // Don't await - we want to start the shutdown process immediately close(); }); }); -export const teardownPgPools = async (verbose: boolean = false) => { - await close(verbose); -} \ No newline at end of file +export const teardownPgPools = async (verbose: boolean = false): Promise => { + return close(verbose); +}; \ No newline at end of file From 618eb95b2365ae80bfb8abdf45373abffeaf093e Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sat, 17 May 2025 18:14:38 -0700 Subject: [PATCH 27/29] rootDb connections --- .../__tests__/postgres-test.records.test.ts | 2 ++ packages/pgsql-test/src/admin.ts | 8 ----- packages/pgsql-test/src/connect.ts | 32 +++++++++++-------- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/packages/pgsql-test/__tests__/postgres-test.records.test.ts b/packages/pgsql-test/__tests__/postgres-test.records.test.ts index 83d23c01f9..c3830fded2 100644 --- a/packages/pgsql-test/__tests__/postgres-test.records.test.ts +++ b/packages/pgsql-test/__tests__/postgres-test.records.test.ts @@ -1,3 +1,5 @@ +import { getPgEnvOptions } from '@launchql/types'; +import { DbAdmin } from '../src/admin'; import { getConnections } from '../src/connect'; import { PgTestClient } from '../src/test-client'; diff --git a/packages/pgsql-test/src/admin.ts b/packages/pgsql-test/src/admin.ts index 47e2ba0d0b..f593297113 100644 --- a/packages/pgsql-test/src/admin.ts +++ b/packages/pgsql-test/src/admin.ts @@ -2,7 +2,6 @@ import { execSync } from 'child_process'; import { getPgEnvOptions, PgConfig } from '@launchql/types'; import { existsSync } from 'fs'; import { streamSql as stream } from './stream'; -import { randomUUID } from 'crypto'; export interface SeedAdapter { seed(db: DbAdmin, dbName: string): Promise | void; @@ -36,7 +35,6 @@ export function compositeSeedAdapter(adapters: SeedAdapter[]): SeedAdapter { }; } - export class DbAdmin { constructor( private config: PgConfig, @@ -120,12 +118,6 @@ export class DbAdmin { } catch { } this.safeDropDb(template); } - - async createRole(role: string, password: string, dbName?: string): Promise { - const db = dbName ?? this.config.database; - const sql = `CREATE ROLE ${role} WITH LOGIN PASSWORD '${password}';`; - await this.streamSql(sql, db); - } async grantRole(role: string, user: string, dbName?: string): Promise { const db = dbName ?? this.config.database; diff --git a/packages/pgsql-test/src/connect.ts b/packages/pgsql-test/src/connect.ts index c023e5c49d..b8e84f2075 100644 --- a/packages/pgsql-test/src/connect.ts +++ b/packages/pgsql-test/src/connect.ts @@ -9,6 +9,7 @@ import { teardownPgPools } from '@launchql/server-utils'; let manager: PgTestConnector; export interface TestConnectionOptions { + rootDb?: string; template?: string; prefix?: string; extensions?: string[]; @@ -22,6 +23,7 @@ export interface TestConnectionOptions { } const defaultTestConnOpts: Partial = { + rootDb: process.env.PGROOTDATABASE || 'postgres', prefix: 'db-', extensions: [], cwd: process.cwd(), @@ -31,32 +33,37 @@ const defaultTestConnOpts: Partial = { password: 'app_password', role: 'anonymous' } +} +const getRootAdmin = (config: PgConfig, connOpts: TestConnectionOptions) => { + const opts = getPgEnvOptions(); + opts.database = connOpts.rootDb; + const admin = new DbAdmin(opts); + return admin; } export const getConnections = async ( _pgConfig: Partial = {}, _opts: TestConnectionOptions = {} ) => { + const connOpts = deepmerge(defaultTestConnOpts, _opts); const config: PgConfig = getPgEnvOptions({ database: `${connOpts.prefix}${randomUUID()}`, ..._pgConfig }); - const admin = new DbAdmin(config); - - const grantUser = () => { - // Set up test role - admin.createUserRole(connOpts.connection.user, connOpts.connection.password, config.database); - admin.grantConnect(connOpts.connection.user, config.database); - - }; + const root = getRootAdmin(config, connOpts); + await root.createUserRole( + connOpts.connection.user, + connOpts.connection.password, + connOpts.rootDb + ); + const admin = new DbAdmin(config); const proj = new LaunchQLProject(connOpts.cwd); if (proj.isInModule()) { admin.create(config.database); - grantUser(); admin.installExtensions(connOpts.extensions); const opts = getEnvOptions({ pg: config @@ -74,7 +81,6 @@ export const getConnections = async ( await deploy(opts, proj.getModuleName(), config.database, proj.modulePath); } } else { - // Create the test database if (process.env.TEST_DB) { config.database = process.env.TEST_DB; @@ -82,12 +88,13 @@ export const getConnections = async ( admin.createFromTemplate(connOpts.template, config.database); } else { admin.create(config.database); - grantUser(); admin.installExtensions(connOpts.extensions); } } + await admin.grantConnect(connOpts.connection.user, config.database); + // Main admin client (optional unless needed elsewhere) manager = PgTestConnector.getInstance(); const pg = manager.getClient(config); @@ -96,8 +103,7 @@ export const getConnections = async ( ...config, user: connOpts.connection.user, password: connOpts.connection.password - }) - // const conn = await getTestConnection(config.database, app_user, app_password); + }); db.setContext({ role: 'anonymous' }); const teardown = async () => { From fb805f3dbbda363051560bf73b88901bd6994136 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sat, 17 May 2025 19:26:26 -0700 Subject: [PATCH 28/29] cleanup --- .github/workflows/run-tests.yaml | 11 +++++ .../__tests__/postgres-test.test.ts | 36 -------------- packages/pgsql-test/src/connect.ts | 48 ++++++------------- packages/pgsql-test/src/manager.ts | 13 ++--- packages/pgsql-test/src/utils.ts | 48 ------------------- packages/types/src/env.ts | 23 ++++++--- packages/types/src/launchql.ts | 35 +++++++++++--- 7 files changed, 77 insertions(+), 137 deletions(-) delete mode 100644 packages/pgsql-test/__tests__/postgres-test.test.ts delete mode 100644 packages/pgsql-test/src/utils.ts diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 0e7e456513..f635f6a178 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -52,6 +52,15 @@ jobs: - name: build run: yarn build + - name: seed app_user + run: | + psql -f ./bootstrap-roles.sql postgres + env: + PGHOST: pg_db + PGPORT: 5432 + PGUSER: postgres + PGPASSWORD: password + - name: launchql/migrate run: cd ./packages/migrate && yarn test @@ -72,6 +81,7 @@ jobs: yarn test env: PGHOST: pg_db + PGPORT: 5432 PGUSER: postgres PGPASSWORD: password @@ -81,6 +91,7 @@ jobs: yarn test env: PGHOST: pg_db + PGPORT: 5432 PGUSER: postgres PGPASSWORD: password diff --git a/packages/pgsql-test/__tests__/postgres-test.test.ts b/packages/pgsql-test/__tests__/postgres-test.test.ts deleted file mode 100644 index d138dcb756..0000000000 --- a/packages/pgsql-test/__tests__/postgres-test.test.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { getPgEnvOptions, PgConfig } from '@launchql/types'; - -import { - getConnection, - Connection -} from '../src'; -import { randomUUID } from 'crypto'; -import { PgTestClient } from '../src/test-client'; - -const TEST_DB_BASE = `postgres_test_${randomUUID()}`; - -const config = getPgEnvOptions({ - database: TEST_DB_BASE -}); - -afterAll(() => { - Connection.getManager().closeAll(); -}); - -describe('Postgres Test Framework', () => { - let db: PgTestClient; - - afterEach(() => { - // if (db) closeConnection(db); - }); - - it('creates a test DB with hot mode (FAST_TEST)', () => { - db = getConnection({ hot: true, extensions: ['uuid-ossp'] }); - expect(db).toBeDefined(); - }); - - it('creates a test DB from scratch (default)', () => { - db = getConnection({}); - expect(db).toBeDefined(); - }); -}); diff --git a/packages/pgsql-test/src/connect.ts b/packages/pgsql-test/src/connect.ts index b8e84f2075..74771ea328 100644 --- a/packages/pgsql-test/src/connect.ts +++ b/packages/pgsql-test/src/connect.ts @@ -1,43 +1,23 @@ import { DbAdmin } from './admin'; -import { getEnvOptions, getPgEnvOptions, PgConfig } from '@launchql/types'; +import { + getEnvOptions, + getPgEnvOptions, + TestConnectionOptions, + PgConfig, + getConnEnvOptions +} from '@launchql/types'; import { deploy, deployFast, LaunchQLProject } from '@launchql/migrate'; import { PgTestConnector } from './manager'; import { randomUUID } from 'crypto'; -import deepmerge from 'deepmerge'; + import { teardownPgPools } from '@launchql/server-utils'; let manager: PgTestConnector; -export interface TestConnectionOptions { - rootDb?: string; - template?: string; - prefix?: string; - extensions?: string[]; - cwd?: string; - deployFast?: boolean; - connection?: { - user?: string; - password?: string; - role?: string; - } -} - -const defaultTestConnOpts: Partial = { - rootDb: process.env.PGROOTDATABASE || 'postgres', - prefix: 'db-', - extensions: [], - cwd: process.cwd(), - deployFast: true, - connection: { - user: 'app_user', - password: 'app_password', - role: 'anonymous' - } -} - -const getRootAdmin = (config: PgConfig, connOpts: TestConnectionOptions) => { - const opts = getPgEnvOptions(); - opts.database = connOpts.rootDb; +export const getPgRootAdmin = (connOpts: TestConnectionOptions={}) => { + const opts = getPgEnvOptions({ + database: connOpts.rootDb + }); const admin = new DbAdmin(opts); return admin; } @@ -47,13 +27,13 @@ export const getConnections = async ( _opts: TestConnectionOptions = {} ) => { - const connOpts = deepmerge(defaultTestConnOpts, _opts); + const connOpts = getConnEnvOptions(_opts); const config: PgConfig = getPgEnvOptions({ database: `${connOpts.prefix}${randomUUID()}`, ..._pgConfig }); - const root = getRootAdmin(config, connOpts); + const root = getPgRootAdmin(connOpts); await root.createUserRole( connOpts.connection.user, connOpts.connection.password, diff --git a/packages/pgsql-test/src/manager.ts b/packages/pgsql-test/src/manager.ts index 633b70faab..6ef9117045 100644 --- a/packages/pgsql-test/src/manager.ts +++ b/packages/pgsql-test/src/manager.ts @@ -1,7 +1,7 @@ import { Pool } from 'pg'; import chalk from 'chalk'; import { DbAdmin } from './admin'; -import { PgConfig } from '@launchql/types'; +import { getConnEnvOptions, getPgEnvOptions, PgConfig } from '@launchql/types'; import { PgTestClient } from './test-client'; const SYS_EVENTS = ['SIGTERM']; @@ -57,10 +57,6 @@ export class PgTestConnector { return `${config.host}:${config.port}/${config.database}`; } - getAdmin(config: PgConfig): DbAdmin { - return new DbAdmin(config, this.verbose); - } - getPool(config: PgConfig): Pool { const key = this.poolKey(config); if (!this.pgPools.has(key)) { @@ -108,7 +104,11 @@ export class PgTestConnector { Array.from(this.seenDbConfigs.values()).map(async (config) => { try { // somehow an "admin" db had app_user creds? - const admin = new DbAdmin({...config, user: 'postgres', password: 'password'}, this.verbose); + const rootPg = getPgEnvOptions(); + const admin = new DbAdmin( + {...config, user: rootPg.user, password: rootPg.password}, + this.verbose + ); // console.log(config); admin.drop(); this.log(chalk.yellow(`🧨 Dropped database: ${chalk.white(config.database)}`)); @@ -128,6 +128,7 @@ export class PgTestConnector { drop(config: PgConfig): void { const key = this.dbKey(config); + // for drop, no need for conn opts const admin = new DbAdmin(config, this.verbose); admin.drop(); this.log(chalk.red(`🧨 Dropped database: ${chalk.white(config.database)}`)); diff --git a/packages/pgsql-test/src/utils.ts b/packages/pgsql-test/src/utils.ts deleted file mode 100644 index 24b04243d9..0000000000 --- a/packages/pgsql-test/src/utils.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { Client, Pool } from 'pg'; -// import { createdb, dropdb, templatedb, installExt, grantConnect } from './db'; -import { connect, close } from './legacy-connect'; -import { getPgEnvOptions, PgConfig } from '@launchql/types'; -import { getEnvOptions } from '@launchql/types'; -import { randomUUID } from 'crypto'; -import { PgTestClient } from './test-client'; -import { DbAdmin } from './admin'; - -export interface TestOptions { - hot?: boolean; - template?: string; - prefix?: string; - extensions?: string[]; -} - -export function getOpts(configOpts: TestOptions = {}): TestOptions { - return { - template: configOpts.template, - prefix: configOpts.prefix || 'testing-db', - extensions: configOpts.extensions || [], - }; -} - -export function getConnection(configOpts: TestOptions, database?: string): PgTestClient { - const opts = getOpts(configOpts); - const dbName = database || `${opts.prefix}-${Date.now()}`; - - const config = getPgEnvOptions({ - database: dbName - }); - - const admin = new DbAdmin(config); - - if (process.env.TEST_DB) { - config.database = process.env.TEST_DB; - } else if (opts.hot) { - admin.create(config.database); - admin.installExtensions(opts.extensions); - } else if (opts.template) { - admin.createFromTemplate(opts.template, config.database); - } else { - admin.create(config.database); - admin.installExtensions(opts.extensions); - } - - return connect(config); -} \ No newline at end of file diff --git a/packages/types/src/env.ts b/packages/types/src/env.ts index ba58e15091..4d1995ef74 100644 --- a/packages/types/src/env.ts +++ b/packages/types/src/env.ts @@ -1,5 +1,5 @@ import deepmerge from 'deepmerge'; -import { getMergedOptions, launchqlDefaults, LaunchQLOptions, PgConfig } from './launchql'; +import { launchqlDefaults, LaunchQLOptions, PgConfig, TestConnectionOptions } from './launchql'; const parseEnvNumber = (val?: string): number | undefined => { const num = Number(val); @@ -13,11 +13,10 @@ const parseEnvBoolean = (val?: string): boolean | undefined => { export const getEnvOptions = (overrides: LaunchQLOptions = {}): LaunchQLOptions => { const envOpts = getEnvVars(); - return getMergedOptions - ({ - ...envOpts, - ...overrides, - }); + const defaults = deepmerge(launchqlDefaults, envOpts); + const options = deepmerge(defaults, overrides); + // if you need to sanitize... + return options; }; export const getPgEnvOptions = (overrides: Partial = {}): PgConfig => { @@ -28,8 +27,17 @@ export const getPgEnvOptions = (overrides: Partial = {}): PgConfig => return options; }; +export const getConnEnvOptions = (overrides: Partial = {}): TestConnectionOptions => { + const opts = getEnvOptions({ + db: overrides + }); + return opts.db; +}; + const getEnvVars = (): LaunchQLOptions => { const { + PGROOTDATABASE, + PORT, SERVER_HOST, SERVER_TRUST_PROXY, @@ -54,6 +62,9 @@ const getEnvVars = (): LaunchQLOptions => { } = process.env; return { + db: { + ...(PGROOTDATABASE && { rootDb: PGROOTDATABASE }), + }, server: { ...(PORT && { port: parseEnvNumber(PORT) }), ...(SERVER_HOST && { host: SERVER_HOST }), diff --git a/packages/types/src/launchql.ts b/packages/types/src/launchql.ts index ea1ba996d1..7ea27cb872 100644 --- a/packages/types/src/launchql.ts +++ b/packages/types/src/launchql.ts @@ -51,7 +51,22 @@ export interface PgConfig { database: string; } +export interface TestConnectionOptions { + rootDb?: string; + template?: string; + prefix?: string; + extensions?: string[]; + cwd?: string; + deployFast?: boolean; + connection?: { + user?: string; + password?: string; + role?: string; + } + } + export interface LaunchQLOptions { + db?: Partial; pg?: Partial; graphile?: { isPublic?: boolean; @@ -84,6 +99,18 @@ export interface LaunchQLOptions { export const launchqlDefaults: LaunchQLOptions = { + db: { + rootDb: 'postgres', + prefix: 'db-', + extensions: [], + cwd: process.cwd(), + deployFast: true, + connection: { + user: 'app_user', + password: 'app_password', + role: 'anonymous' + } + }, pg: { host: 'localhost', port: 5432, @@ -117,10 +144,4 @@ export const launchqlDefaults: LaunchQLOptions = { awsAccessKey: 'minioadmin', awsSecretKey: 'minioadmin' } -}; - -export const getMergedOptions = (options: LaunchQLOptions): LaunchQLOptions => { - options = deepmerge(launchqlDefaults, options ?? {}); - // if you need to sanitize... - return options; -}; +}; \ No newline at end of file From 94c7b42b4770ec4b5fd9520d77514e2a050c594d Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sat, 17 May 2025 19:39:26 -0700 Subject: [PATCH 29/29] env opts --- packages/explorer/src/server.ts | 4 ++-- packages/explorer/src/settings.ts | 4 ++-- packages/graphile-settings/src/index.ts | 4 ++-- packages/server/src/server.ts | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/explorer/src/server.ts b/packages/explorer/src/server.ts index 7bfdd56dd3..cd27ff07cd 100644 --- a/packages/explorer/src/server.ts +++ b/packages/explorer/src/server.ts @@ -13,10 +13,10 @@ import { import { printSchemas, printDatabases } from './render'; import { getGraphileSettings } from './settings'; import { getPgEnvOptions, LaunchQLOptions } from '@launchql/types'; -import { getMergedOptions } from '@launchql/types'; +import { getEnvOptions } from '@launchql/types'; export const LaunchQLExplorer = (rawOpts: LaunchQLOptions = {}): Express => { - const opts = getMergedOptions(rawOpts); + const opts = getEnvOptions(rawOpts); const { pg, diff --git a/packages/explorer/src/settings.ts b/packages/explorer/src/settings.ts index ce27bd7ad2..1f1a503818 100644 --- a/packages/explorer/src/settings.ts +++ b/packages/explorer/src/settings.ts @@ -1,10 +1,10 @@ import { PostGraphileOptions } from 'postgraphile'; import { getGraphileSettings as getSettings } from '@launchql/graphile-settings'; import { LaunchQLOptions } from '@launchql/types'; -import { getMergedOptions } from '@launchql/types'; +import { getEnvOptions } from '@launchql/types'; export const getGraphileSettings = (rawOpts: LaunchQLOptions): PostGraphileOptions => { - const opts = getMergedOptions(rawOpts); + const opts = getEnvOptions(rawOpts); const baseOptions = getSettings(opts); diff --git a/packages/graphile-settings/src/index.ts b/packages/graphile-settings/src/index.ts index f2b6b7c6d9..872bf3b97c 100644 --- a/packages/graphile-settings/src/index.ts +++ b/packages/graphile-settings/src/index.ts @@ -23,10 +23,10 @@ import LqlTypesPlugin from './plugins/types'; import { Uploader } from './resolvers/upload'; import { PostGraphileOptions } from 'postgraphile'; import { LaunchQLOptions } from '@launchql/types'; -import { getMergedOptions } from '@launchql/types'; +import { getEnvOptions } from '@launchql/types'; export const getGraphileSettings = (rawOpts: LaunchQLOptions): PostGraphileOptions => { - const opts = getMergedOptions(rawOpts); + const opts = getEnvOptions(rawOpts); const { server, diff --git a/packages/server/src/server.ts b/packages/server/src/server.ts index 20a938628a..f038eda0c0 100644 --- a/packages/server/src/server.ts +++ b/packages/server/src/server.ts @@ -16,10 +16,10 @@ import { flush, flushService } from './middleware/flush'; import requestIp from 'request-ip'; import { Pool, PoolClient } from 'pg'; -import { LaunchQLOptions, getMergedOptions } from '@launchql/types'; +import { LaunchQLOptions, getEnvOptions } from '@launchql/types'; export const LaunchQLServer = (rawOpts: LaunchQLOptions = {}) => { - const app = new Server(getMergedOptions(rawOpts)); + const app = new Server(getEnvOptions(rawOpts)); app.addEventListener(); app.listen(); };