diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 43531db298..9b1487c091 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -52,11 +52,7 @@ jobs: - name: launchql/client run: cd ./packages/client && yarn test env: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: password - POSTGRES_HOST: pg_db - POSTGRES_PORT: 5432 - DATABASE_URL: postgres://postgres:password@pg_db:5432/postgres + TEST_DATABASE_URL: postgres://postgres:password@pg_db:5432/postgres - name: launchql/orm run: cd ./packages/orm && yarn test @@ -84,15 +80,6 @@ jobs: AWS_SECRET_KEY: minioadmin AWS_REGION: us-east-1 BUCKET_NAME: test-bucket - - - name: launchql/stream-to-s3 - run: cd ./packages/stream-to-s3 && yarn test - env: - MINIO_ENDPOINT: http://minio_cdn:9000 - AWS_ACCESS_KEY: minioadmin - AWS_SECRET_KEY: minioadmin - AWS_REGION: us-east-1 - BUCKET_NAME: test-bucket - name: launchql/upload-names run: cd ./packages/upload-names && yarn test diff --git a/packages/cli/package.json b/packages/cli/package.json index 2212c379d8..d89d3717e8 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -40,6 +40,7 @@ "@launchql/migrate": "^2.0.0", "@launchql/explorer": "^2.0.0", "@launchql/server": "^2.0.0", + "@launchql/types": "^2.0.0", "chalk": "^4.1.0", "deepmerge": "^4.3.1", "inquirerer": "^1.9.1", diff --git a/packages/cli/src/commands/explorer.ts b/packages/cli/src/commands/explorer.ts index 91bff81e5f..cede6b0640 100644 --- a/packages/cli/src/commands/explorer.ts +++ b/packages/cli/src/commands/explorer.ts @@ -1,6 +1,7 @@ -import { LaunchQLExplorer as explorer, ExplorerOptions } from '@launchql/explorer'; +import { LaunchQLExplorer as explorer } from '@launchql/explorer'; import { CLIOptions, Inquirerer, Question } from 'inquirerer'; import chalk from 'chalk'; +import { getEnvOptions, LaunchQLOptions } from '@launchql/types'; const questions: Question[] = [ { @@ -36,7 +37,8 @@ const questions: Question[] = [ type: 'number', // alias: 'p', required: false, - default: 5555 + default: 5555, + useDefault: true }, { name: 'origin', @@ -64,13 +66,17 @@ export default async ( simpleInflection } = await prompter.prompt(argv, questions); - const options: ExplorerOptions = { - oppositeBaseNames, - origin, - port, - postgis, - simpleInflection - }; + const options: LaunchQLOptions = getEnvOptions({ + features: { + oppositeBaseNames, + simpleInflection, + postgis + }, + server: { + origin, + port + } + }); console.log(chalk.green('\nāœ… Selected Configuration:')); for (const [key, value] of Object.entries(options)) { diff --git a/packages/cli/src/commands/server.ts b/packages/cli/src/commands/server.ts index 1c8fba7008..5038fb4b7f 100644 --- a/packages/cli/src/commands/server.ts +++ b/packages/cli/src/commands/server.ts @@ -1,6 +1,7 @@ -import { LaunchQLServer as server, ServerOptions } from '@launchql/server'; +import { LaunchQLServer as server } from '@launchql/server'; import { CLIOptions, Inquirerer, Question } from 'inquirerer'; import chalk from 'chalk'; +import { getEnvOptions, LaunchQLOptions } from '@launchql/types'; const questions: Question[] = [ { @@ -36,7 +37,8 @@ const questions: Question[] = [ type: 'number', // alias: 'p', required: false, - default: 5555 + default: 5555, + useDefault: true }, // { // name: 'origin', @@ -63,12 +65,16 @@ export default async ( simpleInflection } = await prompter.prompt(argv, questions); - const options: ServerOptions = { - oppositeBaseNames, - port, - postgis, - simpleInflection - }; + const options: LaunchQLOptions = getEnvOptions({ + features: { + oppositeBaseNames, + simpleInflection, + postgis + }, + server: { + port + } + }); console.log(chalk.green('\nāœ… Selected Configuration:')); for (const [key, value] of Object.entries(options)) { diff --git a/packages/client/__tests__/client.test.ts b/packages/client/__tests__/client.test.ts index 5c86880a0d..bb1f3f1301 100644 --- a/packages/client/__tests__/client.test.ts +++ b/packages/client/__tests__/client.test.ts @@ -1,11 +1,12 @@ -process.env.DATABASE_URL = process.env.DATABASE_URL ?? 'postgres://postgres:password@localhost:5432/postgres'; import { PoolClient } from 'pg'; import { Database } from '../src'; let client: Database; +const databaseUrl = process.env.TEST_DATABASE_URL || 'postgres://postgres:password@localhost:5432/postgres'; + beforeAll(() => { - client = new Database(); + client = new Database(databaseUrl); }); afterAll(async () => { diff --git a/packages/client/package.json b/packages/client/package.json index e3640f24fa..5c3beb7e43 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -32,7 +32,6 @@ "keywords": [], "dependencies": { "@types/pg": "^8.11.10", - "envalid": "^8.0.0", "pg": "^8.13.1" } } \ No newline at end of file diff --git a/packages/client/src/env.ts b/packages/client/src/env.ts deleted file mode 100644 index 8fcf762bb2..0000000000 --- a/packages/client/src/env.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { cleanEnv, url } from 'envalid'; - -const env = cleanEnv( - process.env, - { - DATABASE_URL: url(), - } -); - -export default env; diff --git a/packages/client/src/index.ts b/packages/client/src/index.ts index 69e4bb5552..5288f0c2b1 100644 --- a/packages/client/src/index.ts +++ b/packages/client/src/index.ts @@ -1,33 +1,18 @@ import { Pool, PoolClient } from 'pg'; -import env from './env'; - export class Database { - private static instance: Database; private pool: Pool; - constructor() { - if (Database.instance) { - return Database.instance; - } - - const pgPoolConfig = { - connectionString: env.DATABASE_URL, - }; - this.pool = new Pool(pgPoolConfig); + constructor(databaseUrl: string) { + this.pool = new Pool({ connectionString: databaseUrl }); - // Ensure the pool is closed on process termination process.on('SIGTERM', async () => { await this.shutdown(); }); - - Database.instance = this; - return this; } /** * Executes a callback function within a database transaction. - * @param fn - A callback function that receives a PoolClient to perform database operations. */ async withTransaction(fn: (client: PoolClient) => Promise): Promise { const client = await this.pool.connect(); @@ -39,7 +24,7 @@ export class Database { } catch (e) { console.error('Error during transaction:', e); await client.query('ROLLBACK'); - throw e; // Re-throw the error to propagate it + throw e; } } finally { client.release(); @@ -52,4 +37,4 @@ export class Database { async shutdown(): Promise { await this.pool.end(); } -} \ No newline at end of file +} diff --git a/packages/explorer/package.json b/packages/explorer/package.json index d00b0c71e3..38d1c3e3fd 100644 --- a/packages/explorer/package.json +++ b/packages/explorer/package.json @@ -39,7 +39,7 @@ "@launchql/server-utils": "^0.4.0", "@launchql/url-domains": "^0.1.0", "@launchql/upload-names": "^0.1.0", - "envalid": "^8.0.0", + "@launchql/types": "^2.0.0", "express": "^5.1.0", "graphile-build": "^4.14.1", "graphql-upload": "^17.0.0", diff --git a/packages/explorer/src/env.ts b/packages/explorer/src/env.ts deleted file mode 100644 index 10cfec8011..0000000000 --- a/packages/explorer/src/env.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { cleanEnv, str, port, bool, url } from 'envalid'; - -export const env = cleanEnv( - process.env, - { - SERVER_PORT: port({ default: 5757 }), - SERVER_HOST: str({ default: 'localhost' }), - PGUSER: str({ default: 'postgres' }), - PGHOST: str({ default: 'localhost' }), - PGPASSWORD: str({ default: 'password' }), - PGPORT: port({ default: 5432 }), - USE_SIMPLE_INFLECTION: bool({ default: true }), - USE_OPPOSITE_BASENAMES: bool({ default: false }), - USE_POSTGIS: bool({ default: true }), - AWS_REGION: str({ default: 'us-east-1' }), - AWS_SECRET_KEY: str({ default: 'minioadmin' }), - AWS_ACCESS_KEY: str({ default: 'minioadmin' }), - MINIO_ENDPOINT: url({ default: undefined }), - BUCKET_NAME: str({ default: 'test-bucket' }) - } -); diff --git a/packages/explorer/src/resolvers/uploads.ts b/packages/explorer/src/resolvers/uploads.ts index 43290cadfe..82063ebdca 100644 --- a/packages/explorer/src/resolvers/uploads.ts +++ b/packages/explorer/src/resolvers/uploads.ts @@ -1,36 +1,14 @@ import Streamer from '@launchql/s3-streamer'; import uploadNames from '@launchql/upload-names'; -import { env } from '../env'; import type { GraphQLResolveInfo } from 'graphql'; import { ReadStream } from 'fs'; -const { - BUCKET_NAME, - AWS_REGION, - AWS_SECRET_KEY, - AWS_ACCESS_KEY, - MINIO_ENDPOINT -} = env; - -function ensureVar(v: any, name: string): void { - if (v) return; - throw new Error(`REQUIRES env var: ${name}`); -} - -let streamer: Streamer; -function getUploader(): Streamer { - if (streamer) return streamer; - ensureVar(BUCKET_NAME, 'BUCKET_NAME'); - ensureVar(AWS_ACCESS_KEY, 'AWS_ACCESS_KEY'); - ensureVar(AWS_SECRET_KEY, 'AWS_SECRET_KEY'); - streamer = new Streamer({ - defaultBucket: BUCKET_NAME, - AWS_REGION, - AWS_SECRET_KEY, - AWS_ACCESS_KEY, - MINIO_ENDPOINT - }); - return streamer; +interface UploaderOptions { + bucketName: string; + awsRegion: string; + awsSecretKey: string; + awsAccessKey: string; + minioEndpoint?: string; } interface Upload { @@ -45,75 +23,73 @@ interface UploadPluginInfo { type: string; } -export default async function resolveUpload( - upload: Upload, - _args: any, - _context: any, - info: GraphQLResolveInfo & { uploadPlugin: UploadPluginInfo } -): Promise { - const { - uploadPlugin: { tags, type } - } = info; - console.log({ tags }); - console.log({ upload }); - - const streamer = getUploader(); - const readStream = upload.createReadStream() as unknown as ReadStream; - const { filename, mimetype, encoding } = upload; - - const rand = - Math.random().toString(36).substring(2, 7) + - Math.random().toString(36).substring(2, 7); - const key = rand + '-' + uploadNames(filename); - - const result = await streamer.upload({ - readStream, - filename, - key, - bucket: BUCKET_NAME - }); - - const url = result.upload.Location; - - console.log({ mimetype, vs: result }); - - const { - contentType, - magic: { charset } - } = result; - - const typ = type || tags.type; - - const mim = tags.mime - ? tags.mime - .trim() - .split(',') - .map((a: string) => a.trim()) - : typ === 'image' - ? ['image/jpg', 'image/jpeg', 'image/png', 'image/svg+xml'] - : []; - - let allowed = true; - if (mim && mim.length) { - allowed = mim.includes(contentType); +export class UploadHandler { + private streamer: Streamer; + + constructor(private options: UploaderOptions) { + this.streamer = new Streamer({ + defaultBucket: options.bucketName, + awsRegion: options.awsRegion, + awsSecretKey: options.awsSecretKey, + awsAccessKey: options.awsAccessKey, + minioEndpoint: options.minioEndpoint + }); } - if (!allowed) { - throw new Error(`UPLOAD_MIMETYPE ${mim.join(',')}`); - } - - console.log({ type }); - - switch (typ) { - case 'image': - case 'upload': - return { - filename, - mime: contentType, - url - }; - case 'attachment': - default: - return url; + async handleUpload( + upload: Upload, + _args: any, + _context: any, + info: GraphQLResolveInfo & { uploadPlugin: UploadPluginInfo } + ): Promise { + const { + uploadPlugin: { tags, type } + } = info; + + const readStream = upload.createReadStream() as ReadStream; + const { filename, mimetype } = upload; + + const rand = + Math.random().toString(36).substring(2, 7) + + Math.random().toString(36).substring(2, 7); + const key = rand + '-' + uploadNames(filename); + + const result = await this.streamer.upload({ + readStream, + filename, + key, + bucket: this.options.bucketName + }); + + const url = result.upload.Location; + const { + contentType, + magic: { charset } + } = result; + + const typ = type || tags.type; + + const mim = tags.mime + ? tags.mime.trim().split(',').map((a: string) => a.trim()) + : typ === 'image' + ? ['image/jpg', 'image/jpeg', 'image/png', 'image/svg+xml'] + : []; + + if (mim.length && !mim.includes(contentType)) { + throw new Error(`UPLOAD_MIMETYPE ${mim.join(',')}`); + } + + switch (typ) { + case 'image': + case 'upload': + return { + filename, + mime: contentType, + url + }; + case 'attachment': + default: + return url; + } } } diff --git a/packages/explorer/src/run.ts b/packages/explorer/src/run.ts index 082da1d678..850cae4a8b 100755 --- a/packages/explorer/src/run.ts +++ b/packages/explorer/src/run.ts @@ -1,5 +1,6 @@ #!/usr/bin/env node +import { getEnvOptions } from '@launchql/types'; import { LaunchQLExplorer as server } from './server'; -server(); \ No newline at end of file +server(getEnvOptions()); \ No newline at end of file diff --git a/packages/explorer/src/server.ts b/packages/explorer/src/server.ts index f5f4b2a9f9..ae8cfdb306 100644 --- a/packages/explorer/src/server.ts +++ b/packages/explorer/src/server.ts @@ -1,5 +1,5 @@ import express, { Request, Response, NextFunction, Express } from 'express'; -import { postgraphile, PostGraphileOptions } from 'postgraphile'; +import { postgraphile } from 'postgraphile'; import graphqlUploadExpress from 'graphql-upload/graphqlUploadExpress.mjs'; import { middleware as parseDomains } from '@launchql/url-domains'; import { @@ -10,26 +10,19 @@ import { poweredBy, GraphileCache } from '@launchql/server-utils'; - import { printSchemas, printDatabases } from './render'; -import { env } from './env'; import { getGraphileSettings } from './settings'; +import { LaunchQLOptions } from '@launchql/types'; +import { getMergedOptions } from '@launchql/types'; + +export const LaunchQLExplorer = (rawOpts: LaunchQLOptions = {}): Express => { + const opts = getMergedOptions(rawOpts); + + const { + pg, + server + } = opts; -export interface ExplorerOptions { - simpleInflection?: boolean; - oppositeBaseNames?: boolean; - port?: number; - postgis?: boolean; - origin?: string; -} - -export const LaunchQLExplorer = ({ - simpleInflection = env.USE_SIMPLE_INFLECTION, - oppositeBaseNames = env.USE_OPPOSITE_BASENAMES, - port = env.SERVER_PORT, - postgis = env.USE_POSTGIS, - origin -}: ExplorerOptions = {}): Express => { const getGraphileInstanceObj = (dbname: string, schemaname: string): GraphileCache => { const key = `${dbname}.${schemaname}`; @@ -37,26 +30,27 @@ export const LaunchQLExplorer = ({ return graphileCache.get(key); } - const opts: PostGraphileOptions = { + const settings = { ...getGraphileSettings({ - simpleInflection, - oppositeBaseNames, - port, - host: env.SERVER_HOST, - schema: schemaname, - postgis + ...opts, + graphile: { schema: schemaname } }), graphqlRoute: '/graphql', graphiqlRoute: '/graphiql' }; - const pgPool = getRootPgPool(dbname); - const handler = postgraphile(pgPool, schemaname, opts) + const pgPool = getRootPgPool({ + ...opts.pg, + database: dbname + }); + const handler = postgraphile(pgPool, schemaname, settings); + const obj = { pgPool, pgPoolKey: dbname, handler }; + graphileCache.set(key, obj); return obj; }; @@ -64,7 +58,7 @@ export const LaunchQLExplorer = ({ const app = express(); healthz(app); - cors(app, origin); + cors(app, server.origin); app.use(parseDomains()); app.use(poweredBy('launchql')); app.use(graphqlUploadExpress()); @@ -73,7 +67,10 @@ export const LaunchQLExplorer = ({ if (req.urlDomains?.subdomains.length === 1) { const [dbName] = req.urlDomains.subdomains; try { - const pgPool = getRootPgPool(dbName); + const pgPool = getRootPgPool({ + ...opts.pg, + database: dbName + }); const results = await pgPool.query(` SELECT s.nspname AS table_schema FROM pg_catalog.pg_namespace s @@ -84,8 +81,8 @@ export const LaunchQLExplorer = ({ dbName, schemas: results.rows, req, - hostname: env.SERVER_HOST, - port + hostname: server.host, + port: server.port }) ); return; @@ -106,7 +103,10 @@ export const LaunchQLExplorer = ({ if (req.urlDomains?.subdomains.length === 2) { const [, dbName] = req.urlDomains.subdomains; try { - const pgPool = getRootPgPool(dbName); + const pgPool = getRootPgPool({ + ...opts.pg, + database: dbName + }); await pgPool.query('SELECT 1;'); } catch (e: any) { if (e.message?.match(/does not exist/)) { @@ -150,12 +150,15 @@ export const LaunchQLExplorer = ({ app.use(async (req: Request, res: Response, next: NextFunction) => { if (req.urlDomains?.subdomains.length === 0) { try { - const rootPgPool = getRootPgPool(env.PGUSER); + const rootPgPool = getRootPgPool({ + ...opts.pg, + database: opts.pg.user // is this to get postgres? + }); const results = await rootPgPool.query(` SELECT * FROM pg_catalog.pg_database WHERE datistemplate = FALSE AND datname != 'postgres' AND datname !~ '^pg_' `); - res.send(printDatabases({ databases: results.rows, req, port })); + res.send(printDatabases({ databases: results.rows, req, port: server.port })); return; } catch (e: any) { if (e.message?.match(/does not exist/)) { @@ -170,8 +173,8 @@ export const LaunchQLExplorer = ({ return next(); }); - app.listen(port, env.SERVER_HOST, () => { - console.log(`app listening at http://${env.SERVER_HOST}:${port}`); + app.listen(server.port, server.host, () => { + console.log(`app listening at http://${server.host}:${server.port}`); }); return app; diff --git a/packages/explorer/src/settings.ts b/packages/explorer/src/settings.ts index 41819e3b40..ce27bd7ad2 100644 --- a/packages/explorer/src/settings.ts +++ b/packages/explorer/src/settings.ts @@ -1,36 +1,16 @@ -import { env } from './env'; import { PostGraphileOptions } from 'postgraphile'; import { getGraphileSettings as getSettings } from '@launchql/graphile-settings'; +import { LaunchQLOptions } from '@launchql/types'; +import { getMergedOptions } from '@launchql/types'; -interface SettingsInput { - host?: string; - port?: number; - schema: string | string[]; - simpleInflection?: boolean; - oppositeBaseNames?: boolean; - postgis?: boolean; -} +export const getGraphileSettings = (rawOpts: LaunchQLOptions): PostGraphileOptions => { + const opts = getMergedOptions(rawOpts); -export const getGraphileSettings = ({ - host, - port, - schema, - simpleInflection, - oppositeBaseNames, - postgis -}: SettingsInput): PostGraphileOptions => { - const options = getSettings({ - host, - port, - schema, - simpleInflection, - oppositeBaseNames, - postgis - }); + const baseOptions = getSettings(opts); - options.pgSettings = async function pgSettings(_req: any) { - return { role: env.PGUSER }; + baseOptions.pgSettings = async function pgSettings(_req: any) { + return { role: opts.pg?.user ?? 'postgres' }; }; - return options; + return baseOptions; }; diff --git a/packages/graphile-query/__tests__/first.test.ts b/packages/graphile-query/__tests__/first.test.ts deleted file mode 100644 index 2d48e8da80..0000000000 --- a/packages/graphile-query/__tests__/first.test.ts +++ /dev/null @@ -1,3 +0,0 @@ -it('works', () => { - console.log('hello test world!'); -}) \ No newline at end of file diff --git a/packages/graphile-settings/package.json b/packages/graphile-settings/package.json index de9b247050..ab30e35080 100644 --- a/packages/graphile-settings/package.json +++ b/packages/graphile-settings/package.json @@ -36,8 +36,8 @@ "@launchql/graphile-settings": "^1.2.0", "@launchql/graphile-query": "^1.2.0", "@launchql/upload-names": "^0.1.0", + "@launchql/types": "^2.0.0", "cors": "^2.8.5", - "dotenv": "^16.5.0", "envalid": "^8.0.0", "express": "^5.1.0", "graphile-build": "^4.14.1", diff --git a/packages/graphile-settings/src/env.ts b/packages/graphile-settings/src/env.ts deleted file mode 100644 index 9fbef996ec..0000000000 --- a/packages/graphile-settings/src/env.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { cleanEnv, str, port, bool } from 'envalid'; - -if (process.env.NODE_ENV !== 'production') { - require('dotenv').config(); -} - -export const env = cleanEnv(process.env, { - PGUSER: str(), - PGPASSWORD: str(), - PGHOST: str(), - PGDATABASE: str(), - PGPORT: port({ default: 5432 }), - DATABASE_URL: str(), - NODE_ENV: str({ choices: ['development', 'production', 'test'] }), - SCHEMAS: str({ default: 'public' }), - SERVER_HOST: str({ default: 'localhost' }), - TRUST_PROXY: bool({ default: false }), - PORT: port({ default: 3000 }), - // is this PORT? - SERVER_PORT: port({ default: 3333 }), - USE_SIMPLE_INFLECTION: bool({ default: true }), - USE_OPPOSITE_BASENAMES: bool({ default: true }), - USE_POSTGIS: bool({ default: true }), - - AWS_REGION: str({ default: 'aws_region' }), - AWS_SECRET_KEY: str({ default: 'aws_secret' }), - AWS_ACCESS_KEY: str({ default: 'aws_secret' }), - MINIO_ENDPOINT: str({ default: 'minio' }), - BUCKET_NAME: str({ default: 'bucket' }), - -}); diff --git a/packages/graphile-settings/src/index.ts b/packages/graphile-settings/src/index.ts index d4252db42b..f2b6b7c6d9 100644 --- a/packages/graphile-settings/src/index.ts +++ b/packages/graphile-settings/src/index.ts @@ -1,4 +1,3 @@ -import { env } from './env'; import { NodePlugin, Plugin } from 'graphile-build'; // @ts-ignore import PgSimpleInflector from 'graphile-simple-inflector'; @@ -21,26 +20,32 @@ import PgManyToMany from '@graphile-contrib/pg-many-to-many'; // @ts-ignore import PgSearch from 'graphile-search-plugin'; import LqlTypesPlugin from './plugins/types'; -import resolveUpload from './resolvers/upload'; +import { Uploader } from './resolvers/upload'; import { PostGraphileOptions } from 'postgraphile'; +import { LaunchQLOptions } from '@launchql/types'; +import { getMergedOptions } from '@launchql/types'; -interface GraphileSettingsInput { - host?: string; - port?: number; - schema: string | string[]; - simpleInflection?: boolean; - oppositeBaseNames?: boolean; - postgis?: boolean; -} +export const getGraphileSettings = (rawOpts: LaunchQLOptions): PostGraphileOptions => { + const opts = getMergedOptions(rawOpts); + + const { + server, + graphile, + features, + cdn + } = opts; + + // Instantiate uploader with merged cdn opts + const uploader = new Uploader({ + bucketName: cdn.bucketName!, + awsRegion: cdn.awsRegion!, + awsAccessKey: cdn.awsAccessKey!, + awsSecretKey: cdn.awsSecretKey!, + minioEndpoint: cdn.minioEndpoint + }); + + const resolveUpload = uploader.resolveUpload.bind(uploader); -export const getGraphileSettings = ({ - host, - port, - schema, - simpleInflection = true, - oppositeBaseNames = false, - postgis = true -}: GraphileSettingsInput): PostGraphileOptions => { const plugins: Plugin[] = [ ConnectionFilterPlugin, FulltextFilterPlugin, @@ -51,11 +56,11 @@ export const getGraphileSettings = ({ PgSearch ]; - if (postgis) { + if (features?.postgis) { plugins.push(PgPostgis, PgPostgisFilter); } - if (simpleInflection) { + if (features?.simpleInflection) { plugins.push(PgSimpleInflector); } @@ -87,7 +92,7 @@ export const getGraphileSettings = ({ resolve: resolveUpload } ], - pgSimplifyOppositeBaseNames: oppositeBaseNames, + pgSimplifyOppositeBaseNames: features?.oppositeBaseNames, connectionFilterComputedColumns: false }, appendPlugins: plugins, @@ -98,9 +103,9 @@ export const getGraphileSettings = ({ enableQueryBatching: true, graphiql: true, watch: false, - port, - host, - schema, + port: server?.port, + host: server?.host, + schema: graphile?.schema, ignoreRBAC: false, legacyRelations: 'omit', showErrorStack: false, @@ -109,14 +114,13 @@ export const getGraphileSettings = ({ disableQueryLog: false, includeExtensionResources: true, setofFunctionsContainNulls: false, - retryOnInitFail: async (error: Error) => { + retryOnInitFail: async (_error: Error) => { return false; }, additionalGraphQLContextFromRequest: (req, res) => ({ ...langAdditional(req, res), req, - res, - env + res }) }; }; diff --git a/packages/graphile-settings/src/plugins/types.ts b/packages/graphile-settings/src/plugins/types.ts index a714e6d8bc..1e3abe40b0 100644 --- a/packages/graphile-settings/src/plugins/types.ts +++ b/packages/graphile-settings/src/plugins/types.ts @@ -3,7 +3,7 @@ import { Plugin } from 'graphile-build'; // formerly lql-types.js const CustomPgTypeMappingsPlugin: Plugin = builder => { - builder.hook('build', (build, context) => { + builder.hook('build', (build, _context) => { const customMappings = [ { name: 'geolocation', namespaceName: 'public', type: 'GeoJSON' }, { name: 'geopolygon', namespaceName: 'public', type: 'GeoJSON' }, diff --git a/packages/graphile-settings/src/resolvers/upload.ts b/packages/graphile-settings/src/resolvers/upload.ts index a1a8a0e53c..980dac5b1e 100644 --- a/packages/graphile-settings/src/resolvers/upload.ts +++ b/packages/graphile-settings/src/resolvers/upload.ts @@ -1,72 +1,84 @@ +import streamer from '@launchql/s3-streamer'; import uploadNames from '@launchql/upload-names'; -import type { ReadStream } from 'fs'; -interface Upload { - filename: string; - mimetype: string; - encoding: string; - createReadStream: () => ReadStream; +interface UploaderOptions { + bucketName: string; + awsRegion: string; + awsSecretKey: string; + awsAccessKey: string; + minioEndpoint?: string; } -interface UploadPluginInfo { - uploadPlugin: { - tags: Record; - type?: string; - }; -} +export class Uploader { + private streamerInstance: any; -export default async function resolveUpload( - upload: Upload, - _args: unknown, - _context: unknown, - info: UploadPluginInfo -): Promise< - | { - filename: string; - mime: string; - url: string; - } - | string -> { - const { - uploadPlugin: { tags, type } - } = info; + constructor(private opts: UploaderOptions) { + const { + bucketName, + awsRegion, + awsSecretKey, + awsAccessKey, + minioEndpoint + } = this.opts; - const { filename, mimetype } = upload; + this.streamerInstance = new streamer({ + defaultBucket: bucketName, + awsRegion, + awsSecretKey, + awsAccessKey, + minioEndpoint, + }); + } - const rand = - Math.random().toString(36).substring(2, 7) + - Math.random().toString(36).substring(2, 7); + async resolveUpload(upload: any, _args: any, _context: any, info: any) { + const { + uploadPlugin: { tags, type } + } = info; - const key = rand + '-' + uploadNames(filename); - const url = `https://mock-bucket.local/${key}`; + const readStream = upload.createReadStream(); + const { filename } = upload; - const typ = type || tags.type; + const rand = + Math.random().toString(36).substring(2, 7) + + Math.random().toString(36).substring(2, 7); - const mimetypes = tags.mime - ? tags.mime.trim().split(',').map(a => a.trim()) - : typ === 'image' - ? ['image/jpg', 'image/jpeg', 'image/png', 'image/svg+xml'] - : []; + const key = `${rand}-${uploadNames(filename)}`; + const result = await this.streamerInstance.upload({ + readStream, + filename, + key, + bucket: this.opts.bucketName + }); - const allowed = !mimetypes.length || mimetypes.includes(mimetype); + const url = result.upload.Location; + const { + contentType, + magic: { charset } + } = result; - if (!allowed) { - throw new Error( - `Upload rejected: MIME type "${mimetype}" is not allowed. Expected one of: ${mimetypes.join(', ')}.` - ); - } + const typ = type || tags.type; - switch (typ) { - case 'image': - case 'upload': - return { - filename, - mime: mimetype, - url - }; - case 'attachment': - default: - return url; + const allowedMimes = tags.mime + ? tags.mime.trim().split(',').map((a: string) => a.trim()) + : typ === 'image' + ? ['image/jpg', 'image/jpeg', 'image/png', 'image/svg+xml'] + : []; + + if (allowedMimes.length && !allowedMimes.includes(contentType)) { + throw new Error(`UPLOAD_MIMETYPE ${allowedMimes.join(',')}`); + } + + switch (typ) { + case 'image': + case 'upload': + return { + filename, + mime: contentType, + url + }; + case 'attachment': + default: + return url; + } } } diff --git a/packages/s3-streamer/__tests__/uploads.test.ts b/packages/s3-streamer/__tests__/uploads.test.ts index f487ac1fae..e4b43a2510 100644 --- a/packages/s3-streamer/__tests__/uploads.test.ts +++ b/packages/s3-streamer/__tests__/uploads.test.ts @@ -61,10 +61,10 @@ describe('uploads', () => { it('upload files via class', async () => { const streamer = new Streamer({ defaultBucket: BUCKET_NAME, - AWS_REGION, - AWS_SECRET_KEY, - AWS_ACCESS_KEY, - MINIO_ENDPOINT + awsRegion: AWS_REGION, + awsSecretKey: AWS_SECRET_KEY, + awsAccessKey: AWS_ACCESS_KEY, + minioEndpoint: MINIO_ENDPOINT }); const res: Record = {}; @@ -89,10 +89,10 @@ describe('uploads', () => { it('upload files via functions', async () => { const client = getClient({ - AWS_REGION, - AWS_SECRET_KEY, - AWS_ACCESS_KEY, - MINIO_ENDPOINT + awsRegion: AWS_REGION, + awsSecretKey: AWS_SECRET_KEY, + awsAccessKey: AWS_ACCESS_KEY, + minioEndpoint: MINIO_ENDPOINT }); const res: Record = {}; diff --git a/packages/s3-streamer/package.json b/packages/s3-streamer/package.json index 2e57a93acd..8410a841e8 100644 --- a/packages/s3-streamer/package.json +++ b/packages/s3-streamer/package.json @@ -30,10 +30,13 @@ "test:watch": "jest --watch" }, "devDependencies": { - "@launchql/s3-utils": "0.0.1" + "@launchql/s3-utils": "0.0.1", + "dotenv": "^16.5.0", + "envalid": "^8.0.0" }, "dependencies": { - "aws-sdk": "^2.1692.0", - "@launchql/content-type-stream": "^0.0.1" + "@launchql/content-type-stream": "^0.0.1", + "@launchql/types": "^2.0.0", + "aws-sdk": "^2.1692.0" } } \ No newline at end of file diff --git a/packages/s3-streamer/src/s3.ts b/packages/s3-streamer/src/s3.ts index 6a854e0f9f..1015bb2513 100644 --- a/packages/s3-streamer/src/s3.ts +++ b/packages/s3-streamer/src/s3.ts @@ -1,30 +1,30 @@ import S3 from 'aws-sdk/clients/s3'; -interface S3Env { - AWS_ACCESS_KEY?: string; - AWS_SECRET_KEY?: string; - AWS_REGION?: string; - MINIO_ENDPOINT?: string; +interface S3Options { + awsAccessKey: string; + awsSecretKey: string; + awsRegion: string; + minioEndpoint?: string; } -export default function getS3(env: S3Env): S3 { - const isMinio = Boolean(env.MINIO_ENDPOINT); +export default function getS3(opts: S3Options): S3 { + const isMinio = Boolean(opts.minioEndpoint); const awsConfig: S3.ClientConfiguration = isMinio ? { - accessKeyId: env.AWS_ACCESS_KEY, - secretAccessKey: env.AWS_SECRET_KEY, - endpoint: env.MINIO_ENDPOINT, + accessKeyId: opts.awsAccessKey, + secretAccessKey: opts.awsSecretKey, + endpoint: opts.minioEndpoint, s3ForcePathStyle: true, signatureVersion: 'v4', } : { apiVersion: '2006-03-01', - region: env.AWS_REGION, - ...(env.AWS_ACCESS_KEY && env.AWS_SECRET_KEY + region: opts.awsRegion, + ...(opts.awsAccessKey && opts.awsSecretKey ? { - accessKeyId: env.AWS_ACCESS_KEY, - secretAccessKey: env.AWS_SECRET_KEY, + accessKeyId: opts.awsAccessKey, + secretAccessKey: opts.awsSecretKey, } : {}), }; diff --git a/packages/s3-streamer/src/streamer.ts b/packages/s3-streamer/src/streamer.ts index e11602e56d..4e49da064f 100644 --- a/packages/s3-streamer/src/streamer.ts +++ b/packages/s3-streamer/src/streamer.ts @@ -4,11 +4,11 @@ import getS3 from './s3'; import { upload as streamUpload, type AsyncUploadResult } from './utils'; interface StreamerOptions { - AWS_REGION?: string; - AWS_SECRET_KEY?: string; - AWS_ACCESS_KEY?: string; - MINIO_ENDPOINT?: string; - defaultBucket?: string; + awsRegion: string; + awsSecretKey: string; + awsAccessKey: string; + minioEndpoint?: string; + defaultBucket: string; } interface UploadParams { @@ -23,17 +23,17 @@ export class Streamer { private defaultBucket?: string; constructor({ - AWS_REGION = 'us-east-1', - AWS_SECRET_KEY, - AWS_ACCESS_KEY, - MINIO_ENDPOINT, + awsRegion, + awsSecretKey, + awsAccessKey, + minioEndpoint, defaultBucket - }: StreamerOptions = {}) { + }: StreamerOptions) { this.s3 = getS3({ - AWS_REGION, - AWS_SECRET_KEY, - AWS_ACCESS_KEY, - MINIO_ENDPOINT + awsRegion, + awsSecretKey, + awsAccessKey, + minioEndpoint }); this.defaultBucket = defaultBucket; } diff --git a/packages/server-utils/__tests__/first.test.ts b/packages/server-utils/__tests__/first.test.ts deleted file mode 100644 index 2d48e8da80..0000000000 --- a/packages/server-utils/__tests__/first.test.ts +++ /dev/null @@ -1,3 +0,0 @@ -it('works', () => { - console.log('hello test world!'); -}) \ No newline at end of file diff --git a/packages/server-utils/package.json b/packages/server-utils/package.json index d66e6c12a3..ffbf9d6417 100644 --- a/packages/server-utils/package.json +++ b/packages/server-utils/package.json @@ -30,6 +30,7 @@ "test:watch": "jest --watch" }, "dependencies": { + "@launchql/types": "^2.0.0", "cors": "^2.8.5", "envalid": "^8.0.0", "express": "^5.1.0", diff --git a/packages/server-utils/src/pg.ts b/packages/server-utils/src/pg.ts index 56b3c4f737..3094a43e27 100644 --- a/packages/server-utils/src/pg.ts +++ b/packages/server-utils/src/pg.ts @@ -1,20 +1,32 @@ -import { env } from './env'; import pg from 'pg'; import { pgCache } from './lru'; -export const getDbString = (db: string): string => - `postgres://${env.PGUSER}:${env.PGPASSWORD}@${env.PGHOST}:${env.PGPORT}/${db}`; +import { PostgresOptions } from '@launchql/types'; -export const getRootPgPool = (dbname: string): pg.Pool => { - if (pgCache.has(dbname)) { - const cached = pgCache.get(dbname); +export const getDbString = ( + user: string, + password: string, + host: string, + port: string | number, + database: string +): string => + `postgres://${user}:${password}@${host}:${port}/${database}`; + +export const getRootPgPool = ({ + user, + password, + host, + port, + database, +}: PostgresOptions): pg.Pool => { + if (pgCache.has(database)) { + const cached = pgCache.get(database); if (cached) return cached; } - const pgPool = new pg.Pool({ - connectionString: getDbString(dbname), - }); + const connectionString = getDbString(user, password, host, port, database); + const pgPool = new pg.Pool({ connectionString }); - pgCache.set(dbname, pgPool); + pgCache.set(database, pgPool); return pgPool; }; diff --git a/packages/server-utils/src/utils.ts b/packages/server-utils/src/utils.ts index b0398155ac..593b4a1eef 100644 --- a/packages/server-utils/src/utils.ts +++ b/packages/server-utils/src/utils.ts @@ -16,8 +16,8 @@ export const poweredBy = (name: string) => { }; }; -export const trustProxy = (app: Express, env: { TRUST_PROXY?: string }): void => { - if (env.TRUST_PROXY) { +export const trustProxy = (app: Express, trustProxy?: boolean): void => { + if (trustProxy) { app.set('trust proxy', (ip: string) => { return true; }); diff --git a/packages/server/package.json b/packages/server/package.json index 7af68e0f68..4b1c46a970 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -26,7 +26,6 @@ "build": "npm run clean; tsc; tsc -p tsconfig.esm.json; npm run copy", "build:dev": "npm run clean; tsc --declarationMap; tsc -p tsconfig.esm.json; npm run copy", "dev": "ts-node src/run.ts", - "simple": "ts-node src/simple-server.ts", "dev:watch": "nodemon --watch src --ext ts --exec ts-node src/run.ts", "lint": "eslint . --fix", "test": "jest", @@ -43,8 +42,8 @@ "@launchql/server-utils": "^0.4.0", "@launchql/url-domains": "^0.1.0", "@launchql/pg-query-context": "^0.1.0", + "@launchql/types": "^2.0.0", "cors": "^2.8.5", - "dotenv": "^16.5.0", "envalid": "^8.0.0", "express": "^5.1.0", "graphile-build": "^4.14.1", diff --git a/packages/server/src/env.ts b/packages/server/src/env.ts deleted file mode 100644 index 5414569199..0000000000 --- a/packages/server/src/env.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { cleanEnv, str, port, bool } from 'envalid'; -import { makeValidator, EnvError } from 'envalid' - -const array = makeValidator((input: string) => { - if (typeof input !== 'string') { - throw new EnvError(`Expected a string but got: ${typeof input}`) - } - return input - .split(',') - .map(s => s.trim()) - .filter(s => s.length > 0) -}) -if (process.env.NODE_ENV !== 'production') { - require('dotenv').config(); -} - -// SERVER_PORT: port({ default: 5555 }), -// SERVER_HOST: str({ default: 'localhost' }), - - -export const env = cleanEnv(process.env, { - META_SCHEMAS: array({ default: ['meta_public', 'collections_public'] }), - SCHEMAS: str({ default: 'collections_public' }), - - IS_PUBLIC: bool({ default: true }), - STRICT_AUTH: bool({ default: true }), - - ///// - PGUSER: str(), - PGPASSWORD: str(), - PGHOST: str(), - PGDATABASE: str(), - PGPORT: port({ default: 5432 }), - DATABASE_URL: str(), - NODE_ENV: str({ choices: ['development', 'production', 'test'] }), - SERVER_HOST: str({ default: 'localhost' }), - TRUST_PROXY: bool({ default: false }), - PORT: port({ default: 3000 }), - // is this PORT? - SERVER_PORT: port({ default: 3333 }), - USE_SIMPLE_INFLECTION: bool({ default: true }), - USE_OPPOSITE_BASENAMES: bool({ default: true }), - USE_POSTGIS: bool({ default: true }), - - AWS_REGION: str({ default: 'aws_region' }), - AWS_SECRET_KEY: str({ default: 'aws_secret' }), - AWS_ACCESS_KEY: str({ default: 'aws_secret' }), - MINIO_ENDPOINT: str({ default: 'minio' }), - BUCKET_NAME: str({ default: 'bucket' }), - -}); diff --git a/packages/server/src/middleware/api.ts b/packages/server/src/middleware/api.ts index 343a1e261d..f177d9fd27 100644 --- a/packages/server/src/middleware/api.ts +++ b/packages/server/src/middleware/api.ts @@ -2,61 +2,45 @@ import { getGraphileSettings } from '@launchql/graphile-settings'; import { GraphileQuery, getSchema } from '@launchql/graphile-query'; import { ApiQuery, ApiByNameQuery } from './gql'; import { svcCache, getRootPgPool } from '@launchql/server-utils'; -import { env } from '../env'; import errorPage404 from '../errors/404'; import errorPage50x from '../errors/50x'; - -const isPublic = env.IS_PUBLIC; +import { LaunchQLOptions } from '@launchql/types'; +import { Response, NextFunction } from 'express'; export const getSubdomain = (reqDomains: string[]): string | null => { const names = reqDomains.filter((name) => !['www'].includes(name)); return !names.length ? null : names.join('.'); }; -export const api = async (req: any, res: any, next: any): Promise => { - try { - const svc = await getApiConfig(req); - if (!svc) { - return res.status(404).send(errorPage404); - } - req.apiInfo = svc; - req.databaseId = svc.data.api.databaseId; - } catch (e: any) { - if (e.message.match(/does not exist/)) { - return res.status(404).send(errorPage404); - } - console.error(e); - return res.status(500).send(errorPage50x); - } - return next(); -}; - -const getSvcKey = (req: any): string => { - const domain = req.urlDomains.domain; - const key = req.urlDomains.subdomains - .filter((name: string) => !['www'].includes(name)) - .concat(domain) - .join('.'); - - if (!isPublic) { - if (req.get('X-Api-Name')) { - return 'api:' + req.get('X-Database-Id') + ':' + req.get('X-Api-Name'); - } - if (req.get('X-Schemata')) { - return 'schemata:' + req.get('X-Database-Id') + ':' + req.get('X-Schemata'); - } - if (req.get('X-Meta-Schema')) { - return 'metaschema:api:' + req.get('X-Database-Id'); +export const createApiMiddleware = (opts: LaunchQLOptions) => { + return async (req: any, res: Response, next: NextFunction): Promise => { + try { + const svc = await getApiConfig(opts, req); + if (!svc) { + res.status(404).send(errorPage404); + return; + } + req.apiInfo = svc; + req.databaseId = svc.data.api.databaseId; + next(); + } catch (e: any) { + if (e.message.match(/does not exist/)) { + res.status(404).send(errorPage404); + } else { + console.error(e); + res.status(500).send(errorPage50x); + } } - } - return key; + }; }; const getHardCodedSchemata = ({ + opts, schemata, databaseId, key }: { + opts: LaunchQLOptions, schemata: string; databaseId: string; key: string; @@ -66,7 +50,7 @@ const getHardCodedSchemata = ({ api: { databaseId, isPublic: false, - dbname: env.PGDATABASE, + dbname: opts.pg.database, anonRole: 'administrator', roleName: 'administrator', schemaNamesFromExt: { @@ -87,19 +71,21 @@ const getHardCodedSchemata = ({ }; const getMetaSchema = ({ + opts, key, databaseId }: { + opts: LaunchQLOptions, key: string; databaseId: string; }): any => { - const schemata = env.META_SCHEMAS; + const schemata = opts.graphile.metaSchemas; const svc = { data: { api: { databaseId, isPublic: false, - dbname: env.PGDATABASE, + dbname: opts.pg.database, anonRole: 'administrator', roleName: 'administrator', schemaNamesFromExt: { @@ -117,11 +103,13 @@ const getMetaSchema = ({ }; const queryServiceByDomainAndSubdomain = async ({ + opts, key, client, domain, subdomain }: { + opts: LaunchQLOptions, key: string; client: any; domain: string; @@ -144,7 +132,7 @@ const queryServiceByDomainAndSubdomain = async ({ const nodes = result?.data?.domains?.nodes; if (nodes?.length) { const data = nodes[0]; - if (!data.api || data.api.isPublic !== isPublic) return null; + if (!data.api || data.api.isPublic !== opts.graphile.isPublic) return null; const svc = { data }; svcCache.set(key, svc); return svc; @@ -153,11 +141,13 @@ const queryServiceByDomainAndSubdomain = async ({ }; const queryServiceByApiName = async ({ + opts, key, client, databaseId, name }: { + opts: LaunchQLOptions, key: string; client: any; databaseId: string; @@ -175,7 +165,7 @@ const queryServiceByApiName = async ({ } const data = result?.data; - if (data?.api && data.api.isPublic === isPublic) { + if (data?.api && data.api.isPublic === opts.graphile.isPublic) { const svc = { data }; svcCache.set(key, svc); return svc; @@ -183,24 +173,45 @@ const queryServiceByApiName = async ({ return null; }; -export const getApiConfig = async (req: any): Promise => { - const rootPgPool = getRootPgPool(env.PGDATABASE); +const getSvcKey = (opts: LaunchQLOptions, req: any): string => { + const domain = req.urlDomains.domain; + const key = req.urlDomains.subdomains + .filter((name: string) => !['www'].includes(name)) + .concat(domain) + .join('.'); + + if (!opts.graphile.isPublic) { + if (req.get('X-Api-Name')) { + return 'api:' + req.get('X-Database-Id') + ':' + req.get('X-Api-Name'); + } + if (req.get('X-Schemata')) { + return 'schemata:' + req.get('X-Database-Id') + ':' + req.get('X-Schemata'); + } + if (req.get('X-Meta-Schema')) { + return 'metaschema:api:' + req.get('X-Database-Id'); + } + } + return key; +}; +export const getApiConfig = async (opts: LaunchQLOptions, req: any): Promise => { + const rootPgPool = getRootPgPool(opts.pg); const subdomain = getSubdomain(req.urlDomains.subdomains); const domain = req.urlDomains.domain; - const key = getSvcKey(req); + const key = getSvcKey(opts, req); req.svc_key = key; - const schemata = env.META_SCHEMAS; + const schemata = opts.graphile.metaSchemas let svc; if (svcCache.has(key)) { svc = svcCache.get(key); } else { const settings = getGraphileSettings({ - simpleInflection: true, - schema: schemata + graphile: { + schema: schemata + } }); console.log('remove THESE ignores!!!'); @@ -209,15 +220,17 @@ export const getApiConfig = async (req: any): Promise => { // @ts-ignore const client = new GraphileQuery({ schema, pool: rootPgPool, settings }); - if (!isPublic) { + if (!opts.graphile.isPublic) { if (req.get('X-Schemata')) { svc = getHardCodedSchemata({ + opts, key, schemata: req.get('X-Schemata'), databaseId: req.get('X-Database-Id') }); } else if (req.get('X-Api-Name')) { svc = await queryServiceByApiName({ + opts, key, client, name: req.get('X-Api-Name'), @@ -225,11 +238,13 @@ export const getApiConfig = async (req: any): Promise => { }); } else if (req.get('X-Meta-Schema')) { svc = getMetaSchema({ + opts, key, databaseId: req.get('X-Database-Id') }); } else { svc = await queryServiceByDomainAndSubdomain({ + opts, key, client, domain, @@ -238,6 +253,7 @@ export const getApiConfig = async (req: any): Promise => { } } else { svc = await queryServiceByDomainAndSubdomain({ + opts, key, client, domain, diff --git a/packages/server/src/middleware/auth.ts b/packages/server/src/middleware/auth.ts index 55b585c9b8..15762dd0f9 100644 --- a/packages/server/src/middleware/auth.ts +++ b/packages/server/src/middleware/auth.ts @@ -1,77 +1,81 @@ -import { Request, Response, NextFunction } from 'express'; +import { Request, Response, NextFunction, RequestHandler } from 'express'; import { getRootPgPool } from '@launchql/server-utils'; -import { env } from '../env'; import pgQueryContext from '@launchql/pg-query-context'; +import { LaunchQLOptions } from '@launchql/types'; -const strictAuth = env.STRICT_AUTH; +export const createAuthenticateMiddleware = (opts: LaunchQLOptions): RequestHandler => { -export const authenticate = async (req: Request, res: Response, next: NextFunction): Promise => { - const api = req.apiInfo.data.api; - const pool = getRootPgPool(api.dbname); + return async (req: Request, res: Response, next: NextFunction): Promise => { + const api = req.apiInfo?.data?.api; + if (!api) { + res.status(500).send('Missing API info'); + return; + } - const rlsModule = api.rlsModule; + const pool = getRootPgPool({ + ...opts.pg, + database: api.dbname + }); + const rlsModule = api.rlsModule; - if (!rlsModule) return next(); + if (!rlsModule) return next(); - const authFn = strictAuth - ? rlsModule.authenticateStrict - : rlsModule.authenticate; + const authFn = opts.server.strictAuth + ? rlsModule.authenticateStrict + : rlsModule.authenticate; - if (authFn && rlsModule.privateSchema.schemaName) { - const { authorization = '' } = req.headers; - const [authType, authToken] = authorization.split(' '); - let token: any = {}; + if (authFn && rlsModule.privateSchema.schemaName) { + const { authorization = '' } = req.headers; + const [authType, authToken] = authorization.split(' '); + let token: any = {}; - if (authType?.toLowerCase?.() === 'bearer' && authToken) { - let result: any = null; + if (authType?.toLowerCase() === 'bearer' && authToken) { + const context: Record = { + 'jwt.claims.ip_address': req.clientIp, + }; - const context: Record = { - [`jwt.claims.ip_address`]: req.clientIp - }; + if (req.get('origin')) { + context['jwt.claims.origin'] = req.get('origin'); + } + if (req.get('User-Agent')) { + context['jwt.claims.user_agent'] = req.get('User-Agent'); + } - if (req.get('origin')) { - context['jwt.claims.origin'] = req.get('origin'); - } - if (req.get('User-Agent')) { - context['jwt.claims.user_agent'] = req.get('User-Agent'); - } + try { + const result = await pgQueryContext({ + client: pool, + context, + query: `SELECT * FROM "${rlsModule.privateSchema.schemaName}"."${authFn}"($1)`, + variables: [authToken], + }); - try { - result = await pgQueryContext({ - client: pool, - context, - query: `SELECT * FROM "${rlsModule.privateSchema.schemaName}"."${authFn}"($1)`, - variables: [authToken] - }); - } catch (e: any) { - return res.status(200).end( - JSON.stringify({ + if (result?.rowCount === 0) { + res.status(200).json({ + errors: [{ extensions: { code: 'UNAUTHENTICATED' } }], + }); + return; + } + + token = result.rows[0]; + } catch (e: any) { + res.status(200).json({ errors: [ { extensions: { code: 'BAD_TOKEN_DEFINITION', - message: e.message - } - } - ] - }) - ); + message: e.message, + }, + }, + ], + }); + return; + } } - if (result?.rowCount === 0) { - return res.status(200).end( - JSON.stringify({ - errors: [{ extensions: { code: 'UNAUTHENTICATED' } }] - }) - ); - } else { - token = result.rows[0]; - } + // @ts-ignore - augment req with `token` + req.token = token; } - // @ts-ignore - req.token = token; - } - - return next(); + next(); + }; }; diff --git a/packages/server/src/middleware/flush.ts b/packages/server/src/middleware/flush.ts index 72ccec03ff..0e5cbadea4 100644 --- a/packages/server/src/middleware/flush.ts +++ b/packages/server/src/middleware/flush.ts @@ -1,6 +1,6 @@ import { Request, Response, NextFunction } from 'express'; import { graphileCache, svcCache, getRootPgPool } from '@launchql/server-utils'; -import { env } from '../env'; +import { LaunchQLOptions } from '@launchql/types'; export const flush = async (req: Request, res: Response, next: NextFunction): Promise => { if (req.url === '/flush') { @@ -13,15 +13,15 @@ export const flush = async (req: Request, res: Response, next: NextFunction): Pr return next(); }; -export const flushService = async (databaseId: string): Promise => { - const pgPool = getRootPgPool(env.PGDATABASE); +export const flushService = async (opts: LaunchQLOptions, databaseId: string): Promise => { + const pgPool = getRootPgPool(opts.pg); console.log('flushing db ' + databaseId); const api = new RegExp(`^api:${databaseId}:.*`); const schemata = new RegExp(`^schemata:${databaseId}:.*`); const meta = new RegExp(`^metaschema:api:${databaseId}`); - if (!env.IS_PUBLIC) { + if (!opts.graphile.isPublic) { graphileCache.forEach((_, k: string) => { if (api.test(k) || schemata.test(k) || meta.test(k)) { graphileCache.delete(k); diff --git a/packages/server/src/middleware/graphile.ts b/packages/server/src/middleware/graphile.ts index 790b2d320c..fa9013974c 100644 --- a/packages/server/src/middleware/graphile.ts +++ b/packages/server/src/middleware/graphile.ts @@ -1,71 +1,12 @@ import { Request, Response, NextFunction, RequestHandler } from 'express'; -import { env } from '../env'; import { graphileCache, getRootPgPool } from '@launchql/server-utils'; import { postgraphile, PostGraphileOptions } from 'postgraphile'; import { getGraphileSettings as getSettings } from '@launchql/graphile-settings'; -import type { Plugin } from 'graphile-build'; import PublicKeySignature from '../plugins/PublicKeySignature'; +import { LaunchQLOptions } from '@launchql/types'; + +export const graphile = (lOpts: LaunchQLOptions): RequestHandler => { -declare module 'express-serve-static-core' { - interface Request { - apiInfo: { - data: { - api: { - dbname: string; - anonRole: string; - roleName: string; - schemaNames: { - nodes: { schemaName: string }[]; - }; - schemaNamesFromExt: { - nodes: { schemaName: string }[]; - }; - apiModules: { - nodes: { - name: string; - data?: any; - }[]; - }; - rlsModule?: { - authenticate?: string; - authenticateStrict?: string; - privateSchema: { - schemaName: string; - }; - }; - }; - }; - }; - svc_key: string; - clientIp?: string; - databaseId?: string; - token?: { - id: string; - user_id: string; - [key: string]: any; - }; - } -} - -interface GraphileMiddlewareOptions { - simpleInflection?: boolean; - oppositeBaseNames?: boolean; - port?: number; - postgis?: boolean; - appendPlugins?: Plugin[]; - graphileBuildOptions?: PostGraphileOptions['graphileBuildOptions']; - overrideSettings?: Partial; -} - -export const graphile = ({ - simpleInflection, - oppositeBaseNames, - port, - postgis, - appendPlugins = [], - graphileBuildOptions = {}, - overrideSettings = {} -}: GraphileMiddlewareOptions): RequestHandler => { // @ts-ignore return async (req: Request, res: Response, next: NextFunction) => { try { @@ -85,12 +26,11 @@ export const graphile = ({ } const options = getSettings({ - host: env.SERVER_HOST, - schema: schemas, - port, - simpleInflection, - oppositeBaseNames, - postgis + ...lOpts, + graphile: { + ...lOpts.graphile, + schema: schemas + } }); const pubkey_challenge = api.apiModules.nodes.find( @@ -102,7 +42,7 @@ export const graphile = ({ } options.appendPlugins = options.appendPlugins ?? []; - options.appendPlugins.push(...appendPlugins); + options.appendPlugins.push(...lOpts.graphile.appendPlugins); // @ts-ignore options.pgSettings = async function pgSettings(req: Request) { @@ -135,15 +75,18 @@ export const graphile = ({ options.graphileBuildOptions = { ...options.graphileBuildOptions, - ...graphileBuildOptions + ...lOpts.graphile.graphileBuildOptions }; const opts: PostGraphileOptions = { ...options, - ...overrideSettings + ...lOpts.graphile.overrideSettings }; - const pgPool = getRootPgPool(dbname); + const pgPool = getRootPgPool({ + ...lOpts.pg, + database: dbname + }); const handler = postgraphile(pgPool, schemas, opts); graphileCache.set(key, { diff --git a/packages/server/src/run.ts b/packages/server/src/run.ts index 003cbd9aa9..279030a97f 100644 --- a/packages/server/src/run.ts +++ b/packages/server/src/run.ts @@ -1,5 +1,6 @@ #!/usr/bin/env node +import { getEnvOptions } from '@launchql/types'; import { LaunchQLServer as server } from './server'; -server(); \ No newline at end of file +server(getEnvOptions()); diff --git a/packages/server/src/server.ts b/packages/server/src/server.ts index 29d6f59ba9..2319f74d7d 100644 --- a/packages/server/src/server.ts +++ b/packages/server/src/server.ts @@ -5,69 +5,38 @@ import { getRootPgPool } from '@launchql/server-utils'; -import { env } from './env'; import graphqlUploadExpress from 'graphql-upload/graphqlUploadExpress.mjs'; import { middleware as parseDomains } from '@launchql/url-domains'; import express, { Express, RequestHandler } from 'express'; -import { authenticate } from './middleware/auth'; +import { createAuthenticateMiddleware } from './middleware/auth'; import { graphile } from './middleware/graphile'; import { cors } from './middleware/cors'; -import { api } from './middleware/api'; +import { createApiMiddleware } from './middleware/api'; import { flush, flushService } from './middleware/flush'; import requestIp from 'request-ip'; import { Pool, PoolClient } from 'pg'; -export interface ServerOptions { - simpleInflection?: boolean; - oppositeBaseNames?: boolean; - port?: number; - postgis?: boolean; - appendPlugins?: any[]; - overrideSettings?: Record; - graphileBuildOptions?: Record; -} +import { LaunchQLOptions, getMergedOptions } from '@launchql/types'; -export const LaunchQLServer = ({ - simpleInflection = env.USE_SIMPLE_INFLECTION, - oppositeBaseNames = env.USE_OPPOSITE_BASENAMES, - port = env.SERVER_PORT, - postgis = env.USE_POSTGIS -}: ServerOptions = {}) => { - const app = new Server({ - simpleInflection, - oppositeBaseNames, - port, - postgis - }); +export const LaunchQLServer = (rawOpts: LaunchQLOptions = {}) => { + const app = new Server(getMergedOptions(rawOpts)); app.addEventListener(); app.listen(); }; -// const middleware = { -// authenticate, -// graphile, -// api -// }; - class Server { private app: Express; - private port: number; - - constructor({ - simpleInflection = env.USE_SIMPLE_INFLECTION, - oppositeBaseNames = env.USE_OPPOSITE_BASENAMES, - port = env.SERVER_PORT, - postgis = env.USE_POSTGIS, - appendPlugins = [], - overrideSettings = {}, - graphileBuildOptions = {} - }: ServerOptions = {}) { - this.port = port!; + private opts: LaunchQLOptions; + + constructor(opts: LaunchQLOptions) { + this.opts = opts; const app = express(); + const api = createApiMiddleware(opts); + const authenticate = createAuthenticateMiddleware(opts); healthz(app); - trustProxy(app, env as any); + trustProxy(app, opts.server.trustProxy); app.use(poweredBy('launchql')); app.use(graphqlUploadExpress()); app.use(parseDomains() as RequestHandler); @@ -75,33 +44,25 @@ class Server { app.use(api); app.use(cors as any); app.use(authenticate); - app.use( - graphile({ - simpleInflection, - oppositeBaseNames, - port, - postgis, - appendPlugins, - overrideSettings, - graphileBuildOptions - }) - ); + app.use(graphile(opts)); app.use(flush); + this.app = app; } listen(): void { - this.app.listen(this.port, env.SERVER_HOST, () => - this.log(`listening at http://${env.SERVER_HOST}:${this.port}`) + const { server } = this.opts; + this.app.listen(server?.port, server?.host, () => + this.log(`listening at http://${server?.host}:${server?.port}`) ); } async flush(databaseId: string): Promise { - await flushService(databaseId); + await flushService(this.opts, databaseId); } getPool(): Pool { - return getRootPgPool(env.PGDATABASE); + return getRootPgPool(this.opts.pg); } addEventListener(): void { @@ -116,8 +77,7 @@ class Server { return; } - client.on('notification', (args: { channel: string; payload?: string }) => { - const { channel, payload } = args; + client.on('notification', ({ channel, payload }) => { if (channel === 'schema:update' && payload) { console.log('schema:update', payload); this.flush(payload); @@ -144,5 +104,4 @@ class Server { } } -// export { middleware, Server }; export { Server }; diff --git a/packages/server/src/simple-server.ts b/packages/server/src/simple-server.ts deleted file mode 100644 index 3e27e2730b..0000000000 --- a/packages/server/src/simple-server.ts +++ /dev/null @@ -1,136 +0,0 @@ -import express from 'express'; -import { postgraphile } from 'postgraphile'; -import { NodePlugin, Plugin } from 'graphile-build'; -import ConnectionFilterPlugin from 'postgraphile-plugin-connection-filter'; -import PgPostgis from '@pyramation/postgis'; -import PgManyToMany from '@graphile-contrib/pg-many-to-many'; - -// @ts-ignore -import PgSimpleInflector from 'graphile-simple-inflector'; -// @ts-ignore -import PgMetaschema from 'graphile-meta-schema'; -// @ts-ignore -import FulltextFilterPlugin from '@pyramation/postgraphile-plugin-fulltext-filter'; -// @ts-ignore -import PostGraphileUploadFieldPlugin from 'postgraphile-derived-upload-field'; -// @ts-ignore -import { LangPlugin, additionalGraphQLContextFromRequest as langAdditional } from 'graphile-i18n'; -// @ts-ignore -import PgPostgisFilter from 'postgraphile-plugin-connection-filter-postgis'; -// @ts-ignore -import PgSearch from 'graphile-search-plugin'; - -import dotenv from 'dotenv'; - -import type { Request, Response } from 'express'; - -const additionalGraphQLContextFromRequest = ( - req: Request, - res: Response - ) => ({ - ...langAdditional(req, res), - req, - res, - env: process.env - }); - -dotenv.config(); - -// Inline resolver for uploads -const resolveUpload = async (input: any) => { - // Implement your actual file handling logic here. - return { success: true, input }; -}; - -const app = express(); -const PORT = process.env.PORT || 5000; -const DATABASE_URL = process.env.DATABASE_URL || 'postgres://postgres:password@localhost:5432/dashboard'; - -const schema = ['dashboard_public']; -const simpleInflection = true; -const oppositeBaseNames = false; -const postgis = true; - -const plugins: Plugin[] = [ - ConnectionFilterPlugin, - FulltextFilterPlugin, - PostGraphileUploadFieldPlugin, - PgMetaschema, - PgManyToMany, - PgSearch -]; - -const options: any = { - appendPlugins: plugins, - skipPlugins: [NodePlugin], - dynamicJson: true, - disableGraphiql: false, - enhanceGraphiql: true, - enableQueryBatching: true, - graphiql: true, - watch: false, - schema, - legacyRelations: 'omit', - showErrorStack: false, - extendedErrors: false, - disableQueryLog: false, - includeExtensionResources: true, - setofFunctionsContainNulls: false, - pgSimplifyOppositeBaseNames: oppositeBaseNames, - connectionFilterComputedColumns: false, - uploadFieldDefinitions: [ - { - name: 'upload', - namespaceName: 'public', - type: 'JSON', - resolve: resolveUpload - }, - { - name: 'attachment', - namespaceName: 'public', - type: 'String', - resolve: resolveUpload - }, - { - name: 'image', - namespaceName: 'public', - type: 'JSON', - resolve: resolveUpload - }, - { - tag: 'upload', - resolve: resolveUpload - } - ], - retryOnInitFail: async (error: Error) => { - return false; - }, - additionalGraphQLContextFromRequest - }; - -if (postgis) { - plugins.push(PgPostgis, PgPostgisFilter); -} - -if (simpleInflection) { - plugins.push(PgSimpleInflector); -} - -plugins.push(LangPlugin); - -app.use( - postgraphile(DATABASE_URL, schema, { - graphiql: true, - enhanceGraphiql: true - }) -); - -app.listen(PORT, () => { - console.log(`šŸš€ Server ready at http://localhost:${PORT}/graphiql`); -}); - -console.log(process.env.PGUSER); -console.log(process.env.PGHOST); -console.log(process.env.PGPORT); -console.log(process.env.PGPASSORD); -console.log(process.env.DATABASE_URL); \ No newline at end of file diff --git a/packages/stream-to-s3/README.md b/packages/stream-to-s3/README.md deleted file mode 100644 index 54c15f4eee..0000000000 --- a/packages/stream-to-s3/README.md +++ /dev/null @@ -1,55 +0,0 @@ -# stream-to-s3 - -```sh -npm install @launchql/stream-to-s3 -``` - -Stream uploads to s3 - -```js -const readStream = createReadStream(filename); -const results = await asyncUpload({ - readStream, - filename, - bucket, - key -}); -``` - -and get detailed payload results - -```js -{ upload: -{ ETag: '"952fd44d14cee87882239b707231609d"', - Location: 'http://localhost:9000/launchql/db1/assets/.gitignore', - key: 'db1/assets/.gitignore', - Key: 'db1/assets/.gitignore', - Bucket: 'launchql' }, -magic: { type: 'text/plain', charset: 'us-ascii' }, -contentType: 'text/plain', -contents: -{ uuid: '278aee01-1404-5725-8f0e-7044c9c16397', - sha: '7d65523f2a5afb69d76824dd1dfa62a34faa3197', - etag: '952fd44d14cee87882239b707231609d' } } -``` - -## variables - -### production - -`AWS_REGION` defaults `us-east-1` -`AWS_SECRET_KEY` -`AWS_ACCESS_KEY` - -### testing - -`BUCKET_NAME`=yourbucket - -### using minio - -values: - -`MINIO_ENDPOINT`=http://localhost:9000 -`AWS_ACCESS_KEY`=minio-access -`AWS_SECRET_KEY`=minio-secret - diff --git a/packages/stream-to-s3/__tests__/__snapshots__/stream.test.ts.snap b/packages/stream-to-s3/__tests__/__snapshots__/stream.test.ts.snap deleted file mode 100644 index 66dfc54121..0000000000 --- a/packages/stream-to-s3/__tests__/__snapshots__/stream.test.ts.snap +++ /dev/null @@ -1,994 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`uploads works 1`] = ` -{ - ".gitignore": { - "contentType": "text/plain", - "contents": { - "etag": "952fd44d14cee87882239b707231609d", - "sha": "7d65523f2a5afb69d76824dd1dfa62a34faa3197", - "uuid": "278aee01-1404-5725-8f0e-7044c9c16397", - }, - "magic": { - "charset": "us-ascii", - "type": "text/plain", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""952fd44d14cee87882239b707231609d"", - "Key": "db1/assets/.gitignore", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/.gitignore", - "key": "db1/assets/.gitignore", - }, - }, - ".travis.yml": { - "contentType": "text/yaml", - "contents": { - "etag": "eed1da4a1867d8af5f529e8668d23fd0", - "sha": "d92d5fb6cf5a031d7036bed36a8447114a23ad02", - "uuid": "f3dc1ee0-18fc-5f9e-af17-ff1ed4698f8c", - }, - "magic": { - "charset": "us-ascii", - "type": "text/plain", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""eed1da4a1867d8af5f529e8668d23fd0"", - "Key": "db1/assets/.travis.yml", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/.travis.yml", - "key": "db1/assets/.travis.yml", - }, - }, - "LICENSE": { - "contentType": "text/plain", - "contents": { - "etag": "17c231f7b93a4b3c5bd4c2c18effb21e", - "sha": "aab4092fc03df815831fada0eefbb36d8e745eb7", - "uuid": "bcf3bf1d-249b-571f-b141-2402736dbd80", - }, - "magic": { - "charset": "us-ascii", - "type": "text/plain", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""17c231f7b93a4b3c5bd4c2c18effb21e"", - "Key": "db1/assets/LICENSE", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/LICENSE", - "key": "db1/assets/LICENSE", - }, - }, - "ai.ai": { - "contentType": "application/pdf", - "contents": { - "etag": "648644bf1b5fcadf5c0a0ac67577b64f", - "sha": "5ff33d36055240efecf6307aff015efacb7de5d8", - "uuid": "32334d6b-424a-58a9-99aa-7037c1aa2f10", - }, - "magic": { - "charset": "iso-8859-1", - "type": "application/pdf", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""648644bf1b5fcadf5c0a0ac67577b64f"", - "Key": "db1/assets/ai.ai", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/ai.ai", - "key": "db1/assets/ai.ai", - }, - }, - "apng.png": { - "contentType": "image/png", - "contents": { - "etag": "425e622d348abc7ea349245e7e8738c2", - "sha": "7bd00aabcf434ec8648276fa443775fa54b7c8e5", - "uuid": "53f6f4a0-7140-5c9b-afa4-b417e51464a5", - }, - "magic": { - "charset": "binary", - "type": "image/png", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""425e622d348abc7ea349245e7e8738c2"", - "Key": "db1/assets/apng.png", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/apng.png", - "key": "db1/assets/apng.png", - }, - }, - "bmp.bmp": { - "contentType": "image/x-ms-bmp", - "contents": { - "etag": "a986c2384e5e2d4ee0d5133f3c27d62d", - "sha": "f45a12e5dcae2710a4e6c841c481363caea648a2", - "uuid": "20f0ad46-a93c-538b-a5d1-c107f98932ac", - }, - "magic": { - "charset": "binary", - "type": "image/x-ms-bmp", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""a986c2384e5e2d4ee0d5133f3c27d62d"", - "Key": "db1/assets/bmp.bmp", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/bmp.bmp", - "key": "db1/assets/bmp.bmp", - }, - }, - "build.js": { - "contentType": "application/javascript", - "contents": { - "etag": "01e41e2830bae8fcc21129ce6095d9be", - "sha": "eb7d3181cd0b6105dade03af525dc1b3fe2a7c0c", - "uuid": "31e59e9a-30ba-510d-afb3-3dc78a7b1b78", - }, - "magic": { - "charset": "us-ascii", - "type": "text/plain", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""01e41e2830bae8fcc21129ce6095d9be"", - "Key": "db1/assets/build.js", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/build.js", - "key": "db1/assets/build.js", - }, - }, - "build.js.map": { - "contentType": "application/json", - "contents": { - "etag": "c8cbb12bb4ef453eceac8c3fb889d082", - "sha": "0d5d3124cfc4fa46a26628b1cb3790ca606503aa", - "uuid": "b929c3e1-004b-59e0-af46-801e04b6ca96", - }, - "magic": { - "charset": "us-ascii", - "type": "text/plain", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""c8cbb12bb4ef453eceac8c3fb889d082"", - "Key": "db1/assets/build.js.map", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/build.js.map", - "key": "db1/assets/build.js.map", - }, - }, - "css.css": { - "contentType": "text/css", - "contents": { - "etag": "216e807dd0b784846799cb18051f250d", - "sha": "e01ef3dfa09c24f2692e0b8b9d57b2e25eeb4f48", - "uuid": "f68dd34c-37a6-53a1-b556-369c18963897", - }, - "magic": { - "charset": "us-ascii", - "type": "text/plain", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""216e807dd0b784846799cb18051f250d"", - "Key": "db1/assets/css.css", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/css.css", - "key": "db1/assets/css.css", - }, - }, - "csv.csv": { - "contentType": "text/csv", - "contents": { - "etag": "fedefb601304bf36f8c5f076191e7d44", - "sha": "3d13c82bf6132b33cefab537bc0e20608339fe33", - "uuid": "d466f82f-b2b4-53a5-92a3-c5324794a2f4", - }, - "magic": { - "charset": "us-ascii", - "type": "text/plain", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""fedefb601304bf36f8c5f076191e7d44"", - "Key": "db1/assets/csv.csv", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/csv.csv", - "key": "db1/assets/csv.csv", - }, - }, - "docs.js": { - "contentType": "application/javascript", - "contents": { - "etag": "fc829c7b7378701e2e5f835ab968bba1", - "sha": "0b01be0b43e824c875a5281b5e9c7602b76e2030", - "uuid": "5f416069-1240-5fe3-b978-f39ca5fc493e", - }, - "magic": { - "charset": "us-ascii", - "type": "text/plain", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""fc829c7b7378701e2e5f835ab968bba1"", - "Key": "db1/assets/docs.js", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/docs.js", - "key": "db1/assets/docs.js", - }, - }, - "docs.jsx": { - "contentType": "text/jsx", - "contents": { - "etag": "fc829c7b7378701e2e5f835ab968bba1", - "sha": "0b01be0b43e824c875a5281b5e9c7602b76e2030", - "uuid": "5f416069-1240-5fe3-b978-f39ca5fc493e", - }, - "magic": { - "charset": "us-ascii", - "type": "text/plain", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""fc829c7b7378701e2e5f835ab968bba1"", - "Key": "db1/assets/docs.jsx", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/docs.jsx", - "key": "db1/assets/docs.jsx", - }, - }, - "docx.docx": { - "contentType": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - "contents": { - "etag": "352c6f638cc9ff63590f818eeea6707d", - "sha": "c2766fc50294aaa0967aaa3c1010edba56ac46d3", - "uuid": "c87b1ebc-a545-581f-b4c8-733a844ebc09", - }, - "magic": { - "charset": "binary", - "type": "application/zip", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""352c6f638cc9ff63590f818eeea6707d"", - "Key": "db1/assets/docx.docx", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/docx.docx", - "key": "db1/assets/docx.docx", - }, - }, - "dwg.dwg": { - "contentType": "image/vnd.dwg", - "contents": { - "etag": "2feedee43f2d4f6a7a6b467d6faf27fa", - "sha": "1507590977dbbc08c8dd72d7ccc14552efa6e4b2", - "uuid": "8c7c2f87-7d31-5d9a-a06f-f15a34d6bb3b", - }, - "magic": { - "charset": "binary", - "type": "image/vnd.dwg", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""2feedee43f2d4f6a7a6b467d6faf27fa"", - "Key": "db1/assets/dwg.dwg", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/dwg.dwg", - "key": "db1/assets/dwg.dwg", - }, - }, - "dxf.dxf": { - "contentType": "image/vnd.dxf", - "contents": { - "etag": "422ffda43f6f6f17a9c0ef5ed62ed9bd", - "sha": "583222ce6b89fa92d2a1aa32ad776d36408e06cc", - "uuid": "a277c9ad-28f7-5076-a08a-07be148c86bd", - }, - "magic": { - "charset": "us-ascii", - "type": "text/plain", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""422ffda43f6f6f17a9c0ef5ed62ed9bd"", - "Key": "db1/assets/dxf.dxf", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/dxf.dxf", - "key": "db1/assets/dxf.dxf", - }, - }, - "emf.emf": { - "contentType": "image/emf", - "contents": { - "etag": "bab54b66b80237d5bb800b1aef9ebe9b", - "sha": "cb634df027769806647358b95b4406c8d0fcb213", - "uuid": "03aa38bc-adf1-528d-a8a8-e2b47ae99370", - }, - "magic": { - "charset": "binary", - "type": "application/octet-stream", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""bab54b66b80237d5bb800b1aef9ebe9b"", - "Key": "db1/assets/emf.emf", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/emf.emf", - "key": "db1/assets/emf.emf", - }, - }, - "epub.epub": { - "contentType": "application/epub+zip", - "contents": { - "etag": "8dc60cd4a41395178a3d80bfa2974c41", - "sha": "2cac547509eb495f781588496c42b13c46d13f7e", - "uuid": "e7c45793-44c9-59c9-8ca9-42e2a838f257", - }, - "magic": { - "charset": "binary", - "type": "application/epub+zip", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""8dc60cd4a41395178a3d80bfa2974c41"", - "Key": "db1/assets/epub.epub", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/epub.epub", - "key": "db1/assets/epub.epub", - }, - }, - "font.otf": { - "contentType": "application/x-font-opentype", - "contents": { - "etag": "945b9e0e38dcfd294b4b26150ccade3c", - "sha": "f6b7979678bdf06d2fdca5093cec308e84c3c2c5", - "uuid": "e0d4ed22-7c80-52aa-b042-241ebafa1e3e", - }, - "magic": { - "charset": "binary", - "type": "application/vnd.ms-opentype", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""945b9e0e38dcfd294b4b26150ccade3c"", - "Key": "db1/assets/font.otf", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/font.otf", - "key": "db1/assets/font.otf", - }, - }, - "font.ttf": { - "contentType": "application/x-font-ttf", - "contents": { - "etag": "3736a705526e0ecd366f70a59f704ec6", - "sha": "f7df25a76f7d79a7682eb9949fff7c2e173cf59f", - "uuid": "62ee870d-67a9-5e1b-89a2-d40bd65e8d6f", - }, - "magic": { - "charset": "binary", - "type": "application/font-sfnt", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""3736a705526e0ecd366f70a59f704ec6"", - "Key": "db1/assets/font.ttf", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/font.ttf", - "key": "db1/assets/font.ttf", - }, - }, - "font.woff": { - "contentType": "application/font-woff", - "contents": { - "etag": "f2af1f64bc0a0e5cfe4b5fee322e4157", - "sha": "fb9679572962f410b5f56d159573debcd5c4eabd", - "uuid": "4014a9c5-832e-58a2-a2ef-15fb5eca009e", - }, - "magic": { - "charset": "binary", - "type": "application/octet-stream", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""f2af1f64bc0a0e5cfe4b5fee322e4157"", - "Key": "db1/assets/font.woff", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/font.woff", - "key": "db1/assets/font.woff", - }, - }, - "font.woff2": { - "contentType": "application/font-woff2", - "contents": { - "etag": "f2af1f64bc0a0e5cfe4b5fee322e4157", - "sha": "fb9679572962f410b5f56d159573debcd5c4eabd", - "uuid": "4014a9c5-832e-58a2-a2ef-15fb5eca009e", - }, - "magic": { - "charset": "binary", - "type": "application/octet-stream", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""f2af1f64bc0a0e5cfe4b5fee322e4157"", - "Key": "db1/assets/font.woff2", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/font.woff2", - "key": "db1/assets/font.woff2", - }, - }, - "gif-w-alpha.gif": { - "contentType": "image/gif", - "contents": { - "etag": "70b8ba9aebd02707cdcc43b6f9061fb0", - "sha": "068a05d7fc36947e1e57905a3244f547cbd04ecf", - "uuid": "9fcbccec-b5b4-5112-bb49-4da5d6535212", - }, - "magic": { - "charset": "binary", - "type": "image/gif", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""70b8ba9aebd02707cdcc43b6f9061fb0"", - "Key": "db1/assets/gif-w-alpha.gif", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/gif-w-alpha.gif", - "key": "db1/assets/gif-w-alpha.gif", - }, - }, - "gif.gif": { - "contentType": "image/gif", - "contents": { - "etag": "b411ab31d73df4637d9ddcd99c03b7e9", - "sha": "f56f613f1fa92d9ba8ec8c33feb1fbaaa3f09c84", - "uuid": "a054dca6-4cac-578b-ba1d-6cc3d6afd10a", - }, - "magic": { - "charset": "binary", - "type": "image/gif", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""b411ab31d73df4637d9ddcd99c03b7e9"", - "Key": "db1/assets/gif.gif", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/gif.gif", - "key": "db1/assets/gif.gif", - }, - }, - "jpg.jpg": { - "contentType": "image/jpeg", - "contents": { - "etag": "0d55576e9c5c346dc0ed4adedfaf0619", - "sha": "c5cb787fc921ec5f9ce65b56024099f349f73d31", - "uuid": "e0bfb599-dba9-58f1-96ae-e15f2b12d764", - }, - "magic": { - "charset": "binary", - "type": "image/jpeg", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""0d55576e9c5c346dc0ed4adedfaf0619"", - "Key": "db1/assets/jpg.jpg", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/jpg.jpg", - "key": "db1/assets/jpg.jpg", - }, - }, - "json.json": { - "contentType": "application/json", - "contents": { - "etag": "64dcb5b3b291074d02c80f600fda3f6e", - "sha": "e6c7c64d292a414941d239c57117b36f24c9f829", - "uuid": "78160718-8dfa-5cb4-bb50-e479c8c58383", - }, - "magic": { - "charset": "us-ascii", - "type": "text/plain", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""64dcb5b3b291074d02c80f600fda3f6e"", - "Key": "db1/assets/json.json", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/json.json", - "key": "db1/assets/json.json", - }, - }, - "less.less": { - "contentType": "text/less", - "contents": { - "etag": "ec5315f0f5af9b39421b6d06512ceb64", - "sha": "7e271b5244414ebe8b0ccc47ad74d1edd58f2b65", - "uuid": "be9337b6-dda5-5ca1-a0ad-4c75aa531cea", - }, - "magic": { - "charset": "us-ascii", - "type": "text/plain", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""ec5315f0f5af9b39421b6d06512ceb64"", - "Key": "db1/assets/less.less", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/less.less", - "key": "db1/assets/less.less", - }, - }, - "lock.lock": { - "contentType": "text/plain", - "contents": { - "etag": "78f1c5a17d4a71257b95061e2432cadc", - "sha": "4a70e2938dcb22b766977ac3dd49940c803ccdd4", - "uuid": "66278343-543b-527c-95b0-08ed0452c0b7", - }, - "magic": { - "charset": "us-ascii", - "type": "text/plain", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""78f1c5a17d4a71257b95061e2432cadc"", - "Key": "db1/assets/lock.lock", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/lock.lock", - "key": "db1/assets/lock.lock", - }, - }, - "md.md": { - "contentType": "text/markdown", - "contents": { - "etag": "2524b9c1caad97610a5149886a7e793f", - "sha": "62c8208c3f148ada7e1a1a3b8972ac5898dfbf89", - "uuid": "89250a3c-a794-5d26-8ee4-d692dff9fc35", - }, - "magic": { - "charset": "us-ascii", - "type": "text/plain", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""2524b9c1caad97610a5149886a7e793f"", - "Key": "db1/assets/md.md", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/md.md", - "key": "db1/assets/md.md", - }, - }, - "mp4.mp4": { - "contentType": "video/mp4", - "contents": { - "etag": "0c147ea51be2c7425677a04e800a4342", - "sha": "3e1eb4cde2d9c61c399899a6034c20b15214caa1", - "uuid": "57826c39-fea3-5588-b4f2-db1958025495", - }, - "magic": { - "charset": "binary", - "type": "video/mp4", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""0c147ea51be2c7425677a04e800a4342"", - "Key": "db1/assets/mp4.mp4", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/mp4.mp4", - "key": "db1/assets/mp4.mp4", - }, - }, - "ods.ods": { - "contentType": "application/vnd.oasis.opendocument.spreadsheet", - "contents": { - "etag": "28f9c41946c806e049bd5e7d1ad07fde", - "sha": "c6abbd7353da3b158965a9a7327a4f57f339f639", - "uuid": "efadabfe-a65f-5235-a6da-353e750c6510", - }, - "magic": { - "charset": "binary", - "type": "application/vnd.oasis.opendocument.spreadsheet", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""28f9c41946c806e049bd5e7d1ad07fde"", - "Key": "db1/assets/ods.ods", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/ods.ods", - "key": "db1/assets/ods.ods", - }, - }, - "odt.odt": { - "contentType": "application/vnd.oasis.opendocument.text", - "contents": { - "etag": "24cac2f0a673e3af35095a109dc6a9fa", - "sha": "ae7b8a3011c7db10633a4ee85cbd61c3ab7a5f27", - "uuid": "f9ac160d-597d-5252-a948-de54365160ed", - }, - "magic": { - "charset": "binary", - "type": "application/vnd.oasis.opendocument.text", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""24cac2f0a673e3af35095a109dc6a9fa"", - "Key": "db1/assets/odt.odt", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/odt.odt", - "key": "db1/assets/odt.odt", - }, - }, - "pct.pct": { - "contentType": "image/x-pict", - "contents": { - "etag": "630620c4f931bd94573cafe460ad4af6", - "sha": "5115a779068096d58af256a49c1935c2b73dee6e", - "uuid": "9a9d7dd2-a146-5972-b899-cb4fa933e4df", - }, - "magic": { - "charset": "binary", - "type": "application/octet-stream", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""630620c4f931bd94573cafe460ad4af6"", - "Key": "db1/assets/pct.pct", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/pct.pct", - "key": "db1/assets/pct.pct", - }, - }, - "pdf.pdf": { - "contentType": "application/pdf", - "contents": { - "etag": "b9aa628349df6cc1c99919fe4a12216c", - "sha": "072c23927a91b19e96b72fc14e99cb66958103b7", - "uuid": "e6f5ba06-d53f-5f5b-b606-6476fe7ea639", - }, - "magic": { - "charset": "binary", - "type": "application/pdf", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""b9aa628349df6cc1c99919fe4a12216c"", - "Key": "db1/assets/pdf.pdf", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/pdf.pdf", - "key": "db1/assets/pdf.pdf", - }, - }, - "png-w-alpha.png": { - "contentType": "image/png", - "contents": { - "etag": "e81f3ad8e657fe1df43548c4a20f7b76", - "sha": "17f794e8c92370f208f0f9892c88e497c24b4b1d", - "uuid": "5c05dc14-8036-5edc-9f26-82f9758d77d4", - }, - "magic": { - "charset": "binary", - "type": "image/png", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""e81f3ad8e657fe1df43548c4a20f7b76"", - "Key": "db1/assets/png-w-alpha.png", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/png-w-alpha.png", - "key": "db1/assets/png-w-alpha.png", - }, - }, - "psd.psd": { - "contentType": "image/vnd.adobe.photoshop", - "contents": { - "etag": "2625e3d597d2bc0d0223169c3b6a7c58", - "sha": "b2c7f8cf7fa2ea7763b0d589d756a01ef1b7ca89", - "uuid": "e66ef9d4-f9e0-54d1-9d6b-90b6f3da6b03", - }, - "magic": { - "charset": "binary", - "type": "image/vnd.adobe.photoshop", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""2625e3d597d2bc0d0223169c3b6a7c58"", - "Key": "db1/assets/psd.psd", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/psd.psd", - "key": "db1/assets/psd.psd", - }, - }, - "scss.scss": { - "contentType": "text/x-scss", - "contents": { - "etag": "cd8759ad9f9530b6f852675d1db3235a", - "sha": "8a5a7fe68e341aea02b05d0d4309b643fca6f73e", - "uuid": "d56076b2-b52a-58a9-bf15-20c6791d8d23", - }, - "magic": { - "charset": "us-ascii", - "type": "text/plain", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""cd8759ad9f9530b6f852675d1db3235a"", - "Key": "db1/assets/scss.scss", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/scss.scss", - "key": "db1/assets/scss.scss", - }, - }, - "shellscript": { - "contentType": "text/x-shellscript", - "contents": { - "etag": "9e73a9fd45d381175f3791bc4f33ec11", - "sha": "8301dee1416543a8ed2756efd4eeeaf83690eac4", - "uuid": "6baad900-76df-55cb-8d32-a73ec827dc8a", - }, - "magic": { - "charset": "us-ascii", - "type": "text/x-shellscript", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""9e73a9fd45d381175f3791bc4f33ec11"", - "Key": "db1/assets/shellscript", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/shellscript", - "key": "db1/assets/shellscript", - }, - }, - "sql.sql": { - "contentType": "application/x-sql", - "contents": { - "etag": "acb01cdf33de89cc35d84d748cf05bc8", - "sha": "c0ebf3f51fa0a296fcf8befe4a5f58d0fe01a199", - "uuid": "3508037e-f358-5774-9cdf-3989d2f769a0", - }, - "magic": { - "charset": "utf-8", - "type": "text/plain", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""acb01cdf33de89cc35d84d748cf05bc8"", - "Key": "db1/assets/sql.sql", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/sql.sql", - "key": "db1/assets/sql.sql", - }, - }, - "svg-with-alpha-and-text.svg": { - "contentType": "image/svg+xml", - "contents": { - "etag": "6036843b30c15b80898bc23174da05f7", - "sha": "137fd8a16a01f3c0de2d5ffd786ce7978de997d3", - "uuid": "1b901d29-7d0a-5eac-ae79-0d794e109907", - }, - "magic": { - "charset": "us-ascii", - "type": "image/svg", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""6036843b30c15b80898bc23174da05f7"", - "Key": "db1/assets/svg-with-alpha-and-text.svg", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/svg-with-alpha-and-text.svg", - "key": "db1/assets/svg-with-alpha-and-text.svg", - }, - }, - "svg.svg": { - "contentType": "image/svg+xml", - "contents": { - "etag": "a82efeaff853f1cf8cf85c2d526c0d9e", - "sha": "e471602538abcec3c8ef4b8dad0e3d9b545e8e56", - "uuid": "296837f5-d3cf-55f4-b3ba-3e388fb0a037", - }, - "magic": { - "charset": "us-ascii", - "type": "image/svg", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""a82efeaff853f1cf8cf85c2d526c0d9e"", - "Key": "db1/assets/svg.svg", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/svg.svg", - "key": "db1/assets/svg.svg", - }, - }, - "svgo.yaml": { - "contentType": "text/yaml", - "contents": { - "etag": "08541d86097e1750cf2d980a9e405ad8", - "sha": "955b0053a46d5fa4b202c275cf58101da506fd13", - "uuid": "7a35a57d-b2ab-57db-bb15-a7a0dfd93902", - }, - "magic": { - "charset": "us-ascii", - "type": "text/plain", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""08541d86097e1750cf2d980a9e405ad8"", - "Key": "db1/assets/svgo.yaml", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/svgo.yaml", - "key": "db1/assets/svgo.yaml", - }, - }, - "swf.swf": { - "contentType": "application/x-shockwave-flash", - "contents": { - "etag": "008b732b6223004db383b3d605b0fe70", - "sha": "3aabad40e5e25f1b10a38674b53a99e2546b391b", - "uuid": "cf63cc7c-406b-542c-a923-6d0584ecf2be", - }, - "magic": { - "charset": "binary", - "type": "application/x-shockwave-flash", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""008b732b6223004db383b3d605b0fe70"", - "Key": "db1/assets/swf.swf", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/swf.swf", - "key": "db1/assets/swf.swf", - }, - }, - "tga.tga": { - "contentType": "image/x-tga", - "contents": { - "etag": "4b4384288a9b5b1a3b544decc957dbe5", - "sha": "3d6584fd6b585a5fc0d2738b680005f2d2b91700", - "uuid": "2ab6572b-c97d-582c-b3dd-2b76891de092", - }, - "magic": { - "charset": "binary", - "type": "image/x-tga", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""4b4384288a9b5b1a3b544decc957dbe5"", - "Key": "db1/assets/tga.tga", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/tga.tga", - "key": "db1/assets/tga.tga", - }, - }, - "tif.tif": { - "contentType": "image/tiff", - "contents": { - "etag": "4171810ca9e1293197110c23f324161f", - "sha": "6e6237ff0e65cbbc3ce372079093edda2a1f950b", - "uuid": "aa3916b5-a7c2-5c10-9c76-aef5f4442901", - }, - "magic": { - "charset": "binary", - "type": "image/tiff", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""4171810ca9e1293197110c23f324161f"", - "Key": "db1/assets/tif.tif", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/tif.tif", - "key": "db1/assets/tif.tif", - }, - }, - "todo-0.1.0.tgz": { - "contentType": "application/x-gzip", - "contents": { - "etag": "b3af984d63447e8173fb565ae3874754", - "sha": "175b3bf38c377bb282bfe28978a3d5641dc61a5f", - "uuid": "cd15f500-6a08-5870-b103-718e0c1ee11c", - }, - "magic": { - "charset": "binary", - "type": "application/x-gzip", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""b3af984d63447e8173fb565ae3874754"", - "Key": "db1/assets/todo-0.1.0.tgz", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/todo-0.1.0.tgz", - "key": "db1/assets/todo-0.1.0.tgz", - }, - }, - "tsv.tsv": { - "contentType": "text/tab-separated-values", - "contents": { - "etag": "2ddf1d91d0e148ba36b1d711d3f15518", - "sha": "bcfb990493b50b9d568274cc79a13eae02a6ac0e", - "uuid": "aa2f950a-cf7a-537a-a800-961b3870319c", - }, - "magic": { - "charset": "us-ascii", - "type": "text/plain", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""2ddf1d91d0e148ba36b1d711d3f15518"", - "Key": "db1/assets/tsv.tsv", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/tsv.tsv", - "key": "db1/assets/tsv.tsv", - }, - }, - "txt.txt": { - "contentType": "text/plain", - "contents": { - "etag": "b9d78e5636ec484481763d0213cc39cb", - "sha": "b241078102f68969da7c8e442229a390fecd3473", - "uuid": "eec733bd-dfee-5647-ae8f-fffaacd7a54a", - }, - "magic": { - "charset": "us-ascii", - "type": "text/plain", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""b9d78e5636ec484481763d0213cc39cb"", - "Key": "db1/assets/txt.txt", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/txt.txt", - "key": "db1/assets/txt.txt", - }, - }, - "typescript.ts": { - "contentType": "text/x-typescript", - "contents": { - "etag": "d318a3bfecfad0286a38ab05f901302e", - "sha": "e0c625144d4b1bbfcbf599dad444f78390f2d094", - "uuid": "4cb57844-fed2-5b51-b56b-c957a3f1f046", - }, - "magic": { - "charset": "us-ascii", - "type": "text/plain", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""d318a3bfecfad0286a38ab05f901302e"", - "Key": "db1/assets/typescript.ts", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/typescript.ts", - "key": "db1/assets/typescript.ts", - }, - }, - "typescript.tsx": { - "contentType": "text/x-typescript", - "contents": { - "etag": "d318a3bfecfad0286a38ab05f901302e", - "sha": "e0c625144d4b1bbfcbf599dad444f78390f2d094", - "uuid": "4cb57844-fed2-5b51-b56b-c957a3f1f046", - }, - "magic": { - "charset": "us-ascii", - "type": "text/plain", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""d318a3bfecfad0286a38ab05f901302e"", - "Key": "db1/assets/typescript.tsx", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/typescript.tsx", - "key": "db1/assets/typescript.tsx", - }, - }, - "wmf.wmf": { - "contentType": "image/wmf", - "contents": { - "etag": "adc25f2984aa0dbaaa6a54fb486ea3e3", - "sha": "c674de1fedd2c5f0d23c77b5275bac2aca53d687", - "uuid": "9c08564f-148d-573e-869f-080cd49a95eb", - }, - "magic": { - "charset": "binary", - "type": "application/octet-stream", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""adc25f2984aa0dbaaa6a54fb486ea3e3"", - "Key": "db1/assets/wmf.wmf", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/wmf.wmf", - "key": "db1/assets/wmf.wmf", - }, - }, - "xlsx.xlsx": { - "contentType": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - "contents": { - "etag": "11ee509d7e7015cbb693787206345c67", - "sha": "e8f3cfd46b660904f6c1040513cf677366997c9e", - "uuid": "f6a1afd3-110a-530e-8279-e4e9291726ae", - }, - "magic": { - "charset": "binary", - "type": "application/zip", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""11ee509d7e7015cbb693787206345c67"", - "Key": "db1/assets/xlsx.xlsx", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/xlsx.xlsx", - "key": "db1/assets/xlsx.xlsx", - }, - }, - "zip.zip": { - "contentType": "application/zip", - "contents": { - "etag": "9dfdfd1ae3cb267b12ef9d63bffa2295", - "sha": "249f27795f59c3c5f4c9a60298c271f261454250", - "uuid": "b23685b4-04f1-5244-bad3-be5fa6a5af7f", - }, - "magic": { - "charset": "binary", - "type": "application/zip", - }, - "upload": { - "Bucket": "test-bucket", - "ETag": ""9dfdfd1ae3cb267b12ef9d63bffa2295"", - "Key": "db1/assets/zip.zip", - "Location": "http://minio_cdn:9000/test-bucket/db1/assets/zip.zip", - "key": "db1/assets/zip.zip", - }, - }, -} -`; diff --git a/packages/stream-to-s3/__tests__/stream.test.ts b/packages/stream-to-s3/__tests__/stream.test.ts deleted file mode 100644 index 9666a4100d..0000000000 --- a/packages/stream-to-s3/__tests__/stream.test.ts +++ /dev/null @@ -1,76 +0,0 @@ -import dotenv from 'dotenv'; -dotenv.config({ path: __dirname + '/../../../.env.minio.env' }); - -import { sync as glob } from 'glob'; -import { createReadStream } from 'fs'; -import { basename } from 'path'; -import S3 from 'aws-sdk/clients/s3'; - -import asyncUpload from '../src'; -import { cleanEnv, str, url } from 'envalid'; -import { createS3Bucket } from '@launchql/s3-utils'; - -export const testEnv = cleanEnv(process.env, { - AWS_REGION: str({ default: 'us-east-1' }), - AWS_SECRET_KEY: str({ default: 'minioadmin' }), - AWS_ACCESS_KEY: str({ default: 'minioadmin' }), - MINIO_ENDPOINT: url({ default: undefined }), - BUCKET_NAME: str({ default: 'test-bucket' }) -}); - -// Initialize S3 client -const s3Client = new S3({ - accessKeyId: testEnv.AWS_ACCESS_KEY, - secretAccessKey: testEnv.AWS_SECRET_KEY, - region: testEnv.AWS_REGION, - endpoint: testEnv.MINIO_ENDPOINT, - s3ForcePathStyle: true, - signatureVersion: 'v4' -}); - -jest.setTimeout(3000000); - -// Create bucket before tests -beforeAll(async () => { - process.env.IS_MINIO = 'true'; // Ensure MinIO behavior in createS3Bucket - const result = await createS3Bucket(s3Client, testEnv.BUCKET_NAME); - if (!result.success) throw new Error('Failed to create test S3 bucket'); -}); - -const files = [] - .concat(glob(__dirname + '/../../../__fixtures__/kitchen-sink/**')) - .concat(glob(__dirname + '/../../../__fixtures__/kitchen-sink/**/.*')) - .filter((file) => { - const key = file.split('kitchen-sink')[1]; - return key != ''; - }) - .map((f) => ({ - key: basename(f), - path: f - })); - -describe('uploads', () => { - it('works', async () => { - const res = {}; - for (const file of files) { - const key = file.key; - const readStream = createReadStream(file.path); - const results = await asyncUpload({ - readStream, - filename: file.path, - bucket: testEnv.BUCKET_NAME, - key: 'db1/assets/' + basename(file.path) - }); - // @ts-ignore - res[key] = results; - } - - Object.keys(res).map((k)=>{ - // CI/CD matching - // @ts-ignore - res[k].upload.Location = res[k].upload.Location.replace(/localhost:9000/g, 'minio_cdn:9000'); - }) - - expect(res).toMatchSnapshot(); - }); -}); diff --git a/packages/stream-to-s3/src/env.ts b/packages/stream-to-s3/src/env.ts deleted file mode 100644 index 79b8ff2fed..0000000000 --- a/packages/stream-to-s3/src/env.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { cleanEnv, str, url } from 'envalid'; - -export const env = cleanEnv(process.env, { - AWS_REGION: str({ default: 'us-east-1' }), - AWS_SECRET_KEY: str(), - AWS_ACCESS_KEY: str(), - MINIO_ENDPOINT: url({ default: undefined }) -}); diff --git a/packages/stream-to-s3/src/index.ts b/packages/stream-to-s3/src/index.ts deleted file mode 100644 index a2de84831e..0000000000 --- a/packages/stream-to-s3/src/index.ts +++ /dev/null @@ -1,137 +0,0 @@ -import stream, { PassThrough, Readable } from 'stream'; -import type S3 from 'aws-sdk/clients/s3'; -import getS3 from './s3'; -import { - streamContentType, - ContentStream -} from '@launchql/content-type-stream'; - -const s3: S3 = getS3(); - -interface UploadFromStreamParams { - key: string; - contentType: string; - bucket: string; -} - -interface AsyncUploadParams extends UploadFromStreamParams { - readStream: Readable; - magic: { charset: string }; -} - -interface AsyncUploadResult { - upload: S3.ManagedUpload.SendData; - magic: { charset: string }; - contentType: string; - contents: unknown; -} - -export function uploadFromStream({ - key, - contentType, - bucket -}: UploadFromStreamParams): PassThrough { - const pass = new stream.PassThrough(); - - const params = { - Body: pass, - Key: key, - ContentType: contentType, - Bucket: bucket - }; - - s3.upload(params, function ( - err: Error | null, - data: S3.ManagedUpload.SendData - ): void { - if (err) { - pass.emit('error', err); - } else { - pass.emit('upload', data); - } - }); - - return pass; -} - -export const asyncUpload = ({ - key, - contentType, - readStream, - magic, - bucket -}: AsyncUploadParams): Promise => { - return new Promise((resolve, reject) => { - // upload stream - let upload: S3.ManagedUpload.SendData | undefined; - - const uploadStream = uploadFromStream({ - key, - contentType, - bucket - }); - - // content stream - let contents: unknown; - const contentStream = new ContentStream(); - - const tryResolve = () => { - if (contents && upload) { - resolve({ - upload, - magic, - contentType, - contents - }); - } - }; - - contentStream - .on('contents', function (results: unknown) { - contents = results; - tryResolve(); - }) - .on('error', (error: Error) => { - reject(error); - }); - - uploadStream - .on('upload', (results: S3.ManagedUpload.SendData) => { - upload = results; - tryResolve(); - }) - .on('error', (error: Error) => { - reject(error); - }); - - readStream.pipe(contentStream); - contentStream.pipe(uploadStream); - }); -}; - -interface UploadInput { - readStream: Readable; - filename: string; - bucket: string; - key: string; -} - -export default async ({ - readStream, - filename, - bucket, - key -}: UploadInput): Promise => { - const { stream: newStream, magic, contentType } = await streamContentType({ - readStream, - filename - }); - - return await asyncUpload({ - key, - contentType, - readStream: newStream, - magic, - bucket - }); -}; diff --git a/packages/stream-to-s3/src/s3.ts b/packages/stream-to-s3/src/s3.ts deleted file mode 100644 index 9fcbc12377..0000000000 --- a/packages/stream-to-s3/src/s3.ts +++ /dev/null @@ -1,28 +0,0 @@ -import S3 from 'aws-sdk/clients/s3'; -import { env } from './env'; - -export default function getS3(): S3 { - const isMinio = Boolean(env.MINIO_ENDPOINT); - - const awsConfig: S3.ClientConfiguration = isMinio - ? { - accessKeyId: env.AWS_ACCESS_KEY, - secretAccessKey: env.AWS_SECRET_KEY, - endpoint: env.MINIO_ENDPOINT, - s3ForcePathStyle: true, - signatureVersion: 'v4', - } - : { - apiVersion: '2006-03-01', - region: env.AWS_REGION, - // Only include credentials if explicitly set ā€” this allows IAM role fallback - ...(env.AWS_ACCESS_KEY && env.AWS_SECRET_KEY - ? { - accessKeyId: env.AWS_ACCESS_KEY, - secretAccessKey: env.AWS_SECRET_KEY, - } - : {}) - }; - - return new S3(awsConfig); -} diff --git a/packages/types/README.md b/packages/types/README.md new file mode 100644 index 0000000000..63ba973bba --- /dev/null +++ b/packages/types/README.md @@ -0,0 +1 @@ +# types diff --git a/packages/stream-to-s3/jest.config.js b/packages/types/jest.config.js similarity index 100% rename from packages/stream-to-s3/jest.config.js rename to packages/types/jest.config.js diff --git a/packages/stream-to-s3/package.json b/packages/types/package.json similarity index 72% rename from packages/stream-to-s3/package.json rename to packages/types/package.json index ba966c78a1..c75674e641 100644 --- a/packages/stream-to-s3/package.json +++ b/packages/types/package.json @@ -1,8 +1,8 @@ { - "name": "@launchql/stream-to-s3", - "version": "0.0.1", - "author": "Dan Lynch ", - "description": "stream to s3", + "name": "@launchql/types", + "version": "2.0.0", + "author": "Dan Lynch ", + "description": "types", "main": "index.js", "module": "esm/index.js", "types": "index.d.ts", @@ -29,13 +29,9 @@ "test": "jest", "test:watch": "jest --watch" }, - "keywords": [], - "devDependencies": { - "@launchql/s3-utils": "0.0.1" - }, "dependencies": { - "aws-sdk": "^2.1692.0", - "envalid": "^8.0.0", - "@launchql/content-type-stream": "^0.0.1" + "deepmerge": "^4.3.1", + "postgraphile": "^4.14.1", + "graphile-build": "^4.14.1" } } \ No newline at end of file diff --git a/packages/types/src/env.ts b/packages/types/src/env.ts new file mode 100644 index 0000000000..62611c1d7f --- /dev/null +++ b/packages/types/src/env.ts @@ -0,0 +1,76 @@ +import { getMergedOptions, LaunchQLOptions } from './launchql'; + +const parseEnvNumber = (val?: string): number | undefined => { + const num = Number(val); + return !isNaN(num) ? num : undefined; +}; + +const parseEnvBoolean = (val?: string): boolean | undefined => { + if (val === undefined) return undefined; + return ['true', '1', 'yes'].includes(val.toLowerCase()); +}; + +export const getEnvOptions = (overrides: LaunchQLOptions = {}): LaunchQLOptions => { + const envOpts = getEnvVars(); + return getMergedOptions + ({ + ...envOpts, + ...overrides, + }); +}; + + +export const getEnvVars = (): LaunchQLOptions => { + const { + PORT, + SERVER_HOST, + SERVER_TRUST_PROXY, + SERVER_ORIGIN, + SERVER_STRICT_AUTH, + + PGHOST, + PGPORT, + PGUSER, + PGPASSWORD, + PGDATABASE, + + FEATURES_SIMPLE_INFLECTION, + FEATURES_OPPOSITE_BASE_NAMES, + FEATURES_POSTGIS, + + BUCKET_NAME, + AWS_REGION, + AWS_ACCESS_KEY, + AWS_SECRET_KEY, + MINIO_ENDPOINT, + } = process.env; + + return { + server: { + ...(PORT && { port: parseEnvNumber(PORT) }), + ...(SERVER_HOST && { host: SERVER_HOST }), + ...(SERVER_TRUST_PROXY && { trustProxy: parseEnvBoolean(SERVER_TRUST_PROXY) }), + ...(SERVER_ORIGIN && { origin: SERVER_ORIGIN }), + ...(SERVER_STRICT_AUTH && { strictAuth: parseEnvBoolean(SERVER_STRICT_AUTH) }), + }, + pg: { + ...(PGHOST && { host: PGHOST }), + ...(PGPORT && { port: parseEnvNumber(PGPORT) }), + ...(PGUSER && { user: PGUSER }), + ...(PGPASSWORD && { password: PGPASSWORD }), + ...(PGDATABASE && { database: PGDATABASE }), + }, + features: { + ...(FEATURES_SIMPLE_INFLECTION && { simpleInflection: parseEnvBoolean(FEATURES_SIMPLE_INFLECTION) }), + ...(FEATURES_OPPOSITE_BASE_NAMES && { oppositeBaseNames: parseEnvBoolean(FEATURES_OPPOSITE_BASE_NAMES) }), + ...(FEATURES_POSTGIS && { postgis: parseEnvBoolean(FEATURES_POSTGIS) }), + }, + cdn: { + ...(BUCKET_NAME && { bucketName: BUCKET_NAME }), + ...(AWS_REGION && { awsRegion: AWS_REGION }), + ...(AWS_ACCESS_KEY && { awsAccessKey: AWS_ACCESS_KEY }), + ...(AWS_SECRET_KEY && { awsSecretKey: AWS_SECRET_KEY }), + ...(MINIO_ENDPOINT && { minioEndpoint: MINIO_ENDPOINT }), + } + }; +}; diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts new file mode 100644 index 0000000000..8bdf5b8724 --- /dev/null +++ b/packages/types/src/index.ts @@ -0,0 +1,2 @@ +export * from './launchql'; +export * from './env'; \ No newline at end of file diff --git a/packages/types/src/launchql.ts b/packages/types/src/launchql.ts new file mode 100644 index 0000000000..f1c61369ac --- /dev/null +++ b/packages/types/src/launchql.ts @@ -0,0 +1,129 @@ +import deepmerge from "deepmerge"; +import { PostGraphileOptions } from 'postgraphile'; +import type { Plugin } from 'graphile-build'; + +declare module 'express-serve-static-core' { + interface Request { + apiInfo: { + data: { + api: { + dbname: string; + anonRole: string; + roleName: string; + schemaNames: { + nodes: { schemaName: string }[]; + }; + schemaNamesFromExt: { + nodes: { schemaName: string }[]; + }; + apiModules: { + nodes: { + name: string; + data?: any; + }[]; + }; + rlsModule?: { + authenticate?: string; + authenticateStrict?: string; + privateSchema: { + schemaName: string; + }; + }; + }; + }; + }; + svc_key: string; + clientIp?: string; + databaseId?: string; + token?: { + id: string; + user_id: string; + [key: string]: any; + }; + } +} + + +export interface PostgresOptions { + host?: string; + port?: number; + user?: string; + password?: string; + database?: string; +} + +export interface LaunchQLOptions { + pg?: PostgresOptions; + graphile?: { + isPublic?: boolean; + schema?: string | string[]; + metaSchemas?: string[]; + appendPlugins?: Plugin[]; + graphileBuildOptions?: PostGraphileOptions['graphileBuildOptions']; + overrideSettings?: Partial; + }; + server?: { + host?: string; + port?: number; + trustProxy?: boolean; + origin?: string; + strictAuth?: boolean; + }; + features?: { + simpleInflection?: boolean; + oppositeBaseNames?: boolean; + postgis?: boolean; + }; + cdn?: { + bucketName?: string; + awsRegion?: string; + awsAccessKey?: string; + awsSecretKey?: string; + minioEndpoint?: string; + }; +} + + +export const launchqlDefaults: LaunchQLOptions = { + pg: { + host: 'localhost', + port: 5432, + user: 'postgres', + password: 'password', + database: 'postgres', + }, + graphile: { + isPublic: true, + schema: ['public'], + // TODO how to handle metaSchemas...? + metaSchemas: ['collections_public', 'meta_public'], + appendPlugins: [], + overrideSettings: {}, + graphileBuildOptions: {}, + }, + server: { + host: 'localhost', + port: 3000, + trustProxy: false, + strictAuth: false, + }, + features: { + simpleInflection: true, + oppositeBaseNames: true, + postgis: true, + }, + cdn: { + bucketName: 'test-bucket', + awsRegion: 'us-east-1', + awsAccessKey: 'minioadmin', + awsSecretKey: 'minioadmin' + } +}; + + + +export const getMergedOptions = (options: LaunchQLOptions): LaunchQLOptions => { + options = deepmerge(launchqlDefaults, options ?? {}); + // if you need to sanitize... + return options; +}; diff --git a/packages/stream-to-s3/tsconfig.esm.json b/packages/types/tsconfig.esm.json similarity index 100% rename from packages/stream-to-s3/tsconfig.esm.json rename to packages/types/tsconfig.esm.json diff --git a/packages/stream-to-s3/tsconfig.json b/packages/types/tsconfig.json similarity index 100% rename from packages/stream-to-s3/tsconfig.json rename to packages/types/tsconfig.json