From 6bfd9da08c5a2ca9cb56d75866fa6b70010f8225 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Tue, 11 Jan 2022 20:43:59 +0900 Subject: [PATCH 1/7] Refactor migrations - Wrap with TRANSACTION --- backend/src/api/database-migration.ts | 279 ++++++++++++++++---------- 1 file changed, 178 insertions(+), 101 deletions(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index c823428d1..61c752302 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -1,3 +1,4 @@ +import { PoolConnection } from 'mysql2/promise'; import config from '../config'; import { DB } from '../database'; import logger from '../logger'; @@ -8,55 +9,63 @@ class DatabaseMigration { constructor() { } + /** + * Entry point + */ public async $initializeOrMigrateDatabase(): Promise { - if (!await this.$checkIfTableExists('statistics')) { - await this.$initializeDatabaseTables(); - } + logger.info("MIGRATIONS: Running migrations"); - if (await this.$checkIfTableExists('state')) { - const databaseSchemaVersion = await this.$getSchemaVersionFromDatabase(); - if (DatabaseMigration.currentVersion > databaseSchemaVersion) { - await this.$migrateTableSchemaFromVersion(databaseSchemaVersion); + // First of all, if the `state` database does not exist, create it so we can track migration version + if (!await this.$checkIfTableExists('state')) { + logger.info("MIGRATIONS: `state` table does not exist. Creating it.") + try { + await this.$createMigrationStateTable(); + } catch (e) { + logger.err("Unable to create `state` table. Aborting migration. Error: " + e); + process.exit(-1); } - } else { - await this.$migrateTableSchemaFromVersion(0); + logger.info("MIGRATIONS: `state` table initialized.") } - } - private async $initializeDatabaseTables(): Promise { - const connection = await DB.pool.getConnection(); - for (const query of this.getInitializeTableQueries()) { - await connection.query({ sql: query, timeout: this.queryTimeout }); + let databaseSchemaVersion = 0; + try { + databaseSchemaVersion = await this.$getSchemaVersionFromDatabase(); + } catch (e) { + logger.err("Unable to get current database migration version, aborting. Error: " + e); + process.exit(-1); } - connection.release(); - logger.info(`Initial database tables have been created`); - } - private async $migrateTableSchemaFromVersion(version: number): Promise { - const connection = await DB.pool.getConnection(); - for (const query of this.getMigrationQueriesFromVersion(version)) { - await connection.query({ sql: query, timeout: this.queryTimeout }); + logger.info("MIGRATIONS: Current state.schema_version " + databaseSchemaVersion.toString()); + logger.info("MIGRATIONS: Latest DatabaseMigration.version is " + DatabaseMigration.currentVersion.toString()); + if (databaseSchemaVersion.toString() === DatabaseMigration.currentVersion.toString()) { + logger.info("MIGRATIONS: Nothing to do."); + return; } - connection.release(); - await this.$updateToLatestSchemaVersion(); - logger.info(`Database schema have been migrated from version ${version} to ${DatabaseMigration.currentVersion} (latest version)`); + + if (DatabaseMigration.currentVersion > databaseSchemaVersion) { + logger.info("MIGRATIONS: Upgrading datababse schema"); + try { + await this.$migrateTableSchemaFromVersion(databaseSchemaVersion); + logger.info(`OK. Database schema have been migrated from version ${databaseSchemaVersion} to ${DatabaseMigration.currentVersion} (latest version)`); + } catch (e) { + logger.err("Unable to migrate database, aborting. Error: " + e); + } + } + + return; } - private async $getSchemaVersionFromDatabase(): Promise { - const connection = await DB.pool.getConnection(); - const query = `SELECT number FROM state WHERE name = 'schema_version';`; - const [rows] = await connection.query({ sql: query, timeout: this.queryTimeout }); - connection.release(); - return rows[0]['number']; - } - - private async $updateToLatestSchemaVersion(): Promise { - const connection = await DB.pool.getConnection(); - const query = `UPDATE state SET number = ${DatabaseMigration.currentVersion} WHERE name = 'schema_version'`; - const [rows] = await connection.query({ sql: query, timeout: this.queryTimeout }); - connection.release(); + /** + * Small query execution wrapper to log all executed queries + */ + private async $executeQuery(connection: PoolConnection, query: string): Promise { + logger.info("MIGRATIONS: Execute query:\n" + query); + return connection.query({ sql: query, timeout: this.queryTimeout }); } + /** + * Check if 'table' exists in the database + */ private async $checkIfTableExists(table: string): Promise { const connection = await DB.pool.getConnection(); const query = `SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = '${config.DATABASE.DATABASE}' AND TABLE_NAME = '${table}'`; @@ -65,11 +74,104 @@ class DatabaseMigration { return rows[0]['COUNT(*)'] === 1; } - private getInitializeTableQueries(): string[] { + /** + * Get current database version + */ + private async $getSchemaVersionFromDatabase(): Promise { + const connection = await DB.pool.getConnection(); + const query = `SELECT number FROM state WHERE name = 'schema_version';`; + const [rows] = await connection.query({ sql: query, timeout: this.queryTimeout }); + connection.release(); + return rows[0]['number']; + } + + /** + * Create the `state` table + */ + private async $createMigrationStateTable(): Promise { + const connection = await DB.pool.getConnection(); + await this.$executeQuery(connection, `START TRANSACTION;`); + await this.$executeQuery(connection, "SET autocommit = 0;"); + + try { + const query = `CREATE TABLE IF NOT EXISTS state ( + name varchar(25) NOT NULL, + number int(11) NULL, + string varchar(100) NULL, + CONSTRAINT name_unique UNIQUE (name) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; + await this.$executeQuery(connection, query); + + // Set initial values + await this.$executeQuery(connection, `INSERT INTO state VALUES('schema_version', 0, NULL);`); + await this.$executeQuery(connection, `INSERT INTO state VALUES('last_elements_block', 0, NULL);`); + } catch (e) { + await this.$executeQuery(connection, `ROLLBACK;`); + connection.release(); + throw e; + } + + await this.$executeQuery(connection, `COMMIT;`); + } + + /** + * We actually run the migrations queries here + */ + private async $migrateTableSchemaFromVersion(version: number): Promise { + let transactionQueries: string[] = []; + for (const query of this.getMigrationQueriesFromVersion(version)) { + transactionQueries.push(query); + } + transactionQueries.push(this.getUpdateToLatestSchemaVersionQuery()); + + const connection = await DB.pool.getConnection(); + try { + await this.$executeQuery(connection, "START TRANSACTION;"); + await this.$executeQuery(connection, "SET autocommit = 0;"); + for (const query of transactionQueries) { + await this.$executeQuery(connection, query); + } + } catch (e) { + await this.$executeQuery(connection, "ROLLBACK;"); + connection.release(); + throw e; + } + + await this.$executeQuery(connection, "COMMIT;"); + } + + /** + * Generate migration queries based on schema version + */ + private getMigrationQueriesFromVersion(version: number): string[] { const queries: string[] = []; - queries.push(`CREATE TABLE IF NOT EXISTS statistics ( - id int(11) NOT NULL, + if (version < 1) { + queries.push(this.getCreateElementsTableQuery()); + queries.push(this.getCreateStatisticsQuery()); + if (config.MEMPOOL.NETWORK !== 'liquid' && config.MEMPOOL.NETWORK !== 'liquidtestnet') { + queries.push(this.getUpdateStatisticsQuery()); + } + } + + if (version < 2) { + queries.push(`CREATE INDEX IF NOT EXISTS added ON statistics (added);`); + } + + return queries; + } + + /** + * Save the schema version in the database + */ + private getUpdateToLatestSchemaVersionQuery(): string { + return `UPDATE state SET number = ${DatabaseMigration.currentVersion} WHERE name = 'schema_version';`; + } + + // Couple of wrappers to clean the main logic + private getCreateStatisticsQuery(): string { + return `CREATE TABLE IF NOT EXISTS statistics ( + id int(11) NOT NULL AUTO_INCREMENT, added datetime NOT NULL, unconfirmed_transactions int(11) UNSIGNED NOT NULL, tx_per_second float UNSIGNED NOT NULL, @@ -114,68 +216,43 @@ class DatabaseMigration { vsize_1400 int(11) NOT NULL, vsize_1600 int(11) NOT NULL, vsize_1800 int(11) NOT NULL, - vsize_2000 int(11) NOT NULL - ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`); - - queries.push(`ALTER TABLE statistics ADD PRIMARY KEY (id);`); - queries.push(`ALTER TABLE statistics MODIFY id int(11) NOT NULL AUTO_INCREMENT;`); - - return queries; + vsize_2000 int(11) NOT NULL, + CONSTRAINT PRIMARY KEY (id) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8;` } - - private getMigrationQueriesFromVersion(version: number): string[] { - const queries: string[] = []; - - if (version < 1) { - if (config.MEMPOOL.NETWORK !== 'liquid' && config.MEMPOOL.NETWORK !== 'liquidtestnet') { - queries.push(`UPDATE statistics SET - vsize_1 = vsize_1 + vsize_2, vsize_2 = vsize_3, - vsize_3 = vsize_4, vsize_4 = vsize_5, - vsize_5 = vsize_6, vsize_6 = vsize_8, - vsize_8 = vsize_10, vsize_10 = vsize_12, - vsize_12 = vsize_15, vsize_15 = vsize_20, - vsize_20 = vsize_30, vsize_30 = vsize_40, - vsize_40 = vsize_50, vsize_50 = vsize_60, - vsize_60 = vsize_70, vsize_70 = vsize_80, - vsize_80 = vsize_90, vsize_90 = vsize_100, - vsize_100 = vsize_125, vsize_125 = vsize_150, - vsize_150 = vsize_175, vsize_175 = vsize_200, - vsize_200 = vsize_250, vsize_250 = vsize_300, - vsize_300 = vsize_350, vsize_350 = vsize_400, - vsize_400 = vsize_500, vsize_500 = vsize_600, - vsize_600 = vsize_700, vsize_700 = vsize_800, - vsize_800 = vsize_900, vsize_900 = vsize_1000, - vsize_1000 = vsize_1200, vsize_1200 = vsize_1400, - vsize_1400 = vsize_1800, vsize_1800 = vsize_2000, vsize_2000 = 0`); - } - - queries.push(`CREATE TABLE IF NOT EXISTS elements_pegs ( - block int(11) NOT NULL, - datetime int(11) NOT NULL, - amount bigint(20) NOT NULL, - txid varchar(65) NOT NULL, - txindex int(11) NOT NULL, - bitcoinaddress varchar(100) NOT NULL, - bitcointxid varchar(65) NOT NULL, - bitcoinindex int(11) NOT NULL, - final_tx int(11) NOT NULL - ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`); - - queries.push(`CREATE TABLE IF NOT EXISTS state ( - name varchar(25) NOT NULL, - number int(11) NULL, - string varchar(100) NULL - ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`); - - queries.push(`INSERT INTO state VALUES('schema_version', 0, NULL);`); - queries.push(`INSERT INTO state VALUES('last_elements_block', 0, NULL);`); - } - - if (version < 2) { - queries.push(`CREATE INDEX IF NOT EXISTS added ON statistics (added);`); - } - - return queries; + private getUpdateStatisticsQuery(): string { + return `UPDATE statistics SET + vsize_1 = vsize_1 + vsize_2, vsize_2 = vsize_3, + vsize_3 = vsize_4, vsize_4 = vsize_5, + vsize_5 = vsize_6, vsize_6 = vsize_8, + vsize_8 = vsize_10, vsize_10 = vsize_12, + vsize_12 = vsize_15, vsize_15 = vsize_20, + vsize_20 = vsize_30, vsize_30 = vsize_40, + vsize_40 = vsize_50, vsize_50 = vsize_60, + vsize_60 = vsize_70, vsize_70 = vsize_80, + vsize_80 = vsize_90, vsize_90 = vsize_100, + vsize_100 = vsize_125, vsize_125 = vsize_150, + vsize_150 = vsize_175, vsize_175 = vsize_200, + vsize_200 = vsize_250, vsize_250 = vsize_300, + vsize_300 = vsize_350, vsize_350 = vsize_400, + vsize_400 = vsize_500, vsize_500 = vsize_600, + vsize_600 = vsize_700, vsize_700 = vsize_800, + vsize_800 = vsize_900, vsize_900 = vsize_1000, + vsize_1000 = vsize_1200, vsize_1200 = vsize_1400, + vsize_1400 = vsize_1800, vsize_1800 = vsize_2000, vsize_2000 = 0;`; + } + private getCreateElementsTableQuery(): string { + return `CREATE TABLE IF NOT EXISTS elements_pegs ( + block int(11) NOT NULL, + datetime int(11) NOT NULL, + amount bigint(20) NOT NULL, + txid varchar(65) NOT NULL, + txindex int(11) NOT NULL, + bitcoinaddress varchar(100) NOT NULL, + bitcointxid varchar(65) NOT NULL, + bitcoinindex int(11) NOT NULL, + final_tx int(11) NOT NULL + ) ENGINE=InnoDB DEFAULT CHARSET=utf8;` } } From fc878b696d4173cffef7e430ccdcf2575b80a4b8 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Wed, 12 Jan 2022 14:10:16 +0900 Subject: [PATCH 2/7] Only create `statistics.index` if needed (supports old mariadb) - Make sure all db connections are released - Fix linter issues - Remove .toString() --- backend/src/api/database-migration.ts | 96 +++++++++++++++++++-------- 1 file changed, 69 insertions(+), 27 deletions(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 61c752302..4078d9092 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -6,6 +6,7 @@ import logger from '../logger'; class DatabaseMigration { private static currentVersion = 2; private queryTimeout = 120000; + private statisticsAddedIndexed = false; constructor() { } @@ -13,53 +14,91 @@ class DatabaseMigration { * Entry point */ public async $initializeOrMigrateDatabase(): Promise { - logger.info("MIGRATIONS: Running migrations"); + logger.info('MIGRATIONS: Running migrations'); // First of all, if the `state` database does not exist, create it so we can track migration version if (!await this.$checkIfTableExists('state')) { - logger.info("MIGRATIONS: `state` table does not exist. Creating it.") + logger.info('MIGRATIONS: `state` table does not exist. Creating it.') try { await this.$createMigrationStateTable(); } catch (e) { - logger.err("Unable to create `state` table. Aborting migration. Error: " + e); + logger.err('Unable to create `state` table. Aborting migration. Error: ' + e); process.exit(-1); } - logger.info("MIGRATIONS: `state` table initialized.") + logger.info('MIGRATIONS: `state` table initialized.') } let databaseSchemaVersion = 0; try { databaseSchemaVersion = await this.$getSchemaVersionFromDatabase(); } catch (e) { - logger.err("Unable to get current database migration version, aborting. Error: " + e); + logger.err('Unable to get current database migration version, aborting. Error: ' + e); process.exit(-1); } - logger.info("MIGRATIONS: Current state.schema_version " + databaseSchemaVersion.toString()); - logger.info("MIGRATIONS: Latest DatabaseMigration.version is " + DatabaseMigration.currentVersion.toString()); - if (databaseSchemaVersion.toString() === DatabaseMigration.currentVersion.toString()) { - logger.info("MIGRATIONS: Nothing to do."); + logger.info('MIGRATIONS: Current state.schema_version ' + databaseSchemaVersion); + logger.info('MIGRATIONS: Latest DatabaseMigration.version is ' + DatabaseMigration.currentVersion); + if (databaseSchemaVersion >= DatabaseMigration.currentVersion) { + logger.info('MIGRATIONS: Nothing to do.'); return; } + // Will create `statistics.added` INDEX if needed for databaseSchemaVersion <= 2 + await this.$setStatisticsAddedIndexedFlag(databaseSchemaVersion); + if (DatabaseMigration.currentVersion > databaseSchemaVersion) { - logger.info("MIGRATIONS: Upgrading datababse schema"); + logger.info('MIGRATIONS: Upgrading datababse schema'); try { await this.$migrateTableSchemaFromVersion(databaseSchemaVersion); logger.info(`OK. Database schema have been migrated from version ${databaseSchemaVersion} to ${DatabaseMigration.currentVersion} (latest version)`); } catch (e) { - logger.err("Unable to migrate database, aborting. Error: " + e); + logger.err('Unable to migrate database, aborting. Error: ' + e); } } return; } + /** + * Special case here for the `statistics` table - It appeared that somehow some dbs already had the `added` field indexed + * while it does not appear in previous schemas. The mariadb command "CREATE INDEX IF NOT EXISTS" is not supported on + * older mariadb version. Therefore we set a flag here in order to know if the index needs to be created or not before + * running the migration process + */ + private async $setStatisticsAddedIndexedFlag(databaseSchemaVersion: number) { + if (databaseSchemaVersion >= 2) { + this.statisticsAddedIndexed = true; + return; + } + + const connection = await DB.pool.getConnection(); + + try { + const query = `SELECT COUNT(1) hasIndex FROM INFORMATION_SCHEMA.STATISTICS + WHERE table_schema=DATABASE() AND table_name='statistics' AND index_name='added';`; + const [rows] = await this.$executeQuery(connection, query); + if (rows[0].hasIndex === 0) { + logger.info('MIGRATIONS: `statistics.added` is not indexed'); + this.statisticsAddedIndexed = false; + } else if (rows[0].hasIndex === 1) { + logger.info('MIGRATIONS: `statistics.added` is already indexed'); + this.statisticsAddedIndexed = true; + } + } catch (e) { + // Should really never happen but just in case it fails, we just don't execute + // any query related to this indexing so it won't fail if the index actually already exists + logger.err('MIGRATIONS: Unable to check if `statistics.added` INDEX exist or not.'); + this.statisticsAddedIndexed = true; + } + + connection.release(); + } + /** * Small query execution wrapper to log all executed queries */ private async $executeQuery(connection: PoolConnection, query: string): Promise { - logger.info("MIGRATIONS: Execute query:\n" + query); + logger.info('MIGRATIONS: Execute query:\n' + query); return connection.query({ sql: query, timeout: this.queryTimeout }); } @@ -90,10 +129,11 @@ class DatabaseMigration { */ private async $createMigrationStateTable(): Promise { const connection = await DB.pool.getConnection(); - await this.$executeQuery(connection, `START TRANSACTION;`); - await this.$executeQuery(connection, "SET autocommit = 0;"); try { + await this.$executeQuery(connection, `START TRANSACTION;`); + await this.$executeQuery(connection, 'SET autocommit = 0;'); + const query = `CREATE TABLE IF NOT EXISTS state ( name varchar(25) NOT NULL, number int(11) NULL, @@ -105,20 +145,21 @@ class DatabaseMigration { // Set initial values await this.$executeQuery(connection, `INSERT INTO state VALUES('schema_version', 0, NULL);`); await this.$executeQuery(connection, `INSERT INTO state VALUES('last_elements_block', 0, NULL);`); + await this.$executeQuery(connection, `COMMIT;`); + + connection.release(); } catch (e) { await this.$executeQuery(connection, `ROLLBACK;`); connection.release(); throw e; } - - await this.$executeQuery(connection, `COMMIT;`); } /** * We actually run the migrations queries here */ private async $migrateTableSchemaFromVersion(version: number): Promise { - let transactionQueries: string[] = []; + const transactionQueries: string[] = []; for (const query of this.getMigrationQueriesFromVersion(version)) { transactionQueries.push(query); } @@ -126,18 +167,19 @@ class DatabaseMigration { const connection = await DB.pool.getConnection(); try { - await this.$executeQuery(connection, "START TRANSACTION;"); - await this.$executeQuery(connection, "SET autocommit = 0;"); + await this.$executeQuery(connection, 'START TRANSACTION;'); + await this.$executeQuery(connection, 'SET autocommit = 0;'); for (const query of transactionQueries) { await this.$executeQuery(connection, query); } + await this.$executeQuery(connection, 'COMMIT;'); + + connection.release(); } catch (e) { - await this.$executeQuery(connection, "ROLLBACK;"); + await this.$executeQuery(connection, 'ROLLBACK;'); connection.release(); throw e; } - - await this.$executeQuery(connection, "COMMIT;"); } /** @@ -150,12 +192,12 @@ class DatabaseMigration { queries.push(this.getCreateElementsTableQuery()); queries.push(this.getCreateStatisticsQuery()); if (config.MEMPOOL.NETWORK !== 'liquid' && config.MEMPOOL.NETWORK !== 'liquidtestnet') { - queries.push(this.getUpdateStatisticsQuery()); + queries.push(this.getShiftStatisticsQuery()); } } - if (version < 2) { - queries.push(`CREATE INDEX IF NOT EXISTS added ON statistics (added);`); + if (version < 2 && this.statisticsAddedIndexed === false) { + queries.push(`CREATE INDEX added ON statistics (added);`); } return queries; @@ -164,7 +206,7 @@ class DatabaseMigration { /** * Save the schema version in the database */ - private getUpdateToLatestSchemaVersionQuery(): string { + private getUpdateToLatestSchemaVersionQuery(): string { return `UPDATE state SET number = ${DatabaseMigration.currentVersion} WHERE name = 'schema_version';`; } @@ -220,7 +262,7 @@ class DatabaseMigration { CONSTRAINT PRIMARY KEY (id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8;` } - private getUpdateStatisticsQuery(): string { + private getShiftStatisticsQuery(): string { return `UPDATE statistics SET vsize_1 = vsize_1 + vsize_2, vsize_2 = vsize_3, vsize_3 = vsize_4, vsize_4 = vsize_5, From cce49bdb7e65c1037d8b5f9e72f287f59f6e4558 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Wed, 12 Jan 2022 14:51:16 +0900 Subject: [PATCH 3/7] MariaDB 10.2 does not supports `CAST as FLOAT` -> Replace with `CAST as DOUBLE` --- backend/src/api/statistics.ts | 84 +++++++++++++++++------------------ 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/backend/src/api/statistics.ts b/backend/src/api/statistics.ts index 9a53e3504..5c66153c8 100644 --- a/backend/src/api/statistics.ts +++ b/backend/src/api/statistics.ts @@ -270,47 +270,47 @@ class Statistics { private getQueryForDaysAvg(div: number, interval: string) { return `SELECT id, UNIX_TIMESTAMP(added) as added, - CAST(avg(unconfirmed_transactions) as FLOAT) as unconfirmed_transactions, - CAST(avg(tx_per_second) as FLOAT) as tx_per_second, - CAST(avg(vbytes_per_second) as FLOAT) as vbytes_per_second, - CAST(avg(vsize_1) as FLOAT) as vsize_1, - CAST(avg(vsize_2) as FLOAT) as vsize_2, - CAST(avg(vsize_3) as FLOAT) as vsize_3, - CAST(avg(vsize_4) as FLOAT) as vsize_4, - CAST(avg(vsize_5) as FLOAT) as vsize_5, - CAST(avg(vsize_6) as FLOAT) as vsize_6, - CAST(avg(vsize_8) as FLOAT) as vsize_8, - CAST(avg(vsize_10) as FLOAT) as vsize_10, - CAST(avg(vsize_12) as FLOAT) as vsize_12, - CAST(avg(vsize_15) as FLOAT) as vsize_15, - CAST(avg(vsize_20) as FLOAT) as vsize_20, - CAST(avg(vsize_30) as FLOAT) as vsize_30, - CAST(avg(vsize_40) as FLOAT) as vsize_40, - CAST(avg(vsize_50) as FLOAT) as vsize_50, - CAST(avg(vsize_60) as FLOAT) as vsize_60, - CAST(avg(vsize_70) as FLOAT) as vsize_70, - CAST(avg(vsize_80) as FLOAT) as vsize_80, - CAST(avg(vsize_90) as FLOAT) as vsize_90, - CAST(avg(vsize_100) as FLOAT) as vsize_100, - CAST(avg(vsize_125) as FLOAT) as vsize_125, - CAST(avg(vsize_150) as FLOAT) as vsize_150, - CAST(avg(vsize_175) as FLOAT) as vsize_175, - CAST(avg(vsize_200) as FLOAT) as vsize_200, - CAST(avg(vsize_250) as FLOAT) as vsize_250, - CAST(avg(vsize_300) as FLOAT) as vsize_300, - CAST(avg(vsize_350) as FLOAT) as vsize_350, - CAST(avg(vsize_400) as FLOAT) as vsize_400, - CAST(avg(vsize_500) as FLOAT) as vsize_500, - CAST(avg(vsize_600) as FLOAT) as vsize_600, - CAST(avg(vsize_700) as FLOAT) as vsize_700, - CAST(avg(vsize_800) as FLOAT) as vsize_800, - CAST(avg(vsize_900) as FLOAT) as vsize_900, - CAST(avg(vsize_1000) as FLOAT) as vsize_1000, - CAST(avg(vsize_1200) as FLOAT) as vsize_1200, - CAST(avg(vsize_1400) as FLOAT) as vsize_1400, - CAST(avg(vsize_1600) as FLOAT) as vsize_1600, - CAST(avg(vsize_1800) as FLOAT) as vsize_1800, - CAST(avg(vsize_2000) as FLOAT) as vsize_2000 \ + CAST(avg(unconfirmed_transactions) as DOUBLE) as unconfirmed_transactions, + CAST(avg(tx_per_second) as DOUBLE) as tx_per_second, + CAST(avg(vbytes_per_second) as DOUBLE) as vbytes_per_second, + CAST(avg(vsize_1) as DOUBLE) as vsize_1, + CAST(avg(vsize_2) as DOUBLE) as vsize_2, + CAST(avg(vsize_3) as DOUBLE) as vsize_3, + CAST(avg(vsize_4) as DOUBLE) as vsize_4, + CAST(avg(vsize_5) as DOUBLE) as vsize_5, + CAST(avg(vsize_6) as DOUBLE) as vsize_6, + CAST(avg(vsize_8) as DOUBLE) as vsize_8, + CAST(avg(vsize_10) as DOUBLE) as vsize_10, + CAST(avg(vsize_12) as DOUBLE) as vsize_12, + CAST(avg(vsize_15) as DOUBLE) as vsize_15, + CAST(avg(vsize_20) as DOUBLE) as vsize_20, + CAST(avg(vsize_30) as DOUBLE) as vsize_30, + CAST(avg(vsize_40) as DOUBLE) as vsize_40, + CAST(avg(vsize_50) as DOUBLE) as vsize_50, + CAST(avg(vsize_60) as DOUBLE) as vsize_60, + CAST(avg(vsize_70) as DOUBLE) as vsize_70, + CAST(avg(vsize_80) as DOUBLE) as vsize_80, + CAST(avg(vsize_90) as DOUBLE) as vsize_90, + CAST(avg(vsize_100) as DOUBLE) as vsize_100, + CAST(avg(vsize_125) as DOUBLE) as vsize_125, + CAST(avg(vsize_150) as DOUBLE) as vsize_150, + CAST(avg(vsize_175) as DOUBLE) as vsize_175, + CAST(avg(vsize_200) as DOUBLE) as vsize_200, + CAST(avg(vsize_250) as DOUBLE) as vsize_250, + CAST(avg(vsize_300) as DOUBLE) as vsize_300, + CAST(avg(vsize_350) as DOUBLE) as vsize_350, + CAST(avg(vsize_400) as DOUBLE) as vsize_400, + CAST(avg(vsize_500) as DOUBLE) as vsize_500, + CAST(avg(vsize_600) as DOUBLE) as vsize_600, + CAST(avg(vsize_700) as DOUBLE) as vsize_700, + CAST(avg(vsize_800) as DOUBLE) as vsize_800, + CAST(avg(vsize_900) as DOUBLE) as vsize_900, + CAST(avg(vsize_1000) as DOUBLE) as vsize_1000, + CAST(avg(vsize_1200) as DOUBLE) as vsize_1200, + CAST(avg(vsize_1400) as DOUBLE) as vsize_1400, + CAST(avg(vsize_1600) as DOUBLE) as vsize_1600, + CAST(avg(vsize_1800) as DOUBLE) as vsize_1800, + CAST(avg(vsize_2000) as DOUBLE) as vsize_2000 \ FROM statistics \ WHERE added BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW() \ GROUP BY UNIX_TIMESTAMP(added) DIV ${div} \ @@ -320,7 +320,7 @@ class Statistics { private getQueryForDays(div: number, interval: string) { return `SELECT id, UNIX_TIMESTAMP(added) as added, unconfirmed_transactions, tx_per_second, - CAST(avg(vbytes_per_second) as FLOAT) as vbytes_per_second, + CAST(avg(vbytes_per_second) as DOUBLE) as vbytes_per_second, vsize_1, vsize_2, vsize_3, From 4e322fe00668e3e47c2975a18a50736f4ee25bf2 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Wed, 12 Jan 2022 16:06:45 +0900 Subject: [PATCH 4/7] Print database engine version when migration script starts --- backend/src/api/database-migration.ts | 38 ++++++++++++++++++++------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 4078d9092..17b34b710 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -16,23 +16,25 @@ class DatabaseMigration { public async $initializeOrMigrateDatabase(): Promise { logger.info('MIGRATIONS: Running migrations'); + await this.$printDatabaseVersion(); + // First of all, if the `state` database does not exist, create it so we can track migration version if (!await this.$checkIfTableExists('state')) { - logger.info('MIGRATIONS: `state` table does not exist. Creating it.') + logger.info('MIGRATIONS: `state` table does not exist. Creating it.'); try { await this.$createMigrationStateTable(); } catch (e) { - logger.err('Unable to create `state` table. Aborting migration. Error: ' + e); + logger.err('Unable to create `state` table. Aborting migration. ' + e); process.exit(-1); } - logger.info('MIGRATIONS: `state` table initialized.') + logger.info('MIGRATIONS: `state` table initialized.'); } let databaseSchemaVersion = 0; try { databaseSchemaVersion = await this.$getSchemaVersionFromDatabase(); } catch (e) { - logger.err('Unable to get current database migration version, aborting. Error: ' + e); + logger.err('Unable to get current database migration version, aborting. ' + e); process.exit(-1); } @@ -76,7 +78,7 @@ class DatabaseMigration { try { const query = `SELECT COUNT(1) hasIndex FROM INFORMATION_SCHEMA.STATISTICS WHERE table_schema=DATABASE() AND table_name='statistics' AND index_name='added';`; - const [rows] = await this.$executeQuery(connection, query); + const [rows] = await this.$executeQuery(connection, query, true); if (rows[0].hasIndex === 0) { logger.info('MIGRATIONS: `statistics.added` is not indexed'); this.statisticsAddedIndexed = false; @@ -97,8 +99,10 @@ class DatabaseMigration { /** * Small query execution wrapper to log all executed queries */ - private async $executeQuery(connection: PoolConnection, query: string): Promise { - logger.info('MIGRATIONS: Execute query:\n' + query); + private async $executeQuery(connection: PoolConnection, query: string, silent: boolean = false): Promise { + if (!silent) { + logger.info('MIGRATIONS: Execute query:\n' + query); + } return connection.query({ sql: query, timeout: this.queryTimeout }); } @@ -119,7 +123,7 @@ class DatabaseMigration { private async $getSchemaVersionFromDatabase(): Promise { const connection = await DB.pool.getConnection(); const query = `SELECT number FROM state WHERE name = 'schema_version';`; - const [rows] = await connection.query({ sql: query, timeout: this.queryTimeout }); + const [rows] = await this.$executeQuery(connection, query, true); connection.release(); return rows[0]['number']; } @@ -210,6 +214,20 @@ class DatabaseMigration { return `UPDATE state SET number = ${DatabaseMigration.currentVersion} WHERE name = 'schema_version';`; } + /** + * Print current database version + */ + private async $printDatabaseVersion() { + const connection = await DB.pool.getConnection(); + try { + const [rows] = await this.$executeQuery(connection, 'SELECT VERSION() as version;', true); + logger.info(`MIGRATIONS: Database engine version '${rows[0].version}'`); + } catch (e) { + logger.info(`MIGRATIONS: Could not fetch database engine version. Error ` + e); + } + connection.release(); + } + // Couple of wrappers to clean the main logic private getCreateStatisticsQuery(): string { return `CREATE TABLE IF NOT EXISTS statistics ( @@ -260,7 +278,7 @@ class DatabaseMigration { vsize_1800 int(11) NOT NULL, vsize_2000 int(11) NOT NULL, CONSTRAINT PRIMARY KEY (id) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8;` + ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; } private getShiftStatisticsQuery(): string { return `UPDATE statistics SET @@ -294,7 +312,7 @@ class DatabaseMigration { bitcointxid varchar(65) NOT NULL, bitcoinindex int(11) NOT NULL, final_tx int(11) NOT NULL - ) ENGINE=InnoDB DEFAULT CHARSET=utf8;` + ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; } } From ae2cb05dc5fd41e71aeaf73fe55135afefabe712 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Wed, 12 Jan 2022 16:41:27 +0900 Subject: [PATCH 5/7] Extract all `CREATE` commands from transaction --- backend/src/api/database-migration.ts | 91 ++++++++++++++++----------- 1 file changed, 54 insertions(+), 37 deletions(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 17b34b710..dc667ad62 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -45,8 +45,13 @@ class DatabaseMigration { return; } - // Will create `statistics.added` INDEX if needed for databaseSchemaVersion <= 2 - await this.$setStatisticsAddedIndexedFlag(databaseSchemaVersion); + // Now, create missing tables. Those queries cannot be wrapped into a transaction unfortunately + try { + await this.$createMissingTablesAndIndexes(databaseSchemaVersion); + } catch (e) { + logger.err('Unable to create required tables, aborting. ' + e); + process.exit(-1); + } if (DatabaseMigration.currentVersion > databaseSchemaVersion) { logger.info('MIGRATIONS: Upgrading datababse schema'); @@ -54,13 +59,33 @@ class DatabaseMigration { await this.$migrateTableSchemaFromVersion(databaseSchemaVersion); logger.info(`OK. Database schema have been migrated from version ${databaseSchemaVersion} to ${DatabaseMigration.currentVersion} (latest version)`); } catch (e) { - logger.err('Unable to migrate database, aborting. Error: ' + e); + logger.err('Unable to migrate database, aborting. ' + e); } } return; } + /** + * Create all missing tables + */ + private async $createMissingTablesAndIndexes(databaseSchemaVersion: number) { + await this.$setStatisticsAddedIndexedFlag(databaseSchemaVersion); + + const connection = await DB.pool.getConnection(); + try { + await this.$executeQuery(connection, this.getCreateElementsTableQuery(), await this.$checkIfTableExists('elements_pegs')); + await this.$executeQuery(connection, this.getCreateStatisticsQuery(), await this.$checkIfTableExists('statistics')); + if (databaseSchemaVersion < 2 && this.statisticsAddedIndexed === false) { + await this.$executeQuery(connection, `CREATE INDEX added ON statistics (added);`); + } + connection.release(); + } catch (e) { + connection.release(); + throw e; + } + } + /** * Special case here for the `statistics` table - It appeared that somehow some dbs already had the `added` field indexed * while it does not appear in previous schemas. The mariadb command "CREATE INDEX IF NOT EXISTS" is not supported on @@ -76,6 +101,7 @@ class DatabaseMigration { const connection = await DB.pool.getConnection(); try { + // We don't use "CREATE INDEX IF NOT EXISTS" because it is not supported on old mariadb version 5.X const query = `SELECT COUNT(1) hasIndex FROM INFORMATION_SCHEMA.STATISTICS WHERE table_schema=DATABASE() AND table_name='statistics' AND index_name='added';`; const [rows] = await this.$executeQuery(connection, query, true); @@ -135,9 +161,6 @@ class DatabaseMigration { const connection = await DB.pool.getConnection(); try { - await this.$executeQuery(connection, `START TRANSACTION;`); - await this.$executeQuery(connection, 'SET autocommit = 0;'); - const query = `CREATE TABLE IF NOT EXISTS state ( name varchar(25) NOT NULL, number int(11) NULL, @@ -149,18 +172,16 @@ class DatabaseMigration { // Set initial values await this.$executeQuery(connection, `INSERT INTO state VALUES('schema_version', 0, NULL);`); await this.$executeQuery(connection, `INSERT INTO state VALUES('last_elements_block', 0, NULL);`); - await this.$executeQuery(connection, `COMMIT;`); connection.release(); } catch (e) { - await this.$executeQuery(connection, `ROLLBACK;`); connection.release(); throw e; } } /** - * We actually run the migrations queries here + * We actually execute the migrations queries here */ private async $migrateTableSchemaFromVersion(version: number): Promise { const transactionQueries: string[] = []; @@ -193,17 +214,11 @@ class DatabaseMigration { const queries: string[] = []; if (version < 1) { - queries.push(this.getCreateElementsTableQuery()); - queries.push(this.getCreateStatisticsQuery()); if (config.MEMPOOL.NETWORK !== 'liquid' && config.MEMPOOL.NETWORK !== 'liquidtestnet') { queries.push(this.getShiftStatisticsQuery()); } } - if (version < 2 && this.statisticsAddedIndexed === false) { - queries.push(`CREATE INDEX added ON statistics (added);`); - } - return queries; } @@ -223,12 +238,34 @@ class DatabaseMigration { const [rows] = await this.$executeQuery(connection, 'SELECT VERSION() as version;', true); logger.info(`MIGRATIONS: Database engine version '${rows[0].version}'`); } catch (e) { - logger.info(`MIGRATIONS: Could not fetch database engine version. Error ` + e); + logger.info(`MIGRATIONS: Could not fetch database engine version. ` + e); } connection.release(); } // Couple of wrappers to clean the main logic + private getShiftStatisticsQuery(): string { + return `UPDATE statistics SET + vsize_1 = vsize_1 + vsize_2, vsize_2 = vsize_3, + vsize_3 = vsize_4, vsize_4 = vsize_5, + vsize_5 = vsize_6, vsize_6 = vsize_8, + vsize_8 = vsize_10, vsize_10 = vsize_12, + vsize_12 = vsize_15, vsize_15 = vsize_20, + vsize_20 = vsize_30, vsize_30 = vsize_40, + vsize_40 = vsize_50, vsize_50 = vsize_60, + vsize_60 = vsize_70, vsize_70 = vsize_80, + vsize_80 = vsize_90, vsize_90 = vsize_100, + vsize_100 = vsize_125, vsize_125 = vsize_150, + vsize_150 = vsize_175, vsize_175 = vsize_200, + vsize_200 = vsize_250, vsize_250 = vsize_300, + vsize_300 = vsize_350, vsize_350 = vsize_400, + vsize_400 = vsize_500, vsize_500 = vsize_600, + vsize_600 = vsize_700, vsize_700 = vsize_800, + vsize_800 = vsize_900, vsize_900 = vsize_1000, + vsize_1000 = vsize_1200, vsize_1200 = vsize_1400, + vsize_1400 = vsize_1800, vsize_1800 = vsize_2000, vsize_2000 = 0;`; + } + private getCreateStatisticsQuery(): string { return `CREATE TABLE IF NOT EXISTS statistics ( id int(11) NOT NULL AUTO_INCREMENT, @@ -280,27 +317,7 @@ class DatabaseMigration { CONSTRAINT PRIMARY KEY (id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; } - private getShiftStatisticsQuery(): string { - return `UPDATE statistics SET - vsize_1 = vsize_1 + vsize_2, vsize_2 = vsize_3, - vsize_3 = vsize_4, vsize_4 = vsize_5, - vsize_5 = vsize_6, vsize_6 = vsize_8, - vsize_8 = vsize_10, vsize_10 = vsize_12, - vsize_12 = vsize_15, vsize_15 = vsize_20, - vsize_20 = vsize_30, vsize_30 = vsize_40, - vsize_40 = vsize_50, vsize_50 = vsize_60, - vsize_60 = vsize_70, vsize_70 = vsize_80, - vsize_80 = vsize_90, vsize_90 = vsize_100, - vsize_100 = vsize_125, vsize_125 = vsize_150, - vsize_150 = vsize_175, vsize_175 = vsize_200, - vsize_200 = vsize_250, vsize_250 = vsize_300, - vsize_300 = vsize_350, vsize_350 = vsize_400, - vsize_400 = vsize_500, vsize_500 = vsize_600, - vsize_600 = vsize_700, vsize_700 = vsize_800, - vsize_800 = vsize_900, vsize_900 = vsize_1000, - vsize_1000 = vsize_1200, vsize_1200 = vsize_1400, - vsize_1400 = vsize_1800, vsize_1800 = vsize_2000, vsize_2000 = 0;`; - } + private getCreateElementsTableQuery(): string { return `CREATE TABLE IF NOT EXISTS elements_pegs ( block int(11) NOT NULL, From f494bd6d6a2fedc727fc58c57300905c481ef197 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Wed, 12 Jan 2022 17:26:10 +0900 Subject: [PATCH 6/7] Sleep 10 seconds before ending the process after critical error in database migration --- backend/src/api/database-migration.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index dc667ad62..64fa93761 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -3,13 +3,14 @@ import config from '../config'; import { DB } from '../database'; import logger from '../logger'; +const sleep = (ms: number) => new Promise( res => setTimeout(res, ms)); + class DatabaseMigration { private static currentVersion = 2; private queryTimeout = 120000; private statisticsAddedIndexed = false; constructor() { } - /** * Entry point */ @@ -24,7 +25,8 @@ class DatabaseMigration { try { await this.$createMigrationStateTable(); } catch (e) { - logger.err('Unable to create `state` table. Aborting migration. ' + e); + logger.err('Unable to create `state` table, aborting in 10 seconds. ' + e); + await sleep(10000); process.exit(-1); } logger.info('MIGRATIONS: `state` table initialized.'); @@ -34,7 +36,8 @@ class DatabaseMigration { try { databaseSchemaVersion = await this.$getSchemaVersionFromDatabase(); } catch (e) { - logger.err('Unable to get current database migration version, aborting. ' + e); + logger.err('Unable to get current database migration version, aborting in 10 seconds. ' + e); + await sleep(10000); process.exit(-1); } @@ -49,7 +52,8 @@ class DatabaseMigration { try { await this.$createMissingTablesAndIndexes(databaseSchemaVersion); } catch (e) { - logger.err('Unable to create required tables, aborting. ' + e); + logger.err('Unable to create required tables, aborting in 10 seconds. ' + e); + await sleep(10000); process.exit(-1); } From 2944f0b80563ae743729c027f95f41990d652ee4 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Wed, 12 Jan 2022 17:43:32 +0900 Subject: [PATCH 7/7] Added missing log tags --- backend/src/api/database-migration.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 64fa93761..2ac97636e 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -25,7 +25,7 @@ class DatabaseMigration { try { await this.$createMigrationStateTable(); } catch (e) { - logger.err('Unable to create `state` table, aborting in 10 seconds. ' + e); + logger.err('MIGRATIONS: Unable to create `state` table, aborting in 10 seconds. ' + e); await sleep(10000); process.exit(-1); } @@ -36,7 +36,7 @@ class DatabaseMigration { try { databaseSchemaVersion = await this.$getSchemaVersionFromDatabase(); } catch (e) { - logger.err('Unable to get current database migration version, aborting in 10 seconds. ' + e); + logger.err('MIGRATIONS: Unable to get current database migration version, aborting in 10 seconds. ' + e); await sleep(10000); process.exit(-1); } @@ -52,7 +52,7 @@ class DatabaseMigration { try { await this.$createMissingTablesAndIndexes(databaseSchemaVersion); } catch (e) { - logger.err('Unable to create required tables, aborting in 10 seconds. ' + e); + logger.err('MIGRATIONS: Unable to create required tables, aborting in 10 seconds. ' + e); await sleep(10000); process.exit(-1); } @@ -61,9 +61,9 @@ class DatabaseMigration { logger.info('MIGRATIONS: Upgrading datababse schema'); try { await this.$migrateTableSchemaFromVersion(databaseSchemaVersion); - logger.info(`OK. Database schema have been migrated from version ${databaseSchemaVersion} to ${DatabaseMigration.currentVersion} (latest version)`); + logger.info(`MIGRATIONS: OK. Database schema have been migrated from version ${databaseSchemaVersion} to ${DatabaseMigration.currentVersion} (latest version)`); } catch (e) { - logger.err('Unable to migrate database, aborting. ' + e); + logger.err('MIGRATIONS: Unable to migrate database, aborting. ' + e); } }