From 772765959b2713899d478712a82581b7f1f8f1ed Mon Sep 17 00:00:00 2001 From: junderw Date: Fri, 19 Aug 2022 22:00:47 +0900 Subject: [PATCH 01/23] Fix Difficulty API (REST) --- backend/src/api/difficulty-adjustment.ts | 36 ++++++++++--------- .../difficulty/difficulty.component.html | 2 +- .../difficulty/difficulty.component.ts | 4 +-- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/backend/src/api/difficulty-adjustment.ts b/backend/src/api/difficulty-adjustment.ts index 1f85fdb80..37a0e7a8d 100644 --- a/backend/src/api/difficulty-adjustment.ts +++ b/backend/src/api/difficulty-adjustment.ts @@ -6,33 +6,37 @@ class DifficultyAdjustmentApi { constructor() { } public getDifficultyAdjustment(): IDifficultyAdjustment { + const ESTIMATE_LAG_BLOCKS = 146; // For first 7.2% of epoch, don't estimate. + const EPOCH_BLOCK_LENGTH = 2016; + const DATime = blocks.getLastDifficultyAdjustmentTime(); const previousRetarget = blocks.getPreviousDifficultyRetarget(); const blockHeight = blocks.getCurrentBlockHeight(); const blocksCache = blocks.getBlocks(); const latestBlock = blocksCache[blocksCache.length - 1]; - const now = new Date().getTime() / 1000; - const diff = now - DATime; - const blocksInEpoch = blockHeight % 2016; - const progressPercent = (blocksInEpoch >= 0) ? blocksInEpoch / 2016 * 100 : 100; - const remainingBlocks = 2016 - blocksInEpoch; - const nextRetargetHeight = blockHeight + remainingBlocks; + const nowSeconds = Math.floor(new Date().getTime() / 1000); + const diffSeconds = nowSeconds - DATime; + const blocksInEpoch = (blockHeight >= 0) ? blockHeight % EPOCH_BLOCK_LENGTH : 0; + const progressPercent = (blockHeight >= 0) ? blocksInEpoch / EPOCH_BLOCK_LENGTH * 100 : 100; + const remainingBlocks = EPOCH_BLOCK_LENGTH - blocksInEpoch; + const nextRetargetHeight = (blockHeight >= 0) ? blockHeight + remainingBlocks : 0; let difficultyChange = 0; - if (remainingBlocks < 1870) { - if (blocksInEpoch > 0) { - difficultyChange = (600 / (diff / blocksInEpoch) - 1) * 100; - } + // Only calculate the estimate once we have 7.2% of blocks in current epoch + if (blocksInEpoch >= ESTIMATE_LAG_BLOCKS) { + difficultyChange = (600 / (diffSeconds / blocksInEpoch) - 1) * 100; + // Max increase is x4 (+300%) if (difficultyChange > 300) { difficultyChange = 300; } + // Max decrease is /4 (-75%) if (difficultyChange < -75) { difficultyChange = -75; } } - let timeAvgMins = blocksInEpoch && blocksInEpoch > 146 ? diff / blocksInEpoch / 60 : 10; + let timeAvgMins = blocksInEpoch >= ESTIMATE_LAG_BLOCKS ? diffSeconds / blocksInEpoch / 60 : 10; // Testnet difficulty is set to 1 after 20 minutes of no blocks, // therefore the time between blocks will always be below 20 minutes (1200s). @@ -41,14 +45,14 @@ class DifficultyAdjustmentApi { if (timeAvgMins > 20) { timeAvgMins = 20; } - if (now - latestBlock.timestamp + timeAvgMins * 60 > 1200) { - timeOffset = -Math.min(now - latestBlock.timestamp, 1200) * 1000; + if (nowSeconds - latestBlock.timestamp + timeAvgMins * 60 > 1200) { + timeOffset = -Math.min(nowSeconds - latestBlock.timestamp, 1200) * 1000; } } - const timeAvg = timeAvgMins * 60 * 1000 ; - const remainingTime = (remainingBlocks * timeAvg) + (now * 1000); - const estimatedRetargetDate = remainingTime + now; + const timeAvg = Math.floor(timeAvgMins * 60 * 1000); + const remainingTime = remainingBlocks * timeAvg; + const estimatedRetargetDate = remainingTime + nowSeconds * 1000; return { progressPercent, diff --git a/frontend/src/app/components/difficulty/difficulty.component.html b/frontend/src/app/components/difficulty/difficulty.component.html index 3684b8de4..e030f74fa 100644 --- a/frontend/src/app/components/difficulty/difficulty.component.html +++ b/frontend/src/app/components/difficulty/difficulty.component.html @@ -10,7 +10,7 @@ {{ i }} blocks {{ i }} block -
+
Estimate
diff --git a/frontend/src/app/components/difficulty/difficulty.component.ts b/frontend/src/app/components/difficulty/difficulty.component.ts index 5d969bf1e..76a996acc 100644 --- a/frontend/src/app/components/difficulty/difficulty.component.ts +++ b/frontend/src/app/components/difficulty/difficulty.component.ts @@ -11,7 +11,7 @@ interface EpochProgress { newDifficultyHeight: number; colorAdjustments: string; colorPreviousAdjustments: string; - remainingTime: number; + estimatedRetargetDate: number; previousRetarget: number; blocksUntilHalving: number; timeUntilHalving: number; @@ -74,7 +74,7 @@ export class DifficultyComponent implements OnInit { colorAdjustments, colorPreviousAdjustments, newDifficultyHeight: da.nextRetargetHeight, - remainingTime: da.remainingTime, + estimatedRetargetDate: da.estimatedRetargetDate, previousRetarget: da.previousRetarget, blocksUntilHalving, timeUntilHalving, From bb1adf41e788c02d949c13ac80a2f43560e5669b Mon Sep 17 00:00:00 2001 From: junderw Date: Fri, 19 Aug 2022 22:01:04 +0900 Subject: [PATCH 02/23] Update API docs for Difficulty API (REST) --- .../src/app/docs/api-docs/api-docs-data.ts | 44 ++++++++++++------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/frontend/src/app/docs/api-docs/api-docs-data.ts b/frontend/src/app/docs/api-docs/api-docs-data.ts index f8f0e23b8..1b02ceceb 100644 --- a/frontend/src/app/docs/api-docs/api-docs-data.ts +++ b/frontend/src/app/docs/api-docs/api-docs-data.ts @@ -114,11 +114,14 @@ export const restApiDocsData = [ curl: [], response: `{ progressPercent: 44.397234501112074, - difficultyChange: 0.9845932018381687, - estimatedRetargetDate: 1627762478.9111245, + difficultyChange: 98.45932018381687, + estimatedRetargetDate: 1627762478, remainingBlocks: 1121, - remainingTime: 665977.6261244365, - previousRetarget: -4.807005268478962 + remainingTime: 665977, + previousRetarget: -4.807005268478962, + nextRetargetHeight: 741888, + timeAvg: 302328, + timeOffset: 0 }` }, codeSampleTestnet: { @@ -127,11 +130,14 @@ export const restApiDocsData = [ curl: [], response: `{ progressPercent: 44.397234501112074, - difficultyChange: 0.9845932018381687, - estimatedRetargetDate: 1627762478.9111245, + difficultyChange: 98.45932018381687, + estimatedRetargetDate: 1627762478, remainingBlocks: 1121, - remainingTime: 665977.6261244365, - previousRetarget: -4.807005268478962 + remainingTime: 665977, + previousRetarget: -4.807005268478962, + nextRetargetHeight: 741888, + timeAvg: 302328, + timeOffset: 0 }` }, codeSampleSignet: { @@ -140,11 +146,14 @@ export const restApiDocsData = [ curl: [], response: `{ progressPercent: 44.397234501112074, - difficultyChange: 0.9845932018381687, - estimatedRetargetDate: 1627762478.9111245, + difficultyChange: 98.45932018381687, + estimatedRetargetDate: 1627762478, remainingBlocks: 1121, - remainingTime: 665977.6261244365, - previousRetarget: -4.807005268478962 + remainingTime: 665977, + previousRetarget: -4.807005268478962, + nextRetargetHeight: 741888, + timeAvg: 302328, + timeOffset: 0 }` }, codeSampleLiquid: { @@ -153,11 +162,14 @@ export const restApiDocsData = [ curl: [], response: `{ progressPercent: 44.397234501112074, - difficultyChange: 0.9845932018381687, - estimatedRetargetDate: 1627762478.9111245, + difficultyChange: 98.45932018381687, + estimatedRetargetDate: 1627762478, remainingBlocks: 1121, - remainingTime: 665977.6261244365, - previousRetarget: -4.807005268478962 + remainingTime: 665977, + previousRetarget: -4.807005268478962, + nextRetargetHeight: 741888, + timeAvg: 302328, + timeOffset: 0 }` } } From a00eb2736b6a52c548691d03f66a55712fd254c8 Mon Sep 17 00:00:00 2001 From: junderw Date: Fri, 19 Aug 2022 23:48:59 +0900 Subject: [PATCH 03/23] Refactor Difficulty Adjustment calc + unit test it --- backend/src/__tests__/config.test.ts | 58 ++++++++++ backend/src/api/difficulty-adjustment.ts | 131 ++++++++++++++--------- 2 files changed, 137 insertions(+), 52 deletions(-) diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index 9453505f4..b469d3e07 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -137,4 +137,62 @@ describe('Mempool Backend Config', () => { }); }); + test('should calculate Difficulty Adjustments properly', () => { + jest.isolateModules(() => { + const { calcDifficultyAdjustment } = jest.requireActual('../api/difficulty-adjustment'); + const dt = dtString => Math.floor(new Date(dtString).getTime() / 1000); + const vectors = [ + [ // Vector 1 + [ // Inputs + dt('2022-08-18T11:07:00.000Z'), // Last DA time (in seconds) + dt('2022-08-19T14:03:53.000Z'), // Current time (now) (in seconds) + 750134, // Current block height + 0.6280047707459726, // Previous retarget % (Passed through) + 'mainnet', // Network (if testnet, next value is non-zero) + 0, // If not testnet, not used + ], + { // Expected Result + progressPercent: 9.027777777777777, + difficultyChange: 12.562233927411782, + estimatedRetargetDate: 1661895424692, + remainingBlocks: 1834, + remainingTime: 977591692, + previousRetarget: 0.6280047707459726, + nextRetargetHeight: 751968, + timeAvg: 533038, + timeOffset: 0, + }, + ], + [ // Vector 2 (testnet) + [ // Inputs + dt('2022-08-18T11:07:00.000Z'), // Last DA time (in seconds) + dt('2022-08-19T14:03:53.000Z'), // Current time (now) (in seconds) + 750134, // Current block height + 0.6280047707459726, // Previous retarget % (Passed through) + 'testnet', // Network + dt('2022-08-19T13:52:46.000Z'), // Latest block timestamp in seconds + ], + { // Expected Result is same other than timeOffset + progressPercent: 9.027777777777777, + difficultyChange: 12.562233927411782, + estimatedRetargetDate: 1661895424692, + remainingBlocks: 1834, + remainingTime: 977591692, + previousRetarget: 0.6280047707459726, + nextRetargetHeight: 751968, + timeAvg: 533038, + timeOffset: -667000, // 11 min 7 seconds since last block (testnet only) + // If we add time avg to abs(timeOffset) it makes exactly 1200000 ms, or 20 minutes + }, + ], + ] as [[number, number, number, number, string, number], any][]; + + for (const vector of vectors) { + const result = calcDifficultyAdjustment(...vector[0]); + // previousRetarget is passed through untouched + expect(result.previousRetarget).toStrictEqual(vector[0][3]); + expect(result).toStrictEqual(vector[1]); + } + }); + }); }); diff --git a/backend/src/api/difficulty-adjustment.ts b/backend/src/api/difficulty-adjustment.ts index 37a0e7a8d..31d525342 100644 --- a/backend/src/api/difficulty-adjustment.ts +++ b/backend/src/api/difficulty-adjustment.ts @@ -2,69 +2,96 @@ import config from '../config'; import { IDifficultyAdjustment } from '../mempool.interfaces'; import blocks from './blocks'; +export interface DifficultyAdjustment { + progressPercent: number; // Percent: 0 to 100 + difficultyChange: number; // Percent: -75 to 300 + estimatedRetargetDate: number; // Unix time in ms + remainingBlocks: number; // Block count + remainingTime: number; // Duration of time in ms + previousRetarget: number; // Percent: -75 to 300 + nextRetargetHeight: number; // Block Height + timeAvg: number; // Duration of time in ms + timeOffset: number; // (Testnet) Time since last block (cap @ 20min) in ms +} + +export function calcDifficultyAdjustment( + DATime: number, + nowSeconds: number, + blockHeight: number, + previousRetarget: number, + network: string, + latestBlockTimestamp: number, +): DifficultyAdjustment { + const ESTIMATE_LAG_BLOCKS = 146; // For first 7.2% of epoch, don't estimate. + const EPOCH_BLOCK_LENGTH = 2016; + + const diffSeconds = nowSeconds - DATime; + const blocksInEpoch = (blockHeight >= 0) ? blockHeight % EPOCH_BLOCK_LENGTH : 0; + const progressPercent = (blockHeight >= 0) ? blocksInEpoch / EPOCH_BLOCK_LENGTH * 100 : 100; + const remainingBlocks = EPOCH_BLOCK_LENGTH - blocksInEpoch; + const nextRetargetHeight = (blockHeight >= 0) ? blockHeight + remainingBlocks : 0; + + let difficultyChange = 0; + // Only calculate the estimate once we have 7.2% of blocks in current epoch + if (blocksInEpoch >= ESTIMATE_LAG_BLOCKS) { + difficultyChange = (600 / (diffSeconds / blocksInEpoch) - 1) * 100; + // Max increase is x4 (+300%) + if (difficultyChange > 300) { + difficultyChange = 300; + } + // Max decrease is /4 (-75%) + if (difficultyChange < -75) { + difficultyChange = -75; + } + } + + let timeAvgMins = blocksInEpoch >= ESTIMATE_LAG_BLOCKS ? diffSeconds / blocksInEpoch / 60 : 10; + + // Testnet difficulty is set to 1 after 20 minutes of no blocks, + // therefore the time between blocks will always be below 20 minutes (1200s). + let timeOffset = 0; + if (network === 'testnet') { + if (timeAvgMins > 20) { + timeAvgMins = 20; + } + + if (nowSeconds - latestBlockTimestamp + timeAvgMins * 60 > 1200) { + timeOffset = -Math.min(nowSeconds - latestBlockTimestamp, 1200) * 1000; + } + } + + const timeAvg = Math.floor(timeAvgMins * 60 * 1000); + const remainingTime = remainingBlocks * timeAvg; + const estimatedRetargetDate = remainingTime + nowSeconds * 1000; + + return { + progressPercent, + difficultyChange, + estimatedRetargetDate, + remainingBlocks, + remainingTime, + previousRetarget, + nextRetargetHeight, + timeAvg, + timeOffset, + }; +} + class DifficultyAdjustmentApi { constructor() { } public getDifficultyAdjustment(): IDifficultyAdjustment { - const ESTIMATE_LAG_BLOCKS = 146; // For first 7.2% of epoch, don't estimate. - const EPOCH_BLOCK_LENGTH = 2016; - const DATime = blocks.getLastDifficultyAdjustmentTime(); const previousRetarget = blocks.getPreviousDifficultyRetarget(); const blockHeight = blocks.getCurrentBlockHeight(); const blocksCache = blocks.getBlocks(); const latestBlock = blocksCache[blocksCache.length - 1]; - const nowSeconds = Math.floor(new Date().getTime() / 1000); - const diffSeconds = nowSeconds - DATime; - const blocksInEpoch = (blockHeight >= 0) ? blockHeight % EPOCH_BLOCK_LENGTH : 0; - const progressPercent = (blockHeight >= 0) ? blocksInEpoch / EPOCH_BLOCK_LENGTH * 100 : 100; - const remainingBlocks = EPOCH_BLOCK_LENGTH - blocksInEpoch; - const nextRetargetHeight = (blockHeight >= 0) ? blockHeight + remainingBlocks : 0; - let difficultyChange = 0; - // Only calculate the estimate once we have 7.2% of blocks in current epoch - if (blocksInEpoch >= ESTIMATE_LAG_BLOCKS) { - difficultyChange = (600 / (diffSeconds / blocksInEpoch) - 1) * 100; - // Max increase is x4 (+300%) - if (difficultyChange > 300) { - difficultyChange = 300; - } - // Max decrease is /4 (-75%) - if (difficultyChange < -75) { - difficultyChange = -75; - } - } - - let timeAvgMins = blocksInEpoch >= ESTIMATE_LAG_BLOCKS ? diffSeconds / blocksInEpoch / 60 : 10; - - // Testnet difficulty is set to 1 after 20 minutes of no blocks, - // therefore the time between blocks will always be below 20 minutes (1200s). - let timeOffset = 0; - if (config.MEMPOOL.NETWORK === 'testnet') { - if (timeAvgMins > 20) { - timeAvgMins = 20; - } - if (nowSeconds - latestBlock.timestamp + timeAvgMins * 60 > 1200) { - timeOffset = -Math.min(nowSeconds - latestBlock.timestamp, 1200) * 1000; - } - } - - const timeAvg = Math.floor(timeAvgMins * 60 * 1000); - const remainingTime = remainingBlocks * timeAvg; - const estimatedRetargetDate = remainingTime + nowSeconds * 1000; - - return { - progressPercent, - difficultyChange, - estimatedRetargetDate, - remainingBlocks, - remainingTime, - previousRetarget, - nextRetargetHeight, - timeAvg, - timeOffset, - }; + return calcDifficultyAdjustment( + DATime, nowSeconds, blockHeight, previousRetarget, + config.MEMPOOL.NETWORK, latestBlock.timestamp + ); } } From 1bc2c1816792f2c52429122d73318dd986e9068d Mon Sep 17 00:00:00 2001 From: junderw Date: Sat, 20 Aug 2022 09:35:47 +0900 Subject: [PATCH 04/23] Fix tests --- backend/jest.config.ts | 7 ++- .../api/difficulty-adjustment.test.ts | 60 +++++++++++++++++++ backend/src/__tests__/config.test.ts | 59 ------------------ backend/testSetup.ts | 1 + 4 files changed, 66 insertions(+), 61 deletions(-) create mode 100644 backend/src/__tests__/api/difficulty-adjustment.test.ts create mode 100644 backend/testSetup.ts diff --git a/backend/jest.config.ts b/backend/jest.config.ts index 5576bfe80..14f932f98 100644 --- a/backend/jest.config.ts +++ b/backend/jest.config.ts @@ -7,11 +7,14 @@ const config: Config.InitialOptions = { automock: false, collectCoverage: true, collectCoverageFrom: ["./src/**/**.ts"], - coverageProvider: "v8", + coverageProvider: "babel", coverageThreshold: { global: { lines: 1 } - } + }, + setupFiles: [ + "./testSetup.ts", + ], } export default config; diff --git a/backend/src/__tests__/api/difficulty-adjustment.test.ts b/backend/src/__tests__/api/difficulty-adjustment.test.ts new file mode 100644 index 000000000..4cee25c2f --- /dev/null +++ b/backend/src/__tests__/api/difficulty-adjustment.test.ts @@ -0,0 +1,60 @@ +import { calcDifficultyAdjustment, DifficultyAdjustment } from '../../api/difficulty-adjustment'; + +test('should calculate Difficulty Adjustments properly', () => { + const dt = (dtString) => { + return Math.floor(new Date(dtString).getTime() / 1000); + }; + + const vectors = [ + [ // Vector 1 + [ // Inputs + dt('2022-08-18T11:07:00.000Z'), // Last DA time (in seconds) + dt('2022-08-19T14:03:53.000Z'), // Current time (now) (in seconds) + 750134, // Current block height + 0.6280047707459726, // Previous retarget % (Passed through) + 'mainnet', // Network (if testnet, next value is non-zero) + 0, // If not testnet, not used + ], + { // Expected Result + progressPercent: 9.027777777777777, + difficultyChange: 12.562233927411782, + estimatedRetargetDate: 1661895424692, + remainingBlocks: 1834, + remainingTime: 977591692, + previousRetarget: 0.6280047707459726, + nextRetargetHeight: 751968, + timeAvg: 533038, + timeOffset: 0, + }, + ], + [ // Vector 2 (testnet) + [ // Inputs + dt('2022-08-18T11:07:00.000Z'), // Last DA time (in seconds) + dt('2022-08-19T14:03:53.000Z'), // Current time (now) (in seconds) + 750134, // Current block height + 0.6280047707459726, // Previous retarget % (Passed through) + 'testnet', // Network + dt('2022-08-19T13:52:46.000Z'), // Latest block timestamp in seconds + ], + { // Expected Result is same other than timeOffset + progressPercent: 9.027777777777777, + difficultyChange: 12.562233927411782, + estimatedRetargetDate: 1661895424692, + remainingBlocks: 1834, + remainingTime: 977591692, + previousRetarget: 0.6280047707459726, + nextRetargetHeight: 751968, + timeAvg: 533038, + timeOffset: -667000, // 11 min 7 seconds since last block (testnet only) + // If we add time avg to abs(timeOffset) it makes exactly 1200000 ms, or 20 minutes + }, + ], + ] as [[number, number, number, number, string, number], DifficultyAdjustment][]; + + for (const vector of vectors) { + const result = calcDifficultyAdjustment(...vector[0]); + // previousRetarget is passed through untouched + expect(result.previousRetarget).toStrictEqual(vector[0][3]); + expect(result).toStrictEqual(vector[1]); + } +}); diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index b469d3e07..7314fde6f 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -136,63 +136,4 @@ describe('Mempool Backend Config', () => { expect(config.EXTERNAL_DATA_SERVER).toStrictEqual(fixture.EXTERNAL_DATA_SERVER); }); }); - - test('should calculate Difficulty Adjustments properly', () => { - jest.isolateModules(() => { - const { calcDifficultyAdjustment } = jest.requireActual('../api/difficulty-adjustment'); - const dt = dtString => Math.floor(new Date(dtString).getTime() / 1000); - const vectors = [ - [ // Vector 1 - [ // Inputs - dt('2022-08-18T11:07:00.000Z'), // Last DA time (in seconds) - dt('2022-08-19T14:03:53.000Z'), // Current time (now) (in seconds) - 750134, // Current block height - 0.6280047707459726, // Previous retarget % (Passed through) - 'mainnet', // Network (if testnet, next value is non-zero) - 0, // If not testnet, not used - ], - { // Expected Result - progressPercent: 9.027777777777777, - difficultyChange: 12.562233927411782, - estimatedRetargetDate: 1661895424692, - remainingBlocks: 1834, - remainingTime: 977591692, - previousRetarget: 0.6280047707459726, - nextRetargetHeight: 751968, - timeAvg: 533038, - timeOffset: 0, - }, - ], - [ // Vector 2 (testnet) - [ // Inputs - dt('2022-08-18T11:07:00.000Z'), // Last DA time (in seconds) - dt('2022-08-19T14:03:53.000Z'), // Current time (now) (in seconds) - 750134, // Current block height - 0.6280047707459726, // Previous retarget % (Passed through) - 'testnet', // Network - dt('2022-08-19T13:52:46.000Z'), // Latest block timestamp in seconds - ], - { // Expected Result is same other than timeOffset - progressPercent: 9.027777777777777, - difficultyChange: 12.562233927411782, - estimatedRetargetDate: 1661895424692, - remainingBlocks: 1834, - remainingTime: 977591692, - previousRetarget: 0.6280047707459726, - nextRetargetHeight: 751968, - timeAvg: 533038, - timeOffset: -667000, // 11 min 7 seconds since last block (testnet only) - // If we add time avg to abs(timeOffset) it makes exactly 1200000 ms, or 20 minutes - }, - ], - ] as [[number, number, number, number, string, number], any][]; - - for (const vector of vectors) { - const result = calcDifficultyAdjustment(...vector[0]); - // previousRetarget is passed through untouched - expect(result.previousRetarget).toStrictEqual(vector[0][3]); - expect(result).toStrictEqual(vector[1]); - } - }); - }); }); diff --git a/backend/testSetup.ts b/backend/testSetup.ts new file mode 100644 index 000000000..dda44ef6e --- /dev/null +++ b/backend/testSetup.ts @@ -0,0 +1 @@ +jest.mock('./mempool-config.json', () => ({}), { virtual: true }); From d700b5f1450127cf4cf7e4c0d5f15b56e5cc1568 Mon Sep 17 00:00:00 2001 From: junderw Date: Sat, 20 Aug 2022 09:46:18 +0900 Subject: [PATCH 05/23] Fix test setup --- .../api/difficulty-adjustment.test.ts | 108 +++++++++--------- backend/testSetup.ts | 4 + 2 files changed, 59 insertions(+), 53 deletions(-) diff --git a/backend/src/__tests__/api/difficulty-adjustment.test.ts b/backend/src/__tests__/api/difficulty-adjustment.test.ts index 4cee25c2f..eb774d445 100644 --- a/backend/src/__tests__/api/difficulty-adjustment.test.ts +++ b/backend/src/__tests__/api/difficulty-adjustment.test.ts @@ -1,60 +1,62 @@ import { calcDifficultyAdjustment, DifficultyAdjustment } from '../../api/difficulty-adjustment'; -test('should calculate Difficulty Adjustments properly', () => { - const dt = (dtString) => { - return Math.floor(new Date(dtString).getTime() / 1000); - }; +describe('Mempool Difficulty Adjustment', () => { + test('should calculate Difficulty Adjustments properly', () => { + const dt = (dtString) => { + return Math.floor(new Date(dtString).getTime() / 1000); + }; - const vectors = [ - [ // Vector 1 - [ // Inputs - dt('2022-08-18T11:07:00.000Z'), // Last DA time (in seconds) - dt('2022-08-19T14:03:53.000Z'), // Current time (now) (in seconds) - 750134, // Current block height - 0.6280047707459726, // Previous retarget % (Passed through) - 'mainnet', // Network (if testnet, next value is non-zero) - 0, // If not testnet, not used + const vectors = [ + [ // Vector 1 + [ // Inputs + dt('2022-08-18T11:07:00.000Z'), // Last DA time (in seconds) + dt('2022-08-19T14:03:53.000Z'), // Current time (now) (in seconds) + 750134, // Current block height + 0.6280047707459726, // Previous retarget % (Passed through) + 'mainnet', // Network (if testnet, next value is non-zero) + 0, // If not testnet, not used + ], + { // Expected Result + progressPercent: 9.027777777777777, + difficultyChange: 12.562233927411782, + estimatedRetargetDate: 1661895424692, + remainingBlocks: 1834, + remainingTime: 977591692, + previousRetarget: 0.6280047707459726, + nextRetargetHeight: 751968, + timeAvg: 533038, + timeOffset: 0, + }, ], - { // Expected Result - progressPercent: 9.027777777777777, - difficultyChange: 12.562233927411782, - estimatedRetargetDate: 1661895424692, - remainingBlocks: 1834, - remainingTime: 977591692, - previousRetarget: 0.6280047707459726, - nextRetargetHeight: 751968, - timeAvg: 533038, - timeOffset: 0, - }, - ], - [ // Vector 2 (testnet) - [ // Inputs - dt('2022-08-18T11:07:00.000Z'), // Last DA time (in seconds) - dt('2022-08-19T14:03:53.000Z'), // Current time (now) (in seconds) - 750134, // Current block height - 0.6280047707459726, // Previous retarget % (Passed through) - 'testnet', // Network - dt('2022-08-19T13:52:46.000Z'), // Latest block timestamp in seconds + [ // Vector 2 (testnet) + [ // Inputs + dt('2022-08-18T11:07:00.000Z'), // Last DA time (in seconds) + dt('2022-08-19T14:03:53.000Z'), // Current time (now) (in seconds) + 750134, // Current block height + 0.6280047707459726, // Previous retarget % (Passed through) + 'testnet', // Network + dt('2022-08-19T13:52:46.000Z'), // Latest block timestamp in seconds + ], + { // Expected Result is same other than timeOffset + progressPercent: 9.027777777777777, + difficultyChange: 12.562233927411782, + estimatedRetargetDate: 1661895424692, + remainingBlocks: 1834, + remainingTime: 977591692, + previousRetarget: 0.6280047707459726, + nextRetargetHeight: 751968, + timeAvg: 533038, + timeOffset: -667000, // 11 min 7 seconds since last block (testnet only) + // If we add time avg to abs(timeOffset) it makes exactly 1200000 ms, or 20 minutes + }, ], - { // Expected Result is same other than timeOffset - progressPercent: 9.027777777777777, - difficultyChange: 12.562233927411782, - estimatedRetargetDate: 1661895424692, - remainingBlocks: 1834, - remainingTime: 977591692, - previousRetarget: 0.6280047707459726, - nextRetargetHeight: 751968, - timeAvg: 533038, - timeOffset: -667000, // 11 min 7 seconds since last block (testnet only) - // If we add time avg to abs(timeOffset) it makes exactly 1200000 ms, or 20 minutes - }, - ], - ] as [[number, number, number, number, string, number], DifficultyAdjustment][]; + ] as [[number, number, number, number, string, number], DifficultyAdjustment][]; - for (const vector of vectors) { - const result = calcDifficultyAdjustment(...vector[0]); - // previousRetarget is passed through untouched - expect(result.previousRetarget).toStrictEqual(vector[0][3]); - expect(result).toStrictEqual(vector[1]); - } + for (const vector of vectors) { + const result = calcDifficultyAdjustment(...vector[0]); + // previousRetarget is passed through untouched + expect(result.previousRetarget).toStrictEqual(vector[0][3]); + expect(result).toStrictEqual(vector[1]); + } + }); }); diff --git a/backend/testSetup.ts b/backend/testSetup.ts index dda44ef6e..ca51bbbe6 100644 --- a/backend/testSetup.ts +++ b/backend/testSetup.ts @@ -1 +1,5 @@ jest.mock('./mempool-config.json', () => ({}), { virtual: true }); +jest.mock('./src/logger.ts', () => ({}), { virtual: true }); +jest.mock('./src/api/rbf-cache.ts', () => ({}), { virtual: true }); +jest.mock('./src/api/mempool.ts', () => ({}), { virtual: true }); +jest.mock('./src/api/memory-cache.ts', () => ({}), { virtual: true }); From 87c9f881c06e6f5b4210c672647c106411521a23 Mon Sep 17 00:00:00 2001 From: junderw Date: Sat, 20 Aug 2022 11:24:48 +0900 Subject: [PATCH 06/23] Refactor difficulty API logic --- backend/src/api/difficulty-adjustment.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/backend/src/api/difficulty-adjustment.ts b/backend/src/api/difficulty-adjustment.ts index 31d525342..ce023931d 100644 --- a/backend/src/api/difficulty-adjustment.ts +++ b/backend/src/api/difficulty-adjustment.ts @@ -23,7 +23,8 @@ export function calcDifficultyAdjustment( latestBlockTimestamp: number, ): DifficultyAdjustment { const ESTIMATE_LAG_BLOCKS = 146; // For first 7.2% of epoch, don't estimate. - const EPOCH_BLOCK_LENGTH = 2016; + const EPOCH_BLOCK_LENGTH = 2016; // Bitcoin mainnet + const BLOCK_SECONDS_TARGET = 600; // Bitcoin mainnet const diffSeconds = nowSeconds - DATime; const blocksInEpoch = (blockHeight >= 0) ? blockHeight % EPOCH_BLOCK_LENGTH : 0; @@ -32,9 +33,12 @@ export function calcDifficultyAdjustment( const nextRetargetHeight = (blockHeight >= 0) ? blockHeight + remainingBlocks : 0; let difficultyChange = 0; + let timeAvgMins = 10; // Only calculate the estimate once we have 7.2% of blocks in current epoch if (blocksInEpoch >= ESTIMATE_LAG_BLOCKS) { - difficultyChange = (600 / (diffSeconds / blocksInEpoch) - 1) * 100; + const secondsPerBlock = diffSeconds / blocksInEpoch; + timeAvgMins = secondsPerBlock / 60; + difficultyChange = (BLOCK_SECONDS_TARGET / secondsPerBlock - 1) * 100; // Max increase is x4 (+300%) if (difficultyChange > 300) { difficultyChange = 300; @@ -45,8 +49,6 @@ export function calcDifficultyAdjustment( } } - let timeAvgMins = blocksInEpoch >= ESTIMATE_LAG_BLOCKS ? diffSeconds / blocksInEpoch / 60 : 10; - // Testnet difficulty is set to 1 after 20 minutes of no blocks, // therefore the time between blocks will always be below 20 minutes (1200s). let timeOffset = 0; From 3a4982c5e65f7d40718144033dadb196d1c9dcb2 Mon Sep 17 00:00:00 2001 From: junderw Date: Sat, 20 Aug 2022 13:02:22 +0900 Subject: [PATCH 07/23] Refactor Difficulty API logic --- backend/src/api/difficulty-adjustment.ts | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/backend/src/api/difficulty-adjustment.ts b/backend/src/api/difficulty-adjustment.ts index ce023931d..43c6e463f 100644 --- a/backend/src/api/difficulty-adjustment.ts +++ b/backend/src/api/difficulty-adjustment.ts @@ -25,6 +25,7 @@ export function calcDifficultyAdjustment( const ESTIMATE_LAG_BLOCKS = 146; // For first 7.2% of epoch, don't estimate. const EPOCH_BLOCK_LENGTH = 2016; // Bitcoin mainnet const BLOCK_SECONDS_TARGET = 600; // Bitcoin mainnet + const TESTNET_MAX_BLOCK_SECONDS = 1200; // Bitcoin testnet const diffSeconds = nowSeconds - DATime; const blocksInEpoch = (blockHeight >= 0) ? blockHeight % EPOCH_BLOCK_LENGTH : 0; @@ -33,12 +34,11 @@ export function calcDifficultyAdjustment( const nextRetargetHeight = (blockHeight >= 0) ? blockHeight + remainingBlocks : 0; let difficultyChange = 0; - let timeAvgMins = 10; + let timeAvgSecs = BLOCK_SECONDS_TARGET; // Only calculate the estimate once we have 7.2% of blocks in current epoch if (blocksInEpoch >= ESTIMATE_LAG_BLOCKS) { - const secondsPerBlock = diffSeconds / blocksInEpoch; - timeAvgMins = secondsPerBlock / 60; - difficultyChange = (BLOCK_SECONDS_TARGET / secondsPerBlock - 1) * 100; + timeAvgSecs = diffSeconds / blocksInEpoch; + difficultyChange = (BLOCK_SECONDS_TARGET / timeAvgSecs - 1) * 100; // Max increase is x4 (+300%) if (difficultyChange > 300) { difficultyChange = 300; @@ -53,16 +53,17 @@ export function calcDifficultyAdjustment( // therefore the time between blocks will always be below 20 minutes (1200s). let timeOffset = 0; if (network === 'testnet') { - if (timeAvgMins > 20) { - timeAvgMins = 20; + if (timeAvgSecs > TESTNET_MAX_BLOCK_SECONDS) { + timeAvgSecs = TESTNET_MAX_BLOCK_SECONDS; } - if (nowSeconds - latestBlockTimestamp + timeAvgMins * 60 > 1200) { - timeOffset = -Math.min(nowSeconds - latestBlockTimestamp, 1200) * 1000; + const secondsSinceLastBlock = nowSeconds - latestBlockTimestamp; + if (secondsSinceLastBlock + timeAvgSecs > TESTNET_MAX_BLOCK_SECONDS) { + timeOffset = -Math.min(secondsSinceLastBlock, TESTNET_MAX_BLOCK_SECONDS) * 1000; } } - const timeAvg = Math.floor(timeAvgMins * 60 * 1000); + const timeAvg = Math.floor(timeAvgSecs * 1000); const remainingTime = remainingBlocks * timeAvg; const estimatedRetargetDate = remainingTime + nowSeconds * 1000; From 755ac276f74b85096a210958297d758f09d5accc Mon Sep 17 00:00:00 2001 From: nymkappa Date: Tue, 23 Aug 2022 18:00:40 +0200 Subject: [PATCH 08/23] Node history chart full width if we can't display channel map --- frontend/src/app/lightning/node/node.component.html | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frontend/src/app/lightning/node/node.component.html b/frontend/src/app/lightning/node/node.component.html index 12ca8eaa1..c81978bbc 100644 --- a/frontend/src/app/lightning/node/node.component.html +++ b/frontend/src/app/lightning/node/node.component.html @@ -119,7 +119,7 @@
-
+
@@ -127,6 +127,9 @@
+
+ +

Active channels map

From 35512bef8d1906982a28dc8e0b9d30750b0fa929 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Tue, 23 Aug 2022 22:47:18 +0200 Subject: [PATCH 09/23] Do not fetch node stats for channel tree graph --- backend/src/api/explorer/channels.api.ts | 45 ++++++++++++++++-------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/backend/src/api/explorer/channels.api.ts b/backend/src/api/explorer/channels.api.ts index a0a617e43..b5eac7499 100644 --- a/backend/src/api/explorer/channels.api.ts +++ b/backend/src/api/explorer/channels.api.ts @@ -288,21 +288,36 @@ class ChannelsApi { const channels: any[] = [] for (const row of allChannels) { - const activeChannelsStats: any = await nodesApi.$getActiveChannelsStats(row.public_key); - channels.push({ - status: row.status, - closing_reason: row.closing_reason, - capacity: row.capacity ?? 0, - short_id: row.short_id, - id: row.id, - fee_rate: row.node1_fee_rate ?? row.node2_fee_rate ?? 0, - node: { - alias: row.alias.length > 0 ? row.alias : row.public_key.slice(0, 20), - public_key: row.public_key, - channels: activeChannelsStats.active_channel_count ?? 0, - capacity: activeChannelsStats.capacity ?? 0, - } - }); + let channel; + if (index >= 0) { + const activeChannelsStats: any = await nodesApi.$getActiveChannelsStats(row.public_key); + channel = { + status: row.status, + closing_reason: row.closing_reason, + capacity: row.capacity ?? 0, + short_id: row.short_id, + id: row.id, + fee_rate: row.node1_fee_rate ?? row.node2_fee_rate ?? 0, + node: { + alias: row.alias.length > 0 ? row.alias : row.public_key.slice(0, 20), + public_key: row.public_key, + channels: activeChannelsStats.active_channel_count ?? 0, + capacity: activeChannelsStats.capacity ?? 0, + } + }; + } else if (index === -1) { + channel = { + capacity: row.capacity ?? 0, + short_id: row.short_id, + id: row.id, + node: { + alias: row.alias.length > 0 ? row.alias : row.public_key.slice(0, 20), + public_key: row.public_key, + } + }; + } + + channels.push(channel); } return channels; From 43cc9499b1dbfdd18ca0c12a1e4e07aa2e9462ef Mon Sep 17 00:00:00 2001 From: nymkappa Date: Wed, 24 Aug 2022 08:35:02 +0200 Subject: [PATCH 10/23] Check query input before running the mysql query --- backend/src/api/explorer/channels.routes.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/backend/src/api/explorer/channels.routes.ts b/backend/src/api/explorer/channels.routes.ts index 0fa91db92..eda3a6168 100644 --- a/backend/src/api/explorer/channels.routes.ts +++ b/backend/src/api/explorer/channels.routes.ts @@ -47,8 +47,17 @@ class ChannelsRoutes { res.status(400).send('Missing parameter: public_key'); return; } + const index = parseInt(typeof req.query.index === 'string' ? req.query.index : '0', 10) || 0; const status: string = typeof req.query.status === 'string' ? req.query.status : ''; + + if (index < -1) { + res.status(400).send('Invalid index'); + } + if (['open', 'active', 'closed'].includes(status) === false) { + res.status(400).send('Invalid status'); + } + const channels = await channelsApi.$getChannelsForNode(req.query.public_key, index, 10, status); const channelsCount = await channelsApi.$getChannelsCountForNode(req.query.public_key, status); res.header('Pragma', 'public'); From a151a90d2fd429edc3f522319e7c6defa9addcd8 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Wed, 24 Aug 2022 12:12:16 +0200 Subject: [PATCH 11/23] Import historical nodes --- .../lightning/sync-tasks/stats-importer.ts | 69 +++++++++++-------- 1 file changed, 42 insertions(+), 27 deletions(-) diff --git a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts index e05ba4ab3..30cfcc62b 100644 --- a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts +++ b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts @@ -7,6 +7,7 @@ import { ILightningApi } from '../../../api/lightning/lightning-api.interface'; import { isIP } from 'net'; import { Common } from '../../../api/common'; import channelsApi from '../../../api/explorer/channels.api'; +import nodesApi from '../../../api/explorer/nodes.api'; const fsPromises = promises; @@ -32,7 +33,26 @@ class LightningStatsImporter { let clearnetTorNodes = 0; let unannouncedNodes = 0; + const [nodesInDbRaw]: any[] = await DB.query(`SELECT public_key FROM nodes`); + const nodesInDb = {}; + for (const node of nodesInDbRaw) { + nodesInDb[node.public_key] = node; + } + for (const node of networkGraph.nodes) { + // If we don't know about this node, insert it in db + if (isHistorical === true && !nodesInDb[node.pub_key]) { + await nodesApi.$saveNode({ + last_update: node.last_update, + pub_key: node.pub_key, + alias: node.alias, + addresses: node.addresses, + color: node.color, + features: node.features, + }); + nodesInDb[node.pub_key] = node; + } + let hasOnion = false; let hasClearnet = false; let isUnnanounced = true; @@ -69,7 +89,7 @@ class LightningStatsImporter { const baseFees: number[] = []; const alreadyCountedChannels = {}; - const [channelsInDbRaw]: any[] = await DB.query(`SELECT short_id, created FROM channels`); + const [channelsInDbRaw]: any[] = await DB.query(`SELECT short_id FROM channels`); const channelsInDb = {}; for (const channel of channelsInDbRaw) { channelsInDb[channel.short_id] = channel; @@ -84,29 +104,19 @@ class LightningStatsImporter { continue; } - // Channel is already in db, check if we need to update 'created' field - if (isHistorical === true) { - //@ts-ignore - if (channelsInDb[short_id] && channel.timestamp < channel.created) { - await DB.query(` - UPDATE channels SET created = FROM_UNIXTIME(?) WHERE channels.short_id = ?`, - //@ts-ignore - [channel.timestamp, short_id] - ); - } else if (!channelsInDb[short_id]) { - await channelsApi.$saveChannel({ - channel_id: short_id, - chan_point: `${tx.txid}:${short_id.split('x')[2]}`, - //@ts-ignore - last_update: channel.timestamp, - node1_pub: channel.node1_pub, - node2_pub: channel.node2_pub, - capacity: (tx.value * 100000000).toString(), - node1_policy: null, - node2_policy: null, - }, 0); - channelsInDb[channel.channel_id] = channel; - } + // If we don't know about this channel, insert it in db + if (isHistorical === true && !channelsInDb[short_id]) { + await channelsApi.$saveChannel({ + channel_id: short_id, + chan_point: `${tx.txid}:${short_id.split('x')[2]}`, + last_update: channel.last_update, + node1_pub: channel.node1_pub, + node2_pub: channel.node2_pub, + capacity: (tx.value * 100000000).toString(), + node1_policy: null, + node2_policy: null, + }, 0); + channelsInDb[channel.channel_id] = channel; } if (!nodeStats[channel.node1_pub]) { @@ -281,6 +291,7 @@ class LightningStatsImporter { * Import topology files LN historical data into the database */ async $importHistoricalLightningStats(): Promise { + logger.debug('Run the historical importer'); try { let fileList: string[] = []; try { @@ -294,7 +305,7 @@ class LightningStatsImporter { fileList.sort().reverse(); const [rows]: any[] = await DB.query(` - SELECT UNIX_TIMESTAMP(added) AS added, node_count + SELECT UNIX_TIMESTAMP(added) AS added FROM lightning_stats ORDER BY added DESC `); @@ -391,12 +402,16 @@ class LightningStatsImporter { }); } + let rgb = node.rgb_color ?? '#000000'; + if (rgb.indexOf('#') === -1) { + rgb = `#${rgb}`; + } newGraph.nodes.push({ last_update: node.timestamp ?? 0, pub_key: node.id ?? null, - alias: node.alias ?? null, + alias: node.alias ?? node.id.slice(0, 20), addresses: addresses, - color: node.rgb_color ?? null, + color: rgb, features: {}, }); } From dc4cd96fc0404b40f04b05fa78757b8ca64281b5 Mon Sep 17 00:00:00 2001 From: Stephan Oeste Date: Wed, 24 Aug 2022 16:08:17 +0200 Subject: [PATCH 12/23] Fix path for newsyslog configs in prod installer --- production/install | 29 +++++++++++++++++++++-------- production/linux/rsyslog.conf | 2 ++ 2 files changed, 23 insertions(+), 8 deletions(-) create mode 100644 production/linux/rsyslog.conf diff --git a/production/install b/production/install index 729ff33e0..db16484e7 100755 --- a/production/install +++ b/production/install @@ -911,15 +911,28 @@ osSudo "${MEMPOOL_USER}" ln -s mempool/production/mempool-kill-all stop osSudo "${MEMPOOL_USER}" ln -s mempool/production/mempool-start-all start osSudo "${MEMPOOL_USER}" ln -s mempool/production/mempool-restart-all restart -echo "[*] Installing syslog configuration" -osSudo "${ROOT_USER}" mkdir -p /usr/local/etc/syslog.d -osSudo "${ROOT_USER}" install -c -m 755 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/mempool-logger" /usr/local/bin/mempool-logger -osSudo "${ROOT_USER}" install -c -m 644 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/syslog.conf" /usr/local/etc/syslog.d/mempool.conf -echo "[*] Installing newsyslog configuration" -osSudo "${ROOT_USER}" mkdir -p /usr/local/etc/newsyslog.conf.d -osSudo "${ROOT_USER}" install -c -m 644 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/newsyslog-mempool-backend.conf" /usr/local/etc/syslog.d/newsyslog-mempool-backend.conf -osSudo "${ROOT_USER}" install -c -m 644 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/newsyslog-mempool-nginx.conf" /usr/local/etc/syslog.d/newsyslog-mempool-nginx.conf +case $OS in + FreeBSD) + echo "[*] Installing syslog configuration" + osSudo "${ROOT_USER}" mkdir -p /usr/local/etc/syslog.d + osSudo "${ROOT_USER}" install -c -m 755 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/mempool-logger" /usr/local/bin/mempool-logger + osSudo "${ROOT_USER}" install -c -m 644 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/syslog.conf" /usr/local/etc/syslog.d/mempool.conf + + echo "[*] Installing newsyslog configuration" + osSudo "${ROOT_USER}" mkdir -p /usr/local/etc/newsyslog.conf.d + osSudo "${ROOT_USER}" install -c -m 644 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/newsyslog-mempool-backend.conf" /usr/local/etc/newsyslog.conf.d/newsyslog-mempool-backend.conf + osSudo "${ROOT_USER}" install -c -m 644 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/newsyslog-mempool-nginx.conf" /usr/local/etc/newsyslog.conf.d/newsyslog-mempool-nginx.conf + + echo "[*] Creating log files" + osSudo "${ROOT_USER}" newsyslog -C + ;; + Debian) + echo "[*] Installing syslog configuration" + osSudo "${ROOT_USER}" install -c -m 644 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/linux/rsyslog.conf" /etc/rsyslog.d/10-mempool.conf + osSudo "${ROOT_USER}" sed -i.orig -e 's/^\*\.\*;auth,authpriv\.none/*\.*;auth,authpriv\.none,local7\.none/' /etc/rsyslog.d/50-default.conf + ;; +esac echo "[*] Installing Mempool crontab" osSudo "${ROOT_USER}" crontab -u "${MEMPOOL_USER}" "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/mempool.crontab" diff --git a/production/linux/rsyslog.conf b/production/linux/rsyslog.conf new file mode 100644 index 000000000..92ce6e591 --- /dev/null +++ b/production/linux/rsyslog.conf @@ -0,0 +1,2 @@ +local7.info /var/log/mempool +local7.* /var/log/mempool.debug From b6bea12bcaf36cab782251f87a58a01228cf2045 Mon Sep 17 00:00:00 2001 From: softsimon Date: Thu, 25 Aug 2022 15:14:54 +0400 Subject: [PATCH 13/23] Numbers not included in node alias search --- backend/src/api/explorer/nodes.api.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/api/explorer/nodes.api.ts b/backend/src/api/explorer/nodes.api.ts index 379df4213..e4730b5e9 100644 --- a/backend/src/api/explorer/nodes.api.ts +++ b/backend/src/api/explorer/nodes.api.ts @@ -260,7 +260,7 @@ class NodesApi { public async $searchNodeByPublicKeyOrAlias(search: string) { try { const publicKeySearch = search.replace('%', '') + '%'; - const aliasSearch = search.replace(/[-_.]/g, ' ').replace(/[^a-zA-Z ]/g, '').split(' ').map((search) => '+' + search + '*').join(' '); + const aliasSearch = search.replace(/[-_.]/g, ' ').replace(/[^a-zA-Z0-9 ]/g, '').split(' ').map((search) => '+' + search + '*').join(' '); const query = `SELECT nodes.public_key, nodes.alias, node_stats.capacity FROM nodes LEFT JOIN node_stats ON node_stats.public_key = nodes.public_key WHERE nodes.public_key LIKE ? OR MATCH nodes.alias_search AGAINST (? IN BOOLEAN MODE) GROUP BY nodes.public_key ORDER BY node_stats.capacity DESC LIMIT 10`; const [rows]: any = await DB.query(query, [publicKeySearch, aliasSearch]); return rows; @@ -555,7 +555,7 @@ class NodesApi { } private aliasToSearchText(str: string): string { - return str.replace(/[-_.]/g, ' ').replace(/[^a-zA-Z ]/g, ''); + return str.replace(/[-_.]/g, ' ').replace(/[^a-zA-Z0-9 ]/g, ''); } } From a5410178c87ef1a40d8b3477f6be8b1697b27e10 Mon Sep 17 00:00:00 2001 From: softsimon Date: Thu, 25 Aug 2022 17:24:42 +0400 Subject: [PATCH 14/23] Fix for empty tx value fixes #2263 --- .../transactions-list/transactions-list.component.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/app/components/transactions-list/transactions-list.component.ts b/frontend/src/app/components/transactions-list/transactions-list.component.ts index 69b65a8a4..2a317e738 100644 --- a/frontend/src/app/components/transactions-list/transactions-list.component.ts +++ b/frontend/src/app/components/transactions-list/transactions-list.component.ts @@ -105,10 +105,10 @@ export class TransactionsListComponent implements OnInit, OnChanges { }, 10); } - this.transactions.forEach((tx, i) => { + this.transactions.forEach((tx) => { tx['@voutLimit'] = true; tx['@vinLimit'] = true; - if (this.outspends[i]) { + if (tx['addressValue'] !== undefined) { return; } From cfb5f2f3b5c6f641d5ad96265881b0fb0f5370f4 Mon Sep 17 00:00:00 2001 From: softsimon Date: Thu, 25 Aug 2022 19:29:21 +0400 Subject: [PATCH 15/23] Search bar placeholder text update --- .../src/app/components/search-form/search-form.component.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/app/components/search-form/search-form.component.html b/frontend/src/app/components/search-form/search-form.component.html index 417414b58..378e38940 100644 --- a/frontend/src/app/components/search-form/search-form.component.html +++ b/frontend/src/app/components/search-form/search-form.component.html @@ -1,7 +1,7 @@
- + From 85c428be250842add2405ade617b1d7dce724e9e Mon Sep 17 00:00:00 2001 From: softsimon Date: Fri, 26 Aug 2022 02:31:14 +0400 Subject: [PATCH 16/23] Store capacity and channels in nodes table --- backend/src/api/database-migration.ts | 8 +++++++- backend/src/api/explorer/nodes.api.ts | 2 +- .../src/tasks/lightning/sync-tasks/stats-importer.ts | 11 +++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 1d6f10a2c..1dc0b9704 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -4,7 +4,7 @@ import logger from '../logger'; import { Common } from './common'; class DatabaseMigration { - private static currentVersion = 39; + private static currentVersion = 40; private queryTimeout = 120000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -342,6 +342,12 @@ class DatabaseMigration { await this.$executeQuery('ALTER TABLE `nodes` ADD alias_search TEXT NULL DEFAULT NULL AFTER `alias`'); await this.$executeQuery('ALTER TABLE nodes ADD FULLTEXT(alias_search)'); } + + if (databaseSchemaVersion < 40 && isBitcoin === true) { + await this.$executeQuery('ALTER TABLE `nodes` ADD capacity bigint(20) unsigned DEFAULT NULL'); + await this.$executeQuery('ALTER TABLE `nodes` ADD channels int(11) unsigned DEFAULT NULL'); + await this.$executeQuery('ALTER TABLE `nodes` ADD INDEX `capacity` (`capacity`);'); + } } /** diff --git a/backend/src/api/explorer/nodes.api.ts b/backend/src/api/explorer/nodes.api.ts index e4730b5e9..f92718be3 100644 --- a/backend/src/api/explorer/nodes.api.ts +++ b/backend/src/api/explorer/nodes.api.ts @@ -261,7 +261,7 @@ class NodesApi { try { const publicKeySearch = search.replace('%', '') + '%'; const aliasSearch = search.replace(/[-_.]/g, ' ').replace(/[^a-zA-Z0-9 ]/g, '').split(' ').map((search) => '+' + search + '*').join(' '); - const query = `SELECT nodes.public_key, nodes.alias, node_stats.capacity FROM nodes LEFT JOIN node_stats ON node_stats.public_key = nodes.public_key WHERE nodes.public_key LIKE ? OR MATCH nodes.alias_search AGAINST (? IN BOOLEAN MODE) GROUP BY nodes.public_key ORDER BY node_stats.capacity DESC LIMIT 10`; + const query = `SELECT public_key, alias, capacity, channels FROM nodes WHERE public_key LIKE ? OR MATCH alias_search AGAINST (? IN BOOLEAN MODE) ORDER BY capacity DESC LIMIT 10`; const [rows]: any = await DB.query(query, [publicKeySearch, aliasSearch]); return rows; } catch (e) { diff --git a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts index 30cfcc62b..7ac1c5885 100644 --- a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts +++ b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts @@ -279,6 +279,17 @@ class LightningStatsImporter { nodeStats[public_key].capacity, nodeStats[public_key].channels, ]); + + if (!isHistorical) { + await DB.query( + `UPDATE nodes SET capacity = ?, channels = ? WHERE public_key = ?`, + [ + nodeStats[public_key].capacity, + nodeStats[public_key].channels, + public_key, + ] + ); + } } return { From ba7b65a9780372bbe513b15469342aefaf248552 Mon Sep 17 00:00:00 2001 From: softsimon Date: Fri, 26 Aug 2022 16:35:24 +0400 Subject: [PATCH 17/23] Fetch capacity and channels from nodes table --- backend/src/api/explorer/nodes.api.ts | 62 ++++++++------------------- 1 file changed, 18 insertions(+), 44 deletions(-) diff --git a/backend/src/api/explorer/nodes.api.ts b/backend/src/api/explorer/nodes.api.ts index f92718be3..c49ed9ac5 100644 --- a/backend/src/api/explorer/nodes.api.ts +++ b/backend/src/api/explorer/nodes.api.ts @@ -115,17 +115,13 @@ class NodesApi { public async $getTopCapacityNodes(full: boolean): Promise { try { - let [rows]: any[] = await DB.query('SELECT UNIX_TIMESTAMP(MAX(added)) as maxAdded FROM node_stats'); - const latestDate = rows[0].maxAdded; - + let rows: any; let query: string; if (full === false) { query = ` SELECT nodes.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias, - node_stats.capacity - FROM node_stats - JOIN nodes ON nodes.public_key = node_stats.public_key - WHERE added = FROM_UNIXTIME(${latestDate}) + nodes.capacity + FROM nodes ORDER BY capacity DESC LIMIT 100 `; @@ -133,16 +129,14 @@ class NodesApi { [rows] = await DB.query(query); } else { query = ` - SELECT node_stats.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(node_stats.public_key, 1, 20), alias) as alias, - CAST(COALESCE(node_stats.capacity, 0) as INT) as capacity, - CAST(COALESCE(node_stats.channels, 0) as INT) as channels, + SELECT nodes.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias, + CAST(COALESCE(nodes.capacity, 0) as INT) as capacity, + CAST(COALESCE(nodes.channels, 0) as INT) as channels, UNIX_TIMESTAMP(nodes.first_seen) as firstSeen, UNIX_TIMESTAMP(nodes.updated_at) as updatedAt, geo_names_city.names as city, geo_names_country.names as country - FROM node_stats - RIGHT JOIN nodes ON nodes.public_key = node_stats.public_key + FROM nodes LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country' LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city' - WHERE added = FROM_UNIXTIME(${latestDate}) ORDER BY capacity DESC LIMIT 100 `; @@ -163,17 +157,13 @@ class NodesApi { public async $getTopChannelsNodes(full: boolean): Promise { try { - let [rows]: any[] = await DB.query('SELECT UNIX_TIMESTAMP(MAX(added)) as maxAdded FROM node_stats'); - const latestDate = rows[0].maxAdded; - + let rows: any; let query: string; if (full === false) { query = ` SELECT nodes.public_key as publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias, - node_stats.channels - FROM node_stats - JOIN nodes ON nodes.public_key = node_stats.public_key - WHERE added = FROM_UNIXTIME(${latestDate}) + nodes.channels + FROM nodes ORDER BY channels DESC LIMIT 100; `; @@ -181,16 +171,14 @@ class NodesApi { [rows] = await DB.query(query); } else { query = ` - SELECT node_stats.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(node_stats.public_key, 1, 20), alias) as alias, - CAST(COALESCE(node_stats.channels, 0) as INT) as channels, - CAST(COALESCE(node_stats.capacity, 0) as INT) as capacity, + SELECT nodes.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias, + CAST(COALESCE(nodes.channels, 0) as INT) as channels, + CAST(COALESCE(nodes.capacity, 0) as INT) as capacity, UNIX_TIMESTAMP(nodes.first_seen) as firstSeen, UNIX_TIMESTAMP(nodes.updated_at) as updatedAt, geo_names_city.names as city, geo_names_country.names as country - FROM node_stats - RIGHT JOIN nodes ON nodes.public_key = node_stats.public_key + FROM nodes LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country' LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city' - WHERE added = FROM_UNIXTIME(${latestDate}) ORDER BY channels DESC LIMIT 100 `; @@ -276,7 +264,7 @@ class NodesApi { // List all channels and the two linked ISP query = ` - SELECT short_id, capacity, + SELECT short_id, channels.capacity, channels.node1_public_key AS node1PublicKey, isp1.names AS isp1, isp1.id as isp1ID, channels.node2_public_key AS node2PublicKey, isp2.names AS isp2, isp2.id as isp2ID FROM channels @@ -391,17 +379,11 @@ class NodesApi { public async $getNodesPerCountry(countryId: string) { try { const query = ` - SELECT nodes.public_key, CAST(COALESCE(node_stats.capacity, 0) as INT) as capacity, CAST(COALESCE(node_stats.channels, 0) as INT) as channels, + SELECT nodes.public_key, CAST(COALESCE(nodes.capacity, 0) as INT) as capacity, CAST(COALESCE(nodes.channels, 0) as INT) as channels, nodes.alias, UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at, geo_names_city.names as city, geo_names_country.names as country, geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision - FROM node_stats - JOIN ( - SELECT public_key, MAX(added) as last_added - FROM node_stats - GROUP BY public_key - ) as b ON b.public_key = node_stats.public_key AND b.last_added = node_stats.added - RIGHT JOIN nodes ON nodes.public_key = node_stats.public_key + FROM nodes LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country' LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city' LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code' @@ -426,17 +408,10 @@ class NodesApi { public async $getNodesPerISP(ISPId: string) { try { const query = ` - SELECT nodes.public_key, CAST(COALESCE(node_stats.capacity, 0) as INT) as capacity, CAST(COALESCE(node_stats.channels, 0) as INT) as channels, + SELECT nodes.public_key, CAST(COALESCE(nodes.capacity, 0) as INT) as capacity, CAST(COALESCE(nodes.channels, 0) as INT) as channels, nodes.alias, UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at, geo_names_city.names as city, geo_names_country.names as country, geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision - FROM node_stats - JOIN ( - SELECT public_key, MAX(added) as last_added - FROM node_stats - GROUP BY public_key - ) as b ON b.public_key = node_stats.public_key AND b.last_added = node_stats.added - RIGHT JOIN nodes ON nodes.public_key = node_stats.public_key LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country' LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city' LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code' @@ -464,7 +439,6 @@ class NodesApi { FROM nodes JOIN geo_names ON geo_names.id = nodes.country_id AND geo_names.type = 'country' JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code' - JOIN channels ON channels.node1_public_key = nodes.public_key OR channels.node2_public_key = nodes.public_key GROUP BY country_id ORDER BY COUNT(DISTINCT nodes.public_key) DESC `; From d051538c6a2ba7f49e214a988c1bdea59a9d92a4 Mon Sep 17 00:00:00 2001 From: softsimon Date: Sat, 27 Aug 2022 09:54:16 +0200 Subject: [PATCH 18/23] Fix for broken SQL to load Lightning nodes on ISP page --- backend/src/api/explorer/nodes.api.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/src/api/explorer/nodes.api.ts b/backend/src/api/explorer/nodes.api.ts index c49ed9ac5..56cfebe9f 100644 --- a/backend/src/api/explorer/nodes.api.ts +++ b/backend/src/api/explorer/nodes.api.ts @@ -412,6 +412,7 @@ class NodesApi { nodes.alias, UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at, geo_names_city.names as city, geo_names_country.names as country, geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision + FROM nodes LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country' LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city' LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code' From bba9f2608ae2643e77f0b2011fd9c9c3ba140122 Mon Sep 17 00:00:00 2001 From: wiz Date: Sat, 27 Aug 2022 13:54:07 +0200 Subject: [PATCH 19/23] [ops] Add nginx placeholders for unfurlbot configuration --- .../nginx/location-api-v1-services.conf | 24 ------------------- .../location-testnet-api-v1-lightning.conf | 2 +- production/nginx/nginx.conf | 8 +++++++ production/nginx/server-common.conf | 12 +++++++--- 4 files changed, 18 insertions(+), 28 deletions(-) diff --git a/production/nginx/location-api-v1-services.conf b/production/nginx/location-api-v1-services.conf index 869036c54..aae49727e 100644 --- a/production/nginx/location-api-v1-services.conf +++ b/production/nginx/location-api-v1-services.conf @@ -70,30 +70,6 @@ location /api/v1/translators { proxy_hide_header content-security-policy; proxy_hide_header x-frame-options; } -location /api/v1/enterprise/images { - proxy_pass $mempoolSpaceServices; - proxy_cache services; - proxy_cache_background_update on; - proxy_cache_use_stale updating; - proxy_cache_valid 200 10m; - expires 10m; - proxy_hide_header onion-location; - proxy_hide_header strict-transport-security; - proxy_hide_header content-security-policy; - proxy_hide_header x-frame-options; -} -location /api/v1/enterprise { - proxy_pass $mempoolSpaceServices; - proxy_cache services; - proxy_cache_background_update on; - proxy_cache_use_stale updating; - proxy_cache_valid 200 5m; - expires 5m; - proxy_hide_header onion-location; - proxy_hide_header strict-transport-security; - proxy_hide_header content-security-policy; - proxy_hide_header x-frame-options; -} location /api/v1/assets { proxy_pass $mempoolSpaceServices; proxy_cache services; diff --git a/production/nginx/location-testnet-api-v1-lightning.conf b/production/nginx/location-testnet-api-v1-lightning.conf index 5319004ee..cc7c617a6 100644 --- a/production/nginx/location-testnet-api-v1-lightning.conf +++ b/production/nginx/location-testnet-api-v1-lightning.conf @@ -4,7 +4,7 @@ location /testnet/api/v1/lightning { try_files /dev/null @mempool-testnet-api-v1-lightning; } location @mempool-testnet-api-v1-lightning { - proxy_pass $mempoolSignetLightning; + proxy_pass $mempoolTestnetLightning; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; diff --git a/production/nginx/nginx.conf b/production/nginx/nginx.conf index 83a794e51..5861067e8 100644 --- a/production/nginx/nginx.conf +++ b/production/nginx/nginx.conf @@ -22,6 +22,11 @@ http { include mempool/production/nginx/http-proxy-cache.conf; include mempool/production/nginx/http-language.conf; + # match preview/unfurl bot user-agents + map $http_user_agent $unfurlbot { + default 0; + } + # mempool configuration include mempool/production/nginx/upstream-mempool.conf; @@ -42,6 +47,7 @@ http { # for services from mempool.space like contributors on about page set $mempoolSpaceServices "https://mempool.space"; + set $mempoolSpaceUnfurler "http://127.0.0.1:8001"; # for mempool daemons, see upstream-mempool.conf set $mempoolMainnet "http://mempool-bitcoin-mainnet"; @@ -77,6 +83,7 @@ http { # for services from mempool.space like contributors on about page set $mempoolSpaceServices "https://mempool.space"; + set $mempoolSpaceUnfurler "http://127.0.0.1:8001"; # for mempool daemons, see upstream-mempool.conf set $mempoolBisq "http://mempool-bitcoin-bisq"; @@ -105,6 +112,7 @@ http { # for services from mempool.space like contributors on about page set $mempoolSpaceServices "https://mempool.space"; + set $mempoolSpaceUnfurler "http://127.0.0.1:8001"; # for mempool daemons, see upstream-mempool.conf set $mempoolMainnet "http://mempool-liquid-mainnet"; diff --git a/production/nginx/server-common.conf b/production/nginx/server-common.conf index f1b21c1e0..c5781692e 100644 --- a/production/nginx/server-common.conf +++ b/production/nginx/server-common.conf @@ -59,7 +59,7 @@ location = / { # cache /resources/** for 1 week since they don't change often location ~ ^/[a-z][a-z]/resources/(.*) { try_files $uri /en-US/resources/$1 =404; - expires 1w; + expires 1w; } # cache //main.f40e91d908a068a2.js forever since they never change location ~ ^/([a-z][a-z])/(.+\..+\.(js|css)) { @@ -69,11 +69,14 @@ location ~ ^/([a-z][a-z])/(.+\..+\.(js|css)) { # cache everything else for 5 minutes location ~ ^/([a-z][a-z])$ { try_files $uri /$1/index.html /en-US/index.html =404; - expires 5m; + expires 5m; } location ~ ^/([a-z][a-z])/ { + if ($unfurlbot) { + proxy_pass $mempoolSpaceUnfurler; + } try_files $uri /$1/index.html /en-US/index.html =404; - expires 5m; + expires 5m; } # cache /resources/** for 1 week since they don't change often @@ -89,6 +92,9 @@ location ~* ^/.+\..+\.(js|css) { # catch-all for all URLs i.e. /address/foo /tx/foo /block/000 # cache 5 minutes since they change frequently location / { + if ($unfurlbot) { + proxy_pass $mempoolSpaceUnfurler; + } try_files /$lang/$uri $uri /en-US/$uri /en-US/index.html =404; expires 5m; } From 004dcebc196bc905a75883cb11c3c570e81db7a1 Mon Sep 17 00:00:00 2001 From: wiz Date: Sat, 27 Aug 2022 14:17:17 +0200 Subject: [PATCH 20/23] [ops] Add nginx entrypoints for unfurler daemon --- production/nginx/server-common.conf | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/production/nginx/server-common.conf b/production/nginx/server-common.conf index c5781692e..6ce34c2c3 100644 --- a/production/nginx/server-common.conf +++ b/production/nginx/server-common.conf @@ -89,6 +89,18 @@ location ~* ^/.+\..+\.(js|css) { try_files /$lang/$uri /en-US/$uri =404; expires 1y; } + +# unfurl preview +location /preview { + try_files /$lang/$uri $uri /en-US/$uri /en-US/index.html =404; + expires 10m; +} +# unfurl renderer +location ^~ /render { + proxy_pass $mempoolSpaceUnfurler; + expires 10m; +} + # catch-all for all URLs i.e. /address/foo /tx/foo /block/000 # cache 5 minutes since they change frequently location / { From 40dc476460769d0d75dc8ddcbb7ff572f89adf82 Mon Sep 17 00:00:00 2001 From: softsimon Date: Sat, 27 Aug 2022 16:00:58 +0200 Subject: [PATCH 21/23] Fixes multiple bugs with outspends and channels fixes #412 --- backend/src/api/explorer/channels.routes.ts | 33 ++++++------ .../transactions-list.component.html | 12 ++--- .../transactions-list.component.ts | 53 +++++++++++-------- .../src/app/interfaces/electrs.interface.ts | 9 ++++ .../src/app/interfaces/node-api.interface.ts | 32 +++++++++++ .../lightning/channel/channel.component.html | 4 +- .../lightning/channel/channel.component.ts | 12 +++-- frontend/src/app/services/api.service.ts | 4 +- 8 files changed, 107 insertions(+), 52 deletions(-) diff --git a/backend/src/api/explorer/channels.routes.ts b/backend/src/api/explorer/channels.routes.ts index eda3a6168..2b7f3fa6d 100644 --- a/backend/src/api/explorer/channels.routes.ts +++ b/backend/src/api/explorer/channels.routes.ts @@ -70,7 +70,7 @@ class ChannelsRoutes { } } - private async $getChannelsByTransactionIds(req: Request, res: Response) { + private async $getChannelsByTransactionIds(req: Request, res: Response): Promise { try { if (!Array.isArray(req.query.txId)) { res.status(400).send('Not an array'); @@ -83,27 +83,26 @@ class ChannelsRoutes { } } const channels = await channelsApi.$getChannelsByTransactionId(txIds); - const inputs: any[] = []; - const outputs: any[] = []; + const result: any[] = []; for (const txid of txIds) { - const foundChannelInputs = channels.find((channel) => channel.closing_transaction_id === txid); - if (foundChannelInputs) { - inputs.push(foundChannelInputs); - } else { - inputs.push(null); + const inputs: any = {}; + const outputs: any = {}; + // Assuming that we only have one lightning close input in each transaction. This may not be true in the future + const foundChannelsFromInput = channels.find((channel) => channel.closing_transaction_id === txid); + if (foundChannelsFromInput) { + inputs[0] = foundChannelsFromInput; } - const foundChannelOutputs = channels.find((channel) => channel.transaction_id === txid); - if (foundChannelOutputs) { - outputs.push(foundChannelOutputs); - } else { - outputs.push(null); + const foundChannelsFromOutputs = channels.filter((channel) => channel.transaction_id === txid); + for (const output of foundChannelsFromOutputs) { + outputs[output.transaction_vout] = output; } + result.push({ + inputs, + outputs, + }); } - res.json({ - inputs: inputs, - outputs: outputs, - }); + res.json(result); } catch (e) { res.status(500).send(e instanceof Error ? e.message : e); } diff --git a/frontend/src/app/components/transactions-list/transactions-list.component.html b/frontend/src/app/components/transactions-list/transactions-list.component.html index 17eb52878..6333b75c2 100644 --- a/frontend/src/app/components/transactions-list/transactions-list.component.html +++ b/frontend/src/app/components/transactions-list/transactions-list.component.html @@ -20,7 +20,7 @@
- + + @@ -172,7 +172,7 @@
- +
@@ -212,15 +212,15 @@
- + - + - + diff --git a/frontend/src/app/components/transactions-list/transactions-list.component.ts b/frontend/src/app/components/transactions-list/transactions-list.component.ts index 2a317e738..a5cfde02f 100644 --- a/frontend/src/app/components/transactions-list/transactions-list.component.ts +++ b/frontend/src/app/components/transactions-list/transactions-list.component.ts @@ -27,7 +27,6 @@ export class TransactionsListComponent implements OnInit, OnChanges { @Input() outputIndex: number; @Input() address: string = ''; @Input() rowLimit = 12; - @Input() channels: { inputs: any[], outputs: any[] }; @Output() loadMore = new EventEmitter(); @@ -36,8 +35,8 @@ export class TransactionsListComponent implements OnInit, OnChanges { refreshOutspends$: ReplaySubject = new ReplaySubject(); refreshChannels$: ReplaySubject = new ReplaySubject(); showDetails$ = new BehaviorSubject(false); - outspends: Outspend[][] = []; assetsMinimal: any; + transactionsLength: number = 0; constructor( public stateService: StateService, @@ -47,7 +46,7 @@ export class TransactionsListComponent implements OnInit, OnChanges { private ref: ChangeDetectorRef, ) { } - ngOnInit() { + ngOnInit(): void { this.latestBlock$ = this.stateService.blocks$.pipe(map(([block]) => block)); this.stateService.networkChanged$.subscribe((network) => this.network = network); @@ -62,14 +61,17 @@ export class TransactionsListComponent implements OnInit, OnChanges { .pipe( switchMap((txIds) => this.apiService.getOutspendsBatched$(txIds)), tap((outspends: Outspend[][]) => { - this.outspends = this.outspends.concat(outspends); + const transactions = this.transactions.filter((tx) => !tx._outspends); + outspends.forEach((outspend, i) => { + transactions[i]._outspends = outspend; + }); }), ), this.stateService.utxoSpent$ .pipe( tap((utxoSpent) => { for (const i in utxoSpent) { - this.outspends[0][i] = { + this.transactions[0]._outspends[i] = { spent: true, txid: utxoSpent[i].txid, vin: utxoSpent[i].vin, @@ -81,21 +83,23 @@ export class TransactionsListComponent implements OnInit, OnChanges { .pipe( filter(() => this.stateService.env.LIGHTNING), switchMap((txIds) => this.apiService.getChannelByTxIds$(txIds)), - map((channels) => { - this.channels = channels; + tap((channels) => { + const transactions = this.transactions.filter((tx) => !tx._channels); + channels.forEach((channel, i) => { + transactions[i]._channels = channel; + }); }), ) , ).subscribe(() => this.ref.markForCheck()); } - ngOnChanges() { + ngOnChanges(): void { if (!this.transactions || !this.transactions.length) { return; } - if (this.paginated) { - this.outspends = []; - } + + this.transactionsLength = this.transactions.length; if (this.outputIndex) { setTimeout(() => { const assetBoxElements = document.getElementsByClassName('assetBox'); @@ -126,14 +130,19 @@ export class TransactionsListComponent implements OnInit, OnChanges { tx['addressValue'] = addressIn - addressOut; } }); - const txIds = this.transactions.map((tx) => tx.txid); - this.refreshOutspends$.next(txIds); - if (!this.channels) { - this.refreshChannels$.next(txIds); + const txIds = this.transactions.filter((tx) => !tx._outspends).map((tx) => tx.txid); + if (txIds.length) { + this.refreshOutspends$.next(txIds); + } + if (this.stateService.env.LIGHTNING) { + const txIds = this.transactions.filter((tx) => !tx._channels).map((tx) => tx.txid); + if (txIds.length) { + this.refreshChannels$.next(txIds); + } } } - onScroll() { + onScroll(): void { const scrollHeight = document.body.scrollHeight; const scrollTop = document.documentElement.scrollTop; if (scrollHeight > 0){ @@ -148,11 +157,11 @@ export class TransactionsListComponent implements OnInit, OnChanges { return tx.vout.some((v: any) => v.value === undefined); } - getTotalTxOutput(tx: Transaction) { + getTotalTxOutput(tx: Transaction): number { return tx.vout.map((v: Vout) => v.value || 0).reduce((a: number, b: number) => a + b); } - switchCurrency() { + switchCurrency(): void { if (this.network === 'liquid' || this.network === 'liquidtestnet') { return; } @@ -164,7 +173,7 @@ export class TransactionsListComponent implements OnInit, OnChanges { return tx.txid + tx.status.confirmed; } - trackByIndexFn(index: number) { + trackByIndexFn(index: number): number { return index; } @@ -177,7 +186,7 @@ export class TransactionsListComponent implements OnInit, OnChanges { return Math.pow(base, exponent); } - toggleDetails() { + toggleDetails(): void { if (this.showDetails$.value === true) { this.showDetails$.next(false); } else { @@ -185,7 +194,7 @@ export class TransactionsListComponent implements OnInit, OnChanges { } } - loadMoreInputs(tx: Transaction) { + loadMoreInputs(tx: Transaction): void { tx['@vinLimit'] = false; this.electrsApiService.getTransaction$(tx.txid) @@ -196,7 +205,7 @@ export class TransactionsListComponent implements OnInit, OnChanges { }); } - ngOnDestroy() { + ngOnDestroy(): void { this.outspendsSubscription.unsubscribe(); } } diff --git a/frontend/src/app/interfaces/electrs.interface.ts b/frontend/src/app/interfaces/electrs.interface.ts index 9c873d2eb..63dec7abd 100644 --- a/frontend/src/app/interfaces/electrs.interface.ts +++ b/frontend/src/app/interfaces/electrs.interface.ts @@ -1,3 +1,5 @@ +import { IChannel } from './node-api.interface'; + export interface Transaction { txid: string; version: number; @@ -19,6 +21,13 @@ export interface Transaction { deleteAfter?: number; _unblinded?: any; _deduced?: boolean; + _outspends?: Outspend[]; + _channels?: TransactionChannels; +} + +export interface TransactionChannels { + inputs: { [vin: number]: IChannel }; + outputs: { [vout: number]: IChannel }; } interface Ancestor { diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index 66ee8179e..5c071a357 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -189,3 +189,35 @@ export interface IOldestNodes { city?: any, country?: any, } + +export interface IChannel { + id: number; + short_id: string; + capacity: number; + transaction_id: string; + transaction_vout: number; + closing_transaction_id: string; + closing_reason: string; + updated_at: string; + created: string; + status: number; + node_left: Node, + node_right: Node, +} + + +export interface INode { + alias: string; + public_key: string; + channels: number; + capacity: number; + base_fee_mtokens: number; + cltv_delta: number; + fee_rate: number; + is_disabled: boolean; + max_htlc_mtokens: number; + min_htlc_mtokens: number; + updated_at: string; + longitude: number; + latitude: number; +} diff --git a/frontend/src/app/lightning/channel/channel.component.html b/frontend/src/app/lightning/channel/channel.component.html index 65866273d..6d0439e76 100644 --- a/frontend/src/app/lightning/channel/channel.component.html +++ b/frontend/src/app/lightning/channel/channel.component.html @@ -65,13 +65,13 @@

Opening transaction

- +

Closing transaction

  
- + diff --git a/frontend/src/app/lightning/channel/channel.component.ts b/frontend/src/app/lightning/channel/channel.component.ts index bbf9be36d..553173052 100644 --- a/frontend/src/app/lightning/channel/channel.component.ts +++ b/frontend/src/app/lightning/channel/channel.component.ts @@ -2,6 +2,7 @@ import { ChangeDetectionStrategy, Component, OnInit } from '@angular/core'; import { ActivatedRoute, ParamMap } from '@angular/router'; import { forkJoin, Observable, of, share, zip } from 'rxjs'; import { catchError, map, shareReplay, switchMap, tap } from 'rxjs/operators'; +import { IChannel } from 'src/app/interfaces/node-api.interface'; import { ApiService } from 'src/app/services/api.service'; import { ElectrsApiService } from 'src/app/services/electrs-api.service'; import { SeoService } from 'src/app/services/seo.service'; @@ -62,10 +63,15 @@ export class ChannelComponent implements OnInit { ); this.transactions$ = this.channel$.pipe( - switchMap((data) => { + switchMap((channel: IChannel) => { return zip([ - data.transaction_id ? this.electrsApiService.getTransaction$(data.transaction_id) : of(null), - data.closing_transaction_id ? this.electrsApiService.getTransaction$(data.closing_transaction_id) : of(null), + channel.transaction_id ? this.electrsApiService.getTransaction$(channel.transaction_id) : of(null), + channel.closing_transaction_id ? this.electrsApiService.getTransaction$(channel.closing_transaction_id).pipe( + map((tx) => { + tx._channels = { inputs: {0: channel}, outputs: {}}; + return tx; + }) + ) : of(null), ]); }), ); diff --git a/frontend/src/app/services/api.service.ts b/frontend/src/app/services/api.service.ts index 5d89a168f..5f036c575 100644 --- a/frontend/src/app/services/api.service.ts +++ b/frontend/src/app/services/api.service.ts @@ -242,12 +242,12 @@ export class ApiService { return this.httpClient.get(this.apiBaseUrl + this.apiBasePath + `/api/v1/enterprise/info/` + name); } - getChannelByTxIds$(txIds: string[]): Observable<{ inputs: any[], outputs: any[] }> { + getChannelByTxIds$(txIds: string[]): Observable { let params = new HttpParams(); txIds.forEach((txId: string) => { params = params.append('txId[]', txId); }); - return this.httpClient.get<{ inputs: any[], outputs: any[] }>(this.apiBaseUrl + this.apiBasePath + '/api/v1/lightning/channels/txids/', { params }); + return this.httpClient.get(this.apiBaseUrl + this.apiBasePath + '/api/v1/lightning/channels/txids/', { params }); } lightningSearch$(searchText: string): Observable { From 6f87fd9c89848724c02711988fe72d0c07614375 Mon Sep 17 00:00:00 2001 From: wiz Date: Sat, 27 Aug 2022 18:57:59 +0200 Subject: [PATCH 22/23] [ops] Add production lightning backend configurations --- .../mempool-config.mainnet-lightning.json | 49 +++++++++++++++++++ .../mempool-config.signet-lightning.json | 44 +++++++++++++++++ .../mempool-config.testnet-lightning.json | 44 +++++++++++++++++ 3 files changed, 137 insertions(+) create mode 100644 production/mempool-config.mainnet-lightning.json create mode 100644 production/mempool-config.signet-lightning.json create mode 100644 production/mempool-config.testnet-lightning.json diff --git a/production/mempool-config.mainnet-lightning.json b/production/mempool-config.mainnet-lightning.json new file mode 100644 index 000000000..99ce8d518 --- /dev/null +++ b/production/mempool-config.mainnet-lightning.json @@ -0,0 +1,49 @@ +{ + "MEMPOOL": { + "NETWORK": "mainnet", + "BACKEND": "esplora", + "HTTP_PORT": 8993, + "INDEXING_BLOCKS_AMOUNT": 0, + "API_URL_PREFIX": "/api/v1/" + }, + "SYSLOG": { + "MIN_PRIORITY": "debug" + }, + "CORE_RPC": { + "PORT": 8332, + "USERNAME": "__BITCOIN_RPC_USER__", + "PASSWORD": "__BITCOIN_RPC_PASS__" + }, + "ESPLORA": { + "REST_API_URL": "http://127.0.0.1:4000" + }, + "LIGHTNING": { + "ENABLED": true, + "BACKEND": "cln", + "GRAPH_REFRESH_INTERVAL": 60, + "TOPOLOGY_FOLDER": "/cln/topology/output" + }, + "LND": { + "REST_API_URL": "https://127.0.0.1:8888", + "TLS_CERT_PATH": "/lnd/.lnd/tls.cert", + "MACAROON_PATH": "/lnd/.lnd/data/chain/bitcoin/mainnet/readonly.macaroon" + }, + "CLIGHTNING": { + "SOCKET": "/cln/.lightning/bitcoin/lightning-rpc" + }, + "MAXMIND": { + "ENABLED": true, + "GEOLITE2_CITY": "/usr/local/share/GeoIP/GeoIP2-City.mmdb" + }, + "STATISTICS": { + "ENABLED": false + }, + "DATABASE": { + "ENABLED": true, + "HOST": "127.0.0.1", + "PORT": 3306, + "DATABASE": "mempool_mainnet_lightning", + "USERNAME": "mempool_mainnet_lightning", + "PASSWORD": "mempool_mainnet_lightning" + } +} diff --git a/production/mempool-config.signet-lightning.json b/production/mempool-config.signet-lightning.json new file mode 100644 index 000000000..08250ffbd --- /dev/null +++ b/production/mempool-config.signet-lightning.json @@ -0,0 +1,44 @@ +{ + "MEMPOOL": { + "NETWORK": "signet", + "BACKEND": "esplora", + "HTTP_PORT": 8991, + "INDEXING_BLOCKS_AMOUNT": 0, + "API_URL_PREFIX": "/api/v1/" + }, + "SYSLOG": { + "MIN_PRIORITY": "debug" + }, + "CORE_RPC": { + "PORT": 38332, + "USERNAME": "__BITCOIN_RPC_USER__", + "PASSWORD": "__BITCOIN_RPC_PASS__" + }, + "ESPLORA": { + "REST_API_URL": "http://127.0.0.1:4003" + }, + "LIGHTNING": { + "ENABLED": true, + "BACKEND": "cln", + "GRAPH_REFRESH_INTERVAL": 60, + "TOPOLOGY_FOLDER": "" + }, + "CLIGHTNING": { + "SOCKET": "/cln/.lightning/signet/lightning-rpc" + }, + "MAXMIND": { + "ENABLED": true, + "GEOLITE2_CITY": "/usr/local/share/GeoIP/GeoIP2-City.mmdb" + }, + "STATISTICS": { + "ENABLED": false + }, + "DATABASE": { + "ENABLED": true, + "HOST": "127.0.0.1", + "PORT": 3306, + "USERNAME": "mempool_signet_lightning", + "PASSWORD": "mempool_signet_lightning", + "DATABASE": "mempool_signet_lightning" + } +} diff --git a/production/mempool-config.testnet-lightning.json b/production/mempool-config.testnet-lightning.json new file mode 100644 index 000000000..c2a42e4bc --- /dev/null +++ b/production/mempool-config.testnet-lightning.json @@ -0,0 +1,44 @@ +{ + "MEMPOOL": { + "NETWORK": "testnet", + "BACKEND": "esplora", + "HTTP_PORT": 8992, + "INDEXING_BLOCKS_AMOUNT": 0, + "API_URL_PREFIX": "/api/v1/" + }, + "SYSLOG": { + "MIN_PRIORITY": "debug" + }, + "CORE_RPC": { + "PORT": 18332, + "USERNAME": "__BITCOIN_RPC_USER__", + "PASSWORD": "__BITCOIN_RPC_PASS__" + }, + "ESPLORA": { + "REST_API_URL": "http://127.0.0.1:4002" + }, + "LIGHTNING": { + "ENABLED": true, + "BACKEND": "cln", + "GRAPH_REFRESH_INTERVAL": 60, + "TOPOLOGY_FOLDER": "" + }, + "CLIGHTNING": { + "SOCKET": "/cln/.lightning/testnet/lightning-rpc" + }, + "MAXMIND": { + "ENABLED": true, + "GEOLITE2_CITY": "/usr/local/share/GeoIP/GeoIP2-City.mmdb" + }, + "STATISTICS": { + "ENABLED": false + }, + "DATABASE": { + "ENABLED": true, + "HOST": "127.0.0.1", + "PORT": 3306, + "USERNAME": "mempool_testnet_lightning", + "PASSWORD": "mempool_testnet_lightning", + "DATABASE": "mempool_testnet_lightning" + } +} From be10cc65f458ccb85ba9e725d9bbc317a7a90c55 Mon Sep 17 00:00:00 2001 From: softsimon Date: Sat, 27 Aug 2022 22:32:56 +0200 Subject: [PATCH 23/23] Fixing missing channel openings --- .../transactions-list/transactions-list.component.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/app/components/transactions-list/transactions-list.component.html b/frontend/src/app/components/transactions-list/transactions-list.component.html index 6333b75c2..81c3dce5c 100644 --- a/frontend/src/app/components/transactions-list/transactions-list.component.html +++ b/frontend/src/app/components/transactions-list/transactions-list.component.html @@ -77,7 +77,7 @@ {{ vin.prevout.scriptpubkey_type?.toUpperCase() }}
- +