machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into nymkappa/bugfix/useless-mysql-query-channel-tree

Browse Source
This commit is contained in:
wiz 2022-08-25 23:32:18 +09:00 committed by GitHub
commit a9e1ce42df
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 115 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
backend/src/tasks/lightning/network-sync.service.ts
View File

@ -83,7 +83,7 @@ class NetworkSyncService {
logger.info(`${progress} nodes updated. ${deletedSockets} sockets deleted`);
// If a channel if not present in the graph, mark it as inactive
nodesApi.$setNodesInactive(graphNodesPubkeys);
await nodesApi.$setNodesInactive(graphNodesPubkeys);
if (config.MAXMIND.ENABLED) {
$lookupNodeLocation();
@ -121,7 +121,7 @@ class NetworkSyncService {
logger.info(`${progress} channels updated`);
// If a channel if not present in the graph, mark it as inactive
channelsApi.$setChannelsInactive(graphChannelsIds);
await channelsApi.$setChannelsInactive(graphChannelsIds);
} catch (e) {
logger.err(`Cannot update channel list. Reason: ${(e instanceof Error ? e.message : e)}`);
}

69
backend/src/tasks/lightning/sync-tasks/stats-importer.ts
View File

@ -7,6 +7,7 @@ import { ILightningApi } from '../../../api/lightning/lightning-api.interface';
import { isIP } from 'net';
import { Common } from '../../../api/common';
import channelsApi from '../../../api/explorer/channels.api';
import nodesApi from '../../../api/explorer/nodes.api';
const fsPromises = promises;
@ -32,7 +33,26 @@ class LightningStatsImporter {
let clearnetTorNodes = 0;
let unannouncedNodes = 0;
const [nodesInDbRaw]: any[] = await DB.query(`SELECT public_key FROM nodes`);
const nodesInDb = {};
for (const node of nodesInDbRaw) {
nodesInDb[node.public_key] = node;
}
for (const node of networkGraph.nodes) {
// If we don't know about this node, insert it in db
if (isHistorical === true && !nodesInDb[node.pub_key]) {
await nodesApi.$saveNode({
last_update: node.last_update,
pub_key: node.pub_key,
alias: node.alias,
addresses: node.addresses,
color: node.color,
features: node.features,
});
nodesInDb[node.pub_key] = node;
}
let hasOnion = false;
let hasClearnet = false;
let isUnnanounced = true;
@ -69,7 +89,7 @@ class LightningStatsImporter {
const baseFees: number[] = [];
const alreadyCountedChannels = {};
const [channelsInDbRaw]: any[] = await DB.query(`SELECT short_id, created FROM channels`);
const [channelsInDbRaw]: any[] = await DB.query(`SELECT short_id FROM channels`);
const channelsInDb = {};
for (const channel of channelsInDbRaw) {
channelsInDb[channel.short_id] = channel;
@ -84,29 +104,19 @@ class LightningStatsImporter {
continue;
}
// Channel is already in db, check if we need to update 'created' field
if (isHistorical === true) {
//@ts-ignore
if (channelsInDb[short_id] && channel.timestamp < channel.created) {
await DB.query(`
UPDATE channels SET created = FROM_UNIXTIME(?) WHERE channels.short_id = ?`,
//@ts-ignore
[channel.timestamp, short_id]
);
} else if (!channelsInDb[short_id]) {
await channelsApi.$saveChannel({
channel_id: short_id,
chan_point: `${tx.txid}:${short_id.split('x')[2]}`,
//@ts-ignore
last_update: channel.timestamp,
node1_pub: channel.node1_pub,
node2_pub: channel.node2_pub,
capacity: (tx.value * 100000000).toString(),
node1_policy: null,
node2_policy: null,
}, 0);
channelsInDb[channel.channel_id] = channel;
}
// If we don't know about this channel, insert it in db
if (isHistorical === true && !channelsInDb[short_id]) {
await channelsApi.$saveChannel({
channel_id: short_id,
chan_point: `${tx.txid}:${short_id.split('x')[2]}`,
last_update: channel.last_update,
node1_pub: channel.node1_pub,
node2_pub: channel.node2_pub,
capacity: (tx.value * 100000000).toString(),
node1_policy: null,
node2_policy: null,
}, 0);
channelsInDb[channel.channel_id] = channel;
}
if (!nodeStats[channel.node1_pub]) {
@ -281,6 +291,7 @@ class LightningStatsImporter {
* Import topology files LN historical data into the database
*/
async $importHistoricalLightningStats(): Promise<void> {
logger.debug('Run the historical importer');
try {
let fileList: string[] = [];
try {
@ -294,7 +305,7 @@ class LightningStatsImporter {
fileList.sort().reverse();
const [rows]: any[] = await DB.query(`
SELECT UNIX_TIMESTAMP(added) AS added, node_count
SELECT UNIX_TIMESTAMP(added) AS added
FROM lightning_stats
ORDER BY added DESC
`);
@ -391,12 +402,16 @@ class LightningStatsImporter {
});
}
let rgb = node.rgb_color ?? '#000000';
if (rgb.indexOf('#') === -1) {
rgb = `#${rgb}`;
}
newGraph.nodes.push({
last_update: node.timestamp ?? 0,
pub_key: node.id ?? null,
alias: node.alias ?? null,
alias: node.alias ?? node.id.slice(0, 20),
addresses: addresses,
color: node.rgb_color ?? null,
color: rgb,
features: {},
});
}

16
frontend/src/app/lightning/node/node.component.html
View File

@ -119,20 +119,30 @@
</div>
<div *ngIf="!error">
<div class="row">
<div class="row" *ngIf="node.as_number">
<div class="col-sm">
<app-nodes-channels-map [style]="'nodepage'" [publicKey]="node.public_key"></app-nodes-channels-map>
<app-nodes-channels-map [style]="'nodepage'" [publicKey]="node.public_key" [hasLocation]="!!node.as_number"></app-nodes-channels-map>
</div>
<div class="col-sm">
<app-node-statistics-chart [publicKey]="node.public_key"></app-node-statistics-chart>
</div>
</div>
<div *ngIf="!node.as_number">
<app-node-statistics-chart [publicKey]="node.public_key"></app-node-statistics-chart>
</div>
<h2 i18n="lightning.active-channels-map">Active channels map</h2>
<app-node-channels style="display:block;margin-bottom: 40px" [publicKey]="node.public_key"></app-node-channels>
<div class="d-flex justify-content-between">
<h2>Channels ({{ channelsListStatus === 'open' ? node.opened_channel_count : node.closed_channel_count }})</h2>
<h2 *ngIf="channelsListStatus === 'open'">
<span i18n="lightning.open-channels">Open channels</span>
<span> ({{ node.opened_channel_count }})</span>
</h2>
<h2 *ngIf="channelsListStatus === 'closed'">
<span i18n="lightning.open-channels">Closed channels</span>
<span> ({{ node.closed_channel_count }})</span>
</h2>
</div>
<app-channels-list [publicKey]="node.public_key"

23
frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.html
View File

@ -1,5 +1,5 @@
<div [style]="style === 'widget' ? 'height: 250px' : ''">
<div *ngIf="channelsObservable | async">
<div class="map-wrapper" [class]="style">
<ng-container *ngIf="channelsObservable | async">
<div *ngIf="chartOptions" [class]="'full-container ' + style + (fitContainer ? ' fit-container' : '')">
<div *ngIf="style === 'graph'" class="card-header">
<div class="d-flex d-md-block align-items-baseline" style="margin-bottom: -5px">
@ -8,14 +8,15 @@
<small style="color: #ffffff66" i18n="lightning.tor-nodes-excluded">(Tor nodes excluded)</small>
</div>
<div class="chart" [class]="style" echarts [initOpts]="chartInitOptions" [options]="chartOptions" (chartInit)="onChartInit($event)"
(chartFinished)="onChartFinished($event)">
<div class="chart" [class]="style" echarts [initOpts]="chartInitOptions" [options]="chartOptions"
(chartInit)="onChartInit($event)" (chartFinished)="onChartFinished($event)">
</div>
<div *ngIf="!chartOptions && style === 'nodepage'" style="padding-top: 30px"></div>
</div>
<div *ngIf="!chartOptions && style === 'nodepage'" style="padding-top: 30px"></div>
</div>
<div class="text-center loading-spinner" [class]="style" *ngIf="isLoading">
<div class="spinner-border text-light"></div>
</div>
</div>
<div class="text-center loading-spinner" [class]="style" *ngIf="isLoading">
<div class="spinner-border text-light"></div>
</div>
</ng-container>
</div>

35
frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.scss
View File

@ -1,3 +1,15 @@
.map-wrapper {
height: 100%;
&.widget {
height: 250px;
}
&.graph {
height: auto;
}
}
.card-header {
border-bottom: 0;
font-size: 18px;
@ -93,6 +105,9 @@
top: 50%;
left: calc(50% - 15px);
z-index: 100;
@media (max-width: 767.98px) {
top: 550px;
}
}
.loading-spinner.widget {
position: absolute;
@ -103,4 +118,22 @@
@media (max-width: 767.98px) {
top: 250px;
}
}
}
.loading-spinner.nodepage {
position: absolute;
top: 200px;
z-index: 100;
width: 100%;
left: 0;
}
.loading-spinner.channelpage {
position: absolute;
top: 400px;
z-index: 100;
width: 100%;
left: 0;
@media (max-width: 767.98px) {
top: 450px;
}
}

8
frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts
View File

@ -21,6 +21,7 @@ export class NodesChannelsMap implements OnInit {
@Input() publicKey: string | undefined;
@Input() channel: any[] = [];
@Input() fitContainer = false;
@Input() hasLocation = true;
@Output() readyEvent = new EventEmitter();
channelsObservable: Observable<any>;
@ -32,7 +33,7 @@ export class NodesChannelsMap implements OnInit {
channelColor = '#466d9d';
channelCurve = 0;
nodeSize = 4;
isLoading = true;
isLoading = false;
chartInstance = undefined;
chartOptions: EChartsOption = {};
@ -73,6 +74,11 @@ export class NodesChannelsMap implements OnInit {
this.channelsObservable = this.activatedRoute.paramMap
.pipe(
switchMap((params: ParamMap) => {
this.isLoading = true;
if (this.style === 'channelpage' && this.channel.length === 0 || !this.hasLocation) {
this.isLoading = false;
}
return zip(
this.assetsService.getWorldMapJson$,
this.style !== 'channelpage' ? this.apiService.getChannelsGeo$(params.get('public_key') ?? undefined, this.style) : [''],

2
frontend/src/app/lightning/nodes-networks-chart/nodes-networks-chart.component.ts
View File

@ -121,7 +121,7 @@ export class NodesNetworksChartComponent implements OnInit {
left: 'center',
top: 'center',
};
} else if (data.tor_nodes.length > 0) {
} else if (this.widget && data.tor_nodes.length > 0) {
title = {
textStyle: {
color: 'grey',

2
frontend/src/app/lightning/statistics-chart/lightning-statistics-chart.component.ts
View File

@ -113,7 +113,7 @@ export class LightningStatisticsChartComponent implements OnInit {
left: 'center',
top: 'center'
};
} else if (data.channel_count.length > 0) {
} else if (this.widget && data.channel_count.length > 0) {
title = {
textStyle: {
color: 'grey',

4
production/install
View File

@ -385,7 +385,7 @@ DEBIAN_UNFURL_PKG+=(libxdamage-dev libxrandr-dev libgbm-dev libpango1.0-dev liba
# packages needed for mempool ecosystem
FREEBSD_PKG=()
FREEBSD_PKG+=(zsh sudo git screen curl wget calc neovim)
FREEBSD_PKG+=(zsh sudo git git-lfs screen curl wget calc neovim)
FREEBSD_PKG+=(openssh-portable py39-pip rust llvm90 jq base64 libzmq4)
FREEBSD_PKG+=(boost-libs autoconf automake gmake gcc libevent libtool pkgconf)
FREEBSD_PKG+=(nginx rsync py39-certbot-nginx mariadb105-server keybase)
@ -1267,7 +1267,7 @@ case $OS in
osGroupCreate "${CLN_GROUP}"
osUserCreate "${CLN_USER}" "${CLN_HOME}" "${CLN_GROUP}"
osSudo "${ROOT_USER}" chsh -s `which zsh` "${CLN_USER}"
osSudo "${CLN_USER}" touch "${CLN_HOME}/.zshrc"
echo "export PATH=$PATH:$HOME/.local/bin" >> "${CLN_HOME}/.zshrc"
osSudo "${ROOT_USER}" chown -R "${CLN_USER}:${CLN_GROUP}" "${CLN_HOME}"
echo "[*] Installing Core Lightning package"

2
production/syslog.conf
View File

@ -1,4 +1,4 @@
local7.>=notice |/usr/local/bin/sudo -u mempool /usr/local/bin/mempool-logger mempool.ops alerts
local7.>=err |/usr/local/bin/sudo -u mempool /usr/local/bin/mempool-logger mempool.ops alerts
local7.>=info |/usr/local/bin/sudo -u mempool /usr/local/bin/mempool-logger mempool.ops node100
local7.>=info /var/log/mempool
local7.* /var/log/mempool.debug