diff --git a/package.json b/package.json index fd64866f..d83e0752 100644 --- a/package.json +++ b/package.json @@ -76,7 +76,8 @@ "lisk-framework": "0.11.0-rc.5", "semver": "7.3.2", "shelljs": "^0.8.5", - "tar": "6.1.13" + "tar": "6.1.13", + "axios": "0.25.0" }, "devDependencies": { "@oclif/dev-cli": "1.22.2", diff --git a/src/index.ts b/src/index.ts index 625fa8bd..4102e838 100644 --- a/src/index.ts +++ b/src/index.ts @@ -71,6 +71,7 @@ import { MigratorException } from './utils/exception'; import { writeCommandsToExec } from './utils/commands'; import { getNetworkIdentifier } from './utils/network'; import { extractTarBall } from './utils/fs'; +import { downloadAndValidate } from './utils/download'; let configCoreV4: PartialApplicationConfig; class LiskMigrator extends Command { @@ -168,7 +169,17 @@ class LiskMigrator extends Command { const dataDir = join(__dirname, '..', DEFAULT_DATA_DIR); try { - if (!useSnapshot) { + if (useSnapshot) { + if (snapshotPath.startsWith('http')) { + cli.action.start(`Downloading snapshot from ${snapshotPath} to ${outputDir}`); + await downloadAndValidate(snapshotPath, outputDir, dataDir); + cli.action.stop(); + } else if (snapshotPath.endsWith('.tar.gz')) { + cli.action.start(`Extracting snapshot at ${dataDir}`); + await extractTarBall(snapshotPath, dataDir); + cli.action.stop(); + } + } else { const client = await getAPIClient(liskCoreV3DataPath); const nodeInfo = (await client.node.getNodeInfo()) as NodeInfo; const { version: appVersion } = nodeInfo; @@ -227,10 +238,6 @@ class LiskMigrator extends Command { delay: 500, isFinal: true, }); - } else if (useSnapshot && snapshotPath.endsWith('.tar.gz')) { - cli.action.start(`Extracting snapshot at ${dataDir}`); - await extractTarBall(snapshotPath, dataDir); - cli.action.stop(); } await setTokenIDLskByNetID(networkIdentifier); @@ -323,94 +330,96 @@ class LiskMigrator extends Command { this.log(`Genesis block tar and SHA256 files have been created at: ${outputDir}.`); cli.action.stop(); - if (autoStartLiskCoreV4 && !useSnapshot) { - try { - if (!autoMigrateUserConfig) { - configCoreV4 = defaultConfigV4; - } + if (!useSnapshot) { + if (autoStartLiskCoreV4) { + try { + if (!autoMigrateUserConfig) { + configCoreV4 = defaultConfigV4; + } - cli.action.start('Copying genesis block to the Lisk Core executable directory'); - const liskCoreExecPath = await execAsync('which lisk-core'); - const liskCoreV4ConfigPath = resolve( - liskCoreExecPath, - '../..', - `lib/node_modules/lisk-core/config/${networkConstant.name}`, - ); + cli.action.start('Copying genesis block to the Lisk Core executable directory'); + const liskCoreExecPath = await execAsync('which lisk-core'); + const liskCoreV4ConfigPath = resolve( + liskCoreExecPath, + '../..', + `lib/node_modules/lisk-core/config/${networkConstant.name}`, + ); - await copyGenesisBlock( - `${outputDir}/genesis_block.blob`, - `${liskCoreV4ConfigPath}/genesis_block.blob`, - ); - this.log(`Genesis block has been copied to: ${liskCoreV4ConfigPath}.`); - cli.action.stop(); + await copyGenesisBlock( + `${outputDir}/genesis_block.blob`, + `${liskCoreV4ConfigPath}/genesis_block.blob`, + ); + this.log(`Genesis block has been copied to: ${liskCoreV4ConfigPath}.`); + cli.action.stop(); - // Ask user to manually stop Lisk Core v3 and continue - const isLiskCoreV3Stopped = await cli.confirm( - "Please stop Lisk Core v3 to continue. Type 'yes' and press Enter when ready. [yes/no]", - ); + // Ask user to manually stop Lisk Core v3 and continue + const isLiskCoreV3Stopped = await cli.confirm( + "Please stop Lisk Core v3 to continue. Type 'yes' and press Enter when ready. [yes/no]", + ); - if (isLiskCoreV3Stopped) { - let numTriesLeft = 3; - while (numTriesLeft) { - numTriesLeft -= 1; + if (isLiskCoreV3Stopped) { + let numTriesLeft = 3; + while (numTriesLeft) { + numTriesLeft -= 1; - const isCoreV3Running = await isLiskCoreV3Running(liskCoreV3DataPath); - if (!isCoreV3Running) break; + const isCoreV3Running = await isLiskCoreV3Running(liskCoreV3DataPath); + if (!isCoreV3Running) break; - if (numTriesLeft >= 0) { - const isStopReconfirmed = await cli.confirm( - "Lisk Core v3 still running. Please stop the node, type 'yes' to proceed and 'no' to exit. [yes/no]", - ); - if (!isStopReconfirmed) { - throw new Error( - `Cannot proceed with Lisk Core v4 auto-start. Please continue manually. In order to access legacy blockchain information posts-migration, please copy the contents of the ${snapshotDirPath} directory to 'data/legacy.db' under the Lisk Core v4 data directory (e.g: ${DEFAULT_LISK_CORE_PATH}/data/legacy.db/). Exiting!!!`, + if (numTriesLeft >= 0) { + const isStopReconfirmed = await cli.confirm( + "Lisk Core v3 still running. Please stop the node, type 'yes' to proceed and 'no' to exit. [yes/no]", ); - } else if (numTriesLeft === 0 && isStopReconfirmed) { - const isCoreV3StillRunning = await isLiskCoreV3Running(liskCoreV3DataPath); - if (isCoreV3StillRunning) { + if (!isStopReconfirmed) { throw new Error( - `Cannot auto-start Lisk Core v4 as Lisk Core v3 is still running. Please continue manually. In order to access legacy blockchain information posts-migration, please copy the contents of the ${snapshotDirPath} directory to 'data/legacy.db' under the Lisk Core v4 data directory (e.g: ${DEFAULT_LISK_CORE_PATH}/data/legacy.db/). Exiting!!!`, + `Cannot proceed with Lisk Core v4 auto-start. Please continue manually. In order to access legacy blockchain information posts-migration, please copy the contents of the ${snapshotDirPath} directory to 'data/legacy.db' under the Lisk Core v4 data directory (e.g: ${DEFAULT_LISK_CORE_PATH}/data/legacy.db/). Exiting!!!`, ); + } else if (numTriesLeft === 0 && isStopReconfirmed) { + const isCoreV3StillRunning = await isLiskCoreV3Running(liskCoreV3DataPath); + if (isCoreV3StillRunning) { + throw new Error( + `Cannot auto-start Lisk Core v4 as Lisk Core v3 is still running. Please continue manually. In order to access legacy blockchain information posts-migration, please copy the contents of the ${snapshotDirPath} directory to 'data/legacy.db' under the Lisk Core v4 data directory (e.g: ${DEFAULT_LISK_CORE_PATH}/data/legacy.db/). Exiting!!!`, + ); + } } } } - } - const isUserConfirmed = await cli.confirm( - `Start Lisk Core with the following configuration? [yes/no] + const isUserConfirmed = await cli.confirm( + `Start Lisk Core with the following configuration? [yes/no] ${util.inspect(configCoreV4, false, 3)} `, - ); - - if (isUserConfirmed) { - cli.action.start('Starting Lisk Core v4'); - const networkName = networkConstant.name as string; - await startLiskCore(this, liskCoreV3DataPath, configCoreV4, networkName, outputDir); - this.log( - `Started Lisk Core v4 at default data directory ('${DEFAULT_LISK_CORE_PATH}').`, ); - cli.action.stop(); + + if (isUserConfirmed) { + cli.action.start('Starting Lisk Core v4'); + const networkName = networkConstant.name as string; + await startLiskCore(this, liskCoreV3DataPath, configCoreV4, networkName, outputDir); + this.log( + `Started Lisk Core v4 at default data directory ('${DEFAULT_LISK_CORE_PATH}').`, + ); + cli.action.stop(); + } else { + this.log( + 'User did not accept the migrated config. Skipping the Lisk Core v4 auto-start process.', + ); + } } else { - this.log( - 'User did not accept the migrated config. Skipping the Lisk Core v4 auto-start process.', + throw new Error( + `User did not confirm Lisk Core v3 node shutdown. Skipping the Lisk Core v4 auto-start process. Please continue manually. In order to access legacy blockchain information posts-migration, please copy the contents of the ${snapshotDirPath} directory to 'data/legacy.db' under the Lisk Core v4 data directory (e.g: ${DEFAULT_LISK_CORE_PATH}/data/legacy.db/). Exiting!!!`, ); } - } else { - throw new Error( - `User did not confirm Lisk Core v3 node shutdown. Skipping the Lisk Core v4 auto-start process. Please continue manually. In order to access legacy blockchain information posts-migration, please copy the contents of the ${snapshotDirPath} directory to 'data/legacy.db' under the Lisk Core v4 data directory (e.g: ${DEFAULT_LISK_CORE_PATH}/data/legacy.db/). Exiting!!!`, + } catch (err) { + const errorMsg = `Failed to auto-start Lisk Core v4.\nError: ${(err as Error).message}`; + throw new MigratorException( + errorMsg, + err instanceof MigratorException ? err.code : ERROR_CODE.LISK_CORE_START, ); } - } catch (err) { - const errorMsg = `Failed to auto-start Lisk Core v4.\nError: ${(err as Error).message}`; - throw new MigratorException( - errorMsg, - err instanceof MigratorException ? err.code : ERROR_CODE.LISK_CORE_START, + } else { + this.log( + `Please copy the contents of ${snapshotDirPath} directory to 'data/legacy.db' under the Lisk Core v4 data directory (e.g: ${DEFAULT_LISK_CORE_PATH}/data/legacy.db/) in order to access legacy blockchain information.`, ); + this.log('Please copy genesis block to the Lisk Core V4 network directory.'); } - } else { - this.log( - `Please copy the contents of ${snapshotDirPath} directory to 'data/legacy.db' under the Lisk Core v4 data directory (e.g: ${DEFAULT_LISK_CORE_PATH}/data/legacy.db/) in order to access legacy blockchain information.`, - ); - this.log('Please copy genesis block to the Lisk Core V4 network directory.'); } } catch (error) { const commandsToExecute: string[] = []; diff --git a/src/types.ts b/src/types.ts index 6b8515c6..a85c8903 100644 --- a/src/types.ts +++ b/src/types.ts @@ -398,3 +398,9 @@ export interface ForgingStatus { readonly maxHeightPrevoted?: number; readonly maxHeightPreviouslyForged?: number; } + +export interface FileInfo { + readonly fileName: string; + readonly fileDir: string; + readonly filePath: string; +} diff --git a/src/utils/download.ts b/src/utils/download.ts new file mode 100644 index 00000000..433957d8 --- /dev/null +++ b/src/utils/download.ts @@ -0,0 +1,104 @@ +/* + * LiskHQ/lisk-commander + * Copyright © 2023 Lisk Foundation + * + * See the LICENSE file at the top-level directory of this distribution + * for licensing information. + * + * Unless otherwise agreed in a custom licensing agreement with the Lisk Foundation, + * no part of this software, including this file, may be copied, modified, + * propagated, or distributed except according to the terms contained in the + * LICENSE file. + * + * Removal or modification of this copyright notice is prohibited. + * + */ +import * as crypto from 'crypto'; +import * as axios from 'axios'; +import * as fs from 'fs-extra'; +import * as path from 'path'; +import { extractTarBall } from './fs'; +import { FileInfo } from '../types'; + +export const getDownloadedFileInfo = (url: string, downloadDir: string): FileInfo => { + const pathWithoutProtocol = url.replace(/(^\w+:|^)\/\//, '').split('/'); + const fileName = pathWithoutProtocol.pop() as string; + const filePath = path.join(downloadDir, fileName); + + return { + fileName, + fileDir: downloadDir, + filePath, + }; +}; + +export const download = async (url: string, dir: string): Promise => { + const { filePath, fileDir } = getDownloadedFileInfo(url, dir); + + if (fs.existsSync(filePath)) { + fs.unlinkSync(filePath); + } + + fs.ensureDirSync(fileDir); + const writeStream = fs.createWriteStream(filePath); + const response = await axios.default({ + url, + method: 'GET', + responseType: 'stream', + maxContentLength: 5000, + }); + + response.data.pipe(writeStream); + + return new Promise((resolve, reject) => { + writeStream.on('finish', resolve); + writeStream.on('error', reject); + }); +}; + +export const verifyChecksum = async (filePath: string, expectedChecksum: string): Promise => { + const fileStream = fs.createReadStream(filePath); + const dataHash = crypto.createHash('sha256'); + const fileHash = await new Promise((resolve, reject) => { + fileStream.on('data', (datum: Buffer) => { + dataHash.update(datum); + }); + fileStream.on('error', error => { + reject(error); + }); + fileStream.on('end', () => { + resolve(dataHash.digest()); + }); + }); + + const fileChecksum = fileHash.toString('hex'); + if (fileChecksum !== expectedChecksum) { + throw new Error( + `File checksum: ${fileChecksum} mismatched with expected checksum: ${expectedChecksum}`, + ); + } +}; + +export const getChecksum = (url: string, dir: string): string => { + const { filePath } = getDownloadedFileInfo(url, dir); + const content = fs.readFileSync(`${filePath}.SHA256`, 'utf8'); + + if (!content) { + throw new Error(`Invalid filepath: ${filePath}`); + } + + return content.split(' ')[0]; +}; + +export const downloadAndValidate = async ( + url: string, + snapshotDirPath: string, + extractSnapshotPath: string, +): Promise => { + await download(url, snapshotDirPath); + await download(`${url}.SHA256`, snapshotDirPath); + const { filePath } = getDownloadedFileInfo(url, snapshotDirPath); + const checksum = getChecksum(url, snapshotDirPath); + await verifyChecksum(filePath, checksum); + await extractTarBall(filePath, extractSnapshotPath); +}; diff --git a/yarn.lock b/yarn.lock index 145a6983..0ef419cd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1848,6 +1848,13 @@ available-typed-arrays@^1.0.5: resolved "https://registry.yarnpkg.com/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz#92f95616501069d07d10edb2fc37d3e1c65123b7" integrity sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw== +axios@0.25.0: + version "0.25.0" + resolved "https://registry.yarnpkg.com/axios/-/axios-0.25.0.tgz#349cfbb31331a9b4453190791760a8d35b093e0a" + integrity sha512-cD8FOb0tRH3uuEe6+evtAbgJtfxr7ly3fQjYcMcuPlgkwVS9xboaVIpcDV+cYQe+yGykgwZCs1pzjntcGa6l5g== + dependencies: + follow-redirects "^1.14.7" + babel-jest@^27.5.1: version "27.5.1" resolved "https://registry.yarnpkg.com/babel-jest/-/babel-jest-27.5.1.tgz#a1bf8d61928edfefd21da27eb86a695bfd691444" @@ -3304,6 +3311,11 @@ flatted@^3.2.9: resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.2.9.tgz#7eb4c67ca1ba34232ca9d2d93e9886e611ad7daf" integrity sha512-36yxDn5H7OFZQla0/jFJmbIKTdZAQHngCedGxiMmpNfEZM0sdEeT+WczLQrjK6D7o2aiyLYDnkw0R3JK0Qv1RQ== +follow-redirects@^1.14.7: + version "1.15.3" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.3.tgz#fe2f3ef2690afce7e82ed0b44db08165b207123a" + integrity sha512-1VzOtuEM8pC9SFU1E+8KfTjZyMztRsgEfwQl44z8A25uy13jSzTj6dyK2Df52iV0vgHCfBwLhDWevLn95w5v6Q== + for-each@^0.3.3: version "0.3.3" resolved "https://registry.yarnpkg.com/for-each/-/for-each-0.3.3.tgz#69b447e88a0a5d32c3e7084f3f1710034b21376e"