From 0d20744de34182e607c79aa5edf39c947b90f1ca Mon Sep 17 00:00:00 2001 From: Sameer Kumar Subudhi Date: Tue, 24 Oct 2023 18:42:52 +0200 Subject: [PATCH] :sparkles: Add separate flag to support snapshot downloads --- .eslintignore | 3 ++- .prettierignore | 5 ++-- docs/migration.md | 12 +++++----- package.json | 4 ++-- src/index.ts | 55 ++++++++++++++++++++++++++++++------------- src/utils/download.ts | 16 ++++++------- src/utils/node.ts | 16 +------------ src/utils/path.ts | 17 ++++++++++++- yarn.lock | 12 +++++----- 9 files changed, 82 insertions(+), 58 deletions(-) diff --git a/.eslintignore b/.eslintignore index 2a9cbeaf..4b979dfd 100644 --- a/.eslintignore +++ b/.eslintignore @@ -8,4 +8,5 @@ coverage benchmark .eslintrc.js dist/ -test/_setup.js \ No newline at end of file +test/_setup.js +data/ diff --git a/.prettierignore b/.prettierignore index b8e1c989..0c4b2471 100644 --- a/.prettierignore +++ b/.prettierignore @@ -46,5 +46,6 @@ browsertest.build/ *.SHA256 blockchain.db/* -# Output directory -output/ \ No newline at end of file +# Output and data directory +output/ +data/ diff --git a/docs/migration.md b/docs/migration.md index 50460239..001dff13 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -82,10 +82,9 @@ OPTIONS -c, --config=config Custom configuration file path for Lisk Core v3.1.x. -d, --lisk-core-v3-data-path=lisk-core-v3-data-path Path where the Lisk Core v3.x instance is running. When not supplied, defaults to the default data directory for Lisk Core. -h, --help show CLI help - -n, --network=network Network to be considered for the migration. Depends on the '--snapshot-path' flag. + -n, --network=(mainnet|testnet) Network to be considered for the migration. Depends on the '--snapshot-path' flag. - -o, --output=output File path to write the genesis block. If not provided, it will default to cwd/output/{v3_networkIdentifier}/genesis_block.blob. Do not use any value - starting with the default data path reserved for Lisk Core: '~/.lisk/lisk-core'. + -o, --output=output File path to write the genesis block. If not provided, it will default to cwd/output/{v3_networkIdentifier}/genesis_block.blob. Do not use any value starting with the default data path reserved for Lisk Core: '~/.lisk/lisk-core'. -p, --page-size=page-size [default: 100000] Maximum number of blocks to be iterated at once for computation. Defaults to 100000. @@ -95,10 +94,11 @@ OPTIONS --auto-migrate-config Migrate user configuration automatically. Defaults to false. - --auto-start-lisk-core-v4 Start Lisk Core v4 automatically. Defaults to false. When using this flag, kindly open another terminal window to stop Lisk Core v3.1.x for when the - migrator prompts. + --auto-start-lisk-core-v4 Start Lisk Core v4 automatically. Defaults to false. When using this flag, kindly open another terminal window to stop Lisk Core v3.1.x for when the migrator prompts. - --snapshot-path=snapshot-path Path/URL to the state snapshot to run the migration offline. It could point either to a directory, a tarball (tar.gz) or a URL ending with tar.gz. Depends on the '--network (-n)' flag. + --snapshot-path=snapshot-path Path to the state snapshot to run the migration offline. It could point either to a directory or a tarball (tar.gz). + + --snapshot-url=snapshot-url URL to download the state snapshot from. Use to run the migration offline. URL must end with tar.gz. EXAMPLES lisk-migrator --snapshot-height 20931763 --lisk-core-path /path/to/data-dir diff --git a/package.json b/package.json index d83e0752..07f451c9 100644 --- a/package.json +++ b/package.json @@ -70,14 +70,14 @@ "@oclif/config": "1.14.0", "@oclif/errors": "1.2.2", "@oclif/plugin-help": "2.2.3", + "axios": "0.25.0", "cli-ux": "5.5.1", "debug": "4.3.1", "fs-extra": "11.1.0", "lisk-framework": "0.11.0-rc.5", "semver": "7.3.2", "shelljs": "^0.8.5", - "tar": "6.1.13", - "axios": "0.25.0" + "tar": "6.1.13" }, "devDependencies": { "@oclif/dev-cli": "1.22.2", diff --git a/src/index.ts b/src/index.ts index 4102e838..633ce747 100644 --- a/src/index.ts +++ b/src/index.ts @@ -62,16 +62,15 @@ import { startLiskCore, isLiskCoreV3Running, getLiskCoreStartCommand, - resolveSnapshotPath, } from './utils/node'; -import { resolveAbsolutePath, verifyOutputPath } from './utils/path'; +import { resolveAbsolutePath, verifyOutputPath, resolveSnapshotPath } from './utils/path'; import { execAsync } from './utils/process'; import { getBlockHeaderByHeight } from './utils/block'; import { MigratorException } from './utils/exception'; import { writeCommandsToExec } from './utils/commands'; import { getNetworkIdentifier } from './utils/network'; import { extractTarBall } from './utils/fs'; -import { downloadAndValidate } from './utils/download'; +import { downloadAndExtract } from './utils/download'; let configCoreV4: PartialApplicationConfig; class LiskMigrator extends Command { @@ -128,8 +127,17 @@ class LiskMigrator extends Command { required: false, env: 'SNAPSHOT_PATH', description: - "Path/URL to the state snapshot to run the migration offline. It could point either to a directory, a tarball (tar.gz) or a URL ending with tar.gz. Depends on the '--network (-n)' flag.", + 'Path to the state snapshot to run the migration offline. It could point either to a directory or a tarball (tar.gz).', dependsOn: ['network'], + exclusive: ['snapshot-url'], + }), + 'snapshot-url': flagsParser.string({ + required: false, + env: 'SNAPSHOT_URL', + description: + 'URL to download the state snapshot from. Use to run the migration offline. URL must end with tar.gz.', + dependsOn: ['network'], + exclusive: ['snapshot-path'], }), network: flagsParser.enum({ char: 'n', @@ -138,7 +146,6 @@ class LiskMigrator extends Command { description: "Network to be considered for the migration. Depends on the '--snapshot-path' flag.", options: ['mainnet', 'testnet'], - dependsOn: ['snapshot-path'], }), }; @@ -153,9 +160,25 @@ class LiskMigrator extends Command { const autoMigrateUserConfig = flags['auto-migrate-config'] ?? false; const autoStartLiskCoreV4 = flags['auto-start-lisk-core-v4']; const pageSize = Number(flags['page-size']); - const snapshotPath = flags['snapshot-path'] as string; + const snapshotPath = flags['snapshot-path'] + ? resolveAbsolutePath(flags['snapshot-path']) + : (flags['snapshot-path'] as string); + const snapshotURL = flags['snapshot-url'] as string; const network = flags.network as string; - const useSnapshot = !!snapshotPath || false; + const useSnapshot = !!(snapshotPath || snapshotURL); + + // Custom flag dependency check because neither exactlyOne or relationships properties are working for network + if (network && !useSnapshot) { + this.error( + 'Either --snapshot-path= or --snapshot-url= must be provided when using --network=', + ); + } + + if (snapshotURL && (!snapshotURL.startsWith('http') || !snapshotURL.endsWith('tar.gz'))) { + this.error( + `Expected --snapshot-url to begin with http(s) and end with 'tar.gz' instead received ${snapshotURL}.`, + ); + } verifyOutputPath(outputPath); @@ -170,14 +193,14 @@ class LiskMigrator extends Command { try { if (useSnapshot) { - if (snapshotPath.startsWith('http')) { - cli.action.start(`Downloading snapshot from ${snapshotPath} to ${outputDir}`); - await downloadAndValidate(snapshotPath, outputDir, dataDir); - cli.action.stop(); - } else if (snapshotPath.endsWith('.tar.gz')) { - cli.action.start(`Extracting snapshot at ${dataDir}`); + if (snapshotURL?.startsWith('http')) { + cli.action.start(`Downloading snapshot from ${snapshotURL} to ${outputDir}`); + await downloadAndExtract(snapshotURL, outputDir, dataDir); + cli.action.stop(`Successfully downloaded snapshot from ${snapshotURL} to ${outputDir}`); + } else if (snapshotPath?.endsWith('.tar.gz')) { + cli.action.start(`Extracting snapshot to ${dataDir}`); await extractTarBall(snapshotPath, dataDir); - cli.action.stop(); + cli.action.stop(`Successfully extracted snapshot to ${dataDir}`); } } else { const client = await getAPIClient(liskCoreV3DataPath); @@ -222,9 +245,7 @@ class LiskMigrator extends Command { ); } - // Using 'gt' instead of 'gte' because the behavior is swapped - // i.e. 'gt' acts as 'gte' and vice-versa - if (semver.gt(MIN_SUPPORTED_LISK_CORE_VERSION, appVersion)) { + if (semver.lt(appVersion, MIN_SUPPORTED_LISK_CORE_VERSION)) { this.error( `Lisk Migrator is not compatible with Lisk Core version ${appVersion}. Minimum supported version is ${MIN_SUPPORTED_LISK_CORE_VERSION}.`, ); diff --git a/src/utils/download.ts b/src/utils/download.ts index 433957d8..73bc7f7d 100644 --- a/src/utils/download.ts +++ b/src/utils/download.ts @@ -90,15 +90,15 @@ export const getChecksum = (url: string, dir: string): string => { return content.split(' ')[0]; }; -export const downloadAndValidate = async ( +export const downloadAndExtract = async ( url: string, - snapshotDirPath: string, - extractSnapshotPath: string, + downloadPath: string, + extractionPath: string, ): Promise => { - await download(url, snapshotDirPath); - await download(`${url}.SHA256`, snapshotDirPath); - const { filePath } = getDownloadedFileInfo(url, snapshotDirPath); - const checksum = getChecksum(url, snapshotDirPath); + await download(url, downloadPath); + await download(`${url}.SHA256`, downloadPath); + const { filePath } = getDownloadedFileInfo(url, downloadPath); + const checksum = getChecksum(url, downloadPath); await verifyChecksum(filePath, checksum); - await extractTarBall(filePath, extractSnapshotPath); + await extractTarBall(filePath, extractionPath); }; diff --git a/src/utils/node.ts b/src/utils/node.ts index b05cb2bf..9540b8da 100644 --- a/src/utils/node.ts +++ b/src/utils/node.ts @@ -21,7 +21,7 @@ import { renameSync } from 'fs-extra'; import { PartialApplicationConfig } from 'lisk-framework'; import { execAsync } from './process'; -import { copyDir, exists, getFiles } from './fs'; +import { copyDir, exists } from './fs'; import { isPortAvailable } from './network'; import { resolveAbsolutePath } from './path'; import { Port } from '../types'; @@ -253,17 +253,3 @@ export const startLiskCore = async ( ); } }; - -export const resolveSnapshotPath = async ( - useSnapshot: boolean, - snapshotPath: string, - dataDir: string, - liskCoreV3DataPath: string, -) => { - if (!useSnapshot) return path.join(liskCoreV3DataPath, SNAPSHOT_DIR); - if (!snapshotPath.endsWith('.tar.gz')) return snapshotPath; - - const [snapshotDirNameExtracted] = (await getFiles(dataDir)) as string[]; - const snapshotFilePathExtracted = path.join(dataDir, snapshotDirNameExtracted); - return snapshotFilePathExtracted; -}; diff --git a/src/utils/path.ts b/src/utils/path.ts index 55ee471b..5fc385b3 100644 --- a/src/utils/path.ts +++ b/src/utils/path.ts @@ -13,7 +13,8 @@ */ import { homedir } from 'os'; import { isAbsolute, join } from 'path'; -import { DEFAULT_LISK_CORE_PATH } from '../constants'; +import { getFiles } from './fs'; +import { DEFAULT_LISK_CORE_PATH, SNAPSHOT_DIR } from '../constants'; export const resolveAbsolutePath = (path: string) => { if (isAbsolute(path)) { @@ -37,3 +38,17 @@ export const verifyOutputPath = (_outputPath: string): void | Error => { ); } }; + +export const resolveSnapshotPath = async ( + useSnapshot: boolean, + snapshotPath: string, + dataDir: string, + liskCoreV3DataPath: string, +) => { + if (!useSnapshot) return join(liskCoreV3DataPath, SNAPSHOT_DIR); + if (snapshotPath && !snapshotPath.endsWith('.tar.gz')) return snapshotPath; + + const [snapshotDirNameExtracted] = (await getFiles(dataDir)) as string[]; + const snapshotFilePathExtracted = join(dataDir, snapshotDirNameExtracted); + return snapshotFilePathExtracted; +}; diff --git a/yarn.lock b/yarn.lock index 0ef419cd..0c9c234a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1223,9 +1223,9 @@ integrity sha512-K0VQKziLUWkVKiRVrx4a40iPaxTUefQmjtkQofBkYRcoaaL/8rhwDWww9qWbrgicNOgnpIsMxyNIUM4+n6dUIA== "@types/node@*": - version "20.8.7" - resolved "https://registry.yarnpkg.com/@types/node/-/node-20.8.7.tgz#ad23827850843de973096edfc5abc9e922492a25" - integrity sha512-21TKHHh3eUHIi2MloeptJWALuCu5H7HQTdTrWIFReA8ad+aggoX+lRes3ex7/FtpC+sVUpFMQ+QTfYr74mruiQ== + version "20.8.8" + resolved "https://registry.yarnpkg.com/@types/node/-/node-20.8.8.tgz#adee050b422061ad5255fc38ff71b2bb96ea2a0e" + integrity sha512-YRsdVxq6OaLfmR9Hy816IMp33xOBjfyOgUd77ehqg96CFywxAPbDbXvAsuN2KVg2HOT8Eh6uAfU+l4WffwPVrQ== dependencies: undici-types "~5.25.1" @@ -2662,9 +2662,9 @@ ed2curve@0.3.0: tweetnacl "1.x.x" electron-to-chromium@^1.4.535: - version "1.4.564" - resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.564.tgz#9c6ada8ec7b43c65d8629300a0916a346ac5c0c2" - integrity sha512-bGAx9+teIzL5I4esQwCMtiXtb78Ysc8xOKTPOvmafbJZ4SQ40kDO1ym3yRcGSkfaBtV81fGgHOgPoe6DsmpmkA== + version "1.4.565" + resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.565.tgz#205f3746a759ec3c43bce98b9eef5445f2721ea9" + integrity sha512-XbMoT6yIvg2xzcbs5hCADi0dXBh4//En3oFXmtPX+jiyyiCTiM9DGFT2SLottjpEs9Z8Mh8SqahbR96MaHfuSg== emittery@^0.8.1: version "0.8.1"