✨ Add separate flag to support snapshot downloads

.eslintignore

@@ -8,4 +8,5 @@ coverage
 benchmark
 .eslintrc.js
 dist/
-test/_setup.js
+test/_setup.js
+data/

.prettierignore

@@ -46,5 +46,6 @@ browsertest.build/
 *.SHA256
 blockchain.db/*
 
-# Output directory
-output/
+# Output and data directory
+output/
+data/

docs/migration.md

@@ -82,10 +82,9 @@ OPTIONS
   -c, --config=config                                  Custom configuration file path for Lisk Core v3.1.x.
   -d, --lisk-core-v3-data-path=lisk-core-v3-data-path  Path where the Lisk Core v3.x instance is running. When not supplied, defaults to the default data directory for Lisk Core.
   -h, --help                                           show CLI help
-  -n, --network=network                                Network to be considered for the migration. Depends on the '--snapshot-path' flag.
+  -n, --network=(mainnet|testnet)                      Network to be considered for the migration. Depends on the '--snapshot-path' flag.
 
-  -o, --output=output                                  File path to write the genesis block. If not provided, it will default to cwd/output/{v3_networkIdentifier}/genesis_block.blob. Do not use any value
-                                                       starting with the default data path reserved for Lisk Core: '~/.lisk/lisk-core'.
+  -o, --output=output                                  File path to write the genesis block. If not provided, it will default to cwd/output/{v3_networkIdentifier}/genesis_block.blob. Do not use any value starting with the default data path reserved for Lisk Core: '~/.lisk/lisk-core'.
 
   -p, --page-size=page-size                            [default: 100000] Maximum number of blocks to be iterated at once for computation. Defaults to 100000.
 
@@ -95,10 +94,11 @@ OPTIONS
 
   --auto-migrate-config                                Migrate user configuration automatically. Defaults to false.
 
-  --auto-start-lisk-core-v4                            Start Lisk Core v4 automatically. Defaults to false. When using this flag, kindly open another terminal window to stop Lisk Core v3.1.x for when the
-                                                       migrator prompts.
+  --auto-start-lisk-core-v4                            Start Lisk Core v4 automatically. Defaults to false. When using this flag, kindly open another terminal window to stop Lisk Core v3.1.x for when the migrator prompts.
 
-  --snapshot-path=snapshot-path                        Path/URL to the state snapshot to run the migration offline. It could point either to a directory, a tarball (tar.gz) or a URL ending with tar.gz. Depends on the '--network (-n)' flag.
+  --snapshot-path=snapshot-path                        Path to the state snapshot to run the migration offline. It could point either to a directory or a tarball (tar.gz).
+
+  --snapshot-url=snapshot-url                          URL to download the state snapshot from. Use to run the migration offline. URL must end with tar.gz.
 
 EXAMPLES
   lisk-migrator --snapshot-height 20931763 --lisk-core-path /path/to/data-dir

package.json

@@ -70,14 +70,14 @@
 		"@oclif/config": "1.14.0",
 		"@oclif/errors": "1.2.2",
 		"@oclif/plugin-help": "2.2.3",
+		"axios": "0.25.0",
 		"cli-ux": "5.5.1",
 		"debug": "4.3.1",
 		"fs-extra": "11.1.0",
 		"lisk-framework": "0.11.0-rc.5",
 		"semver": "7.3.2",
 		"shelljs": "^0.8.5",
-		"tar": "6.1.13",
-		"axios": "0.25.0"
+		"tar": "6.1.13"
 	},
 	"devDependencies": {
 		"@oclif/dev-cli": "1.22.2",

src/index.ts

@@ -62,16 +62,15 @@ import {
 	startLiskCore,
 	isLiskCoreV3Running,
 	getLiskCoreStartCommand,
-	resolveSnapshotPath,
 } from './utils/node';
-import { resolveAbsolutePath, verifyOutputPath } from './utils/path';
+import { resolveAbsolutePath, verifyOutputPath, resolveSnapshotPath } from './utils/path';
 import { execAsync } from './utils/process';
 import { getBlockHeaderByHeight } from './utils/block';
 import { MigratorException } from './utils/exception';
 import { writeCommandsToExec } from './utils/commands';
 import { getNetworkIdentifier } from './utils/network';
 import { extractTarBall } from './utils/fs';
-import { downloadAndValidate } from './utils/download';
+import { downloadAndExtract } from './utils/download';
 
 let configCoreV4: PartialApplicationConfig;
 class LiskMigrator extends Command {
@@ -128,8 +127,17 @@ class LiskMigrator extends Command {
 			required: false,
 			env: 'SNAPSHOT_PATH',
 			description:
-				"Path/URL to the state snapshot to run the migration offline. It could point either to a directory, a tarball (tar.gz) or a URL ending with tar.gz. Depends on the '--network (-n)' flag.",
+				'Path to the state snapshot to run the migration offline. It could point either to a directory or a tarball (tar.gz).',
 			dependsOn: ['network'],
+			exclusive: ['snapshot-url'],
+		}),
+		'snapshot-url': flagsParser.string({
+			required: false,
+			env: 'SNAPSHOT_URL',
+			description:
+				'URL to download the state snapshot from. Use to run the migration offline. URL must end with tar.gz.',
+			dependsOn: ['network'],
+			exclusive: ['snapshot-path'],
 		}),
 		network: flagsParser.enum({
 			char: 'n',
@@ -138,7 +146,6 @@ class LiskMigrator extends Command {
 			description:
 				"Network to be considered for the migration. Depends on the '--snapshot-path' flag.",
 			options: ['mainnet', 'testnet'],
-			dependsOn: ['snapshot-path'],
 		}),
 	};
 
@@ -153,9 +160,25 @@ class LiskMigrator extends Command {
 		const autoMigrateUserConfig = flags['auto-migrate-config'] ?? false;
 		const autoStartLiskCoreV4 = flags['auto-start-lisk-core-v4'];
 		const pageSize = Number(flags['page-size']);
-		const snapshotPath = flags['snapshot-path'] as string;
+		const snapshotPath = flags['snapshot-path']
+			? resolveAbsolutePath(flags['snapshot-path'])
+			: (flags['snapshot-path'] as string);
+		const snapshotURL = flags['snapshot-url'] as string;
 		const network = flags.network as string;
-		const useSnapshot = !!snapshotPath || false;
+		const useSnapshot = !!(snapshotPath || snapshotURL);
+
+		// Custom flag dependency check because neither exactlyOne or relationships properties are working for network
+		if (network && !useSnapshot) {
+			this.error(
+				'Either --snapshot-path= or --snapshot-url= must be provided when using --network=',
+			);
+		}
+
+		if (snapshotURL && (!snapshotURL.startsWith('http') || !snapshotURL.endsWith('tar.gz'))) {
+			this.error(
+				`Expected --snapshot-url to begin with http(s) and end with 'tar.gz' instead received ${snapshotURL}.`,
+			);
+		}
 
 		verifyOutputPath(outputPath);
 
@@ -170,14 +193,14 @@ class LiskMigrator extends Command {
 
 		try {
 			if (useSnapshot) {
-				if (snapshotPath.startsWith('http')) {
-					cli.action.start(`Downloading snapshot from ${snapshotPath} to ${outputDir}`);
-					await downloadAndValidate(snapshotPath, outputDir, dataDir);
-					cli.action.stop();
-				} else if (snapshotPath.endsWith('.tar.gz')) {
-					cli.action.start(`Extracting snapshot at ${dataDir}`);
+				if (snapshotURL?.startsWith('http')) {
+					cli.action.start(`Downloading snapshot from ${snapshotURL} to ${outputDir}`);
+					await downloadAndExtract(snapshotURL, outputDir, dataDir);
+					cli.action.stop(`Successfully downloaded snapshot from ${snapshotURL} to ${outputDir}`);
+				} else if (snapshotPath?.endsWith('.tar.gz')) {
+					cli.action.start(`Extracting snapshot to ${dataDir}`);
 					await extractTarBall(snapshotPath, dataDir);
-					cli.action.stop();
+					cli.action.stop(`Successfully extracted snapshot to ${dataDir}`);
 				}
 			} else {
 				const client = await getAPIClient(liskCoreV3DataPath);
@@ -222,9 +245,7 @@ class LiskMigrator extends Command {
 					);
 				}
 
-				// Using 'gt' instead of 'gte' because the behavior is swapped
-				// i.e. 'gt' acts as 'gte' and vice-versa
-				if (semver.gt(MIN_SUPPORTED_LISK_CORE_VERSION, appVersion)) {
+				if (semver.lt(appVersion, MIN_SUPPORTED_LISK_CORE_VERSION)) {
 					this.error(
 						`Lisk Migrator is not compatible with Lisk Core version ${appVersion}. Minimum supported version is ${MIN_SUPPORTED_LISK_CORE_VERSION}.`,
 					);

src/utils/download.ts

@@ -90,15 +90,15 @@ export const getChecksum = (url: string, dir: string): string => {
 	return content.split(' ')[0];
 };
 
-export const downloadAndValidate = async (
+export const downloadAndExtract = async (
 	url: string,
-	snapshotDirPath: string,
-	extractSnapshotPath: string,
+	downloadPath: string,
+	extractionPath: string,
 ): Promise<void> => {
-	await download(url, snapshotDirPath);
-	await download(`${url}.SHA256`, snapshotDirPath);
-	const { filePath } = getDownloadedFileInfo(url, snapshotDirPath);
-	const checksum = getChecksum(url, snapshotDirPath);
+	await download(url, downloadPath);
+	await download(`${url}.SHA256`, downloadPath);
+	const { filePath } = getDownloadedFileInfo(url, downloadPath);
+	const checksum = getChecksum(url, downloadPath);
 	await verifyChecksum(filePath, checksum);
-	await extractTarBall(filePath, extractSnapshotPath);
+	await extractTarBall(filePath, extractionPath);
 };

src/utils/node.ts

@@ -21,7 +21,7 @@ import { renameSync } from 'fs-extra';
 import { PartialApplicationConfig } from 'lisk-framework';
 
 import { execAsync } from './process';
-import { copyDir, exists, getFiles } from './fs';
+import { copyDir, exists } from './fs';
 import { isPortAvailable } from './network';
 import { resolveAbsolutePath } from './path';
 import { Port } from '../types';
@@ -253,17 +253,3 @@ export const startLiskCore = async (
 		);
 	}
 };
-
-export const resolveSnapshotPath = async (
-	useSnapshot: boolean,
-	snapshotPath: string,
-	dataDir: string,
-	liskCoreV3DataPath: string,
-) => {
-	if (!useSnapshot) return path.join(liskCoreV3DataPath, SNAPSHOT_DIR);
-	if (!snapshotPath.endsWith('.tar.gz')) return snapshotPath;
-
-	const [snapshotDirNameExtracted] = (await getFiles(dataDir)) as string[];
-	const snapshotFilePathExtracted = path.join(dataDir, snapshotDirNameExtracted);
-	return snapshotFilePathExtracted;
-};

src/utils/path.ts

@@ -13,7 +13,8 @@
  */
 import { homedir } from 'os';
 import { isAbsolute, join } from 'path';
-import { DEFAULT_LISK_CORE_PATH } from '../constants';
+import { getFiles } from './fs';
+import { DEFAULT_LISK_CORE_PATH, SNAPSHOT_DIR } from '../constants';
 
 export const resolveAbsolutePath = (path: string) => {
 	if (isAbsolute(path)) {
@@ -37,3 +38,17 @@ export const verifyOutputPath = (_outputPath: string): void | Error => {
 		);
 	}
 };
+
+export const resolveSnapshotPath = async (
+	useSnapshot: boolean,
+	snapshotPath: string,
+	dataDir: string,
+	liskCoreV3DataPath: string,
+) => {
+	if (!useSnapshot) return join(liskCoreV3DataPath, SNAPSHOT_DIR);
+	if (snapshotPath && !snapshotPath.endsWith('.tar.gz')) return snapshotPath;
+
+	const [snapshotDirNameExtracted] = (await getFiles(dataDir)) as string[];
+	const snapshotFilePathExtracted = join(dataDir, snapshotDirNameExtracted);
+	return snapshotFilePathExtracted;
+};

yarn.lock

@@ -1223,9 +1223,9 @@
   integrity sha512-K0VQKziLUWkVKiRVrx4a40iPaxTUefQmjtkQofBkYRcoaaL/8rhwDWww9qWbrgicNOgnpIsMxyNIUM4+n6dUIA==
 
 "@types/node@*":
-  version "20.8.7"
-  resolved "https://registry.yarnpkg.com/@types/node/-/node-20.8.7.tgz#ad23827850843de973096edfc5abc9e922492a25"
-  integrity sha512-21TKHHh3eUHIi2MloeptJWALuCu5H7HQTdTrWIFReA8ad+aggoX+lRes3ex7/FtpC+sVUpFMQ+QTfYr74mruiQ==
+  version "20.8.8"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-20.8.8.tgz#adee050b422061ad5255fc38ff71b2bb96ea2a0e"
+  integrity sha512-YRsdVxq6OaLfmR9Hy816IMp33xOBjfyOgUd77ehqg96CFywxAPbDbXvAsuN2KVg2HOT8Eh6uAfU+l4WffwPVrQ==
   dependencies:
     undici-types "~5.25.1"
 
@@ -2662,9 +2662,9 @@ ed2curve@0.3.0:
     tweetnacl "1.x.x"
 
 electron-to-chromium@^1.4.535:
-  version "1.4.564"
-  resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.564.tgz#9c6ada8ec7b43c65d8629300a0916a346ac5c0c2"
-  integrity sha512-bGAx9+teIzL5I4esQwCMtiXtb78Ysc8xOKTPOvmafbJZ4SQ40kDO1ym3yRcGSkfaBtV81fGgHOgPoe6DsmpmkA==
+  version "1.4.565"
+  resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.565.tgz#205f3746a759ec3c43bce98b9eef5445f2721ea9"
+  integrity sha512-XbMoT6yIvg2xzcbs5hCADi0dXBh4//En3oFXmtPX+jiyyiCTiM9DGFT2SLottjpEs9Z8Mh8SqahbR96MaHfuSg==
 
 emittery@^0.8.1:
   version "0.8.1"