key-migration: control key scripts (#2294)

# Goal
The goal of this PR is to create scripts to be able to fetch control
keys

Related to #2277 

# Details
- added a tool to be able to fetch all msa ids and their control keys
from `publicKeyToMsaId` storage
- added a tool to be able to fetch all msa ids and their control keys
from offchain indexer
- the output is in compatible so that after sorting we can compare them
for any discrepancies
- added a tool to be able to fetch all token keys

# Control Keys with Tokens
The script should be executed close to the migration to get the latest
state but for now we have these control keys with tokens on main-net.
```
"0xca94146d680192fbeff69aa6080e2f9daff8d8d238b9a76f136e35d5d3083d3c"
"0x748a616c3171d049c7f331df9918e010ff27625d22c016ce6a6d3c2f24004166"
"0x54fa640b48cbfdea066434189cecdd11f76c26b2499ece246fc1c69ea0e74761"
"0x94c602c107c5c11a5e9fed59f65140701bb48678a10e667c32c8a03914a0ac0c"
"0x5e1fb4d1e2faebbf2ffa2cb50718b57064cfd3c6b1445948f87fcef923108f4d"
"0x068cbd7799b593960797a2c698f7943dd60ec8720bb28ed4019f0387444a7862"
"0xaed8aa458599d23d83ba2bd13d5e9afb4aacb826e54d6876a41e01c65f72da4e"
"0xae139214802b320dff4f202d37554e6899d37c5913927b9af440f783f8ec5a38"
"0x58083af872b860bb71a359b92f1d2c67b9c5c970a64b5f264130e02533b0f372"
"0x861b30dae9083e3bebaf427d08ec20b4b71282a987342c226cfc3bfbdeabaf61"
```

Author: aramikm

tools/eth-migration/README.md

@@ -0,0 +1,20 @@
+# Eth migrations tools
+Some scripts and tools to facilitate the key migration
+
+## Get Onchain msa id and keys
+`npm run control-keys --silent`
+
+
+## Get Offchain Indexed msa id and keys
+`npm run offchain-keys --silent`
+
+### Sort and Compare to see any discrepancies
+1. `sort -t',' -k1,1n -k2,2 onchain-keys.txt >> onchain-keys-sorted.txt`
+2. `sort -t',' -k1,1n -k2,2 offchain-keys.txt >> offchain-keys-sorted.txt`
+3. `diff onchain-keys-sorted.txt offchain-keys-sorted.txt`
+
+## Get Token keys
+`npm run token-keys --silent`
+
+### Select token keys which are also control keys
+`jq -c 'select(.msaId != 0)' tokens-keys.txt`

tools/eth-migration/fetch-control-keys.mjs

@@ -0,0 +1,71 @@
+// Only way to silence PolkadotJS API warnings we don't want
+console.warn = () => {};
+
+import { ApiPromise, WsProvider } from "@polkadot/api";
+
+const SOURCE_URL = process.env["FREQUENCY_URL"] || "wss://1.rpc.frequency.xyz";
+const BATCH_SIZE = 1000;
+
+const options = {
+	rpc: {
+		msa: {
+			getKeysByMsaId: {
+				description: 'Fetch Keys for an MSA Id',
+				params: [
+					{
+						name: 'msa_id',
+						type: 'MessageSourceId',
+					},
+				],
+				type: 'Option<KeyInfoResponse>',
+			},
+		},
+	},
+	types: {
+		MessageSourceId: 'u64',
+		KeyInfoResponse: {
+			msa_keys: 'Vec<AccountId>',
+			msa_id: 'MessageSourceId',
+		},
+	},
+};
+
+export async function fetchControlKeysFromState(sourceUrl) {
+	// Connect to the state source
+	const sourceProvider = new WsProvider(sourceUrl);
+	const sourceApi = await ApiPromise.create({provider: sourceProvider, ...options});
+
+	let startKey;
+	let result;
+	do {
+		result = await sourceApi.query.msa.publicKeyToMsaId.entriesPaged({
+			args: [],
+			pageSize: BATCH_SIZE,
+			startKey
+		});
+
+		result.forEach(([key, value]) => {
+			if (!value.isSome) {
+				console.error(`No MsaId for ${key.args[0].toHex()}`);
+			} else {
+				console.log(`${value.unwrap().toString()},${key.args[0].toHex()}`);
+			}
+		});
+
+		if (result.length > 0) {
+			startKey = result[result.length - 1][0];
+		}
+	} while (result.length > 0);
+}
+
+async function main() {
+	try {
+		await fetchControlKeysFromState(SOURCE_URL);
+		process.exit(0);
+	} catch (error) {
+		console.error("Error:", error);
+		process.exit(1);
+	}
+}
+
+main();

tools/eth-migration/fetch-offchain-index-keys.mjs

@@ -0,0 +1,85 @@
+// Only way to silence PolkadotJS API warnings we don't want
+console.warn = () => {};
+
+import { ApiPromise, WsProvider } from "@polkadot/api";
+
+const SOURCE_URL = process.env["FREQUENCY_URL"] || "wss://1.rpc.frequency.xyz";
+const PROMISE_BATCH_SIZE = 500;
+
+const options = {
+	rpc: {
+		msa: {
+			getKeysByMsaId: {
+				description: 'Fetch Keys for an MSA Id',
+				params: [
+					{
+						name: 'msa_id',
+						type: 'MessageSourceId',
+					},
+				],
+				type: 'Option<KeyInfoResponse>',
+			},
+		},
+	},
+	types: {
+		MessageSourceId: 'u64',
+		KeyInfoResponse: {
+			msa_keys: 'Vec<AccountId>',
+			msa_id: 'MessageSourceId',
+		},
+	},
+};
+
+export async function fetchControlKeysFromOffchainRpc(sourceUrl) {
+	// Connect to the state source
+	const sourceProvider = new WsProvider(sourceUrl);
+	const sourceApi = await ApiPromise.create({provider: sourceProvider, ...options});
+
+	// Get the max schema id
+	let msaMaxId = (await sourceApi.query.msa.currentMsaIdentifierMaximum()).toNumber();
+	let promises = [];
+	let id = 1;
+	let batchNumber = 0;
+	for (; id <= msaMaxId ; id++){
+		promises.push(sourceApi.rpc.msa.getKeysByMsaId(id));
+		if (id % PROMISE_BATCH_SIZE === 0 && promises.length > 0) {
+			printRpcResults(await Promise.all(promises), batchNumber);
+			promises = [];
+			msaMaxId = (await sourceApi.query.msa.currentMsaIdentifierMaximum()).toNumber();
+			batchNumber++;
+		}
+	}
+
+	if (promises.length > 0) {
+		printRpcResults(await Promise.all(promises), batchNumber);
+	}
+}
+
+function printRpcResults(results, batchNumber) {
+	let index = 0;
+	for (const r of results) {
+		if (!r.isSome) {
+			console.error(`No keys for MsaId: ${(batchNumber * PROMISE_BATCH_SIZE + index +1)}!`);
+		} else {
+			const keys = r.unwrap();
+			for (const key of keys.msa_keys){
+				console.log(`${keys.msa_id},${key.toHex()}`);
+			}
+		}
+		index ++;
+	}
+}
+
+
+
+async function main() {
+	try {
+		await fetchControlKeysFromOffchainRpc(SOURCE_URL);
+		process.exit(0);
+	} catch (error) {
+		console.error("Error:", error);
+		process.exit(1);
+	}
+}
+
+main();

tools/eth-migration/fetch-token-keys.mjs

@@ -0,0 +1,79 @@
+// Only way to silence PolkadotJS API warnings we don't want
+console.warn = () => {};
+
+import { ApiPromise, WsProvider } from "@polkadot/api";
+
+const SOURCE_URL = process.env["FREQUENCY_URL"] || "wss://1.rpc.frequency.xyz";
+const BATCH_SIZE = 1000;
+
+const options = {
+	rpc: {
+		msa: {
+			getKeysByMsaId: {
+				description: 'Fetch Keys for an MSA Id',
+				params: [
+					{
+						name: 'msa_id',
+						type: 'MessageSourceId',
+					},
+				],
+				type: 'Option<KeyInfoResponse>',
+			},
+		},
+	},
+	types: {
+		MessageSourceId: 'u64',
+		KeyInfoResponse: {
+			msa_keys: 'Vec<AccountId>',
+			msa_id: 'MessageSourceId',
+		},
+	},
+};
+
+// {
+// 	"publicKey": "0x488e14e049fd59b77e5821c4bb71fc9c4b8ced4c19ed27b3949e91b6274f1400",
+//  "msaId": 1,
+// 	"values": {
+// 		"nonce": 0,
+// 			"consumers": 0,
+// 			"providers": 1,
+// 			"sufficients": 0,
+// 			"data": {
+// 			"free": 5000000000,
+// 			"reserved": 0,
+// 			"frozen": 0,
+// 			"flags": "0x80000000000000000000000000000000"
+// 		}
+// 	}
+// }
+export async function fetchTokenKeysFromState(sourceUrl) {
+	// Connect to the state source
+	const sourceProvider = new WsProvider(sourceUrl);
+	const sourceApi = await ApiPromise.create({provider: sourceProvider, ...options});
+
+	let result;
+	do {
+		result = await sourceApi.query.system.account.entriesPaged({ args: [], pageSize: BATCH_SIZE, startKey: result && result[result.length - 1][0] });
+		let msaResult = await sourceApi.query.msa.publicKeyToMsaId.multi(result.map(([key, _]) => key.args[0]));
+		result.forEach(([key, account], index) => {
+			const obj = {
+				publicKey: key.args[0].toHex(),
+				msaId: msaResult[index].isSome ? msaResult[index].unwrap().toNumber() : 0,
+				values: account,
+			}
+			console.log(`${JSON.stringify(obj)}`);
+		});
+	} while (result.length === BATCH_SIZE);
+}
+
+async function main() {
+	try {
+		await fetchTokenKeysFromState(SOURCE_URL);
+		process.exit(0);
+	} catch (error) {
+		console.error("Error:", error);
+		process.exit(1);
+	}
+}
+
+main();