Skip to content

Commit f29d8f4

Browse files
wilwadewilwade
authored
Set Dev Genesis Starting Schema (#2160)
# Goal The goal of this PR is to update the development genesis to start at `16_000` so that schemas from mainnet do not collide. Closes #2126 # Discussion - Update Genesis schemas from mainnet - Updates to the state copy tool - Set the Genesis value of `CurrentSchemaIdentifierMaximum` to `16_000` # Checklist - [x] Unit Tests added? - [x] Spec version incremented?
1 parent c02cbe9 commit f29d8f4

File tree

8 files changed

+168
-10
lines changed

8 files changed

+168
-10
lines changed

‎pallets/schemas/src/lib.rs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -232,6 +232,8 @@ pub mod pallet {
232232

233233
#[pallet::genesis_config]
234234
pub struct GenesisConfig<T: Config> {
235+
/// Maximum schema size in bytes at genesis
236+
pub initial_schema_identifier_max: u16,
235237
/// Maximum schema size in bytes at genesis
236238
pub initial_max_schema_model_size: u32,
237239
/// Genesis Schemas to load for development
@@ -244,6 +246,7 @@ pub mod pallet {
244246
impl<T: Config> sp_std::default::Default for GenesisConfig<T> {
245247
fn default() -> Self {
246248
Self {
249+
initial_schema_identifier_max: 16_000,
247250
initial_max_schema_model_size: 1024,
248251
initial_schemas: Default::default(),
249252
_config: Default::default(),
@@ -286,6 +289,9 @@ pub mod pallet {
286289
)
287290
.expect("Failed to set Schema in Genesis!");
288291
}
292+
293+
// Set the maximum manually
294+
CurrentSchemaIdentifierMaximum::<T>::put(self.initial_schema_identifier_max);
289295
}
290296
}
291297

‎pallets/schemas/src/tests/other_tests.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1098,8 +1098,8 @@ fn genesis_config_build_genesis_schemas() {
10981098
System::set_block_number(1);
10991099
let res = CurrentSchemaIdentifierMaximum::<Test>::get();
11001100

1101-
// We should have at least 10
1102-
assert!(res >= 10);
1101+
// Should be set to 16_000
1102+
assert!(res == 16_000);
11031103

11041104
// Check that the first schema exists
11051105
let res = SchemasPallet::get_schema_by_id(1);

‎resources/genesis-schemas.json

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,5 +71,70 @@
7171
"settings": [],
7272
"model": "{\"type\":\"record\",\"name\":\"UserPrivateConnectionsChunk\",\"namespace\":\"org.dsnp\",\"fields\":[{\"name\":\"keyId\",\"type\":\"long\",\"doc\":\"User-Assigned Key Identifier\"},{\"name\":\"pridList\",\"type\":{\"type\":\"array\",\"items\":{\"name\":\"prid\",\"type\":\"fixed\",\"size\":8,\"doc\":\"Pseudonymous Relationship Identifier\"}}},{\"doc\":\"lib_sodium sealed box\",\"name\":\"encryptedCompressedPrivateGraph\",\"type\":\"bytes\"}],\"types\":[{\"type\":\"array\",\"name\":\"PrivateGraph\",\"namespace\":\"org.dsnp\",\"items\":{\"type\":\"record\",\"name\":\"GraphEdge\",\"fields\":[{\"name\":\"userId\",\"type\":\"long\",\"doc\":\"DSNP User Id of object of relationship\"},{\"name\":\"since\",\"type\":\"long\",\"doc\":\"Unix epoch in seconds when this relationship was originally established rounded to the nearest 1000\"}]}}]}",
7373
"name": ""
74+
},
75+
{
76+
"model_type": "Parquet",
77+
"payload_location": "IPFS",
78+
"settings": [],
79+
"model": "[{\"name\":\"announcementType\",\"column_type\":{\"INTEGER\":{\"bit_width\":32,\"sign\":true}},\"compression\":\"GZIP\",\"bloom_filter\":false},{\"name\":\"contentHash\",\"column_type\":\"STRING\",\"compression\":\"GZIP\",\"bloom_filter\":true},{\"name\":\"fromId\",\"column_type\":{\"INTEGER\":{\"bit_width\":64,\"sign\":false}},\"compression\":\"GZIP\",\"bloom_filter\":true},{\"name\":\"url\",\"column_type\":\"STRING\",\"compression\":\"GZIP\",\"bloom_filter\":false},{\"name\":\"targetAnnouncementType\",\"column_type\":{\"INTEGER\":{\"bit_width\":32,\"sign\":true}},\"compression\":\"GZIP\",\"bloom_filter\":false},{\"name\":\"targetContentHash\",\"column_type\":\"STRING\",\"compression\":\"GZIP\",\"bloom_filter\":true}]",
80+
"name": "dsnp.user-attribute-set"
81+
},
82+
{
83+
"model_type": "Parquet",
84+
"payload_location": "IPFS",
85+
"settings": [],
86+
"model": "[{\"name\":\"announcementType\",\"column_type\":{\"INTEGER\":{\"bit_width\":32,\"sign\":true}},\"compression\":\"GZIP\",\"bloom_filter\":false},{\"name\":\"fromId\",\"column_type\":{\"INTEGER\":{\"bit_width\":64,\"sign\":false}},\"compression\":\"GZIP\",\"bloom_filter\":true},{\"name\":\"subject\",\"column_type\":\"STRING\",\"compression\":\"GZIP\",\"bloom_filter\":true},{\"name\":\"url\",\"column_type\":\"STRING\",\"compression\":\"GZIP\",\"bloom_filter\":false},{\"name\":\"contentHash\",\"column_type\":\"STRING\",\"compression\":\"GZIP\",\"bloom_filter\":true},{\"name\":\"attributeSetType\",\"column_type\":\"STRING\",\"compression\":\"GZIP\",\"bloom_filter\":true},{\"name\":\"issuer\",\"column_type\":\"STRING\",\"compression\":\"GZIP\",\"bloom_filter\":true}]",
87+
"name": "dsnp.dsnp-content-attribute-set"
88+
},
89+
{
90+
"model_type": "Parquet",
91+
"payload_location": "IPFS",
92+
"settings": [],
93+
"model": "[{\"name\":\"announcementType\",\"column_type\":{\"INTEGER\":{\"bit_width\":32,\"sign\":true}},\"compression\":\"GZIP\",\"bloom_filter\":false},{\"name\":\"fromId\",\"column_type\":{\"INTEGER\":{\"bit_width\":64,\"sign\":false}},\"compression\":\"GZIP\",\"bloom_filter\":true},{\"name\":\"subject\",\"column_type\":\"STRING\",\"compression\":\"GZIP\",\"bloom_filter\":true},{\"name\":\"subjectContentHash\",\"column_type\":\"STRING\",\"compression\":\"GZIP\",\"bloom_filter\":true},{\"name\":\"url\",\"column_type\":\"STRING\",\"compression\":\"GZIP\",\"bloom_filter\":false},{\"name\":\"contentHash\",\"column_type\":\"STRING\",\"compression\":\"GZIP\",\"bloom_filter\":true},{\"name\":\"attributeSetType\",\"column_type\":\"STRING\",\"compression\":\"GZIP\",\"bloom_filter\":true},{\"name\":\"issuer\",\"column_type\":\"STRING\",\"compression\":\"GZIP\",\"bloom_filter\":true}]",
94+
"name": "dsnp.ext-content-attribute-set"
95+
},
96+
{
97+
"model_type": "AvroBinary",
98+
"payload_location": "Itemized",
99+
"settings": [
100+
"SignatureRequired"
101+
],
102+
"model": "{\"type\":\"record\",\"name\":\"PublicKey\",\"namespace\":\"org.dsnp\",\"fields\":[{\"name\":\"publicKey\",\"doc\":\"Multicodec public key\",\"type\":\"bytes\"}]}",
103+
"name": "dsnp.public-key-assertion-method"
104+
},
105+
{
106+
"model_type": "AvroBinary",
107+
"payload_location": "Itemized",
108+
"settings": [],
109+
"model": "{\"namespace\":\"org.dsnp\",\"name\":\"ProfileResource\",\"type\":\"record\",\"doc\":\"Profile-linked resource\",\"fields\":[{\"name\":\"type\",\"type\":\"int\",\"doc\":\"Type of resource\"},{\"name\":\"contentAddress\",\"type\":\"string\",\"doc\":\"Content address for the resource\"}]}",
110+
"name": "dsnp.profile-resources"
111+
},
112+
{
113+
"model_type": "Parquet",
114+
"payload_location": "IPFS",
115+
"settings": [],
116+
"model": "[{\"name\":\"announcementType\",\"column_type\":{\"INTEGER\":{\"bit_width\":32,\"sign\":true}},\"compression\":\"GZIP\",\"bloom_filter\":false},{\"name\":\"fromId\",\"column_type\":{\"INTEGER\":{\"bit_width\":64,\"sign\":false}},\"compression\":\"GZIP\",\"bloom_filter\":true},{\"name\":\"targetAnnouncementType\",\"column_type\":{\"INTEGER\":{\"bit_width\":32,\"sign\":true}},\"compression\":\"GZIP\",\"bloom_filter\":false},{\"name\":\"targetContentHash\",\"column_type\":\"STRING\",\"compression\":\"GZIP\",\"bloom_filter\":true}]",
117+
"name": "dsnp.tombstone"
118+
},
119+
{
120+
"model_type": "Parquet",
121+
"payload_location": "IPFS",
122+
"settings": [],
123+
"model": "[{\"name\":\"announcementType\",\"column_type\":{\"INTEGER\":{\"bit_width\":32,\"sign\":true}},\"compression\":\"GZIP\",\"bloom_filter\":false},{\"name\":\"contentHash\",\"column_type\":\"STRING\",\"compression\":\"GZIP\",\"bloom_filter\":true},{\"name\":\"fromId\",\"column_type\":{\"INTEGER\":{\"bit_width\":64,\"sign\":false}},\"compression\":\"GZIP\",\"bloom_filter\":true},{\"name\":\"url\",\"column_type\":\"STRING\",\"compression\":\"GZIP\",\"bloom_filter\":false}]",
124+
"name": "dsnp.broadcast"
125+
},
126+
{
127+
"model_type": "Parquet",
128+
"payload_location": "IPFS",
129+
"settings": [],
130+
"model": "[{\"name\":\"announcementType\",\"column_type\":{\"INTEGER\":{\"bit_width\":32,\"sign\":true}},\"compression\":\"GZIP\",\"bloom_filter\":false},{\"name\":\"contentHash\",\"column_type\":\"STRING\",\"compression\":\"GZIP\",\"bloom_filter\":true},{\"name\":\"fromId\",\"column_type\":{\"INTEGER\":{\"bit_width\":64,\"sign\":false}},\"compression\":\"GZIP\",\"bloom_filter\":true},{\"name\":\"inReplyTo\",\"column_type\":\"STRING\",\"compression\":\"GZIP\",\"bloom_filter\":true},{\"name\":\"url\",\"column_type\":\"STRING\",\"compression\":\"GZIP\",\"bloom_filter\":false}]",
131+
"name": "dsnp.reply"
132+
},
133+
{
134+
"model_type": "Parquet",
135+
"payload_location": "IPFS",
136+
"settings": [],
137+
"model": "[{\"name\":\"announcementType\",\"column_type\":{\"INTEGER\":{\"bit_width\":32,\"sign\":true}},\"compression\":\"GZIP\",\"bloom_filter\":false},{\"name\":\"contentHash\",\"column_type\":\"STRING\",\"compression\":\"GZIP\",\"bloom_filter\":true},{\"name\":\"fromId\",\"column_type\":{\"INTEGER\":{\"bit_width\":64,\"sign\":false}},\"compression\":\"GZIP\",\"bloom_filter\":true},{\"name\":\"url\",\"column_type\":\"STRING\",\"compression\":\"GZIP\",\"bloom_filter\":false},{\"name\":\"targetAnnouncementType\",\"column_type\":{\"INTEGER\":{\"bit_width\":32,\"sign\":true}},\"compression\":\"GZIP\",\"bloom_filter\":false},{\"name\":\"targetContentHash\",\"column_type\":\"STRING\",\"compression\":\"GZIP\",\"bloom_filter\":true}]",
138+
"name": "dsnp.update"
74139
}
75140
]

‎runtime/frequency/src/lib.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -375,7 +375,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion {
375375
spec_name: create_runtime_str!("frequency"),
376376
impl_name: create_runtime_str!("frequency"),
377377
authoring_version: 1,
378-
spec_version: 115,
378+
spec_version: 116,
379379
impl_version: 0,
380380
apis: apis::RUNTIME_API_VERSIONS,
381381
transaction_version: 1,
@@ -389,7 +389,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion {
389389
spec_name: create_runtime_str!("frequency-testnet"),
390390
impl_name: create_runtime_str!("frequency"),
391391
authoring_version: 1,
392-
spec_version: 115,
392+
spec_version: 116,
393393
impl_version: 0,
394394
apis: apis::RUNTIME_API_VERSIONS,
395395
transaction_version: 1,

‎tools/state-copy/README.md

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# State Copy Tool
2+
3+
Tools to help copy state from one Frequency Chain to another.
4+
5+
## Testnet Schemas Match
6+
7+
To maintain alignment with Mainnet, when a new schema is deployed on Mainnet, Testnet data and Dev Genesis are set to have those exact same schemas.
8+
9+
1. In the Frequency codebase: `cd tools/state-copy`
10+
2. `npm i`
11+
3. Edit `schemas.mjs`
12+
- Use the Testnet `DEST_URL`: `const DEST_URL = "wss://0.rpc.testnet.amplica.io";`
13+
- Update the`const SUDO_URI = "//Alice";` to be the SUDO key for Testnet
14+
4. `npm run schemas`
15+
5. Remove the changes

‎tools/state-copy/copy.mjs

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ async function getKeys(api, storageKey) {
1919
return result;
2020
}
2121

22-
export async function copy(sourceUrl, destUrl, storageKey, filterKeys = []) {
22+
export async function copy(sourceUrl, destUrl, storageKey, sudoUri = "//Alice", filterKeys = []) {
2323
// Connect to the state source
2424
const sourceProvider = new WsProvider(sourceUrl);
2525
const sourceApi = await ApiPromise.create({ provider: sourceProvider });
@@ -44,8 +44,7 @@ export async function copy(sourceUrl, destUrl, storageKey, filterKeys = []) {
4444

4545
console.log("Fetched all values", storageKV);
4646

47-
// Set up sudo account (assumes //Alice has sudo access)
48-
const sudoAccount = keyring.createFromUri("//Alice");
47+
const sudoAccount = keyring.createFromUri(sudoUri);
4948

5049
// Prepare and send sudo.sudo(system.setStorage()) call
5150
const sudoCall = destApi.tx.sudo.sudo(destApi.tx.system.setStorage(storageKV));

‎tools/state-copy/remove.mjs

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
import { ApiPromise, WsProvider } from "@polkadot/api";
2+
import { Keyring } from "@polkadot/keyring";
3+
4+
const keyring = new Keyring({ type: "sr25519" });
5+
6+
async function getKeys(api, storageKey) {
7+
const pageSize = 500;
8+
const result = [];
9+
let startKey = "";
10+
while (true) {
11+
const page = await api.rpc.state.getKeysPaged(storageKey, pageSize, startKey);
12+
result.push(...page.map((x) => x.toString()));
13+
if (page.length === 0) {
14+
break;
15+
}
16+
17+
startKey = page[page.length - 1].toString();
18+
}
19+
return result;
20+
}
21+
22+
export async function removeStorageKeys(destUrl, storageKey, sudoUri = "//Alice", filterKeys = []) {
23+
// Connect to destination source
24+
const destProvider = new WsProvider(destUrl);
25+
const destApi = await ApiPromise.create({ provider: destProvider });
26+
27+
console.log("Connected to destination network");
28+
29+
// Get all keys from the specified storage key prefix
30+
const keys = (await getKeys(destApi, storageKey)).filter((k) => !filterKeys.includes(k));
31+
console.log(`Found ${keys.length} keys under ${storageKey}...`);
32+
33+
const sudoAccount = keyring.createFromUri(sudoUri);
34+
35+
// Prepare and send sudo.sudo(system.killStorage()) call
36+
const sudoCall = destApi.tx.sudo.sudo(destApi.tx.system.killStorage(keys));
37+
38+
console.log("Submitting sudo call to remove storage...");
39+
await new Promise(async (resolve, reject) => {
40+
const unsub = await sudoCall.signAndSend(sudoAccount, ({ status, events }) => {
41+
if (status.isInBlock || status.isFinalized) {
42+
console.log(
43+
`Block hash: ${(status.isInBlock && status.asInBlock) || (status.isFinalized && status.asFinalized)}`,
44+
);
45+
const success = events.find((x) => destApi.events.system.ExtrinsicSuccess.is(x.event));
46+
const failure = events.find((x) => destApi.events.system.ExtrinsicFailed.is(x.event));
47+
unsub();
48+
if (success && !failure) {
49+
console.log("State removal successful");
50+
resolve();
51+
} else {
52+
console.error("State removal FAILED!");
53+
reject();
54+
}
55+
}
56+
});
57+
});
58+
}

‎tools/state-copy/schemas.mjs

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,33 @@
1-
// This small nodejs script will pull the schemas from mainnet and then load them onto local
1+
// This small nodejs script will pull the schema pallet storage from mainnet and then load it onto another chain.
2+
//
3+
// WARNING: This will move ALL storage values, so if there are some you do NOT want, use the FILTER_OUT or another setup.
24

35
import { copy } from "./copy.mjs";
46

7+
// Set up sudo account (assumes //Alice has sudo access)
8+
const SUDO_URI = "//Alice";
9+
10+
// Testnet
11+
// const DEST_URL = "wss://0.rpc.testnet.amplica.io";
12+
// const SOURCE_URL = "wss://0.rpc.testnet.amplica.io";
13+
14+
// Localhost
15+
const DEST_URL = "ws://localhost:9944";
16+
17+
// Mainnet
518
const SOURCE_URL = "wss://1.rpc.frequency.xyz";
6-
const DEST_URL = "ws://127.0.0.1:9944";
719
const STORAGE_KEY = "0xeec6f3c13d26ae2507c99b6751e19e76";
820
const FILTER_OUT = [
921
"0xeec6f3c13d26ae2507c99b6751e19e76d5d9c370c6c8aee1116ee09d6811b0d5", // governanceSchemaModelMaxBytes
1022
"0xeec6f3c13d26ae2507c99b6751e19e764e7b9012096b41c4eb3aaf947f6ea429", // palletVersion
23+
// Comment this out to INCLUDE setting the currentSchemaIdentifierMaximum from the SOURCE
24+
"0xeec6f3c13d26ae2507c99b6751e19e765b81a4f27a1e406724e3a53d909f29cd", // currentSchemaIdentifierMaximum
1125
];
1226

1327
async function main() {
1428
try {
15-
await copy(SOURCE_URL, DEST_URL, STORAGE_KEY, FILTER_OUT);
29+
await copy(SOURCE_URL, DEST_URL, STORAGE_KEY, SUDO_URI, FILTER_OUT);
30+
1631
process.exit(0);
1732
} catch (error) {
1833
console.error("Error:", error);

0 commit comments

Comments
 (0)