offchain: double index to overcome overrides (#2360)

# Goal
The goal of this PR is to fix the bug in offchain worker
Closes #2293 

There are 2 kinds of errors:
- Some MsaIds have multiple indexed keys which only one of them is valid
(ex: 1394377)
- Some MsaIds have no indexed keys (ex: 1414115)

The main issue is that forks cause overwrites on offchain indexed events
so we had to find a way to fix these overwrites.

# Attempted solution

We are now double indexing any event so that if one of them gets
overwritten the other one would most likely get consumed. To do this we
are getting a pseudo random index from 1 to 1000 and store that event
inside a bucket associated with that block and index. The only tradeoff
is that we have to look at all the buckets for each block inside
offchain worker to fetch these stored events since there is no
`iter_prefix` from inside offchain worker.

# Discussion
- To fix the existing wrong indices I tried to add a runtime but any
code not called from offchain worker context can not access to any
offchain worker related features such as locks and timestamps and ...
- Since adding an extrinsic was the easiest way to handle it added a new
extrinsic (similar to `remark`) which only adds an event to reindexing a
certain msa id. This is not the best solution since we have to spend
some token to reindex but it was the fastest and easiest to do.
Currently it's not free but if desired we can make it free or capacity
supported but will need to ensure it's not going to be abused
- The other way to fix this is to add a new rpc and refactor the code
such that it can be called from this rpc but it was a heavier lift

# Further attempts that didn't work
- Using the Remark event to inject the reindex data doesn't work since
the event only stores the hash of the remark not the data
- Using the Remark Extrinsic to inject the reindex data also doesn't
work since it is only stored temporarily and if the offchain worker does
not run in the the exact time-window that it's available it would be
lost.


# Checklist
- [x] Unit Tests added?
- [x] Benchmarks added?
- [x] Spec version incremented?

Author: aramikm

pallets/msa/src/benchmarking.rs

@@ -353,6 +353,22 @@ mod benchmarks {
 		Ok(())
 	}
 
+	#[benchmark]
+	fn reindex_offchain() -> Result<(), BenchmarkError> {
+		let key = create_account::<T>("account", 0);
+		let caller = whitelisted_caller();
+		let msa_id = 1u64;
+		let event = OffchainReplayEvent::MsaPallet(MsaOffchainReplayEvent::KeyReIndex {
+			msa_id,
+			index_key: Some(key),
+		});
+
+		#[extrinsic_call]
+		_(RawOrigin::Signed(caller), event);
+
+		Ok(())
+	}
+
 	impl_benchmark_test_suite!(
 		Msa,
 		crate::tests::mock::new_test_ext_keystore(),

pallets/msa/src/lib.rs

@@ -427,7 +427,7 @@ pub mod pallet {
 				Self::create_account(public_key, |_| -> DispatchResult { Ok(()) })?;
 
 			let event = Event::MsaCreated { msa_id: new_msa_id, key: new_public_key };
-			offchain_index_event::<T>(&event, new_msa_id);
+			offchain_index_event::<T>(Some(&event), new_msa_id);
 			Self::deposit_event(event);
 			Ok(())
 		}
@@ -496,7 +496,7 @@ pub mod pallet {
 
 			let event =
 				Event::MsaCreated { msa_id: new_delegator_msa_id, key: new_delegator_public_key };
-			offchain_index_event::<T>(&event, new_delegator_msa_id);
+			offchain_index_event::<T>(Some(&event), new_delegator_msa_id);
 			Self::deposit_event(event);
 			Self::deposit_event(Event::DelegationGranted {
 				delegator_id: DelegatorId(new_delegator_msa_id),
@@ -688,7 +688,7 @@ pub mod pallet {
 						msa_id,
 						key: add_key_payload.new_public_key.clone(),
 					};
-					offchain_index_event::<T>(&event, msa_id);
+					offchain_index_event::<T>(Some(&event), msa_id);
 					Self::deposit_event(event);
 					Ok(())
 				},
@@ -725,7 +725,7 @@ pub mod pallet {
 
 					// Deposit the event
 					let event = Event::PublicKeyDeleted { key: public_key_to_delete };
-					offchain_index_event::<T>(&event, who_msa_id);
+					offchain_index_event::<T>(Some(&event), who_msa_id);
 					Self::deposit_event(event);
 				},
 				None => {
@@ -811,7 +811,7 @@ pub mod pallet {
 				Some(msa_id) => {
 					Self::delete_key_for_msa(msa_id, &who)?;
 					let event = Event::PublicKeyDeleted { key: who };
-					offchain_index_event::<T>(&event, msa_id);
+					offchain_index_event::<T>(Some(&event), msa_id);
 					Self::deposit_event(event);
 					Self::deposit_event(Event::MsaRetired { msa_id });
 				},
@@ -877,6 +877,35 @@ pub mod pallet {
 			});
 			Ok(())
 		}
+
+		/// A generic endpoint to replay any offchain related event to fix any potential issues
+		#[pallet::call_index(13)]
+		#[pallet::weight(T::WeightInfo::reindex_offchain())]
+		pub fn reindex_offchain(
+			origin: OriginFor<T>,
+			event: OffchainReplayEvent<T>,
+		) -> DispatchResult {
+			let _ = ensure_signed(origin)?;
+			match event {
+				OffchainReplayEvent::MsaPallet(MsaOffchainReplayEvent::KeyReIndex {
+					msa_id,
+					index_key,
+				}) => {
+					// don't need to check existence of msa_id since it would get checked on offchain side
+					match index_key {
+						Some(key) => {
+							let event = Event::PublicKeyAdded { msa_id, key };
+							offchain_index_event::<T>(Some(&event), msa_id);
+						},
+						None => {
+							offchain_index_event::<T>(None, msa_id);
+						},
+					}
+				},
+			}
+
+			Ok(())
+		}
 	}
 }