refactor related venues + event fetching for performance/efficiency (#1368)

* add fetchVenue + refactor fetchSovereignVenueId to use it + track previously checked venues to avoid circular references
* add getVenueCollectionRef + getVenueRef
* add fetchChildVenues
* add rough implementation plan for fetchRelatedVenues
* move everything from api/sovereignVenue into api/venue
* Implements sovereign venue search; Implements all child venues search
* refactor fetchAllChildVenues into fetchDirectChildVenues + fetchDescendantVenues
* add FIRESTORE_QUERY_IN_ARRAY_MAX_ITEMS to settings
* refactor fetchDirectChildVenues to handle multiple venueIds with 'chunked firestore in array' query pattern
* rename fetchSovereignVenueId -> fetchSovereignVenue
* refactor fetchSovereignVenue to take options prop with named keys
* add maxDepth option to fetchSovereignVenue
* refactor fetchDescendantVenues to make use of fetchDirectChildVenues's chunked query pattern
* fix fetchDescendantVenues to properly recurse
* ensure fetchDirectChildVenues returns quickly when no ids to check
* refactor fetchDescendantVenues to take options prop + add maxDepth
* add anyVenueWithIdConverter + refactor code to use it
* rename useConnectRelatedVenues -> useLegacyConnectRelatedVenues
* refactor useSovereignVenueId to accept venueId prop directly
* refactor useSovereignVenueId to expose errorMsg from state
* refactor useSovereignVenueId to wrap main code in useEffect
* add useRelatedVenues
* ensure react-hooks/exhaustive-deps correctly lints useAsync's deps array
* ensure our default emptyArray has a stable signature
* cleanup reactions context/provider pattern
* rename hooks/useRelatedVenues.ts -> hooks/useRelatedVenues.tsx
* refactor useRelatedVenues to use context/provider pattern
* refactor RelatedVenuesProvider to make venueId optional
* stub out usage of useLegacyConnectRelatedVenues in unused legacy LiveSchedule component
* add findVenueInRelatedVenues to RelatedVenuesProvider
* add sovereignVenue to RelatedVenuesProvider
* refactor useRelatedVenues to split into useRelatedVenues / useRelatedVenuesContext + add currentVenue / parentVenue
* refactor VenuePartygoers to accept venueId as a prop
* add RelatedVenuesProvider to WithNavigationBar
* refactor VenuePartygoers to replace useLegacyConnectRelatedVenues with useRelatedVenues
* add RelatedVenuesProvider to TemplateWrapper
* stub out the seemingly legacy SplashPage route as it doesn't have a venueId in scope anyway
* fix infinite recursion bug in fetchDescendantVenues
* allow createPerformanceTrace to accept a string as the traceName
* add tracePromise to utils/performance
* wrap fetchRelatedVenues with tracePromise in RelatedVenuesProvider
* add asArray helper + use it to simplify our param conversions
* add getVenueEventCollectionRef to api/events
* add fetchVenueEvents / venueEventWithIdConverter to api/events
* add fetchAllVenueEvents to api/events
* add useVenueEvents hook
* refactor ArtPiece to pass venue as prop + cleanup legacy default export
* cleanup ArtPiece imports
* refactor SchedulePageModal to accept venueId prop + fix usages
* add relatedVenueIds to RelatedVenuesProvider
* refactor SchedulePageModal to use useRelatedVenues + useVenueEvents
* remove useLegacyConnectRelatedVenues + related selectors/types/etc
* refactor PartyMap + related types to accept venue prop
* wrap fetchSovereignVenue with tracePromise in useSovereignVenueId
* use venueEventsSelector in VenueLandingPage
* cleanup firestore connections/selectors in VenuePage to keep related together
* fix all occurrences of updateTheme to use useEffect
* add @debt comments to places that try and load the entire venues collection into memory
* use orderedVenuesSelector in VenueList
* add @debt comments to places that try to select the entire venues collection from memory
* add @debt / @deprecated comments to venuesSelector / orderedVenuesSelector / makeVenueSelector
* refactor UserProfileModal to accept venue prop
* refactor stripePromise to use tracePromise
* pass venue to SuspectedLocation as prop
* replace fetchSovereignVenueId with useSovereignVenueId in ProfilePictureInput

Author: 0xdevalias

.eslintrc

@@ -52,6 +52,13 @@
       }
     ],
     "react/prop-types": "off",
-    "react/display-name": "warn"
+    "react/display-name": "warn",
+    // @see https://github.com/facebook/react/tree/master/packages/eslint-plugin-react-hooks#advanced-configuration
+    "react-hooks/exhaustive-deps": [
+      "warn",
+      {
+        "additionalHooks": "(useAsync)"
+      }
+    ]
   }
 }

functions/stats.js

@@ -19,6 +19,7 @@ const eventIsNow = (event, now) => {
   );
 };
 
+// @debt This function will currently load all venues in firebase into memory.. not very efficient
 exports.getOnlineStats = functions.https.onCall(async () => {
   const now = new Date().getTime();
 
@@ -70,6 +71,7 @@ exports.getAllEvents = functions.https.onCall(async () => {
   );
 });
 
+// @debt This function will currently load all venues in firebase into memory.. not very efficient
 exports.getLiveAndFutureEvents = functions.https.onCall(async () => {
   try {
     const now = new Date().getTime();

scripts/get-events.ts

@@ -43,6 +43,7 @@ initFirebaseAdminApp(projectId, {
       .join(",")
   );
 
+  // @debt This function will currently load all venues in firebase into memory.. not very efficient
   const firestoreVenues = await admin.firestore().collection("venues").get();
 
   firestoreVenues.docs.forEach((doc) => {

scripts/get-venue-owners.ts

@@ -39,6 +39,7 @@ initFirebaseAdminApp(projectId, {
     nextPageToken = pageToken;
   }
 
+  // @debt This function will currently load all venues in firebase into memory.. not very efficient
   const firestoreVenues = await admin.firestore().collection("venues").get();
   const venues = firestoreVenues.docs.filter(
     (doc) =>

scripts/get-venues.ts

@@ -58,6 +58,7 @@ initFirebaseAdminApp(projectId, {
       .join(",")
   );
 
+  // @debt This function will currently load all venues in firebase into memory.. not very efficient
   const firestoreVenues = await admin.firestore().collection("venues").get();
 
   firestoreVenues.docs.forEach((doc) => {

scripts/rescale-placement.ts

@@ -25,6 +25,7 @@ const dryRun = dryRunFlag === "true";
 initFirebaseAdminApp(projectId);
 
 (async () => {
+  // @debt This function will currently load all venues in firebase into memory.. not very efficient
   const firestoreVenues = await admin.firestore().collection("venues").get();
 
   for (const doc of firestoreVenues.docs) {

src/api/events.ts

@@ -0,0 +1,55 @@
+import firebase from "firebase/app";
+
+import { VenueEvent } from "types/venues";
+
+import { withId, WithId, withVenueId, WithVenueId } from "utils/id";
+import { asArray } from "utils/types";
+
+import { getVenueRef } from "./venue";
+
+export const getVenueEventCollectionRef = (venueId: string) =>
+  getVenueRef(venueId).collection("events");
+
+export const fetchVenueEvents = async (
+  venueId: string
+): Promise<WithVenueId<WithId<VenueEvent>>[]> =>
+  getVenueEventCollectionRef(venueId)
+    .withConverter(venueEventWithIdConverter)
+    .get()
+    .then((docSnapshot) =>
+      docSnapshot.docs.map((venueEvent) =>
+        withVenueId(venueEvent.data(), venueId)
+      )
+    );
+
+export const fetchAllVenueEvents = async (
+  venueIdOrIds: string | string[]
+): Promise<WithVenueId<WithId<VenueEvent>>[]> =>
+  Promise.all(asArray(venueIdOrIds).map(fetchVenueEvents)).then((result) =>
+    result.flat()
+  );
+
+/**
+ * Convert VenueEvent objects between the app/firestore formats (@debt:, including validation).
+ */
+export const venueEventWithIdConverter: firebase.firestore.FirestoreDataConverter<
+  WithId<VenueEvent>
+> = {
+  toFirestore: (
+    venueEvent: WithId<VenueEvent>
+  ): firebase.firestore.DocumentData => {
+    // @debt Properly check/validate this data
+    //   return VenueEventSchema.validateSync(venueEvent);
+
+    return venueEvent;
+  },
+
+  fromFirestore: (
+    snapshot: firebase.firestore.QueryDocumentSnapshot
+  ): WithId<VenueEvent> => {
+    // @debt Properly check/validate this data rather than using 'as'
+    //   return withId(VenueEventSchema.validateSync(snapshot.data(), snapshot.id);
+
+    return withId(snapshot.data() as VenueEvent, snapshot.id);
+  },
+};

src/api/sovereignVenue.ts

@@ -1,5 +1,13 @@
 import Bugsnag from "@bugsnag/js";
 import firebase from "firebase/app";
+import { chunk } from "lodash";
+
+import { FIRESTORE_QUERY_IN_ARRAY_MAX_ITEMS } from "settings";
+
+import { AnyVenue } from "types/venues";
+
+import { withId, WithId } from "utils/id";
+import { asArray } from "utils/types";
 
 export const getVenueCollectionRef = () =>
   firebase.firestore().collection("venues");
@@ -40,3 +48,144 @@ export const setVenueLiveStatus = async ({
     })
     .finally(onFinish);
 };
+
+export interface FetchSovereignVenueOptions {
+  previouslyCheckedVenueIds?: readonly string[];
+  maxDepth?: number;
+}
+
+export interface FetchSovereignVenueReturn {
+  sovereignVenue: WithId<AnyVenue>;
+  checkedVenueIds: readonly string[];
+}
+
+export const fetchSovereignVenue = async (
+  venueId: string,
+  options?: FetchSovereignVenueOptions
+): Promise<FetchSovereignVenueReturn> => {
+  const { previouslyCheckedVenueIds = [], maxDepth } = options ?? {};
+
+  const venue = await fetchVenue(venueId);
+
+  if (!venue) throw new Error(`The '${venueId}' venue doesn't exist`);
+
+  if (!venue.parentId)
+    return {
+      sovereignVenue: venue,
+      checkedVenueIds: previouslyCheckedVenueIds,
+    };
+
+  if (previouslyCheckedVenueIds.includes(venueId))
+    throw new Error(
+      `Circular reference detected. '${venueId}' has already been checked`
+    );
+
+  if (maxDepth && maxDepth <= 0)
+    throw new Error("Maximum depth reached before finding the sovereignVenue.");
+
+  return fetchSovereignVenue(venue.parentId, {
+    ...options,
+    previouslyCheckedVenueIds: [...previouslyCheckedVenueIds, venueId],
+    maxDepth: maxDepth ? maxDepth - 1 : undefined,
+  });
+};
+
+export const fetchVenue = async (
+  venueId: string
+): Promise<WithId<AnyVenue> | undefined> =>
+  getVenueRef(venueId)
+    .withConverter(anyVenueWithIdConverter)
+    .get()
+    .then((docSnapshot) => docSnapshot.data());
+
+export const fetchDirectChildVenues = async (
+  venueIdOrIds: string | string[]
+): Promise<WithId<AnyVenue>[]> => {
+  const venueIds = asArray(venueIdOrIds);
+
+  if (venueIds.length <= 0) return [];
+
+  return Promise.all(
+    chunk(venueIds, FIRESTORE_QUERY_IN_ARRAY_MAX_ITEMS).map(
+      async (venueIdsChunk: string[]) => {
+        const childVenuesSnapshot = await getVenueCollectionRef()
+          .where("parentId", "in", venueIdsChunk)
+          .withConverter(anyVenueWithIdConverter)
+          .get();
+
+        return childVenuesSnapshot.docs
+          .filter((docSnapshot) => docSnapshot.exists)
+          .map((docSnapshot) => docSnapshot.data());
+      }
+    )
+  ).then((result) => result.flat());
+};
+
+export interface FetchDescendantVenuesOptions {
+  maxDepth?: number;
+}
+
+export const fetchDescendantVenues = async (
+  venueIdOrIds: string | string[],
+  options?: FetchDescendantVenuesOptions
+): Promise<WithId<AnyVenue>[]> => {
+  const venueIds = asArray(venueIdOrIds);
+
+  const { maxDepth } = options ?? {};
+
+  if (venueIds.length <= 0) return [];
+
+  const directChildVenues: WithId<AnyVenue>[] = await fetchDirectChildVenues(
+    venueIds
+  );
+
+  if (maxDepth && maxDepth <= 0 && directChildVenues.length > 0)
+    throw new Error(
+      "Maximum depth reached before fetching all descendant venues."
+    );
+
+  const descendantVenues: WithId<AnyVenue>[] = await fetchDescendantVenues(
+    directChildVenues.map((venue) => venue.id),
+    {
+      ...options,
+      maxDepth: maxDepth ? maxDepth - 1 : undefined,
+    }
+  );
+
+  return [...directChildVenues, ...descendantVenues];
+};
+
+export const fetchRelatedVenues = async (
+  venueId: string
+): Promise<WithId<AnyVenue>[]> => {
+  const { sovereignVenue } = await fetchSovereignVenue(venueId);
+
+  const descendantVenues = await fetchDescendantVenues(sovereignVenue.id);
+
+  return [sovereignVenue, ...descendantVenues];
+};
+
+/**
+ * Convert Venue objects between the app/firestore formats (@debt:, including validation).
+ */
+export const anyVenueWithIdConverter: firebase.firestore.FirestoreDataConverter<
+  WithId<AnyVenue>
+> = {
+  toFirestore: (
+    anyVenue: WithId<AnyVenue>
+  ): firebase.firestore.DocumentData => {
+    // @debt Properly check/validate this data
+    //   return AnyVenueSchema.validateSync(anyVenue);
+
+    return anyVenue;
+  },
+
+  fromFirestore: (
+    snapshot: firebase.firestore.QueryDocumentSnapshot
+  ): WithId<AnyVenue> => {
+    // @debt Properly check/validate this data rather than using 'as'
+    //   return withId(AnyVenueSchema.validateSync(snapshot.data(), snapshot.id);
+
+    return withId(snapshot.data() as AnyVenue, snapshot.id);
+  },
+};

src/api/venue.ts

@@ -1,29 +1,35 @@
 import React, { FC, useCallback, useMemo } from "react";
 
-import { VenueEvent } from "types/venues";
+import { AnyVenue, VenueEvent } from "types/venues";
 
 import { isEventLive } from "utils/event";
 import { venueSelector } from "utils/selectors";
-import { WithVenueId } from "utils/id";
+import { WithId, WithVenueId } from "utils/id";
 
-import { useConnectRelatedVenues } from "hooks/useConnectRelatedVenues";
+// import { useLegacyConnectRelatedVenues } from "hooks/useRelatedVenues";
 import { useSelector } from "hooks/useSelector";
-import { useVenueId } from "hooks/useVenueId";
+// import { useVenueId } from "hooks/useVenueId";
 
 import { EventDisplay } from "../EventDisplay";
 
 import "./LiveSchedule.scss";
 import { hasElements } from "utils/types";
 
+const emptyArray: never[] = [];
+
 const LiveSchedule: FC = () => {
-  const venueId = useVenueId();
+  // const venueId = useVenueId();
   const currentVenue = useSelector(venueSelector);
-  useConnectRelatedVenues({ venueId });
 
-  const { relatedVenueEvents, relatedVenues } = useConnectRelatedVenues({
-    venueId,
-    withEvents: true,
-  });
+  // @debt Stubbing out legacy code as this component isn't used anymore and is getting deleted in a different PR.
+  // useLegacyConnectRelatedVenues({ venueId });
+  //
+  // const { relatedVenueEvents, relatedVenues } = useLegacyConnectRelatedVenues({
+  //   venueId,
+  //   withEvents: true,
+  // });
+  const relatedVenues: WithId<AnyVenue>[] = emptyArray;
+  const relatedVenueEvents: WithVenueId<VenueEvent>[] = emptyArray;
 
   const relatedVenueFor = useCallback(
     (event: WithVenueId<VenueEvent>) =>

src/components/molecules/LiveSchedule/LiveSchedule.tsx

@@ -201,7 +201,7 @@ const NavBar: React.FC<NavBarPropsType> = ({
               >
                 {navbarTitle} <span className="schedule-text">Schedule</span>
               </div>
-              <VenuePartygoers />
+              <VenuePartygoers venueId={venueId} />
             </div>
 
             {!user && <NavBarLogin />}
@@ -315,7 +315,10 @@ const NavBar: React.FC<NavBarPropsType> = ({
         }`}
         onClick={hideEventSchedule}
       >
-        <SchedulePageModal isVisible={isEventScheduleVisible} />
+        <SchedulePageModal
+          venueId={venueId}
+          isVisible={isEventScheduleVisible}
+        />
       </div>
 
       {/* @debt Remove back button from Navbar */}