dashboard: split out standby guardians

evan-gray · evan-gray · commit d63b975cb464 · 2024-08-22T10:38:41.000-06:00
diff --git a/common/src/consts.ts b/common/src/consts.ts
@@ -311,6 +311,13 @@ export const GUARDIAN_SET_4 = [
   },
 ];
 
+export const STANDBY_GUARDIANS = [
+  {
+    pubkey: '0x68c16a92903c4c74ffddc730582ba53d967d3dac',
+    name: 'Google Cloud',
+  },
+];
+
 export type GuardianSetInfo = {
   timestamp: string;
   contract: string;
diff --git a/dashboard/src/components/Alerts.tsx b/dashboard/src/components/Alerts.tsx
@@ -18,7 +18,11 @@ import { useCallback, useMemo, useState } from 'react';
 import { Environment, useCurrentEnvironment } from '../contexts/NetworkContext';
 import { ChainIdToHeartbeats } from '../hooks/useChainHeartbeats';
 import { Heartbeat } from '../utils/getLastHeartbeats';
-import { GUARDIAN_SET_4, chainIdToName } from '@wormhole-foundation/wormhole-monitor-common';
+import {
+  GUARDIAN_SET_4,
+  STANDBY_GUARDIANS,
+  chainIdToName,
+} from '@wormhole-foundation/wormhole-monitor-common';
 
 export const BEHIND_DIFF = 1000;
 export const CHAIN_LESS_THAN_MAX_WARNING_THRESHOLD = 2;
@@ -56,15 +60,23 @@ function chainDownAlerts(
   chainIdsToHeartbeats: ChainIdToHeartbeats,
   environment: Environment
 ): AlertEntry[] {
+  // don't count standby guardians in chain down alerts
+  const filteredHeartbeats = heartbeats.filter(
+    (hb) => !STANDBY_GUARDIANS.find((g) => g.pubkey.toLowerCase() === hb.guardianAddr.toLowerCase())
+  );
   const downChains: { [chainId: string]: string[] } = {};
   Object.entries(chainIdsToHeartbeats)
     // Aurora is known to be disabled, no need to alert on it
     .filter(([chainId]) => chainId !== chainToChainId('Aurora').toString())
     .forEach(([chainId, chainHeartbeats]) => {
+      // don't count standby guardians in chain down alerts
+      const filteredChainHeartbeats = chainHeartbeats.filter(
+        (hb) => !STANDBY_GUARDIANS.find((g) => g.pubkey.toLowerCase() === hb.guardian.toLowerCase())
+      );
       // Search for known guardians without heartbeats
-      const missingGuardians = heartbeats.filter(
+      const missingGuardians = filteredHeartbeats.filter(
         (guardianHeartbeat) =>
-          chainHeartbeats.findIndex(
+          filteredChainHeartbeats.findIndex(
             (chainHeartbeat) => chainHeartbeat.guardian === guardianHeartbeat.guardianAddr
           ) === -1
       );
@@ -78,7 +90,7 @@ function chainDownAlerts(
       // Could be disconnected or erroring post initial checks
       // Track highest height to check for lagging guardians
       let highest = BigInt(0);
-      chainHeartbeats.forEach((chainHeartbeat) => {
+      filteredChainHeartbeats.forEach((chainHeartbeat) => {
         const height = BigInt(chainHeartbeat.network.height);
         if (height > highest) {
           highest = height;
@@ -91,7 +103,7 @@ function chainDownAlerts(
         }
       });
       // Search for guardians which are lagging significantly behind
-      chainHeartbeats.forEach((chainHeartbeat) => {
+      filteredChainHeartbeats.forEach((chainHeartbeat) => {
         if (chainHeartbeat.network.height !== '0') {
           const height = BigInt(chainHeartbeat.network.height);
           const diff = highest - height;
diff --git a/dashboard/src/components/Chains.tsx b/dashboard/src/components/Chains.tsx
@@ -23,16 +23,19 @@ import {
   useMediaQuery,
 } from '@mui/material';
 import {
-  SortingState,
   createColumnHelper,
   getCoreRowModel,
   getSortedRowModel,
+  SortingState,
   useReactTable,
 } from '@tanstack/react-table';
+import { chainToChainId } from '@wormhole-foundation/sdk-base';
+import { chainIdToName, STANDBY_GUARDIANS } from '@wormhole-foundation/wormhole-monitor-common';
 import { useCallback, useMemo, useState } from 'react';
 import { Environment, useCurrentEnvironment } from '../contexts/NetworkContext';
 import { useSettingsContext } from '../contexts/SettingsContext';
 import { ChainIdToHeartbeats, HeartbeatInfo } from '../hooks/useChainHeartbeats';
+import { CHAIN_ICON_MAP } from '../utils/consts';
 import {
   BEHIND_DIFF,
   CHAIN_LESS_THAN_MAX_WARNING_THRESHOLD,
@@ -42,9 +45,6 @@ import {
 } from './Alerts';
 import CollapsibleSection from './CollapsibleSection';
 import Table from './Table';
-import { CHAIN_ICON_MAP } from '../utils/consts';
-import { chainToChainId } from '@wormhole-foundation/sdk-base';
-import { chainIdToName } from '@wormhole-foundation/wormhole-monitor-common';
 
 const columnHelper = createColumnHelper<HeartbeatInfo>();
 
@@ -111,6 +111,19 @@ function Chain({
   conditionalRowStyle?: ((a: HeartbeatInfo) => SxProps<Theme> | undefined) | undefined;
   environment: Environment;
 }) {
+  const [guardianHeartbeats, standbyHeartbeats] = useMemo(
+    () =>
+      heartbeats.reduce(
+        ([guardian, standby], hb) => {
+          if (STANDBY_GUARDIANS.find((g) => g.pubkey.toLowerCase() === hb.guardian.toLowerCase())) {
+            return [guardian, [...standby, hb]];
+          }
+          return [[...guardian, hb], standby];
+        },
+        [[] as HeartbeatInfo[], [] as HeartbeatInfo[]]
+      ),
+    [heartbeats]
+  );
   const smUp = useMediaQuery((theme: any) => theme.breakpoints.up('sm'));
   const {
     settings: { showChainName },
@@ -132,7 +145,7 @@ function Chain({
                 {chainIdToName(Number(chainId))} ({chainId})
               </Typography>
               <Typography>
-                {healthyCount} / {heartbeats.length}
+                {healthyCount} / {guardianHeartbeats.length}
               </Typography>
             </Box>
           }
@@ -150,7 +163,7 @@ function Chain({
             <Box sx={{ position: 'relative', display: 'inline-flex' }}>
               <CircularProgress
                 variant="determinate"
-                value={healthyCount === 0 ? 100 : (healthyCount / heartbeats.length) * 100}
+                value={healthyCount === 0 ? 100 : (healthyCount / guardianHeartbeats.length) * 100}
                 color={
                   healthyCount < getQuorumCount(environment)
                     ? 'error'
@@ -206,7 +219,11 @@ function Chain({
           {chainIdToName(Number(chainId))} ({chainId})
         </DialogTitle>
         <DialogContent>
-          <ChainDetails heartbeats={heartbeats} conditionalRowStyle={conditionalRowStyle} />
+          <ChainDetails heartbeats={guardianHeartbeats} conditionalRowStyle={conditionalRowStyle} />
+          <Typography variant="subtitle1" sx={{ mt: 2, mb: 1 }}>
+            Standby Guardians
+          </Typography>
+          <ChainDetails heartbeats={standbyHeartbeats} conditionalRowStyle={conditionalRowStyle} />
         </DialogContent>
       </Dialog>
     </>
@@ -238,15 +255,18 @@ function Chains({ chainIdsToHeartbeats }: { chainIdsToHeartbeats: ChainIdToHeart
     let numErrors = 0;
     const helpers = Object.entries(chainIdsToHeartbeats).reduce((obj, [chainId, heartbeats]) => {
       let highest = BigInt(0);
-      heartbeats.forEach((heartbeat) => {
+      const filteredHeartbeats = heartbeats.filter(
+        (hb) => !STANDBY_GUARDIANS.find((g) => g.pubkey.toLowerCase() === hb.guardian.toLowerCase())
+      );
+      filteredHeartbeats.forEach((heartbeat) => {
         const height = BigInt(heartbeat.network.height);
         if (height > highest) {
           highest = height;
         }
       });
       const conditionalRowStyle = (heartbeat: HeartbeatInfo) =>
         isHeartbeatUnhealthy(heartbeat, highest) ? { backgroundColor: 'rgba(100,0,0,.2)' } : {};
-      const healthyCount = heartbeats.reduce(
+      const healthyCount = filteredHeartbeats.reduce(
         (count, heartbeat) => count + (isHeartbeatUnhealthy(heartbeat, highest) ? 0 : 1),
         0
       );
diff --git a/dashboard/src/components/Guardians.tsx b/dashboard/src/components/Guardians.tsx
@@ -33,20 +33,20 @@ import {
   Typography,
 } from '@mui/material';
 import {
-  SortingState,
   createColumnHelper,
   getCoreRowModel,
   getSortedRowModel,
+  SortingState,
   useReactTable,
 } from '@tanstack/react-table';
+import { chainIdToName, STANDBY_GUARDIANS } from '@wormhole-foundation/wormhole-monitor-common';
 import { useCallback, useMemo, useState } from 'react';
 import TimeAgo from 'react-timeago';
 import { ChainIdToHeartbeats } from '../hooks/useChainHeartbeats';
 import { Heartbeat } from '../utils/getLastHeartbeats';
 import { isHeartbeatUnhealthy } from './Chains';
 import CollapsibleSection from './CollapsibleSection';
 import Table from './Table';
-import { chainIdToName } from '@wormhole-foundation/wormhole-monitor-common';
 
 const columnHelper = createColumnHelper<Heartbeat>();
 
@@ -298,6 +298,21 @@ function Guardians({
   chainIdsToHeartbeats: ChainIdToHeartbeats;
   latestRelease: string | null;
 }) {
+  const [guardianHeartbeats, standbyHeartbeats] = useMemo(
+    () =>
+      heartbeats.reduce(
+        ([guardian, standby], hb) => {
+          if (
+            STANDBY_GUARDIANS.find((g) => g.pubkey.toLowerCase() === hb.guardianAddr.toLowerCase())
+          ) {
+            return [guardian, [...standby, hb]];
+          }
+          return [[...guardian, hb], standby];
+        },
+        [[] as Heartbeat[], [] as Heartbeat[]]
+      ),
+    [heartbeats]
+  );
   const highestByChain = useMemo(
     () =>
       Object.entries(chainIdsToHeartbeats).reduce((obj, [chainId, heartbeats]) => {
@@ -316,7 +331,18 @@ function Guardians({
   const [sorting, setSorting] = useState<SortingState>([]);
   const table = useReactTable({
     columns,
-    data: heartbeats,
+    data: guardianHeartbeats,
+    state: {
+      sorting,
+    },
+    getRowId: (heartbeat) => `${heartbeat.guardianAddr}-${heartbeat.nodeName}`,
+    getCoreRowModel: getCoreRowModel(),
+    getSortedRowModel: getSortedRowModel(),
+    onSortingChange: setSorting,
+  });
+  const standbyTable = useReactTable({
+    columns,
+    data: standbyHeartbeats,
     state: {
       sorting,
     },
@@ -348,7 +374,7 @@ function Guardians({
     let numWarnings = 0;
     let numErrors = 0;
     const chainCount = Object.keys(highestByChain).length;
-    for (const heartbeat of heartbeats) {
+    for (const heartbeat of guardianHeartbeats) {
       const healthyCount = heartbeat.networks.reduce(
         (count, network) =>
           isHeartbeatUnhealthy(
@@ -373,7 +399,7 @@ function Guardians({
       numWarnings,
       numErrors,
     };
-  }, [heartbeats, highestByChain]);
+  }, [guardianHeartbeats, highestByChain]);
   return (
     <CollapsibleSection
       header={
@@ -464,20 +490,43 @@ function Guardians({
       }
     >
       {display === 'cards' ? (
-        <Box display="flex" flexWrap="wrap" alignItems="center" justifyContent={'center'}>
-          {heartbeats.map((hb) => (
-            <GuardianCard
-              key={`${hb.guardianAddr}-${hb.nodeName}`}
-              heartbeat={hb}
-              highestByChain={highestByChain}
-              latestRelease={latestRelease}
-            />
-          ))}
-        </Box>
+        <>
+          <Box display="flex" flexWrap="wrap" alignItems="center" justifyContent={'center'}>
+            {guardianHeartbeats.map((hb) => (
+              <GuardianCard
+                key={`${hb.guardianAddr}-${hb.nodeName}`}
+                heartbeat={hb}
+                highestByChain={highestByChain}
+                latestRelease={latestRelease}
+              />
+            ))}
+          </Box>
+          <Typography variant="subtitle1" sx={{ mt: 2, mb: 1 }}>
+            Standby Guardians
+          </Typography>
+          <Box display="flex" flexWrap="wrap" alignItems="center" justifyContent={'center'}>
+            {standbyHeartbeats.map((hb) => (
+              <GuardianCard
+                key={`${hb.guardianAddr}-${hb.nodeName}`}
+                heartbeat={hb}
+                highestByChain={highestByChain}
+                latestRelease={latestRelease}
+              />
+            ))}
+          </Box>
+        </>
       ) : (
-        <Card>
-          <Table<Heartbeat> table={table} showRowCount />
-        </Card>
+        <>
+          <Card>
+            <Table<Heartbeat> table={table} showRowCount />
+          </Card>
+          <Typography variant="subtitle1" sx={{ mt: 2, mb: 1 }}>
+            Standby Guardians
+          </Typography>
+          <Card>
+            <Table<Heartbeat> table={standbyTable} showRowCount />
+          </Card>
+        </>
       )}
     </CollapsibleSection>
   );
