cloud_functions: add alarming guardian heartbeats

Author: panoel

cloud_functions/scripts/deploy.sh

@@ -68,6 +68,11 @@ if [ -z "$CLOUD_FUNCTIONS_BLOCK_INCREMENT" ]; then
     exit 1
 fi
 
+if [ -z "$FIRESTORE_GUARDIAN_HEARTBEAT_COLLECTION" ]; then
+    echo "FIRESTORE_GUARDIAN_HEARTBEAT_COLLECTION must be specified"
+    exit 1
+fi
+
 if [ -z "$FIRESTORE_LATEST_COLLECTION" ]; then
     echo "FIRESTORE_LATEST_COLLECTION must be specified"
     exit 1
@@ -121,7 +126,7 @@ fi
 # Context: Of all the cloud functions, there are some that only go into MAINNET and some that go into both MAINNET and TESTNET.
 #          There are no cloud functions that only go into TESTNET.
 # First, deploy the functions that are common to both MAINNET and TESTNET
-gcloud functions --project "$GCP_PROJECT" deploy alarm-missing-vaas --entry-point alarmMissingVaas --gen2 --runtime nodejs18 --trigger-http --no-allow-unauthenticated --timeout 300 --memory 1GB --region europe-west3 --set-env-vars MISSING_VAA_SLACK_CHANNEL_ID=$MISSING_VAA_SLACK_CHANNEL_ID,MISSING_VAA_SLACK_POST_URL=$MISSING_VAA_SLACK_POST_URL,MISSING_VAA_SLACK_BOT_TOKEN=$MISSING_VAA_SLACK_BOT_TOKEN,FIRESTORE_ALARM_MISSING_VAAS_COLLECTION=$FIRESTORE_ALARM_MISSING_VAAS_COLLECTION,FIRESTORE_GOVERNOR_STATUS_COLLECTION=$FIRESTORE_GOVERNOR_STATUS_COLLECTION,FIRESTORE_LATEST_COLLECTION=$FIRESTORE_LATEST_COLLECTION,NETWORK=$NETWORK,FUNCTION=alarmMissingVaas
+gcloud functions --project "$GCP_PROJECT" deploy alarm-missing-vaas --entry-point alarmMissingVaas --gen2 --runtime nodejs18 --trigger-http --no-allow-unauthenticated --timeout 300 --memory 1GB --region europe-west3 --set-env-vars MISSING_VAA_SLACK_CHANNEL_ID=$MISSING_VAA_SLACK_CHANNEL_ID,MISSING_VAA_SLACK_POST_URL=$MISSING_VAA_SLACK_POST_URL,MISSING_VAA_SLACK_BOT_TOKEN=$MISSING_VAA_SLACK_BOT_TOKEN,FIRESTORE_ALARM_MISSING_VAAS_COLLECTION=$FIRESTORE_ALARM_MISSING_VAAS_COLLECTION,FIRESTORE_GOVERNOR_STATUS_COLLECTION=$FIRESTORE_GOVERNOR_STATUS_COLLECTION,FIRESTORE_LATEST_COLLECTION=$FIRESTORE_LATEST_COLLECTION,FIRESTORE_GUARDIAN_HEARTBEAT_COLLECTION=$FIRESTORE_GUARDIAN_HEARTBEAT_COLLECTION,NETWORK=$NETWORK,FUNCTION=alarmMissingVaas
 gcloud functions --project "$GCP_PROJECT" deploy compute-message-count-history --entry-point computeMessageCountHistory --gen2 --runtime nodejs18 --trigger-http --no-allow-unauthenticated --timeout 300 --memory 1GB --region europe-west3 --set-env-vars BIGTABLE_INSTANCE_ID=$BIGTABLE_INSTANCE_ID,BIGTABLE_SIGNED_VAAS_TABLE_ID=$BIGTABLE_SIGNED_VAAS_TABLE_ID,FIRESTORE_MESSAGE_COUNT_HISTORY_COLLECTION=$FIRESTORE_MESSAGE_COUNT_HISTORY_COLLECTION,NETWORK=$NETWORK,FUNCTION=computeMessageCountHistory
 gcloud functions --project "$GCP_PROJECT" deploy compute-message-counts --entry-point computeMessageCounts --gen2 --runtime nodejs18 --trigger-http --allow-unauthenticated --timeout 300 --memory 4GB --region europe-west3 --set-env-vars BIGTABLE_TABLE_ID=$BIGTABLE_TABLE_ID,BIGTABLE_INSTANCE_ID=$BIGTABLE_INSTANCE_ID,CLOUD_FUNCTIONS_REFRESH_TIME_INTERVAL=$CLOUD_FUNCTIONS_REFRESH_TIME_INTERVAL,NETWORK=$NETWORK,FUNCTION=computeMessageCounts
 gcloud functions --project "$GCP_PROJECT" deploy compute-missing-vaas --entry-point computeMissingVaas --gen2 --runtime nodejs18 --trigger-http --allow-unauthenticated --timeout 300 --memory 4GB --region europe-west3 --set-env-vars BIGTABLE_TABLE_ID=$BIGTABLE_TABLE_ID,BIGTABLE_INSTANCE_ID=$BIGTABLE_INSTANCE_ID,CLOUD_FUNCTIONS_REFRESH_TIME_INTERVAL=$CLOUD_FUNCTIONS_REFRESH_TIME_INTERVAL,NETWORK=$NETWORK,FUNCTION=computeMissingVaas

cloud_functions/src/alarmMissingVaas.ts

@@ -88,6 +88,9 @@ export async function alarmMissingVaas(req: any, res: any) {
     // Alarm any watchers that are behind by more than 24 hours
     await alarmOldBlockTimes(refTimes);
 
+    // Alarm any guardians that have not sent a heartbeat in more than 1 hour
+    await alarmOldHeartbeats();
+
     // attempting to retrieve missing VAAs...
     const messages: MissingVaasByChain = await commonGetMissingVaas();
     if (messages) {
@@ -399,6 +402,40 @@ async function storeAlarmedChains(alarms: AlarmedChainTime[]): Promise<void> {
   await alarmedChains.set({ times: alarms });
 }
 
+async function alarmOldHeartbeats(): Promise<void> {
+  // Get Guardian Heartbeats from firestore
+  const now: Date = new Date();
+  const firestore = new Firestore();
+  const collectionRef = firestore.collection(
+    assertEnvironmentVariable('FIRESTORE_GUARDIAN_HEARTBEAT_COLLECTION')
+  );
+  const snapshot = await collectionRef.get();
+  // Walk all the documents in the collection
+  const documents = snapshot.docs;
+  for (const doc of documents) {
+    const data = doc.data();
+    if (data) {
+      // Only need to look at the timestamp field, which is in nanoseconds
+      const timestamp: string = data.timestamp;
+      // Convert the timestamp to a milliseconds
+      const timestampMs: number = Math.floor(Number(timestamp) / 1_000_000);
+      const heartbeatTime: Date = new Date(timestampMs);
+      const deltaTime: number = (now.getTime() - heartbeatTime.getTime()) / (1000 * 60 * 60); // hours
+      if (deltaTime >= 1) {
+        // Send a message to slack
+        const alarmSlackInfo: SlackInfo = {
+          channelId: assertEnvironmentVariable('MISSING_VAA_SLACK_CHANNEL_ID'),
+          postUrl: assertEnvironmentVariable('MISSING_VAA_SLACK_POST_URL'),
+          botToken: assertEnvironmentVariable('MISSING_VAA_SLACK_BOT_TOKEN'),
+          bannerTxt: 'Wormhole Missing VAA Alarm',
+          msg: `The guardian ${doc.id} has not sent a heartbeat in ${deltaTime} hours.`,
+        };
+        await formatAndSendToSlack(alarmSlackInfo);
+      }
+    }
+  }
+}
+
 type FirestoreVAA = {
   chain: string;
   txHash: string;