Node: Add reobservation metric (#4303)

Author: bruce-riley

node/pkg/watchers/algorand/watcher.go

@@ -17,6 +17,7 @@ import (
 	gossipv1 "github.com/certusone/wormhole/node/pkg/proto/gossip/v1"
 	"github.com/certusone/wormhole/node/pkg/readiness"
 	"github.com/certusone/wormhole/node/pkg/supervisor"
+	"github.com/certusone/wormhole/node/pkg/watchers"
 	eth_common "github.com/ethereum/go-ethereum/common"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
@@ -169,6 +170,9 @@ func lookAtTxn(e *Watcher, t types.SignedTxnInBlock, b types.Block, logger *zap.
 		}
 
 		algorandMessagesConfirmed.Inc()
+		if isReobservation {
+			watchers.ReobservationsByChain.WithLabelValues("algorand", "std").Inc()
+		}
 
 		logger.Info("message observed",
 			zap.Time("timestamp", observation.Timestamp),

node/pkg/watchers/aptos/watcher.go

@@ -337,6 +337,9 @@ func (e *Watcher) observeData(logger *zap.Logger, data gjson.Result, nativeSeq u
 	}
 
 	aptosMessagesConfirmed.WithLabelValues(e.networkID).Inc()
+	if isReobservation {
+		watchers.ReobservationsByChain.WithLabelValues(e.chainID.String(), "std").Inc()
+	}
 
 	logger.Info("message observed",
 		zap.String("txHash", observation.TxIDString()),

node/pkg/watchers/cosmwasm/watcher.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/certusone/wormhole/node/pkg/p2p"
 	gossipv1 "github.com/certusone/wormhole/node/pkg/proto/gossip/v1"
+	"github.com/certusone/wormhole/node/pkg/watchers"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 
 	"github.com/prometheus/client_golang/prometheus"
@@ -303,6 +304,7 @@ func (e *Watcher) Run(ctx context.Context) error {
 					msg.IsReobservation = true
 					e.msgC <- msg
 					messagesConfirmed.WithLabelValues(networkName).Inc()
+					watchers.ReobservationsByChain.WithLabelValues(networkName, "std").Inc()
 				}
 			}
 		}

node/pkg/watchers/evm/watcher.go

@@ -12,6 +12,7 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/certusone/wormhole/node/pkg/watchers"
 	"github.com/certusone/wormhole/node/pkg/watchers/evm/connectors"
 	"github.com/certusone/wormhole/node/pkg/watchers/evm/connectors/ethabi"
 	"github.com/certusone/wormhole/node/pkg/watchers/interfaces"
@@ -620,8 +621,6 @@ func (w *Watcher) Run(parentCtx context.Context) error {
 								zap.String("txHash", txHash.String()),
 								zap.Error(pubErr),
 							)
-						} else {
-							ethMessagesConfirmed.WithLabelValues(w.networkName).Inc()
 						}
 					}
 				}
@@ -812,8 +811,6 @@ func (w *Watcher) postMessage(
 				zap.String("txHash", msg.TxIDString()),
 				zap.Error(pubErr),
 			)
-		} else {
-			ethMessagesConfirmed.WithLabelValues(w.networkName).Inc()
 		}
 		return
 	}
@@ -888,6 +885,10 @@ func (w *Watcher) verifyAndPublish(
 	}
 
 	w.msgC <- msg
+	ethMessagesConfirmed.WithLabelValues(w.networkName).Inc()
+	if msg.IsReobservation {
+		watchers.ReobservationsByChain.WithLabelValues(w.chainID.String(), "std").Inc()
+	}
 	return nil
 
 }

node/pkg/watchers/ibc/watcher.go

@@ -18,6 +18,7 @@ import (
 	"github.com/certusone/wormhole/node/pkg/p2p"
 	"github.com/certusone/wormhole/node/pkg/readiness"
 	"github.com/certusone/wormhole/node/pkg/supervisor"
+	"github.com/certusone/wormhole/node/pkg/watchers"
 	"github.com/certusone/wormhole/node/pkg/watchers/cosmwasm"
 	"github.com/wormhole-foundation/wormhole/sdk/vaa"
 
@@ -673,6 +674,9 @@ func (w *Watcher) processIbcReceivePublishEvent(evt *ibcReceivePublishEvent, obs
 
 	ce.msgC <- evt.Msg
 	messagesConfirmed.WithLabelValues(ce.chainName).Inc()
+	if evt.Msg.IsReobservation {
+		watchers.ReobservationsByChain.WithLabelValues(evt.Msg.EmitterChain.String(), "std").Inc()
+	}
 	return nil
 }
 

node/pkg/watchers/near/tx_processing.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/certusone/wormhole/node/pkg/common"
+	"github.com/certusone/wormhole/node/pkg/watchers"
 	"github.com/certusone/wormhole/node/pkg/watchers/near/nearapi"
 	eth_common "github.com/ethereum/go-ethereum/common"
 	"github.com/mr-tron/base58"
@@ -246,6 +247,10 @@ func (e *Watcher) processWormholeLog(logger *zap.Logger, _ context.Context, job
 		IsReobservation:  job.isReobservation,
 	}
 
+	if job.isReobservation {
+		watchers.ReobservationsByChain.WithLabelValues("near", "std").Inc()
+	}
+
 	// tell everyone about it
 	job.hasWormholeMsg = true
 

node/pkg/watchers/solana/client.go

@@ -18,6 +18,7 @@ import (
 	"github.com/certusone/wormhole/node/pkg/query"
 	"github.com/certusone/wormhole/node/pkg/readiness"
 	"github.com/certusone/wormhole/node/pkg/supervisor"
+	"github.com/certusone/wormhole/node/pkg/watchers"
 	eth_common "github.com/ethereum/go-ethereum/common"
 	"github.com/gagliardetto/solana-go"
 	lookup "github.com/gagliardetto/solana-go/programs/address-lookup-table"
@@ -1062,6 +1063,9 @@ func (s *SolanaWatcher) processMessageAccount(logger *zap.Logger, data []byte, a
 	}
 
 	solanaMessagesConfirmed.WithLabelValues(s.networkName).Inc()
+	if isReobservation {
+		watchers.ReobservationsByChain.WithLabelValues(s.chainID.String(), "std").Inc()
+	}
 
 	if logger.Level().Enabled(s.msgObservedLogLevel) {
 		logger.Log(s.msgObservedLogLevel, "message observed",

node/pkg/watchers/solana/shim.go

@@ -34,6 +34,7 @@ import (
 	"time"
 
 	"github.com/certusone/wormhole/node/pkg/common"
+	"github.com/certusone/wormhole/node/pkg/watchers"
 	"github.com/gagliardetto/solana-go"
 	"github.com/gagliardetto/solana-go/rpc"
 	"github.com/near/borsh-go"
@@ -365,6 +366,9 @@ func (s *SolanaWatcher) shimProcessRest(
 	}
 
 	solanaMessagesConfirmed.WithLabelValues(s.networkName).Inc()
+	if isReobservation {
+		watchers.ReobservationsByChain.WithLabelValues(s.chainID.String(), "shim").Inc()
+	}
 
 	if logger.Level().Enabled(s.msgObservedLogLevel) {
 		logger.Log(s.msgObservedLogLevel, "message observed from shim",

node/pkg/watchers/sui/watcher.go

@@ -17,6 +17,7 @@ import (
 	gossipv1 "github.com/certusone/wormhole/node/pkg/proto/gossip/v1"
 	"github.com/certusone/wormhole/node/pkg/readiness"
 	"github.com/certusone/wormhole/node/pkg/supervisor"
+	"github.com/certusone/wormhole/node/pkg/watchers"
 
 	eth_common "github.com/ethereum/go-ethereum/common"
 	"github.com/prometheus/client_golang/prometheus"
@@ -274,6 +275,9 @@ func (e *Watcher) inspectBody(logger *zap.Logger, body SuiResult, isReobservatio
 	}
 
 	suiMessagesConfirmed.Inc()
+	if isReobservation {
+		watchers.ReobservationsByChain.WithLabelValues("sui", "std").Inc()
+	}
 
 	logger.Info("message observed",
 		zap.String("txHash", observation.TxIDString()),

node/pkg/watchers/watchers.go

@@ -6,6 +6,8 @@ import (
 	"github.com/certusone/wormhole/node/pkg/query"
 	"github.com/certusone/wormhole/node/pkg/supervisor"
 	"github.com/certusone/wormhole/node/pkg/watchers/interfaces"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/wormhole-foundation/wormhole/sdk/vaa"
 )
 
@@ -27,3 +29,11 @@ type WatcherConfig interface {
 		env common.Environment,
 	) (interfaces.L1Finalizer, supervisor.Runnable, interfaces.Reobserver, error)
 }
+
+var (
+	ReobservationsByChain = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "wormhole_reobservations_by_chain",
+			Help: "Total number of reobservations completed by chain and observation type",
+		}, []string{"chain", "type"})
+)