Node: Don't hang on post to Loki (#4267)

Author: bruce-riley

node/pkg/telemetry/loki.go

@@ -23,6 +23,7 @@ import (
 	"github.com/grafana/loki/pkg/logproto"
 	lokiflag "github.com/grafana/loki/pkg/util/flagext"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 )
@@ -39,6 +40,20 @@ type ExternalLoggerLoki struct {
 	localLogger *zap.Logger
 }
 
+var (
+	lokiMessagesSent = promauto.NewCounter(
+		prometheus.CounterOpts{
+			Name: "wormhole_loki_messages_sent",
+			Help: "Total number of log messages posted to Loki",
+		})
+
+	lokiMessagesDropped = promauto.NewCounter(
+		prometheus.CounterOpts{
+			Name: "wormhole_loki_messages_dropped",
+			Help: "Total number of log messages dropped while posting to Loki",
+		})
+)
+
 func (logger *ExternalLoggerLoki) log(time time.Time, message json.RawMessage, level zapcore.Level) {
 	lokiLabels := logger.labels[level]
 
@@ -56,7 +71,12 @@ func (logger *ExternalLoggerLoki) log(time time.Time, message json.RawMessage, l
 		Labels: lokiLabels,
 	}
 
-	logger.c.Chan() <- entry
+	select {
+	case logger.c.Chan() <- entry:
+		lokiMessagesSent.Inc()
+	default:
+		lokiMessagesDropped.Inc()
+	}
 
 	// A fatal error exits, which can cause us to lose messages. Flush everything.
 	if level == zapcore.FatalLevel {