This change enhances Flipt's Git-based feature flag backend observability by adding detailed synchronization metrics. Currently, failures during Git sync are only logged without metric visibility, limiting proactive monitoring and alerting capabilities.

    - Introduce new OpenTelemetry metrics for Git sync operations:
      - Last sync time as an observable gauge (timestamp).
      - Sync duration histogram.
      - Counters for number of flags fetched.
      - Success and failure counts with failure reason attributes.

    - Instrument the `SnapshotStore.update` method, the core sync loop, to record these metrics accurately on every sync attempt, including partial failures and cleanups.

    - Extend the `Snapshot` type with `TotalFlagsCount()` to count all flags across namespaces for metric reporting.

    - Integrate metrics initialization in app startup ensuring consistent telemetry setup.

    - Improve test coverage by suggesting strategies to verify metric emission and sync behavior.

    These metric additions enable operators to monitor Git sync health, detect failures promptly, and troubleshoot issues efficiently, significantly improving runtime observability and system reliability.

Signed-off-by: Rohit Jaiswal <rohit.jaiswal@indexexchange.com>

Author: Rohit Jaiswal

DEVELOPMENT.md

@@ -28,7 +28,7 @@ If you run into errors such as:
 undefined: sqlite3.Error
 ```
 
-Then you need to enable CGO. 
+Then you need to enable CGO.
 
 ### Windows
 
@@ -43,10 +43,10 @@ Then you need to enable CGO.
 ## Setup
 
 1. Clone this repo: `git clone https://github.com/flipt-io/flipt`.
-1. Run `mage bootstrap` to install required development tools. See [#bootstrap](#bootstrap) below.
-1. Run `mage go:test` to execute the Go test suite. For more information on tests, see also [here](build/README.md)
-1. Run `mage` to build the binary with embedded assets.
-1. Run `mage -l` to see a full list of possible commands.
+2. Run `mage bootstrap` to install required development tools. See [#bootstrap](#bootstrap) below.
+3. Run `mage go:test` to execute the Go test suite. For more information on tests, see also [here](build/README.md)
+4. Run `mage` to build the binary with embedded assets.
+5. Run `mage -l` to see a full list of possible commands.
 
 ## Conventional Commits
 
@@ -112,7 +112,7 @@ These ports will be forwarded to your local machine automatically if you are dev
 
 ## Docker Compose
 
-If you want to develop Flipt using Docker Compose, you can use the `docker-compose.yml` file in the root of this repository. 
+If you want to develop Flipt using Docker Compose, you can use the `docker-compose.yml` file in the root of this repository.
 
 This will start two Docker containers:
 

cmd/flipt/main.go

@@ -23,6 +23,7 @@ import (
 	"go.flipt.io/flipt/internal/cmd"
 	"go.flipt.io/flipt/internal/config"
 	"go.flipt.io/flipt/internal/info"
+	"go.flipt.io/flipt/internal/metrics"
 	"go.flipt.io/flipt/internal/release"
 	"go.flipt.io/flipt/internal/telemetry"
 	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
@@ -105,6 +106,8 @@ func exec() error {
 					return err
 				}
 
+				metrics.InitMetrics()
+
 				defer func() {
 					_ = logger.Sync()
 				}()

go.work.sum

@@ -19,12 +19,76 @@ import (
 	sdkmetric "go.opentelemetry.io/otel/sdk/metric"
 )
 
+var (
+	GitSyncLastTime     metric.Int64ObservableGauge
+	GitSyncDuration     metric.Float64Histogram
+	GitSyncFlagsFetched metric.Int64Counter
+	GitSyncSuccess      metric.Int64Counter
+	GitSyncFailure      metric.Int64Counter
+
+	// internal storage for last sync time value
+	lastSyncTimeValue int64
+	lastSyncTimeMu    sync.RWMutex
+)
+
 func init() {
 	if otel.GetMeterProvider() == nil {
 		otel.SetMeterProvider(metricnoop.NewMeterProvider())
 	}
 }
 
+func InitMetrics() {
+	InitGitSyncMetrics(meter())
+}
+
+func InitGitSyncMetrics(meter metric.Meter) {
+	var err error
+
+	// Create counter and histogram instruments
+	GitSyncDuration, err = meter.Float64Histogram("git_sync_duration_seconds")
+	if err != nil {
+		panic(fmt.Errorf("creating git_sync_duration_seconds histogram: %w", err))
+	}
+	GitSyncFlagsFetched, err = meter.Int64Counter("git_sync_flags_fetched")
+	if err != nil {
+		panic(fmt.Errorf("creating git_sync_flags_fetched counter: %w", err))
+	}
+	GitSyncSuccess, err = meter.Int64Counter("git_sync_success_count")
+	if err != nil {
+		panic(fmt.Errorf("creating git_sync_success_count counter: %w", err))
+	}
+	GitSyncFailure, err = meter.Int64Counter("git_sync_failure_count")
+	if err != nil {
+		panic(fmt.Errorf("creating git_sync_failure_count counter: %w", err))
+	}
+
+	// Create ObservableGauge for last sync time and register callback
+	GitSyncLastTime, err = meter.Int64ObservableGauge("git_sync_last_time_unix")
+	if err != nil {
+		panic(fmt.Errorf("creating git_sync_last_time_unix observable gauge: %w", err))
+	}
+
+	_, err = meter.RegisterCallback(
+		func(ctx context.Context, observer metric.Observer) error {
+			lastSyncTimeMu.RLock()
+			value := lastSyncTimeValue
+			lastSyncTimeMu.RUnlock()
+			observer.ObserveInt64(GitSyncLastTime, value)
+			return nil
+		},
+		GitSyncLastTime,
+	)
+	if err != nil {
+		panic(fmt.Errorf("registering callback for git_sync_last_time_unix: %w", err))
+	}
+}
+
+func SetGitSyncLastTime(ts int64) {
+	lastSyncTimeMu.Lock()
+	lastSyncTimeValue = ts
+	lastSyncTimeMu.Unlock()
+}
+
 // This is memoized in the OTEL library to avoid creating multiple instances of the same exporter.
 func meter() metric.Meter {
 	return otel.Meter("github.com/flipt-io/flipt")

internal/metrics/metrics.go

@@ -4,10 +4,13 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
 	"io/fs"
 	"os"
 	"slices"
 	"sync"
+	"time"
 
 	"github.com/go-git/go-billy/v5/osfs"
 	"github.com/go-git/go-git/v5"
@@ -20,6 +23,7 @@ import (
 	"github.com/go-git/go-git/v5/storage/memory"
 	"go.flipt.io/flipt/internal/containers"
 	"go.flipt.io/flipt/internal/gitfs"
+	"go.flipt.io/flipt/internal/metrics"
 	"go.flipt.io/flipt/internal/storage"
 	storagefs "go.flipt.io/flipt/internal/storage/fs"
 	"go.uber.org/zap"
@@ -294,6 +298,10 @@ func (s *SnapshotStore) View(ctx context.Context, storeRef storage.Reference, fn
 	return fn(snap)
 }
 
+func (s *SnapshotStore) Resolve(ref string) (plumbing.Hash, error) {
+	return s.resolve(ref)
+}
+
 // listRemoteRefs returns a set of branch and tag names present on the remote.
 func (s *SnapshotStore) listRemoteRefs(ctx context.Context) (map[string]struct{}, error) {
 	remotes, err := s.repo.Remotes()
@@ -335,14 +343,27 @@ func (s *SnapshotStore) listRemoteRefs(ctx context.Context) (map[string]struct{}
 // HEAD updates to a new revision, it builds a snapshot and updates it
 // on the store.
 func (s *SnapshotStore) update(ctx context.Context) (bool, error) {
+	syncStart := time.Now()
 	updated, fetchErr := s.fetch(ctx, s.snaps.References())
 
 	if !updated && fetchErr == nil {
+		// No update and no error: record metrics for a successful no-change sync
+		duration := time.Since(syncStart).Seconds()
+		metrics.SetGitSyncLastTime(time.Now().Unix())
+		metrics.GitSyncDuration.Record(ctx, duration)
+		metrics.GitSyncSuccess.Add(ctx, 1)
 		return false, nil
 	}
 
-	// If we can't fetch, we need to check if the remote refs have changed
-	// and remove any references that are no longer present
+	if fetchErr != nil {
+		// Record failure early to capture fetch errors
+		duration := time.Since(syncStart).Seconds()
+		metrics.SetGitSyncLastTime(time.Now().Unix())
+		metrics.GitSyncDuration.Record(ctx, duration)
+		metrics.GitSyncFailure.Add(ctx, 1, metric.WithAttributes(attribute.String("reason", fetchErr.Error())))
+	}
+
+	// If fetchErr exists, try cleaning up refs but do not declare full failure yet
 	if fetchErr != nil {
 		remoteRefs, listErr := s.listRemoteRefs(ctx)
 		if listErr != nil {
@@ -364,6 +385,8 @@ func (s *SnapshotStore) update(ctx context.Context) (bool, error) {
 	}
 
 	var errs []error
+	flagsFetched := 0
+
 	if fetchErr != nil {
 		errs = append(errs, fetchErr)
 	}
@@ -373,11 +396,29 @@ func (s *SnapshotStore) update(ctx context.Context) (bool, error) {
 			errs = append(errs, err)
 			continue
 		}
-		if _, err := s.snaps.AddOrBuild(ctx, ref, hash, s.buildSnapshot); err != nil {
+
+		snap, err := s.snaps.AddOrBuild(ctx, ref, hash, s.buildSnapshot)
+		if err != nil {
 			errs = append(errs, err)
+			continue
+		}
+		if snap != nil {
+			flagsFetched += snap.TotalFlagsCount()
 		}
 	}
-	return true, errors.Join(errs...)
+
+	duration := time.Since(syncStart).Seconds()
+	metrics.SetGitSyncLastTime(time.Now().Unix())
+	metrics.GitSyncDuration.Record(ctx, duration)
+	metrics.GitSyncFlagsFetched.Add(ctx, int64(flagsFetched))
+
+	if fetchErr != nil || len(errs) > 0 {
+		metrics.GitSyncFailure.Add(ctx, 1, metric.WithAttributes(attribute.String("reason", fmt.Sprintf("fetchErr: %v, buildErrors: %v", fetchErr, errs))))
+		return true, errors.Join(append(errs, fetchErr)...)
+	}
+
+	metrics.GitSyncSuccess.Add(ctx, 1)
+	return true, nil
 }
 
 func (s *SnapshotStore) fetch(ctx context.Context, heads []string) (bool, error) {