From 49fa6a777bcd5ed2418b1b927afeae53a8d33427 Mon Sep 17 00:00:00 2001 From: Gregory Newman-Smith <109068393+gregns1@users.noreply.github.com> Date: Thu, 9 Jan 2025 12:21:20 +0000 Subject: [PATCH] CBG-4336: add updated at field for persisted configs (#7265) * CBG-4336: add updated at field for persisted configs * change time to utc * fix default collection test issues * flakyness with tests * add delay to other tests too * updates to add created_at field to configs * remove unnecessary method --- auth/auth.go | 6 ++ auth/principal.go | 6 ++ auth/role.go | 10 +++ db/sg_replicate_cfg.go | 7 ++ db/sg_replicate_cfg_test.go | 1 + db/users.go | 2 + rest/api_collections_test.go | 2 + rest/config.go | 2 + rest/config_database_test.go | 60 +++++++++++++++ rest/config_manager.go | 11 +++ rest/config_registry.go | 4 + rest/config_test.go | 101 +++++++++++++++++++++++++ rest/replicatortest/replicator_test.go | 38 ++++++++++ rest/session_api.go | 1 + 14 files changed, 251 insertions(+) diff --git a/auth/auth.go b/auth/auth.go index 4b2eea7b65..b470d98af3 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -631,6 +631,8 @@ func (auth *Authenticator) UpdateUserEmail(u User, email string) error { if err != nil { return nil, err } + currentUser.SetUpdatedAt() + return currentUser, nil } @@ -662,6 +664,7 @@ func (auth *Authenticator) rehashPassword(user User, password string) error { if err != nil { return nil, err } + currentUserImpl.SetUpdatedAt() return currentUserImpl, nil } else { return nil, base.ErrUpdateCancel @@ -740,6 +743,7 @@ func (auth *Authenticator) DeleteRole(role Role, purge bool, deleteSeq uint64) e } p.setDeleted(true) p.SetSequence(deleteSeq) + p.SetUpdatedAt() // Update channel history for default collection channelHistory := auth.calculateHistory(p.Name(), deleteSeq, p.Channels(), nil, p.ChannelHistory()) @@ -953,6 +957,8 @@ func (auth *Authenticator) RegisterNewUser(username, email string) (User, error) base.WarnfCtx(auth.LogCtx, "Skipping SetEmail for user %q - Invalid email address provided: %q", base.UD(username), base.UD(email)) } } + user.SetUpdatedAt() + user.SetCreatedAt(time.Now().UTC()) err = auth.Save(user) if base.IsCasMismatch(err) { diff --git a/auth/principal.go b/auth/principal.go index ffe6c6535b..d1b3e46b7c 100644 --- a/auth/principal.go +++ b/auth/principal.go @@ -52,6 +52,12 @@ type Principal interface { setDeleted(bool) IsDeleted() bool + // Sets the updated time for the principal document + SetUpdatedAt() + + // Sets the created time for the principal document + SetCreatedAt(t time.Time) + // Principal includes the PrincipalCollectionAccess interface for operations against // the _default._default collection (stored directly on the principal for backward // compatibility) diff --git a/auth/role.go b/auth/role.go index bacfe185bf..30d828669f 100644 --- a/auth/role.go +++ b/auth/role.go @@ -33,6 +33,8 @@ type roleImpl struct { ChannelInvalSeq uint64 `json:"channel_inval_seq,omitempty"` // Sequence at which the channels were invalidated. Data remains in Channels_ for history calculation. Deleted bool `json:"deleted,omitempty"` CollectionsAccess map[string]map[string]*CollectionAccess `json:"collection_access,omitempty"` // Nested maps of CollectionAccess, indexed by scope and collection name + UpdatedAt time.Time `json:"updated_at"` + CreatedAt time.Time `json:"created_at"` cas uint64 docID string // key used to store the roleImpl } @@ -277,6 +279,14 @@ func (role *roleImpl) Name() string { return role.Name_ } +func (role *roleImpl) SetUpdatedAt() { + role.UpdatedAt = time.Now().UTC() +} + +func (role *roleImpl) SetCreatedAt(t time.Time) { + role.CreatedAt = t +} + func (role *roleImpl) Sequence() uint64 { return role.Sequence_ } diff --git a/db/sg_replicate_cfg.go b/db/sg_replicate_cfg.go index b79cdc2706..68cd2b7d40 100644 --- a/db/sg_replicate_cfg.go +++ b/db/sg_replicate_cfg.go @@ -115,6 +115,8 @@ type ReplicationConfig struct { Adhoc bool `json:"adhoc,omitempty"` BatchSize int `json:"batch_size,omitempty"` RunAs string `json:"run_as,omitempty"` + UpdatedAt *time.Time `json:"updated_at,omitempty"` + CreatedAt *time.Time `json:"created_at,omitempty"` } func DefaultReplicationConfig() ReplicationConfig { @@ -335,6 +337,9 @@ func (rc *ReplicationConfig) Upsert(ctx context.Context, c *ReplicationUpsertCon rc.RunAs = *c.RunAs } + timeNow := time.Now().UTC() + rc.UpdatedAt = &timeNow + if c.QueryParams != nil { // QueryParams can be either []interface{} or map[string]interface{}, so requires type-specific copying // avoid later mutating c.QueryParams @@ -1106,6 +1111,8 @@ func (m *sgReplicateManager) UpsertReplication(ctx context.Context, replication } else { // Add a new replication to the cfg. Set targetState based on initialState when specified. replicationConfig := DefaultReplicationConfig() + createdAt := time.Now().UTC() + replicationConfig.CreatedAt = &createdAt replicationConfig.ID = replication.ID targetState := ReplicationStateRunning if replication.InitialState != nil && *replication.InitialState == ReplicationStateStopped { diff --git a/db/sg_replicate_cfg_test.go b/db/sg_replicate_cfg_test.go index bec50ed039..7518946754 100644 --- a/db/sg_replicate_cfg_test.go +++ b/db/sg_replicate_cfg_test.go @@ -536,6 +536,7 @@ func TestUpsertReplicationConfig(t *testing.T) { for _, testCase := range testCases { t.Run(fmt.Sprintf("%s", testCase.name), func(t *testing.T) { testCase.existingConfig.Upsert(base.TestCtx(t), testCase.updatedConfig) + testCase.existingConfig.UpdatedAt = nil // remove updated at field for comparison below equal, err := testCase.existingConfig.Equals(testCase.expectedConfig) assert.NoError(t, err) assert.True(t, equal) diff --git a/db/users.go b/db/users.go index fae475c77e..9d82fbffa6 100644 --- a/db/users.go +++ b/db/users.go @@ -86,6 +86,7 @@ func (dbc *DatabaseContext) UpdatePrincipal(ctx context.Context, updates *auth.P if err != nil { return replaced, princ, fmt.Errorf("Error creating user/role: %w", err) } + princ.SetCreatedAt(time.Now().UTC()) changed = true } else if !allowReplace { err = base.HTTPErrorf(http.StatusConflict, "Already exists") @@ -214,6 +215,7 @@ func (dbc *DatabaseContext) UpdatePrincipal(ctx context.Context, updates *auth.P user.SetJWTLastUpdated(time.Now()) } } + princ.SetUpdatedAt() err = authenticator.Save(princ) // On cas error, retry. Otherwise break out of loop if base.IsCasMismatch(err) { diff --git a/rest/api_collections_test.go b/rest/api_collections_test.go index 82587f7e02..0bd31f9f9b 100644 --- a/rest/api_collections_test.go +++ b/rest/api_collections_test.go @@ -1001,6 +1001,8 @@ func TestRuntimeConfigUpdateAfterConfigUpdateConflict(t *testing.T) { delete(scopesConfig[scope].Collections, collection1) assert.Equal(t, scopesConfig, dbCfg.Scopes) originalDBCfg.Server = nil + dbCfg.UpdatedAt = nil // originalDBCfg fetch is from memory so has no update/create at time + dbCfg.CreatedAt = nil assert.Equal(t, originalDBCfg, dbCfg) // now assert that _config shows the same diff --git a/rest/config.go b/rest/config.go index 979907ef0c..d194da69e7 100644 --- a/rest/config.go +++ b/rest/config.go @@ -193,6 +193,8 @@ type DbConfig struct { ChangesRequestPlus *bool `json:"changes_request_plus,omitempty"` // If set, is used as the default value of request_plus for non-continuous replications CORS *auth.CORSConfig `json:"cors,omitempty"` // Per-database CORS config Logging *DbLoggingConfig `json:"logging,omitempty"` // Per-database Logging config + UpdatedAt *time.Time `json:"updated_at,omitempty"` // Time at which the database config was last updated + CreatedAt *time.Time `json:"created_at,omitempty"` // Time at which the database config was created } type ScopesConfig map[string]ScopeConfig diff --git a/rest/config_database_test.go b/rest/config_database_test.go index 93f55595c7..74be5a2827 100644 --- a/rest/config_database_test.go +++ b/rest/config_database_test.go @@ -9,10 +9,14 @@ package rest import ( + "encoding/json" + "net/http" "testing" "time" + "github.com/couchbase/sync_gateway/base" "github.com/couchbase/sync_gateway/db" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -22,3 +26,59 @@ func TestDefaultDbConfig(t *testing.T) { compactIntervalDays := *(DefaultDbConfig(&sc, useXattrs).CompactIntervalDays) require.Equal(t, db.DefaultCompactInterval, time.Duration(compactIntervalDays)*time.Hour*24) } + +func TestDbConfigUpdatedAtField(t *testing.T) { + b := base.GetTestBucket(t) + rt := NewRestTester(t, &RestTesterConfig{ + CustomTestBucket: b, + PersistentConfig: true, + }) + defer rt.Close() + ctx := base.TestCtx(t) + + dbConfig := rt.NewDbConfig() + RequireStatus(t, rt.CreateDatabase("db1", dbConfig), http.StatusCreated) + + sc := rt.ServerContext() + + resp := rt.SendAdminRequest(http.MethodGet, "/db1/_config", "") + RequireStatus(t, resp, http.StatusOK) + var unmarshaledConfig DbConfig + require.NoError(t, json.Unmarshal(resp.BodyBytes(), &unmarshaledConfig)) + + registry := &GatewayRegistry{} + bName := b.GetName() + _, err := sc.BootstrapContext.Connection.GetMetadataDocument(ctx, bName, base.SGRegistryKey, registry) + require.NoError(t, err) + + // Check that the config has an updatedAt field + require.NotNil(t, unmarshaledConfig.UpdatedAt) + require.NotNil(t, unmarshaledConfig.CreatedAt) + currUpdatedTime := unmarshaledConfig.UpdatedAt + currCreatedTime := unmarshaledConfig.CreatedAt + registryUpdated := registry.UpdatedAt + registryCreated := registry.CreatedAt + + // avoid flake where update at seems to be the same (possibly running to fast) + time.Sleep(500 * time.Nanosecond) + + // Update the config + dbConfig = rt.NewDbConfig() + RequireStatus(t, rt.UpsertDbConfig("db1", dbConfig), http.StatusCreated) + + resp = rt.SendAdminRequest(http.MethodGet, "/db1/_config", "") + RequireStatus(t, resp, http.StatusOK) + unmarshaledConfig = DbConfig{} + require.NoError(t, json.Unmarshal(resp.BodyBytes(), &unmarshaledConfig)) + + registry = &GatewayRegistry{} + _, err = sc.BootstrapContext.Connection.GetMetadataDocument(ctx, b.GetName(), base.SGRegistryKey, registry) + require.NoError(t, err) + + // asser that the db config timestamps are as expected + assert.Greater(t, unmarshaledConfig.UpdatedAt.UnixNano(), currUpdatedTime.UnixNano()) + assert.Equal(t, unmarshaledConfig.CreatedAt.UnixNano(), currCreatedTime.UnixNano()) + // assert that registry timestamps are as expected + assert.Equal(t, registry.CreatedAt.UnixNano(), registryCreated.UnixNano()) + assert.Greater(t, registry.UpdatedAt.UnixNano(), registryUpdated.UnixNano()) +} diff --git a/rest/config_manager.go b/rest/config_manager.go index dbb8be15d4..6f6d08c542 100644 --- a/rest/config_manager.go +++ b/rest/config_manager.go @@ -11,6 +11,7 @@ package rest import ( "context" "fmt" + "time" "github.com/couchbase/sync_gateway/base" "github.com/couchbase/sync_gateway/db" @@ -105,6 +106,7 @@ func (b *bootstrapContext) InsertConfig(ctx context.Context, bucketName, groupID } // Persist registry + registry.UpdatedAt = time.Now().UTC() writeErr := b.setGatewayRegistry(ctx, bucketName, registry) if writeErr == nil { base.DebugfCtx(ctx, base.KeyConfig, "Registry updated successfully") @@ -131,6 +133,9 @@ func (b *bootstrapContext) InsertConfig(ctx context.Context, bucketName, groupID return 0, fmt.Errorf("InsertConfig failed to persist registry after %d attempts", configUpdateMaxRetryAttempts) } // Step 3. Write the database config + timeUpdated := time.Now().UTC() + config.UpdatedAt = &timeUpdated + config.CreatedAt = &timeUpdated cas, configErr := b.Connection.InsertMetadataDocument(ctx, bucketName, PersistentConfigKey(ctx, groupID, dbName), config) if configErr != nil { base.InfofCtx(ctx, base.KeyConfig, "Insert for database config returned error %v", configErr) @@ -150,6 +155,7 @@ func (b *bootstrapContext) UpdateConfig(ctx context.Context, bucketName, groupID var updatedConfig *DatabaseConfig var registry *GatewayRegistry var previousVersion string + var createdAtTime *time.Time registryUpdated := false for attempt := 1; attempt <= configUpdateMaxRetryAttempts; attempt++ { @@ -167,6 +173,7 @@ func (b *bootstrapContext) UpdateConfig(ctx context.Context, bucketName, groupID if existingConfig == nil { return 0, base.ErrNotFound } + createdAtTime = existingConfig.CreatedAt base.DebugfCtx(ctx, base.KeyConfig, "UpdateConfig fetched registry and database successfully") @@ -195,6 +202,7 @@ func (b *bootstrapContext) UpdateConfig(ctx context.Context, bucketName, groupID } // Persist registry + registry.UpdatedAt = time.Now().UTC() writeErr := b.setGatewayRegistry(ctx, bucketName, registry) if writeErr == nil { base.DebugfCtx(ctx, base.KeyConfig, "UpdateConfig persisted updated registry successfully") @@ -222,6 +230,9 @@ func (b *bootstrapContext) UpdateConfig(ctx context.Context, bucketName, groupID } // Step 2. Update the config document + timeUpdated := time.Now().UTC() + updatedConfig.UpdatedAt = &timeUpdated + updatedConfig.CreatedAt = createdAtTime docID := PersistentConfigKey(ctx, groupID, dbName) casOut, err := b.Connection.WriteMetadataDocument(ctx, bucketName, docID, updatedConfig.cfgCas, updatedConfig) if err != nil { diff --git a/rest/config_registry.go b/rest/config_registry.go index a6534b3907..6dcc699689 100644 --- a/rest/config_registry.go +++ b/rest/config_registry.go @@ -12,6 +12,7 @@ import ( "context" "fmt" "net/http" + "time" "github.com/couchbase/sync_gateway/base" ) @@ -47,6 +48,8 @@ type GatewayRegistry struct { Version string `json:"version"` // Registry version ConfigGroups map[string]*RegistryConfigGroup `json:"config_groups"` // Map of config groups, keyed by config group ID SGVersion base.ComparableBuildVersion `json:"sg_version"` // Latest patch version of Sync Gateway that touched the registry + UpdatedAt time.Time `json:"updated_at"` // Time the registry was last updated + CreatedAt time.Time `json:"created_at"` // Time the registry was created } const GatewayRegistryVersion = "1.0" @@ -111,6 +114,7 @@ func NewGatewayRegistry(syncGatewayVersion base.ComparableBuildVersion) *Gateway ConfigGroups: make(map[string]*RegistryConfigGroup), Version: GatewayRegistryVersion, SGVersion: syncGatewayVersion, + CreatedAt: time.Now().UTC(), } } diff --git a/rest/config_test.go b/rest/config_test.go index 8ac718550c..f0431d5901 100644 --- a/rest/config_test.go +++ b/rest/config_test.go @@ -3139,3 +3139,104 @@ func TestRevCacheMemoryLimitConfig(t *testing.T) { assert.Equal(t, uint32(100), *dbConfig.CacheConfig.RevCacheConfig.MaxItemCount) assert.Equal(t, uint32(0), *dbConfig.CacheConfig.RevCacheConfig.MaxMemoryCountMB) } + +func TestUserUpdatedAtField(t *testing.T) { + rt := NewRestTester(t, &RestTesterConfig{ + CustomTestBucket: base.GetTestBucket(t), + PersistentConfig: true, + }) + defer rt.Close() + + dbConfig := rt.NewDbConfig() + RequireStatus(t, rt.CreateDatabase("db1", dbConfig), http.StatusCreated) + + metaKeys := rt.GetDatabase().MetadataKeys + + resp := rt.SendAdminRequest(http.MethodPost, "/db1/_user/", `{"name":"user1","password":"password"}`) + RequireStatus(t, resp, http.StatusCreated) + + ds := rt.MetadataStore() + var user map[string]interface{} + userKey := metaKeys.UserKey("user1") + _, err := ds.Get(userKey, &user) + require.NoError(t, err) + + // Check that the user has an updatedAt field + require.NotNil(t, user["updated_at"]) + currTimeStr := user["updated_at"].(string) + currTime, err := time.Parse(time.RFC3339, currTimeStr) + require.NoError(t, err) + require.NotNil(t, user["created_at"]) + currTimeCreatedStr := user["created_at"].(string) + timeCreated, err := time.Parse(time.RFC3339, currTimeCreatedStr) + require.NoError(t, err) + + // avoid flake where update at seems to be the same (possibly running to fast) + time.Sleep(500 * time.Nanosecond) + + resp = rt.SendAdminRequest(http.MethodPut, "/db1/_user/user1", `{"name":"user1","password":"password1"}`) + RequireStatus(t, resp, http.StatusOK) + + user = map[string]interface{}{} + _, err = ds.Get(userKey, &user) + require.NoError(t, err) + newTimeStr := user["updated_at"].(string) + newTime, err := time.Parse(time.RFC3339, newTimeStr) + require.NoError(t, err) + newCreatedStr := user["created_at"].(string) + newCreated, err := time.Parse(time.RFC3339, newCreatedStr) + require.NoError(t, err) + + assert.Greater(t, newTime.UnixNano(), currTime.UnixNano()) + assert.Equal(t, timeCreated.UnixNano(), newCreated.UnixNano()) +} + +func TestRoleUpdatedAtField(t *testing.T) { + rt := NewRestTester(t, &RestTesterConfig{ + CustomTestBucket: base.GetTestBucket(t), + PersistentConfig: true, + }) + defer rt.Close() + + dbConfig := rt.NewDbConfig() + RequireStatus(t, rt.CreateDatabase("db1", dbConfig), http.StatusCreated) + + resp := rt.SendAdminRequest(http.MethodPost, "/db1/_role/", `{"name":"role1","admin_channels":["test"]}`) + RequireStatus(t, resp, http.StatusCreated) + + ds := rt.MetadataStore() + metaKeys := rt.GetDatabase().MetadataKeys + roleKey := metaKeys.RoleKey("role1") + var user map[string]interface{} + _, err := ds.Get(roleKey, &user) + require.NoError(t, err) + + // Check that the user has an updatedAt field + require.NotNil(t, user["updated_at"]) + currTimeStr := user["updated_at"].(string) + currTime, err := time.Parse(time.RFC3339, currTimeStr) + require.NoError(t, err) + require.NotNil(t, user["created_at"]) + currTimeCreatedStr := user["created_at"].(string) + timeCreated, err := time.Parse(time.RFC3339, currTimeCreatedStr) + require.NoError(t, err) + + // avoid flake where update at seems to be the same (possibly running to fast) + time.Sleep(500 * time.Nanosecond) + + resp = rt.SendAdminRequest(http.MethodPut, "/db1/_role/role1", `{"name":"role1","admin_channels":["ABC"]}`) + RequireStatus(t, resp, http.StatusOK) + + user = map[string]interface{}{} + _, err = ds.Get(roleKey, &user) + require.NoError(t, err) + newTimeStr := user["updated_at"].(string) + newTime, err := time.Parse(time.RFC3339, newTimeStr) + require.NoError(t, err) + newCreatedStr := user["created_at"].(string) + newCreated, err := time.Parse(time.RFC3339, newCreatedStr) + require.NoError(t, err) + + assert.Greater(t, newTime.UnixNano(), currTime.UnixNano()) + assert.Equal(t, timeCreated.UnixNano(), newCreated.UnixNano()) +} diff --git a/rest/replicatortest/replicator_test.go b/rest/replicatortest/replicator_test.go index 9e95610638..a5e1d577a2 100644 --- a/rest/replicatortest/replicator_test.go +++ b/rest/replicatortest/replicator_test.go @@ -8560,3 +8560,41 @@ func requireBodyEqual(t *testing.T, expected string, doc *db.Document) { require.NoError(t, base.JSONUnmarshal([]byte(expected), &expectedBody)) require.Equal(t, expectedBody, doc.Body(base.TestCtx(t))) } + +func TestReplicationConfigUpdatedAt(t *testing.T) { + base.RequireNumTestBuckets(t, 2) + + activeRT, _, remoteURLString, teardown := rest.SetupSGRPeers(t) + defer teardown() + + // create a replication and assert the updated at field is present in the config + activeRT.CreateReplication("replication1", remoteURLString, db.ActiveReplicatorTypePush, nil, true, db.ConflictResolverDefault) + + resp := activeRT.SendAdminRequest(http.MethodGet, "/{{.db}}/_replication/replication1", "") + var configResponse db.ReplicationConfig + require.NoError(t, json.Unmarshal(resp.BodyBytes(), &configResponse)) + + // Check that the config has an updated_at field + require.NotNil(t, configResponse.UpdatedAt) + require.NotNil(t, configResponse.CreatedAt) + currTime := configResponse.UpdatedAt + createdAtTime := configResponse.CreatedAt + + // avoid flake where update at seems to be the same (possibly running to fast) + time.Sleep(500 * time.Nanosecond) + + resp = activeRT.SendAdminRequest("PUT", "/{{.db}}/_replicationStatus/replication1?action=stop", "") + rest.RequireStatus(t, resp, http.StatusOK) + + // update the config + resp = activeRT.SendAdminRequest(http.MethodPut, "/{{.db}}/_replication/replication1", fmt.Sprintf(`{"name":"replication1","source":"%s","type":"push", "continuous":true}`, remoteURLString)) + rest.RequireStatus(t, resp, http.StatusOK) + + // Check that the updated_at field is updated when the config is updated + resp = activeRT.SendAdminRequest(http.MethodGet, "/{{.db}}/_replication/replication1", "") + configResponse = db.ReplicationConfig{} + require.NoError(t, json.Unmarshal(resp.BodyBytes(), &configResponse)) + + assert.Greater(t, configResponse.UpdatedAt.UnixNano(), currTime.UnixNano()) + assert.Equal(t, configResponse.CreatedAt.UnixNano(), createdAtTime.UnixNano()) +} diff --git a/rest/session_api.go b/rest/session_api.go index a4d2e476fe..ababefb4d6 100644 --- a/rest/session_api.go +++ b/rest/session_api.go @@ -278,6 +278,7 @@ func (h *handler) deleteUserSessions() error { return nil } user.UpdateSessionUUID() + user.SetUpdatedAt() err = auth.Save(user) if err == nil { base.Audit(h.ctx(), base.AuditIDPublicUserSessionDeleteAll, base.AuditFields{base.AuditFieldUserName: userName})