Skip to content

Commit

Permalink
Merge pull request #154 from Lixuhuilll/optimize-redicache
Browse files Browse the repository at this point in the history
优化 RedisCache 的删除性能
  • Loading branch information
dragove authored Dec 27, 2023
2 parents 13781ad + 9ea4bcb commit 7b7963d
Show file tree
Hide file tree
Showing 2 changed files with 61 additions and 9 deletions.
64 changes: 58 additions & 6 deletions src/main/java/plus/maa/backend/repository/RedisCache.java
Original file line number Diff line number Diff line change
Expand Up @@ -12,17 +12,21 @@
import org.jetbrains.annotations.Nullable;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.ClassPathResource;
import org.springframework.dao.InvalidDataAccessApiUsageException;
import org.springframework.data.redis.RedisSystemException;
import org.springframework.data.redis.core.Cursor;
import org.springframework.data.redis.core.ScanOptions;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.data.redis.core.script.RedisScript;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Component;

import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Function;
import java.util.function.Supplier;

Expand Down Expand Up @@ -50,6 +54,9 @@ public class RedisCache {
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
.build();


private final AtomicBoolean supportUnlink = new AtomicBoolean(true);

/*
使用 lua 脚本插入数据,维持 ZSet 的相对大小(size <= 实际大小 <= size + 50)以及过期时间
实际大小这么设计是为了避免频繁的 ZREMRANGEBYRANK 操作
Expand Down Expand Up @@ -275,7 +282,38 @@ public void setCacheLevelCommit(String commit) {
}

public void removeCache(String key) {
redisTemplate.delete(key);
removeCache(key, false);
}

public void removeCache(String key, boolean notUseUnlink) {
removeCache(List.of(key), notUseUnlink);
}

public void removeCache(Collection<String> keys) {
removeCache(keys, false);
}

public void removeCache(Collection<String> keys, boolean notUseUnlink) {

if (!notUseUnlink && supportUnlink.get()) {
try {
redisTemplate.unlink(keys);
return;
} catch (InvalidDataAccessApiUsageException | RedisSystemException e) {
// Redisson、Jedis、Lettuce
Throwable cause = e.getCause();
if (cause == null || !StringUtils.containsAny(
cause.getMessage(), "unknown command", "not support")) {
throw e;
}
if (supportUnlink.compareAndSet(true, false)) {
log.warn("当前连接的 Redis Service 可能不支持 Unlink 命令,切换为 Del");
}
}
}

// 兜底的 Del 命令
redisTemplate.delete(keys);
}

/**
Expand All @@ -294,22 +332,36 @@ public <T> boolean removeKVIfEquals(String key, T value) {
}

/**
* 模糊删除缓存。
* 模糊删除缓存。不保证立即删除,不保证完全删除。<br>
* 异步,因为 Scan 虽然不会阻塞 Redis,但客户端会阻塞
*
* @param pattern 待删除的 Key 表达式,例如 "home:*" 表示删除 Key 以 "home:" 开头的所有缓存
* @author Lixuhuilll
*/
@Async
public void removeCacheByPattern(String pattern) {
syncRemoveCacheByPattern(pattern);
}

/**
* 模糊删除缓存。不保证立即删除,不保证完全删除。<br>
* 同步调用 Scan,不会长时间阻塞 Redis,但会阻塞客户端,阻塞时间视 Redis 中 key 的数量而定。
* 删除期间,其他线程或客户端可对 Redis 进行 CURD(因为不阻塞 Redis),因此不保证删除的时机,也不保证完全删除干净
*
* @param pattern 待删除的 Key 表达式,例如 "home:*" 表示删除 Key 以 "home:" 开头的所有缓存
* @author Lixuhuilll
*/
public void syncRemoveCacheByPattern(String pattern) {
// 批量删除的阈值
final int batchSize = 10000;
final int batchSize = 2000;
// 构造 ScanOptions
ScanOptions scanOptions = ScanOptions.scanOptions()
.count(batchSize)
.match(pattern)
.build();

// 保存要删除的键
List<String> keysToDelete = new ArrayList<>();
List<String> keysToDelete = new ArrayList<>(batchSize);

// try-with-resources 自动关闭 SCAN
try (Cursor<String> cursor = redisTemplate.scan(scanOptions)) {
Expand All @@ -320,15 +372,15 @@ public void removeCacheByPattern(String pattern) {

// 如果达到批量删除的阈值,则执行批量删除
if (keysToDelete.size() >= batchSize) {
redisTemplate.delete(keysToDelete);
removeCache(keysToDelete);
keysToDelete.clear();
}
}
}

// 删除剩余的键(不足 batchSize 的最后一批)
if (!keysToDelete.isEmpty()) {
redisTemplate.delete(keysToDelete);
removeCache(keysToDelete);
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ public void refreshHotScores() {
}

// 移除首页热度缓存
redisCache.removeCacheByPattern("home:hot:*");
redisCache.syncRemoveCacheByPattern("home:hot:*");
}

/**
Expand All @@ -91,9 +91,9 @@ public void refreshTop100HotScores() {
refresh(copilotIdSTRs, copilots);

// 移除近期评分变化量缓存
redisCache.removeCacheByPattern("rate:hot:copilotIds");
redisCache.removeCache("rate:hot:copilotIds");
// 移除首页热度缓存
redisCache.removeCacheByPattern("home:hot:*");
redisCache.syncRemoveCacheByPattern("home:hot:*");
}

private void refresh(Collection<String> copilotIdSTRs, Iterable<Copilot> copilots) {
Expand Down

0 comments on commit 7b7963d

Please sign in to comment.