内存管理与对象系统
学习目标
- 深入理解 Redis 对象系统的设计原理
- 掌握引用计数和对象共享机制
- 实现 LRU/LFU 淘汰策略
- 理解内存碎片分析和整理技术
- 掌握 Go 内存分配器优化技巧
️ Redis 对象系统设计
1. 对象系统架构
Redis 使用统一的对象系统管理所有数据类型,通过引用计数和对象共享优化内存使用:
┌─────────────────────────────────────────────────────────────┐
│ Redis 对象系统架构 │
├─────────────────────────────────────────────────────────────┤
│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │
│ │ 字符串 │ │ 列表 │ │ 集合 │ │
│ │ Object │ │ Object │ │ Object │ │
│ └─────────────┘ └─────────────┘ └─────────────┘ │
│ │ │ │ │
│ └───────────────────┼───────────────────┘ │
│ │ │
│ ┌─────────────────────────────────────────────────────────┐ │
│ │ 通用对象结构 │ │
│ │ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ │
│ │ │ 类型 │ │ 编码 │ │ 引用计数 │ │ │
│ │ └─────────────┘ └─────────────┘ └─────────────┘ │ │
│ │ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ │
│ │ │ 指针 │ │ 过期时间 │ │ 访问时间 │ │ │
│ │ └─────────────┘ └─────────────┘ └─────────────┘ │ │
│ └─────────────────────────────────────────────────────────┘ │
│ │ │
│ ┌─────────────────────────────────────────────────────────┐ │
│ │ 内存管理策略 │ │
│ │ • 引用计数:自动内存回收 │ │
│ │ • 对象共享:减少重复数据 │ │
│ │ • 淘汰策略:LRU/LFU/TTL │ │
│ │ • 内存整理:减少碎片化 │ │
│ └─────────────────────────────────────────────────────────┘ │
└─────────────────────────────────────────────────────────────┘
2. 对象类型定义
// Redis 对象类型(简化版)
typedef enum {
REDIS_STRING = 0,
REDIS_LIST = 1,
REDIS_SET = 2,
REDIS_ZSET = 3,
REDIS_HASH = 4
} redisObjectType;
typedef enum {
REDIS_ENCODING_RAW = 0,
REDIS_ENCODING_INT = 1,
REDIS_ENCODING_HT = 2,
REDIS_ENCODING_ZIPLIST = 3,
REDIS_ENCODING_SKIPLIST = 4
} redisEncoding;
typedef struct redisObject {
unsigned type:4; // 对象类型
unsigned encoding:4; // 编码方式
unsigned lru:24; // LRU 时间戳
int refcount; // 引用计数
void *ptr; // 指向实际数据的指针
} redisObject;
️ Go 语言对象系统实现
1. 基础对象结构
// object/redis_object.go
package object
import (
"fmt"
"sync"
"time"
)
// 对象类型
type ObjectType int
const (
STRING ObjectType = iota
LIST
SET
ZSET
HASH
)
// 编码类型
type Encoding int
const (
RAW Encoding = iota
INT
HT
ZIPLIST
SKIPLIST
)
// Redis 对象
type RedisObject struct {
Type ObjectType
Encoding Encoding
Lru int64
RefCount int32
Ptr interface{}
ExpireAt int64
mu sync.RWMutex
}
// 创建对象
func NewRedisObject(objType ObjectType, encoding Encoding, ptr interface{}) *RedisObject {
return &RedisObject{
Type: objType,
Encoding: encoding,
Lru: time.Now().Unix(),
RefCount: 1,
Ptr: ptr,
ExpireAt: 0,
}
}
// 增加引用计数
func (obj *RedisObject) IncrRefCount() {
obj.mu.Lock()
defer obj.mu.Unlock()
obj.RefCount++
}
// 减少引用计数
func (obj *RedisObject) DecrRefCount() {
obj.mu.Lock()
defer obj.mu.Unlock()
obj.RefCount--
if obj.RefCount <= 0 {
obj.free()
}
}
// 释放对象
func (obj *RedisObject) free() {
// 根据类型释放内存
switch obj.Type {
case STRING:
// 字符串对象通常不需要特殊释放
case LIST:
// 列表对象需要释放所有节点
if list, ok := obj.Ptr.(*List); ok {
list.Clear()
}
case SET:
// 集合对象需要释放所有成员
if set, ok := obj.Ptr.(*Set); ok {
set.Clear()
}
case ZSET:
// 有序集合对象需要释放跳表和字典
if zset, ok := obj.Ptr.(*ZSet); ok {
zset.Clear()
}
case HASH:
// 哈希对象需要释放所有键值对
if hash, ok := obj.Ptr.(*Hash); ok {
hash.Clear()
}
}
obj.Ptr = nil
}
// 获取对象类型
func (obj *RedisObject) GetType() ObjectType {
obj.mu.RLock()
defer obj.mu.RUnlock()
return obj.Type
}
// 获取编码类型
func (obj *RedisObject) GetEncoding() Encoding {
obj.mu.RLock()
defer obj.mu.RUnlock()
return obj.Encoding
}
// 获取引用计数
func (obj *RedisObject) GetRefCount() int32 {
obj.mu.RLock()
defer obj.mu.RUnlock()
return obj.RefCount
}
// 更新 LRU 时间
func (obj *RedisObject) UpdateLru() {
obj.mu.Lock()
defer obj.mu.Unlock()
obj.Lru = time.Now().Unix()
}
// 获取 LRU 时间
func (obj *RedisObject) GetLru() int64 {
obj.mu.RLock()
defer obj.mu.RUnlock()
return obj.Lru
}
// 设置过期时间
func (obj *RedisObject) SetExpire(expireAt int64) {
obj.mu.Lock()
defer obj.mu.Unlock()
obj.ExpireAt = expireAt
}
// 获取过期时间
func (obj *RedisObject) GetExpire() int64 {
obj.mu.RLock()
defer obj.mu.RUnlock()
return obj.ExpireAt
}
// 检查是否过期
func (obj *RedisObject) IsExpired() bool {
obj.mu.RLock()
defer obj.mu.RUnlock()
if obj.ExpireAt == 0 {
return false
}
return time.Now().Unix() > obj.ExpireAt
}
2. 对象池实现
// object/object_pool.go
package object
import (
"sync"
)
// 对象池
type ObjectPool struct {
pools map[ObjectType]*sync.Pool
mu sync.RWMutex
}
// 创建对象池
func NewObjectPool() *ObjectPool {
return &ObjectPool{
pools: make(map[ObjectType]*sync.Pool),
}
}
// 获取对象池
func (op *ObjectPool) GetPool(objType ObjectType) *sync.Pool {
op.mu.RLock()
pool, exists := op.pools[objType]
op.mu.RUnlock()
if exists {
return pool
}
op.mu.Lock()
defer op.mu.Unlock()
// 双重检查
if pool, exists := op.pools[objType]; exists {
return pool
}
// 创建新的对象池
pool = &sync.Pool{
New: func() interface{} {
return op.createObject(objType)
},
}
op.pools[objType] = pool
return pool
}
// 创建对象
func (op *ObjectPool) createObject(objType ObjectType) *RedisObject {
switch objType {
case STRING:
return NewRedisObject(STRING, RAW, "")
case LIST:
return NewRedisObject(LIST, ZIPLIST, NewList())
case SET:
return NewRedisObject(SET, HT, NewSet())
case ZSET:
return NewRedisObject(ZSET, SKIPLIST, NewZSet())
case HASH:
return NewRedisObject(HASH, HT, NewHash())
default:
return NewRedisObject(STRING, RAW, "")
}
}
// 获取对象
func (op *ObjectPool) Get(objType ObjectType) *RedisObject {
pool := op.GetPool(objType)
obj := pool.Get().(*RedisObject)
obj.RefCount = 1
obj.Lru = time.Now().Unix()
obj.ExpireAt = 0
return obj
}
// 归还对象
func (op *ObjectPool) Put(obj *RedisObject) {
if obj.RefCount > 1 {
return // 还有引用,不能归还
}
pool := op.GetPool(obj.Type)
obj.Reset()
pool.Put(obj)
}
// 重置对象
func (obj *RedisObject) Reset() {
obj.mu.Lock()
defer obj.mu.Unlock()
obj.RefCount = 0
obj.Lru = 0
obj.ExpireAt = 0
// 根据类型重置
switch obj.Type {
case STRING:
obj.Ptr = ""
case LIST:
if list, ok := obj.Ptr.(*List); ok {
list.Clear()
}
case SET:
if set, ok := obj.Ptr.(*Set); ok {
set.Clear()
}
case ZSET:
if zset, ok := obj.Ptr.(*ZSet); ok {
zset.Clear()
}
case HASH:
if hash, ok := obj.Ptr.(*Hash); ok {
hash.Clear()
}
}
}
3. 引用计数管理
// object/ref_count.go
package object
import (
"sync"
"sync/atomic"
)
// 引用计数管理器
type RefCountManager struct {
objects map[string]*RedisObject
mu sync.RWMutex
}
// 创建引用计数管理器
func NewRefCountManager() *RefCountManager {
return &RefCountManager{
objects: make(map[string]*RedisObject),
}
}
// 设置对象
func (rcm *RefCountManager) Set(key string, obj *RedisObject) {
rcm.mu.Lock()
defer rcm.mu.Unlock()
// 减少旧对象的引用计数
if oldObj, exists := rcm.objects[key]; exists {
oldObj.DecrRefCount()
}
// 增加新对象的引用计数
obj.IncrRefCount()
rcm.objects[key] = obj
}
// 获取对象
func (rcm *RefCountManager) Get(key string) *RedisObject {
rcm.mu.RLock()
defer rcm.mu.RUnlock()
if obj, exists := rcm.objects[key]; exists {
obj.UpdateLru()
return obj
}
return nil
}
// 删除对象
func (rcm *RefCountManager) Delete(key string) {
rcm.mu.Lock()
defer rcm.mu.Unlock()
if obj, exists := rcm.objects[key]; exists {
obj.DecrRefCount()
delete(rcm.objects, key)
}
}
// 获取所有对象
func (rcm *RefCountManager) GetAll() map[string]*RedisObject {
rcm.mu.RLock()
defer rcm.mu.RUnlock()
result := make(map[string]*RedisObject)
for k, v := range rcm.objects {
result[k] = v
}
return result
}
// 清理过期对象
func (rcm *RefCountManager) CleanExpired() int {
rcm.mu.Lock()
defer rcm.mu.Unlock()
count := 0
for key, obj := range rcm.objects {
if obj.IsExpired() {
obj.DecrRefCount()
delete(rcm.objects, key)
count++
}
}
return count
}
// 获取内存使用统计
func (rcm *RefCountManager) GetMemoryStats() map[string]interface{} {
rcm.mu.RLock()
defer rcm.mu.RUnlock()
stats := make(map[string]interface{})
stats["total_objects"] = len(rcm.objects)
typeCount := make(map[ObjectType]int)
for _, obj := range rcm.objects {
typeCount[obj.Type]++
}
stats["type_count"] = typeCount
return stats
}
4. LRU 淘汰策略实现
// object/lru_eviction.go
package object
import (
"container/heap"
"sync"
"time"
)
// LRU 节点
type LRUNode struct {
key string
obj *RedisObject
lastUsed int64
index int
}
// LRU 堆
type LRUHeap []*LRUNode
func (h LRUHeap) Len() int { return len(h) }
func (h LRUHeap) Less(i, j int) bool { return h[i].lastUsed < h[j].lastUsed }
func (h LRUHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i]; h[i].index = i; h[j].index = j }
func (h *LRUHeap) Push(x interface{}) {
n := len(*h)
item := x.(*LRUNode)
item.index = n
*h = append(*h, item)
}
func (h *LRUHeap) Pop() interface{} {
old := *h
n := len(old)
item := old[n-1]
old[n-1] = nil
item.index = -1
*h = old[0 : n-1]
return item
}
// LRU 淘汰器
type LRUEviction struct {
heap *LRUHeap
objects map[string]*LRUNode
mu sync.RWMutex
}
// 创建 LRU 淘汰器
func NewLRUEviction() *LRUEviction {
h := &LRUHeap{}
heap.Init(h)
return &LRUEviction{
heap: h,
objects: make(map[string]*LRUNode),
}
}
// 添加对象
func (lru *LRUEviction) Add(key string, obj *RedisObject) {
lru.mu.Lock()
defer lru.mu.Unlock()
// 如果已存在,更新
if node, exists := lru.objects[key]; exists {
node.lastUsed = time.Now().Unix()
heap.Fix(lru.heap, node.index)
return
}
// 创建新节点
node := &LRUNode{
key: key,
obj: obj,
lastUsed: time.Now().Unix(),
}
heap.Push(lru.heap, node)
lru.objects[key] = node
}
// 更新对象
func (lru *LRUEviction) Update(key string) {
lru.mu.Lock()
defer lru.mu.Unlock()
if node, exists := lru.objects[key]; exists {
node.lastUsed = time.Now().Unix()
heap.Fix(lru.heap, node.index)
}
}
// 删除对象
func (lru *LRUEviction) Remove(key string) {
lru.mu.Lock()
defer lru.mu.Unlock()
if node, exists := lru.objects[key]; exists {
heap.Remove(lru.heap, node.index)
delete(lru.objects, key)
}
}
// 获取最久未使用的对象
func (lru *LRUEviction) GetLRU() (string, *RedisObject) {
lru.mu.RLock()
defer lru.mu.RUnlock()
if lru.heap.Len() == 0 {
return "", nil
}
node := (*lru.heap)[0]
return node.key, node.obj
}
// 淘汰对象
func (lru *LRUEviction) Evict() (string, *RedisObject) {
lru.mu.Lock()
defer lru.mu.Unlock()
if lru.heap.Len() == 0 {
return "", nil
}
node := heap.Pop(lru.heap).(*LRUNode)
delete(lru.objects, node.key)
return node.key, node.obj
}
// 获取淘汰候选对象
func (lru *LRUEviction) GetEvictionCandidates(count int) []string {
lru.mu.RLock()
defer lru.mu.RUnlock()
if count <= 0 || lru.heap.Len() == 0 {
return nil
}
if count > lru.heap.Len() {
count = lru.heap.Len()
}
candidates := make([]string, count)
for i := 0; i < count; i++ {
candidates[i] = (*lru.heap)[i].key
}
return candidates
}
5. LFU 淘汰策略实现
// object/lfu_eviction.go
package object
import (
"container/heap"
"sync"
"time"
)
// LFU 节点
type LFUNode struct {
key string
obj *RedisObject
frequency int
lastUsed int64
index int
}
// LFU 堆
type LFUHeap []*LFUNode
func (h LFUHeap) Len() int { return len(h) }
func (h LFUHeap) Less(i, j int) bool {
if h[i].frequency == h[j].frequency {
return h[i].lastUsed < h[j].lastUsed
}
return h[i].frequency < h[j].frequency
}
func (h LFUHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i]; h[i].index = i; h[j].index = j }
func (h *LFUHeap) Push(x interface{}) {
n := len(*h)
item := x.(*LFUNode)
item.index = n
*h = append(*h, item)
}
func (h *LFUHeap) Pop() interface{} {
old := *h
n := len(old)
item := old[n-1]
old[n-1] = nil
item.index = -1
*h = old[0 : n-1]
return item
}
// LFU 淘汰器
type LFUEviction struct {
heap *LFUHeap
objects map[string]*LFUNode
mu sync.RWMutex
}
// 创建 LFU 淘汰器
func NewLFUEviction() *LFUEviction {
h := &LFUHeap{}
heap.Init(h)
return &LFUEviction{
heap: h,
objects: make(map[string]*LFUNode),
}
}
// 添加对象
func (lfu *LFUEviction) Add(key string, obj *RedisObject) {
lfu.mu.Lock()
defer lfu.mu.Unlock()
// 如果已存在,更新
if node, exists := lfu.objects[key]; exists {
node.frequency++
node.lastUsed = time.Now().Unix()
heap.Fix(lfu.heap, node.index)
return
}
// 创建新节点
node := &LFUNode{
key: key,
obj: obj,
frequency: 1,
lastUsed: time.Now().Unix(),
}
heap.Push(lfu.heap, node)
lfu.objects[key] = node
}
// 更新对象
func (lfu *LFUEviction) Update(key string) {
lfu.mu.Lock()
defer lfu.mu.Unlock()
if node, exists := lfu.objects[key]; exists {
node.frequency++
node.lastUsed = time.Now().Unix()
heap.Fix(lfu.heap, node.index)
}
}
// 删除对象
func (lfu *LFUEviction) Remove(key string) {
lfu.mu.Lock()
defer lfu.mu.Unlock()
if node, exists := lfu.objects[key]; exists {
heap.Remove(lfu.heap, node.index)
delete(lfu.objects, key)
}
}
// 获取最少使用的对象
func (lfu *LFUEviction) GetLFU() (string, *RedisObject) {
lfu.mu.RLock()
defer lfu.mu.RUnlock()
if lfu.heap.Len() == 0 {
return "", nil
}
node := (*lfu.heap)[0]
return node.key, node.obj
}
// 淘汰对象
func (lfu *LFUEviction) Evict() (string, *RedisObject) {
lfu.mu.Lock()
defer lfu.mu.Unlock()
if lfu.heap.Len() == 0 {
return "", nil
}
node := heap.Pop(lfu.heap).(*LFUNode)
delete(lfu.objects, node.key)
return node.key, node.obj
}
// 获取淘汰候选对象
func (lfu *LFUEviction) GetEvictionCandidates(count int) []string {
lfu.mu.RLock()
defer lfu.mu.RUnlock()
if count <= 0 || lfu.heap.Len() == 0 {
return nil
}
if count > lfu.heap.Len() {
count = lfu.heap.Len()
}
candidates := make([]string, count)
for i := 0; i < count; i++ {
candidates[i] = (*lfu.heap)[i].key
}
return candidates
}
6. 内存碎片管理
// object/memory_fragmentation.go
package object
import (
"runtime"
"sync"
"time"
)
// 内存碎片管理器
type MemoryFragmentationManager struct {
totalAllocated int64
totalUsed int64
fragmentation float64
mu sync.RWMutex
}
// 创建内存碎片管理器
func NewMemoryFragmentationManager() *MemoryFragmentationManager {
return &MemoryFragmentationManager{
totalAllocated: 0,
totalUsed: 0,
fragmentation: 0.0,
}
}
// 更新内存统计
func (mfm *MemoryFragmentationManager) UpdateStats(allocated, used int64) {
mfm.mu.Lock()
defer mfm.mu.Unlock()
mfm.totalAllocated = allocated
mfm.totalUsed = used
if allocated > 0 {
mfm.fragmentation = float64(allocated-used) / float64(allocated)
} else {
mfm.fragmentation = 0.0
}
}
// 获取内存统计
func (mfm *MemoryFragmentationManager) GetStats() map[string]interface{} {
mfm.mu.RLock()
defer mfm.mu.RUnlock()
return map[string]interface{}{
"total_allocated": mfm.totalAllocated,
"total_used": mfm.totalUsed,
"fragmentation": mfm.fragmentation,
"fragmentation_pct": mfm.fragmentation * 100,
}
}
// 检查是否需要内存整理
func (mfm *MemoryFragmentationManager) ShouldDefragment() bool {
mfm.mu.RLock()
defer mfm.mu.RUnlock()
// 当碎片率超过 30% 时建议整理
return mfm.fragmentation > 0.3
}
// 内存整理
func (mfm *MemoryFragmentationManager) Defragment() {
// 强制垃圾回收
runtime.GC()
// 等待 GC 完成
time.Sleep(100 * time.Millisecond)
// 更新统计
var m runtime.MemStats
runtime.ReadMemStats(&m)
mfm.UpdateStats(int64(m.HeapAlloc), int64(m.HeapInuse))
}
// 获取内存使用详情
func (mfm *MemoryFragmentationManager) GetMemoryDetails() map[string]interface{} {
var m runtime.MemStats
runtime.ReadMemStats(&m)
return map[string]interface{}{
"heap_alloc": m.HeapAlloc,
"heap_sys": m.HeapSys,
"heap_idle": m.HeapIdle,
"heap_inuse": m.HeapInuse,
"heap_released": m.HeapReleased,
"heap_objects": m.HeapObjects,
"gc_cycles": m.NumGC,
"gc_pause_total": m.PauseTotalNs,
"gc_pause_avg": m.PauseNs[(m.NumGC+255)%256],
}
}
7. 对象生命周期管理
// object/lifecycle_manager.go
package object
import (
"sync"
"time"
)
// 对象生命周期管理器
type LifecycleManager struct {
objects map[string]*RedisObject
lruEviction *LRUEviction
lfuEviction *LFUEviction
fragmentation *MemoryFragmentationManager
mu sync.RWMutex
maxMemory int64
evictionPolicy string
}
// 创建生命周期管理器
func NewLifecycleManager(maxMemory int64, evictionPolicy string) *LifecycleManager {
return &LifecycleManager{
objects: make(map[string]*RedisObject),
lruEviction: NewLRUEviction(),
lfuEviction: NewLFUEviction(),
fragmentation: NewMemoryFragmentationManager(),
maxMemory: maxMemory,
evictionPolicy: evictionPolicy,
}
}
// 设置对象
func (lm *LifecycleManager) Set(key string, obj *RedisObject) {
lm.mu.Lock()
defer lm.mu.Unlock()
// 检查内存限制
if lm.shouldEvict() {
lm.evictObjects()
}
// 设置对象
lm.objects[key] = obj
// 添加到淘汰器
switch lm.evictionPolicy {
case "lru":
lm.lruEviction.Add(key, obj)
case "lfu":
lm.lfuEviction.Add(key, obj)
}
// 更新内存统计
lm.updateMemoryStats()
}
// 获取对象
func (lm *LifecycleManager) Get(key string) *RedisObject {
lm.mu.RLock()
defer lm.mu.RUnlock()
if obj, exists := lm.objects[key]; exists {
// 更新访问时间
obj.UpdateLru()
// 更新淘汰器
switch lm.evictionPolicy {
case "lru":
lm.lruEviction.Update(key)
case "lfu":
lm.lfuEviction.Update(key)
}
return obj
}
return nil
}
// 删除对象
func (lm *LifecycleManager) Delete(key string) {
lm.mu.Lock()
defer lm.mu.Unlock()
if obj, exists := lm.objects[key]; exists {
// 减少引用计数
obj.DecrRefCount()
// 从淘汰器中删除
switch lm.evictionPolicy {
case "lru":
lm.lruEviction.Remove(key)
case "lfu":
lm.lfuEviction.Remove(key)
}
delete(lm.objects, key)
// 更新内存统计
lm.updateMemoryStats()
}
}
// 检查是否需要淘汰
func (lm *LifecycleManager) shouldEvict() bool {
if lm.maxMemory <= 0 {
return false
}
var m runtime.MemStats
runtime.ReadMemStats(&m)
return int64(m.HeapAlloc) > lm.maxMemory
}
// 淘汰对象
func (lm *LifecycleManager) evictObjects() {
// 获取淘汰候选对象
var candidates []string
switch lm.evictionPolicy {
case "lru":
candidates = lm.lruEviction.GetEvictionCandidates(10)
case "lfu":
candidates = lm.lfuEviction.GetEvictionCandidates(10)
default:
return
}
// 淘汰对象
for _, key := range candidates {
if obj, exists := lm.objects[key]; exists {
// 检查是否过期
if obj.IsExpired() {
lm.Delete(key)
continue
}
// 检查引用计数
if obj.GetRefCount() <= 1 {
lm.Delete(key)
}
}
}
}
// 更新内存统计
func (lm *LifecycleManager) updateMemoryStats() {
var m runtime.MemStats
runtime.ReadMemStats(&m)
lm.fragmentation.UpdateStats(int64(m.HeapAlloc), int64(m.HeapInuse))
}
// 清理过期对象
func (lm *LifecycleManager) CleanExpired() int {
lm.mu.Lock()
defer lm.mu.Unlock()
count := 0
for key, obj := range lm.objects {
if obj.IsExpired() {
lm.Delete(key)
count++
}
}
return count
}
// 获取内存统计
func (lm *LifecycleManager) GetMemoryStats() map[string]interface{} {
lm.mu.RLock()
defer lm.mu.RUnlock()
stats := lm.fragmentation.GetStats()
stats["total_objects"] = len(lm.objects)
stats["max_memory"] = lm.maxMemory
stats["eviction_policy"] = lm.evictionPolicy
return stats
}
// 强制内存整理
func (lm *LifecycleManager) Defragment() {
lm.mu.Lock()
defer lm.mu.Unlock()
lm.fragmentation.Defragment()
}
测试验证
1. 单元测试
// object/redis_object_test.go
package object
import (
"testing"
"time"
)
func TestRedisObject(t *testing.T) {
obj := NewRedisObject(STRING, RAW, "test")
// 测试基本属性
if obj.GetType() != STRING {
t.Errorf("Expected type STRING, got %v", obj.GetType())
}
if obj.GetEncoding() != RAW {
t.Errorf("Expected encoding RAW, got %v", obj.GetEncoding())
}
if obj.GetRefCount() != 1 {
t.Errorf("Expected ref count 1, got %d", obj.GetRefCount())
}
// 测试引用计数
obj.IncrRefCount()
if obj.GetRefCount() != 2 {
t.Errorf("Expected ref count 2, got %d", obj.GetRefCount())
}
obj.DecrRefCount()
if obj.GetRefCount() != 1 {
t.Errorf("Expected ref count 1, got %d", obj.GetRefCount())
}
// 测试过期时间
expireAt := time.Now().Unix() + 3600
obj.SetExpire(expireAt)
if obj.GetExpire() != expireAt {
t.Errorf("Expected expire time %d, got %d", expireAt, obj.GetExpire())
}
if obj.IsExpired() {
t.Error("Object should not be expired")
}
// 测试过期
obj.SetExpire(time.Now().Unix() - 1)
if !obj.IsExpired() {
t.Error("Object should be expired")
}
}
func TestObjectPool(t *testing.T) {
pool := NewObjectPool()
// 获取对象
obj1 := pool.Get(STRING)
if obj1.GetType() != STRING {
t.Errorf("Expected type STRING, got %v", obj1.GetType())
}
// 归还对象
pool.Put(obj1)
// 再次获取对象
obj2 := pool.Get(STRING)
if obj2.GetType() != STRING {
t.Errorf("Expected type STRING, got %v", obj2.GetType())
}
// 验证对象被重置
if obj2.GetRefCount() != 1 {
t.Errorf("Expected ref count 1, got %d", obj2.GetRefCount())
}
}
func TestRefCountManager(t *testing.T) {
rcm := NewRefCountManager()
// 创建对象
obj1 := NewRedisObject(STRING, RAW, "test1")
obj2 := NewRedisObject(STRING, RAW, "test2")
// 设置对象
rcm.Set("key1", obj1)
rcm.Set("key2", obj2)
// 验证引用计数
if obj1.GetRefCount() != 2 { // 1 + 1 from manager
t.Errorf("Expected ref count 2, got %d", obj1.GetRefCount())
}
// 获取对象
retrieved := rcm.Get("key1")
if retrieved != obj1 {
t.Error("Retrieved object should be the same as original")
}
// 删除对象
rcm.Delete("key1")
if obj1.GetRefCount() != 1 { // 1 from manager
t.Errorf("Expected ref count 1, got %d", obj1.GetRefCount())
}
}
func TestLRUEviction(t *testing.T) {
lru := NewLRUEviction()
// 创建对象
obj1 := NewRedisObject(STRING, RAW, "test1")
obj2 := NewRedisObject(STRING, RAW, "test2")
obj3 := NewRedisObject(STRING, RAW, "test3")
// 添加对象
lru.Add("key1", obj1)
lru.Add("key2", obj2)
lru.Add("key3", obj3)
// 更新对象
lru.Update("key1")
// 获取最久未使用的对象
key, obj := lru.GetLRU()
if key != "key2" && key != "key3" {
t.Errorf("Expected key2 or key3, got %s", key)
}
// 淘汰对象
evictedKey, evictedObj := lru.Evict()
if evictedKey == "" || evictedObj == nil {
t.Error("Should have evicted an object")
}
}
func TestLFUEviction(t *testing.T) {
lfu := NewLFUEviction()
// 创建对象
obj1 := NewRedisObject(STRING, RAW, "test1")
obj2 := NewRedisObject(STRING, RAW, "test2")
obj3 := NewRedisObject(STRING, RAW, "test3")
// 添加对象
lfu.Add("key1", obj1)
lfu.Add("key2", obj2)
lfu.Add("key3", obj3)
// 更新对象使用频率
lfu.Update("key1")
lfu.Update("key1")
lfu.Update("key2")
// 获取最少使用的对象
key, obj := lfu.GetLFU()
if key != "key3" {
t.Errorf("Expected key3, got %s", key)
}
// 淘汰对象
evictedKey, evictedObj := lfu.Evict()
if evictedKey == "" || evictedObj == nil {
t.Error("Should have evicted an object")
}
}
func TestLifecycleManager(t *testing.T) {
lm := NewLifecycleManager(1024*1024, "lru") // 1MB max memory
// 创建对象
obj1 := NewRedisObject(STRING, RAW, "test1")
obj2 := NewRedisObject(STRING, RAW, "test2")
// 设置对象
lm.Set("key1", obj1)
lm.Set("key2", obj2)
// 获取对象
retrieved := lm.Get("key1")
if retrieved != obj1 {
t.Error("Retrieved object should be the same as original")
}
// 删除对象
lm.Delete("key1")
// 验证对象被删除
retrieved = lm.Get("key1")
if retrieved != nil {
t.Error("Object should be deleted")
}
// 获取内存统计
stats := lm.GetMemoryStats()
if stats["total_objects"].(int) != 1 {
t.Errorf("Expected 1 object, got %d", stats["total_objects"])
}
}
2. 性能基准测试
// object/benchmark_test.go
package object
import (
"testing"
)
func BenchmarkRedisObjectCreation(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
obj := NewRedisObject(STRING, RAW, "test")
obj.DecrRefCount()
}
}
func BenchmarkObjectPoolGetPut(b *testing.B) {
pool := NewObjectPool()
b.ResetTimer()
for i := 0; i < b.N; i++ {
obj := pool.Get(STRING)
pool.Put(obj)
}
}
func BenchmarkRefCountManager(b *testing.B) {
rcm := NewRefCountManager()
b.ResetTimer()
for i := 0; i < b.N; i++ {
key := fmt.Sprintf("key%d", i%1000)
obj := NewRedisObject(STRING, RAW, "test")
rcm.Set(key, obj)
rcm.Get(key)
}
}
func BenchmarkLRUEviction(b *testing.B) {
lru := NewLRUEviction()
// 预填充数据
for i := 0; i < 1000; i++ {
key := fmt.Sprintf("key%d", i)
obj := NewRedisObject(STRING, RAW, "test")
lru.Add(key, obj)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
key := fmt.Sprintf("key%d", i%1000)
lru.Update(key)
}
}
func BenchmarkLFUEviction(b *testing.B) {
lfu := NewLFUEviction()
// 预填充数据
for i := 0; i < 1000; i++ {
key := fmt.Sprintf("key%d", i)
obj := NewRedisObject(STRING, RAW, "test")
lfu.Add(key, obj)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
key := fmt.Sprintf("key%d", i%1000)
lfu.Update(key)
}
}
func BenchmarkLifecycleManager(b *testing.B) {
lm := NewLifecycleManager(1024*1024, "lru")
b.ResetTimer()
for i := 0; i < b.N; i++ {
key := fmt.Sprintf("key%d", i%1000)
obj := NewRedisObject(STRING, RAW, "test")
lm.Set(key, obj)
lm.Get(key)
}
}
3. 内存使用测试
// object/memory_test.go
package object
import (
"runtime"
"testing"
)
func TestMemoryUsage(t *testing.T) {
var m1, m2 runtime.MemStats
runtime.ReadMemStats(&m1)
// 创建大量对象
objects := make([]*RedisObject, 10000)
for i := 0; i < 10000; i++ {
objects[i] = NewRedisObject(STRING, RAW, "test")
}
runtime.ReadMemStats(&m2)
heapGrowth := m2.HeapAlloc - m1.HeapAlloc
t.Logf("Memory usage: %d bytes", heapGrowth)
if heapGrowth > 10*1024*1024 { // 10MB
t.Errorf("Memory usage too high: %d bytes", heapGrowth)
}
}
func TestObjectPoolMemoryReuse(t *testing.T) {
pool := NewObjectPool()
var m1, m2 runtime.MemStats
runtime.ReadMemStats(&m1)
// 获取和归还对象
for i := 0; i < 10000; i++ {
obj := pool.Get(STRING)
pool.Put(obj)
}
runtime.ReadMemStats(&m2)
heapGrowth := m2.HeapAlloc - m1.HeapAlloc
t.Logf("Memory usage with pool: %d bytes", heapGrowth)
if heapGrowth > 1024*1024 { // 1MB
t.Errorf("Memory usage too high: %d bytes", heapGrowth)
}
}
func TestMemoryFragmentation(t *testing.T) {
mfm := NewMemoryFragmentationManager()
// 模拟内存分配
mfm.UpdateStats(1000, 800)
stats := mfm.GetStats()
if stats["fragmentation"].(float64) != 0.2 {
t.Errorf("Expected fragmentation 0.2, got %f", stats["fragmentation"])
}
// 检查是否需要整理
if !mfm.ShouldDefragment() {
t.Error("Should recommend defragmentation")
}
}
性能对比分析
1. 内存管理策略对比
策略 | 优点 | 缺点 | 适用场景 |
---|---|---|---|
引用计数 | 简单、实时回收 | 循环引用、性能开销 | 简单对象 |
标记清除 | 无循环引用问题 | 停顿时间长 | 复杂对象 |
分代回收 | 性能好 | 实现复杂 | 长期运行 |
对象池 | 减少分配开销 | 内存占用 | 频繁创建销毁 |
2. 淘汰策略对比
策略 | 优点 | 缺点 | 适用场景 |
---|---|---|---|
LRU | 简单、直观 | 可能误删热点数据 | 访问模式稳定 |
LFU | 准确识别热点 | 实现复杂、内存开销 | 访问模式变化 |
TTL | 自动过期 | 需要额外存储 | 有时效性要求 |
随机 | 实现简单 | 可能误删重要数据 | 对准确性要求不高 |
3. 性能测试结果
// 基准测试结果示例
func BenchmarkComparison(b *testing.B) {
// 对象创建性能
b.Run("ObjectCreation", func(b *testing.B) {
for i := 0; i < b.N; i++ {
obj := NewRedisObject(STRING, RAW, "test")
obj.DecrRefCount()
}
})
// 对象池性能
b.Run("ObjectPool", func(b *testing.B) {
pool := NewObjectPool()
for i := 0; i < b.N; i++ {
obj := pool.Get(STRING)
pool.Put(obj)
}
})
// 引用计数性能
b.Run("RefCount", func(b *testing.B) {
obj := NewRedisObject(STRING, RAW, "test")
for i := 0; i < b.N; i++ {
obj.IncrRefCount()
obj.DecrRefCount()
}
})
}
面试要点
1. Redis 对象系统的优势
答案要点:
- 统一管理:所有数据类型使用统一的对象结构
- 引用计数:自动内存回收,避免内存泄漏
- 对象共享:减少重复数据,节省内存
- 类型安全:编译时类型检查,运行时类型验证
2. 引用计数的优缺点
答案要点:
- 优点:实时回收、简单实现、无停顿
- 缺点:循环引用、性能开销、内存碎片
- 解决方案:弱引用、循环检测、定期整理
3. LRU vs LFU 的选择
答案要点:
- LRU:适合访问模式稳定的场景
- LFU:适合访问模式变化的场景
- 混合策略:结合两种策略的优势
- 自适应:根据访问模式动态调整
4. 内存优化的技巧
答案要点:
- 对象池:减少频繁分配和释放
- 内存对齐:提高访问效率
- 批量操作:减少锁竞争
- 预分配:避免动态扩容
总结
通过本章学习,我们深入理解了:
- Redis 对象系统的设计原理和实现
- 引用计数和对象共享机制的优化
- LRU/LFU 淘汰策略的完整实现
- 内存碎片分析和整理技术
- Go 内存分配器优化技巧
内存管理与对象系统为 Redis 提供了高效的内存使用和对象生命周期管理。在下一章中,我们将学习 AOF 持久化机制,了解 Redis 如何实现数据持久化。