基于Golang构建YoOSF-API高并发架构的最佳实践

1func NewFiberServer() *fiber.App {
2    app := fiber.New(fiber.Config{
3        Prefork:       true,  // 启用SO_REUSEPORT，提升多核利用率
4        Concurrency:   1024 * 1024, // 最大并发连接数
5        ReadTimeout:   5 * time.Second,
6        WriteTimeout:  5 * time.Second,
7    })
8    // 跨域中间件
9    app.Use(cors.New(cors.Config{
10        AllowOrigins:     "*",
11        AllowMethods:     "GET,POST,PUT,DELETE",
12        AllowHeaders:     "Content-Type,Authorization",
13        ExposeHeaders:    "X-Cache-Hit", // 透传缓存命中状态
14        MaxAge:           300,
15    }))
16    return app
17}

1type LocalCache struct {
2    mu       sync.RWMutex
3    cache    *lru.Cache
4    threshold int // 热点数据访问阈值
5    stats    CacheStats
6}
7
8func (lc *LocalCache) Get(key string) (interface{}, bool) {
9    lc.mu.RLock()
10    defer lc.mu.RUnlock()
11    val, ok := lc.cache.Get(key)
12    if ok {
13        lc.stats.Hits++
14    } else {
15        lc.stats.Misses++
16    }
17    return val, ok
18}

1func GetFileDataWithLock(key string) (interface{}, error) {
2    // 1. 先查本地缓存
3    if val, ok := localCache.Get(key); ok {
4        return val, nil
5    }
6    // 2. 查Redis缓存
7    val, ok := redisClient.Get(key).Result()
8    if ok == nil {
9        localCache.Set(key, val) // 回写本地缓存
10        return val, nil
11    }
12    // 3. 获取分布式锁，防止缓存击穿
13    lockKey := "lock:" + key
14    lock, err := redisClient.SetNX(lockKey, "1", 5*time.Second).Result()
15    if err != nil || !lock {
16        // 锁竞争失败，等待后重试或返回降级结果
17        time.Sleep(10 * time.Millisecond)
18        return GetFileDataWithLock(key)
19    }
20    defer redisClient.Del(lockKey) // 释放锁
21    
22    // 4. 从S3获取数据并更新缓存
23    data, err := s3Client.GetObject(key)
24    if err != nil {
25        return nil, err
26    }
27    // 双写缓存
28    localCache.Set(key, data)
29    redisClient.Set(key, data, 5*time.Minute)
30    return data, nil
31}

1type GoroutinePool struct {
2    workerNum int
3    taskChan  chan func()
4    wg        sync.WaitGroup
5}
6
7func NewGoroutinePool(workerNum int) *GoroutinePool {
8    pool := &GoroutinePool{
9        workerNum: workerNum,
10        taskChan:  make(chan func(), 1000), // 任务队列缓冲
11    }
12    pool.start()
13    return pool
14}
15
16func (p *GoroutinePool) start() {
17    for i := 0; i < p.workerNum; i++ {
18        p.wg.Add(1)
19        go func() {
20            defer p.wg.Done()
21            for task := range p.taskChan {
22                task() // 执行任务
23            }
24        }()
25    }
26}
27
28// 提交任务，队列满时降级处理
29func (p *GoroutinePool) Submit(task func()) bool {
30    select {
31    case p.taskChan <- task:
32        return true
33    default:
34        // 队列满，直接执行（降级策略）
35        task()
36        return false
37    }
38}

1// 业务层调用
2func (s *DownloadURLService) GenerateURLAsync(fileKey string) (string, error) {
3    resultChan := make(chan string, 1)
4    // 提交异步任务
5    ok := goroutinePool.Submit(func() {
6        url, err := s.s3Client.GeneratePresignedURL(fileKey)
7        if err != nil {
8            resultChan <- ""
9            return
10        }
11        // 缓存结果
12        redisClient.Set("url:"+fileKey, url, 30*time.Minute)
13        resultChan <- url
14    })
15    if !ok {
16        // 协程池满，同步生成（降级）
17        return s.s3Client.GeneratePresignedURL(fileKey)
18    }
19    return <-resultChan, nil
20}