实验管理

// experiment_models.go
package experiment

import (
    "time"
)

// Project 项目
type Project struct {
    ID          string            `json:"id" db:"id"`
    Name        string            `json:"name" db:"name"`
    Description string            `json:"description" db:"description"`
    Owner       string            `json:"owner" db:"owner"`
    Team        string            `json:"team" db:"team"`
    Tags        []string          `json:"tags" db:"tags"`
    Settings    ProjectSettings   `json:"settings" db:"settings"`
    CreatedAt   time.Time         `json:"created_at" db:"created_at"`
    UpdatedAt   time.Time         `json:"updated_at" db:"updated_at"`
}

// ProjectSettings 项目设置
type ProjectSettings struct {
    // 默认实验配置
    DefaultExperimentConfig map[string]interface{} `json:"default_experiment_config"`
    // 指标可视化配置
    MetricDisplayConfig     MetricDisplayConfig    `json:"metric_display_config"`
    // 归档策略
    ArchivePolicy           ArchivePolicy          `json:"archive_policy"`
}

// MetricDisplayConfig 指标显示配置
type MetricDisplayConfig struct {
    PrimaryMetrics   []string `json:"primary_metrics"`   // 主要指标
    HigherIsBetter   []string `json:"higher_is_better"`  // 越高越好的指标
    ChartTypes       map[string]string `json:"chart_types"` // 图表类型
}

// ArchivePolicy 归档策略
type ArchivePolicy struct {
    AutoArchiveDays int  `json:"auto_archive_days"` // 自动归档天数
    KeepBestRuns    int  `json:"keep_best_runs"`    // 保留最佳运行数
    KeepRecentRuns  int  `json:"keep_recent_runs"`  // 保留最近运行数
}

// Experiment 实验
type Experiment struct {
    ID          string           `json:"id" db:"id"`
    ProjectID   string           `json:"project_id" db:"project_id"`
    Name        string           `json:"name" db:"name"`
    Description string           `json:"description" db:"description"`
    Hypothesis  string           `json:"hypothesis" db:"hypothesis"` // 实验假设
    Status      ExperimentStatus `json:"status" db:"status"`
    Tags        []string         `json:"tags" db:"tags"`
    CreatedBy   string           `json:"created_by" db:"created_by"`
    CreatedAt   time.Time        `json:"created_at" db:"created_at"`
    UpdatedAt   time.Time        `json:"updated_at" db:"updated_at"`
    ArchivedAt  *time.Time       `json:"archived_at" db:"archived_at"`
}

// ExperimentStatus 实验状态
type ExperimentStatus string

const (
    ExperimentStatusActive    ExperimentStatus = "active"
    ExperimentStatusCompleted ExperimentStatus = "completed"
    ExperimentStatusArchived  ExperimentStatus = "archived"
)

// Run 运行
type Run struct {
    ID           string        `json:"id" db:"id"`
    ExperimentID string        `json:"experiment_id" db:"experiment_id"`
    Name         string        `json:"name" db:"name"`
    Status       RunStatus     `json:"status" db:"status"`

    // 来源追踪
    Source       RunSource     `json:"source" db:"source"`

    // 配置
    Parameters   Parameters    `json:"parameters" db:"parameters"`
    Environment  Environment   `json:"environment" db:"environment"`

    // 运行时信息
    StartTime    *time.Time    `json:"start_time" db:"start_time"`
    EndTime      *time.Time    `json:"end_time" db:"end_time"`
    Duration     int64         `json:"duration" db:"duration"` // 秒

    // 资源使用
    Resources    ResourceUsage `json:"resources" db:"resources"`

    // 元信息
    Tags         []string      `json:"tags" db:"tags"`
    Notes        string        `json:"notes" db:"notes"`
    CreatedBy    string        `json:"created_by" db:"created_by"`
    CreatedAt    time.Time     `json:"created_at" db:"created_at"`
}

// RunStatus 运行状态
type RunStatus string

const (
    RunStatusPending   RunStatus = "pending"
    RunStatusRunning   RunStatus = "running"
    RunStatusCompleted RunStatus = "completed"
    RunStatusFailed    RunStatus = "failed"
    RunStatusKilled    RunStatus = "killed"
)

// RunSource 运行来源
type RunSource struct {
    // Git 信息
    GitRepoURL    string `json:"git_repo_url"`
    GitCommit     string `json:"git_commit"`
    GitBranch     string `json:"git_branch"`
    GitTag        string `json:"git_tag"`
    GitDirty      bool   `json:"git_dirty"` // 是否有未提交更改

    // 代码入口
    EntryPoint    string `json:"entry_point"`    // 执行脚本
    SourceType    string `json:"source_type"`    // "local", "git", "notebook"

    // 代码快照（小项目）
    CodeSnapshot  string `json:"code_snapshot"`  // 代码归档路径
}

// Parameters 参数
type Parameters struct {
    // 模型参数
    ModelConfig   map[string]interface{} `json:"model_config"`

    // 训练参数
    TrainingConfig map[string]interface{} `json:"training_config"`

    // 数据参数
    DataConfig    map[string]interface{} `json:"data_config"`

    // 原始参数（全部展开）
    Flat          map[string]interface{} `json:"flat"`
}

// Environment 环境信息
type Environment struct {
    // Python 环境
    PythonVersion    string            `json:"python_version"`
    PipPackages      map[string]string `json:"pip_packages"`
    CondaEnvironment string            `json:"conda_environment"`

    // 硬件信息
    GPUType          string   `json:"gpu_type"`
    GPUCount         int      `json:"gpu_count"`
    GPUMemory        int64    `json:"gpu_memory"`
    CPUCount         int      `json:"cpu_count"`
    MemoryGB         int      `json:"memory_gb"`

    // 系统信息
    Platform         string   `json:"platform"`
    Hostname         string   `json:"hostname"`
    DockerImage      string   `json:"docker_image"`

    // 框架版本
    FrameworkVersions map[string]string `json:"framework_versions"`
}

// ResourceUsage 资源使用
type ResourceUsage struct {
    GPUHours     float64 `json:"gpu_hours"`
    CPUHours     float64 `json:"cpu_hours"`
    MemoryGBHours float64 `json:"memory_gb_hours"`
    StorageGB    float64 `json:"storage_gb"`
    Cost         float64 `json:"cost"` // 预估成本
}

// Metric 指标
type Metric struct {
    RunID     string    `json:"run_id" db:"run_id"`
    Key       string    `json:"key" db:"key"`
    Value     float64   `json:"value" db:"value"`
    Step      int64     `json:"step" db:"step"`
    Timestamp time.Time `json:"timestamp" db:"timestamp"`
    Context   string    `json:"context" db:"context"` // train, eval, test
}

// Artifact 产物
type Artifact struct {
    ID        string    `json:"id" db:"id"`
    RunID     string    `json:"run_id" db:"run_id"`
    Name      string    `json:"name" db:"name"`
    Type      string    `json:"type" db:"type"` // model, data, figure, etc.
    Path      string    `json:"path" db:"path"`
    Size      int64     `json:"size" db:"size"`
    Checksum  string    `json:"checksum" db:"checksum"`
    Metadata  map[string]interface{} `json:"metadata" db:"metadata"`
    CreatedAt time.Time `json:"created_at" db:"created_at"`
}

// tracking_client.go
package experiment

import (
    "bytes"
    "context"
    "encoding/json"
    "fmt"
    "net/http"
    "os"
    "os/exec"
    "runtime"
    "strings"
    "sync"
    "time"
)

// TrackingClient 追踪客户端
type TrackingClient struct {
    serverURL    string
    httpClient   *http.Client

    // 当前运行上下文
    activeRun    *Run
    runMu        sync.RWMutex

    // 批量写入缓冲
    metricBuffer []*Metric
    bufferMu     sync.Mutex
    flushTicker  *time.Ticker

    // 配置
    config       ClientConfig
}

// ClientConfig 客户端配置
type ClientConfig struct {
    BatchSize      int           // 批量大小
    FlushInterval  time.Duration // 刷新间隔
    AutoLogEnv     bool          // 自动记录环境
    AutoLogGit     bool          // 自动记录 Git 信息
}

// NewTrackingClient 创建追踪客户端
func NewTrackingClient(serverURL string, config ClientConfig) *TrackingClient {
    client := &TrackingClient{
        serverURL:    serverURL,
        httpClient:   &http.Client{Timeout: 30 * time.Second},
        metricBuffer: make([]*Metric, 0, config.BatchSize),
        config:       config,
    }

    // 启动定时刷新
    client.flushTicker = time.NewTicker(config.FlushInterval)
    go client.flushLoop()

    return client
}

// StartRun 开始运行
func (c *TrackingClient) StartRun(
    ctx context.Context,
    experimentID string,
    runName string,
    params map[string]interface{},
) (*Run, error) {

    // 创建运行
    run := &Run{
        ExperimentID: experimentID,
        Name:         runName,
        Status:       RunStatusRunning,
        Parameters: Parameters{
            Flat: params,
        },
        CreatedAt: time.Now(),
    }

    now := time.Now()
    run.StartTime = &now

    // 自动收集信息
    if c.config.AutoLogEnv {
        run.Environment = c.collectEnvironment()
    }

    if c.config.AutoLogGit {
        run.Source = c.collectGitInfo()
    }

    // 发送到服务器
    resp, err := c.post(ctx, "/api/v1/runs", run)
    if err != nil {
        return nil, fmt.Errorf("create run: %w", err)
    }

    if err := json.NewDecoder(resp.Body).Decode(run); err != nil {
        return nil, fmt.Errorf("decode response: %w", err)
    }

    // 设置为活跃运行
    c.runMu.Lock()
    c.activeRun = run
    c.runMu.Unlock()

    return run, nil
}

// LogParam 记录参数
func (c *TrackingClient) LogParam(key string, value interface{}) error {
    c.runMu.RLock()
    run := c.activeRun
    c.runMu.RUnlock()

    if run == nil {
        return fmt.Errorf("no active run")
    }

    return c.LogParams(map[string]interface{}{key: value})
}

// LogParams 批量记录参数
func (c *TrackingClient) LogParams(params map[string]interface{}) error {
    c.runMu.RLock()
    run := c.activeRun
    c.runMu.RUnlock()

    if run == nil {
        return fmt.Errorf("no active run")
    }

    _, err := c.post(context.Background(), fmt.Sprintf("/api/v1/runs/%s/params", run.ID), params)
    return err
}

// LogMetric 记录指标
func (c *TrackingClient) LogMetric(key string, value float64, step int64) error {
    c.runMu.RLock()
    run := c.activeRun
    c.runMu.RUnlock()

    if run == nil {
        return fmt.Errorf("no active run")
    }

    metric := &Metric{
        RunID:     run.ID,
        Key:       key,
        Value:     value,
        Step:      step,
        Timestamp: time.Now(),
    }

    // 添加到缓冲
    c.bufferMu.Lock()
    c.metricBuffer = append(c.metricBuffer, metric)

    // 如果达到批量大小，立即刷新
    if len(c.metricBuffer) >= c.config.BatchSize {
        go c.flush()
    }
    c.bufferMu.Unlock()

    return nil
}

// LogMetrics 批量记录指标
func (c *TrackingClient) LogMetrics(metrics map[string]float64, step int64) error {
    for key, value := range metrics {
        if err := c.LogMetric(key, value, step); err != nil {
            return err
        }
    }
    return nil
}

// LogArtifact 记录产物
func (c *TrackingClient) LogArtifact(localPath string, artifactPath string) error {
    c.runMu.RLock()
    run := c.activeRun
    c.runMu.RUnlock()

    if run == nil {
        return fmt.Errorf("no active run")
    }

    // 读取文件
    data, err := os.ReadFile(localPath)
    if err != nil {
        return fmt.Errorf("read file: %w", err)
    }

    // 上传
    url := fmt.Sprintf("%s/api/v1/runs/%s/artifacts?path=%s", c.serverURL, run.ID, artifactPath)
    req, err := http.NewRequest("PUT", url, bytes.NewReader(data))
    if err != nil {
        return err
    }

    resp, err := c.httpClient.Do(req)
    if err != nil {
        return err
    }
    defer resp.Body.Close()

    if resp.StatusCode != http.StatusOK {
        return fmt.Errorf("upload failed: %s", resp.Status)
    }

    return nil
}

// LogModel 记录模型
func (c *TrackingClient) LogModel(modelPath string, modelName string, metadata map[string]interface{}) error {
    c.runMu.RLock()
    run := c.activeRun
    c.runMu.RUnlock()

    if run == nil {
        return fmt.Errorf("no active run")
    }

    // 创建模型产物记录
    artifact := &Artifact{
        RunID:    run.ID,
        Name:     modelName,
        Type:     "model",
        Path:     modelPath,
        Metadata: metadata,
    }

    _, err := c.post(context.Background(), fmt.Sprintf("/api/v1/runs/%s/models", run.ID), artifact)
    return err
}

// SetTag 设置标签
func (c *TrackingClient) SetTag(key, value string) error {
    c.runMu.RLock()
    run := c.activeRun
    c.runMu.RUnlock()

    if run == nil {
        return fmt.Errorf("no active run")
    }

    _, err := c.post(context.Background(),
        fmt.Sprintf("/api/v1/runs/%s/tags", run.ID),
        map[string]string{key: value})
    return err
}

// EndRun 结束运行
func (c *TrackingClient) EndRun(status RunStatus) error {
    c.runMu.Lock()
    run := c.activeRun
    c.activeRun = nil
    c.runMu.Unlock()

    if run == nil {
        return fmt.Errorf("no active run")
    }

    // 刷新剩余指标
    c.flush()

    // 更新运行状态
    now := time.Now()
    run.EndTime = &now
    run.Status = status
    run.Duration = int64(now.Sub(*run.StartTime).Seconds())

    _, err := c.post(context.Background(), fmt.Sprintf("/api/v1/runs/%s/end", run.ID), run)
    return err
}

// flush 刷新指标缓冲
func (c *TrackingClient) flush() {
    c.bufferMu.Lock()
    if len(c.metricBuffer) == 0 {
        c.bufferMu.Unlock()
        return
    }

    metrics := c.metricBuffer
    c.metricBuffer = make([]*Metric, 0, c.config.BatchSize)
    c.bufferMu.Unlock()

    c.runMu.RLock()
    run := c.activeRun
    c.runMu.RUnlock()

    if run == nil {
        return
    }

    // 批量发送
    c.post(context.Background(), fmt.Sprintf("/api/v1/runs/%s/metrics/batch", run.ID), metrics)
}

func (c *TrackingClient) flushLoop() {
    for range c.flushTicker.C {
        c.flush()
    }
}

// collectEnvironment 收集环境信息
func (c *TrackingClient) collectEnvironment() Environment {
    env := Environment{
        PythonVersion:     getPythonVersion(),
        PipPackages:       getPipPackages(),
        Platform:          runtime.GOOS,
        CPUCount:          runtime.NumCPU(),
        FrameworkVersions: make(map[string]string),
    }

    // 获取 GPU 信息
    if gpuInfo := getGPUInfo(); gpuInfo != nil {
        env.GPUType = gpuInfo.Type
        env.GPUCount = gpuInfo.Count
        env.GPUMemory = gpuInfo.Memory
    }

    // 获取主机名
    env.Hostname, _ = os.Hostname()

    // 获取 Docker 镜像（如果在容器中）
    env.DockerImage = os.Getenv("DOCKER_IMAGE")

    return env
}

// collectGitInfo 收集 Git 信息
func (c *TrackingClient) collectGitInfo() RunSource {
    source := RunSource{
        SourceType: "git",
    }

    // 获取远程仓库 URL
    if out, err := exec.Command("git", "remote", "get-url", "origin").Output(); err == nil {
        source.GitRepoURL = strings.TrimSpace(string(out))
    }

    // 获取当前提交
    if out, err := exec.Command("git", "rev-parse", "HEAD").Output(); err == nil {
        source.GitCommit = strings.TrimSpace(string(out))
    }

    // 获取当前分支
    if out, err := exec.Command("git", "rev-parse", "--abbrev-ref", "HEAD").Output(); err == nil {
        source.GitBranch = strings.TrimSpace(string(out))
    }

    // 检查是否有未提交更改
    if out, err := exec.Command("git", "status", "--porcelain").Output(); err == nil {
        source.GitDirty = len(out) > 0
    }

    return source
}

func (c *TrackingClient) post(ctx context.Context, path string, body interface{}) (*http.Response, error) {
    data, err := json.Marshal(body)
    if err != nil {
        return nil, err
    }

    req, err := http.NewRequestWithContext(ctx, "POST", c.serverURL+path, bytes.NewReader(data))
    if err != nil {
        return nil, err
    }
    req.Header.Set("Content-Type", "application/json")

    return c.httpClient.Do(req)
}

// 辅助函数
func getPythonVersion() string {
    out, err := exec.Command("python", "--version").Output()
    if err != nil {
        return ""
    }
    return strings.TrimSpace(string(out))
}

func getPipPackages() map[string]string {
    packages := make(map[string]string)
    out, err := exec.Command("pip", "freeze").Output()
    if err != nil {
        return packages
    }

    for _, line := range strings.Split(string(out), "\n") {
        parts := strings.Split(line, "==")
        if len(parts) == 2 {
            packages[parts[0]] = parts[1]
        }
    }
    return packages
}

type GPUInfo struct {
    Type   string
    Count  int
    Memory int64
}

func getGPUInfo() *GPUInfo {
    // 使用 nvidia-smi 获取 GPU 信息
    out, err := exec.Command("nvidia-smi", "--query-gpu=name,count,memory.total", "--format=csv,noheader,nounits").Output()
    if err != nil {
        return nil
    }

    lines := strings.Split(strings.TrimSpace(string(out)), "\n")
    if len(lines) == 0 {
        return nil
    }

    // 解析第一行
    parts := strings.Split(lines[0], ", ")
    if len(parts) < 3 {
        return nil
    }

    info := &GPUInfo{
        Type:  parts[0],
        Count: len(lines),
    }

    fmt.Sscanf(parts[2], "%d", &info.Memory)
    return info
}

// experiment_service.go
package service

import (
    "context"
    "fmt"
    "time"
)

// ExperimentService 实验服务
type ExperimentService struct {
    // 存储
    projectRepo    ProjectRepository
    experimentRepo ExperimentRepository
    runRepo        RunRepository
    metricRepo     MetricRepository
    artifactRepo   ArtifactRepository

    // 存储后端
    objectStorage  ObjectStorage

    // 缓存
    cache          Cache

    // 事件发布
    eventPublisher EventPublisher
}

// NewExperimentService 创建服务
func NewExperimentService(
    projectRepo ProjectRepository,
    experimentRepo ExperimentRepository,
    runRepo RunRepository,
    metricRepo MetricRepository,
    artifactRepo ArtifactRepository,
    objectStorage ObjectStorage,
    cache Cache,
    eventPublisher EventPublisher,
) *ExperimentService {
    return &ExperimentService{
        projectRepo:    projectRepo,
        experimentRepo: experimentRepo,
        runRepo:        runRepo,
        metricRepo:     metricRepo,
        artifactRepo:   artifactRepo,
        objectStorage:  objectStorage,
        cache:          cache,
        eventPublisher: eventPublisher,
    }
}

// CreateProject 创建项目
func (s *ExperimentService) CreateProject(ctx context.Context, project *Project) error {
    project.ID = generateID("proj")
    project.CreatedAt = time.Now()
    project.UpdatedAt = time.Now()

    if err := s.projectRepo.Create(ctx, project); err != nil {
        return fmt.Errorf("create project: %w", err)
    }

    s.eventPublisher.Publish(ctx, "project.created", project)
    return nil
}

// CreateExperiment 创建实验
func (s *ExperimentService) CreateExperiment(ctx context.Context, experiment *Experiment) error {
    // 验证项目存在
    if _, err := s.projectRepo.Get(ctx, experiment.ProjectID); err != nil {
        return fmt.Errorf("project not found: %w", err)
    }

    experiment.ID = generateID("exp")
    experiment.Status = ExperimentStatusActive
    experiment.CreatedAt = time.Now()
    experiment.UpdatedAt = time.Now()

    if err := s.experimentRepo.Create(ctx, experiment); err != nil {
        return fmt.Errorf("create experiment: %w", err)
    }

    s.eventPublisher.Publish(ctx, "experiment.created", experiment)
    return nil
}

// CreateRun 创建运行
func (s *ExperimentService) CreateRun(ctx context.Context, run *Run) error {
    // 验证实验存在
    if _, err := s.experimentRepo.Get(ctx, run.ExperimentID); err != nil {
        return fmt.Errorf("experiment not found: %w", err)
    }

    run.ID = generateID("run")
    run.Status = RunStatusRunning
    run.CreatedAt = time.Now()

    if err := s.runRepo.Create(ctx, run); err != nil {
        return fmt.Errorf("create run: %w", err)
    }

    s.eventPublisher.Publish(ctx, "run.started", run)
    return nil
}

// LogParams 记录参数
func (s *ExperimentService) LogParams(ctx context.Context, runID string, params map[string]interface{}) error {
    run, err := s.runRepo.Get(ctx, runID)
    if err != nil {
        return fmt.Errorf("run not found: %w", err)
    }

    // 合并参数
    if run.Parameters.Flat == nil {
        run.Parameters.Flat = make(map[string]interface{})
    }
    for k, v := range params {
        run.Parameters.Flat[k] = v
    }

    return s.runRepo.Update(ctx, run)
}

// LogMetrics 记录指标
func (s *ExperimentService) LogMetrics(ctx context.Context, metrics []*Metric) error {
    if len(metrics) == 0 {
        return nil
    }

    // 批量写入
    if err := s.metricRepo.BatchCreate(ctx, metrics); err != nil {
        return fmt.Errorf("batch create metrics: %w", err)
    }

    // 更新缓存中的最新指标
    runID := metrics[0].RunID
    latestMetrics := make(map[string]*Metric)
    for _, m := range metrics {
        key := fmt.Sprintf("%s:%s", m.Key, m.Context)
        if existing, ok := latestMetrics[key]; !ok || m.Step > existing.Step {
            latestMetrics[key] = m
        }
    }

    for _, m := range latestMetrics {
        cacheKey := fmt.Sprintf("run:%s:metric:%s:%s:latest", runID, m.Key, m.Context)
        s.cache.Set(ctx, cacheKey, m, time.Hour)
    }

    return nil
}

// LogArtifact 记录产物
func (s *ExperimentService) LogArtifact(ctx context.Context, runID string, name string, data []byte, artifactType string) error {
    run, err := s.runRepo.Get(ctx, runID)
    if err != nil {
        return fmt.Errorf("run not found: %w", err)
    }

    // 上传到对象存储
    storagePath := fmt.Sprintf("runs/%s/artifacts/%s", runID, name)
    if err := s.objectStorage.Upload(ctx, storagePath, data); err != nil {
        return fmt.Errorf("upload artifact: %w", err)
    }

    // 计算校验和
    checksum := calculateChecksum(data)

    // 创建产物记录
    artifact := &Artifact{
        ID:        generateID("art"),
        RunID:     run.ID,
        Name:      name,
        Type:      artifactType,
        Path:      storagePath,
        Size:      int64(len(data)),
        Checksum:  checksum,
        CreatedAt: time.Now(),
    }

    if err := s.artifactRepo.Create(ctx, artifact); err != nil {
        return fmt.Errorf("create artifact record: %w", err)
    }

    return nil
}

// EndRun 结束运行
func (s *ExperimentService) EndRun(ctx context.Context, runID string, status RunStatus) error {
    run, err := s.runRepo.Get(ctx, runID)
    if err != nil {
        return fmt.Errorf("run not found: %w", err)
    }

    now := time.Now()
    run.Status = status
    run.EndTime = &now

    if run.StartTime != nil {
        run.Duration = int64(now.Sub(*run.StartTime).Seconds())
    }

    // 计算资源使用
    run.Resources = s.calculateResourceUsage(ctx, run)

    if err := s.runRepo.Update(ctx, run); err != nil {
        return fmt.Errorf("update run: %w", err)
    }

    s.eventPublisher.Publish(ctx, "run.ended", run)
    return nil
}

// GetRunMetrics 获取运行指标
func (s *ExperimentService) GetRunMetrics(
    ctx context.Context,
    runID string,
    keys []string,
    stepRange *StepRange,
) ([]*Metric, error) {
    return s.metricRepo.Query(ctx, MetricQuery{
        RunID:     runID,
        Keys:      keys,
        StepRange: stepRange,
    })
}

// CompareRuns 比较运行
func (s *ExperimentService) CompareRuns(
    ctx context.Context,
    runIDs []string,
    metricKeys []string,
) (*RunComparison, error) {
    comparison := &RunComparison{
        Runs:    make([]*Run, 0, len(runIDs)),
        Metrics: make(map[string]map[string]float64),
        Params:  make(map[string]map[string]interface{}),
    }

    for _, runID := range runIDs {
        run, err := s.runRepo.Get(ctx, runID)
        if err != nil {
            return nil, err
        }
        comparison.Runs = append(comparison.Runs, run)
        comparison.Params[runID] = run.Parameters.Flat

        // 获取最终指标
        for _, key := range metricKeys {
            cacheKey := fmt.Sprintf("run:%s:metric:%s:train:latest", runID, key)
            var metric *Metric
            if err := s.cache.Get(ctx, cacheKey, &metric); err == nil && metric != nil {
                if comparison.Metrics[key] == nil {
                    comparison.Metrics[key] = make(map[string]float64)
                }
                comparison.Metrics[key][runID] = metric.Value
            }
        }
    }

    return comparison, nil
}

// SearchRuns 搜索运行
func (s *ExperimentService) SearchRuns(
    ctx context.Context,
    query RunSearchQuery,
) ([]*Run, int64, error) {
    return s.runRepo.Search(ctx, query)
}

// calculateResourceUsage 计算资源使用
func (s *ExperimentService) calculateResourceUsage(ctx context.Context, run *Run) ResourceUsage {
    usage := ResourceUsage{}

    if run.StartTime == nil || run.EndTime == nil {
        return usage
    }

    duration := run.EndTime.Sub(*run.StartTime)
    hours := duration.Hours()

    // GPU 小时
    usage.GPUHours = float64(run.Environment.GPUCount) * hours

    // CPU 小时
    usage.CPUHours = float64(run.Environment.CPUCount) * hours

    // 内存 GB 小时
    usage.MemoryGBHours = float64(run.Environment.MemoryGB) * hours

    // 预估成本（示例价格）
    gpuPrice := 2.0  // $/GPU/hour
    cpuPrice := 0.05 // $/CPU/hour
    memPrice := 0.01 // $/GB/hour

    usage.Cost = usage.GPUHours*gpuPrice + usage.CPUHours*cpuPrice + usage.MemoryGBHours*memPrice

    return usage
}

// RunSearchQuery 运行搜索查询
type RunSearchQuery struct {
    ExperimentID string
    Status       []RunStatus
    Tags         map[string]string
    ParamFilter  map[string]interface{}
    MetricFilter *MetricFilter
    TimeRange    *TimeRange
    OrderBy      string
    Limit        int
    Offset       int
}

// MetricFilter 指标过滤
type MetricFilter struct {
    Key      string
    Operator string // ">", "<", ">=", "<=", "="
    Value    float64
}

// TimeRange 时间范围
type TimeRange struct {
    Start time.Time
    End   time.Time
}

// StepRange 步数范围
type StepRange struct {
    Start int64
    End   int64
}

// RunComparison 运行比较结果
type RunComparison struct {
    Runs    []*Run
    Metrics map[string]map[string]float64      // metric -> runID -> value
    Params  map[string]map[string]interface{} // runID -> params
}

func generateID(prefix string) string {
    return fmt.Sprintf("%s_%d", prefix, time.Now().UnixNano())
}

func calculateChecksum(data []byte) string {
    // MD5 或 SHA256
    return ""
}