diff --git a/api/handlers/asset.go b/api/handlers/asset.go index baa1cd5..ebf6585 100644 --- a/api/handlers/asset.go +++ b/api/handlers/asset.go @@ -20,7 +20,7 @@ type AssetHandler struct { func NewAssetHandler(db *gorm.DB, cfg *config.Config, log *logger.Logger) *AssetHandler { return &AssetHandler{ - assetService: services.NewAssetService(db, log), + assetService: services.NewAssetService(db, log, cfg), log: log, } } @@ -218,3 +218,27 @@ func (h *AssetHandler) ImportFromVideoGen(c *gin.Context) { response.Success(c, asset) } + +// ExtractFrame 从视频素材中提取帧 +func (h *AssetHandler) ExtractFrame(c *gin.Context) { + assetID, err := strconv.ParseUint(c.Param("id"), 10, 32) + if err != nil { + response.BadRequest(c, "无效的ID") + return + } + + var req services.ExtractFrameRequest + if err := c.ShouldBindJSON(&req); err != nil { + response.BadRequest(c, err.Error()) + return + } + + imageGen, err := h.assetService.ExtractFrameFromAsset(uint(assetID), &req) + if err != nil { + h.log.Errorw("Failed to extract frame", "error", err, "asset_id", assetID) + response.InternalError(c, err.Error()) + return + } + + response.Success(c, imageGen) +} diff --git a/api/routes/routes.go b/api/routes/routes.go index 295e7d4..4b5c04a 100644 --- a/api/routes/routes.go +++ b/api/routes/routes.go @@ -180,6 +180,7 @@ func SetupRouter(cfg *config.Config, db *gorm.DB, log *logger.Logger, localStora assets.DELETE("/:id", assetHandler.DeleteAsset) assets.POST("/import/image/:image_gen_id", assetHandler.ImportFromImageGen) assets.POST("/import/video/:video_gen_id", assetHandler.ImportFromVideoGen) + assets.POST("/:id/extract-frame", assetHandler.ExtractFrame) } storyboards := api.Group("/storyboards") diff --git a/application/services/asset_service.go b/application/services/asset_service.go index c630c5c..e479ea3 100644 --- a/application/services/asset_service.go +++ b/application/services/asset_service.go @@ -2,27 +2,48 @@ package services import ( "fmt" + "os" + "path/filepath" "strconv" "strings" + "time" models "github.com/drama-generator/backend/domain/models" "github.com/drama-generator/backend/infrastructure/external/ffmpeg" + "github.com/drama-generator/backend/infrastructure/storage" + "github.com/drama-generator/backend/pkg/config" "github.com/drama-generator/backend/pkg/logger" "gorm.io/gorm" ) type AssetService struct { - db *gorm.DB - log *logger.Logger - ffmpeg *ffmpeg.FFmpeg + db *gorm.DB + log *logger.Logger + ffmpeg *ffmpeg.FFmpeg + cfg *config.Config + ossStorage *storage.OssStorage } -func NewAssetService(db *gorm.DB, log *logger.Logger) *AssetService { - return &AssetService{ +func NewAssetService(db *gorm.DB, log *logger.Logger, cfg ...*config.Config) *AssetService { + service := &AssetService{ db: db, log: log, ffmpeg: ffmpeg.NewFFmpeg(log), } + if len(cfg) > 0 { + service.cfg = cfg[0] + // 如果配置了 OSS，初始化 OSS 存储 + if cfg[0].Storage.Type == "oss" && storage.IsOssConfigured(&cfg[0].Storage.Oss) { + ossStorage, err := storage.NewOssStorage(&cfg[0].Storage.Oss) + if err != nil { + log.Warnw("Failed to initialize OSS storage, falling back to local", "error", err) + } else { + service.ossStorage = ossStorage + log.Infow("OSS storage initialized", "bucket", cfg[0].Storage.Oss.BucketName) + } + } + } + return service } type CreateAssetRequest struct { @@ -264,6 +285,23 @@ func (s *AssetService) ImportFromVideoGen(videoGenID uint) (*models.Asset, error storyboardNum = &videoGen.Storyboard.StoryboardNumber } + // 如果 duration 为空，尝试使用 FFmpeg 探测 + duration := videoGen.Duration + if duration == nil || *duration == 0 { + s.log.Infow("Duration is empty, probing video duration", "video_gen_id", videoGenID) + probedDuration, err := s.ffmpeg.GetVideoDuration(*videoGen.VideoURL) + if err == nil && probedDuration > 0 { + durationInt := int(probedDuration + 0.5) // 四舍五入 + duration = &durationInt + s.log.Infow("Probed video duration", "video_gen_id", videoGenID, "duration", durationInt) + + // 同时更新 VideoGeneration 表 + s.db.Model(&videoGen).Update("duration", durationInt) + } else { + s.log.Warnw("Failed to probe video duration", "video_gen_id", videoGenID, "error", err) + } + } + asset := &models.Asset{ Name: fmt.Sprintf("Video_%d", videoGen.ID), Type: models.AssetTypeVideo, @@ -273,7 +311,7 @@ func (s *AssetService) ImportFromVideoGen(videoGenID uint) (*models.Asset, error StoryboardID: videoGen.StoryboardID, StoryboardNum: storyboardNum, VideoGenID: &videoGenID, - Duration: videoGen.Duration, + Duration: duration, Width: videoGen.Width, Height: videoGen.Height, } @@ -283,8 +321,125 @@ func (s *AssetService) ImportFromVideoGen(videoGenID uint) (*models.Asset, error } if err := s.db.Create(asset).Error; err != nil { - return nil, fmt.Errorf("failed to create asset: %w", err) + return nil, fmt.Errorf("failed to create asset: %w\n", err) } return asset, nil } + +// ExtractFrameRequest 视频帧提取请求 +type ExtractFrameRequest struct { + Position string `json:"position"` // "first" 或 "last" + StoryboardID uint `json:"storyboard_id"` // 关联的分镜 ID + FrameType string `json:"frame_type"` // image_generations 的帧类型，默认 "first" +} + +// ExtractFrameFromAsset 从视频素材中提取帧并保存到 image_generations 表 +func (s *AssetService) ExtractFrameFromAsset(assetID uint, req *ExtractFrameRequest) (*models.ImageGeneration, error) { + // 获取素材 + var asset models.Asset + if err := s.db.First(&asset, assetID).Error; err != nil { + return nil, fmt.Errorf("asset not found: %w", err) + } + + // 验证是否为视频素材 + if asset.Type != models.AssetTypeVideo { + return nil, fmt.Errorf("asset is not a video") + } + + // 默认值 + position := req.Position + if position == "" { + position = "last" + } + frameType := req.FrameType + if frameType == "" { + frameType = "first" // 提取的尾帧用作下一个分镜的首帧 + } + + // 生成唯一文件名 + timestamp := time.Now().Format("20060102_150405") + fileName := fmt.Sprintf("frame_%s_%d_%s.jpg", position, assetID, timestamp) + + // 创建临时目录 + tmpDir := "./data/tmp" + if err := os.MkdirAll(tmpDir, 0755); err != nil { + return nil, fmt.Errorf("failed to create temp directory: %w", err) + } + outputPath := filepath.Join(tmpDir, fileName) + + // 使用 FFmpeg 提取帧 + _, err := s.ffmpeg.ExtractFrame(asset.URL, outputPath, position) + if err != nil { + return nil, fmt.Errorf("failed to extract frame: %w", err) + } + + var imageURL string + + // 根据存储类型上传 + if s.ossStorage != nil { + // 上传到 OSS + url, err := s.ossStorage.UploadWithFilename(outputPath, "extracted_frames", fileName) + if err != nil { + s.log.Errorw("Failed to upload to OSS, falling back to local", "error", err) + } else { + imageURL = url + s.log.Infow("Frame uploaded to OSS", "url", url) + // 删除临时文件 + os.Remove(outputPath) + } + } + + // 如果 OSS 上传失败或未配置，使用本地存储 + if imageURL == "" { + localPath := "./data/storage" + baseURL := "/static" + if s.cfg != nil && s.cfg.Storage.LocalPath != "" { + localPath = s.cfg.Storage.LocalPath + } + if s.cfg != nil && s.cfg.Storage.BaseURL != "" { + baseURL = s.cfg.Storage.BaseURL + } + + // 移动文件到最终位置 + finalDir := filepath.Join(localPath, "extracted_frames") + if err := os.MkdirAll(finalDir, 0755); err != nil { + return nil, fmt.Errorf("failed to create output directory: %w", err) + } + finalPath := filepath.Join(finalDir, fileName) + if err := os.Rename(outputPath, finalPath); err != nil { + return nil, fmt.Errorf("failed to move file: %w", err) + } + + imageURL = fmt.Sprintf("%s/extracted_frames/%s", baseURL, fileName) + } + + // 获取 DramaID + var dramaID uint + if asset.DramaID != nil { + dramaID = *asset.DramaID + } + + // 创建 image_generation 记录 + imageGen := &models.ImageGeneration{ + DramaID: dramaID, + StoryboardID: &req.StoryboardID, + Prompt: fmt.Sprintf("Extracted %s frame from video asset #%d", position, assetID), + Status: models.ImageStatusCompleted, + ImageURL: &imageURL, + FrameType: &frameType, + } + + if err := s.db.Create(imageGen).Error; err != nil { + return nil, fmt.Errorf("failed to create image generation record: %w", err) + } + + s.log.Infow("Frame extracted and saved", + "asset_id", assetID, + "position", position, + "image_gen_id", imageGen.ID, + "frame_type", frameType, + "image_url", imageURL) + + return imageGen, nil +} diff --git a/configs/config.example.yaml b/configs/config.example.yaml index 080e768..78c8836 100644 --- a/configs/config.example.yaml +++ b/configs/config.example.yaml @@ -19,9 +19,17 @@ database: max_open: 100 storage: - type: "local" + type: "local" # 存储类型：local（本地）或 oss（阿里云OSS） + # 本地存储配置 local_path: "./data/storage" base_url: "http://localhost:5678/static" + # 阿里云 OSS 配置（type 为 oss 时生效） + oss: + endpoint: "oss-cn-hangzhou.aliyuncs.com" # OSS 服务地址 + access_key_id: "" # AccessKey ID + access_key_secret: "" # AccessKey Secret + bucket_name: "" # Bucket 名称 + custom_domain: "" # 自定义域名（可选，用于 CDN 加速） ai: default_text_provider: "openai" diff --git a/data/extracted_frames/9/frame_last_9.jpg b/data/extracted_frames/9/frame_last_9.jpg new file mode 100644 index 0000000..33e5ecb Binary files /dev/null and b/data/extracted_frames/9/frame_last_9.jpg differ diff --git a/go.mod b/go.mod index 07052f7..88140e2 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( ) require ( + github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible // indirect github.com/bytedance/sonic v1.9.1 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/dustin/go-humanize v1.0.1 // indirect @@ -61,6 +62,7 @@ require ( golang.org/x/net v0.38.0 // indirect golang.org/x/sys v0.34.0 // indirect golang.org/x/text v0.26.0 // indirect + golang.org/x/time v0.3.0 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index e4e8281..041592a 100644 --- a/go.sum +++ b/go.sum @@ -38,6 +38,8 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g= +github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= @@ -431,6 +433,8 @@ golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= diff --git a/infrastructure/database/database.go b/infrastructure/database/database.go index 87c9d9a..4d019d1 100644 --- a/infrastructure/database/database.go +++ b/infrastructure/database/database.go @@ -82,6 +82,7 @@ func AutoMigrate(db *gorm.DB) error { &models.ImageGeneration{}, &models.VideoGeneration{}, &models.VideoMerge{}, + &models.FramePrompt{}, // AI配置 &models.AIServiceConfig{}, diff --git a/infrastructure/external/ffmpeg/ffmpeg.go b/infrastructure/external/ffmpeg/ffmpeg.go index 9985296..4eb6442 100644 --- a/infrastructure/external/ffmpeg/ffmpeg.go +++ b/infrastructure/external/ffmpeg/ffmpeg.go @@ -853,3 +853,72 @@ func (f *FFmpeg) generateSilence(outputPath string, duration float64) (string, e f.log.Infow("Silence audio generated successfully", "output", outputPath) return outputPath, nil } + +// ExtractFrame 从视频中提取指定位置的帧 +// position: "first" 提取首帧, "last" 提取尾帧 +// 返回提取的图片文件路径 +func (f *FFmpeg) ExtractFrame(videoURL, outputPath, position string) (string, error) { + f.log.Infow("Extracting frame from video", "url", videoURL, "position", position, "output", outputPath) + + // 下载视频文件 + downloadPath := filepath.Join(f.tempDir, fmt.Sprintf("video_%d.mp4", time.Now().Unix())) + localVideoPath, err := f.downloadVideo(videoURL, downloadPath) + if err != nil { + return "", fmt.Errorf("failed to download video: %w", err) + } + defer os.Remove(localVideoPath) + + // 确保输出目录存在 + outputDir := filepath.Dir(outputPath) + if err := os.MkdirAll(outputDir, 0755); err != nil { + return "", fmt.Errorf("failed to create output directory: %w", err) + } + + var cmd *exec.Cmd + + if position == "last" { + // 提取尾帧：先获取视频时长，然后提取最后一帧 + duration, err := f.GetVideoDuration(localVideoPath) + if err != nil { + return "", fmt.Errorf("failed to get video duration: %w", err) + } + + // 提取最后一帧（时长减去0.1秒的位置） + seekTime := duration - 0.1 + if seekTime < 0 { + seekTime = 0 + } + + cmd = exec.Command("ffmpeg", + "-ss", fmt.Sprintf("%.2f", seekTime), + "-i", localVideoPath, + "-vframes", "1", + "-q:v", "2", + "-y", + outputPath, + ) + } else { + // 默认提取首帧 + cmd = exec.Command("ffmpeg", + "-i", localVideoPath, + "-vframes", "1", + "-q:v", "2", + "-y", + outputPath, + ) + } + + output, err := cmd.CombinedOutput() + if err != nil { + f.log.Errorw("FFmpeg frame extraction failed", "error", err, "output", string(output)) + return "", fmt.Errorf("ffmpeg frame extraction failed: %w, output: %s", err, string(output)) + } + + // 检查输出文件是否存在 + if _, err := os.Stat(outputPath); os.IsNotExist(err) { + return "", fmt.Errorf("frame extraction failed: output file not created") + } + + f.log.Infow("Frame extracted successfully", "output", outputPath, "position", position) + return outputPath, nil +} diff --git a/pkg/config/config.go b/pkg/config/config.go index 081db5d..373a5c4 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -43,9 +43,18 @@ type DatabaseConfig struct { } type StorageConfig struct { - Type string `mapstructure:"type"` // local, minio - LocalPath string `mapstructure:"local_path"` // 本地存储路径 - BaseURL string `mapstructure:"base_url"` // 访问URL前缀 + Type string `mapstructure:"type"` // local, oss + LocalPath string `mapstructure:"local_path"` // 本地存储路径 + BaseURL string `mapstructure:"base_url"` // 访问URL前缀 + Oss OssConfig `mapstructure:"oss"` // 阿里云 OSS 配置 +} + +type OssConfig struct { + Endpoint string `mapstructure:"endpoint"` // OSS 服务地址 + AccessKeyID string `mapstructure:"access_key_id"` // AccessKey ID + AccessKeySecret string `mapstructure:"access_key_secret"` // AccessKey Secret + BucketName string `mapstructure:"bucket_name"` // Bucket 名称 + CustomDomain string `mapstructure:"custom_domain"` // 自定义域名（可选） } type AIConfig struct { diff --git a/server b/server new file mode 100755 index 0000000..312624d Binary files /dev/null and b/server differ diff --git a/web/src/api/asset.ts b/web/src/api/asset.ts index 3e86d87..6278c15 100644 --- a/web/src/api/asset.ts +++ b/web/src/api/asset.ts @@ -1,10 +1,10 @@ import type { - Asset, - AssetCollection, - AssetTag, - CreateAssetRequest, - ListAssetsParams, - UpdateAssetRequest + Asset, + AssetCollection, + AssetTag, + CreateAssetRequest, + ListAssetsParams, + UpdateAssetRequest } from '../types/asset' import request from '../utils/request' @@ -43,5 +43,9 @@ export const assetAPI = { importFromVideo(videoGenId: number) { return request.post(`/assets/import/video/${videoGenId}`) + }, + + extractFrame(assetId: number, data: { position: string; storyboard_id: number; frame_type?: string }) { + return request.post(`/assets/${assetId}/extract-frame`, data) } } diff --git a/web/src/views/drama/ProfessionalEditor.vue b/web/src/views/drama/ProfessionalEditor.vue index 4ccd6bc..31e2ebe 100644 --- a/web/src/views/drama/ProfessionalEditor.vue +++ b/web/src/views/drama/ProfessionalEditor.vue @@ -239,6 +239,24 @@ }} + +
+
从素材提取尾帧（用于画面连续性）
+
+ + + + + 提取尾帧 + +
+
+
@@ -952,6 +970,8 @@ const generatingImage = ref(false) const generatedImages = ref([]) const isSwitchingFrameType = ref(false) // 标志位：是否正在切换帧类型 const loadingImages = ref(false) +const selectedAssetForExtract = ref(null) // 选中的素材用于提取尾帧 +const extractingFrame = ref(false) // 是否正在提取尾帧 let pollingTimer: any = null let pollingFrameType: FrameType | null = null // 记录正在轮询的帧类型 @@ -1603,6 +1623,35 @@ const generateFrameImage = async () => { } } +// 从素材提取尾帧 +const extractLastFrame = async () => { + if (!selectedAssetForExtract.value || !currentStoryboard.value) { + ElMessage.warning('请先选择视频素材') + return + } + + extractingFrame.value = true + try { + const result = await assetAPI.extractFrame(selectedAssetForExtract.value, { + position: 'last', + storyboard_id: currentStoryboard.value.id, + frame_type: 'first' // 提取的尾帧用作当前分镜的首帧 + }) + + ElMessage.success('尾帧提取成功，已添加到首帧图片列表') + + // 刷新图片列表 + await loadStoryboardImages(currentStoryboard.value!.id, 'first') + + // 清空选择 + selectedAssetForExtract.value = null + } catch (error: any) { + ElMessage.error('提取失败: ' + (error.message || '未知错误')) + } finally { + extractingFrame.value = false + } +} + // 获取状态标签类型 const getStatusType = (status: string) => { const statusMap: Record = { @@ -2154,7 +2203,13 @@ const handleTimelineSelect = (sceneId: number) => { } const handleAddStoryboard = async () => { - ElMessage.info('添加分镜功能开发中') + if (!currentStoryboard.value) { + ElMessage.warning('请先选择分镜') + return + } + + // 调用现有的 generateVideo 函数为当前分镜生成视频 + await generateVideo() } const togglePlay = () => {