From 4c68e415ee459c8e90d6c8639e01758e79186d7f Mon Sep 17 00:00:00 2001 From: mangmang <362165265@qq.com> Date: Sun, 19 Jul 2026 23:29:55 +0800 Subject: [PATCH] Fix M4A fallback ingestion --- .../java/com/music/config/AsyncConfig.java | 61 +++ .../com/music/service/AggregatorService.java | 3 +- .../music/service/AudioValidationService.java | 197 +++++++- .../com/music/service/ConvertService.java | 3 +- .../java/com/music/service/DedupService.java | 3 +- .../java/com/music/service/IngestService.java | 449 +++++++++++++----- .../music/service/LibraryMergeService.java | 3 +- .../com/music/service/OrganizeService.java | 3 +- .../com/music/service/ZhConvertService.java | 3 +- .../com/music/config/AsyncConfigTest.java | 72 +++ .../service/AudioValidationServiceTest.java | 110 +++++ .../music/service/IngestServiceE2ETest.java | 443 +++++++++++++++++ 12 files changed, 1224 insertions(+), 126 deletions(-) create mode 100644 backend/src/main/java/com/music/config/AsyncConfig.java create mode 100644 backend/src/test/java/com/music/config/AsyncConfigTest.java diff --git a/backend/src/main/java/com/music/config/AsyncConfig.java b/backend/src/main/java/com/music/config/AsyncConfig.java new file mode 100644 index 0000000..52738a9 --- /dev/null +++ b/backend/src/main/java/com/music/config/AsyncConfig.java @@ -0,0 +1,61 @@ +package com.music.config; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.aop.interceptor.AsyncUncaughtExceptionHandler; +import org.springframework.aop.interceptor.SimpleAsyncUncaughtExceptionHandler; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.annotation.AsyncConfigurer; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; + +import java.util.concurrent.Executor; + +/** + * 应用级异步执行配置。 + * + *

显式声明名为 {@value #TASK_EXECUTOR_BEAN_NAME} 的 {@link ThreadPoolTaskExecutor}， + * 并作为 {@link AsyncConfigurer#getAsyncExecutor()} 的默认 {@code @Async} 执行器。 + * 这样可消除多个候选 {@code TaskExecutor}（如 WebSocket message broker 通道执行器、 + * Spring Boot 自动配置的 {@code applicationTaskExecutor}）导致 {@code @Async} 无法选择 + * 而记录的警告，同时保留任务终态与异常可观测性。

+ * + *

各 {@code @Async} 方法显式引用 {@value #TASK_EXECUTOR_BEAN_NAME}， + * 避免依赖隐式的按类型/按名称解析。

+ */ +@Configuration +public class AsyncConfig implements AsyncConfigurer { + + private static final Logger log = LoggerFactory.getLogger(AsyncConfig.class); + + /** 应用级异步执行器 bean 名称（供 {@code @Async("...")} 引用） */ + public static final String TASK_EXECUTOR_BEAN_NAME = "mangtoolTaskExecutor"; + + @Bean(name = TASK_EXECUTOR_BEAN_NAME) + public ThreadPoolTaskExecutor mangtoolTaskExecutor() { + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + executor.setCorePoolSize(2); + executor.setMaxPoolSize(4); + executor.setQueueCapacity(50); + executor.setThreadNamePrefix("mangtool-async-"); + // 关闭时等待正在执行的任务完成，保证任务终态可被观测 + executor.setWaitForTasksToCompleteOnShutdown(true); + executor.setAwaitTerminationSeconds(30); + executor.initialize(); + return executor; + } + + @Override + public Executor getAsyncExecutor() { + return mangtoolTaskExecutor(); + } + + @Override + public AsyncUncaughtExceptionHandler getAsyncUncaughtExceptionHandler() { + return (throwable, method, params) -> { + log.error("异步任务 {} 执行抛出未捕获异常", method.getName(), throwable); + new SimpleAsyncUncaughtExceptionHandler() + .handleUncaughtException(throwable, method, params); + }; + } +} diff --git a/backend/src/main/java/com/music/service/AggregatorService.java b/backend/src/main/java/com/music/service/AggregatorService.java index 90d6f90..fa0cf4c 100644 --- a/backend/src/main/java/com/music/service/AggregatorService.java +++ b/backend/src/main/java/com/music/service/AggregatorService.java @@ -3,6 +3,7 @@ package com.music.service; import com.music.common.FileTransferUtils; import com.music.dto.ProgressMessage; import org.springframework.messaging.simp.SimpMessagingTemplate; +import com.music.config.AsyncConfig; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; @@ -30,7 +31,7 @@ public class AggregatorService { /** * 异步执行音频文件汇聚任务 */ - @Async + @Async(AsyncConfig.TASK_EXECUTOR_BEAN_NAME) public void aggregate(String taskId, String srcDir, String dstDir, String mode) { Path sourcePath = Paths.get(srcDir); Path targetPath = Paths.get(dstDir); diff --git a/backend/src/main/java/com/music/service/AudioValidationService.java b/backend/src/main/java/com/music/service/AudioValidationService.java index 75bcc34..699b6bb 100644 --- a/backend/src/main/java/com/music/service/AudioValidationService.java +++ b/backend/src/main/java/com/music/service/AudioValidationService.java @@ -1,5 +1,7 @@ package com.music.service; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; @@ -10,7 +12,10 @@ import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.concurrent.TimeUnit; /** @@ -51,11 +56,16 @@ public class AudioValidationService { static final int DEFAULT_DECODE_TIMEOUT_SECONDS = 300; /** 保留诊断信息最大长度 */ static final int MAX_DIAGNOSTIC_LENGTH = 512; + /** FFprobe 元数据 JSON 输出最大捕获字节数（远大于诊断上限，容纳完整机器可读输出） */ + static final int MAX_METADATA_JSON_LENGTH = 1024 * 1024; /** 流读取缓冲区大小 */ private static final int BUFFER_SIZE = 4096; /** 回收子进程 / 合并 drainer 线程的最长等待秒数 */ private static final int CLEANUP_WAIT_SECONDS = 5; + /** 解析 FFprobe 结构化 JSON 输出的线程安全共享实例 */ + private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); + // ========== 嵌套结果类 ========== /** @@ -90,6 +100,59 @@ public class AudioValidationService { } } + // ========== 嵌套元数据结果类 ========== + + /** + * FFprobe 后备元数据读取结果。 + *

{@link #isAvailable()} 表示 FFprobe 成功识别音频流并返回了 format tag 元数据； + * 仅在此情况下 {@link #getTags()} 才有意义。字段值均取自 FFprobe 结构化 JSON 输出， + * 不做任何文件名猜测或人类可读文本解析。

+ */ + public static class ProbeMetadata { + private final boolean available; + private final String diagnostic; + private final Map tags; + + ProbeMetadata(boolean available, String diagnostic, Map tags) { + this.available = available; + this.diagnostic = diagnostic != null ? diagnostic : ""; + this.tags = tags != null ? tags : new LinkedHashMap<>(); + } + + public boolean isAvailable() { + return available; + } + + public String getDiagnostic() { + return diagnostic; + } + + /** + * 返回小写键 → 值的标签映射（键取自 FFprobe format.tags）。 + */ + public Map getTags() { + return tags; + } + + /** + * 按候选键列表返回第一个非空标签值（键不区分大小写）。 + */ + public String getFirst(String... keys) { + for (String key : keys) { + if (key == null) continue; + String v = tags.get(key.toLowerCase()); + if (v != null && !v.trim().isEmpty()) { + return v.trim(); + } + } + return ""; + } + + static ProbeMetadata unavailable(String diagnostic) { + return new ProbeMetadata(false, diagnostic, null); + } + } + // ========== 嵌套进程运行结果 ========== /** @@ -195,6 +258,116 @@ public class AudioValidationService { } } + // ========== 元数据后备读取（FFprobe 结构化 JSON） ========== + + /** + * 返回 FFprobe 元数据读取命令参数列表（结构化 JSON 输出，用于测试观察）。 + *

使用 {@code -print_format json} + {@code -show_format} + {@code -show_streams}， + * 仅解析机器可读 JSON，绝不解析人类可读文本或文件名。

+ */ + List getMetadataCommandArgs(Path audioFile) { + List cmd = new ArrayList<>(); + cmd.add(getFfprobeCommand()); + cmd.add("-v"); + cmd.add("error"); + cmd.add("-print_format"); + cmd.add("json"); + cmd.add("-show_format"); + cmd.add("-show_streams"); + cmd.add(audioFile.toAbsolutePath().toString()); + return cmd; + } + + /** + * 使用 FFprobe 结构化 JSON 输出读取音频元数据（jaudiotagger 无法解析时的后备路径）。 + *

要求文件至少含一个可识别的音频流，且能返回 {@code format.tags} 或 + * {@code streams[].tags}。标签键统一小写化后存入 {@link ProbeMetadata#getTags()}。 + * 仅解析 JSON，绝不进行文件名猜测或人类可读文本解析。

+ * + * @param audioFile 待读取的音频文件路径 + * @return 后备元数据结果；{@link ProbeMetadata#isAvailable()} 为 false 表示 + * FFprobe 无法识别音频或读取失败（调用方应归类为 Unreadable） + */ + public ProbeMetadata readMetadata(Path audioFile) { + int timeout = resolveProbeTimeout(); + ProcessResult result; + try { + result = runProcess(getMetadataCommandArgs(audioFile), timeout, MAX_METADATA_JSON_LENGTH); + } catch (IOException e) { + log.warn("FFprobe 元数据读取执行失败: {} - {}", audioFile, e.getMessage()); + return ProbeMetadata.unavailable("FFprobe 执行失败: " + e.getMessage()); + } + + if (result.timedOut) { + log.warn("FFprobe 元数据读取超时: {} ({}s)", audioFile, timeout); + return ProbeMetadata.unavailable("元数据读取超时"); + } + if (result.exitCode != 0) { + String diag = capDiagnostic(result.output); + log.warn("FFprobe 元数据读取失败: {} (exit={}) diag={}", audioFile, result.exitCode, diag); + return ProbeMetadata.unavailable("FFprobe 读取失败: " + diag); + } + + try { + JsonNode root = JSON_MAPPER.readTree(result.output); + if (!hasAudioStream(root)) { + log.warn("FFprobe 后备未发现音频流: {}", audioFile); + return ProbeMetadata.unavailable("文件中未发现可识别的音频流"); + } + + Map tags = new LinkedHashMap<>(); + // 优先 format.tags + collectTags(root.path("format").path("tags"), tags); + // 再补充第一个音频流的 tags（不覆盖已有 format 值） + JsonNode streams = root.path("streams"); + if (streams.isArray()) { + for (JsonNode stream : streams) { + if ("audio".equals(stream.path("codec_type").asText())) { + collectTags(stream.path("tags"), tags); + break; + } + } + } + return new ProbeMetadata(true, "", tags); + } catch (IOException e) { + log.warn("FFprobe JSON 解析失败: {} - {}", audioFile, e.getMessage()); + return ProbeMetadata.unavailable("FFprobe JSON 解析失败: " + e.getMessage()); + } + } + + /** + * 判断 FFprobe JSON 中是否存在音频流。 + */ + private static boolean hasAudioStream(JsonNode root) { + JsonNode streams = root.path("streams"); + if (!streams.isArray()) return false; + for (JsonNode stream : streams) { + if ("audio".equals(stream.path("codec_type").asText())) { + return true; + } + } + return false; + } + + /** + * 将 JSON tags 对象的字段收集到映射中（键小写化，已存在的键不覆盖）。 + */ + private static void collectTags(JsonNode tagsNode, Map target) { + if (tagsNode == null || !tagsNode.isObject()) return; + Iterator> fields = tagsNode.fields(); + while (fields.hasNext()) { + Map.Entry entry = fields.next(); + String key = entry.getKey(); + if (key == null) continue; + String lower = key.toLowerCase(); + if (target.containsKey(lower)) continue; + String value = entry.getValue().asText(""); + if (value != null && !value.trim().isEmpty()) { + target.put(lower, value.trim()); + } + } + } + // ========== 解码（FFmpeg，严格模式） ========== /** @@ -261,10 +434,20 @@ public class AudioValidationService { */ private static ProcessResult runProcess(List command, int timeoutSeconds) throws IOException { + return runProcess(command, timeoutSeconds, MAX_DIAGNOSTIC_LENGTH); + } + + /** + * 运行一个外部进程，并发读取合并后的标准输出，最多保留 {@code maxCapture} 字节。 + *

诊断类调用使用 {@link #MAX_DIAGNOSTIC_LENGTH}；需要完整机器可读输出 + * （如 FFprobe JSON）时传入更大的上限。超出上限的数据仍被读取以排空管道。

+ */ + private static ProcessResult runProcess(List command, int timeoutSeconds, int maxCapture) + throws IOException { ProcessBuilder pb = new ProcessBuilder(command); pb.redirectErrorStream(true); Process p = pb.start(); - DrainResult drain = startConcurrentDrainer(p); + DrainResult drain = startConcurrentDrainer(p, maxCapture); boolean interrupted = false; boolean finished; @@ -311,19 +494,19 @@ public class AudioValidationService { } /** - * 启动守护线程并发读取进程标准输出（合并后），保留前 {@link #MAX_DIAGNOSTIC_LENGTH} - * 字节为诊断信息，继续读取剩余数据以排空管道，防止子进程因管道写满而阻塞或收到 SIGPIPE。 + * 启动守护线程并发读取进程标准输出（合并后），保留前 {@code maxCapture} + * 字节，继续读取剩余数据以排空管道，防止子进程因管道写满而阻塞或收到 SIGPIPE。 */ - private static DrainResult startConcurrentDrainer(Process p) { - ByteArrayOutputStream buffer = new ByteArrayOutputStream(MAX_DIAGNOSTIC_LENGTH); + private static DrainResult startConcurrentDrainer(Process p, int maxCapture) { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(Math.min(maxCapture, BUFFER_SIZE)); Thread t = new Thread(() -> { try { byte[] buf = new byte[BUFFER_SIZE]; int n; InputStream is = p.getInputStream(); while ((n = is.read(buf)) != -1) { - if (buffer.size() < MAX_DIAGNOSTIC_LENGTH) { - int remaining = MAX_DIAGNOSTIC_LENGTH - buffer.size(); + if (buffer.size() < maxCapture) { + int remaining = maxCapture - buffer.size(); buffer.write(buf, 0, Math.min(n, remaining)); } // buffer 已满后继续读取以排空管道，丢弃多余数据 diff --git a/backend/src/main/java/com/music/service/ConvertService.java b/backend/src/main/java/com/music/service/ConvertService.java index 38da759..d9b7b5f 100644 --- a/backend/src/main/java/com/music/service/ConvertService.java +++ b/backend/src/main/java/com/music/service/ConvertService.java @@ -4,6 +4,7 @@ import com.music.dto.ProgressMessage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.messaging.simp.SimpMessagingTemplate; +import com.music.config.AsyncConfig; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; @@ -46,7 +47,7 @@ public class ConvertService { this.progressStore = progressStore; } - @Async + @Async(AsyncConfig.TASK_EXECUTOR_BEAN_NAME) public void convert(String taskId, String srcDir, String dstDir, String mode) { Path srcPath = Paths.get(srcDir); Path dstPath = Paths.get(dstDir); diff --git a/backend/src/main/java/com/music/service/DedupService.java b/backend/src/main/java/com/music/service/DedupService.java index 562743f..8d24be3 100644 --- a/backend/src/main/java/com/music/service/DedupService.java +++ b/backend/src/main/java/com/music/service/DedupService.java @@ -9,6 +9,7 @@ import org.jaudiotagger.tag.Tag; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.messaging.simp.SimpMessagingTemplate; +import com.music.config.AsyncConfig; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; @@ -52,7 +53,7 @@ public class DedupService { /** * 异步执行去重任务 */ - @Async + @Async(AsyncConfig.TASK_EXECUTOR_BEAN_NAME) public void dedup(String taskId, String libraryDir, String trashDir, diff --git a/backend/src/main/java/com/music/service/IngestService.java b/backend/src/main/java/com/music/service/IngestService.java index 721d87c..a21747e 100644 --- a/backend/src/main/java/com/music/service/IngestService.java +++ b/backend/src/main/java/com/music/service/IngestService.java @@ -12,6 +12,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.messaging.simp.SimpMessagingTemplate; +import com.music.config.AsyncConfig; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; @@ -77,6 +78,10 @@ public class IngestService { private static final int FFMPEG_COMPRESSION_LEVEL = 5; private static final int FFMPEG_CHECK_TIMEOUT_SECONDS = 10; private static final int FFMPEG_CONVERT_TIMEOUT_SECONDS = 600; + /** FFmpeg 合并输出保留的诊断字节上限（超出继续读取以排空管道） */ + private static final int FFMPEG_DIAGNOSTIC_CAP = 4096; + /** 强制终止后回收子进程 / 合并 drainer 的最长等待秒数 */ + private static final int FFMPEG_CLEANUP_WAIT_SECONDS = 5; private static final String FFMPEG_BIN_PROPERTY = "mangtool.ffmpeg.bin"; private static final String REPORT_SUBDIR = "Reports"; @@ -122,7 +127,7 @@ public class IngestService { * @param taskId 唯一任务标识 * @param runningLock 由控制器传入的 AtomicBoolean，任务结束时置为 false */ - @Async + @Async(AsyncConfig.TASK_EXECUTOR_BEAN_NAME) public void ingest(String taskId, AtomicBoolean runningLock) { try { String inputDir = configService.getInputDir(); @@ -306,80 +311,103 @@ public class IngestService { String fileName = srcFile.getFileName().toString(); String baseName = getBaseName(fileName); - // 1. 读取元数据 - AudioFile audioFile; - Tag tag; + // 1. 读取元数据：优先 jaudiotagger；受支持的 M4A/MP4 无法解析时回退 FFprobe 结构化输出 + AudioFile audioFile = null; + Tag tag = null; + boolean usedFallback = false; + + String title; + String artist; + String album; + String albumArtist; + String yearStr; + String trackRaw; + String discRaw; + + boolean jaudioOk = false; + String jaudioError = null; try { audioFile = AudioFileIO.read(srcFile.toFile()); tag = audioFile.getTag(); - if (tag == null) { - unreadable.incrementAndGet(); - moveToRejected(srcFile, rejectedPath, "Unreadable", fileName); - return "rejected:unreadable"; + jaudioOk = (tag != null); + if (!jaudioOk) { + jaudioError = "标签为空"; } } catch (Exception e) { - unreadable.incrementAndGet(); - moveToRejected(srcFile, rejectedPath, "Unreadable", fileName); - log.warn("无法读取文件元数据: {} - {}", fileName, e.getMessage()); - return "rejected:unreadable"; + jaudioError = e.getMessage(); } - // 2. 读取关键字段 - String title = trim(tag.getFirst(FieldKey.TITLE)); - String artist = trim(tag.getFirst(FieldKey.ARTIST)); - String album = trim(tag.getFirst(FieldKey.ALBUM)); + if (jaudioOk) { + title = trim(tag.getFirst(FieldKey.TITLE)); + artist = trim(tag.getFirst(FieldKey.ARTIST)); + album = trim(tag.getFirst(FieldKey.ALBUM)); + albumArtist = trim(tag.getFirst(FieldKey.ALBUM_ARTIST)); + yearStr = safeGetFirst(tag, FieldKey.YEAR); + trackRaw = safeGetFirst(tag, FieldKey.TRACK); + discRaw = safeGetFirst(tag, FieldKey.DISC_NO); + } else { + // jaudiotagger 无法解析：仅对受支持的 M4A/MP4 且启用了 FFprobe 校验服务时回退 + if (audioValidationService == null || !isFallbackEligible(srcFile)) { + unreadable.incrementAndGet(); + moveToRejected(srcFile, rejectedPath, "Unreadable", fileName); + log.warn("无法读取文件元数据: {} - {}", fileName, jaudioError); + return "rejected:unreadable"; + } + AudioValidationService.ProbeMetadata meta = audioValidationService.readMetadata(srcFile); + if (!meta.isAvailable()) { + unreadable.incrementAndGet(); + moveToRejected(srcFile, rejectedPath, "Unreadable", fileName); + log.warn("FFprobe 后备元数据读取失败: {} - {}", fileName, meta.getDiagnostic()); + return "rejected:unreadable"; + } + usedFallback = true; + title = meta.getFirst("title"); + artist = meta.getFirst("artist"); + album = meta.getFirst("album"); + albumArtist = meta.getFirst("album_artist", "albumartist"); + yearStr = meta.getFirst("date", "year"); + trackRaw = meta.getFirst("track"); + discRaw = meta.getFirst("disc", "discnumber"); + log.info("jaudiotagger 无法解析，改用 FFprobe 后备读取元数据: {}", fileName); + } - // 3. 校验 Title/Artist/Album 非空 + // 2. 校验 Title/Artist/Album 非空（任一为空 → MissingMetadata） if (title.isEmpty() || artist.isEmpty() || album.isEmpty()) { missingMeta.incrementAndGet(); moveToRejected(srcFile, rejectedPath, "MissingMetadata", fileName); return "rejected:missing-metadata"; } - // 4. 繁简转换文本标签 - boolean tagsModified = false; - for (FieldKey key : TEXT_FIELDS) { - String value = trim(tag.getFirst(key)); - if (!value.isEmpty()) { - String converted = traditionalFilterService.toSimplified(value); - if (!converted.equals(value)) { - tag.setField(key, converted); - tagsModified = true; + // 3. 繁简转换文本标签并写回源文件（仅 jaudiotagger 路径； + // FFprobe 后备路径稍后用 FFmpeg 无损 remux 写入简体标签） + if (jaudioOk) { + boolean tagsModified = false; + for (FieldKey key : TEXT_FIELDS) { + String value = trim(tag.getFirst(key)); + if (!value.isEmpty()) { + String converted = traditionalFilterService.toSimplified(value); + if (!converted.equals(value)) { + tag.setField(key, converted); + tagsModified = true; + } + } + } + if (tagsModified) { + try { + audioFile.commit(); + } catch (Exception e) { + otherRejected.incrementAndGet(); + moveToRejected(srcFile, rejectedPath, "Other", fileName); + log.warn("标签写入失败且无法恢复: {} - {}", fileName, e.getMessage()); + return "rejected:tag-write-failed"; } } } - // 对 albumArtist 也进行繁简转换 - String albumArtist = trim(tag.getFirst(FieldKey.ALBUM_ARTIST)); - if (!albumArtist.isEmpty()) { - String converted = traditionalFilterService.toSimplified(albumArtist); - if (!converted.equals(albumArtist)) { - tag.setField(FieldKey.ALBUM_ARTIST, converted); - tagsModified = true; - } - } - - // 5. 将简化后的标签写回源文件 - if (tagsModified) { - try { - audioFile.commit(); - } catch (Exception e) { - otherRejected.incrementAndGet(); - moveToRejected(srcFile, rejectedPath, "Other", fileName); - log.warn("标签写入失败且无法恢复: {} - {}", fileName, e.getMessage()); - return "rejected:tag-write-failed"; - } - } - - // 6. 提取其他可选字段（WavTag 等实现可能不支持 YEAR/TRACK/DISC_NO，使用安全读取） - String yearStr = safeGetFirst(tag, FieldKey.YEAR); - String trackRaw = safeGetFirst(tag, FieldKey.TRACK); - String discRaw = safeGetFirst(tag, FieldKey.DISC_NO); - int trackNum = parseInt(trackRaw, 0); int discNum = parseInt(discRaw, 0); - // 7. 计算 MD5 用于去重兜底 + // 4. 计算 MD5 用于去重兜底 String fileMd5 = computeMd5(srcFile); // 8. 构建归一化身份标识 @@ -399,7 +427,7 @@ public class IngestService { return "rejected:duplicate"; } - // 10. 格式转换（如需要） + // 5. 格式转换 / 后备 remux（如需要） Path effectiveFile = srcFile; boolean needsConversion = isLosslessFormat(srcFile); if (needsConversion) { @@ -412,13 +440,33 @@ public class IngestService { log.warn("转码失败: {} - {}", fileName, e.getMessage()); return "rejected:conversion-failed"; } + } else if (usedFallback) { + // jaudiotagger 无法写回此容器的标签：用 FFmpeg 无损 remux（-c copy） + // 生成保留音频质量与原有元数据的 Navidrome 兼容输出，并写入简体 Title/Artist/Album。 + String simpTitle = traditionalFilterService.toSimplified(title); + String simpArtist = traditionalFilterService.toSimplified(artist); + String simpAlbum = traditionalFilterService.toSimplified(album); + String simpAlbumArtist = albumArtist.isEmpty() + ? "" : traditionalFilterService.toSimplified(albumArtist); + try { + Path remuxed = remuxWithSimplifiedTags(srcFile, simpTitle, simpArtist, + simpAlbum, simpAlbumArtist); + effectiveFile = remuxed; + } catch (Exception e) { + convFailed.incrementAndGet(); + moveToRejected(srcFile, rejectedPath, "ConversionFailed", fileName); + log.warn("后备 remux 失败: {} - {}", fileName, e.getMessage()); + return "rejected:conversion-failed"; + } } - // 11. 音频完整性验证：确保有效文件可被 FFprobe 识别且 FFmpeg 可完整解码 + boolean derivedFile = !effectiveFile.equals(srcFile); + + // 6. 音频完整性验证：确保有效文件可被 FFprobe 识别且 FFmpeg 可完整解码 if (audioValidationService != null) { ValidationResult vr = audioValidationService.validate(effectiveFile); if (!vr.isValid()) { - if (needsConversion && !effectiveFile.equals(srcFile)) { + if (derivedFile) { deleteIfExists(effectiveFile); } unreadable.incrementAndGet(); @@ -428,50 +476,84 @@ public class IngestService { } } - // 12. 重新打开有效文件，重新检查 Title/Artist/Album - // （确保转换过程或标签持久化未导致 Navidrome 必需的元数据丢失） + // 7. 重新检查有效文件的 Title/Artist/Album（确保转换/remux/标签持久化未丢失必需元数据） if (audioValidationService != null) { - try { - AudioFile validatedAudio = AudioFileIO.read(effectiveFile.toFile()); - Tag validatedTag = validatedAudio.getTag(); - String vTitle = trim(validatedTag != null ? validatedTag.getFirst(FieldKey.TITLE) : ""); - String vArtist = trim(validatedTag != null ? validatedTag.getFirst(FieldKey.ARTIST) : ""); - String vAlbum = trim(validatedTag != null ? validatedTag.getFirst(FieldKey.ALBUM) : ""); + if (usedFallback) { + // 后备路径：jaudiotagger 仍无法解析该容器，改用 FFprobe 再次读取校验 + AudioValidationService.ProbeMetadata reMeta = + audioValidationService.readMetadata(effectiveFile); + String vTitle = reMeta.isAvailable() ? reMeta.getFirst("title") : ""; + String vArtist = reMeta.isAvailable() ? reMeta.getFirst("artist") : ""; + String vAlbum = reMeta.isAvailable() ? reMeta.getFirst("album") : ""; if (vTitle.isEmpty() || vArtist.isEmpty() || vAlbum.isEmpty()) { - if (needsConversion && !effectiveFile.equals(srcFile)) { + if (derivedFile) { deleteIfExists(effectiveFile); } unreadable.incrementAndGet(); moveToRejected(srcFile, rejectedPath, "Unreadable", fileName); - log.warn("完整性验证后元数据缺失: {}（Title='{}' Artist='{}' Album='{}'）", + log.warn("后备 remux 后元数据缺失: {}（Title='{}' Artist='{}' Album='{}'）", fileName, vTitle, vArtist, vAlbum); return "rejected:unreadable"; } - // 使用有效文件的标签更新后续处理引用 - tag = validatedTag; title = vTitle; artist = vArtist; album = vAlbum; - albumArtist = trim(validatedTag.getFirst(FieldKey.ALBUM_ARTIST)); - yearStr = safeGetFirst(validatedTag, FieldKey.YEAR); - trackRaw = safeGetFirst(validatedTag, FieldKey.TRACK); - discRaw = safeGetFirst(validatedTag, FieldKey.DISC_NO); + albumArtist = reMeta.getFirst("album_artist", "albumartist"); + yearStr = reMeta.getFirst("date", "year"); + trackRaw = reMeta.getFirst("track"); + discRaw = reMeta.getFirst("disc", "discnumber"); trackNum = parseInt(trackRaw, 0); discNum = parseInt(discRaw, 0); - } catch (Exception e) { - if (needsConversion && !effectiveFile.equals(srcFile)) { - deleteIfExists(effectiveFile); + } else { + try { + AudioFile validatedAudio = AudioFileIO.read(effectiveFile.toFile()); + Tag validatedTag = validatedAudio.getTag(); + String vTitle = trim(validatedTag != null ? validatedTag.getFirst(FieldKey.TITLE) : ""); + String vArtist = trim(validatedTag != null ? validatedTag.getFirst(FieldKey.ARTIST) : ""); + String vAlbum = trim(validatedTag != null ? validatedTag.getFirst(FieldKey.ALBUM) : ""); + if (vTitle.isEmpty() || vArtist.isEmpty() || vAlbum.isEmpty()) { + if (derivedFile) { + deleteIfExists(effectiveFile); + } + unreadable.incrementAndGet(); + moveToRejected(srcFile, rejectedPath, "Unreadable", fileName); + log.warn("完整性验证后元数据缺失: {}（Title='{}' Artist='{}' Album='{}'）", + fileName, vTitle, vArtist, vAlbum); + return "rejected:unreadable"; + } + // 使用有效文件的标签更新后续处理引用 + tag = validatedTag; + title = vTitle; + artist = vArtist; + album = vAlbum; + albumArtist = trim(validatedTag.getFirst(FieldKey.ALBUM_ARTIST)); + yearStr = safeGetFirst(validatedTag, FieldKey.YEAR); + trackRaw = safeGetFirst(validatedTag, FieldKey.TRACK); + discRaw = safeGetFirst(validatedTag, FieldKey.DISC_NO); + trackNum = parseInt(trackRaw, 0); + discNum = parseInt(discRaw, 0); + } catch (Exception e) { + if (derivedFile) { + deleteIfExists(effectiveFile); + } + unreadable.incrementAndGet(); + moveToRejected(srcFile, rejectedPath, "Unreadable", fileName); + log.warn("完整性验证后重新读取元数据失败: {} - {}", fileName, e.getMessage()); + return "rejected:unreadable"; } - unreadable.incrementAndGet(); - moveToRejected(srcFile, rejectedPath, "Unreadable", fileName); - log.warn("完整性验证后重新读取元数据失败: {} - {}", fileName, e.getMessage()); - return "rejected:unreadable"; } } - // 13. 确定最终格式与文件名 - String ext = needsConversion ? "flac" : getExtension(fileName); - if (ext == null) ext = "flac"; + // 确定最终格式与文件名（后备 remux 统一输出 m4a 容器） + String ext; + if (needsConversion) { + ext = "flac"; + } else if (usedFallback) { + ext = "m4a"; + } else { + ext = getExtension(fileName); + if (ext == null) ext = "flac"; + } // 12. 构建目标路径 String effectiveArtist = !albumArtist.isEmpty() @@ -491,40 +573,39 @@ public class IngestService { String destFileName = trackStr + " - " + safeTitle + "." + ext; Path targetFile = resolveUniqueFile(targetDir, destFileName); - // 13. 移动/复制到目标位置 + // 8. 移动/复制到目标位置 try { FileTransferUtils.moveWithFallback(effectiveFile, targetFile); } catch (IOException e) { otherRejected.incrementAndGet(); - // 如果经过转换，删除临时 FLAC 并隔离源文件 - if (needsConversion && !effectiveFile.equals(srcFile)) { + // 如果经过转换/remux，删除派生文件并隔离源文件 + if (derivedFile) { deleteIfExists(effectiveFile); - moveToRejected(srcFile, rejectedPath, "Other", fileName); - } else { - moveToRejected(srcFile, rejectedPath, "Other", fileName); } + moveToRejected(srcFile, rejectedPath, "Other", fileName); log.warn("移动文件到 Library 失败: {} - {}", fileName, e.getMessage()); return "rejected:move-failed"; } - // 14. 如果进行了格式转换且成功，删除原文件 - if (needsConversion && !effectiveFile.equals(srcFile)) { + // 9. 如果生成了派生文件（转换或后备 remux）且成功，删除原文件 + if (derivedFile) { deleteIfExists(srcFile); } - // 15. 更新身份标识集合 + // 10. 更新身份标识集合 libraryIdentities.add(identity); batchIdentities.add(identity); ingested.incrementAndGet(); - // 16. 处理关联 LRC 文件（可选） + // 11. 处理关联 LRC 文件（可选） handleAssociatedLrc(srcFile, targetDir, baseName, destFileName); - // 17. 提取嵌入式歌词（可选）写入 LRC - extractEmbeddedLyrics(tag, targetDir, trackStr, safeTitle, title, effectiveArtist); - - // 18. 提取封面（可选） - extractCover(tag, targetDir); + // 12. 提取嵌入式歌词与封面（可选，仅 jaudiotagger 路径有 tag 对象； + // 后备 remux 已保留原始嵌入式封面/歌词于输出容器中） + if (tag != null) { + extractEmbeddedLyrics(tag, targetDir, trackStr, safeTitle, title, effectiveArtist); + extractCover(tag, targetDir); + } return "ingested"; } @@ -732,30 +813,19 @@ public class IngestService { Path output = outputDir.resolve(baseName + ".flac"); output = resolveUniqueFile(outputDir, baseName + ".flac"); - ProcessBuilder pb = new ProcessBuilder( + List cmd = new ArrayList<>(Arrays.asList( getFfmpegCommand(), "-y", "-i", input.toAbsolutePath().toString(), "-compression_level", String.valueOf(FFMPEG_COMPRESSION_LEVEL), output.toAbsolutePath().toString() - ); - pb.redirectErrorStream(true); - Process p = pb.start(); - boolean finished = p.waitFor(FFMPEG_CONVERT_TIMEOUT_SECONDS, TimeUnit.SECONDS); - if (!finished) { - p.destroyForcibly(); - deleteIfExists(output); - throw new RuntimeException("ffmpeg 转码超时（" + FFMPEG_CONVERT_TIMEOUT_SECONDS + "s）"); - } - int exit = p.exitValue(); - if (exit != 0) { - deleteIfExists(output); - throw new RuntimeException("ffmpeg 退出码: " + exit); - } + )); + runFfmpegDrained(cmd, FFMPEG_CONVERT_TIMEOUT_SECONDS, output, "转码"); return output; } private void deleteIfExists(Path path) { + if (path == null) return; try { Files.deleteIfExists(path); } catch (IOException e) { @@ -763,6 +833,159 @@ public class IngestService { } } + /** FFprobe 后备读取仅对这些容器格式启用（jaudiotagger 常拒绝的合法 AAC/M4A/MP4） */ + private static final Set FALLBACK_ELIGIBLE_EXTENSIONS = new HashSet<>(Arrays.asList( + "m4a", "mp4", "aac", "m4b", "m4p" + )); + + /** + * 判断文件是否适用 FFprobe 元数据后备读取路径。 + * 仅限受支持的 M4A/MP4/AAC 容器，避免对其他格式做无意义的后备。 + */ + private boolean isFallbackEligible(Path file) { + String ext = getExtension(file.getFileName().toString()); + return ext != null && FALLBACK_ELIGIBLE_EXTENSIONS.contains(ext); + } + + /** + * 使用 FFmpeg 无损 remux（{@code -c copy}）生成保留音频质量与原有元数据的 + * Navidrome 兼容 M4A 输出，并写入简体 Title/Artist/Album（及可选 Album Artist）。 + *

用于 jaudiotagger 无法安全写回该容器标签、且不能修改原文件的场景， + * 避免因写回失败而误拒绝合法音频。不伪造缺失的必要元数据。

+ *

输出为独立临时文件；调用方负责在失败或验证不通过时清理。

+ * + * @return 生成的 M4A 输出路径 + */ + List buildRemuxCommand(Path input, Path output, String title, String artist, + String album, String albumArtist) { + List cmd = new ArrayList<>(); + cmd.add(getFfmpegCommand()); + cmd.add("-y"); + cmd.add("-i"); + cmd.add(input.toAbsolutePath().toString()); + // 仅映射预期的第一个音频流，避免把普通视频/字幕/数据等流带入生成的 M4A， + // 确保输出是干净的 Navidrome 兼容音乐资产。 + cmd.add("-map"); + cmd.add("0:a:0"); + cmd.add("-c"); + cmd.add("copy"); + // 保留原有 format 级元数据（不含未映射的流），随后覆盖简体标签 + cmd.add("-map_metadata"); + cmd.add("0"); + // 覆盖简体标签（不伪造缺失字段：调用方已确保三项必需字段非空） + cmd.add("-metadata"); + cmd.add("title=" + title); + cmd.add("-metadata"); + cmd.add("artist=" + artist); + cmd.add("-metadata"); + cmd.add("album=" + album); + if (albumArtist != null && !albumArtist.isEmpty()) { + cmd.add("-metadata"); + cmd.add("album_artist=" + albumArtist); + } + cmd.add(output.toAbsolutePath().toString()); + return cmd; + } + + private Path remuxWithSimplifiedTags(Path input, String title, String artist, + String album, String albumArtist) + throws IOException, InterruptedException { + String baseName = getBaseName(input.getFileName().toString()); + Path output = resolveUniqueFile(input.getParent(), baseName + ".fallback.m4a"); + + List cmd = buildRemuxCommand(input, output, title, artist, album, albumArtist); + runFfmpegDrained(cmd, FFMPEG_CONVERT_TIMEOUT_SECONDS, output, "remux"); + return output; + } + + /** + * 运行一个 FFmpeg 子进程，并发排空其合并后的标准输出，避免管道写满导致死锁； + * 超时或中断时强制终止并有界回收，失败时清理输出文件。 + *

合并 stderr→stdout 后由守护线程持续读取（仅保留前 {@value #FFMPEG_DIAGNOSTIC_CAP} + * 字节作诊断），超出部分丢弃以排空管道。非零退出或超时抛出 {@link RuntimeException}。

+ * + * @param command 完整命令行 + * @param timeoutSeconds 超时秒数 + * @param outputToClean 失败/超时时需要删除的输出文件（可为 null） + * @param label 日志/异常用的操作标签（如 "remux" / "转码"） + */ + private void runFfmpegDrained(List command, int timeoutSeconds, + Path outputToClean, String label) + throws IOException, InterruptedException { + ProcessBuilder pb = new ProcessBuilder(command); + pb.redirectErrorStream(true); + Process p = pb.start(); + + final StringBuilder diag = new StringBuilder(); + Thread drainer = new Thread(() -> { + try (InputStream is = p.getInputStream()) { + byte[] buf = new byte[4096]; + int n; + while ((n = is.read(buf)) != -1) { + synchronized (diag) { + int remaining = FFMPEG_DIAGNOSTIC_CAP - diag.length(); + if (remaining > 0) { + diag.append(new String(buf, 0, Math.min(n, remaining), + StandardCharsets.UTF_8)); + } + // 已达上限后继续读取以排空管道，丢弃多余数据 + } + } + } catch (IOException ignored) { + // 进程终止后流关闭属正常情况 + } + }, "ffmpeg-" + label + "-drainer"); + drainer.setDaemon(true); + drainer.start(); + + boolean interrupted = false; + boolean finished; + try { + finished = p.waitFor(timeoutSeconds, TimeUnit.SECONDS); + } catch (InterruptedException e) { + interrupted = true; + finished = false; + } + + if (!finished) { + p.destroyForcibly(); + try { + p.waitFor(FFMPEG_CLEANUP_WAIT_SECONDS, TimeUnit.SECONDS); + } catch (InterruptedException e) { + interrupted = true; + } + } + + // 先 join drainer 确保输出读尽，再判定结果 + try { + drainer.join(TimeUnit.SECONDS.toMillis(FFMPEG_CLEANUP_WAIT_SECONDS)); + } catch (InterruptedException e) { + interrupted = true; + } + + if (interrupted) { + deleteIfExists(outputToClean); + Thread.currentThread().interrupt(); + throw new InterruptedException("ffmpeg " + label + " 被中断"); + } + + if (!finished) { + deleteIfExists(outputToClean); + throw new RuntimeException("ffmpeg " + label + " 超时（" + timeoutSeconds + "s）"); + } + + int exit = p.exitValue(); + if (exit != 0) { + String tail; + synchronized (diag) { + tail = diag.toString().trim(); + } + deleteIfExists(outputToClean); + throw new RuntimeException("ffmpeg " + label + " 退出码: " + exit + + (tail.isEmpty() ? "" : " - " + tail)); + } + } + /** * 扫描 Library 目录中已有的音频文件，构建身份标识集合 * （所有文本字段经过 t2s + lowercase 归一化，与 incoming 文件一致） diff --git a/backend/src/main/java/com/music/service/LibraryMergeService.java b/backend/src/main/java/com/music/service/LibraryMergeService.java index e9e95b8..4d8055d 100644 --- a/backend/src/main/java/com/music/service/LibraryMergeService.java +++ b/backend/src/main/java/com/music/service/LibraryMergeService.java @@ -5,6 +5,7 @@ import com.music.dto.ProgressMessage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.messaging.simp.SimpMessagingTemplate; +import com.music.config.AsyncConfig; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; @@ -42,7 +43,7 @@ public class LibraryMergeService { this.progressStore = progressStore; } - @Async + @Async(AsyncConfig.TASK_EXECUTOR_BEAN_NAME) public void merge(String taskId, String srcDir, String dstDir, boolean smartUpgrade, boolean keepBackup) { Path srcPath = Paths.get(srcDir); Path dstPath = Paths.get(dstDir); diff --git a/backend/src/main/java/com/music/service/OrganizeService.java b/backend/src/main/java/com/music/service/OrganizeService.java index 78da35a..6ac5281 100644 --- a/backend/src/main/java/com/music/service/OrganizeService.java +++ b/backend/src/main/java/com/music/service/OrganizeService.java @@ -16,6 +16,7 @@ import org.jaudiotagger.tag.flac.FlacTag; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.messaging.simp.SimpMessagingTemplate; +import com.music.config.AsyncConfig; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; @@ -66,7 +67,7 @@ public class OrganizeService { this.pinyinFormat.setToneType(HanyuPinyinToneType.WITHOUT_TONE); } - @Async + @Async(AsyncConfig.TASK_EXECUTOR_BEAN_NAME) public void organize(String taskId, String srcDir, String dstDir, String mode, boolean extractCover, boolean extractLyrics, boolean generateReport) { Path srcPath = Paths.get(srcDir); diff --git a/backend/src/main/java/com/music/service/ZhConvertService.java b/backend/src/main/java/com/music/service/ZhConvertService.java index b2cc5aa..4d868d8 100644 --- a/backend/src/main/java/com/music/service/ZhConvertService.java +++ b/backend/src/main/java/com/music/service/ZhConvertService.java @@ -10,6 +10,7 @@ import org.jaudiotagger.tag.Tag; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.messaging.simp.SimpMessagingTemplate; +import com.music.config.AsyncConfig; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; @@ -64,7 +65,7 @@ public class ZhConvertService { * @param mode preview / execute * @param thresholdRatio 触发阈值（0.0-1.0） */ - @Async + @Async(AsyncConfig.TASK_EXECUTOR_BEAN_NAME) public void process(String taskId, String scanDir, String outputDir, diff --git a/backend/src/test/java/com/music/config/AsyncConfigTest.java b/backend/src/test/java/com/music/config/AsyncConfigTest.java new file mode 100644 index 0000000..ce35802 --- /dev/null +++ b/backend/src/test/java/com/music/config/AsyncConfigTest.java @@ -0,0 +1,72 @@ +package com.music.config; + +import org.junit.jupiter.api.Test; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; + +import java.lang.reflect.Method; +import java.util.concurrent.Executor; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * {@link AsyncConfig} 单元测试：验证 {@code @Async} 使用明确的应用 taskExecutor， + * 从而消除多个 TaskExecutor 无法选择的警告，并保留异常处理器。 + */ +class AsyncConfigTest { + + private final AsyncConfig config = new AsyncConfig(); + + @Test + void taskExecutorBeanIsConfiguredThreadPool() { + ThreadPoolTaskExecutor executor = config.mangtoolTaskExecutor(); + assertNotNull(executor); + assertEquals("mangtool-async-", executor.getThreadNamePrefix()); + assertTrue(executor.getCorePoolSize() >= 1); + assertTrue(executor.getMaxPoolSize() >= executor.getCorePoolSize()); + executor.shutdown(); + } + + @Test + void getAsyncExecutorReturnsExplicitExecutor() { + Executor executor = config.getAsyncExecutor(); + assertNotNull(executor, "@Async 应有明确的默认执行器，避免多候选无法选择"); + assertTrue(executor instanceof ThreadPoolTaskExecutor); + ((ThreadPoolTaskExecutor) executor).shutdown(); + } + + @Test + void getAsyncUncaughtExceptionHandlerPresent() { + assertNotNull(config.getAsyncUncaughtExceptionHandler(), + "应提供异常处理器以保留异步任务异常可观测性"); + } + + @Test + void beanNameConstantMatchesFactoryMethod() throws Exception { + // @Bean 方法名即 bean 名称，需与供 @Async 引用的常量一致 + Method m = AsyncConfig.class.getMethod(AsyncConfig.TASK_EXECUTOR_BEAN_NAME); + assertNotNull(m, "bean 名称常量应对应工厂方法名: " + AsyncConfig.TASK_EXECUTOR_BEAN_NAME); + } + + @Test + void asyncAnnotationsReferenceExplicitExecutor() { + // 抽查关键的 @Async 服务方法，确认显式引用了应用级 executor + assertAsyncQualifier(com.music.service.IngestService.class, "ingest"); + assertAsyncQualifier(com.music.service.ConvertService.class, "convert"); + } + + private static void assertAsyncQualifier(Class serviceClass, String methodName) { + Method target = null; + for (Method m : serviceClass.getDeclaredMethods()) { + if (m.getName().equals(methodName)) { + target = m; + break; + } + } + assertNotNull(target, "应找到方法 " + methodName); + org.springframework.scheduling.annotation.Async async = + target.getAnnotation(org.springframework.scheduling.annotation.Async.class); + assertNotNull(async, methodName + " 应带 @Async 注解"); + assertEquals(AsyncConfig.TASK_EXECUTOR_BEAN_NAME, async.value(), + methodName + " 的 @Async 应显式引用应用级 taskExecutor"); + } +} diff --git a/backend/src/test/java/com/music/service/AudioValidationServiceTest.java b/backend/src/test/java/com/music/service/AudioValidationServiceTest.java index 911ff73..3dd18b6 100644 --- a/backend/src/test/java/com/music/service/AudioValidationServiceTest.java +++ b/backend/src/test/java/com/music/service/AudioValidationServiceTest.java @@ -10,6 +10,7 @@ import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Arrays; import java.util.List; import java.util.concurrent.TimeUnit; @@ -436,6 +437,115 @@ class AudioValidationServiceTest { } } + // ========== FFprobe 后备元数据读取 ========== + + @Test + void getMetadataCommandArgs_usesStructuredJson() { + Path f = Paths.get("/tmp/song.m4a"); + List args = service.getMetadataCommandArgs(f); + assertTrue(args.contains("-print_format")); + assertTrue(args.contains("json")); + assertTrue(args.contains("-show_format")); + assertTrue(args.contains("-show_streams")); + // 绝不使用任何人类可读文本或文件名猜测选项 + assertFalse(args.contains("-of")); + assertEquals(f.toAbsolutePath().toString(), args.get(args.size() - 1)); + } + + @Test + void readMetadata_returnsFullTags() throws Exception { + assumeFfmpegFfprobe(); + Path tmpDir = Files.createTempDirectory("probe-meta-"); + try { + Path m4a = createTaggedM4a(tmpDir, "song.m4a", + "ProbeTitle", "ProbeArtist", "ProbeAlbum"); + + AudioValidationService.ProbeMetadata meta = service.readMetadata(m4a); + assertTrue(meta.isAvailable(), "应成功读取元数据"); + assertEquals("ProbeTitle", meta.getFirst("title")); + assertEquals("ProbeArtist", meta.getFirst("artist")); + assertEquals("ProbeAlbum", meta.getFirst("album")); + } finally { + deleteDirectory(tmpDir); + } + } + + @Test + void readMetadata_missingAlbumReturnsEmpty() throws Exception { + assumeFfmpegFfprobe(); + Path tmpDir = Files.createTempDirectory("probe-meta-"); + try { + // 仅写 title/artist，故意不写 album + Path m4a = tmpDir.resolve("noalbum.m4a"); + ProcessBuilder pb = new ProcessBuilder( + "ffmpeg", "-y", "-f", "lavfi", "-i", "anullsrc=r=44100:cl=mono", + "-t", "0.3", "-c:a", "aac", + "-metadata", "title=T", "-metadata", "artist=A", + m4a.toAbsolutePath().toString()); + pb.redirectErrorStream(true); + Process p = pb.start(); + readAllBytes(p.getInputStream()); + p.waitFor(15, TimeUnit.SECONDS); + + AudioValidationService.ProbeMetadata meta = service.readMetadata(m4a); + assertTrue(meta.isAvailable()); + assertEquals("", meta.getFirst("album"), "缺失的 album 应返回空字符串"); + } finally { + deleteDirectory(tmpDir); + } + } + + @Test + void readMetadata_nonAudioFileUnavailable() throws Exception { + assumeFfmpegFfprobe(); + Path tmpDir = Files.createTempDirectory("probe-meta-"); + try { + Path bogus = tmpDir.resolve("bogus.m4a"); + Files.write(bogus, "not an audio file".getBytes(StandardCharsets.UTF_8)); + + AudioValidationService.ProbeMetadata meta = service.readMetadata(bogus); + assertFalse(meta.isAvailable(), "无法识别的文件应返回 unavailable"); + } finally { + deleteDirectory(tmpDir); + } + } + + @Test + void readMetadata_ffprobeFailureUnavailable() { + // 强制 ffprobe 指向失败命令 + System.setProperty(AudioValidationService.FFPROBE_BIN_PROPERTY, "/bin/false"); + AudioValidationService.ProbeMetadata meta = + service.readMetadata(Paths.get("/tmp/whatever.m4a")); + assertFalse(meta.isAvailable(), "ffprobe 失败应返回 unavailable"); + } + + /** + * 使用 FFmpeg 创建一个带标签的有效短 M4A（AAC）文件。 + */ + private static Path createTaggedM4a(Path dir, String fileName, + String title, String artist, String album) throws Exception { + Path out = dir.resolve(fileName); + ProcessBuilder pb = new ProcessBuilder( + "ffmpeg", "-y", "-f", "lavfi", "-i", "anullsrc=r=44100:cl=mono", + "-t", "0.3", "-c:a", "aac", + "-metadata", "title=" + title, + "-metadata", "artist=" + artist, + "-metadata", "album=" + album, + out.toAbsolutePath().toString()); + pb.redirectErrorStream(true); + Process p = pb.start(); + readAllBytes(p.getInputStream()); + boolean finished = p.waitFor(15, TimeUnit.SECONDS); + if (!finished) { + p.destroyForcibly(); + throw new RuntimeException("ffmpeg 创建 M4A 超时"); + } + if (p.exitValue() != 0) { + throw new RuntimeException("ffmpeg 创建 M4A 失败 (exit=" + p.exitValue() + ")"); + } + return out; + } + // ========== 帮助方法 ========== /** diff --git a/backend/src/test/java/com/music/service/IngestServiceE2ETest.java b/backend/src/test/java/com/music/service/IngestServiceE2ETest.java index 4205b48..09f6b6a 100644 --- a/backend/src/test/java/com/music/service/IngestServiceE2ETest.java +++ b/backend/src/test/java/com/music/service/IngestServiceE2ETest.java @@ -18,6 +18,7 @@ import java.lang.reflect.Method; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Arrays; import java.util.HashSet; import java.util.Set; @@ -28,8 +29,11 @@ import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.*; import static org.junit.jupiter.api.Assumptions.assumeTrue; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; +import static org.mockito.ArgumentMatchers.any; /** * 一键导入服务端到端测试 —— 使用临时文件系统验证完整的导入管线行为。 @@ -1210,8 +1214,447 @@ class IngestServiceE2ETest { } } + // ========== 5. FFprobe 元数据后备（jaudiotagger 无法解析的合法 M4A/AAC） ========== + + @Test + void jaudiotaggerUnreadableM4a_ffprobeFallbackIngested() throws Exception { + assumeFfmpeg(); + Path tmpDir = Files.createTempDirectory("ingest-fallback-"); + try { + Path inputDir = tmpDir.resolve("Input"); + Path libDir = tmpDir.resolve("Library"); + Path rejDir = tmpDir.resolve("Rejected"); + Files.createDirectories(inputDir); + Files.createDirectories(libDir); + Files.createDirectories(rejDir); + + // M4A 容器但扩展名为 .aac → jaudiotagger 无 AAC reader 而拒绝， + // FFprobe 能内容探测并读取完整 Title/Artist/Album + Path src = createMp4ContainerAsAac(inputDir, "legit.aac", + "Fb Title", "Fb Artist", "Fb Album"); + assertJaudiotaggerCannotRead(src); + + IngestService service = buildServiceWithValidation(); + AtomicInteger ingested = new AtomicInteger(); + AtomicInteger duplicates = new AtomicInteger(); + AtomicInteger missingMeta = new AtomicInteger(); + AtomicInteger unreadable = new AtomicInteger(); + AtomicInteger convFailed = new AtomicInteger(); + AtomicInteger otherRejected = new AtomicInteger(); + + String result = invokeProcessSingleFile(service, src, libDir, rejDir, + new HashSet<>(), new HashSet<>(), + ingested, duplicates, missingMeta, unreadable, + convFailed, otherRejected); + + assertEquals("ingested", result, + "jaudiotagger 无法解析但 FFprobe 提供完整元数据的 M4A 应入库"); + assertEquals(1, ingested.get()); + assertEquals(0, unreadable.get()); + assertEquals(0, missingMeta.get()); + + Path expectedDir = libDir.resolve("Fb Artist").resolve("Fb Album"); + assertTrue(Files.isDirectory(expectedDir), "应创建 Artist/Album 目录"); + boolean found = Files.list(expectedDir).anyMatch(p -> + p.getFileName().toString().equals("01 - Fb Title.m4a")); + assertTrue(found, "后备 remux 输出应为 01 - Fb Title.m4a"); + assertFalse(Files.exists(src), "源文件应在 remux 后被删除"); + } finally { + deleteDirectory(tmpDir); + } + } + + @Test + void jaudiotaggerUnreadableM4a_missingAlbumRejectedMissingMetadata() throws Exception { + assumeFfmpeg(); + Path tmpDir = Files.createTempDirectory("ingest-fallback-"); + try { + Path inputDir = tmpDir.resolve("Input"); + Path libDir = tmpDir.resolve("Library"); + Path rejDir = tmpDir.resolve("Rejected"); + Files.createDirectories(inputDir); + Files.createDirectories(libDir); + Files.createDirectories(rejDir); + + // 仅 title/artist，无 album → 后备读取后仍应归类 MissingMetadata + Path src = createMp4ContainerAsAac(inputDir, "noalbum.aac", + "Only Title", "Only Artist", null); + assertJaudiotaggerCannotRead(src); + + IngestService service = buildServiceWithValidation(); + AtomicInteger ingested = new AtomicInteger(); + AtomicInteger duplicates = new AtomicInteger(); + AtomicInteger missingMeta = new AtomicInteger(); + AtomicInteger unreadable = new AtomicInteger(); + AtomicInteger convFailed = new AtomicInteger(); + AtomicInteger otherRejected = new AtomicInteger(); + + String result = invokeProcessSingleFile(service, src, libDir, rejDir, + new HashSet<>(), new HashSet<>(), + ingested, duplicates, missingMeta, unreadable, + convFailed, otherRejected); + + assertEquals("rejected:missing-metadata", result, + "后备读取缺 Album 应归类 MissingMetadata"); + assertEquals(1, missingMeta.get()); + assertEquals(0, unreadable.get()); + assertTrue(Files.exists(rejDir.resolve("MissingMetadata").resolve("noalbum.aac"))); + } finally { + deleteDirectory(tmpDir); + } + } + + @Test + void jaudiotaggerUnreadable_ffprobeAlsoFails_rejectedUnreadable() throws Exception { + assumeFfmpeg(); + Path tmpDir = Files.createTempDirectory("ingest-fallback-"); + try { + Path inputDir = tmpDir.resolve("Input"); + Path libDir = tmpDir.resolve("Library"); + Path rejDir = tmpDir.resolve("Rejected"); + Files.createDirectories(inputDir); + Files.createDirectories(libDir); + Files.createDirectories(rejDir); + + // 非音频内容但扩展名 .m4a → jaudiotagger 与 FFprobe 都无法识别 + Path src = inputDir.resolve("garbage.m4a"); + Files.write(src, "definitely not audio".getBytes(StandardCharsets.UTF_8)); + + IngestService service = buildServiceWithValidation(); + AtomicInteger ingested = new AtomicInteger(); + AtomicInteger duplicates = new AtomicInteger(); + AtomicInteger missingMeta = new AtomicInteger(); + AtomicInteger unreadable = new AtomicInteger(); + AtomicInteger convFailed = new AtomicInteger(); + AtomicInteger otherRejected = new AtomicInteger(); + + String result = invokeProcessSingleFile(service, src, libDir, rejDir, + new HashSet<>(), new HashSet<>(), + ingested, duplicates, missingMeta, unreadable, + convFailed, otherRejected); + + assertEquals("rejected:unreadable", result, + "FFprobe 也无法识别时应归类 Unreadable"); + assertEquals(1, unreadable.get()); + assertTrue(Files.exists(rejDir.resolve("Unreadable").resolve("garbage.m4a"))); + } finally { + deleteDirectory(tmpDir); + } + } + + @Test + void fallbackPath_stillRunsDuplicateDetection() throws Exception { + assumeFfmpeg(); + Path tmpDir = Files.createTempDirectory("ingest-fallback-"); + try { + Path inputDir = tmpDir.resolve("Input"); + Path libDir = tmpDir.resolve("Library"); + Path rejDir = tmpDir.resolve("Rejected"); + Files.createDirectories(inputDir); + Files.createDirectories(libDir); + Files.createDirectories(rejDir); + + Path src1 = createMp4ContainerAsAac(inputDir, "dup1.aac", + "Dup Title", "Dup Artist", "Dup Album"); + Path src2 = createMp4ContainerAsAac(inputDir, "dup2.aac", + "Dup Title", "Dup Artist", "Dup Album"); + + IngestService service = buildServiceWithValidation(); + Set libraryIds = new HashSet<>(); + Set batchIds = new HashSet<>(); + + String r1 = invokeProcessSingleFile(service, src1, libDir, rejDir, + libraryIds, batchIds, + new AtomicInteger(), new AtomicInteger(), + new AtomicInteger(), new AtomicInteger(), + new AtomicInteger(), new AtomicInteger()); + assertEquals("ingested", r1, "首个后备文件应入库"); + + AtomicInteger unreadable2 = new AtomicInteger(); + AtomicInteger duplicates2 = new AtomicInteger(); + String r2 = invokeProcessSingleFile(service, src2, libDir, rejDir, + libraryIds, batchIds, + new AtomicInteger(), duplicates2, + new AtomicInteger(), unreadable2, + new AtomicInteger(), new AtomicInteger()); + assertEquals("rejected:duplicate", r2, + "后备路径应继续执行去重检测"); + assertEquals(1, duplicates2.get()); + } finally { + deleteDirectory(tmpDir); + } + } + + @Test + void fallbackPath_stillRunsDecodeValidation() throws Exception { + assumeFfmpeg(); + Path tmpDir = Files.createTempDirectory("ingest-fallback-"); + try { + Path inputDir = tmpDir.resolve("Input"); + Path libDir = tmpDir.resolve("Library"); + Path rejDir = tmpDir.resolve("Rejected"); + Files.createDirectories(inputDir); + Files.createDirectories(libDir); + Files.createDirectories(rejDir); + + // 合法 M4A（jaudiotagger 无法读，FFprobe 元数据齐全）→ 后备路径应被触发 + Path src = createMp4ContainerAsAac(inputDir, "legit.aac", + "Dec Title", "Dec Artist", "Dec Album"); + assertJaudiotaggerCannotRead(src); + + // 用受控的 AudioValidationService 替身： + // - readMetadata 走真实实现，确保后备元数据“确实可用”（前置成立） + // - validate() 强制失败，隔离出“完整解码校验”这一步作为唯一失败点 + AudioValidationService realAvs = new AudioValidationService(); + AudioValidationService avs = spy(realAvs); + doReturn(AudioValidationService.ValidationResult.invalid("模拟解码失败")) + .when(avs).validate(any(Path.class)); + + // 前置断言：后备元数据确实可用且完整 + AudioValidationService.ProbeMetadata pre = avs.readMetadata(src); + assertTrue(pre.isAvailable(), "前置条件：FFprobe 后备元数据应可用"); + assertEquals("Dec Title", pre.getFirst("title")); + assertEquals("Dec Artist", pre.getFirst("artist")); + assertEquals("Dec Album", pre.getFirst("album")); + + IngestService service = new IngestService( + mock(SimpMessagingTemplate.class), + new ProgressStore(), + new TraditionalFilterService(), + mock(ConfigService.class), + avs); + + AtomicInteger ingested = new AtomicInteger(); + AtomicInteger unreadable = new AtomicInteger(); + String result = invokeProcessSingleFile(service, src, libDir, rejDir, + new HashSet<>(), new HashSet<>(), + ingested, new AtomicInteger(), + new AtomicInteger(), unreadable, + new AtomicInteger(), new AtomicInteger()); + + assertEquals("rejected:unreadable", result, + "后备/remux 成功但完整解码校验失败时应归类 Unreadable"); + assertEquals(1, unreadable.get()); + assertEquals(0, ingested.get()); + assertTrue(Files.exists(rejDir.resolve("Unreadable").resolve("legit.aac")), + "解码校验失败的源文件应进入 Rejected/Unreadable"); + // 派生的临时 remux 产物应被清理 + assertTrue(Files.list(inputDir).noneMatch(p -> + p.getFileName().toString().endsWith(".fallback.m4a")), + "解码校验失败后派生的 remux 产物应被清理"); + } finally { + deleteDirectory(tmpDir); + } + } + + @Test + void fallbackRemux_producesAudioOnlyOutputDroppingVideo() throws Exception { + assumeFfmpeg(); + Path tmpDir = Files.createTempDirectory("ingest-remux-map-"); + try { + Path inputDir = tmpDir.resolve("Input"); + Path libDir = tmpDir.resolve("Library"); + Path rejDir = tmpDir.resolve("Rejected"); + Files.createDirectories(inputDir); + Files.createDirectories(libDir); + Files.createDirectories(rejDir); + + // 构造包含真实视频流 + 音频流的 .aac（MP4 容器）文件 + Path src = createMp4WithVideoAndAudio(inputDir, "withvideo.aac", + "Map Title", "Map Artist", "Map Album"); + assertJaudiotaggerCannotRead(src); + // 前置：源文件确有视频流 + assertTrue(streamCodecTypes(src).contains("video"), + "前置条件：源文件应包含视频流"); + + IngestService service = buildServiceWithValidation(); + AtomicInteger ingested = new AtomicInteger(); + String result = invokeProcessSingleFile(service, src, libDir, rejDir, + new HashSet<>(), new HashSet<>(), + ingested, new AtomicInteger(), + new AtomicInteger(), new AtomicInteger(), + new AtomicInteger(), new AtomicInteger()); + + assertEquals("ingested", result, "含视频流的合法音频应经后备 remux 入库"); + Path out = libDir.resolve("Map Artist").resolve("Map Album") + .resolve("01 - Map Title.m4a"); + assertTrue(Files.exists(out), "应生成入库的 M4A: " + out); + + java.util.List outTypes = streamCodecTypes(out); + assertTrue(outTypes.contains("audio"), "输出应保留音频流"); + assertFalse(outTypes.contains("video"), + "输出不应携带普通视频流（应为干净的音乐资产），实际: " + outTypes); + } finally { + deleteDirectory(tmpDir); + } + } + + @Test + void buildRemuxCommand_mapsOnlyFirstAudioStream() throws Exception { + IngestService service = buildServiceWithValidation(); + Method m = IngestService.class.getDeclaredMethod("buildRemuxCommand", + Path.class, Path.class, String.class, String.class, String.class, String.class); + m.setAccessible(true); + + @SuppressWarnings("unchecked") + java.util.List cmd = (java.util.List) m.invoke(service, + Paths.get("/tmp/in.aac"), Paths.get("/tmp/out.m4a"), + "T", "A", "Al", "AA"); + + // 必须显式仅映射第一个音频流，且不得使用无限制的 -map 0 + int mapIdx = cmd.indexOf("-map"); + assertTrue(mapIdx >= 0, "应包含 -map 参数"); + assertEquals("0:a:0", cmd.get(mapIdx + 1), "应仅映射第一个音频流"); + // 确认没有任何 '-map 0' 无限制映射 + for (int i = 0; i < cmd.size() - 1; i++) { + if ("-map".equals(cmd.get(i))) { + assertNotEquals("0", cmd.get(i + 1), + "不得使用无限制的 -map 0（会带入视频/字幕/数据流）"); + } + } + assertTrue(cmd.contains("copy"), "应无损复制音频（-c copy）"); + } + + @Test + void runFfmpegDrained_doesNotDeadlockOnVerboseOutput() throws Exception { + // 使用产生大量输出的可控进程验证并发排空不会因管道写满而死锁。 + // 该测试不依赖 ffmpeg，直接反射调用 runFfmpegDrained。 + Path tmpDir = Files.createTempDirectory("ffmpeg-drain-"); + try { + IngestService service = buildService(); + Method m = IngestService.class.getDeclaredMethod("runFfmpegDrained", + java.util.List.class, int.class, Path.class, String.class); + m.setAccessible(true); + + // 产生远超管道缓冲区（通常 64KB）的输出：约 2MB + java.util.List cmd = Arrays.asList( + "sh", "-c", + "i=0; while [ $i -lt 40000 ]; do " + + "echo 'noisy ffmpeg style diagnostic line to fill the pipe buffer'; " + + "i=$((i+1)); done"); + + long start = System.currentTimeMillis(); + // 成功退出（exit 0）→ 不抛异常；若死锁则会一直阻塞直到超时 + m.invoke(service, cmd, 30, null, "test"); + long elapsed = System.currentTimeMillis() - start; + + assertTrue(elapsed < 25_000, + "并发排空应快速完成而非因管道写满死锁，耗时(ms)=" + elapsed); + } finally { + deleteDirectory(tmpDir); + } + } + + @Test + void runFfmpegDrained_nonZeroExitCleansOutputAndThrows() throws Exception { + Path tmpDir = Files.createTempDirectory("ffmpeg-drain-fail-"); + try { + IngestService service = buildService(); + Method m = IngestService.class.getDeclaredMethod("runFfmpegDrained", + java.util.List.class, int.class, Path.class, String.class); + m.setAccessible(true); + + Path output = tmpDir.resolve("out.m4a"); + Files.write(output, "partial".getBytes(StandardCharsets.UTF_8)); + + // 命令先产生大量输出再以非零退出，验证：不死锁、清理输出、抛异常 + java.util.List cmd = Arrays.asList( + "sh", "-c", + "i=0; while [ $i -lt 20000 ]; do echo 'noise line'; i=$((i+1)); done; exit 3"); + + try { + m.invoke(service, cmd, 30, output, "test"); + fail("非零退出应抛出异常"); + } catch (java.lang.reflect.InvocationTargetException e) { + assertTrue(e.getCause() instanceof RuntimeException, + "应抛出 RuntimeException，实际: " + e.getCause()); + assertTrue(e.getCause().getMessage().contains("退出码"), + "异常应包含退出码信息"); + } + assertFalse(Files.exists(output), "失败时应清理输出文件"); + } finally { + deleteDirectory(tmpDir); + } + } + // ========== 工具方法 ========== + /** + * 创建一个 MP4/M4A 容器但使用 .aac 扩展名的文件。 + *

jaudiotagger 按扩展名分派 reader，会因缺少 AAC reader 而拒绝； + * FFprobe 通过内容探测能识别并读取完整元数据。用于触发后备读取路径。

+ */ + private static Path createMp4ContainerAsAac(Path dir, String name, + String title, String artist, String album) throws Exception { + Path file = dir.resolve(name); + java.util.List cmd = new java.util.ArrayList<>(Arrays.asList( + "ffmpeg", "-y", "-f", "lavfi", "-i", "anullsrc=r=44100:cl=mono", + "-t", "0.3", "-c:a", "aac", "-f", "mp4")); + if (title != null) { cmd.add("-metadata"); cmd.add("title=" + title); } + if (artist != null) { cmd.add("-metadata"); cmd.add("artist=" + artist); } + if (album != null) { cmd.add("-metadata"); cmd.add("album=" + album); } + cmd.add(file.toAbsolutePath().toString()); + runProcessChecked(new ProcessBuilder(cmd), "ffmpeg"); + return file; + } + + /** + * 创建一个同时包含真实视频流与音频流的 MP4/M4A 容器（.aac 扩展名）。 + * 用于验证后备 remux 仅保留音频、丢弃普通视频流。 + */ + private static Path createMp4WithVideoAndAudio(Path dir, String name, + String title, String artist, String album) throws Exception { + Path file = dir.resolve(name); + java.util.List cmd = new java.util.ArrayList<>(Arrays.asList( + "ffmpeg", "-y", + "-f", "lavfi", "-i", "anullsrc=r=44100:cl=mono", + "-f", "lavfi", "-i", "testsrc=s=64x64:d=0.3", + "-map", "0:a", "-map", "1:v", + "-c:a", "aac", "-c:v", "libx264", + "-shortest", "-t", "0.3", "-f", "mp4")); + if (title != null) { cmd.add("-metadata"); cmd.add("title=" + title); } + if (artist != null) { cmd.add("-metadata"); cmd.add("artist=" + artist); } + if (album != null) { cmd.add("-metadata"); cmd.add("album=" + album); } + cmd.add(file.toAbsolutePath().toString()); + runProcessChecked(new ProcessBuilder(cmd), "ffmpeg"); + return file; + } + + /** + * 用 ffprobe 返回文件中所有流的 codec_type 列表。 + */ + private static java.util.List streamCodecTypes(Path file) throws Exception { + ProcessBuilder pb = new ProcessBuilder( + "ffprobe", "-v", "error", + "-show_entries", "stream=codec_type", + "-of", "default=nw=1:nk=1", + file.toAbsolutePath().toString()); + pb.redirectErrorStream(true); + Process p = pb.start(); + String out = readStream(p.getInputStream()); + p.waitFor(10, TimeUnit.SECONDS); + java.util.List types = new java.util.ArrayList<>(); + for (String line : out.split("\\R")) { + String t = line.trim(); + if (!t.isEmpty()) types.add(t); + } + return types; + } + + /** + * 断言 jaudiotagger 无法读取该文件（无 tag 或抛异常），确认后备路径确会被触发。 + */ + private static void assertJaudiotaggerCannotRead(Path file) { + boolean readable; + try { + AudioFile af = AudioFileIO.read(file.toFile()); + readable = af.getTag() != null; + } catch (Exception e) { + readable = false; + } + assertFalse(readable, "前置条件：jaudiotagger 应无法读取该文件的标签"); + } + private IngestService buildService() { return new IngestService( mock(SimpMessagingTemplate.class),