feat: upload local music directories

This commit is contained in:
2026-07-21 19:25:20 +08:00
parent b21a5d0ed1
commit 59cf2ea67c
15 changed files with 1024 additions and 29 deletions
+4
View File
@@ -111,3 +111,7 @@ cd docker && docker compose up -d --build
- `MANGTOOL_MB_BASE_URL` (default: https://musicbrainz.org/ws/2) - `MANGTOOL_MB_BASE_URL` (default: https://musicbrainz.org/ws/2)
- `MANGTOOL_CAA_BASE_URL` (default: https://coverartarchive.org) - `MANGTOOL_CAA_BASE_URL` (default: https://coverartarchive.org)
- `MANGTOOL_MB_USER_AGENT` (default: MangTool/1.0 (https://gitea.mangmang.fun/LiuMangMang/MyTool)) - User-Agent for MusicBrainz API requests - `MANGTOOL_MB_USER_AGENT` (default: MangTool/1.0 (https://gitea.mangmang.fun/LiuMangMang/MyTool)) - User-Agent for MusicBrainz API requests
### 环境变量说明
- `MANGTOOL_UPLOAD_TEMP`: 前端文件上传的临时存放目录,默认值为 `/home/mangtool/MusicWork/.upload-tmp`
@@ -0,0 +1,14 @@
package com.music.common;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
public class Constants {
public static final Set<String> SUPPORTED_AUDIO_EXTENSIONS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(
"flac", "mp3", "m4a", "aac", "ogg", "opus", "wma", "wav", "ape", "aiff", "aif", "wv", "tta"
)));
public static final String EXTENSION_LRC = "lrc";
}
@@ -11,9 +11,21 @@ import com.music.service.IngestTaskStore;
import com.music.service.ProgressStore; import com.music.service.ProgressStore;
import org.springframework.validation.annotation.Validated; import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*; import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.beans.factory.annotation.Value;
import javax.annotation.PostConstruct;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
import java.util.UUID; import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
/** /**
* 一键导入控制器 * 一键导入控制器
@@ -28,20 +40,51 @@ public class IngestController {
private final ConfigService configService; private final ConfigService configService;
private final IngestTaskStore taskStore; private final IngestTaskStore taskStore;
// Removed ALLOWED_EXTENSIONS, use com.music.common.Constants
private final Path tempDir;
/** 防止并发执行的生命周期锁,由 IngestService 在 finally 中释放 */ /** 防止并发执行的生命周期锁,由 IngestService 在 finally 中释放 */
private final AtomicBoolean running = new AtomicBoolean(false); private final AtomicBoolean running = new AtomicBoolean(false);
private final AtomicInteger activeUploads = new AtomicInteger(0);
/** 当前/最近一次任务的 taskId,用于 GET /api/ingest/status 恢复 */ /** 当前/最近一次任务的 taskId,用于 GET /api/ingest/status 恢复 */
private volatile String currentTaskId; private volatile String currentTaskId;
private final Object lifecycleLock = new Object();
public IngestController(IngestService ingestService, public IngestController(IngestService ingestService,
ProgressStore progressStore, ProgressStore progressStore,
ConfigService configService, ConfigService configService,
IngestTaskStore taskStore) { IngestTaskStore taskStore,
@Value("${spring.servlet.multipart.location:/tmp/mangtool-upload-tmp}") String tempLocation) {
this.ingestService = ingestService; this.ingestService = ingestService;
this.progressStore = progressStore; this.progressStore = progressStore;
this.configService = configService; this.configService = configService;
this.taskStore = taskStore; this.taskStore = taskStore;
this.tempDir = Paths.get(tempLocation);
}
@PostConstruct
public void init() {
try {
if (!Files.exists(tempDir)) {
Files.createDirectories(tempDir);
}
// Clean stale .part files
try (java.util.stream.Stream<Path> stream = Files.list(tempDir)) {
stream.filter(p -> p.toString().endsWith(".part"))
.forEach(p -> {
try {
Files.deleteIfExists(p);
} catch (IOException ignored) {}
});
}
} catch (IOException e) {
throw new RuntimeException("Failed to initialize temp directory for uploads", e);
}
} }
/** /**
@@ -54,9 +97,15 @@ public class IngestController {
throw new BusinessException(400, "请先配置工作根目录"); throw new BusinessException(400, "请先配置工作根目录");
} }
synchronized (lifecycleLock) {
if (activeUploads.get() > 0) {
throw new BusinessException(409, "有文件正在上传,请等待完成后再试");
}
if (!running.compareAndSet(false, true)) { if (!running.compareAndSet(false, true)) {
throw new BusinessException(409, "导入任务已在运行中,请等待完成"); throw new BusinessException(409, "导入任务已在运行中,请等待完成");
} }
}
String taskId = UUID.randomUUID().toString(); String taskId = UUID.randomUUID().toString();
currentTaskId = taskId; currentTaskId = taskId;
@@ -190,6 +239,127 @@ public class IngestController {
throw new BusinessException(404, "未找到该任务的报告"); throw new BusinessException(404, "未找到该任务的报告");
} }
@PostMapping("/upload")
public Result<com.music.dto.UploadFileResponse> upload(@RequestParam("file") MultipartFile file,
@RequestParam("relativePath") String relativePath) {
synchronized (lifecycleLock) {
if (running.get()) {
throw new BusinessException(409, "导入任务正在运行中,无法上传文件");
}
activeUploads.incrementAndGet();
}
try {
String inputDir = configService.getInputDir();
if (inputDir == null || inputDir.trim().isEmpty()) {
throw new BusinessException(400, "请先配置工作根目录");
}
if (file.isEmpty()) {
throw new BusinessException(400, "文件为空");
}
if (relativePath == null || relativePath.trim().isEmpty()) {
throw new BusinessException(400, "相对路径不能为空");
}
if (relativePath.contains("\\")) {
throw new BusinessException(400, "路径中不能包含反斜杠");
}
// Path Security
if (relativePath.startsWith("/") || relativePath.contains(":") ||
relativePath.contains("..") || relativePath.contains("./") ||
relativePath.endsWith("/.")) {
throw new BusinessException(400, "非法的相对路径");
}
String ext = "";
int dotIdx = relativePath.lastIndexOf('.');
int slashIdx = relativePath.lastIndexOf('/');
if (dotIdx > slashIdx && dotIdx < relativePath.length() - 1) {
ext = relativePath.substring(dotIdx + 1).toLowerCase();
}
if (!com.music.common.Constants.SUPPORTED_AUDIO_EXTENSIONS.contains(ext) && !com.music.common.Constants.EXTENSION_LRC.equals(ext)) {
throw new BusinessException(400, "不支持的文件类型: " + ext);
}
String normalizedRelativePath = relativePath;
Path targetBase = Paths.get(inputDir).normalize();
Path targetPath = targetBase.resolve(normalizedRelativePath).normalize();
// 检查是否逃出 inputDir,或者发生路径遍历
if (!targetPath.startsWith(targetBase)) {
throw new BusinessException(400, "非法的相对路径,禁止目录遍历");
}
try {
Path current = targetBase;
String[] parts = relativePath.split("/");
for (String part : parts) {
if (part.isEmpty()) continue;
current = current.resolve(part);
if (Files.exists(current, java.nio.file.LinkOption.NOFOLLOW_LINKS) && Files.isSymbolicLink(current)) {
throw new BusinessException(400, "路径中不允许存在符号链接");
}
}
} catch (SecurityException e) {
throw new BusinessException(500, "安全检查失败: " + e.getMessage());
}
try {
Files.createDirectories(targetPath.getParent());
if (!Files.exists(tempDir)) {
Files.createDirectories(tempDir);
}
} catch (IOException e) {
throw new BusinessException(500, "无法创建目录: " + e.getMessage());
}
Path partFile = null;
try {
try {
if (Files.getFileStore(tempDir).equals(Files.getFileStore(targetPath.getParent()))) {
partFile = tempDir.resolve(UUID.randomUUID().toString() + ".part");
} else {
partFile = targetPath.getParent().resolve(UUID.randomUUID().toString() + ".part");
}
} catch (IOException e) {
partFile = targetPath.getParent().resolve(UUID.randomUUID().toString() + ".part");
}
if (partFile.getParent().toFile().getUsableSpace() < file.getSize()) {
throw new BusinessException(507, "磁盘空间不足");
}
file.transferTo(partFile.toFile());
synchronized (lifecycleLock) {
if (Files.exists(targetPath)) {
return Result.success(new com.music.dto.UploadFileResponse("skipped", normalizedRelativePath, targetPath.toFile().length()));
}
try {
Files.move(partFile, targetPath, StandardCopyOption.ATOMIC_MOVE);
} catch (java.nio.file.AtomicMoveNotSupportedException e) {
Files.move(partFile, targetPath);
}
}
return Result.success(new com.music.dto.UploadFileResponse("uploaded", normalizedRelativePath, targetPath.toFile().length()));
} catch (IOException e) {
throw new BusinessException(500, "文件上传失败: " + e.getMessage());
} finally {
if (partFile != null) {
try {
Files.deleteIfExists(partFile);
} catch (IOException ignored) {}
}
}
} finally {
activeUploads.decrementAndGet();
}
}
private long lastModified(java.nio.file.Path p) { private long lastModified(java.nio.file.Path p) {
try { try {
return java.nio.file.Files.getLastModifiedTime(p).toMillis(); return java.nio.file.Files.getLastModifiedTime(p).toMillis();
@@ -0,0 +1,40 @@
package com.music.dto;
public class UploadFileResponse {
private String status; // "uploaded" or "skipped"
private String relativePath;
private long size;
public UploadFileResponse() {
}
public UploadFileResponse(String status, String relativePath, long size) {
this.status = status;
this.relativePath = relativePath;
this.size = size;
}
public String getStatus() {
return status;
}
public void setStatus(String status) {
this.status = status;
}
public String getRelativePath() {
return relativePath;
}
public void setRelativePath(String relativePath) {
this.relativePath = relativePath;
}
public long getSize() {
return size;
}
public void setSize(long size) {
this.size = size;
}
}
@@ -23,6 +23,10 @@ public class GlobalExceptionHandler {
public Result<Void> handleValidationException(Exception ex) { public Result<Void> handleValidationException(Exception ex) {
return Result.failure(400, "请求参数错误:" + ex.getMessage()); return Result.failure(400, "请求参数错误:" + ex.getMessage());
} }
@ExceptionHandler(org.springframework.web.multipart.MaxUploadSizeExceededException.class)
public Result<Void> handleMaxUploadSizeExceededException(org.springframework.web.multipart.MaxUploadSizeExceededException ex) {
return Result.failure(413, "文件大小超出限制");
}
@ExceptionHandler(Exception.class) @ExceptionHandler(Exception.class)
public Result<Void> handleOther(Exception ex) { public Result<Void> handleOther(Exception ex) {
@@ -57,19 +57,6 @@ public class IngestService {
"wav", "ape", "aiff", "aif", "wv", "tta" "wav", "ape", "aiff", "aif", "wv", "tta"
)); ));
/** 可直接保留的格式 */
private static final Set<String> COMPATIBLE_EXTENSIONS = new HashSet<>(Arrays.asList(
"flac", "mp3", "m4a", "aac", "ogg", "opus", "wma"
));
/** 所有支持的音频格式 */
private static final Set<String> ALL_AUDIO_EXTENSIONS = new HashSet<>();
static {
ALL_AUDIO_EXTENSIONS.addAll(LOSSLESS_EXTENSIONS);
ALL_AUDIO_EXTENSIONS.addAll(COMPATIBLE_EXTENSIONS);
}
/** 需要繁简转换的标签字段 */ /** 需要繁简转换的标签字段 */
private static final FieldKey[] TEXT_FIELDS = { private static final FieldKey[] TEXT_FIELDS = {
FieldKey.TITLE, FieldKey.ARTIST, FieldKey.ALBUM, FieldKey.ALBUM_ARTIST FieldKey.TITLE, FieldKey.ARTIST, FieldKey.ALBUM, FieldKey.ALBUM_ARTIST
@@ -1501,7 +1488,7 @@ public class IngestService {
private boolean isAudioFile(Path file) { private boolean isAudioFile(Path file) {
String ext = getExtension(file.getFileName().toString()); String ext = getExtension(file.getFileName().toString());
return ext != null && ALL_AUDIO_EXTENSIONS.contains(ext); return ext != null && com.music.common.Constants.SUPPORTED_AUDIO_EXTENSIONS.contains(ext);
} }
private boolean isLosslessFormat(Path file) { private boolean isLosslessFormat(Path file) {
@@ -9,6 +9,12 @@ spring:
serialization: serialization:
write-dates-as-timestamps: false write-dates-as-timestamps: false
servlet:
multipart:
max-file-size: 4096MB
max-request-size: 4096MB
location: ${MANGTOOL_UPLOAD_TEMP:/tmp/mangtool-upload-tmp}
logging: logging:
level: level:
root: INFO root: INFO
@@ -0,0 +1,366 @@
package com.music;
import com.music.common.Constants;
import com.music.controller.IngestController;
import com.music.exception.BusinessException;
import com.music.dto.IngestRequest;
import com.music.service.ConfigService;
import com.music.service.IngestService;
import com.music.service.IngestTaskStore;
import com.music.service.ProgressStore;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Mock;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
import org.mockito.MockitoAnnotations;
import org.springframework.mock.web.MockMultipartFile;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.Mockito.when;
import static org.mockito.ArgumentMatchers.any;
public class IngestUploadTest {
@Mock
private IngestService ingestService;
@Mock
private ProgressStore progressStore;
@Mock
private ConfigService configService;
@Mock
private IngestTaskStore taskStore;
private IngestController ingestController;
private Path tempDir;
private Path inputDir;
@BeforeEach
public void setUp() throws Exception {
MockitoAnnotations.openMocks(this);
tempDir = Files.createTempDirectory("upload-tmp");
inputDir = Files.createTempDirectory("music-input");
ingestController = new IngestController(ingestService, progressStore, configService, taskStore, tempDir.toString());
when(configService.getInputDir()).thenReturn(inputDir.toString());
when(configService.getBasePath()).thenReturn(inputDir.toString());
}
@AfterEach
public void tearDown() throws Exception {
if (Files.exists(tempDir)) {
try (java.util.stream.Stream<Path> s = Files.walk(tempDir)) {
s.sorted(java.util.Comparator.reverseOrder()).forEach(p -> {
try { Files.delete(p); } catch (IOException ignored) {}
});
}
}
if (Files.exists(inputDir)) {
try (java.util.stream.Stream<Path> s = Files.walk(inputDir)) {
s.sorted(java.util.Comparator.reverseOrder()).forEach(p -> {
try { Files.delete(p); } catch (IOException ignored) {}
});
}
}
}
@Test
public void testInitCleansStalePartFiles() throws Exception {
Path part1 = tempDir.resolve("test1.part");
Path part2 = tempDir.resolve("test2.txt");
Files.write(part1, "test".getBytes());
Files.write(part2, "test".getBytes());
ingestController.init();
assertFalse(Files.exists(part1), "Stale .part file should be cleaned");
assertTrue(Files.exists(part2), "Non-.part file should not be cleaned");
}
@Test
public void testUploadSuccess() throws Exception {
MockMultipartFile file = new MockMultipartFile("file", "test.mp3", "audio/mpeg", "testdata".getBytes());
com.music.common.Result<com.music.dto.UploadFileResponse> result = ingestController.upload(file, "album/test.mp3");
assertEquals(0, result.getCode());
assertEquals("uploaded", result.getData().getStatus());
assertEquals("album/test.mp3", result.getData().getRelativePath());
assertTrue(result.getData().getSize() > 0);
Path targetPath = inputDir.resolve("album/test.mp3");
assertTrue(Files.exists(targetPath));
assertEquals("testdata", new String(Files.readAllBytes(targetPath)));
}
@Test
public void testUploadExtensions() throws Exception {
for (String ext : Constants.SUPPORTED_AUDIO_EXTENSIONS) {
MockMultipartFile file = new MockMultipartFile("file", "test." + ext, "audio/mpeg", "testdata".getBytes());
com.music.common.Result<com.music.dto.UploadFileResponse> result = ingestController.upload(file, "album/test." + ext);
assertEquals(0, result.getCode());
}
MockMultipartFile file = new MockMultipartFile("file", "test.lrc", "text/plain", "testdata".getBytes());
com.music.common.Result<com.music.dto.UploadFileResponse> result = ingestController.upload(file, "album/test.lrc");
assertEquals(0, result.getCode());
}
@Test
public void testUploadSkipExisting() throws Exception {
Path targetPath = inputDir.resolve("album/test.mp3");
Files.createDirectories(targetPath.getParent());
Files.write(targetPath, "existing".getBytes());
MockMultipartFile file = new MockMultipartFile("file", "test.mp3", "audio/mpeg", "testdata".getBytes());
com.music.common.Result<com.music.dto.UploadFileResponse> result = ingestController.upload(file, "album/test.mp3");
assertEquals(0, result.getCode());
assertEquals("skipped", result.getData().getStatus());
// Assert content not overwritten
assertEquals("existing", new String(Files.readAllBytes(targetPath)));
}
@Test
public void testUploadInvalidPaths() {
MockMultipartFile file = new MockMultipartFile("file", "test.mp3", "audio/mpeg", "testdata".getBytes());
String[] badPaths = { "../test.mp3", "/etc/passwd", "C:\\test.mp3", "album\\test.mp3", "./test.mp3", "album/.", "C:/test.mp3" };
for (String path : badPaths) {
BusinessException ex = assertThrows(BusinessException.class, () -> {
ingestController.upload(file, path);
});
assertEquals(400, ex.getCode());
}
}
@Test
public void testUploadEmptyFile() {
MockMultipartFile file = new MockMultipartFile("file", "empty.mp3", "audio/mpeg", new byte[0]);
BusinessException ex = assertThrows(BusinessException.class, () -> {
ingestController.upload(file, "album/empty.mp3");
});
assertEquals(400, ex.getCode());
assertTrue(ex.getMessage().contains("文件为空"));
}
@Test
public void testUploadUnsupportedExtension() {
MockMultipartFile file = new MockMultipartFile("file", "test.exe", "application/octet-stream", "testdata".getBytes());
BusinessException ex = assertThrows(BusinessException.class, () -> {
ingestController.upload(file, "test.exe");
});
assertEquals(400, ex.getCode());
assertTrue(ex.getMessage().contains("不支持"));
}
@Test
public void testUploadSymlinkRejected() throws Exception {
Path targetDir = inputDir.resolve("album");
Files.createDirectories(targetDir);
Path realFile = tempDir.resolve("real.txt");
Files.write(realFile, "test".getBytes());
Path symlink = targetDir.resolve("link");
try {
Files.createSymbolicLink(symlink, realFile);
} catch (UnsupportedOperationException | IOException e) {
// Ignore on platforms without symlink support
return;
}
MockMultipartFile file = new MockMultipartFile("file", "test.mp3", "audio/mpeg", "testdata".getBytes());
BusinessException ex = assertThrows(BusinessException.class, () -> {
ingestController.upload(file, "album/link/test.mp3");
});
assertEquals(400, ex.getCode());
assertTrue(ex.getMessage().contains("符号链接"));
}
@Test
public void testUploadExtensionFromRelativePath() throws Exception {
MockMultipartFile file = new MockMultipartFile("file", "test.txt", "audio/mpeg", "testdata".getBytes());
com.music.common.Result<com.music.dto.UploadFileResponse> result = ingestController.upload(file, "album/test.mp3");
assertEquals(0, result.getCode());
assertEquals("uploaded", result.getData().getStatus());
}
static class BlockingMultipartFile extends MockMultipartFile {
private final CountDownLatch startLatch;
private final CountDownLatch endLatch;
public BlockingMultipartFile(String name, String originalFilename, String contentType, byte[] content, CountDownLatch startLatch, CountDownLatch endLatch) {
super(name, originalFilename, contentType, content);
this.startLatch = startLatch;
this.endLatch = endLatch;
}
@Override
public byte[] getBytes() throws IOException {
block();
return super.getBytes();
}
@Override
public java.io.InputStream getInputStream() throws IOException {
block();
return super.getInputStream();
}
@Override
public void transferTo(java.io.File dest) throws IOException, IllegalStateException {
block();
super.transferTo(dest);
}
@Override
public void transferTo(Path dest) throws IOException, IllegalStateException {
block();
super.transferTo(dest);
}
private void block() {
startLatch.countDown();
try {
endLatch.await();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
}
@Test
public void testConcurrencyStartUpload() throws InterruptedException, Exception {
CountDownLatch startLatch = new CountDownLatch(1);
CountDownLatch endLatch = new CountDownLatch(1);
ExecutorService executor = Executors.newFixedThreadPool(1);
BlockingMultipartFile blockingFile = new BlockingMultipartFile(
"file", "blocking.mp3", "audio/mpeg", "data".getBytes(), startLatch, endLatch);
executor.submit(() -> {
try {
ingestController.upload(blockingFile, "blocking.mp3");
} catch (Exception ignored) {
}
});
assertTrue(startLatch.await(5, java.util.concurrent.TimeUnit.SECONDS));
BusinessException ex = assertThrows(BusinessException.class, () -> {
ingestController.start(new IngestRequest());
});
assertEquals(409, ex.getCode());
endLatch.countDown();
executor.shutdown();
executor.awaitTermination(5, java.util.concurrent.TimeUnit.SECONDS);
com.music.common.Result<?> res = ingestController.start(new IngestRequest());
assertEquals(0, res.getCode());
}
@Test
public void testEarlyFailureDoesNotBlockStart() throws Exception {
// missing config
when(configService.getInputDir()).thenReturn(null);
MockMultipartFile file = new MockMultipartFile("file", "test.mp3", "audio/mpeg", "testdata".getBytes());
assertThrows(BusinessException.class, () -> ingestController.upload(file, "album/test.mp3"));
when(configService.getInputDir()).thenReturn(inputDir.toString());
// empty file
MockMultipartFile emptyFile = new MockMultipartFile("file", "test.mp3", "audio/mpeg", new byte[0]);
assertThrows(BusinessException.class, () -> ingestController.upload(emptyFile, "album/test.mp3"));
// invalid path
assertThrows(BusinessException.class, () -> ingestController.upload(file, "../test.mp3"));
// symlink rejected
Path targetDir = inputDir.resolve("album");
Files.createDirectories(targetDir);
Path realFile = tempDir.resolve("real.txt");
Files.write(realFile, "test".getBytes());
Path symlink = targetDir.resolve("link");
try {
Files.createSymbolicLink(symlink, realFile);
assertThrows(BusinessException.class, () -> ingestController.upload(file, "album/link/test.mp3"));
} catch (UnsupportedOperationException | IOException ignored) {
}
// skipped existing
Path targetPath = inputDir.resolve("album/test.mp3");
Files.createDirectories(targetPath.getParent());
Files.write(targetPath, "existing".getBytes());
ingestController.upload(file, "album/test.mp3");
// Verify start is not blocked
com.music.common.Result<?> res = ingestController.start(new IngestRequest());
assertEquals(0, res.getCode());
}
@Test
public void testConcurrentUploadNoOverwrite() throws Exception {
CountDownLatch transferLatch = new CountDownLatch(2);
CountDownLatch releaseLatch = new CountDownLatch(1);
class SyncFile extends MockMultipartFile {
public SyncFile(String name, String originalFilename, String contentType, byte[] content) {
super(name, originalFilename, contentType, content);
}
@Override
public void transferTo(java.io.File dest) throws IOException, IllegalStateException {
super.transferTo(dest);
transferLatch.countDown();
try { releaseLatch.await(); } catch (InterruptedException e) {}
}
@Override
public void transferTo(Path dest) throws IOException, IllegalStateException {
super.transferTo(dest);
transferLatch.countDown();
try { releaseLatch.await(); } catch (InterruptedException e) {}
}
}
SyncFile file1 = new SyncFile("file", "test.mp3", "audio/mpeg", "content_A".getBytes());
SyncFile file2 = new SyncFile("file", "test.mp3", "audio/mpeg", "content_B".getBytes());
ExecutorService executor = Executors.newFixedThreadPool(2);
java.util.concurrent.Future<com.music.common.Result<com.music.dto.UploadFileResponse>> f1 = executor.submit(() -> ingestController.upload(file1, "concurrent/test.mp3"));
java.util.concurrent.Future<com.music.common.Result<com.music.dto.UploadFileResponse>> f2 = executor.submit(() -> ingestController.upload(file2, "concurrent/test.mp3"));
assertTrue(transferLatch.await(5, java.util.concurrent.TimeUnit.SECONDS));
releaseLatch.countDown();
com.music.common.Result<com.music.dto.UploadFileResponse> res1 = f1.get();
com.music.common.Result<com.music.dto.UploadFileResponse> res2 = f2.get();
executor.shutdown();
boolean r1Uploaded = "uploaded".equals(res1.getData().getStatus());
boolean r2Uploaded = "uploaded".equals(res2.getData().getStatus());
assertTrue(r1Uploaded ^ r2Uploaded, "Exactly one upload should succeed");
Path targetPath = inputDir.resolve("concurrent/test.mp3");
String content = new String(Files.readAllBytes(targetPath));
if (r1Uploaded) assertEquals("content_A", content);
else assertEquals("content_B", content);
}
}
@@ -114,7 +114,8 @@ class ProgressMessageMappingTest {
mock(IngestService.class), mock(IngestService.class),
new ProgressStore(), new ProgressStore(),
cfgService, cfgService,
mock(com.music.service.IngestTaskStore.class)); mock(com.music.service.IngestTaskStore.class),
"/tmp");
// 模拟配置已设置 // 模拟配置已设置
com.music.dto.ConfigResponse cfgResp = new com.music.dto.ConfigResponse(); com.music.dto.ConfigResponse cfgResp = new com.music.dto.ConfigResponse();
@@ -148,7 +149,8 @@ class ProgressMessageMappingTest {
mock(IngestService.class), mock(IngestService.class),
store, store,
cfgService, cfgService,
mock(com.music.service.IngestTaskStore.class)); mock(com.music.service.IngestTaskStore.class),
"/tmp");
java.lang.reflect.Field runningField = com.music.controller.IngestController.class.getDeclaredField("running"); java.lang.reflect.Field runningField = com.music.controller.IngestController.class.getDeclaredField("running");
runningField.setAccessible(true); runningField.setAccessible(true);
@@ -169,7 +171,7 @@ class ProgressMessageMappingTest {
ProgressStore store = new ProgressStore(); ProgressStore store = new ProgressStore();
com.music.controller.IngestController controller = new com.music.controller.IngestController( com.music.controller.IngestController controller = new com.music.controller.IngestController(
mock(IngestService.class), store, cfgService, mock(IngestService.class), store, cfgService,
mock(com.music.service.IngestTaskStore.class)); mock(com.music.service.IngestTaskStore.class), "/tmp");
com.music.common.Result<com.music.dto.IngestStatusResponse> result = com.music.common.Result<com.music.dto.IngestStatusResponse> result =
controller.statusById("expired-task"); controller.statusById("expired-task");
@@ -203,8 +205,7 @@ class ProgressMessageMappingTest {
// 新进程的控制器:ProgressStore 为空(内存进度已丢失),currentTaskId 为 null // 新进程的控制器:ProgressStore 为空(内存进度已丢失),currentTaskId 为 null
ProgressStore store = new ProgressStore(); ProgressStore store = new ProgressStore();
com.music.controller.IngestController controller = new com.music.controller.IngestController( com.music.controller.IngestController controller = new com.music.controller.IngestController(mock(IngestService.class), store, cfgService, fresh, "/tmp");
mock(IngestService.class), store, cfgService, fresh);
com.music.common.Result<com.music.dto.IngestStatusResponse> result = controller.statusCurrent(); com.music.common.Result<com.music.dto.IngestStatusResponse> result = controller.statusCurrent();
com.music.dto.IngestStatusResponse resp = result.getData(); com.music.dto.IngestStatusResponse resp = result.getData();
+4
View File
@@ -186,3 +186,7 @@ docker compose logs -f mangtool
- `MANGTOOL_MB_BASE_URL` (default: https://musicbrainz.org/ws/2) - `MANGTOOL_MB_BASE_URL` (default: https://musicbrainz.org/ws/2)
- `MANGTOOL_CAA_BASE_URL` (default: https://coverartarchive.org) - `MANGTOOL_CAA_BASE_URL` (default: https://coverartarchive.org)
- `MANGTOOL_MB_USER_AGENT` (default: MangTool/1.0 (https://gitea.mangmang.fun/LiuMangMang/MyTool)) - User-Agent for MusicBrainz API requests - `MANGTOOL_MB_USER_AGENT` (default: MangTool/1.0 (https://gitea.mangmang.fun/LiuMangMang/MyTool)) - User-Agent for MusicBrainz API requests
### 环境变量说明
- `MANGTOOL_UPLOAD_TEMP`: 前端文件上传的临时存放目录,默认值为 `/home/mangtool/MusicWork/.upload-tmp`。
+1
View File
@@ -13,6 +13,7 @@ services:
volumes: volumes:
- ${MANGTOOL_DATA_DIR:-./data}:/home/mangtool/MusicWork - ${MANGTOOL_DATA_DIR:-./data}:/home/mangtool/MusicWork
environment: environment:
- MANGTOOL_UPLOAD_TEMP=${MANGTOOL_UPLOAD_TEMP:-/home/mangtool/MusicWork/.upload-tmp}
- MANGTOOL_COVER_ENABLED=${MANGTOOL_COVER_ENABLED:-false} - MANGTOOL_COVER_ENABLED=${MANGTOOL_COVER_ENABLED:-false}
- MANGTOOL_MB_BASE_URL=${MANGTOOL_MB_BASE_URL:-https://musicbrainz.org/ws/2} - MANGTOOL_MB_BASE_URL=${MANGTOOL_MB_BASE_URL:-https://musicbrainz.org/ws/2}
- MANGTOOL_CAA_BASE_URL=${MANGTOOL_CAA_BASE_URL:-https://coverartarchive.org} - MANGTOOL_CAA_BASE_URL=${MANGTOOL_CAA_BASE_URL:-https://coverartarchive.org}
+17
View File
@@ -1,4 +1,5 @@
import request from './request'; import request from './request';
import type { AxiosRequestConfig } from 'axios';
export interface IngestResponse { export interface IngestResponse {
taskId: string; taskId: string;
@@ -59,3 +60,19 @@ export function getIngestCurrentStatus(): Promise<IngestStatusResponse> {
export function getIngestStatus(taskId: string): Promise<IngestStatusResponse> { export function getIngestStatus(taskId: string): Promise<IngestStatusResponse> {
return request.get(`/api/ingest/status/${taskId}`); return request.get(`/api/ingest/status/${taskId}`);
} }
export interface UploadFileResponse {
status: 'uploaded' | 'skipped';
relativePath: string;
size: number;
}
/**
* 上传文件到输入目录
*/
export function uploadFile(file: File, relativePath: string, config?: AxiosRequestConfig): Promise<UploadFileResponse> {
const formData = new FormData();
formData.append('file', file);
formData.append('relativePath', relativePath);
return request.post('/api/ingest/upload', formData, config);
}
+11 -5
View File
@@ -1,12 +1,18 @@
import axios from 'axios'; import axios from 'axios';
import { ElMessage } from 'element-plus'; import { ElMessage } from 'element-plus';
export class BusinessError extends Error {
code: number;
constructor(message: string, code: number) {
super(message);
this.code = code;
this.name = 'BusinessError';
}
}
const request = axios.create({ const request = axios.create({
baseURL: import.meta.env.VITE_API_BASE_URL ?? 'http://localhost:8080', baseURL: import.meta.env.VITE_API_BASE_URL ?? 'http://localhost:8080',
timeout: 30000, timeout: 30000
headers: {
'Content-Type': 'application/json'
}
}); });
// 响应拦截器:统一处理 Result<T> 格式 // 响应拦截器:统一处理 Result<T> 格式
@@ -21,7 +27,7 @@ request.interceptors.response.use(
} else { } else {
// 失败,显示错误信息并 reject // 失败,显示错误信息并 reject
ElMessage.error(result.message || '请求失败'); ElMessage.error(result.message || '请求失败');
return Promise.reject(new Error(result.message || '请求失败')); return Promise.reject(new BusinessError(result.message || '请求失败', result.code));
} }
} }
// 如果不是 Result 格式,直接返回 // 如果不是 Result 格式,直接返回
+374 -2
View File
@@ -108,6 +108,66 @@
<!-- 操作按钮 --> <!-- 操作按钮 -->
<div class="control-actions"> <div class="control-actions">
<div class="upload-area">
<input type="file" webkitdirectory multiple ref="uploadInput" style="display: none" @change="onFilesSelected" />
<button
v-if="!uploadState.uploading && !running"
class="btn btn-primary upload-btn"
type="button"
:disabled="!canStart"
@click="openDirectoryPicker"
>
<app-icon name="upload" :size="16" />
<span>选择本地目录上传并入库</span>
</button>
<div v-if="uploadState.uploading" class="upload-progress-box">
<div class="upload-header">
<span>上传进度 ({{ uploadState.uploaded + uploadState.skipped }}/{{ uploadState.totalAudio + uploadState.totalLrc }})</span>
<button class="icon-btn cancel-btn" @click="cancelUpload" title="取消上传">
<app-icon name="close" :size="14" />
</button>
</div>
<div class="progress-track">
<div class="progress-fill tone-ok" :style="{ width: uploadState.percentage + '%' }"></div>
</div>
<div class="upload-stats">
<span>当前: {{ uploadState.currentFile }}</span>
<div class="stats-row">
<span>音频: {{ uploadState.totalAudio }}</span>
<span>歌词: {{ uploadState.totalLrc }}</span>
<span>忽略: {{ uploadState.ignored }}</span>
<span>大小: {{ uploadState.formattedSize }}</span>
</div>
<div class="stats-row">
<span class="c-ok">上传: {{ uploadState.uploaded }}</span>
<span class="c-warn">跳过: {{ uploadState.skipped }}</span>
<span class="c-error">失败: {{ uploadState.failed }}</span>
</div>
</div>
</div>
<!-- 失败文件列表及重试 -->
<div v-if="!uploadState.uploading && !running && failedUploads.length > 0" class="failed-uploads-box">
<div class="failed-header">
<span class="c-error">上传失败 ({{ failedUploads.length }})</span>
<button
class="btn btn-ghost btn-sm"
type="button"
@click="retryFailedUploads"
>
<app-icon name="refresh" :size="14" />
<span>重试</span>
</button>
</div>
<div class="failed-list c-scroll">
<div v-for="fail in failedUploads" :key="fail.relPath" class="failed-item" :title="fail.relPath">
{{ fail.relPath }}
</div>
</div>
</div>
</div>
<button <button
v-if="!running" v-if="!running"
class="btn btn-primary" class="btn btn-primary"
@@ -332,6 +392,7 @@
<script setup lang="ts"> <script setup lang="ts">
import { computed, nextTick, onMounted, onUnmounted, reactive, ref } from 'vue'; import { computed, nextTick, onMounted, onUnmounted, reactive, ref } from 'vue';
import { ElMessage } from 'element-plus'; import { ElMessage } from 'element-plus';
import axios from 'axios';
import AppIcon from './common/AppIcon.vue'; import AppIcon from './common/AppIcon.vue';
import { import {
startIngest as apiStartIngest, startIngest as apiStartIngest,
@@ -340,10 +401,15 @@ import {
getIngestStatus, getIngestStatus,
getIngestCurrentStatus getIngestCurrentStatus
} from '../api/ingest'; } from '../api/ingest';
import type { IngestStatusResponse } from '../api/ingest'; import type { IngestStatusResponse, UploadFileResponse } from '../api/ingest';
import { useWebSocket } from '../composables/useWebSocket'; import { useWebSocket } from '../composables/useWebSocket';
import type { ProgressMessage } from '../composables/useWebSocket'; import type { ProgressMessage } from '../composables/useWebSocket';
import { useAppState } from '../composables/useAppState'; import { useAppState } from '../composables/useAppState';
import { BusinessError } from '../api/request';
import type { AxiosProgressEvent } from 'axios';
import { uploadFile } from '../api/ingest';
interface IngestProgressMessage extends ProgressMessage { interface IngestProgressMessage extends ProgressMessage {
lyricsFound?: number; lyricsFound?: number;
@@ -393,7 +459,7 @@ const progress = reactive<ProgressMessage>({
otherRejectedFiles: 0 otherRejectedFiles: 0
}); });
const canStart = computed(() => isConfigured.value && !submitting.value); const canStart = computed(() => isConfigured.value && !submitting.value && !uploadState.uploading);
/** /**
* 歌词统计(独立于 ProgressMessage 类型,避免修改共享类型)。 * 歌词统计(独立于 ProgressMessage 类型,避免修改共享类型)。
@@ -696,6 +762,240 @@ function stopPolling() {
} }
} }
/* ---------------- 上传与自动入库逻辑 ---------------- */
const uploadInput = ref<HTMLInputElement | null>(null);
function openDirectoryPicker() {
if (uploadInput.value) {
uploadInput.value.click();
}
}
const uploadState = reactive({
uploading: false,
totalAudio: 0,
totalLrc: 0,
ignored: 0,
totalBytes: 0,
uploadedBytes: 0,
uploaded: 0,
skipped: 0,
failed: 0,
currentFile: '',
percentage: 0,
formattedSize: '0 B'
});
let uploadAbortController: AbortController | null = null;
function formatBytes(bytes: number) {
if (bytes === 0) return '0 B';
const k = 1024;
const sizes = ['B', 'KB', 'MB', 'GB'];
const i = Math.floor(Math.log(bytes) / Math.log(k));
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
}
const SUPPORTED_AUDIO_EXTS = ['.mp3', '.flac', '.wav', '.m4a', '.ogg', '.opus', '.wma', '.ape', '.aiff', '.aif', '.wv', '.tta'];
const failedUploads = ref<{ file: File, relPath: string }[]>([]);
async function onFilesSelected(e: Event) {
const target = e.target as HTMLInputElement;
const files = Array.from(target.files || []);
if (!files.length) return;
target.value = ''; // reset
// 1. 过滤和统计
const toUpload: { file: File, relPath: string }[] = [];
let totalAudio = 0;
let totalLrc = 0;
let ignored = 0;
let totalBytes = 0;
for (const file of files) {
const name = file.name.toLowerCase();
const isAudio = SUPPORTED_AUDIO_EXTS.some(ext => name.endsWith(ext));
const isLrc = name.endsWith('.lrc');
if (isAudio || isLrc) {
if (isAudio) totalAudio++;
if (isLrc) totalLrc++;
totalBytes += file.size;
// Strip root directory
const pathParts = file.webkitRelativePath.split('/');
pathParts.shift(); // remove root dir
const relPath = pathParts.join('/');
toUpload.push({ file, relPath });
} else {
ignored++;
}
}
if (totalAudio === 0) {
ElMessage.warning('选中的目录未包含支持的音频文件');
return;
}
// 2. 初始化状态
uploadState.uploading = true;
uploadState.totalAudio = totalAudio;
uploadState.totalLrc = totalLrc;
uploadState.ignored = ignored;
uploadState.totalBytes = totalBytes;
uploadState.uploadedBytes = 0;
uploadState.uploaded = 0;
uploadState.skipped = 0;
uploadState.failed = 0;
uploadState.percentage = 0;
uploadState.formattedSize = formatBytes(totalBytes);
uploadAbortController = new AbortController();
failedUploads.value = []; // Reset previous failures on new full upload
await doUploadLoop(toUpload, totalAudio, totalLrc, totalBytes);
}
async function doUploadLoop(items: { file: File, relPath: string }[], totalAudio: number, totalLrc: number, totalBytes: number) {
let hasAudioSuccess = false;
let isCancelled = false;
let previousFilesBytes = uploadState.uploadedBytes;
// 3. 串行上传
for (const item of items) {
if (uploadAbortController.signal.aborted) {
isCancelled = true;
break;
}
uploadState.currentFile = item.relPath;
let success = false;
let attempts = 0;
while (attempts < 3) {
if (uploadAbortController.signal.aborted) break;
attempts++;
try {
const resp = await uploadFile(item.file, item.relPath, {
signal: uploadAbortController.signal,
timeout: 0, // No timeout
onUploadProgress: (pEvent: AxiosProgressEvent) => {
if (pEvent.loaded) {
uploadState.uploadedBytes = previousFilesBytes + pEvent.loaded;
if (totalBytes > 0) {
uploadState.percentage = Math.floor((uploadState.uploadedBytes / totalBytes) * 100);
}
}
}
});
if (resp && resp.status === 'skipped') {
uploadState.skipped++;
} else {
uploadState.uploaded++;
}
previousFilesBytes += item.file.size;
uploadState.uploadedBytes = previousFilesBytes;
if (totalBytes > 0) {
uploadState.percentage = Math.floor((uploadState.uploadedBytes / totalBytes) * 100);
}
if (SUPPORTED_AUDIO_EXTS.some(ext => item.file.name.toLowerCase().endsWith(ext))) {
hasAudioSuccess = true;
}
success = true;
break; // Success
} catch (err: unknown) {
if (axios.isCancel(err) || uploadAbortController.signal.aborted) {
break;
}
let status = 0;
if (axios.isAxiosError(err)) {
status = err.response?.status || 0;
} else if (err instanceof BusinessError) {
status = err.code;
}
if (status === 409) {
ElMessage.error('存在冲突(409),有文件正在处理中,将跳过当前文件');
break;
}
if (status && ((status >= 400 && status < 500 && status !== 408 && status !== 429) || status === 507)) {
// Deterministic error, do not retry
break;
}
// Transient error, wait and retry
if (attempts === 1) await new Promise(r => setTimeout(r, 1000));
else if (attempts === 2) await new Promise(r => setTimeout(r, 3000));
}
}
if (!success && !uploadAbortController?.signal.aborted) {
uploadState.failed++;
failedUploads.value.push(item);
}
}
if (uploadAbortController?.signal.aborted) {
isCancelled = true;
}
uploadState.uploading = false;
uploadAbortController = null;
if (isCancelled) {
ElMessage.info('上传已取消,部分完成的文件已保留在目录中');
} else {
if (uploadState.failed > 0) {
ElMessage.warning(`上传结束:失败 ${uploadState.failed} 个文件,自动入库已取消`);
} else if (hasAudioSuccess) {
ElMessage.success(`上传成功 (音频:${totalAudio}, 歌词:${totalLrc}),即将开始自动入库`);
await startIngest();
} else {
ElMessage.warning('上传结束,但没有成功的音频文件,未自动触发入库');
}
}
}
async function retryFailedUploads() {
if (failedUploads.value.length === 0) return;
const itemsToRetry = [...failedUploads.value];
failedUploads.value = []; // clear current failures list
uploadState.uploading = true;
uploadState.failed = 0; // reset for this retry session
uploadState.uploadedBytes = 0;
uploadState.percentage = 0;
let totalAudio = 0;
let totalLrc = 0;
let totalBytes = 0;
for (const item of itemsToRetry) {
const isLrc = item.file.name.toLowerCase().endsWith('.lrc');
if (isLrc) totalLrc++; else totalAudio++;
totalBytes += item.file.size;
}
uploadState.totalAudio = totalAudio;
uploadState.totalLrc = totalLrc;
uploadState.totalBytes = totalBytes;
uploadState.formattedSize = formatBytes(totalBytes);
uploadState.uploaded = 0;
uploadState.skipped = 0;
uploadAbortController = new AbortController();
await doUploadLoop(itemsToRetry, totalAudio, totalLrc, totalBytes);
}
function cancelUpload() {
if (uploadAbortController) {
uploadAbortController.abort();
}
}
/* ---------------- 启动导入 ---------------- */ /* ---------------- 启动导入 ---------------- */
async function startIngest() { async function startIngest() {
@@ -1680,3 +1980,75 @@ onUnmounted(() => {
} }
} }
</style> </style>
<style scoped>
.upload-area {
margin-bottom: 10px;
}
.upload-progress-box {
padding: 10px;
border: 1px solid var(--c-border);
border-radius: var(--c-radius-sm);
background: var(--c-bg-inset);
margin-top: 10px;
}
.upload-header {
display: flex;
justify-content: space-between;
align-items: center;
font-size: 12px;
font-weight: bold;
margin-bottom: 8px;
}
.upload-stats {
font-size: 11px;
color: var(--c-text-secondary);
margin-top: 8px;
display: flex;
flex-direction: column;
gap: 4px;
}
.stats-row {
display: flex;
gap: 12px;
}
.c-ok { color: var(--c-accent-text); }
.c-warn { color: var(--c-warning-text); }
.c-error { color: var(--c-error-text); }
.failed-uploads-box {
margin-top: 10px;
padding: 10px;
border: 1px solid rgba(244, 63, 94, 0.25);
border-radius: var(--c-radius-sm);
background: rgba(244, 63, 94, 0.05);
display: flex;
flex-direction: column;
gap: 8px;
}
.failed-header {
display: flex;
justify-content: space-between;
align-items: center;
font-size: 12px;
font-weight: bold;
}
.btn-sm {
padding: 4px 8px;
font-size: 11px;
}
.failed-list {
max-height: 120px;
overflow-y: auto;
display: flex;
flex-direction: column;
gap: 4px;
}
.failed-item {
font-size: 11px;
color: var(--c-text-secondary);
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
}
</style>
@@ -62,6 +62,9 @@ const ICONS: Record<string, IconDef> = {
folder: { folder: {
paths: ['M3 7v10a2 2 0 002 2h14a2 2 0 002-2V9a2 2 0 00-2-2h-6l-2-2H5a2 2 0 00-2 2z'] paths: ['M3 7v10a2 2 0 002 2h14a2 2 0 002-2V9a2 2 0 00-2-2h-6l-2-2H5a2 2 0 00-2 2z']
}, },
upload: {
paths: ['M4 16v1a3 3 0 003 3h10a3 3 0 003-3v-1m-4-8l-4-4m0 0L8 8m4-4v12']
},
login: { login: {
paths: [ paths: [
'M11 16l-4-4m0 0l4-4m-4 4h14m-5 4v1a3 3 0 01-3 3H6a3 3 0 01-3-3V7a3 3 0 013-3h7a3 3 0 013 3v1' 'M11 16l-4-4m0 0l4-4m-4 4h14m-5 4v1a3 3 0 01-3 3H6a3 3 0 01-3-3V7a3 3 0 013-3h7a3 3 0 013 3v1'