Template
fix: reject metadata-only lyrics and refresh stale sidecars
This commit is contained in:
@@ -964,7 +964,10 @@ public class IngestService {
|
|||||||
String safeTitle, String originalTitle, String artist) {
|
String safeTitle, String originalTitle, String artist) {
|
||||||
try {
|
try {
|
||||||
String lyrics = tag.getFirst(FieldKey.LYRICS);
|
String lyrics = tag.getFirst(FieldKey.LYRICS);
|
||||||
if (lyrics == null || lyrics.trim().isEmpty()) return false;
|
if (!LyricsService.hasMeaningfulLyrics(lyrics)) {
|
||||||
|
log.debug("忽略无正文的嵌入式歌词: {}", safeTitle);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
// 构建简易 LRC 内容(无时间戳,仅作为歌词文本)
|
// 构建简易 LRC 内容(无时间戳,仅作为歌词文本)
|
||||||
StringBuilder lrcContent = new StringBuilder();
|
StringBuilder lrcContent = new StringBuilder();
|
||||||
|
|||||||
@@ -125,7 +125,9 @@ public class LibraryHealthService {
|
|||||||
String name = f.getFileName().toString();
|
String name = f.getFileName().toString();
|
||||||
if (name.toLowerCase(Locale.ROOT).endsWith(".lrc")) {
|
if (name.toLowerCase(Locale.ROOT).endsWith(".lrc")) {
|
||||||
String base = baseName(name).toLowerCase(Locale.ROOT);
|
String base = baseName(name).toLowerCase(Locale.ROOT);
|
||||||
lrcBaseNames.add(base);
|
if (LyricsService.hasMeaningfulLyrics(f)) {
|
||||||
|
lrcBaseNames.add(base);
|
||||||
|
}
|
||||||
if (!audioBaseNames.contains(base)) {
|
if (!audioBaseNames.contains(base)) {
|
||||||
report.setOrphanSidecars(report.getOrphanSidecars() + 1);
|
report.setOrphanSidecars(report.getOrphanSidecars() + 1);
|
||||||
addExample(report.getExamplesOrphanSidecars(), rel(libraryPath, f));
|
addExample(report.getExamplesOrphanSidecars(), rel(libraryPath, f));
|
||||||
@@ -404,7 +406,7 @@ public class LibraryHealthService {
|
|||||||
|
|
||||||
// ========== 工具 ==========
|
// ========== 工具 ==========
|
||||||
|
|
||||||
/** 大小写不敏感判定音频是否有同名 .lrc 侧车(.lrc/.LRC 等)。 */
|
/** 大小写不敏感判定音频是否有同名且包含正文的 .lrc 侧车。 */
|
||||||
private boolean hasLrcSidecar(Path audio) {
|
private boolean hasLrcSidecar(Path audio) {
|
||||||
Path dir = audio.getParent();
|
Path dir = audio.getParent();
|
||||||
if (dir == null) return false;
|
if (dir == null) return false;
|
||||||
@@ -412,7 +414,8 @@ public class LibraryHealthService {
|
|||||||
try (DirectoryStream<Path> ds = Files.newDirectoryStream(dir)) {
|
try (DirectoryStream<Path> ds = Files.newDirectoryStream(dir)) {
|
||||||
for (Path f : ds) {
|
for (Path f : ds) {
|
||||||
if (Files.isRegularFile(f)
|
if (Files.isRegularFile(f)
|
||||||
&& f.getFileName().toString().toLowerCase(Locale.ROOT).equals(target)) {
|
&& f.getFileName().toString().toLowerCase(Locale.ROOT).equals(target)
|
||||||
|
&& LyricsService.hasMeaningfulLyrics(f)) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import org.slf4j.LoggerFactory;
|
|||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
import java.io.ByteArrayOutputStream;
|
import java.io.ByteArrayOutputStream;
|
||||||
|
import java.io.IOException;
|
||||||
import java.io.InputStream;
|
import java.io.InputStream;
|
||||||
import java.net.HttpURLConnection;
|
import java.net.HttpURLConnection;
|
||||||
import java.net.URL;
|
import java.net.URL;
|
||||||
@@ -19,6 +20,8 @@ import java.util.ArrayList;
|
|||||||
import java.util.Comparator;
|
import java.util.Comparator;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Locale;
|
import java.util.Locale;
|
||||||
|
import java.util.regex.Matcher;
|
||||||
|
import java.util.regex.Pattern;
|
||||||
|
|
||||||
/** Optional lyric lookup. All failures are intentionally non-fatal to ingest. */
|
/** Optional lyric lookup. All failures are intentionally non-fatal to ingest. */
|
||||||
@Service
|
@Service
|
||||||
@@ -26,6 +29,8 @@ public class LyricsService {
|
|||||||
private static final Logger log = LoggerFactory.getLogger(LyricsService.class);
|
private static final Logger log = LoggerFactory.getLogger(LyricsService.class);
|
||||||
private static final ObjectMapper MAPPER = new ObjectMapper();
|
private static final ObjectMapper MAPPER = new ObjectMapper();
|
||||||
private static final int MAX_BYTES = 256 * 1024;
|
private static final int MAX_BYTES = 256 * 1024;
|
||||||
|
private static final Pattern TIMED_LINE = Pattern.compile("^\\s*\\[(?:\\d{1,2}:)?\\d{1,2}:\\d{2}(?:\\.\\d+)?\\]\\s*(.*)$");
|
||||||
|
private static final Pattern META_LINE = Pattern.compile("^(?:作词|作曲|词曲|编曲|制作人?|混音(?:师|助理)?|录音(?:师)?|工程师|弦乐|鼓和钢琴|电贝斯|额外编程|数字编辑|原唱|演唱|出品|发行|版权|produced|written|composer|arranged|lyrics?|music|artist|title)\\s*[::\\-].*$", Pattern.CASE_INSENSITIVE);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 远程歌词解析结果(供入库报告统计使用)。
|
* 远程歌词解析结果(供入库报告统计使用)。
|
||||||
@@ -53,7 +58,7 @@ public class LyricsService {
|
|||||||
public RemoteOutcome resolveRemote(Path targetFile, String title, String artist) {
|
public RemoteOutcome resolveRemote(Path targetFile, String title, String artist) {
|
||||||
if (targetFile == null || title == null || artist == null) return RemoteOutcome.MISSING;
|
if (targetFile == null || title == null || artist == null) return RemoteOutcome.MISSING;
|
||||||
Path lrc = targetFile.resolveSibling(baseName(targetFile.getFileName().toString()) + ".lrc");
|
Path lrc = targetFile.resolveSibling(baseName(targetFile.getFileName().toString()) + ".lrc");
|
||||||
if (Files.isRegularFile(lrc)) return RemoteOutcome.EXISTS;
|
if (Files.isRegularFile(lrc) && hasMeaningfulLyrics(lrc)) return RemoteOutcome.EXISTS;
|
||||||
if (!enabled()) return RemoteOutcome.DISABLED;
|
if (!enabled()) return RemoteOutcome.DISABLED;
|
||||||
try {
|
try {
|
||||||
Candidate c = lookup(title, artist);
|
Candidate c = lookup(title, artist);
|
||||||
@@ -87,7 +92,7 @@ public class LyricsService {
|
|||||||
List<Candidate> candidates = new ArrayList<>();
|
List<Candidate> candidates = new ArrayList<>();
|
||||||
if (root != null && root.isArray()) for (JsonNode n : root) {
|
if (root != null && root.isArray()) for (JsonNode n : root) {
|
||||||
String lyrics = text(n, "lyrics");
|
String lyrics = text(n, "lyrics");
|
||||||
if (lyrics.trim().isEmpty()) continue;
|
if (!hasMeaningfulLyrics(lyrics)) continue;
|
||||||
String ct = text(n, "title"), ca = text(n, "artist");
|
String ct = text(n, "title"), ca = text(n, "artist");
|
||||||
int score = score(title, artist, ct, ca, lyrics, text(n, "type"), n.path("isComplete").asBoolean(false));
|
int score = score(title, artist, ct, ca, lyrics, text(n, "type"), n.path("isComplete").asBoolean(false));
|
||||||
candidates.add(new Candidate(lyrics, text(n, "source"), score));
|
candidates.add(new Candidate(lyrics, text(n, "source"), score));
|
||||||
@@ -104,6 +109,35 @@ public class LyricsService {
|
|||||||
}
|
}
|
||||||
private static String norm(String s) { return s == null ? "" : s.toLowerCase(Locale.ROOT).replaceAll("[((](网易云音乐|酷狗音乐)[))]", "").replaceAll("[^\\p{L}\\p{N}]", ""); }
|
private static String norm(String s) { return s == null ? "" : s.toLowerCase(Locale.ROOT).replaceAll("[((](网易云音乐|酷狗音乐)[))]", "").replaceAll("[^\\p{L}\\p{N}]", ""); }
|
||||||
private static String text(JsonNode n, String k) { return n.path(k).asText(""); }
|
private static String text(JsonNode n, String k) { return n.path(k).asText(""); }
|
||||||
|
static boolean hasMeaningfulLyrics(Path file) {
|
||||||
|
if (file == null || !Files.isRegularFile(file)) return false;
|
||||||
|
try {
|
||||||
|
return hasMeaningfulLyrics(new String(Files.readAllBytes(file), StandardCharsets.UTF_8));
|
||||||
|
} catch (IOException e) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reject metadata-only placeholders such as title/artist lines while accepting
|
||||||
|
* both timestamped LRC and plain embedded lyric text.
|
||||||
|
*/
|
||||||
|
static boolean hasMeaningfulLyrics(String lyrics) {
|
||||||
|
if (lyrics == null) return false;
|
||||||
|
String normalized = lyrics.replace("\uFEFF", "").replace("\r", "");
|
||||||
|
int lines = 0;
|
||||||
|
int chars = 0;
|
||||||
|
for (String raw : normalized.split("\\n")) {
|
||||||
|
String line = raw.trim();
|
||||||
|
if (line.isEmpty() || line.matches("^\\[(?:ti|ar|al|by|re|ve|offset):.*$")) continue;
|
||||||
|
Matcher timed = TIMED_LINE.matcher(line);
|
||||||
|
String body = timed.matches() ? timed.group(1).trim() : line;
|
||||||
|
if (body.isEmpty() || META_LINE.matcher(body).matches()) continue;
|
||||||
|
lines++;
|
||||||
|
chars += body.replaceAll("\\s+", "").length();
|
||||||
|
}
|
||||||
|
return lines >= 2 && chars >= 8;
|
||||||
|
}
|
||||||
private static byte[] readLimited(InputStream in) throws Exception { try (InputStream x = in; ByteArrayOutputStream b = new ByteArrayOutputStream()) { byte[] buf = new byte[8192]; int n, total = 0; while ((n = x.read(buf)) >= 0) { total += n; if (total > MAX_BYTES) throw new IllegalArgumentException("歌词响应过大"); b.write(buf, 0, n); } return b.toByteArray(); } }
|
private static byte[] readLimited(InputStream in) throws Exception { try (InputStream x = in; ByteArrayOutputStream b = new ByteArrayOutputStream()) { byte[] buf = new byte[8192]; int n, total = 0; while ((n = x.read(buf)) >= 0) { total += n; if (total > MAX_BYTES) throw new IllegalArgumentException("歌词响应过大"); b.write(buf, 0, n); } return b.toByteArray(); } }
|
||||||
private static String value(String env, String prop, String def) { String v = System.getenv(env); if (v == null || v.isEmpty()) v = System.getProperty(prop); return v == null || v.isEmpty() ? def : v; }
|
private static String value(String env, String prop, String def) { String v = System.getenv(env); if (v == null || v.isEmpty()) v = System.getProperty(prop); return v == null || v.isEmpty() ? def : v; }
|
||||||
private static String baseName(String n) { int i = n.lastIndexOf('.'); return i > 0 ? n.substring(0, i) : n; }
|
private static String baseName(String n) { int i = n.lastIndexOf('.'); return i > 0 ? n.substring(0, i) : n; }
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ class LibraryHealthServiceTest {
|
|||||||
Path album = root.resolve("Artist").resolve("Album");
|
Path album = root.resolve("Artist").resolve("Album");
|
||||||
Files.createDirectories(album);
|
Files.createDirectories(album);
|
||||||
Files.write(album.resolve("01 - Song.mp3"), new byte[]{1, 2, 3});
|
Files.write(album.resolve("01 - Song.mp3"), new byte[]{1, 2, 3});
|
||||||
Files.write(album.resolve("01 - Song.lrc"), "[00:01]hi".getBytes(StandardCharsets.UTF_8));
|
Files.write(album.resolve("01 - Song.lrc"), "[00:01]first line\n[00:02]second line".getBytes(StandardCharsets.UTF_8));
|
||||||
Files.write(album.resolve("orphan.lrc"), "[00:01]orphan".getBytes(StandardCharsets.UTF_8));
|
Files.write(album.resolve("orphan.lrc"), "[00:01]orphan".getBytes(StandardCharsets.UTF_8));
|
||||||
Files.write(album.resolve("cover.jpg"), new byte[]{(byte) 0xFF, (byte) 0xD8, (byte) 0xFF});
|
Files.write(album.resolve("cover.jpg"), new byte[]{(byte) 0xFF, (byte) 0xD8, (byte) 0xFF});
|
||||||
return album;
|
return album;
|
||||||
@@ -244,7 +244,7 @@ class LibraryHealthServiceTest {
|
|||||||
Files.createDirectories(album);
|
Files.createDirectories(album);
|
||||||
// 大写 .LRC 侧车:既不应算缺歌词,也不应算孤立(与 orphan 判定保持一致)
|
// 大写 .LRC 侧车:既不应算缺歌词,也不应算孤立(与 orphan 判定保持一致)
|
||||||
Files.write(album.resolve("01 - Song.mp3"), new byte[]{1, 2, 3});
|
Files.write(album.resolve("01 - Song.mp3"), new byte[]{1, 2, 3});
|
||||||
Files.write(album.resolve("01 - Song.LRC"), "[00:01]hi".getBytes(StandardCharsets.UTF_8));
|
Files.write(album.resolve("01 - Song.LRC"), "[00:01]first line\n[00:02]second line".getBytes(StandardCharsets.UTF_8));
|
||||||
|
|
||||||
LibraryHealthService svc = newService(root);
|
LibraryHealthService svc = newService(root);
|
||||||
LibraryHealthReport report = svc.scan(false);
|
LibraryHealthReport report = svc.scan(false);
|
||||||
|
|||||||
@@ -22,8 +22,8 @@ class LyricsServiceTest {
|
|||||||
@Test
|
@Test
|
||||||
void selectsExactArtistInsteadOfFirstCoverCandidate() throws Exception {
|
void selectsExactArtistInsteadOfFirstCoverCandidate() throws Exception {
|
||||||
String json = "["
|
String json = "["
|
||||||
+ "{\"title\":\"夜曲 (网易云音乐)\",\"artist\":\"Xai小爱\",\"lyrics\":\"[00:01]翻唱\",\"type\":\"lrc\",\"source\":\"netease\"},"
|
+ "{\"title\":\"夜曲 (网易云音乐)\",\"artist\":\"Xai小爱\",\"lyrics\":\"[00:01]翻唱第一句\\n[00:02]翻唱第二句\",\"type\":\"lrc\",\"source\":\"netease\"},"
|
||||||
+ "{\"title\":\"夜曲 (酷狗音乐)\",\"artist\":\"周杰伦\",\"lyrics\":\"[00:01]一群嗜血的蚂蚁\",\"type\":\"lrc\",\"source\":\"kugou\",\"isComplete\":true}]";
|
+ "{\"title\":\"夜曲 (酷狗音乐)\",\"artist\":\"周杰伦\",\"lyrics\":\"[00:01]一群嗜血的蚂蚁\\n[00:02]被腐肉所吸引\",\"type\":\"lrc\",\"source\":\"kugou\",\"isComplete\":true}]";
|
||||||
start(json, 200);
|
start(json, 200);
|
||||||
LyricsService.Candidate result = new LyricsService().lookup("夜曲", "周杰伦");
|
LyricsService.Candidate result = new LyricsService().lookup("夜曲", "周杰伦");
|
||||||
assertNotNull(result);
|
assertNotNull(result);
|
||||||
@@ -37,6 +37,17 @@ class LyricsServiceTest {
|
|||||||
assertNull(new LyricsService().lookup("夜曲", "周杰伦"));
|
assertNull(new LyricsService().lookup("夜曲", "周杰伦"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void rejectsMetadataOnlyLyrics() {
|
||||||
|
assertFalse(LyricsService.hasMeaningfulLyrics("[00:00.00]特别的人\n[00:00.00]方大同\n"));
|
||||||
|
assertFalse(LyricsService.hasMeaningfulLyrics("[ti:特别的人]\n[ar:方大同]\n"));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void acceptsTimestampedLyrics() {
|
||||||
|
assertTrue(LyricsService.hasMeaningfulLyrics("[00:01.00]第一句歌词\n[00:03.00]第二句歌词\n"));
|
||||||
|
}
|
||||||
|
|
||||||
private void start(String body, int status) throws Exception {
|
private void start(String body, int status) throws Exception {
|
||||||
server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
|
server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
|
||||||
server.createContext("/lrc", exchange -> {
|
server.createContext("/lrc", exchange -> {
|
||||||
|
|||||||
@@ -7,7 +7,25 @@ AUTH="${MANGTOOL_LYRICS_AUTH:-}"
|
|||||||
export BASE_URL AUTH
|
export BASE_URL AUTH
|
||||||
find "$LIBRARY_DIR" -type f \( -iname '*.mp3' -o -iname '*.flac' -o -iname '*.m4a' -o -iname '*.aac' -o -iname '*.ogg' -o -iname '*.opus' -o -iname '*.wma' \) -print0 |
|
find "$LIBRARY_DIR" -type f \( -iname '*.mp3' -o -iname '*.flac' -o -iname '*.m4a' -o -iname '*.aac' -o -iname '*.ogg' -o -iname '*.opus' -o -iname '*.wma' \) -print0 |
|
||||||
while IFS= read -r -d '' audio; do
|
while IFS= read -r -d '' audio; do
|
||||||
lrc="${audio%.*}.lrc"; [[ -s "$lrc" ]] && { echo "SKIP $audio"; continue; }
|
lrc="${audio%.*}.lrc"
|
||||||
|
if [[ -s "$lrc" ]] && python3 - "$lrc" <<'PY'
|
||||||
|
import re,sys
|
||||||
|
text=open(sys.argv[1],encoding='utf-8-sig',errors='replace').read().replace('\r','')
|
||||||
|
meta=re.compile(r'^\[(?:ti|ar|al|by|re|ve|offset):',re.I)
|
||||||
|
timed=re.compile(r'^\s*\[(?:\d{1,2}:)?\d{1,2}:\d{2}(?:\.\d+)?\]\s*(.*)$')
|
||||||
|
labels=re.compile(r'^(?:作词|作曲|词曲|编曲|制作人?|混音(?:师|助理)?|录音(?:师)?|工程师|弦乐|鼓和钢琴|电贝斯|额外编程|数字编辑|原唱|演唱|出品|发行|版权|produced|written|composer|arranged|lyrics?|music|artist|title)\s*[::\-].*$',re.I)
|
||||||
|
body=[]
|
||||||
|
for raw in text.split('\n'):
|
||||||
|
line=raw.strip()
|
||||||
|
if not line or meta.match(line): continue
|
||||||
|
match=timed.match(line); line=(match.group(1).strip() if match else line)
|
||||||
|
if line and not labels.match(line): body.append(line)
|
||||||
|
sys.exit(0 if len(body)>=2 and len(''.join(body).replace(' ',''))>=8 else 1)
|
||||||
|
PY
|
||||||
|
then
|
||||||
|
echo "SKIP $audio"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
meta=$(ffprobe -v error -show_entries format_tags=title,artist -of json "$audio" 2>/dev/null || true)
|
meta=$(ffprobe -v error -show_entries format_tags=title,artist -of json "$audio" 2>/dev/null || true)
|
||||||
result=$(AUDIO_META="$meta" python3 - <<'PY'
|
result=$(AUDIO_META="$meta" python3 - <<'PY'
|
||||||
import json,os,sys,urllib.parse,urllib.request
|
import json,os,sys,urllib.parse,urllib.request
|
||||||
@@ -19,10 +37,22 @@ try:
|
|||||||
with urllib.request.urlopen(req,timeout=15) as r: rows=json.load(r)
|
with urllib.request.urlopen(req,timeout=15) as r: rows=json.load(r)
|
||||||
except Exception: sys.exit(0)
|
except Exception: sys.exit(0)
|
||||||
def n(x): return ''.join(c.lower() for c in x if c.isalnum())
|
def n(x): return ''.join(c.lower() for c in x if c.isalnum())
|
||||||
|
def meaningful(text):
|
||||||
|
import re
|
||||||
|
meta=re.compile(r'^\[(?:ti|ar|al|by|re|ve|offset):',re.I)
|
||||||
|
timed=re.compile(r'^\s*\[(?:\d{1,2}:)?\d{1,2}:\d{2}(?:\.\d+)?\]\s*(.*)$')
|
||||||
|
labels=re.compile(r'^(?:作词|作曲|词曲|编曲|制作人?|混音(?:师|助理)?|录音(?:师)?|工程师|弦乐|鼓和钢琴|电贝斯|额外编程|数字编辑|原唱|演唱|出品|发行|版权|produced|written|composer|arranged|lyrics?|music|artist|title)\s*[::\-].*$',re.I)
|
||||||
|
body=[]
|
||||||
|
for raw in text.replace('\r','').split('\n'):
|
||||||
|
line=raw.strip()
|
||||||
|
if not line or meta.match(line): continue
|
||||||
|
match=timed.match(line); line=(match.group(1).strip() if match else line)
|
||||||
|
if line and not labels.match(line): body.append(line)
|
||||||
|
return len(body)>=2 and len(''.join(body).replace(' ',''))>=8
|
||||||
best=None
|
best=None
|
||||||
for x in rows if isinstance(rows,list) else []:
|
for x in rows if isinstance(rows,list) else []:
|
||||||
l=x.get('lyrics','').strip(); ct=x.get('title',''); ca=x.get('artist','')
|
l=x.get('lyrics','').strip(); ct=x.get('title',''); ca=x.get('artist','')
|
||||||
if not l: continue
|
if not meaningful(l): continue
|
||||||
s=(45 if n(t)==n(ct) else 25 if n(t) in n(ct) or n(ct) in n(t) else 0)+(45 if n(a)==n(ca) else 20 if n(a) in n(ca) or n(ca) in n(a) else -40)
|
s=(45 if n(t)==n(ct) else 25 if n(t) in n(ct) or n(ct) in n(t) else 0)+(45 if n(a)==n(ca) else 20 if n(a) in n(ca) or n(ca) in n(a) else -40)
|
||||||
s+=10 if '[' in l and ':' in l else 0
|
s+=10 if '[' in l and ':' in l else 0
|
||||||
if s>=60 and (best is None or s>best[0]): best=(s,l)
|
if s>=60 and (best is None or s>best[0]): best=(s,l)
|
||||||
|
|||||||
Reference in New Issue
Block a user