liumangmang
313 lines
7.7 KiB
Python
313 lines
7.7 KiB
Python
from datetime import datetime, timezone
|
|
|
|
|
|
TASK_STATUS_PENDING = 'pending'
|
|
TASK_STATUS_RUNNING = 'running'
|
|
TASK_STATUS_COMPLETED = 'completed'
|
|
TASK_STATUS_FAILED = 'failed'
|
|
TASK_TYPE_INGEST = 'ingest'
|
|
TASK_TYPE_REPAIR = 'repair'
|
|
|
|
STAGE_STATUS_PENDING = 'pending'
|
|
STAGE_STATUS_RUNNING = 'running'
|
|
STAGE_STATUS_COMPLETED = 'completed'
|
|
STAGE_STATUS_SKIPPED = 'skipped'
|
|
STAGE_STATUS_FAILED = 'failed'
|
|
|
|
ACTIVE_TASK_STATUSES = (TASK_STATUS_PENDING, TASK_STATUS_RUNNING)
|
|
SCAN_PROGRESS_LOG_LIMIT = 50
|
|
SCAN_PROGRESS_BATCH_SIZE = 20
|
|
SCAN_PROGRESS_INTERVAL_SECONDS = 1.0
|
|
PREPROCESS_PROGRESS_BATCH_SIZE = 20
|
|
PREPROCESS_PROGRESS_INTERVAL_SECONDS = 1.0
|
|
MATCH_PROGRESS_BATCH_SIZE = 20
|
|
MATCH_PROGRESS_INTERVAL_SECONDS = 1.0
|
|
DEDUPE_PROGRESS_BATCH_SIZE = 20
|
|
DEDUPE_PROGRESS_INTERVAL_SECONDS = 1.0
|
|
ORGANIZE_PROGRESS_BATCH_SIZE = 20
|
|
ORGANIZE_PROGRESS_INTERVAL_SECONDS = 1.0
|
|
TASK_WORKSPACE_ROOT = '/tmp/musicworkshop/tasks'
|
|
|
|
STAGE_IDS = (
|
|
'scan',
|
|
'preprocess',
|
|
'match',
|
|
'dedupe',
|
|
'organize',
|
|
'complete'
|
|
)
|
|
|
|
REPAIR_STAGE_IDS = (
|
|
'prepare',
|
|
'execute',
|
|
'complete'
|
|
)
|
|
|
|
|
|
def current_timestamp() -> str:
|
|
return (
|
|
datetime.now(timezone.utc)
|
|
.replace(microsecond=0)
|
|
.isoformat()
|
|
.replace('+00:00', 'Z')
|
|
)
|
|
|
|
|
|
def create_pending_stage_states() -> dict[str, str]:
|
|
return {stage_id: STAGE_STATUS_PENDING for stage_id in STAGE_IDS}
|
|
|
|
|
|
def create_pending_repair_stage_states() -> dict[str, str]:
|
|
return {stage_id: STAGE_STATUS_PENDING for stage_id in REPAIR_STAGE_IDS}
|
|
|
|
|
|
def create_scan_failed_stage_states() -> dict[str, str]:
|
|
return {
|
|
'scan': STAGE_STATUS_FAILED,
|
|
'preprocess': STAGE_STATUS_PENDING,
|
|
'match': STAGE_STATUS_PENDING,
|
|
'dedupe': STAGE_STATUS_PENDING,
|
|
'organize': STAGE_STATUS_PENDING,
|
|
'complete': STAGE_STATUS_FAILED
|
|
}
|
|
|
|
|
|
def create_scan_completed_stage_states() -> dict[str, str]:
|
|
return {
|
|
'scan': STAGE_STATUS_COMPLETED,
|
|
'preprocess': STAGE_STATUS_PENDING,
|
|
'match': STAGE_STATUS_PENDING,
|
|
'dedupe': STAGE_STATUS_PENDING,
|
|
'organize': STAGE_STATUS_PENDING,
|
|
'complete': STAGE_STATUS_PENDING
|
|
}
|
|
|
|
|
|
def create_preprocess_running_stage_states() -> dict[str, str]:
|
|
return {
|
|
'scan': STAGE_STATUS_COMPLETED,
|
|
'preprocess': STAGE_STATUS_RUNNING,
|
|
'match': STAGE_STATUS_PENDING,
|
|
'dedupe': STAGE_STATUS_PENDING,
|
|
'organize': STAGE_STATUS_PENDING,
|
|
'complete': STAGE_STATUS_PENDING
|
|
}
|
|
|
|
|
|
def create_preprocess_completed_stage_states() -> dict[str, str]:
|
|
return {
|
|
'scan': STAGE_STATUS_COMPLETED,
|
|
'preprocess': STAGE_STATUS_COMPLETED,
|
|
'match': STAGE_STATUS_PENDING,
|
|
'dedupe': STAGE_STATUS_PENDING,
|
|
'organize': STAGE_STATUS_PENDING,
|
|
'complete': STAGE_STATUS_PENDING
|
|
}
|
|
|
|
|
|
def create_preprocess_failed_stage_states() -> dict[str, str]:
|
|
return {
|
|
'scan': STAGE_STATUS_COMPLETED,
|
|
'preprocess': STAGE_STATUS_FAILED,
|
|
'match': STAGE_STATUS_PENDING,
|
|
'dedupe': STAGE_STATUS_PENDING,
|
|
'organize': STAGE_STATUS_PENDING,
|
|
'complete': STAGE_STATUS_FAILED
|
|
}
|
|
|
|
|
|
def create_match_running_stage_states() -> dict[str, str]:
|
|
return {
|
|
'scan': STAGE_STATUS_COMPLETED,
|
|
'preprocess': STAGE_STATUS_COMPLETED,
|
|
'match': STAGE_STATUS_RUNNING,
|
|
'dedupe': STAGE_STATUS_PENDING,
|
|
'organize': STAGE_STATUS_PENDING,
|
|
'complete': STAGE_STATUS_PENDING
|
|
}
|
|
|
|
|
|
def create_match_completed_stage_states() -> dict[str, str]:
|
|
return {
|
|
'scan': STAGE_STATUS_COMPLETED,
|
|
'preprocess': STAGE_STATUS_COMPLETED,
|
|
'match': STAGE_STATUS_COMPLETED,
|
|
'dedupe': STAGE_STATUS_PENDING,
|
|
'organize': STAGE_STATUS_PENDING,
|
|
'complete': STAGE_STATUS_PENDING
|
|
}
|
|
|
|
|
|
def create_match_failed_stage_states() -> dict[str, str]:
|
|
return {
|
|
'scan': STAGE_STATUS_COMPLETED,
|
|
'preprocess': STAGE_STATUS_COMPLETED,
|
|
'match': STAGE_STATUS_FAILED,
|
|
'dedupe': STAGE_STATUS_PENDING,
|
|
'organize': STAGE_STATUS_PENDING,
|
|
'complete': STAGE_STATUS_FAILED
|
|
}
|
|
|
|
|
|
def create_dedupe_running_stage_states() -> dict[str, str]:
|
|
return {
|
|
'scan': STAGE_STATUS_COMPLETED,
|
|
'preprocess': STAGE_STATUS_COMPLETED,
|
|
'match': STAGE_STATUS_COMPLETED,
|
|
'dedupe': STAGE_STATUS_RUNNING,
|
|
'organize': STAGE_STATUS_PENDING,
|
|
'complete': STAGE_STATUS_PENDING
|
|
}
|
|
|
|
|
|
def create_dedupe_completed_stage_states() -> dict[str, str]:
|
|
return {
|
|
'scan': STAGE_STATUS_COMPLETED,
|
|
'preprocess': STAGE_STATUS_COMPLETED,
|
|
'match': STAGE_STATUS_COMPLETED,
|
|
'dedupe': STAGE_STATUS_COMPLETED,
|
|
'organize': STAGE_STATUS_PENDING,
|
|
'complete': STAGE_STATUS_PENDING
|
|
}
|
|
|
|
|
|
def create_dedupe_failed_stage_states() -> dict[str, str]:
|
|
return {
|
|
'scan': STAGE_STATUS_COMPLETED,
|
|
'preprocess': STAGE_STATUS_COMPLETED,
|
|
'match': STAGE_STATUS_COMPLETED,
|
|
'dedupe': STAGE_STATUS_FAILED,
|
|
'organize': STAGE_STATUS_PENDING,
|
|
'complete': STAGE_STATUS_FAILED
|
|
}
|
|
|
|
|
|
def create_organize_running_stage_states() -> dict[str, str]:
|
|
return {
|
|
'scan': STAGE_STATUS_COMPLETED,
|
|
'preprocess': STAGE_STATUS_COMPLETED,
|
|
'match': STAGE_STATUS_COMPLETED,
|
|
'dedupe': STAGE_STATUS_COMPLETED,
|
|
'organize': STAGE_STATUS_RUNNING,
|
|
'complete': STAGE_STATUS_PENDING
|
|
}
|
|
|
|
|
|
def create_organize_completed_stage_states() -> dict[str, str]:
|
|
return {
|
|
'scan': STAGE_STATUS_COMPLETED,
|
|
'preprocess': STAGE_STATUS_COMPLETED,
|
|
'match': STAGE_STATUS_COMPLETED,
|
|
'dedupe': STAGE_STATUS_COMPLETED,
|
|
'organize': STAGE_STATUS_COMPLETED,
|
|
'complete': STAGE_STATUS_PENDING
|
|
}
|
|
|
|
|
|
def create_organize_failed_stage_states() -> dict[str, str]:
|
|
return {
|
|
'scan': STAGE_STATUS_COMPLETED,
|
|
'preprocess': STAGE_STATUS_COMPLETED,
|
|
'match': STAGE_STATUS_COMPLETED,
|
|
'dedupe': STAGE_STATUS_COMPLETED,
|
|
'organize': STAGE_STATUS_FAILED,
|
|
'complete': STAGE_STATUS_FAILED
|
|
}
|
|
|
|
|
|
def create_task_completed_stage_states() -> dict[str, str]:
|
|
return {
|
|
'scan': STAGE_STATUS_COMPLETED,
|
|
'preprocess': STAGE_STATUS_COMPLETED,
|
|
'match': STAGE_STATUS_COMPLETED,
|
|
'dedupe': STAGE_STATUS_COMPLETED,
|
|
'organize': STAGE_STATUS_COMPLETED,
|
|
'complete': STAGE_STATUS_COMPLETED
|
|
}
|
|
|
|
|
|
def create_empty_scan_stats() -> dict[str, int]:
|
|
return {
|
|
'total_found': 0,
|
|
'queued': 0,
|
|
'skipped_locked': 0,
|
|
'skipped_invalid': 0,
|
|
'ignored_non_audio': 0
|
|
}
|
|
|
|
|
|
def create_empty_preprocess_stats() -> dict[str, int]:
|
|
return {
|
|
'input_items': 0,
|
|
'output_items': 0,
|
|
'split_parents': 0,
|
|
'generated_children': 0,
|
|
'converted_items': 0,
|
|
'metadata_snapshots': 0,
|
|
'fingerprints_ok': 0,
|
|
'fingerprints_failed': 0,
|
|
'failed_items': 0,
|
|
'warning_items': 0
|
|
}
|
|
|
|
|
|
def create_empty_match_stats() -> dict[str, int]:
|
|
return {
|
|
'input_items': 0,
|
|
'matched_authoritative': 0,
|
|
'matched_fallback': 0,
|
|
'low_score': 0,
|
|
'not_found': 0,
|
|
'provider_warnings': 0,
|
|
'failed_items': 0
|
|
}
|
|
|
|
|
|
def create_empty_dedupe_stats() -> dict[str, int]:
|
|
return {
|
|
'input_items': 0,
|
|
'library_candidates': 0,
|
|
'batch_duplicates': 0,
|
|
'library_duplicates': 0,
|
|
'replaced_library_items': 0,
|
|
'kept_items': 0,
|
|
'failed_items': 0
|
|
}
|
|
|
|
|
|
def create_empty_organize_stats() -> dict[str, int]:
|
|
return {
|
|
'input_items': 0,
|
|
'moved_items': 0,
|
|
'renamed_items': 0,
|
|
'collision_resolved': 0,
|
|
'trashed_items': 0,
|
|
'failed_items': 0
|
|
}
|
|
|
|
|
|
def create_empty_task_stats() -> dict[str, dict[str, int]]:
|
|
return {
|
|
'scan': create_empty_scan_stats(),
|
|
'preprocess': create_empty_preprocess_stats(),
|
|
'match': create_empty_match_stats(),
|
|
'dedupe': create_empty_dedupe_stats(),
|
|
'organize': create_empty_organize_stats()
|
|
}
|
|
|
|
|
|
def create_empty_repair_stats() -> dict[str, dict[str, int]]:
|
|
return {
|
|
'prepare': {
|
|
'previewed_items': 0,
|
|
'rejected_items': 0
|
|
},
|
|
'execute': {
|
|
'succeeded_items': 0,
|
|
'failed_items': 0,
|
|
'moved_items': 0,
|
|
'updated_metadata_items': 0,
|
|
'ignored_items': 0
|
|
}
|
|
}
|