refactor: async parallel fetching

This commit is contained in:
2026-04-03 18:39:31 +02:00
parent aab23e6b8d
commit 0eabea6a50
13 changed files with 452 additions and 256 deletions
+38 -1
View File
@@ -4,7 +4,8 @@ Date: 2026-03-25 02:33:26
Description: Fetcher pipeline — registry and types
"""
from typing import Literal
from typing import Literal, Optional
from loguru import logger
from .base import BaseFetcher
from .local import LocalFetcher
@@ -15,6 +16,7 @@ from .lrclib_search import LrclibSearchFetcher
from .netease import NeteaseFetcher
from .qqmusic import QQMusicFetcher
from ..cache import CacheEngine
from ..models import TrackMeta
FetcherMethodType = Literal[
"local",
@@ -26,6 +28,15 @@ FetcherMethodType = Literal[
"qqmusic",
]
# Fetchers within a group run in parallel; groups run sequentially.
# A group that produces any positive result stops the pipeline.
_FETCHER_GROUPS: list[list[FetcherMethodType]] = [
["local"],
["cache-search"],
["spotify", "lrclib"],
["lrclib-search", "netease", "qqmusic"],
]
def create_fetchers(cache: CacheEngine) -> dict[FetcherMethodType, BaseFetcher]:
"""Instantiate all fetchers. Returns a dict keyed by source name."""
@@ -39,3 +50,29 @@ def create_fetchers(cache: CacheEngine) -> dict[FetcherMethodType, BaseFetcher]:
"qqmusic": QQMusicFetcher(),
}
return fetchers
def build_plan(
fetchers: dict[FetcherMethodType, BaseFetcher],
track: TrackMeta,
force_method: Optional[FetcherMethodType] = None,
) -> list[list[BaseFetcher]]:
"""Return the fetch plan as a list of groups (each group runs in parallel)."""
if force_method:
if force_method not in fetchers:
logger.error(f"Unknown method: {force_method}")
return []
return [[fetchers[force_method]]]
plan: list[list[BaseFetcher]] = []
for group_methods in _FETCHER_GROUPS:
group = [
fetchers[m]
for m in group_methods
if m in fetchers and fetchers[m].is_available(track)
]
if group:
plan.append(group)
logger.debug(f"Fetch plan: {[[f.source_name for f in g] for g in plan]}")
return plan