refactor: async parallel fetching
This commit is contained in:
@@ -4,7 +4,8 @@ Date: 2026-03-25 02:33:26
|
||||
Description: Fetcher pipeline — registry and types
|
||||
"""
|
||||
|
||||
from typing import Literal
|
||||
from typing import Literal, Optional
|
||||
from loguru import logger
|
||||
|
||||
from .base import BaseFetcher
|
||||
from .local import LocalFetcher
|
||||
@@ -15,6 +16,7 @@ from .lrclib_search import LrclibSearchFetcher
|
||||
from .netease import NeteaseFetcher
|
||||
from .qqmusic import QQMusicFetcher
|
||||
from ..cache import CacheEngine
|
||||
from ..models import TrackMeta
|
||||
|
||||
FetcherMethodType = Literal[
|
||||
"local",
|
||||
@@ -26,6 +28,15 @@ FetcherMethodType = Literal[
|
||||
"qqmusic",
|
||||
]
|
||||
|
||||
# Fetchers within a group run in parallel; groups run sequentially.
|
||||
# A group that produces any positive result stops the pipeline.
|
||||
_FETCHER_GROUPS: list[list[FetcherMethodType]] = [
|
||||
["local"],
|
||||
["cache-search"],
|
||||
["spotify", "lrclib"],
|
||||
["lrclib-search", "netease", "qqmusic"],
|
||||
]
|
||||
|
||||
|
||||
def create_fetchers(cache: CacheEngine) -> dict[FetcherMethodType, BaseFetcher]:
|
||||
"""Instantiate all fetchers. Returns a dict keyed by source name."""
|
||||
@@ -39,3 +50,29 @@ def create_fetchers(cache: CacheEngine) -> dict[FetcherMethodType, BaseFetcher]:
|
||||
"qqmusic": QQMusicFetcher(),
|
||||
}
|
||||
return fetchers
|
||||
|
||||
|
||||
def build_plan(
|
||||
fetchers: dict[FetcherMethodType, BaseFetcher],
|
||||
track: TrackMeta,
|
||||
force_method: Optional[FetcherMethodType] = None,
|
||||
) -> list[list[BaseFetcher]]:
|
||||
"""Return the fetch plan as a list of groups (each group runs in parallel)."""
|
||||
if force_method:
|
||||
if force_method not in fetchers:
|
||||
logger.error(f"Unknown method: {force_method}")
|
||||
return []
|
||||
return [[fetchers[force_method]]]
|
||||
|
||||
plan: list[list[BaseFetcher]] = []
|
||||
for group_methods in _FETCHER_GROUPS:
|
||||
group = [
|
||||
fetchers[m]
|
||||
for m in group_methods
|
||||
if m in fetchers and fetchers[m].is_available(track)
|
||||
]
|
||||
if group:
|
||||
plan.append(group)
|
||||
|
||||
logger.debug(f"Fetch plan: {[[f.source_name for f in g] for g in plan]}")
|
||||
return plan
|
||||
|
||||
Reference in New Issue
Block a user