Upload 22 files
Browse files
backend/services/auto_discovery_service.py
CHANGED
|
@@ -8,6 +8,7 @@ Auto Discovery Service
|
|
| 8 |
from __future__ import annotations
|
| 9 |
|
| 10 |
import asyncio
|
|
|
|
| 11 |
import json
|
| 12 |
import logging
|
| 13 |
import os
|
|
@@ -182,18 +183,59 @@ class AutoDiscoveryService:
|
|
| 182 |
async with AsyncDDGS() as ddgs:
|
| 183 |
for query in query_list:
|
| 184 |
try:
|
| 185 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 186 |
query,
|
| 187 |
max_results=self.max_candidates_per_query,
|
| 188 |
-
)
|
| 189 |
-
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
|
| 193 |
-
|
| 194 |
-
|
| 195 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 196 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 197 |
except Exception as exc: # pragma: no cover - وابسته به اینترنت
|
| 198 |
logger.warning("Failed to fetch results for query '%s': %s", query, exc)
|
| 199 |
|
|
|
|
| 8 |
from __future__ import annotations
|
| 9 |
|
| 10 |
import asyncio
|
| 11 |
+
import inspect
|
| 12 |
import json
|
| 13 |
import logging
|
| 14 |
import os
|
|
|
|
| 183 |
async with AsyncDDGS() as ddgs:
|
| 184 |
for query in query_list:
|
| 185 |
try:
|
| 186 |
+
text_method = getattr(ddgs, "atext", None)
|
| 187 |
+
if callable(text_method):
|
| 188 |
+
async for entry in text_method(
|
| 189 |
+
query,
|
| 190 |
+
max_results=self.max_candidates_per_query,
|
| 191 |
+
):
|
| 192 |
+
results.append(
|
| 193 |
+
{
|
| 194 |
+
"query": query,
|
| 195 |
+
"title": entry.get("title", ""),
|
| 196 |
+
"url": entry.get("href") or entry.get("url") or "",
|
| 197 |
+
"snippet": entry.get("body", ""),
|
| 198 |
+
}
|
| 199 |
+
)
|
| 200 |
+
continue
|
| 201 |
+
|
| 202 |
+
text_method = getattr(ddgs, "text", None)
|
| 203 |
+
if not callable(text_method):
|
| 204 |
+
raise AttributeError("AsyncDDGS has no 'atext' or 'text' method")
|
| 205 |
+
|
| 206 |
+
search_result = text_method(
|
| 207 |
query,
|
| 208 |
max_results=self.max_candidates_per_query,
|
| 209 |
+
)
|
| 210 |
+
|
| 211 |
+
if inspect.isawaitable(search_result):
|
| 212 |
+
search_result = await search_result
|
| 213 |
+
|
| 214 |
+
if hasattr(search_result, "__aiter__"):
|
| 215 |
+
async for entry in search_result:
|
| 216 |
+
results.append(
|
| 217 |
+
{
|
| 218 |
+
"query": query,
|
| 219 |
+
"title": entry.get("title", ""),
|
| 220 |
+
"url": entry.get("href") or entry.get("url") or "",
|
| 221 |
+
"snippet": entry.get("body", ""),
|
| 222 |
+
}
|
| 223 |
+
)
|
| 224 |
+
else:
|
| 225 |
+
iterable = (
|
| 226 |
+
search_result
|
| 227 |
+
if isinstance(search_result, list)
|
| 228 |
+
else list(search_result or [])
|
| 229 |
)
|
| 230 |
+
for entry in iterable:
|
| 231 |
+
results.append(
|
| 232 |
+
{
|
| 233 |
+
"query": query,
|
| 234 |
+
"title": entry.get("title", ""),
|
| 235 |
+
"url": entry.get("href") or entry.get("url") or "",
|
| 236 |
+
"snippet": entry.get("body", ""),
|
| 237 |
+
}
|
| 238 |
+
)
|
| 239 |
except Exception as exc: # pragma: no cover - وابسته به اینترنت
|
| 240 |
logger.warning("Failed to fetch results for query '%s': %s", query, exc)
|
| 241 |
|
provider_manager.py
CHANGED
|
@@ -31,15 +31,54 @@ class RotationStrategy(Enum):
|
|
| 31 |
FASTEST_RESPONSE = "fastest_response"
|
| 32 |
|
| 33 |
|
| 34 |
-
@dataclass
|
| 35 |
class RateLimitInfo:
|
| 36 |
"""اطلاعات محدودیت نرخ"""
|
| 37 |
requests_per_second: Optional[int] = None
|
| 38 |
requests_per_minute: Optional[int] = None
|
| 39 |
requests_per_hour: Optional[int] = None
|
| 40 |
requests_per_day: Optional[int] = None
|
|
|
|
|
|
|
|
|
|
| 41 |
current_usage: int = 0
|
| 42 |
reset_time: Optional[float] = None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 43 |
|
| 44 |
def is_limited(self) -> bool:
|
| 45 |
"""بررسی محدودیت نرخ"""
|
|
@@ -90,7 +129,9 @@ class Provider:
|
|
| 90 |
def __post_init__(self):
|
| 91 |
"""مقداردهی اولیه"""
|
| 92 |
if isinstance(self.rate_limit, dict):
|
| 93 |
-
self.rate_limit = RateLimitInfo(
|
|
|
|
|
|
|
| 94 |
|
| 95 |
@property
|
| 96 |
def success_rate(self) -> float:
|
|
@@ -277,7 +318,7 @@ class ProviderManager:
|
|
| 277 |
# بارگذاری ارائهدهندگان
|
| 278 |
for provider_id, provider_data in config.get('providers', {}).items():
|
| 279 |
rate_limit_data = provider_data.get('rate_limit', {})
|
| 280 |
-
rate_limit = RateLimitInfo(
|
| 281 |
|
| 282 |
provider = Provider(
|
| 283 |
provider_id=provider_id,
|
|
|
|
| 31 |
FASTEST_RESPONSE = "fastest_response"
|
| 32 |
|
| 33 |
|
| 34 |
+
@dataclass(init=False)
|
| 35 |
class RateLimitInfo:
|
| 36 |
"""اطلاعات محدودیت نرخ"""
|
| 37 |
requests_per_second: Optional[int] = None
|
| 38 |
requests_per_minute: Optional[int] = None
|
| 39 |
requests_per_hour: Optional[int] = None
|
| 40 |
requests_per_day: Optional[int] = None
|
| 41 |
+
requests_per_week: Optional[int] = None
|
| 42 |
+
requests_per_month: Optional[int] = None
|
| 43 |
+
weight_per_minute: Optional[int] = None
|
| 44 |
current_usage: int = 0
|
| 45 |
reset_time: Optional[float] = None
|
| 46 |
+
extra_limits: Dict[str, Any] = field(default_factory=dict)
|
| 47 |
+
|
| 48 |
+
def __init__(
|
| 49 |
+
self,
|
| 50 |
+
requests_per_second: Optional[int] = None,
|
| 51 |
+
requests_per_minute: Optional[int] = None,
|
| 52 |
+
requests_per_hour: Optional[int] = None,
|
| 53 |
+
requests_per_day: Optional[int] = None,
|
| 54 |
+
requests_per_week: Optional[int] = None,
|
| 55 |
+
requests_per_month: Optional[int] = None,
|
| 56 |
+
weight_per_minute: Optional[int] = None,
|
| 57 |
+
current_usage: int = 0,
|
| 58 |
+
reset_time: Optional[float] = None,
|
| 59 |
+
**extra: Any,
|
| 60 |
+
):
|
| 61 |
+
self.requests_per_second = requests_per_second
|
| 62 |
+
self.requests_per_minute = requests_per_minute
|
| 63 |
+
self.requests_per_hour = requests_per_hour
|
| 64 |
+
self.requests_per_day = requests_per_day
|
| 65 |
+
self.requests_per_week = requests_per_week
|
| 66 |
+
self.requests_per_month = requests_per_month
|
| 67 |
+
self.weight_per_minute = weight_per_minute
|
| 68 |
+
self.current_usage = current_usage
|
| 69 |
+
self.reset_time = reset_time
|
| 70 |
+
self.extra_limits = extra
|
| 71 |
+
|
| 72 |
+
@classmethod
|
| 73 |
+
def from_dict(cls, data: Optional[Dict[str, Any]]) -> "RateLimitInfo":
|
| 74 |
+
"""ساخت نمونه از دیکشنری و مدیریت کلیدهای ناشناخته."""
|
| 75 |
+
if isinstance(data, cls):
|
| 76 |
+
return data
|
| 77 |
+
|
| 78 |
+
if not data:
|
| 79 |
+
return cls()
|
| 80 |
+
|
| 81 |
+
return cls(**data)
|
| 82 |
|
| 83 |
def is_limited(self) -> bool:
|
| 84 |
"""بررسی محدودیت نرخ"""
|
|
|
|
| 129 |
def __post_init__(self):
|
| 130 |
"""مقداردهی اولیه"""
|
| 131 |
if isinstance(self.rate_limit, dict):
|
| 132 |
+
self.rate_limit = RateLimitInfo.from_dict(self.rate_limit)
|
| 133 |
+
elif not isinstance(self.rate_limit, RateLimitInfo):
|
| 134 |
+
self.rate_limit = RateLimitInfo()
|
| 135 |
|
| 136 |
@property
|
| 137 |
def success_rate(self) -> float:
|
|
|
|
| 318 |
# بارگذاری ارائهدهندگان
|
| 319 |
for provider_id, provider_data in config.get('providers', {}).items():
|
| 320 |
rate_limit_data = provider_data.get('rate_limit', {})
|
| 321 |
+
rate_limit = RateLimitInfo.from_dict(rate_limit_data)
|
| 322 |
|
| 323 |
provider = Provider(
|
| 324 |
provider_id=provider_id,
|