diff --git a/config.example.json b/config.example.json index 05f172e..923e027 100644 --- a/config.example.json +++ b/config.example.json @@ -6,7 +6,7 @@ "mail.google.com", "accounts.google.com" ], - "script_id": "YOUR_APPS_SCRIPT_DEPLOYMENT_ID", + "script_id": "YOUR_APPS_SCRIPT_DEPLOYMENT_ID", // Or a list for round-robin: ["ID1", "ID2"], "auth_key": "CHANGE_ME_TO_A_STRONG_SECRET", "listen_host": "127.0.0.1", "http_port": 8085, diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 188bf12..1acfc03 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -45,7 +45,7 @@ The network sees a Google-facing connection. The relay request carries the real - Warm TLS connection pool for H1 fallback. - HTTP/2 multiplexing when the `h2` package is installed. - Batching for static sub-resource bursts. -- Optional multiple `script_ids` for load balancing. +- Optional multiple `script_id` for round-robin load balancing. - Optional range-parallel download acceleration for large files. - Optional exit node for destinations that block Google egress. diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 479fbab..7cef32c 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -6,11 +6,10 @@ Most users only need `script_id`, `auth_key`, and the default local ports. This | Setting | Meaning | |---------|---------| -| `script_id` | Your Google Apps Script Deployment ID. Use this for one deployment. | -| `script_ids` | Array of Deployment IDs for load balancing. Use instead of `script_id`. | +| `script_id` | Your Google Apps Script Deployment ID. Can be a single string or an array of strings for round-robin load balancing. | | `auth_key` | Shared password. Must match `AUTH_KEY` inside [apps_script/Code.gs](../apps_script/Code.gs). | -If you use `script_ids`, every deployed copy of [apps_script/Code.gs](../apps_script/Code.gs) must use the same `AUTH_KEY`. +If you use multiple IDs in `script_id`, every deployed copy of [apps_script/Code.gs](../apps_script/Code.gs) must use the same `AUTH_KEY`. ## Proxy Binding diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index 35dcef8..67e87c5 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -60,7 +60,7 @@ Fix: 1. Create a new Apps Script deployment. 2. Copy the new Deployment ID into `config.json`. 3. Confirm the deployment is a Web App with **Execute as: Me** and **Who has access: Anyone**. -4. If quota is exhausted, wait for the quota reset or add more deployments with `script_ids`. +4. If quota is exhausted, wait for the quota reset or add more deployments by providing a list of IDs in `script_id`. ## Page Looks Like Random Characters @@ -105,7 +105,7 @@ Try these in order: 1. Install all dependencies from [requirements.txt](../requirements.txt), especially `h2`. 2. Run `python main.py --scan` and update `google_ip`. -3. Deploy multiple Apps Script projects and use `script_ids`. +3. Deploy multiple Apps Script projects and use a list of IDs in `script_id`. 4. Keep `log_level` at `INFO` unless debugging. 5. Use an exit node only when needed, because it adds another hop. diff --git a/docs/fa/ARCHITECTURE.md b/docs/fa/ARCHITECTURE.md index 48c3d00..4e874b6 100644 --- a/docs/fa/ARCHITECTURE.md +++ b/docs/fa/ARCHITECTURE.md @@ -45,7 +45,7 @@ Browser یا app - pool گرم اتصال TLS برای fallback H1. - HTTP/2 multiplexing وقتی package `h2` نصب باشد. - batch کردن درخواست‌های static در burst ها. -- چند `script_ids` اختیاری برای load balancing. +- چند `script_id` اختیاری برای load balancing به صورت round-robin. - دانلود موازی range برای فایل‌های بزرگ. - Exit Node اختیاری برای مقصدهایی که خروجی Google را مسدود می‌کنند. diff --git a/docs/fa/CONFIGURATION.md b/docs/fa/CONFIGURATION.md index 5cb8e87..68cf5ea 100644 --- a/docs/fa/CONFIGURATION.md +++ b/docs/fa/CONFIGURATION.md @@ -6,11 +6,10 @@ | تنظیم | معنی | |-------|------| -| `script_id` | Deployment ID مربوط به Google Apps Script. برای یک deployment استفاده می‌شود. | -| `script_ids` | آرایه‌ای از چند Deployment ID برای load balancing. به جای `script_id` استفاده می‌شود. | +| `script_id` | Deployment ID مربوط به Google Apps Script. می‌تواند یک رشته متنی باشد یا آرایه‌ای از رشته‌ها برای load balancing به صورت round-robin. | | `auth_key` | رمز مشترک. باید دقیقا با `AUTH_KEY` داخل [apps_script/Code.gs](../../apps_script/Code.gs) یکی باشد. | -اگر از `script_ids` استفاده می‌کنید، همه deployment ها باید `AUTH_KEY` یکسان داشته باشند. +اگر از چندین ID در `script_id` استفاده می‌کنید، همه deployment ها باید `AUTH_KEY` یکسان داشته باشند. ## اتصال پراکسی diff --git a/main.py b/main.py index 76e7370..27dfaba 100644 --- a/main.py +++ b/main.py @@ -166,7 +166,8 @@ def main(): if os.environ.get("DFT_AUTH_KEY"): config["auth_key"] = os.environ["DFT_AUTH_KEY"] if os.environ.get("DFT_SCRIPT_ID"): - config["script_id"] = os.environ["DFT_SCRIPT_ID"] + val = os.environ["DFT_SCRIPT_ID"] + config["script_id"] = [x.strip() for x in val.split(",") if x.strip()] if "," in val else val # CLI argument overrides if args.port is not None: @@ -218,7 +219,14 @@ def main(): # Always Apps Script mode — force-set for backward-compat configs. config["mode"] = "apps_script" - sid = config.get("script_ids") or config.get("script_id") + + # Consolidate script_id and script_ids into a single smart field. + sids = config.get("script_ids") or config.get("script_id") + if sids: + config["script_id"] = sids + config.pop("script_ids", None) + + sid = config.get("script_id") if not sid or (isinstance(sid, str) and sid == "YOUR_APPS_SCRIPT_DEPLOYMENT_ID"): print("Missing 'script_id' in config.") print("Deploy the Apps Script from Code.gs and paste the Deployment ID.") @@ -241,9 +249,9 @@ def main(): log.info("Apps Script relay : SNI=%s → script.google.com", config.get("front_domain", "www.google.com")) - script_ids = config.get("script_ids") or config.get("script_id") + script_ids = config.get("script_id") if isinstance(script_ids, list): - log.info("Script IDs : %d scripts (sticky per-host)", len(script_ids)) + log.info("Script IDs : %d scripts (round-robin)", len(script_ids)) for i, sid in enumerate(script_ids): _s = str(sid) masked = f"{_s[:6]}…{_s[-4:]}" if len(_s) > 12 else _s diff --git a/setup.py b/setup.py index 0bb2f48..874c7da 100644 --- a/setup.py +++ b/setup.py @@ -111,10 +111,10 @@ def configure_apps_script(cfg: dict) -> dict: ids = [x.strip() for x in ids_raw.split(",") if x.strip()] if len(ids) == 1: cfg["script_id"] = ids[0] - cfg.pop("script_ids", None) - else: - cfg["script_ids"] = ids - cfg.pop("script_id", None) + elif len(ids) > 1: + cfg["script_id"] = ids + + cfg.pop("script_ids", None) return cfg diff --git a/src/relay/domain_fronter.py b/src/relay/domain_fronter.py index da2a3a1..40a589e 100644 --- a/src/relay/domain_fronter.py +++ b/src/relay/domain_fronter.py @@ -14,7 +14,6 @@ import hashlib import json import logging -import random import re import socket import ssl @@ -39,9 +38,6 @@ POOL_MIN_IDLE, RELAY_TIMEOUT, SCRIPT_BLACKLIST_TTL, - SCRIPT_PROBE_INTERVAL_MAX, - SCRIPT_PROBE_INTERVAL_MIN, - SCRIPT_PROBE_TIMEOUT, SEMAPHORE_MAX, STATEFUL_HEADER_NAMES, STATIC_EXTS, @@ -63,7 +59,6 @@ validate_range_response, ) from .relay_response import ( - classify_relay_envelope, classify_relay_error, error_response, extract_apps_script_user_html, @@ -93,32 +88,6 @@ def _mask_sid(sid: str) -> str: return f"{sid[:6]}\u2026{sid[-4:]}" -class ScriptDeploymentError(Exception): - """Internal signal that a permanent Apps Script envelope was detected. - - Raised by per-SID parse hooks (``_relay_single_h2_with_sid``, - ``_relay_single_h2``, ``_relay_single``, ``_parse_batch_body``) when - :func:`relay_response.classify_relay_envelope` returns a permanent - category (``"quota" | "auth" | "deploy" | "admin"``). Caught by - ``_relay_fanout`` and ``_relay_with_retry`` so the corresponding - script ID is dropped from rotation via ``_blacklist_sid`` and the - next attempt picks a different deployment. Never propagates to - client code paths — the relay always converts it back into a normal - response (a healthy racer's bytes, a retry result, or an upstream - 502) before returning to callers outside this module. - """ - - def __init__(self, sid: str, category: str, raw: str): - self.sid = sid - self.category = category - self.raw = raw - super().__init__(f"ScriptDeploymentError(sid={_mask_sid(sid)}, " - f"category={category}, raw={raw[:120]!r})") - - def __str__(self) -> str: - return f"{self.category}: {self.raw[:120]}" - - class DomainFronter: _STATIC_EXTS = STATIC_EXTS _H2_FAILURE_COOLDOWN = 15.0 # reduced: DPI token bucket refills in ~8-10s @@ -156,7 +125,7 @@ def __init__(self, config: dict): self._sni_probe_task: asyncio.Task | None = None self.http_host = "script.google.com" # Multi-script round-robin for higher throughput - script = config.get("script_ids") or config.get("script_id") + script = config.get("script_id") self._script_ids = script if isinstance(script, list) else [script] self._script_idx = 0 self.script_id = self._script_ids[0] # backward compat / logging @@ -182,8 +151,6 @@ def __init__(self, config: dict): len(self._script_ids))) self._sid_blacklist: dict[str, float] = {} self._blacklist_ttl = SCRIPT_BLACKLIST_TTL - self._probe_task: asyncio.Task | None = None - self._probe_semaphore: asyncio.Semaphore = asyncio.Semaphore(1) # Per-host stats (requests, cache hits, bytes, cumulative latency). self._per_site: dict[str, HostStat] = {} @@ -778,119 +745,6 @@ def _prune_blacklist(self, force: bool = False) -> None: if force or until <= now: self._sid_blacklist.pop(sid, None) - async def _probe_one_sid(self, sid: str) -> None: - """Re-validate a blacklisted SID with one low-cost relay request. - - Healthy → drop from `_sid_blacklist` and log recovery. - Permanent envelope or transport error → leave blacklisted, no TTL change. - Never propagates exceptions to client paths (2.10). - """ - payload = {"m": "GET", "u": "http://example.com/", "k": self.auth_key} - body_bytes = json.dumps(payload).encode() - path = self._exec_path_for_sid(sid) - async with self._probe_semaphore: - try: - # Probe also consumes Apps Script quota — record it. - self._record_execution(sid) - transport = self._pick_h2() or self._h2 - if transport is None: - # No H2 transport available — skip this probe cycle. The - # H1 fallback path is intentionally not used here so a - # saturated client semaphore can't starve probes. - return - status, headers, body = await asyncio.wait_for( - transport.request( - method="POST", path=path, host=self.http_host, - headers=self._apps_script_headers(), - body=body_bytes, - timeout=SCRIPT_PROBE_TIMEOUT, - ), - timeout=SCRIPT_PROBE_TIMEOUT, - ) - except Exception as exc: - log.debug("Probe %s failed (%s) — keeping blacklisted", - _mask_sid(sid), exc) - return - - category, _raw = classify_relay_envelope(body) - if category is not None: - log.debug("Probe %s still %s — keeping blacklisted", - _mask_sid(sid), category) - return - - # Healthy — recover. - self._sid_blacklist.pop(sid, None) - log.info("Re-validated script %s — recovered", _mask_sid(sid)) - - async def _probe_tick(self) -> None: - """One pass of the probe loop — re-validate every still-blacklisted SID. - - No-op when only one script ID is configured (3.7) or when the - blacklist is empty (3.8). - """ - if len(self._script_ids) <= 1: - return - sids = [s for s in list(self._sid_blacklist) - if self._is_sid_blacklisted(s)] - if not sids: - return - await asyncio.gather( - *(self._probe_one_sid(s) for s in sids), - return_exceptions=True, - ) - - async def _probe_blacklisted_sids(self) -> None: - """Background loop: every uniform(MIN, MAX) seconds re-probe blacklisted SIDs.""" - while True: - try: - interval = random.uniform( - SCRIPT_PROBE_INTERVAL_MIN, SCRIPT_PROBE_INTERVAL_MAX, - ) - await asyncio.sleep(interval) - await self._probe_tick() - except asyncio.CancelledError: - break - except Exception as exc: - log.debug("Probe loop error: %s", exc) - - def _is_failed_relay_result(self, sid: str, result: bytes) -> bool: - """Return True if a successful racer's bytes are actually an Apps - Script permanent-failure envelope (or a 502 synthesised from one). - - The parse-site hooks in ``_relay_single_h2_with_sid`` / - ``_relay_single_h2`` / ``_relay_single`` / ``_parse_batch_body`` - normally raise :class:`ScriptDeploymentError` before ``_relay_fanout`` - ever sees the body, so this inspection is a defence-in-depth check - against bypass paths (test monkeypatches that replace - ``_relay_single_h2_with_sid`` directly, future refactors that - remove the per-SID hook). When this returns True the caller - blacklists the SID (idempotent if already done by the parse hook) - and keeps waiting on the remaining racers instead of forwarding - the 502 to the client (requirement 2.4). - """ - # Case 1: caller bypassed the parse hook and handed back the raw - # Apps Script envelope. classify_relay_envelope decides whether - # the envelope error is permanent (quota/auth/deploy/admin) or - # transient/healthy. - category, _raw = classify_relay_envelope(result) - if category is not None: - if not self._is_sid_blacklisted(sid): - self._blacklist_sid(sid, reason=category) - return True - # Case 2: caller bypassed the parse hook but already invoked - # parse_relay_response, so the original envelope has been - # replaced by a synthesised ``HTTP/1.1 502`` response. Without - # the JSON envelope we cannot distinguish permanent from - # transient classes, but in fan-out we always prefer to keep - # waiting on the remaining racers rather than forward a 502 — - # if every racer ends up here we still surface the last - # exception via ``winner_exc``. - if result.startswith(b"HTTP/1.1 502"): - if not self._is_sid_blacklisted(sid): - self._blacklist_sid(sid, reason="parsed_502") - return True - return False - def _pick_fanout_sids(self, key: str | None) -> list[str]: """Pick up to `parallel_relay` distinct non-blacklisted script IDs. @@ -1076,28 +930,18 @@ async def _stats_logger(self): log.debug("Stats logger error: %s", e) def _script_id_for_key(self, key: str | None = None) -> str: - """Pick a stable Apps Script ID for a host or fallback to round-robin. - - When multiple deployments are configured, using a stable mapping per - host reduces IP/session churn for sites that are sensitive to endpoint - changes. If no key is available, we keep the older round-robin fallback - so warmup/keepalive traffic still distributes normally. + """Pick an Apps Script ID using round-robin distribution. + When multiple deployments are configured, we use pure round-robin to + distribute load evenly across all available script IDs. + Blacklisted IDs are skipped by probing forward in the list until a - healthy one is found; if none, the stable pick is returned anyway. + healthy one is found; if none, the next round-robin pick is returned anyway. """ if len(self._script_ids) == 1: return self._script_ids[0] - if not key: - return self._next_script_id() - digest = hashlib.sha1(key.encode("utf-8")).digest() - base = int.from_bytes(digest[:4], "big") % len(self._script_ids) - n = len(self._script_ids) - for offset in range(n): - sid = self._script_ids[(base + offset) % n] - if not self._is_sid_blacklisted(sid): - return sid - return self._script_ids[base] + + return self._next_script_id() def _exec_path(self, url_or_host: str | None = None) -> str: """Get the Apps Script endpoint path (/dev or /exec).""" @@ -1200,8 +1044,6 @@ async def _warm_pool(self): self._stats_task = self._spawn(self._stats_logger()) if self._execution_task is None: self._execution_task = self._spawn(self._execution_logger()) - if self._probe_task is None: - self._probe_task = self._spawn(self._probe_blacklisted_sids()) # Start H2 connection (runs alongside H1 pool) if self._h2: self._spawn(self._h2_connect_and_warm()) @@ -1247,7 +1089,6 @@ async def close(self): self._maintenance_task = None self._stats_task = None self._execution_task = None - self._probe_task = None self._keepalive_task = None await self._flush_pool() @@ -2712,26 +2553,9 @@ async def _relay_fanout(self, payload: dict) -> bytes: exc = t.exception() if exc is None: winner_result = t.result() - if self._is_failed_relay_result(sid, winner_result): - # Defence-in-depth: the per-SID parse hook - # normally raises ScriptDeploymentError before - # we get here, but a bypass path (test - # monkeypatch on _relay_single_h2_with_sid, - # future refactor) could still surface a - # permanent envelope as a "successful" body. - # Treat this racer as failed and keep waiting - # on the remaining racers (requirement 2.4). - winner_exc = RuntimeError( - f"fan-out racer {_mask_sid(sid)} " - f"returned permanent envelope" - ) - continue return winner_result - # This racer failed — blacklist (unless the parse - # hook already did with a sharper category reason) - # and keep waiting for others (requirement 3.3). - if not isinstance(exc, ScriptDeploymentError): - self._blacklist_sid(sid, reason=type(exc).__name__) + # This racer failed — blacklist and keep waiting for others + self._blacklist_sid(sid, reason=type(exc).__name__) winner_exc = exc # All racers failed if winner_exc is not None: @@ -2774,11 +2598,6 @@ async def _relay_single_h2(self, payload: dict) -> bytes: len(body), ) - category, raw = classify_relay_envelope(body) - if category is not None: - self._blacklist_sid(sid, reason=category) - raise ScriptDeploymentError(sid, category, raw) - return parse_relay_response(body, self._max_response_body_bytes) async def _relay_single_h2_with_sid(self, payload: dict, @@ -2802,11 +2621,6 @@ async def _relay_single_h2_with_sid(self, payload: dict, timeout=self._relay_timeout, ) - category, raw = classify_relay_envelope(body) - if category is not None: - self._blacklist_sid(sid, reason=category) - raise ScriptDeploymentError(sid, category, raw) - return parse_relay_response(body, self._max_response_body_bytes) async def _follow_redirects( @@ -2890,6 +2704,7 @@ async def _relay_single(self, payload: dict) -> bytes: ) await self._release(reader, writer, created) + return parse_relay_response(resp_body, self._max_response_body_bytes) except Exception: try: @@ -2898,13 +2713,6 @@ async def _relay_single(self, payload: dict) -> bytes: pass raise - category, raw = classify_relay_envelope(resp_body) - if category is not None: - self._blacklist_sid(sid, reason=category) - raise ScriptDeploymentError(sid, category, raw) - - return parse_relay_response(resp_body, self._max_response_body_bytes) - async def _relay_batch(self, payloads: list[dict]) -> list[bytes]: """Send multiple requests in one POST using Apps Script fetchAll.""" batch_payload = { @@ -2945,7 +2753,7 @@ async def _relay_batch(self, payloads: list[dict]) -> list[bytes]: len(body), ) self._record_h2_success() - return self._parse_batch_body(body, payloads, sid) + return self._parse_batch_body(body, payloads) except Exception as e: if self._is_h2_transport_error(e): self._record_h2_failure(e) @@ -2985,23 +2793,11 @@ async def _relay_batch(self, payloads: list[dict]) -> list[bytes]: pass raise - return self._parse_batch_body(resp_body, payloads, sid) + return self._parse_batch_body(resp_body, payloads) def _parse_batch_body(self, resp_body: bytes, - payloads: list[dict], - sid: str) -> list[bytes]: + payloads: list[dict]) -> list[bytes]: """Parse a batch response body into individual results.""" - # Classify the *outer* batch envelope before per-item parsing. - # A permanent error here ("e" at the top level) means the - # deployment itself failed the whole batch — blacklist the SID - # and raise so _relay_with_retry can pick a different one. - # Per-item "e" fields below are target-origin errors and stay - # untouched (requirement 3.6). - category, raw = classify_relay_envelope(resp_body) - if category is not None: - self._blacklist_sid(sid, reason=category) - raise ScriptDeploymentError(sid, category, raw) - text = resp_body.decode(errors="replace").strip() # Apps Script can wrap JSON inside an HTML shell; reuse the same # robust loader used by single-response parsing.