| """Zen Guard for Windows: watches OpenCode's log for quota walls and rotates /64.""" |
| |
| from __future__ import annotations |
| |
| import json |
| import os |
| import sys |
| import time |
| from pathlib import Path |
| |
| |
| _HERE = Path(__file__).resolve().parent |
| if str(_HERE) not in sys.path: |
| sys.path.insert(0, str(_HERE)) |
| _SCRIPTS = Path(os.environ.get("LOCALAPPDATA", "")) / "openhop" / "scripts" |
| if _SCRIPTS.is_dir() and str(_SCRIPTS) not in sys.path: |
| sys.path.insert(0, str(_SCRIPTS)) |
| |
| from ouc_common import ( |
| GUARD_LOG_FILE, |
| GUARD_STATE_FILE, |
| MUTEX_GUARD, |
| OPENCODE_LOG_FILE, |
| acquire_mutex, |
| append_log, |
| ensure_dirs, |
| find_warp_cli, |
| get_egress_v6, |
| get_warp_local_v6, |
| hop_once, |
| bump_egress_epoch, |
| keeper_pause_ctx, |
| next_free_reset, |
| normalize_v6_64, |
| trace_via_socks, |
| warp_path_leaking, |
| ) |
| |
| POLL_SECONDS = 10 |
| COOLDOWN_SECONDS = 75 |
| |
| |
| |
| |
| REWALL_SECONDS = 180 |
| FAILED_HOPS_MAX = 15 |
| HUNT_GAP_SECONDS = 12 |
| BACKOFF_SECONDS = 3 * 60 |
| TOR_TRIES = 4 |
| HEARTBEAT_FILE = GUARD_LOG_FILE.parent / "heartbeat.txt" |
| |
| |
| def _heartbeat() -> None: |
| """Cheap liveness signal the app checks: a running but hung guard stops writing this.""" |
| try: |
| HEARTBEAT_FILE.write_text(str(int(time.time())), encoding="utf-8") |
| except Exception: |
| pass |
| MAX_HOP_TRIES = 8 |
| BLACKLIST_MAX = 200 |
| MAX_READ_BYTES = 50_000_000 |
| |
| |
| def _default_state(offset: int = 0) -> dict: |
| return { |
| "blacklist": [], |
| "offset": offset, |
| "last_hop": 0, |
| "hops": 0, |
| "current": "", |
| "pending": {}, |
| "resumed": {}, |
| "failed_hops": 0, |
| "backoff_until": 0, |
| "hop_counted": 0, |
| "backoff_logged": 0, |
| } |
| |
| |
| def _load_state() -> dict: |
| if not GUARD_STATE_FILE.is_file(): |
| offset = OPENCODE_LOG_FILE.stat().st_size if OPENCODE_LOG_FILE.is_file() else 0 |
| state = _default_state(offset) |
| _save_state(state) |
| return state |
| try: |
| data = json.loads(GUARD_STATE_FILE.read_text(encoding="utf-8")) |
| if isinstance(data, dict): |
| return {**_default_state(), **data} |
| except Exception: |
| pass |
| return _default_state() |
| |
| |
| def _save_state(state: dict) -> None: |
| try: |
| GUARD_STATE_FILE.write_text(json.dumps(state), encoding="utf-8") |
| except Exception: |
| pass |
| |
| |
| PENDING_MAX_AGE = 30 * 60 |
| |
| |
| def _resumable(state: dict) -> list[str]: |
| """Chats that hit the limit and haven't had a 'continue' since the last new IP.""" |
| last_hop = float(state.get("last_ok_hop", 0) or 0) |
| resumed = state.get("resumed") or {} |
| return [sid for sid in (state.get("pending") or {}) if float(resumed.get(sid, 0) or 0) < last_hop] |
| |
| |
| def _prune(state: dict) -> None: |
| cutoff = time.time() - PENDING_MAX_AGE |
| for key in ("pending", "resumed"): |
| state[key] = {k: v for k, v in (state.get(key) or {}).items() if float(v or 0) >= cutoff} |
| |
| |
| def _dispatch_auto_resume(state: dict) -> None: |
| """After a new IP: send 'continue' to every chat (tab) that hit the limit, not just the latest one.""" |
| try: |
| from auto_resume import resume_active_session, is_auto_resume_enabled, clear_opencode_free_limit |
| |
| |
| clear_opencode_free_limit() |
| |
| sids = _resumable(state) |
| if is_auto_resume_enabled(): |
| time.sleep(3.0) |
| ok, res_msg = resume_active_session(sids or None) |
| append_log(GUARD_LOG_FILE, f"[auto-resume] Limit bypassed -> {res_msg}") |
| now = time.time() |
| for sid in sids: |
| state.setdefault("resumed", {})[sid] = now |
| (state.get("pending") or {}).pop(sid, None) |
| _prune(state) |
| _save_state(state) |
| except Exception as exc: |
| append_log(GUARD_LOG_FILE, f"[auto-resume] note: {exc}") |
| |
| |
| def _blacklist_add(state: dict, addr: str) -> None: |
| """Remember a used-up address until OpenCode's daily reset (a bit after 01:00 UTC).""" |
| prefix = normalize_v6_64(addr) |
| if not prefix: |
| return |
| now = time.time() |
| until = {k: v for k, v in (state.get("burnt_until") or {}).items() if float(v or 0) > now} |
| until[prefix] = next_free_reset(now) |
| state["burnt_until"] = dict(list(until.items())[-BLACKLIST_MAX:]) |
| state["blacklist"] = list(state["burnt_until"].keys()) |
| |
| |
| def _is_burnt(state: dict, addr: str) -> bool: |
| prefix = normalize_v6_64(addr) |
| if not prefix: |
| return False |
| return float((state.get("burnt_until") or {}).get(prefix, 0) or 0) > time.time() |
| |
| |
| def _note_wall(state: dict, now: int) -> bool: |
| """Track walls that return right after a new IP and pause rotation when it's clearly a loop. |
| |
| Returns True while auto-rotation is paused (the caller should not rotate or resume). |
| """ |
| backoff_until = int(state.get("backoff_until", 0) or 0) |
| if backoff_until and now >= backoff_until: |
| state["backoff_until"] = 0 |
| state["failed_hops"] = 0 |
| append_log(GUARD_LOG_FILE, "rotation pause is over — trying again on the next limit") |
| if now < int(state.get("backoff_until", 0) or 0): |
| if now - int(state.get("backoff_logged", 0) or 0) >= 60: |
| state["backoff_logged"] = now |
| _save_state(state) |
| left = max(0, (int(state["backoff_until"]) - now) // 60) |
| append_log( |
| GUARD_LOG_FILE, |
| f"limit seen again, but auto-rotation is paused for ~{left} more min " |
| f"(new IPs kept hitting the limit immediately)", |
| ) |
| return True |
| |
| last_hop = int(state.get("last_ok_hop", 0) or 0) |
| if last_hop and now - last_hop <= REWALL_SECONDS: |
| if int(state.get("hop_counted", 0) or 0) != last_hop: |
| state["hop_counted"] = last_hop |
| state["failed_hops"] = int(state.get("failed_hops", 0) or 0) + 1 |
| append_log( |
| GUARD_LOG_FILE, |
| f"limit came back {now - last_hop}s after the last IP change " |
| f"(failed attempt {state['failed_hops']} of {FAILED_HOPS_MAX})", |
| ) |
| elif last_hop: |
| |
| state["failed_hops"] = 0 |
| |
| if int(state.get("failed_hops", 0) or 0) >= FAILED_HOPS_MAX: |
| state["backoff_until"] = now + BACKOFF_SECONDS |
| state["backoff_logged"] = now |
| append_log( |
| GUARD_LOG_FILE, |
| f"pausing auto-rotation for {BACKOFF_SECONDS // 60} min: the last " |
| f"{FAILED_HOPS_MAX} new addresses hit the limit again immediately. " |
| f"It will try one more address by itself after that; you can also press " |
| f"'Get a new IP' or try Tor now.", |
| ) |
| _save_state(state) |
| return now < int(state.get("backoff_until", 0) or 0) |
| |
| |
| def _read_new_log_bytes(state: dict) -> tuple[bool, list[str]]: |
| """(wall seen since the last offset?, sessionIDs of the chats that hit it).""" |
| if not OPENCODE_LOG_FILE.is_file(): |
| return False, [] |
| try: |
| size = OPENCODE_LOG_FILE.stat().st_size |
| except OSError: |
| return False, [] |
| |
| offset = int(state.get("offset", 0) or 0) |
| if size < offset: |
| offset = 0 |
| |
| delta = size - offset |
| wall = False |
| sessions: list[str] = [] |
| if 0 < delta < MAX_READ_BYTES: |
| try: |
| with OPENCODE_LOG_FILE.open("rb") as fh: |
| fh.seek(offset) |
| chunk = fh.read(delta).decode("utf-8", errors="replace") |
| chunk_lower = chunk.lower() |
| wall_patterns = ( |
| "quotaexceeded", |
| "quota exceeded", |
| "quota_exceeded", |
| "free usage limit", |
| "free usage exceeded", |
| "rate limit exceeded", |
| "rate_limit_exceeded", |
| "429 too many requests", |
| "free-tier usage limit", |
| "too many requests", |
| "exceeded your current quota", |
| "usage limit reached", |
| "credits exhausted", |
| "status=429", |
| "status: 429", |
| "status 429", |
| "code=429", |
| "resource_exhausted", |
| "ratelimit", |
| ) |
| if any(p in chunk_lower for p in wall_patterns): |
| wall = True |
| try: |
| from auto_resume import session_id_in |
| except Exception: |
| session_id_in = None |
| if session_id_in: |
| for line in chunk.splitlines(): |
| low = line.lower() |
| if any(p in low for p in wall_patterns): |
| sid = session_id_in(line) |
| if sid and sid not in sessions: |
| sessions.append(sid) |
| except OSError: |
| pass |
| |
| state["offset"] = size |
| _save_state(state) |
| return wall, sessions |
| |
| |
| def _rotate(state: dict, warp_bin: str | None) -> None: |
| now = int(time.time()) |
| _heartbeat() |
| |
| |
| mode = "warp" |
| try: |
| from proxy_chain import get_egress_config |
| mode = get_egress_config().get("mode", "warp") |
| except Exception: |
| pass |
| |
| |
| if mode == "tor": |
| try: |
| from tor_manager import get_active_tor_target, rotate_tor_circuit |
| target = get_active_tor_target() |
| old_ip = trace_via_socks(target[0], target[1]).get("ip", "") if target else "" |
| append_log(GUARD_LOG_FILE, f"wall detected in Tor mode (exit {old_ip or '?'}) — new circuit") |
| if old_ip: |
| _blacklist_add(state, old_ip) |
| new_ip, msg = "", "" |
| for attempt in range(1, TOR_TRIES + 1): |
| _heartbeat() |
| ok, msg = rotate_tor_circuit() |
| target = get_active_tor_target() |
| new_ip = trace_via_socks(target[0], target[1]).get("ip", "") if target else "" |
| if not new_ip or (not _is_burnt(state, new_ip) and new_ip != old_ip): |
| break |
| append_log(GUARD_LOG_FILE, f"try {attempt}: Tor exit {new_ip} was already used up today — another one") |
| time.sleep(10) |
| bump_egress_epoch() |
| state["last_hop"] = now |
| state["hops"] = int(state.get("hops", 0) or 0) + 1 |
| if new_ip: |
| state["current"] = new_ip |
| _save_state(state) |
| append_log(GUARD_LOG_FILE, f"Tor rotated #{state['hops']}: {old_ip or '?'} -> {new_ip or '?'} ({msg})") |
| state["last_ok_hop"] = time.time() |
| _dispatch_auto_resume(state) |
| return |
| except Exception as exc: |
| append_log(GUARD_LOG_FILE, f"Tor rotation error: {exc}") |
| return |
| |
| |
| if mode == "custom": |
| append_log(GUARD_LOG_FILE, "wall detected in Custom Proxy mode — cycling proxy node") |
| try: |
| from proxy_chain import rotate_custom_proxy |
| ok, msg = rotate_custom_proxy() |
| bump_egress_epoch() |
| state["last_hop"] = now |
| state["hops"] = int(state.get("hops", 0) or 0) + 1 |
| _save_state(state) |
| append_log(GUARD_LOG_FILE, f"Custom proxy rotated #{state['hops']}: {msg}") |
| state["last_ok_hop"] = time.time() |
| _dispatch_auto_resume(state) |
| return |
| except Exception as exc: |
| append_log(GUARD_LOG_FILE, f"Custom proxy rotation error: {exc}") |
| state["last_hop"] = now |
| state["hops"] = int(state.get("hops", 0) or 0) + 1 |
| _save_state(state) |
| return |
| |
| |
| if not warp_bin: |
| append_log(GUARD_LOG_FILE, "wall detected but warp-cli is not installed for WARP rotation") |
| return |
| |
| old_v6 = get_egress_v6(timeout=8.0) or (get_warp_local_v6() or "") |
| old_prefix = normalize_v6_64(old_v6) |
| append_log(GUARD_LOG_FILE, f"wall detected — rotating WARP /64 (was: {old_v6 or '?'})") |
| _blacklist_add(state, old_v6) |
| _save_state(state) |
| |
| new_v6 = "" |
| with keeper_pause_ctx("zen-guard", seconds=300): |
| for attempt in range(1, MAX_HOP_TRIES + 1): |
| _heartbeat() |
| cand = hop_once(warp_bin) |
| if not cand: |
| if warp_path_leaking(max_age=60): |
| |
| |
| append_log( |
| GUARD_LOG_FILE, |
| f"try {attempt}: WARP is connected but traffic leaves with your own IP — " |
| f"not rotating further; OpenCode stays blocked until WARP is repaired", |
| ) |
| break |
| append_log(GUARD_LOG_FILE, f"try {attempt}: tunnel repair failed") |
| continue |
| if old_prefix and normalize_v6_64(cand) == old_prefix: |
| append_log( |
| GUARD_LOG_FILE, |
| f"try {attempt}: WARP returned the same address again ({cand}) — not a new IP", |
| ) |
| continue |
| if _is_burnt(state, cand): |
| append_log( |
| GUARD_LOG_FILE, |
| f"try {attempt}: cloudflare returned a blacklisted /64 ({cand}) — again", |
| ) |
| continue |
| new_v6 = cand |
| break |
| |
| if not new_v6 and not warp_path_leaking(max_age=60): |
| append_log(GUARD_LOG_FILE, "everything blacklisted — recycling oldest block") |
| cand = hop_once(warp_bin) |
| if cand: |
| new_v6 = cand |
| |
| state["last_hop"] = now |
| same_address = bool(old_prefix) and bool(new_v6) and normalize_v6_64(new_v6) == old_prefix |
| if new_v6 and not same_address: |
| state["current"] = new_v6 |
| state["hops"] = int(state.get("hops", 0) or 0) + 1 |
| _save_state(state) |
| append_log(GUARD_LOG_FILE, f"rotated #{state['hops']}: {old_v6} -> {new_v6} ✓") |
| try: |
| from ouc_common import LANE_DIR |
| (LANE_DIR / "edge_reroll").write_text(str(time.time()), encoding="utf-8") |
| except Exception: |
| pass |
| state["last_ok_hop"] = time.time() |
| _dispatch_auto_resume(state) |
| elif same_address: |
| |
| state["last_ok_hop"] = time.time() |
| state["hop_counted"] = state["last_ok_hop"] |
| state["failed_hops"] = int(state.get("failed_hops", 0) or 0) + 1 |
| _save_state(state) |
| append_log( |
| GUARD_LOG_FILE, |
| f"rotation ended on the same address ({new_v6}) — chat left paused instead of looping", |
| ) |
| else: |
| _save_state(state) |
| append_log( |
| GUARD_LOG_FILE, |
| "rotation failed — keeper will keep the tunnel alive, retrying next wall", |
| ) |
| |
| |
| def _end_pause(state: dict, warp_bin: str | None, now: int) -> bool: |
| """Called every tick: if the rotation pause is over, try one fresh IP for the chats still waiting. |
| |
| Returns True when a rotation was attempted. |
| """ |
| if not state.get("backoff_until") or now < int(state["backoff_until"]): |
| return False |
| state["backoff_until"] = 0 |
| state["failed_hops"] = 0 |
| state["backoff_logged"] = 0 |
| state["hop_counted"] = 0 |
| _save_state(state) |
| append_log(GUARD_LOG_FILE, "rotation pause is over") |
| if not state.get("pending"): |
| return False |
| append_log( |
| GUARD_LOG_FILE, |
| f"trying a fresh IP for {len(state['pending'])} chat(s) that are still waiting", |
| ) |
| _rotate(state, warp_bin) |
| return True |
| |
| |
| def main() -> int: |
| ensure_dirs() |
| mutex = acquire_mutex(MUTEX_GUARD) |
| if mutex is None: |
| return 0 |
| |
| warp_bin = find_warp_cli() |
| state = _load_state() |
| append_log( |
| GUARD_LOG_FILE, |
| f"guard up (poll {POLL_SECONDS}s, cooldown {COOLDOWN_SECONDS}s, warp={'found' if warp_bin else 'none'})", |
| ) |
| _heartbeat() |
| |
| while True: |
| time.sleep(POLL_SECONDS) |
| try: |
| _heartbeat() |
| now = int(time.time()) |
| |
| |
| _end_pause(state, warp_bin, now) |
| wall, sessions = _read_new_log_bytes(state) |
| if not wall: |
| continue |
| if _note_wall(state, now): |
| continue |
| pending = state.setdefault("pending", {}) |
| for sid in sessions: |
| pending.setdefault(sid, now) |
| _save_state(state) |
| last_hop = int(state.get("last_hop", 0) or 0) |
| age = now - last_hop |
| |
| hunting = ( |
| int(state.get("failed_hops", 0) or 0) > 0 |
| and now - int(state.get("last_ok_hop", 0) or 0) <= REWALL_SECONDS |
| ) |
| if age >= COOLDOWN_SECONDS or (hunting and age >= HUNT_GAP_SECONDS): |
| if hunting and age < COOLDOWN_SECONDS: |
| append_log(GUARD_LOG_FILE, "that address was already used up — trying the next one right away") |
| _rotate(state, warp_bin) |
| elif _resumable(state): |
| |
| |
| append_log( |
| GUARD_LOG_FILE, |
| f"{len(_resumable(state))} more chat(s) hit the limit around the last IP change " |
| f"({age}s ago) — sending them 'continue'", |
| ) |
| _dispatch_auto_resume(state) |
| else: |
| append_log( |
| GUARD_LOG_FILE, |
| f"wall seen but in cooldown ({age}s < {COOLDOWN_SECONDS}s) — waiting", |
| ) |
| except Exception as exc: |
| append_log(GUARD_LOG_FILE, f"guard error: {exc}") |
| |
| |
| if __name__ == "__main__": |
| raise SystemExit(main()) |