Agent Reverse Tunneling - Agent Role Implementation

This commit is contained in:
2025-12-01 02:40:23 -07:00
parent 478237f487
commit fe4511ecaf
8 changed files with 1155 additions and 11 deletions

View File

@@ -0,0 +1,2 @@
"""Reverse tunnel protocol modules (placeholder package)."""

View File

@@ -0,0 +1,226 @@
"""PowerShell channel implementation for reverse tunnel (Agent side)."""
from __future__ import annotations
import asyncio
import os
import sys
from typing import Any, Dict, Optional
# Message types mirrored from the tunnel framing (kept local to avoid import cycles).
MSG_DATA = 0x05
MSG_WINDOW_UPDATE = 0x06
MSG_CONTROL = 0x09
MSG_CLOSE = 0x08
# Close codes (mirrored from engine framing)
CLOSE_OK = 0
CLOSE_PROTOCOL_ERROR = 3
CLOSE_AGENT_SHUTDOWN = 6
class PowershellChannel:
def __init__(self, role, tunnel, channel_id: int, metadata: Optional[Dict[str, Any]]):
self.role = role
self.tunnel = tunnel
self.channel_id = channel_id
self.metadata = metadata or {}
self.loop = getattr(role, "loop", None) or asyncio.get_event_loop()
self._closed = False
self._reader_task = None
self._writer_task = None
self._stdin_queue: asyncio.Queue = asyncio.Queue()
self._pty = None
self._exit_code: Optional[int] = None
self._frame_cls = getattr(role, "_frame_cls", None)
# ------------------------------------------------------------------ Helpers
def _make_frame(self, msg_type: int, payload: bytes = b"", *, flags: int = 0):
frame_cls = self._frame_cls
if frame_cls is None:
return None
try:
return frame_cls(msg_type=msg_type, channel_id=self.channel_id, payload=payload or b"", flags=flags)
except Exception:
return None
async def _send_frame(self, frame) -> None:
if frame is None:
return
await self.role._send_frame(self.tunnel, frame)
async def _send_close(self, code: int, reason: str) -> None:
try:
close_frame = getattr(self.role, "close_frame")
if callable(close_frame):
await self._send_frame(close_frame(self.channel_id, code, reason))
return
except Exception:
pass
frame = self._make_frame(
MSG_CLOSE,
payload=f'{{"code":{code},"reason":"{reason}"}}'.encode("utf-8"),
)
await self._send_frame(frame)
def _powershell_path(self) -> str:
preferred = self.metadata.get("shell") if isinstance(self.metadata, dict) else None
if isinstance(preferred, str) and preferred.strip():
return preferred.strip()
# Default to Windows PowerShell; fallback to pwsh if provided later.
return "powershell.exe"
def _initial_size(self) -> tuple:
cols = int(self.metadata.get("cols") or self.metadata.get("columns") or 120) if isinstance(self.metadata, dict) else 120
rows = int(self.metadata.get("rows") or 32) if isinstance(self.metadata, dict) else 32
cols = max(20, min(cols, 300))
rows = max(10, min(rows, 200))
return cols, rows
# ------------------------------------------------------------------ Lifecycle
async def start(self) -> None:
if sys.platform.lower().startswith("win") is False:
await self._send_close(CLOSE_PROTOCOL_ERROR, "windows_only")
return
try:
import pywinpty # type: ignore
except Exception as exc: # pragma: no cover - dependency guard
self.role._log(f"reverse_tunnel ps channel missing pywinpty: {exc}", error=True)
await self._send_close(CLOSE_PROTOCOL_ERROR, "pywinpty_missing")
return
shell = self._powershell_path()
cols, rows = self._initial_size()
try:
self._pty = pywinpty.Process(
spawn_cmd=shell,
dimensions=(cols, rows),
)
except Exception as exc:
self.role._log(f"reverse_tunnel ps channel failed to spawn {shell}: {exc}", error=True)
await self._send_close(CLOSE_PROTOCOL_ERROR, "spawn_failed")
return
self._reader_task = self.loop.create_task(self._pump_stdout())
self._writer_task = self.loop.create_task(self._pump_stdin())
self.role._log(f"reverse_tunnel ps channel started shell={shell} cols={cols} rows={rows}")
async def on_frame(self, frame) -> None:
if self._closed:
return
if frame.msg_type == MSG_DATA:
if frame.payload:
try:
self._stdin_queue.put_nowait(frame.payload)
except Exception:
await self._stdin_queue.put(frame.payload)
elif frame.msg_type == MSG_CONTROL:
await self._handle_control(frame.payload)
elif frame.msg_type == MSG_CLOSE:
await self.stop(code=CLOSE_AGENT_SHUTDOWN, reason="operator_close")
elif frame.msg_type == MSG_WINDOW_UPDATE:
# Reserved for back-pressure; ignore for now.
return
async def _handle_control(self, payload: bytes) -> None:
try:
import json
data = json.loads(payload.decode("utf-8"))
except Exception:
return
cols = data.get("cols") or data.get("columns")
rows = data.get("rows")
if cols is None and rows is None:
return
try:
cols_int = int(cols) if cols is not None else None
rows_int = int(rows) if rows is not None else None
except Exception:
return
await self._resize(cols_int, rows_int)
async def _resize(self, cols: Optional[int], rows: Optional[int]) -> None:
if self._pty is None:
return
try:
cur_cols, cur_rows = self._initial_size()
if cols is None:
cols = cur_cols
if rows is None:
rows = cur_rows
cols = max(20, min(int(cols), 300))
rows = max(10, min(int(rows), 200))
self._pty.set_size(cols, rows)
self.role._log(f"reverse_tunnel ps channel resized cols={cols} rows={rows}")
except Exception:
self.role._log("reverse_tunnel ps channel resize failed", error=True)
async def _pump_stdout(self) -> None:
loop = asyncio.get_event_loop()
try:
while not self._closed and self._pty:
chunk = await loop.run_in_executor(None, self._pty.read, 4096)
if chunk is None:
break
if isinstance(chunk, str):
data = chunk.encode("utf-8", errors="replace")
else:
data = bytes(chunk)
if not data:
break
frame = self._make_frame(MSG_DATA, payload=data)
await self._send_frame(frame)
except asyncio.CancelledError:
pass
except Exception:
self.role._log("reverse_tunnel ps stdout pump error", error=True)
finally:
await self.stop(reason="stdout_closed")
async def _pump_stdin(self) -> None:
loop = asyncio.get_event_loop()
try:
while not self._closed and self._pty:
try:
data = await self._stdin_queue.get()
except asyncio.CancelledError:
break
if data is None:
break
if isinstance(data, (bytes, bytearray)):
text = data.decode("utf-8", errors="replace")
else:
text = str(data)
try:
await loop.run_in_executor(None, self._pty.write, text)
except Exception:
break
except asyncio.CancelledError:
pass
except Exception:
self.role._log("reverse_tunnel ps stdin pump error", error=True)
finally:
await self.stop(reason="stdin_closed")
async def stop(self, code: int = CLOSE_OK, reason: str = "") -> None:
if self._closed:
return
self._closed = True
if self._pty is not None:
try:
self._pty.terminate()
except Exception:
pass
current = asyncio.current_task()
if self._reader_task and self._reader_task is not current:
try:
self._reader_task.cancel()
except Exception:
pass
if self._writer_task and self._writer_task is not current:
try:
self._writer_task.cancel()
except Exception:
pass
await self._send_close(code, reason or "powershell_exit")
self.role._log(f"reverse_tunnel ps channel stopped channel={self.channel_id} reason={reason or 'exit'}")

View File

@@ -0,0 +1,654 @@
import asyncio
import base64
import json
import os
import struct
import time
from dataclasses import dataclass, field
from typing import Any, Dict, Optional
from urllib.parse import urlparse
import aiohttp
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric import ed25519
try:
from .ReverseTunnel import tunnel_Powershell
except Exception:
tunnel_Powershell = None
ROLE_NAME = "reverse_tunnel"
ROLE_CONTEXTS = ["interactive", "system"]
# Message types (keep in sync with Engine service)
MSG_CONNECT = 0x01
MSG_CONNECT_ACK = 0x02
MSG_CHANNEL_OPEN = 0x03
MSG_CHANNEL_ACK = 0x04
MSG_DATA = 0x05
MSG_WINDOW_UPDATE = 0x06
MSG_HEARTBEAT = 0x07
MSG_CLOSE = 0x08
MSG_CONTROL = 0x09
# Close codes
CLOSE_OK = 0
CLOSE_IDLE_TIMEOUT = 1
CLOSE_GRACE_EXPIRED = 2
CLOSE_PROTOCOL_ERROR = 3
CLOSE_AUTH_FAILED = 4
CLOSE_SERVER_SHUTDOWN = 5
CLOSE_AGENT_SHUTDOWN = 6
CLOSE_DOMAIN_LIMIT = 7
CLOSE_UNEXPECTED_DISCONNECT = 8
FRAME_VERSION = 1
FRAME_HEADER_STRUCT = struct.Struct("<BBBBII") # version, msg_type, flags, reserved, channel_id, length
@dataclass
class TunnelFrame:
msg_type: int
channel_id: int
payload: bytes = field(default_factory=bytes)
flags: int = 0
version: int = FRAME_VERSION
reserved: int = 0
def encode(self) -> bytes:
payload = self.payload or b""
header = FRAME_HEADER_STRUCT.pack(
self.version,
self.msg_type,
self.flags,
self.reserved,
int(self.channel_id),
len(payload),
)
return header + payload
def decode_frame(raw: bytes) -> TunnelFrame:
if len(raw) < FRAME_HEADER_STRUCT.size:
raise ValueError("frame_too_small")
version, msg_type, flags, reserved, channel_id, length = FRAME_HEADER_STRUCT.unpack_from(raw, 0)
if version != FRAME_VERSION:
raise ValueError(f"unsupported_version:{version}")
if length < 0 or len(raw) < FRAME_HEADER_STRUCT.size + length:
raise ValueError("invalid_length")
payload = raw[FRAME_HEADER_STRUCT.size : FRAME_HEADER_STRUCT.size + length]
if len(payload) != length:
raise ValueError("length_mismatch")
return TunnelFrame(msg_type=msg_type, channel_id=channel_id, payload=payload, flags=flags, version=version, reserved=reserved)
def heartbeat_frame(channel_id: int = 0, *, is_ack: bool = False) -> TunnelFrame:
flags = 0x1 if is_ack else 0x0
return TunnelFrame(msg_type=MSG_HEARTBEAT, channel_id=channel_id, flags=flags, payload=b"")
def close_frame(channel_id: int, code: int, reason: str = "") -> TunnelFrame:
payload = json.dumps({"code": code, "reason": reason}, separators=(",", ":")).encode("utf-8")
return TunnelFrame(msg_type=MSG_CLOSE, channel_id=channel_id, payload=payload)
def _norm_text(value: Any) -> str:
if value is None:
return ""
try:
return str(value).strip()
except Exception:
return ""
def _is_literal_ip(value: str) -> bool:
try:
import ipaddress
ipaddress.ip_address(value.strip().strip("[]"))
return True
except Exception:
return False
@dataclass
class ActiveTunnel:
tunnel_id: str
domain: str
protocol: str
port: int
token: str
url: str
heartbeat_seconds: int
idle_seconds: int
grace_seconds: int
expires_at: Optional[int]
signing_key_hint: Optional[str] = None
session: Optional[aiohttp.ClientSession] = None
websocket: Optional[aiohttp.ClientWebSocketResponse] = None
tasks: list = field(default_factory=list)
send_queue: asyncio.Queue = field(default_factory=asyncio.Queue)
channels: Dict[int, Any] = field(default_factory=dict)
last_activity: float = field(default_factory=lambda: time.time())
connected: bool = False
stopping: bool = False
stop_reason: Optional[str] = None
class BaseChannel:
"""Placeholder channel handler; protocol-specific handlers plug in later."""
def __init__(self, role: "Role", tunnel: ActiveTunnel, channel_id: int, metadata: Optional[dict]):
self.role = role
self.tunnel = tunnel
self.channel_id = channel_id
self.metadata = metadata or {}
async def start(self) -> None:
# Nothing to prime for placeholder channels.
return
async def on_frame(self, frame: TunnelFrame) -> None:
# Drop frames until protocol module is provided.
return
async def stop(self, code: int = CLOSE_OK, reason: str = "") -> None:
await self.role._send_frame(self.tunnel, close_frame(self.channel_id, code, reason))
class Role:
def __init__(self, ctx):
self.ctx = ctx
self.sio = ctx.sio
self.loop = ctx.loop or asyncio.get_event_loop()
self.hooks = ctx.hooks or {}
self._http_client_factory = self.hooks.get("http_client")
self._log_hook = self.hooks.get("log_agent")
self._active: Dict[str, ActiveTunnel] = {}
self._domain_claims: Dict[str, str] = {}
self._domain_limits: Dict[str, Optional[int]] = {
"ps": 1,
"rdp": 1,
"vnc": 1,
"webrtc": 1,
"ssh": None,
"winrm": None,
}
self._default_heartbeat = 20
self._protocol_handlers: Dict[str, Any] = {}
self._frame_cls = TunnelFrame
self.close_frame = close_frame
if tunnel_Powershell and hasattr(tunnel_Powershell, "PowershellChannel"):
self._protocol_handlers["ps"] = tunnel_Powershell.PowershellChannel
# ------------------------------------------------------------------ Logging
def _log(self, message: str, *, error: bool = False) -> None:
fname = "reverse_tunnel.log"
try:
if callable(self._log_hook):
self._log_hook(message, fname=fname)
if error:
self._log_hook(message, fname="agent.error.log")
except Exception:
pass
# ------------------------------------------------------------------ Event wiring
def register_events(self):
@self.sio.on("reverse_tunnel_start")
async def _reverse_tunnel_start(payload):
await self._handle_tunnel_start(payload)
@self.sio.on("reverse_tunnel_stop")
async def _reverse_tunnel_stop(payload):
tid = ""
if isinstance(payload, dict):
tid = _norm_text(payload.get("tunnel_id"))
await self._stop_tunnel(tid, code=CLOSE_AGENT_SHUTDOWN, reason="server_stop")
# ------------------------------------------------------------------ Token helpers
def _decode_token_payload(self, token: str, *, signing_key_hint: Optional[str] = None) -> Dict[str, Any]:
if not token:
raise ValueError("token_missing")
def _b64decode(segment: str) -> bytes:
padding = "=" * (-len(segment) % 4)
return base64.urlsafe_b64decode(segment + padding)
parts = token.split(".")
payload_segment = parts[0]
payload_bytes = _b64decode(payload_segment)
try:
payload = json.loads(payload_bytes.decode("utf-8"))
except Exception as exc:
raise ValueError("token_decode_error") from exc
if len(parts) == 2:
candidates = []
hint = _norm_text(signing_key_hint)
if hint:
candidates.append(hint)
client = self._http_client()
if client and hasattr(client, "load_server_signing_key"):
try:
stored = client.load_server_signing_key()
except Exception:
stored = None
if isinstance(stored, str) and stored.strip():
candidates.append(stored.strip())
signature = _b64decode(parts[1])
verified = False
for candidate in candidates:
try:
key_bytes = base64.b64decode(candidate, validate=True)
public_key = serialization.load_der_public_key(key_bytes)
except Exception:
continue
if not isinstance(public_key, ed25519.Ed25519PublicKey):
continue
try:
public_key.verify(signature, payload_bytes)
verified = True
if client and hasattr(client, "store_server_signing_key"):
try:
client.store_server_signing_key(candidate)
except Exception:
pass
break
except Exception:
continue
if not verified:
raise ValueError("token_signature_invalid")
return payload
def _validate_token(self, token: str, *, expected_agent: str, expected_domain: str, expected_protocol: str, expected_tunnel: str, signing_key_hint: Optional[str]) -> Dict[str, Any]:
payload = self._decode_token_payload(token, signing_key_hint=signing_key_hint)
def _matches(expected: str, actual: Any) -> bool:
return _norm_text(expected).lower() == _norm_text(actual).lower()
if expected_agent and not _matches(expected_agent, payload.get("agent_id")):
raise ValueError("token_agent_mismatch")
if expected_tunnel and not _matches(expected_tunnel, payload.get("tunnel_id")):
raise ValueError("token_id_mismatch")
if expected_domain and not _matches(expected_domain, payload.get("domain")):
raise ValueError("token_domain_mismatch")
if expected_protocol and not _matches(expected_protocol, payload.get("protocol")):
raise ValueError("token_protocol_mismatch")
expires_at = payload.get("expires_at")
try:
expires_ts = int(expires_at) if expires_at is not None else None
except Exception:
expires_ts = None
if expires_ts is not None and expires_ts < int(time.time()):
raise ValueError("token_expired")
return payload
# ------------------------------------------------------------------ Utility
def _http_client(self):
try:
if callable(self._http_client_factory):
return self._http_client_factory()
except Exception:
return None
return None
def _port_allowed(self, port: int) -> bool:
return isinstance(port, int) and 1 <= port <= 65535
def _domain_allowed(self, domain: str) -> bool:
limit = self._domain_limits.get(domain)
if limit is None:
return True
active = [tid for tid, t in self._active.items() if t.domain == domain and not t.stopping]
pending = [tid for dom, tid in self._domain_claims.items() if dom == domain and tid not in active]
count = len(set(active + pending))
return count < limit
def _build_ws_url(self, port: int) -> str:
client = self._http_client()
if client:
try:
client.refresh_base_url()
except Exception:
pass
base_url = getattr(client, "base_url", "") or "https://localhost:5000"
else:
base_url = "https://localhost:5000"
parsed = urlparse(base_url)
host = parsed.hostname or "localhost"
scheme = "wss" if (parsed.scheme or "").lower() == "https" else "ws"
return f"{scheme}://{host}:{port}/"
def _ssl_context(self, host: str):
client = self._http_client()
verify = getattr(getattr(client, "session", None), "verify", True)
if verify is False:
return False
if isinstance(verify, str) and os.path.isfile(verify) and client and hasattr(client, "key_store"):
try:
ctx = client.key_store.build_ssl_context()
if ctx and _is_literal_ip(host):
ctx.check_hostname = False
return ctx
except Exception:
return None
return None
async def _emit_status(self, payload: Dict[str, Any]) -> None:
try:
await self.sio.emit("reverse_tunnel_status", payload)
except Exception:
pass
def _mark_activity(self, tunnel: ActiveTunnel) -> None:
tunnel.last_activity = time.time()
# ------------------------------------------------------------------ Event handlers
async def _handle_tunnel_start(self, payload: Any) -> None:
if not isinstance(payload, dict):
self._log("reverse_tunnel_start ignored: payload not a dict", error=True)
return
tunnel_id = _norm_text(payload.get("tunnel_id"))
token = _norm_text(payload.get("token"))
port = payload.get("port") or payload.get("assigned_port")
protocol = _norm_text(payload.get("protocol") or "ps").lower() or "ps"
domain = _norm_text(payload.get("domain") or protocol).lower() or protocol
heartbeat_seconds = int(payload.get("heartbeat_seconds") or self._default_heartbeat or 20)
idle_seconds = int(payload.get("idle_seconds") or 3600)
grace_seconds = int(payload.get("grace_seconds") or 3600)
signing_key_hint = _norm_text(payload.get("signing_key"))
if not token:
self._log("reverse_tunnel_start rejected: missing token", error=True)
await self._emit_status({"tunnel_id": tunnel_id, "agent_id": self.ctx.agent_id, "status": "error", "reason": "token_missing"})
return
if not tunnel_id:
tunnel_id = _norm_text(payload.get("lease_id")) # fallback alias
try:
claims = self._validate_token(
token,
expected_agent=self.ctx.agent_id,
expected_domain=domain,
expected_protocol=protocol,
expected_tunnel=tunnel_id,
signing_key_hint=signing_key_hint,
)
except Exception as exc:
self._log(f"reverse_tunnel_start rejected: token validation failed ({exc})", error=True)
await self._emit_status({"tunnel_id": tunnel_id, "agent_id": self.ctx.agent_id, "status": "error", "reason": "token_invalid"})
return
tunnel_id = _norm_text(claims.get("tunnel_id") or tunnel_id)
if not tunnel_id:
self._log("reverse_tunnel_start rejected: tunnel_id missing after token parse", error=True)
return
try:
port = int(port or claims.get("assigned_port") or 0)
except Exception:
port = 0
if not self._port_allowed(port):
self._log(f"reverse_tunnel_start rejected: invalid port {port}", error=True)
await self._emit_status({"tunnel_id": tunnel_id, "agent_id": self.ctx.agent_id, "status": "error", "reason": "invalid_port"})
return
domain = _norm_text(claims.get("domain") or domain).lower()
protocol = _norm_text(claims.get("protocol") or protocol).lower()
expires_at = claims.get("expires_at")
if tunnel_id in self._active:
self._log(f"reverse_tunnel_start ignored: tunnel already active tunnel_id={tunnel_id}")
return
if not self._domain_allowed(domain):
self._log(f"reverse_tunnel_start rejected: domain limit for {domain}", error=True)
await self._emit_status({"tunnel_id": tunnel_id, "agent_id": self.ctx.agent_id, "status": "error", "reason": "domain_limit"})
return
url = self._build_ws_url(port)
parsed = urlparse(url)
heartbeat_seconds = max(5, min(heartbeat_seconds, 120))
idle_seconds = max(30, idle_seconds)
grace_seconds = max(60, grace_seconds)
tunnel = ActiveTunnel(
tunnel_id=tunnel_id,
domain=domain,
protocol=protocol,
port=port,
token=token,
url=url,
heartbeat_seconds=heartbeat_seconds,
idle_seconds=idle_seconds,
grace_seconds=grace_seconds,
expires_at=int(expires_at) if expires_at is not None else None,
signing_key_hint=signing_key_hint or None,
)
self._active[tunnel_id] = tunnel
self._domain_claims[domain] = tunnel_id
self._log(f"reverse_tunnel_start accepted tunnel_id={tunnel_id} domain={domain} protocol={protocol} url={url}")
await self._emit_status({"tunnel_id": tunnel_id, "agent_id": self.ctx.agent_id, "status": "connecting", "url": url})
task = self.loop.create_task(self._run_tunnel(tunnel, host=parsed.hostname or "localhost"))
tunnel.tasks.append(task)
# ------------------------------------------------------------------ Core tunnel handling
async def _run_tunnel(self, tunnel: ActiveTunnel, *, host: str) -> None:
ssl_ctx = self._ssl_context(host)
timeout = aiohttp.ClientTimeout(total=None, sock_connect=10, sock_read=None)
try:
tunnel.session = aiohttp.ClientSession(timeout=timeout)
tunnel.websocket = await tunnel.session.ws_connect(
tunnel.url,
ssl=ssl_ctx,
heartbeat=None,
max_msg_size=0,
timeout=timeout,
)
self._mark_activity(tunnel)
await tunnel.websocket.send_bytes(
TunnelFrame(
msg_type=MSG_CONNECT,
channel_id=0,
payload=json.dumps(
{
"agent_id": self.ctx.agent_id,
"tunnel_id": tunnel.tunnel_id,
"token": tunnel.token,
"protocol": tunnel.protocol,
"domain": tunnel.domain,
"version": FRAME_VERSION,
},
separators=(",", ":"),
).encode("utf-8"),
).encode()
)
sender = self.loop.create_task(self._pump_sender(tunnel))
receiver = self.loop.create_task(self._pump_receiver(tunnel))
heartbeats = self.loop.create_task(self._heartbeat_loop(tunnel))
watchdog = self.loop.create_task(self._watchdog(tunnel))
tunnel.tasks.extend([sender, receiver, heartbeats, watchdog])
await asyncio.wait([sender, receiver, heartbeats, watchdog], return_when=asyncio.FIRST_COMPLETED)
except Exception as exc:
self._log(f"reverse_tunnel connection failed tunnel_id={tunnel.tunnel_id}: {exc}", error=True)
await self._emit_status({"tunnel_id": tunnel.tunnel_id, "agent_id": self.ctx.agent_id, "status": "error", "reason": "connect_failed"})
finally:
await self._shutdown_tunnel(tunnel)
async def _pump_sender(self, tunnel: ActiveTunnel) -> None:
try:
while tunnel.websocket and not tunnel.websocket.closed:
frame: TunnelFrame = await tunnel.send_queue.get()
try:
await tunnel.websocket.send_bytes(frame.encode())
self._mark_activity(tunnel)
except Exception:
break
except asyncio.CancelledError:
pass
except Exception:
self._log(f"reverse_tunnel sender failed tunnel_id={tunnel.tunnel_id}", error=True)
async def _pump_receiver(self, tunnel: ActiveTunnel) -> None:
ws = tunnel.websocket
if ws is None:
return
try:
async for msg in ws:
if msg.type == aiohttp.WSMsgType.BINARY:
try:
frame = decode_frame(msg.data)
except Exception:
self._log(f"reverse_tunnel frame decode failed tunnel_id={tunnel.tunnel_id}", error=True)
continue
self._mark_activity(tunnel)
await self._handle_frame(tunnel, frame)
elif msg.type in (aiohttp.WSMsgType.CLOSED, aiohttp.WSMsgType.ERROR, aiohttp.WSMsgType.CLOSE):
break
except asyncio.CancelledError:
pass
except Exception:
self._log(f"reverse_tunnel receiver failed tunnel_id={tunnel.tunnel_id}", error=True)
async def _heartbeat_loop(self, tunnel: ActiveTunnel) -> None:
try:
while tunnel.websocket and not tunnel.websocket.closed:
await asyncio.sleep(tunnel.heartbeat_seconds)
await self._send_frame(tunnel, heartbeat_frame())
except asyncio.CancelledError:
pass
except Exception:
self._log(f"reverse_tunnel heartbeat failed tunnel_id={tunnel.tunnel_id}", error=True)
async def _watchdog(self, tunnel: ActiveTunnel) -> None:
try:
while tunnel.websocket and not tunnel.websocket.closed:
await asyncio.sleep(10)
now = time.time()
if tunnel.idle_seconds and (now - tunnel.last_activity) >= tunnel.idle_seconds:
await self._send_frame(tunnel, close_frame(0, CLOSE_IDLE_TIMEOUT, "idle_timeout"))
break
if tunnel.expires_at and (now - tunnel.expires_at) >= tunnel.grace_seconds:
await self._send_frame(tunnel, close_frame(0, CLOSE_GRACE_EXPIRED, "grace_expired"))
break
except asyncio.CancelledError:
pass
except Exception:
self._log(f"reverse_tunnel watchdog failed tunnel_id={tunnel.tunnel_id}", error=True)
async def _handle_frame(self, tunnel: ActiveTunnel, frame: TunnelFrame) -> None:
if frame.msg_type == MSG_HEARTBEAT:
if frame.flags & 0x1:
return
await self._send_frame(tunnel, heartbeat_frame(channel_id=frame.channel_id, is_ack=True))
return
if frame.msg_type == MSG_CONNECT_ACK:
tunnel.connected = True
await self._emit_status({"tunnel_id": tunnel.tunnel_id, "agent_id": self.ctx.agent_id, "status": "connected"})
return
if frame.msg_type == MSG_CHANNEL_OPEN:
await self._handle_channel_open(tunnel, frame)
return
if frame.msg_type == MSG_CLOSE:
try:
reason = json.loads(frame.payload.decode("utf-8"))
except Exception:
reason = {"code": CLOSE_UNEXPECTED_DISCONNECT, "reason": "close"}
tunnel.stop_reason = reason.get("reason") if isinstance(reason, dict) else None
await self._shutdown_tunnel(tunnel, send_close=False)
return
handler = tunnel.channels.get(frame.channel_id)
if handler:
try:
await handler.on_frame(frame)
except Exception:
self._log(f"reverse_tunnel channel handler failed tunnel_id={tunnel.tunnel_id} channel={frame.channel_id}", error=True)
else:
if frame.msg_type in (MSG_DATA, MSG_CONTROL, MSG_WINDOW_UPDATE):
await self._send_frame(tunnel, close_frame(frame.channel_id, CLOSE_PROTOCOL_ERROR, "unknown_channel"))
async def _handle_channel_open(self, tunnel: ActiveTunnel, frame: TunnelFrame) -> None:
try:
payload = json.loads(frame.payload.decode("utf-8"))
except Exception:
payload = {}
protocol = _norm_text(payload.get("protocol") or tunnel.protocol)
metadata = payload.get("metadata") if isinstance(payload, dict) else {}
if frame.channel_id in tunnel.channels:
await self._send_frame(tunnel, close_frame(frame.channel_id, CLOSE_PROTOCOL_ERROR, "channel_exists"))
return
handler_cls = self._protocol_handlers.get(protocol.lower()) or BaseChannel
try:
handler = handler_cls(self, tunnel, frame.channel_id, metadata)
except Exception:
handler = BaseChannel(self, tunnel, frame.channel_id, metadata)
tunnel.channels[frame.channel_id] = handler
await handler.start()
await self._send_frame(
tunnel,
TunnelFrame(
msg_type=MSG_CHANNEL_ACK,
channel_id=frame.channel_id,
payload=json.dumps({"status": "ok", "protocol": protocol}, separators=(",", ":")).encode("utf-8"),
),
)
async def _send_frame(self, tunnel: ActiveTunnel, frame: TunnelFrame) -> None:
if tunnel.stopping:
return
try:
tunnel.send_queue.put_nowait(frame)
except Exception:
await tunnel.send_queue.put(frame)
async def _stop_tunnel(self, tunnel_id: str, *, code: int = CLOSE_AGENT_SHUTDOWN, reason: str = "requested") -> None:
tunnel = self._active.get(tunnel_id)
if not tunnel:
return
await self._send_frame(tunnel, close_frame(0, code, reason))
await self._shutdown_tunnel(tunnel, send_close=False)
async def _shutdown_tunnel(self, tunnel: ActiveTunnel, *, send_close: bool = True) -> None:
if tunnel.stopping:
return
tunnel.stopping = True
if send_close:
try:
await self._send_frame(tunnel, close_frame(0, CLOSE_AGENT_SHUTDOWN, "agent_shutdown"))
except Exception:
pass
for task in list(tunnel.tasks):
try:
task.cancel()
except Exception:
pass
if tunnel.websocket is not None:
try:
await tunnel.websocket.close()
except Exception:
pass
if tunnel.session is not None:
try:
await tunnel.session.close()
except Exception:
pass
self._active.pop(tunnel.tunnel_id, None)
if tunnel.domain in self._domain_claims and self._domain_claims.get(tunnel.domain) == tunnel.tunnel_id:
self._domain_claims.pop(tunnel.domain, None)
self._log(f"reverse_tunnel stopped tunnel_id={tunnel.tunnel_id} reason={tunnel.stop_reason or 'closed'}")
await self._emit_status({"tunnel_id": tunnel.tunnel_id, "agent_id": self.ctx.agent_id, "status": "closed", "reason": tunnel.stop_reason or "closed"})
# ------------------------------------------------------------------ Lifecycle
def stop_all(self):
for tunnel_id in list(self._active.keys()):
try:
self.loop.create_task(self._stop_tunnel(tunnel_id, code=CLOSE_AGENT_SHUTDOWN, reason="agent_shutdown"))
except Exception:
pass