From 8fa7bd4fb0d23b8d2f4fb798b13c7068f7deab93 Mon Sep 17 00:00:00 2001 From: Nicole Rappe Date: Wed, 29 Oct 2025 15:19:29 -0600 Subject: [PATCH] ENGINE: Migrated Logs to Runtime Folders --- AGENTS.md | 15 +++++++-------- Borealis.ps1 | 7 ++++--- Borealis.sh | 6 +++--- Data/Agent/Roles/role_PlaybookExec_SYSTEM.py | 8 ++++---- Data/Agent/Roles/role_ScriptExec_SYSTEM.py | 2 +- Data/Agent/agent.py | 11 ++++++----- Data/Agent/agent_deployment.py | 2 +- Data/Agent/launch_service.ps1 | 6 ++++-- Data/Engine/config.py | 6 +++--- Data/Engine/server.py | 2 +- readme.md | 4 ++-- 11 files changed, 36 insertions(+), 33 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 11a76766..2ed2460f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,9 +4,9 @@ - **Runtime Paths**: Do not edit `/Agent`; make changes in `Data/Agent` so the runtime copy stays ephemeral. Runtime folders are wiped regularly. ### Logging -- General log: `Logs/Agent/agent.log`; rotate daily to `agent.log.YYYY-MM-DD` and never delete automatically. -- Subsystems (e.g., `ansible`, `webrtc`, `scheduler`) must log to `Logs/Agent/.log` and follow the same rotation policy. -- Installation output writes to `Logs/Agent/install.log`. +- General log: `Agent/Logs/agent.log`; rotate daily to `agent.log.YYYY-MM-DD` and never delete automatically. +- Subsystems (e.g., `ansible`, `webrtc`, `scheduler`) must log to `Agent/Logs/.log` and follow the same rotation policy. +- Installation output writes to `Agent/Logs/install.log`; keep ad-hoc diagnostics (e.g., `system_last.ps1`, ansible traces) under `Agent/Logs/` so runtime state stays self-contained. - When troubleshooting with operators, prepend each line with `--` and confirm whether to keep or remove verbose logging after resolution. ### Security @@ -15,7 +15,7 @@ - Uses a dedicated `ssl.SSLContext` seeded with the Engine’s TLS bundle for REST and Socket.IO traffic. - Validates all script payloads with Ed25519 signatures issued by the backend before execution. - Enforces outbound-only communication; every API/WebSocket call flows through `AgentHttpClient.ensure_authenticated` to refresh tokens proactively. -- Logs bootstrap, enrollment, token refresh, and signature events under `Logs/Agent/`. +- Logs bootstrap, enrollment, token refresh, and signature events under `Agent/Logs/`. ### Execution Contexts & Roles - Roles auto-discover from `Data/Agent/Roles/` and require no loader changes. @@ -43,9 +43,9 @@ - Reference the migration tracker before making Engine changes to avoid jumping ahead of the approved stage. ### Logging -- General log: `Logs/Engine/engine.log` with daily rotation (`engine.log.YYYY-MM-DD`); do not auto-delete rotated files. -- Subsystems should log to `Logs/Engine/.log`; installation output belongs in `Logs/Engine/install.log`. -- Adhere to the centralized logging policy and keep all log files inside the project root. +- General log: `Engine/Logs/engine.log` with daily rotation (`engine.log.YYYY-MM-DD`); do not auto-delete rotated files. +- Subsystems should log to `Engine/Logs/.log`; installation output belongs in `Engine/Logs/install.log`. +- Adhere to the centralized logging policy and keep Engine-specific artifacts within `Engine/Logs/` to preserve the runtime boundary. ### Security & API Parity - Shares the mutual trust model with the legacy server: Ed25519 device identities, EdDSA-signed access tokens, pinned Borealis root CA, TLS 1.3-only serving, and Authorization headers plus service-context markers on every device API. @@ -78,4 +78,3 @@ ### Platform Notes - Exists primarily to document past behaviour and assist the Engine migration. Future platform parity work should target the Engine; the legacy server will be deprecated once feature parity is confirmed. - diff --git a/Borealis.ps1 b/Borealis.ps1 index 42bbcae6..ba0a779d 100644 --- a/Borealis.ps1 +++ b/Borealis.ps1 @@ -165,8 +165,9 @@ function Request-AgentElevation { # Ensure log directories function Ensure-AgentLogDir { - $logRoot = Join-Path $scriptDir 'Logs' - $agentLogDir = Join-Path $logRoot 'Agent' + $agentRoot = Join-Path $scriptDir 'Agent' + if (-not (Test-Path $agentRoot)) { New-Item -ItemType Directory -Path $agentRoot -Force | Out-Null } + $agentLogDir = Join-Path $agentRoot 'Logs' if (-not (Test-Path $agentLogDir)) { New-Item -ItemType Directory -Path $agentLogDir -Force | Out-Null } return $agentLogDir } @@ -1036,7 +1037,7 @@ function InstallOrUpdate-BorealisAgent { } } catch { Write-AgentLog -FileName 'Install.log' -Message ("[CONFIG] Failed to persist agent_settings.json: {0}" -f $_.Exception.Message) - Write-Host "Failed to update agent_settings.json. Check Logs/Agent/install.log for details." -ForegroundColor Red + Write-Host "Failed to update agent_settings.json. Check Agent/Logs/install.log for details." -ForegroundColor Red } } diff --git a/Borealis.sh b/Borealis.sh index 8a01512b..2b5c24b8 100644 --- a/Borealis.sh +++ b/Borealis.sh @@ -96,8 +96,8 @@ detect_distro() { fi } -ensure_log_dir() { mkdir -p "${SCRIPT_DIR}/Logs/Agent"; } -log_agent() { ensure_log_dir; printf "[%s] %s\n" "$(date +%F\ %T)" "$1" >> "${SCRIPT_DIR}/Logs/Agent/$2"; } +ensure_log_dir() { mkdir -p "${SCRIPT_DIR}/Agent/Logs"; } +log_agent() { ensure_log_dir; printf "[%s] %s\n" "$(date +%F\ %T)" "$1" >> "${SCRIPT_DIR}/Agent/Logs/$2"; } need_sudo() { [ "${EUID:-$(id -u)}" -ne 0 ]; } @@ -231,7 +231,7 @@ set -o nounset set -o pipefail ROOT_DIR="$(cd -- "$(dirname -- "$0")" && pwd)" cd "$ROOT_DIR" -LOG_DIR="$(cd -- "$ROOT_DIR/../../Logs/Agent" && pwd 2>/dev/null || echo "$ROOT_DIR/../../Logs/Agent")" +LOG_DIR="$(cd -- "$ROOT_DIR/../Logs" && pwd 2>/dev/null || echo "$ROOT_DIR/../Logs")" mkdir -p "$LOG_DIR" PY_BIN="${ROOT_DIR}/../bin/python3" exec "$PY_BIN" "$ROOT_DIR/agent.py" --system-service --config SYSTEM >>"$LOG_DIR/svc.out.log" 2>>"$LOG_DIR/svc.err.log" diff --git a/Data/Agent/Roles/role_PlaybookExec_SYSTEM.py b/Data/Agent/Roles/role_PlaybookExec_SYSTEM.py index 62a7fb45..db1df154 100644 --- a/Data/Agent/Roles/role_PlaybookExec_SYSTEM.py +++ b/Data/Agent/Roles/role_PlaybookExec_SYSTEM.py @@ -242,7 +242,7 @@ class Role: self._ansible_ready = False self._ansible_bootstrap_lock = None try: - base = os.path.join(_project_root(), 'Logs', 'Agent') + base = os.path.join(_project_root(), 'Agent', 'Logs') os.makedirs(base, exist_ok=True) self._ansible_log(f"[init] PlaybookExec role init agent_id={ctx.agent_id}") except Exception: @@ -580,7 +580,7 @@ class Role: def _log_local(self, msg: str, error: bool = False): try: - base = os.path.join(_project_root(), 'Logs', 'Agent') + base = os.path.join(_project_root(), 'Agent', 'Logs') os.makedirs(base, exist_ok=True) fn = 'agent.error.log' if error else 'agent.log' ts = time.strftime('%Y-%m-%d %H:%M:%S') @@ -600,7 +600,7 @@ class Role: def _ansible_log(self, msg: str, error: bool = False, run_id: str = None): try: - d = os.path.join(_project_root(), 'Logs', 'Agent') + d = os.path.join(_project_root(), 'Agent', 'Logs') ts = time.strftime('%Y-%m-%d %H:%M:%S') path = os.path.join(d, 'ansible.log') try: @@ -716,7 +716,7 @@ class Role: if os.name != 'nt': return mod = self._ps_module_path() - log_dir = os.path.join(_project_root(), 'Logs', 'Agent') + log_dir = os.path.join(_project_root(), 'Agent', 'Logs') try: os.makedirs(log_dir, exist_ok=True) except Exception: diff --git a/Data/Agent/Roles/role_ScriptExec_SYSTEM.py b/Data/Agent/Roles/role_ScriptExec_SYSTEM.py index 0e367c01..0c262285 100644 --- a/Data/Agent/Roles/role_ScriptExec_SYSTEM.py +++ b/Data/Agent/Roles/role_ScriptExec_SYSTEM.py @@ -219,7 +219,7 @@ def _run_powershell_via_system_task(content: str, env_map: Dict[str, str], timeo with os.fdopen(script_fd, 'w', encoding='utf-8', newline='\n') as f: f.write(final_content) try: - log_dir = os.path.join(_project_root(), 'Logs', 'Agent') + log_dir = os.path.join(_project_root(), 'Agent', 'Logs') os.makedirs(log_dir, exist_ok=True) with open(os.path.join(log_dir, 'system_last.ps1'), 'w', encoding='utf-8', newline='\n') as df: df.write(content or '') diff --git a/Data/Agent/agent.py b/Data/Agent/agent.py index f2a506c6..60d84569 100644 --- a/Data/Agent/agent.py +++ b/Data/Agent/agent.py @@ -57,9 +57,10 @@ def _iter_exception_chain(exc: BaseException): def _agent_logs_root() -> str: try: root = _find_project_root() - return os.path.abspath(os.path.join(root, 'Logs', 'Agent')) + return os.path.abspath(os.path.join(root, 'Agent', 'Logs')) except Exception: - return os.path.abspath(os.path.join(os.path.dirname(__file__), 'Logs', 'Agent')) + base_dir = os.path.abspath(os.path.dirname(__file__)) + return os.path.abspath(os.path.join(base_dir, 'Logs')) def _rotate_daily(path: str): @@ -517,7 +518,7 @@ def _find_project_root(): # Heuristic fallback: two levels up from Agent/Borealis return os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -# Simple file logger under Logs/Agent +# Simple file logger under Agent/Logs def _log_agent(message: str, fname: str = 'agent.log', *, scope: Optional[str] = None): try: log_dir = _agent_logs_root() @@ -2856,7 +2857,7 @@ def _run_powershell_via_system_task(content: str): with os.fdopen(fd, 'w', encoding='utf-8', newline='\n') as f: f.write(content or '') try: - log_dir = os.path.join(_project_root_for_temp(), 'Logs', 'Agent') + log_dir = os.path.join(_project_root_for_temp(), 'Agent', 'Logs') os.makedirs(log_dir, exist_ok=True) debug_copy = os.path.join(log_dir, 'system_last.ps1') with open(debug_copy, 'w', encoding='utf-8', newline='\n') as df: @@ -3260,7 +3261,7 @@ if __name__=='__main__': return try: # Save last SYSTEM script for debugging - dbg_dir = os.path.join(_find_project_root(), 'Logs', 'Agent') + dbg_dir = os.path.join(_find_project_root(), 'Agent', 'Logs') os.makedirs(dbg_dir, exist_ok=True) with open(os.path.join(dbg_dir, 'system_last.ps1'), 'w', encoding='utf-8', newline='\n') as df: df.write(content or '') diff --git a/Data/Agent/agent_deployment.py b/Data/Agent/agent_deployment.py index 332d8ba9..4c6f117d 100644 --- a/Data/Agent/agent_deployment.py +++ b/Data/Agent/agent_deployment.py @@ -14,7 +14,7 @@ def project_paths(): venv_root = os.path.abspath(os.path.join(venv_scripts, os.pardir)) project_root = os.path.abspath(os.path.join(venv_root, os.pardir)) borealis_dir = os.path.join(venv_root, "Borealis") - logs_dir = os.path.join(project_root, "Logs", "Agent") + logs_dir = os.path.join(project_root, "Agent", "Logs") temp_dir = os.path.join(project_root, "Temp") return { "project_root": project_root, diff --git a/Data/Agent/launch_service.ps1 b/Data/Agent/launch_service.ps1 index 4c7a48ac..29a3b80d 100644 --- a/Data/Agent/launch_service.ps1 +++ b/Data/Agent/launch_service.ps1 @@ -9,9 +9,11 @@ try { $scriptDir = Split-Path -Path $PSCommandPath -Parent Set-Location -Path $scriptDir - # Centralized logs under \Logs\Agent + # Centralized logs under \Agent\Logs $projRoot = Resolve-Path (Join-Path $scriptDir '..\..') - $logsAgent = Join-Path $projRoot 'Logs\Agent' + $agentRoot = Join-Path $projRoot 'Agent' + if (-not (Test-Path $agentRoot)) { New-Item -ItemType Directory -Path $agentRoot -Force | Out-Null } + $logsAgent = Join-Path $agentRoot 'Logs' if (-not (Test-Path $logsAgent)) { New-Item -ItemType Directory -Path $logsAgent -Force | Out-Null } $wrapperLog = Join-Path $logsAgent 'service_wrapper.log' diff --git a/Data/Engine/config.py b/Data/Engine/config.py index 89bba6bb..c716ed35 100644 --- a/Data/Engine/config.py +++ b/Data/Engine/config.py @@ -34,8 +34,8 @@ defaults that mirror the legacy server runtime. Key environment variables are When TLS values are not provided explicitly the Engine provisions certificates under ``Engine/Certificates`` (migrating any legacy material) so the runtime remains self-contained. -Logs are written to ``Logs/Engine/engine.log`` with daily rotation and -errors are additionally duplicated to ``Logs/Engine/error.log`` so the +Logs are written to ``Engine/Logs/engine.log`` with daily rotation and +errors are additionally duplicated to ``Engine/Logs/error.log`` so the runtime integrates with the platform's logging policy. """ @@ -73,7 +73,7 @@ def _discover_project_root() -> Path: PROJECT_ROOT = _discover_project_root() DEFAULT_DATABASE_PATH = PROJECT_ROOT / "database.db" -LOG_ROOT = PROJECT_ROOT / "Logs" / "Engine" +LOG_ROOT = PROJECT_ROOT / "Engine" / "Logs" LOG_FILE_PATH = LOG_ROOT / "engine.log" ERROR_LOG_FILE_PATH = LOG_ROOT / "error.log" API_LOG_FILE_PATH = LOG_ROOT / "api.log" diff --git a/Data/Engine/server.py b/Data/Engine/server.py index 6cde58b4..d895492f 100644 --- a/Data/Engine/server.py +++ b/Data/Engine/server.py @@ -11,7 +11,7 @@ Stage 1 introduced the structural skeleton for the Engine runtime. Stage 2 builds upon that foundation by centralising configuration handling and logging initialisation so the Engine mirrors the legacy server's start-up behaviour. The factory delegates configuration resolution to :mod:`Data.Engine.config` -and emits structured logs to ``Logs/Engine/engine.log`` (with an accompanying +and emits structured logs to ``Engine/Logs/engine.log`` (with an accompanying error log) to align with the project's operational practices. """ from __future__ import annotations diff --git a/readme.md b/readme.md index 42db8eff..4c1e3f39 100644 --- a/readme.md +++ b/readme.md @@ -90,7 +90,7 @@ The process that agents go through when authenticating securely with a Borealis - Device enrollment is gated by enrollment/installer codes (*They have configurable expiration and usage limits*) and an operator approval queue; replay-resistant nonces plus rate limits (40 req/min/IP, 12 req/min/fingerprint) prevent brute force or code reuse. - All device APIs now require Authorization: Bearer headers and a service-context (e.g. SYSTEM or CURRENTUSER) marker; missing, expired, mismatched, or revoked credentials are rejected before any business logic runs. Operator-driven revoking / device quarantining logic is not yet implemented. - Replay and credential theft defenses layer in DPoP proof validation (thumbprint binding) on the server side and short-lived access tokens (15 min) with 30-day refresh tokens hashed via SHA-256. -- Centralized logging under Logs/Server and Logs/Agent captures enrollment approvals, rate-limit hits, signature failures, and auth anomalies for post-incident review. +- Centralized logging under Logs/Server and Agent/Logs captures enrollment approvals, rate-limit hits, signature failures, and auth anomalies for post-incident review. #### Server Security - Auto-manages PKI: a persistent Borealis root CA (ECDSA SECP384R1) signs leaf certificates that include localhost SANs, tightened filesystem permissions, and a combined bundle for agent identity / cert pinning. - Script delivery is code-signed with an Ed25519 key stored under Certificates/Server/Code-Signing; agents refuse any payload whose signature or hash does not match the pinned public key. @@ -104,7 +104,7 @@ The process that agents go through when authenticating securely with a Borealis - Imports the server’s TLS bundle into a dedicated ssl.SSLContext, reuses it for the REST session, and injects it into the Socket.IO engine so WebSockets enjoy the same pinning and hostname checks. - Treats every script payload as hostile until verified: only Ed25519 signatures from the server are accepted, missing/invalid signatures are logged and dropped, and the trusted signing key is updated only after successful verification between the agent and the server. - Operates outbound-only; there are no listener ports, and every API/WebSocket call flows through AgentHttpClient.ensure_authenticated, forcing token refresh logic before retrying. -- Logs bootstrap, enrollment, token refresh, and signature events to daily-rotated files under Logs/Agent, giving operators visibility without leaking secrets outside the project root. +- Logs bootstrap, enrollment, token refresh, and signature events to daily-rotated files under Agent/Logs, giving operators visibility without leaking secrets outside the project root. ### Agent/Server Enrollment ```mermaid