feat(engine): scaffold runtime skeleton

Data/Engine/CODE_MIGRATION_TRACKER.md

@@ -0,0 +1,40 @@
+# Borealis Engine Migration Tracker
+
+## Current Focus
+- **Stage:** Stage 1 — Establish the Engine skeleton and bootstrapper
+- **Active Task:** Stage 1 tasks completed; awaiting direction to proceed to Stage 2.
+
+## Task Ledger
+- [x] **Stage 1 — Establish the Engine skeleton and bootstrapper**
+  - [x] Add Data/Engine/__init__.py plus service subpackages with placeholder modules and docstrings.
+  - [x] Scaffold Data/Engine/server.py with the create_app(config) factory and stub service registration hooks.
+  - [x] Return a shared context object containing handles such as the database path, logger, and scheduler.
+  - [x] Update project tooling so the Engine runtime can be launched alongside the legacy path.
+- [ ] **Stage 2 — Port configuration and dependency loading into the Engine factory**
+  - [ ] Extract configuration loading logic from Data/Server/server.py into Data/Engine/config.py helpers.
+  - [ ] Verify context parity between Engine and legacy startup.
+  - [ ] Initialize logging to Logs/Server/server.log when Engine mode is active.
+  - [ ] Document Engine launch paths and configuration requirements in module docstrings.
+- [ ] **Stage 3 — Introduce API blueprints and service adapters**
+  - [ ] Create domain-focused API blueprints and register_api entry point.
+  - [ ] Mirror route behaviour from the legacy server via service adapters.
+  - [ ] Add configuration toggles for enabling API groups incrementally.
+- [ ] **Stage 4 — Build unit and smoke tests for Engine APIs**
+  - [ ] Add pytest modules under Data/Engine/Unit_Tests exercising API blueprints.
+  - [ ] Provide fixtures that mirror the legacy SQLite schema and seed data.
+  - [ ] Assert HTTP status codes, payloads, and side effects for parity.
+  - [ ] Integrate Engine API tests into CI/local workflows.
+- [ ] **Stage 5 — Bridge the legacy server to Engine APIs**
+  - [ ] Delegate API blueprint registration to the Engine factory from the legacy server.
+  - [ ] Replace legacy API routes with Engine-provided blueprints gated by a flag.
+  - [ ] Emit transitional logging when Engine handles requests.
+- [ ] **Stage 6 — Plan WebUI migration**
+  - [ ] Move static/template handling into Data/Engine/services/WebUI.
+  - [ ] Preserve TLS-aware URL generation and caching.
+  - [ ] Add migration switch in the legacy server for WebUI delegation.
+  - [ ] Extend tests to cover critical WebUI routes.
+- [ ] **Stage 7 — Plan WebSocket migration**
+  - [ ] Extract Socket.IO handlers into Data/Engine/services/WebSocket.
+  - [ ] Provide register_realtime hook for the Engine factory.
+  - [ ] Add integration tests or smoke checks for key events.
+  - [ ] Update legacy server to consume Engine WebSocket registration.

Data/Engine/__init__.py

@@ -0,0 +1,11 @@
+"""Borealis Engine runtime package.
+
+This package houses the next-generation server runtime that will gradually
+replace :mod:`Data.Server.server`. Stage 1 focuses on providing a skeleton
+application factory and service placeholders so later stages can port
+features incrementally.
+"""
+
+from .server import create_app, EngineContext  # re-export for convenience
+
+__all__ = ["create_app", "EngineContext"]

Data/Engine/bootstrapper.py

@@ -0,0 +1,39 @@
+"""Command-line bootstrapper for the Stage 1 Engine runtime."""
+from __future__ import annotations
+
+import os
+from typing import Any, Dict
+
+from .server import create_app
+
+
+DEFAULT_HOST = "0.0.0.0"
+DEFAULT_PORT = 5001
+
+
+def _build_runtime_config() -> Dict[str, Any]:
+    return {
+        "HOST": os.environ.get("BOREALIS_ENGINE_HOST", DEFAULT_HOST),
+        "PORT": int(os.environ.get("BOREALIS_ENGINE_PORT", DEFAULT_PORT)),
+        "TLS_CERT_PATH": os.environ.get("BOREALIS_TLS_CERT"),
+        "TLS_KEY_PATH": os.environ.get("BOREALIS_TLS_KEY"),
+        "TLS_BUNDLE_PATH": os.environ.get("BOREALIS_TLS_BUNDLE"),
+    }
+
+
+def main() -> None:
+    config = _build_runtime_config()
+    app, socketio, context = create_app(config)
+
+    host = config.get("HOST", DEFAULT_HOST)
+    port = int(config.get("PORT", DEFAULT_PORT))
+
+    run_kwargs: Dict[str, Any] = {"host": host, "port": port}
+    if context.tls_bundle_path and context.tls_key_path:
+        run_kwargs.update({"certfile": context.tls_bundle_path, "keyfile": context.tls_key_path})
+
+    socketio.run(app, **run_kwargs)
+
+
+if __name__ == "__main__":  # pragma: no cover - manual launch helper
+    main()

Data/Engine/server.py

@@ -0,0 +1,240 @@
+"""Stage 1 Borealis Engine application factory.
+
+This module establishes the foundational structure for the Engine runtime so
+subsequent migration stages can progressively assume responsibility for the
+API, WebUI, and WebSocket layers.
+"""
+from __future__ import annotations
+
+import logging
+import os
+import ssl
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Mapping, MutableMapping, Optional, Tuple
+
+import eventlet
+from flask import Flask
+from flask_cors import CORS
+from flask_socketio import SocketIO
+from werkzeug.middleware.proxy_fix import ProxyFix
+
+# Eventlet ensures Socket.IO long-polling and WebSocket support parity with the
+# legacy server. We keep thread pools enabled for compatibility with blocking
+# filesystem/database operations.
+eventlet.monkey_patch(thread=False)
+
+try:  # pragma: no-cover - defensive import mirroring the legacy runtime.
+    from eventlet.wsgi import HttpProtocol  # type: ignore
+except Exception:  # pragma: no-cover - the Engine should still operate without it.
+    HttpProtocol = None  # type: ignore[assignment]
+else:
+    _original_handle_one_request = HttpProtocol.handle_one_request
+
+    def _quiet_tls_http_mismatch(self):  # type: ignore[override]
+        """Mirror the legacy suppression of noisy TLS handshake errors."""
+
+        def _close_connection_quietly():
+            try:
+                self.close_connection = True  # type: ignore[attr-defined]
+            except Exception:
+                pass
+            try:
+                conn = getattr(self, "socket", None) or getattr(self, "connection", None)
+                if conn:
+                    conn.close()
+            except Exception:
+                pass
+
+        try:
+            return _original_handle_one_request(self)
+        except ssl.SSLError as exc:  # type: ignore[arg-type]
+            reason = getattr(exc, "reason", "")
+            reason_text = str(reason).lower() if reason else ""
+            message = " ".join(str(arg) for arg in exc.args if arg).lower()
+            if (
+                "http_request" in message
+                or reason_text == "http request"
+                or "unknown ca" in message
+                or reason_text == "unknown ca"
+                or "unknown_ca" in message
+            ):
+                _close_connection_quietly()
+                return None
+            raise
+        except ssl.SSLEOFError:
+            _close_connection_quietly()
+            return None
+        except ConnectionAbortedError:
+            _close_connection_quietly()
+            return None
+
+    HttpProtocol.handle_one_request = _quiet_tls_http_mismatch  # type: ignore[assignment]
+
+
+# Ensure the legacy ``Modules`` package is importable when running from the
+# Engine deployment directory.
+_ENGINE_DIR = Path(__file__).resolve().parent
+_SEARCH_ROOTS = [
+    _ENGINE_DIR.parent / "Server",
+    _ENGINE_DIR.parent.parent / "Data" / "Server",
+]
+for root in _SEARCH_ROOTS:
+    modules_dir = root / "Modules"
+    if modules_dir.is_dir():
+        root_str = str(root)
+        if root_str not in sys.path:
+            sys.path.insert(0, root_str)
+
+try:  # pragma: no-cover - optional during Stage 1 scaffolding.
+    from Modules.crypto import certificates  # type: ignore
+except Exception:  # pragma: no-cover - Engine can start without certificate helpers.
+    certificates = None  # type: ignore[assignment]
+
+
+@dataclass
+class EngineContext:
+    """Shared handles that Engine services will consume."""
+
+    database_path: str
+    logger: logging.Logger
+    scheduler: Any
+    tls_cert_path: Optional[str]
+    tls_key_path: Optional[str]
+    tls_bundle_path: Optional[str]
+    config: Mapping[str, Any]
+
+
+__all__ = ["EngineContext", "create_app"]
+
+
+def _resolve_static_folder() -> str:
+    candidates = [
+        _ENGINE_DIR / "web-interface" / "build",
+        _ENGINE_DIR / "web-interface" / "dist",
+        _ENGINE_DIR / "web-interface",
+    ]
+    for candidate in candidates:
+        absolute = candidate.resolve()
+        if absolute.is_dir():
+            return str(absolute)
+    # Fall back to the first candidate to maintain parity with the legacy
+    # behaviour where the folder may not exist yet.
+    return str(candidates[0].resolve())
+
+
+def _initialise_logger(name: str = "borealis.engine") -> logging.Logger:
+    logger = logging.getLogger(name)
+    if not logger.handlers:
+        handler = logging.StreamHandler()
+        formatter = logging.Formatter("%(asctime)s-%(name)s-%(levelname)s: %(message)s")
+        handler.setFormatter(formatter)
+        logger.addHandler(handler)
+    logger.setLevel(logging.INFO)
+    logger.propagate = False
+    return logger
+
+
+def _discover_tls_material(config: Mapping[str, Any]) -> Tuple[Optional[str], Optional[str], Optional[str]]:
+    cert_path = str(config.get("TLS_CERT_PATH") or os.environ.get("BOREALIS_TLS_CERT") or "") or None
+    key_path = str(config.get("TLS_KEY_PATH") or os.environ.get("BOREALIS_TLS_KEY") or "") or None
+    bundle_path = str(config.get("TLS_BUNDLE_PATH") or os.environ.get("BOREALIS_TLS_BUNDLE") or "") or None
+
+    if certificates and not all([cert_path, key_path, bundle_path]):
+        try:
+            auto_cert, auto_key, auto_bundle = certificates.certificate_paths()
+        except Exception:
+            auto_cert = auto_key = auto_bundle = None
+        else:
+            cert_path = cert_path or auto_cert
+            key_path = key_path or auto_key
+            bundle_path = bundle_path or auto_bundle
+
+    if cert_path:
+        os.environ.setdefault("BOREALIS_TLS_CERT", cert_path)
+    if key_path:
+        os.environ.setdefault("BOREALIS_TLS_KEY", key_path)
+    if bundle_path:
+        os.environ.setdefault("BOREALIS_TLS_BUNDLE", bundle_path)
+    return cert_path, key_path, bundle_path
+
+
+def _coerce_config(source: Optional[Mapping[str, Any]]) -> MutableMapping[str, Any]:
+    if source is None:
+        return {}
+    return dict(source)
+
+
+def create_app(config: Optional[Mapping[str, Any]] = None) -> Tuple[Flask, SocketIO, EngineContext]:
+    """Create the Stage 1 Engine Flask application."""
+
+    runtime_config: MutableMapping[str, Any] = _coerce_config(config)
+    logger = _initialise_logger()
+
+    database_path = runtime_config.get("DATABASE_PATH") or os.path.abspath(
+        os.path.join(_ENGINE_DIR, "..", "..", "database.db")
+    )
+    os.makedirs(os.path.dirname(database_path), exist_ok=True)
+
+    static_folder = _resolve_static_folder()
+    app = Flask(__name__, static_folder=static_folder, static_url_path="")
+    app.wsgi_app = ProxyFix(app.wsgi_app, x_proto=1, x_host=1)
+
+    cors_origins = runtime_config.get("CORS_ORIGINS") or os.environ.get("BOREALIS_CORS_ORIGINS")
+    if cors_origins:
+        origins = [origin.strip() for origin in str(cors_origins).split(",") if origin.strip()]
+        CORS(app, supports_credentials=True, origins=origins)
+    else:
+        CORS(app, supports_credentials=True)
+
+    app.secret_key = runtime_config.get("SECRET_KEY") or os.environ.get("BOREALIS_SECRET", "borealis-dev-secret")
+
+    app.config.update(
+        SESSION_COOKIE_HTTPONLY=True,
+        SESSION_COOKIE_SAMESITE=runtime_config.get(
+            "SESSION_COOKIE_SAMESITE",
+            os.environ.get("BOREALIS_COOKIE_SAMESITE", "Lax"),
+        ),
+        SESSION_COOKIE_SECURE=bool(
+            str(runtime_config.get("SESSION_COOKIE_SECURE", os.environ.get("BOREALIS_COOKIE_SECURE", "0"))).lower()
+            in {"1", "true", "yes"}
+        ),
+    )
+    app.config.setdefault("PREFERRED_URL_SCHEME", "https")
+
+    cookie_domain = runtime_config.get("SESSION_COOKIE_DOMAIN") or os.environ.get("BOREALIS_COOKIE_DOMAIN")
+    if cookie_domain:
+        app.config["SESSION_COOKIE_DOMAIN"] = cookie_domain
+
+    socketio = SocketIO(
+        app,
+        cors_allowed_origins="*",
+        async_mode="eventlet",
+        engineio_options={
+            "max_http_buffer_size": 100_000_000,
+            "max_websocket_message_size": 100_000_000,
+        },
+    )
+
+    tls_cert_path, tls_key_path, tls_bundle_path = _discover_tls_material(runtime_config)
+
+    context = EngineContext(
+        database_path=database_path,
+        logger=logger,
+        scheduler=None,
+        tls_cert_path=tls_cert_path,
+        tls_key_path=tls_key_path,
+        tls_bundle_path=tls_bundle_path,
+        config=runtime_config,
+    )
+
+    from .services import API, WebSocket, WebUI  # Local import to avoid circular deps during bootstrap
+
+    API.register_api(app, context)
+    WebUI.register_web_ui(app, context)
+    WebSocket.register_realtime(socketio, context)
+
+    logger.debug("Engine application factory completed initialisation.")
+
+    return app, socketio, context

Data/Engine/services/API/__init__.py

@@ -0,0 +1,21 @@
+"""API service stubs for the Borealis Engine runtime.
+
+Stage 1 only establishes the package layout. Future stages will populate this
+module with blueprint factories that wrap the legacy API helpers.
+"""
+from __future__ import annotations
+
+from flask import Flask
+
+from ...server import EngineContext
+
+
+def register_api(app: Flask, context: EngineContext) -> None:
+    """Placeholder hook for API blueprint registration.
+
+    Later migration stages will import domain-specific blueprint modules and
+    attach them to ``app`` using the shared :class:`EngineContext`. For now we
+    simply log the intent so tooling can verify the hook is wired.
+    """
+
+    context.logger.debug("Engine API services are not yet implemented.")

Data/Engine/services/WebSocket/__init__.py

@@ -0,0 +1,17 @@
+"""WebSocket service stubs for the Borealis Engine runtime.
+
+Future stages will move Socket.IO namespaces and event handlers here. Stage 1
+only keeps a placeholder so the Engine bootstrapper can stub registration
+without touching legacy behaviour.
+"""
+from __future__ import annotations
+
+from flask_socketio import SocketIO
+
+from ...server import EngineContext
+
+
+def register_realtime(socket_server: SocketIO, context: EngineContext) -> None:
+    """Placeholder hook for Socket.IO namespace registration."""
+
+    context.logger.debug("Engine WebSocket services are not yet implemented.")

Data/Engine/services/WebUI/__init__.py

@@ -0,0 +1,17 @@
+"""WebUI service stubs for the Borealis Engine runtime.
+
+The future WebUI migration will centralise static asset serving, template
+rendering, and dev-server proxying here. Stage 1 keeps the placeholder so the
+application factory can stub out registration calls.
+"""
+from __future__ import annotations
+
+from flask import Flask
+
+from ...server import EngineContext
+
+
+def register_web_ui(app: Flask, context: EngineContext) -> None:
+    """Placeholder hook for WebUI route registration."""
+
+    context.logger.debug("Engine WebUI services are not yet implemented.")

Data/Engine/services/__init__.py

@@ -0,0 +1,5 @@
+"""Service registration hooks for the Borealis Engine runtime."""
+
+from . import API, WebSocket, WebUI
+
+__all__ = ["API", "WebSocket", "WebUI"]