Assembly Management Rework - Stage 5 & 6 Complete (Stage 4 Pending)

Data/Engine/tests/assemblies/test_cache.py

@@ -0,0 +1,152 @@
+# ======================================================
+# Data\Engine\tests\assemblies\test_cache.py
+# Description: Validates AssemblyCache lifecycle behaviour, including dirty flags and background flushing.
+#
+# API Endpoints (if applicable): None
+# ======================================================
+
+from __future__ import annotations
+
+import base64
+import logging
+import threading
+import time
+from typing import Iterator, Tuple
+
+import pytest
+
+from Data.Engine.assembly_management.databases import AssemblyDatabaseManager
+from Data.Engine.assembly_management.models import AssemblyDomain
+from Data.Engine.assembly_management.payloads import PayloadManager
+from Data.Engine.assembly_management.bootstrap import AssemblyCache
+from Data.Engine.services.assemblies.service import AssemblyRuntimeService
+
+
+@pytest.fixture()
+def assembly_runtime(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Tuple[AssemblyRuntimeService, AssemblyCache, AssemblyDatabaseManager]]:
+    staging_root = tmp_path_factory.mktemp("assemblies_staging")
+    runtime_root = tmp_path_factory.mktemp("assemblies_runtime")
+
+    db_manager = AssemblyDatabaseManager(staging_root=staging_root, runtime_root=runtime_root, logger=logging.getLogger("test.assemblies.db"))
+    db_manager.initialise()
+    payload_manager = PayloadManager(
+        staging_root=staging_root / "Payloads",
+        runtime_root=runtime_root / "Payloads",
+        logger=logging.getLogger("test.assemblies.payload"),
+    )
+    cache = AssemblyCache(
+        database_manager=db_manager,
+        payload_manager=payload_manager,
+        flush_interval_seconds=5.0,
+        logger=logging.getLogger("test.assemblies.cache"),
+    )
+    service = AssemblyRuntimeService(cache, logger=logging.getLogger("test.assemblies.runtime"))
+    try:
+        yield service, cache, db_manager
+    finally:
+        cache.shutdown(flush=True)
+
+
+def _script_payload(display_name: str = "Cache Test Script") -> dict:
+    body = 'Write-Host "Hello from cache tests"'
+    encoded = base64.b64encode(body.encode("utf-8")).decode("ascii")
+    return {
+        "domain": "user",
+        "assembly_guid": None,
+        "assembly_kind": "script",
+        "display_name": display_name,
+        "summary": "Cache test fixture payload.",
+        "category": "script",
+        "assembly_type": "powershell",
+        "version": 1,
+        "metadata": {
+            "sites": {"mode": "all", "values": []},
+            "variables": [],
+            "files": [],
+            "timeout_seconds": 120,
+            "script_encoding": "base64",
+        },
+        "tags": {},
+        "payload": {
+            "version": 1,
+            "name": display_name,
+            "description": "Cache test fixture payload.",
+            "category": "script",
+            "type": "powershell",
+            "script": encoded,
+            "timeout_seconds": 120,
+            "sites": {"mode": "all", "values": []},
+            "variables": [],
+            "files": [],
+            "script_encoding": "base64",
+        },
+    }
+
+
+def test_cache_flush_marks_entries_clean(assembly_runtime) -> None:
+    service, cache, db_manager = assembly_runtime
+    record = service.create_assembly(_script_payload())
+    guid = record["assembly_guid"]
+
+    snapshot = {entry["assembly_guid"]: entry for entry in cache.describe()}
+    assert snapshot[guid]["is_dirty"] == "true"
+
+    cache.flush_now()
+    cache.reload()
+    snapshot = {entry["assembly_guid"]: entry for entry in cache.describe()}
+    assert snapshot[guid]["is_dirty"] == "false"
+    persisted = db_manager.load_all(AssemblyDomain.USER)
+    assert any(item.assembly_guid == guid for item in persisted)
+
+
+def test_cache_worker_flushes_on_event(assembly_runtime) -> None:
+    service, cache, _db_manager = assembly_runtime
+    record = service.create_assembly(_script_payload(display_name="Worker Flush Script"))
+    guid = record["assembly_guid"]
+
+    cache._flush_event.set()  # Trigger worker loop without waiting for full interval.
+    time.sleep(0.2)
+    cache.reload()
+    snapshot = {entry["assembly_guid"]: entry for entry in cache.describe()}
+    assert snapshot[guid]["is_dirty"] == "false"
+
+
+def test_cache_flush_waits_for_locked_database(assembly_runtime) -> None:
+    service, cache, db_manager = assembly_runtime
+    if cache._worker.is_alive():  # type: ignore[attr-defined]
+        cache._stop_event.set()  # type: ignore[attr-defined]
+        cache._flush_event.set()  # type: ignore[attr-defined]
+        cache._worker.join(timeout=1.0)  # type: ignore[attr-defined]
+        cache._stop_event.clear()  # type: ignore[attr-defined]
+    record = service.create_assembly(_script_payload(display_name="Concurrency Script"))
+    guid = record["assembly_guid"]
+    cache.flush_now()
+
+    update_payload = _script_payload(display_name="Concurrency Script")
+    update_payload["assembly_guid"] = guid
+    update_payload["summary"] = "Updated summary after lock."
+    service.update_assembly(guid, update_payload)
+
+    conn = db_manager._open_connection(AssemblyDomain.USER)  # type: ignore[attr-defined]
+    cur = conn.cursor()
+    cur.execute("BEGIN IMMEDIATE")
+
+    flush_thread = threading.Thread(target=cache.flush_now)
+    flush_thread.start()
+
+    time.sleep(0.2)
+    snapshot = {entry["assembly_guid"]: entry for entry in cache.describe()}
+    assert snapshot[guid]["is_dirty"] == "true"
+    conn.rollback()
+    conn.close()
+
+    flush_thread.join(timeout=5.0)
+    assert not flush_thread.is_alive(), "Initial flush attempt did not return after releasing database lock."
+    cache.flush_now()
+    cache.reload()
+    snapshot = {entry["assembly_guid"]: entry for entry in cache.describe()}
+    assert snapshot[guid]["is_dirty"] == "false"
+
+    records = db_manager.load_all(AssemblyDomain.USER)
+    summaries = {entry.assembly_guid: entry.summary for entry in records}
+    assert summaries[guid] == "Updated summary after lock."

Data/Engine/tests/assemblies/test_import_export.py

@@ -0,0 +1,99 @@
+# ======================================================
+# Data\Engine\tests\assemblies\test_import_export.py
+# Description: Ensures assembly import/export endpoints round-trip legacy JSON documents.
+#
+# API Endpoints (if applicable): None
+# ======================================================
+
+from __future__ import annotations
+
+import base64
+
+from flask.testing import FlaskClient
+
+from Data.Engine.Unit_Tests.conftest import EngineTestHarness
+
+
+def _user_client(harness: EngineTestHarness) -> FlaskClient:
+    client = harness.app.test_client()
+    with client.session_transaction() as sess:
+        sess["username"] = "importer"
+        sess["role"] = "User"
+    return client
+
+
+def _script_document(name: str = "Import Script") -> dict:
+    payload = 'Write-Host "round trip export"'
+    encoded = base64.b64encode(payload.encode("utf-8")).decode("ascii")
+    return {
+        "version": 2,
+        "name": name,
+        "description": "Import/export test script.",
+        "category": "script",
+        "type": "powershell",
+        "script": encoded,
+        "timeout_seconds": 45,
+        "sites": {"mode": "all", "values": []},
+        "variables": [{"name": "example", "label": "Example", "type": "string", "default": ""}],
+        "files": [],
+        "script_encoding": "base64",
+    }
+
+
+def _workflow_document(name: str = "Import Workflow") -> dict:
+    return {
+        "tab_name": name,
+        "description": "Import/export workflow test.",
+        "nodes": [
+            {
+                "id": "node-1",
+                "type": "DataNode",
+                "position": {"x": 10, "y": 20},
+                "data": {"label": "Input", "value": "example"},
+            }
+        ],
+        "edges": [],
+    }
+
+
+def test_script_import_export_round_trip(engine_harness: EngineTestHarness) -> None:
+    client = _user_client(engine_harness)
+    document = _script_document()
+
+    import_response = client.post(
+        "/api/assemblies/import",
+        json={"domain": "user", "document": document},
+    )
+    assert import_response.status_code == 201
+    imported = import_response.get_json()
+    assembly_guid = imported["assembly_guid"]
+    assert imported["payload_json"]["script"] == document["script"]
+
+    export_response = client.get(f"/api/assemblies/{assembly_guid}/export")
+    assert export_response.status_code == 200
+    exported = export_response.get_json()
+    assert exported["assembly_guid"] == assembly_guid
+    assert exported["payload"]["script"] == document["script"]
+    assert exported["metadata"]["display_name"] == document["name"]
+    assert exported["payload"]["variables"][0]["name"] == "example"
+    assert isinstance(exported["queue"], list)
+
+
+def test_workflow_import_export_round_trip(engine_harness: EngineTestHarness) -> None:
+    client = _user_client(engine_harness)
+    document = _workflow_document()
+
+    response = client.post(
+        "/api/assemblies/import",
+        json={"domain": "user", "document": document},
+    )
+    assert response.status_code == 201
+    payload = response.get_json()
+    assembly_guid = payload["assembly_guid"]
+    assert payload["payload_json"]["nodes"][0]["id"] == "node-1"
+
+    export_response = client.get(f"/api/assemblies/{assembly_guid}/export")
+    assert export_response.status_code == 200
+    exported = export_response.get_json()
+    assert exported["payload"]["nodes"][0]["id"] == "node-1"
+    assert exported["metadata"]["display_name"] == document["tab_name"]

Data/Engine/tests/assemblies/test_payloads.py

@@ -0,0 +1,37 @@
+# ======================================================
+# Data\Engine\tests\assemblies\test_payloads.py
+# Description: Exercises PayloadManager storage, mirroring, and deletion behaviours.
+#
+# API Endpoints (if applicable): None
+# ======================================================
+
+from __future__ import annotations
+
+import base64
+from pathlib import Path
+
+from Data.Engine.assembly_management.models import PayloadType
+from Data.Engine.assembly_management.payloads import PayloadManager
+
+
+def test_payload_manager_store_update_delete(tmp_path: Path) -> None:
+    staging_root = tmp_path / "staging"
+    runtime_root = tmp_path / "runtime"
+    manager = PayloadManager(staging_root=staging_root, runtime_root=runtime_root)
+
+    content = base64.b64encode(b"payload-bytes").decode("ascii")
+    descriptor = manager.store_payload(PayloadType.SCRIPT, content, assembly_guid="abc123", extension=".json")
+
+    staging_path = staging_root / "abc123" / descriptor.file_name
+    runtime_path = runtime_root / "abc123" / descriptor.file_name
+    assert staging_path.is_file()
+    assert runtime_path.is_file()
+    assert descriptor.size_bytes == len(content)
+
+    updated = manager.update_payload(descriptor, content + "-v2")
+    assert updated.size_bytes == len(content + "-v2")
+    assert staging_path.read_text(encoding="utf-8").endswith("-v2")
+
+    manager.delete_payload(descriptor)
+    assert not staging_path.exists()
+    assert not runtime_path.exists()

Data/Engine/tests/assemblies/test_permissions.py

@@ -0,0 +1,122 @@
+# ======================================================
+# Data\Engine\tests\assemblies\test_permissions.py
+# Description: Verifies Assembly API domain guards and Dev Mode permissions.
+#
+# API Endpoints (if applicable): None
+# ======================================================
+
+from __future__ import annotations
+
+import base64
+
+from flask.testing import FlaskClient
+
+from Data.Engine.assembly_management.models import AssemblyDomain
+
+from Data.Engine.Unit_Tests.conftest import EngineTestHarness
+
+
+def _script_document(name: str = "Permission Script") -> dict:
+    script = 'Write-Host "permissions"'
+    encoded = base64.b64encode(script.encode("utf-8")).decode("ascii")
+    return {
+        "version": 1,
+        "name": name,
+        "description": "Permission test script.",
+        "category": "script",
+        "type": "powershell",
+        "script": encoded,
+        "timeout_seconds": 60,
+        "sites": {"mode": "all", "values": []},
+        "variables": [],
+        "files": [],
+        "script_encoding": "base64",
+    }
+
+
+def _user_client(harness: EngineTestHarness) -> FlaskClient:
+    client = harness.app.test_client()
+    with client.session_transaction() as sess:
+        sess["username"] = "operator"
+        sess["role"] = "User"
+    return client
+
+
+def _admin_client(harness: EngineTestHarness) -> FlaskClient:
+    client = harness.app.test_client()
+    with client.session_transaction() as sess:
+        sess["username"] = "admin"
+        sess["role"] = "Admin"
+    return client
+
+
+def test_non_admin_cannot_write_official_domain(engine_harness: EngineTestHarness) -> None:
+    client = _user_client(engine_harness)
+    response = client.post(
+        "/api/assemblies",
+        json={
+            "domain": AssemblyDomain.OFFICIAL.value,
+            "assembly_kind": "script",
+            "display_name": "User Attempt",
+            "summary": "Should fail",
+            "category": "script",
+            "assembly_type": "powershell",
+            "version": 1,
+            "metadata": {},
+            "payload": _script_document("User Attempt"),
+        },
+    )
+    assert response.status_code == 403
+    payload = response.get_json()
+    assert payload["error"] == "forbidden"
+
+
+def test_admin_requires_dev_mode_for_official_mutation(engine_harness: EngineTestHarness) -> None:
+    client = _admin_client(engine_harness)
+    response = client.post(
+        "/api/assemblies",
+        json={
+            "domain": AssemblyDomain.OFFICIAL.value,
+            "assembly_kind": "script",
+            "display_name": "Dev Mode Required",
+            "summary": "Should request dev mode",
+            "category": "script",
+            "assembly_type": "powershell",
+            "version": 1,
+            "metadata": {},
+            "payload": _script_document("Dev Mode Required"),
+        },
+    )
+    assert response.status_code == 403
+    payload = response.get_json()
+    assert payload["error"] == "dev_mode_required"
+
+
+def test_admin_with_dev_mode_can_mutate_official(engine_harness: EngineTestHarness) -> None:
+    client = _admin_client(engine_harness)
+    response = client.post("/api/assemblies/dev-mode/switch", json={"enabled": True})
+    assert response.status_code == 200
+    assert response.get_json()["dev_mode"] is True
+
+    create_response = client.post(
+        "/api/assemblies",
+        json={
+            "domain": AssemblyDomain.OFFICIAL.value,
+            "assembly_kind": "script",
+            "display_name": "Official Dev Mode Script",
+            "summary": "Created while Dev Mode enabled",
+            "category": "script",
+            "assembly_type": "powershell",
+            "version": 1,
+            "metadata": {},
+            "payload": _script_document("Official Dev Mode Script"),
+        },
+    )
+    assert create_response.status_code == 201
+    record = create_response.get_json()
+    assert record["source"] == AssemblyDomain.OFFICIAL.value
+    assert record["is_dirty"] is True
+
+    flush_response = client.post("/api/assemblies/dev-mode/write")
+    assert flush_response.status_code == 200
+    assert flush_response.get_json()["status"] == "flushed"

Data/Engine/tests/conftest.py

@@ -0,0 +1,11 @@
+# ======================================================
+# Data\Engine\tests\conftest.py
+# Description: Re-exports shared Engine test fixtures for assembly-specific test suites.
+#
+# API Endpoints (if applicable): None
+# ======================================================
+
+from __future__ import annotations
+
+from Data.Engine.Unit_Tests.conftest import engine_harness  # noqa: F401
+