Misc Changes

.gitignore

@@ -23,4 +23,16 @@ database.db
 # Misc Files/Folders
 .vs/s
 __pycache__
-/Update_Staging/
+/Update_Staging/
+
+# Assembly Databases
+/Data/Engine/Assemblies/community.db
+/Data/Engine/Assemblies/community.db-shm
+/Data/Engine/Assemblies/community.db-wal
+/Data/Engine/Assemblies/official.db
+/Data/Engine/Assemblies/official.db-shm
+/Data/Engine/Assemblies/official.db-wal
+/Data/Engine/Assemblies/user_created.db
+/Data/Engine/Assemblies/user_created.db-shm
+/Data/Engine/Assemblies/user_created.db-wal
+/Data/Engine/Assemblies/Payloads/

Data/Engine/Assemblies/DB_MIGRATION_TRACKER.md

@@ -48,10 +48,10 @@
 - `initialise_assembly_runtime()` is invoked from both `create_app` and `register_engine_api`, wiring the cache onto `EngineContext` and ensuring graceful shutdown flushing.
 
 ## 2. Update Engine services and APIs for multi-domain assemblies
-[ ] Refactor existing assembly REST endpoints to read from the cache instead of filesystem JSON.
+[x] Refactor existing assembly REST endpoints to read from the cache instead of filesystem JSON.
-[ ] Add source metadata (`official`, `community`, `user`) to API responses.
+[x] Add source metadata (`official`, `community`, `user`) to API responses.
-[ ] Introduce administrative endpoints to support Dev Mode overrides: clone between domains, force writes to read-only DBs, bulk sync official DB from staging.
+[x] Introduce administrative endpoints to support Dev Mode overrides: clone between domains, force writes to read-only DBs, bulk sync official DB from staging.
-[ ] Provide queue status (dirty vs. persisted) in list/detail responses for UI pill rendering.
+[x] Provide queue status (dirty vs. persisted) in list/detail responses for UI pill rendering.
 ### Details
 ```
 1. Inspect `Data/Engine/services/assemblies/` (create if absent) and replace filesystem access with calls into `AssemblyCache`.
@@ -71,8 +71,10 @@
 6. Adjust error handling to surface concurrency/locking issues with retries and user-friendly messages.
 ```
 
-**Stage Notes (in progress)**
+**Stage Notes**
-- Refactoring work is underway to move REST endpoints onto `AssemblyCache` while we align on assembly GUID usage and domain permissions. Pending operator testing before tasks can be completed.
+- Assembly REST stack now proxies through `AssemblyRuntimeService`, sourcing list/detail data from `AssemblyCache` with GUID-based payload resolution (`Data/Engine/services/assemblies/service.py`, `Data/Engine/services/API/assemblies/management.py`).
+- Responses include `source`, `is_dirty`, and `payload_guid`, and expose the queue snapshot for dirty-pill rendering; domain guards enforce user vs Admin+Dev Mode write access.
+- Added Dev Mode toggle/flush endpoints plus official-domain sync, all wired to the cache write queue and importer; verified via operator testing of the API list/update/clone paths.
 
 ## 3. Implement Dev Mode authorization and UX toggles
 [ ] Gate privileged writes behind Admin role + Dev Mode toggle.

Data/Engine/assembly_management/databases.py

@@ -308,6 +308,7 @@ class AssemblyDatabaseManager:
 
     def _apply_schema(self, conn: sqlite3.Connection) -> None:
         cur = conn.cursor()
+        self._migrate_legacy_schema(cur)
         for statement in _SCHEMA_STATEMENTS:
             cur.execute(statement)
         conn.commit()
@@ -331,3 +332,147 @@ class AssemblyDatabaseManager:
                         runtime_candidate,
                         exc,
                     )
+    
+    def _migrate_legacy_schema(self, cur: sqlite3.Cursor) -> None:
+        """Upgrade legacy assembly/payload tables to the consolidated schema."""
+
+        cur.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='assemblies'")
+        if not cur.fetchone():
+            return
+
+        cur.execute("PRAGMA table_info('assemblies')")
+        legacy_columns = {row[1] for row in cur.fetchall()}
+        if "assembly_guid" in legacy_columns:
+            return  # Already migrated
+
+        self._logger.info("Migrating legacy assemblies schema to assembly_guid layout.")
+        cur.execute(
+            """
+            CREATE TABLE IF NOT EXISTS assemblies_new (
+                assembly_guid TEXT PRIMARY KEY,
+                display_name TEXT NOT NULL,
+                summary TEXT,
+                category TEXT,
+                assembly_kind TEXT NOT NULL,
+                assembly_type TEXT,
+                version INTEGER NOT NULL DEFAULT 1,
+                metadata_json TEXT,
+                tags_json TEXT,
+                checksum TEXT,
+                payload_type TEXT NOT NULL,
+                payload_file_name TEXT NOT NULL,
+                payload_file_extension TEXT NOT NULL,
+                payload_size_bytes INTEGER NOT NULL DEFAULT 0,
+                payload_checksum TEXT,
+                payload_created_at TEXT NOT NULL,
+                payload_updated_at TEXT NOT NULL,
+                created_at TEXT NOT NULL,
+                updated_at TEXT NOT NULL
+            )
+            """
+        )
+
+        cur.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='payloads'")
+        has_payloads = cur.fetchone() is not None
+
+        if has_payloads:
+            cur.execute(
+                """
+                INSERT INTO assemblies_new (
+                    assembly_guid,
+                    display_name,
+                    summary,
+                    category,
+                    assembly_kind,
+                    assembly_type,
+                    version,
+                    metadata_json,
+                    tags_json,
+                    checksum,
+                    payload_type,
+                    payload_file_name,
+                    payload_file_extension,
+                    payload_size_bytes,
+                    payload_checksum,
+                    payload_created_at,
+                    payload_updated_at,
+                    created_at,
+                    updated_at
+                )
+                SELECT
+                    a.assembly_id AS assembly_guid,
+                    a.display_name,
+                    a.summary,
+                    a.category,
+                    a.assembly_kind,
+                    a.assembly_type,
+                    a.version,
+                    COALESCE(a.metadata_json, '{}') AS metadata_json,
+                    COALESCE(a.tags_json, '{}') AS tags_json,
+                    a.checksum,
+                    COALESCE(p.payload_type, 'unknown') AS payload_type,
+                    COALESCE(p.file_name, 'payload.json') AS payload_file_name,
+                    COALESCE(p.file_extension, '.json') AS payload_file_extension,
+                    COALESCE(p.size_bytes, 0) AS payload_size_bytes,
+                    COALESCE(p.checksum, '') AS payload_checksum,
+                    COALESCE(p.created_at, a.created_at) AS payload_created_at,
+                    COALESCE(p.updated_at, a.updated_at) AS payload_updated_at,
+                    a.created_at,
+                    a.updated_at
+                FROM assemblies AS a
+                LEFT JOIN payloads AS p ON p.payload_guid = a.payload_guid
+                """
+            )
+        else:
+            cur.execute(
+                """
+                INSERT INTO assemblies_new (
+                    assembly_guid,
+                    display_name,
+                    summary,
+                    category,
+                    assembly_kind,
+                    assembly_type,
+                    version,
+                    metadata_json,
+                    tags_json,
+                    checksum,
+                    payload_type,
+                    payload_file_name,
+                    payload_file_extension,
+                    payload_size_bytes,
+                    payload_checksum,
+                    payload_created_at,
+                    payload_updated_at,
+                    created_at,
+                    updated_at
+                )
+                SELECT
+                    assembly_id AS assembly_guid,
+                    display_name,
+                    summary,
+                    category,
+                    assembly_kind,
+                    assembly_type,
+                    version,
+                    COALESCE(metadata_json, '{}'),
+                    COALESCE(tags_json, '{}'),
+                    checksum,
+                    'unknown' AS payload_type,
+                    'payload.json' AS payload_file_name,
+                    '.json' AS payload_file_extension,
+                    0 AS payload_size_bytes,
+                    '' AS payload_checksum,
+                    created_at AS payload_created_at,
+                    updated_at AS payload_updated_at,
+                    created_at,
+                    updated_at
+                FROM assemblies
+                """
+            )
+
+        cur.execute("DROP TABLE assemblies")
+        if has_payloads:
+            cur.execute("DROP TABLE payloads")
+        cur.execute("ALTER TABLE assemblies_new RENAME TO assemblies")
+        self._logger.info("Legacy assemblies schema migration completed.")

Data/Engine/assembly_management/sync.py

@@ -110,21 +110,21 @@ def _record_from_file(
         return None
 
     payload_type = _payload_type_for_kind(kind)
-    guid = hashlib.sha1(rel_path.encode("utf-8")).hexdigest()
+    assembly_guid = hashlib.sha1(rel_path.encode("utf-8")).hexdigest()
-    descriptor = payload_manager.store_payload(payload_type, text, assembly_guid=guid, extension=".json")
+    descriptor = payload_manager.store_payload(payload_type, text, assembly_guid=assembly_guid, extension=".json")
 
     file_stat = file_path.stat()
     timestamp = _dt.datetime.utcfromtimestamp(file_stat.st_mtime).replace(microsecond=0)
     descriptor.created_at = timestamp
     descriptor.updated_at = timestamp
 
-    assembly_id = _assembly_id_from_path(rel_path)
+    assembly_path = _assembly_id_from_path(rel_path)
     document_metadata = _metadata_from_document(kind, document, rel_path)
     tags = _coerce_dict(document.get("tags"))
 
     record = AssemblyRecord(
-        assembly_id=assembly_id,
+        assembly_guid=assembly_guid,
-        display_name=document_metadata.get("display_name") or assembly_id.rsplit("/", 1)[-1],
+        display_name=document_metadata.get("display_name") or assembly_path.rsplit("/", 1)[-1],
         summary=document_metadata.get("summary"),
         category=document_metadata.get("category"),
         assembly_kind=kind,

Data/Engine/services/API/assemblies/management.py

@@ -25,8 +25,8 @@ from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple
 from flask import Blueprint, jsonify, request, session
 from itsdangerous import BadSignature, SignatureExpired, URLSafeTimedSerializer
 
-from Data.Engine.assembly_management.models import AssemblyDomain
+from ....assembly_management.models import AssemblyDomain
-from ..assemblies.service import AssemblyRuntimeService
+from ...assemblies.service import AssemblyRuntimeService
 
 if TYPE_CHECKING:  # pragma: no cover - typing aide
     from .. import EngineServiceAdapters

Data/Engine/services/assemblies/service.py

@@ -17,9 +17,9 @@ import logging
 import uuid
 from typing import Any, Dict, List, Mapping, Optional
 
-from Data.Engine.assembly_management.bootstrap import AssemblyCache
+from ...assembly_management.bootstrap import AssemblyCache
-from Data.Engine.assembly_management.models import AssemblyDomain, AssemblyRecord, CachedAssembly, PayloadType
+from ...assembly_management.models import AssemblyDomain, AssemblyRecord, CachedAssembly, PayloadType
-from Data.Engine.assembly_management.sync import sync_official_domain
+from ...assembly_management.sync import sync_official_domain
 
 
 class AssemblyRuntimeService:
@@ -329,4 +329,3 @@ def _coerce_int(value: Any, *, default: int = 0) -> int:
 
 def _utcnow() -> _dt.datetime:
     return _dt.datetime.utcnow().replace(microsecond=0)
-