From 8309a55e6c26dbfb2704d8dff3daf6ace7113ab2 Mon Sep 17 00:00:00 2001 From: Hongming Wang Date: Tue, 28 Apr 2026 12:02:33 -0700 Subject: [PATCH] feat(validator): runtime-load check for adapter.py contract (#17) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the third workstream of #90 (eliminate template repo drift): a strong contract check that exercises adapter.py the same way the runtime does at workspace boot. Without this, a template can have a syntactically-valid Dockerfile + an adapter.py that ImportErrors at runtime, build clean through Docker smoke, and crash on first user prompt — exactly the human-error class #90 is meant to eliminate. Existing checks ranked from weakest to strongest: 1. check_adapter() — text-grep for legacy `molecule_ai` imports. Catches one specific footgun. 2. Docker build smoke — `docker build` succeeds. Doesn't RUN the image, so adapter.py is never imported. Misses every adapter-load bug. 3. (NEW) check_adapter_runtime_load — imports adapter.py via the same `importlib.spec_from_file_location` path the runtime uses, and asserts at least one class inherits from molecule_runtime.adapters.base.BaseAdapter. Hard-error conditions: - adapter.py raises any exception during import (SyntaxError, ImportError, NameError, etc.). Same exception would crash the workspace at boot. - No class in the module inherits from BaseAdapter. The runtime's class-discovery silently falls through to the default langgraph executor in this case — exactly the silent-failure shape the contract is meant to catch. Skip conditions: - No adapter.py exists. Templates without one inherit the default executor by design (policy, not drift). - molecule-ai-workspace-runtime not importable in the validator env. Warns loudly so the CI-config bug surfaces, but doesn't hard-fail (we'd be reporting "your adapter is broken" when the actual cause is missing infra). Workflow update: validate-workspace-template.yml now installs the template's requirements.txt before invoking the validator (or falls back to installing molecule-ai-workspace-runtime alone if the template has no requirements.txt). This satisfies the runtime-load check's import dependencies the same way the workspace container does at boot — `pip install -r requirements.txt`. Verified locally: - 20/20 tests in test_validate_workspace_template.py pass (14 existing + 6 new). - Real langgraph template passes the full new validator including runtime-load (0 warnings, 0 errors). - Surveyed all 8 production templates' adapter.py shapes; every one already inherits from BaseAdapter, so this check turns green on first run with zero per-template fixups needed. Co-authored-by: Claude Opus 4.7 (1M context) --- .../workflows/validate-workspace-template.yml | 11 ++ .../scripts/validate-workspace-template.py | 102 +++++++++++++ scripts/test_validate_workspace_template.py | 143 ++++++++++++++++++ scripts/validate-workspace-template.py | 102 +++++++++++++ 4 files changed, 358 insertions(+) diff --git a/.github/workflows/validate-workspace-template.yml b/.github/workflows/validate-workspace-template.yml index b2f82f6..6175d07 100644 --- a/.github/workflows/validate-workspace-template.yml +++ b/.github/workflows/validate-workspace-template.yml @@ -26,6 +26,17 @@ jobs: cache: "pip" cache-dependency-path: .molecule-ci-canonical/.molecule-ci/scripts/requirements.txt - run: pip install pyyaml -q + # Install the template's runtime dependencies so the validator's + # `check_adapter_runtime_load()` can import adapter.py the same way + # the workspace container does at boot. Without this, a + # syntactically-valid adapter that ImportErrors on a missing + # transitive dep would build clean and crash on first user prompt. + # The fallback (no requirements.txt) installs the runtime alone so + # BaseAdapter is at least importable for the class-discovery check. + - if: hashFiles('requirements.txt') != '' + run: pip install -q -r requirements.txt + - if: hashFiles('requirements.txt') == '' + run: pip install -q molecule-ai-workspace-runtime - run: python3 .molecule-ci-canonical/scripts/validate-workspace-template.py - name: Docker build smoke test if: hashFiles('Dockerfile') != '' diff --git a/.molecule-ci/scripts/validate-workspace-template.py b/.molecule-ci/scripts/validate-workspace-template.py index 6db8cd1..9c0b236 100644 --- a/.molecule-ci/scripts/validate-workspace-template.py +++ b/.molecule-ci/scripts/validate-workspace-template.py @@ -172,6 +172,7 @@ def check_requirements() -> None: # ───────────────────────────────────────────────────────────── adapter.py def check_adapter() -> None: + """Static-text adapter checks. Fast — no imports.""" if not os.path.isfile("adapter.py"): warn("no adapter.py — runtime will use the default langgraph executor from the wheel") return @@ -186,11 +187,112 @@ def check_adapter() -> None: ) +def check_adapter_runtime_load() -> None: + """Strong adapter contract: import adapter.py the same way the runtime + does at workspace boot, and assert at least one class in it inherits + from molecule_runtime.adapters.base.BaseAdapter. + + The Docker build smoke test in validate-workspace-template.yml builds + the image but doesn't RUN it — adapter.py is only imported at + container startup. So a template with a syntactically-valid Dockerfile + + a broken adapter.py (wrong base class, ImportError on a missing + framework dep, typo) builds clean and fails on first user prompt. + This check exercises the same class-resolution path the runtime uses, + so a passing validator means a passing workspace boot for the + adapter-load step. + + Skip conditions: + - No adapter.py exists. Templates without one inherit the default + langgraph executor from the wheel (intentional, not drift). + - molecule-ai-workspace-runtime not importable in the validator + environment. That's a CI-config bug — the workflow that runs + this validator must `pip install molecule-ai-workspace-runtime` + first. Warn loudly so the misconfiguration surfaces, but don't + hard-fail (we'd be saying "your adapter is broken" when the + actual cause is missing infra). The `pip install -r + requirements.txt` step in validate-workspace-template.yml + normally satisfies this transitively. + + Hard-error conditions: + - adapter.py raises any exception during import. The same + exception would crash workspace boot. + - No class in the module inherits from BaseAdapter. The runtime's + adapter-discovery would silently fall through to the default + executor, ignoring this file — exactly the kind of human-error + mode this contract is supposed to eliminate. + """ + if not os.path.isfile("adapter.py"): + return # check_adapter() already warned; don't double-warn + + try: + from molecule_runtime.adapters.base import BaseAdapter # noqa: PLC0415 + except ImportError: + warn( + "adapter.py: skipping runtime-load check — " + "`molecule-ai-workspace-runtime` not installed in the validator " + "environment. The CI workflow that invokes this script must " + "`pip install molecule-ai-workspace-runtime` (or `pip install " + "-r requirements.txt`) first; otherwise this critical check is " + "silently bypassed." + ) + return + + # Load adapter.py as a module under a unique name so it doesn't + # collide with any installed `adapter` package or with a previous + # invocation in the same Python process. + import importlib.util # noqa: PLC0415 + import sys # noqa: PLC0415 + + module_name = "_template_adapter_under_validation" + spec = importlib.util.spec_from_file_location(module_name, "adapter.py") + if spec is None or spec.loader is None: + err("adapter.py: cannot construct an import spec — file may be unreadable") + return + + mod = importlib.util.module_from_spec(spec) + sys.modules[module_name] = mod # required so dataclass / pydantic refs resolve + + try: + spec.loader.exec_module(mod) + except Exception as e: + err( + f"adapter.py: failed to import — `{type(e).__name__}: {e}`. " + f"This is the same failure mode that crashes workspace boot at " + f"runtime; the cure is to fix the adapter, not skip this check. " + f"If the import fails because a transitive dep isn't installed in " + f"this CI env, add it to the template's requirements.txt — that's " + f"what the workspace container does, and the validator job " + f"installs requirements.txt before running this check." + ) + sys.modules.pop(module_name, None) + return + + adapter_classes = [ + obj + for name, obj in vars(mod).items() + if isinstance(obj, type) + and obj is not BaseAdapter + and issubclass(obj, BaseAdapter) + ] + sys.modules.pop(module_name, None) + + if not adapter_classes: + err( + "adapter.py: no class inheriting from " + "`molecule_runtime.adapters.base.BaseAdapter` found. " + "The runtime resolves the adapter via class discovery — " + "without a BaseAdapter subclass, workspace boot falls " + "through to the default langgraph executor and ignores " + "this file silently. If that's intentional, delete adapter.py." + ) + + def main() -> None: check_dockerfile() check_config_yaml() check_requirements() check_adapter() + check_adapter_runtime_load() for w in WARNINGS: print(f"::warning::{w}") diff --git a/scripts/test_validate_workspace_template.py b/scripts/test_validate_workspace_template.py index 610c302..e9e9844 100644 --- a/scripts/test_validate_workspace_template.py +++ b/scripts/test_validate_workspace_template.py @@ -273,3 +273,146 @@ def test_modern_molecule_runtime_import_does_not_warn(validator, tmp_path, monke validator.check_adapter() legacy_warnings = [w for w in validator.WARNINGS if "molecule_ai" in w] assert legacy_warnings == [], legacy_warnings + + +# ──────────────────── adapter.py runtime-load (strong contract) +# +# These tests pin the contract that adapter.py must be importable AND +# define at least one BaseAdapter subclass — the same path the runtime +# uses at workspace boot. Skipped when molecule-ai-workspace-runtime +# isn't installed in the test environment (the validator's CI workflow +# guarantees it via `pip install -r requirements.txt` before invoking +# the validator; local pytest can run with or without it). + +def _has_runtime_installed() -> bool: + """True if molecule-ai-workspace-runtime is importable. Used to skip + the runtime-load tests when running pytest locally without the + runtime in the venv.""" + try: + import molecule_runtime.adapters.base # noqa: F401, PLC0415 + return True + except ImportError: + return False + + +_RUNTIME_AVAILABLE = _has_runtime_installed() +_skip_no_runtime = pytest.mark.skipif( + not _RUNTIME_AVAILABLE, + reason="molecule-ai-workspace-runtime not installed in test env", +) + + +def test_no_adapter_skips_runtime_load_silently(validator, tmp_path, monkeypatch): + """No adapter.py = use default langgraph executor from the wheel. + That's policy, not drift, so runtime-load check should not fire.""" + monkeypatch.chdir(tmp_path) + validator.check_adapter_runtime_load() + # No ERRORS, no runtime-load WARNINGS specifically. + runtime_load_warnings = [ + w for w in validator.WARNINGS if "runtime-load check" in w + ] + assert validator.ERRORS == [], validator.ERRORS + assert runtime_load_warnings == [], runtime_load_warnings + + +@_skip_no_runtime +def test_valid_baseadapter_subclass_passes(validator, tmp_path, monkeypatch): + """The happy path: adapter.py defines a class inheriting from + BaseAdapter. All 8 production templates match this shape.""" + adapter = ( + "from molecule_runtime.adapters.base import BaseAdapter\n" + "\n" + "class MyAdapter(BaseAdapter):\n" + " @staticmethod\n" + " def name():\n" + " return 'test-adapter'\n" + ) + _materialise(tmp_path, adapter_py=adapter) + monkeypatch.chdir(tmp_path) + validator.check_adapter_runtime_load() + assert validator.ERRORS == [], validator.ERRORS + + +@_skip_no_runtime +def test_adapter_with_no_baseadapter_subclass_errors(validator, tmp_path, monkeypatch): + """The most insidious silent-failure mode: adapter.py imports + cleanly, defines classes, but NONE inherit from BaseAdapter. The + runtime's class-discovery would silently skip this file and fall + through to the default executor — workspace would 'work' but with + the wrong runtime. Must hard-error.""" + adapter = ( + "class JustSomePlainClass:\n" + " def run(self): pass\n" + ) + _materialise(tmp_path, adapter_py=adapter) + monkeypatch.chdir(tmp_path) + validator.check_adapter_runtime_load() + assert any( + "no class inheriting from" in e and "BaseAdapter" in e + for e in validator.ERRORS + ), validator.ERRORS + + +@_skip_no_runtime +def test_adapter_with_syntax_error_errors(validator, tmp_path, monkeypatch): + """SyntaxError at import is the same failure mode that crashes + workspace boot. Catch it here.""" + adapter = "this is not valid python at all\n" + _materialise(tmp_path, adapter_py=adapter) + monkeypatch.chdir(tmp_path) + validator.check_adapter_runtime_load() + assert any("failed to import" in e for e in validator.ERRORS), validator.ERRORS + + +@_skip_no_runtime +def test_adapter_with_import_error_errors(validator, tmp_path, monkeypatch): + """ImportError during adapter.py exec — same failure mode as + workspace boot. The error message should point the contributor at + requirements.txt as the right fix.""" + adapter = ( + "import this_package_definitely_does_not_exist_0xdeadbeef\n" + "from molecule_runtime.adapters.base import BaseAdapter\n" + ) + _materialise(tmp_path, adapter_py=adapter) + monkeypatch.chdir(tmp_path) + validator.check_adapter_runtime_load() + assert any( + "failed to import" in e and "ModuleNotFoundError" in e + for e in validator.ERRORS + ), validator.ERRORS + + +def test_runtime_not_installed_warns_not_errors(validator, tmp_path, monkeypatch): + """If the validator runs in an env without molecule-ai-workspace-runtime, + we WARN (loud) but don't error — hard-erroring would say 'your adapter + is broken' when the actual issue is the CI infra. Mock the import to + simulate this regardless of what's installed locally.""" + adapter = ( + "from molecule_runtime.adapters.base import BaseAdapter\n" + "class A(BaseAdapter): pass\n" + ) + _materialise(tmp_path, adapter_py=adapter) + monkeypatch.chdir(tmp_path) + + # Force the runtime import to fail by hiding the module. + import sys + saved = {k: sys.modules.pop(k) for k in list(sys.modules) + if k.startswith("molecule_runtime")} + saved_meta = sys.meta_path[:] + class _Block: + def find_spec(self, name, path=None, target=None): + if name == "molecule_runtime" or name.startswith("molecule_runtime."): + raise ImportError(f"blocked for test: {name}") + return None + sys.meta_path.insert(0, _Block()) + try: + validator.check_adapter_runtime_load() + finally: + sys.meta_path[:] = saved_meta + sys.modules.update(saved) + + assert validator.ERRORS == [], validator.ERRORS + assert any( + "skipping runtime-load check" in w + for w in validator.WARNINGS + ), validator.WARNINGS diff --git a/scripts/validate-workspace-template.py b/scripts/validate-workspace-template.py index 6db8cd1..9c0b236 100644 --- a/scripts/validate-workspace-template.py +++ b/scripts/validate-workspace-template.py @@ -172,6 +172,7 @@ def check_requirements() -> None: # ───────────────────────────────────────────────────────────── adapter.py def check_adapter() -> None: + """Static-text adapter checks. Fast — no imports.""" if not os.path.isfile("adapter.py"): warn("no adapter.py — runtime will use the default langgraph executor from the wheel") return @@ -186,11 +187,112 @@ def check_adapter() -> None: ) +def check_adapter_runtime_load() -> None: + """Strong adapter contract: import adapter.py the same way the runtime + does at workspace boot, and assert at least one class in it inherits + from molecule_runtime.adapters.base.BaseAdapter. + + The Docker build smoke test in validate-workspace-template.yml builds + the image but doesn't RUN it — adapter.py is only imported at + container startup. So a template with a syntactically-valid Dockerfile + + a broken adapter.py (wrong base class, ImportError on a missing + framework dep, typo) builds clean and fails on first user prompt. + This check exercises the same class-resolution path the runtime uses, + so a passing validator means a passing workspace boot for the + adapter-load step. + + Skip conditions: + - No adapter.py exists. Templates without one inherit the default + langgraph executor from the wheel (intentional, not drift). + - molecule-ai-workspace-runtime not importable in the validator + environment. That's a CI-config bug — the workflow that runs + this validator must `pip install molecule-ai-workspace-runtime` + first. Warn loudly so the misconfiguration surfaces, but don't + hard-fail (we'd be saying "your adapter is broken" when the + actual cause is missing infra). The `pip install -r + requirements.txt` step in validate-workspace-template.yml + normally satisfies this transitively. + + Hard-error conditions: + - adapter.py raises any exception during import. The same + exception would crash workspace boot. + - No class in the module inherits from BaseAdapter. The runtime's + adapter-discovery would silently fall through to the default + executor, ignoring this file — exactly the kind of human-error + mode this contract is supposed to eliminate. + """ + if not os.path.isfile("adapter.py"): + return # check_adapter() already warned; don't double-warn + + try: + from molecule_runtime.adapters.base import BaseAdapter # noqa: PLC0415 + except ImportError: + warn( + "adapter.py: skipping runtime-load check — " + "`molecule-ai-workspace-runtime` not installed in the validator " + "environment. The CI workflow that invokes this script must " + "`pip install molecule-ai-workspace-runtime` (or `pip install " + "-r requirements.txt`) first; otherwise this critical check is " + "silently bypassed." + ) + return + + # Load adapter.py as a module under a unique name so it doesn't + # collide with any installed `adapter` package or with a previous + # invocation in the same Python process. + import importlib.util # noqa: PLC0415 + import sys # noqa: PLC0415 + + module_name = "_template_adapter_under_validation" + spec = importlib.util.spec_from_file_location(module_name, "adapter.py") + if spec is None or spec.loader is None: + err("adapter.py: cannot construct an import spec — file may be unreadable") + return + + mod = importlib.util.module_from_spec(spec) + sys.modules[module_name] = mod # required so dataclass / pydantic refs resolve + + try: + spec.loader.exec_module(mod) + except Exception as e: + err( + f"adapter.py: failed to import — `{type(e).__name__}: {e}`. " + f"This is the same failure mode that crashes workspace boot at " + f"runtime; the cure is to fix the adapter, not skip this check. " + f"If the import fails because a transitive dep isn't installed in " + f"this CI env, add it to the template's requirements.txt — that's " + f"what the workspace container does, and the validator job " + f"installs requirements.txt before running this check." + ) + sys.modules.pop(module_name, None) + return + + adapter_classes = [ + obj + for name, obj in vars(mod).items() + if isinstance(obj, type) + and obj is not BaseAdapter + and issubclass(obj, BaseAdapter) + ] + sys.modules.pop(module_name, None) + + if not adapter_classes: + err( + "adapter.py: no class inheriting from " + "`molecule_runtime.adapters.base.BaseAdapter` found. " + "The runtime resolves the adapter via class discovery — " + "without a BaseAdapter subclass, workspace boot falls " + "through to the default langgraph executor and ignores " + "this file silently. If that's intentional, delete adapter.py." + ) + + def main() -> None: check_dockerfile() check_config_yaml() check_requirements() check_adapter() + check_adapter_runtime_load() for w in WARNINGS: print(f"::warning::{w}")