From 6910d651c28c2ab81e327b2c4b4af19422ff2e69 Mon Sep 17 00:00:00 2001 From: John Pangas Date: Fri, 24 Jul 2026 17:47:27 -0600 Subject: [PATCH] Add test_rails action --- .../test_plan_generator/__main__.py | 7 +- .../hackbot_runtime/actions/__init__.py | 10 +- .../actions/handlers/registry.py | 2 + .../actions/handlers/testrail_handler.py | 193 ++++++++++++ .../hackbot_runtime/actions/testrail.py | 28 ++ .../tests/test_testrail_action.py | 22 ++ .../tests/test_testrail_handler.py | 279 ++++++++++++++++++ services/hackbot-api/app/agents.py | 1 + services/hackbot-api/tests/test_agents.py | 1 + 9 files changed, 540 insertions(+), 3 deletions(-) create mode 100644 libs/hackbot-runtime/hackbot_runtime/actions/handlers/testrail_handler.py create mode 100644 libs/hackbot-runtime/hackbot_runtime/actions/testrail.py create mode 100644 libs/hackbot-runtime/tests/test_testrail_action.py create mode 100644 libs/hackbot-runtime/tests/test_testrail_handler.py diff --git a/agents/test-plan-generator/hackbot_agents/test_plan_generator/__main__.py b/agents/test-plan-generator/hackbot_agents/test_plan_generator/__main__.py index af67eac60b..88494ae6da 100644 --- a/agents/test-plan-generator/hackbot_agents/test_plan_generator/__main__.py +++ b/agents/test-plan-generator/hackbot_agents/test_plan_generator/__main__.py @@ -1,4 +1,5 @@ from hackbot_runtime import HackbotContext, run_async +from hackbot_runtime.actions.testrail import record_test_plan from pydantic_settings import BaseSettings, SettingsConfigDict from .agent import TestPlanGeneratorResult, run_test_plan_generator @@ -21,7 +22,7 @@ async def main(ctx: HackbotContext) -> TestPlanGeneratorResult: firefox_path = str(install_firefox_nightly()) - return await run_test_plan_generator( + result = await run_test_plan_generator( feature_name=inputs.feature_name, feature_description=inputs.feature_description, test_scope=inputs.test_scope, @@ -32,6 +33,10 @@ async def main(ctx: HackbotContext) -> TestPlanGeneratorResult: log=ctx.log_path, verbose=True, ) + if result.result is None: + raise RuntimeError("Cannot submit an empty test plan to TestRail") + record_test_plan(ctx.actions, result.result.model_dump(mode="json")) + return result if __name__ == "__main__": diff --git a/libs/hackbot-runtime/hackbot_runtime/actions/__init__.py b/libs/hackbot-runtime/hackbot_runtime/actions/__init__.py index e38385139b..a620c7b37c 100644 --- a/libs/hackbot-runtime/hackbot_runtime/actions/__init__.py +++ b/libs/hackbot-runtime/hackbot_runtime/actions/__init__.py @@ -7,9 +7,15 @@ claude-sdk adapter is ``hackbot_runtime.actions.claude_sdk.actions_server_for``. """ -from hackbot_runtime.actions import bugzilla, phabricator +from hackbot_runtime.actions import bugzilla, phabricator, testrail from hackbot_runtime.actions.recorder import ActionsRecorder ACTIONS_SERVER_NAME = "actions" -__all__ = ["ACTIONS_SERVER_NAME", "ActionsRecorder", "bugzilla", "phabricator"] +__all__ = [ + "ACTIONS_SERVER_NAME", + "ActionsRecorder", + "bugzilla", + "phabricator", + "testrail", +] diff --git a/libs/hackbot-runtime/hackbot_runtime/actions/handlers/registry.py b/libs/hackbot-runtime/hackbot_runtime/actions/handlers/registry.py index d9835e9cf3..e41ed4325e 100644 --- a/libs/hackbot-runtime/hackbot_runtime/actions/handlers/registry.py +++ b/libs/hackbot-runtime/hackbot_runtime/actions/handlers/registry.py @@ -6,6 +6,7 @@ UpdateBugHandler, ) from hackbot_runtime.actions.handlers.phabricator_handler import SubmitPatchHandler +from hackbot_runtime.actions.handlers.testrail_handler import SubmitTestCasesHandler # Maps a recorded action's dotted `type` to the handler that applies it. # Adding a new action type later is a one-line addition here — the dispatch @@ -16,6 +17,7 @@ "bugzilla.add_attachment": AddAttachmentHandler(), "bugzilla.create_bug": CreateBugHandler(), "phabricator.submit_patch": SubmitPatchHandler(), + "testrail.submit_test_cases": SubmitTestCasesHandler(), } diff --git a/libs/hackbot-runtime/hackbot_runtime/actions/handlers/testrail_handler.py b/libs/hackbot-runtime/hackbot_runtime/actions/handlers/testrail_handler.py new file mode 100644 index 0000000000..6469c4b051 --- /dev/null +++ b/libs/hackbot-runtime/hackbot_runtime/actions/handlers/testrail_handler.py @@ -0,0 +1,193 @@ +"""Apply-side TestRail action for generated test cases.""" + +from __future__ import annotations + +import logging +import os +from typing import Any + +import requests + +from hackbot_runtime.actions.handlers.base import ActionResult, ApplyContext + +log = logging.getLogger(__name__) + +_DEFAULT_TESTRAIL_URL = "https://mozilla.testrail.io" +_CASE_TYPE_NAME = "Functional" +_CASE_TEMPLATE_NAME = "Test Case (Steps)" +_CASE_LABEL = "AI Generated" +_SECTION_NAME = "Test Cases" +_TIMEOUT_SECONDS = 30 + + +def _required_env(name: str) -> str: + value = os.environ.get(name, "").strip() + if not value: + raise RuntimeError(f"{name} is not configured") + return value + + +def _base_url() -> str: + return os.environ.get("TESTRAIL_URL", _DEFAULT_TESTRAIL_URL).rstrip("/") + + +def _required_int_env(name: str) -> int: + raw_value = _required_env(name) + try: + return int(raw_value) + except ValueError as exc: + raise RuntimeError(f"{name} must be an integer") from exc + + +def _project_id() -> int: + return _required_int_env("TESTRAIL_PROJECT_ID") + + +def _api_url(endpoint: str) -> str: + return f"{_base_url()}/index.php?/api/v2/{endpoint.lstrip('/')}" + + +def _api_request( + method: str, endpoint: str, data: dict[str, Any] | None = None +) -> dict[str, Any] | list[Any]: + response = requests.request( + method, + _api_url(endpoint), + auth=( + _required_env("TESTRAIL_USERNAME"), + _required_env("TESTRAIL_API_KEY"), + ), + json=data, + headers={"Content-Type": "application/json"}, + timeout=_TIMEOUT_SECONDS, + ) + response.raise_for_status() + if not response.content: + return {} + result = response.json() + if not isinstance(result, (dict, list)): + raise RuntimeError("TestRail returned an unexpected response") + return result + + +def _require_id(response: object, object_name: str) -> int: + if not isinstance(response, dict): + raise RuntimeError(f"TestRail did not return a {object_name} object") + value = response.get("id") + try: + return int(value) + except (TypeError, ValueError) as exc: + raise RuntimeError( + f"TestRail did not return an id for the created {object_name}" + ) from exc + + +def _resolve_case_type_id() -> int: + response = _api_request("GET", "get_case_types") + case_types = ( + response.get("case_types", []) if isinstance(response, dict) else response + ) + for case_type in case_types: + if ( + isinstance(case_type, dict) + and str(case_type.get("name", "")).strip().casefold() + == _CASE_TYPE_NAME.casefold() + ): + return _require_id(case_type, "case type") + raise RuntimeError(f'TestRail has no case type named "{_CASE_TYPE_NAME}"') + + +def _resolve_template_id(project_id: int) -> int: + response = _api_request("GET", f"get_templates/{project_id}") + templates = ( + response.get("templates", []) if isinstance(response, dict) else response + ) + for template in templates: + if ( + isinstance(template, dict) + and str(template.get("name", "")).strip().casefold() + == _CASE_TEMPLATE_NAME.casefold() + ): + return _require_id(template, "template") + raise RuntimeError(f'TestRail has no template named "{_CASE_TEMPLATE_NAME}"') + + +def _case_payload( + test_case: dict[str, Any], case_type_id: int, template_id: int +) -> dict[str, Any]: + steps = [str(step) for step in test_case.get("steps", [])] + payload: dict[str, Any] = { + "title": test_case["title"], + "type_id": case_type_id, + "template_id": template_id, + "labels": [_CASE_LABEL], + "custom_steps_separated": [{"content": step, "expected": ""} for step in steps], + } + if test_case.get("preconditions"): + payload["custom_preconds"] = test_case["preconditions"] + return payload + + +def _suite_url(suite_id: int) -> str: + return f"{_base_url()}/index.php?/suites/view/{suite_id}" + + +class SubmitTestCasesHandler: + async def apply(self, params: dict[str, Any], ctx: ApplyContext) -> ActionResult: + feature = str(params.get("feature") or "").strip() + test_cases = params.get("generated_test_cases") or [] + + if not feature: + return ActionResult.failed("TestRail submission requires a feature name") + if not test_cases: + return ActionResult.failed("TestRail submission requires test cases") + + try: + project_id = _project_id() + case_type_id = _resolve_case_type_id() + template_id = _resolve_template_id(project_id) + suite_name = f"[Hackbot] - {feature}" + suite_id = _require_id( + _api_request( + "POST", + f"add_suite/{project_id}", + {"name": suite_name}, + ), + "suite", + ) + section_id = _require_id( + _api_request( + "POST", + f"add_section/{project_id}", + {"suite_id": suite_id, "name": _SECTION_NAME}, + ), + "section", + ) + + created_case_ids: dict[int, int] = {} + for test_case in test_cases: + generated_id = int(test_case["id"]) + case_id = _require_id( + _api_request( + "POST", + f"add_case/{section_id}", + _case_payload(test_case, case_type_id, template_id), + ), + "case", + ) + created_case_ids[generated_id] = case_id + + except Exception as exc: + log.exception( + "Failed to submit test plan to TestRail for run %s", ctx.run_id + ) + return ActionResult.failed(str(exc)) + + return ActionResult.ok( + { + "suite_id": suite_id, + "suite_url": _suite_url(suite_id), + "section_id": section_id, + "case_ids": list(created_case_ids.values()), + } + ) diff --git a/libs/hackbot-runtime/hackbot_runtime/actions/testrail.py b/libs/hackbot-runtime/hackbot_runtime/actions/testrail.py new file mode 100644 index 0000000000..4bc1fdb8fa --- /dev/null +++ b/libs/hackbot-runtime/hackbot_runtime/actions/testrail.py @@ -0,0 +1,28 @@ +"""TestRail-domain recordable actions. + +The test plan generator records this action deterministically after its +structured result has been validated. The external TestRail mutation still +happens only in the apply side handler. +""" + +from __future__ import annotations + +from typing import Any + +from hackbot_runtime.actions.recorder import ActionsRecorder + +ACTION_TYPE = "testrail.submit_test_cases" + + +def record_test_plan( + recorder: ActionsRecorder, + test_plan: dict[str, Any], + *, + reasoning: str = "Upload the generated test cases to TestRail.", +) -> dict: + """Record validated generated cases for deferred TestRail submission.""" + params = { + "feature": test_plan["feature"], + "generated_test_cases": test_plan["generated_test_cases"], + } + return recorder.record(ACTION_TYPE, params, reasoning=reasoning) diff --git a/libs/hackbot-runtime/tests/test_testrail_action.py b/libs/hackbot-runtime/tests/test_testrail_action.py new file mode 100644 index 0000000000..0fd9bd2026 --- /dev/null +++ b/libs/hackbot-runtime/tests/test_testrail_action.py @@ -0,0 +1,22 @@ +from hackbot_runtime.actions import ActionsRecorder +from hackbot_runtime.actions.handlers import get_handler +from hackbot_runtime.actions.handlers.testrail_handler import SubmitTestCasesHandler +from hackbot_runtime.actions.testrail import ACTION_TYPE, record_test_plan + + +def test_record_test_plan_records_deferred_action(): + recorder = ActionsRecorder() + plan = {"feature": "Feature", "generated_test_cases": [], "results": []} + + action = record_test_plan(recorder, plan) + + assert action["type"] == ACTION_TYPE + assert action["params"] == { + "feature": "Feature", + "generated_test_cases": [], + } + assert recorder.actions == [action] + + +def test_submit_test_cases_handler_is_registered(): + assert isinstance(get_handler(ACTION_TYPE), SubmitTestCasesHandler) diff --git a/libs/hackbot-runtime/tests/test_testrail_handler.py b/libs/hackbot-runtime/tests/test_testrail_handler.py new file mode 100644 index 0000000000..f6d6e7ea49 --- /dev/null +++ b/libs/hackbot-runtime/tests/test_testrail_handler.py @@ -0,0 +1,279 @@ +"""Tests for the apply-side TestRail action handler.""" + +import pytest +from hackbot_runtime.actions.handlers import ApplyContext, testrail_handler + + +@pytest.fixture(autouse=True) +def configure_project(monkeypatch): + monkeypatch.setenv("TESTRAIL_PROJECT_ID", "73") + + +def _ctx(): + async def download(_key): + raise AssertionError("TestRail submissions do not use artifacts") + + return ApplyContext(run_id="run-1", download_artifact=download) + + +def _plan(): + return { + "feature": "PDF Improvements", + "generated_test_cases": [ + { + "id": 1, + "title": "The PDF opens", + "context": "content", + "preconditions": "A PDF is available.", + "steps": ["Open the PDF", "Select some text"], + }, + { + "id": 2, + "title": "The toolbar remains available", + "context": "chrome", + "preconditions": None, + "steps": ["Open the toolbar"], + }, + ], + "results": [ + { + "id": 1, + "status": "passed", + "summary": "The PDF behaved as expected.", + "failure_reason": None, + "step_results": [ + { + "step_number": 1, + "status": "passed", + "observation": "The PDF opened.", + "failure_reason": None, + }, + { + "step_number": 2, + "status": "passed", + "observation": "Text was selected.", + "failure_reason": None, + }, + ], + }, + { + "id": 2, + "status": "unsuitable", + "summary": "The toolbar could not be inspected.", + "failure_reason": "No available tool can inspect it.", + "step_results": [ + { + "step_number": 1, + "status": "not_run", + "observation": "Not run.", + "failure_reason": None, + } + ], + }, + ], + "summary": "One passed and one was unsuitable.", + } + + +async def test_submit_test_cases_creates_suite_section_and_cases(monkeypatch): + calls = [] + responses = iter( + [ + [{"id": 6, "name": "Functional"}], + [{"id": 2, "name": "Test Case (Steps)"}], + {"id": 10}, + {"id": 20}, + {"id": 101}, + {"id": 102}, + ] + ) + + def fake_request(method, endpoint, data=None): + calls.append((method, endpoint, data)) + return next(responses) + + monkeypatch.setattr(testrail_handler, "_api_request", fake_request) + monkeypatch.setattr( + testrail_handler, "_base_url", lambda: "https://testrail.example" + ) + + result = await testrail_handler.SubmitTestCasesHandler().apply(_plan(), _ctx()) + + assert result.status == "applied" + assert result.result == { + "suite_id": 10, + "suite_url": "https://testrail.example/index.php?/suites/view/10", + "section_id": 20, + "case_ids": [101, 102], + } + + assert calls[0] == ("GET", "get_case_types", None) + assert calls[1] == ("GET", "get_templates/73", None) + assert calls[2] == ( + "POST", + "add_suite/73", + {"name": "[Hackbot] - PDF Improvements"}, + ) + assert calls[3] == ( + "POST", + "add_section/73", + {"suite_id": 10, "name": "Test Cases"}, + ) + + first_case = calls[4] + assert first_case[1] == "add_case/20" + assert first_case[2] == { + "title": "The PDF opens", + "type_id": 6, + "template_id": 2, + "labels": ["AI Generated"], + "custom_preconds": "A PDF is available.", + "custom_steps_separated": [ + {"content": "Open the PDF", "expected": ""}, + {"content": "Select some text", "expected": ""}, + ], + } + + +async def test_submit_test_cases_reports_api_failure(monkeypatch): + def fail(*_args, **_kwargs): + raise RuntimeError("TestRail is unavailable") + + monkeypatch.setattr(testrail_handler, "_api_request", fail) + result = await testrail_handler.SubmitTestCasesHandler().apply(_plan(), _ctx()) + + assert result.status == "failed" + assert result.error == "TestRail is unavailable" + + +def test_api_request_uses_basic_authentication(monkeypatch): + monkeypatch.setenv("TESTRAIL_URL", "https://testrail.example/") + monkeypatch.setenv("TESTRAIL_USERNAME", "qa@example.com") + monkeypatch.setenv("TESTRAIL_API_KEY", "secret") + captured = {} + + class Response: + content = b'{"id": 10}' + + def raise_for_status(self): + return None + + def json(self): + return {"id": 10} + + def fake_request(method, url, **kwargs): + captured.update(method=method, url=url, **kwargs) + return Response() + + monkeypatch.setattr(testrail_handler.requests, "request", fake_request) + + result = testrail_handler._api_request("POST", "add_suite/73", {"name": "Suite"}) + + assert result == {"id": 10} + assert captured["url"] == ( + "https://testrail.example/index.php?/api/v2/add_suite/73" + ) + assert captured["auth"] == ("qa@example.com", "secret") + assert captured["json"] == {"name": "Suite"} + assert captured["timeout"] == 30 + + +def test_base_url_defaults_to_mozilla_testrail(monkeypatch): + monkeypatch.delenv("TESTRAIL_URL", raising=False) + + assert testrail_handler._base_url() == "https://mozilla.testrail.io" + + +def test_api_request_requires_credentials(monkeypatch): + monkeypatch.delenv("TESTRAIL_USERNAME", raising=False) + + try: + testrail_handler._api_request("GET", "get_projects") + except RuntimeError as exc: + assert str(exc) == "TESTRAIL_USERNAME is not configured" + else: + raise AssertionError("missing TestRail configuration did not fail") + + +def test_project_id_can_be_overridden(monkeypatch): + monkeypatch.setenv("TESTRAIL_PROJECT_ID", "99") + + assert testrail_handler._project_id() == 99 + + +def test_project_id_is_required(monkeypatch): + monkeypatch.delenv("TESTRAIL_PROJECT_ID") + + try: + testrail_handler._project_id() + except RuntimeError as exc: + assert str(exc) == "TESTRAIL_PROJECT_ID is not configured" + else: + raise AssertionError("missing TestRail project id did not fail") + + +def test_configured_ids_must_be_integers(monkeypatch): + monkeypatch.setenv("TESTRAIL_PROJECT_ID", "grave-yard") + + try: + testrail_handler._project_id() + except RuntimeError as exc: + assert str(exc) == "TESTRAIL_PROJECT_ID must be an integer" + else: + raise AssertionError("invalid TestRail project id did not fail") + + +def test_resolve_case_type_id_by_name(monkeypatch): + monkeypatch.setattr( + testrail_handler, + "_api_request", + lambda *_args: [ + {"id": 3, "name": "Automated"}, + {"id": 12, "name": " functional "}, + ], + ) + + assert testrail_handler._resolve_case_type_id() == 12 + + +def test_resolve_case_type_id_requires_functional_type(monkeypatch): + monkeypatch.setattr( + testrail_handler, + "_api_request", + lambda *_args: [{"id": 3, "name": "Automated"}], + ) + + try: + testrail_handler._resolve_case_type_id() + except RuntimeError as exc: + assert str(exc) == 'TestRail has no case type named "Functional"' + else: + raise AssertionError("missing Functional case type did not fail") + + +def test_resolve_template_id_by_name(monkeypatch): + monkeypatch.setattr( + testrail_handler, + "_api_request", + lambda *_args: [ + {"id": 1, "name": "Test Case (Text)"}, + {"id": 2, "name": " test case (steps) "}, + ], + ) + + assert testrail_handler._resolve_template_id(73) == 2 + + +def test_resolve_template_id_requires_steps_template(monkeypatch): + monkeypatch.setattr( + testrail_handler, + "_api_request", + lambda *_args: [{"id": 1, "name": "Test Case (Text)"}], + ) + + try: + testrail_handler._resolve_template_id(73) + except RuntimeError as exc: + assert str(exc) == 'TestRail has no template named "Test Case (Steps)"' + else: + raise AssertionError("missing Test Case (Steps) template did not fail") diff --git a/services/hackbot-api/app/agents.py b/services/hackbot-api/app/agents.py index eb5aa804bb..ccf2149c45 100644 --- a/services/hackbot-api/app/agents.py +++ b/services/hackbot-api/app/agents.py @@ -87,5 +87,6 @@ def model_to_env(inputs: BaseModel) -> dict[str, str]: ), job_name="hackbot-agent-test-plan-generator", input_schema=TestPlanGeneratorInputs, + auto_apply_actions=True, ), } diff --git a/services/hackbot-api/tests/test_agents.py b/services/hackbot-api/tests/test_agents.py index 1e782b9963..d901f89049 100644 --- a/services/hackbot-api/tests/test_agents.py +++ b/services/hackbot-api/tests/test_agents.py @@ -91,3 +91,4 @@ def test_test_plan_generator_registry_uses_default_env_serializer(): assert spec.build_env is None assert spec.job_name == "hackbot-agent-test-plan-generator" assert spec.input_schema is PlanGeneratorInputs + assert spec.auto_apply_actions is True