From 2d858b1a561ae1e039944e6c4153ddd89b361a29 Mon Sep 17 00:00:00 2001 From: Kavya Sree Kaitepalli Date: Mon, 24 Aug 2026 07:08:02 +0000 Subject: [PATCH 1/8] feat: add training agent and command-line interface --- .../auto_memory/training/__init__.py | 3 + src/microbots/auto_memory/training/cli.py | 38 +++++++ src/microbots/auto_memory/training/runner.py | 107 ++++++++++++++++++ .../training/training_instructions.md | 79 +++++++++++++ 4 files changed, 227 insertions(+) create mode 100644 src/microbots/auto_memory/training/__init__.py create mode 100644 src/microbots/auto_memory/training/cli.py create mode 100644 src/microbots/auto_memory/training/runner.py create mode 100644 src/microbots/auto_memory/training/training_instructions.md diff --git a/src/microbots/auto_memory/training/__init__.py b/src/microbots/auto_memory/training/__init__.py new file mode 100644 index 0000000..fbdc5c1 --- /dev/null +++ b/src/microbots/auto_memory/training/__init__.py @@ -0,0 +1,3 @@ +"""Utilities for training an agent to build repository memory.""" + +from .runner import run_training \ No newline at end of file diff --git a/src/microbots/auto_memory/training/cli.py b/src/microbots/auto_memory/training/cli.py new file mode 100644 index 0000000..176be41 --- /dev/null +++ b/src/microbots/auto_memory/training/cli.py @@ -0,0 +1,38 @@ +"""Command-line interface for the repository training agent.""" + +import argparse +from .runner import run_training + + +def parse_args(): + """Parse command-line arguments for a training run. + + Returns + ------- + argparse.Namespace + Parsed command-line arguments. + """ + parser = argparse.ArgumentParser(description="Training agent") + parser.add_argument("--repo", required=True, help="Path or URL to the repo to learn from") + parser.add_argument("--feedback", default="", help="Optional feedback text (can be empty)") + parser.add_argument("--memory-dir", default="./memory", help="Directory to store the memory file") + parser.add_argument("--model", required=True, help="Model identifier, e.g. azure-openai/gpt-4o") + return parser.parse_args() + + +def main(): + """Run repository training from command-line arguments.""" + args = parse_args() + result = run_training( + repo_path=args.repo, + feedback=args.feedback, + memory_dir=args.memory_dir, + model=args.model, + ) + print(f"status={result.status} memory_dir={args.memory_dir}") + if not result.status: + print(f"error={result.error}") + + +if __name__ == "__main__": + main() diff --git a/src/microbots/auto_memory/training/runner.py b/src/microbots/auto_memory/training/runner.py new file mode 100644 index 0000000..dca9b0c --- /dev/null +++ b/src/microbots/auto_memory/training/runner.py @@ -0,0 +1,107 @@ +"""Run repository training with a reading bot and persistent memory tool.""" + +from pathlib import Path +import subprocess +import tempfile +from urllib.parse import urlparse +from microbots.bot.ReadingBot import ReadingBot +from microbots.tools.tool_definitions.memory_tool import MemoryTool +from microbots.MicroBot import BotRunResult + +_INSTRUCTIONS_PATH = Path(__file__).parent / "training_instructions.md" + +def _is_git_url(repo: str) -> bool: + """Determine whether a repository reference looks like a Git remote. + + Parameters + ---------- + repo : str + Repository path or URL. + + Returns + ------- + bool + ``True`` when the reference looks like a Git remote. + """ + parsed = urlparse(repo) + return parsed.scheme in ("http", "https", "git", "ssh") or repo.endswith(".git") + +def _prepare_source_dir(repo: str, workdir: Path) -> Path: + """Ensure a local directory exists for the agent to read from. + + Parameters + ---------- + repo : str + Local repository path or Git URL. + workdir : pathlib.Path + Working directory in which a remote repository can be cloned. + + Returns + ------- + pathlib.Path + Existing local repository path or the path to the cloned repository. + """ + if not _is_git_url(repo): + return Path(repo) + + dest = workdir / "source" + if dest.exists(): + return dest # reuse existing clone across iterations + + dest.parent.mkdir(parents=True, exist_ok=True) + subprocess.run( + ["git", "clone", "--depth", "1", repo, str(dest)], + check=True, + ) + return dest + + +def run_training( + repo_path: str, + feedback: str, + memory_dir: str, + model: str, + max_iterations: int = 20, + timeout_in_seconds: int = 600, +) -> BotRunResult: + """Run one training pass over a repository and update its memory. + + Parameters + ---------- + repo_path : str + Local repository path or Git URL to learn from. + feedback : str + Optional feedback to include in the training prompt. + memory_dir : str + Directory in which the memory tool stores its memory. + model : str + Model identifier used by the reading bot. + max_iterations : int, default=20 + Maximum number of bot iterations. + timeout_in_seconds : int, default=600 + Maximum duration of the bot run in seconds. + + Returns + ------- + microbots.MicroBot.BotRunResult + Result of the training bot run. + """ + + workdir = Path(tempfile.mkdtemp(prefix="training_workdir_")) + source_dir = _prepare_source_dir(repo_path, workdir) + + instructions = _INSTRUCTIONS_PATH.read_text(encoding="utf-8") + feedback_section = feedback.strip() or "No feedback provided for this run." + prompt = f"{instructions}\n\n## Feedback\n{feedback_section}\n" + + bot = ReadingBot( + model=model, + folder_to_mount=str(source_dir), + additional_tools=[MemoryTool(memory_dir=memory_dir)], + ) + + return bot.run( + prompt, + max_iterations=max_iterations, + timeout_in_seconds=timeout_in_seconds, + ) \ No newline at end of file diff --git a/src/microbots/auto_memory/training/training_instructions.md b/src/microbots/auto_memory/training/training_instructions.md new file mode 100644 index 0000000..baa68dc --- /dev/null +++ b/src/microbots/auto_memory/training/training_instructions.md @@ -0,0 +1,79 @@ +# Repo-Learning Agent Instructions + +You are a **package-maintainer agent** in a training phase. Your **only** job +is to **learn the repository** and write down what you learn as durable notes +in memory. You are not here to fix bugs, implement features, close tickets, +land patches, or make any change to the repository itself. + +A future evaluation loop will reuse the notes you leave behind. If it isn't in +memory, it doesn't exist. Optimise every action for "what will the next agent, +starting cold, need in order to act as maintainer of this repo?" + +--- + +## Mission + +For the repository under study, build up a **maintainer's mental model** and +persist it to `/memories/` using the `memory` tool. + +--- + +## Memory Protocol (non-negotiable) + +You have a `memory` tool that persists files under `/memories/`. Follow this +protocol every iteration: + +1. **Always start with** `memory view /memories` to see what prior iterations + already learned. Do not re-derive facts that are already recorded. +2. **Read before you write.** If a note already covers the area you're + exploring, extend or correct it instead of creating a parallel note. +3. **Write as you go.** Record each non-trivial finding immediately, in the + iteration you discovered it — do not batch discoveries until "the end". +4. **Cite sources.** Every claim should be traceable to a file path (and, when + useful, a symbol or line range) or an exact command + observed output. +5. **Prefer facts over prose.** Short bullets, tables, and code snippets beat + paragraphs. Notes are read by another agent, not a human reviewer. +6. **Keep memory tidy.** Rename vague files, delete stale ones, and merge + duplicates. A messy `/memories/` is worse than a small one. +7. **Never invent.** If you don't know, say so and (if possible) record the + next investigation step. Speculation poisons the next agent. + +Choose your own file structure inside `/memories/`. Organise it in +whatever way best fits the repo you are studying — just keep it discoverable, +non-duplicative, and easy for a cold-start agent to navigate. + +--- + +## Working Loop + +For each iteration: + +1. `memory view /memories` — recover prior state. +2. Pick the **highest-value gap** in the maintainer mental model above. +3. Investigate read-only: browse code, inspect tests, and run read-only + commands (e.g. listing files, viewing history, running an existing test + suite to observe behaviour). Do **not** modify repository files. +4. Record findings into the appropriate memory file(s), creating or + reorganising files as needed. +5. Before ending the iteration, do a final `memory view /memories` sanity + check: is your latest finding actually saved, cited, and discoverable? + +--- + +## What NOT to Record + +- Raw dumps of large files. Summarise and link by path instead. +- Transient reasoning ("I'm going to look at X next") — only keep it if it + survives the iteration as a real open question. +- Anything you are only guessing. Mark uncertainty explicitly or omit it. +- Secrets, tokens, or environment-specific absolute paths that won't + generalise to the next agent's machine. + +--- + +## Definition of Done (per iteration) + +An iteration is "done" when `/memories/` is strictly more useful to a +cold-start maintainer than it was when the iteration began — new facts +added, stale facts corrected or removed. If memory did not improve, the +iteration is not done. Update memory, then stop. \ No newline at end of file From ab5155960f0e387d6115d799c4f51f1e871473a7 Mon Sep 17 00:00:00 2001 From: Kavya Sree Kaitepalli Date: Mon, 24 Aug 2026 08:33:19 +0000 Subject: [PATCH 2/8] add tests for training agent --- test/auto_memory/training/test_cli.py | 78 +++++++ test/auto_memory/training/test_runner.py | 260 +++++++++++++++++++++++ 2 files changed, 338 insertions(+) create mode 100644 test/auto_memory/training/test_cli.py create mode 100644 test/auto_memory/training/test_runner.py diff --git a/test/auto_memory/training/test_cli.py b/test/auto_memory/training/test_cli.py new file mode 100644 index 0000000..d50d537 --- /dev/null +++ b/test/auto_memory/training/test_cli.py @@ -0,0 +1,78 @@ +"""Unit tests for microbots.auto_memory.training.cli.""" + +import sys +from unittest.mock import patch + +import pytest + +from microbots.auto_memory.training.cli import main, parse_args +from microbots.MicroBot import BotRunResult + + +@pytest.mark.unit +def test_parse_args_defaults(): + argv = [ + "cli.py", + "--repo", + "/some/repo", + "--model", + "azure-openai/gpt-4o", + ] + with patch.object(sys, "argv", argv): + args = parse_args() + + assert args.repo == "/some/repo" + assert args.model == "azure-openai/gpt-4o" + assert args.feedback == "" + assert args.memory_dir == "./memory" + + +@pytest.mark.unit +def test_main_calls_run_training_with_parsed_args(): + argv = [ + "cli.py", + "--repo", + "/some/repo", + "--feedback", + "some feedback", + "--memory-dir", + "/some/memory", + "--model", + "azure-openai/gpt-4o", + ] + fake_result = BotRunResult(status=True, result="ok", error=None) + + with patch.object(sys, "argv", argv), patch( + "microbots.auto_memory.training.cli.run_training", + return_value=fake_result, + ) as mock_run_training: + main() + + mock_run_training.assert_called_once_with( + repo_path="/some/repo", + feedback="some feedback", + memory_dir="/some/memory", + model="azure-openai/gpt-4o", + ) + + +@pytest.mark.unit +def test_main_prints_error_on_failure(capsys): + argv = [ + "cli.py", + "--repo", + "/some/repo", + "--model", + "azure-openai/gpt-4o", + ] + fake_result = BotRunResult(status=False, result=None, error="boom") + + with patch.object(sys, "argv", argv), patch( + "microbots.auto_memory.training.cli.run_training", + return_value=fake_result, + ): + main() + + captured = capsys.readouterr() + assert "status=False" in captured.out + assert "error=boom" in captured.out diff --git a/test/auto_memory/training/test_runner.py b/test/auto_memory/training/test_runner.py new file mode 100644 index 0000000..9ce3751 --- /dev/null +++ b/test/auto_memory/training/test_runner.py @@ -0,0 +1,260 @@ +"""Unit tests for microbots.auto_memory.training.runner. + +All external dependencies (subprocess/git, ReadingBot, MemoryTool) are +mocked so these tests run without Docker, network access, or an LLM. +The one exception is test_run_training_end_to_end, which is a real +integration test (marked accordingly) that exercises Docker and a live +model deployment. +""" + +import os +from pathlib import Path +from subprocess import CalledProcessError +from unittest.mock import MagicMock, patch + +import pytest + +from microbots.auto_memory.training.runner import ( + _is_git_url, + _prepare_source_dir, + run_training, +) +from microbots.MicroBot import BotRunResult + + +# --------------------------------------------------------------------------- +# _is_git_url +# --------------------------------------------------------------------------- + +@pytest.mark.unit +@pytest.mark.parametrize( + "repo, expected", + [ + ("https://github.com/pytest-dev/pytest.git", True), + ("git@github.com:pytest-dev/pytest.git", True), + ("ssh://git@github.com/pytest-dev/pytest.git", True), + ("/home/user/some/local/repo", False), + ("some-local-dir-without-scheme", False), + ("relative/local/path.git", True), # ends with .git -> treated as git + ], +) +def test_is_git_url(repo, expected): + assert _is_git_url(repo) is expected + + +# --------------------------------------------------------------------------- +# _prepare_source_dir +# --------------------------------------------------------------------------- + +@pytest.mark.unit +def test_prepare_source_dir_local_path_passthrough(tmp_path): + local_repo = tmp_path / "local_repo" + local_repo.mkdir() + + with patch("microbots.auto_memory.training.runner.subprocess.run") as mock_run: + result = _prepare_source_dir(str(local_repo), tmp_path / "workdir") + + assert result == Path(local_repo) + mock_run.assert_not_called() + + +@pytest.mark.unit +def test_prepare_source_dir_clones_git_url(tmp_path): + workdir = tmp_path / "workdir" + repo_url = "https://github.com/pytest-dev/pytest.git" + expected_dest = workdir / "source" + + with patch("microbots.auto_memory.training.runner.subprocess.run") as mock_run: + result = _prepare_source_dir(repo_url, workdir) + + mock_run.assert_called_once_with( + ["git", "clone", "--depth", "1", repo_url, str(expected_dest)], + check=True, + ) + assert result == expected_dest + + +@pytest.mark.unit +def test_prepare_source_dir_reuses_existing_clone(tmp_path): + workdir = tmp_path / "workdir" + dest = workdir / "source" + dest.mkdir(parents=True) + repo_url = "https://github.com/pytest-dev/pytest.git" + + with patch("microbots.auto_memory.training.runner.subprocess.run") as mock_run: + result = _prepare_source_dir(repo_url, workdir) + + mock_run.assert_not_called() + assert result == dest + + +@pytest.mark.unit +def test_prepare_source_dir_clone_failure_propagates(tmp_path): + workdir = tmp_path / "workdir" + repo_url = "https://github.com/pytest-dev/pytest.git" + + with patch( + "microbots.auto_memory.training.runner.subprocess.run", + side_effect=CalledProcessError(returncode=1, cmd=["git", "clone"]), + ): + with pytest.raises(CalledProcessError): + _prepare_source_dir(repo_url, workdir) + + +# --------------------------------------------------------------------------- +# run_training +# --------------------------------------------------------------------------- + +@pytest.mark.unit +def test_run_training_prompt_includes_feedback(tmp_path): + local_repo = tmp_path / "repo" + local_repo.mkdir() + memory_dir = tmp_path / "memory" + + mock_bot_instance = MagicMock() + mock_bot_instance.run.return_value = BotRunResult( + status=True, result="ok", error=None + ) + + with patch( + "microbots.auto_memory.training.runner.ReadingBot", + return_value=mock_bot_instance, + ), patch("microbots.auto_memory.training.runner.MemoryTool"): + run_training( + repo_path=str(local_repo), + feedback="Focus on error handling paths.", + memory_dir=str(memory_dir), + model="azure-openai/gpt-4o", + ) + + prompt_arg = mock_bot_instance.run.call_args.args[0] + assert "Focus on error handling paths." in prompt_arg + + +@pytest.mark.unit +def test_run_training_prompt_handles_empty_feedback(tmp_path): + local_repo = tmp_path / "repo" + local_repo.mkdir() + memory_dir = tmp_path / "memory" + + mock_bot_instance = MagicMock() + mock_bot_instance.run.return_value = BotRunResult( + status=True, result="ok", error=None + ) + + with patch( + "microbots.auto_memory.training.runner.ReadingBot", + return_value=mock_bot_instance, + ), patch("microbots.auto_memory.training.runner.MemoryTool"): + run_training( + repo_path=str(local_repo), + feedback="", + memory_dir=str(memory_dir), + model="azure-openai/gpt-4o", + ) + + prompt_arg = mock_bot_instance.run.call_args.args[0] + assert "No feedback provided for this run." in prompt_arg + + +@pytest.mark.unit +def test_run_training_passes_correct_args_to_reading_bot(tmp_path): + local_repo = tmp_path / "repo" + local_repo.mkdir() + memory_dir = tmp_path / "memory" + + mock_bot_instance = MagicMock() + mock_bot_instance.run.return_value = BotRunResult( + status=True, result="ok", error=None + ) + mock_memory_tool_instance = MagicMock() + + with patch( + "microbots.auto_memory.training.runner.ReadingBot", + return_value=mock_bot_instance, + ) as mock_reading_bot, patch( + "microbots.auto_memory.training.runner.MemoryTool", + return_value=mock_memory_tool_instance, + ) as mock_memory_tool: + run_training( + repo_path=str(local_repo), + feedback="", + memory_dir=str(memory_dir), + model="azure-openai/gpt-4o", + ) + + mock_memory_tool.assert_called_once_with(memory_dir=str(memory_dir)) + + _, kwargs = mock_reading_bot.call_args + assert kwargs["model"] == "azure-openai/gpt-4o" + assert kwargs["folder_to_mount"] == str(local_repo) + assert kwargs["additional_tools"] == [mock_memory_tool_instance] + + +@pytest.mark.unit +def test_run_training_returns_bot_result(tmp_path): + local_repo = tmp_path / "repo" + local_repo.mkdir() + memory_dir = tmp_path / "memory" + + expected_result = BotRunResult(status=True, result="done", error=None) + mock_bot_instance = MagicMock() + mock_bot_instance.run.return_value = expected_result + + with patch( + "microbots.auto_memory.training.runner.ReadingBot", + return_value=mock_bot_instance, + ), patch("microbots.auto_memory.training.runner.MemoryTool"): + result = run_training( + repo_path=str(local_repo), + feedback="", + memory_dir=str(memory_dir), + model="azure-openai/gpt-4o", + ) + + assert result is expected_result + + +# --------------------------------------------------------------------------- +# End-to-end integration test (real Docker + real LLM deployment required) +# --------------------------------------------------------------------------- + +@pytest.mark.integration +@pytest.mark.slow +@pytest.mark.docker +def test_run_training_end_to_end(test_repo, tmp_path): + """Smoke-test the training flow against a small fixture repo. + + Requires Docker and a working model deployment (same env vars used by + test/bot/test_reading_bot.py). This is a smoke test, not a + completion test: max_iterations is intentionally kept small so it + 's fast to run locally. It only asserts the flow executes end-to-end + (clone -> mount -> bot run) without asserting the agent reached + task_done, since that may need more iterations than we want to spend + here. + """ + memory_dir = tmp_path / "memory" + model = f"azure-openai/{os.getenv('AZURE_OPENAI_DEPLOYMENT_NAME', 'mini-swe-agent-gpt5')}" + + result: BotRunResult = run_training( + repo_path=str(test_repo), + feedback="", + memory_dir=str(memory_dir), + model=model, + max_iterations=8, + timeout_in_seconds=600, + ) + + # Accept either a completed run, or a run that stopped only because it + # hit the (intentionally low) iteration cap - both prove the flow works. + acceptable_errors = (None, "Max iterations 8 reached") + assert result.status or result.error in acceptable_errors, ( + f"Training run failed unexpectedly: {result.error}" + ) + + # With only 5 iterations the agent may not fully finish the task, but + # it should still persist at least one memory file along the way. + memory_files = [f for f in memory_dir.rglob("*") if f.is_file()] + assert memory_files, ( + f"Expected at least one memory file under {memory_dir}, found none" + ) From 87f581af180da720b7f1dce3548cecdbc5a966ee Mon Sep 17 00:00:00 2001 From: Kavya Sree Kaitepalli Date: Mon, 24 Aug 2026 09:14:46 +0000 Subject: [PATCH 3/8] enhance logging in training CLI and runner, improve cleanup of temporary directories --- src/microbots/auto_memory/training/cli.py | 10 ++- src/microbots/auto_memory/training/runner.py | 61 +++++++++----- test/auto_memory/training/test_cli.py | 9 +-- test/auto_memory/training/test_runner.py | 83 +++++++++++++++++++- 4 files changed, 134 insertions(+), 29 deletions(-) diff --git a/src/microbots/auto_memory/training/cli.py b/src/microbots/auto_memory/training/cli.py index 176be41..96f10bb 100644 --- a/src/microbots/auto_memory/training/cli.py +++ b/src/microbots/auto_memory/training/cli.py @@ -1,8 +1,12 @@ """Command-line interface for the repository training agent.""" import argparse +import logging + from .runner import run_training +logger = logging.getLogger(__name__) + def parse_args(): """Parse command-line arguments for a training run. @@ -22,6 +26,7 @@ def parse_args(): def main(): """Run repository training from command-line arguments.""" + logging.basicConfig(level=logging.INFO) args = parse_args() result = run_training( repo_path=args.repo, @@ -29,10 +34,11 @@ def main(): memory_dir=args.memory_dir, model=args.model, ) - print(f"status={result.status} memory_dir={args.memory_dir}") + logger.info("status=%s memory_dir=%s", result.status, args.memory_dir) if not result.status: - print(f"error={result.error}") + logger.error("error=%s", result.error) if __name__ == "__main__": main() + diff --git a/src/microbots/auto_memory/training/runner.py b/src/microbots/auto_memory/training/runner.py index dca9b0c..a147742 100644 --- a/src/microbots/auto_memory/training/runner.py +++ b/src/microbots/auto_memory/training/runner.py @@ -1,15 +1,27 @@ """Run repository training with a reading bot and persistent memory tool.""" -from pathlib import Path +import logging +import re +import shutil import subprocess import tempfile +from pathlib import Path from urllib.parse import urlparse + from microbots.bot.ReadingBot import ReadingBot from microbots.tools.tool_definitions.memory_tool import MemoryTool from microbots.MicroBot import BotRunResult +logger = logging.getLogger(__name__) + _INSTRUCTIONS_PATH = Path(__file__).parent / "training_instructions.md" +# Matches SCP-style SSH remotes, e.g. "git@github.com:org/repo" or +# "git@github.com:org/repo.git". urlparse() alone can't detect these since +# they have no scheme, and they don't always end in ".git". +_SCP_STYLE_RE = re.compile(r"^[\w.\-]+@[\w.\-]+:") + + def _is_git_url(repo: str) -> bool: """Determine whether a repository reference looks like a Git remote. @@ -24,7 +36,11 @@ def _is_git_url(repo: str) -> bool: ``True`` when the reference looks like a Git remote. """ parsed = urlparse(repo) - return parsed.scheme in ("http", "https", "git", "ssh") or repo.endswith(".git") + return ( + parsed.scheme in ("http", "https", "git", "ssh") + or repo.endswith(".git") + or bool(_SCP_STYLE_RE.match(repo)) + ) def _prepare_source_dir(repo: str, workdir: Path) -> Path: """Ensure a local directory exists for the agent to read from. @@ -88,20 +104,27 @@ def run_training( """ workdir = Path(tempfile.mkdtemp(prefix="training_workdir_")) - source_dir = _prepare_source_dir(repo_path, workdir) - - instructions = _INSTRUCTIONS_PATH.read_text(encoding="utf-8") - feedback_section = feedback.strip() or "No feedback provided for this run." - prompt = f"{instructions}\n\n## Feedback\n{feedback_section}\n" - - bot = ReadingBot( - model=model, - folder_to_mount=str(source_dir), - additional_tools=[MemoryTool(memory_dir=memory_dir)], - ) - - return bot.run( - prompt, - max_iterations=max_iterations, - timeout_in_seconds=timeout_in_seconds, - ) \ No newline at end of file + try: + source_dir = _prepare_source_dir(repo_path, workdir) + + instructions = _INSTRUCTIONS_PATH.read_text(encoding="utf-8") + feedback_section = feedback.strip() or "No feedback provided for this run." + prompt = f"{instructions}\n\n## Feedback\n{feedback_section}\n" + + bot = ReadingBot( + model=model, + folder_to_mount=str(source_dir), + additional_tools=[MemoryTool(memory_dir=memory_dir)], + ) + + return bot.run( + prompt, + max_iterations=max_iterations, + timeout_in_seconds=timeout_in_seconds, + ) + finally: + # Only remove workdir when we actually cloned into it; a local + # repo_path is used directly and must never be deleted here. + if _is_git_url(repo_path): + logger.info("Cleaning up training workdir %s", workdir) + shutil.rmtree(workdir, ignore_errors=True) \ No newline at end of file diff --git a/test/auto_memory/training/test_cli.py b/test/auto_memory/training/test_cli.py index d50d537..7c40929 100644 --- a/test/auto_memory/training/test_cli.py +++ b/test/auto_memory/training/test_cli.py @@ -57,7 +57,7 @@ def test_main_calls_run_training_with_parsed_args(): @pytest.mark.unit -def test_main_prints_error_on_failure(capsys): +def test_main_logs_status_and_error_on_failure(caplog): argv = [ "cli.py", "--repo", @@ -70,9 +70,8 @@ def test_main_prints_error_on_failure(capsys): with patch.object(sys, "argv", argv), patch( "microbots.auto_memory.training.cli.run_training", return_value=fake_result, - ): + ), caplog.at_level("INFO", logger="microbots.auto_memory.training.cli"): main() - captured = capsys.readouterr() - assert "status=False" in captured.out - assert "error=boom" in captured.out + assert "status=False" in caplog.text + assert "error=boom" in caplog.text diff --git a/test/auto_memory/training/test_runner.py b/test/auto_memory/training/test_runner.py index 9ce3751..4b6c66c 100644 --- a/test/auto_memory/training/test_runner.py +++ b/test/auto_memory/training/test_runner.py @@ -10,6 +10,7 @@ import os from pathlib import Path from subprocess import CalledProcessError +import tempfile from unittest.mock import MagicMock, patch import pytest @@ -36,6 +37,7 @@ ("/home/user/some/local/repo", False), ("some-local-dir-without-scheme", False), ("relative/local/path.git", True), # ends with .git -> treated as git + ("git@github.com:pytest-dev/pytest", True), # SCP-style, no .git suffix ], ) def test_is_git_url(repo, expected): @@ -215,6 +217,81 @@ def test_run_training_returns_bot_result(tmp_path): assert result is expected_result +@pytest.mark.unit +def test_run_training_cleans_up_workdir_after_git_clone(tmp_path): + """When repo_path is a git URL, the temp workdir it clones into should + be removed once the run finishes.""" + memory_dir = tmp_path / "memory" + repo_url = "https://github.com/pytest-dev/pytest.git" + + created_workdirs = [] + + def fake_clone(cmd, check): + # cmd = ["git", "clone", "--depth", "1", repo_url, dest] + dest = Path(cmd[-1]) + dest.mkdir(parents=True, exist_ok=True) + + mock_bot_instance = MagicMock() + mock_bot_instance.run.return_value = BotRunResult( + status=True, result="ok", error=None + ) + + real_mkdtemp = tempfile.mkdtemp + + def tracking_mkdtemp(*args, **kwargs): + path = real_mkdtemp(*args, **kwargs) + created_workdirs.append(Path(path)) + return path + + with patch( + "microbots.auto_memory.training.runner.subprocess.run", + side_effect=fake_clone, + ), patch( + "microbots.auto_memory.training.runner.tempfile.mkdtemp", + side_effect=tracking_mkdtemp, + ), patch( + "microbots.auto_memory.training.runner.ReadingBot", + return_value=mock_bot_instance, + ), patch("microbots.auto_memory.training.runner.MemoryTool"): + run_training( + repo_path=repo_url, + feedback="", + memory_dir=str(memory_dir), + model="azure-openai/gpt-4o", + ) + + assert len(created_workdirs) == 1 + assert not created_workdirs[0].exists() + + +@pytest.mark.unit +def test_run_training_keeps_local_repo_untouched(tmp_path): + """When repo_path is a local directory, it must never be deleted, + even though the (unused) temp workdir is still cleaned up.""" + local_repo = tmp_path / "repo" + local_repo.mkdir() + (local_repo / "marker.txt").write_text("keep me") + memory_dir = tmp_path / "memory" + + mock_bot_instance = MagicMock() + mock_bot_instance.run.return_value = BotRunResult( + status=True, result="ok", error=None + ) + + with patch( + "microbots.auto_memory.training.runner.ReadingBot", + return_value=mock_bot_instance, + ), patch("microbots.auto_memory.training.runner.MemoryTool"): + run_training( + repo_path=str(local_repo), + feedback="", + memory_dir=str(memory_dir), + model="azure-openai/gpt-4o", + ) + + assert (local_repo / "marker.txt").exists() + + # --------------------------------------------------------------------------- # End-to-end integration test (real Docker + real LLM deployment required) # --------------------------------------------------------------------------- @@ -227,8 +304,8 @@ def test_run_training_end_to_end(test_repo, tmp_path): Requires Docker and a working model deployment (same env vars used by test/bot/test_reading_bot.py). This is a smoke test, not a - completion test: max_iterations is intentionally kept small so it - 's fast to run locally. It only asserts the flow executes end-to-end + completion test: max_iterations is intentionally kept small so it's + fast to run locally. It only asserts the flow executes end-to-end (clone -> mount -> bot run) without asserting the agent reached task_done, since that may need more iterations than we want to spend here. @@ -252,7 +329,7 @@ def test_run_training_end_to_end(test_repo, tmp_path): f"Training run failed unexpectedly: {result.error}" ) - # With only 5 iterations the agent may not fully finish the task, but + # With only 8 iterations the agent may not fully finish the task, but # it should still persist at least one memory file along the way. memory_files = [f for f in memory_dir.rglob("*") if f.is_file()] assert memory_files, ( From 5dbb4510713596741605debf2c7bc5f518604879 Mon Sep 17 00:00:00 2001 From: Kavya Sree Kaitepalli Date: Wed, 26 Aug 2026 07:11:59 +0000 Subject: [PATCH 4/8] Add loop for multiple training iterations and enhance feedback handling in training instructions --- src/microbots/auto_memory/training/cli.py | 33 ++++++++++++++----- .../training/training_instructions.md | 25 ++++++++++++++ test/auto_memory/training/test_cli.py | 26 +++++++++++++++ 3 files changed, 75 insertions(+), 9 deletions(-) diff --git a/src/microbots/auto_memory/training/cli.py b/src/microbots/auto_memory/training/cli.py index 96f10bb..1ecc23f 100644 --- a/src/microbots/auto_memory/training/cli.py +++ b/src/microbots/auto_memory/training/cli.py @@ -21,6 +21,12 @@ def parse_args(): parser.add_argument("--feedback", default="", help="Optional feedback text (can be empty)") parser.add_argument("--memory-dir", default="./memory", help="Directory to store the memory file") parser.add_argument("--model", required=True, help="Model identifier, e.g. azure-openai/gpt-4o") + parser.add_argument( + "--iterations", + type=int, + default=1, + help="Number of training passes to run in sequence over the same memory_dir", + ) return parser.parse_args() @@ -28,15 +34,24 @@ def main(): """Run repository training from command-line arguments.""" logging.basicConfig(level=logging.INFO) args = parse_args() - result = run_training( - repo_path=args.repo, - feedback=args.feedback, - memory_dir=args.memory_dir, - model=args.model, - ) - logger.info("status=%s memory_dir=%s", result.status, args.memory_dir) - if not result.status: - logger.error("error=%s", result.error) + + for iteration in range(1, args.iterations + 1): + logger.info("training iteration %d/%d starting", iteration, args.iterations) + result = run_training( + repo_path=args.repo, + feedback=args.feedback, + memory_dir=args.memory_dir, + model=args.model, + ) + logger.info( + "training iteration %d/%d: status=%s memory_dir=%s", + iteration, + args.iterations, + result.status, + args.memory_dir, + ) + if not result.status: + logger.error("iteration %d/%d error=%s", iteration, args.iterations, result.error) if __name__ == "__main__": diff --git a/src/microbots/auto_memory/training/training_instructions.md b/src/microbots/auto_memory/training/training_instructions.md index baa68dc..a202ec3 100644 --- a/src/microbots/auto_memory/training/training_instructions.md +++ b/src/microbots/auto_memory/training/training_instructions.md @@ -44,6 +44,31 @@ non-duplicative, and easy for a cold-start agent to navigate. --- +## Handling Feedback + +Each run may include a `## Feedback` section appended after these +instructions. This feedback comes from a prior evaluation attempt that +**failed** using the memory notes as they currently stand. + +1. **Treat feedback as the highest-priority gap.** It is direct evidence that + something in `/memories/` was missing, wrong, or misleading — investigate + and correct it before exploring anything else this iteration. +2. **Read the current memory first.** Check whether the failure relates to an + existing note (it was incomplete or incorrect) or a missing note (the area + was never covered). +3. **Update memory to close the gap**, not just to acknowledge the feedback. + Add the missing fact, correct the wrong one, or add a caveat/edge case + that explains why the previous approach failed. +4. **If feedback is empty or missing** (`"No feedback provided for this run."`), + there is no prior failure to address — proceed with the normal working + loop below and continue building out the maintainer mental model. +5. **Never leave feedback unaddressed.** If you cannot fully resolve the + underlying gap this iteration, record what you learned and what remains + unresolved as an explicit open question in memory, so the next iteration + picks it up. + +--- + ## Working Loop For each iteration: diff --git a/test/auto_memory/training/test_cli.py b/test/auto_memory/training/test_cli.py index 7c40929..6944ad3 100644 --- a/test/auto_memory/training/test_cli.py +++ b/test/auto_memory/training/test_cli.py @@ -56,6 +56,32 @@ def test_main_calls_run_training_with_parsed_args(): ) +@pytest.mark.unit +def test_main_runs_multiple_iterations_with_same_memory_dir(): + argv = [ + "cli.py", + "--repo", + "/some/repo", + "--memory-dir", + "/some/memory", + "--model", + "azure-openai/gpt-4o", + "--iterations", + "3", + ] + fake_result = BotRunResult(status=True, result="ok", error=None) + + with patch.object(sys, "argv", argv), patch( + "microbots.auto_memory.training.cli.run_training", + return_value=fake_result, + ) as mock_run_training: + main() + + assert mock_run_training.call_count == 3 + for call in mock_run_training.call_args_list: + assert call.kwargs["memory_dir"] == "/some/memory" + + @pytest.mark.unit def test_main_logs_status_and_error_on_failure(caplog): argv = [ From a919d273d2fa3c4b766bf9f849b33a5925580736 Mon Sep 17 00:00:00 2001 From: Kavya Sree Kaitepalli Date: Wed, 26 Aug 2026 09:15:26 +0000 Subject: [PATCH 5/8] implement run_training_loop for multiple training iterations and update CLI --- src/microbots/auto_memory/training/cli.py | 28 ++---- src/microbots/auto_memory/training/runner.py | 68 +++++++++++++- test/auto_memory/training/test_cli.py | 39 ++++---- test/auto_memory/training/test_runner.py | 96 +++++++++++++++++++- 4 files changed, 190 insertions(+), 41 deletions(-) diff --git a/src/microbots/auto_memory/training/cli.py b/src/microbots/auto_memory/training/cli.py index 1ecc23f..f6a4436 100644 --- a/src/microbots/auto_memory/training/cli.py +++ b/src/microbots/auto_memory/training/cli.py @@ -3,7 +3,7 @@ import argparse import logging -from .runner import run_training +from .runner import run_training_loop logger = logging.getLogger(__name__) @@ -35,23 +35,15 @@ def main(): logging.basicConfig(level=logging.INFO) args = parse_args() - for iteration in range(1, args.iterations + 1): - logger.info("training iteration %d/%d starting", iteration, args.iterations) - result = run_training( - repo_path=args.repo, - feedback=args.feedback, - memory_dir=args.memory_dir, - model=args.model, - ) - logger.info( - "training iteration %d/%d: status=%s memory_dir=%s", - iteration, - args.iterations, - result.status, - args.memory_dir, - ) - if not result.status: - logger.error("iteration %d/%d error=%s", iteration, args.iterations, result.error) + result = run_training_loop( + repo_path=args.repo, + feedback=args.feedback, + memory_dir=args.memory_dir, + model=args.model, + iterations=args.iterations, + ) + if not result.status: + logger.error("training loop finished with failure: %s", result.error) if __name__ == "__main__": diff --git a/src/microbots/auto_memory/training/runner.py b/src/microbots/auto_memory/training/runner.py index a147742..1dc1f99 100644 --- a/src/microbots/auto_memory/training/runner.py +++ b/src/microbots/auto_memory/training/runner.py @@ -127,4 +127,70 @@ def run_training( # repo_path is used directly and must never be deleted here. if _is_git_url(repo_path): logger.info("Cleaning up training workdir %s", workdir) - shutil.rmtree(workdir, ignore_errors=True) \ No newline at end of file + shutil.rmtree(workdir, ignore_errors=True) + + +def run_training_loop( + repo_path: str, + feedback: str, + memory_dir: str, + model: str, + iterations: int = 1, + max_iterations: int = 20, + timeout_in_seconds: int = 600, +) -> BotRunResult: + """Run repeated training passes over the same memory directory. + + Each pass reuses ``memory_dir`` from the previous one, allowing the + agent to progressively refine its notes. The real output of this + function is the accumulated state on disk under ``memory_dir``; the + returned :class:`~microbots.MicroBot.BotRunResult` is only a health + signal for the *last* iteration, since individual iteration failures + are logged and do not stop the loop. + + Parameters + ---------- + repo_path : str + Local repository path or Git URL to learn from. + feedback : str + Optional feedback to include in the training prompt, applied to + every iteration. + memory_dir : str + Directory in which the memory tool stores its memory, shared and + accumulated across all iterations. + model : str + Model identifier used by the reading bot. + iterations : int, default=1 + Number of training passes to run in sequence. + max_iterations : int, default=20 + Maximum number of bot iterations per training pass. + timeout_in_seconds : int, default=600 + Maximum duration of each training pass in seconds. + + Returns + ------- + microbots.MicroBot.BotRunResult + Result of the last training pass. + """ + result: BotRunResult | None = None + for iteration in range(1, iterations + 1): + logger.info("training iteration %d/%d starting", iteration, iterations) + result = run_training( + repo_path=repo_path, + feedback=feedback, + memory_dir=memory_dir, + model=model, + max_iterations=max_iterations, + timeout_in_seconds=timeout_in_seconds, + ) + logger.info( + "training iteration %d/%d: status=%s memory_dir=%s", + iteration, + iterations, + result.status, + memory_dir, + ) + if not result.status: + logger.error("iteration %d/%d error=%s", iteration, iterations, result.error) + + return result \ No newline at end of file diff --git a/test/auto_memory/training/test_cli.py b/test/auto_memory/training/test_cli.py index 6944ad3..55bb254 100644 --- a/test/auto_memory/training/test_cli.py +++ b/test/auto_memory/training/test_cli.py @@ -25,10 +25,11 @@ def test_parse_args_defaults(): assert args.model == "azure-openai/gpt-4o" assert args.feedback == "" assert args.memory_dir == "./memory" + assert args.iterations == 1 @pytest.mark.unit -def test_main_calls_run_training_with_parsed_args(): +def test_main_calls_run_training_loop_with_parsed_args(): argv = [ "cli.py", "--repo", @@ -39,51 +40,48 @@ def test_main_calls_run_training_with_parsed_args(): "/some/memory", "--model", "azure-openai/gpt-4o", + "--iterations", + "3", ] fake_result = BotRunResult(status=True, result="ok", error=None) with patch.object(sys, "argv", argv), patch( - "microbots.auto_memory.training.cli.run_training", + "microbots.auto_memory.training.cli.run_training_loop", return_value=fake_result, - ) as mock_run_training: + ) as mock_run_training_loop: main() - mock_run_training.assert_called_once_with( + mock_run_training_loop.assert_called_once_with( repo_path="/some/repo", feedback="some feedback", memory_dir="/some/memory", model="azure-openai/gpt-4o", + iterations=3, ) @pytest.mark.unit -def test_main_runs_multiple_iterations_with_same_memory_dir(): +def test_main_logs_error_when_loop_fails_on_last_iteration(caplog): argv = [ "cli.py", "--repo", "/some/repo", - "--memory-dir", - "/some/memory", "--model", "azure-openai/gpt-4o", - "--iterations", - "3", ] - fake_result = BotRunResult(status=True, result="ok", error=None) + fake_result = BotRunResult(status=False, result=None, error="boom") with patch.object(sys, "argv", argv), patch( - "microbots.auto_memory.training.cli.run_training", + "microbots.auto_memory.training.cli.run_training_loop", return_value=fake_result, - ) as mock_run_training: + ), caplog.at_level("ERROR", logger="microbots.auto_memory.training.cli"): main() - assert mock_run_training.call_count == 3 - for call in mock_run_training.call_args_list: - assert call.kwargs["memory_dir"] == "/some/memory" + assert "boom" in caplog.text @pytest.mark.unit -def test_main_logs_status_and_error_on_failure(caplog): +def test_main_does_not_log_error_when_loop_succeeds(caplog): argv = [ "cli.py", "--repo", @@ -91,13 +89,12 @@ def test_main_logs_status_and_error_on_failure(caplog): "--model", "azure-openai/gpt-4o", ] - fake_result = BotRunResult(status=False, result=None, error="boom") + fake_result = BotRunResult(status=True, result="ok", error=None) with patch.object(sys, "argv", argv), patch( - "microbots.auto_memory.training.cli.run_training", + "microbots.auto_memory.training.cli.run_training_loop", return_value=fake_result, - ), caplog.at_level("INFO", logger="microbots.auto_memory.training.cli"): + ), caplog.at_level("ERROR", logger="microbots.auto_memory.training.cli"): main() - assert "status=False" in caplog.text - assert "error=boom" in caplog.text + assert caplog.text == "" diff --git a/test/auto_memory/training/test_runner.py b/test/auto_memory/training/test_runner.py index 4b6c66c..0ea5654 100644 --- a/test/auto_memory/training/test_runner.py +++ b/test/auto_memory/training/test_runner.py @@ -19,6 +19,7 @@ _is_git_url, _prepare_source_dir, run_training, + run_training_loop, ) from microbots.MicroBot import BotRunResult @@ -293,9 +294,102 @@ def test_run_training_keeps_local_repo_untouched(tmp_path): # --------------------------------------------------------------------------- -# End-to-end integration test (real Docker + real LLM deployment required) +# run_training_loop # --------------------------------------------------------------------------- +@pytest.mark.unit +def test_run_training_loop_calls_run_training_n_times_with_same_memory_dir(): + fake_result = BotRunResult(status=True, result="ok", error=None) + + with patch( + "microbots.auto_memory.training.runner.run_training", + return_value=fake_result, + ) as mock_run_training: + run_training_loop( + repo_path="/some/repo", + feedback="fb", + memory_dir="/some/memory", + model="azure-openai/gpt-4o", + iterations=3, + ) + + assert mock_run_training.call_count == 3 + for call in mock_run_training.call_args_list: + assert call.kwargs["memory_dir"] == "/some/memory" + assert call.kwargs["feedback"] == "fb" + assert call.kwargs["repo_path"] == "/some/repo" + assert call.kwargs["model"] == "azure-openai/gpt-4o" + + +@pytest.mark.unit +def test_run_training_loop_returns_last_result(): + results = [ + BotRunResult(status=True, result="first", error=None), + BotRunResult(status=False, result=None, error="second failed"), + BotRunResult(status=True, result="third", error=None), + ] + + with patch( + "microbots.auto_memory.training.runner.run_training", + side_effect=results, + ): + result = run_training_loop( + repo_path="/some/repo", + feedback="", + memory_dir="/some/memory", + model="azure-openai/gpt-4o", + iterations=3, + ) + + assert result is results[-1] + + +@pytest.mark.unit +def test_run_training_loop_continues_after_a_failed_iteration(): + """A failure on iteration N must not stop iteration N+1 from running.""" + results = [ + BotRunResult(status=False, result=None, error="boom"), + BotRunResult(status=True, result="ok", error=None), + ] + + with patch( + "microbots.auto_memory.training.runner.run_training", + side_effect=results, + ) as mock_run_training: + result = run_training_loop( + repo_path="/some/repo", + feedback="", + memory_dir="/some/memory", + model="azure-openai/gpt-4o", + iterations=2, + ) + + assert mock_run_training.call_count == 2 + assert result is results[-1] + + +@pytest.mark.unit +def test_run_training_loop_default_single_iteration(): + fake_result = BotRunResult(status=True, result="ok", error=None) + + with patch( + "microbots.auto_memory.training.runner.run_training", + return_value=fake_result, + ) as mock_run_training: + result = run_training_loop( + repo_path="/some/repo", + feedback="", + memory_dir="/some/memory", + model="azure-openai/gpt-4o", + ) + + mock_run_training.assert_called_once() + assert result is fake_result + + +# --------------------------------------------------------------------------- +# End-to-end integration test (real Docker + real LLM deployment required) +# --------------------------------------------------------------------------- @pytest.mark.integration @pytest.mark.slow @pytest.mark.docker From 52f0829a96881558d860bd6fed4879011b026c46 Mon Sep 17 00:00:00 2001 From: Kavya Sree Kaitepalli Date: Thu, 27 Aug 2026 05:18:13 +0000 Subject: [PATCH 6/8] refactor: remove end-to-end integration test for training loop --- test/auto_memory/training/test_runner.py | 45 ------------------------ 1 file changed, 45 deletions(-) diff --git a/test/auto_memory/training/test_runner.py b/test/auto_memory/training/test_runner.py index 0ea5654..003756b 100644 --- a/test/auto_memory/training/test_runner.py +++ b/test/auto_memory/training/test_runner.py @@ -7,7 +7,6 @@ model deployment. """ -import os from pathlib import Path from subprocess import CalledProcessError import tempfile @@ -385,47 +384,3 @@ def test_run_training_loop_default_single_iteration(): mock_run_training.assert_called_once() assert result is fake_result - - -# --------------------------------------------------------------------------- -# End-to-end integration test (real Docker + real LLM deployment required) -# --------------------------------------------------------------------------- -@pytest.mark.integration -@pytest.mark.slow -@pytest.mark.docker -def test_run_training_end_to_end(test_repo, tmp_path): - """Smoke-test the training flow against a small fixture repo. - - Requires Docker and a working model deployment (same env vars used by - test/bot/test_reading_bot.py). This is a smoke test, not a - completion test: max_iterations is intentionally kept small so it's - fast to run locally. It only asserts the flow executes end-to-end - (clone -> mount -> bot run) without asserting the agent reached - task_done, since that may need more iterations than we want to spend - here. - """ - memory_dir = tmp_path / "memory" - model = f"azure-openai/{os.getenv('AZURE_OPENAI_DEPLOYMENT_NAME', 'mini-swe-agent-gpt5')}" - - result: BotRunResult = run_training( - repo_path=str(test_repo), - feedback="", - memory_dir=str(memory_dir), - model=model, - max_iterations=8, - timeout_in_seconds=600, - ) - - # Accept either a completed run, or a run that stopped only because it - # hit the (intentionally low) iteration cap - both prove the flow works. - acceptable_errors = (None, "Max iterations 8 reached") - assert result.status or result.error in acceptable_errors, ( - f"Training run failed unexpectedly: {result.error}" - ) - - # With only 8 iterations the agent may not fully finish the task, but - # it should still persist at least one memory file along the way. - memory_files = [f for f in memory_dir.rglob("*") if f.is_file()] - assert memory_files, ( - f"Expected at least one memory file under {memory_dir}, found none" - ) From 790d1b5646f96c7507af498b6580db83732957fe Mon Sep 17 00:00:00 2001 From: Kavya Sree Kaitepalli Date: Mon, 31 Aug 2026 05:55:40 +0000 Subject: [PATCH 7/8] refactor: remove CLI and related tests for training agent; resolving comments --- src/microbots/auto_memory/training/cli.py | 51 ---- src/microbots/auto_memory/training/runner.py | 174 ++--------- test/auto_memory/training/test_cli.py | 100 ------- test/auto_memory/training/test_runner.py | 290 +++++-------------- 4 files changed, 89 insertions(+), 526 deletions(-) delete mode 100644 src/microbots/auto_memory/training/cli.py delete mode 100644 test/auto_memory/training/test_cli.py diff --git a/src/microbots/auto_memory/training/cli.py b/src/microbots/auto_memory/training/cli.py deleted file mode 100644 index f6a4436..0000000 --- a/src/microbots/auto_memory/training/cli.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Command-line interface for the repository training agent.""" - -import argparse -import logging - -from .runner import run_training_loop - -logger = logging.getLogger(__name__) - - -def parse_args(): - """Parse command-line arguments for a training run. - - Returns - ------- - argparse.Namespace - Parsed command-line arguments. - """ - parser = argparse.ArgumentParser(description="Training agent") - parser.add_argument("--repo", required=True, help="Path or URL to the repo to learn from") - parser.add_argument("--feedback", default="", help="Optional feedback text (can be empty)") - parser.add_argument("--memory-dir", default="./memory", help="Directory to store the memory file") - parser.add_argument("--model", required=True, help="Model identifier, e.g. azure-openai/gpt-4o") - parser.add_argument( - "--iterations", - type=int, - default=1, - help="Number of training passes to run in sequence over the same memory_dir", - ) - return parser.parse_args() - - -def main(): - """Run repository training from command-line arguments.""" - logging.basicConfig(level=logging.INFO) - args = parse_args() - - result = run_training_loop( - repo_path=args.repo, - feedback=args.feedback, - memory_dir=args.memory_dir, - model=args.model, - iterations=args.iterations, - ) - if not result.status: - logger.error("training loop finished with failure: %s", result.error) - - -if __name__ == "__main__": - main() - diff --git a/src/microbots/auto_memory/training/runner.py b/src/microbots/auto_memory/training/runner.py index 1dc1f99..94f6700 100644 --- a/src/microbots/auto_memory/training/runner.py +++ b/src/microbots/auto_memory/training/runner.py @@ -1,76 +1,13 @@ """Run repository training with a reading bot and persistent memory tool.""" -import logging -import re -import shutil -import subprocess -import tempfile from pathlib import Path -from urllib.parse import urlparse from microbots.bot.ReadingBot import ReadingBot from microbots.tools.tool_definitions.memory_tool import MemoryTool from microbots.MicroBot import BotRunResult -logger = logging.getLogger(__name__) - _INSTRUCTIONS_PATH = Path(__file__).parent / "training_instructions.md" -# Matches SCP-style SSH remotes, e.g. "git@github.com:org/repo" or -# "git@github.com:org/repo.git". urlparse() alone can't detect these since -# they have no scheme, and they don't always end in ".git". -_SCP_STYLE_RE = re.compile(r"^[\w.\-]+@[\w.\-]+:") - - -def _is_git_url(repo: str) -> bool: - """Determine whether a repository reference looks like a Git remote. - - Parameters - ---------- - repo : str - Repository path or URL. - - Returns - ------- - bool - ``True`` when the reference looks like a Git remote. - """ - parsed = urlparse(repo) - return ( - parsed.scheme in ("http", "https", "git", "ssh") - or repo.endswith(".git") - or bool(_SCP_STYLE_RE.match(repo)) - ) - -def _prepare_source_dir(repo: str, workdir: Path) -> Path: - """Ensure a local directory exists for the agent to read from. - - Parameters - ---------- - repo : str - Local repository path or Git URL. - workdir : pathlib.Path - Working directory in which a remote repository can be cloned. - - Returns - ------- - pathlib.Path - Existing local repository path or the path to the cloned repository. - """ - if not _is_git_url(repo): - return Path(repo) - - dest = workdir / "source" - if dest.exists(): - return dest # reuse existing clone across iterations - - dest.parent.mkdir(parents=True, exist_ok=True) - subprocess.run( - ["git", "clone", "--depth", "1", repo, str(dest)], - check=True, - ) - return dest - def run_training( repo_path: str, @@ -85,7 +22,11 @@ def run_training( Parameters ---------- repo_path : str - Local repository path or Git URL to learn from. + Absolute path to a local repository to learn from. The caller + (e.g. the orchestrator or an ``EvalTask``'s ``setup``) is + responsible for ensuring this is a ready, checked-out local + directory; this function does not clone or otherwise manage + the repo. feedback : str Optional feedback to include in the training prompt. memory_dir : str @@ -102,95 +43,18 @@ def run_training( microbots.MicroBot.BotRunResult Result of the training bot run. """ + instructions = _INSTRUCTIONS_PATH.read_text(encoding="utf-8") + feedback_section = feedback.strip() or "No feedback provided for this run." + prompt = f"{instructions}\n\n## Feedback\n{feedback_section}\n" + + bot = ReadingBot( + model=model, + folder_to_mount=repo_path, + additional_tools=[MemoryTool(memory_dir=memory_dir)], + ) - workdir = Path(tempfile.mkdtemp(prefix="training_workdir_")) - try: - source_dir = _prepare_source_dir(repo_path, workdir) - - instructions = _INSTRUCTIONS_PATH.read_text(encoding="utf-8") - feedback_section = feedback.strip() or "No feedback provided for this run." - prompt = f"{instructions}\n\n## Feedback\n{feedback_section}\n" - - bot = ReadingBot( - model=model, - folder_to_mount=str(source_dir), - additional_tools=[MemoryTool(memory_dir=memory_dir)], - ) - - return bot.run( - prompt, - max_iterations=max_iterations, - timeout_in_seconds=timeout_in_seconds, - ) - finally: - # Only remove workdir when we actually cloned into it; a local - # repo_path is used directly and must never be deleted here. - if _is_git_url(repo_path): - logger.info("Cleaning up training workdir %s", workdir) - shutil.rmtree(workdir, ignore_errors=True) - - -def run_training_loop( - repo_path: str, - feedback: str, - memory_dir: str, - model: str, - iterations: int = 1, - max_iterations: int = 20, - timeout_in_seconds: int = 600, -) -> BotRunResult: - """Run repeated training passes over the same memory directory. - - Each pass reuses ``memory_dir`` from the previous one, allowing the - agent to progressively refine its notes. The real output of this - function is the accumulated state on disk under ``memory_dir``; the - returned :class:`~microbots.MicroBot.BotRunResult` is only a health - signal for the *last* iteration, since individual iteration failures - are logged and do not stop the loop. - - Parameters - ---------- - repo_path : str - Local repository path or Git URL to learn from. - feedback : str - Optional feedback to include in the training prompt, applied to - every iteration. - memory_dir : str - Directory in which the memory tool stores its memory, shared and - accumulated across all iterations. - model : str - Model identifier used by the reading bot. - iterations : int, default=1 - Number of training passes to run in sequence. - max_iterations : int, default=20 - Maximum number of bot iterations per training pass. - timeout_in_seconds : int, default=600 - Maximum duration of each training pass in seconds. - - Returns - ------- - microbots.MicroBot.BotRunResult - Result of the last training pass. - """ - result: BotRunResult | None = None - for iteration in range(1, iterations + 1): - logger.info("training iteration %d/%d starting", iteration, iterations) - result = run_training( - repo_path=repo_path, - feedback=feedback, - memory_dir=memory_dir, - model=model, - max_iterations=max_iterations, - timeout_in_seconds=timeout_in_seconds, - ) - logger.info( - "training iteration %d/%d: status=%s memory_dir=%s", - iteration, - iterations, - result.status, - memory_dir, - ) - if not result.status: - logger.error("iteration %d/%d error=%s", iteration, iterations, result.error) - - return result \ No newline at end of file + return bot.run( + prompt, + max_iterations=max_iterations, + timeout_in_seconds=timeout_in_seconds, + ) \ No newline at end of file diff --git a/test/auto_memory/training/test_cli.py b/test/auto_memory/training/test_cli.py deleted file mode 100644 index 55bb254..0000000 --- a/test/auto_memory/training/test_cli.py +++ /dev/null @@ -1,100 +0,0 @@ -"""Unit tests for microbots.auto_memory.training.cli.""" - -import sys -from unittest.mock import patch - -import pytest - -from microbots.auto_memory.training.cli import main, parse_args -from microbots.MicroBot import BotRunResult - - -@pytest.mark.unit -def test_parse_args_defaults(): - argv = [ - "cli.py", - "--repo", - "/some/repo", - "--model", - "azure-openai/gpt-4o", - ] - with patch.object(sys, "argv", argv): - args = parse_args() - - assert args.repo == "/some/repo" - assert args.model == "azure-openai/gpt-4o" - assert args.feedback == "" - assert args.memory_dir == "./memory" - assert args.iterations == 1 - - -@pytest.mark.unit -def test_main_calls_run_training_loop_with_parsed_args(): - argv = [ - "cli.py", - "--repo", - "/some/repo", - "--feedback", - "some feedback", - "--memory-dir", - "/some/memory", - "--model", - "azure-openai/gpt-4o", - "--iterations", - "3", - ] - fake_result = BotRunResult(status=True, result="ok", error=None) - - with patch.object(sys, "argv", argv), patch( - "microbots.auto_memory.training.cli.run_training_loop", - return_value=fake_result, - ) as mock_run_training_loop: - main() - - mock_run_training_loop.assert_called_once_with( - repo_path="/some/repo", - feedback="some feedback", - memory_dir="/some/memory", - model="azure-openai/gpt-4o", - iterations=3, - ) - - -@pytest.mark.unit -def test_main_logs_error_when_loop_fails_on_last_iteration(caplog): - argv = [ - "cli.py", - "--repo", - "/some/repo", - "--model", - "azure-openai/gpt-4o", - ] - fake_result = BotRunResult(status=False, result=None, error="boom") - - with patch.object(sys, "argv", argv), patch( - "microbots.auto_memory.training.cli.run_training_loop", - return_value=fake_result, - ), caplog.at_level("ERROR", logger="microbots.auto_memory.training.cli"): - main() - - assert "boom" in caplog.text - - -@pytest.mark.unit -def test_main_does_not_log_error_when_loop_succeeds(caplog): - argv = [ - "cli.py", - "--repo", - "/some/repo", - "--model", - "azure-openai/gpt-4o", - ] - fake_result = BotRunResult(status=True, result="ok", error=None) - - with patch.object(sys, "argv", argv), patch( - "microbots.auto_memory.training.cli.run_training_loop", - return_value=fake_result, - ), caplog.at_level("ERROR", logger="microbots.auto_memory.training.cli"): - main() - - assert caplog.text == "" diff --git a/test/auto_memory/training/test_runner.py b/test/auto_memory/training/test_runner.py index 003756b..65995dd 100644 --- a/test/auto_memory/training/test_runner.py +++ b/test/auto_memory/training/test_runner.py @@ -1,108 +1,25 @@ """Unit tests for microbots.auto_memory.training.runner. -All external dependencies (subprocess/git, ReadingBot, MemoryTool) are -mocked so these tests run without Docker, network access, or an LLM. -The one exception is test_run_training_end_to_end, which is a real -integration test (marked accordingly) that exercises Docker and a live -model deployment. +All external dependencies (ReadingBot, MemoryTool) are mocked so these +tests run without Docker, network access, or an LLM. The one exception +is test_run_training_end_to_end, which is a real integration test +(marked accordingly) that exercises Docker and a live model deployment. + +runner.py no longer manages repo cloning or looping: repo_path is +always assumed to be a ready local directory prepared by the caller +(e.g. the orchestrator or an EvalTask's setup), and iterating training +passes is the orchestrator's responsibility. """ -from pathlib import Path -from subprocess import CalledProcessError -import tempfile +import os from unittest.mock import MagicMock, patch import pytest -from microbots.auto_memory.training.runner import ( - _is_git_url, - _prepare_source_dir, - run_training, - run_training_loop, -) +from microbots.auto_memory.training.runner import run_training from microbots.MicroBot import BotRunResult -# --------------------------------------------------------------------------- -# _is_git_url -# --------------------------------------------------------------------------- - -@pytest.mark.unit -@pytest.mark.parametrize( - "repo, expected", - [ - ("https://github.com/pytest-dev/pytest.git", True), - ("git@github.com:pytest-dev/pytest.git", True), - ("ssh://git@github.com/pytest-dev/pytest.git", True), - ("/home/user/some/local/repo", False), - ("some-local-dir-without-scheme", False), - ("relative/local/path.git", True), # ends with .git -> treated as git - ("git@github.com:pytest-dev/pytest", True), # SCP-style, no .git suffix - ], -) -def test_is_git_url(repo, expected): - assert _is_git_url(repo) is expected - - -# --------------------------------------------------------------------------- -# _prepare_source_dir -# --------------------------------------------------------------------------- - -@pytest.mark.unit -def test_prepare_source_dir_local_path_passthrough(tmp_path): - local_repo = tmp_path / "local_repo" - local_repo.mkdir() - - with patch("microbots.auto_memory.training.runner.subprocess.run") as mock_run: - result = _prepare_source_dir(str(local_repo), tmp_path / "workdir") - - assert result == Path(local_repo) - mock_run.assert_not_called() - - -@pytest.mark.unit -def test_prepare_source_dir_clones_git_url(tmp_path): - workdir = tmp_path / "workdir" - repo_url = "https://github.com/pytest-dev/pytest.git" - expected_dest = workdir / "source" - - with patch("microbots.auto_memory.training.runner.subprocess.run") as mock_run: - result = _prepare_source_dir(repo_url, workdir) - - mock_run.assert_called_once_with( - ["git", "clone", "--depth", "1", repo_url, str(expected_dest)], - check=True, - ) - assert result == expected_dest - - -@pytest.mark.unit -def test_prepare_source_dir_reuses_existing_clone(tmp_path): - workdir = tmp_path / "workdir" - dest = workdir / "source" - dest.mkdir(parents=True) - repo_url = "https://github.com/pytest-dev/pytest.git" - - with patch("microbots.auto_memory.training.runner.subprocess.run") as mock_run: - result = _prepare_source_dir(repo_url, workdir) - - mock_run.assert_not_called() - assert result == dest - - -@pytest.mark.unit -def test_prepare_source_dir_clone_failure_propagates(tmp_path): - workdir = tmp_path / "workdir" - repo_url = "https://github.com/pytest-dev/pytest.git" - - with patch( - "microbots.auto_memory.training.runner.subprocess.run", - side_effect=CalledProcessError(returncode=1, cmd=["git", "clone"]), - ): - with pytest.raises(CalledProcessError): - _prepare_source_dir(repo_url, workdir) - - # --------------------------------------------------------------------------- # run_training # --------------------------------------------------------------------------- @@ -189,85 +106,68 @@ def test_run_training_passes_correct_args_to_reading_bot(tmp_path): _, kwargs = mock_reading_bot.call_args assert kwargs["model"] == "azure-openai/gpt-4o" + # repo_path is used directly as folder_to_mount now; no cloning/staging. assert kwargs["folder_to_mount"] == str(local_repo) assert kwargs["additional_tools"] == [mock_memory_tool_instance] @pytest.mark.unit -def test_run_training_returns_bot_result(tmp_path): +def test_run_training_passes_max_iterations_and_timeout_to_bot_run(tmp_path): local_repo = tmp_path / "repo" local_repo.mkdir() memory_dir = tmp_path / "memory" - expected_result = BotRunResult(status=True, result="done", error=None) mock_bot_instance = MagicMock() - mock_bot_instance.run.return_value = expected_result + mock_bot_instance.run.return_value = BotRunResult( + status=True, result="ok", error=None + ) with patch( "microbots.auto_memory.training.runner.ReadingBot", return_value=mock_bot_instance, ), patch("microbots.auto_memory.training.runner.MemoryTool"): - result = run_training( + run_training( repo_path=str(local_repo), feedback="", memory_dir=str(memory_dir), model="azure-openai/gpt-4o", + max_iterations=5, + timeout_in_seconds=42, ) - assert result is expected_result + _, kwargs = mock_bot_instance.run.call_args + assert kwargs["max_iterations"] == 5 + assert kwargs["timeout_in_seconds"] == 42 @pytest.mark.unit -def test_run_training_cleans_up_workdir_after_git_clone(tmp_path): - """When repo_path is a git URL, the temp workdir it clones into should - be removed once the run finishes.""" +def test_run_training_returns_bot_result(tmp_path): + local_repo = tmp_path / "repo" + local_repo.mkdir() memory_dir = tmp_path / "memory" - repo_url = "https://github.com/pytest-dev/pytest.git" - - created_workdirs = [] - - def fake_clone(cmd, check): - # cmd = ["git", "clone", "--depth", "1", repo_url, dest] - dest = Path(cmd[-1]) - dest.mkdir(parents=True, exist_ok=True) + expected_result = BotRunResult(status=True, result="done", error=None) mock_bot_instance = MagicMock() - mock_bot_instance.run.return_value = BotRunResult( - status=True, result="ok", error=None - ) - - real_mkdtemp = tempfile.mkdtemp - - def tracking_mkdtemp(*args, **kwargs): - path = real_mkdtemp(*args, **kwargs) - created_workdirs.append(Path(path)) - return path + mock_bot_instance.run.return_value = expected_result with patch( - "microbots.auto_memory.training.runner.subprocess.run", - side_effect=fake_clone, - ), patch( - "microbots.auto_memory.training.runner.tempfile.mkdtemp", - side_effect=tracking_mkdtemp, - ), patch( "microbots.auto_memory.training.runner.ReadingBot", return_value=mock_bot_instance, ), patch("microbots.auto_memory.training.runner.MemoryTool"): - run_training( - repo_path=repo_url, + result = run_training( + repo_path=str(local_repo), feedback="", memory_dir=str(memory_dir), model="azure-openai/gpt-4o", ) - assert len(created_workdirs) == 1 - assert not created_workdirs[0].exists() + assert result is expected_result @pytest.mark.unit -def test_run_training_keeps_local_repo_untouched(tmp_path): - """When repo_path is a local directory, it must never be deleted, - even though the (unused) temp workdir is still cleaned up.""" +def test_run_training_does_not_modify_repo_path(tmp_path): + """run_training must never delete/modify repo_path itself - it does + not own the repo's lifecycle anymore (no cloning, no cleanup).""" local_repo = tmp_path / "repo" local_repo.mkdir() (local_repo / "marker.txt").write_text("keep me") @@ -293,94 +193,44 @@ def test_run_training_keeps_local_repo_untouched(tmp_path): # --------------------------------------------------------------------------- -# run_training_loop +# End-to-end integration test (real Docker + real LLM deployment required) # --------------------------------------------------------------------------- -@pytest.mark.unit -def test_run_training_loop_calls_run_training_n_times_with_same_memory_dir(): - fake_result = BotRunResult(status=True, result="ok", error=None) - - with patch( - "microbots.auto_memory.training.runner.run_training", - return_value=fake_result, - ) as mock_run_training: - run_training_loop( - repo_path="/some/repo", - feedback="fb", - memory_dir="/some/memory", - model="azure-openai/gpt-4o", - iterations=3, - ) - - assert mock_run_training.call_count == 3 - for call in mock_run_training.call_args_list: - assert call.kwargs["memory_dir"] == "/some/memory" - assert call.kwargs["feedback"] == "fb" - assert call.kwargs["repo_path"] == "/some/repo" - assert call.kwargs["model"] == "azure-openai/gpt-4o" - - -@pytest.mark.unit -def test_run_training_loop_returns_last_result(): - results = [ - BotRunResult(status=True, result="first", error=None), - BotRunResult(status=False, result=None, error="second failed"), - BotRunResult(status=True, result="third", error=None), - ] - - with patch( - "microbots.auto_memory.training.runner.run_training", - side_effect=results, - ): - result = run_training_loop( - repo_path="/some/repo", - feedback="", - memory_dir="/some/memory", - model="azure-openai/gpt-4o", - iterations=3, - ) - - assert result is results[-1] - - -@pytest.mark.unit -def test_run_training_loop_continues_after_a_failed_iteration(): - """A failure on iteration N must not stop iteration N+1 from running.""" - results = [ - BotRunResult(status=False, result=None, error="boom"), - BotRunResult(status=True, result="ok", error=None), - ] - - with patch( - "microbots.auto_memory.training.runner.run_training", - side_effect=results, - ) as mock_run_training: - result = run_training_loop( - repo_path="/some/repo", - feedback="", - memory_dir="/some/memory", - model="azure-openai/gpt-4o", - iterations=2, - ) - - assert mock_run_training.call_count == 2 - assert result is results[-1] - - -@pytest.mark.unit -def test_run_training_loop_default_single_iteration(): - fake_result = BotRunResult(status=True, result="ok", error=None) +@pytest.mark.integration +@pytest.mark.slow +@pytest.mark.docker +def test_run_training_end_to_end(test_repo, tmp_path): + """Smoke-test the training flow against a small fixture repo. + + Requires Docker and a working model deployment (same env vars used by + test/bot/test_reading_bot.py). This is a smoke test, not a + completion test: max_iterations is intentionally kept small so it's + fast to run locally. It only asserts the flow executes end-to-end + (mount -> bot run) without asserting the agent reached task_done, + since that may need more iterations than we want to spend here. + """ + memory_dir = tmp_path / "memory" + model = f"azure-openai/{os.getenv('AZURE_OPENAI_DEPLOYMENT_NAME', 'mini-swe-agent-gpt5')}" + + result: BotRunResult = run_training( + repo_path=str(test_repo), + feedback="", + memory_dir=str(memory_dir), + model=model, + max_iterations=8, + timeout_in_seconds=600, + ) - with patch( - "microbots.auto_memory.training.runner.run_training", - return_value=fake_result, - ) as mock_run_training: - result = run_training_loop( - repo_path="/some/repo", - feedback="", - memory_dir="/some/memory", - model="azure-openai/gpt-4o", - ) + # Accept either a completed run, or a run that stopped only because it + # hit the (intentionally low) iteration cap - both prove the flow works. + acceptable_errors = (None, "Max iterations 8 reached") + assert result.status or result.error in acceptable_errors, ( + f"Training run failed unexpectedly: {result.error}" + ) - mock_run_training.assert_called_once() - assert result is fake_result + # With only 8 iterations the agent may not fully finish the task, but + # it should still persist at least one memory file along the way. + memory_files = [f for f in memory_dir.rglob("*") if f.is_file()] + assert memory_files, ( + f"Expected at least one memory file under {memory_dir}, found none" + ) From e02abaff150f9175d8ad4cb16591a63dc953e6ca Mon Sep 17 00:00:00 2001 From: Kavya Sree Kaitepalli Date: Mon, 31 Aug 2026 08:08:24 +0000 Subject: [PATCH 8/8] refactor: remove end-to-end integration test for training flow and clean up comments --- test/auto_memory/training/test_runner.py | 49 +----------------------- 1 file changed, 1 insertion(+), 48 deletions(-) diff --git a/test/auto_memory/training/test_runner.py b/test/auto_memory/training/test_runner.py index 65995dd..ed15e55 100644 --- a/test/auto_memory/training/test_runner.py +++ b/test/auto_memory/training/test_runner.py @@ -1,9 +1,7 @@ """Unit tests for microbots.auto_memory.training.runner. All external dependencies (ReadingBot, MemoryTool) are mocked so these -tests run without Docker, network access, or an LLM. The one exception -is test_run_training_end_to_end, which is a real integration test -(marked accordingly) that exercises Docker and a live model deployment. +tests run without Docker, network access, or an LLM. runner.py no longer manages repo cloning or looping: repo_path is always assumed to be a ready local directory prepared by the caller @@ -11,7 +9,6 @@ passes is the orchestrator's responsibility. """ -import os from unittest.mock import MagicMock, patch import pytest @@ -190,47 +187,3 @@ def test_run_training_does_not_modify_repo_path(tmp_path): ) assert (local_repo / "marker.txt").exists() - - -# --------------------------------------------------------------------------- -# End-to-end integration test (real Docker + real LLM deployment required) -# --------------------------------------------------------------------------- - -@pytest.mark.integration -@pytest.mark.slow -@pytest.mark.docker -def test_run_training_end_to_end(test_repo, tmp_path): - """Smoke-test the training flow against a small fixture repo. - - Requires Docker and a working model deployment (same env vars used by - test/bot/test_reading_bot.py). This is a smoke test, not a - completion test: max_iterations is intentionally kept small so it's - fast to run locally. It only asserts the flow executes end-to-end - (mount -> bot run) without asserting the agent reached task_done, - since that may need more iterations than we want to spend here. - """ - memory_dir = tmp_path / "memory" - model = f"azure-openai/{os.getenv('AZURE_OPENAI_DEPLOYMENT_NAME', 'mini-swe-agent-gpt5')}" - - result: BotRunResult = run_training( - repo_path=str(test_repo), - feedback="", - memory_dir=str(memory_dir), - model=model, - max_iterations=8, - timeout_in_seconds=600, - ) - - # Accept either a completed run, or a run that stopped only because it - # hit the (intentionally low) iteration cap - both prove the flow works. - acceptable_errors = (None, "Max iterations 8 reached") - assert result.status or result.error in acceptable_errors, ( - f"Training run failed unexpectedly: {result.error}" - ) - - # With only 8 iterations the agent may not fully finish the task, but - # it should still persist at least one memory file along the way. - memory_files = [f for f in memory_dir.rglob("*") if f.is_file()] - assert memory_files, ( - f"Expected at least one memory file under {memory_dir}, found none" - )