diff --git a/src/microbots/auto_memory/training/__init__.py b/src/microbots/auto_memory/training/__init__.py new file mode 100644 index 0000000..fbdc5c1 --- /dev/null +++ b/src/microbots/auto_memory/training/__init__.py @@ -0,0 +1,3 @@ +"""Utilities for training an agent to build repository memory.""" + +from .runner import run_training \ No newline at end of file diff --git a/src/microbots/auto_memory/training/runner.py b/src/microbots/auto_memory/training/runner.py new file mode 100644 index 0000000..94f6700 --- /dev/null +++ b/src/microbots/auto_memory/training/runner.py @@ -0,0 +1,60 @@ +"""Run repository training with a reading bot and persistent memory tool.""" + +from pathlib import Path + +from microbots.bot.ReadingBot import ReadingBot +from microbots.tools.tool_definitions.memory_tool import MemoryTool +from microbots.MicroBot import BotRunResult + +_INSTRUCTIONS_PATH = Path(__file__).parent / "training_instructions.md" + + +def run_training( + repo_path: str, + feedback: str, + memory_dir: str, + model: str, + max_iterations: int = 20, + timeout_in_seconds: int = 600, +) -> BotRunResult: + """Run one training pass over a repository and update its memory. + + Parameters + ---------- + repo_path : str + Absolute path to a local repository to learn from. The caller + (e.g. the orchestrator or an ``EvalTask``'s ``setup``) is + responsible for ensuring this is a ready, checked-out local + directory; this function does not clone or otherwise manage + the repo. + feedback : str + Optional feedback to include in the training prompt. + memory_dir : str + Directory in which the memory tool stores its memory. + model : str + Model identifier used by the reading bot. + max_iterations : int, default=20 + Maximum number of bot iterations. + timeout_in_seconds : int, default=600 + Maximum duration of the bot run in seconds. + + Returns + ------- + microbots.MicroBot.BotRunResult + Result of the training bot run. + """ + instructions = _INSTRUCTIONS_PATH.read_text(encoding="utf-8") + feedback_section = feedback.strip() or "No feedback provided for this run." + prompt = f"{instructions}\n\n## Feedback\n{feedback_section}\n" + + bot = ReadingBot( + model=model, + folder_to_mount=repo_path, + additional_tools=[MemoryTool(memory_dir=memory_dir)], + ) + + return bot.run( + prompt, + max_iterations=max_iterations, + timeout_in_seconds=timeout_in_seconds, + ) \ No newline at end of file diff --git a/src/microbots/auto_memory/training/training_instructions.md b/src/microbots/auto_memory/training/training_instructions.md new file mode 100644 index 0000000..a202ec3 --- /dev/null +++ b/src/microbots/auto_memory/training/training_instructions.md @@ -0,0 +1,104 @@ +# Repo-Learning Agent Instructions + +You are a **package-maintainer agent** in a training phase. Your **only** job +is to **learn the repository** and write down what you learn as durable notes +in memory. You are not here to fix bugs, implement features, close tickets, +land patches, or make any change to the repository itself. + +A future evaluation loop will reuse the notes you leave behind. If it isn't in +memory, it doesn't exist. Optimise every action for "what will the next agent, +starting cold, need in order to act as maintainer of this repo?" + +--- + +## Mission + +For the repository under study, build up a **maintainer's mental model** and +persist it to `/memories/` using the `memory` tool. + +--- + +## Memory Protocol (non-negotiable) + +You have a `memory` tool that persists files under `/memories/`. Follow this +protocol every iteration: + +1. **Always start with** `memory view /memories` to see what prior iterations + already learned. Do not re-derive facts that are already recorded. +2. **Read before you write.** If a note already covers the area you're + exploring, extend or correct it instead of creating a parallel note. +3. **Write as you go.** Record each non-trivial finding immediately, in the + iteration you discovered it — do not batch discoveries until "the end". +4. **Cite sources.** Every claim should be traceable to a file path (and, when + useful, a symbol or line range) or an exact command + observed output. +5. **Prefer facts over prose.** Short bullets, tables, and code snippets beat + paragraphs. Notes are read by another agent, not a human reviewer. +6. **Keep memory tidy.** Rename vague files, delete stale ones, and merge + duplicates. A messy `/memories/` is worse than a small one. +7. **Never invent.** If you don't know, say so and (if possible) record the + next investigation step. Speculation poisons the next agent. + +Choose your own file structure inside `/memories/`. Organise it in +whatever way best fits the repo you are studying — just keep it discoverable, +non-duplicative, and easy for a cold-start agent to navigate. + +--- + +## Handling Feedback + +Each run may include a `## Feedback` section appended after these +instructions. This feedback comes from a prior evaluation attempt that +**failed** using the memory notes as they currently stand. + +1. **Treat feedback as the highest-priority gap.** It is direct evidence that + something in `/memories/` was missing, wrong, or misleading — investigate + and correct it before exploring anything else this iteration. +2. **Read the current memory first.** Check whether the failure relates to an + existing note (it was incomplete or incorrect) or a missing note (the area + was never covered). +3. **Update memory to close the gap**, not just to acknowledge the feedback. + Add the missing fact, correct the wrong one, or add a caveat/edge case + that explains why the previous approach failed. +4. **If feedback is empty or missing** (`"No feedback provided for this run."`), + there is no prior failure to address — proceed with the normal working + loop below and continue building out the maintainer mental model. +5. **Never leave feedback unaddressed.** If you cannot fully resolve the + underlying gap this iteration, record what you learned and what remains + unresolved as an explicit open question in memory, so the next iteration + picks it up. + +--- + +## Working Loop + +For each iteration: + +1. `memory view /memories` — recover prior state. +2. Pick the **highest-value gap** in the maintainer mental model above. +3. Investigate read-only: browse code, inspect tests, and run read-only + commands (e.g. listing files, viewing history, running an existing test + suite to observe behaviour). Do **not** modify repository files. +4. Record findings into the appropriate memory file(s), creating or + reorganising files as needed. +5. Before ending the iteration, do a final `memory view /memories` sanity + check: is your latest finding actually saved, cited, and discoverable? + +--- + +## What NOT to Record + +- Raw dumps of large files. Summarise and link by path instead. +- Transient reasoning ("I'm going to look at X next") — only keep it if it + survives the iteration as a real open question. +- Anything you are only guessing. Mark uncertainty explicitly or omit it. +- Secrets, tokens, or environment-specific absolute paths that won't + generalise to the next agent's machine. + +--- + +## Definition of Done (per iteration) + +An iteration is "done" when `/memories/` is strictly more useful to a +cold-start maintainer than it was when the iteration began — new facts +added, stale facts corrected or removed. If memory did not improve, the +iteration is not done. Update memory, then stop. \ No newline at end of file diff --git a/test/auto_memory/training/test_runner.py b/test/auto_memory/training/test_runner.py new file mode 100644 index 0000000..ed15e55 --- /dev/null +++ b/test/auto_memory/training/test_runner.py @@ -0,0 +1,189 @@ +"""Unit tests for microbots.auto_memory.training.runner. + +All external dependencies (ReadingBot, MemoryTool) are mocked so these +tests run without Docker, network access, or an LLM. + +runner.py no longer manages repo cloning or looping: repo_path is +always assumed to be a ready local directory prepared by the caller +(e.g. the orchestrator or an EvalTask's setup), and iterating training +passes is the orchestrator's responsibility. +""" + +from unittest.mock import MagicMock, patch + +import pytest + +from microbots.auto_memory.training.runner import run_training +from microbots.MicroBot import BotRunResult + + +# --------------------------------------------------------------------------- +# run_training +# --------------------------------------------------------------------------- + +@pytest.mark.unit +def test_run_training_prompt_includes_feedback(tmp_path): + local_repo = tmp_path / "repo" + local_repo.mkdir() + memory_dir = tmp_path / "memory" + + mock_bot_instance = MagicMock() + mock_bot_instance.run.return_value = BotRunResult( + status=True, result="ok", error=None + ) + + with patch( + "microbots.auto_memory.training.runner.ReadingBot", + return_value=mock_bot_instance, + ), patch("microbots.auto_memory.training.runner.MemoryTool"): + run_training( + repo_path=str(local_repo), + feedback="Focus on error handling paths.", + memory_dir=str(memory_dir), + model="azure-openai/gpt-4o", + ) + + prompt_arg = mock_bot_instance.run.call_args.args[0] + assert "Focus on error handling paths." in prompt_arg + + +@pytest.mark.unit +def test_run_training_prompt_handles_empty_feedback(tmp_path): + local_repo = tmp_path / "repo" + local_repo.mkdir() + memory_dir = tmp_path / "memory" + + mock_bot_instance = MagicMock() + mock_bot_instance.run.return_value = BotRunResult( + status=True, result="ok", error=None + ) + + with patch( + "microbots.auto_memory.training.runner.ReadingBot", + return_value=mock_bot_instance, + ), patch("microbots.auto_memory.training.runner.MemoryTool"): + run_training( + repo_path=str(local_repo), + feedback="", + memory_dir=str(memory_dir), + model="azure-openai/gpt-4o", + ) + + prompt_arg = mock_bot_instance.run.call_args.args[0] + assert "No feedback provided for this run." in prompt_arg + + +@pytest.mark.unit +def test_run_training_passes_correct_args_to_reading_bot(tmp_path): + local_repo = tmp_path / "repo" + local_repo.mkdir() + memory_dir = tmp_path / "memory" + + mock_bot_instance = MagicMock() + mock_bot_instance.run.return_value = BotRunResult( + status=True, result="ok", error=None + ) + mock_memory_tool_instance = MagicMock() + + with patch( + "microbots.auto_memory.training.runner.ReadingBot", + return_value=mock_bot_instance, + ) as mock_reading_bot, patch( + "microbots.auto_memory.training.runner.MemoryTool", + return_value=mock_memory_tool_instance, + ) as mock_memory_tool: + run_training( + repo_path=str(local_repo), + feedback="", + memory_dir=str(memory_dir), + model="azure-openai/gpt-4o", + ) + + mock_memory_tool.assert_called_once_with(memory_dir=str(memory_dir)) + + _, kwargs = mock_reading_bot.call_args + assert kwargs["model"] == "azure-openai/gpt-4o" + # repo_path is used directly as folder_to_mount now; no cloning/staging. + assert kwargs["folder_to_mount"] == str(local_repo) + assert kwargs["additional_tools"] == [mock_memory_tool_instance] + + +@pytest.mark.unit +def test_run_training_passes_max_iterations_and_timeout_to_bot_run(tmp_path): + local_repo = tmp_path / "repo" + local_repo.mkdir() + memory_dir = tmp_path / "memory" + + mock_bot_instance = MagicMock() + mock_bot_instance.run.return_value = BotRunResult( + status=True, result="ok", error=None + ) + + with patch( + "microbots.auto_memory.training.runner.ReadingBot", + return_value=mock_bot_instance, + ), patch("microbots.auto_memory.training.runner.MemoryTool"): + run_training( + repo_path=str(local_repo), + feedback="", + memory_dir=str(memory_dir), + model="azure-openai/gpt-4o", + max_iterations=5, + timeout_in_seconds=42, + ) + + _, kwargs = mock_bot_instance.run.call_args + assert kwargs["max_iterations"] == 5 + assert kwargs["timeout_in_seconds"] == 42 + + +@pytest.mark.unit +def test_run_training_returns_bot_result(tmp_path): + local_repo = tmp_path / "repo" + local_repo.mkdir() + memory_dir = tmp_path / "memory" + + expected_result = BotRunResult(status=True, result="done", error=None) + mock_bot_instance = MagicMock() + mock_bot_instance.run.return_value = expected_result + + with patch( + "microbots.auto_memory.training.runner.ReadingBot", + return_value=mock_bot_instance, + ), patch("microbots.auto_memory.training.runner.MemoryTool"): + result = run_training( + repo_path=str(local_repo), + feedback="", + memory_dir=str(memory_dir), + model="azure-openai/gpt-4o", + ) + + assert result is expected_result + + +@pytest.mark.unit +def test_run_training_does_not_modify_repo_path(tmp_path): + """run_training must never delete/modify repo_path itself - it does + not own the repo's lifecycle anymore (no cloning, no cleanup).""" + local_repo = tmp_path / "repo" + local_repo.mkdir() + (local_repo / "marker.txt").write_text("keep me") + memory_dir = tmp_path / "memory" + + mock_bot_instance = MagicMock() + mock_bot_instance.run.return_value = BotRunResult( + status=True, result="ok", error=None + ) + + with patch( + "microbots.auto_memory.training.runner.ReadingBot", + return_value=mock_bot_instance, + ), patch("microbots.auto_memory.training.runner.MemoryTool"): + run_training( + repo_path=str(local_repo), + feedback="", + memory_dir=str(memory_dir), + model="azure-openai/gpt-4o", + ) + + assert (local_repo / "marker.txt").exists()