implement task evaluation framework

.github/workflows/test.yml

@@ -165,7 +165,7 @@ jobs:
 
       - name: Install package in development mode
         run: |
-          pip install -e .
+          pip install -e ".[training]"
 
       - name: Install GitHub Copilot SDK dependencies for GHCP tests
         if: matrix.test-type == 'ghcp'

pyproject.toml

@@ -25,6 +25,7 @@ requires-python = ">=3.11"
 ghcp = ["github-copilot-sdk==0.3.0"]
 azure_ad = ["azure-identity>=1.15.0"]
 dev = ["pre-commit>=3.7", "numpydoc>=1.8"]
+training = ["datasets==4.5.0", "swebench==4.1.0"]
 
 [tool.setuptools.dynamic]
 dependencies = { file = ["requirements.txt"] }

requirements.txt

@@ -10,36 +10,26 @@ certifi==2025.8.3
 charset-normalizer==3.4.3
 click==8.3.0
 coverage==7.11.3
-datasets==4.5.0
-dill==0.4.0
 distro==1.9.0
 docker==7.1.0
 docstring_parser==0.17.0
 fastapi==0.116.1
-filelock==3.20.3
 frozenlist==1.7.0
-fsspec==2025.10.0
 h11==0.16.0
-hf-xet==1.2.0
 httpcore==1.0.9
 httpx==0.28.1
-huggingface_hub==1.3.2
 idna==3.10
 iniconfig==2.1.0
 jiter==0.11.0
 markdown-it-py==4.0.0
 mdurl==0.1.2
 multidict==6.6.4
-multiprocess==0.70.18
-numpy==1.26.4
 openai==1.107.3
 packaging==25.0
-pandas==3.0.0
 pexpect==4.9.0
 pluggy==1.6.0
 propcache==0.3.2
 ptyprocess==0.7.0
-pyarrow==23.0.0
 pydantic==2.11.9
 pydantic_core==2.33.2
 Pygments==2.19.2
@@ -62,5 +52,4 @@ typing-inspection==0.4.1
 typing_extensions==4.15.0
 urllib3==2.5.0
 uvicorn==0.35.0
-xxhash==3.6.0
 yarl==1.20.1

src/microbots/auto_memory/__init__.py

@@ -0,0 +1,8 @@
+"""Train <-> eval loop for repo-learning agents.
+
+Re-exports the public task, outcome, and orchestrator types used to define
+an evaluation task and run it in a loop against a training agent.
+"""
+
+from .evalTask import CallbackResult, EvalOutcome, EvalTask
+from .orchestrator import LoopResult, run_train_eval_loop

src/microbots/auto_memory/cli.py

@@ -0,0 +1,92 @@
+"""Command-line entry point for the auto-memory train/eval loop.
+
+Two modes, selected by ``--task``:
+
+- ``--task <name>`` given: run the full train <-> eval loop for that
+  task.
+- ``--task`` omitted: train only, no eval task, with empty feedback.
+
+Both modes are dispatched via ``orchestrator.run``.
+"""
+
+import argparse
+import logging
+from pathlib import Path
+
+from microbots.auto_memory.orchestrator import run
+from microbots.auto_memory.task_registry import TASK_REGISTRY, discover_tasks
+from microbots.auto_memory.workdir import load_config, require_workdir, resolve_workdir
+
+logger = logging.getLogger(__name__)
+
+# Import every task module so their @register_task decorators fire.
+discover_tasks()
+
+def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
+    """Parse the CLI's top-level args.
+
+    Task-specific values (e.g. an eval task's instance ID) are not
+    parsed here; they come from the workdir's config file instead.
+
+    Parameters
+    ----------
+    argv : list[str] | None
+        Args to parse. Defaults to ``sys.argv[1:]`` when ``None``.
+
+    Returns
+    -------
+    argparse.Namespace
+        The parsed args.
+    """
+    parser = argparse.ArgumentParser(description="Run the auto-memory train/eval loop.")
+    parser.add_argument("--model", required=True, help='Model, e.g. "azure-openai/gpt-5.5".')
+    parser.add_argument(
+        "--workdir",
+        help="Directory holding this run's files (repo clone, logs, memory, "
+        "config). Defaults to './workdir' relative to the current directory.",
+    )
+    parser.add_argument(
+        "--task",
+        choices=sorted(TASK_REGISTRY),
+        help="Eval task to run. Omit to only run training, with no eval task.",
+    )
+    parser.add_argument("--max-rounds", type=int, default=5)
+    parser.add_argument("--training-iterations", type=int, default=10)
+
+    return parser.parse_args(argv)
+
+def main(argv: list[str] | None = None) -> None:
+    """CLI entry point: run training only, or the full train/eval loop.
+
+    Parameters
+    ----------
+    argv : list[str] | None
+        Args to parse. Defaults to ``sys.argv[1:]`` when ``None``.
+    """
+    args = parse_args(argv)
+
+    workdir = Path(args.workdir) if args.workdir else resolve_workdir()
+    require_workdir(workdir)
+
+    config = load_config(workdir)
+    tasks = (
+        TASK_REGISTRY[args.task].from_config(config.get("task_args", {}))
+        if args.task
+        else [None]
+    )
+    for task in tasks:
+        result = run(
+            workdir=workdir,
+            model=args.model,
+            task=task,
+            max_rounds=args.max_rounds,
+            training_iterations=args.training_iterations,
+            config=config,
+        )
+        if result is not None:
+            logger.info(
+                "task=%s passed=%s rounds_run=%d", args.task, result.passed, result.rounds_run
+            )
+
+if __name__ == "__main__":
+    main()