diff --git a/.gitignore b/.gitignore index 1a470a1e..923a087f 100644 --- a/.gitignore +++ b/.gitignore @@ -223,6 +223,9 @@ marimo/_static/ marimo/_lsp/ __marimo__/ +# Singularity +*.sif + # >>>>>>>>> CUSTOM THINGS ON TOP OF FILE <<<<<<<<< docs/visualization/*.json diff --git a/README.md b/README.md index f9bc5b23..e84b11d9 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ Check out our [arXiv paper](https://arxiv.org/abs/2511.00839) and [website](http - **Python 3.11+** - **[uv](https://docs.astral.sh/uv/)** - Fast Python package manager -- **Docker** - For running games in containers +- **Docker** or **Singularity** - For running games in containers - **Git** ### Installation @@ -58,8 +58,9 @@ uv run python main.py configs/test/battlesnake.yaml ``` > [!TIP] -> CodeClash requires Docker to create execution environments. CodeClash was developed and tested on Ubuntu 22.04.4 LTS. +> CodeClash requires Docker or Singularity to create execution environments. CodeClash was developed and tested on Ubuntu 22.04.4 LTS. > The same instructions should work for Mac. If not, check out [#81](https://github.com/CodeClash-ai/CodeClash/issues/81) for an alternative solution. +> To use Singularity instead of Docker, set the environment variable: `export CODECLASH_RUNTIME=singularity`
Alternative: Using pip (not recommended) diff --git a/codeclash/agents/__init__.py b/codeclash/agents/__init__.py index 11d326ad..683afe08 100644 --- a/codeclash/agents/__init__.py +++ b/codeclash/agents/__init__.py @@ -1,12 +1,11 @@ -from minisweagent.environments.docker import DockerEnvironment - from codeclash.agents.dummy_agent import Dummy from codeclash.agents.minisweagent import MiniSWEAgent from codeclash.agents.player import Player from codeclash.agents.utils import GameContext +from codeclash.utils.environment import ContainerEnvironment -def get_agent(config: dict, game_context: GameContext, environment: DockerEnvironment) -> Player: +def get_agent(config: dict, game_context: GameContext, environment: ContainerEnvironment) -> Player: agents = { "dummy": Dummy, "mini": MiniSWEAgent, diff --git a/codeclash/agents/minisweagent.py b/codeclash/agents/minisweagent.py index 5adeb041..6b48d695 100644 --- a/codeclash/agents/minisweagent.py +++ b/codeclash/agents/minisweagent.py @@ -5,7 +5,6 @@ from minisweagent import Model from minisweagent.agents.default import AgentConfig, DefaultAgent -from minisweagent.environments.docker import DockerEnvironment from minisweagent.models import get_model from minisweagent.models.test_models import DeterministicModel from minisweagent.run.utils.save import save_traj @@ -13,7 +12,7 @@ from codeclash import REPO_DIR from codeclash.agents.player import Player from codeclash.agents.utils import GameContext -from codeclash.utils.environment import copy_to_container +from codeclash.utils.environment import ContainerEnvironment, copy_to_container os.environ["MSWEA_MODEL_RETRY_STOP_AFTER_ATTEMPT"] = "90" os.environ["LITELLM_MODEL_REGISTRY_PATH"] = str( @@ -30,7 +29,7 @@ class ClashAgent(DefaultAgent): def __init__( self, model: Model, - env: DockerEnvironment, + env: ContainerEnvironment, *, logger: logging.Logger, config_class: Callable = AgentConfig, @@ -47,7 +46,7 @@ def add_message(self, role: str, content: str, **kwargs): class MiniSWEAgent(Player): """Player with agentic code editing capabilities""" - def __init__(self, config: dict, environment: DockerEnvironment, game_context: GameContext): + def __init__(self, config: dict, environment: ContainerEnvironment, game_context: GameContext): super().__init__(config, environment=environment, game_context=game_context) def run(self): diff --git a/codeclash/agents/player.py b/codeclash/agents/player.py index c6fd48f1..ceceea4b 100644 --- a/codeclash/agents/player.py +++ b/codeclash/agents/player.py @@ -5,12 +5,11 @@ from abc import ABC, abstractmethod from dotenv import load_dotenv -from minisweagent.environments.docker import DockerEnvironment from codeclash.agents.utils import GameContext from codeclash.constants import GH_ORG from codeclash.tournaments.utils.git_utils import extract_modified_code_file_paths_from_diff, filter_git_diff -from codeclash.utils.environment import assert_zero_exit_code, create_file_in_container +from codeclash.utils.environment import ContainerEnvironment, assert_zero_exit_code, create_file_in_container from codeclash.utils.log import get_logger load_dotenv() @@ -20,7 +19,7 @@ class Player(ABC): def __init__( self, config: dict, - environment: DockerEnvironment, + environment: ContainerEnvironment, game_context: GameContext, ) -> None: self.config = config @@ -170,7 +169,7 @@ def _get_commit_hash(self) -> str: self.environment.execute("git rev-parse HEAD"), logger=self.logger, ) - return out["output"].strip() + return out["output"].strip().splitlines()[-1] # take last line only to strip any Singularity warnings def _commit(self) -> None: """Commit changes to the agent's codebase.""" diff --git a/codeclash/arenas/arena.py b/codeclash/arenas/arena.py index 9bba83f6..a84a1720 100644 --- a/codeclash/arenas/arena.py +++ b/codeclash/arenas/arena.py @@ -8,14 +8,22 @@ from typing import Any from minisweagent.environments.docker import DockerEnvironment +from minisweagent.environments.singularity import SingularityEnvironment from codeclash.agents.player import Player from codeclash.constants import DIR_LOGS, DIR_WORK, GH_ORG, RESULT_TIE from codeclash.utils.aws import is_running_in_aws_batch, pull_game_container_aws_ecr -from codeclash.utils.environment import assert_zero_exit_code, copy_between_containers, copy_from_container +from codeclash.utils.environment import ContainerEnvironment, assert_zero_exit_code, copy_between_containers, copy_from_container from codeclash.utils.log import get_logger +def get_runtime() -> str: + """Get the container runtime from the CODECLASH_RUNTIME environment variable. + Defaults to 'docker' if not set. + """ + return os.environ.get("CODECLASH_RUNTIME", "docker").lower() + + class PlayerStats: def __init__(self, name: str): self.name = name @@ -99,8 +107,8 @@ def __init__(self, config: dict, *, tournament_id: str, local_output_dir: Path, self.log_env: Path = DIR_LOGS self.log_local: Path = local_output_dir self.logger = get_logger(self.name, log_path=self.log_local / "game.log", emoji="🏓") - self.environment: DockerEnvironment = self.get_environment() - """The running docker environment for executing the game""" + self.environment: ContainerEnvironment = self.get_environment() + """The running container environment for executing the game""" @property def game_config(self) -> dict: @@ -114,11 +122,51 @@ def game_id(self) -> str: def image_name(self) -> str: return f"codeclash/{self.name.lower()}" + @property + def sif_path(self) -> Path: + """Path to the Singularity .sif image file, located next to the arena definition.""" + arena_file = Path(inspect.getfile(self.__class__)) + return arena_file.parent / f"{self.name.lower()}.sif" + def build_image(self): """ - Build a Docker image for the game using the Dockerfile in the codebase. - If running in AWS, pull the image from the AWS Docker registry instead. + Build a container image for the game. + + For Docker: builds from the Dockerfile in the codebase. If running in AWS, + pulls the image from the AWS Docker registry instead. + For Singularity: builds a .sif file from the .def file if it doesn't already exist. """ + if get_runtime() == "singularity": + self._build_singularity_image() + else: + self._build_docker_image() + + def _build_singularity_image(self): + """Build a Singularity .sif image from the .def file.""" + if self.sif_path.exists(): + self.logger.debug(f"Singularity image {self.sif_path} already exists") + return + + arena_file = Path(inspect.getfile(self.__class__)) + def_path = arena_file.parent / f"{self.name}.def" + if not def_path.exists(): + raise RuntimeError(f"Singularity definition file not found: {def_path}") + + self.logger.info(f"Building Singularity image {self.sif_path} from {def_path}") + result = subprocess.run( + f"singularity build --fakeroot {self.sif_path} {def_path}", + shell=True, + capture_output=True, + text=True, + ) + if result.returncode == 0: + self.logger.info(f"Built Singularity image {self.sif_path}") + else: + self.logger.error(f"Failed to build Singularity image: {result.stderr}\n{result.stdout}") + raise RuntimeError(f"Failed to build Singularity image: {result.stderr}") + + def _build_docker_image(self): + """Build a Docker image from the Dockerfile.""" if is_running_in_aws_batch(): pull_game_container_aws_ecr(game_name=self.name, image_name=self.image_name, logger=self.logger) @@ -148,9 +196,9 @@ def build_image(self): text=True, ) if result.returncode == 0: - self.logger.info(f"✅ Built Docker image {self.image_name}") + self.logger.info(f"Built Docker image {self.image_name}") else: - self.logger.error(f"❌ Failed to build Docker image: {result.stderr}\n{result.stdout}{result.stderr}") + self.logger.error(f"Failed to build Docker image: {result.stderr}\n{result.stdout}{result.stderr}") raise RuntimeError(f"Failed to build Docker image: {result.stderr}") def copy_logs_from_env(self, round_num: int) -> None: @@ -178,39 +226,54 @@ def end(self, cleanup: bool = False): def log_round(self, round_num: int) -> Path: return self.log_local / "rounds" / str(round_num) - def get_environment(self, branch_name: str | None = None) -> DockerEnvironment: - """Get docker container ID with the game code installed.""" + def get_environment(self, branch_name: str | None = None) -> ContainerEnvironment: + """Get a container environment with the game code installed.""" self.build_image() - if not self._keep_containers: - run_args = ["--rm"] + + env_vars = { + "GITHUB_TOKEN": os.getenv("GITHUB_TOKEN", ""), + "PAGER": "cat", + "MANPAGER": "cat", + "LESS": "-R", + "PIP_PROGRESS_BAR": "off", + "TQDM_DISABLE": "1", + } + + if get_runtime() == "singularity": + environment = SingularityEnvironment( + image=str(self.sif_path), + cwd=str(DIR_WORK), + env=env_vars, + timeout=36000, # 10h in seconds + logger=self.logger, + ) else: - run_args = [] - environment = DockerEnvironment( - image=self.image_name, - cwd=str(DIR_WORK), - env={ - "GITHUB_TOKEN": os.getenv("GITHUB_TOKEN", ""), - "PAGER": "cat", - "MANPAGER": "cat", - "LESS": "-R", - "PIP_PROGRESS_BAR": "off", - "TQDM_DISABLE": "1", - }, - container_timeout="10h", - logger=self.logger, - run_args=run_args, - ) + if not self._keep_containers: + run_args = ["--rm"] + else: + run_args = [] + environment = DockerEnvironment( + image=self.image_name, + cwd=str(DIR_WORK), + env=env_vars, + container_timeout="10h", + logger=self.logger, + run_args=run_args, + ) branch_name = self.game_id if branch_name is None else branch_name # Logger setting will likely not take effect for initial container creation logs environment.logger = get_logger("environment", emoji="ðŸŠī") + # Use local (not --global) git config so it persists in the repo's .git/config. + # Singularity's --contain flag creates an ephemeral home per exec call, + # so --global config written to ~/.gitconfig is lost between calls. for cmd in [ f"git branch {branch_name}", f"git checkout {branch_name}", - 'git config --global user.email "player@codeclash.com"', - 'git config --global user.name "Player"', - "git config --global commit.gpgsign false", + 'git config user.email "player@codeclash.com"', + 'git config user.name "Player"', + "git config commit.gpgsign false", ]: assert_zero_exit_code(environment.execute(cmd), logger=self.logger) return environment diff --git a/codeclash/arenas/battlecode23/BattleCode23.def b/codeclash/arenas/battlecode23/BattleCode23.def new file mode 100644 index 00000000..a40fd115 --- /dev/null +++ b/codeclash/arenas/battlecode23/BattleCode23.def @@ -0,0 +1,20 @@ +Bootstrap: docker +From: eclipse-temurin:8-jdk + +%environment + export JAVA_HOME=/opt/java/openjdk + export GRADLE_USER_HOME=/root/.gradle + +%post + apt-get update && apt-get install -y --no-install-recommends \ + git curl unzip && \ + rm -rf /var/lib/apt/lists/* + + git clone https://github.com/CodeClash-ai/BattleCode2023.git /workspace \ + && cd /workspace \ + && git remote set-url origin https://github.com/CodeClash-ai/BattleCode2023.git + + cd /workspace && chmod +x gradlew && ./gradlew update + +%labels + Author CodeClash diff --git a/codeclash/arenas/battlecode23/battlecode23.py b/codeclash/arenas/battlecode23/battlecode23.py index 4e503a52..3410c1be 100644 --- a/codeclash/arenas/battlecode23/battlecode23.py +++ b/codeclash/arenas/battlecode23/battlecode23.py @@ -128,7 +128,8 @@ def _compile_agent(self, agent: Player, idx: int) -> str | None: return None # Save compiled classes outside build/ (gradle clean deletes build/) - classes_dir = f"/tmp/agent{idx}_classes" + # Use /opt as Singularity's --contain clears /tmp across execute() calls + classes_dir = f"/opt/agent{idx}_classes" self.environment.execute( f"rm -rf {classes_dir}; mkdir -p {classes_dir}; cp -r build/classes/* {classes_dir}/" ) diff --git a/codeclash/arenas/battlecode24/BattleCode24.def b/codeclash/arenas/battlecode24/BattleCode24.def new file mode 100644 index 00000000..b2336ce6 --- /dev/null +++ b/codeclash/arenas/battlecode24/BattleCode24.def @@ -0,0 +1,20 @@ +Bootstrap: docker +From: eclipse-temurin:8-jdk + +%environment + export JAVA_HOME=/opt/java/openjdk + export GRADLE_USER_HOME=/root/.gradle + +%post + apt-get update && apt-get install -y --no-install-recommends \ + git curl unzip && \ + rm -rf /var/lib/apt/lists/* + + git clone https://github.com/CodeClash-ai/BattleCode2024.git /workspace \ + && cd /workspace \ + && git remote set-url origin https://github.com/CodeClash-ai/BattleCode2024.git + + cd /workspace && chmod +x gradlew && ./gradlew update + +%labels + Author CodeClash diff --git a/codeclash/arenas/battlecode24/battlecode24.py b/codeclash/arenas/battlecode24/battlecode24.py index 50827ba0..1bb0febd 100644 --- a/codeclash/arenas/battlecode24/battlecode24.py +++ b/codeclash/arenas/battlecode24/battlecode24.py @@ -130,7 +130,8 @@ def _compile_agent(self, agent: Player, idx: int) -> str | None: return None # Save compiled classes outside build/ (gradle clean deletes build/) - classes_dir = f"/tmp/agent{idx}_classes" + # Use /opt as Singularity's --contain clears /tmp across execute() calls + classes_dir = f"/opt/agent{idx}_classes" self.environment.execute( f"rm -rf {classes_dir}; mkdir -p {classes_dir}; cp -r build/classes/* {classes_dir}/" ) diff --git a/codeclash/arenas/battlecode25/BattleCode25.def b/codeclash/arenas/battlecode25/BattleCode25.def new file mode 100644 index 00000000..2715556e --- /dev/null +++ b/codeclash/arenas/battlecode25/BattleCode25.def @@ -0,0 +1,20 @@ +Bootstrap: docker +From: python:3.12-slim-bookworm + +%environment + export PYTHONDONTWRITEBYTECODE=1 + export PYTHONUNBUFFERED=1 + +%post + apt-get update && apt-get install -y --no-install-recommends \ + build-essential git curl && \ + rm -rf /var/lib/apt/lists/* + + git clone https://github.com/CodeClash-ai/BattleCode2025.git /workspace \ + && cd /workspace \ + && git remote set-url origin https://github.com/CodeClash-ai/BattleCode2025.git + + cd /workspace && python run.py update + +%labels + Author CodeClash diff --git a/codeclash/arenas/battlesnake/BattleSnake.def b/codeclash/arenas/battlesnake/BattleSnake.def new file mode 100644 index 00000000..e09f9ac0 --- /dev/null +++ b/codeclash/arenas/battlesnake/BattleSnake.def @@ -0,0 +1,35 @@ +Bootstrap: docker +From: ubuntu:22.04 + +%environment + export DEBIAN_FRONTEND=noninteractive + export GO_VERSION=1.22.0 + export PATH=/usr/local/go/bin:$PATH + +%post + export DEBIAN_FRONTEND=noninteractive + export GO_VERSION=1.22.0 + + apt-get update \ + && apt-get install -y --no-install-recommends \ + curl ca-certificates python3.10 python3.10-venv \ + python3-pip python-is-python3 wget git build-essential jq locales \ + && rm -rf /var/lib/apt/lists/* + + ARCH=$(dpkg --print-architecture) \ + && echo "Building for architecture: $ARCH" \ + && curl -fsSL https://go.dev/dl/go${GO_VERSION}.linux-${ARCH}.tar.gz -o /tmp/go.tar.gz \ + && tar -C /usr/local -xzf /tmp/go.tar.gz \ + && rm /tmp/go.tar.gz + + export PATH=/usr/local/go/bin:$PATH + + git clone https://github.com/CodeClash-ai/BattleSnake.git /workspace \ + && cd /workspace \ + && git remote set-url origin https://github.com/CodeClash-ai/BattleSnake.git + + cd /workspace/game && go build -o battlesnake ./cli/battlesnake/main.go + cd /workspace && pip install -r requirements.txt + +%labels + Author CodeClash diff --git a/codeclash/arenas/battlesnake/battlesnake.py b/codeclash/arenas/battlesnake/battlesnake.py index a61167e1..5c51b498 100644 --- a/codeclash/arenas/battlesnake/battlesnake.py +++ b/codeclash/arenas/battlesnake/battlesnake.py @@ -1,10 +1,12 @@ import json +import os import random import subprocess import time from collections import defaultdict from concurrent.futures import ThreadPoolExecutor, as_completed +from minisweagent.environments.singularity import SingularityEnvironment from tqdm.auto import tqdm from codeclash.agents.player import Player @@ -34,6 +36,26 @@ def __init__(self, config, **kwargs): self.run_cmd_round += f" --{arg} {val}" self._failed_to_start_player = [] + def _start_server_popen(self, command: str, cwd: str) -> subprocess.Popen: + """Start a long-running server process via subprocess.Popen. + + In Singularity, each environment.execute() call is a separate `singularity exec` + invocation. Background processes (&) don't reliably survive after the call returns. + This method uses Popen to keep the singularity exec process alive for the duration + of the server, so the server persists across subsequent execute() calls. + """ + env = self.environment + cmd = [env.config.executable, "exec", "--contain", "--cleanenv"] + if cwd and cwd != "/": + cmd.extend(["--pwd", cwd]) + for key in env.config.forward_env: + if (value := os.getenv(key)) is not None: + cmd.extend(["--env", f"{key}={value}"]) + for key, value in env.config.env.items(): + cmd.extend(["--env", f"{key}={value}"]) + cmd.extend(["--writable", str(env.sandbox_dir), "bash", "-c", command]) + return subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + def _wait_for_ports(self, requested_ports: list[int], timeout: float = 60.0) -> list[int]: """Wait for ports to be served, up to timeout seconds. @@ -89,12 +111,21 @@ def execute_round(self, agents: list[Player]): assert len(agents) > 1, "Battlesnake requires at least two players" self.logger.debug("Starting game servers") player2port = {} + server_procs: list[subprocess.Popen] = [] + use_popen = isinstance(self.environment, SingularityEnvironment) + for idx, agent in enumerate(agents): port = 8001 + idx player2port[agent.name] = port - # Surprisingly slow despite using & - # Start server in background - just add & to run in background! - self.environment.execute(f"PORT={port} python {self.submission} &", cwd=f"/{agent.name}") + if use_popen: + # In Singularity, background processes don't persist across execute() calls. + # Use Popen to keep the singularity exec process (and the server) alive. + proc = self._start_server_popen( + f"PORT={port} python {self.submission}", cwd=f"/{agent.name}" + ) + server_procs.append(proc) + else: + self.environment.execute(f"PORT={port} python {self.submission} &", cwd=f"/{agent.name}") self.logger.debug(f"Waiting for ports: {player2port}") available_ports = self._wait_for_ports(list(player2port.values())) @@ -129,8 +160,13 @@ def execute_round(self, agents: list[Player]): for future in tqdm(as_completed(futures), total=len(futures)): future.result() finally: - # Kill all python servers when done - self.environment.execute(f"pkill -f 'python {self.submission}' || true") + if server_procs: + for proc in server_procs: + proc.terminate() + for proc in server_procs: + proc.wait() + else: + self.environment.execute(f"pkill -f 'python {self.submission}' || true") def get_results(self, agents: list[Player], round_num: int, stats: RoundStats): scores = defaultdict(int) diff --git a/codeclash/arenas/bridge/Bridge.def b/codeclash/arenas/bridge/Bridge.def new file mode 100644 index 00000000..c21c682c --- /dev/null +++ b/codeclash/arenas/bridge/Bridge.def @@ -0,0 +1,21 @@ +Bootstrap: docker +From: ubuntu:22.04 + +%environment + export DEBIAN_FRONTEND=noninteractive + +%post + export DEBIAN_FRONTEND=noninteractive + + apt-get update \ + && apt-get install -y --no-install-recommends \ + curl ca-certificates python3.10 python3.10-venv \ + python3-pip python-is-python3 wget git build-essential jq locales \ + && rm -rf /var/lib/apt/lists/* + + git clone https://github.com/CodeClash-ai/Bridge.git /workspace \ + && cd /workspace \ + && git remote set-url origin https://github.com/CodeClash-ai/Bridge.git + +%labels + Author CodeClash diff --git a/codeclash/arenas/chess/Chess.def b/codeclash/arenas/chess/Chess.def new file mode 100644 index 00000000..c10ef29a --- /dev/null +++ b/codeclash/arenas/chess/Chess.def @@ -0,0 +1,30 @@ +Bootstrap: docker +From: ubuntu:22.04 + +%environment + export DEBIAN_FRONTEND=noninteractive + +%post + export DEBIAN_FRONTEND=noninteractive + + apt-get update \ + && apt-get install -y --no-install-recommends \ + curl ca-certificates python3.10 python3.10-venv \ + python3-pip python-is-python3 wget git build-essential \ + g++ make jq locales \ + && rm -rf /var/lib/apt/lists/* + + git clone https://github.com/Babak-SSH/Kojiro.git /workspace \ + && cd /workspace \ + && git remote set-url origin https://github.com/Babak-SSH/Kojiro.git + + rm -rf /opt/fastchess-build \ + && git clone https://github.com/Disservin/fastchess.git /opt/fastchess-build \ + && cd /opt/fastchess-build \ + && make -j \ + && install -d /usr/local/bin \ + && install fastchess /usr/local/bin/fastchess \ + && rm -rf /opt/fastchess-build + +%labels + Author CodeClash diff --git a/codeclash/arenas/corewar/CoreWar.def b/codeclash/arenas/corewar/CoreWar.def new file mode 100644 index 00000000..cd7d6c38 --- /dev/null +++ b/codeclash/arenas/corewar/CoreWar.def @@ -0,0 +1,20 @@ +Bootstrap: docker +From: ubuntu:22.04 + +%post + apt-get update \ + && apt-get install -y \ + curl ca-certificates wget git build-essential jq locales \ + && rm -rf /var/lib/apt/lists/* + + git clone https://github.com/CodeClash-ai/CoreWar.git /workspace \ + && cd /workspace \ + && git remote set-url origin https://github.com/CodeClash-ai/CoreWar.git + + cd /workspace/src/ && make CFLAGS="-O -DEXT94 -DPERMUTATE -DRWLIMIT" LIB="" + + # Use /opt as Singularity's --contain bind-mounts host /home, hiding container paths + cp /workspace/doc/examples/dwarf.red /opt/dwarf.red + +%labels + Author CodeClash diff --git a/codeclash/arenas/corewar/corewar.py b/codeclash/arenas/corewar/corewar.py index 6843ebd2..ba0b9ed9 100644 --- a/codeclash/arenas/corewar/corewar.py +++ b/codeclash/arenas/corewar/corewar.py @@ -98,7 +98,7 @@ def validate_code(self, agent: Player) -> tuple[bool, str | None]: if self.submission not in agent.environment.execute("ls")["output"]: return False, f"There should be a `{self.submission}` file" # Play game against a simple default bot to ensure it runs - test_run_cmd = f"{self.run_cmd_round} {self.submission} /home/dwarf.red" + test_run_cmd = f"{self.run_cmd_round} {self.submission} /opt/dwarf.red" test_run = agent.environment.execute(test_run_cmd, timeout=60)["output"] if any([l.startswith("Error") for l in test_run.split("\n")]): return False, f"The `{self.submission}` file is malformed (Ran `{test_run_cmd}`):\n{test_run}" diff --git a/codeclash/arenas/dummy/Dummy.def b/codeclash/arenas/dummy/Dummy.def new file mode 100644 index 00000000..21b496d4 --- /dev/null +++ b/codeclash/arenas/dummy/Dummy.def @@ -0,0 +1,21 @@ +Bootstrap: docker +From: ubuntu:22.04 + +%environment + export DEBIAN_FRONTEND=noninteractive + +%post + export DEBIAN_FRONTEND=noninteractive + + apt-get update \ + && apt-get install -y --no-install-recommends \ + curl ca-certificates python3.10 python3.10-venv \ + python3-pip python-is-python3 wget git build-essential jq locales \ + && rm -rf /var/lib/apt/lists/* + + git clone https://github.com/CodeClash-ai/DummyArena.git /workspace \ + && cd /workspace \ + && git remote set-url origin https://github.com/CodeClash-ai/DummyArena.git + +%labels + Author CodeClash diff --git a/codeclash/arenas/figgie/Figgie.def b/codeclash/arenas/figgie/Figgie.def new file mode 100644 index 00000000..5eee2de0 --- /dev/null +++ b/codeclash/arenas/figgie/Figgie.def @@ -0,0 +1,21 @@ +Bootstrap: docker +From: ubuntu:22.04 + +%environment + export DEBIAN_FRONTEND=noninteractive + +%post + export DEBIAN_FRONTEND=noninteractive + + apt-get update \ + && apt-get install -y --no-install-recommends \ + curl ca-certificates python3.10 python3.10-venv \ + python3-pip python-is-python3 wget git build-essential jq locales \ + && rm -rf /var/lib/apt/lists/* + + git clone https://github.com/CodeClash-ai/Figgie.git /workspace \ + && cd /workspace \ + && git remote set-url origin https://github.com/CodeClash-ai/Figgie.git + +%labels + Author CodeClash diff --git a/codeclash/arenas/figgie/figgie.py b/codeclash/arenas/figgie/figgie.py index c6fe64c8..9b4355d6 100644 --- a/codeclash/arenas/figgie/figgie.py +++ b/codeclash/arenas/figgie/figgie.py @@ -58,10 +58,10 @@ def get_action(state: dict) -> dict """ def __init__(self, config, **kwargs): - super().__init__(config, **kwargs) num_players = len(config.get("players", [])) if num_players not in [4, 5]: raise ValueError(f"Figgie requires 4 or 5 players, got {num_players}") + super().__init__(config, **kwargs) def execute_round(self, agents: list[Player]) -> None: args = [f"/{agent.name}/{self.submission}" for agent in agents] diff --git a/codeclash/arenas/gomoku/Gomoku.def b/codeclash/arenas/gomoku/Gomoku.def new file mode 100644 index 00000000..f7b34590 --- /dev/null +++ b/codeclash/arenas/gomoku/Gomoku.def @@ -0,0 +1,21 @@ +Bootstrap: docker +From: ubuntu:22.04 + +%environment + export DEBIAN_FRONTEND=noninteractive + +%post + export DEBIAN_FRONTEND=noninteractive + + apt-get update \ + && apt-get install -y --no-install-recommends \ + curl ca-certificates python3.10 python3.10-venv \ + python3-pip python-is-python3 wget git build-essential jq locales \ + && rm -rf /var/lib/apt/lists/* + + git clone https://github.com/CodeClash-ai/Gomoku.git /workspace \ + && cd /workspace \ + && git remote set-url origin https://github.com/CodeClash-ai/Gomoku.git + +%labels + Author CodeClash diff --git a/codeclash/arenas/halite/Halite.def b/codeclash/arenas/halite/Halite.def new file mode 100644 index 00000000..b7a2c47d --- /dev/null +++ b/codeclash/arenas/halite/Halite.def @@ -0,0 +1,33 @@ +Bootstrap: docker +From: ubuntu:22.04 + +%environment + export DEBIAN_FRONTEND=noninteractive + export PATH="/root/.cargo/bin:${PATH}" + export RUSTUP_HOME=/root/.rustup + export CARGO_HOME=/root/.cargo + +%post + export DEBIAN_FRONTEND=noninteractive + + apt-get update \ + && apt-get install -y --no-install-recommends \ + curl ca-certificates python3.10 python3.10-venv \ + python3-pip python-is-python3 wget git build-essential jq locales \ + && rm -rf /var/lib/apt/lists/* + + curl https://sh.rustup.rs -sSf | sh -s -- -y + . "$HOME/.cargo/env" + + apt-get update && apt-get install -y ocaml ocamlbuild \ + && rm -rf /var/lib/apt/lists/* + + git clone https://github.com/CodeClash-ai/Halite.git /workspace \ + && cd /workspace \ + && git remote set-url origin https://github.com/CodeClash-ai/Halite.git + + export PATH="/root/.cargo/bin:${PATH}" + cd /workspace/environment && make + +%labels + Author CodeClash diff --git a/codeclash/arenas/halite/halite.py b/codeclash/arenas/halite/halite.py index 6afd28dc..9dfdb4d6 100644 --- a/codeclash/arenas/halite/halite.py +++ b/codeclash/arenas/halite/halite.py @@ -83,7 +83,7 @@ def _run_single_simulation(self, agents: list[Player], idx: int, cmd: str): def execute_round(self, agents: list[Player]): entries = [] for agent in agents: - executable = agent.environment.execute(f"cat {HALITE_HIDDEN_EXEC}")["output"].strip() + executable = agent.environment.execute(f"cat {HALITE_HIDDEN_EXEC}")["output"].strip().splitlines()[-1] entries.append(executable) cmd = f"{self.run_cmd_round} {shlex.join(entries)}" self.logger.info(f"Running game: {cmd}") @@ -137,7 +137,8 @@ def validate_code( ) -> tuple[bool, str | None]: # Check that the `submission/` folder exists exists_output = agent.environment.execute("test -d submission && echo 'exists'")["output"] - if "exists" != exists_output.strip(): + lines = exists_output.strip().splitlines() + if not lines or "exists" != lines[-1]: return False, f"Submission folder `{self.submission}/` does not exist" # Check that there is a *single* file called "main." in the submission folder diff --git a/codeclash/arenas/halite2/Halite2.def b/codeclash/arenas/halite2/Halite2.def new file mode 100644 index 00000000..f1403d6e --- /dev/null +++ b/codeclash/arenas/halite2/Halite2.def @@ -0,0 +1,40 @@ +Bootstrap: docker +From: ubuntu:22.04 + +%environment + export DEBIAN_FRONTEND=noninteractive + export PATH="/root/.cargo/bin:/root/.ghcup/bin:${PATH}" + export RUSTUP_HOME=/root/.rustup + export CARGO_HOME=/root/.cargo + +%post + export DEBIAN_FRONTEND=noninteractive + + apt-get update \ + && apt-get install -y --no-install-recommends \ + curl ca-certificates python3.10 python3.10-venv \ + python3-pip python-is-python3 wget git build-essential jq locales cmake \ + && rm -rf /var/lib/apt/lists/* + + curl https://sh.rustup.rs -sSf | sh -s -- -y + . "$HOME/.cargo/env" + + apt-get update && apt-get install -y ocaml ocamlbuild \ + && rm -rf /var/lib/apt/lists/* + + apt-get update && apt-get install -y libgmp-dev \ + && rm -rf /var/lib/apt/lists/* + + curl --proto '=https' --tlsv1.2 -sSf https://get-ghcup.haskell.org | \ + BOOTSTRAP_HASKELL_NONINTERACTIVE=1 BOOTSTRAP_HASKELL_ADJUST_BASHRC=1 sh + + export PATH="/root/.ghcup/bin:/root/.cargo/bin:${PATH}" + + git clone https://github.com/CodeClash-ai/Halite2.git /workspace \ + && cd /workspace \ + && git remote set-url origin https://github.com/CodeClash-ai/Halite2.git + + cd /workspace/environment && cmake . && make + +%labels + Author CodeClash diff --git a/codeclash/arenas/halite3/Halite3.def b/codeclash/arenas/halite3/Halite3.def new file mode 100644 index 00000000..8f75b4ae --- /dev/null +++ b/codeclash/arenas/halite3/Halite3.def @@ -0,0 +1,33 @@ +Bootstrap: docker +From: ubuntu:22.04 + +%environment + export DEBIAN_FRONTEND=noninteractive + export PATH="/root/.cargo/bin:${PATH}" + export RUSTUP_HOME=/root/.rustup + export CARGO_HOME=/root/.cargo + +%post + export DEBIAN_FRONTEND=noninteractive + + apt-get update \ + && apt-get install -y --no-install-recommends \ + curl ca-certificates python3.10 python3.10-venv \ + python3-pip python-is-python3 wget git build-essential jq locales cmake \ + && rm -rf /var/lib/apt/lists/* + + curl https://sh.rustup.rs -sSf | sh -s -- -y + . "$HOME/.cargo/env" + + apt-get update && apt-get install -y ocaml ocamlbuild \ + && rm -rf /var/lib/apt/lists/* + + git clone https://github.com/CodeClash-ai/Halite3.git /workspace \ + && cd /workspace \ + && git remote set-url origin https://github.com/CodeClash-ai/Halite3.git + + export PATH="/root/.cargo/bin:${PATH}" + cd /workspace/game_engine && cmake . && make + +%labels + Author CodeClash diff --git a/codeclash/arenas/huskybench/HuskyBench.def b/codeclash/arenas/huskybench/HuskyBench.def new file mode 100644 index 00000000..ecbd7ba1 --- /dev/null +++ b/codeclash/arenas/huskybench/HuskyBench.def @@ -0,0 +1,24 @@ +Bootstrap: docker +From: python:3.10-slim + +%environment + export DEBIAN_FRONTEND=noninteractive + export TZ=Etc/UTC + +%post + export DEBIAN_FRONTEND=noninteractive + export TZ=Etc/UTC + + apt-get update && apt-get install -y \ + wget git build-essential unzip lsof \ + && rm -rf /var/lib/apt/lists/* + + git clone https://github.com/CodeClash-ai/HuskyBench.git /workspace \ + && cd /workspace \ + && git remote set-url origin https://github.com/CodeClash-ai/HuskyBench.git + + cd /workspace && pip install -r engine/requirements.txt + mkdir -p /workspace/engine/output + +%labels + Author CodeClash diff --git a/codeclash/arenas/robocode/RoboCode.def b/codeclash/arenas/robocode/RoboCode.def new file mode 100644 index 00000000..93c51d02 --- /dev/null +++ b/codeclash/arenas/robocode/RoboCode.def @@ -0,0 +1,24 @@ +Bootstrap: docker +From: maven:3.9-eclipse-temurin-24 + +%environment + export DEBIAN_FRONTEND=noninteractive + export TZ=Etc/UTC + +%post + export DEBIAN_FRONTEND=noninteractive + export TZ=Etc/UTC + + apt-get update && apt-get install -y \ + wget git build-essential ant unzip \ + python3 python3-pip python3-venv \ + && ln -sf /usr/bin/python3 /usr/bin/python \ + && ln -sf /usr/bin/pip3 /usr/bin/pip \ + && rm -rf /var/lib/apt/lists/* + + git clone https://github.com/CodeClash-ai/RoboCode.git /workspace \ + && cd /workspace \ + && git remote set-url origin https://github.com/CodeClash-ai/RoboCode.git + +%labels + Author CodeClash diff --git a/codeclash/arenas/robotrumble/RobotRumble.def b/codeclash/arenas/robotrumble/RobotRumble.def new file mode 100644 index 00000000..42d736fe --- /dev/null +++ b/codeclash/arenas/robotrumble/RobotRumble.def @@ -0,0 +1,21 @@ +Bootstrap: docker +From: ubuntu:22.04 + +%environment + export DEBIAN_FRONTEND=noninteractive + +%post + export DEBIAN_FRONTEND=noninteractive + + apt-get update \ + && apt-get install -y --no-install-recommends \ + curl ca-certificates python3.10 python3.10-venv \ + python3-pip python-is-python3 wget git build-essential jq locales \ + && rm -rf /var/lib/apt/lists/* + + git clone https://github.com/CodeClash-ai/RobotRumble.git /workspace \ + && cd /workspace \ + && git remote set-url origin https://github.com/CodeClash-ai/RobotRumble.git + +%labels + Author CodeClash diff --git a/codeclash/arenas/robotrumble/robotrumble.py b/codeclash/arenas/robotrumble/robotrumble.py index d5d831fb..b538c44b 100644 --- a/codeclash/arenas/robotrumble/robotrumble.py +++ b/codeclash/arenas/robotrumble/robotrumble.py @@ -58,7 +58,7 @@ def execute_round(self, agents: list[Player]): self.logger.info(f"Running game with players: {[agent.name for agent in agents]}") args = [] for agent in agents: - executable = agent.environment.execute(f"cat {ROBOTRUMBLE_HIDDEN_EXEC}")["output"].strip() + executable = agent.environment.execute(f"cat {ROBOTRUMBLE_HIDDEN_EXEC}")["output"].strip().splitlines()[-1] args.append(f"/{agent.name}/{executable}") cmd = f"{self.run_cmd_round} {shlex.join(args)}" self.logger.info(f"Running game: {cmd}") @@ -147,7 +147,8 @@ def validate_code(self, agent: Player) -> tuple[bool, str | None]: ext, exists = None, False for possible_ext in MAP_EXT_TO_HEADER.keys(): exists_output = agent.environment.execute(f"test -f robot.{possible_ext} && echo 'exists'")["output"] - if "exists" == exists_output.strip(): + lines = exists_output.strip().splitlines() + if lines and "exists" == lines[-1]: ext = possible_ext exists = True break diff --git a/codeclash/tournaments/pvp.py b/codeclash/tournaments/pvp.py index 65c4e1d0..e61a71f1 100644 --- a/codeclash/tournaments/pvp.py +++ b/codeclash/tournaments/pvp.py @@ -179,7 +179,10 @@ def _compress_round_logs(self) -> None: ] self.logger.info(f"Compressing round logs, this might take a while... ({' '.join(cmd)})") result = subprocess.run(cmd, capture_output=True, text=True) - if result.returncode != 0: + if result.returncode == 1: + # Allow "some files differ" warning + self.logger.warning(f"tar reported file changes during compression (exit code 1): {result.stderr.strip()}") + elif result.returncode > 1: raise RuntimeError(f"Command failed with exit code {result.returncode}:\n{result.stderr}") # Remove the original round logs shutil.rmtree(self.game.log_local / "rounds") @@ -203,7 +206,10 @@ def _compress_round_folder(self, round_num_zero_indexed: int) -> None: f"Compressing round {round_num_zero_indexed} logs, this might take a while... ({' '.join(cmd)})" ) result = subprocess.run(cmd, capture_output=True, text=True) - if result.returncode != 0: + if result.returncode == 1: + # Allow "some files differ" warning + self.logger.warning(f"tar reported file changes during compression (exit code 1): {result.stderr.strip()}") + elif result.returncode > 1: raise RuntimeError(f"Command failed with exit code {result.returncode}:\n{result.stderr}") self.logger.debug("Removing %s", round_dir) shutil.rmtree(round_dir) diff --git a/codeclash/tournaments/utils/git_utils.py b/codeclash/tournaments/utils/git_utils.py index 5bca9e54..a443b467 100644 --- a/codeclash/tournaments/utils/git_utils.py +++ b/codeclash/tournaments/utils/git_utils.py @@ -10,7 +10,6 @@ def filter_git_diff(diff: str) -> str: out: list[str] = [] block: list[str] = [] in_block = False - prelude_copied = False def is_binary_block(bl: list[str]) -> bool: for ln in bl: @@ -41,14 +40,10 @@ def extract_file_path_from_block(bl: list[str]) -> str: else: out.extend(block) block = [] - else: - if not prelude_copied: - prelude_copied = True in_block = True if in_block: block.append(ln) - else: - out.append(ln) + if in_block and block: if is_binary_block(block): @@ -124,15 +119,11 @@ def split_git_diff_by_files(diff: str) -> dict[str, str]: current_file = None current_block = [] - # Store any prelude (content before first diff --git line) - prelude = [] - found_first_diff = False - for line in lines: if line.startswith("diff --git "): # Save previous file's diff if we have one if current_file and current_block: - files_diffs[current_file] = "".join(prelude + current_block) + files_diffs[current_file] = "".join(current_block) current_block = [] # Extract file path from the diff line @@ -149,16 +140,12 @@ def split_git_diff_by_files(diff: str) -> dict[str, str]: current_file = "unknown_file" current_block.append(line) - found_first_diff = True - else: - if found_first_diff and current_file: - current_block.append(line) - else: - # This is prelude content before any diff - prelude.append(line) + elif current_file: + current_block.append(line) + # Handle the last file if current_file and current_block: - files_diffs[current_file] = "".join(prelude + current_block) + files_diffs[current_file] = "".join(current_block) return files_diffs diff --git a/codeclash/utils/environment.py b/codeclash/utils/environment.py index bbba0d8b..f6878283 100644 --- a/codeclash/utils/environment.py +++ b/codeclash/utils/environment.py @@ -5,6 +5,9 @@ from pathlib import Path from minisweagent.environments.docker import DockerEnvironment +from minisweagent.environments.singularity import SingularityEnvironment + +ContainerEnvironment = DockerEnvironment | SingularityEnvironment # Patterns to exclude when copying between containers COPY_EXCLUDE_PATTERNS = [".git", "__pycache__"] @@ -20,17 +23,46 @@ def assert_zero_exit_code(result: dict, *, logger: logging.Logger | None = None) def copy_between_containers( - src_container: DockerEnvironment, - dest_container: DockerEnvironment, + src_container: ContainerEnvironment, + dest_container: ContainerEnvironment, src_path: str | Path, dest_path: str | Path, ): """ - Copy files from one Docker container to another via a temporary local directory. + Copy files from one container to another. + + For Docker: copies via a temporary local directory using docker cp. + For Singularity: copies directly between sandbox directories. Be extremely careful with trailing slashes in src_path and dest_path, the behavior of docker cp is also different depending on whether the destination exists. """ + if isinstance(src_container, SingularityEnvironment) and isinstance(dest_container, SingularityEnvironment): + src_full = src_container.sandbox_dir / str(src_path).lstrip("/") + dest_full = dest_container.sandbox_dir / str(dest_path).lstrip("/") + print(f"Copy between containers (singularity): {src_full} -> {dest_full}") + + dest_full.parent.mkdir(parents=True, exist_ok=True) + if dest_full.exists() and dest_full.is_dir(): + # docker cp copies src dir INTO existing dest as dest/basename(src)/... + actual_dest = dest_full / src_full.name + if actual_dest.exists(): + shutil.rmtree(actual_dest) + shutil.copytree( + src_full, + actual_dest, + symlinks=True, + ignore=shutil.ignore_patterns(*COPY_EXCLUDE_PATTERNS), + ) + else: + shutil.copytree( + src_full, + dest_full, + symlinks=True, + ignore=shutil.ignore_patterns(*COPY_EXCLUDE_PATTERNS), + ) + return + print( f"Copy between containers: {src_container.container_id}:{src_path} -> {dest_container.container_id}:{dest_path}" ) @@ -80,18 +112,41 @@ def copy_between_containers( def copy_to_container( - container: DockerEnvironment, + container: ContainerEnvironment, src_path: str | Path, dest_path: str | Path, ): """ - Copy a file or directory from the local filesystem to a Docker container. + Copy a file or directory from the local filesystem to a container. + + For Docker: uses docker cp. + For Singularity: copies directly to the sandbox directory. The copy operation is recursive for directories. Be extremely careful with trailing slashes in src_path and dest_path, the behavior of docker cp is also different depending on whether the destination exists. """ + if isinstance(container, SingularityEnvironment): + if not str(dest_path).startswith("/"): + dest_path = f"{container.config.cwd}/{dest_path}" + dest_full = container.sandbox_dir / str(dest_path).lstrip("/") + print(f"Copy to container (singularity): {src_path} -> {dest_full}") + dest_full.parent.mkdir(parents=True, exist_ok=True) + src_path = Path(src_path) + if src_path.is_dir(): + if dest_full.exists() and dest_full.is_dir(): + # docker cp copies src dir INTO existing dest as dest/basename(src)/... + actual_dest = dest_full / src_path.name + if actual_dest.exists(): + shutil.rmtree(actual_dest) + shutil.copytree(src_path, actual_dest) + else: + shutil.copytree(src_path, dest_full) + else: + shutil.copy2(src_path, dest_full) + return + if not str(dest_path).startswith("/"): # If not an absolute path, assume relative to container's cwd dest_path = f"{container.config.cwd}/{dest_path}" @@ -113,18 +168,58 @@ def copy_to_container( def copy_from_container( - container: DockerEnvironment, + container: ContainerEnvironment, src_path: str | Path, dest_path: str | Path, ): """ - Copy a file or directory from a Docker container to the local filesystem. + Copy a file or directory from a container to the local filesystem. + + For Docker: uses docker cp. + For Singularity: copies directly from the sandbox directory. The copy operation is recursive for directories. Be extremely careful with trailing slashes in src_path and dest_path, the behavior of docker cp is also different depending on whether the destination exists. """ + if isinstance(container, SingularityEnvironment): + # Handle trailing "/." which means "contents of directory" + src_str = str(src_path) + copy_contents = src_str.endswith("/.") + if copy_contents: + src_str = src_str.removesuffix("/.") + src_full = container.sandbox_dir / src_str.lstrip("/") + print(f"Copy from container (singularity): {src_full} -> {dest_path}") + Path(dest_path).parent.mkdir(parents=True, exist_ok=True) + if src_full.is_dir(): + if copy_contents: + # Copy contents of directory into dest_path + dest_path = Path(dest_path) + dest_path.mkdir(parents=True, exist_ok=True) + for item in src_full.iterdir(): + s = src_full / item.name + d = dest_path / item.name + if s.is_dir(): + if d.exists(): + shutil.rmtree(d) + shutil.copytree(s, d) + else: + shutil.copy2(s, d) + else: + dest_path = Path(dest_path) + if dest_path.exists() and dest_path.is_dir(): + # docker cp copies src dir INTO existing dest as dest/basename(src)/... + actual_dest = dest_path / src_full.name + if actual_dest.exists(): + shutil.rmtree(actual_dest) + shutil.copytree(src_full, actual_dest) + else: + shutil.copytree(src_full, dest_path) + else: + shutil.copy2(src_full, dest_path) + return + cmd = [ "docker", "cp", @@ -142,15 +237,26 @@ def copy_from_container( def create_file_in_container( - container: DockerEnvironment, + container: ContainerEnvironment, *, content: str, dest_path: str | Path, ): """ - Create a file with given content on a Docker container. - Uses a temporary file on the local filesystem for the transfer. + Create a file with given content in a container. + + For Docker: uses a temporary file on the local filesystem for the transfer. + For Singularity: writes directly to the sandbox directory. """ + if isinstance(container, SingularityEnvironment): + if not str(dest_path).startswith("/"): + dest_path = f"{container.config.cwd}/{dest_path}" + dest_full = container.sandbox_dir / str(dest_path).lstrip("/") + print(f"Create file in container (singularity): {dest_full}") + dest_full.parent.mkdir(parents=True, exist_ok=True) + dest_full.write_text(content) + return + # Some weird stuff happening on AWS where /tmp doesn't work properly dir = Path.home() / "tmp" dir.mkdir(parents=True, exist_ok=True) diff --git a/tests/arenas/test_corewar.py b/tests/arenas/test_corewar.py index 7f7446b9..a58e43d5 100644 --- a/tests/arenas/test_corewar.py +++ b/tests/arenas/test_corewar.py @@ -39,7 +39,7 @@ def test_valid_submission(self, arena, mock_player_factory): files={"warrior.red": VALID_WARRIOR}, command_outputs={ "ls": {"output": "warrior.red\n", "returncode": 0}, - "./src/pmars warrior.red /home/dwarf.red": { + "./src/pmars warrior.red /opt/dwarf.red": { "output": "warrior.red by Imp scores 10\ndwarf.red by Dwarf scores 5", "returncode": 0, }, @@ -69,7 +69,7 @@ def test_malformed_warrior_file(self, arena, mock_player_factory): files={"warrior.red": "invalid redcode syntax"}, command_outputs={ "ls": {"output": "warrior.red\n", "returncode": 0}, - "./src/pmars warrior.red /home/dwarf.red": { + "./src/pmars warrior.red /opt/dwarf.red": { "output": "Error: Invalid instruction at line 1\n", "returncode": 0, # pmars returns 0 even on parse errors }, diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..b6022d5a --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,11 @@ +import os +import shutil + + +def pytest_configure(config): + """Auto-detect container runtime if CODECLASH_RUNTIME is not already set.""" + if "CODECLASH_RUNTIME" not in os.environ: + if shutil.which("docker"): + os.environ["CODECLASH_RUNTIME"] = "docker" + elif shutil.which("singularity"): + os.environ["CODECLASH_RUNTIME"] = "singularity" diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py new file mode 100644 index 00000000..f7692fc6 --- /dev/null +++ b/tests/utils/__init__.py @@ -0,0 +1 @@ +# Container environment parity unit tests diff --git a/tests/utils/test_container_environment.py b/tests/utils/test_container_environment.py new file mode 100644 index 00000000..a6425e98 --- /dev/null +++ b/tests/utils/test_container_environment.py @@ -0,0 +1,318 @@ +""" +Tests that Singularity copy operations in environment.py match docker cp semantics. + +docker cp rules (from Docker docs): +1. SRC is a file: + - DEST doesn't exist -> create file at DEST + - DEST exists as file -> overwrite + - DEST exists as dir -> copy file INTO dir as DEST/basename(SRC) + +2. SRC is a directory (no /. suffix): + - DEST doesn't exist -> create DEST, copy CONTENTS into it + - DEST exists as dir -> copy SRC dir INTO DEST as DEST/basename(SRC)/... + +3. SRC is a directory with /. suffix: + - DEST doesn't exist -> create DEST, copy CONTENTS into it + - DEST exists as dir -> merge CONTENTS into DEST (no subdirectory created) +""" + +import tempfile +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from minisweagent.environments.singularity import SingularityEnvironment, SingularityEnvironmentConfig + +from codeclash.utils.environment import ( + copy_between_containers, + copy_from_container, + copy_to_container, + create_file_in_container, +) + + +@pytest.fixture +def mock_env(tmp_path): + """Create a SingularityEnvironment with a fake sandbox_dir (no actual build).""" + + def _make(cwd="/workspace"): + _make.counter = getattr(_make, "counter", 0) + 1 + sandbox = tmp_path / f"sandbox-{_make.counter}" + sandbox.mkdir() + (sandbox / cwd.lstrip("/")).mkdir(parents=True, exist_ok=True) + env = object.__new__(SingularityEnvironment) + env.config = SingularityEnvironmentConfig(image="dummy", cwd=cwd) + env.sandbox_dir = sandbox + env.logger = MagicMock() + return env + + return _make + + +def populate_dir(d: Path, files: dict[str, str]): + for relpath, content in files.items(): + p = d / relpath + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(content) + + +def dir_contents(d: Path) -> dict[str, str]: + result = {} + if d.exists() and d.is_dir(): + for p in sorted(d.rglob("*")): + if p.is_file(): + result[str(p.relative_to(d))] = p.read_text() + return result + + +# ============================================================ +# copy_to_container +# ============================================================ + + +class TestCopyToContainer: + def test_file_to_nonexistent_dest(self, mock_env, tmp_path): + env = mock_env() + src = tmp_path / "src_file.txt" + src.write_text("hello") + + copy_to_container(env, src, "/dest/file.txt") + assert (env.sandbox_dir / "dest/file.txt").read_text() == "hello" + + def test_file_overwrites_existing_file(self, mock_env, tmp_path): + env = mock_env() + (env.sandbox_dir / "dest").mkdir(parents=True) + (env.sandbox_dir / "dest/file.txt").write_text("old") + + src = tmp_path / "src_file.txt" + src.write_text("new") + + copy_to_container(env, src, "/dest/file.txt") + assert (env.sandbox_dir / "dest/file.txt").read_text() == "new" + + def test_file_to_existing_directory(self, mock_env, tmp_path): + """docker cp copies file INTO the directory.""" + env = mock_env() + (env.sandbox_dir / "dest_dir").mkdir(parents=True) + (env.sandbox_dir / "dest_dir/existing.txt").write_text("keep") + + src = tmp_path / "myfile.txt" + src.write_text("new_content") + + copy_to_container(env, src, "/dest_dir") + assert (env.sandbox_dir / "dest_dir/myfile.txt").read_text() == "new_content" + assert (env.sandbox_dir / "dest_dir/existing.txt").read_text() == "keep" + + def test_dir_to_nonexistent_dest(self, mock_env, tmp_path): + env = mock_env() + src_dir = tmp_path / "src_dir" + populate_dir(src_dir, {"a.txt": "aaa", "sub/b.txt": "bbb"}) + + copy_to_container(env, src_dir, "/newdest") + assert dir_contents(env.sandbox_dir / "newdest") == {"a.txt": "aaa", "sub/b.txt": "bbb"} + + def test_dir_to_existing_directory(self, mock_env, tmp_path): + """docker cp copies src dir INTO existing dest as dest/basename(src)/...""" + env = mock_env() + (env.sandbox_dir / "existing_dest").mkdir(parents=True) + (env.sandbox_dir / "existing_dest/old.txt").write_text("old") + + src_dir = tmp_path / "mydir" + populate_dir(src_dir, {"a.txt": "aaa", "sub/b.txt": "bbb"}) + + copy_to_container(env, src_dir, "/existing_dest") + assert dir_contents(env.sandbox_dir / "existing_dest") == { + "old.txt": "old", + "mydir/a.txt": "aaa", + "mydir/sub/b.txt": "bbb", + } + + def test_dir_to_dest_with_trailing_slash(self, mock_env, tmp_path): + env = mock_env() + src_dir = tmp_path / "mydir" + populate_dir(src_dir, {"a.txt": "aaa"}) + + copy_to_container(env, src_dir, "/newdest/") + assert dir_contents(env.sandbox_dir / "newdest") == {"a.txt": "aaa"} + + def test_file_with_relative_dest_path(self, mock_env, tmp_path): + env = mock_env(cwd="/workspace") + src = tmp_path / "file.txt" + src.write_text("relative") + + copy_to_container(env, src, "subdir/file.txt") + assert (env.sandbox_dir / "workspace/subdir/file.txt").read_text() == "relative" + + +# ============================================================ +# copy_from_container +# ============================================================ + + +class TestCopyFromContainer: + def test_file_to_nonexistent_dest(self, mock_env, tmp_path): + env = mock_env() + populate_dir(env.sandbox_dir / "data", {"file.txt": "from_container"}) + + dest = tmp_path / "output/file.txt" + copy_from_container(env, "/data/file.txt", dest) + assert dest.read_text() == "from_container" + + def test_file_to_existing_directory(self, mock_env, tmp_path): + """shutil.copy2(file, dir) copies file INTO dir.""" + env = mock_env() + populate_dir(env.sandbox_dir / "data", {"file.txt": "from_container"}) + + dest_dir = tmp_path / "output" + dest_dir.mkdir() + + copy_from_container(env, "/data/file.txt", dest_dir) + assert (dest_dir / "file.txt").read_text() == "from_container" + + def test_dir_to_nonexistent_dest(self, mock_env, tmp_path): + env = mock_env() + populate_dir(env.sandbox_dir / "data/mydir", {"a.txt": "aaa", "sub/b.txt": "bbb"}) + + dest = tmp_path / "output" + copy_from_container(env, "/data/mydir", dest) + assert dir_contents(dest) == {"a.txt": "aaa", "sub/b.txt": "bbb"} + + def test_dir_to_existing_directory(self, mock_env, tmp_path): + """docker cp copies src dir INTO existing dest as dest/basename(src)/...""" + env = mock_env() + populate_dir(env.sandbox_dir / "data/mydir", {"a.txt": "aaa", "sub/b.txt": "bbb"}) + + dest = tmp_path / "output" + dest.mkdir() + (dest / "old.txt").write_text("old") + + copy_from_container(env, "/data/mydir", dest) + assert dir_contents(dest) == { + "old.txt": "old", + "mydir/a.txt": "aaa", + "mydir/sub/b.txt": "bbb", + } + + def test_dir_dot_to_nonexistent_dest(self, mock_env, tmp_path): + """/. on non-existent dest creates dest with contents.""" + env = mock_env() + populate_dir(env.sandbox_dir / "data/mydir", {"a.txt": "aaa", "sub/b.txt": "bbb"}) + + dest = tmp_path / "output" + copy_from_container(env, "/data/mydir/.", dest) + assert dir_contents(dest) == {"a.txt": "aaa", "sub/b.txt": "bbb"} + + def test_dir_dot_to_existing_directory(self, mock_env, tmp_path): + """/. merges contents into existing dest.""" + env = mock_env() + populate_dir(env.sandbox_dir / "data/mydir", {"a.txt": "aaa", "sub/b.txt": "bbb"}) + + dest = tmp_path / "output" + dest.mkdir() + (dest / "old.txt").write_text("old") + + copy_from_container(env, "/data/mydir/.", dest) + assert dir_contents(dest) == {"old.txt": "old", "a.txt": "aaa", "sub/b.txt": "bbb"} + + def test_removesuffix_with_dotted_dir_name(self, mock_env, tmp_path): + """Ensure /. suffix stripping doesn't eat dots in directory names.""" + env = mock_env() + populate_dir(env.sandbox_dir / "data/v1.0.0", {"release.txt": "v1"}) + + dest = tmp_path / "output" + copy_from_container(env, "/data/v1.0.0/.", dest) + assert dir_contents(dest) == {"release.txt": "v1"} + + def test_removesuffix_with_trailing_dots_in_name(self, mock_env, tmp_path): + """Directory name ending with dots must not be corrupted by /. stripping.""" + env = mock_env() + populate_dir(env.sandbox_dir / "data/logs...", {"out.txt": "log data"}) + + dest = tmp_path / "output" + copy_from_container(env, "/data/logs.../.", dest) + assert dir_contents(dest) == {"out.txt": "log data"} + + def test_trailing_slash_on_src(self, mock_env, tmp_path): + """Trailing / on src (not /.) should not change behavior.""" + env = mock_env() + populate_dir(env.sandbox_dir / "data/mydir", {"a.txt": "aaa"}) + + dest = tmp_path / "output" + copy_from_container(env, "/data/mydir/", dest) + assert dir_contents(dest) == {"a.txt": "aaa"} + + +# ============================================================ +# copy_between_containers +# ============================================================ + + +class TestCopyBetweenContainers: + def test_dir_to_nonexistent_dest(self, mock_env): + env1 = mock_env() + env2 = mock_env() + populate_dir( + env1.sandbox_dir / "workspace", + {"a.txt": "aaa", "sub/b.txt": "bbb", ".git/config": "gitdata", "__pycache__/cache.pyc": "cache"}, + ) + + copy_between_containers(env1, env2, "/workspace", "/player1") + assert dir_contents(env2.sandbox_dir / "player1") == {"a.txt": "aaa", "sub/b.txt": "bbb"} + + def test_dir_to_existing_dest(self, mock_env): + """docker cp copies src dir INTO existing dest.""" + env1 = mock_env() + env2 = mock_env() + populate_dir(env1.sandbox_dir / "workspace", {"a.txt": "aaa"}) + populate_dir(env2.sandbox_dir / "player1", {"old.txt": "old"}) + + copy_between_containers(env1, env2, "/workspace", "/player1") + assert dir_contents(env2.sandbox_dir / "player1") == { + "old.txt": "old", + "workspace/a.txt": "aaa", + } + + def test_trailing_slash_on_nonexistent_dest(self, mock_env): + env1 = mock_env() + env2 = mock_env() + populate_dir(env1.sandbox_dir / "workspace", {"a.txt": "aaa"}) + + copy_between_containers(env1, env2, "/workspace", "/opponent_codebases/agent1/") + assert dir_contents(env2.sandbox_dir / "opponent_codebases/agent1") == {"a.txt": "aaa"} + + def test_excludes_git_and_pycache(self, mock_env): + env1 = mock_env() + env2 = mock_env() + populate_dir( + env1.sandbox_dir / "workspace", + {"code.py": "x", ".git/HEAD": "ref", "__pycache__/mod.pyc": "bytecode"}, + ) + + copy_between_containers(env1, env2, "/workspace", "/dest") + assert dir_contents(env2.sandbox_dir / "dest") == {"code.py": "x"} + + +# ============================================================ +# create_file_in_container +# ============================================================ + + +class TestCreateFileInContainer: + def test_absolute_path(self, mock_env): + env = mock_env() + create_file_in_container(env, content="hello world", dest_path="/data/test.txt") + assert (env.sandbox_dir / "data/test.txt").read_text() == "hello world" + + def test_relative_path(self, mock_env): + env = mock_env(cwd="/workspace") + create_file_in_container(env, content="relative content", dest_path="sub/test.txt") + assert (env.sandbox_dir / "workspace/sub/test.txt").read_text() == "relative content" + + def test_overwrite_existing(self, mock_env): + env = mock_env() + (env.sandbox_dir / "data").mkdir() + (env.sandbox_dir / "data/test.txt").write_text("old") + + create_file_in_container(env, content="new", dest_path="/data/test.txt") + assert (env.sandbox_dir / "data/test.txt").read_text() == "new"