diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml new file mode 100644 index 0000000..83a2e0b --- /dev/null +++ b/.github/workflows/checks.yml @@ -0,0 +1,25 @@ +name: checks + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +jobs: + python: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check out repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.11" + - name: Compile project sources + run: python -m compileall -q src scripts configs + - name: Run dependency-free unit tests + run: python -m unittest discover -s tests -v diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md index 6d9e77c..515c3fb 100644 --- a/README.md +++ b/README.md @@ -1,220 +1,212 @@ -# Agent0: A Reproduction Study +# Agent0 Evaluation Harness -> **Reproducing Self-Evolving Agents via Tool-Integrated Reasoning** +> OpenCompass benchmark tooling for evaluating `gpt-4o-mini` on GSM8K and MATH. -[![arXiv](https://img.shields.io/badge/arXiv-2511.16043-b31b1b.svg)](https://arxiv.org/abs/2511.16043) +[![Agent0 paper](https://img.shields.io/badge/arXiv-2511.16043-b31b1b.svg)](https://arxiv.org/abs/2511.16043) [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE) [![OpenCompass](https://img.shields.io/badge/Eval-OpenCompass-green.svg)](https://github.com/open-compass/opencompass) -## Abstract +> [!IMPORTANT] +> This repository does **not** reproduce Agent0's curriculum/executor co-training. Its recorded scores are evaluations of an off-the-shelf `gpt-4o-mini` endpoint, not gains produced by the Agent0 training method. -This repository provides an independent reproduction of the **Agent0** framework introduced by Xia et al. (2025)[^1]. Agent0 proposes a fully autonomous paradigm for training language model agents without external data through multi-step co-evolution between a *curriculum agent* (task proposer) and an *executor agent* (task solver), augmented with tool-integrated reasoning. +## Scope -Our reproduction validates the mathematical reasoning improvements reported in the original work using OpenAI's `gpt-4o-mini` as the base model, achieving **82.79%** on GSM8K and **69.62%** on MATH benchmarks - consistent with the gains demonstrated in the paper. +This project is an independent evaluation harness inspired by the +[Agent0 paper](https://arxiv.org/abs/2511.16043). It provides: -## 1. Introduction +- OpenCompass dataset configurations for GSM8K and MATH; +- an OpenAI-compatible model shim configured through environment variables; +- scripts for launching, monitoring, and promoting long-running benchmark results; and +- compact, versioned summaries from two `gpt-4o-mini` evaluation runs. -Large Language Model (LLM) agents trained with reinforcement learning face a fundamental constraint: dependence on human-curated data limits scalability and tethers model capabilities to existing human knowledge. Xia et al. address this limitation by introducing Agent0, which establishes a self-reinforcing loop between two co-evolving agents: +The repository also contains experimental curriculum/executor orchestration, +frontier-filtering, reward-shaping, and trainer interfaces. Those modules are +prototype scaffolding. They have not demonstrated end-to-end co-training, +produced a trained Agent0 checkpoint, or reproduced the paper's reported +training improvements. The current FlexRL backend logs batch sizes rather than +performing optimizer updates, and optional TRL bridges require separately +configured local models. -1. **Curriculum Agent**: Proposes increasingly challenging *frontier tasks* calibrated to the executor's current skill level. -2. **Executor Agent**: Learns to solve these tasks using external tools (e.g., Python interpreter, calculators). +For the authors' implementation of the training method, see +[aiming-lab/Agent0](https://github.com/aiming-lab/Agent0). -This symbiotic competition - where executor improvement pressures the curriculum to propose harder tasks - yields a self-sustaining training signal without external supervision[^1]. +## What the benchmark establishes -### 1.1 Contributions of This Reproduction +The checked-in summaries show that the harness can send GSM8K and MATH prompts +through OpenCompass to an OpenAI-compatible `gpt-4o-mini` endpoint and collect +the corresponding accuracy metrics. -- Full-fidelity evaluation harness using OpenCompass[^2] against GSM8K[^3] and MATH[^4] benchmarks. -- Automated monitoring infrastructure for long-running (~20h) evaluation sweeps. -- Reproducible environment configuration targeting real OpenAI endpoints (no mock layers). +They do **not** establish: -## 2. Repository Structure +- curriculum-agent or executor-agent parameter updates; +- multi-step co-evolution between those agents; +- a causal improvement over a frozen-model baseline; +- parity with the paper's Qwen-based training setup; or +- independent verification of the historical runs from raw artifacts. -``` +The full OpenCompass work directories are gitignored; this repository retains +the configurations and concise result summaries. + +## Repository layout + +```text Agent0/ -├── configs/ # Hyperparameters (GRPO, ADPO, tool rewards) -│ └── opencompass/ # Model shims and dataset configurations -├── data/ # Frontier buffers, rollouts, judge responses -├── docs/ # Research methodology notes -├── reports/ # Evaluation summaries, iteration reports -│ └── evals/ # Promoted benchmark results -├── scripts/ # Entrypoints and utilities -│ ├── run_eval.py # Primary evaluation driver -│ ├── run_opencompass_eval.py # Low-level OpenCompass wrapper -│ ├── monitor_opencompass.py # Rich-based progress dashboard -│ └── promote_eval_results.py # Result archival utility -├── src/ -│ ├── agents/ # Curriculum/executor wrappers -│ ├── pipeline/ # Filtering, self-consistency, judge clients -│ ├── tools/ # Sandbox orchestration -│ └── training/ # GRPO, ADPO, rollout managers -└── outputs/ # OpenCompass artifacts +├── configs/ +│ └── opencompass/ # Model and GSM8K/MATH dataset configs +├── docs/ # Research planning notes +├── reports/ +│ └── evals/ # Checked-in benchmark summaries +├── scripts/ +│ ├── run_eval.py # Evaluation entry point +│ ├── run_opencompass_eval.py # OpenCompass command builder +│ ├── monitor_opencompass.py # Progress dashboard +│ └── promote_eval_results.py # Summary promotion utility +└── src/ + ├── agents/ # Experimental agent clients + ├── pipeline/ # Filtering, judging, and rewards + ├── tools/ # Sandbox integration + └── training/ # Experimental trainer scaffolding ``` -## 3. Environment Setup +## Setup -### 3.1 Prerequisites +### Prerequisites -- **Python**: 3.11+ -- **macOS**: Tested on Apple Silicon; Linux should work with minor path adjustments. -- **API Access**: Valid OpenAI API key with access to `gpt-4o-mini`. +- Python 3.11+ +- an OpenAI API key with access to `gpt-4o-mini` +- macOS or Linux (the recorded long runs used Apple Silicon macOS) -### 3.2 Installation +Create an environment and install the evaluation dependencies: ```bash -cd Agent0 python -m venv .venv source .venv/bin/activate -pip install --upgrade pip -pip install torch==2.4.1 opencompass==0.5.1 wandb==0.17.8 rich pyyaml python-dotenv +python -m pip install --upgrade pip +python -m pip install opencompass==0.5.1 rich pyyaml python-dotenv ``` -### 3.3 Configuration - -Create a `.env` file (gitignored) with your credentials: +Create a gitignored `.env` file: -```env -OPENAI_API_KEY=sk-... - -# Evaluation endpoint configuration +```dotenv +OPENAI_API_KEY=replace-with-your-key AGENT0_VLLM_BASE=https://api.openai.com/v1 AGENT0_EVAL_MODEL=gpt-4o-mini -AGENT0_EVAL_API_KEY=${OPENAI_API_KEY} - -# Optional: W&B telemetry -AGENT0_USE_WANDB=0 -AGENT0_WANDB_PROJECT=agent0-repro +AGENT0_EVAL_API_KEY=replace-with-your-key ``` -## 4. Evaluation Protocol +Never commit real credentials. OpenCompass evaluation calls can incur API +charges; review the selected datasets and endpoint before starting a run. -We evaluate using OpenCompass[^2], following the same benchmark suite as the original paper. +## Evaluation Harness (OpenCompass) -### 4.1 Running Evaluations +Inspect the generated command without making API calls: -**Standard execution:** ```bash -python scripts/run_eval.py --suite math-lite --max-workers 1 +python scripts/run_eval.py --suite math-lite --dry-run ``` -**Production run with monitoring and auto-promotion (recommended):** +Run the GSM8K and MATH suite: + ```bash -caffeinate -di sh -c 'source .venv/bin/activate && \ - python scripts/run_eval.py --suite math-lite \ - --work-dir outputs/opencompass/$(date +%Y%m%d) \ - --max-workers 1 --monitor --promote' +python scripts/run_eval.py --suite math-lite --max-workers 1 ``` -The `--monitor` flag launches a Rich-powered dashboard displaying per-dataset progress, ETAs, and stall detection. The `--promote` flag automatically archives results to `reports/evals/` and updates this README upon completion. - -### 4.2 Monitoring Long Runs +Use the built-in monitor and promote a completed summary: -For runs already in progress: ```bash -python scripts/monitor_opencompass.py +python scripts/run_eval.py \ + --suite math-lite \ + --work-dir outputs/opencompass \ + --max-workers 1 \ + --monitor \ + --promote ``` -The monitor displays: -- Per-dataset completion percentage -- Problems per minute throughput -- Estimated time remaining -- Stall detection (5-minute threshold) +For an existing work directory: -## 5. Results - -### 5.1 Benchmark Scores +```bash +python scripts/monitor_opencompass.py outputs/opencompass +``` -We report results across two independent runs to assess variance: +## Recorded results -| Run | Date | GSM8K | MATH | Runtime | -|-----|------|-------|------|---------| -| 1 | 2025-11-28 | 82.79 | 70.38 | ~20h | -| 2 | 2025-11-29 | 82.79 | 69.62 | ~19.5h | -| **Mean** | - | **82.79** | **70.00** | - | -| **Std** | - | ±0.00 | ±0.38 | - | +| Run | Date | Endpoint model | GSM8K | MATH | Report | +| --- | --- | --- | ---: | ---: | --- | +| 1 | 2025-11-28 | `gpt-4o-mini` | 82.79 | 70.38 | [summary](reports/evals/20251128_mathlite.md) | +| 2 | 2025-11-29 | `gpt-4o-mini` | 82.79 | 69.62 | [summary](reports/evals/20251129_151443.md) | -### 5.2 Latest Benchmark Snapshot +### Latest Benchmark Snapshot | Dataset | Config | Metric | Mode | Score | -|---------|--------|--------|------|-------| +| --- | --- | --- | --- | ---: | | GSM8K | `gsm8k_gen_1d7fe4` | accuracy | `gen` | **82.79** | -| MATH | `math_gen_393424` | accuracy | `gen` | **69.62** | +| MATH | `math_0shot_gen_393424` | accuracy | `gen` | **69.62** | -> *Evaluated using `gpt-4o-mini` via OpenAI API. Single-worker inference, ~13s/problem average on MATH.* +These are historical endpoint-evaluation results. A direct numerical comparison +with the paper is not valid because the model, training state, and evaluation +protocol differ. API-backed results can also vary across provider revisions +even when temperature is set to zero. -### 5.3 Comparison with Original Results +## Experimental co-evolution code -The original Agent0 paper reports improvements over the Qwen3-8B-Base model[^1]: +`scripts/run_demo.py` wires together prototype curriculum and executor +clients, filtering, reward calculations, sandbox calls, and trainer adapters. +By default, the backend in `src/training/backends.py` records batch sizes +rather than performing a real optimizer update. Optional TRL adapters are +activated only when separately supplied model settings are present. -| Model | GSM8K | MATH (avg) | -|-------|-------|------------| -| Qwen3-8B-Base | 89.1 | 52.0 | -| Qwen3-8B-Base + Tool | 90.7 | 60.3 | -| **+ Agent0** | **94.5** | **62.4** | +Treat this path as exploratory code, not as evidence that the Agent0 algorithm +has been reproduced. -Our reproduction uses `gpt-4o-mini` rather than fine-tuned Qwen checkpoints, so direct numerical comparison is not applicable. However, our results demonstrate that the evaluation infrastructure correctly interfaces with OpenAI-compatible endpoints and produces stable, reproducible scores. +## Security and data handling -## 6. Implementation Notes +- Keep endpoint credentials in the gitignored `.env` file and scope them to the + minimum models and spend required for a run. +- Model-generated Python is delegated to the external SandFuzz executable. This + repository does not treat that boundary as hardened isolation; run it in a + disposable environment without sensitive files or unrestricted network access. +- The sandbox child process receives only a small allowlist of non-secret + environment variables. Tool code, stdout, stderr, prompts, and rollout data can + still be sensitive and are written beneath gitignored local artifact paths. +- Review generated artifacts before sharing them. The configured telemetry + labels are documentation, not a general-purpose content redaction system. -### 6.1 Model Shim +## Known limitations -The OpenCompass model shim (`configs/opencompass/models/agent0_vllm.py`) wraps any OpenAI-compatible endpoint: +1. There is no end-to-end, validated curriculum/executor co-training run. +2. No trained checkpoint or controlled before/after comparison is published. +3. Raw OpenCompass output directories are not included in version control. +4. The environment is documented but not lockfile-pinned. +5. The recorded runs use a hosted model that can change independently of this repository. -```python -# Key environment variables: -# AGENT0_VLLM_BASE - API base URL -# AGENT0_EVAL_MODEL - Model identifier -# AGENT0_EVAL_API_KEY - Authentication token -``` - -### 6.2 Dataset Configuration - -The `math-lite` suite includes: -- **GSM8K**: 1,319 grade-school math word problems[^3] -- **MATH**: 5,000 competition-level problems across algebra, geometry, and calculus[^4] - -### 6.3 Known Limitations +## Roadmap -1. **Single-worker throughput**: API rate limits constrain parallelism; expect ~20h for full `math-lite` suite. -2. **macOS sleep prevention**: Long runs require `caffeinate -di` to prevent system sleep. -3. **Progress visibility**: OpenCompass master progress bar updates only after first dataset completes; use `--monitor` for real-time visibility. +- [ ] Implement and test real curriculum/executor optimizer updates. +- [ ] Add unit and integration tests for the project-owned modules. +- [ ] Publish sanitized run manifests with model, prompt, dependency, and commit metadata. +- [ ] Add frozen-model baselines and controlled ablations before making training claims. -## 7. Roadmap +## Citation -- [ ] Integrate BBH (Big-Bench Hard) benchmark suite -- [ ] Add curriculum-executor co-training loop with GRPO/ADPO -- [ ] Implement self-consistency band filtering from original paper -- [ ] Scale to multi-GPU vLLM endpoints for improved throughput - -## 8. Citation - -If you use this reproduction in your research, please cite the original Agent0 paper: +If this harness informs research on Agent0, cite the original paper: ```bibtex @article{xia2025agent0, title={Agent0: Unleashing Self-Evolving Agents from Zero Data via Tool-Integrated Reasoning}, - author={Xia, Peng and Zeng, Kaide and Liu, Jiaqi and Qin, Can and Wu, Fang and + author={Xia, Peng and Zeng, Kaide and Liu, Jiaqi and Qin, Can and Wu, Fang and Zhou, Yiyang and Xiong, Caiming and Yao, Huaxiu}, journal={arXiv preprint arXiv:2511.16043}, year={2025} } ``` -## 9. Acknowledgments - -We thank the authors of Agent0 for releasing their methodology and the [AIMING Lab](https://aiming-lab.github.io/Agent0) for maintaining the official repository. This reproduction uses [OpenCompass](https://github.com/open-compass/opencompass) for evaluation infrastructure. - -## References - -[^1]: Xia, P., Zeng, K., Liu, J., Qin, C., Wu, F., Zhou, Y., Xiong, C., & Yao, H. (2025). Agent0: Unleashing Self-Evolving Agents from Zero Data via Tool-Integrated Reasoning. *arXiv preprint arXiv:2511.16043*. https://arxiv.org/abs/2511.16043 - -[^2]: OpenCompass Contributors. (2024). OpenCompass: A Universal Evaluation Platform for Foundation Models. https://github.com/open-compass/opencompass - -[^3]: Cobbe, K., Kosaraju, V., Bavarian, M., Chen, M., Jun, H., Kaiser, L., Plappert, M., Tworek, J., Hilton, J., Nakano, R., Hesse, C., & Schulman, J. (2021). Training Verifiers to Solve Math Word Problems. *arXiv preprint arXiv:2110.14168*. - -[^4]: Hendrycks, D., Burns, C., Kadavath, S., Arber, A., Basart, S., Tang, E., Song, D., & Steinhardt, J. (2021). Measuring Mathematical Problem Solving With the MATH Dataset. *arXiv preprint arXiv:2103.03874*. - ---- +## License -*This reproduction study is not affiliated with the original Agent0 authors. For the official implementation, see [aiming-lab/Agent0](https://github.com/aiming-lab/Agent0).* +Licensed under the [Apache License 2.0](LICENSE). +## Acknowledgments +This repository is not affiliated with the Agent0 authors. It uses +[OpenCompass](https://github.com/open-compass/opencompass) for evaluation and +credits the [Agent0 authors](https://aiming-lab.github.io/Agent0) for the +research direction. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..819ca2e --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,20 @@ +# Security policy + +## Scope + +This repository is a research prototype. Its model-generated code adapter is not +a hardened isolation boundary, and the project is not intended to process +untrusted workloads or production secrets. + +Security fixes are maintained on the default branch only. Historical experiment +artifacts and upstream services are outside this repository's support scope. + +## Reporting + +Please report a suspected vulnerability through GitHub's private vulnerability +reporting flow for this repository. Do not include credentials, private prompts, +or user data in a public issue. + +Include the affected commit, a minimal reproduction, expected impact, and any +safe mitigation you have identified. No bug-bounty program or authorization to +test third-party systems is implied. diff --git a/configs/tooling.yaml b/configs/tooling.yaml index cc24d64..50b2c20 100644 --- a/configs/tooling.yaml +++ b/configs/tooling.yaml @@ -7,7 +7,7 @@ sandbox: memory_limit_mb: 768 artifact_dir: data/tool_runs stop_go: - trigger_regex: "```python[\\s\\S]*?```" + trigger_regex: "```python\\s*\\n([\\s\\S]*?)```" max_code_blocks: 5 capture_stdout: true capture_stderr: true diff --git a/docs/research_methodology.md b/docs/research_methodology.md index e0ecc67..3793703 100644 --- a/docs/research_methodology.md +++ b/docs/research_methodology.md @@ -1,37 +1,54 @@ -# Agent0 Demo Research Methodology - -## Objectives -- Reproduce the Agent0 co-evolution paradigm (curriculum + executor) with tool-augmented RL on commodity hardware. -- Deliver an auditable, extensible demo that mirrors the paper's academic rigor and enables rapid iteration on reward design, tool orchestration, and evaluation. - -## Guiding Principles -1. **Problem Framing**: Maintain a living positioning memo capturing hypotheses, assumptions, and success metrics (pass@1 uplift, tool-usage efficiency, curriculum diversity). -2. **Feasibility Spikes**: Run small pilot jobs (toy math tasks, 1–2 tool calls) before scaling RL loops. Log every spike with configs + seeds. -3. **Modular Infrastructure**: Config-first code (YAML), reproducible env via `uv` + Brew, W&B tracking, and deterministic data pipelines. -4. **Iterative Validation**: Unit tests for sandbox/tooling, ablations for algorithms, baseline comparisons against frozen Qwen3 with/without tool. -5. **Transparent Collaboration**: Weekly memos, structured PRs, TODO backlog, and disciplined experiment logging. -6. **Reproducibility**: Version datasets/checkpoints (DVC-ready), archive prompts, judge scripts, and hardware notes. -7. **Reflection Loops**: Post-iteration retros, adjust hypotheses, document learnings before expanding scope. - -## Workflow Stages -1. **Literature + Benchmark Review**: Summaries of Agent0, R-Zero, Socratic-Zero, SPIRAL, plus current (Nov 2025) tool ecosystems. -2. **Environment Bring-Up**: Brew installs (`uv`, `sandfuzz`, `qwen-vllm`), `uv` env, GPU/remote access notes. -3. **Tooling Validation**: Deterministic SandFuzz sandbox tests, stop-go protocol harness, error capture. -4. **Curriculum Agent Prototyping**: Prompt tuning, reward signal simulation (uncertainty/tool/repetition) using synthetic executor responses. -5. **Executor Pilot**: Self-consistency voting, ADPO math, stress tests on ambiguous tasks. -6. **Full Co-Evolution Loop**: Alternating curriculum/executor training with frontier filtering, logging, and judge integration. -7. **Evaluation + Reporting**: OpenCompass harness, GPT-4o Mini verifier, pass@k metrics, difficulty drift analysis, and ablation schedule. - -## Milestones -- M0: Repo scaffold + tooling smoke tests. -- M1: Curriculum/executor prompts finalized; sandbox loop demo. -- M2: Frontier filtering + ADPO training over mini dataset. -- M3: Full demo iteration with telemetry dashboards and report. -- M4: Benchmark sweep + ablations aligned with paper tables. - -## Risk Mitigation -- **Judge Dependence**: Provide pluggable verifier (GPT-4o Mini vs local sympy grader) to avoid single point of failure. -- **Tool Abuse**: Hard cap tool rewards, monitor SandFuzz logs, add anti-loop heuristics. -- **Hardware Limits**: Parameterize batch sizes and rollout counts; support CPU-only debug mode. - -This document serves as the Stanford-style methodology reference for all subsequent workstreams. +# Historical implementation plan + +> Planning note, written before the prototype was evaluated. Items below are +> proposed work, not completed experiments or evidence of Agent0 reproduction. + +## Research question + +Can a small, auditable harness reproduce parts of Agent0's curriculum/executor +orchestration and eventually test co-training claims under controlled baselines? + +The current repository does not answer that question. It contains endpoint +clients, filtering and reward prototypes, trainer interfaces, and an independent +OpenCompass evaluation of an off-the-shelf model. It does not contain a trained +checkpoint or a validated optimizer-backed co-evolution run. + +## Proposed method + +1. Define hypotheses and success criteria before each experiment, including a + frozen-model baseline and a clear primary metric. +2. Validate tool execution and logging on small synthetic tasks before running + model-generated code or paid endpoints at scale. +3. Version the exact configuration, commit, prompt templates, dataset revisions, + seeds, checkpoint identifiers, and sanitized run artifacts. +4. Separate orchestration signals from parameter updates. A reward calculation, + accepted frontier sample, or backend callback is not evidence of learning. +5. Compare trained and frozen baselines under the same model, tools, prompts, + token budget, and evaluation protocol. +6. Report failed runs and uncertainty alongside successful results. + +## Proposed milestones + +- **M0 — Harness checks:** unit tests for filtering, rewards, telemetry, and the + sandbox adapter. +- **M1 — Orchestration smoke test:** versioned traces from a small curriculum and + executor loop, without a training claim. +- **M2 — Optimizer integration:** prove that configured trainer steps update a + checkpoint and record before/after hashes. +- **M3 — Controlled experiment:** compare trained and frozen baselines with + retained manifests and repeated runs. +- **M4 — External evaluation:** run a preregistered benchmark and publish enough + sanitized evidence for independent review. + +## Primary risks + +- **Untrusted code:** run model-generated code in a disposable, least-privilege + environment; the adapter is not itself a hardened security boundary. +- **Judge dependence:** use deterministic graders where possible and measure + sensitivity to judge choice. +- **Hosted-model drift:** record provider model identifiers and dates, and avoid + claiming deterministic reproduction from API-backed scores alone. +- **Missing provenance:** treat a metric without its raw manifest and artifacts as + a historical observation, not a reproducible result. +- **Hardware constraints:** keep pilot runs small and publish the actual compute + and wall-clock budget. diff --git a/reports/evals/20251128_mathlite.md b/reports/evals/20251128_mathlite.md index dc98547..f7e2a0d 100644 --- a/reports/evals/20251128_mathlite.md +++ b/reports/evals/20251128_mathlite.md @@ -1,23 +1,35 @@ -# Math-Lite Benchmark (2025-11-28 Run) - -- **Command**: `caffeinate -di sh -c 'source .venv/bin/activate && python scripts/run_eval.py --suite math-lite --work-dir outputs/opencompass/progress_smoke --max-workers 1'` -- **Work dir**: `outputs/opencompass/progress_smoke/20251128_180834` -- **Executor endpoint**: `agent0_vllm` shim pointing at the live OpenAI-compatible gateway (same settings as `.env`). -- **Elapsed time**: ~20 h 32 m for inference + 15 s for evaluation aggregation (single worker). -- **Notes**: - - SandFuzz tool-workers were kept alive throughout; `caffeinate -di` prevented macOS sleep. - - `scripts/run_eval.py` streamed per-dataset progress while OpenCompass wrote detailed logs to `logs/infer/`. - - GSM8K and MATH were executed back-to-back with real OpenAI API calls (no mocks). - -## Results -| Dataset | Config | Metric | Mode | Score | -| --- | --- | --- | --- | --- | -| GSM8K | `gsm8k_gen_1d7fe4` | Accuracy | `gen` | **82.79** | -| MATH | `math_0shot_gen_393424` | Accuracy | `gen` | **70.38** | +# Historical OpenCompass summary — 2025-11-28 + +> This file preserves a concise report from an API-backed evaluation. The raw +> OpenCompass work directory, prompts, responses, dependency lock, and provider +> snapshot are not versioned here, so the scores are not independently +> reproducible from this repository alone. + +## Recorded run metadata + +- **Endpoint model:** `gpt-4o-mini` +- **Harness:** OpenCompass through the repository's `agent0_vllm` shim +- **Worker count:** 1 +- **Reported elapsed time:** approximately 20 h 32 m for inference +- **Recorded command:** -> Source files: `summary/summary_20251128_180834.(txt|csv|md)` in the same work directory. + ```bash + python scripts/run_eval.py \ + --suite math-lite \ + --work-dir outputs/opencompass/progress_smoke \ + --max-workers 1 + ``` + +The original local work directory was recorded as +`outputs/opencompass/progress_smoke/20251128_180834`; it is not committed. + +## Recorded results + +| Dataset | Config | Metric | Mode | Score | +| --- | --- | --- | --- | ---: | +| GSM8K | `gsm8k_gen_1d7fe4` | accuracy | `gen` | 82.79 | +| MATH | `math_0shot_gen_393424` | accuracy | `gen` | 70.38 | -## Follow-Ups -1. Mirror these scores into the README so new runs have a reference point. -2. Trigger downstream reporting (`scripts/generate_iteration_report.py`) once the next co-evolution loop finishes. -3. Consider running with `--max-workers 2` if rate limits permit; current single-worker throughput yields ~18 h wall-clock for MATH. +These values describe the hosted model at the time of the run. They do not show +an Agent0 training gain, a curriculum/executor parameter update, or parity with +the paper's model and evaluation protocol. diff --git a/reports/evals/20251129_151443.md b/reports/evals/20251129_151443.md index ef71a79..f05d792 100644 --- a/reports/evals/20251129_151443.md +++ b/reports/evals/20251129_151443.md @@ -1,4 +1,13 @@ -| dataset | version | metric | mode | agent0-vllm | -|----- | ----- | ----- | ----- | -----| -| gsm8k | 1d7fe4 | accuracy | gen | 82.79 | -| math | 393424 | accuracy | gen | 69.62 | +# Historical OpenCompass summary — 2025-11-29 + +> Retained metric summary from an API-backed `gpt-4o-mini` evaluation. Raw +> prompts, responses, logs, dependency state, and provider snapshot are not +> committed, so this file is provenance-limited and not an independent +> reproduction package. + +| Dataset | Config version | Metric | Mode | Recorded score | +| --- | --- | --- | --- | ---: | +| GSM8K | `1d7fe4` | accuracy | `gen` | 82.79 | +| MATH | `393424` | accuracy | `gen` | 69.62 | + +The scores do not establish an Agent0 training gain or a model parameter update. diff --git a/reports/iter_openai.md b/reports/iter_openai.md index 0423af2..23bcd30 100644 --- a/reports/iter_openai.md +++ b/reports/iter_openai.md @@ -1,46 +1,57 @@ -# Agent0 Iteration Report Template +# Prototype Orchestration Smoke Test -> Duplicate this file into `reports/iter_XXX.md` and replace bracketed tokens. +> Historical, partial log from 2025-11-26. This was not a completed +> curriculum/executor co-training run. Raw telemetry and rollout artifacts are +> not versioned, so the retained summary values are not independently +> reproducible. + +## Run metadata -## Run Metadata - **Date:** 2025-11-26 -- **Run Name:** iter_openai -- **Telemetry File:** reports/telemetry_openai.jsonl -- **Git Commit:** unknown -- **Data Sources:** N/A - -## Curriculum Summary -- Total prompts sampled: 2 -- Mean reward: 0.4830 -- Reward breakdown plot: attach from W&B or `reports/figures/reward_breakdown.png`. -- Notable task seeds: - 1. GSM8K seed - 2. MATH seed - -## Frontier Filtering +- **Run name:** `iter_openai` +- **Git commit:** Not recorded +- **Source artifacts:** Not retained in version control + +## Curriculum summary + +- Prompts sampled: 2 +- Reported mean reward: 0.4830 +- Reward breakdown figure: Not captured +- Task seed labels: GSM8K and MATH + +The reward is a computed orchestration signal; it is not evidence of a model +parameter update. + +## Frontier filtering + - Accepted / total: 0 / 2 - Consistency band: [0.30, 0.80] - Judge pass rate: 0.0000 -- Common rejection reasons: repetition=0, out-of-band=0 - -## Executor Rollouts -- Mean turns per rollout: 5.33 -- Tool events per rollout (p50 / p90): 2.00 / 2.50 -- Example trace: link to `data/rollouts/(attach rollout file)` - -## Training Signals -- GRPO reward mean/std: 0.4830 / 0.0010 -- ADPO adv mean/std: N/A / N/A -- TRL backend status: not configured - -## OpenCompass Snapshot (optional) -- Suite: math-lite -- Key metrics: - - GSM8K acc: N/A - - MATH acc: N/A - - BBH avg: N/A - -## Postmortem / Next Actions -- Successes: TBD -- Issues: TBD -- Next iteration focus: TBD +- Detailed rejection reasons: Not retained + +## Executor rollouts + +- Reported mean turns per rollout: 5.33 +- Reported tool events per rollout (p50 / p90): 2.00 / 2.50 +- Example trace: Not retained + +## Training signals + +- Reported GRPO reward mean / standard deviation: 0.4830 / 0.0010 +- ADPO advantage: Not produced because the frontier was empty +- TRL backend: Not configured +- Optimizer update: Not demonstrated + +## OpenCompass evaluation + +OpenCompass evaluation was not part of this smoke test. See the separate +[2025-11-28](evals/20251128_mathlite.md) and +[2025-11-29](evals/20251129_151443.md) benchmark summaries. + +## Outcome + +- The prototype generated and scored two curriculum prompts. +- No samples entered the executor-training frontier. +- No configured training backend or retained artifacts establish a model update. +- A future training claim would require a versioned commit, sanitized run + manifest, retained traces, checkpoint identifiers, and controlled baselines. diff --git a/reports/templates/iteration_report.md b/reports/templates/iteration_report.md index 3dcf4ed..8444464 100644 --- a/reports/templates/iteration_report.md +++ b/reports/templates/iteration_report.md @@ -8,6 +8,8 @@ - **Telemetry File:** {{TELEMETRY_PATH}} - **Git Commit:** {{GIT_SHA}} - **Data Sources:** {{DATASETS}} +- **Base model / checkpoint:** {{MODEL_CHECKPOINT}} +- **Dependency lock or environment digest:** {{ENV_DIGEST}} ## Curriculum Summary - Total prompts sampled: {{CURRICULUM_BATCH}} @@ -32,6 +34,7 @@ - GRPO reward mean/std: {{GRPO_MEAN}} / {{GRPO_STD}} - ADPO adv mean/std: {{ADPO_MEAN}} / {{ADPO_STD}} - TRL backend status: {{TRL_STATUS}} +- Optimizer update evidence (before/after checkpoint hashes): {{UPDATE_EVIDENCE}} ## OpenCompass Snapshot (optional) - Suite: {{EVAL_SUITE}} diff --git a/scripts/generate_iteration_report.py b/scripts/generate_iteration_report.py index e47666c..a4e73f1 100644 --- a/scripts/generate_iteration_report.py +++ b/scripts/generate_iteration_report.py @@ -134,6 +134,9 @@ def _parse_args() -> argparse.Namespace: parser.add_argument("--date", default=_dt.date.today().isoformat()) parser.add_argument("--git-sha", default=None) parser.add_argument("--datasets", nargs="*", default=None) + parser.add_argument("--model-checkpoint", default="not recorded") + parser.add_argument("--env-digest", default="not recorded") + parser.add_argument("--update-evidence", default="not demonstrated") parser.add_argument("--curriculum-batch", type=int, default=None) parser.add_argument("--tasks", nargs="*", default=None, help="Notable curriculum seeds (ordered)") parser.add_argument("--trace-file", default="", help="Relative path to exemplar rollout JSON") @@ -181,6 +184,8 @@ def _build_placeholder_map(args: argparse.Namespace, stats) -> Dict[str, str]: "TELEMETRY_PATH": str(args.telemetry), "GIT_SHA": git_sha, "DATASETS": datasets, + "MODEL_CHECKPOINT": args.model_checkpoint, + "ENV_DIGEST": args.env_digest, "CURRICULUM_BATCH": _format_int(args.curriculum_batch or stats.counts.get("curriculum/reward")), "MEAN_REWARD": _format_float(grpo_mean), "TASK_1": task_1, @@ -202,6 +207,7 @@ def _build_placeholder_map(args: argparse.Namespace, stats) -> Dict[str, str]: "ADPO_MEAN": _format_float(adpo_mean), "ADPO_STD": _format_float(adpo_std), "TRL_STATUS": args.trl_status, + "UPDATE_EVIDENCE": args.update_evidence, "EVAL_SUITE": args.eval_suite, "GSM8K_ACC": _format_float(args.gsm8k_acc, precision=2), "MATH_ACC": _format_float(args.math_acc, precision=2), diff --git a/src/tools/python_sandbox.py b/src/tools/python_sandbox.py index b97c2f6..f3b48af 100644 --- a/src/tools/python_sandbox.py +++ b/src/tools/python_sandbox.py @@ -3,7 +3,7 @@ from dataclasses import dataclass from pathlib import Path -from typing import Dict, List, Optional +from typing import Dict, List, Optional, TypedDict import json import os @@ -12,6 +12,21 @@ import time +_SANDBOX_ENV_ALLOWLIST = ("PATH", "LANG", "LC_ALL", "LC_CTYPE", "TMPDIR", "SYSTEMROOT") +_UNSAFE_TASK_ID_CHARS = re.compile(r"[^A-Za-z0-9_.-]+") + + +def _sandbox_environment() -> Dict[str, str]: + """Return the small, non-secret environment passed to the sandbox CLI.""" + return {name: os.environ[name] for name in _SANDBOX_ENV_ALLOWLIST if name in os.environ} + + +def _safe_artifact_id(task_id: str) -> str: + """Keep artifact names inside the configured directory.""" + normalized = _UNSAFE_TASK_ID_CHARS.sub("_", task_id).strip("._") + return (normalized or "task")[:96] + + @dataclass class SandboxConfig: engine: str @@ -31,6 +46,20 @@ class StopGoConfig: capture_stderr: bool +class SandboxResult(TypedDict): + task_id: str + code: str + stdout: str + stderr: str + status: str + latency_s: float + + +class StopGoResult(TypedDict): + patched_response: str + tool_events: Optional[List[SandboxResult]] + + class PythonSandbox: """Delegates code blocks to SandFuzz CLI and captures stdout/stderr.""" @@ -40,15 +69,15 @@ def __init__(self, config: SandboxConfig) -> None: if self.config.log_path: self.config.log_path.parent.mkdir(parents=True, exist_ok=True) - def _log_payload(self, payload: Dict[str, str]) -> None: + def _log_payload(self, payload: SandboxResult) -> None: if not self.config.log_path: return with self.config.log_path.open("a", encoding="utf-8") as sink: sink.write(json.dumps(payload) + "\n") - def execute(self, code: str, task_id: str) -> Dict[str, str]: + def execute(self, code: str, task_id: str) -> SandboxResult: cmd = [ - "sandfuzz", + self.config.engine, "run", "--lang", self.config.python_runtime, @@ -65,6 +94,7 @@ def execute(self, code: str, task_id: str) -> Dict[str, str]: capture_output=True, timeout=self.config.execution_timeout_s + 2, check=False, + env=_sandbox_environment(), ) status = "ok" if proc.returncode == 0 else f"exit-{proc.returncode}" stdout = proc.stdout.decode("utf-8", errors="replace") @@ -72,13 +102,13 @@ def execute(self, code: str, task_id: str) -> Dict[str, str]: except FileNotFoundError: status = "missing-binary" stdout = "" - stderr = "sandfuzz binary not found" + stderr = f"{self.config.engine} binary not found" except subprocess.TimeoutExpired: status = "timeout" stdout = "" stderr = "execution exceeded timeout" latency = time.time() - start - payload = { + payload: SandboxResult = { "task_id": task_id, "code": code, "stdout": stdout, @@ -86,7 +116,7 @@ def execute(self, code: str, task_id: str) -> Dict[str, str]: "status": status, "latency_s": latency, } - artifact_path = self.config.artifact_dir / f"{task_id}.json" + artifact_path = self.config.artifact_dir / f"{_safe_artifact_id(task_id)}.json" artifact_path.write_text(json.dumps(payload, indent=2), encoding="utf-8") self._log_payload(payload) return payload @@ -100,13 +130,15 @@ def __init__(self, sandbox: PythonSandbox, cfg: StopGoConfig) -> None: self._cfg = cfg self._regex = re.compile(self._cfg.trigger_regex, re.MULTILINE) - def run(self, response: str, task_id: str) -> Dict[str, Optional[List[Dict[str, str]]]]: + def run(self, response: str, task_id: str) -> StopGoResult: matches = list(self._regex.finditer(response))[: self._cfg.max_code_blocks] if not matches: return {"patched_response": response, "tool_events": None} - patched = response - events: List[Dict[str, str]] = [] + patched_parts: List[str] = [] + cursor = 0 + events: List[SandboxResult] = [] for idx, match in enumerate(matches): + patched_parts.append(response[cursor : match.end()]) code = match.group(1) if match.groups() else match.group(0) exec_id = f"{task_id}_code{idx}" result = self._sandbox.execute(code, exec_id) @@ -117,5 +149,8 @@ def run(self, response: str, task_id: str) -> Dict[str, Optional[List[Dict[str, if self._cfg.capture_stderr and result.get("stderr"): snippets.append(f"[tool stderr]\n{result['stderr']}") trailer = "\n\n" + "\n".join(snippets) if snippets else "" - patched = patched.replace(match.group(0), match.group(0) + trailer, 1) + patched_parts.append(trailer) + cursor = match.end() + patched_parts.append(response[cursor:]) + patched = "".join(patched_parts) return {"patched_response": patched, "tool_events": events} diff --git a/tests/test_python_sandbox.py b/tests/test_python_sandbox.py new file mode 100644 index 0000000..e13acc7 --- /dev/null +++ b/tests/test_python_sandbox.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +import json +import os +import subprocess +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from src.tools.python_sandbox import ( + PythonSandbox, + SandboxConfig, + StopGoConfig, + StopGoController, +) + + +class PythonSandboxTests(unittest.TestCase): + def test_child_environment_excludes_credentials_and_artifact_stays_local(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + artifact_dir = Path(temp_dir) + config = SandboxConfig( + engine="sandbox-cli", + worker_pool_size=1, + python_runtime="python3", + execution_timeout_s=2, + memory_limit_mb=64, + artifact_dir=artifact_dir, + ) + sandbox = PythonSandbox(config) + completed = subprocess.CompletedProcess( + args=[], returncode=0, stdout=b"ok\n", stderr=b"" + ) + + with patch.dict( + os.environ, + {"OPENAI_API_KEY": "must-not-pass", "PATH": "/usr/bin"}, + clear=True, + ), patch("src.tools.python_sandbox.subprocess.run", return_value=completed) as run: + result = sandbox.execute("print('ok')", "../../outside") + + self.assertEqual(result["status"], "ok") + self.assertEqual(run.call_args.args[0][0], "sandbox-cli") + self.assertEqual(run.call_args.kwargs["env"], {"PATH": "/usr/bin"}) + artifacts = list(artifact_dir.glob("*.json")) + self.assertEqual(len(artifacts), 1) + self.assertEqual(artifacts[0].parent, artifact_dir) + self.assertNotIn("..", artifacts[0].name) + self.assertEqual(json.loads(artifacts[0].read_text())["stdout"], "ok\n") + + def test_controller_sends_only_python_body_to_sandbox(self) -> None: + class RecordingSandbox: + def __init__(self) -> None: + self.code = "" + + def execute(self, code: str, task_id: str): + self.code = code + return {"stdout": "2\n", "stderr": "", "status": "ok"} + + sandbox = RecordingSandbox() + controller = StopGoController( + sandbox, # type: ignore[arg-type] + StopGoConfig( + trigger_regex=r"```python\s*\n([\s\S]*?)```", + max_code_blocks=1, + capture_stdout=True, + capture_stderr=True, + ), + ) + + result = controller.run("```python\nprint(1 + 1)\n```", "example") + + self.assertEqual(sandbox.code, "print(1 + 1)\n") + self.assertIn("[tool stdout]\n2", result["patched_response"]) + + def test_controller_annotates_identical_code_blocks_in_order(self) -> None: + class RecordingSandbox: + def __init__(self) -> None: + self.calls = 0 + + def execute(self, code: str, task_id: str): + self.calls += 1 + return { + "stdout": f"result-{self.calls}\n", + "stderr": "", + "status": "ok", + } + + sandbox = RecordingSandbox() + controller = StopGoController( + sandbox, # type: ignore[arg-type] + StopGoConfig( + trigger_regex=r"```python\s*\n([\s\S]*?)```", + max_code_blocks=2, + capture_stdout=True, + capture_stderr=True, + ), + ) + block = "```python\nprint(1)\n```" + + result = controller.run(f"{block}\nthen\n{block}", "duplicate") + + patched = result["patched_response"] + self.assertIsInstance(patched, str) + assert isinstance(patched, str) + self.assertEqual(sandbox.calls, 2) + self.assertLess(patched.index("result-1"), patched.index("then")) + self.assertGreater(patched.index("result-2"), patched.index("then")) + + +if __name__ == "__main__": + unittest.main() diff --git a/vendor/pyext-0.7/PKG-INFO b/vendor/pyext-0.7/PKG-INFO deleted file mode 100644 index ca4c301..0000000 --- a/vendor/pyext-0.7/PKG-INFO +++ /dev/null @@ -1,73 +0,0 @@ -Metadata-Version: 1.1 -Name: pyext -Version: 0.7 -Summary: Simple Python extensions. -Home-page: UNKNOWN -Author: Ryan Gonzalez -Author-email: kirbyfan64sos@gmail.com -License: UNKNOWN -Description: PyExt - ===== - - .. image:: https://travis-ci.org/kirbyfan64/PyExt.png - :target: https://travis-ci.org/kirbyfan64/PyExt - - Several simple extensions that add some nifty features to Python. - - Links: - ****** - - ========= ============================================= - GitHub https://github.com/kirbyfan64/PyExt - PyPI https://pypi.python.org/pypi/pyext - Newsgroup https://groups.google.com/forum/#!forum/pyext - ========= ============================================= - - Features: - ********* - - - Function overloading - - Switch statement - - Runtime module creation - - Tail recursion removal - - Python 2 function annotations - - Python 2 safe tuple unpacking - - Assignment if condition is true - - Examples: - ********* - - Function overloading:: - - @overload.argc(1) - def f(a): print 'Function 1 called' - @overload.argc(2) - def f(a, b): print 'Function 2 called' - - f(1) - f(1, 2) - - Switch statement:: - - with switch(1): - if case(0): print 'Awkward...'; case.quit() # case.quit() is the same as break - if case(2): print '???' - if case(1): print 'Phew! It works!' - if case.default(): print 'Ummmm...' - - Function annotations:: - - @fannotate('Return annotation', a=1, b=2) - def x(a, b): - return 0 - - Assign if condition is true:: - - compare_and_swap('my_var', None, 2) # set my_var to 2 if it equals None - - .. note:: Please ignore this project's messy commit history(several commits under invalid_email_address, about 20 commits labeled Initial). I was trying to use hg-git and kept goofing stuff up. - -Platform: UNKNOWN -Classifier: License :: OSI Approved :: MIT License -Classifier: Programming Language :: Python :: 2 -Classifier: Programming Language :: Python :: 3 diff --git a/vendor/pyext-0.7/README.rst b/vendor/pyext-0.7/README.rst deleted file mode 100644 index 085a5a1..0000000 --- a/vendor/pyext-0.7/README.rst +++ /dev/null @@ -1,60 +0,0 @@ -PyExt -===== - -.. image:: https://travis-ci.org/kirbyfan64/PyExt.png - :target: https://travis-ci.org/kirbyfan64/PyExt - -Several simple extensions that add some nifty features to Python. - -Links: -****** - -========= ============================================= -GitHub https://github.com/kirbyfan64/PyExt -PyPI https://pypi.python.org/pypi/pyext -Newsgroup https://groups.google.com/forum/#!forum/pyext -========= ============================================= - -Features: -********* - -- Function overloading -- Switch statement -- Runtime module creation -- Tail recursion removal -- Python 2 function annotations -- Python 2 safe tuple unpacking -- Assignment if condition is true - -Examples: -********* - -Function overloading:: - - @overload.argc(1) - def f(a): print 'Function 1 called' - @overload.argc(2) - def f(a, b): print 'Function 2 called' - - f(1) - f(1, 2) - -Switch statement:: - - with switch(1): - if case(0): print 'Awkward...'; case.quit() # case.quit() is the same as break - if case(2): print '???' - if case(1): print 'Phew! It works!' - if case.default(): print 'Ummmm...' - -Function annotations:: - - @fannotate('Return annotation', a=1, b=2) - def x(a, b): - return 0 - -Assign if condition is true:: - - compare_and_swap('my_var', None, 2) # set my_var to 2 if it equals None - -.. note:: Please ignore this project's messy commit history(several commits under invalid_email_address, about 20 commits labeled Initial). I was trying to use hg-git and kept goofing stuff up. diff --git a/vendor/pyext-0.7/build/lib/pyext.py b/vendor/pyext-0.7/build/lib/pyext.py deleted file mode 100644 index 34e6b94..0000000 --- a/vendor/pyext-0.7/build/lib/pyext.py +++ /dev/null @@ -1,496 +0,0 @@ -''' -Copyright (C) 2014 Ryan Gonzalez - - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to use, -copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the -Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -''' - -g_backup = globals().copy() - -__version__ = '0.7' - -__all__ = ['overload', 'RuntimeModule', 'switch', 'tail_recurse', 'copyfunc', 'set_docstring', 'annotate', 'safe_unpack', 'modify_function', 'assign', 'fannotate', 'compare_and_swap', 'is_main', 'call_if_main', 'run_main'] - -import sys, inspect, types -from collections import namedtuple - -_ArgSpec = namedtuple('ArgSpec', 'args varargs keywords defaults') - -def __targspec(func, specs, attr='__orig_arg__'): - if hasattr(func, '__is_overload__') and func.__is_overload__: - return getattr(func, attr) - return specs(func) - -def set_docstring(doc): - '''A simple decorator to set docstrings. - - :param doc: The docstring to tie to the function. - - Example:: - - @set_docstring('This is a docstring') - def myfunc(x): - pass''' - def _wrap(f): - f.__doc__ = doc - return f - return _wrap - -__modify_function_doc = ''' -Creates a copy of a function, changing its attributes. - -:param globals: Will be added to the function's globals. - -:param name: The new function name. Set to ``None`` to use the function's original name. - -:param code: The new function code object. Set to ``None`` to use the function's original code object. - -:param defaults: The new function defaults. Set to ``None`` to use the function's original defaults. - -:param closure: The new function closure. Set to ``None`` to use the function's original closure. - -.. warning:: This function can be potentially dangerous. -''' - -def copyfunc(f): - '''Copies a funcion. - - :param f: The function to copy. - - :return: The copied function. - - .. deprecated:: 0.4 - Use :func:`modify_function` instead. - ''' - return modify_function(f) - -if sys.version_info.major == 3: - @set_docstring(__modify_function_doc) - def modify_function(f, globals={}, name=None, code=None, defaults=None, - closure=None): - if code is None: code = f.__code__ - if name is None: name = f.__name__ - if defaults is None: defaults = f.__defaults__ - if closure is None: closure = f.__closure__ - newf = types.FunctionType(code, dict(f.__globals__, **globals), name=name, - argdefs=defaults, closure=closure) - newf.__dict__.update(f.__dict__) - return newf - def argspec(f): - return inspect.getfullargspec(f) - ofullargspec = inspect.getfullargspec - def _fullargspec(func): - return __targspec(func, ofullargspec) - inspect.getfullargspec = _fullargspec - def _exec(m,g): exec(m,g) -else: - @set_docstring(__modify_function_doc) - def modify_function(f, globals={}, name=None, code=None, defaults=None, - closure=None): - if code is None: code = f.func_code - if name is None: name = f.__name__ - if defaults is None: defaults = f.func_defaults - if closure is None: closure = f.func_closure - newf = types.FunctionType(code, dict(f.func_globals, **globals), name=name, - argdefs=defaults, closure=closure) - newf.__dict__.update(f.__dict__) - return newf - def argspec(f): - return inspect.getargspec(f) - eval(compile('def _exec(m,g): exec m in g', '', 'exec')) - -def _gettypes(args): - return tuple(map(type, args)) - -try: - oargspec = inspect.getargspec -except AttributeError: - def oargspec(func): - spec = inspect.getfullargspec(func) - return _ArgSpec(spec.args, spec.varargs, spec.varkw, spec.defaults) - -def _argspec(func): - return __targspec(func, oargspec) - -setattr(inspect, 'getargspec', _argspec) - -try: - import IPython -except ImportError: - IPython = None -else: - # Replace IPython's argspec - if hasattr(IPython.core.oinspect, 'getargspec'): - oipyargspec = IPython.core.oinspect.getargspec - else: - def oipyargspec(func): - spec = inspect.getfullargspec(func) - return _ArgSpec(spec.args, spec.varargs, spec.varkw, spec.defaults) - - def _ipyargspec(func): - return __targspec(func, oipyargspec, '__orig_arg_ipy__') - - IPython.core.oinspect.getargspec = _ipyargspec - -class overload(object): - '''Simple function overloading in Python.''' - _items = {} - _types = {} - @classmethod - def argc(self, argc=None): - '''Overloads a function based on the specified argument count. - - :param argc: The argument count. Defaults to ``None``. If ``None`` is given, automatically compute the argument count from the given function. - - .. note:: - - Keyword argument counts are NOT checked! In addition, when the argument count is automatically calculated, the keyword argument count is also ignored! - - Example:: - - @overload.argc() - def func(a): - print 'Function 1 called' - - @overload.argc() - def func(a, b): - print 'Function 2 called' - - func(1) # Calls first function - func(1, 2) # Calls second function - func() # Raises error - ''' - # Python 2 UnboundLocalError fix - argc = {'argc': argc} - def _wrap(f): - def _newf(*args, **kwargs): - if len(args) not in self._items[f.__name__]: - raise TypeError("No overload of function '%s' that takes %d args" % (f.__name__, len(args))) - return self._items[f.__name__][len(args)](*args, **kwargs) - if f.__name__ not in self._items: - self._items[f.__name__] = {} - if argc['argc'] is None: - argc['argc'] = len(argspec(f).args) - self._items[f.__name__][argc['argc']] = f - _newf.__name__ = f.__name__ - _newf.__doc__ = f.__doc__ - _newf.__is_overload__ = True - _newf.__orig_arg__ = argspec(f) - if IPython: - _newf.__orig_arg_ipy__ = IPython.core.oinspect.getargspec(f) - return _newf - return _wrap - @classmethod - def args(self, *argtypes, **kw): - '''Overload a function based on the specified argument types. - - :param argtypes: The argument types. If None is given, get the argument types from the function annotations(Python 3 only) - :param kw: Can only contain 1 argument, `is_cls`. If True, the function is assumed to be part of a class. - - Example:: - - @overload.args(str) - def func(s): - print 'Got string' - - @overload.args(int, str) - def func(i, s): - print 'Got int and string' - - @overload.args() - def func(i:int): # A function annotation example - print 'Got int' - - func('s') - func(1) - func(1, 's') - func(True) # Raises error - ''' - - # Python 2 UnboundLocalError fix...again! - argtypes = {'args': tuple(argtypes)} - def _wrap(f): - def _newf(*args): - if len(kw) == 0: - cargs = args - elif len(kw) == 1 and 'is_cls' in kw and kw['is_cls']: - cargs = args[1:] - else: - raise ValueError('Invalid keyword args specified') - if _gettypes(cargs) not in self._types[f.__name__]: - raise TypeError("No overload of function '%s' that takes '%s' types and %d arg(s)" % (f.__name__, _gettypes(cargs), len(cargs))) - return self._types[f.__name__][_gettypes(cargs)](*args) - if f.__name__ not in self._types: - self._types[f.__name__] = {} - if len(argtypes['args']) == 1 and argtypes['args'][0] is None: - aspec = argspec(f) - argtypes['args'] = tuple(map(lambda x: x[1], sorted( - aspec.annotations.items(), key=lambda x: aspec.args.index(x[0])))) - self._types[f.__name__][argtypes['args']] = f - _newf.__name__ = f.__name__ - _newf.__doc__ = f.__doc__ - _newf.__is_overload__ = True - _newf.__orig_arg__ = argspec(f) - if IPython: - _newf.__orig_arg_ipy__ = IPython.core.oinspect.getargspec(f) - return _newf - return _wrap - -class _RuntimeModule(object): - 'Create a module object at runtime and insert it into sys.path. If called, same as :py:func:`from_objects`.' - def __call__(self, *args, **kwargs): - return self.from_objects(*args, **kwargs) - @staticmethod - @overload.argc(1) - def from_objects(name, **d): - return _RuntimeModule.from_objects(name, '', **d) - @staticmethod - @overload.argc(2) - def from_objects(name, docstring, **d): - '''Create a module at runtime from `d`. - - :param name: The module name. - - :param docstring: Optional. The module's docstring. - - :param \*\*d: All the keyword args, mapped from name->value. - - Example: ``RuntimeModule.from_objects('name', 'doc', a=1, b=2)``''' - module = types.ModuleType(name, docstring) - module.__dict__.update(d) - module.__file__ = '' - sys.modules[name] = module - return module - @staticmethod - @overload.argc(2) - def from_string(name, s): - return _RuntimeModule.from_string(name, '', s) - @staticmethod - @overload.argc(3) - def from_string(name, docstring, s): - '''Create a module at runtime from `s``. - - :param name: The module name. - - :param docstring: Optional. The module docstring. - - :param s: A string containing the module definition.''' - g = {} - _exec(s, g) - return _RuntimeModule.from_objects(name, docstring, **dict(filter(lambda x: x[0] not in g_backup, g.items()))) - -RuntimeModule = _RuntimeModule() - -class CaseObject(object): - 'The object returned by a switch statement. When called, it will return True if the given argument equals its value, else False. It can be called with multiple parameters, in which case it checks if its value equals any of the arguments.' - def __init__(self, value): - self.value = value - self.did_match = False - self.did_pass = False - def __call__(self, *args): - if assign('res', not self.did_pass and any([self.value == rhs for rhs in args])): - self.did_match = True - return res - def quit(self): - 'Forces all other calls to return False. Equilavent of a ``break`` statement.' - self.did_pass = True - def default(self): - "Executed if quit wasn't called." - return not self.did_match and not self.did_pass - def __iter__(self): - yield self - def __enter__(self): - return self - def __exit__(self, *args): - pass - -def switch(value): - '''A Python switch statement implementation that is used with a ``with`` statement. - - :param value: The value to "switch". - - ``with`` statement example:: - - with switch('x'): - if case(1): print 'Huh?' - if case('x'): print 'It works!!!' - - .. warning:: If you modify a variable named "case" in the same scope that you use the ``with`` statement version, you will get an UnboundLocalError. The soluction is to use ``with switch('x') as case:`` instead of ``with switch('x'):``.''' - res = CaseObject(value) - inspect.stack()[1][0].f_globals['case'] = res - return res - -def tail_recurse(spec=None): - '''Remove tail recursion from a function. - - :param spec: A function that, when given the arguments, returns a bool indicating whether or not to exit. If ``None,`` tail recursion is always called unless the function returns a value. - - .. note:: - - This function has a slight overhead that is noticable when using timeit. Only use it if the function has a possibility of going over the recursion limit. - - .. warning:: - - This function will BREAK any code that either uses any recursion other than tail recursion or calls itself multiple times. For example, ``def x(): return x()+1`` will fail. - - Example:: - - @tail_recurse() - def add(a, b): - if a == 0: return b - return add(a-1, b+1) - - add(10000000, 1) # Doesn't max the recursion limit. - ''' - def _wrap(f): - class TailRecursion(Exception): - def __init__(self, args, kwargs): - self.args = args - self.kwargs = kwargs - def _newf(*args, **kwargs): - if inspect.stack()[1][3] == f.__name__: - if (spec and spec(args)) or not spec: - raise TailRecursion(args, kwargs) - while True: - try: - res = f(*args, **kwargs) - except TailRecursion as ex: - args = ex.args - kwargs = ex.kwargs - continue - else: - return res - _newf.__doc__ = f.__doc__ - return _newf - return _wrap - -def annotate(*args, **kwargs): - '''Set function annotations using decorators. - - :param args: This is a list of annotations for the function, in the order of the function's parameters. For example, ``annotate('Annotation 1', 'Annotation 2')`` will set the annotations of parameter 1 of the function to ``Annotation 1``. - - :param kwargs: This is a mapping of argument names to annotations. Note that these are applied *after* the argument list, so any args set that way will be overriden by this mapping. If there is a key named `ret`, that will be the annotation for the function's return value. - - .. deprecated:: 0.5 - Use :func:`fannotate` instead. -''' - def _wrap(f): - if not hasattr(f, '__annotations__'): - f.__annotations__ = {} - if 'ret' in kwargs: - f.__annotations__['return'] = kwargs.pop('ret') - f.__annotations__.update(dict(zip(argspec(f).args, args))) - f.__annotations__.update(kwargs) - return f - return _wrap - -def fannotate(*args, **kwargs): - '''Set function annotations using decorators. - - :param \*args: The first positional argument is used for the function's return value; all others are discarded. - - :param \**kwargs: This is a mapping of argument names to annotations. - - Example:: - - @fannotate('This for the return value', a='Parameter a', b='Parameter b') - def x(a, b): - pass - - ''' - def _wrap(f): - if not hasattr(f, '__annotations__'): - f.__annotations__ = {} - if len(args) >= 1: - f.__annotations__['return'] = args[0] - f.__annotations__.update(kwargs) - return f - return _wrap - -def safe_unpack(seq, ln, fill=None): - '''Safely unpack a sequence to length `ln`, without raising ValueError. Based on Lua's method of unpacking. Empty values will be filled in with `fill`, while any extra values will be cut off. - - :param seq: The sequence to unpack. - - :param ln: The expected length of the sequence. - - :param fill: The value to substitute if the sequence is too small. Defaults to ``None``. - - Example:: - - s = 'a:b' - a, b = safe_unpack(s.split(':'), 2) - # a = 'a' - # b = 'b' - s = 'a' - a, b = safe_unpack(s.split(':'), 2) - # a = 'a' - # b = None''' - if len(seq) > ln: - return seq[:ln] - elif len(seq) < ln: - return seq + type(seq)([fill]*(ln-len(seq))) - else: - return seq - -def assign(varname, value): - '''Assign `value` to `varname` and return it. If `varname` is an attribute and the instance name it belongs to is not defined, a NameError is raised. - This can be used to emulate assignment as an expression. For example, this:: - - if assign('x', 7): ... - - is equilavent to this C code:: - - if (x = 7) ... - - .. warning:: - - When assigning an attribute, the instance it belongs to MUST be declared as global prior to the assignment. Otherwise, the assignment will not work. - ''' - fd = inspect.stack()[1][0].f_globals - if '.' not in varname: - fd[varname] = value - else: - vsplit = list(map(str.strip, varname.split('.'))) - if vsplit[0] not in fd: - raise NameError('Unknown object: %s'%vsplit[0]) - base = fd[vsplit[0]] - for x in vsplit[1:-1]: - base = getattr(base, x) - setattr(base, vsplit[-1], value) - return value - -def is_main(frame=1): - "Return if the caller is main. Equilavent to ``__name__ == '__main__'``." - return inspect.stack()[frame][0].f_globals['__name__'] == '__main__' - -def _call_if_main(frame, f, args): - if is_main(frame): return f(*args) - -def call_if_main(f,*args): - "Call the `f` with `args` if the caller's module is main." - return _call_if_main(3,f,args) - -def run_main(f,*args): - "Call `f` with the `args` and terminate the program with its return code if the caller's module is main." - sys.exit(_call_if_main(3,f,args)) - -def compare_and_swap(var, compare, new): - "If `var` is equal to `compare`, set it to `new`." - if assign('v', inspect.stack()[1][0].f_globals)[var] == compare: - v[var] = new diff --git a/vendor/pyext-0.7/pyext.egg-info/PKG-INFO b/vendor/pyext-0.7/pyext.egg-info/PKG-INFO deleted file mode 100644 index 82ae6ae..0000000 --- a/vendor/pyext-0.7/pyext.egg-info/PKG-INFO +++ /dev/null @@ -1,75 +0,0 @@ -Metadata-Version: 2.4 -Name: pyext -Version: 0.7 -Summary: Simple Python extensions. -Author: Ryan Gonzalez -Author-email: kirbyfan64sos@gmail.com -Classifier: License :: OSI Approved :: MIT License -Classifier: Programming Language :: Python :: 2 -Classifier: Programming Language :: Python :: 3 -Dynamic: author -Dynamic: author-email -Dynamic: classifier -Dynamic: description -Dynamic: summary - -PyExt -===== - -.. image:: https://travis-ci.org/kirbyfan64/PyExt.png - :target: https://travis-ci.org/kirbyfan64/PyExt - -Several simple extensions that add some nifty features to Python. - -Links: -****** - -========= ============================================= -GitHub https://github.com/kirbyfan64/PyExt -PyPI https://pypi.python.org/pypi/pyext -Newsgroup https://groups.google.com/forum/#!forum/pyext -========= ============================================= - -Features: -********* - -- Function overloading -- Switch statement -- Runtime module creation -- Tail recursion removal -- Python 2 function annotations -- Python 2 safe tuple unpacking -- Assignment if condition is true - -Examples: -********* - -Function overloading:: - - @overload.argc(1) - def f(a): print 'Function 1 called' - @overload.argc(2) - def f(a, b): print 'Function 2 called' - - f(1) - f(1, 2) - -Switch statement:: - - with switch(1): - if case(0): print 'Awkward...'; case.quit() # case.quit() is the same as break - if case(2): print '???' - if case(1): print 'Phew! It works!' - if case.default(): print 'Ummmm...' - -Function annotations:: - - @fannotate('Return annotation', a=1, b=2) - def x(a, b): - return 0 - -Assign if condition is true:: - - compare_and_swap('my_var', None, 2) # set my_var to 2 if it equals None - -.. note:: Please ignore this project's messy commit history(several commits under invalid_email_address, about 20 commits labeled Initial). I was trying to use hg-git and kept goofing stuff up. diff --git a/vendor/pyext-0.7/pyext.egg-info/SOURCES.txt b/vendor/pyext-0.7/pyext.egg-info/SOURCES.txt deleted file mode 100644 index cb61a77..0000000 --- a/vendor/pyext-0.7/pyext.egg-info/SOURCES.txt +++ /dev/null @@ -1,9 +0,0 @@ -README.rst -pyext.py -setup.cfg -setup.py -pyext.egg-info/PKG-INFO -pyext.egg-info/SOURCES.txt -pyext.egg-info/dependency_links.txt -pyext.egg-info/top_level.txt -test/test_pyext.py \ No newline at end of file diff --git a/vendor/pyext-0.7/pyext.egg-info/dependency_links.txt b/vendor/pyext-0.7/pyext.egg-info/dependency_links.txt deleted file mode 100644 index 8b13789..0000000 --- a/vendor/pyext-0.7/pyext.egg-info/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/vendor/pyext-0.7/pyext.egg-info/top_level.txt b/vendor/pyext-0.7/pyext.egg-info/top_level.txt deleted file mode 100644 index f6420fb..0000000 --- a/vendor/pyext-0.7/pyext.egg-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -pyext diff --git a/vendor/pyext-0.7/pyext.py b/vendor/pyext-0.7/pyext.py deleted file mode 100644 index 34e6b94..0000000 --- a/vendor/pyext-0.7/pyext.py +++ /dev/null @@ -1,496 +0,0 @@ -''' -Copyright (C) 2014 Ryan Gonzalez - - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to use, -copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the -Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -''' - -g_backup = globals().copy() - -__version__ = '0.7' - -__all__ = ['overload', 'RuntimeModule', 'switch', 'tail_recurse', 'copyfunc', 'set_docstring', 'annotate', 'safe_unpack', 'modify_function', 'assign', 'fannotate', 'compare_and_swap', 'is_main', 'call_if_main', 'run_main'] - -import sys, inspect, types -from collections import namedtuple - -_ArgSpec = namedtuple('ArgSpec', 'args varargs keywords defaults') - -def __targspec(func, specs, attr='__orig_arg__'): - if hasattr(func, '__is_overload__') and func.__is_overload__: - return getattr(func, attr) - return specs(func) - -def set_docstring(doc): - '''A simple decorator to set docstrings. - - :param doc: The docstring to tie to the function. - - Example:: - - @set_docstring('This is a docstring') - def myfunc(x): - pass''' - def _wrap(f): - f.__doc__ = doc - return f - return _wrap - -__modify_function_doc = ''' -Creates a copy of a function, changing its attributes. - -:param globals: Will be added to the function's globals. - -:param name: The new function name. Set to ``None`` to use the function's original name. - -:param code: The new function code object. Set to ``None`` to use the function's original code object. - -:param defaults: The new function defaults. Set to ``None`` to use the function's original defaults. - -:param closure: The new function closure. Set to ``None`` to use the function's original closure. - -.. warning:: This function can be potentially dangerous. -''' - -def copyfunc(f): - '''Copies a funcion. - - :param f: The function to copy. - - :return: The copied function. - - .. deprecated:: 0.4 - Use :func:`modify_function` instead. - ''' - return modify_function(f) - -if sys.version_info.major == 3: - @set_docstring(__modify_function_doc) - def modify_function(f, globals={}, name=None, code=None, defaults=None, - closure=None): - if code is None: code = f.__code__ - if name is None: name = f.__name__ - if defaults is None: defaults = f.__defaults__ - if closure is None: closure = f.__closure__ - newf = types.FunctionType(code, dict(f.__globals__, **globals), name=name, - argdefs=defaults, closure=closure) - newf.__dict__.update(f.__dict__) - return newf - def argspec(f): - return inspect.getfullargspec(f) - ofullargspec = inspect.getfullargspec - def _fullargspec(func): - return __targspec(func, ofullargspec) - inspect.getfullargspec = _fullargspec - def _exec(m,g): exec(m,g) -else: - @set_docstring(__modify_function_doc) - def modify_function(f, globals={}, name=None, code=None, defaults=None, - closure=None): - if code is None: code = f.func_code - if name is None: name = f.__name__ - if defaults is None: defaults = f.func_defaults - if closure is None: closure = f.func_closure - newf = types.FunctionType(code, dict(f.func_globals, **globals), name=name, - argdefs=defaults, closure=closure) - newf.__dict__.update(f.__dict__) - return newf - def argspec(f): - return inspect.getargspec(f) - eval(compile('def _exec(m,g): exec m in g', '', 'exec')) - -def _gettypes(args): - return tuple(map(type, args)) - -try: - oargspec = inspect.getargspec -except AttributeError: - def oargspec(func): - spec = inspect.getfullargspec(func) - return _ArgSpec(spec.args, spec.varargs, spec.varkw, spec.defaults) - -def _argspec(func): - return __targspec(func, oargspec) - -setattr(inspect, 'getargspec', _argspec) - -try: - import IPython -except ImportError: - IPython = None -else: - # Replace IPython's argspec - if hasattr(IPython.core.oinspect, 'getargspec'): - oipyargspec = IPython.core.oinspect.getargspec - else: - def oipyargspec(func): - spec = inspect.getfullargspec(func) - return _ArgSpec(spec.args, spec.varargs, spec.varkw, spec.defaults) - - def _ipyargspec(func): - return __targspec(func, oipyargspec, '__orig_arg_ipy__') - - IPython.core.oinspect.getargspec = _ipyargspec - -class overload(object): - '''Simple function overloading in Python.''' - _items = {} - _types = {} - @classmethod - def argc(self, argc=None): - '''Overloads a function based on the specified argument count. - - :param argc: The argument count. Defaults to ``None``. If ``None`` is given, automatically compute the argument count from the given function. - - .. note:: - - Keyword argument counts are NOT checked! In addition, when the argument count is automatically calculated, the keyword argument count is also ignored! - - Example:: - - @overload.argc() - def func(a): - print 'Function 1 called' - - @overload.argc() - def func(a, b): - print 'Function 2 called' - - func(1) # Calls first function - func(1, 2) # Calls second function - func() # Raises error - ''' - # Python 2 UnboundLocalError fix - argc = {'argc': argc} - def _wrap(f): - def _newf(*args, **kwargs): - if len(args) not in self._items[f.__name__]: - raise TypeError("No overload of function '%s' that takes %d args" % (f.__name__, len(args))) - return self._items[f.__name__][len(args)](*args, **kwargs) - if f.__name__ not in self._items: - self._items[f.__name__] = {} - if argc['argc'] is None: - argc['argc'] = len(argspec(f).args) - self._items[f.__name__][argc['argc']] = f - _newf.__name__ = f.__name__ - _newf.__doc__ = f.__doc__ - _newf.__is_overload__ = True - _newf.__orig_arg__ = argspec(f) - if IPython: - _newf.__orig_arg_ipy__ = IPython.core.oinspect.getargspec(f) - return _newf - return _wrap - @classmethod - def args(self, *argtypes, **kw): - '''Overload a function based on the specified argument types. - - :param argtypes: The argument types. If None is given, get the argument types from the function annotations(Python 3 only) - :param kw: Can only contain 1 argument, `is_cls`. If True, the function is assumed to be part of a class. - - Example:: - - @overload.args(str) - def func(s): - print 'Got string' - - @overload.args(int, str) - def func(i, s): - print 'Got int and string' - - @overload.args() - def func(i:int): # A function annotation example - print 'Got int' - - func('s') - func(1) - func(1, 's') - func(True) # Raises error - ''' - - # Python 2 UnboundLocalError fix...again! - argtypes = {'args': tuple(argtypes)} - def _wrap(f): - def _newf(*args): - if len(kw) == 0: - cargs = args - elif len(kw) == 1 and 'is_cls' in kw and kw['is_cls']: - cargs = args[1:] - else: - raise ValueError('Invalid keyword args specified') - if _gettypes(cargs) not in self._types[f.__name__]: - raise TypeError("No overload of function '%s' that takes '%s' types and %d arg(s)" % (f.__name__, _gettypes(cargs), len(cargs))) - return self._types[f.__name__][_gettypes(cargs)](*args) - if f.__name__ not in self._types: - self._types[f.__name__] = {} - if len(argtypes['args']) == 1 and argtypes['args'][0] is None: - aspec = argspec(f) - argtypes['args'] = tuple(map(lambda x: x[1], sorted( - aspec.annotations.items(), key=lambda x: aspec.args.index(x[0])))) - self._types[f.__name__][argtypes['args']] = f - _newf.__name__ = f.__name__ - _newf.__doc__ = f.__doc__ - _newf.__is_overload__ = True - _newf.__orig_arg__ = argspec(f) - if IPython: - _newf.__orig_arg_ipy__ = IPython.core.oinspect.getargspec(f) - return _newf - return _wrap - -class _RuntimeModule(object): - 'Create a module object at runtime and insert it into sys.path. If called, same as :py:func:`from_objects`.' - def __call__(self, *args, **kwargs): - return self.from_objects(*args, **kwargs) - @staticmethod - @overload.argc(1) - def from_objects(name, **d): - return _RuntimeModule.from_objects(name, '', **d) - @staticmethod - @overload.argc(2) - def from_objects(name, docstring, **d): - '''Create a module at runtime from `d`. - - :param name: The module name. - - :param docstring: Optional. The module's docstring. - - :param \*\*d: All the keyword args, mapped from name->value. - - Example: ``RuntimeModule.from_objects('name', 'doc', a=1, b=2)``''' - module = types.ModuleType(name, docstring) - module.__dict__.update(d) - module.__file__ = '' - sys.modules[name] = module - return module - @staticmethod - @overload.argc(2) - def from_string(name, s): - return _RuntimeModule.from_string(name, '', s) - @staticmethod - @overload.argc(3) - def from_string(name, docstring, s): - '''Create a module at runtime from `s``. - - :param name: The module name. - - :param docstring: Optional. The module docstring. - - :param s: A string containing the module definition.''' - g = {} - _exec(s, g) - return _RuntimeModule.from_objects(name, docstring, **dict(filter(lambda x: x[0] not in g_backup, g.items()))) - -RuntimeModule = _RuntimeModule() - -class CaseObject(object): - 'The object returned by a switch statement. When called, it will return True if the given argument equals its value, else False. It can be called with multiple parameters, in which case it checks if its value equals any of the arguments.' - def __init__(self, value): - self.value = value - self.did_match = False - self.did_pass = False - def __call__(self, *args): - if assign('res', not self.did_pass and any([self.value == rhs for rhs in args])): - self.did_match = True - return res - def quit(self): - 'Forces all other calls to return False. Equilavent of a ``break`` statement.' - self.did_pass = True - def default(self): - "Executed if quit wasn't called." - return not self.did_match and not self.did_pass - def __iter__(self): - yield self - def __enter__(self): - return self - def __exit__(self, *args): - pass - -def switch(value): - '''A Python switch statement implementation that is used with a ``with`` statement. - - :param value: The value to "switch". - - ``with`` statement example:: - - with switch('x'): - if case(1): print 'Huh?' - if case('x'): print 'It works!!!' - - .. warning:: If you modify a variable named "case" in the same scope that you use the ``with`` statement version, you will get an UnboundLocalError. The soluction is to use ``with switch('x') as case:`` instead of ``with switch('x'):``.''' - res = CaseObject(value) - inspect.stack()[1][0].f_globals['case'] = res - return res - -def tail_recurse(spec=None): - '''Remove tail recursion from a function. - - :param spec: A function that, when given the arguments, returns a bool indicating whether or not to exit. If ``None,`` tail recursion is always called unless the function returns a value. - - .. note:: - - This function has a slight overhead that is noticable when using timeit. Only use it if the function has a possibility of going over the recursion limit. - - .. warning:: - - This function will BREAK any code that either uses any recursion other than tail recursion or calls itself multiple times. For example, ``def x(): return x()+1`` will fail. - - Example:: - - @tail_recurse() - def add(a, b): - if a == 0: return b - return add(a-1, b+1) - - add(10000000, 1) # Doesn't max the recursion limit. - ''' - def _wrap(f): - class TailRecursion(Exception): - def __init__(self, args, kwargs): - self.args = args - self.kwargs = kwargs - def _newf(*args, **kwargs): - if inspect.stack()[1][3] == f.__name__: - if (spec and spec(args)) or not spec: - raise TailRecursion(args, kwargs) - while True: - try: - res = f(*args, **kwargs) - except TailRecursion as ex: - args = ex.args - kwargs = ex.kwargs - continue - else: - return res - _newf.__doc__ = f.__doc__ - return _newf - return _wrap - -def annotate(*args, **kwargs): - '''Set function annotations using decorators. - - :param args: This is a list of annotations for the function, in the order of the function's parameters. For example, ``annotate('Annotation 1', 'Annotation 2')`` will set the annotations of parameter 1 of the function to ``Annotation 1``. - - :param kwargs: This is a mapping of argument names to annotations. Note that these are applied *after* the argument list, so any args set that way will be overriden by this mapping. If there is a key named `ret`, that will be the annotation for the function's return value. - - .. deprecated:: 0.5 - Use :func:`fannotate` instead. -''' - def _wrap(f): - if not hasattr(f, '__annotations__'): - f.__annotations__ = {} - if 'ret' in kwargs: - f.__annotations__['return'] = kwargs.pop('ret') - f.__annotations__.update(dict(zip(argspec(f).args, args))) - f.__annotations__.update(kwargs) - return f - return _wrap - -def fannotate(*args, **kwargs): - '''Set function annotations using decorators. - - :param \*args: The first positional argument is used for the function's return value; all others are discarded. - - :param \**kwargs: This is a mapping of argument names to annotations. - - Example:: - - @fannotate('This for the return value', a='Parameter a', b='Parameter b') - def x(a, b): - pass - - ''' - def _wrap(f): - if not hasattr(f, '__annotations__'): - f.__annotations__ = {} - if len(args) >= 1: - f.__annotations__['return'] = args[0] - f.__annotations__.update(kwargs) - return f - return _wrap - -def safe_unpack(seq, ln, fill=None): - '''Safely unpack a sequence to length `ln`, without raising ValueError. Based on Lua's method of unpacking. Empty values will be filled in with `fill`, while any extra values will be cut off. - - :param seq: The sequence to unpack. - - :param ln: The expected length of the sequence. - - :param fill: The value to substitute if the sequence is too small. Defaults to ``None``. - - Example:: - - s = 'a:b' - a, b = safe_unpack(s.split(':'), 2) - # a = 'a' - # b = 'b' - s = 'a' - a, b = safe_unpack(s.split(':'), 2) - # a = 'a' - # b = None''' - if len(seq) > ln: - return seq[:ln] - elif len(seq) < ln: - return seq + type(seq)([fill]*(ln-len(seq))) - else: - return seq - -def assign(varname, value): - '''Assign `value` to `varname` and return it. If `varname` is an attribute and the instance name it belongs to is not defined, a NameError is raised. - This can be used to emulate assignment as an expression. For example, this:: - - if assign('x', 7): ... - - is equilavent to this C code:: - - if (x = 7) ... - - .. warning:: - - When assigning an attribute, the instance it belongs to MUST be declared as global prior to the assignment. Otherwise, the assignment will not work. - ''' - fd = inspect.stack()[1][0].f_globals - if '.' not in varname: - fd[varname] = value - else: - vsplit = list(map(str.strip, varname.split('.'))) - if vsplit[0] not in fd: - raise NameError('Unknown object: %s'%vsplit[0]) - base = fd[vsplit[0]] - for x in vsplit[1:-1]: - base = getattr(base, x) - setattr(base, vsplit[-1], value) - return value - -def is_main(frame=1): - "Return if the caller is main. Equilavent to ``__name__ == '__main__'``." - return inspect.stack()[frame][0].f_globals['__name__'] == '__main__' - -def _call_if_main(frame, f, args): - if is_main(frame): return f(*args) - -def call_if_main(f,*args): - "Call the `f` with `args` if the caller's module is main." - return _call_if_main(3,f,args) - -def run_main(f,*args): - "Call `f` with the `args` and terminate the program with its return code if the caller's module is main." - sys.exit(_call_if_main(3,f,args)) - -def compare_and_swap(var, compare, new): - "If `var` is equal to `compare`, set it to `new`." - if assign('v', inspect.stack()[1][0].f_globals)[var] == compare: - v[var] = new diff --git a/vendor/pyext-0.7/setup.cfg b/vendor/pyext-0.7/setup.cfg deleted file mode 100644 index 861a9f5..0000000 --- a/vendor/pyext-0.7/setup.cfg +++ /dev/null @@ -1,5 +0,0 @@ -[egg_info] -tag_build = -tag_date = 0 -tag_svn_revision = 0 - diff --git a/vendor/pyext-0.7/setup.py b/vendor/pyext-0.7/setup.py deleted file mode 100644 index 503e3db..0000000 --- a/vendor/pyext-0.7/setup.py +++ /dev/null @@ -1,23 +0,0 @@ -try: - from setuptools import setup -except ImportError: - from distutils.core import setup - -import pyext - -with open('README.rst', 'r') as f: - readme = f.read() - -setup(name='pyext', - version=str(pyext.__version__), - author='Ryan Gonzalez', - author_email='kirbyfan64sos@gmail.com', - py_modules=['pyext'], - description='Simple Python extensions.', - long_description=readme, - classifiers=[ - 'License :: OSI Approved :: MIT License', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 3'] - ) - diff --git a/vendor/pyext-0.7/test/test_pyext.py b/vendor/pyext-0.7/test/test_pyext.py deleted file mode 100644 index 12da3f4..0000000 --- a/vendor/pyext-0.7/test/test_pyext.py +++ /dev/null @@ -1,94 +0,0 @@ -import sys, inspect, types, unittest -from pyext import * - -class TestPyExt(unittest.TestCase): - def test_overload_argc(self): - @overload.argc(1) - def f(a): return 1 - @overload.argc(2) - def f(a, b): return 2 - @overload.argc() - def f(): return 0 - self.assertEqual(f(), 0) - self.assertEqual(f(1), 1) - self.assertEqual(f(1, 2), 2) - self.assertRaises(TypeError, f, 1, 2, 3) - self.assertEqual(len(inspect.getargspec(f).args), 0) - def test_overload_args(self): - @overload.args(str, int) - def f(a, b): return str, int - @overload.args(int) - def f(a): return int - @overload.args(str) - def f(a): return str - @overload.args() - def f(): return - self.assertEqual(f(), None) - self.assertEqual(f(0), int) - self.assertEqual(f('s'), str) - self.assertEqual(f('s', 0), (str, int)) - self.assertRaises(TypeError, f, 0, 's') - self.assertEqual(len(inspect.getargspec(f).args), 0) - class x(object): - @overload.args(str, is_cls=True) - def f(self, s): return 1 - @overload.args(int, is_cls=True) - def f(self, i): return 2 - self.assertEqual(x().f('s'), 1) - self.assertEqual(x().f(1), 2) - def test_module(self): - m = RuntimeModule('s', 'doc', x=1, f=2) - self.assertEqual(m.x, 1) - self.assertEqual(m.f, 2) - self.assertTrue(isinstance(m, types.ModuleType)) - self.assertEqual(m.__doc__, 'doc') - m2 = RuntimeModule.from_string('s', 'doc', 'a=7; b=6') - self.assertEqual(m2.a, 7) - self.assertEqual(m2.b, 6) - def test_switch(self): - with switch('x'): - if case('x'): x = 4; case.quit() - if case('b'): x = 2 - if case(1): x = 3 - if case('a'): x = 1 - if case('x'): x = 0 - self.assertEqual(x, 4) - with switch(1): - if case.default(): x = 7 - self.assertEqual(x, 7) - with switch(2): - if case(1,2): x = 9 - self.assertEqual(x, 9) - def test_annot(self): - @fannotate('r', a='a', b=1, c=2) - def x(a, b, c): pass - self.assertEqual(x.__annotations__, {'a': 'a', 'b': 1, 'c': 2, 'return': 'r'}) - def test_unpack(self): - t = (1, 2, 3) - self.assertEqual(safe_unpack(t,2), (1,2)) - self.assertEqual(safe_unpack(t,4), (1,2,3,None)) - self.assertEqual(safe_unpack(t,4,fill=0), (1,2,3,0)) - def test_assign(self): - self.assertEqual(assign('x', 7), 7) - self.assertEqual(x, 7) - global f - def f(): pass - self.assertEqual(assign('f.__annotations__', {'a': 1}), {'a': 1}) - self.assertEqual(f.__annotations__, {'a': 1}) - def test_compare_and_swap(self): - global v - v = None - compare_and_swap('v', None, 7) - self.assertEqual(v, 7) - compare_and_swap('v', None, 8) - self.assertEqual(v, 7) - if sys.version_info.major == 3: - def test_overload_args_annot(self): - def x(a, b): return 0 - x.__annotations__ = {'a': int, 'b': str} - x = overload.args(None)(x) - self.assertEqual(x(1, 's'), 0) - self.assertRaises(TypeError, x, 1, 2) - -if __name__ == '__main__': - unittest.main()