4.0

.github/workflows/install.yml

@@ -17,7 +17,6 @@ jobs:
         py:
           - "3.11"
           - "3.10"
-          - "3.9"
         env:
           - pip
           - conda

.gitignore

@@ -2,6 +2,10 @@
 c_*.c
 pufferlib/extensions.c
 pufferlib/puffernet.c
+logs/
+
+# Build dir
+build/
 
 # hipified cuda extensions dir [HIP/ROCM]
 pufferlib/extensions/hip/
@@ -18,6 +22,7 @@ cy_*.c
 
 # C extensions
 *.so
+*.o
 
 # Distribution / packaging
 .Python
@@ -162,3 +167,8 @@ pufferlib/ocean/impulse_wars/*-release/
 pufferlib/ocean/impulse_wars/debug-*/
 pufferlib/ocean/impulse_wars/release-*/
 pufferlib/ocean/impulse_wars/benchmark/
+
+# Data
+resources/drive/data/*
+resources/drive/binaries/*
+

README.md

@@ -1,16 +1,11 @@
 ![figure](https://pufferai.github.io/source/resource/header.png)
 
-[![PyPI version](https://badge.fury.io/py/pufferlib.svg)](https://badge.fury.io/py/pufferlib)
-![PyPI - Python Version](https://img.shields.io/pypi/pyversions/pufferlib)
-![Github Actions](https://github.com/PufferAI/PufferLib/actions/workflows/install.yml/badge.svg)
-[![](https://dcbadge.vercel.app/api/server/spT4huaGYV?style=plastic)](https://discord.gg/spT4huaGYV)
-[![Twitter](https://img.shields.io/twitter/url/https/twitter.com/cloudposse.svg?style=social&label=Follow%20%40jsuarez5341)](https://twitter.com/jsuarez5341)
+[![Discord](https://dcbadge.vercel.app/api/server/spT4huaGYV?style=plastic)](https://discord.gg/spT4huaGYV)
+[![Twitter](https://img.shields.io/twitter/url/https/twitter.com/cloudposse.svg?style=social&label=Follow%20%40jsuarez)](https://twitter.com/jsuarez)
 
-PufferLib is the reinforcement learning library I wish existed during my PhD. It started as a compatibility layer to make working with complex environments a breeze. Now, it's a high-performance toolkit for research and industry with optimized parallel simulation, environments that run and train at 1M+ steps/second, and tons of quality of life improvements for practitioners. All our tools are free and open source. We also offer priority service for companies, startups, and labs!
+PufferLib is a fast and sane reinforcement learning library that can train tiny, super-human models in seconds. The included learning algorithm, hyperparameter tuning, and simulation methods are the product of our own research. All our tools are free and open source. Need a high performance environment for your application? We build them professionally and offer training + extended support. Contact jsuarez🐡puffer🐡ai.
 
-![Trailer](https://github.com/PufferAI/puffer.ai/blob/main/docs/assets/puffer_2.gif?raw=true)
-
-All of our documentation is hosted at [puffer.ai](https://puffer.ai "PufferLib Documentation"). @jsuarez5341 on [Discord](https://discord.gg/puffer) for support -- post here before opening issues. We're always looking for new contributors, too!
+All of our documentation is hosted at [puffer.ai](https://puffer.ai "PufferLib Documentation"). @jsuarez5341 on [Discord](https://discord.gg/puffer) for support. Post there before opening issues. We're always looking for new contributors!
 
 ## Star to puff up the project!
 

build.sh

@@ -0,0 +1,300 @@
+#!/bin/bash
+set -e
+
+# Usage:
+#   ./build.sh breakout              # Build _C.so with breakout statically linked
+#   ./build.sh breakout --float      # float32 precision (required for --slowly)
+#   ./build.sh breakout --cpu        # CPU fallback, torch only
+#   ./build.sh breakout --debug      # Debug build
+#   ./build.sh breakout --local      # Standalone executable (debug, sanitizers)
+#   ./build.sh breakout --fast       # Standalone executable (optimized)
+#   ./build.sh breakout --web        # Emscripten web build
+#   ./build.sh breakout --profile    # Kernel profiling binary
+#   ./build.sh all                   # Build all envs with default and --float
+
+if [ -z "$1" ]; then
+    echo "Usage: ./build.sh ENV_NAME [--float] [--debug] [--local|--fast|--web|--profile|--cpu|--all]"
+    exit 1
+fi
+ENV=$1
+shift
+
+for arg in "$@"; do
+    case $arg in
+        --float) PRECISION="-DPRECISION_FLOAT" ;;
+        --debug) DEBUG=1 ;;
+        --local) MODE=local ;;
+        --fast)  MODE=fast ;;
+        --web)   MODE=web ;;
+        --profile) MODE=profile ;;
+        --cpu)   MODE=cpu; PRECISION="-DPRECISION_FLOAT" ;;
+        *) echo "Error: unknown argument '$arg'" && exit 1 ;;
+    esac
+done
+
+if [ "$ENV" = "all" ]; then
+    FAILED=""
+    for env_dir in ocean/*/; do
+        env=$(basename "$env_dir")
+        if bash "$0" "$env" && bash "$0" "$env" --float; then
+            echo "OK: $env"
+        else
+            echo "FAIL: $env"
+            FAILED="$FAILED\n  $env"
+        fi
+    done
+
+    if [ -n "$FAILED" ]; then
+        echo -e "\nFailed builds:$FAILED"
+    fi
+    exit 0
+fi
+
+# Linux/mac
+PLATFORM="$(uname -s)"
+if [ "$PLATFORM" = "Linux" ]; then
+    RAYLIB_NAME='raylib-5.5_linux_amd64'
+    OMP_LIB=-lomp5
+    SANITIZE_FLAGS=(-fsanitize=address,undefined,bounds,pointer-overflow,leak -fno-omit-frame-pointer)
+    STANDALONE_LDFLAGS=(-lGL)
+    SHARED_LDFLAGS=(-Bsymbolic-functions)
+else
+    RAYLIB_NAME='raylib-5.5_macos'
+    OMP_LIB=-lomp
+    SANITIZE_FLAGS=()
+    STANDALONE_LDFLAGS=(-framework Cocoa -framework IOKit -framework CoreVideo -framework OpenGL)
+    SHARED_LDFLAGS=(-framework Cocoa -framework OpenGL -framework IOKit -undefined dynamic_lookup)
+fi
+
+CLANG_WARN=(
+    -Wall
+    -ferror-limit=3
+    -Werror=incompatible-pointer-types
+    -Werror=return-type
+    -Wno-error=incompatible-pointer-types-discards-qualifiers
+    -Wno-incompatible-pointer-types-discards-qualifiers
+    -Wno-error=array-parameter
+)
+
+download() {
+    local name=$1 url=$2
+    [ -d "$name" ] && return
+    echo "Downloading $name..."
+    case "$url" in
+        *.zip) curl -sL "$url" -o "$name.zip" && unzip -q "$name.zip" && rm "$name.zip" ;;
+        *)     curl -sL "$url" -o "$name.tar.gz" && tar xf "$name.tar.gz" && rm "$name.tar.gz" ;;
+    esac
+}
+
+RAYLIB_URL="https://github.com/raysan5/raylib/releases/download/5.5"
+if [ "$MODE" = "web" ]; then
+    RAYLIB_NAME='raylib-5.5_webassembly'
+    download "$RAYLIB_NAME" "$RAYLIB_URL/$RAYLIB_NAME.zip"
+else
+    download "$RAYLIB_NAME" "$RAYLIB_URL/$RAYLIB_NAME.tar.gz"
+fi
+
+RAYLIB_A="$RAYLIB_NAME/lib/libraylib.a"
+INCLUDES=(-I./$RAYLIB_NAME/include -I./src -I./vendor)
+LINK_ARCHIVES=("$RAYLIB_A")
+EXTRA_SRC=""
+
+if [ "$ENV" = "constellation" ]; then
+    SRC_DIR="constellation"
+    EXTRA_SRC="vendor/cJSON.c"
+    OUTPUT_NAME="seethestars"
+elif [ "$ENV" = "trailer" ]; then
+    SRC_DIR="trailer"
+    OUTPUT_NAME="trailer/trailer"
+elif [ "$ENV" = "impulse_wars" ]; then
+    SRC_DIR="ocean/$ENV"
+    if [ "$MODE" = "web" ]; then BOX2D_NAME='box2d-web'
+    elif [ "$PLATFORM" = "Linux" ]; then BOX2D_NAME='box2d-linux-amd64'
+    else BOX2D_NAME='box2d-macos-arm64'
+    fi
+    BOX2D_URL="https://github.com/capnspacehook/box2d/releases/latest/download"
+    download "$BOX2D_NAME" "$BOX2D_URL/$BOX2D_NAME.tar.gz"
+    INCLUDES+=(-I./$BOX2D_NAME/include -I./$BOX2D_NAME/src)
+    LINK_ARCHIVES+=("./$BOX2D_NAME/libbox2d.a")
+elif [ -d "ocean/$ENV" ]; then
+    SRC_DIR="ocean/$ENV"
+else
+    echo "Error: environment '$ENV' not found" && exit 1
+fi
+
+OUTPUT_NAME=${OUTPUT_NAME:-$ENV}
+
+# Standalone environment build
+if [ -n "$DEBUG" ] || [ "$MODE" = "local" ]; then
+    CLANG_OPT=(-g -O0 "${CLANG_WARN[@]}" "${SANITIZE_FLAGS[@]}")
+    NVCC_OPT="-O0 -g"
+    LINK_OPT="-g"
+else
+    CLANG_OPT=(-O2 -DNDEBUG "${CLANG_WARN[@]}")
+    NVCC_OPT="-O2 --threads 0"
+    LINK_OPT="-O2"
+fi
+if [ "$MODE" = "local" ] || [ "$MODE" = "fast" ]; then
+    FLAGS=(
+        "${INCLUDES[@]}"
+        "$SRC_DIR/$ENV.c" $EXTRA_SRC -o "$OUTPUT_NAME"
+        "${LINK_ARCHIVES[@]}"
+        "${STANDALONE_LDFLAGS[@]}"
+        -lm -lpthread -fopenmp
+        -DPLATFORM_DESKTOP
+    )
+    echo "Compiling $ENV..."
+    ${CC:-clang} "${CLANG_OPT[@]}" "${FLAGS[@]}"
+    echo "Built: ./$OUTPUT_NAME"
+    exit 0
+elif [ "$MODE" = "web" ]; then
+    mkdir -p "build/web/$ENV"
+    echo "Compiling $ENV for web..."
+    emcc \
+        -o "build/web/$ENV/game.html" \
+        "$SRC_DIR/$ENV.c" $EXTRA_SRC \
+        -O3 -Wall \
+        "${LINK_ARCHIVES[@]}" \
+        "${INCLUDES[@]}" \
+        -L. -L./$RAYLIB_NAME/lib \
+        -sASSERTIONS=2 -gsource-map \
+        -sUSE_GLFW=3 -sUSE_WEBGL2=1 -sASYNCIFY -sFILESYSTEM -sFORCE_FILESYSTEM=1 \
+        --shell-file vendor/minshell.html \
+        -sINITIAL_MEMORY=512MB -sALLOW_MEMORY_GROWTH -sSTACK_SIZE=512KB \
+        -DNDEBUG -DPLATFORM_WEB -DGRAPHICS_API_OPENGL_ES3 \
+        --preload-file resources/$ENV@resources/$ENV \
+        --preload-file resources/shared@resources/shared
+    echo "Built: build/web/$ENV/game.html"
+    exit 0
+fi
+
+# Find cuDNN path
+CUDA_HOME=${CUDA_HOME:-${CUDA_PATH:-$(dirname "$(dirname "$(which nvcc)")")}}
+CUDNN_IFLAG=""
+CUDNN_LFLAG=""
+for dir in /usr/local/cuda/include /usr/include; do
+    if [ -f "$dir/cudnn.h" ]; then
+        CUDNN_IFLAG="-I$dir"
+        break
+    fi
+done
+for dir in /usr/local/cuda/lib64 /usr/lib/x86_64-linux-gnu; do
+    if [ -f "$dir/libcudnn.so" ]; then
+        CUDNN_LFLAG="-L$dir"
+        break
+    fi
+done
+if [ -z "$CUDNN_IFLAG" ]; then
+    CUDNN_IFLAG=$(python -c "import nvidia.cudnn, os; print('-I' + os.path.join(nvidia.cudnn.__path__[0], 'include'))" 2>/dev/null || echo "")
+fi
+if [ -z "$CUDNN_LFLAG" ]; then
+    CUDNN_LFLAG=$(python -c "import nvidia.cudnn, os; print('-L' + os.path.join(nvidia.cudnn.__path__[0], 'lib'))" 2>/dev/null || echo "")
+fi
+
+export CCACHE_DIR="${CCACHE_DIR:-$HOME/.ccache}"
+export CCACHE_BASEDIR="$(pwd)"
+export CCACHE_COMPILERCHECK=content
+NVCC="ccache $CUDA_HOME/bin/nvcc"
+CC="${CC:-$(command -v ccache >/dev/null && echo 'ccache clang' || echo 'clang')}"
+ARCH=${NVCC_ARCH:-native}
+
+PYTHON_INCLUDE=$(python -c "import sysconfig; print(sysconfig.get_path('include'))")
+PYBIND_INCLUDE=$(python -c "import pybind11; print(pybind11.get_include())")
+NUMPY_INCLUDE=$(python -c "import numpy; print(numpy.get_include())")
+EXT_SUFFIX=$(python -c "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))")
+OUTPUT="pufferlib/_C${EXT_SUFFIX}"
+
+BINDING_SRC="$SRC_DIR/binding.c"
+mkdir -p build
+STATIC_OBJ="build/libstatic_${ENV}.o"
+STATIC_LIB="build/libstatic_${ENV}.a"
+
+if [ ! -f "$BINDING_SRC" ]; then
+    echo "Error: $BINDING_SRC not found"
+    exit 1
+fi
+
+echo "Compiling static library for $ENV..."
+${CC:-clang} -c "${CLANG_OPT[@]}" \
+    -I. -Isrc -I$SRC_DIR -Ivendor \
+    -I./$RAYLIB_NAME/include -I$CUDA_HOME/include \
+    -DPLATFORM_DESKTOP \
+    -fno-semantic-interposition -fvisibility=hidden \
+    -fPIC -fopenmp \
+    "$BINDING_SRC" -o "$STATIC_OBJ"
+ar rcs "$STATIC_LIB" "$STATIC_OBJ"
+
+# Brittle hack: have to extract the tensor type from the static lib to build trainer
+OBS_TENSOR_T=$(awk '/^#define OBS_TENSOR_T/{print $3}' "$BINDING_SRC")
+if [ -z "$OBS_TENSOR_T" ]; then
+    echo "Error: Could not find OBS_TENSOR_T in $BINDING_SRC"
+    exit 1
+fi
+
+if [ -z "$MODE" ]; then
+    echo "Compiling CUDA ($ARCH) training backend..."
+    $NVCC -c -arch=$ARCH -Xcompiler -fPIC \
+        -Xcompiler=-D_GLIBCXX_USE_CXX11_ABI=1 \
+        -Xcompiler=-DNPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION \
+        -Xcompiler=-DPLATFORM_DESKTOP \
+        -std=c++17 \
+        -I. -Isrc \
+        -I$PYTHON_INCLUDE -I$PYBIND_INCLUDE -I$NUMPY_INCLUDE \
+        -I$CUDA_HOME/include $CUDNN_IFLAG -I$RAYLIB_NAME/include \
+        -Xcompiler=-fopenmp \
+        -DOBS_TENSOR_T=$OBS_TENSOR_T \
+        -DENV_NAME=$ENV \
+        $PRECISION $NVCC_OPT \
+        src/bindings.cu -o build/bindings.o
+
+    LINK_CMD=(
+        ${CXX:-g++} -shared -fPIC -fopenmp
+        build/bindings.o "$STATIC_LIB" "$RAYLIB_A"
+        -L$CUDA_HOME/lib64 $CUDNN_LFLAG
+        -lcudart -lnccl -lnvidia-ml -lcublas -lcusolver -lcurand -lcudnn
+        $OMP_LIB $LINK_OPT
+        "${SHARED_LDFLAGS[@]}"
+        -o "$OUTPUT"
+    )
+    "${LINK_CMD[@]}"
+    echo "Built: $OUTPUT"
+
+elif [ "$MODE" = "cpu" ]; then
+    echo "Compiling CPU training backend..."
+    ${CXX:-g++} -c -fPIC -fopenmp \
+        -D_GLIBCXX_USE_CXX11_ABI=1 \
+        -DPLATFORM_DESKTOP \
+        -std=c++17 \
+        -I. -Isrc \
+        -I$PYTHON_INCLUDE -I$PYBIND_INCLUDE \
+        -DOBS_TENSOR_T=$OBS_TENSOR_T \
+        -DENV_NAME=$ENV \
+        $PRECISION $LINK_OPT \
+        src/bindings_cpu.cpp -o build/bindings_cpu.o
+    LINK_CMD=(
+        ${CXX:-g++} -shared -fPIC -fopenmp
+        build/bindings_cpu.o "$STATIC_LIB" "$RAYLIB_A"
+        -lm -lpthread $OMP_LIB $LINK_OPT
+        "${SHARED_LDFLAGS[@]}"
+        -o "$OUTPUT"
+    )
+    "${LINK_CMD[@]}"
+    echo "Built: $OUTPUT"
+
+elif [ "$MODE" = "profile" ]; then
+    echo "Compiling profile binary ($ARCH)..."
+    $NVCC $NVCC_OPT -arch=$ARCH -std=c++17 \
+        -I. -Isrc -I$SRC_DIR -Ivendor \
+        -I$CUDA_HOME/include $CUDNN_IFLAG -I$RAYLIB_NAME/include \
+        -DOBS_TENSOR_T=$OBS_TENSOR_T \
+        -DENV_NAME=$ENV \
+        -Xcompiler=-DPLATFORM_DESKTOP \
+        $PRECISION \
+        -Xcompiler=-fopenmp \
+        tests/profile_kernels.cu vendor/ini.c \
+        "$STATIC_LIB" "$RAYLIB_A" \
+        -lnccl -lnvidia-ml -lcublas -lcurand -lcudnn \
+        -lGL -lm -lpthread $OMP_LIB \
+        -o profile
+    echo "Built: ./profile"
+fi

pufferlib/config/ocean/asteroids.ini → config/asteroids.ini

@@ -1,8 +1,5 @@
 [base]
-package = ocean
-env_name = puffer_asteroids
-policy_name = Policy
-rnn_name = Recurrent
+env_name = asteroids
 
 [vec]
 num_envs = 8
@@ -17,7 +14,7 @@ adam_beta2 = 0.9999436458974764
 adam_eps = 6.915036275112011e-08
 anneal_lr = true
 batch_size = auto
-bptt_horizon = 64
+horizon = 64
 checkpoint_interval = 200
 clip_coef = 0.18588778503512546
 ent_coef = 0.0016620361911332262