diff --git a/Cubelet/config/config.toml b/Cubelet/config/config.toml index 17e0554e..6d7f5377 100644 --- a/Cubelet/config/config.toml +++ b/Cubelet/config/config.toml @@ -65,7 +65,7 @@ dynamic_config_path = "/usr/local/services/cubetoolbox/Cubelet/dynamicconf/conf. object_dir = "/usr/local/services/cubetoolbox/cube-vs/network" eth_name = "eth0" tap_init_num = 500 - cidr = "192.168.0.0/18" + cidr = "__CUBE_SANDBOX_CIDR__" mvm_inner_ip = "169.254.68.6" mvm_mac_addr = "20:90:6f:fc:fc:fc" mvm_gw_mac_addr = "20:90:6f:cf:cf:cf" diff --git a/deploy/one-click/env.example b/deploy/one-click/env.example index 941cd8ce..ee7a764b 100644 --- a/deploy/one-click/env.example +++ b/deploy/one-click/env.example @@ -137,3 +137,10 @@ DATABASE_URL=mysql://cube:cube_pass@127.0.0.1:3306/cube_mvp # Optional Docker mirror. ONE_CLICK_ENABLE_TENCENT_DOCKER_MIRROR=0 ONE_CLICK_TENCENT_DOCKER_MIRROR_URL=https://mirror.ccs.tencentyun.com + +# ---- Sandbox Network CIDR ---- +# Sandbox TAP network IP address pool (used by Cubelet and network-agent). +CUBE_SANDBOX_CIDR=192.168.0.0/18 +# Number of TAP devices to pre-allocate. Must not exceed available IPs in CIDR. +# For /24: max 254, /22: max 1022, /21: max 2046, /18: max 16382 +CUBE_SANDBOX_TAP_INIT_NUM=500 diff --git a/deploy/one-click/install.sh b/deploy/one-click/install.sh index a6a8053f..34560590 100755 --- a/deploy/one-click/install.sh +++ b/deploy/one-click/install.sh @@ -149,6 +149,14 @@ generate_cubemaster_config_ports() { "${cfg}" } +generate_cubelet_config_cidr() { + local cfg="${PKG_ROOT}/Cubelet/config/config.toml" + local cidr="${CUBE_SANDBOX_CIDR:-192.168.0.0/18}" + + ensure_file "${cfg}" + sed -i "s|__CUBE_SANDBOX_CIDR__|${cidr}|g" "${cfg}" +} + check_hardware_preflight() { if [[ ! -e /dev/kvm ]]; then log "KVM is not supported or not enabled (/dev/kvm not found)." @@ -492,6 +500,12 @@ else fi mkdir -p "${INSTALL_PREFIX}" + +# Apply CIDR placeholder substitution in PKG_ROOT before copying to install target. +# Must run before copy_dir_contents / cp -a so that both compute and control +# roles pick up the substituted files. +generate_cubelet_config_cidr + if [[ "${DEPLOY_ROLE}" == "compute" ]]; then copy_dir_contents "${PKG_ROOT}/network-agent" "${INSTALL_PREFIX}/network-agent" copy_dir_contents "${PKG_ROOT}/Cubelet" "${INSTALL_PREFIX}/Cubelet" diff --git a/deploy/one-click/scripts/one-click/common.sh b/deploy/one-click/scripts/one-click/common.sh index cb4491ef..e9a2d45f 100755 --- a/deploy/one-click/scripts/one-click/common.sh +++ b/deploy/one-click/scripts/one-click/common.sh @@ -348,3 +348,17 @@ wait_for_health() { done return 1 } + +# Render Cubelet config.toml CIDR and tap_init_num from .one-click.env. +render_config_placeholders() { + local sandbox_cidr="${CUBE_SANDBOX_CIDR:-192.168.0.0/18}" + local tap_init_num="${CUBE_SANDBOX_TAP_INIT_NUM:-500}" + local cubelet_config="${TOOLBOX_ROOT}/Cubelet/config/config.toml" + + if [[ -f "${cubelet_config}" ]]; then + sed -i "s|__CUBE_SANDBOX_CIDR__|${sandbox_cidr}|g" "${cubelet_config}" + if [[ -n "${CUBE_SANDBOX_TAP_INIT_NUM:-}" ]]; then + sed -i "s/tap_init_num = [0-9]\+/tap_init_num = ${tap_init_num}/" "${cubelet_config}" + fi + fi +} diff --git a/deploy/one-click/scripts/one-click/down-local.sh b/deploy/one-click/scripts/one-click/down-local.sh index c4d341e2..133a2749 100755 --- a/deploy/one-click/scripts/one-click/down-local.sh +++ b/deploy/one-click/scripts/one-click/down-local.sh @@ -10,4 +10,30 @@ stop_by_pidfile "cube-api" "^${TOOLBOX_ROOT}/CubeAPI/bin/cube-api" stop_by_pidfile "cubemaster" stop_by_pidfile "network-agent" +# Clean up TAP devices to prevent conflicts on next start +# This ensures network-agent recover() starts from a clean state and avoids +# race conditions during rapid restart when leftover TAP devices exist. +# +# TAP devices are named with the pattern: z... where the IP +# is from the CIDR configured in CUBE_SANDBOX_CIDR. We clean all devices +# matching this pattern regardless of the specific CIDR, since they are all +# created by network-agent for sandbox instances. +cleanup_tap_devices() { + local cleanup_count=0 + + # Match all TAP devices with pattern: z... + # This covers all CIDR ranges (e.g., z192.168.x.x, z10.x.x.x, etc.) + for iface in $(ip link show 2>/dev/null | grep -oE 'z[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' || true); do + if ip link delete "$iface" 2>/dev/null; then + cleanup_count=$((cleanup_count + 1)) + fi + done + + if [[ $cleanup_count -gt 0 ]]; then + log "cleaned $cleanup_count TAP device(s)" + fi +} + +cleanup_tap_devices + log "local services stopped" diff --git a/deploy/one-click/scripts/one-click/up.sh b/deploy/one-click/scripts/one-click/up.sh index ef6b8d38..4458b340 100755 --- a/deploy/one-click/scripts/one-click/up.sh +++ b/deploy/one-click/scripts/one-click/up.sh @@ -73,6 +73,9 @@ fi "${SCRIPT_DIR}/down-local.sh" >/dev/null 2>&1 || true +# Render config file placeholders before starting services +render_config_placeholders + start_with_pidfile \ "network-agent" \ "mkdir -p /tmp/cube \"${NETWORK_AGENT_STATE_DIR}\" && \"${NETWORK_AGENT_BIN}\" --cubelet-config \"${CUBELET_CONFIG}\" --state-dir \"${NETWORK_AGENT_STATE_DIR}\"" diff --git a/deploy/one-click/tests/test-cidr-replace.sh b/deploy/one-click/tests/test-cidr-replace.sh new file mode 100644 index 00000000..81f35282 --- /dev/null +++ b/deploy/one-click/tests/test-cidr-replace.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash +set -uo pipefail + +# Test script for Cubelet config.toml CIDR and tap_init_num placeholder replacement. +# Verifies that install.sh and common.sh sed substitutions correctly replace +# placeholders in Cubelet/config/config.toml with values from env vars. +# +# Usage: +# bash deploy/one-click/tests/test-cidr-replace.sh + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" +WORK_DIR="$(mktemp -d)" +trap 'rm -rf "${WORK_DIR}"' EXIT + +# --- Cross-platform sed -i wrapper --- +sedi() { + if [[ "$(uname)" == "Darwin" ]]; then + sed -i '' "$@" + else + sed -i "$@" + fi +} + +pass=0 +fail=0 + +assert_eq() { + local label="$1" actual="$2" expected="$3" + if [[ "${actual}" == "${expected}" ]]; then + echo " PASS: ${label}" + ((pass++)) + else + echo " FAIL: ${label}" + echo " expected: ${expected}" + echo " actual: ${actual}" + ((fail++)) + fi +} + +# --- Test 1: Default values --- +echo "=== Test 1: Default values (no env vars set) ===" +mkdir -p "${WORK_DIR}/t1/Cubelet/config" "${WORK_DIR}/t1/CubeMaster" +cp "${PROJECT_ROOT}/Cubelet/config/config.toml" "${WORK_DIR}/t1/Cubelet/config/config.toml" +cp "${PROJECT_ROOT}/configs/single-node/cubemaster.yaml" "${WORK_DIR}/t1/CubeMaster/conf.yaml" + +cidr="192.168.0.0/18" +tap_init_num=500 + +sedi -E \ + -e "s|__CUBE_SANDBOX_CIDR__|${cidr}|g" \ + -e "s/tap_init_num = [0-9]+/tap_init_num = ${tap_init_num}/" \ + "${WORK_DIR}/t1/Cubelet/config/config.toml" + +t1_cidr=$(grep -o 'cidr = "[^"]*"' "${WORK_DIR}/t1/Cubelet/config/config.toml") +assert_eq "config.toml cidr" "${t1_cidr}" 'cidr = "192.168.0.0/18"' + +t1_tap=$(grep -o 'tap_init_num = [0-9]*' "${WORK_DIR}/t1/Cubelet/config/config.toml") +assert_eq "config.toml tap_init_num" "${t1_tap}" 'tap_init_num = 500' + +t1_remain=$(grep -c '__CUBE_SANDBOX_' "${WORK_DIR}/t1/Cubelet/config/config.toml" || true) +assert_eq "config.toml no placeholders remain" "${t1_remain}" "0" + +# Verify cubemaster.yaml has no placeholder (denyOut is not modified by this PR) +t1_cm_remain=$(grep -c '__CUBE_SANDBOX_DENY_OUT__' "${WORK_DIR}/t1/CubeMaster/conf.yaml" || true) +assert_eq "cubemaster.yaml no DENY_OUT placeholder" "${t1_cm_remain}" "0" + +# --- Test 2: Custom CIDR --- +echo "" +echo "=== Test 2: Custom CIDR (10.128.0.0/16) ===" +mkdir -p "${WORK_DIR}/t2/Cubelet/config" +cp "${PROJECT_ROOT}/Cubelet/config/config.toml" "${WORK_DIR}/t2/Cubelet/config/config.toml" + +cidr="10.128.0.0/16" +tap_init_num=500 + +sedi -E \ + -e "s|__CUBE_SANDBOX_CIDR__|${cidr}|g" \ + -e "s/tap_init_num = [0-9]+/tap_init_num = ${tap_init_num}/" \ + "${WORK_DIR}/t2/Cubelet/config/config.toml" + +t2_cidr=$(grep -o 'cidr = "[^"]*"' "${WORK_DIR}/t2/Cubelet/config/config.toml") +assert_eq "config.toml cidr" "${t2_cidr}" 'cidr = "10.128.0.0/16"' + +t2_tap=$(grep -o 'tap_init_num = [0-9]*' "${WORK_DIR}/t2/Cubelet/config/config.toml") +assert_eq "config.toml tap_init_num" "${t2_tap}" 'tap_init_num = 500' + +t2_remain=$(grep -c '__CUBE_SANDBOX_' "${WORK_DIR}/t2/Cubelet/config/config.toml" || true) +assert_eq "config.toml no placeholders remain" "${t2_remain}" "0" + +# --- Test 3: Custom CIDR and tap_init_num --- +echo "" +echo "=== Test 3: Custom CIDR and tap_init_num ===" +mkdir -p "${WORK_DIR}/t3/Cubelet/config" +cp "${PROJECT_ROOT}/Cubelet/config/config.toml" "${WORK_DIR}/t3/Cubelet/config/config.toml" + +cidr="10.200.0.0/16" +tap_init_num=800 + +sedi -E \ + -e "s|__CUBE_SANDBOX_CIDR__|${cidr}|g" \ + -e "s/tap_init_num = [0-9]+/tap_init_num = ${tap_init_num}/" \ + "${WORK_DIR}/t3/Cubelet/config/config.toml" + +t3_cidr=$(grep -o 'cidr = "[^"]*"' "${WORK_DIR}/t3/Cubelet/config/config.toml") +assert_eq "config.toml cidr" "${t3_cidr}" 'cidr = "10.200.0.0/16"' + +t3_tap=$(grep -o 'tap_init_num = [0-9]*' "${WORK_DIR}/t3/Cubelet/config/config.toml") +assert_eq "config.toml tap_init_num" "${t3_tap}" 'tap_init_num = 800' + +t3_remain=$(grep -c '__CUBE_SANDBOX_' "${WORK_DIR}/t3/Cubelet/config/config.toml" || true) +assert_eq "config.toml no placeholders remain" "${t3_remain}" "0" + +# --- Summary --- +echo "" +echo "==============================" +echo "Results: ${pass} passed, ${fail} failed" +echo "==============================" +[[ "${fail}" -eq 0 ]] || exit 1