Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ This will:
cargo run -- init \
--curio local:/home/user/code/curio \
--filecoin-services local:/home/user/code/filecoin-services \
--pdp local:/home/user/code/pdp \
--lotus local:/home/user/code/lotus \
```

Expand Down Expand Up @@ -124,8 +125,9 @@ Only needs three things on your machine:
Everything else (Lotus, Curio, dependencies) is built inside Docker.

### ⚙️ Configurable Repositories
Depends on 3 repositories, all configurable:
Depends on these repositories, all configurable:
- `filecoin-services` - FOC smart contracts
- `pdp` - PDP contracts, optionally pinned independently from filecoin-services
- `curio` - Next-gen storage provider
- `lotus` - Filecoin daemon

Expand Down
14 changes: 12 additions & 2 deletions README_ADVANCED.md
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ foc-devnet init [OPTIONS]
- `--curio <SOURCE>` - Curio source location
- `--lotus <SOURCE>` - Lotus source location
- `--filecoin-services <SOURCE>` - Filecoin Services source location
- `--pdp <SOURCE>` - PDP source location. If omitted, uses the PDP submodule bundled with Filecoin Services.
- `--proof-params-dir <PATH>` - Local proof params directory
- `--rand` - Use random mnemonic instead of deterministic one. Use this for unique test scenarios.

Expand Down Expand Up @@ -257,6 +258,12 @@ branch = "pdpv0"
url = "https://github.com/FilOzone/filecoin-services.git"
tag = "v1.0.0"

# Optional. When omitted, filecoin-services' service_contracts/lib/pdp
# submodule is used.
[pdp]
url = "https://github.com/FilOzone/pdp.git"
tag = "v3.4.0"

[multicall3]
url = "https://github.com/mds1/multicall3.git"
branch = "main"
Expand Down Expand Up @@ -809,6 +816,7 @@ port_range_count = 100
- **[lotus](https://github.com/filecoin-project/lotus)** - Filecoin daemon
- **[curio](https://github.com/filecoin-project/curio)** - Storage provider (PDP)
- **[filecoin-services](https://github.com/FilOzone/filecoin-services)** - FOC smart contracts
- **[pdp](https://github.com/FilOzone/pdp)** - PDP contracts; optional independent override mounted over `service_contracts/lib/pdp`
- **[multicall3](https://github.com/mds1/multicall3)** - Multicall3 contract
### Dependent Version Strategy

Expand All @@ -830,7 +838,8 @@ Default versions for these repositories are defined in code (see [`src/config.rs
foc-devnet init \
--lotus local:/home/user/dev/lotus \
--curio local:/home/user/dev/curio \
--filecoin-services local:/home/user/dev/filecoin-services
--filecoin-services local:/home/user/dev/filecoin-services \
--pdp local:/home/user/dev/pdp
```

**Mixed approach:**
Expand Down Expand Up @@ -1327,4 +1336,5 @@ the latest final releases and nightly `frontier` runs pin current development
branch heads to immutable commits. The resolved metadata path is exposed to
scenarios as `CI_DEPENDENCY_METADATA`; Synapse SDK and filecoin-pin also receive
their exact source, version/ref, and commit through `SYNAPSE_SDK_*` and
`FILECOIN_PIN_*` environment variables.
`FILECOIN_PIN_*` environment variables. PDP receives `PDP_SOURCE`, `PDP_REF`,
and `PDP_COMMIT` when dependency profiles are resolved.
2 changes: 1 addition & 1 deletion ci/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ components.
Installation currently lives in three places (which consume the resolved
metadata):

- `foc-devnet init`: Lotus, Curio, and filecoin-services.
- `foc-devnet init`: Lotus, Curio, filecoin-services, and optionally PDP.
- `scenarios/synapse.py`: Synapse SDK scenario dependency.
- `scenarios/test_multi_copy_upload.py`: filecoin-pin scenario dependency.

Expand Down
14 changes: 14 additions & 0 deletions ci/dependency-profiles.json
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,20 @@
"branch": "main"
}
},
"pdp": {
"repository": "https://github.com/FilOzone/pdp.git",
"default": {
"strategy": "config_default"
},
"stability": {
"strategy": "git_tag",
"tag": "v*"
},
Comment thread
galargh marked this conversation as resolved.
"frontier": {
"strategy": "git_branch",
"branch": "main"
}
},
"synapse-sdk": {
"repository": "https://github.com/FilOzone/synapse-sdk.git",
"npm_package": "@filoz/synapse-sdk",
Expand Down
6 changes: 6 additions & 0 deletions scripts/resolve-ci-dependencies.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
"lotus": "--lotus",
"curio": "--curio",
"filecoin-services": "--filecoin-services",
"pdp": "--pdp",
}
VERSION_RE = re.compile(r"^\d+(?:\.\d+)*(?:[-+][0-9A-Za-z.-]+)?$")
STABLE_VERSION_RE = re.compile(r"^\d+(?:\.\d+)*$")
Expand Down Expand Up @@ -306,6 +307,9 @@ def scenario_environment(metadata_path: Path, components: dict) -> dict:
"FILECOIN_PIN_SOURCE": filecoin_pin["source"],
"FILECOIN_PIN_VERSION": filecoin_pin.get("version", ""),
"FILECOIN_PIN_COMMIT": filecoin_pin.get("commit", ""),
"PDP_SOURCE": components["pdp"]["source"],
"PDP_REF": components["pdp"].get("ref", ""),
"PDP_COMMIT": components["pdp"].get("commit", ""),
}


Expand Down Expand Up @@ -335,6 +339,8 @@ def verify(args) -> None:
metadata = json.loads(args.metadata.read_text())
components = metadata["components"]
for name in INIT_COMPONENT_FLAGS:
if name == "pdp" and components[name]["source"] == "config_default":
continue
repository_path = args.code_dir / name
actual = run_command(["git", "-C", str(repository_path), "rev-parse", "HEAD"])
expected = components[name].get("commit")
Expand Down
58 changes: 57 additions & 1 deletion scripts/tests/test_resolve_ci_dependencies.py
Original file line number Diff line number Diff line change
Expand Up @@ -314,6 +314,11 @@ def test_init_args_skip_config_defaults_and_pin_other_sources(self):
"repository": "https://example.test/services.git",
"commit": "def",
},
"pdp": {
"source": "git",
"repository": "https://example.test/pdp.git",
"commit": "123",
},
}
self.assertEqual(
resolver.build_init_args(components),
Expand All @@ -322,9 +327,20 @@ def test_init_args_skip_config_defaults_and_pin_other_sources(self):
"gitcommit:https://example.test/curio.git:abc",
"--filecoin-services",
"gitcommit:https://example.test/services.git:def",
"--pdp",
"gitcommit:https://example.test/pdp.git:123",
],
)

def test_init_args_skip_default_pdp(self):
components = {
"lotus": {"source": "config_default"},
"curio": {"source": "config_default"},
"filecoin-services": {"source": "config_default"},
"pdp": {"source": "config_default"},
}
self.assertEqual(resolver.build_init_args(components), [])

def test_cache_hash_depends_only_on_lotus_and_curio_commits(self):
base = {
"lotus": {"commit": "aaa"},
Expand All @@ -348,7 +364,10 @@ def test_verify_records_checkouts_and_writes_cache_key(self, run_command):
metadata = {
"schema_version": 1,
"profile": "default",
"components": {name: {"source": "config_default"} for name in commits},
"components": {
**{name: {"source": "config_default"} for name in commits},
"pdp": {"source": "config_default"},
},
}
with tempfile.TemporaryDirectory() as directory:
directory = Path(directory)
Expand All @@ -370,8 +389,45 @@ def test_verify_records_checkouts_and_writes_cache_key(self, run_command):

self.assertEqual(verified["components"]["lotus"]["commit"], "aaa")
self.assertTrue(verified["components"]["curio"]["verified"])
self.assertFalse(verified["components"]["pdp"].get("verified", False))
self.assertIn("source-hash=", outputs)

@patch.object(resolver, "run_command")
def test_verify_checks_independent_pdp_checkout(self, run_command):
commits = {
"lotus": "aaa",
"curio": "bbb",
"filecoin-services": "ccc",
"pdp": "ddd",
}
run_command.side_effect = lambda command: commits[Path(command[2]).name]
metadata = {
"schema_version": 1,
"profile": "frontier",
"components": {
name: {"source": "git", "commit": commit}
for name, commit in commits.items()
},
}
with tempfile.TemporaryDirectory() as directory:
directory = Path(directory)
metadata_path = directory / "metadata.json"
metadata_path.write_text(json.dumps(metadata))
args = type(
"Args",
(),
{
"metadata": metadata_path,
"code_dir": directory / "code",
"github_output": None,
},
)
resolver.verify(args)
verified = json.loads(metadata_path.read_text())

self.assertTrue(verified["components"]["pdp"]["verified"])
self.assertEqual(verified["components"]["pdp"]["commit"], "ddd")


if __name__ == "__main__":
unittest.main()
8 changes: 8 additions & 0 deletions src/cli.rs
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,14 @@ pub enum Commands {
/// 'local:/path/to/filecoin-services'.
#[arg(long)]
filecoin_services: Option<String>,
/// PDP source location. If omitted, uses filecoin-services' bundled lib/pdp submodule.
/// Latest tag: 'latesttag' (newest), 'latesttag:<selector>' (e.g. 'latesttag:v*'),
/// 'latesttag:<url>:<selector>' (custom repo). Resolved once at init.
/// Explicit: 'gittag:<tag>', 'gittag:<url>:<tag>', 'gitcommit:<sha>',
/// 'gitcommit:<url>:<sha>', 'gitbranch:<branch>', 'gitbranch:<url>:<branch>',
/// 'local:/path/to/pdp'.
#[arg(long)]
pdp: Option<String>,
/// Path to local filecoin-proof-params directory to use instead of downloading
#[arg(long)]
proof_params_dir: Option<String>,
Expand Down
18 changes: 18 additions & 0 deletions src/commands/init/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,15 @@ use crate::paths::foc_devnet_config;
/// * `curio_location` - Optional override for Curio repository location
/// * `lotus_location` - Optional override for Lotus repository location
/// * `filecoin_services_location` - Optional override for Filecoin Services repository location
/// * `pdp_location` - Optional override for PDP repository location
///
/// # Returns
/// Returns `Ok(())` on successful config generation, or an error if generation fails.
pub fn generate_default_config(
curio_location: Option<String>,
lotus_location: Option<String>,
filecoin_services_location: Option<String>,
pdp_location: Option<String>,
) -> Result<(), Box<dyn std::error::Error>> {
let config_path = foc_devnet_config();

Expand All @@ -37,6 +39,7 @@ pub fn generate_default_config(
if curio_location.is_some()
|| lotus_location.is_some()
|| filecoin_services_location.is_some()
|| pdp_location.is_some()
{
let content = fs::read_to_string(&config_path)?;
let mut config: Config = toml::from_str(&content)
Expand All @@ -46,6 +49,7 @@ pub fn generate_default_config(
curio_location,
lotus_location,
filecoin_services_location,
pdp_location,
)?;
let updated = toml::to_string(&config)
.map_err(|e| format!("Failed to serialize config: {}", e))?;
Expand All @@ -66,6 +70,7 @@ pub fn generate_default_config(
curio_location,
lotus_location,
filecoin_services_location,
pdp_location,
)?;

let default_config = toml::to_string(&config)
Expand All @@ -83,6 +88,7 @@ fn apply_overrides(
curio_location: Option<String>,
lotus_location: Option<String>,
filecoin_services_location: Option<String>,
pdp_location: Option<String>,
) -> Result<(), Box<dyn std::error::Error>> {
apply_location_override(
&mut config.lotus,
Expand All @@ -99,6 +105,18 @@ fn apply_overrides(
filecoin_services_location,
"https://github.com/FilOzone/filecoin-services.git",
)?;
if let Some(pdp_location) = pdp_location {
let mut pdp = config.pdp.clone().unwrap_or_else(|| Location::GitBranch {
url: "https://github.com/FilOzone/pdp.git".to_string(),
branch: "main".to_string(),
});
apply_location_override(
&mut pdp,
Some(pdp_location),
"https://github.com/FilOzone/pdp.git",
)?;
config.pdp = Some(pdp);
}

Ok(())
}
Expand Down
2 changes: 2 additions & 0 deletions src/commands/init/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ pub struct InitOptions {
pub curio_location: Option<String>,
pub lotus_location: Option<String>,
pub filecoin_services_location: Option<String>,
pub pdp_location: Option<String>,
pub proof_params_dir: Option<String>,
pub use_random_mnemonic: bool,
pub no_docker_build: bool,
Expand All @@ -47,6 +48,7 @@ pub fn init_environment(options: InitOptions) -> Result<(), Box<dyn std::error::
options.curio_location.clone(),
options.lotus_location.clone(),
options.filecoin_services_location.clone(),
options.pdp_location.clone(),
)?;

// Generate keys
Expand Down
6 changes: 6 additions & 0 deletions src/commands/init/repositories.rs
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,12 @@ pub fn download_code_repositories() -> Result<(), Box<dyn std::error::Error>> {
// Download filecoin-services repository if Git-based
download_repository("filecoin-services", &config.filecoin_services)?;

// Download PDP repository if independently configured. Otherwise use the
// filecoin-services submodule.
if let Some(pdp) = &config.pdp {
download_repository("pdp", pdp)?;
}

// Download multicall3 repository if Git-based
download_repository("multicall3", &config.multicall3)?;

Expand Down
20 changes: 11 additions & 9 deletions src/commands/start/foc_deploy/deployment.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
//! including contract deployment and output parsing.

use crate::commands::start::foc_deploy::contract_addresses::ContractAddresses;
use crate::commands::start::foc_deployer::deploy_foc_contracts;
use crate::commands::start::foc_deployer::{deploy_foc_contracts, DeployFocContractsParams};
use crate::paths::{contract_addresses_file, foc_metadata_file};
use std::error::Error;
use tracing::{info, warn};
Expand Down Expand Up @@ -51,22 +51,24 @@ pub fn perform_deployment(
super::helpers::check_required_addresses(context)?;

let services_repo = super::helpers::get_filecoin_services_repo_path()?;
let pdp_repo = super::helpers::get_pdp_repo_path()?;

// Get Lotus container name and RPC URL
let run_id = context.run_id();
let lotus_container = crate::docker::containers::lotus_container_name(run_id);
let lotus_rpc_url = crate::commands::start::lotus_utils::get_lotus_rpc_url(context)?;

// Deploy FOC contracts using deployment script
let contract_addresses = deploy_foc_contracts(
&foc_deployer,
&foc_deployer_eth,
&mock_usdfc_address,
&services_repo,
&lotus_container,
&lotus_rpc_url,
let contract_addresses = deploy_foc_contracts(DeployFocContractsParams {
foc_deployer: &foc_deployer,
deployer_eth_addr: &foc_deployer_eth,
mock_usdfc_address: &mock_usdfc_address,
services_repo_path: &services_repo,
pdp_repo_path: pdp_repo.as_deref(),
lotus_container: &lotus_container,
lotus_rpc_url: &lotus_rpc_url,
run_id,
)?;
})?;

// Store contract addresses in context
for (name, addr) in &contract_addresses.addresses {
Expand Down
13 changes: 13 additions & 0 deletions src/commands/start/foc_deploy/foc_deploy_step.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
use super::deployment::{check_existing_deployment, perform_deployment, post_execute_verification};
use super::helpers::{
check_lotus_running, check_required_addresses, get_filecoin_services_repo_path,
get_pdp_repo_path,
};
use crate::commands::start::step::{SetupContext, Step};
use std::error::Error;
Expand Down Expand Up @@ -56,6 +57,18 @@ impl Step for FOCDeployStep {
}
info!("filecoin-services repository found");

if let Some(pdp_repo) = get_pdp_repo_path()? {
if !pdp_repo.exists() {
return Err(format!(
"PDP repository not found at {}. \
Please run 'foc-devnet init' to clone the repository.",
pdp_repo.display()
)
.into());
}
info!("PDP repository found");
}

// Check if deployment script exists
let deploy_script = services_repo
.join("service_contracts")
Expand Down
Loading