From d5926ea42b607bd3efddc6129600ab3a977d7388 Mon Sep 17 00:00:00 2001 From: Olivier Desenfans Date: Fri, 16 Feb 2024 15:15:53 +0100 Subject: [PATCH] Internal: build Stone separately from the SDK (#4) Problem: the build time of the SDK reaches ~20 minutes, which is not acceptable to users. This is due to the compilation of Stone in the `build.rs` file. Solution: we now build and install Stone from a Shell script that is isolated from the Rust compilation process. Developers must now run this script before attempting to run tests, but this accelerates the build process significantly. --- .github/workflows/ci.yml | 48 +++++++------- Cargo.toml | 1 - README.md | 47 ++++++++++++++ build.rs | 131 --------------------------------------- scripts/build-stone.sh | 32 ++++++++++ scripts/install-stone.sh | 43 +++++++++++++ src/prover.rs | 19 ++---- src/test_utils.rs | 12 ---- src/verifier.rs | 29 +++------ 9 files changed, 160 insertions(+), 202 deletions(-) delete mode 100644 build.rs create mode 100644 scripts/build-stone.sh create mode 100644 scripts/install-stone.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1d02038..90998c8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,7 +33,7 @@ jobs: components: rustfmt, clippy - name: Set up cargo cache - uses: actions/cache@v3 + uses: actions/cache@v4 continue-on-error: false with: path: | @@ -45,39 +45,43 @@ jobs: key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} restore-keys: ${{ runner.os }}-cargo- - - name: Log in to Github container registry - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - - - name: Set cache image environment variables + - name: Get Stone submodule version + id: get-stone-version run: | - # Uppercase characters are not allowed in Docker tags - cache_image=$(echo ghcr.io/${GITHUB_REPOSITORY}/build-cache | tr '[:upper:]' '[:lower:]') - echo "STONE_PROVER_DOCKER_CACHE=$(echo ${cache_image})" >> $GITHUB_ENV + echo "stone_version=$(cat .git/modules/dependencies/stone-prover/HEAD)" >> $GITHUB_OUTPUT + - name: Cache Stone prover and verifier + id: cache-stone + uses: actions/cache@v4 + with: + path: dependencies/stone + key: stone-${{ runner.os }}-${{ steps.get-stone-version.outputs.stone_version }} - - name: Download Docker cache image (if available) - run: docker pull ${STONE_PROVER_DOCKER_CACHE} || true + - name: Build Stone + if: steps.cache-stone.outputs.cache-hit != 'true' + run: | + bash -x scripts/install-stone.sh --install-dir ./dependencies/stone + + - name: Set Stone in PATH + run: | + echo "$(pwd)/dependencies/stone" >> $GITHUB_PATH - name: Build - run: cargo build --verbose + run: | + cargo build --verbose - name: Lint with Clippy - run: cargo clippy -- -D warnings + run: | + cargo clippy -- -D warnings - name: Run tests - run: cargo test --verbose + run: | + cargo test --verbose - name: Set release artifacts uses: softprops/action-gh-release@v1 if: startsWith(github.ref, 'refs/tags/') with: files: | - target/debug/cpu_air_prover - target/debug/cpu_air_verifier - - - name: Push the image to the cache - # It's not possible to push packages from fork PRs. - if: github.event.pull_request.head.repo.full_name == github.repository - run: | - docker tag stone-prover-build:latest ${STONE_PROVER_DOCKER_CACHE} - docker push ${STONE_PROVER_DOCKER_CACHE} + dependencies/stone/cpu_air_prover + dependencies/stone/cpu_air_verifier diff --git a/Cargo.toml b/Cargo.toml index d756ece..7f0f979 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,6 @@ name = "stone-prover-sdk" version = "0.1.0" edition = "2021" description = "Rust SDK for the Starkware Stone prover and verifier." -build = "build.rs" [dependencies] bincode = "2.0.0-rc.3" diff --git a/README.md b/README.md index 250d0a0..f25d7ca 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,50 @@ # Stone Prover SDK Rust SDK for the Starkware Stone prover and verifier. + +## Install + +To use this SDK, you will need the Stone prover and verifier binaries. +You can either follow the instructions on the [Stone repository](https://github.com/starkware-libs/stone-prover), +download them from the [latest SDK release](https://github.com/Moonsong-Labs/stone-prover-sdk/releases/latest) +or run the following commands: + +```shell +git clone --recurse-submodules https://github.com/Moonsong-Labs/stone-prover-sdk.git +cd stone-prover-sdk +bash scripts/install-stone.sh +``` + +This will install the prover and verifier in `${HOME}/.stone` and add this directory to your `PATH`. + +## Features + +### Prove and verify Cairo programs + +The `prover` and `verifier` modules contain thin abstractions on top of the prover and verifier. +They allow the user to prove and verify the execution of any Cairo program from Rust code. + +### Execute Cairo programs + +The `cairo_vm` module provides utility functions over the [cairo-vm](https://github.com/Moonsong-Labs/cairo-vm) +crate to execute Cairo programs using the Rust Cairo VM. + +## Contribute + +### Set up the development environment + +First, clone the repository and install Stone: + +```shell +git clone --recurse-submodules https://github.com/Moonsong-Labs/stone-prover-sdk.git +cd stone-prover-sdk +bash scripts/install-stone.sh +``` + +This step takes several minutes. The script adds the install directory (`$HOME/.stone` by default) to your `PATH` +for supported shells. Make sure that this is the case: + +```shell +which cpu_air_prover +# Should print /cpu_air_prover +``` diff --git a/build.rs b/build.rs deleted file mode 100644 index 8fb87a8..0000000 --- a/build.rs +++ /dev/null @@ -1,131 +0,0 @@ -/// Builds the Stone Prover C++ submodule to make it callable from the wrapper. -use std::path::{Path, PathBuf}; - -#[derive(Debug)] -enum CommandError { - /// The command failed with a non-zero return code. - CommandFailed(std::process::Output), - /// The command could not be launched. - IoError(std::io::Error), -} - -impl From for CommandError { - fn from(value: std::io::Error) -> Self { - Self::IoError(value) - } -} - -/// Run any shell command line and retrieve its output. -fn run_command(command: &str) -> Result { - let output = std::process::Command::new("sh") - .arg("-c") - .arg(command) - .output()?; - - if !output.status.success() { - return Err(CommandError::CommandFailed(output)); - } - Ok(output) -} - -/// Copy a file from a running Docker container. -fn copy_file_from_container( - container_name: &str, - container_file: &Path, - target: &Path, -) -> Result<(), CommandError> { - let docker_copy_command = format!( - "docker cp -L {container_name}:{} {}", - container_file.to_string_lossy(), - target.to_string_lossy() - ); - let _ = run_command(&docker_copy_command); - Ok(()) -} - -/// Copy the prover and verifier binary files from the prover build container. -fn copy_prover_files_from_container( - container_name: &str, - output_dir: &Path, -) -> Result<(), CommandError> { - copy_file_from_container(container_name, Path::new("/bin/cpu_air_prover"), output_dir)?; - copy_file_from_container( - container_name, - Path::new("/bin/cpu_air_verifier"), - output_dir, - )?; - - Ok(()) -} - -fn make_docker_build_command(repo_dir: &Path, image_name: &str) -> String { - let mut docker_build_command = format!( - "docker build -t {image_name} {}", - repo_dir.to_string_lossy() - ); - - // Check if the platform is Mac - #[cfg(target_os = "macos")] - { - // Add build args - docker_build_command.push_str(" --build-arg CMAKE_ARGS=-DNO_AVX=1"); - } - - // Check if a cache image exists. Used by the CI/CD pipeline. - if let Ok(cache_image) = std::env::var("STONE_PROVER_DOCKER_CACHE") { - docker_build_command.push_str(&format!(" --cache-from {cache_image}")); - } - - docker_build_command -} - -/// Build the Stone Prover and copy binaries to `output_dir`. -/// -/// The prover repository contains a Dockerfile to build the prover. This function: -/// 1. Builds the Dockerfile -/// 2. Starts a container based on the generated image -/// 3. Extracts the binaries from the container -/// 4. Stops the container. -fn build_stone_prover(repo_dir: &Path, output_dir: &Path) { - // Build the Stone Prover build Docker image - let image_name = "stone-prover-build:latest"; - let docker_build_command = make_docker_build_command(repo_dir, image_name); - run_command(&docker_build_command).expect("Failed to build Stone Prover using Dockerfile"); - - // Run a container based on the Docker image - let docker_create_command = format!("docker create {image_name}"); - let docker_create_output = run_command(&docker_create_command) - .expect("Failed to start container to copy prover files"); - let container_name = String::from_utf8_lossy(&docker_create_output.stdout) - .trim() - .to_owned(); - println!("Started container {container_name}"); - - // Copy the files - let copy_result = copy_prover_files_from_container(&container_name, output_dir); - - // Stop the container - let docker_delete_command = format!("docker rm {container_name}"); - run_command(&docker_delete_command).expect("Failed to stop and delete prover build container"); - - // Handle a potential error during copy - if let Err(e) = copy_result { - panic!( - "Failed to copy files from the prover build container: {:?}", - e - ); - } -} - -fn main() { - let out_dir = PathBuf::from(std::env::var("OUT_DIR").unwrap()); - // Copy the prover and verifier to the root of the target directory - // to make them easy to find for all the crates in the workspace. - let target_dir = out_dir.join("../../..").canonicalize().unwrap(); - - let stone_prover_repo_dir = Path::new("dependencies/stone-prover"); - build_stone_prover(stone_prover_repo_dir, target_dir.as_path()); - - // Rerun if the submodule is updated - println!("cargo:rerun-if-changed=.git/modules/dependencies/stone-prover/HEAD"); -} diff --git a/scripts/build-stone.sh b/scripts/build-stone.sh new file mode 100644 index 0000000..eb731e4 --- /dev/null +++ b/scripts/build-stone.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +set -eo pipefail + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +OUTPUT_DIR="$(pwd)" + +while true; do + case "$1" in + -o | --output-dir ) OUTPUT_DIR="$2"; shift 2 ;; + * ) break ;; + esac +done + +mkdir -p "${OUTPUT_DIR}" + +STONE_DIR="${SCRIPT_DIR}/../dependencies/stone-prover" + +STONE_GIT_TAG=$(cat .git/modules/dependencies/stone-prover/HEAD) +IMAGE_NAME="stone-prover-build:${STONE_GIT_TAG}" +ARCH_NAME=$(uname -m) + +BUILD_FLAGS="" +if [ "${ARCH_NAME}" == "arm64" ]; then BUILD_FLAGS="--build-arg CMAKE_ARGS=-DNO_AVX=1"; fi + +docker build -t "${IMAGE_NAME}" ${BUILD_FLAGS} "${STONE_DIR}" + +CONTAINER=$(docker create "${IMAGE_NAME}") +docker cp -L "${CONTAINER}:/bin/cpu_air_prover" "${OUTPUT_DIR}" +docker cp -L "${CONTAINER}:/bin/cpu_air_verifier" "${OUTPUT_DIR}" + +docker rm "${CONTAINER}" \ No newline at end of file diff --git a/scripts/install-stone.sh b/scripts/install-stone.sh new file mode 100644 index 0000000..dbe8b3c --- /dev/null +++ b/scripts/install-stone.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +set -eo pipefail + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +INSTALL_DIR="${HOME}/.stone" + +while true; do + case "$1" in + -i | --install-dir ) INSTALL_DIR="$2"; shift 2 ;; + * ) break ;; + esac +done + +echo "Installing Stone in ${INSTALL_DIR}..." +mkdir -p "${INSTALL_DIR}" +bash "${SCRIPT_DIR}/build-stone.sh" --output-dir "${INSTALL_DIR}" + +# Add the tool to the PATH + +echo "Configuring PATH..." +if [[ ":$PATH:" != *":${INSTALL_DIR}:"* ]]; then + PROFILE_FILE="" + # ZSH_NAME is set on zsh + if [ -v ZSH_NAME ]; then + PROFILE_FILE="${HOME}/.zsh" + elif [ -v BASH ]; then + PROFILE_FILE="${HOME}/.bashrc" + else + echo "Unsupported shell, you will need to add the export PATH statement in the right configuration file manually." + fi + + if [ -n "${PROFILE_FILE}" ]; then + echo -e "\n# Stone prover and verifier\nexport PATH=\"${INSTALL_DIR}:\$PATH\"" >> "${PROFILE_FILE}" + fi +fi + +# Notify the user to update the PATH immediately +echo "Done!" +echo "Stone was added to ${PROFILE_FILE} and will be available the next time you open a shell." +echo "To add Stone to your PATH immediately, run the following command:" +echo "export PATH=\"${INSTALL_DIR}:\$PATH\"" \ No newline at end of file diff --git a/src/prover.rs b/src/prover.rs index 67c70e2..b61740b 100644 --- a/src/prover.rs +++ b/src/prover.rs @@ -232,18 +232,15 @@ mod test { use tempfile::NamedTempFile; use crate::test_utils::{ - parsed_prover_test_case, prover_cli_test_case, prover_in_path, read_proof_file, - ParsedProverTestCase, ProverCliTestCase, + parsed_prover_test_case, prover_cli_test_case, read_proof_file, ParsedProverTestCase, + ProverCliTestCase, }; use super::*; /// Check that the Stone Prover command-line wrapper works. #[rstest] - fn test_run_prover_from_command_line( - prover_cli_test_case: ProverCliTestCase, - #[from(prover_in_path)] _path: (), - ) { + fn test_run_prover_from_command_line(prover_cli_test_case: ProverCliTestCase) { let output_file = NamedTempFile::new().expect("Creating output file failed"); run_prover_from_command_line( &prover_cli_test_case.public_input_file, @@ -259,10 +256,7 @@ mod test { } #[rstest] - fn test_run_prover( - parsed_prover_test_case: ParsedProverTestCase, - #[from(prover_in_path)] _path: (), - ) { + fn test_run_prover(parsed_prover_test_case: ParsedProverTestCase) { let proof = run_prover( &parsed_prover_test_case.public_input, &parsed_prover_test_case.private_input, @@ -278,10 +272,7 @@ mod test { #[rstest] #[tokio::test] - async fn test_run_prover_async( - parsed_prover_test_case: ParsedProverTestCase, - #[from(prover_in_path)] _path: (), - ) { + async fn test_run_prover_async(parsed_prover_test_case: ParsedProverTestCase) { let proof = run_prover_async( &parsed_prover_test_case.public_input, &parsed_prover_test_case.private_input, diff --git a/src/test_utils.rs b/src/test_utils.rs index 2f99cab..ae57c69 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -10,18 +10,6 @@ use tempfile::NamedTempFile; use crate::json::read_json_from_file; use crate::models::{Proof, ProverConfig, ProverParameters, PublicInput}; -#[fixture] -pub fn prover_in_path() { - // Add build dir to path for the duration of the test - let path = std::env::var("PATH").unwrap_or_default(); - let build_dir = Path::new(env!("OUT_DIR")); - // This will find the root of the target directory where the prover binaries - // are put after compilation. - let target_dir = build_dir.join("../../..").canonicalize().unwrap(); - - std::env::set_var("PATH", format!("{}:{path}", target_dir.to_string_lossy())); -} - /// Reads and deserializes a JSON proof file. pub fn read_proof_file>(proof_file: P) -> Proof { let proof: Proof = read_json_from_file(proof_file).expect("Could not open proof file"); diff --git a/src/verifier.rs b/src/verifier.rs index 76746f1..365c03a 100644 --- a/src/verifier.rs +++ b/src/verifier.rs @@ -145,32 +145,26 @@ pub async fn run_verifier_from_command_line_async( mod test { use rstest::rstest; - use crate::test_utils::{prover_in_path, prover_test_case, ProverTestCase}; + use crate::test_utils::{prover_test_case, ProverTestCase}; use super::*; /// Check that the Stone Verifier command-line wrapper works. #[rstest] - fn test_run_verifier_from_command_line( - prover_test_case: ProverTestCase, - #[from(prover_in_path)] _path: (), - ) { + fn test_run_verifier_from_command_line(prover_test_case: ProverTestCase) { let proof_file = prover_test_case.proof_file; run_verifier_from_command_line(proof_file.as_path(), None, None) .expect("Proof file is valid"); } #[rstest] - fn test_run_verifier(prover_test_case: ProverTestCase, #[from(prover_in_path)] _path: ()) { + fn test_run_verifier(prover_test_case: ProverTestCase) { let proof_file = prover_test_case.proof_file; run_verifier(proof_file.as_path()).expect("Proof file is valid"); } #[rstest] - fn test_run_verifier_with_annotations( - prover_test_case: ProverTestCase, - #[from(prover_in_path)] _path: (), - ) { + fn test_run_verifier_with_annotations(prover_test_case: ProverTestCase) { let output_dir = tempfile::tempdir().unwrap(); let annotation_file = output_dir.path().join("annotations.json"); let extra_output_file = output_dir.path().join("extra_output_file.json"); @@ -189,10 +183,7 @@ mod test { /// Check that the Stone Verifier command-line wrapper works. #[rstest] #[tokio::test] - async fn test_run_verifier_from_command_line_async( - prover_test_case: ProverTestCase, - #[from(prover_in_path)] _path: (), - ) { + async fn test_run_verifier_from_command_line_async(prover_test_case: ProverTestCase) { let proof_file = prover_test_case.proof_file; run_verifier_from_command_line_async(proof_file.as_path(), None, None) .await @@ -201,10 +192,7 @@ mod test { #[rstest] #[tokio::test] - async fn test_run_verifier_async( - prover_test_case: ProverTestCase, - #[from(prover_in_path)] _path: (), - ) { + async fn test_run_verifier_async(prover_test_case: ProverTestCase) { let proof_file = prover_test_case.proof_file; run_verifier_async(proof_file.as_path()) .await @@ -213,10 +201,7 @@ mod test { #[rstest] #[tokio::test] - async fn test_run_verifier_with_annotations_async( - prover_test_case: ProverTestCase, - #[from(prover_in_path)] _path: (), - ) { + async fn test_run_verifier_with_annotations_async(prover_test_case: ProverTestCase) { let output_dir = tempfile::tempdir().unwrap(); let annotation_file = output_dir.path().join("annotations.json"); let extra_output_file = output_dir.path().join("extra_output_file.json");