From b048418efcaea07bd3d9de239386fe6854c9db4a Mon Sep 17 00:00:00 2001 From: josh Date: Wed, 1 Jan 2025 17:17:34 -0600 Subject: [PATCH 1/4] feat: macos builds and local testnet --- Dockerfile | 28 ++++++++--------- docker-compose.testnet.yml | 64 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 14 deletions(-) create mode 100644 docker-compose.testnet.yml diff --git a/Dockerfile b/Dockerfile index edceab0e8..bee17439f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,6 @@ -ARG BASE_IMAGE=ubuntu:24.04 +ARG BASE_IMAGE=rust:1.83 FROM $BASE_IMAGE AS builder -SHELL ["/bin/bash", "-c"] - -# Set noninteractive mode for apt-get -ARG DEBIAN_FRONTEND=noninteractive LABEL ai.opentensor.image.authors="operations@opentensor.ai" \ ai.opentensor.image.vendor="Opentensor Foundation" \ @@ -12,17 +8,15 @@ LABEL ai.opentensor.image.authors="operations@opentensor.ai" \ ai.opentensor.image.description="Opentensor Subtensor Blockchain" \ ai.opentensor.image.documentation="https://docs.bittensor.com" -# Set up Rust environment -ENV RUST_BACKTRACE=1 -RUN apt-get update && \ - apt-get install -y curl build-essential protobuf-compiler clang git pkg-config libssl-dev && \ - rm -rf /var/lib/apt/lists/* - -RUN set -o pipefail && curl https://sh.rustup.rs -sSf | sh -s -- -y -ENV PATH="/root/.cargo/bin:${PATH}" RUN rustup update stable RUN rustup target add wasm32-unknown-unknown --toolchain stable + +# Set up Rust environment +ENV RUST_BACKTRACE=1 +RUN apt-get update && apt-get install -y protobuf-compiler curl clang git +RUN rm -rf /var/lib/apt/lists/* + # Copy entire repository COPY . /build WORKDIR /build @@ -38,7 +32,13 @@ EXPOSE 30333 9933 9944 FROM $BASE_IMAGE AS subtensor # Copy all chainspec files -COPY --from=builder /build/chainspecs/*.json / +COPY --from=builder /build/*.json / # Copy final binary COPY --from=builder /build/target/production/node-subtensor /usr/local/bin + + +FROM subtensor AS subtensor-local + + +RUN "node-subtensor" build-spec --disable-default-bootnode --raw --chain local > /localnet.json diff --git a/docker-compose.testnet.yml b/docker-compose.testnet.yml new file mode 100644 index 000000000..96beb2725 --- /dev/null +++ b/docker-compose.testnet.yml @@ -0,0 +1,64 @@ +volumes: + subtensor-alice: + subtensor-bob: + +services: + common: &common + image: ghcr.io/opentensor/subtensor:latest-local + cpu_count: 4 + mem_limit: 40000000000 + memswap_limit: 80000000000 + environment: + - CARGO_HOME=/var/www/node-subtensor/.cargo + + alice: + <<: *common + container_name: subtensor-alice + build: + context: . + dockerfile: Dockerfile + target: subtensor-local + ports: + - "9944:9944" + - "30334:30334" + volumes: + - subtensor-alice:/tmp/blockchain + command: + - /bin/bash + - -c + - | + node-subtensor \ + --base-path /tmp/blockchain \ + --chain localnet.json \ + --alice \ + --port 30334 \ + --rpc-port 9944 \ + --validator \ + --rpc-cors=all \ + --allow-private-ipv4 \ + --discover-local \ + --unsafe-force-node-key-generation + + bob: + <<: *common + container_name: subtensor-bob + ports: + - "9945:9945" + - "30335:30335" + volumes: + - subtensor-bob:/tmp/blockchain + command: + - /bin/bash + - -c + - | + node-subtensor \ + --base-path /tmp/blockchain \ + --chain localnet.json \ + --bob \ + --port 30335 \ + --rpc-port 9945 \ + --validator \ + --rpc-cors=all \ + --allow-private-ipv4 \ + --discover-local \ + --unsafe-force-node-key-generation From 573f67896b6ae24812cf363dba07e64a2ad40511 Mon Sep 17 00:00:00 2001 From: josh Date: Wed, 1 Jan 2025 17:48:39 -0600 Subject: [PATCH 2/4] fix: docker containers need rpc external mode --- README.md | 6 +++--- ...-compose.testnet.yml => docker-compose.localnet.yml | 10 ++++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) rename docker-compose.testnet.yml => docker-compose.localnet.yml (87%) diff --git a/README.md b/README.md index 30f36ffab..25a4b73d6 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ Requirements: --- -## For Subnet Development +## For Subnet Development If you are developing and testing subnet incentive mechanism, you will need to run a local subtensor node. Follow the detailed step-by-step instructions provided in the [**Subtensor Nodes** section in Bittensor Developer Documentation](https://docs.bittensor.com/subtensor-nodes). @@ -216,7 +216,7 @@ If you want to see the multi-node consensus algorithm in action, refer to our A Substrate project such as this consists of a number of components that are spread across a few directories. -### Node Capabilities +### Node Capabilities A blockchain node is an application that allows users to participate in a blockchain network. Substrate-based blockchain nodes expose a number of capabilities: @@ -232,7 +232,7 @@ Substrate-based blockchain nodes expose a number of capabilities: **Directory structure** -There are several files in the [`node`](./node/) directory. Make a note of the following important files: +There are several files in the [`node`](./node/) directory. Make a note of the following important files: - [`chain_spec.rs`](./node/src/chain_spec.rs): A [chain specification](https://docs.substrate.io/main-docs/build/chain-spec/) is a diff --git a/docker-compose.testnet.yml b/docker-compose.localnet.yml similarity index 87% rename from docker-compose.testnet.yml rename to docker-compose.localnet.yml index 96beb2725..30acb2fd9 100644 --- a/docker-compose.testnet.yml +++ b/docker-compose.localnet.yml @@ -21,6 +21,9 @@ services: ports: - "9944:9944" - "30334:30334" + expose: + - 9944 + - 30334 volumes: - subtensor-alice:/tmp/blockchain command: @@ -30,6 +33,8 @@ services: node-subtensor \ --base-path /tmp/blockchain \ --chain localnet.json \ + --rpc-external \ + --rpc-methods=unsafe \ --alice \ --port 30334 \ --rpc-port 9944 \ @@ -42,6 +47,9 @@ services: bob: <<: *common container_name: subtensor-bob + expose: + - 9945 + - 30335 ports: - "9945:9945" - "30335:30335" @@ -55,6 +63,8 @@ services: --base-path /tmp/blockchain \ --chain localnet.json \ --bob \ + --rpc-methods=unsafe \ + --rpc-external \ --port 30335 \ --rpc-port 9945 \ --validator \ From f94c19a27d2a0f8e16103341d31b7e948a5dc65d Mon Sep 17 00:00:00 2001 From: josh Date: Wed, 1 Jan 2025 18:07:02 -0600 Subject: [PATCH 3/4] feat: documentation on running a subtensor localnet, and testing it. --- docs/running-subtensor-locally.md | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/docs/running-subtensor-locally.md b/docs/running-subtensor-locally.md index 82bb87356..070d4edb5 100644 --- a/docs/running-subtensor-locally.md +++ b/docs/running-subtensor-locally.md @@ -1,3 +1,32 @@ # Running subtensor node locally -See the [**Subtensor Nodes** section in Bittensor Developer Documentation](https://docs.bittensor.com/subtensor-nodes). +For General information on running Subtensors, see +[**Subtensor Nodes** section in Bittensor Developer Documentation](https://docs.bittensor.com/subtensor-nodes). + +### Running a localnet subtensor node + +Running a localnet in docker compose is the easiest way to quickly iterate on +chain state, like building on the evm. + +1. install docker and docker compose, along with cloning this repository. + +1. build the images from source on the desired branch using + `docker compose -f docker-compose.localnet.yml build`. Note this will take + quite a while. + +1. Run the docker compose file via + `docker compose -f docker-compose.localnet.yml up -d` + +Now you should have a full local validator running. To test your connection, you +can use the following script to check `//Alice`'s balance. Alice is a sudo +account in localnet. + +```py +# pip install substrate-interface +from substrateinterface import Keypair, SubstrateInterface + +substrate = SubstrateInterface(url="ws://127.0.0.1:9945") +hotkey = Keypair.create_from_uri('//Alice') +result = substrate.query("System", "Account", [hotkey.ss58_address]) +print(result.value) +``` From 8d5a1ae7405ee22c98acf3270ea82d0d61b9e669 Mon Sep 17 00:00:00 2001 From: josh Date: Mon, 6 Jan 2025 15:07:33 -0600 Subject: [PATCH 4/4] fix: build localnet with proper flags --- Dockerfile | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index bee17439f..e924d9723 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ ARG BASE_IMAGE=rust:1.83 -FROM $BASE_IMAGE AS builder +FROM $BASE_IMAGE AS base_builder LABEL ai.opentensor.image.authors="operations@opentensor.ai" \ ai.opentensor.image.vendor="Opentensor Foundation" \ @@ -21,24 +21,44 @@ RUN rm -rf /var/lib/apt/lists/* COPY . /build WORKDIR /build + +# +# Image for building prod +# +FROM base_builder as prod_builder # Build the project RUN cargo build -p node-subtensor --profile production --features="metadata-hash" --locked - # Verify the binary was produced RUN test -e /build/target/production/node-subtensor - EXPOSE 30333 9933 9944 +# +# Final prod image +# FROM $BASE_IMAGE AS subtensor - # Copy all chainspec files -COPY --from=builder /build/*.json / - +COPY --from=prod_builder /build/*.json / # Copy final binary -COPY --from=builder /build/target/production/node-subtensor /usr/local/bin +COPY --from=prod_builder /build/target/production/node-subtensor /usr/local/bin -FROM subtensor AS subtensor-local +# +# Image for building local +# +FROM base_builder as local_builder +# Build the project +RUN cargo build --workspace --profile release --features="pow-faucet" +# Verify the binary was produced +RUN test -e /build/target/release/node-subtensor +EXPOSE 30333 9933 9944 +# +# Final local image +# +FROM $BASE_IMAGE AS subtensor-local +# Copy all chainspec files +COPY --from=local_builder /build/*.json / +# Copy final binary +COPY --from=local_builder /build/target/release/node-subtensor /usr/local/bin RUN "node-subtensor" build-spec --disable-default-bootnode --raw --chain local > /localnet.json