From d51cbdaf7838f72ce3551685ab11f96931524dab Mon Sep 17 00:00:00 2001 From: Bohdan Khorolets Date: Tue, 12 Mar 2024 18:23:48 +0200 Subject: [PATCH] chore: Improve observability by addind a few additional metrics, expose the binary build info (#88) --- .cargo/config.toml | 2 ++ Cargo.lock | 6 ++++-- Cargo.toml | 6 ++++++ build.rs | 20 ++++++++++++++++++++ src/main.rs | 14 ++++++++++++++ src/metrics.rs | 30 ++++++++++++++++++++++++++++++ 6 files changed, 76 insertions(+), 2 deletions(-) create mode 100644 .cargo/config.toml create mode 100644 build.rs create mode 100644 src/metrics.rs diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000..57c55d3 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,2 @@ +[env] +NEARCORE_VERSION = "1.37.1" diff --git a/Cargo.lock b/Cargo.lock index 9d0c8ba..34058f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3666,7 +3666,9 @@ dependencies = [ "near-indexer", "near-indexer-primitives", "near-o11y", + "once_cell", "openssl-probe", + "rustc_version 0.4.0", "serde", "serde_json", "tokio", @@ -4403,9 +4405,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "opaque-debug" diff --git a/Cargo.toml b/Cargo.toml index 23dada5..8a9119e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,10 @@ version = "0.1.29" authors = ["Near Inc "] edition = "2021" +[build-dependencies] +anyhow = "1.0.51" +rustc_version = "0.4" + [dependencies] actix = "0.13.0" anyhow = "1.0.51" @@ -18,6 +22,7 @@ http = "0.2" humantime = "2.1.0" itertools = "0.10.0" openssl-probe = "0.1.5" +once_cell = "1.19.0" serde = { version = "1", features = ["derive"] } serde_json = "1.0.55" tokio = { version = "1.1", features = ["sync", "time"] } @@ -25,6 +30,7 @@ tokio-stream = { version = "0.1" } tracing = "0.1.34" tracing-subscriber = "0.2.4" +# Please, update the supported nearcore version in .cargo/config.toml file near-indexer = { git = "https://github.com/near/nearcore", rev = "0f8e073b0be5c2df4d9382bd359edb4d1f88b632" } near-indexer-primitives = { git = "https://github.com/near/nearcore", rev = "0f8e073b0be5c2df4d9382bd359edb4d1f88b632" } near-client = { git = "https://github.com/near/nearcore", rev = "0f8e073b0be5c2df4d9382bd359edb4d1f88b632" } diff --git a/build.rs b/build.rs new file mode 100644 index 0000000..777e594 --- /dev/null +++ b/build.rs @@ -0,0 +1,20 @@ +/// This build.rs script is used to generate build-time information and set environment variables for the build process. +/// It retrieves the Rust compiler version and sets it as the `RUSTC_VERSION` environment variable. +/// It also sets the `BUILD_VERSION` environment variable to the value of `NEARCORE_VERSION` defined in the project. +/// Additionally, it prints messages to indicate which files should trigger a rebuild when changed. +fn get_rustc_version() -> anyhow::Result { + let version = rustc_version::version()?; + Ok(version.to_string()) +} + +fn main() -> anyhow::Result<()> { + println!("cargo:rerun-if-changed=.git/HEAD"); + println!("cargo:rerun-if-changed=.git/index"); + + println!("cargo:rustc-env=BUILD_VERSION={}", env!("NEARCORE_VERSION")); + + let rustc_version = get_rustc_version()?; + println!("cargo:rustc-env=RUSTC_VERSION={}", rustc_version); + + Ok(()) +} diff --git a/src/main.rs b/src/main.rs index 4493bdb..ec4c103 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,6 +9,7 @@ use tokio::sync::Mutex; use tracing_subscriber::EnvFilter; mod configs; +mod metrics; mod utils; const INDEXER: &str = "near_lake"; @@ -49,6 +50,17 @@ fn main() { env!("CARGO_PKG_VERSION") ); + // This will set the near_lake_build_info metric + // e.g. near_lake_build_info{build="1.37.1",release="0.1.29",rustc_version="1.75.0"} + metrics::NODE_BUILD_INFO.reset(); + metrics::NODE_BUILD_INFO + .with_label_values(&[ + env!("CARGO_PKG_VERSION"), + env!("BUILD_VERSION"), + env!("RUSTC_VERSION"), + ]) + .inc(); + let system = actix::System::new(); system.block_on(async move { let indexer_config = args.clone().to_indexer_config(home_dir); @@ -226,6 +238,7 @@ async fn handle_message( let mut stats_lock = stats.lock().await; stats_lock.block_heights_processing.remove(&block_height); stats_lock.blocks_processed_count += 1; + metrics::BLOCKS_DONE.inc(); stats_lock.last_processed_block_height = block_height; drop(stats_lock); Ok(()) @@ -252,6 +265,7 @@ async fn put_object_or_retry( tokio::time::sleep(std::time::Duration::from_secs(2)).await; } } + metrics::RETRY_COUNT.inc(); tracing::warn!( target: INDEXER, "Failed to put {} to S3, retrying", diff --git a/src/metrics.rs b/src/metrics.rs new file mode 100644 index 0000000..70a7ad5 --- /dev/null +++ b/src/metrics.rs @@ -0,0 +1,30 @@ +use near_o11y::metrics::*; +use once_cell::sync::Lazy; + +pub static BLOCKS_DONE: Lazy = Lazy::new(|| { + try_create_int_counter( + "near_lake_block_done_total", + "Total number of indexed blocks", + ) + .unwrap() +}); + +pub static RETRY_COUNT: Lazy = Lazy::new(|| { + try_create_int_counter( + "near_lake_block_retry_count_total", + "Total number of retries for storing indexing blocks to S3", + ) + .unwrap() +}); + +// This metric is present in the near_o11y crate but it's not public +// so we can't use it directly. We have to redefine it here. +pub static NODE_BUILD_INFO: Lazy = Lazy::new(|| { + try_create_int_counter_vec( + "near_lake_build_info", + "Metric whose labels indicate node’s version; see \ + .", + &["release", "build", "rustc_version"], + ) + .unwrap() +});