docs: document functions and structs

Author: charmful0x

src/core/blobscan.rs

@@ -1,11 +1,11 @@
 //! Module to handle blobscan API operations.
-//! 
+//!
 //! Functionalities:
 //! - Fetch blobs from a given Ethereum block number
 //! - Fetch blobs' versioned hashes for a given Ethereum block number
 //! - Fetch blob data for a given blob versioned hash
 //! - Serialize BlobInfo as Arweave's ANS-104 DataItem
-//! - Send the stored blob to Blobscan's API
+//! - Send the stored blob to Blobscan's API - /weavevm-references endpoint
 
 use crate::core::{env_var::get_env_var, types::BlobInfo};
 use anyhow::Error;
@@ -16,6 +16,7 @@ use bundles_rs::{
 use reqwest;
 use serde_json::{self, Value};
 
+/// Get a vector of the blobs versioned hashes in a given Ethereum block number.
 pub async fn get_blobs_versioned_hashes_of_block(block_id: u64) -> Result<Vec<String>, Error> {
  let url = format!("https://api.blobscan.com/blocks/{}?type=canonical", block_id);
  let req: Value = reqwest::Client::new().get(url).send().await.unwrap().json().await?;
@@ -40,12 +41,14 @@ pub async fn get_blobs_versioned_hashes_of_block(block_id: u64) -> Result<Vec<St
  Ok(versioned_hashes)
 }
 
+/// Get the blob's data field (hex) for a given blob's versioned hash.
 async fn get_blob_data(versioned_hash: &str) -> Result<String, Error> {
  let url = format!("https://api.blobscan.com/blobs/{}/data", versioned_hash);
  let res = reqwest::Client::new().get(url).send().await?.text().await.unwrap_or_default();
  Ok(res)
 }
 
+/// Get a vector of blobs as BlobInfo for a given Ethereum block number.
 pub async fn get_blobs_of_block(block_id: u64) -> Result<Vec<BlobInfo>, Error> {
  let versioned_hashes = get_blobs_versioned_hashes_of_block(block_id).await.unwrap_or_default();
  let mut res: Vec<BlobInfo> = Vec::new();
@@ -64,6 +67,8 @@ pub async fn get_blobs_of_block(block_id: u64) -> Result<Vec<BlobInfo>, Error> {
  Ok(res)
 }
 
+/// Serialize the BlobInfo as ANS-104 DataItem, sign it using the agent's Arweave JWK
+/// and return DataItem raw bytes and its deterministic ID.
 pub fn serialize_blobscan_block(block: &BlobInfo) -> Result<(Vec<u8>, String), Error> {
  let data = serde_json::to_vec(&block)?;
  let tags =
@@ -74,7 +79,8 @@ pub fn serialize_blobscan_block(block: &BlobInfo) -> Result<(Vec<u8>, String), E
  Ok((dataitem.to_bytes().unwrap(), dataitem.arweave_id()))
 }
 
-pub async fn send_blob_to_blobscan(blob_hash: &str) -> Result<(), Error> {
+/// Index the blob's versioned_hash -> data_item_id on Blobscan
+pub(crate) async fn send_blob_to_blobscan(blob_hash: &str) -> Result<(), Error> {
  let client = reqwest::Client::new();
  let key = get_env_var("blobscan_api_key").unwrap();
  let response = client

src/core/constants.rs

@@ -2,3 +2,4 @@
 pub const FIRST_ETH_L1_EIP4844_BLOCK: u64 = 19426596; // test: 19426596 -- correct: 19_426_589
 pub const ETH_RPC_URL: &str = "https://eth.llamarpc.com";
 pub const BLOBSCAN_AGENT_ADDRESS: &str = "nb0VFuKAdWNvsw-H8bktG89uWaZuC-DFg4b2EcUlFI0";
+pub const HYPERBEAM_S3_HYBRID_GATEWAY_ENDPOINT: &str = "https://s3-node-0.load.network";

src/core/eth.rs

@@ -1,8 +1,8 @@
 //! Module to interact with an Ethereum JSON-RPC client
-//! 
+//!
 //! Functionalities:
 //! - Get the latest Ethereum block number
-//! 
+//!
 //! Note: this modules should migrate to alloy-rs instead of using ethers
 use crate::core::constants::ETH_RPC_URL;
 use anyhow::Error;

src/core/indexer.rs

@@ -1,7 +1,7 @@
 //! Agent's indexer module - uses planetscale to index
 //! metadata about the Ethereum blobs stored as ANS-104 offchain
 //! DataItems on Load's HyperBEAM S3 node.
-//! 
+//!
 //! Functionalities:
 //! - Map a blob versioned hash to the corresponding offchain ANS-104 DataItem ID
 //! - Get the DataItem ID for a given blob versioned hash
@@ -14,19 +14,25 @@ use planetscale_driver::{query, Database, PSConnection};
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
 
+/// Structure used by the indexer to parse the metadata's pair.
 #[derive(Debug, Default, Database, Serialize, Deserialize)]
 pub(crate) struct GetVersionedHash {
+ /// Blob's versioned hash
  pub versioned_hash: String,
+ /// ~s3@1.0 ANS-104 DataItem ID
  pub arweave_txid: String,
 }
-
+/// Structure to parse the indexer's stats response.
 #[derive(Debug, Default, Database, Serialize, Deserialize)]
 pub(crate) struct GetIndexerStats {
+ /// Latest processed blob's versioned hash
  pub versioned_hash: String,
+ /// Latest processed blob's DataItem ID
  pub arweave_txid: String,
+ /// Latest processed Etehreum block number containing blobs
  pub ethereum_block_number: u64,
 }
-
+/// Initialize Planetscale connection
 async fn ps_init() -> PSConnection {
  let host = get_env_var("DATABASE_HOST").unwrap();
  let username = get_env_var("DATABASE_USERNAME").unwrap();
@@ -36,7 +42,8 @@ async fn ps_init() -> PSConnection {
 
  conn
 }
-
+/// Insert the pair of metadata consisting of blob's versioned hash, its
+/// corresponding offchain DataItem ID, and the carrier ethereum block number.
 pub async fn insert_kv(
  versioned_hash: &str,
  arweave_txid: &str,
@@ -53,7 +60,7 @@ pub async fn insert_kv(
 
  Ok(res)
 }
-
+/// Get the metadata pair (versioned hash, DataItem ID) for a given blob's versioned hash.
 pub async fn get_versioned_hash_value(versioned_hash: &str) -> Result<Value, Error> {
  let client = ps_init().await;
 
@@ -66,7 +73,7 @@ pub async fn get_versioned_hash_value(versioned_hash: &str) -> Result<Value, Err
 
  Ok(serde_json::to_value(res)?)
 }
-
+/// Get the latest processed Ethereum block number that contains an EIP-4844 tx.
 pub async fn get_latest_block_id() -> u64 {
  let client = ps_init().await;
  let res: u64 =
@@ -76,7 +83,8 @@ pub async fn get_latest_block_id() -> u64 {
  .unwrap_or(FIRST_ETH_L1_EIP4844_BLOCK);
  return res;
 }
-
+/// Get the indexer's stats - it returns the latest fields that contains
+/// a blob (a block with EIP-4844 tx).
 pub async fn get_indexer_stats() -> Result<Value, Error> {
  let client = ps_init().await;
  let res: GetIndexerStats = query("SELECT versioned_hash, arweave_txid, ethereum_block_number FROM blobscan_arweave_mapping WHERE ethereum_block_number = (SELECT MAX(ethereum_block_number) FROM blobscan_arweave_mapping) LIMIT 1;").fetch_one(&client).await.unwrap();

src/core/mod.rs

@@ -1,9 +1,9 @@
 //! Core modules for the Blobscan Agent.
-//! 
+//!
 //! This module contains all the core functionality organized into specialized
 //! submodules for handling different aspects of the blob retrieval and storage
 //! pipeline.
-//! 
+//!
 //! ## Module Organization:
 //! - `blobscan` - Blobscan API integration and blob fetching
 //! - `constants` - Application-wide constants and configuration

src/core/s3.rs

@@ -1,11 +1,11 @@
 //! Module to interact with Load's S3 HyperBEAM device (~s3@1.0).
 //! The module store the serialized Ethereum blobs as ANS-104 DataItems
-//! in a a location in the HyperBEAM device where it can be retrieved back from 
+//! in a a location in the HyperBEAM device where it can be retrieved back from
 //! the Load HyperBEAM Hybrid Gateway as if it is an onchain Arweave DataItem
 //! To learn more about Hybrid Gateway and retrieval logic, check the load_hb
-//! documentation: https://github.com/loadnetwork/load_hb/tree/s3-edge/native/s3_nif#hybrid-gateway 
-//! 
-//! Functionalities: 
+//! documentation: https://github.com/loadnetwork/load_hb/tree/s3-edge/native/s3_nif#hybrid-gateway
+//!
+//! Functionalities:
 //! - Initialize ~s3@1.0 device connection
 //! - Store Ethereum blob as BlobInfo struct, serialized as ANS-104 DataItem
 //! - Retrieve a blob and its data (deserialized) back from the ~s3@1.0 for a given versione hash
@@ -16,7 +16,7 @@ use anyhow::{anyhow, Error};
 use aws_config::{BehaviorVersion, Region};
 use aws_sdk_s3::Client;
 use serde_json::Value;
-
+/// Initialize the ~s3@1.0 device connection using the aws s3 sdk.
 async fn s3_client() -> Result<Client, Error> {
  let config = aws_config::defaults(BehaviorVersion::latest())
  .endpoint_url(get_env_var("AWS_ENDPOINT_URL").unwrap())
@@ -32,7 +32,12 @@ async fn s3_client() -> Result<Client, Error> {
  .await;
  Ok(Client::new(&config))
 }
-
+/// Store a blob's BlobInfo as a signed valid ANS-104 DataItem. The DataItem is stored
+/// as an object in the HyperBEAM S3 device and is retrievable from the Load hyperbeam
+/// Hybrid Gateway as it was an onchain DataItem, providing a full backward compatibility
+/// with the Arweave's ecosystem and integration techstack - and most importantly, it offers
+/// a deterministic route for the offchain S3 DataItems to be posted onchain to Arweave while
+/// maintaining blob's DataItem integrity and provenance.
 pub async fn store_blob(versioned_hash: &str, blob_data: &str, block_id: u64) -> Result<(), Error> {
  let client = s3_client().await;
  let s3_bucket_name = get_env_var("S3_BUCKET_NAME").unwrap();
@@ -56,7 +61,8 @@ pub async fn store_blob(versioned_hash: &str, blob_data: &str, block_id: u64) ->
 
  Ok(())
 }
-
+/// Get a JSON serialized BlobInfo object for a given blob;s versioned hash - used
+/// by the agent's HTTP API.
 pub async fn get_blob_by_versioned_hash(versioned_hash: &str) -> Result<Value, Error> {
  let client = s3_client().await;
  let s3_bucket_name = get_env_var("S3_BUCKET_NAME").unwrap();
@@ -70,8 +76,7 @@ pub async fn get_blob_by_versioned_hash(versioned_hash: &str) -> Result<Value, E
  let res = serde_json::to_value(&data)?;
  Ok(res)
 }
-
-pub async fn insert_block(block_id: u64, blobs: Vec<BlobInfo>) -> Result<(), Error> {
+pub(crate) async fn insert_block(block_id: u64, blobs: Vec<BlobInfo>) -> Result<(), Error> {
  let mut hashes: Vec<String> = Vec::new();
  for blob in blobs {
  if !hashes.contains(&blob.versioned_hash) {

src/core/server_handlers.rs

@@ -1,22 +1,22 @@
 //! Module that handles the agent's HTTP API server
-//! 
+//!
 //! Endpoints:
 //! - GET route: `/`
 //! - GET blob's associated ANS-104 DataItem ID: `/v1/blob/:versioned_hash`
 //! - GET indexer stats: `/v1/stats`
 use crate::core::indexer::{get_indexer_stats, get_versioned_hash_value};
 use axum::{extract::Path, response::Json};
 use serde_json::Value;
-
+/// Agent's server health check.
 pub async fn handle_route() -> &'static str {
  "load it up [^^]"
 }
-
+/// Agent's server blob's metadata keys retrieval.
 pub async fn handle_get_blob(Path(versioned_hash): Path<String>) -> Json<Value> {
  let res = get_versioned_hash_value(&versioned_hash).await;
  Json(res.unwrap_or_else(|_| serde_json::json!({"error": "Blob not found"})))
 }
-
+/// Agent's stats retrieval
 pub async fn handle_get_stats() -> Json<Value> {
  let res = get_indexer_stats().await;
  Json(res.unwrap_or_default())

src/core/types.rs

@@ -2,25 +2,34 @@
 //! Data types mostly associated with the Ethereum blobs
 use serde::{Deserialize, Serialize};
 
+/// Used by the agent's indexer to handle the
+/// versioned hash DB response.
 #[derive(Debug, Serialize, Deserialize)]
 pub struct VersionedHashOnly {
+ /// Blob's versioned hash
  pub versioned_hash: String,
 }
-
+/// Blob's main data structure. Represents the
+/// ANS-104 DataItem data field.
 #[derive(Debug, Serialize, Deserialize, Default)]
 pub struct BlobInfo {
+ /// EIP-4844 tx's block number
  pub ethereum_block_number: u64,
+ /// Blob's versioned hash
  pub versioned_hash: String,
+ /// Blob's hex data field
  pub data: String,
 }
 
 impl BlobInfo {
+ /// Creates a BlobInfo instance from given struct field params.
  pub fn from(ethereum_block_number: u64, versioned_hash: String, data: String) -> Self {
  Self { ethereum_block_number, versioned_hash, data }
  }
 }
 
 impl std::fmt::Display for BlobInfo {
+ /// BlobInfo custom display formatted function
  fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
  write!(
  f,

src/main.rs

@@ -1,16 +1,16 @@
 //! Blobscan Agent - Main application entry point.
-//! 
+//!
 //! This application serves as a bridge between Ethereum's EIP-4844 blob data
-//! served by blobscab.com and Load's HyperBEAM ~s3@1.0 device (temporal storage). 
+//! served by blobscab.com and Load's HyperBEAM ~s3@1.0 device (temporal storage).
 //! It continuously monitors Ethereum blocks for blob transactions, processes them
 //! into ANS-104 DataItems, and stores them on Load's HyperBEAM S3 device for hybrid retrieval.
-//! 
+//!
 //! ## Architecture:
 //! - HTTP API server for blob metadata retrieval and stats
-//! - Background indexer for continuous Ethereum block monitoring 
+//! - Background indexer for continuous Ethereum block monitoring
 //! - HyperBEAM ~s3@1.0 storage integration with ANS-104 data format
 //! - Indexer's database PlanetScalefor blob metadata indexing
-//! 
+//!
 //! ## API Endpoints:
 //! - `GET /` - Health check
 //! - `GET /v1/blob/:versioned_hash` - Get blob ANS-104 DataItem ID