fear: migrate from PS to object storage model

Author: charmful0x

Cargo.lock

@@ -18,10 +18,12 @@ eyre = "0.6.12"
 brotli = "7.0.0"
 serde = "1.0.215"
 serde_json = "1.0.133"
-planetscale-driver = "0.5.1"
 dotenv = "0.15.0"
-shuttle-runtime = "0.50.0"
-shuttle-axum = "0.50.0"
 axum = "0.7.9"
+tokio-util = "0.7"
+tower = "0.4"
 reqwest = {version = "0.12.12", features=["json"]}
 anyhow = "1.0.98"
+bundles_rs = { git = "https://github.com/loadnetwork/bundles-rs.git", branch = "main"}
+aws-config = { version= "1.8.3", features = ["behavior-version-latest"] }
+aws-sdk-s3= { version = "1.100.0", features = ["rt-tokio"] }

Cargo.toml

@@ -3,27 +3,23 @@ use {
  std::sync::Arc,
  tokio::sync::RwLock,
  utils::{
- blobscan::{get_blobs_of_block, insert_block},
+ blobscan::get_blobs_of_block,
  constants::FIRST_ETH_L1_EIP4844_BLOCK,
  eth::Ethereum,
- planetscale::get_latest_block_id,
+ s3::{get_latest_block_id, insert_block},
  server_handlers::{handle_get_blob, handle_get_stats, handle_weave_gm},
+ env_var::load_env_vars,
  },
 };
 
 mod utils;
 
-#[shuttle_runtime::main]
-async fn main(
- #[shuttle_runtime::Secrets] secrets: shuttle_runtime::SecretStore,
-) -> shuttle_axum::ShuttleAxum {
- // load secrets from Shuttle.toml into env var;
- secrets.into_iter().for_each(|(key, val)| {
- std::env::set_var(key, val);
- });
+#[tokio::main]
+async fn main() {
+ load_env_vars();
  let router = Router::new()
  .route("/", get(handle_weave_gm))
- .route("/v1/blob/:versioned_hash", get(handle_get_blob))
+ .route("/v1/blob/{versioned_hash}", get(handle_get_blob))
  .route("/v1/stats", get(handle_get_stats));
 
  let block_number = Ethereum::get_latest_eth_block().await.unwrap();
@@ -45,10 +41,11 @@ async fn main(
  match blobs {
  Ok(blobs) => {
  println!("INSERTING: {:?} BLOBS", blobs.len());
+ println!("BLOBS: {:?}\n\n\n", blobs);
  let res = insert_block(target_block_id, blobs).await;
  match res {
- Ok(res) => latest_archived_block += 1,
- _ => eprintln!("error updating planetscale"),
+ Ok(_) => latest_archived_block += 1,
+ Err(e) => eprintln!("error updating s3: {}", e),
  }
  }
  Err(e) => {
@@ -69,5 +66,7 @@ async fn main(
  }
  });
 
- Ok(router.into())
+ let listener = tokio::net::TcpListener::bind("0.0.0.0:3000").await.unwrap();
+ println!("Server running on http://0.0.0.0:3000");
+ axum::serve(listener, router).await.unwrap();
 }

Shuttle.toml

@@ -1,14 +1,11 @@
 use {
  crate::utils::{
  env_var::get_env_var,
- planetscale::{ps_archive_block, ps_get_all_versioned_hashes_paginated},
  types::BlobInfo,
- wvm::send_wvm_calldata,
  },
- eyre::{eyre, Error, Result},
+ eyre::{Error, Result},
  reqwest,
  serde_json::{self, Value},
- std::io::{Read, Write},
 };
 
 pub async fn get_blobs_versioned_hashes_of_block(
@@ -80,37 +77,7 @@ pub async fn get_blobs_of_block(block_id: u32) -> Result<Vec<BlobInfo>> {
 
 pub fn serialize_blobscan_block(block: &BlobInfo) -> Result<Vec<u8>> {
  let data = serde_json::to_vec(&block)?;
- let compressed_data = brotli_compress(&data);
- Ok(compressed_data)
-}
-
-pub async fn insert_block(block_id: u32, blobs: Vec<BlobInfo>) -> Result<(), Error> {
- for blob in blobs {
- let wvm_data_input = serialize_blobscan_block(&blob)?;
- let wvm_txid = send_wvm_calldata(wvm_data_input).await.unwrap();
- let _res = ps_archive_block(&block_id, &wvm_txid, &blob.versioned_hash, &blob.data)
- .await
- .unwrap();
- let _send_to_blobscan = send_blob_to_blobscan(&blob.versioned_hash).await.unwrap();
- }
-
- Ok(())
-}
-
-fn brotli_compress(input: &[u8]) -> Vec<u8> {
- let mut writer = brotli::CompressorWriter::new(Vec::new(), 4096, 11, 22);
- writer.write_all(input).unwrap();
- writer.into_inner()
-}
-
-fn brotli_decompress(input: Vec<u8>) -> Vec<u8> {
- let mut decompressed_data = Vec::new();
- let mut decompressor = brotli::Decompressor::new(input.as_slice(), 4096); // 4096 is the buffer size
-
- decompressor
- .read_to_end(&mut decompressed_data)
- .expect("Decompression failed");
- decompressed_data
+ Ok(data)
 }
 
 pub async fn send_blob_to_blobscan(blob_hash: &str) -> Result<(), Error> {

src/main.rs

@@ -1,7 +1,10 @@
 use {dotenv::dotenv, std::env};
 
-pub fn get_env_var(key: &str) -> Result<String, env::VarError> {
+pub fn load_env_vars() {
  dotenv().ok();
+}
+
+pub fn get_env_var(key: &str) -> Result<String, env::VarError> {
  match env::var(key) {
  Ok(val) => Ok(val),
  Err(e) => Err(e),

src/utils/blobscan.rs

@@ -2,7 +2,7 @@ pub mod blobscan;
 pub mod constants;
 pub mod env_var;
 pub mod eth;
-pub mod planetscale;
+pub mod s3;
 pub mod server_handlers;
 pub mod types;
 pub mod wvm;