From 3975fed4a1c0468ece40fd16f2120b98f01e2aa2 Mon Sep 17 00:00:00 2001 From: z2trillion Date: Fri, 27 Sep 2024 14:55:33 -0400 Subject: [PATCH 01/35] Add data_availibility directory --- Cargo.lock | 1 + Cargo.toml | 1 + aggregator/Cargo.toml | 5 ++++- aggregator/src/data_availability.rs | 21 +++++++++++++++++++++ aggregator/src/data_availability/eip4844.rs | 0 aggregator/src/lib.rs | 2 ++ 6 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 aggregator/src/data_availability.rs create mode 100644 aggregator/src/data_availability/eip4844.rs diff --git a/Cargo.lock b/Cargo.lock index b68269b8b3..dd03b39eb5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -45,6 +45,7 @@ dependencies = [ "ark-std 0.3.0", "bitstream-io", "c-kzg", + "cfg-if 1.0.0", "csv", "ctor", "encoder", diff --git a/Cargo.toml b/Cargo.toml index af433433c5..115c9dc404 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,7 @@ anyhow = "1.0" ark-std = "0.3" base64 = "0.13.0" bincode = "1" +cfg-if = "1" ctor = "0.1" env_logger = "0.10" ethers = { version = "=2.0.7", features = ["ethers-solc"] } diff --git a/aggregator/Cargo.toml b/aggregator/Cargo.toml index ca3748bf50..4e1a8d0143 100644 --- a/aggregator/Cargo.toml +++ b/aggregator/Cargo.toml @@ -11,6 +11,7 @@ gadgets = { path = "../gadgets" } zkevm-circuits = { path = "../zkevm-circuits", default-features=false, features = ["debug-annotations", "parallel_syn"] } ark-std.workspace = true +cfg-if.workspace = true ctor.workspace = true env_logger.workspace = true ethers-core.workspace = true @@ -44,6 +45,8 @@ zstd-encoder = { package = "encoder", git = "https://github.com/scroll-tech/da-c csv = "1.1" [features] -default = ["revm-precompile/c-kzg", "halo2_proofs/circuit-params"] +default = ["revm-precompile/c-kzg", "halo2_proofs/circuit-params", "da-4844"] display = [] print-trace = ["ark-std/print-trace"] +da-avail = [] +da-4844 = [] diff --git a/aggregator/src/data_availability.rs b/aggregator/src/data_availability.rs new file mode 100644 index 0000000000..2ee1cbca73 --- /dev/null +++ b/aggregator/src/data_availability.rs @@ -0,0 +1,21 @@ +use cfg_if::cfg_if; + +enum DataAvailibility { + Eip4844, + Avail, +} + +cfg_if! { + if #[cfg(feature = "da-4844")] { + const DATA_AVAILABILITY: DataAvailibility = DataAvailibility::Eip4844; + } else if #[cfg(feature = "da-avail")] { + const DATA_AVAILABILITY: DataAvailibility = DataAvailibility::Avail; + } else { + unimplemented!("no da feature flag set"); + } +} + +#[cfg(feature = "da-4844")] +mod eip4844; +#[cfg(feature = "da-avail")] +mod avail; \ No newline at end of file diff --git a/aggregator/src/data_availability/eip4844.rs b/aggregator/src/data_availability/eip4844.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/aggregator/src/lib.rs b/aggregator/src/lib.rs index bf608e7cdc..b1b18ee221 100644 --- a/aggregator/src/lib.rs +++ b/aggregator/src/lib.rs @@ -23,6 +23,8 @@ pub mod eip4844; mod param; /// utilities mod util; +/// Data availibility checks +mod data_availability; #[cfg(test)] mod tests; From 570a23441857203d65eb6bce0f6ce6169eb2275a Mon Sep 17 00:00:00 2001 From: z2trillion Date: Fri, 27 Sep 2024 16:24:32 -0400 Subject: [PATCH 02/35] move files around and change paths --- aggregator/Cargo.toml | 4 +- aggregator/src/aggregation.rs | 10 +- aggregator/src/aggregation/batch_data.rs | 8 +- aggregator/src/aggregation/circuit.rs | 8 +- aggregator/src/aggregation/config.rs | 5 +- aggregator/src/batch.rs | 4 +- aggregator/src/data_availability.rs | 10 +- aggregator/src/data_availability/avail.rs | 1 + aggregator/src/data_availability/eip4844.rs | 101 ++++++++++++++++++ .../eip4844}/barycentric.rs | 6 +- .../{ => data_availability/eip4844}/blob.rs | 9 +- .../eip4844}/blob_data.rs | 4 +- .../data_availability/eip4844/constants.rs | 0 aggregator/src/eip4844.rs | 89 --------------- aggregator/src/lib.rs | 10 +- aggregator/src/tests/aggregation.rs | 2 +- aggregator/src/tests/blob.rs | 13 ++- prover/src/aggregator.rs | 2 +- prover/src/aggregator/prover.rs | 2 +- 19 files changed, 147 insertions(+), 141 deletions(-) create mode 100644 aggregator/src/data_availability/avail.rs rename aggregator/src/{aggregation => data_availability/eip4844}/barycentric.rs (98%) rename aggregator/src/{ => data_availability/eip4844}/blob.rs (99%) rename aggregator/src/{aggregation => data_availability/eip4844}/blob_data.rs (99%) create mode 100644 aggregator/src/data_availability/eip4844/constants.rs delete mode 100644 aggregator/src/eip4844.rs diff --git a/aggregator/Cargo.toml b/aggregator/Cargo.toml index 4e1a8d0143..6a1f867ac6 100644 --- a/aggregator/Cargo.toml +++ b/aggregator/Cargo.toml @@ -45,8 +45,8 @@ zstd-encoder = { package = "encoder", git = "https://github.com/scroll-tech/da-c csv = "1.1" [features] -default = ["revm-precompile/c-kzg", "halo2_proofs/circuit-params", "da-4844"] +default = ["revm-precompile/c-kzg", "halo2_proofs/circuit-params", "da-eip4844"] display = [] print-trace = ["ark-std/print-trace"] da-avail = [] -da-4844 = [] +da-eip4844 = [] diff --git a/aggregator/src/aggregation.rs b/aggregator/src/aggregation.rs index 25f7ddafec..6a1ec9ff7e 100644 --- a/aggregator/src/aggregation.rs +++ b/aggregator/src/aggregation.rs @@ -1,9 +1,5 @@ -/// Config to evaluate blob polynomial at a random challenge. -mod barycentric; /// Config to constrain batch data (decoded blob data) mod batch_data; -/// Config to constrain blob data (encoded batch data) -mod blob_data; /// Circuit implementation of aggregation circuit. mod circuit; /// Config for aggregation circuit @@ -15,13 +11,9 @@ mod rlc; /// Utility module mod util; -pub(crate) use barycentric::{ - interpolate, AssignedBarycentricEvaluationConfig, BarycentricEvaluationConfig, BLS_MODULUS, -}; pub(crate) use batch_data::BatchDataConfig; -pub(crate) use blob_data::BlobDataConfig; pub(crate) use decoder::{witgen, DecoderConfig, DecoderConfigArgs}; -pub(crate) use rlc::RlcConfig; +pub(crate) use rlc::{RlcConfig, POWS_OF_256}; pub use circuit::BatchCircuit; pub use config::BatchCircuitConfig; diff --git a/aggregator/src/aggregation/batch_data.rs b/aggregator/src/aggregation/batch_data.rs index 5bed4c2d78..c6ffafe5f2 100644 --- a/aggregator/src/aggregation/batch_data.rs +++ b/aggregator/src/aggregation/batch_data.rs @@ -13,11 +13,9 @@ use zkevm_circuits::{ util::{Challenges, Expr}, }; -use crate::{ - aggregation::rlc::POWS_OF_256, - blob::{BatchData, BLOB_WIDTH, N_BYTES_U256}, - RlcConfig, -}; +#[cfg(feature = "da-eip4844")] +use crate::data_availability::eip4844::blob::{BLOB_WIDTH, N_BYTES_U256}; +use crate::{aggregation::rlc::POWS_OF_256, BatchData, RlcConfig}; #[derive(Clone, Debug)] pub struct BatchDataConfig { diff --git a/aggregator/src/aggregation/circuit.rs b/aggregator/src/aggregation/circuit.rs index 535439a975..3a2c6133f0 100644 --- a/aggregator/src/aggregation/circuit.rs +++ b/aggregator/src/aggregation/circuit.rs @@ -25,14 +25,14 @@ use zkevm_circuits::util::Challenges; use crate::{ aggregation::{decoder::WORKED_EXAMPLE, witgen::process, BatchCircuitConfig}, batch::BatchHash, - blob::BatchData, constants::{ACC_LEN, DIGEST_LEN}, core::{assign_batch_hashes, extract_proof_and_instances_with_pairing_check}, + data_availability::eip4844::barycentric::AssignedBarycentricEvaluationConfig, + data_availability::eip4844::blob::BatchData, util::parse_hash_digest_cells, witgen::{zstd_encode, MultiBlockProcessResult}, - AssignedBarycentricEvaluationConfig, ConfigParams, LOG_DEGREE, PI_CHAIN_ID, - PI_CURRENT_BATCH_HASH, PI_CURRENT_STATE_ROOT, PI_CURRENT_WITHDRAW_ROOT, PI_PARENT_BATCH_HASH, - PI_PARENT_STATE_ROOT, + ConfigParams, LOG_DEGREE, PI_CHAIN_ID, PI_CURRENT_BATCH_HASH, PI_CURRENT_STATE_ROOT, + PI_CURRENT_WITHDRAW_ROOT, PI_PARENT_BATCH_HASH, PI_PARENT_STATE_ROOT, }; /// Batch circuit, the chunk aggregation routine below recursion circuit diff --git a/aggregator/src/aggregation/config.rs b/aggregator/src/aggregation/config.rs index 0d744ec060..978803ea2e 100644 --- a/aggregator/src/aggregation/config.rs +++ b/aggregator/src/aggregation/config.rs @@ -18,9 +18,10 @@ use zkevm_circuits::{ use crate::{ constants::{BITS, LIMBS}, + data_availability::eip4844::barycentric::BarycentricEvaluationConfig, + data_availability::eip4844::blob_data::BlobDataConfig, param::ConfigParams, - BarycentricEvaluationConfig, BatchDataConfig, BlobDataConfig, DecoderConfig, DecoderConfigArgs, - RlcConfig, + BatchDataConfig, DecoderConfig, DecoderConfigArgs, RlcConfig, }; #[derive(Debug, Clone)] diff --git a/aggregator/src/batch.rs b/aggregator/src/batch.rs index b43d788c0b..3a5ade39a3 100644 --- a/aggregator/src/batch.rs +++ b/aggregator/src/batch.rs @@ -7,9 +7,9 @@ use gadgets::{util::split_h256, Field}; use serde::{Deserialize, Serialize}; use crate::{ - blob::{BatchData, PointEvaluationAssignments}, chunk::ChunkInfo, - eip4844::{get_coefficients, get_versioned_hash}, + data_availability::eip4844::blob::{BatchData, PointEvaluationAssignments}, + data_availability::eip4844::{get_coefficients, get_versioned_hash}, }; /// Batch header provides additional fields from the context (within recursion) diff --git a/aggregator/src/data_availability.rs b/aggregator/src/data_availability.rs index 2ee1cbca73..1754231f5c 100644 --- a/aggregator/src/data_availability.rs +++ b/aggregator/src/data_availability.rs @@ -6,16 +6,16 @@ enum DataAvailibility { } cfg_if! { - if #[cfg(feature = "da-4844")] { + if #[cfg(feature = "da-eip4844")] { const DATA_AVAILABILITY: DataAvailibility = DataAvailibility::Eip4844; } else if #[cfg(feature = "da-avail")] { const DATA_AVAILABILITY: DataAvailibility = DataAvailibility::Avail; } else { - unimplemented!("no da feature flag set"); + compile_error!("no da feature flag set"); } } -#[cfg(feature = "da-4844")] -mod eip4844; #[cfg(feature = "da-avail")] -mod avail; \ No newline at end of file +mod avail; +#[cfg(feature = "da-eip4844")] +pub mod eip4844; diff --git a/aggregator/src/data_availability/avail.rs b/aggregator/src/data_availability/avail.rs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/aggregator/src/data_availability/avail.rs @@ -0,0 +1 @@ + diff --git a/aggregator/src/data_availability/eip4844.rs b/aggregator/src/data_availability/eip4844.rs index e69de29bb2..b5288f199e 100644 --- a/aggregator/src/data_availability/eip4844.rs +++ b/aggregator/src/data_availability/eip4844.rs @@ -0,0 +1,101 @@ +/// Config to evaluate blob polynomial at a random challenge. +pub mod barycentric; +pub use barycentric::{ + interpolate, AssignedBarycentricEvaluationConfig, BarycentricEvaluationConfig, BLS_MODULUS, +}; + +/// blob struct and constants +pub mod blob; +use blob::{BLOB_WIDTH, KZG_TRUSTED_SETUP, N_BLOB_BYTES, N_BYTES_U256}; + +/// Config to constrain blob data (encoded batch data) +pub mod blob_data; +pub use blob_data::BlobDataConfig; + +use eth_types::{ToBigEndian, H256, U256}; +use ethers_core::k256::sha2::{Digest, Sha256}; +use revm_primitives::VERSIONED_HASH_VERSION_KZG; + +/// Get the BLOB_WIDTH number of scalar field elements, as 32-bytes unsigned integers. +pub(crate) fn get_coefficients(blob_bytes: &[u8]) -> [U256; BLOB_WIDTH] { + let mut coefficients = [[0u8; N_BYTES_U256]; BLOB_WIDTH]; + + assert!( + blob_bytes.len() <= N_BLOB_BYTES, + "too many bytes in batch data" + ); + + for (i, &byte) in blob_bytes.iter().enumerate() { + coefficients[i / 31][1 + (i % 31)] = byte; + } + + coefficients.map(|coeff| U256::from_big_endian(&coeff)) +} + +/// Get the versioned hash as per EIP-4844. +pub(crate) fn get_versioned_hash(coefficients: &[U256; BLOB_WIDTH]) -> H256 { + let blob = c_kzg::Blob::from_bytes( + &coefficients + .iter() + .cloned() + .flat_map(|coeff| coeff.to_be_bytes()) + .collect::>(), + ) + .expect("blob-coefficients to 4844 blob should succeed"); + let c = c_kzg::KzgCommitment::blob_to_kzg_commitment(&blob, &KZG_TRUSTED_SETUP) + .expect("blob to kzg commitment should succeed"); + kzg_to_versioned_hash(&c) +} + +fn kzg_to_versioned_hash(commitment: &c_kzg::KzgCommitment) -> H256 { + let mut res = Sha256::digest(commitment.as_slice()); + res[0] = VERSIONED_HASH_VERSION_KZG; + H256::from_slice(&res[..]) +} + +/// Get the blob data bytes that will be populated in BlobDataConfig. +pub fn get_blob_bytes(batch_bytes: &[u8]) -> Vec { + let mut blob_bytes = crate::witgen::zstd_encode(batch_bytes); + + // Whether we encode batch -> blob or not. + let enable_encoding = blob_bytes.len() < batch_bytes.len(); + if !enable_encoding { + blob_bytes = batch_bytes.to_vec(); + } + blob_bytes.insert(0, enable_encoding as u8); + + blob_bytes +} + +/// Given the blob's bytes, take into account the first byte, i.e. enable_encoding? and either spit +/// out the raw bytes or zstd decode them. +pub fn decode_blob(blob_bytes: &[u8]) -> std::io::Result> { + let enable_encoding = blob_bytes[0].eq(&1); + + // If not encoded, spit out the rest of the bytes, as it is. + if !enable_encoding { + return Ok(blob_bytes[1..].to_vec()); + } + + // The bytes following the first byte represent the zstd-encoded bytes. + let mut encoded_bytes = blob_bytes[1..].to_vec(); + let mut encoded_len = encoded_bytes.len(); + let mut decoded_bytes = Vec::with_capacity(5 * 4096 * 32); + loop { + let mut decoder = zstd_encoder::zstd::stream::read::Decoder::new(encoded_bytes.as_slice())?; + decoder.include_magicbytes(false)?; + decoder.window_log_max(30)?; + + decoded_bytes.clear(); + + if std::io::copy(&mut decoder, &mut decoded_bytes).is_ok() { + break; + } + + // The error above means we need to truncate the suffix 0-byte. + encoded_len -= 1; + encoded_bytes.truncate(encoded_len); + } + + Ok(decoded_bytes) +} diff --git a/aggregator/src/aggregation/barycentric.rs b/aggregator/src/data_availability/eip4844/barycentric.rs similarity index 98% rename from aggregator/src/aggregation/barycentric.rs rename to aggregator/src/data_availability/eip4844/barycentric.rs index f485eac9b8..1417f79c15 100644 --- a/aggregator/src/aggregation/barycentric.rs +++ b/aggregator/src/data_availability/eip4844/barycentric.rs @@ -20,6 +20,7 @@ use std::{iter::successors, sync::LazyLock}; use crate::{ blob::{BLOB_WIDTH, N_BYTES_U256}, constants::{BITS, LIMBS}, + data_availability::eip4844::blob::BLOB_WIDTH, }; /// Base 2 logarithm of BLOB_WIDTH. @@ -29,6 +30,7 @@ pub static BLS_MODULUS: LazyLock = LazyLock::new(|| { U256::from_str_radix(Scalar::MODULUS, 16).expect("BLS_MODULUS from bls crate") }); +// does this need to be pub? pub static ROOTS_OF_UNITY: LazyLock> = LazyLock::new(|| { // https://github.com/ethereum/consensus-specs/blob/dev/specs/deneb/polynomial-commitments.md#constants let primitive_root_of_unity = Scalar::from(7); @@ -354,8 +356,8 @@ pub fn interpolate(z: Scalar, coefficients: &[Scalar; BLOB_WIDTH]) -> Scalar { mod tests { use super::*; use crate::{ - blob::{BatchData, KZG_TRUSTED_SETUP}, - eip4844::{get_blob_bytes, get_coefficients}, + data_availability::eip4844::blob::{BatchData, KZG_TRUSTED_SETUP}, + data_availability::eip4844::{get_blob_bytes, get_coefficients}, MAX_AGG_SNARKS, }; use c_kzg::{Blob as RethBlob, KzgProof}; diff --git a/aggregator/src/blob.rs b/aggregator/src/data_availability/eip4844/blob.rs similarity index 99% rename from aggregator/src/blob.rs rename to aggregator/src/data_availability/eip4844/blob.rs index d52c0c4333..115d6b0d23 100644 --- a/aggregator/src/blob.rs +++ b/aggregator/src/data_availability/eip4844/blob.rs @@ -1,8 +1,5 @@ -use crate::{ - aggregation::{interpolate, BLS_MODULUS}, - eip4844::get_coefficients, - BatchHash, ChunkInfo, -}; +use super::{get_coefficients, interpolate, BLS_MODULUS}; +use crate::{BatchHash, ChunkInfo}; use eth_types::{H256, U256}; use ethers_core::utils::keccak256; @@ -692,7 +689,7 @@ impl BatchDataRow { mod tests { use super::*; use crate::{ - eip4844::{get_blob_bytes, get_versioned_hash}, + data_availability::eip4844::{get_blob_bytes, get_versioned_hash}, MAX_AGG_SNARKS, }; diff --git a/aggregator/src/aggregation/blob_data.rs b/aggregator/src/data_availability/eip4844/blob_data.rs similarity index 99% rename from aggregator/src/aggregation/blob_data.rs rename to aggregator/src/data_availability/eip4844/blob_data.rs index 4d02c55db9..d56e78ce65 100644 --- a/aggregator/src/aggregation/blob_data.rs +++ b/aggregator/src/data_availability/eip4844/blob_data.rs @@ -10,8 +10,8 @@ use itertools::Itertools; use zkevm_circuits::{table::U8Table, util::Challenges}; use crate::{ - aggregation::rlc::POWS_OF_256, - blob::{BLOB_WIDTH, N_BLOB_BYTES, N_DATA_BYTES_PER_COEFFICIENT}, + aggregation::POWS_OF_256, + data_availability::eip4844::blob::{BLOB_WIDTH, N_BLOB_BYTES, N_DATA_BYTES_PER_COEFFICIENT}, RlcConfig, }; diff --git a/aggregator/src/data_availability/eip4844/constants.rs b/aggregator/src/data_availability/eip4844/constants.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/aggregator/src/eip4844.rs b/aggregator/src/eip4844.rs deleted file mode 100644 index 5f4a29a5dc..0000000000 --- a/aggregator/src/eip4844.rs +++ /dev/null @@ -1,89 +0,0 @@ -use eth_types::{ToBigEndian, H256, U256}; -use ethers_core::k256::sha2::{Digest, Sha256}; -use revm_primitives::VERSIONED_HASH_VERSION_KZG; - -use crate::blob::{BLOB_WIDTH, KZG_TRUSTED_SETUP, N_BLOB_BYTES, N_BYTES_U256}; - -/// Get the BLOB_WIDTH number of scalar field elements, as 32-bytes unsigned integers. -pub(crate) fn get_coefficients(blob_bytes: &[u8]) -> [U256; BLOB_WIDTH] { - let mut coefficients = [[0u8; N_BYTES_U256]; BLOB_WIDTH]; - - assert!( - blob_bytes.len() <= N_BLOB_BYTES, - "too many bytes in batch data" - ); - - for (i, &byte) in blob_bytes.iter().enumerate() { - coefficients[i / 31][1 + (i % 31)] = byte; - } - - coefficients.map(|coeff| U256::from_big_endian(&coeff)) -} - -/// Get the versioned hash as per EIP-4844. -pub(crate) fn get_versioned_hash(coefficients: &[U256; BLOB_WIDTH]) -> H256 { - let blob = c_kzg::Blob::from_bytes( - &coefficients - .iter() - .cloned() - .flat_map(|coeff| coeff.to_be_bytes()) - .collect::>(), - ) - .expect("blob-coefficients to 4844 blob should succeed"); - let c = c_kzg::KzgCommitment::blob_to_kzg_commitment(&blob, &KZG_TRUSTED_SETUP) - .expect("blob to kzg commitment should succeed"); - kzg_to_versioned_hash(&c) -} - -fn kzg_to_versioned_hash(commitment: &c_kzg::KzgCommitment) -> H256 { - let mut res = Sha256::digest(commitment.as_slice()); - res[0] = VERSIONED_HASH_VERSION_KZG; - H256::from_slice(&res[..]) -} - -/// Get the blob data bytes that will be populated in BlobDataConfig. -pub fn get_blob_bytes(batch_bytes: &[u8]) -> Vec { - let mut blob_bytes = crate::witgen::zstd_encode(batch_bytes); - - // Whether we encode batch -> blob or not. - let enable_encoding = blob_bytes.len() < batch_bytes.len(); - if !enable_encoding { - blob_bytes = batch_bytes.to_vec(); - } - blob_bytes.insert(0, enable_encoding as u8); - - blob_bytes -} - -/// Given the blob's bytes, take into account the first byte, i.e. enable_encoding? and either spit -/// out the raw bytes or zstd decode them. -pub fn decode_blob(blob_bytes: &[u8]) -> std::io::Result> { - let enable_encoding = blob_bytes[0].eq(&1); - - // If not encoded, spit out the rest of the bytes, as it is. - if !enable_encoding { - return Ok(blob_bytes[1..].to_vec()); - } - - // The bytes following the first byte represent the zstd-encoded bytes. - let mut encoded_bytes = blob_bytes[1..].to_vec(); - let mut encoded_len = encoded_bytes.len(); - let mut decoded_bytes = Vec::with_capacity(5 * 4096 * 32); - loop { - let mut decoder = zstd_encoder::zstd::stream::read::Decoder::new(encoded_bytes.as_slice())?; - decoder.include_magicbytes(false)?; - decoder.window_log_max(30)?; - - decoded_bytes.clear(); - - if std::io::copy(&mut decoder, &mut decoded_bytes).is_ok() { - break; - } - - // The error above means we need to truncate the suffix 0-byte. - encoded_len -= 1; - encoded_bytes.truncate(encoded_len); - } - - Ok(decoded_bytes) -} diff --git a/aggregator/src/lib.rs b/aggregator/src/lib.rs index b1b18ee221..ed34c8c140 100644 --- a/aggregator/src/lib.rs +++ b/aggregator/src/lib.rs @@ -4,8 +4,6 @@ mod aggregation; /// This module implements `Batch` related data types. /// A batch is a list of chunk. mod batch; -/// blob struct and constants -mod blob; /// Config to recursive aggregate multiple aggregations mod recursion; // This module implements `Chunk` related data types. @@ -17,14 +15,12 @@ mod compression; mod constants; /// Core module for circuit assignment mod core; -/// EIP-4844 related utils. -pub mod eip4844; +/// Data availibility checks +pub mod data_availability; /// Parameters for compression circuit mod param; /// utilities mod util; -/// Data availibility checks -mod data_availability; #[cfg(test)] mod tests; @@ -32,10 +28,10 @@ mod tests; pub use self::core::extract_proof_and_instances_with_pairing_check; pub use aggregation::*; pub use batch::{BatchHash, BatchHeader}; -pub use blob::BatchData; pub use chunk::ChunkInfo; pub use compression::*; pub use constants::MAX_AGG_SNARKS; pub(crate) use constants::*; +pub use data_availability::eip4844::blob::BatchData; pub use param::*; pub use recursion::*; diff --git a/aggregator/src/tests/aggregation.rs b/aggregator/src/tests/aggregation.rs index cfac56a830..5426a863a2 100644 --- a/aggregator/src/tests/aggregation.rs +++ b/aggregator/src/tests/aggregation.rs @@ -10,7 +10,7 @@ use crate::{ aggregation::BatchCircuit, batch::{BatchHash, BatchHeader}, constants::MAX_AGG_SNARKS, - eip4844::get_blob_bytes, + data_availability::eip4844::get_blob_bytes, layer_0, tests::mock_chunk::MockChunkCircuit, BatchData, ChunkInfo, diff --git a/aggregator/src/tests/blob.rs b/aggregator/src/tests/blob.rs index b2d98c329f..83e976d2a6 100644 --- a/aggregator/src/tests/blob.rs +++ b/aggregator/src/tests/blob.rs @@ -19,11 +19,18 @@ use zkevm_circuits::{ use crate::{ aggregation::{ witgen::{process, MultiBlockProcessResult}, - AssignedBarycentricEvaluationConfig, BarycentricEvaluationConfig, BlobDataConfig, RlcConfig, }, - blob::{BatchData, PointEvaluationAssignments, N_BLOB_BYTES, N_BYTES_U256}, - eip4844::{decode_blob, get_blob_bytes, get_coefficients, get_versioned_hash}, + data_availability::eip4844::barycentric::{ + AssignedBarycentricEvaluationConfig, BarycentricEvaluationConfig, + }, + data_availability::eip4844::blob::{ + BatchData, PointEvaluationAssignments, N_BLOB_BYTES, N_BYTES_U256, + }, + data_availability::eip4844::{ + blob_data::BlobDataConfig, decode_blob, get_blob_bytes, get_coefficients, + get_versioned_hash, + }, param::ConfigParams, BatchDataConfig, ChunkInfo, MAX_AGG_SNARKS, }; diff --git a/prover/src/aggregator.rs b/prover/src/aggregator.rs index 0a5a63f9e3..177c313c1a 100644 --- a/prover/src/aggregator.rs +++ b/prover/src/aggregator.rs @@ -2,5 +2,5 @@ mod prover; mod verifier; pub use self::prover::{check_chunk_hashes, Prover}; -pub use aggregator::{eip4844, BatchData, BatchHash, BatchHeader, MAX_AGG_SNARKS}; +pub use aggregator::{BatchData, BatchHash, BatchHeader, MAX_AGG_SNARKS}; pub use verifier::Verifier; diff --git a/prover/src/aggregator/prover.rs b/prover/src/aggregator/prover.rs index bb18fa0874..f5b0227521 100644 --- a/prover/src/aggregator/prover.rs +++ b/prover/src/aggregator/prover.rs @@ -1,7 +1,7 @@ use std::{collections::BTreeMap, env, iter::repeat}; use aggregator::{ - eip4844::decode_blob, BatchData, BatchHash, BatchHeader, ChunkInfo, MAX_AGG_SNARKS, + data_availability::eip4844::decode_blob, BatchData, BatchHash, BatchHeader, ChunkInfo, MAX_AGG_SNARKS, }; use anyhow::{bail, Result}; use eth_types::H256; From 52b2ac0cf8097527da653ff13d9960a4e7c83cbe Mon Sep 17 00:00:00 2001 From: z2trillion Date: Fri, 27 Sep 2024 16:35:40 -0400 Subject: [PATCH 03/35] rename and move decode_blob --- aggregator/src/aggregation.rs | 1 + aggregator/src/aggregation/config.rs | 3 +- aggregator/src/aggregation/decoder.rs | 33 +++++++++++++++++++++ aggregator/src/data_availability/eip4844.rs | 33 --------------------- aggregator/src/tests/blob.rs | 6 ++-- prover/src/aggregator/prover.rs | 6 ++-- 6 files changed, 40 insertions(+), 42 deletions(-) diff --git a/aggregator/src/aggregation.rs b/aggregator/src/aggregation.rs index 6a1ec9ff7e..07c25e2b4e 100644 --- a/aggregator/src/aggregation.rs +++ b/aggregator/src/aggregation.rs @@ -12,6 +12,7 @@ mod rlc; mod util; pub(crate) use batch_data::BatchDataConfig; +pub use decoder::decode_bytes; pub(crate) use decoder::{witgen, DecoderConfig, DecoderConfigArgs}; pub(crate) use rlc::{RlcConfig, POWS_OF_256}; diff --git a/aggregator/src/aggregation/config.rs b/aggregator/src/aggregation/config.rs index 978803ea2e..941dff4d26 100644 --- a/aggregator/src/aggregation/config.rs +++ b/aggregator/src/aggregation/config.rs @@ -18,8 +18,7 @@ use zkevm_circuits::{ use crate::{ constants::{BITS, LIMBS}, - data_availability::eip4844::barycentric::BarycentricEvaluationConfig, - data_availability::eip4844::blob_data::BlobDataConfig, + data_availability::eip4844::{barycentric::BarycentricEvaluationConfig, blob_data::BlobDataConfig}, param::ConfigParams, BatchDataConfig, DecoderConfig, DecoderConfigArgs, RlcConfig, }; diff --git a/aggregator/src/aggregation/decoder.rs b/aggregator/src/aggregation/decoder.rs index bb9c7661f0..de2f32ace9 100644 --- a/aggregator/src/aggregation/decoder.rs +++ b/aggregator/src/aggregation/decoder.rs @@ -5794,3 +5794,36 @@ mod tests { Ok(()) } } + +/// Given the blob's bytes, take into account the first byte, i.e. enable_encoding? and either spit +/// out the raw bytes or zstd decode them. +pub fn decode_bytes(bytes: &[u8]) -> std::io::Result> { + let enable_encoding = bytes[0].eq(&1); + + // If not encoded, spit out the rest of the bytes, as it is. + if !enable_encoding { + return Ok(bytes[1..].to_vec()); + } + + // The bytes following the first byte represent the zstd-encoded bytes. + let mut encoded_bytes = bytes[1..].to_vec(); + let mut encoded_len = encoded_bytes.len(); + let mut decoded_bytes = Vec::with_capacity(5 * 4096 * 32); + loop { + let mut decoder = zstd_encoder::zstd::stream::read::Decoder::new(encoded_bytes.as_slice())?; + decoder.include_magicbytes(false)?; + decoder.window_log_max(30)?; + + decoded_bytes.clear(); + + if std::io::copy(&mut decoder, &mut decoded_bytes).is_ok() { + break; + } + + // The error above means we need to truncate the suffix 0-byte. + encoded_len -= 1; + encoded_bytes.truncate(encoded_len); + } + + Ok(decoded_bytes) +} diff --git a/aggregator/src/data_availability/eip4844.rs b/aggregator/src/data_availability/eip4844.rs index b5288f199e..f78fe64ee9 100644 --- a/aggregator/src/data_availability/eip4844.rs +++ b/aggregator/src/data_availability/eip4844.rs @@ -66,36 +66,3 @@ pub fn get_blob_bytes(batch_bytes: &[u8]) -> Vec { blob_bytes } - -/// Given the blob's bytes, take into account the first byte, i.e. enable_encoding? and either spit -/// out the raw bytes or zstd decode them. -pub fn decode_blob(blob_bytes: &[u8]) -> std::io::Result> { - let enable_encoding = blob_bytes[0].eq(&1); - - // If not encoded, spit out the rest of the bytes, as it is. - if !enable_encoding { - return Ok(blob_bytes[1..].to_vec()); - } - - // The bytes following the first byte represent the zstd-encoded bytes. - let mut encoded_bytes = blob_bytes[1..].to_vec(); - let mut encoded_len = encoded_bytes.len(); - let mut decoded_bytes = Vec::with_capacity(5 * 4096 * 32); - loop { - let mut decoder = zstd_encoder::zstd::stream::read::Decoder::new(encoded_bytes.as_slice())?; - decoder.include_magicbytes(false)?; - decoder.window_log_max(30)?; - - decoded_bytes.clear(); - - if std::io::copy(&mut decoder, &mut decoded_bytes).is_ok() { - break; - } - - // The error above means we need to truncate the suffix 0-byte. - encoded_len -= 1; - encoded_bytes.truncate(encoded_len); - } - - Ok(decoded_bytes) -} diff --git a/aggregator/src/tests/blob.rs b/aggregator/src/tests/blob.rs index 83e976d2a6..6d3c71ff18 100644 --- a/aggregator/src/tests/blob.rs +++ b/aggregator/src/tests/blob.rs @@ -28,9 +28,9 @@ use crate::{ BatchData, PointEvaluationAssignments, N_BLOB_BYTES, N_BYTES_U256, }, data_availability::eip4844::{ - blob_data::BlobDataConfig, decode_blob, get_blob_bytes, get_coefficients, - get_versioned_hash, + blob_data::BlobDataConfig, get_blob_bytes, get_coefficients, get_versioned_hash, }, + decode_bytes, param::ConfigParams, BatchDataConfig, ChunkInfo, MAX_AGG_SNARKS, }; @@ -630,7 +630,7 @@ fn test_decode_blob() { // case 2: yes encode assert_eq!( - decode_blob(&conditional_encode(batch_bytes.as_slice(), true)).expect("should decode"), + decode_bytes(&conditional_encode(batch_bytes.as_slice(), true)).expect("should decode"), batch_bytes, ); } diff --git a/prover/src/aggregator/prover.rs b/prover/src/aggregator/prover.rs index f5b0227521..3d52b07f91 100644 --- a/prover/src/aggregator/prover.rs +++ b/prover/src/aggregator/prover.rs @@ -1,8 +1,6 @@ use std::{collections::BTreeMap, env, iter::repeat}; -use aggregator::{ - data_availability::eip4844::decode_blob, BatchData, BatchHash, BatchHeader, ChunkInfo, MAX_AGG_SNARKS, -}; +use aggregator::{decode_bytes, BatchData, BatchHash, BatchHeader, ChunkInfo, MAX_AGG_SNARKS}; use anyhow::{bail, Result}; use eth_types::H256; use halo2_proofs::{halo2curves::bn256::Bn256, poly::kzg::commitment::ParamsKZG}; @@ -208,7 +206,7 @@ impl<'params> Prover<'params> { // sanity check: // - conditionally decoded blob should match batch data. let batch_bytes = batch_data.get_batch_data_bytes(); - let decoded_blob_bytes = decode_blob(&batch.blob_bytes)?; + let decoded_blob_bytes = decode_bytes(&batch.blob_bytes)?; assert_eq!( batch_bytes, decoded_blob_bytes, "BatchProvingTask(sanity) mismatch batch bytes and decoded blob bytes", From 0c568b8fd80be3531dc92adb104961f287bd8408 Mon Sep 17 00:00:00 2001 From: z2trillion Date: Fri, 27 Sep 2024 16:38:56 -0400 Subject: [PATCH 04/35] unpub barycentric --- aggregator/src/aggregation/circuit.rs | 2 +- aggregator/src/aggregation/config.rs | 2 +- aggregator/src/data_availability/eip4844.rs | 2 +- aggregator/src/tests/blob.rs | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/aggregator/src/aggregation/circuit.rs b/aggregator/src/aggregation/circuit.rs index 3a2c6133f0..665d3431ab 100644 --- a/aggregator/src/aggregation/circuit.rs +++ b/aggregator/src/aggregation/circuit.rs @@ -27,8 +27,8 @@ use crate::{ batch::BatchHash, constants::{ACC_LEN, DIGEST_LEN}, core::{assign_batch_hashes, extract_proof_and_instances_with_pairing_check}, - data_availability::eip4844::barycentric::AssignedBarycentricEvaluationConfig, data_availability::eip4844::blob::BatchData, + data_availability::eip4844::AssignedBarycentricEvaluationConfig, util::parse_hash_digest_cells, witgen::{zstd_encode, MultiBlockProcessResult}, ConfigParams, LOG_DEGREE, PI_CHAIN_ID, PI_CURRENT_BATCH_HASH, PI_CURRENT_STATE_ROOT, diff --git a/aggregator/src/aggregation/config.rs b/aggregator/src/aggregation/config.rs index 941dff4d26..83afad14d2 100644 --- a/aggregator/src/aggregation/config.rs +++ b/aggregator/src/aggregation/config.rs @@ -18,7 +18,7 @@ use zkevm_circuits::{ use crate::{ constants::{BITS, LIMBS}, - data_availability::eip4844::{barycentric::BarycentricEvaluationConfig, blob_data::BlobDataConfig}, + data_availability::eip4844::{blob_data::BlobDataConfig, BarycentricEvaluationConfig}, param::ConfigParams, BatchDataConfig, DecoderConfig, DecoderConfigArgs, RlcConfig, }; diff --git a/aggregator/src/data_availability/eip4844.rs b/aggregator/src/data_availability/eip4844.rs index f78fe64ee9..9e6ff11c3e 100644 --- a/aggregator/src/data_availability/eip4844.rs +++ b/aggregator/src/data_availability/eip4844.rs @@ -1,5 +1,5 @@ /// Config to evaluate blob polynomial at a random challenge. -pub mod barycentric; +mod barycentric; pub use barycentric::{ interpolate, AssignedBarycentricEvaluationConfig, BarycentricEvaluationConfig, BLS_MODULUS, }; diff --git a/aggregator/src/tests/blob.rs b/aggregator/src/tests/blob.rs index 6d3c71ff18..8c77be7a73 100644 --- a/aggregator/src/tests/blob.rs +++ b/aggregator/src/tests/blob.rs @@ -21,15 +21,15 @@ use crate::{ witgen::{process, MultiBlockProcessResult}, RlcConfig, }, - data_availability::eip4844::barycentric::{ - AssignedBarycentricEvaluationConfig, BarycentricEvaluationConfig, - }, data_availability::eip4844::blob::{ BatchData, PointEvaluationAssignments, N_BLOB_BYTES, N_BYTES_U256, }, data_availability::eip4844::{ blob_data::BlobDataConfig, get_blob_bytes, get_coefficients, get_versioned_hash, }, + data_availability::eip4844::{ + AssignedBarycentricEvaluationConfig, BarycentricEvaluationConfig, + }, decode_bytes, param::ConfigParams, BatchDataConfig, ChunkInfo, MAX_AGG_SNARKS, From 4732f8efea60855f82dbeede338a81b417a5d59d Mon Sep 17 00:00:00 2001 From: z2trillion Date: Mon, 30 Sep 2024 23:35:54 -0400 Subject: [PATCH 05/35] move eip4844 files into data_availability/ --- aggregator/src/aggregation.rs | 3 +- aggregator/src/aggregation/batch_data.rs | 605 +++++++++++++++++- aggregator/src/aggregation/circuit.rs | 3 +- aggregator/src/batch.rs | 3 +- aggregator/src/data_availability/eip4844.rs | 3 +- .../data_availability/eip4844/barycentric.rs | 3 +- .../src/data_availability/eip4844/blob.rs | 600 +---------------- .../data_availability/eip4844/blob_data.rs | 7 +- aggregator/src/lib.rs | 1 - aggregator/src/tests/blob.rs | 7 +- 10 files changed, 620 insertions(+), 615 deletions(-) diff --git a/aggregator/src/aggregation.rs b/aggregator/src/aggregation.rs index 07c25e2b4e..ba527a43e9 100644 --- a/aggregator/src/aggregation.rs +++ b/aggregator/src/aggregation.rs @@ -1,5 +1,5 @@ /// Config to constrain batch data (decoded blob data) -mod batch_data; +pub mod batch_data; /// Circuit implementation of aggregation circuit. mod circuit; /// Config for aggregation circuit @@ -11,6 +11,7 @@ mod rlc; /// Utility module mod util; +pub use batch_data::BatchData; pub(crate) use batch_data::BatchDataConfig; pub use decoder::decode_bytes; pub(crate) use decoder::{witgen, DecoderConfig, DecoderConfigArgs}; diff --git a/aggregator/src/aggregation/batch_data.rs b/aggregator/src/aggregation/batch_data.rs index c6ffafe5f2..10ccb73d16 100644 --- a/aggregator/src/aggregation/batch_data.rs +++ b/aggregator/src/aggregation/batch_data.rs @@ -1,4 +1,4 @@ -use eth_types::H256; +use eth_types::{H256, U256}; use ethers_core::utils::keccak256; use halo2_ecc::bigint::CRTInteger; use halo2_proofs::{ @@ -8,14 +8,36 @@ use halo2_proofs::{ poly::Rotation, }; use itertools::Itertools; +use std::iter::{once, repeat}; use zkevm_circuits::{ table::{KeccakTable, LookupTable, RangeTable, U8Table}, util::{Challenges, Expr}, }; #[cfg(feature = "da-eip4844")] -use crate::data_availability::eip4844::blob::{BLOB_WIDTH, N_BYTES_U256}; -use crate::{aggregation::rlc::POWS_OF_256, BatchData, RlcConfig}; +use crate::data_availability::eip4844::blob::BLOB_WIDTH; +use crate::{aggregation::rlc::POWS_OF_256, BatchHash, ChunkInfo, RlcConfig}; + +/// The number of bytes to represent an unsigned 256 bit number. +pub const N_BYTES_U256: usize = 32; + +/// The number data bytes we pack each BLS12-381 scalar into. The most-significant byte is 0. +pub const N_DATA_BYTES_PER_COEFFICIENT: usize = 31; + +/// The number of bytes that we can fit in a blob. Note that each coefficient is represented in 32 +/// bytes, however, since those 32 bytes must represent a BLS12-381 scalar in its canonical form, +/// we explicitly set the most-significant byte to 0, effectively utilising only 31 bytes. +pub const N_BLOB_BYTES: usize = BLOB_WIDTH * N_DATA_BYTES_PER_COEFFICIENT; + +/// Allow up to 5x compression via zstd encoding of the batch data. +const N_BATCH_BYTES: usize = N_BLOB_BYTES * 5; + +/// The number of rows to encode number of valid chunks (num_valid_snarks) in a batch, in the Blob +/// Data config. Since num_valid_chunks is u16, we use 2 bytes/rows. +const N_ROWS_NUM_CHUNKS: usize = 2; + +/// The number of rows to encode chunk size (u32). +const N_ROWS_CHUNK_SIZE: usize = 4; #[derive(Clone, Debug)] pub struct BatchDataConfig { @@ -1016,3 +1038,580 @@ impl BatchDataConfig { Ok(export) } } + +/// Helper struct to generate witness for the Batch Data Config. +#[derive(Clone, Debug)] +pub struct BatchData { + /// The number of valid chunks in the batch. This could be any number between: + /// [1, N_SNARKS] + pub num_valid_chunks: u16, + /// The size of each chunk. The chunk size can be zero if: + /// - The chunk is a padded chunk (not a valid chunk). + /// - The chunk has no L2 transactions, but only L1 msg txs. + pub chunk_sizes: [u32; N_SNARKS], + /// Flattened L2 signed transaction data, for each chunk. + /// + /// Note that in BatchData struct, only `num_valid_chunks` number of chunks' bytes are supposed + /// to be read (for witness generation). For simplicity, the last valid chunk's bytes are + /// copied over for the padded chunks. The `chunk_data_digest` for padded chunks is the + /// `chunk_data_digest` of the last valid chunk (from Aggregation Circuit's perspective). + pub chunk_data: [Vec; N_SNARKS], +} + +impl BatchData { + /// For raw batch bytes with metadata, this function segments the byte stream into chunk segments. + /// Metadata will be removed from the result. + pub fn segment_with_metadata(batch_bytes_with_metadata: Vec) -> Vec> { + let n_bytes_metadata = Self::n_rows_metadata(); + let metadata_bytes = batch_bytes_with_metadata + .clone() + .into_iter() + .take(n_bytes_metadata) + .collect::>(); + let batch_bytes = batch_bytes_with_metadata + .clone() + .into_iter() + .skip(n_bytes_metadata) + .collect::>(); + + // Decoded batch bytes require segmentation based on chunk length + let batch_data_len = batch_bytes.len(); + let chunk_lens = metadata_bytes[N_ROWS_NUM_CHUNKS..] + .chunks(N_ROWS_CHUNK_SIZE) + .map(|chunk| { + chunk + .iter() + .fold(0usize, |acc, &d| acc * 256usize + d as usize) + }) + .collect::>(); + + // length segments sanity check + let valid_chunks = metadata_bytes + .iter() + .take(N_ROWS_NUM_CHUNKS) + .fold(0usize, |acc, &d| acc * 256usize + d as usize); + let calculated_len = chunk_lens.iter().take(valid_chunks).sum::(); + assert_eq!( + batch_data_len, calculated_len, + "chunk segmentation len must add up to the correct value" + ); + + // reconstruct segments + let mut segmented_batch_data: Vec> = Vec::new(); + let mut offset: usize = 0; + let mut segment: usize = 0; + while offset < batch_data_len { + segmented_batch_data.push( + batch_bytes + .clone() + .into_iter() + .skip(offset) + .take(chunk_lens[segment]) + .collect::>(), + ); + + offset += chunk_lens[segment]; + segment += 1; + } + + segmented_batch_data + } +} + +impl From<&BatchHash> for BatchData { + fn from(batch_hash: &BatchHash) -> Self { + Self::new( + batch_hash.number_of_valid_chunks, + &batch_hash.chunks_with_padding, + ) + } +} + +// If the chunk data is represented as a vector of u8's this implementation converts data from +// dynamic number of chunks into BatchData. +impl From<&Vec>> for BatchData { + fn from(chunks: &Vec>) -> Self { + let num_valid_chunks = chunks.len(); + assert!(num_valid_chunks > 0); + assert!(num_valid_chunks <= N_SNARKS); + + let chunk_sizes: [u32; N_SNARKS] = chunks + .iter() + .map(|chunk| chunk.len() as u32) + .chain(repeat(0)) + .take(N_SNARKS) + .collect::>() + .try_into() + .expect("we have N_SNARKS chunks"); + assert!(chunk_sizes.iter().sum::() <= Self::n_rows_data().try_into().unwrap()); + + let last_chunk_data = chunks.last().expect("last chunk exists"); + let chunk_data = chunks + .iter() + .chain(repeat(last_chunk_data)) + .take(N_SNARKS) + .cloned() + .collect::>() + .try_into() + .expect("we have N_SNARKS chunks"); + + Self { + num_valid_chunks: num_valid_chunks.try_into().unwrap(), + chunk_sizes, + chunk_data, + } + } +} + +impl Default for BatchData { + fn default() -> Self { + // default value corresponds to a batch with 1 chunk with no transactions + Self::from(&vec![vec![]]) + } +} + +impl BatchData { + /// The number of rows in Blob Data config's layout to represent the "digest rlc" section. + /// - metadata digest RLC (1 row) + /// - chunk_digests RLC for each chunk (MAX_AGG_SNARKS rows) + /// - blob versioned hash RLC (1 row) + /// - challenge digest RLC (1 row) + pub const fn n_rows_digest_rlc() -> usize { + 1 + N_SNARKS + 1 + 1 + } + + /// The number of rows in Blob Data config's layout to represent the "digest bytes" section. + pub const fn n_rows_digest_bytes() -> usize { + Self::n_rows_digest_rlc() * N_BYTES_U256 + } + + /// The number of rows to encode the size of each chunk in a batch, in the Blob Data config. + /// chunk_size is u32, we use 4 bytes/rows. + const fn n_rows_chunk_sizes() -> usize { + N_SNARKS * N_ROWS_CHUNK_SIZE + } + + /// The total number of rows in "digest rlc" and "digest bytes" sections. + const fn n_rows_digest() -> usize { + Self::n_rows_digest_rlc() + Self::n_rows_digest_bytes() + } + + /// The number of rows in Blob Data config's layout to represent the "blob metadata" section. + pub const fn n_rows_metadata() -> usize { + N_ROWS_NUM_CHUNKS + Self::n_rows_chunk_sizes() + } + + /// The number of rows in Blob Data config's layout to represent the "chunk data" section. + pub const fn n_rows_data() -> usize { + N_BATCH_BYTES - Self::n_rows_metadata() + } + + /// The total number of rows used in Blob Data config's layout. + pub const fn n_rows() -> usize { + N_BATCH_BYTES + Self::n_rows_digest() + } + + /// Construct BatchData from chunks + pub fn new(num_valid_chunks: usize, chunks_with_padding: &[ChunkInfo]) -> Self { + assert!(num_valid_chunks > 0); + assert!(num_valid_chunks <= N_SNARKS); + + // padded chunk has 0 size, valid chunk's size is the number of bytes consumed by the + // flattened data from signed L2 transactions. + let chunk_sizes: [u32; N_SNARKS] = chunks_with_padding + .iter() + .map(|chunk| { + if chunk.is_padding { + 0 + } else { + chunk.tx_bytes.len() as u32 + } + }) + .collect::>() + .try_into() + .unwrap(); + + if chunk_sizes.iter().sum::() > Self::n_rows_data() as u32 { + panic!( + "invalid chunk_sizes {}, n_rows_data {}", + chunk_sizes.iter().sum::(), + Self::n_rows_data() + ) + } + + // chunk data of the "last valid chunk" is repeated over the padded chunks for simplicity + // in calculating chunk_data_digest for those padded chunks. However, for the "chunk data" + // section rows (self.to_data_rows()) we only consider `num_valid_chunks` chunks. + let chunk_data = chunks_with_padding + .iter() + .map(|chunk| chunk.tx_bytes.to_vec()) + .collect::>>() + .try_into() + .unwrap(); + + Self { + num_valid_chunks: num_valid_chunks as u16, + chunk_sizes, + chunk_data, + } + } + + /// Get the preimage of the challenge digest. + pub(crate) fn get_challenge_digest_preimage(&self, versioned_hash: H256) -> Vec { + let metadata_digest = keccak256(self.to_metadata_bytes()); + let chunk_digests = self.chunk_data.iter().map(keccak256); + + // preimage = + // metadata_digest || + // chunk[0].chunk_data_digest || ... + // chunk[N_SNARKS-1].chunk_data_digest || + // blob_versioned_hash + // + // where chunk_data_digest for a padded chunk is set equal to the "last valid chunk"'s + // chunk_data_digest. + metadata_digest + .into_iter() + .chain(chunk_digests.flatten()) + .chain(versioned_hash.to_fixed_bytes()) + .collect::>() + } + + /// Compute the challenge digest from blob bytes. + pub(crate) fn get_challenge_digest(&self, versioned_hash: H256) -> U256 { + let challenge_digest = keccak256(self.get_challenge_digest_preimage(versioned_hash)); + U256::from_big_endian(&challenge_digest) + } + + /// Get the batch data bytes that will be populated in BatchDataConfig. + pub fn get_batch_data_bytes(&self) -> Vec { + let metadata_bytes = self.to_metadata_bytes(); + metadata_bytes + .iter() + .chain( + self.chunk_data + .iter() + .take(self.num_valid_chunks as usize) + .flatten(), + ) + .cloned() + .collect() + } + + /// Get the list of preimages that need to go through the keccak hashing function, and + /// eventually required to be checked for the consistency of blob's metadata, its chunks' bytes + /// and the final blob preimage. + pub fn preimages(&self, versioned_hash: H256) -> Vec> { + let mut preimages = Vec::with_capacity(2 + N_SNARKS); + + // metadata + preimages.push(self.to_metadata_bytes()); + + // each valid chunk's data + for chunk in self.chunk_data.iter().take(self.num_valid_chunks as usize) { + preimages.push(chunk.to_vec()); + } + + // preimage for challenge digest + preimages.push(self.get_challenge_digest_preimage(versioned_hash)); + + preimages + } + + /// Get the witness rows for assignment to the BlobDataConfig. + pub(crate) fn to_rows( + &self, + versioned_hash: H256, + challenge: Challenges>, + ) -> Vec> { + let metadata_rows = self.to_metadata_rows(challenge); + assert_eq!(metadata_rows.len(), Self::n_rows_metadata()); + + let data_rows = self.to_data_rows(challenge); + assert_eq!(data_rows.len(), Self::n_rows_data()); + + let digest_rows = self.to_digest_rows(versioned_hash, challenge); + assert_eq!(digest_rows.len(), Self::n_rows_digest()); + + metadata_rows + .into_iter() + .chain(data_rows) + .chain(digest_rows) + .collect::>>() + } + + /// Get the blob bytes that encode the batch's metadata. + /// + /// metadata_bytes = + /// be_bytes(num_valid_chunks) || + /// be_bytes(chunks[0].chunk_size) || ... + /// be_bytes(chunks[N_SNARKS-1].chunk_size) + /// + /// where: + /// - chunk_size of a padded chunk is 0 + /// - num_valid_chunks is u16 + /// - each chunk_size is u32 + fn to_metadata_bytes(&self) -> Vec { + self.num_valid_chunks + .to_be_bytes() + .into_iter() + .chain( + self.chunk_sizes + .iter() + .flat_map(|chunk_size| chunk_size.to_be_bytes()), + ) + .collect() + } + + /// Get the witness rows for the "metadata" section of Blob data config. + fn to_metadata_rows(&self, challenge: Challenges>) -> Vec> { + // metadata bytes. + let bytes = self.to_metadata_bytes(); + + // accumulators represent the running linear combination of bytes. + let accumulators_iter = self + .num_valid_chunks + .to_be_bytes() + .into_iter() + .scan(0u64, |acc, x| { + *acc = *acc * 256 + (x as u64); + Some(*acc) + }) + .chain(self.chunk_sizes.into_iter().flat_map(|chunk_size| { + chunk_size.to_be_bytes().into_iter().scan(0u64, |acc, x| { + *acc = *acc * 256 + (x as u64); + Some(*acc) + }) + })); + + // digest_rlc is set only for the last row in the "metadata" section, and it denotes the + // RLC of the metadata_digest bytes. + let digest_rlc_iter = { + let digest = keccak256(&bytes); + let digest_rlc = digest.iter().fold(Value::known(Fr::zero()), |acc, &x| { + acc * challenge.evm_word() + Value::known(Fr::from(x as u64)) + }); + repeat(Value::known(Fr::zero())) + .take(Self::n_rows_metadata() - 1) + .chain(once(digest_rlc)) + }; + + // preimage_rlc is the running RLC over bytes in the "metadata" section. + let preimage_rlc_iter = bytes.iter().scan(Value::known(Fr::zero()), |acc, &x| { + *acc = *acc * challenge.keccak_input() + Value::known(Fr::from(x as u64)); + Some(*acc) + }); + + bytes + .iter() + .zip_eq(accumulators_iter) + .zip_eq(preimage_rlc_iter) + .zip_eq(digest_rlc_iter) + .enumerate() + .map( + |(i, (((&byte, accumulator), preimage_rlc), digest_rlc))| BatchDataRow { + byte, + accumulator, + preimage_rlc, + digest_rlc, + // we set boundary on the last row of the "metadata" section to enable a lookup + // to the keccak table. + is_boundary: i == bytes.len() - 1, + ..Default::default() + }, + ) + .collect() + } + + /// Get the witness rows for the "chunk data" section of Blob data config. + fn to_data_rows(&self, challenge: Challenges>) -> Vec> { + // consider only the `valid` chunks while constructing rows for the "chunk data" section. + self.chunk_data + .iter() + .take(self.num_valid_chunks as usize) + .enumerate() + .flat_map(|(i, bytes)| { + let chunk_idx = (i + 1) as u64; + let chunk_size = bytes.len(); + let chunk_digest = keccak256(bytes); + let digest_rlc = chunk_digest + .iter() + .fold(Value::known(Fr::zero()), |acc, &byte| { + acc * challenge.evm_word() + Value::known(Fr::from(byte as u64)) + }); + bytes.iter().enumerate().scan( + (0u64, Value::known(Fr::zero())), + move |acc, (j, &byte)| { + acc.0 += 1; + acc.1 = + acc.1 * challenge.keccak_input() + Value::known(Fr::from(byte as u64)); + Some(BatchDataRow { + byte, + accumulator: acc.0, + chunk_idx, + is_boundary: j == chunk_size - 1, + is_padding: false, + preimage_rlc: acc.1, + digest_rlc: if j == chunk_size - 1 { + digest_rlc + } else { + Value::known(Fr::zero()) + }, + }) + }, + ) + }) + .chain(repeat(BatchDataRow::padding_row())) + .take(Self::n_rows_data()) + .collect() + } + + /// Get the witness rows for both "digest rlc" and "digest bytes" sections of Blob data config. + fn to_digest_rows( + &self, + versioned_hash: H256, + challenge: Challenges>, + ) -> Vec> { + let zero = Value::known(Fr::zero()); + + // metadata + let metadata_bytes = self.to_metadata_bytes(); + let metadata_digest = keccak256(metadata_bytes); + let metadata_digest_rlc = metadata_digest.iter().fold(zero, |acc, &byte| { + acc * challenge.evm_word() + Value::known(Fr::from(byte as u64)) + }); + + // chunk data + // Note: here we don't restrict to considering only `valid` chunks, as the + // `chunk_data_digest` gets repeated for the padded chunks, copying the last valid chunk's + // `chunk_data_digest`. + let (chunk_digests, chunk_digest_rlcs): (Vec<[u8; 32]>, Vec>) = self + .chunk_data + .iter() + .map(|chunk| { + let digest = keccak256(chunk); + let digest_rlc = digest.iter().fold(zero, |acc, &byte| { + acc * challenge.evm_word() + Value::known(Fr::from(byte as u64)) + }); + (digest, digest_rlc) + }) + .unzip(); + + // challenge digest + let challenge_digest_preimage = self.get_challenge_digest_preimage(versioned_hash); + let challenge_digest_preimage_rlc = + challenge_digest_preimage.iter().fold(zero, |acc, &byte| { + acc * challenge.keccak_input() + Value::known(Fr::from(byte as u64)) + }); + let challenge_digest = keccak256(&challenge_digest_preimage); + let challenge_digest_rlc = challenge_digest.iter().fold(zero, |acc, &byte| { + acc * challenge.evm_word() + Value::known(Fr::from(byte as u64)) + }); + + // blob versioned hash + let versioned_hash_rlc = versioned_hash.as_bytes().iter().fold(zero, |acc, &byte| { + acc * challenge.evm_word() + Value::known(Fr::from(byte as u64)) + }); + + // - metadata digest rlc + // - chunks[i].chunk_data_digest rlc for each chunk + // - versioned hash rlc + // - challenge digest rlc + // - metadata digest bytes + // - chunks[i].chunk_data_digest bytes for each chunk + // - versioned hash bytes + // - challenge digest bytes + once(BatchDataRow { + preimage_rlc: Value::known(Fr::zero()), + digest_rlc: metadata_digest_rlc, + // this is_padding assignment does not matter as we have already crossed the "chunk + // data" section. This assignment to 1 is simply to allow the custom gate to check: + // - padding transitions from 0 -> 1 only once. + is_padding: true, + ..Default::default() + }) + .chain( + chunk_digest_rlcs + .iter() + .zip_eq(self.chunk_sizes.iter()) + .enumerate() + .map(|(i, (&digest_rlc, &chunk_size))| BatchDataRow { + preimage_rlc: Value::known(Fr::zero()), + digest_rlc, + chunk_idx: (i + 1) as u64, + accumulator: chunk_size as u64, + ..Default::default() + }), + ) + // versioned hash RLC + .chain(once(BatchDataRow { + preimage_rlc: Value::known(Fr::zero()), + digest_rlc: versioned_hash_rlc, + ..Default::default() + })) + .chain(once(BatchDataRow { + preimage_rlc: challenge_digest_preimage_rlc, + digest_rlc: challenge_digest_rlc, + accumulator: 32 * (N_SNARKS + 1 + 1) as u64, + is_boundary: true, + ..Default::default() + })) + .chain(metadata_digest.iter().map(|&byte| BatchDataRow { + preimage_rlc: Value::known(Fr::zero()), + digest_rlc: Value::known(Fr::zero()), + byte, + ..Default::default() + })) + .chain(chunk_digests.iter().flat_map(|digest| { + digest.iter().map(|&byte| BatchDataRow { + preimage_rlc: Value::known(Fr::zero()), + digest_rlc: Value::known(Fr::zero()), + byte, + ..Default::default() + }) + })) + // bytes of versioned hash + .chain(versioned_hash.as_bytes().iter().map(|&byte| BatchDataRow { + preimage_rlc: Value::known(Fr::zero()), + digest_rlc: Value::known(Fr::zero()), + byte, + ..Default::default() + })) + .chain(challenge_digest.iter().map(|&byte| BatchDataRow { + preimage_rlc: Value::known(Fr::zero()), + digest_rlc: Value::known(Fr::zero()), + byte, + ..Default::default() + })) + .collect() + } +} + +/// Witness row to the Blob Data Config. +#[derive(Clone, Copy, Default, Debug)] +struct BatchDataRow { + /// Byte value at this row. + pub byte: u8, + /// Multi-purpose accumulator value. + pub accumulator: u64, + /// The chunk index we are currently traversing. + pub chunk_idx: u64, + /// Whether this marks the end of a chunk. + pub is_boundary: bool, + /// Whether the row represents right-padded 0s to the blob data. + pub is_padding: bool, + /// A running accumulator of RLC of preimages. + pub preimage_rlc: Value, + /// RLC of the digest. + pub digest_rlc: Value, +} + +impl BatchDataRow { + fn padding_row() -> Self { + Self { + is_padding: true, + preimage_rlc: Value::known(Fr::zero()), + digest_rlc: Value::known(Fr::zero()), + ..Default::default() + } + } +} diff --git a/aggregator/src/aggregation/circuit.rs b/aggregator/src/aggregation/circuit.rs index 665d3431ab..f793025007 100644 --- a/aggregator/src/aggregation/circuit.rs +++ b/aggregator/src/aggregation/circuit.rs @@ -23,11 +23,10 @@ use std::{env, fs::File, rc::Rc}; use zkevm_circuits::util::Challenges; use crate::{ - aggregation::{decoder::WORKED_EXAMPLE, witgen::process, BatchCircuitConfig}, + aggregation::{decoder::WORKED_EXAMPLE, witgen::process, BatchCircuitConfig, BatchData}, batch::BatchHash, constants::{ACC_LEN, DIGEST_LEN}, core::{assign_batch_hashes, extract_proof_and_instances_with_pairing_check}, - data_availability::eip4844::blob::BatchData, data_availability::eip4844::AssignedBarycentricEvaluationConfig, util::parse_hash_digest_cells, witgen::{zstd_encode, MultiBlockProcessResult}, diff --git a/aggregator/src/batch.rs b/aggregator/src/batch.rs index 3a5ade39a3..046c72211d 100644 --- a/aggregator/src/batch.rs +++ b/aggregator/src/batch.rs @@ -7,8 +7,9 @@ use gadgets::{util::split_h256, Field}; use serde::{Deserialize, Serialize}; use crate::{ + aggregation::BatchData, chunk::ChunkInfo, - data_availability::eip4844::blob::{BatchData, PointEvaluationAssignments}, + data_availability::eip4844::blob::PointEvaluationAssignments, data_availability::eip4844::{get_coefficients, get_versioned_hash}, }; diff --git a/aggregator/src/data_availability/eip4844.rs b/aggregator/src/data_availability/eip4844.rs index 9e6ff11c3e..93d811fed5 100644 --- a/aggregator/src/data_availability/eip4844.rs +++ b/aggregator/src/data_availability/eip4844.rs @@ -6,12 +6,13 @@ pub use barycentric::{ /// blob struct and constants pub mod blob; -use blob::{BLOB_WIDTH, KZG_TRUSTED_SETUP, N_BLOB_BYTES, N_BYTES_U256}; /// Config to constrain blob data (encoded batch data) pub mod blob_data; pub use blob_data::BlobDataConfig; +use crate::aggregation::batch_data::{N_BLOB_BYTES, N_BYTES_U256}; +use blob::{BLOB_WIDTH, KZG_TRUSTED_SETUP}; use eth_types::{ToBigEndian, H256, U256}; use ethers_core::k256::sha2::{Digest, Sha256}; use revm_primitives::VERSIONED_HASH_VERSION_KZG; diff --git a/aggregator/src/data_availability/eip4844/barycentric.rs b/aggregator/src/data_availability/eip4844/barycentric.rs index 1417f79c15..e0e63ba64c 100644 --- a/aggregator/src/data_availability/eip4844/barycentric.rs +++ b/aggregator/src/data_availability/eip4844/barycentric.rs @@ -356,7 +356,8 @@ pub fn interpolate(z: Scalar, coefficients: &[Scalar; BLOB_WIDTH]) -> Scalar { mod tests { use super::*; use crate::{ - data_availability::eip4844::blob::{BatchData, KZG_TRUSTED_SETUP}, + aggregation::BatchData, + data_availability::eip4844::blob::KZG_TRUSTED_SETUP, data_availability::eip4844::{get_blob_bytes, get_coefficients}, MAX_AGG_SNARKS, }; diff --git a/aggregator/src/data_availability/eip4844/blob.rs b/aggregator/src/data_availability/eip4844/blob.rs index 115d6b0d23..0417d30dc0 100644 --- a/aggregator/src/data_availability/eip4844/blob.rs +++ b/aggregator/src/data_availability/eip4844/blob.rs @@ -1,5 +1,5 @@ use super::{get_coefficients, interpolate, BLS_MODULUS}; -use crate::{BatchHash, ChunkInfo}; +use crate::{BatchData, BatchHash, ChunkInfo}; use eth_types::{H256, U256}; use ethers_core::utils::keccak256; @@ -19,27 +19,6 @@ use zkevm_circuits::util::Challenges; /// form. pub const BLOB_WIDTH: usize = 4096; -/// The number of bytes to represent an unsigned 256 bit number. -pub const N_BYTES_U256: usize = 32; - -/// The number data bytes we pack each BLS12-381 scalar into. The most-significant byte is 0. -pub const N_DATA_BYTES_PER_COEFFICIENT: usize = 31; - -/// The number of rows to encode number of valid chunks (num_valid_snarks) in a batch, in the Blob -/// Data config. Since num_valid_chunks is u16, we use 2 bytes/rows. -pub const N_ROWS_NUM_CHUNKS: usize = 2; - -/// The number of rows to encode chunk size (u32). -pub const N_ROWS_CHUNK_SIZE: usize = 4; - -/// The number of bytes that we can fit in a blob. Note that each coefficient is represented in 32 -/// bytes, however, since those 32 bytes must represent a BLS12-381 scalar in its canonical form, -/// we explicitly set the most-significant byte to 0, effectively utilising only 31 bytes. -pub const N_BLOB_BYTES: usize = BLOB_WIDTH * N_DATA_BYTES_PER_COEFFICIENT; - -/// Allow up to 5x compression via zstd encoding of the batch data. -pub const N_BATCH_BYTES: usize = N_BLOB_BYTES * 5; - /// KZG trusted setup pub static KZG_TRUSTED_SETUP: Lazy> = Lazy::new(|| { Arc::new( @@ -51,553 +30,6 @@ pub static KZG_TRUSTED_SETUP: Lazy> = Lazy::new(|| { ) }); -/// Helper struct to generate witness for the Batch Data Config. -#[derive(Clone, Debug)] -pub struct BatchData { - /// The number of valid chunks in the batch. This could be any number between: - /// [1, N_SNARKS] - pub num_valid_chunks: u16, - /// The size of each chunk. The chunk size can be zero if: - /// - The chunk is a padded chunk (not a valid chunk). - /// - The chunk has no L2 transactions, but only L1 msg txs. - pub chunk_sizes: [u32; N_SNARKS], - /// Flattened L2 signed transaction data, for each chunk. - /// - /// Note that in BatchData struct, only `num_valid_chunks` number of chunks' bytes are supposed - /// to be read (for witness generation). For simplicity, the last valid chunk's bytes are - /// copied over for the padded chunks. The `chunk_data_digest` for padded chunks is the - /// `chunk_data_digest` of the last valid chunk (from Aggregation Circuit's perspective). - pub chunk_data: [Vec; N_SNARKS], -} - -impl BatchData { - /// For raw batch bytes with metadata, this function segments the byte stream into chunk segments. - /// Metadata will be removed from the result. - pub fn segment_with_metadata(batch_bytes_with_metadata: Vec) -> Vec> { - let n_bytes_metadata = Self::n_rows_metadata(); - let metadata_bytes = batch_bytes_with_metadata - .clone() - .into_iter() - .take(n_bytes_metadata) - .collect::>(); - let batch_bytes = batch_bytes_with_metadata - .clone() - .into_iter() - .skip(n_bytes_metadata) - .collect::>(); - - // Decoded batch bytes require segmentation based on chunk length - let batch_data_len = batch_bytes.len(); - let chunk_lens = metadata_bytes[N_ROWS_NUM_CHUNKS..] - .chunks(N_ROWS_CHUNK_SIZE) - .map(|chunk| { - chunk - .iter() - .fold(0usize, |acc, &d| acc * 256usize + d as usize) - }) - .collect::>(); - - // length segments sanity check - let valid_chunks = metadata_bytes - .iter() - .take(N_ROWS_NUM_CHUNKS) - .fold(0usize, |acc, &d| acc * 256usize + d as usize); - let calculated_len = chunk_lens.iter().take(valid_chunks).sum::(); - assert_eq!( - batch_data_len, calculated_len, - "chunk segmentation len must add up to the correct value" - ); - - // reconstruct segments - let mut segmented_batch_data: Vec> = Vec::new(); - let mut offset: usize = 0; - let mut segment: usize = 0; - while offset < batch_data_len { - segmented_batch_data.push( - batch_bytes - .clone() - .into_iter() - .skip(offset) - .take(chunk_lens[segment]) - .collect::>(), - ); - - offset += chunk_lens[segment]; - segment += 1; - } - - segmented_batch_data - } -} - -impl From<&BatchHash> for BatchData { - fn from(batch_hash: &BatchHash) -> Self { - Self::new( - batch_hash.number_of_valid_chunks, - &batch_hash.chunks_with_padding, - ) - } -} - -// If the chunk data is represented as a vector of u8's this implementation converts data from -// dynamic number of chunks into BatchData. -impl From<&Vec>> for BatchData { - fn from(chunks: &Vec>) -> Self { - let num_valid_chunks = chunks.len(); - assert!(num_valid_chunks > 0); - assert!(num_valid_chunks <= N_SNARKS); - - let chunk_sizes: [u32; N_SNARKS] = chunks - .iter() - .map(|chunk| chunk.len() as u32) - .chain(repeat(0)) - .take(N_SNARKS) - .collect::>() - .try_into() - .expect("we have N_SNARKS chunks"); - assert!(chunk_sizes.iter().sum::() <= Self::n_rows_data().try_into().unwrap()); - - let last_chunk_data = chunks.last().expect("last chunk exists"); - let chunk_data = chunks - .iter() - .chain(repeat(last_chunk_data)) - .take(N_SNARKS) - .cloned() - .collect::>() - .try_into() - .expect("we have N_SNARKS chunks"); - - Self { - num_valid_chunks: num_valid_chunks.try_into().unwrap(), - chunk_sizes, - chunk_data, - } - } -} - -impl Default for BatchData { - fn default() -> Self { - // default value corresponds to a batch with 1 chunk with no transactions - Self::from(&vec![vec![]]) - } -} - -impl BatchData { - /// The number of rows in Blob Data config's layout to represent the "digest rlc" section. - /// - metadata digest RLC (1 row) - /// - chunk_digests RLC for each chunk (MAX_AGG_SNARKS rows) - /// - blob versioned hash RLC (1 row) - /// - challenge digest RLC (1 row) - pub const fn n_rows_digest_rlc() -> usize { - 1 + N_SNARKS + 1 + 1 - } - - /// The number of rows in Blob Data config's layout to represent the "digest bytes" section. - pub const fn n_rows_digest_bytes() -> usize { - Self::n_rows_digest_rlc() * N_BYTES_U256 - } - - /// The number of rows to encode the size of each chunk in a batch, in the Blob Data config. - /// chunk_size is u32, we use 4 bytes/rows. - const fn n_rows_chunk_sizes() -> usize { - N_SNARKS * N_ROWS_CHUNK_SIZE - } - - /// The total number of rows in "digest rlc" and "digest bytes" sections. - const fn n_rows_digest() -> usize { - Self::n_rows_digest_rlc() + Self::n_rows_digest_bytes() - } - - /// The number of rows in Blob Data config's layout to represent the "blob metadata" section. - pub const fn n_rows_metadata() -> usize { - N_ROWS_NUM_CHUNKS + Self::n_rows_chunk_sizes() - } - - /// The number of rows in Blob Data config's layout to represent the "chunk data" section. - pub const fn n_rows_data() -> usize { - N_BATCH_BYTES - Self::n_rows_metadata() - } - - /// The total number of rows used in Blob Data config's layout. - pub const fn n_rows() -> usize { - N_BATCH_BYTES + Self::n_rows_digest() - } - - /// Construct BatchData from chunks - pub fn new(num_valid_chunks: usize, chunks_with_padding: &[ChunkInfo]) -> Self { - assert!(num_valid_chunks > 0); - assert!(num_valid_chunks <= N_SNARKS); - - // padded chunk has 0 size, valid chunk's size is the number of bytes consumed by the - // flattened data from signed L2 transactions. - let chunk_sizes: [u32; N_SNARKS] = chunks_with_padding - .iter() - .map(|chunk| { - if chunk.is_padding { - 0 - } else { - chunk.tx_bytes.len() as u32 - } - }) - .collect::>() - .try_into() - .unwrap(); - - if chunk_sizes.iter().sum::() > Self::n_rows_data() as u32 { - panic!( - "invalid chunk_sizes {}, n_rows_data {}", - chunk_sizes.iter().sum::(), - Self::n_rows_data() - ) - } - - // chunk data of the "last valid chunk" is repeated over the padded chunks for simplicity - // in calculating chunk_data_digest for those padded chunks. However, for the "chunk data" - // section rows (self.to_data_rows()) we only consider `num_valid_chunks` chunks. - let chunk_data = chunks_with_padding - .iter() - .map(|chunk| chunk.tx_bytes.to_vec()) - .collect::>>() - .try_into() - .unwrap(); - - Self { - num_valid_chunks: num_valid_chunks as u16, - chunk_sizes, - chunk_data, - } - } - - /// Get the preimage of the challenge digest. - pub(crate) fn get_challenge_digest_preimage(&self, versioned_hash: H256) -> Vec { - let metadata_digest = keccak256(self.to_metadata_bytes()); - let chunk_digests = self.chunk_data.iter().map(keccak256); - - // preimage = - // metadata_digest || - // chunk[0].chunk_data_digest || ... - // chunk[N_SNARKS-1].chunk_data_digest || - // blob_versioned_hash - // - // where chunk_data_digest for a padded chunk is set equal to the "last valid chunk"'s - // chunk_data_digest. - metadata_digest - .into_iter() - .chain(chunk_digests.flatten()) - .chain(versioned_hash.to_fixed_bytes()) - .collect::>() - } - - /// Compute the challenge digest from blob bytes. - pub(crate) fn get_challenge_digest(&self, versioned_hash: H256) -> U256 { - let challenge_digest = keccak256(self.get_challenge_digest_preimage(versioned_hash)); - U256::from_big_endian(&challenge_digest) - } - - /// Get the batch data bytes that will be populated in BatchDataConfig. - pub fn get_batch_data_bytes(&self) -> Vec { - let metadata_bytes = self.to_metadata_bytes(); - metadata_bytes - .iter() - .chain( - self.chunk_data - .iter() - .take(self.num_valid_chunks as usize) - .flatten(), - ) - .cloned() - .collect() - } - - /// Get the list of preimages that need to go through the keccak hashing function, and - /// eventually required to be checked for the consistency of blob's metadata, its chunks' bytes - /// and the final blob preimage. - pub fn preimages(&self, versioned_hash: H256) -> Vec> { - let mut preimages = Vec::with_capacity(2 + N_SNARKS); - - // metadata - preimages.push(self.to_metadata_bytes()); - - // each valid chunk's data - for chunk in self.chunk_data.iter().take(self.num_valid_chunks as usize) { - preimages.push(chunk.to_vec()); - } - - // preimage for challenge digest - preimages.push(self.get_challenge_digest_preimage(versioned_hash)); - - preimages - } - - /// Get the witness rows for assignment to the BlobDataConfig. - pub(crate) fn to_rows( - &self, - versioned_hash: H256, - challenge: Challenges>, - ) -> Vec> { - let metadata_rows = self.to_metadata_rows(challenge); - assert_eq!(metadata_rows.len(), Self::n_rows_metadata()); - - let data_rows = self.to_data_rows(challenge); - assert_eq!(data_rows.len(), Self::n_rows_data()); - - let digest_rows = self.to_digest_rows(versioned_hash, challenge); - assert_eq!(digest_rows.len(), Self::n_rows_digest()); - - metadata_rows - .into_iter() - .chain(data_rows) - .chain(digest_rows) - .collect::>>() - } - - /// Get the blob bytes that encode the batch's metadata. - /// - /// metadata_bytes = - /// be_bytes(num_valid_chunks) || - /// be_bytes(chunks[0].chunk_size) || ... - /// be_bytes(chunks[N_SNARKS-1].chunk_size) - /// - /// where: - /// - chunk_size of a padded chunk is 0 - /// - num_valid_chunks is u16 - /// - each chunk_size is u32 - fn to_metadata_bytes(&self) -> Vec { - self.num_valid_chunks - .to_be_bytes() - .into_iter() - .chain( - self.chunk_sizes - .iter() - .flat_map(|chunk_size| chunk_size.to_be_bytes()), - ) - .collect() - } - - /// Get the witness rows for the "metadata" section of Blob data config. - fn to_metadata_rows(&self, challenge: Challenges>) -> Vec> { - // metadata bytes. - let bytes = self.to_metadata_bytes(); - - // accumulators represent the running linear combination of bytes. - let accumulators_iter = self - .num_valid_chunks - .to_be_bytes() - .into_iter() - .scan(0u64, |acc, x| { - *acc = *acc * 256 + (x as u64); - Some(*acc) - }) - .chain(self.chunk_sizes.into_iter().flat_map(|chunk_size| { - chunk_size.to_be_bytes().into_iter().scan(0u64, |acc, x| { - *acc = *acc * 256 + (x as u64); - Some(*acc) - }) - })); - - // digest_rlc is set only for the last row in the "metadata" section, and it denotes the - // RLC of the metadata_digest bytes. - let digest_rlc_iter = { - let digest = keccak256(&bytes); - let digest_rlc = digest.iter().fold(Value::known(Fr::zero()), |acc, &x| { - acc * challenge.evm_word() + Value::known(Fr::from(x as u64)) - }); - repeat(Value::known(Fr::zero())) - .take(Self::n_rows_metadata() - 1) - .chain(once(digest_rlc)) - }; - - // preimage_rlc is the running RLC over bytes in the "metadata" section. - let preimage_rlc_iter = bytes.iter().scan(Value::known(Fr::zero()), |acc, &x| { - *acc = *acc * challenge.keccak_input() + Value::known(Fr::from(x as u64)); - Some(*acc) - }); - - bytes - .iter() - .zip_eq(accumulators_iter) - .zip_eq(preimage_rlc_iter) - .zip_eq(digest_rlc_iter) - .enumerate() - .map( - |(i, (((&byte, accumulator), preimage_rlc), digest_rlc))| BatchDataRow { - byte, - accumulator, - preimage_rlc, - digest_rlc, - // we set boundary on the last row of the "metadata" section to enable a lookup - // to the keccak table. - is_boundary: i == bytes.len() - 1, - ..Default::default() - }, - ) - .collect() - } - - /// Get the witness rows for the "chunk data" section of Blob data config. - fn to_data_rows(&self, challenge: Challenges>) -> Vec> { - // consider only the `valid` chunks while constructing rows for the "chunk data" section. - self.chunk_data - .iter() - .take(self.num_valid_chunks as usize) - .enumerate() - .flat_map(|(i, bytes)| { - let chunk_idx = (i + 1) as u64; - let chunk_size = bytes.len(); - let chunk_digest = keccak256(bytes); - let digest_rlc = chunk_digest - .iter() - .fold(Value::known(Fr::zero()), |acc, &byte| { - acc * challenge.evm_word() + Value::known(Fr::from(byte as u64)) - }); - bytes.iter().enumerate().scan( - (0u64, Value::known(Fr::zero())), - move |acc, (j, &byte)| { - acc.0 += 1; - acc.1 = - acc.1 * challenge.keccak_input() + Value::known(Fr::from(byte as u64)); - Some(BatchDataRow { - byte, - accumulator: acc.0, - chunk_idx, - is_boundary: j == chunk_size - 1, - is_padding: false, - preimage_rlc: acc.1, - digest_rlc: if j == chunk_size - 1 { - digest_rlc - } else { - Value::known(Fr::zero()) - }, - }) - }, - ) - }) - .chain(repeat(BatchDataRow::padding_row())) - .take(Self::n_rows_data()) - .collect() - } - - /// Get the witness rows for both "digest rlc" and "digest bytes" sections of Blob data config. - fn to_digest_rows( - &self, - versioned_hash: H256, - challenge: Challenges>, - ) -> Vec> { - let zero = Value::known(Fr::zero()); - - // metadata - let metadata_bytes = self.to_metadata_bytes(); - let metadata_digest = keccak256(metadata_bytes); - let metadata_digest_rlc = metadata_digest.iter().fold(zero, |acc, &byte| { - acc * challenge.evm_word() + Value::known(Fr::from(byte as u64)) - }); - - // chunk data - // Note: here we don't restrict to considering only `valid` chunks, as the - // `chunk_data_digest` gets repeated for the padded chunks, copying the last valid chunk's - // `chunk_data_digest`. - let (chunk_digests, chunk_digest_rlcs): (Vec<[u8; 32]>, Vec>) = self - .chunk_data - .iter() - .map(|chunk| { - let digest = keccak256(chunk); - let digest_rlc = digest.iter().fold(zero, |acc, &byte| { - acc * challenge.evm_word() + Value::known(Fr::from(byte as u64)) - }); - (digest, digest_rlc) - }) - .unzip(); - - // challenge digest - let challenge_digest_preimage = self.get_challenge_digest_preimage(versioned_hash); - let challenge_digest_preimage_rlc = - challenge_digest_preimage.iter().fold(zero, |acc, &byte| { - acc * challenge.keccak_input() + Value::known(Fr::from(byte as u64)) - }); - let challenge_digest = keccak256(&challenge_digest_preimage); - let challenge_digest_rlc = challenge_digest.iter().fold(zero, |acc, &byte| { - acc * challenge.evm_word() + Value::known(Fr::from(byte as u64)) - }); - - // blob versioned hash - let versioned_hash_rlc = versioned_hash.as_bytes().iter().fold(zero, |acc, &byte| { - acc * challenge.evm_word() + Value::known(Fr::from(byte as u64)) - }); - - // - metadata digest rlc - // - chunks[i].chunk_data_digest rlc for each chunk - // - versioned hash rlc - // - challenge digest rlc - // - metadata digest bytes - // - chunks[i].chunk_data_digest bytes for each chunk - // - versioned hash bytes - // - challenge digest bytes - once(BatchDataRow { - preimage_rlc: Value::known(Fr::zero()), - digest_rlc: metadata_digest_rlc, - // this is_padding assignment does not matter as we have already crossed the "chunk - // data" section. This assignment to 1 is simply to allow the custom gate to check: - // - padding transitions from 0 -> 1 only once. - is_padding: true, - ..Default::default() - }) - .chain( - chunk_digest_rlcs - .iter() - .zip_eq(self.chunk_sizes.iter()) - .enumerate() - .map(|(i, (&digest_rlc, &chunk_size))| BatchDataRow { - preimage_rlc: Value::known(Fr::zero()), - digest_rlc, - chunk_idx: (i + 1) as u64, - accumulator: chunk_size as u64, - ..Default::default() - }), - ) - // versioned hash RLC - .chain(once(BatchDataRow { - preimage_rlc: Value::known(Fr::zero()), - digest_rlc: versioned_hash_rlc, - ..Default::default() - })) - .chain(once(BatchDataRow { - preimage_rlc: challenge_digest_preimage_rlc, - digest_rlc: challenge_digest_rlc, - accumulator: 32 * (N_SNARKS + 1 + 1) as u64, - is_boundary: true, - ..Default::default() - })) - .chain(metadata_digest.iter().map(|&byte| BatchDataRow { - preimage_rlc: Value::known(Fr::zero()), - digest_rlc: Value::known(Fr::zero()), - byte, - ..Default::default() - })) - .chain(chunk_digests.iter().flat_map(|digest| { - digest.iter().map(|&byte| BatchDataRow { - preimage_rlc: Value::known(Fr::zero()), - digest_rlc: Value::known(Fr::zero()), - byte, - ..Default::default() - }) - })) - // bytes of versioned hash - .chain(versioned_hash.as_bytes().iter().map(|&byte| BatchDataRow { - preimage_rlc: Value::known(Fr::zero()), - digest_rlc: Value::known(Fr::zero()), - byte, - ..Default::default() - })) - .chain(challenge_digest.iter().map(|&byte| BatchDataRow { - preimage_rlc: Value::known(Fr::zero()), - digest_rlc: Value::known(Fr::zero()), - byte, - ..Default::default() - })) - .collect() - } -} - #[derive(Clone, Debug)] pub struct PointEvaluationAssignments { /// The random challenge scalar z. @@ -655,36 +87,6 @@ impl PointEvaluationAssignments { } } -/// Witness row to the Blob Data Config. -#[derive(Clone, Copy, Default, Debug)] -pub struct BatchDataRow { - /// Byte value at this row. - pub byte: u8, - /// Multi-purpose accumulator value. - pub accumulator: u64, - /// The chunk index we are currently traversing. - pub chunk_idx: u64, - /// Whether this marks the end of a chunk. - pub is_boundary: bool, - /// Whether the row represents right-padded 0s to the blob data. - pub is_padding: bool, - /// A running accumulator of RLC of preimages. - pub preimage_rlc: Value, - /// RLC of the digest. - pub digest_rlc: Value, -} - -impl BatchDataRow { - fn padding_row() -> Self { - Self { - is_padding: true, - preimage_rlc: Value::known(Fr::zero()), - digest_rlc: Value::known(Fr::zero()), - ..Default::default() - } - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/aggregator/src/data_availability/eip4844/blob_data.rs b/aggregator/src/data_availability/eip4844/blob_data.rs index d56e78ce65..c4490ebd65 100644 --- a/aggregator/src/data_availability/eip4844/blob_data.rs +++ b/aggregator/src/data_availability/eip4844/blob_data.rs @@ -10,8 +10,11 @@ use itertools::Itertools; use zkevm_circuits::{table::U8Table, util::Challenges}; use crate::{ - aggregation::POWS_OF_256, - data_availability::eip4844::blob::{BLOB_WIDTH, N_BLOB_BYTES, N_DATA_BYTES_PER_COEFFICIENT}, + aggregation::{ + batch_data::{N_BLOB_BYTES, N_DATA_BYTES_PER_COEFFICIENT}, + POWS_OF_256, + }, + data_availability::eip4844::blob::BLOB_WIDTH, RlcConfig, }; diff --git a/aggregator/src/lib.rs b/aggregator/src/lib.rs index ed34c8c140..b4bdd55fd2 100644 --- a/aggregator/src/lib.rs +++ b/aggregator/src/lib.rs @@ -32,6 +32,5 @@ pub use chunk::ChunkInfo; pub use compression::*; pub use constants::MAX_AGG_SNARKS; pub(crate) use constants::*; -pub use data_availability::eip4844::blob::BatchData; pub use param::*; pub use recursion::*; diff --git a/aggregator/src/tests/blob.rs b/aggregator/src/tests/blob.rs index 8c77be7a73..a6e50add6f 100644 --- a/aggregator/src/tests/blob.rs +++ b/aggregator/src/tests/blob.rs @@ -18,12 +18,11 @@ use zkevm_circuits::{ use crate::{ aggregation::{ + batch_data::{N_BLOB_BYTES, N_BYTES_U256}, witgen::{process, MultiBlockProcessResult}, - RlcConfig, - }, - data_availability::eip4844::blob::{ - BatchData, PointEvaluationAssignments, N_BLOB_BYTES, N_BYTES_U256, + BatchData, RlcConfig, }, + data_availability::eip4844::blob::PointEvaluationAssignments, data_availability::eip4844::{ blob_data::BlobDataConfig, get_blob_bytes, get_coefficients, get_versioned_hash, }, From aa24c73360944936c5ee5ed28add487d3840273e Mon Sep 17 00:00:00 2001 From: z2trillion Date: Mon, 7 Oct 2024 21:20:34 -0400 Subject: [PATCH 06/35] export functions and structs only --- aggregator/src/aggregation/batch_data.rs | 11 +++---- aggregator/src/aggregation/circuit.rs | 7 +---- aggregator/src/aggregation/config.rs | 2 +- aggregator/src/batch.rs | 7 +++-- aggregator/src/constants.rs | 3 ++ aggregator/src/data_availability.rs | 13 +++++++- aggregator/src/data_availability/eip4844.rs | 31 ++++++++++++++++--- .../data_availability/eip4844/barycentric.rs | 2 +- .../src/data_availability/eip4844/blob.rs | 17 +--------- .../data_availability/eip4844/blob_data.rs | 7 ++--- .../eip4844/tests.rs} | 3 +- aggregator/src/tests.rs | 1 - aggregator/src/tests/aggregation.rs | 2 +- 13 files changed, 58 insertions(+), 48 deletions(-) rename aggregator/src/{tests/blob.rs => data_availability/eip4844/tests.rs} (99%) diff --git a/aggregator/src/aggregation/batch_data.rs b/aggregator/src/aggregation/batch_data.rs index 10ccb73d16..cd352a2fd6 100644 --- a/aggregator/src/aggregation/batch_data.rs +++ b/aggregator/src/aggregation/batch_data.rs @@ -1,3 +1,7 @@ +use crate::{ + aggregation::rlc::POWS_OF_256, constants::N_BYTES_U256, data_availability::BLOB_WIDTH, + BatchHash, ChunkInfo, RlcConfig, +}; use eth_types::{H256, U256}; use ethers_core::utils::keccak256; use halo2_ecc::bigint::CRTInteger; @@ -14,13 +18,6 @@ use zkevm_circuits::{ util::{Challenges, Expr}, }; -#[cfg(feature = "da-eip4844")] -use crate::data_availability::eip4844::blob::BLOB_WIDTH; -use crate::{aggregation::rlc::POWS_OF_256, BatchHash, ChunkInfo, RlcConfig}; - -/// The number of bytes to represent an unsigned 256 bit number. -pub const N_BYTES_U256: usize = 32; - /// The number data bytes we pack each BLS12-381 scalar into. The most-significant byte is 0. pub const N_DATA_BYTES_PER_COEFFICIENT: usize = 31; diff --git a/aggregator/src/aggregation/circuit.rs b/aggregator/src/aggregation/circuit.rs index f793025007..47963a88a7 100644 --- a/aggregator/src/aggregation/circuit.rs +++ b/aggregator/src/aggregation/circuit.rs @@ -27,7 +27,6 @@ use crate::{ batch::BatchHash, constants::{ACC_LEN, DIGEST_LEN}, core::{assign_batch_hashes, extract_proof_and_instances_with_pairing_check}, - data_availability::eip4844::AssignedBarycentricEvaluationConfig, util::parse_hash_digest_cells, witgen::{zstd_encode, MultiBlockProcessResult}, ConfigParams, LOG_DEGREE, PI_CHAIN_ID, PI_CURRENT_BATCH_HASH, PI_CURRENT_STATE_ROOT, @@ -188,11 +187,7 @@ impl Circuit for BatchCircuit { |region| { if first_pass { first_pass = false; - return Ok(( - vec![], - vec![], - AssignedBarycentricEvaluationConfig::default(), - )); + return Ok(Default::default()); } // stores accumulators for all snarks, including the padded ones diff --git a/aggregator/src/aggregation/config.rs b/aggregator/src/aggregation/config.rs index 83afad14d2..2c2b6c4aa6 100644 --- a/aggregator/src/aggregation/config.rs +++ b/aggregator/src/aggregation/config.rs @@ -18,7 +18,7 @@ use zkevm_circuits::{ use crate::{ constants::{BITS, LIMBS}, - data_availability::eip4844::{blob_data::BlobDataConfig, BarycentricEvaluationConfig}, + data_availability::{BarycentricEvaluationConfig, BlobDataConfig}, param::ConfigParams, BatchDataConfig, DecoderConfig, DecoderConfigArgs, RlcConfig, }; diff --git a/aggregator/src/batch.rs b/aggregator/src/batch.rs index 046c72211d..8269ad385a 100644 --- a/aggregator/src/batch.rs +++ b/aggregator/src/batch.rs @@ -9,8 +9,7 @@ use serde::{Deserialize, Serialize}; use crate::{ aggregation::BatchData, chunk::ChunkInfo, - data_availability::eip4844::blob::PointEvaluationAssignments, - data_availability::eip4844::{get_coefficients, get_versioned_hash}, + data_availability::{get_coefficients, get_versioned_hash, PointEvaluationAssignments}, }; /// Batch header provides additional fields from the context (within recursion) @@ -31,6 +30,8 @@ pub struct BatchHeader { pub last_block_timestamp: u64, /// The data hash of the batch pub data_hash: H256, + // /// information needed to check that the blobs in the circuit match the blob the data availibility provided. + // pub blob_consistency_witness: BlobConsistencyWitness, /// The versioned hash of the blob with this batch's data pub blob_versioned_hash: H256, /// The blob data proof: z (32), y (32) @@ -84,7 +85,7 @@ impl BatchHeader { let blob_versioned_hash = get_versioned_hash(&coeffs); let point_evaluation_assignments = PointEvaluationAssignments::new(&batch_data, blob_bytes, blob_versioned_hash); - + // Self { version, batch_index, diff --git a/aggregator/src/constants.rs b/aggregator/src/constants.rs index f7cdc73692..7d5d633172 100644 --- a/aggregator/src/constants.rs +++ b/aggregator/src/constants.rs @@ -85,3 +85,6 @@ pub(crate) const BITS: usize = 88; /// If the input size is less than this, dummy snarks /// will be padded. pub const MAX_AGG_SNARKS: usize = 45; + +// Number of bytes in a u256. +pub const N_BYTES_U256: usize = 32; diff --git a/aggregator/src/data_availability.rs b/aggregator/src/data_availability.rs index 1754231f5c..ada21fbea2 100644 --- a/aggregator/src/data_availability.rs +++ b/aggregator/src/data_availability.rs @@ -17,5 +17,16 @@ cfg_if! { #[cfg(feature = "da-avail")] mod avail; +#[cfg(feature = "da-avail")] +pub use eip4844::{ + get_blob_bytes, get_coefficients, get_versioned_hash, BarycentricEvaluationConfig, + BlobDataConfig, PointEvaluationAssignments, BLOB_WIDTH, +}; + +#[cfg(feature = "da-eip4844")] +mod eip4844; #[cfg(feature = "da-eip4844")] -pub mod eip4844; +pub use eip4844::{ + get_blob_bytes, get_coefficients, get_versioned_hash, BarycentricEvaluationConfig, + BlobDataConfig, PointEvaluationAssignments, BLOB_WIDTH, +}; diff --git a/aggregator/src/data_availability/eip4844.rs b/aggregator/src/data_availability/eip4844.rs index 93d811fed5..01a3cef5de 100644 --- a/aggregator/src/data_availability/eip4844.rs +++ b/aggregator/src/data_availability/eip4844.rs @@ -5,20 +5,30 @@ pub use barycentric::{ }; /// blob struct and constants -pub mod blob; +mod blob; +pub use blob::PointEvaluationAssignments; /// Config to constrain blob data (encoded batch data) pub mod blob_data; pub use blob_data::BlobDataConfig; -use crate::aggregation::batch_data::{N_BLOB_BYTES, N_BYTES_U256}; -use blob::{BLOB_WIDTH, KZG_TRUSTED_SETUP}; +#[cfg(test)] +mod tests; + +use crate::constants::N_BYTES_U256; use eth_types::{ToBigEndian, H256, U256}; use ethers_core::k256::sha2::{Digest, Sha256}; +use once_cell::sync::Lazy; use revm_primitives::VERSIONED_HASH_VERSION_KZG; +use std::sync::Arc; + +pub const BLOB_WIDTH: usize = 4096; +/// The number data bytes we pack each BLS12-381 scalar into. The most-significant byte is 0. +pub const N_DATA_BYTES_PER_COEFFICIENT: usize = 31; +pub const N_BLOB_BYTES: usize = BLOB_WIDTH * N_DATA_BYTES_PER_COEFFICIENT; /// Get the BLOB_WIDTH number of scalar field elements, as 32-bytes unsigned integers. -pub(crate) fn get_coefficients(blob_bytes: &[u8]) -> [U256; BLOB_WIDTH] { +pub fn get_coefficients(blob_bytes: &[u8]) -> [U256; BLOB_WIDTH] { let mut coefficients = [[0u8; N_BYTES_U256]; BLOB_WIDTH]; assert!( @@ -33,8 +43,19 @@ pub(crate) fn get_coefficients(blob_bytes: &[u8]) -> [U256; BLOB_WIDTH] { coefficients.map(|coeff| U256::from_big_endian(&coeff)) } +/// KZG trusted setup +static KZG_TRUSTED_SETUP: Lazy> = Lazy::new(|| { + Arc::new( + c_kzg::KzgSettings::load_trusted_setup( + &revm_primitives::kzg::G1_POINTS.0, + &revm_primitives::kzg::G2_POINTS.0, + ) + .expect("failed to load trusted setup"), + ) +}); + /// Get the versioned hash as per EIP-4844. -pub(crate) fn get_versioned_hash(coefficients: &[U256; BLOB_WIDTH]) -> H256 { +pub fn get_versioned_hash(coefficients: &[U256; BLOB_WIDTH]) -> H256 { let blob = c_kzg::Blob::from_bytes( &coefficients .iter() diff --git a/aggregator/src/data_availability/eip4844/barycentric.rs b/aggregator/src/data_availability/eip4844/barycentric.rs index e0e63ba64c..262f1f0647 100644 --- a/aggregator/src/data_availability/eip4844/barycentric.rs +++ b/aggregator/src/data_availability/eip4844/barycentric.rs @@ -357,7 +357,7 @@ mod tests { use super::*; use crate::{ aggregation::BatchData, - data_availability::eip4844::blob::KZG_TRUSTED_SETUP, + data_availability::eip4844::KZG_TRUSTED_SETUP, data_availability::eip4844::{get_blob_bytes, get_coefficients}, MAX_AGG_SNARKS, }; diff --git a/aggregator/src/data_availability/eip4844/blob.rs b/aggregator/src/data_availability/eip4844/blob.rs index 0417d30dc0..3558421c87 100644 --- a/aggregator/src/data_availability/eip4844/blob.rs +++ b/aggregator/src/data_availability/eip4844/blob.rs @@ -8,28 +8,13 @@ use halo2_proofs::{ halo2curves::{bls12_381::Scalar, bn256::Fr}, }; use itertools::Itertools; -use once_cell::sync::Lazy; -use std::{ - iter::{once, repeat}, - sync::Arc, -}; +use std::iter::{once, repeat}; use zkevm_circuits::util::Challenges; /// The number of coefficients (BLS12-381 scalars) to represent the blob polynomial in evaluation /// form. pub const BLOB_WIDTH: usize = 4096; -/// KZG trusted setup -pub static KZG_TRUSTED_SETUP: Lazy> = Lazy::new(|| { - Arc::new( - c_kzg::KzgSettings::load_trusted_setup( - &revm_primitives::kzg::G1_POINTS.0, - &revm_primitives::kzg::G2_POINTS.0, - ) - .expect("failed to load trusted setup"), - ) -}); - #[derive(Clone, Debug)] pub struct PointEvaluationAssignments { /// The random challenge scalar z. diff --git a/aggregator/src/data_availability/eip4844/blob_data.rs b/aggregator/src/data_availability/eip4844/blob_data.rs index c4490ebd65..25a0f3de21 100644 --- a/aggregator/src/data_availability/eip4844/blob_data.rs +++ b/aggregator/src/data_availability/eip4844/blob_data.rs @@ -10,11 +10,8 @@ use itertools::Itertools; use zkevm_circuits::{table::U8Table, util::Challenges}; use crate::{ - aggregation::{ - batch_data::{N_BLOB_BYTES, N_DATA_BYTES_PER_COEFFICIENT}, - POWS_OF_256, - }, - data_availability::eip4844::blob::BLOB_WIDTH, + aggregation::{batch_data::N_BLOB_BYTES, POWS_OF_256}, + data_availability::eip4844::{blob::BLOB_WIDTH, N_DATA_BYTES_PER_COEFFICIENT}, RlcConfig, }; diff --git a/aggregator/src/tests/blob.rs b/aggregator/src/data_availability/eip4844/tests.rs similarity index 99% rename from aggregator/src/tests/blob.rs rename to aggregator/src/data_availability/eip4844/tests.rs index a6e50add6f..3c7dce5bf0 100644 --- a/aggregator/src/tests/blob.rs +++ b/aggregator/src/data_availability/eip4844/tests.rs @@ -18,10 +18,11 @@ use zkevm_circuits::{ use crate::{ aggregation::{ - batch_data::{N_BLOB_BYTES, N_BYTES_U256}, + batch_data::N_BLOB_BYTES, witgen::{process, MultiBlockProcessResult}, BatchData, RlcConfig, }, + constants::N_BYTES_U256, data_availability::eip4844::blob::PointEvaluationAssignments, data_availability::eip4844::{ blob_data::BlobDataConfig, get_blob_bytes, get_coefficients, get_versioned_hash, diff --git a/aggregator/src/tests.rs b/aggregator/src/tests.rs index deeb6f0ab0..8c98351b61 100644 --- a/aggregator/src/tests.rs +++ b/aggregator/src/tests.rs @@ -1,5 +1,4 @@ mod aggregation; -mod blob; mod compression; mod mock_chunk; mod recursion; diff --git a/aggregator/src/tests/aggregation.rs b/aggregator/src/tests/aggregation.rs index 5426a863a2..354ba5321e 100644 --- a/aggregator/src/tests/aggregation.rs +++ b/aggregator/src/tests/aggregation.rs @@ -10,7 +10,7 @@ use crate::{ aggregation::BatchCircuit, batch::{BatchHash, BatchHeader}, constants::MAX_AGG_SNARKS, - data_availability::eip4844::get_blob_bytes, + data_availability::get_blob_bytes, layer_0, tests::mock_chunk::MockChunkCircuit, BatchData, ChunkInfo, From 95f812cd063f6537aad627b35f9e5e9c455678c3 Mon Sep 17 00:00:00 2001 From: z2trillion Date: Mon, 7 Oct 2024 22:19:10 -0400 Subject: [PATCH 07/35] move get_bytes to decoder --- aggregator/src/aggregation.rs | 2 +- aggregator/src/aggregation/decoder.rs | 14 ++++++++++++++ aggregator/src/data_availability.rs | 8 ++++---- aggregator/src/tests/aggregation.rs | 6 +++--- 4 files changed, 22 insertions(+), 8 deletions(-) diff --git a/aggregator/src/aggregation.rs b/aggregator/src/aggregation.rs index ba527a43e9..bb067bfb0c 100644 --- a/aggregator/src/aggregation.rs +++ b/aggregator/src/aggregation.rs @@ -13,7 +13,7 @@ mod util; pub use batch_data::BatchData; pub(crate) use batch_data::BatchDataConfig; -pub use decoder::decode_bytes; +pub use decoder::{decode_bytes, encode_bytes}; pub(crate) use decoder::{witgen, DecoderConfig, DecoderConfigArgs}; pub(crate) use rlc::{RlcConfig, POWS_OF_256}; diff --git a/aggregator/src/aggregation/decoder.rs b/aggregator/src/aggregation/decoder.rs index de2f32ace9..e1b3b7f80e 100644 --- a/aggregator/src/aggregation/decoder.rs +++ b/aggregator/src/aggregation/decoder.rs @@ -5827,3 +5827,17 @@ pub fn decode_bytes(bytes: &[u8]) -> std::io::Result> { Ok(decoded_bytes) } + +/// The inverse of decode_bytes. +pub fn encode_bytes(batch_bytes: &[u8]) -> Vec { + let mut blob_bytes = crate::witgen::zstd_encode(batch_bytes); + + // Whether we encode batch -> blob or not. + let enable_encoding = blob_bytes.len() < batch_bytes.len(); + if !enable_encoding { + blob_bytes = batch_bytes.to_vec(); + } + blob_bytes.insert(0, enable_encoding as u8); + + blob_bytes +} diff --git a/aggregator/src/data_availability.rs b/aggregator/src/data_availability.rs index ada21fbea2..bb0347584c 100644 --- a/aggregator/src/data_availability.rs +++ b/aggregator/src/data_availability.rs @@ -19,14 +19,14 @@ cfg_if! { mod avail; #[cfg(feature = "da-avail")] pub use eip4844::{ - get_blob_bytes, get_coefficients, get_versioned_hash, BarycentricEvaluationConfig, - BlobDataConfig, PointEvaluationAssignments, BLOB_WIDTH, + get_coefficients, get_versioned_hash, BarycentricEvaluationConfig, BlobDataConfig, + PointEvaluationAssignments, BLOB_WIDTH, }; #[cfg(feature = "da-eip4844")] mod eip4844; #[cfg(feature = "da-eip4844")] pub use eip4844::{ - get_blob_bytes, get_coefficients, get_versioned_hash, BarycentricEvaluationConfig, - BlobDataConfig, PointEvaluationAssignments, BLOB_WIDTH, + get_coefficients, get_versioned_hash, BarycentricEvaluationConfig, BlobDataConfig, + PointEvaluationAssignments, BLOB_WIDTH, }; diff --git a/aggregator/src/tests/aggregation.rs b/aggregator/src/tests/aggregation.rs index 354ba5321e..ec374c420b 100644 --- a/aggregator/src/tests/aggregation.rs +++ b/aggregator/src/tests/aggregation.rs @@ -7,10 +7,10 @@ use snark_verifier::loader::halo2::halo2_ecc::halo2_base::utils::fs::gen_srs; use snark_verifier_sdk::{gen_pk, gen_snark_shplonk, verify_snark_shplonk, CircuitExt}; use crate::{ + aggregation::encode_bytes, aggregation::BatchCircuit, batch::{BatchHash, BatchHeader}, constants::MAX_AGG_SNARKS, - data_availability::get_blob_bytes, layer_0, tests::mock_chunk::MockChunkCircuit, BatchData, ChunkInfo, @@ -180,7 +180,7 @@ fn build_new_batch_circuit( .concat(); let batch_data = BatchData::::new(num_real_chunks, &chunks_with_padding); let batch_bytes = batch_data.get_batch_data_bytes(); - let blob_bytes = get_blob_bytes(&batch_bytes); + let blob_bytes = encode_bytes(&batch_bytes); let batch_header = BatchHeader::construct_from_chunks( batch_proving_task.batch_header.version, batch_proving_task.batch_header.batch_index, @@ -261,7 +261,7 @@ fn build_batch_circuit_skip_encoding() -> BatchCircuit::new(num_chunks, &chunks_with_padding); let batch_bytes = batch_data.get_batch_data_bytes(); - let blob_bytes = get_blob_bytes(&batch_bytes); + let blob_bytes = encode_bytes(&batch_bytes); let corrected_batch_header = BatchHeader::construct_from_chunks( batch_proving_task.batch_header.version, batch_proving_task.batch_header.batch_index, From 62e8f2c0360b209d0d34fbd95228f6997b0f0702 Mon Sep 17 00:00:00 2001 From: z2trillion Date: Tue, 8 Oct 2024 00:19:32 -0400 Subject: [PATCH 08/35] Add BlobConsistencyWitness and BlobConsistencyConfig --- aggregator/src/aggregation/circuit.rs | 26 ++-- aggregator/src/aggregation/config.rs | 23 ++-- aggregator/src/batch.rs | 141 ++++++++++---------- aggregator/src/data_availability.rs | 6 +- aggregator/src/data_availability/eip4844.rs | 83 ++++++++++++ prover/src/aggregator/prover.rs | 18 ++- 6 files changed, 200 insertions(+), 97 deletions(-) diff --git a/aggregator/src/aggregation/circuit.rs b/aggregator/src/aggregation/circuit.rs index 47963a88a7..192979f12a 100644 --- a/aggregator/src/aggregation/circuit.rs +++ b/aggregator/src/aggregation/circuit.rs @@ -27,10 +27,17 @@ use crate::{ batch::BatchHash, constants::{ACC_LEN, DIGEST_LEN}, core::{assign_batch_hashes, extract_proof_and_instances_with_pairing_check}, + data_availability::get_coefficients, // TODO: fix this util::parse_hash_digest_cells, witgen::{zstd_encode, MultiBlockProcessResult}, - ConfigParams, LOG_DEGREE, PI_CHAIN_ID, PI_CURRENT_BATCH_HASH, PI_CURRENT_STATE_ROOT, - PI_CURRENT_WITHDRAW_ROOT, PI_PARENT_BATCH_HASH, PI_PARENT_STATE_ROOT, + ConfigParams, + LOG_DEGREE, + PI_CHAIN_ID, + PI_CURRENT_BATCH_HASH, + PI_CURRENT_STATE_ROOT, + PI_CURRENT_WITHDRAW_ROOT, + PI_PARENT_BATCH_HASH, + PI_PARENT_STATE_ROOT, }; /// Batch circuit, the chunk aggregation routine below recursion circuit @@ -244,12 +251,13 @@ impl Circuit for BatchCircuit { let mut ctx = Rc::into_inner(loader).unwrap().into_ctx(); log::debug!("batching: assigning barycentric"); - let barycentric = config.barycentric.assign( + let coefficients = get_coefficients(&self.batch_hash.blob_bytes); + let barycentric = config.blob_consistency_config.assign_barycentric( &mut ctx, - &self.batch_hash.point_evaluation_assignments.coefficients, - self.batch_hash - .point_evaluation_assignments - .challenge_digest, + &coefficients, + // todo: check that the byte order is correct!!!! + self.batch_hash.blob_consistency_witness.challenge.0.into(), + self.batch_hash.blob_consistency_witness.evaluation.0.into(), ); ctx.print_stats(&["barycentric"]); @@ -401,7 +409,7 @@ impl Circuit for BatchCircuit { let batch_data = BatchData::from(&self.batch_hash); - let blob_data_exports = config.blob_data_config.assign( + let blob_data_exports = config.blob_consistency_config.assign_blob_data( &mut layouter, challenges, &config.rlc_config, @@ -415,7 +423,7 @@ impl Circuit for BatchCircuit { &config.rlc_config, &assigned_batch_hash.chunks_are_padding, &batch_data, - self.batch_hash.versioned_hash, + self.batch_hash.blob_consistency_witness.id, barycentric_assignments, )?; diff --git a/aggregator/src/aggregation/config.rs b/aggregator/src/aggregation/config.rs index 2c2b6c4aa6..6e6052c5c0 100644 --- a/aggregator/src/aggregation/config.rs +++ b/aggregator/src/aggregation/config.rs @@ -18,7 +18,7 @@ use zkevm_circuits::{ use crate::{ constants::{BITS, LIMBS}, - data_availability::{BarycentricEvaluationConfig, BlobDataConfig}, + data_availability::BlobConsistencyConfig, param::ConfigParams, BatchDataConfig, DecoderConfig, DecoderConfigArgs, RlcConfig, }; @@ -34,14 +34,12 @@ pub struct BatchCircuitConfig { pub keccak_circuit_config: KeccakCircuitConfig, /// RLC config pub rlc_config: RlcConfig, - /// The blob data's config. - pub blob_data_config: BlobDataConfig, /// The batch data's config. pub batch_data_config: BatchDataConfig, /// The zstd decoder's config. pub decoder_config: DecoderConfig<1024, 512>, - /// Config to do the barycentric evaluation on blob polynomial. - pub barycentric: BarycentricEvaluationConfig, + /// Config check witness blob matches commitment to blob obtained from data availability provider. + pub blob_consistency_config: BlobConsistencyConfig, /// Instance for public input; stores /// - accumulator from aggregation (12 elements) /// - chain id (1 element) @@ -99,9 +97,6 @@ impl BatchCircuitConfig { params.degree as usize, ); - // Barycentric. - let barycentric = BarycentricEvaluationConfig::construct(base_field_config.range.clone()); - let columns = keccak_circuit_config.cell_manager.columns(); log::info!("keccak uses {} columns", columns.len(),); @@ -120,7 +115,6 @@ impl BatchCircuitConfig { let u8_table = U8Table::construct(meta); let range_table = RangeTable::construct(meta); let challenges_expr = challenges.exprs(meta); - let blob_data_config = BlobDataConfig::configure(meta, &challenges_expr, u8_table); let batch_data_config = BatchDataConfig::configure( meta, &challenges_expr, @@ -132,6 +126,14 @@ impl BatchCircuitConfig { // Zstd decoder. let pow_rand_table = PowOfRandTable::construct(meta, &challenges_expr); + // Blob consistency. + let blob_consistency_config = BlobConsistencyConfig::construct( + meta, + &challenges_expr, + u8_table, + base_field_config.range.clone(), + ); + let pow2_table = Pow2Table::construct(meta); let range8 = RangeTable::construct(meta); let range16 = RangeTable::construct(meta); @@ -172,12 +174,11 @@ impl BatchCircuitConfig { Self { base_field_config, rlc_config, - blob_data_config, keccak_circuit_config, instance, - barycentric, batch_data_config, decoder_config, + blob_consistency_config, } } diff --git a/aggregator/src/batch.rs b/aggregator/src/batch.rs index 8269ad385a..5e4379487e 100644 --- a/aggregator/src/batch.rs +++ b/aggregator/src/batch.rs @@ -6,11 +6,7 @@ use ethers_core::utils::keccak256; use gadgets::{util::split_h256, Field}; use serde::{Deserialize, Serialize}; -use crate::{ - aggregation::BatchData, - chunk::ChunkInfo, - data_availability::{get_coefficients, get_versioned_hash, PointEvaluationAssignments}, -}; +use crate::{aggregation::BatchData, chunk::ChunkInfo, data_availability::BlobConsistencyWitness}; /// Batch header provides additional fields from the context (within recursion) /// for constructing the preimage of the batch hash. @@ -31,11 +27,8 @@ pub struct BatchHeader { /// The data hash of the batch pub data_hash: H256, // /// information needed to check that the blobs in the circuit match the blob the data availibility provided. - // pub blob_consistency_witness: BlobConsistencyWitness, - /// The versioned hash of the blob with this batch's data - pub blob_versioned_hash: H256, - /// The blob data proof: z (32), y (32) - pub blob_data_proof: [H256; 2], + // TODO: #[serde(flatten)] etc. for backwards compatibility? + pub blob_consistency_witness: BlobConsistencyWitness, } impl BatchHeader { @@ -81,11 +74,12 @@ impl BatchHeader { let batch_data_hash = keccak256(batch_data_hash_preimage); let batch_data = BatchData::::new(number_of_valid_chunks, &chunks_with_padding); - let coeffs = get_coefficients(blob_bytes); - let blob_versioned_hash = get_versioned_hash(&coeffs); - let point_evaluation_assignments = - PointEvaluationAssignments::new(&batch_data, blob_bytes, blob_versioned_hash); - // + // let coeffs = get_coefficients(blob_bytes); + // let blob_versioned_hash = get_versioned_hash(&coeffs); + // let point_evaluation_assignments = + // PointEvaluationAssignments::new(&batch_data, blob_bytes, blob_versioned_hash); + // // + let blob_consistency_witness = BlobConsistencyWitness::new(blob_bytes, &batch_data); Self { version, batch_index, @@ -94,11 +88,12 @@ impl BatchHeader { parent_batch_hash, last_block_timestamp, data_hash: batch_data_hash.into(), - blob_versioned_hash, - blob_data_proof: [ - H256::from_slice(&point_evaluation_assignments.challenge.to_be_bytes()), - H256::from_slice(&point_evaluation_assignments.evaluation.to_be_bytes()), - ], + blob_consistency_witness, + // blob_versioned_hash, + // blob_data_proof: [ + // H256::from_slice(&point_evaluation_assignments.challenge.to_be_bytes()), + // H256::from_slice(&point_evaluation_assignments.evaluation.to_be_bytes()), + // ], } } @@ -123,11 +118,17 @@ impl BatchHeader { self.l1_message_popped.to_be_bytes().as_ref(), self.total_l1_message_popped.to_be_bytes().as_ref(), self.data_hash.as_bytes(), - self.blob_versioned_hash.as_bytes(), + self.blob_consistency_witness.id.as_bytes(), self.parent_batch_hash.as_bytes(), self.last_block_timestamp.to_be_bytes().as_ref(), - self.blob_data_proof[0].to_fixed_bytes().as_ref(), - self.blob_data_proof[1].to_fixed_bytes().as_ref(), + self.blob_consistency_witness + .challenge + .to_fixed_bytes() + .as_ref(), + self.blob_consistency_witness + .evaluation + .to_fixed_bytes() + .as_ref(), ] .concat(); keccak256(batch_hash_preimage).into() @@ -165,14 +166,16 @@ pub struct BatchHash { pub(crate) current_batch_hash: H256, /// The number of chunks that contain meaningful data, i.e. not padded chunks. pub(crate) number_of_valid_chunks: usize, - /// 4844 point evaluation check related assignments. - pub(crate) point_evaluation_assignments: PointEvaluationAssignments, - /// The 4844 versioned hash for the blob. - pub(crate) versioned_hash: H256, + // /// 4844 point evaluation check related assignments. + // pub(crate) point_evaluation_assignments: PointEvaluationAssignments, + // /// The 4844 versioned hash for the blob. + // pub(crate) versioned_hash: H256, /// The context batch header pub(crate) batch_header: BatchHeader, /// The blob bytes (may be encoded batch bytes, or may be raw batch bytes). pub(crate) blob_bytes: Vec, + + pub blob_consistency_witness: BlobConsistencyWitness, } impl BatchHash { @@ -287,37 +290,40 @@ impl BatchHash { ); let batch_data = BatchData::::new(number_of_valid_chunks, chunks_with_padding); - let coeffs = get_coefficients(blob_bytes); - let versioned_hash = get_versioned_hash(&coeffs); - let point_evaluation_assignments = - PointEvaluationAssignments::new(&batch_data, blob_bytes, versioned_hash); - - assert_eq!( - batch_header.blob_data_proof[0], - H256::from_slice(&point_evaluation_assignments.challenge.to_be_bytes()), - "Expect provided BatchHeader's blob_data_proof field 0 to be correct" - ); - assert_eq!( - batch_header.blob_data_proof[1], - H256::from_slice(&point_evaluation_assignments.evaluation.to_be_bytes()), - "Expect provided BatchHeader's blob_data_proof field 1 to be correct" - ); - - assert_eq!( - batch_header.blob_versioned_hash, versioned_hash, - "Expect provided BatchHeader's blob_versioned_hash field to be correct" - ); + // let coeffs = get_coefficients(blob_bytes); + // let versioned_hash = get_versioned_hash(&coeffs); + + // TODO: moved these checks down. + // let point_evaluation_assignments = + // PointEvaluationAssignments::new(&batch_data, blob_bytes, versioned_hash); + + // assert_eq!( + // batch_header.blob_consistency_witness.challenge, + // H256::from_slice(&point_evaluation_assignments.challenge.to_be_bytes()), + // "Expect provided BatchHeader's blob_data_proof field 0 to be correct" + // ); + // assert_eq!( + // batch_header.blob_consistency_witness.evaluation, + // H256::from_slice(&point_evaluation_assignments.evaluation.to_be_bytes()), + // "Expect provided BatchHeader's blob_data_proof field 1 to be correct" + // ); + + // assert_eq!( + // batch_header.blob_consistency_witness.id, versioned_hash, + // "Expect provided BatchHeader's blob_versioned_hash field to be correct" + // ); let current_batch_hash = batch_header.batch_hash(); - log::info!( - "batch hash {:?}, datahash {}, z {}, y {}, versioned hash {:x}", - current_batch_hash, - hex::encode(batch_data_hash), - hex::encode(point_evaluation_assignments.challenge.to_be_bytes()), - hex::encode(point_evaluation_assignments.evaluation.to_be_bytes()), - versioned_hash, - ); + // log::info!( + // "batch hash {:?}, datahash {}, z {}, y {}, versioned hash {:x}", + // current_batch_hash, + // hex::encode(batch_data_hash), + // hex::encode(point_evaluation_assignments.challenge.to_be_bytes()), + // hex::encode(point_evaluation_assignments.evaluation.to_be_bytes()), + // versioned_hash, + // ); + let blob_consistency_witness = BlobConsistencyWitness::new(&blob_bytes, &batch_data); Self { chain_id: chunks_with_padding[0].chain_id, @@ -328,17 +334,16 @@ impl BatchHash { data_hash: batch_data_hash.into(), current_batch_hash, number_of_valid_chunks, - point_evaluation_assignments, - versioned_hash, batch_header, blob_bytes: blob_bytes.to_vec(), + blob_consistency_witness, } } - /// Return the blob polynomial and its evaluation at challenge - pub fn point_evaluation_assignments(&self) -> PointEvaluationAssignments { - self.point_evaluation_assignments.clone() - } + // /// Return the blob polynomial and its evaluation at challenge + // pub fn point_evaluation_assignments(&self) -> PointEvaluationAssignments { + // self.point_evaluation_assignments.clone() + // } /// Extract all the hash inputs that will ever be used. /// There are N_SNARKS + 2 hashes. @@ -375,19 +380,21 @@ impl BatchHash { .to_be_bytes() .as_ref(), self.data_hash.as_bytes(), - self.versioned_hash.as_bytes(), + self.batch_header.blob_consistency_witness.id.as_bytes(), self.batch_header.parent_batch_hash.as_bytes(), self.batch_header .last_block_timestamp .to_be_bytes() .as_ref(), - self.point_evaluation_assignments + self.batch_header + .blob_consistency_witness .challenge - .to_be_bytes() + .to_fixed_bytes() .as_ref(), - self.point_evaluation_assignments + self.batch_header + .blob_consistency_witness .evaluation - .to_be_bytes() + .to_fixed_bytes() .as_ref(), ] .concat(); @@ -425,7 +432,7 @@ impl BatchHash { // - preimage for each chunk's flattened L2 signed tx data // - preimage for the challenge digest let batch_data = BatchData::from(self); - let dynamic_preimages = batch_data.preimages(self.versioned_hash); + let dynamic_preimages = batch_data.preimages(self.batch_header.blob_consistency_witness.id); for dynamic_preimage in dynamic_preimages { res.push(dynamic_preimage); } diff --git a/aggregator/src/data_availability.rs b/aggregator/src/data_availability.rs index bb0347584c..7e5049ebd5 100644 --- a/aggregator/src/data_availability.rs +++ b/aggregator/src/data_availability.rs @@ -19,14 +19,12 @@ cfg_if! { mod avail; #[cfg(feature = "da-avail")] pub use eip4844::{ - get_coefficients, get_versioned_hash, BarycentricEvaluationConfig, BlobDataConfig, - PointEvaluationAssignments, BLOB_WIDTH, + get_coefficients, get_versioned_hash, BlobConsistencyConfig, BlobConsistencyWitness, BLOB_WIDTH, }; #[cfg(feature = "da-eip4844")] mod eip4844; #[cfg(feature = "da-eip4844")] pub use eip4844::{ - get_coefficients, get_versioned_hash, BarycentricEvaluationConfig, BlobDataConfig, - PointEvaluationAssignments, BLOB_WIDTH, + get_coefficients, get_versioned_hash, BlobConsistencyConfig, BlobConsistencyWitness, BLOB_WIDTH, }; diff --git a/aggregator/src/data_availability/eip4844.rs b/aggregator/src/data_availability/eip4844.rs index 01a3cef5de..40a8729b78 100644 --- a/aggregator/src/data_availability/eip4844.rs +++ b/aggregator/src/data_availability/eip4844.rs @@ -16,11 +16,24 @@ pub use blob_data::BlobDataConfig; mod tests; use crate::constants::N_BYTES_U256; +use crate::BatchData; +use crate::RlcConfig; +use blob_data::AssignedBlobDataExport; use eth_types::{ToBigEndian, H256, U256}; use ethers_core::k256::sha2::{Digest, Sha256}; +use halo2_base::gates::range::RangeConfig; +use halo2_base::Context; +use halo2_ecc::bigint::CRTInteger; +use halo2_proofs::circuit::Layouter; +use halo2_proofs::circuit::Value; +use halo2_proofs::halo2curves::bn256::Fr; +use halo2_proofs::plonk::{ConstraintSystem, Error, Expression}; use once_cell::sync::Lazy; use revm_primitives::VERSIONED_HASH_VERSION_KZG; +use serde::{Deserialize, Serialize}; use std::sync::Arc; +use zkevm_circuits::table::U8Table; +use zkevm_circuits::util::Challenges; pub const BLOB_WIDTH: usize = 4096; /// The number data bytes we pack each BLS12-381 scalar into. The most-significant byte is 0. @@ -88,3 +101,73 @@ pub fn get_blob_bytes(batch_bytes: &[u8]) -> Vec { blob_bytes } + +#[derive(Debug, Clone)] +pub struct BlobConsistencyConfig { + data: BlobDataConfig, + barycentric_evaluation: BarycentricEvaluationConfig, +} + +impl BlobConsistencyConfig { + pub fn construct( + meta: &mut ConstraintSystem, + challenges: &Challenges>, + u8_table: U8Table, + range: RangeConfig, + ) -> Self { + Self { + data: BlobDataConfig::configure(meta, challenges, u8_table), + barycentric_evaluation: BarycentricEvaluationConfig::construct(range), + } + } + + pub fn assign_barycentric( + &self, + ctx: &mut Context, + blob: &[U256; BLOB_WIDTH], + challenge: U256, + evaluation: U256, + ) -> AssignedBarycentricEvaluationConfig { + self.barycentric_evaluation + .assign(ctx, blob, challenge, evaluation) + } + + pub fn assign_blob_data( + &self, + layouter: &mut impl Layouter, + challenge_value: Challenges>, + rlc_config: &RlcConfig, + blob_bytes: &[u8], + barycentric_assignments: &[CRTInteger], + ) -> Result { + self.data.assign( + layouter, + challenge_value, + rlc_config, + blob_bytes, + barycentric_assignments, + ) + } +} + +#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)] +pub struct BlobConsistencyWitness { + pub id: H256, + pub challenge: H256, + pub evaluation: H256, +} + +impl BlobConsistencyWitness { + pub fn new(bytes: &[u8], batch_data: &BatchData) -> Self { + let coeffs = get_coefficients(bytes); + let versioned_hash = get_versioned_hash(&coeffs); + let point_evaluation_assignments = + PointEvaluationAssignments::new(&batch_data, bytes, versioned_hash); + + Self { + id: versioned_hash, + challenge: H256::from_slice(&point_evaluation_assignments.challenge.to_be_bytes()), + evaluation: H256::from_slice(&point_evaluation_assignments.evaluation.to_be_bytes()), + } + } +} diff --git a/prover/src/aggregator/prover.rs b/prover/src/aggregator/prover.rs index 3d52b07f91..da680b62d8 100644 --- a/prover/src/aggregator/prover.rs +++ b/prover/src/aggregator/prover.rs @@ -183,19 +183,25 @@ impl<'params> Prover<'params> { batch.batch_header.data_hash, batch_header.data_hash ); assert_eq!( - batch_header.blob_data_proof[0], batch.batch_header.blob_data_proof[0], + batch_header.blob_consistency_witness.challenge, + batch.batch_header.blob_consistency_witness.challenge, "BatchHeader(sanity) mismatch blob data proof (z) expected={}, got={}", - batch.batch_header.blob_data_proof[0], batch_header.blob_data_proof[0], + batch_header.blob_consistency_witness.challenge, + batch.batch_header.blob_consistency_witness.challenge, ); assert_eq!( - batch_header.blob_data_proof[1], batch.batch_header.blob_data_proof[1], + batch_header.blob_consistency_witness.evaluation, + batch.batch_header.blob_consistency_witness.evaluation, "BatchHeader(sanity) mismatch blob data proof (y) expected={}, got={}", - batch.batch_header.blob_data_proof[1], batch_header.blob_data_proof[1], + batch_header.blob_consistency_witness.evaluation, + batch.batch_header.blob_consistency_witness.evaluation, ); assert_eq!( - batch_header.blob_versioned_hash, batch.batch_header.blob_versioned_hash, + batch_header.blob_consistency_witness.id, + batch.batch_header.blob_consistency_witness.id, "BatchHeader(sanity) mismatch blob versioned hash expected={}, got={}", - batch.batch_header.blob_versioned_hash, batch_header.blob_versioned_hash, + batch_header.blob_consistency_witness.id, + batch.batch_header.blob_consistency_witness.id, ); let batch_hash = batch_header.batch_hash(); From b7d94103a288d1d0f5edee38782a580d7406c878 Mon Sep 17 00:00:00 2001 From: z2trillion Date: Tue, 8 Oct 2024 00:25:45 -0400 Subject: [PATCH 09/35] don't export get_versioned_hash --- aggregator/src/data_availability.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aggregator/src/data_availability.rs b/aggregator/src/data_availability.rs index 7e5049ebd5..dc6160953f 100644 --- a/aggregator/src/data_availability.rs +++ b/aggregator/src/data_availability.rs @@ -19,12 +19,12 @@ cfg_if! { mod avail; #[cfg(feature = "da-avail")] pub use eip4844::{ - get_coefficients, get_versioned_hash, BlobConsistencyConfig, BlobConsistencyWitness, BLOB_WIDTH, + get_coefficients, BlobConsistencyConfig, BlobConsistencyWitness, BLOB_WIDTH, }; #[cfg(feature = "da-eip4844")] mod eip4844; #[cfg(feature = "da-eip4844")] pub use eip4844::{ - get_coefficients, get_versioned_hash, BlobConsistencyConfig, BlobConsistencyWitness, BLOB_WIDTH, + get_coefficients, BlobConsistencyConfig, BlobConsistencyWitness, BLOB_WIDTH, }; From e62240550c31e92b78b44cb4da6c5116c440d664 Mon Sep 17 00:00:00 2001 From: z2trillion Date: Tue, 8 Oct 2024 00:29:47 -0400 Subject: [PATCH 10/35] cleanup --- aggregator/src/batch.rs | 51 +++++------------------------ aggregator/src/data_availability.rs | 8 ++--- 2 files changed, 11 insertions(+), 48 deletions(-) diff --git a/aggregator/src/batch.rs b/aggregator/src/batch.rs index 5e4379487e..52f054ba87 100644 --- a/aggregator/src/batch.rs +++ b/aggregator/src/batch.rs @@ -74,11 +74,6 @@ impl BatchHeader { let batch_data_hash = keccak256(batch_data_hash_preimage); let batch_data = BatchData::::new(number_of_valid_chunks, &chunks_with_padding); - // let coeffs = get_coefficients(blob_bytes); - // let blob_versioned_hash = get_versioned_hash(&coeffs); - // let point_evaluation_assignments = - // PointEvaluationAssignments::new(&batch_data, blob_bytes, blob_versioned_hash); - // // let blob_consistency_witness = BlobConsistencyWitness::new(blob_bytes, &batch_data); Self { version, @@ -290,41 +285,18 @@ impl BatchHash { ); let batch_data = BatchData::::new(number_of_valid_chunks, chunks_with_padding); - // let coeffs = get_coefficients(blob_bytes); - // let versioned_hash = get_versioned_hash(&coeffs); - - // TODO: moved these checks down. - // let point_evaluation_assignments = - // PointEvaluationAssignments::new(&batch_data, blob_bytes, versioned_hash); - - // assert_eq!( - // batch_header.blob_consistency_witness.challenge, - // H256::from_slice(&point_evaluation_assignments.challenge.to_be_bytes()), - // "Expect provided BatchHeader's blob_data_proof field 0 to be correct" - // ); - // assert_eq!( - // batch_header.blob_consistency_witness.evaluation, - // H256::from_slice(&point_evaluation_assignments.evaluation.to_be_bytes()), - // "Expect provided BatchHeader's blob_data_proof field 1 to be correct" - // ); - - // assert_eq!( - // batch_header.blob_consistency_witness.id, versioned_hash, - // "Expect provided BatchHeader's blob_versioned_hash field to be correct" - // ); - let current_batch_hash = batch_header.batch_hash(); - - // log::info!( - // "batch hash {:?}, datahash {}, z {}, y {}, versioned hash {:x}", - // current_batch_hash, - // hex::encode(batch_data_hash), - // hex::encode(point_evaluation_assignments.challenge.to_be_bytes()), - // hex::encode(point_evaluation_assignments.evaluation.to_be_bytes()), - // versioned_hash, - // ); let blob_consistency_witness = BlobConsistencyWitness::new(&blob_bytes, &batch_data); + log::info!( + "batch hash {:?}, datahash {}, z {}, y {}, versioned hash {:x}", + current_batch_hash, + hex::encode(batch_data_hash), + hex::encode(blob_consistency_witness.challenge.to_fixed_bytes()), + hex::encode(blob_consistency_witness.evaluation.to_fixed_bytes()), + blob_consistency_witness.id, + ); + Self { chain_id: chunks_with_padding[0].chain_id, chunks_with_padding: chunks_with_padding.to_vec(), @@ -340,11 +312,6 @@ impl BatchHash { } } - // /// Return the blob polynomial and its evaluation at challenge - // pub fn point_evaluation_assignments(&self) -> PointEvaluationAssignments { - // self.point_evaluation_assignments.clone() - // } - /// Extract all the hash inputs that will ever be used. /// There are N_SNARKS + 2 hashes. /// diff --git a/aggregator/src/data_availability.rs b/aggregator/src/data_availability.rs index dc6160953f..8c55e0c69a 100644 --- a/aggregator/src/data_availability.rs +++ b/aggregator/src/data_availability.rs @@ -18,13 +18,9 @@ cfg_if! { #[cfg(feature = "da-avail")] mod avail; #[cfg(feature = "da-avail")] -pub use eip4844::{ - get_coefficients, BlobConsistencyConfig, BlobConsistencyWitness, BLOB_WIDTH, -}; +pub use eip4844::{get_coefficients, BlobConsistencyConfig, BlobConsistencyWitness, BLOB_WIDTH}; #[cfg(feature = "da-eip4844")] mod eip4844; #[cfg(feature = "da-eip4844")] -pub use eip4844::{ - get_coefficients, BlobConsistencyConfig, BlobConsistencyWitness, BLOB_WIDTH, -}; +pub use eip4844::{get_coefficients, BlobConsistencyConfig, BlobConsistencyWitness, BLOB_WIDTH}; From 28d43b2537991298e4adc465e016a7b3605c94a3 Mon Sep 17 00:00:00 2001 From: z2trillion Date: Tue, 8 Oct 2024 01:08:26 -0400 Subject: [PATCH 11/35] Move blob_data up to data_availability --- .../src/data_availability/{eip4844 => }/blob_data.rs | 2 +- aggregator/src/data_availability/eip4844.rs | 10 +++------- aggregator/src/data_availability/eip4844/tests.rs | 6 ++---- 3 files changed, 6 insertions(+), 12 deletions(-) rename aggregator/src/data_availability/{eip4844 => }/blob_data.rs (99%) diff --git a/aggregator/src/data_availability/eip4844/blob_data.rs b/aggregator/src/data_availability/blob_data.rs similarity index 99% rename from aggregator/src/data_availability/eip4844/blob_data.rs rename to aggregator/src/data_availability/blob_data.rs index 25a0f3de21..1d9711b507 100644 --- a/aggregator/src/data_availability/eip4844/blob_data.rs +++ b/aggregator/src/data_availability/blob_data.rs @@ -11,7 +11,7 @@ use zkevm_circuits::{table::U8Table, util::Challenges}; use crate::{ aggregation::{batch_data::N_BLOB_BYTES, POWS_OF_256}, - data_availability::eip4844::{blob::BLOB_WIDTH, N_DATA_BYTES_PER_COEFFICIENT}, + data_availability::{eip4844::N_DATA_BYTES_PER_COEFFICIENT, BLOB_WIDTH}, RlcConfig, }; diff --git a/aggregator/src/data_availability/eip4844.rs b/aggregator/src/data_availability/eip4844.rs index 40a8729b78..5419a843c5 100644 --- a/aggregator/src/data_availability/eip4844.rs +++ b/aggregator/src/data_availability/eip4844.rs @@ -1,24 +1,20 @@ /// Config to evaluate blob polynomial at a random challenge. mod barycentric; -pub use barycentric::{ +use barycentric::{ interpolate, AssignedBarycentricEvaluationConfig, BarycentricEvaluationConfig, BLS_MODULUS, }; /// blob struct and constants mod blob; -pub use blob::PointEvaluationAssignments; - -/// Config to constrain blob data (encoded batch data) -pub mod blob_data; -pub use blob_data::BlobDataConfig; +use blob::PointEvaluationAssignments; #[cfg(test)] mod tests; +use super::{AssignedBlobDataExport, BlobDataConfig}; use crate::constants::N_BYTES_U256; use crate::BatchData; use crate::RlcConfig; -use blob_data::AssignedBlobDataExport; use eth_types::{ToBigEndian, H256, U256}; use ethers_core::k256::sha2::{Digest, Sha256}; use halo2_base::gates::range::RangeConfig; diff --git a/aggregator/src/data_availability/eip4844/tests.rs b/aggregator/src/data_availability/eip4844/tests.rs index 3c7dce5bf0..f36a646d59 100644 --- a/aggregator/src/data_availability/eip4844/tests.rs +++ b/aggregator/src/data_availability/eip4844/tests.rs @@ -23,13 +23,11 @@ use crate::{ BatchData, RlcConfig, }, constants::N_BYTES_U256, - data_availability::eip4844::blob::PointEvaluationAssignments, - data_availability::eip4844::{ - blob_data::BlobDataConfig, get_blob_bytes, get_coefficients, get_versioned_hash, - }, + data_availability::eip4844::{get_blob_bytes, get_coefficients, get_versioned_hash}, data_availability::eip4844::{ AssignedBarycentricEvaluationConfig, BarycentricEvaluationConfig, }, + data_availability::{eip4844::blob::PointEvaluationAssignments, BlobDataConfig}, decode_bytes, param::ConfigParams, BatchDataConfig, ChunkInfo, MAX_AGG_SNARKS, From 91c3a92931fd65c245be48a8c76b0053d2bae258 Mon Sep 17 00:00:00 2001 From: z2trillion Date: Tue, 8 Oct 2024 01:53:14 -0400 Subject: [PATCH 12/35] fix backwards compatibility --- Makefile | 3 + aggregator/src/aggregation/circuit.rs | 6 +- aggregator/src/batch.rs | 22 ++--- aggregator/src/data_availability.rs | 27 +++--- aggregator/src/data_availability/avail.rs | 102 ++++++++++++++++++++ aggregator/src/data_availability/eip4844.rs | 22 ++++- prover/src/aggregator/prover.rs | 24 ++--- 7 files changed, 162 insertions(+), 44 deletions(-) diff --git a/Makefile b/Makefile index 47883bc0c3..dd664b7419 100644 --- a/Makefile +++ b/Makefile @@ -30,6 +30,9 @@ test-benches: ## Compiles the benchmarks test-all: fmt doc clippy test-doc test-benches test ## Run all the CI checks locally (in your actual toolchain) +test-da-avail: ## Run light tests + @cargo test --release --workspace --exclude integration-tests --exclude circuit-benchmarks --features da-avail batch_circuit + super_bench: ## Run Super Circuit benchmarks @cargo test --profile bench bench_super_circuit_prover -p circuit-benchmarks --features benches -- --nocapture diff --git a/aggregator/src/aggregation/circuit.rs b/aggregator/src/aggregation/circuit.rs index 192979f12a..8eb89c8426 100644 --- a/aggregator/src/aggregation/circuit.rs +++ b/aggregator/src/aggregation/circuit.rs @@ -256,8 +256,8 @@ impl Circuit for BatchCircuit { &mut ctx, &coefficients, // todo: check that the byte order is correct!!!! - self.batch_hash.blob_consistency_witness.challenge.0.into(), - self.batch_hash.blob_consistency_witness.evaluation.0.into(), + self.batch_hash.blob_consistency_witness.challenge().0.into(), + self.batch_hash.blob_consistency_witness.evaluation().0.into(), ); ctx.print_stats(&["barycentric"]); @@ -423,7 +423,7 @@ impl Circuit for BatchCircuit { &config.rlc_config, &assigned_batch_hash.chunks_are_padding, &batch_data, - self.batch_hash.blob_consistency_witness.id, + self.batch_hash.blob_consistency_witness.id(), barycentric_assignments, )?; diff --git a/aggregator/src/batch.rs b/aggregator/src/batch.rs index 52f054ba87..11adb0933d 100644 --- a/aggregator/src/batch.rs +++ b/aggregator/src/batch.rs @@ -27,7 +27,7 @@ pub struct BatchHeader { /// The data hash of the batch pub data_hash: H256, // /// information needed to check that the blobs in the circuit match the blob the data availibility provided. - // TODO: #[serde(flatten)] etc. for backwards compatibility? + #[serde(flatten)] pub blob_consistency_witness: BlobConsistencyWitness, } @@ -113,15 +113,15 @@ impl BatchHeader { self.l1_message_popped.to_be_bytes().as_ref(), self.total_l1_message_popped.to_be_bytes().as_ref(), self.data_hash.as_bytes(), - self.blob_consistency_witness.id.as_bytes(), + self.blob_consistency_witness.id().as_bytes(), self.parent_batch_hash.as_bytes(), self.last_block_timestamp.to_be_bytes().as_ref(), self.blob_consistency_witness - .challenge + .challenge() .to_fixed_bytes() .as_ref(), self.blob_consistency_witness - .evaluation + .evaluation() .to_fixed_bytes() .as_ref(), ] @@ -292,9 +292,9 @@ impl BatchHash { "batch hash {:?}, datahash {}, z {}, y {}, versioned hash {:x}", current_batch_hash, hex::encode(batch_data_hash), - hex::encode(blob_consistency_witness.challenge.to_fixed_bytes()), - hex::encode(blob_consistency_witness.evaluation.to_fixed_bytes()), - blob_consistency_witness.id, + hex::encode(blob_consistency_witness.challenge().to_fixed_bytes()), + hex::encode(blob_consistency_witness.evaluation().to_fixed_bytes()), + blob_consistency_witness.id(), ); Self { @@ -347,7 +347,7 @@ impl BatchHash { .to_be_bytes() .as_ref(), self.data_hash.as_bytes(), - self.batch_header.blob_consistency_witness.id.as_bytes(), + self.batch_header.blob_consistency_witness.id().as_bytes(), self.batch_header.parent_batch_hash.as_bytes(), self.batch_header .last_block_timestamp @@ -355,12 +355,12 @@ impl BatchHash { .as_ref(), self.batch_header .blob_consistency_witness - .challenge + .challenge() .to_fixed_bytes() .as_ref(), self.batch_header .blob_consistency_witness - .evaluation + .evaluation() .to_fixed_bytes() .as_ref(), ] @@ -399,7 +399,7 @@ impl BatchHash { // - preimage for each chunk's flattened L2 signed tx data // - preimage for the challenge digest let batch_data = BatchData::from(self); - let dynamic_preimages = batch_data.preimages(self.batch_header.blob_consistency_witness.id); + let dynamic_preimages = batch_data.preimages(self.batch_header.blob_consistency_witness.id()); for dynamic_preimage in dynamic_preimages { res.push(dynamic_preimage); } diff --git a/aggregator/src/data_availability.rs b/aggregator/src/data_availability.rs index 8c55e0c69a..f9952cca0b 100644 --- a/aggregator/src/data_availability.rs +++ b/aggregator/src/data_availability.rs @@ -5,22 +5,23 @@ enum DataAvailibility { Avail, } -cfg_if! { - if #[cfg(feature = "da-eip4844")] { - const DATA_AVAILABILITY: DataAvailibility = DataAvailibility::Eip4844; - } else if #[cfg(feature = "da-avail")] { - const DATA_AVAILABILITY: DataAvailibility = DataAvailibility::Avail; - } else { - compile_error!("no da feature flag set"); - } -} +mod blob_data; +use blob_data::{AssignedBlobDataExport, BlobDataConfig}; #[cfg(feature = "da-avail")] mod avail; -#[cfg(feature = "da-avail")] -pub use eip4844::{get_coefficients, BlobConsistencyConfig, BlobConsistencyWitness, BLOB_WIDTH}; #[cfg(feature = "da-eip4844")] mod eip4844; -#[cfg(feature = "da-eip4844")] -pub use eip4844::{get_coefficients, BlobConsistencyConfig, BlobConsistencyWitness, BLOB_WIDTH}; + +cfg_if! { + if #[cfg(feature = "da-avail")] { + const DATA_AVAILABILITY: DataAvailibility = DataAvailibility::Avail; + pub use avail::{get_coefficients, BlobConsistencyConfig, BlobConsistencyWitness, BLOB_WIDTH}; + } else if #[cfg(feature = "da-eip4844")] { + const DATA_AVAILABILITY: DataAvailibility = DataAvailibility::Eip4844; + pub use eip4844::{get_coefficients, BlobConsistencyConfig, BlobConsistencyWitness, BLOB_WIDTH}; + } else { + compile_error!("no da feature flag set"); + } +} diff --git a/aggregator/src/data_availability/avail.rs b/aggregator/src/data_availability/avail.rs index 8b13789179..a5180b3a62 100644 --- a/aggregator/src/data_availability/avail.rs +++ b/aggregator/src/data_availability/avail.rs @@ -1 +1,103 @@ +use crate::{constants::N_BYTES_U256, BatchData, RlcConfig}; +use eth_types::{ToBigEndian, H256, U256}; +use ethers_core::k256::sha2::{Digest, Sha256}; +use halo2_base::{gates::range::RangeConfig, AssignedValue, Context}; +use halo2_ecc::bigint::CRTInteger; +use halo2_proofs::{ + circuit::{AssignedCell, Layouter, Value}, + halo2curves::bn256::Fr, + plonk::{ConstraintSystem, Error, Expression}, +}; +use serde::{Deserialize, Serialize}; +use super::{AssignedBlobDataExport, BlobDataConfig}; +use zkevm_circuits::{table::U8Table, util::Challenges}; +pub const BLOB_WIDTH: usize = 4096; + +pub fn get_coefficients(bytes: &[u8]) -> [U256; BLOB_WIDTH] { + [U256::zero(); BLOB_WIDTH] +} + +#[derive(Debug, Clone)] +pub struct BlobConsistencyConfig { + data: BlobDataConfig, +} + +impl BlobConsistencyConfig { + pub fn construct( + meta: &mut ConstraintSystem, + challenges: &Challenges>, + u8_table: U8Table, + _: RangeConfig, + ) -> Self { + Self{ + data: BlobDataConfig::configure(meta, challenges, u8_table), + } + } + + pub fn assign_barycentric( + &self, + ctx: &mut Context, + blob: &[U256; BLOB_WIDTH], + challenge: U256, + evaluation: U256, + ) -> AssignedBarycentricEvaluationConfig { + Default::default() + } + + pub fn assign_blob_data( + &self, + layouter: &mut impl Layouter, + challenge_value: Challenges>, + rlc_config: &RlcConfig, + blob_bytes: &[u8], + barycentric_assignments: &[CRTInteger], + ) -> Result { + self.data.assign( + layouter, + challenge_value, + rlc_config, + blob_bytes, + barycentric_assignments, + ) + } +} + +#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)] +pub struct BlobConsistencyWitness { + #[serde(rename = "blob_versioned_hash")] + id: H256, + blob_data_proof: [H256; 2] +} + +impl BlobConsistencyWitness { + pub fn new(bytes: &[u8], _: &BatchData) -> Self { + Self { + id: H256::default(), // should be keccak of bytes + blob_data_proof: Default::default(), + } + } + + pub fn id(&self) -> H256 { + self.id + } + + pub fn challenge(&self) -> H256 { + self.blob_data_proof[0] + } + + pub fn evaluation(&self) -> H256 { + self.blob_data_proof[1] + } +} + +#[derive(Default)] +pub struct AssignedBarycentricEvaluationConfig { + /// CRTIntegers for the BLOB_WIDTH number of blob polynomial coefficients, followed by a + /// CRTInteger for the challenge digest. + pub(crate) barycentric_assignments: Vec>, + /// 32 Assigned cells representing the LE-bytes of challenge z. + pub(crate) z_le: Vec>, + /// 32 Assigned cells representing the LE-bytes of evaluation y. + pub(crate) y_le: Vec>, +} diff --git a/aggregator/src/data_availability/eip4844.rs b/aggregator/src/data_availability/eip4844.rs index 5419a843c5..2c5b050127 100644 --- a/aggregator/src/data_availability/eip4844.rs +++ b/aggregator/src/data_availability/eip4844.rs @@ -148,9 +148,9 @@ impl BlobConsistencyConfig { #[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)] pub struct BlobConsistencyWitness { - pub id: H256, - pub challenge: H256, - pub evaluation: H256, + #[serde(rename = "blob_versioned_hash")] + id: H256, + blob_data_proof: [H256; 2] } impl BlobConsistencyWitness { @@ -159,11 +159,23 @@ impl BlobConsistencyWitness { let versioned_hash = get_versioned_hash(&coeffs); let point_evaluation_assignments = PointEvaluationAssignments::new(&batch_data, bytes, versioned_hash); + let blob_data_proof = [point_evaluation_assignments.challenge, point_evaluation_assignments.evaluation].map(|x| H256::from_slice(&x.to_be_bytes())); Self { id: versioned_hash, - challenge: H256::from_slice(&point_evaluation_assignments.challenge.to_be_bytes()), - evaluation: H256::from_slice(&point_evaluation_assignments.evaluation.to_be_bytes()), + blob_data_proof, } } + + pub fn id(&self) -> H256 { + self.id + } + + pub fn challenge(&self) -> H256 { + self.blob_data_proof[0] + } + + pub fn evaluation(&self) -> H256 { + self.blob_data_proof[1] + } } diff --git a/prover/src/aggregator/prover.rs b/prover/src/aggregator/prover.rs index da680b62d8..5721bab293 100644 --- a/prover/src/aggregator/prover.rs +++ b/prover/src/aggregator/prover.rs @@ -183,25 +183,25 @@ impl<'params> Prover<'params> { batch.batch_header.data_hash, batch_header.data_hash ); assert_eq!( - batch_header.blob_consistency_witness.challenge, - batch.batch_header.blob_consistency_witness.challenge, + batch_header.blob_consistency_witness.challenge(), + batch.batch_header.blob_consistency_witness.challenge(), "BatchHeader(sanity) mismatch blob data proof (z) expected={}, got={}", - batch_header.blob_consistency_witness.challenge, - batch.batch_header.blob_consistency_witness.challenge, + batch_header.blob_consistency_witness.challenge(), + batch.batch_header.blob_consistency_witness.challenge(), ); assert_eq!( - batch_header.blob_consistency_witness.evaluation, - batch.batch_header.blob_consistency_witness.evaluation, + batch_header.blob_consistency_witness.evaluation(), + batch.batch_header.blob_consistency_witness.evaluation(), "BatchHeader(sanity) mismatch blob data proof (y) expected={}, got={}", - batch_header.blob_consistency_witness.evaluation, - batch.batch_header.blob_consistency_witness.evaluation, + batch_header.blob_consistency_witness.evaluation(), + batch.batch_header.blob_consistency_witness.evaluation(), ); assert_eq!( - batch_header.blob_consistency_witness.id, - batch.batch_header.blob_consistency_witness.id, + batch_header.blob_consistency_witness.id(), + batch.batch_header.blob_consistency_witness.id(), "BatchHeader(sanity) mismatch blob versioned hash expected={}, got={}", - batch_header.blob_consistency_witness.id, - batch.batch_header.blob_consistency_witness.id, + batch_header.blob_consistency_witness.id(), + batch.batch_header.blob_consistency_witness.id(), ); let batch_hash = batch_header.batch_hash(); From 085d4174f4860bfe749dc188dd701040a1d753fd Mon Sep 17 00:00:00 2001 From: z2trillion Date: Mon, 14 Oct 2024 14:55:39 -0400 Subject: [PATCH 13/35] fix rebase --- aggregator/src/aggregation/circuit.rs | 12 ++++++++++-- aggregator/src/batch.rs | 3 ++- aggregator/src/data_availability/avail.rs | 10 +++++----- aggregator/src/data_availability/eip4844.rs | 15 +++++++++------ .../src/data_availability/eip4844/barycentric.rs | 3 +-- 5 files changed, 27 insertions(+), 16 deletions(-) diff --git a/aggregator/src/aggregation/circuit.rs b/aggregator/src/aggregation/circuit.rs index 8eb89c8426..ae048c4e3b 100644 --- a/aggregator/src/aggregation/circuit.rs +++ b/aggregator/src/aggregation/circuit.rs @@ -256,8 +256,16 @@ impl Circuit for BatchCircuit { &mut ctx, &coefficients, // todo: check that the byte order is correct!!!! - self.batch_hash.blob_consistency_witness.challenge().0.into(), - self.batch_hash.blob_consistency_witness.evaluation().0.into(), + self.batch_hash + .blob_consistency_witness + .challenge() + .0 + .into(), + self.batch_hash + .blob_consistency_witness + .evaluation() + .0 + .into(), ); ctx.print_stats(&["barycentric"]); diff --git a/aggregator/src/batch.rs b/aggregator/src/batch.rs index 11adb0933d..1ceea79392 100644 --- a/aggregator/src/batch.rs +++ b/aggregator/src/batch.rs @@ -399,7 +399,8 @@ impl BatchHash { // - preimage for each chunk's flattened L2 signed tx data // - preimage for the challenge digest let batch_data = BatchData::from(self); - let dynamic_preimages = batch_data.preimages(self.batch_header.blob_consistency_witness.id()); + let dynamic_preimages = + batch_data.preimages(self.batch_header.blob_consistency_witness.id()); for dynamic_preimage in dynamic_preimages { res.push(dynamic_preimage); } diff --git a/aggregator/src/data_availability/avail.rs b/aggregator/src/data_availability/avail.rs index a5180b3a62..93e4f23e05 100644 --- a/aggregator/src/data_availability/avail.rs +++ b/aggregator/src/data_availability/avail.rs @@ -1,3 +1,4 @@ +use super::{AssignedBlobDataExport, BlobDataConfig}; use crate::{constants::N_BYTES_U256, BatchData, RlcConfig}; use eth_types::{ToBigEndian, H256, U256}; use ethers_core::k256::sha2::{Digest, Sha256}; @@ -9,7 +10,6 @@ use halo2_proofs::{ plonk::{ConstraintSystem, Error, Expression}, }; use serde::{Deserialize, Serialize}; -use super::{AssignedBlobDataExport, BlobDataConfig}; use zkevm_circuits::{table::U8Table, util::Challenges}; pub const BLOB_WIDTH: usize = 4096; @@ -30,8 +30,8 @@ impl BlobConsistencyConfig { u8_table: U8Table, _: RangeConfig, ) -> Self { - Self{ - data: BlobDataConfig::configure(meta, challenges, u8_table), + Self { + data: BlobDataConfig::configure(meta, challenges, u8_table), } } @@ -65,9 +65,9 @@ impl BlobConsistencyConfig { #[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)] pub struct BlobConsistencyWitness { - #[serde(rename = "blob_versioned_hash")] + #[serde(rename = "blob_versioned_hash")] id: H256, - blob_data_proof: [H256; 2] + blob_data_proof: [H256; 2], } impl BlobConsistencyWitness { diff --git a/aggregator/src/data_availability/eip4844.rs b/aggregator/src/data_availability/eip4844.rs index 2c5b050127..10ea8e9567 100644 --- a/aggregator/src/data_availability/eip4844.rs +++ b/aggregator/src/data_availability/eip4844.rs @@ -122,10 +122,9 @@ impl BlobConsistencyConfig { ctx: &mut Context, blob: &[U256; BLOB_WIDTH], challenge: U256, - evaluation: U256, + _: U256, ) -> AssignedBarycentricEvaluationConfig { - self.barycentric_evaluation - .assign(ctx, blob, challenge, evaluation) + self.barycentric_evaluation.assign(ctx, blob, challenge) } pub fn assign_blob_data( @@ -148,9 +147,9 @@ impl BlobConsistencyConfig { #[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)] pub struct BlobConsistencyWitness { - #[serde(rename = "blob_versioned_hash")] + #[serde(rename = "blob_versioned_hash")] id: H256, - blob_data_proof: [H256; 2] + blob_data_proof: [H256; 2], } impl BlobConsistencyWitness { @@ -159,7 +158,11 @@ impl BlobConsistencyWitness { let versioned_hash = get_versioned_hash(&coeffs); let point_evaluation_assignments = PointEvaluationAssignments::new(&batch_data, bytes, versioned_hash); - let blob_data_proof = [point_evaluation_assignments.challenge, point_evaluation_assignments.evaluation].map(|x| H256::from_slice(&x.to_be_bytes())); + let blob_data_proof = [ + point_evaluation_assignments.challenge, + point_evaluation_assignments.evaluation, + ] + .map(|x| H256::from_slice(&x.to_be_bytes())); Self { id: versioned_hash, diff --git a/aggregator/src/data_availability/eip4844/barycentric.rs b/aggregator/src/data_availability/eip4844/barycentric.rs index 262f1f0647..c323da3619 100644 --- a/aggregator/src/data_availability/eip4844/barycentric.rs +++ b/aggregator/src/data_availability/eip4844/barycentric.rs @@ -18,8 +18,7 @@ use num_bigint::{BigInt, Sign}; use std::{iter::successors, sync::LazyLock}; use crate::{ - blob::{BLOB_WIDTH, N_BYTES_U256}, - constants::{BITS, LIMBS}, + constants::{BITS, LIMBS, N_BYTES_U256}, data_availability::eip4844::blob::BLOB_WIDTH, }; From 95933a6f0d0f2dd3983631b8716624688bfc3fbb Mon Sep 17 00:00:00 2001 From: z2trillion Date: Mon, 14 Oct 2024 15:12:15 -0400 Subject: [PATCH 14/35] fix warnings --- aggregator/src/aggregation/batch_data.rs | 2 +- aggregator/src/batch.rs | 2 +- aggregator/src/data_availability.rs | 12 ++++++------ aggregator/src/data_availability/eip4844.rs | 1 + .../data_availability/eip4844/barycentric.rs | 3 +-- .../src/data_availability/eip4844/blob.rs | 18 ++++++------------ 6 files changed, 16 insertions(+), 22 deletions(-) diff --git a/aggregator/src/aggregation/batch_data.rs b/aggregator/src/aggregation/batch_data.rs index cd352a2fd6..deeb819fc0 100644 --- a/aggregator/src/aggregation/batch_data.rs +++ b/aggregator/src/aggregation/batch_data.rs @@ -1315,7 +1315,7 @@ impl BatchData { } /// Get the witness rows for assignment to the BlobDataConfig. - pub(crate) fn to_rows( + fn to_rows( &self, versioned_hash: H256, challenge: Challenges>, diff --git a/aggregator/src/batch.rs b/aggregator/src/batch.rs index 1ceea79392..99d9c7e3fc 100644 --- a/aggregator/src/batch.rs +++ b/aggregator/src/batch.rs @@ -1,7 +1,7 @@ //! This module implements related functions that aggregates public inputs of many chunks into a //! single one. -use eth_types::{ToBigEndian, H256}; +use eth_types::H256; use ethers_core::utils::keccak256; use gadgets::{util::split_h256, Field}; use serde::{Deserialize, Serialize}; diff --git a/aggregator/src/data_availability.rs b/aggregator/src/data_availability.rs index f9952cca0b..3fc8a249d6 100644 --- a/aggregator/src/data_availability.rs +++ b/aggregator/src/data_availability.rs @@ -1,9 +1,9 @@ use cfg_if::cfg_if; -enum DataAvailibility { - Eip4844, - Avail, -} +// enum DataAvailibility { +// Eip4844, +// Avail, +// } mod blob_data; use blob_data::{AssignedBlobDataExport, BlobDataConfig}; @@ -16,10 +16,10 @@ mod eip4844; cfg_if! { if #[cfg(feature = "da-avail")] { - const DATA_AVAILABILITY: DataAvailibility = DataAvailibility::Avail; + // const DATA_AVAILABILITY: DataAvailibility = DataAvailibility::Avail; pub use avail::{get_coefficients, BlobConsistencyConfig, BlobConsistencyWitness, BLOB_WIDTH}; } else if #[cfg(feature = "da-eip4844")] { - const DATA_AVAILABILITY: DataAvailibility = DataAvailibility::Eip4844; + // const DATA_AVAILABILITY: DataAvailibility = DataAvailibility::Eip4844; pub use eip4844::{get_coefficients, BlobConsistencyConfig, BlobConsistencyWitness, BLOB_WIDTH}; } else { compile_error!("no da feature flag set"); diff --git a/aggregator/src/data_availability/eip4844.rs b/aggregator/src/data_availability/eip4844.rs index 10ea8e9567..bd3cc2a434 100644 --- a/aggregator/src/data_availability/eip4844.rs +++ b/aggregator/src/data_availability/eip4844.rs @@ -84,6 +84,7 @@ fn kzg_to_versioned_hash(commitment: &c_kzg::KzgCommitment) -> H256 { H256::from_slice(&res[..]) } +#[cfg(test)] /// Get the blob data bytes that will be populated in BlobDataConfig. pub fn get_blob_bytes(batch_bytes: &[u8]) -> Vec { let mut blob_bytes = crate::witgen::zstd_encode(batch_bytes); diff --git a/aggregator/src/data_availability/eip4844/barycentric.rs b/aggregator/src/data_availability/eip4844/barycentric.rs index c323da3619..96ba6d7267 100644 --- a/aggregator/src/data_availability/eip4844/barycentric.rs +++ b/aggregator/src/data_availability/eip4844/barycentric.rs @@ -356,8 +356,7 @@ mod tests { use super::*; use crate::{ aggregation::BatchData, - data_availability::eip4844::KZG_TRUSTED_SETUP, - data_availability::eip4844::{get_blob_bytes, get_coefficients}, + data_availability::eip4844::{get_blob_bytes, get_coefficients, KZG_TRUSTED_SETUP}, MAX_AGG_SNARKS, }; use c_kzg::{Blob as RethBlob, KzgProof}; diff --git a/aggregator/src/data_availability/eip4844/blob.rs b/aggregator/src/data_availability/eip4844/blob.rs index 3558421c87..bb37501b4d 100644 --- a/aggregator/src/data_availability/eip4844/blob.rs +++ b/aggregator/src/data_availability/eip4844/blob.rs @@ -1,15 +1,7 @@ use super::{get_coefficients, interpolate, BLS_MODULUS}; -use crate::{BatchData, BatchHash, ChunkInfo}; - +use crate::BatchData; use eth_types::{H256, U256}; -use ethers_core::utils::keccak256; -use halo2_proofs::{ - circuit::Value, - halo2curves::{bls12_381::Scalar, bn256::Fr}, -}; -use itertools::Itertools; -use std::iter::{once, repeat}; -use zkevm_circuits::util::Challenges; +use halo2_proofs::halo2curves::bls12_381::Scalar; /// The number of coefficients (BLS12-381 scalars) to represent the blob polynomial in evaluation /// form. @@ -77,8 +69,10 @@ mod tests { use super::*; use crate::{ data_availability::eip4844::{get_blob_bytes, get_versioned_hash}, - MAX_AGG_SNARKS, + BatchHash, ChunkInfo, MAX_AGG_SNARKS, }; + use ethers_core::utils::keccak256; + use std::iter::{once, repeat}; #[test] #[ignore = "only required for logging challenge digest"] @@ -154,7 +148,7 @@ mod tests { let blob_bytes = get_blob_bytes(&batch_bytes); let coeffs = get_coefficients(&blob_bytes); let versioned_hash = get_versioned_hash(&coeffs); - let chunks_without_padding = crate::chunk::ChunkInfo::mock_chunk_infos(tcase); + let chunks_without_padding = ChunkInfo::mock_chunk_infos(tcase); let batch_hash = BatchHash::::construct_with_unpadded( &chunks_without_padding, batch_header, From 2e1f8d1ea09061a39a689b26233e08b667e65bd9 Mon Sep 17 00:00:00 2001 From: z2trillion Date: Mon, 14 Oct 2024 15:44:06 -0400 Subject: [PATCH 15/35] remove empty file --- aggregator/src/data_availability/eip4844/constants.rs | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 aggregator/src/data_availability/eip4844/constants.rs diff --git a/aggregator/src/data_availability/eip4844/constants.rs b/aggregator/src/data_availability/eip4844/constants.rs deleted file mode 100644 index e69de29bb2..0000000000 From a8782909bd632c6cf54dcbe231a49e7832e84442 Mon Sep 17 00:00:00 2001 From: z2trillion Date: Wed, 16 Oct 2024 14:23:19 -0400 Subject: [PATCH 16/35] add and use blob_crts_limbs function --- aggregator/src/data_availability/blob_data.rs | 97 +++++++++++-------- 1 file changed, 58 insertions(+), 39 deletions(-) diff --git a/aggregator/src/data_availability/blob_data.rs b/aggregator/src/data_availability/blob_data.rs index 1d9711b507..d1febe74d2 100644 --- a/aggregator/src/data_availability/blob_data.rs +++ b/aggregator/src/data_availability/blob_data.rs @@ -1,3 +1,8 @@ +use crate::{ + aggregation::{batch_data::N_BLOB_BYTES, POWS_OF_256}, + data_availability::{eip4844::N_DATA_BYTES_PER_COEFFICIENT, BLOB_WIDTH}, + RlcConfig, +}; use gadgets::util::Expr; use halo2_ecc::bigint::CRTInteger; use halo2_proofs::{ @@ -7,14 +12,9 @@ use halo2_proofs::{ poly::Rotation, }; use itertools::Itertools; +use snark_verifier_sdk::{BITS, LIMBS}; use zkevm_circuits::{table::U8Table, util::Challenges}; -use crate::{ - aggregation::{batch_data::N_BLOB_BYTES, POWS_OF_256}, - data_availability::{eip4844::N_DATA_BYTES_PER_COEFFICIENT, BLOB_WIDTH}, - RlcConfig, -}; - /// Blob is represented by 4096 BLS12-381 scalar field elements, where each element is represented /// by 32 bytes. The scalar field element is required to be in the canonical form, i.e. its value /// MUST BE less than the BLS_MODULUS. In order to ensure this, we hard-code the most-significant @@ -50,6 +50,7 @@ pub struct AssignedBlobDataExport { pub bytes_rlc: AssignedCell, pub bytes_len: AssignedCell, pub cooked_len: AssignedCell, + // pub blob_crt_limbs: [[AssignedCell; LIMBS]; BLOB_WIDTH], } impl BlobDataConfig { @@ -181,6 +182,7 @@ impl BlobDataConfig { bytes_rlc, bytes_len, cooked_len, + // blob_crt_limbs, }) } @@ -333,39 +335,21 @@ impl BlobDataConfig { //////////////////////////////////////////////////////////////////////////////// assert_eq!(barycentric_assignments.len(), BLOB_WIDTH + 1); - let blob_crts = barycentric_assignments - .iter() - .take(BLOB_WIDTH) - .collect::>(); - let mut blob_fields: Vec>> = Vec::with_capacity(BLOB_WIDTH); - for chunk in assigned_bytes.chunks_exact(N_DATA_BYTES_PER_COEFFICIENT) { - // blob bytes are supposed to be deserialised in big-endianness. However, we - // have the export from BarycentricConfig in little-endian bytes. - blob_fields.push(chunk.iter().rev().cloned().collect()); - } - - for (blob_crt, blob_field) in blob_crts.iter().zip_eq(blob_fields.iter()) { - let limb1 = rlc_config.inner_product( - region, - &blob_field[0..11], - &pows_of_256, - &mut rlc_config_offset, - )?; - let limb2 = rlc_config.inner_product( - region, - &blob_field[11..22], - &pows_of_256, - &mut rlc_config_offset, - )?; - let limb3 = rlc_config.inner_product( - region, - &blob_field[22..31], - &pows_of_256[0..9], - &mut rlc_config_offset, - )?; - region.constrain_equal(limb1.cell(), blob_crt.truncation.limbs[0].cell())?; - region.constrain_equal(limb2.cell(), blob_crt.truncation.limbs[1].cell())?; - region.constrain_equal(limb3.cell(), blob_crt.truncation.limbs[2].cell())?; + for (blob_crt_limbs, barycentric_crt) in blob_crts_limbs( + region, + rlc_config, + assigned_bytes, + &pows_of_256, + &mut rlc_config_offset, + ) + .iter() + .zip_eq(barycentric_assignments.iter().take(BLOB_WIDTH)) + { + for (blob_limb, barycentric_limb) in + blob_crt_limbs.iter().zip_eq(barycentric_crt.limbs()) + { + region.constrain_equal(blob_limb.cell(), barycentric_limb.cell())?; + } } // The zstd decoder (DecoderConfig) exports an encoded length that is 1 more than the @@ -376,3 +360,38 @@ impl BlobDataConfig { Ok(cooked_bytes_len) } } + +fn blob_crts_limbs( + region: &mut Region, + rlc_config: &RlcConfig, + bytes: &[AssignedCell], + powers_of_256: &[AssignedCell], + offset: &mut usize, +) -> [[AssignedCell; LIMBS]; BLOB_WIDTH] { + bytes + .chunks_exact(N_DATA_BYTES_PER_COEFFICIENT) + .map(|coefficient_bytes| { + coefficient_bytes + .iter() + .rev() // reverse bytes to match endianess of crt limbs + .chunks(BITS / 8) + .into_iter() + .map(|chunk_bytes| { + let chunk_bytes = chunk_bytes.into_iter().cloned().collect_vec(); + rlc_config + .inner_product( + region, + &chunk_bytes, + &powers_of_256[..chunk_bytes.len()], + offset, + ) + .unwrap() + }) + .collect_vec() + .try_into() + .unwrap() + }) + .collect_vec() + .try_into() + .unwrap() +} From 11c6e24cf6f446c58222fb31fa9672f8e5882e0a Mon Sep 17 00:00:00 2001 From: z2trillion Date: Wed, 16 Oct 2024 17:33:55 -0400 Subject: [PATCH 17/35] add blob_crts_limbs to AssignedBlobDataExport --- aggregator/src/data_availability/blob_data.rs | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/aggregator/src/data_availability/blob_data.rs b/aggregator/src/data_availability/blob_data.rs index d1febe74d2..3fb3441c5b 100644 --- a/aggregator/src/data_availability/blob_data.rs +++ b/aggregator/src/data_availability/blob_data.rs @@ -50,7 +50,7 @@ pub struct AssignedBlobDataExport { pub bytes_rlc: AssignedCell, pub bytes_len: AssignedCell, pub cooked_len: AssignedCell, - // pub blob_crt_limbs: [[AssignedCell; LIMBS]; BLOB_WIDTH], + pub blob_crts_limbs: [[AssignedCell; LIMBS]; BLOB_WIDTH], } impl BlobDataConfig { @@ -163,7 +163,7 @@ impl BlobDataConfig { )?; let enable_encoding = assigned_bytes[0].clone(); - let cooked_len = layouter.assign_region( + let (cooked_len, blob_crts_limbs) = layouter.assign_region( || "BlobData internal checks", |mut region| { self.assign_internal_checks( @@ -182,7 +182,7 @@ impl BlobDataConfig { bytes_rlc, bytes_len, cooked_len, - // blob_crt_limbs, + blob_crts_limbs, }) } @@ -299,7 +299,13 @@ impl BlobDataConfig { barycentric_assignments: &[CRTInteger], assigned_bytes: &[AssignedCell], bytes_len: &AssignedCell, - ) -> Result, Error> { + ) -> Result< + ( + AssignedCell, + [[AssignedCell; LIMBS]; BLOB_WIDTH], + ), + Error, + > { rlc_config.init(region)?; let mut rlc_config_offset = 0; @@ -335,15 +341,16 @@ impl BlobDataConfig { //////////////////////////////////////////////////////////////////////////////// assert_eq!(barycentric_assignments.len(), BLOB_WIDTH + 1); - for (blob_crt_limbs, barycentric_crt) in blob_crts_limbs( + let blob_crts_limbs = blob_crts_limbs( region, rlc_config, assigned_bytes, &pows_of_256, &mut rlc_config_offset, - ) - .iter() - .zip_eq(barycentric_assignments.iter().take(BLOB_WIDTH)) + ); + for (blob_crt_limbs, barycentric_crt) in blob_crts_limbs + .iter() + .zip_eq(barycentric_assignments.iter().take(BLOB_WIDTH)) { for (blob_limb, barycentric_limb) in blob_crt_limbs.iter().zip_eq(barycentric_crt.limbs()) @@ -357,7 +364,7 @@ impl BlobDataConfig { // by adding +1 to it before exporting. let cooked_bytes_len = rlc_config.add(region, bytes_len, &one, &mut rlc_config_offset)?; - Ok(cooked_bytes_len) + Ok((cooked_bytes_len, blob_crts_limbs)) } } From 4fe6c774e4b992ca88911607d208aa97d01fb694 Mon Sep 17 00:00:00 2001 From: z2trillion Date: Wed, 16 Oct 2024 20:23:11 -0400 Subject: [PATCH 18/35] move link out of blob data circuit into aggregation circuit --- aggregator/src/aggregation/circuit.rs | 21 +++++---- aggregator/src/data_availability/blob_data.rs | 10 ---- aggregator/src/data_availability/eip4844.rs | 46 ++++++++++++++----- .../src/data_availability/eip4844/tests.rs | 10 ++-- 4 files changed, 53 insertions(+), 34 deletions(-) diff --git a/aggregator/src/aggregation/circuit.rs b/aggregator/src/aggregation/circuit.rs index ae048c4e3b..e73dd2e400 100644 --- a/aggregator/src/aggregation/circuit.rs +++ b/aggregator/src/aggregation/circuit.rs @@ -27,17 +27,14 @@ use crate::{ batch::BatchHash, constants::{ACC_LEN, DIGEST_LEN}, core::{assign_batch_hashes, extract_proof_and_instances_with_pairing_check}, - data_availability::get_coefficients, // TODO: fix this + data_availability::{ + get_coefficients, // TODO: fix this + BlobConsistencyConfig, + }, util::parse_hash_digest_cells, witgen::{zstd_encode, MultiBlockProcessResult}, - ConfigParams, - LOG_DEGREE, - PI_CHAIN_ID, - PI_CURRENT_BATCH_HASH, - PI_CURRENT_STATE_ROOT, - PI_CURRENT_WITHDRAW_ROOT, - PI_PARENT_BATCH_HASH, - PI_PARENT_STATE_ROOT, + ConfigParams, LOG_DEGREE, PI_CHAIN_ID, PI_CURRENT_BATCH_HASH, PI_CURRENT_STATE_ROOT, + PI_CURRENT_WITHDRAW_ROOT, PI_PARENT_BATCH_HASH, PI_PARENT_STATE_ROOT, }; /// Batch circuit, the chunk aggregation routine below recursion circuit @@ -425,6 +422,12 @@ impl Circuit for BatchCircuit { barycentric_assignments, )?; + BlobConsistencyConfig::::link( + &mut layouter, + &blob_data_exports.blob_crts_limbs, + barycentric_assignments, + )?; + let batch_data_exports = config.batch_data_config.assign( &mut layouter, challenges, diff --git a/aggregator/src/data_availability/blob_data.rs b/aggregator/src/data_availability/blob_data.rs index 3fb3441c5b..62a5a64659 100644 --- a/aggregator/src/data_availability/blob_data.rs +++ b/aggregator/src/data_availability/blob_data.rs @@ -348,16 +348,6 @@ impl BlobDataConfig { &pows_of_256, &mut rlc_config_offset, ); - for (blob_crt_limbs, barycentric_crt) in blob_crts_limbs - .iter() - .zip_eq(barycentric_assignments.iter().take(BLOB_WIDTH)) - { - for (blob_limb, barycentric_limb) in - blob_crt_limbs.iter().zip_eq(barycentric_crt.limbs()) - { - region.constrain_equal(blob_limb.cell(), barycentric_limb.cell())?; - } - } // The zstd decoder (DecoderConfig) exports an encoded length that is 1 more than the // actual number of bytes in encoded data. Accordingly we "cook" the actual len(bytes) here diff --git a/aggregator/src/data_availability/eip4844.rs b/aggregator/src/data_availability/eip4844.rs index bd3cc2a434..0dec908b0d 100644 --- a/aggregator/src/data_availability/eip4844.rs +++ b/aggregator/src/data_availability/eip4844.rs @@ -12,24 +12,23 @@ use blob::PointEvaluationAssignments; mod tests; use super::{AssignedBlobDataExport, BlobDataConfig}; -use crate::constants::N_BYTES_U256; -use crate::BatchData; -use crate::RlcConfig; +use crate::{constants::N_BYTES_U256, BatchData, RlcConfig}; use eth_types::{ToBigEndian, H256, U256}; use ethers_core::k256::sha2::{Digest, Sha256}; -use halo2_base::gates::range::RangeConfig; -use halo2_base::Context; +use halo2_base::{gates::range::RangeConfig, Context}; use halo2_ecc::bigint::CRTInteger; -use halo2_proofs::circuit::Layouter; -use halo2_proofs::circuit::Value; -use halo2_proofs::halo2curves::bn256::Fr; -use halo2_proofs::plonk::{ConstraintSystem, Error, Expression}; +use halo2_proofs::{ + circuit::{AssignedCell, Layouter, Value}, + halo2curves::bn256::Fr, + plonk::{ConstraintSystem, Error, Expression}, +}; +use itertools::Itertools; use once_cell::sync::Lazy; use revm_primitives::VERSIONED_HASH_VERSION_KZG; use serde::{Deserialize, Serialize}; +use snark_verifier_sdk::LIMBS; use std::sync::Arc; -use zkevm_circuits::table::U8Table; -use zkevm_circuits::util::Challenges; +use zkevm_circuits::{table::U8Table, util::Challenges}; pub const BLOB_WIDTH: usize = 4096; /// The number data bytes we pack each BLS12-381 scalar into. The most-significant byte is 0. @@ -144,6 +143,31 @@ impl BlobConsistencyConfig { barycentric_assignments, ) } + + pub fn link( + layouter: &mut impl Layouter, + blob_crts_limbs: &[[AssignedCell; LIMBS]], + barycentric_crts: &[CRTInteger], + ) -> Result<(), Error> { + assert_eq!(blob_crts_limbs.len(), BLOB_WIDTH); + + layouter.assign_region( + || "constrain barycentric inputs to match blob", + |mut region| { + for (blob_crt_limbs, barycentric_crt) in blob_crts_limbs + .iter() + .zip_eq(barycentric_crts.iter().take(BLOB_WIDTH)) + { + for (blob_limb, barycentric_limb) in + blob_crt_limbs.iter().zip_eq(barycentric_crt.limbs()) + { + region.constrain_equal(blob_limb.cell(), barycentric_limb.cell())?; + } + } + Ok(()) + }, + ) + } } #[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)] diff --git a/aggregator/src/data_availability/eip4844/tests.rs b/aggregator/src/data_availability/eip4844/tests.rs index f36a646d59..8badec99e6 100644 --- a/aggregator/src/data_availability/eip4844/tests.rs +++ b/aggregator/src/data_availability/eip4844/tests.rs @@ -23,11 +23,13 @@ use crate::{ BatchData, RlcConfig, }, constants::N_BYTES_U256, - data_availability::eip4844::{get_blob_bytes, get_coefficients, get_versioned_hash}, - data_availability::eip4844::{ - AssignedBarycentricEvaluationConfig, BarycentricEvaluationConfig, + data_availability::{ + eip4844::{ + blob::PointEvaluationAssignments, get_blob_bytes, get_coefficients, get_versioned_hash, + AssignedBarycentricEvaluationConfig, BarycentricEvaluationConfig, + }, + BlobDataConfig, }, - data_availability::{eip4844::blob::PointEvaluationAssignments, BlobDataConfig}, decode_bytes, param::ConfigParams, BatchDataConfig, ChunkInfo, MAX_AGG_SNARKS, From 1b710b232b904d57c0f44f3851bcd7e30a924ba8 Mon Sep 17 00:00:00 2001 From: z2trillion Date: Wed, 16 Oct 2024 20:29:51 -0400 Subject: [PATCH 19/35] remove unused arguments --- aggregator/src/aggregation/circuit.rs | 7 ------- aggregator/src/data_availability/blob_data.rs | 18 +++--------------- aggregator/src/data_availability/eip4844.rs | 11 ++--------- .../src/data_availability/eip4844/tests.rs | 10 +++------- 4 files changed, 8 insertions(+), 38 deletions(-) diff --git a/aggregator/src/aggregation/circuit.rs b/aggregator/src/aggregation/circuit.rs index e73dd2e400..dc2d4c613e 100644 --- a/aggregator/src/aggregation/circuit.rs +++ b/aggregator/src/aggregation/circuit.rs @@ -252,17 +252,11 @@ impl Circuit for BatchCircuit { let barycentric = config.blob_consistency_config.assign_barycentric( &mut ctx, &coefficients, - // todo: check that the byte order is correct!!!! self.batch_hash .blob_consistency_witness .challenge() .0 .into(), - self.batch_hash - .blob_consistency_witness - .evaluation() - .0 - .into(), ); ctx.print_stats(&["barycentric"]); @@ -419,7 +413,6 @@ impl Circuit for BatchCircuit { challenges, &config.rlc_config, &self.batch_hash.blob_bytes, - barycentric_assignments, )?; BlobConsistencyConfig::::link( diff --git a/aggregator/src/data_availability/blob_data.rs b/aggregator/src/data_availability/blob_data.rs index 62a5a64659..e2820f20e9 100644 --- a/aggregator/src/data_availability/blob_data.rs +++ b/aggregator/src/data_availability/blob_data.rs @@ -4,7 +4,6 @@ use crate::{ RlcConfig, }; use gadgets::util::Expr; -use halo2_ecc::bigint::CRTInteger; use halo2_proofs::{ circuit::{AssignedCell, Layouter, Region, Value}, halo2curves::bn256::Fr, @@ -155,7 +154,6 @@ impl BlobDataConfig { challenge_value: Challenges>, rlc_config: &RlcConfig, blob_bytes: &[u8], - barycentric_assignments: &[CRTInteger], ) -> Result { let (assigned_bytes, bytes_rlc, bytes_len, enable_encoding_bool) = layouter.assign_region( || "BlobData bytes", @@ -166,13 +164,7 @@ impl BlobDataConfig { let (cooked_len, blob_crts_limbs) = layouter.assign_region( || "BlobData internal checks", |mut region| { - self.assign_internal_checks( - &mut region, - rlc_config, - barycentric_assignments, - &assigned_bytes, - &bytes_len, - ) + self.assign_internal_checks(&mut region, rlc_config, &assigned_bytes, &bytes_len) }, )?; @@ -296,7 +288,6 @@ impl BlobDataConfig { &self, region: &mut Region, rlc_config: &RlcConfig, - barycentric_assignments: &[CRTInteger], assigned_bytes: &[AssignedCell], bytes_len: &AssignedCell, ) -> Result< @@ -336,11 +327,8 @@ impl BlobDataConfig { // form of the batch. rlc_config.enforce_binary(region, &assigned_bytes[0], &mut rlc_config_offset)?; - //////////////////////////////////////////////////////////////////////////////// - //////////////////////////////////// LINKING /////////////////////////////////// - //////////////////////////////////////////////////////////////////////////////// - - assert_eq!(barycentric_assignments.len(), BLOB_WIDTH + 1); + // Compute 88 bit limbs so we can later check to equality to the inputs in the barycentric + // evaluation circuit. let blob_crts_limbs = blob_crts_limbs( region, rlc_config, diff --git a/aggregator/src/data_availability/eip4844.rs b/aggregator/src/data_availability/eip4844.rs index 0dec908b0d..d4b01236eb 100644 --- a/aggregator/src/data_availability/eip4844.rs +++ b/aggregator/src/data_availability/eip4844.rs @@ -122,7 +122,6 @@ impl BlobConsistencyConfig { ctx: &mut Context, blob: &[U256; BLOB_WIDTH], challenge: U256, - _: U256, ) -> AssignedBarycentricEvaluationConfig { self.barycentric_evaluation.assign(ctx, blob, challenge) } @@ -133,15 +132,9 @@ impl BlobConsistencyConfig { challenge_value: Challenges>, rlc_config: &RlcConfig, blob_bytes: &[u8], - barycentric_assignments: &[CRTInteger], ) -> Result { - self.data.assign( - layouter, - challenge_value, - rlc_config, - blob_bytes, - barycentric_assignments, - ) + self.data + .assign(layouter, challenge_value, rlc_config, blob_bytes) } pub fn link( diff --git a/aggregator/src/data_availability/eip4844/tests.rs b/aggregator/src/data_availability/eip4844/tests.rs index 8badec99e6..eeae13acef 100644 --- a/aggregator/src/data_availability/eip4844/tests.rs +++ b/aggregator/src/data_availability/eip4844/tests.rs @@ -183,13 +183,9 @@ impl Circuit for BlobCircuit { config.batch_data_config.load_range_tables(&mut layouter)?; - config.blob_data.assign( - &mut layouter, - challenge_values, - &config.rlc, - &blob_bytes, - &barycentric_assignments.barycentric_assignments, - )?; + config + .blob_data + .assign(&mut layouter, challenge_values, &config.rlc, &blob_bytes)?; layouter.assign_region( || "BatchDataConfig", From f949f983236ba23bacb0096eb82df0ea295de031 Mon Sep 17 00:00:00 2001 From: z2trillion Date: Mon, 21 Oct 2024 20:37:16 -0400 Subject: [PATCH 20/35] unpub get_coefficients --- aggregator/src/aggregation/circuit.rs | 4 +--- aggregator/src/data_availability.rs | 4 ++-- aggregator/src/data_availability/eip4844.rs | 6 +++--- aggregator/src/data_availability/eip4844/barycentric.rs | 6 ++++-- aggregator/src/data_availability/eip4844/tests.rs | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/aggregator/src/aggregation/circuit.rs b/aggregator/src/aggregation/circuit.rs index dc2d4c613e..b2b28a4d91 100644 --- a/aggregator/src/aggregation/circuit.rs +++ b/aggregator/src/aggregation/circuit.rs @@ -28,7 +28,6 @@ use crate::{ constants::{ACC_LEN, DIGEST_LEN}, core::{assign_batch_hashes, extract_proof_and_instances_with_pairing_check}, data_availability::{ - get_coefficients, // TODO: fix this BlobConsistencyConfig, }, util::parse_hash_digest_cells, @@ -248,10 +247,9 @@ impl Circuit for BatchCircuit { let mut ctx = Rc::into_inner(loader).unwrap().into_ctx(); log::debug!("batching: assigning barycentric"); - let coefficients = get_coefficients(&self.batch_hash.blob_bytes); let barycentric = config.blob_consistency_config.assign_barycentric( &mut ctx, - &coefficients, + &self.batch_hash.blob_bytes, self.batch_hash .blob_consistency_witness .challenge() diff --git a/aggregator/src/data_availability.rs b/aggregator/src/data_availability.rs index 3fc8a249d6..1f58745de3 100644 --- a/aggregator/src/data_availability.rs +++ b/aggregator/src/data_availability.rs @@ -17,10 +17,10 @@ mod eip4844; cfg_if! { if #[cfg(feature = "da-avail")] { // const DATA_AVAILABILITY: DataAvailibility = DataAvailibility::Avail; - pub use avail::{get_coefficients, BlobConsistencyConfig, BlobConsistencyWitness, BLOB_WIDTH}; + pub use avail::{BlobConsistencyConfig, BlobConsistencyWitness, BLOB_WIDTH}; } else if #[cfg(feature = "da-eip4844")] { // const DATA_AVAILABILITY: DataAvailibility = DataAvailibility::Eip4844; - pub use eip4844::{get_coefficients, BlobConsistencyConfig, BlobConsistencyWitness, BLOB_WIDTH}; + pub use eip4844::{BlobConsistencyConfig, BlobConsistencyWitness, BLOB_WIDTH}; } else { compile_error!("no da feature flag set"); } diff --git a/aggregator/src/data_availability/eip4844.rs b/aggregator/src/data_availability/eip4844.rs index d4b01236eb..2fcae133f5 100644 --- a/aggregator/src/data_availability/eip4844.rs +++ b/aggregator/src/data_availability/eip4844.rs @@ -36,7 +36,7 @@ pub const N_DATA_BYTES_PER_COEFFICIENT: usize = 31; pub const N_BLOB_BYTES: usize = BLOB_WIDTH * N_DATA_BYTES_PER_COEFFICIENT; /// Get the BLOB_WIDTH number of scalar field elements, as 32-bytes unsigned integers. -pub fn get_coefficients(blob_bytes: &[u8]) -> [U256; BLOB_WIDTH] { +fn get_coefficients(blob_bytes: &[u8]) -> [U256; BLOB_WIDTH] { let mut coefficients = [[0u8; N_BYTES_U256]; BLOB_WIDTH]; assert!( @@ -120,10 +120,10 @@ impl BlobConsistencyConfig { pub fn assign_barycentric( &self, ctx: &mut Context, - blob: &[U256; BLOB_WIDTH], + bytes: &[u8], challenge: U256, ) -> AssignedBarycentricEvaluationConfig { - self.barycentric_evaluation.assign(ctx, blob, challenge) + self.barycentric_evaluation.assign(ctx, bytes, challenge) } pub fn assign_blob_data( diff --git a/aggregator/src/data_availability/eip4844/barycentric.rs b/aggregator/src/data_availability/eip4844/barycentric.rs index 96ba6d7267..4187867069 100644 --- a/aggregator/src/data_availability/eip4844/barycentric.rs +++ b/aggregator/src/data_availability/eip4844/barycentric.rs @@ -19,7 +19,7 @@ use std::{iter::successors, sync::LazyLock}; use crate::{ constants::{BITS, LIMBS, N_BYTES_U256}, - data_availability::eip4844::blob::BLOB_WIDTH, + data_availability::eip4844::{get_coefficients, blob::BLOB_WIDTH}, }; /// Base 2 logarithm of BLOB_WIDTH. @@ -102,9 +102,11 @@ impl BarycentricEvaluationConfig { pub fn assign( &self, ctx: &mut Context, - blob: &[U256; BLOB_WIDTH], + bytes: &[u8], challenge_digest: U256, ) -> AssignedBarycentricEvaluationConfig { + let blob = get_coefficients(bytes); + // some constants for later use. let one = self.scalar.load_constant(ctx, fe_to_biguint(&Fr::one())); let blob_width = self diff --git a/aggregator/src/data_availability/eip4844/tests.rs b/aggregator/src/data_availability/eip4844/tests.rs index eeae13acef..29afb2d313 100644 --- a/aggregator/src/data_availability/eip4844/tests.rs +++ b/aggregator/src/data_availability/eip4844/tests.rs @@ -154,7 +154,7 @@ impl Circuit for BlobCircuit { PointEvaluationAssignments::new(&self.data, &blob_bytes, versioned_hash); Ok(config.barycentric.assign( &mut ctx, - &point_eval.coefficients, + &blob_bytes, point_eval.challenge_digest, )) }, From 6e6ced89b47f9a6553c6ff16b9a01d3d5305cfdf Mon Sep 17 00:00:00 2001 From: z2trillion Date: Mon, 21 Oct 2024 21:06:54 -0400 Subject: [PATCH 21/35] rename data_availability to blob_consistency --- aggregator/src/aggregation/batch_data.rs | 2 +- aggregator/src/aggregation/circuit.rs | 4 +--- aggregator/src/aggregation/config.rs | 2 +- aggregator/src/batch.rs | 2 +- .../src/{data_availability.rs => blob_consistency.rs} | 0 .../{data_availability => blob_consistency}/avail.rs | 0 .../blob_data.rs | 2 +- .../{data_availability => blob_consistency}/eip4844.rs | 0 .../eip4844/barycentric.rs | 9 +++------ .../eip4844/blob.rs | 2 +- .../eip4844/tests.rs | 10 ++++------ aggregator/src/lib.rs | 4 ++-- 12 files changed, 15 insertions(+), 22 deletions(-) rename aggregator/src/{data_availability.rs => blob_consistency.rs} (100%) rename aggregator/src/{data_availability => blob_consistency}/avail.rs (100%) rename aggregator/src/{data_availability => blob_consistency}/blob_data.rs (99%) rename aggregator/src/{data_availability => blob_consistency}/eip4844.rs (100%) rename aggregator/src/{data_availability => blob_consistency}/eip4844/barycentric.rs (98%) rename aggregator/src/{data_availability => blob_consistency}/eip4844/blob.rs (98%) rename aggregator/src/{data_availability => blob_consistency}/eip4844/tests.rs (98%) diff --git a/aggregator/src/aggregation/batch_data.rs b/aggregator/src/aggregation/batch_data.rs index deeb819fc0..221a059e5b 100644 --- a/aggregator/src/aggregation/batch_data.rs +++ b/aggregator/src/aggregation/batch_data.rs @@ -1,5 +1,5 @@ use crate::{ - aggregation::rlc::POWS_OF_256, constants::N_BYTES_U256, data_availability::BLOB_WIDTH, + aggregation::rlc::POWS_OF_256, blob_consistency::BLOB_WIDTH, constants::N_BYTES_U256, BatchHash, ChunkInfo, RlcConfig, }; use eth_types::{H256, U256}; diff --git a/aggregator/src/aggregation/circuit.rs b/aggregator/src/aggregation/circuit.rs index b2b28a4d91..29fa0873a2 100644 --- a/aggregator/src/aggregation/circuit.rs +++ b/aggregator/src/aggregation/circuit.rs @@ -25,11 +25,9 @@ use zkevm_circuits::util::Challenges; use crate::{ aggregation::{decoder::WORKED_EXAMPLE, witgen::process, BatchCircuitConfig, BatchData}, batch::BatchHash, + blob_consistency::BlobConsistencyConfig, constants::{ACC_LEN, DIGEST_LEN}, core::{assign_batch_hashes, extract_proof_and_instances_with_pairing_check}, - data_availability::{ - BlobConsistencyConfig, - }, util::parse_hash_digest_cells, witgen::{zstd_encode, MultiBlockProcessResult}, ConfigParams, LOG_DEGREE, PI_CHAIN_ID, PI_CURRENT_BATCH_HASH, PI_CURRENT_STATE_ROOT, diff --git a/aggregator/src/aggregation/config.rs b/aggregator/src/aggregation/config.rs index 6e6052c5c0..e6eab4930d 100644 --- a/aggregator/src/aggregation/config.rs +++ b/aggregator/src/aggregation/config.rs @@ -17,8 +17,8 @@ use zkevm_circuits::{ }; use crate::{ + blob_consistency::BlobConsistencyConfig, constants::{BITS, LIMBS}, - data_availability::BlobConsistencyConfig, param::ConfigParams, BatchDataConfig, DecoderConfig, DecoderConfigArgs, RlcConfig, }; diff --git a/aggregator/src/batch.rs b/aggregator/src/batch.rs index 99d9c7e3fc..5b411ac6b9 100644 --- a/aggregator/src/batch.rs +++ b/aggregator/src/batch.rs @@ -6,7 +6,7 @@ use ethers_core::utils::keccak256; use gadgets::{util::split_h256, Field}; use serde::{Deserialize, Serialize}; -use crate::{aggregation::BatchData, chunk::ChunkInfo, data_availability::BlobConsistencyWitness}; +use crate::{aggregation::BatchData, blob_consistency::BlobConsistencyWitness, chunk::ChunkInfo}; /// Batch header provides additional fields from the context (within recursion) /// for constructing the preimage of the batch hash. diff --git a/aggregator/src/data_availability.rs b/aggregator/src/blob_consistency.rs similarity index 100% rename from aggregator/src/data_availability.rs rename to aggregator/src/blob_consistency.rs diff --git a/aggregator/src/data_availability/avail.rs b/aggregator/src/blob_consistency/avail.rs similarity index 100% rename from aggregator/src/data_availability/avail.rs rename to aggregator/src/blob_consistency/avail.rs diff --git a/aggregator/src/data_availability/blob_data.rs b/aggregator/src/blob_consistency/blob_data.rs similarity index 99% rename from aggregator/src/data_availability/blob_data.rs rename to aggregator/src/blob_consistency/blob_data.rs index e2820f20e9..fccb97e6ba 100644 --- a/aggregator/src/data_availability/blob_data.rs +++ b/aggregator/src/blob_consistency/blob_data.rs @@ -1,6 +1,6 @@ use crate::{ aggregation::{batch_data::N_BLOB_BYTES, POWS_OF_256}, - data_availability::{eip4844::N_DATA_BYTES_PER_COEFFICIENT, BLOB_WIDTH}, + blob_consistency::{eip4844::N_DATA_BYTES_PER_COEFFICIENT, BLOB_WIDTH}, RlcConfig, }; use gadgets::util::Expr; diff --git a/aggregator/src/data_availability/eip4844.rs b/aggregator/src/blob_consistency/eip4844.rs similarity index 100% rename from aggregator/src/data_availability/eip4844.rs rename to aggregator/src/blob_consistency/eip4844.rs diff --git a/aggregator/src/data_availability/eip4844/barycentric.rs b/aggregator/src/blob_consistency/eip4844/barycentric.rs similarity index 98% rename from aggregator/src/data_availability/eip4844/barycentric.rs rename to aggregator/src/blob_consistency/eip4844/barycentric.rs index 4187867069..af8afaba83 100644 --- a/aggregator/src/data_availability/eip4844/barycentric.rs +++ b/aggregator/src/blob_consistency/eip4844/barycentric.rs @@ -16,11 +16,8 @@ use halo2_proofs::{ use itertools::Itertools; use num_bigint::{BigInt, Sign}; use std::{iter::successors, sync::LazyLock}; - -use crate::{ - constants::{BITS, LIMBS, N_BYTES_U256}, - data_availability::eip4844::{get_coefficients, blob::BLOB_WIDTH}, -}; +use super::{blob::BLOB_WIDTH, get_coefficients}; +use crate::constants::{BITS, LIMBS, N_BYTES_U256}; /// Base 2 logarithm of BLOB_WIDTH. const LOG_BLOB_WIDTH: usize = 12; @@ -358,7 +355,7 @@ mod tests { use super::*; use crate::{ aggregation::BatchData, - data_availability::eip4844::{get_blob_bytes, get_coefficients, KZG_TRUSTED_SETUP}, + blob_consistency::eip4844::{get_blob_bytes, get_coefficients, KZG_TRUSTED_SETUP}, MAX_AGG_SNARKS, }; use c_kzg::{Blob as RethBlob, KzgProof}; diff --git a/aggregator/src/data_availability/eip4844/blob.rs b/aggregator/src/blob_consistency/eip4844/blob.rs similarity index 98% rename from aggregator/src/data_availability/eip4844/blob.rs rename to aggregator/src/blob_consistency/eip4844/blob.rs index bb37501b4d..aa153e2689 100644 --- a/aggregator/src/data_availability/eip4844/blob.rs +++ b/aggregator/src/blob_consistency/eip4844/blob.rs @@ -68,7 +68,7 @@ impl PointEvaluationAssignments { mod tests { use super::*; use crate::{ - data_availability::eip4844::{get_blob_bytes, get_versioned_hash}, + blob_consistency::eip4844::{get_blob_bytes, get_versioned_hash}, BatchHash, ChunkInfo, MAX_AGG_SNARKS, }; use ethers_core::utils::keccak256; diff --git a/aggregator/src/data_availability/eip4844/tests.rs b/aggregator/src/blob_consistency/eip4844/tests.rs similarity index 98% rename from aggregator/src/data_availability/eip4844/tests.rs rename to aggregator/src/blob_consistency/eip4844/tests.rs index 29afb2d313..976ff10eed 100644 --- a/aggregator/src/data_availability/eip4844/tests.rs +++ b/aggregator/src/blob_consistency/eip4844/tests.rs @@ -23,7 +23,7 @@ use crate::{ BatchData, RlcConfig, }, constants::N_BYTES_U256, - data_availability::{ + blob_consistency::{ eip4844::{ blob::PointEvaluationAssignments, get_blob_bytes, get_coefficients, get_versioned_hash, AssignedBarycentricEvaluationConfig, BarycentricEvaluationConfig, @@ -152,11 +152,9 @@ impl Circuit for BlobCircuit { let point_eval = PointEvaluationAssignments::new(&self.data, &blob_bytes, versioned_hash); - Ok(config.barycentric.assign( - &mut ctx, - &blob_bytes, - point_eval.challenge_digest, - )) + Ok(config + .barycentric + .assign(&mut ctx, &blob_bytes, point_eval.challenge_digest)) }, )?; diff --git a/aggregator/src/lib.rs b/aggregator/src/lib.rs index b4bdd55fd2..681611e877 100644 --- a/aggregator/src/lib.rs +++ b/aggregator/src/lib.rs @@ -8,6 +8,8 @@ mod batch; mod recursion; // This module implements `Chunk` related data types. // A chunk is a list of blocks. +/// Blob consistency checks +pub mod blob_consistency; mod chunk; /// proof compression mod compression; @@ -15,8 +17,6 @@ mod compression; mod constants; /// Core module for circuit assignment mod core; -/// Data availibility checks -pub mod data_availability; /// Parameters for compression circuit mod param; /// utilities From de4c88ff5d468fc59ce0eb87ec36c752bb5885c4 Mon Sep 17 00:00:00 2001 From: z2trillion Date: Mon, 21 Oct 2024 21:13:03 -0400 Subject: [PATCH 22/35] remove da-eip4844 feature flag --- aggregator/Cargo.toml | 3 +-- aggregator/src/blob_consistency.rs | 6 ++---- aggregator/src/blob_consistency/eip4844/barycentric.rs | 4 ++-- aggregator/src/blob_consistency/eip4844/tests.rs | 2 +- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/aggregator/Cargo.toml b/aggregator/Cargo.toml index 6a1f867ac6..17cd9b7d5b 100644 --- a/aggregator/Cargo.toml +++ b/aggregator/Cargo.toml @@ -45,8 +45,7 @@ zstd-encoder = { package = "encoder", git = "https://github.com/scroll-tech/da-c csv = "1.1" [features] -default = ["revm-precompile/c-kzg", "halo2_proofs/circuit-params", "da-eip4844"] +default = ["revm-precompile/c-kzg", "halo2_proofs/circuit-params"] display = [] print-trace = ["ark-std/print-trace"] da-avail = [] -da-eip4844 = [] diff --git a/aggregator/src/blob_consistency.rs b/aggregator/src/blob_consistency.rs index 1f58745de3..70e35771a3 100644 --- a/aggregator/src/blob_consistency.rs +++ b/aggregator/src/blob_consistency.rs @@ -11,17 +11,15 @@ use blob_data::{AssignedBlobDataExport, BlobDataConfig}; #[cfg(feature = "da-avail")] mod avail; -#[cfg(feature = "da-eip4844")] +#[cfg(not(feature = "da-avail"))] mod eip4844; cfg_if! { if #[cfg(feature = "da-avail")] { // const DATA_AVAILABILITY: DataAvailibility = DataAvailibility::Avail; pub use avail::{BlobConsistencyConfig, BlobConsistencyWitness, BLOB_WIDTH}; - } else if #[cfg(feature = "da-eip4844")] { + } else { // const DATA_AVAILABILITY: DataAvailibility = DataAvailibility::Eip4844; pub use eip4844::{BlobConsistencyConfig, BlobConsistencyWitness, BLOB_WIDTH}; - } else { - compile_error!("no da feature flag set"); } } diff --git a/aggregator/src/blob_consistency/eip4844/barycentric.rs b/aggregator/src/blob_consistency/eip4844/barycentric.rs index af8afaba83..c46f20c068 100644 --- a/aggregator/src/blob_consistency/eip4844/barycentric.rs +++ b/aggregator/src/blob_consistency/eip4844/barycentric.rs @@ -1,3 +1,5 @@ +use super::{blob::BLOB_WIDTH, get_coefficients}; +use crate::constants::{BITS, LIMBS, N_BYTES_U256}; use eth_types::{ToLittleEndian, U256}; use halo2_base::{ gates::{range::RangeConfig, GateInstructions}, @@ -16,8 +18,6 @@ use halo2_proofs::{ use itertools::Itertools; use num_bigint::{BigInt, Sign}; use std::{iter::successors, sync::LazyLock}; -use super::{blob::BLOB_WIDTH, get_coefficients}; -use crate::constants::{BITS, LIMBS, N_BYTES_U256}; /// Base 2 logarithm of BLOB_WIDTH. const LOG_BLOB_WIDTH: usize = 12; diff --git a/aggregator/src/blob_consistency/eip4844/tests.rs b/aggregator/src/blob_consistency/eip4844/tests.rs index 976ff10eed..fa828060a2 100644 --- a/aggregator/src/blob_consistency/eip4844/tests.rs +++ b/aggregator/src/blob_consistency/eip4844/tests.rs @@ -22,7 +22,6 @@ use crate::{ witgen::{process, MultiBlockProcessResult}, BatchData, RlcConfig, }, - constants::N_BYTES_U256, blob_consistency::{ eip4844::{ blob::PointEvaluationAssignments, get_blob_bytes, get_coefficients, get_versioned_hash, @@ -30,6 +29,7 @@ use crate::{ }, BlobDataConfig, }, + constants::N_BYTES_U256, decode_bytes, param::ConfigParams, BatchDataConfig, ChunkInfo, MAX_AGG_SNARKS, From 9706829ed5aaaa8fa1327cc89037522cf9d8170f Mon Sep 17 00:00:00 2001 From: z2trillion Date: Mon, 21 Oct 2024 21:46:32 -0400 Subject: [PATCH 23/35] define N_DATA_BYTES_PER_COEFFICIENT once --- aggregator/src/blob_consistency/blob_data.rs | 7 +++++-- aggregator/src/blob_consistency/eip4844.rs | 7 ++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/aggregator/src/blob_consistency/blob_data.rs b/aggregator/src/blob_consistency/blob_data.rs index fccb97e6ba..fd1aa8ded0 100644 --- a/aggregator/src/blob_consistency/blob_data.rs +++ b/aggregator/src/blob_consistency/blob_data.rs @@ -1,6 +1,9 @@ use crate::{ - aggregation::{batch_data::N_BLOB_BYTES, POWS_OF_256}, - blob_consistency::{eip4844::N_DATA_BYTES_PER_COEFFICIENT, BLOB_WIDTH}, + aggregation::{ + batch_data::{N_BLOB_BYTES, N_DATA_BYTES_PER_COEFFICIENT}, + POWS_OF_256, + }, + blob_consistency::BLOB_WIDTH, RlcConfig, }; use gadgets::util::Expr; diff --git a/aggregator/src/blob_consistency/eip4844.rs b/aggregator/src/blob_consistency/eip4844.rs index 2fcae133f5..1683a5a491 100644 --- a/aggregator/src/blob_consistency/eip4844.rs +++ b/aggregator/src/blob_consistency/eip4844.rs @@ -12,7 +12,10 @@ use blob::PointEvaluationAssignments; mod tests; use super::{AssignedBlobDataExport, BlobDataConfig}; -use crate::{constants::N_BYTES_U256, BatchData, RlcConfig}; +use crate::{ + aggregation::batch_data::N_DATA_BYTES_PER_COEFFICIENT, constants::N_BYTES_U256, BatchData, + RlcConfig, +}; use eth_types::{ToBigEndian, H256, U256}; use ethers_core::k256::sha2::{Digest, Sha256}; use halo2_base::{gates::range::RangeConfig, Context}; @@ -31,8 +34,6 @@ use std::sync::Arc; use zkevm_circuits::{table::U8Table, util::Challenges}; pub const BLOB_WIDTH: usize = 4096; -/// The number data bytes we pack each BLS12-381 scalar into. The most-significant byte is 0. -pub const N_DATA_BYTES_PER_COEFFICIENT: usize = 31; pub const N_BLOB_BYTES: usize = BLOB_WIDTH * N_DATA_BYTES_PER_COEFFICIENT; /// Get the BLOB_WIDTH number of scalar field elements, as 32-bytes unsigned integers. From 2fc92637abf7809786fe196c27262c2cd85c3d69 Mon Sep 17 00:00:00 2001 From: z2trillion Date: Mon, 21 Oct 2024 21:49:48 -0400 Subject: [PATCH 24/35] fix da-avail build --- aggregator/src/blob_consistency/avail.rs | 35 ++++++++++-------------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/aggregator/src/blob_consistency/avail.rs b/aggregator/src/blob_consistency/avail.rs index 93e4f23e05..5df1b1b9f1 100644 --- a/aggregator/src/blob_consistency/avail.rs +++ b/aggregator/src/blob_consistency/avail.rs @@ -11,17 +11,12 @@ use halo2_proofs::{ }; use serde::{Deserialize, Serialize}; use zkevm_circuits::{table::U8Table, util::Challenges}; +use snark_verifier_sdk::LIMBS; pub const BLOB_WIDTH: usize = 4096; -pub fn get_coefficients(bytes: &[u8]) -> [U256; BLOB_WIDTH] { - [U256::zero(); BLOB_WIDTH] -} - #[derive(Debug, Clone)] -pub struct BlobConsistencyConfig { - data: BlobDataConfig, -} +pub struct BlobConsistencyConfig {} impl BlobConsistencyConfig { pub fn construct( @@ -30,19 +25,16 @@ impl BlobConsistencyConfig { u8_table: U8Table, _: RangeConfig, ) -> Self { - Self { - data: BlobDataConfig::configure(meta, challenges, u8_table), - } + unimplemented!() } pub fn assign_barycentric( &self, ctx: &mut Context, - blob: &[U256; BLOB_WIDTH], + bytes: &[u8], challenge: U256, - evaluation: U256, ) -> AssignedBarycentricEvaluationConfig { - Default::default() + unimplemented!() } pub fn assign_blob_data( @@ -51,15 +43,16 @@ impl BlobConsistencyConfig { challenge_value: Challenges>, rlc_config: &RlcConfig, blob_bytes: &[u8], - barycentric_assignments: &[CRTInteger], ) -> Result { - self.data.assign( - layouter, - challenge_value, - rlc_config, - blob_bytes, - barycentric_assignments, - ) + unimplemented!() + } + + pub fn link( + layouter: &mut impl Layouter, + blob_crts_limbs: &[[AssignedCell; LIMBS]], + barycentric_crts: &[CRTInteger], + ) -> Result<(), Error> { + unimplemented!() } } From 76a6a70374200a6a666e1ad47085f09dd6dc5fe9 Mon Sep 17 00:00:00 2001 From: z2trillion Date: Mon, 21 Oct 2024 22:22:53 -0400 Subject: [PATCH 25/35] fix typos --- aggregator/src/batch.rs | 2 +- aggregator/src/blob_consistency.rs | 6 +++--- aggregator/src/blob_consistency/avail.rs | 2 +- aggregator/src/blob_consistency/blob_data.rs | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/aggregator/src/batch.rs b/aggregator/src/batch.rs index 5b411ac6b9..d55f24aa97 100644 --- a/aggregator/src/batch.rs +++ b/aggregator/src/batch.rs @@ -26,7 +26,7 @@ pub struct BatchHeader { pub last_block_timestamp: u64, /// The data hash of the batch pub data_hash: H256, - // /// information needed to check that the blobs in the circuit match the blob the data availibility provided. + // /// information needed to check that the blobs in the circuit match the blob the data availability provided. #[serde(flatten)] pub blob_consistency_witness: BlobConsistencyWitness, } diff --git a/aggregator/src/blob_consistency.rs b/aggregator/src/blob_consistency.rs index 70e35771a3..055ad2fe08 100644 --- a/aggregator/src/blob_consistency.rs +++ b/aggregator/src/blob_consistency.rs @@ -1,6 +1,6 @@ use cfg_if::cfg_if; -// enum DataAvailibility { +// enum DataAvailability { // Eip4844, // Avail, // } @@ -16,10 +16,10 @@ mod eip4844; cfg_if! { if #[cfg(feature = "da-avail")] { - // const DATA_AVAILABILITY: DataAvailibility = DataAvailibility::Avail; + // const DATA_AVAILABILITY: DataAvailability = DataAvailability::Avail; pub use avail::{BlobConsistencyConfig, BlobConsistencyWitness, BLOB_WIDTH}; } else { - // const DATA_AVAILABILITY: DataAvailibility = DataAvailibility::Eip4844; + // const DATA_AVAILABILITY: DatayAvailability = DataAvailability::Eip4844; pub use eip4844::{BlobConsistencyConfig, BlobConsistencyWitness, BLOB_WIDTH}; } } diff --git a/aggregator/src/blob_consistency/avail.rs b/aggregator/src/blob_consistency/avail.rs index 5df1b1b9f1..6615eb859a 100644 --- a/aggregator/src/blob_consistency/avail.rs +++ b/aggregator/src/blob_consistency/avail.rs @@ -10,8 +10,8 @@ use halo2_proofs::{ plonk::{ConstraintSystem, Error, Expression}, }; use serde::{Deserialize, Serialize}; -use zkevm_circuits::{table::U8Table, util::Challenges}; use snark_verifier_sdk::LIMBS; +use zkevm_circuits::{table::U8Table, util::Challenges}; pub const BLOB_WIDTH: usize = 4096; diff --git a/aggregator/src/blob_consistency/blob_data.rs b/aggregator/src/blob_consistency/blob_data.rs index fd1aa8ded0..0325f1813d 100644 --- a/aggregator/src/blob_consistency/blob_data.rs +++ b/aggregator/src/blob_consistency/blob_data.rs @@ -361,7 +361,7 @@ fn blob_crts_limbs( .map(|coefficient_bytes| { coefficient_bytes .iter() - .rev() // reverse bytes to match endianess of crt limbs + .rev() // reverse bytes to match endianness of crt limbs .chunks(BITS / 8) .into_iter() .map(|chunk_bytes| { From 7994fe0f9a7853af073c0fe9a939104b75fdddc3 Mon Sep 17 00:00:00 2001 From: z2trillion Date: Tue, 22 Oct 2024 09:38:23 -0400 Subject: [PATCH 26/35] allow dead code for now --- aggregator/src/blob_consistency.rs | 6 ++-- aggregator/src/blob_consistency/avail.rs | 35 ++++++++++++------------ 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/aggregator/src/blob_consistency.rs b/aggregator/src/blob_consistency.rs index 055ad2fe08..b49f676011 100644 --- a/aggregator/src/blob_consistency.rs +++ b/aggregator/src/blob_consistency.rs @@ -8,10 +8,12 @@ use cfg_if::cfg_if; mod blob_data; use blob_data::{AssignedBlobDataExport, BlobDataConfig}; -#[cfg(feature = "da-avail")] +// TODO: remove dead code instead +#[allow(dead_code)] mod avail; -#[cfg(not(feature = "da-avail"))] +// TODO: remove dead code instead +#[allow(dead_code)] mod eip4844; cfg_if! { diff --git a/aggregator/src/blob_consistency/avail.rs b/aggregator/src/blob_consistency/avail.rs index 6615eb859a..0b62d28745 100644 --- a/aggregator/src/blob_consistency/avail.rs +++ b/aggregator/src/blob_consistency/avail.rs @@ -1,7 +1,6 @@ -use super::{AssignedBlobDataExport, BlobDataConfig}; -use crate::{constants::N_BYTES_U256, BatchData, RlcConfig}; -use eth_types::{ToBigEndian, H256, U256}; -use ethers_core::k256::sha2::{Digest, Sha256}; +use super::{AssignedBlobDataExport}; +use crate::{BatchData, RlcConfig}; +use eth_types::{H256, U256}; use halo2_base::{gates::range::RangeConfig, AssignedValue, Context}; use halo2_ecc::bigint::CRTInteger; use halo2_proofs::{ @@ -20,9 +19,9 @@ pub struct BlobConsistencyConfig {} impl BlobConsistencyConfig { pub fn construct( - meta: &mut ConstraintSystem, - challenges: &Challenges>, - u8_table: U8Table, + _meta: &mut ConstraintSystem, + _challenges: &Challenges>, + _u8_table: U8Table, _: RangeConfig, ) -> Self { unimplemented!() @@ -30,27 +29,27 @@ impl BlobConsistencyConfig { pub fn assign_barycentric( &self, - ctx: &mut Context, - bytes: &[u8], - challenge: U256, + _ctx: &mut Context, + _bytes: &[u8], + _challenge: U256, ) -> AssignedBarycentricEvaluationConfig { unimplemented!() } pub fn assign_blob_data( &self, - layouter: &mut impl Layouter, - challenge_value: Challenges>, - rlc_config: &RlcConfig, - blob_bytes: &[u8], + _layouter: &mut impl Layouter, + _challenge_value: Challenges>, + _rlc_config: &RlcConfig, + _blob_bytes: &[u8], ) -> Result { unimplemented!() } pub fn link( - layouter: &mut impl Layouter, - blob_crts_limbs: &[[AssignedCell; LIMBS]], - barycentric_crts: &[CRTInteger], + _layouter: &mut impl Layouter, + _blob_crts_limbs: &[[AssignedCell; LIMBS]], + _barycentric_crts: &[CRTInteger], ) -> Result<(), Error> { unimplemented!() } @@ -64,7 +63,7 @@ pub struct BlobConsistencyWitness { } impl BlobConsistencyWitness { - pub fn new(bytes: &[u8], _: &BatchData) -> Self { + pub fn new(_bytes: &[u8], _: &BatchData) -> Self { Self { id: H256::default(), // should be keccak of bytes blob_data_proof: Default::default(), From ca9111089c5892805725f2fcb6c49b05aaf0fb3c Mon Sep 17 00:00:00 2001 From: z2trillion Date: Mon, 28 Oct 2024 22:03:13 -0400 Subject: [PATCH 27/35] cleanup comments --- aggregator/src/batch.rs | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/aggregator/src/batch.rs b/aggregator/src/batch.rs index d55f24aa97..a48bbc323e 100644 --- a/aggregator/src/batch.rs +++ b/aggregator/src/batch.rs @@ -84,11 +84,6 @@ impl BatchHeader { last_block_timestamp, data_hash: batch_data_hash.into(), blob_consistency_witness, - // blob_versioned_hash, - // blob_data_proof: [ - // H256::from_slice(&point_evaluation_assignments.challenge.to_be_bytes()), - // H256::from_slice(&point_evaluation_assignments.evaluation.to_be_bytes()), - // ], } } @@ -161,15 +156,11 @@ pub struct BatchHash { pub(crate) current_batch_hash: H256, /// The number of chunks that contain meaningful data, i.e. not padded chunks. pub(crate) number_of_valid_chunks: usize, - // /// 4844 point evaluation check related assignments. - // pub(crate) point_evaluation_assignments: PointEvaluationAssignments, - // /// The 4844 versioned hash for the blob. - // pub(crate) versioned_hash: H256, /// The context batch header pub(crate) batch_header: BatchHeader, /// The blob bytes (may be encoded batch bytes, or may be raw batch bytes). pub(crate) blob_bytes: Vec, - + /// Witness data to prove that the blob used as advice in the circuit matches the blob from the data availability layer. pub blob_consistency_witness: BlobConsistencyWitness, } From 36d44aa0e569658a43bbe933b2e069903b3c3890 Mon Sep 17 00:00:00 2001 From: z2trillion Date: Wed, 30 Oct 2024 13:42:29 -0400 Subject: [PATCH 28/35] Don't use BlobConsistencyWitness in BatchHeader --- aggregator/src/batch.rs | 38 ++++++++++-------------- aggregator/src/blob_consistency/avail.rs | 2 +- prover/src/aggregator/prover.rs | 18 ++++------- 3 files changed, 23 insertions(+), 35 deletions(-) diff --git a/aggregator/src/batch.rs b/aggregator/src/batch.rs index a48bbc323e..e5e53d626a 100644 --- a/aggregator/src/batch.rs +++ b/aggregator/src/batch.rs @@ -26,9 +26,10 @@ pub struct BatchHeader { pub last_block_timestamp: u64, /// The data hash of the batch pub data_hash: H256, - // /// information needed to check that the blobs in the circuit match the blob the data availability provided. - #[serde(flatten)] - pub blob_consistency_witness: BlobConsistencyWitness, + /// The versioned hash of the blob with this batch's data + pub blob_versioned_hash: H256, + /// The blob data proof: z (32), y (32) + pub blob_data_proof: [H256; 2], } impl BatchHeader { @@ -83,7 +84,11 @@ impl BatchHeader { parent_batch_hash, last_block_timestamp, data_hash: batch_data_hash.into(), - blob_consistency_witness, + blob_versioned_hash: blob_consistency_witness.id(), + blob_data_proof: [ + blob_consistency_witness.challenge(), + blob_consistency_witness.evaluation(), + ], } } @@ -108,17 +113,11 @@ impl BatchHeader { self.l1_message_popped.to_be_bytes().as_ref(), self.total_l1_message_popped.to_be_bytes().as_ref(), self.data_hash.as_bytes(), - self.blob_consistency_witness.id().as_bytes(), + self.blob_versioned_hash.as_bytes(), self.parent_batch_hash.as_bytes(), self.last_block_timestamp.to_be_bytes().as_ref(), - self.blob_consistency_witness - .challenge() - .to_fixed_bytes() - .as_ref(), - self.blob_consistency_witness - .evaluation() - .to_fixed_bytes() - .as_ref(), + self.blob_data_proof[0].to_fixed_bytes().as_ref(), + self.blob_data_proof[1].to_fixed_bytes().as_ref(), ] .concat(); keccak256(batch_hash_preimage).into() @@ -338,20 +337,16 @@ impl BatchHash { .to_be_bytes() .as_ref(), self.data_hash.as_bytes(), - self.batch_header.blob_consistency_witness.id().as_bytes(), + self.batch_header.blob_versioned_hash.as_bytes(), self.batch_header.parent_batch_hash.as_bytes(), self.batch_header .last_block_timestamp .to_be_bytes() .as_ref(), - self.batch_header - .blob_consistency_witness - .challenge() + self.batch_header.blob_data_proof[0] .to_fixed_bytes() .as_ref(), - self.batch_header - .blob_consistency_witness - .evaluation() + self.batch_header.blob_data_proof[1] .to_fixed_bytes() .as_ref(), ] @@ -390,8 +385,7 @@ impl BatchHash { // - preimage for each chunk's flattened L2 signed tx data // - preimage for the challenge digest let batch_data = BatchData::from(self); - let dynamic_preimages = - batch_data.preimages(self.batch_header.blob_consistency_witness.id()); + let dynamic_preimages = batch_data.preimages(self.batch_header.blob_versioned_hash); for dynamic_preimage in dynamic_preimages { res.push(dynamic_preimage); } diff --git a/aggregator/src/blob_consistency/avail.rs b/aggregator/src/blob_consistency/avail.rs index 0b62d28745..2b21c4c339 100644 --- a/aggregator/src/blob_consistency/avail.rs +++ b/aggregator/src/blob_consistency/avail.rs @@ -1,4 +1,4 @@ -use super::{AssignedBlobDataExport}; +use super::AssignedBlobDataExport; use crate::{BatchData, RlcConfig}; use eth_types::{H256, U256}; use halo2_base::{gates::range::RangeConfig, AssignedValue, Context}; diff --git a/prover/src/aggregator/prover.rs b/prover/src/aggregator/prover.rs index 5721bab293..556fa14b13 100644 --- a/prover/src/aggregator/prover.rs +++ b/prover/src/aggregator/prover.rs @@ -183,25 +183,19 @@ impl<'params> Prover<'params> { batch.batch_header.data_hash, batch_header.data_hash ); assert_eq!( - batch_header.blob_consistency_witness.challenge(), - batch.batch_header.blob_consistency_witness.challenge(), + batch_header.blob_data_proof[0], batch.batch_header.blob_data_proof[0], "BatchHeader(sanity) mismatch blob data proof (z) expected={}, got={}", - batch_header.blob_consistency_witness.challenge(), - batch.batch_header.blob_consistency_witness.challenge(), + batch_header.blob_data_proof[0], batch.batch_header.blob_data_proof[0], ); assert_eq!( - batch_header.blob_consistency_witness.evaluation(), - batch.batch_header.blob_consistency_witness.evaluation(), + batch_header.blob_data_proof[1], batch.batch_header.blob_data_proof[1], "BatchHeader(sanity) mismatch blob data proof (y) expected={}, got={}", - batch_header.blob_consistency_witness.evaluation(), - batch.batch_header.blob_consistency_witness.evaluation(), + batch_header.blob_data_proof[1], batch.batch_header.blob_data_proof[1], ); assert_eq!( - batch_header.blob_consistency_witness.id(), - batch.batch_header.blob_consistency_witness.id(), + batch_header.blob_versioned_hash, batch.batch_header.blob_versioned_hash, "BatchHeader(sanity) mismatch blob versioned hash expected={}, got={}", - batch_header.blob_consistency_witness.id(), - batch.batch_header.blob_consistency_witness.id(), + batch_header.blob_versioned_hash, batch.batch_header.blob_versioned_hash, ); let batch_hash = batch_header.batch_hash(); From 1581f5adf7f1d0805cfd44842b51481d4c45ac42 Mon Sep 17 00:00:00 2001 From: z2trillion Date: Wed, 30 Oct 2024 13:44:11 -0400 Subject: [PATCH 29/35] don't derive (De)serialize --- aggregator/src/blob_consistency/eip4844.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/aggregator/src/blob_consistency/eip4844.rs b/aggregator/src/blob_consistency/eip4844.rs index 1683a5a491..ba5848dc0b 100644 --- a/aggregator/src/blob_consistency/eip4844.rs +++ b/aggregator/src/blob_consistency/eip4844.rs @@ -28,7 +28,6 @@ use halo2_proofs::{ use itertools::Itertools; use once_cell::sync::Lazy; use revm_primitives::VERSIONED_HASH_VERSION_KZG; -use serde::{Deserialize, Serialize}; use snark_verifier_sdk::LIMBS; use std::sync::Arc; use zkevm_circuits::{table::U8Table, util::Challenges}; @@ -164,9 +163,8 @@ impl BlobConsistencyConfig { } } -#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, Default)] pub struct BlobConsistencyWitness { - #[serde(rename = "blob_versioned_hash")] id: H256, blob_data_proof: [H256; 2], } From 67e21072d305769ff92b24ad875f64c0c86e6cc0 Mon Sep 17 00:00:00 2001 From: z2trillion Date: Wed, 30 Oct 2024 15:19:15 -0400 Subject: [PATCH 30/35] add conversion functions --- aggregator/src/aggregation/circuit.rs | 13 ++-- aggregator/src/batch.rs | 9 +-- aggregator/src/blob_consistency/eip4844.rs | 66 ++++++++++++++----- .../src/blob_consistency/eip4844/tests.rs | 16 +++++ 4 files changed, 75 insertions(+), 29 deletions(-) diff --git a/aggregator/src/aggregation/circuit.rs b/aggregator/src/aggregation/circuit.rs index 29fa0873a2..cf1d41cdf4 100644 --- a/aggregator/src/aggregation/circuit.rs +++ b/aggregator/src/aggregation/circuit.rs @@ -1,4 +1,5 @@ use ark_std::{end_timer, start_timer}; +use eth_types::U256; use halo2_proofs::{ circuit::{Layouter, SimpleFloorPlanner, Value}, halo2curves::bn256::{Bn256, Fr, G1Affine}, @@ -248,11 +249,13 @@ impl Circuit for BatchCircuit { let barycentric = config.blob_consistency_config.assign_barycentric( &mut ctx, &self.batch_hash.blob_bytes, - self.batch_hash - .blob_consistency_witness - .challenge() - .0 - .into(), + U256::from_big_endian( + &self + .batch_hash + .blob_consistency_witness + .challenge() + .to_bytes(), + ), ); ctx.print_stats(&["barycentric"]); diff --git a/aggregator/src/batch.rs b/aggregator/src/batch.rs index e5e53d626a..d82a2b4272 100644 --- a/aggregator/src/batch.rs +++ b/aggregator/src/batch.rs @@ -85,10 +85,7 @@ impl BatchHeader { last_block_timestamp, data_hash: batch_data_hash.into(), blob_versioned_hash: blob_consistency_witness.id(), - blob_data_proof: [ - blob_consistency_witness.challenge(), - blob_consistency_witness.evaluation(), - ], + blob_data_proof: blob_consistency_witness.blob_data_proof(), } } @@ -282,8 +279,8 @@ impl BatchHash { "batch hash {:?}, datahash {}, z {}, y {}, versioned hash {:x}", current_batch_hash, hex::encode(batch_data_hash), - hex::encode(blob_consistency_witness.challenge().to_fixed_bytes()), - hex::encode(blob_consistency_witness.evaluation().to_fixed_bytes()), + hex::encode(blob_consistency_witness.challenge().to_bytes()), + hex::encode(blob_consistency_witness.evaluation().to_bytes()), blob_consistency_witness.id(), ); diff --git a/aggregator/src/blob_consistency/eip4844.rs b/aggregator/src/blob_consistency/eip4844.rs index ba5848dc0b..d6ffba0cf8 100644 --- a/aggregator/src/blob_consistency/eip4844.rs +++ b/aggregator/src/blob_consistency/eip4844.rs @@ -16,10 +16,11 @@ use crate::{ aggregation::batch_data::N_DATA_BYTES_PER_COEFFICIENT, constants::N_BYTES_U256, BatchData, RlcConfig, }; -use eth_types::{ToBigEndian, H256, U256}; +use eth_types::{ToBigEndian, ToLittleEndian, H256, U256}; use ethers_core::k256::sha2::{Digest, Sha256}; use halo2_base::{gates::range::RangeConfig, Context}; use halo2_ecc::bigint::CRTInteger; +use halo2_proofs::halo2curves::bls12_381::Scalar; use halo2_proofs::{ circuit::{AssignedCell, Layouter, Value}, halo2curves::bn256::Fr, @@ -165,37 +166,66 @@ impl BlobConsistencyConfig { #[derive(Debug, Clone, Copy, Default)] pub struct BlobConsistencyWitness { - id: H256, - blob_data_proof: [H256; 2], + blob_versioned_hash: H256, + challenge_digest: H256, + evaluation: Scalar, } impl BlobConsistencyWitness { pub fn new(bytes: &[u8], batch_data: &BatchData) -> Self { let coeffs = get_coefficients(bytes); - let versioned_hash = get_versioned_hash(&coeffs); + let blob_versioned_hash = get_versioned_hash(&coeffs); let point_evaluation_assignments = - PointEvaluationAssignments::new(&batch_data, bytes, versioned_hash); - let blob_data_proof = [ - point_evaluation_assignments.challenge, - point_evaluation_assignments.evaluation, - ] - .map(|x| H256::from_slice(&x.to_be_bytes())); - + PointEvaluationAssignments::new(&batch_data, bytes, blob_versioned_hash); Self { - id: versioned_hash, - blob_data_proof, + blob_versioned_hash, + challenge_digest: digest_from_word(point_evaluation_assignments.challenge_digest), + evaluation: scalar_from_word(point_evaluation_assignments.evaluation), } } pub fn id(&self) -> H256 { - self.id + self.blob_versioned_hash + } + + pub fn challenge_digest(&self) -> H256 { + self.challenge_digest + } + + pub fn challenge(&self) -> Scalar { + scalar_from_digest(self.challenge_digest) } - pub fn challenge(&self) -> H256 { - self.blob_data_proof[0] + pub fn evaluation(&self) -> Scalar { + self.evaluation } - pub fn evaluation(&self) -> H256 { - self.blob_data_proof[1] + pub fn blob_data_proof(&self) -> [H256; 2] { + [self.challenge(), self.evaluation].map(digest_from_scalar) } } + +fn digest_from_word(x: U256) -> H256 { + H256::from_slice(&x.to_be_bytes()) +} + +fn digest_from_scalar(x: Scalar) -> H256 { + let mut bytes = x.to_bytes(); + bytes.reverse(); + H256::from_slice(&bytes) +} + +fn scalar_from_word(x: U256) -> Scalar { + let (_quotient, remainder) = x.div_mod(*BLS_MODULUS); + Scalar::from_bytes(&remainder.to_le_bytes()).expect("non-canonical bytes") +} + +fn scalar_from_digest(x: H256) -> Scalar { + scalar_from_word(word_from_digest(x)) +} + +fn word_from_digest(x: H256) -> U256 { + U256::from_big_endian(&x.to_fixed_bytes()) +} + +// word_from scalar would not be used. diff --git a/aggregator/src/blob_consistency/eip4844/tests.rs b/aggregator/src/blob_consistency/eip4844/tests.rs index fa828060a2..891b492ec1 100644 --- a/aggregator/src/blob_consistency/eip4844/tests.rs +++ b/aggregator/src/blob_consistency/eip4844/tests.rs @@ -628,3 +628,19 @@ fn test_decode_blob() { batch_bytes, ); } + +use super::*; + +#[test] +fn test_conversions() { + let scalar = Scalar::one(); + let word = U256::one(); + let mut digest = H256::zero(); + digest.0[31] = 1; + + assert_eq!(digest_from_word(word), digest); + assert_eq!(digest_from_scalar(scalar), digest); + assert_eq!(scalar_from_word(word), scalar); + assert_eq!(scalar_from_digest(digest), scalar); + assert_eq!(word_from_digest(digest), word); +} From 2ec09ae0f740ca3cfacaaf02c30cedea17843da1 Mon Sep 17 00:00:00 2001 From: z2trillion Date: Wed, 30 Oct 2024 15:22:38 -0400 Subject: [PATCH 31/35] cleanup --- aggregator/src/aggregation/circuit.rs | 9 +-------- aggregator/src/blob_consistency/eip4844.rs | 6 +++--- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/aggregator/src/aggregation/circuit.rs b/aggregator/src/aggregation/circuit.rs index cf1d41cdf4..b9a3cc7c79 100644 --- a/aggregator/src/aggregation/circuit.rs +++ b/aggregator/src/aggregation/circuit.rs @@ -1,5 +1,4 @@ use ark_std::{end_timer, start_timer}; -use eth_types::U256; use halo2_proofs::{ circuit::{Layouter, SimpleFloorPlanner, Value}, halo2curves::bn256::{Bn256, Fr, G1Affine}, @@ -249,13 +248,7 @@ impl Circuit for BatchCircuit { let barycentric = config.blob_consistency_config.assign_barycentric( &mut ctx, &self.batch_hash.blob_bytes, - U256::from_big_endian( - &self - .batch_hash - .blob_consistency_witness - .challenge() - .to_bytes(), - ), + self.batch_hash.blob_consistency_witness.challenge_digest(), ); ctx.print_stats(&["barycentric"]); diff --git a/aggregator/src/blob_consistency/eip4844.rs b/aggregator/src/blob_consistency/eip4844.rs index d6ffba0cf8..91fd57e638 100644 --- a/aggregator/src/blob_consistency/eip4844.rs +++ b/aggregator/src/blob_consistency/eip4844.rs @@ -188,8 +188,8 @@ impl BlobConsistencyWitness { self.blob_versioned_hash } - pub fn challenge_digest(&self) -> H256 { - self.challenge_digest + pub fn challenge_digest(&self) -> U256 { + word_from_digest(self.challenge_digest) } pub fn challenge(&self) -> Scalar { @@ -228,4 +228,4 @@ fn word_from_digest(x: H256) -> U256 { U256::from_big_endian(&x.to_fixed_bytes()) } -// word_from scalar would not be used. +// word_from_scalar would not be used. From b50708e99928e47f7f24a77b9c4014684644b8e0 Mon Sep 17 00:00:00 2001 From: z2trillion Date: Wed, 30 Oct 2024 17:43:22 -0400 Subject: [PATCH 32/35] fix da-avail build --- aggregator/src/blob_consistency/avail.rs | 35 ++++++++++++++---------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/aggregator/src/blob_consistency/avail.rs b/aggregator/src/blob_consistency/avail.rs index 2b21c4c339..7c7474c8be 100644 --- a/aggregator/src/blob_consistency/avail.rs +++ b/aggregator/src/blob_consistency/avail.rs @@ -3,12 +3,12 @@ use crate::{BatchData, RlcConfig}; use eth_types::{H256, U256}; use halo2_base::{gates::range::RangeConfig, AssignedValue, Context}; use halo2_ecc::bigint::CRTInteger; +use halo2_proofs::halo2curves::bls12_381::Scalar; use halo2_proofs::{ circuit::{AssignedCell, Layouter, Value}, halo2curves::bn256::Fr, plonk::{ConstraintSystem, Error, Expression}, }; -use serde::{Deserialize, Serialize}; use snark_verifier_sdk::LIMBS; use zkevm_circuits::{table::U8Table, util::Challenges}; @@ -55,31 +55,36 @@ impl BlobConsistencyConfig { } } -#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, Default)] pub struct BlobConsistencyWitness { - #[serde(rename = "blob_versioned_hash")] - id: H256, - blob_data_proof: [H256; 2], + blob_versioned_hash: H256, + challenge_digest: H256, + evaluation: Scalar, } impl BlobConsistencyWitness { - pub fn new(_bytes: &[u8], _: &BatchData) -> Self { - Self { - id: H256::default(), // should be keccak of bytes - blob_data_proof: Default::default(), - } + pub fn new(_bytes: &[u8], _batch_data: &BatchData) -> Self { + unimplemented!() } pub fn id(&self) -> H256 { - self.id + unimplemented!() + } + + pub fn challenge_digest(&self) -> U256 { + unimplemented!() } - pub fn challenge(&self) -> H256 { - self.blob_data_proof[0] + pub fn challenge(&self) -> Scalar { + unimplemented!() } - pub fn evaluation(&self) -> H256 { - self.blob_data_proof[1] + pub fn evaluation(&self) -> Scalar { + unimplemented!() + } + + pub fn blob_data_proof(&self) -> [H256; 2] { + unimplemented!() } } From 415d5bcdce157555408253ed42b0817c7b5b7c49 Mon Sep 17 00:00:00 2001 From: z2trillion Date: Wed, 30 Oct 2024 20:31:53 -0400 Subject: [PATCH 33/35] fix clippy --- aggregator/src/batch.rs | 2 +- aggregator/src/blob_consistency/blob_data.rs | 1 + aggregator/src/blob_consistency/eip4844.rs | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/aggregator/src/batch.rs b/aggregator/src/batch.rs index d82a2b4272..5e3f5bbe6b 100644 --- a/aggregator/src/batch.rs +++ b/aggregator/src/batch.rs @@ -273,7 +273,7 @@ impl BatchHash { let batch_data = BatchData::::new(number_of_valid_chunks, chunks_with_padding); let current_batch_hash = batch_header.batch_hash(); - let blob_consistency_witness = BlobConsistencyWitness::new(&blob_bytes, &batch_data); + let blob_consistency_witness = BlobConsistencyWitness::new(blob_bytes, &batch_data); log::info!( "batch hash {:?}, datahash {}, z {}, y {}, versioned hash {:x}", diff --git a/aggregator/src/blob_consistency/blob_data.rs b/aggregator/src/blob_consistency/blob_data.rs index 0325f1813d..e604eb34b9 100644 --- a/aggregator/src/blob_consistency/blob_data.rs +++ b/aggregator/src/blob_consistency/blob_data.rs @@ -287,6 +287,7 @@ impl BlobDataConfig { )) } + #[allow(clippy::type_complexity)] pub fn assign_internal_checks( &self, region: &mut Region, diff --git a/aggregator/src/blob_consistency/eip4844.rs b/aggregator/src/blob_consistency/eip4844.rs index 91fd57e638..8958f0ffd5 100644 --- a/aggregator/src/blob_consistency/eip4844.rs +++ b/aggregator/src/blob_consistency/eip4844.rs @@ -176,7 +176,7 @@ impl BlobConsistencyWitness { let coeffs = get_coefficients(bytes); let blob_versioned_hash = get_versioned_hash(&coeffs); let point_evaluation_assignments = - PointEvaluationAssignments::new(&batch_data, bytes, blob_versioned_hash); + PointEvaluationAssignments::new(batch_data, bytes, blob_versioned_hash); Self { blob_versioned_hash, challenge_digest: digest_from_word(point_evaluation_assignments.challenge_digest), From f90269ef7dd2de2bce2cdf6391f9405c3e7142d7 Mon Sep 17 00:00:00 2001 From: z2trillion Date: Wed, 30 Oct 2024 20:46:00 -0400 Subject: [PATCH 34/35] move function to be before tests --- aggregator/src/aggregation/decoder.rs | 96 +++++++++++++-------------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/aggregator/src/aggregation/decoder.rs b/aggregator/src/aggregation/decoder.rs index e1b3b7f80e..96483034da 100644 --- a/aggregator/src/aggregation/decoder.rs +++ b/aggregator/src/aggregation/decoder.rs @@ -5438,6 +5438,53 @@ impl DecoderConfig { } } +/// Given the blob's bytes, take into account the first byte, i.e. enable_encoding? and either spit +/// out the raw bytes or zstd decode them. +pub fn decode_bytes(bytes: &[u8]) -> std::io::Result> { + let enable_encoding = bytes[0].eq(&1); + + // If not encoded, spit out the rest of the bytes, as it is. + if !enable_encoding { + return Ok(bytes[1..].to_vec()); + } + + // The bytes following the first byte represent the zstd-encoded bytes. + let mut encoded_bytes = bytes[1..].to_vec(); + let mut encoded_len = encoded_bytes.len(); + let mut decoded_bytes = Vec::with_capacity(5 * 4096 * 32); + loop { + let mut decoder = zstd_encoder::zstd::stream::read::Decoder::new(encoded_bytes.as_slice())?; + decoder.include_magicbytes(false)?; + decoder.window_log_max(30)?; + + decoded_bytes.clear(); + + if std::io::copy(&mut decoder, &mut decoded_bytes).is_ok() { + break; + } + + // The error above means we need to truncate the suffix 0-byte. + encoded_len -= 1; + encoded_bytes.truncate(encoded_len); + } + + Ok(decoded_bytes) +} + +/// The inverse of decode_bytes. +pub fn encode_bytes(batch_bytes: &[u8]) -> Vec { + let mut blob_bytes = crate::witgen::zstd_encode(batch_bytes); + + // Whether we encode batch -> blob or not. + let enable_encoding = blob_bytes.len() < batch_bytes.len(); + if !enable_encoding { + blob_bytes = batch_bytes.to_vec(); + } + blob_bytes.insert(0, enable_encoding as u8); + + blob_bytes +} + #[cfg(test)] mod tests { use crate::{ @@ -5793,51 +5840,4 @@ mod tests { Ok(()) } -} - -/// Given the blob's bytes, take into account the first byte, i.e. enable_encoding? and either spit -/// out the raw bytes or zstd decode them. -pub fn decode_bytes(bytes: &[u8]) -> std::io::Result> { - let enable_encoding = bytes[0].eq(&1); - - // If not encoded, spit out the rest of the bytes, as it is. - if !enable_encoding { - return Ok(bytes[1..].to_vec()); - } - - // The bytes following the first byte represent the zstd-encoded bytes. - let mut encoded_bytes = bytes[1..].to_vec(); - let mut encoded_len = encoded_bytes.len(); - let mut decoded_bytes = Vec::with_capacity(5 * 4096 * 32); - loop { - let mut decoder = zstd_encoder::zstd::stream::read::Decoder::new(encoded_bytes.as_slice())?; - decoder.include_magicbytes(false)?; - decoder.window_log_max(30)?; - - decoded_bytes.clear(); - - if std::io::copy(&mut decoder, &mut decoded_bytes).is_ok() { - break; - } - - // The error above means we need to truncate the suffix 0-byte. - encoded_len -= 1; - encoded_bytes.truncate(encoded_len); - } - - Ok(decoded_bytes) -} - -/// The inverse of decode_bytes. -pub fn encode_bytes(batch_bytes: &[u8]) -> Vec { - let mut blob_bytes = crate::witgen::zstd_encode(batch_bytes); - - // Whether we encode batch -> blob or not. - let enable_encoding = blob_bytes.len() < batch_bytes.len(); - if !enable_encoding { - blob_bytes = batch_bytes.to_vec(); - } - blob_bytes.insert(0, enable_encoding as u8); - - blob_bytes -} +} \ No newline at end of file From 7c9b73770bfa00cc497b79187b31f53d0bcee362 Mon Sep 17 00:00:00 2001 From: z2trillion Date: Thu, 31 Oct 2024 01:34:53 -0400 Subject: [PATCH 35/35] cargo fmt --- aggregator/src/aggregation/decoder.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aggregator/src/aggregation/decoder.rs b/aggregator/src/aggregation/decoder.rs index 96483034da..1bf8ac72ea 100644 --- a/aggregator/src/aggregation/decoder.rs +++ b/aggregator/src/aggregation/decoder.rs @@ -5840,4 +5840,4 @@ mod tests { Ok(()) } -} \ No newline at end of file +}