From 487c83efbbe0314284a1b296701498e6a1c3fdbe Mon Sep 17 00:00:00 2001 From: z2trillion Date: Mon, 4 Nov 2024 20:02:52 -0500 Subject: [PATCH] Add da-avail feature flag (#1433) * Add data_availibility directory * move files around and change paths * rename and move decode_blob * unpub barycentric * move eip4844 files into data_availability/ * export functions and structs only * move get_bytes to decoder * Add BlobConsistencyWitness and BlobConsistencyConfig * don't export get_versioned_hash * cleanup * Move blob_data up to data_availability * fix backwards compatibility * fix rebase * fix warnings * remove empty file * add and use blob_crts_limbs function * add blob_crts_limbs to AssignedBlobDataExport * move link out of blob data circuit into aggregation circuit * remove unused arguments * unpub get_coefficients * rename data_availability to blob_consistency * remove da-eip4844 feature flag * define N_DATA_BYTES_PER_COEFFICIENT once * fix da-avail build * fix typos * allow dead code for now * cleanup comments * Don't use BlobConsistencyWitness in BatchHeader * don't derive (De)serialize * add conversion functions * cleanup * fix da-avail build * fix clippy * move function to be before tests * cargo fmt --- Cargo.lock | 1 + Cargo.toml | 1 + Makefile | 3 + aggregator/Cargo.toml | 2 + aggregator/src/aggregation.rs | 14 +- aggregator/src/aggregation/batch_data.rs | 606 ++++++++++++- aggregator/src/aggregation/circuit.rs | 32 +- aggregator/src/aggregation/config.rs | 25 +- aggregator/src/aggregation/decoder.rs | 47 + aggregator/src/batch.rs | 77 +- aggregator/src/blob.rs | 818 ------------------ aggregator/src/blob_consistency.rs | 27 + aggregator/src/blob_consistency/avail.rs | 100 +++ .../blob_data.rs | 124 +-- aggregator/src/blob_consistency/eip4844.rs | 231 +++++ .../eip4844}/barycentric.rs | 16 +- .../src/blob_consistency/eip4844/blob.rs | 196 +++++ .../eip4844/tests.rs} | 51 +- aggregator/src/constants.rs | 3 + aggregator/src/eip4844.rs | 89 -- aggregator/src/lib.rs | 7 +- aggregator/src/tests.rs | 1 - aggregator/src/tests/aggregation.rs | 6 +- prover/src/aggregator.rs | 2 +- prover/src/aggregator/prover.rs | 12 +- 25 files changed, 1380 insertions(+), 1111 deletions(-) delete mode 100644 aggregator/src/blob.rs create mode 100644 aggregator/src/blob_consistency.rs create mode 100644 aggregator/src/blob_consistency/avail.rs rename aggregator/src/{aggregation => blob_consistency}/blob_data.rs (83%) create mode 100644 aggregator/src/blob_consistency/eip4844.rs rename aggregator/src/{aggregation => blob_consistency/eip4844}/barycentric.rs (98%) create mode 100644 aggregator/src/blob_consistency/eip4844/blob.rs rename aggregator/src/{tests/blob.rs => blob_consistency/eip4844/tests.rs} (94%) delete mode 100644 aggregator/src/eip4844.rs diff --git a/Cargo.lock b/Cargo.lock index 72509f65b0..6155537ce5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -45,6 +45,7 @@ dependencies = [ "ark-std 0.3.0", "bitstream-io", "c-kzg", + "cfg-if 1.0.0", "csv", "ctor", "encoder", diff --git a/Cargo.toml b/Cargo.toml index c059b621e7..5c23c7c6e4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,7 @@ anyhow = "1.0" ark-std = "0.3" base64 = "0.13.0" bincode = "1" +cfg-if = "1" ctor = "0.1" env_logger = "0.10" ethers = { version = "=2.0.7", features = ["ethers-solc"] } diff --git a/Makefile b/Makefile index 47883bc0c3..dd664b7419 100644 --- a/Makefile +++ b/Makefile @@ -30,6 +30,9 @@ test-benches: ## Compiles the benchmarks test-all: fmt doc clippy test-doc test-benches test ## Run all the CI checks locally (in your actual toolchain) +test-da-avail: ## Run light tests + @cargo test --release --workspace --exclude integration-tests --exclude circuit-benchmarks --features da-avail batch_circuit + super_bench: ## Run Super Circuit benchmarks @cargo test --profile bench bench_super_circuit_prover -p circuit-benchmarks --features benches -- --nocapture diff --git a/aggregator/Cargo.toml b/aggregator/Cargo.toml index 1b1afdc5dd..08b9cb7b62 100644 --- a/aggregator/Cargo.toml +++ b/aggregator/Cargo.toml @@ -11,6 +11,7 @@ gadgets = { path = "../gadgets" } zkevm-circuits = { path = "../zkevm-circuits", default-features=false, features = ["debug-annotations", "parallel_syn"] } ark-std.workspace = true +cfg-if.workspace = true ctor.workspace = true env_logger.workspace = true ethers-core.workspace = true @@ -47,6 +48,7 @@ csv = "1.1" default = ["revm-precompile/c-kzg", "halo2_proofs/circuit-params"] display = [] print-trace = ["ark-std/print-trace"] +da-avail = [] [lints] workspace = true diff --git a/aggregator/src/aggregation.rs b/aggregator/src/aggregation.rs index 25f7ddafec..bb067bfb0c 100644 --- a/aggregator/src/aggregation.rs +++ b/aggregator/src/aggregation.rs @@ -1,9 +1,5 @@ -/// Config to evaluate blob polynomial at a random challenge. -mod barycentric; /// Config to constrain batch data (decoded blob data) -mod batch_data; -/// Config to constrain blob data (encoded batch data) -mod blob_data; +pub mod batch_data; /// Circuit implementation of aggregation circuit. mod circuit; /// Config for aggregation circuit @@ -15,13 +11,11 @@ mod rlc; /// Utility module mod util; -pub(crate) use barycentric::{ - interpolate, AssignedBarycentricEvaluationConfig, BarycentricEvaluationConfig, BLS_MODULUS, -}; +pub use batch_data::BatchData; pub(crate) use batch_data::BatchDataConfig; -pub(crate) use blob_data::BlobDataConfig; +pub use decoder::{decode_bytes, encode_bytes}; pub(crate) use decoder::{witgen, DecoderConfig, DecoderConfigArgs}; -pub(crate) use rlc::RlcConfig; +pub(crate) use rlc::{RlcConfig, POWS_OF_256}; pub use circuit::BatchCircuit; pub use config::BatchCircuitConfig; diff --git a/aggregator/src/aggregation/batch_data.rs b/aggregator/src/aggregation/batch_data.rs index 5bed4c2d78..221a059e5b 100644 --- a/aggregator/src/aggregation/batch_data.rs +++ b/aggregator/src/aggregation/batch_data.rs @@ -1,4 +1,8 @@ -use eth_types::H256; +use crate::{ + aggregation::rlc::POWS_OF_256, blob_consistency::BLOB_WIDTH, constants::N_BYTES_U256, + BatchHash, ChunkInfo, RlcConfig, +}; +use eth_types::{H256, U256}; use ethers_core::utils::keccak256; use halo2_ecc::bigint::CRTInteger; use halo2_proofs::{ @@ -8,16 +12,29 @@ use halo2_proofs::{ poly::Rotation, }; use itertools::Itertools; +use std::iter::{once, repeat}; use zkevm_circuits::{ table::{KeccakTable, LookupTable, RangeTable, U8Table}, util::{Challenges, Expr}, }; -use crate::{ - aggregation::rlc::POWS_OF_256, - blob::{BatchData, BLOB_WIDTH, N_BYTES_U256}, - RlcConfig, -}; +/// The number data bytes we pack each BLS12-381 scalar into. The most-significant byte is 0. +pub const N_DATA_BYTES_PER_COEFFICIENT: usize = 31; + +/// The number of bytes that we can fit in a blob. Note that each coefficient is represented in 32 +/// bytes, however, since those 32 bytes must represent a BLS12-381 scalar in its canonical form, +/// we explicitly set the most-significant byte to 0, effectively utilising only 31 bytes. +pub const N_BLOB_BYTES: usize = BLOB_WIDTH * N_DATA_BYTES_PER_COEFFICIENT; + +/// Allow up to 5x compression via zstd encoding of the batch data. +const N_BATCH_BYTES: usize = N_BLOB_BYTES * 5; + +/// The number of rows to encode number of valid chunks (num_valid_snarks) in a batch, in the Blob +/// Data config. Since num_valid_chunks is u16, we use 2 bytes/rows. +const N_ROWS_NUM_CHUNKS: usize = 2; + +/// The number of rows to encode chunk size (u32). +const N_ROWS_CHUNK_SIZE: usize = 4; #[derive(Clone, Debug)] pub struct BatchDataConfig { @@ -1018,3 +1035,580 @@ impl BatchDataConfig { Ok(export) } } + +/// Helper struct to generate witness for the Batch Data Config. +#[derive(Clone, Debug)] +pub struct BatchData { + /// The number of valid chunks in the batch. This could be any number between: + /// [1, N_SNARKS] + pub num_valid_chunks: u16, + /// The size of each chunk. The chunk size can be zero if: + /// - The chunk is a padded chunk (not a valid chunk). + /// - The chunk has no L2 transactions, but only L1 msg txs. + pub chunk_sizes: [u32; N_SNARKS], + /// Flattened L2 signed transaction data, for each chunk. + /// + /// Note that in BatchData struct, only `num_valid_chunks` number of chunks' bytes are supposed + /// to be read (for witness generation). For simplicity, the last valid chunk's bytes are + /// copied over for the padded chunks. The `chunk_data_digest` for padded chunks is the + /// `chunk_data_digest` of the last valid chunk (from Aggregation Circuit's perspective). + pub chunk_data: [Vec; N_SNARKS], +} + +impl BatchData { + /// For raw batch bytes with metadata, this function segments the byte stream into chunk segments. + /// Metadata will be removed from the result. + pub fn segment_with_metadata(batch_bytes_with_metadata: Vec) -> Vec> { + let n_bytes_metadata = Self::n_rows_metadata(); + let metadata_bytes = batch_bytes_with_metadata + .clone() + .into_iter() + .take(n_bytes_metadata) + .collect::>(); + let batch_bytes = batch_bytes_with_metadata + .clone() + .into_iter() + .skip(n_bytes_metadata) + .collect::>(); + + // Decoded batch bytes require segmentation based on chunk length + let batch_data_len = batch_bytes.len(); + let chunk_lens = metadata_bytes[N_ROWS_NUM_CHUNKS..] + .chunks(N_ROWS_CHUNK_SIZE) + .map(|chunk| { + chunk + .iter() + .fold(0usize, |acc, &d| acc * 256usize + d as usize) + }) + .collect::>(); + + // length segments sanity check + let valid_chunks = metadata_bytes + .iter() + .take(N_ROWS_NUM_CHUNKS) + .fold(0usize, |acc, &d| acc * 256usize + d as usize); + let calculated_len = chunk_lens.iter().take(valid_chunks).sum::(); + assert_eq!( + batch_data_len, calculated_len, + "chunk segmentation len must add up to the correct value" + ); + + // reconstruct segments + let mut segmented_batch_data: Vec> = Vec::new(); + let mut offset: usize = 0; + let mut segment: usize = 0; + while offset < batch_data_len { + segmented_batch_data.push( + batch_bytes + .clone() + .into_iter() + .skip(offset) + .take(chunk_lens[segment]) + .collect::>(), + ); + + offset += chunk_lens[segment]; + segment += 1; + } + + segmented_batch_data + } +} + +impl From<&BatchHash> for BatchData { + fn from(batch_hash: &BatchHash) -> Self { + Self::new( + batch_hash.number_of_valid_chunks, + &batch_hash.chunks_with_padding, + ) + } +} + +// If the chunk data is represented as a vector of u8's this implementation converts data from +// dynamic number of chunks into BatchData. +impl From<&Vec>> for BatchData { + fn from(chunks: &Vec>) -> Self { + let num_valid_chunks = chunks.len(); + assert!(num_valid_chunks > 0); + assert!(num_valid_chunks <= N_SNARKS); + + let chunk_sizes: [u32; N_SNARKS] = chunks + .iter() + .map(|chunk| chunk.len() as u32) + .chain(repeat(0)) + .take(N_SNARKS) + .collect::>() + .try_into() + .expect("we have N_SNARKS chunks"); + assert!(chunk_sizes.iter().sum::() <= Self::n_rows_data().try_into().unwrap()); + + let last_chunk_data = chunks.last().expect("last chunk exists"); + let chunk_data = chunks + .iter() + .chain(repeat(last_chunk_data)) + .take(N_SNARKS) + .cloned() + .collect::>() + .try_into() + .expect("we have N_SNARKS chunks"); + + Self { + num_valid_chunks: num_valid_chunks.try_into().unwrap(), + chunk_sizes, + chunk_data, + } + } +} + +impl Default for BatchData { + fn default() -> Self { + // default value corresponds to a batch with 1 chunk with no transactions + Self::from(&vec![vec![]]) + } +} + +impl BatchData { + /// The number of rows in Blob Data config's layout to represent the "digest rlc" section. + /// - metadata digest RLC (1 row) + /// - chunk_digests RLC for each chunk (MAX_AGG_SNARKS rows) + /// - blob versioned hash RLC (1 row) + /// - challenge digest RLC (1 row) + pub const fn n_rows_digest_rlc() -> usize { + 1 + N_SNARKS + 1 + 1 + } + + /// The number of rows in Blob Data config's layout to represent the "digest bytes" section. + pub const fn n_rows_digest_bytes() -> usize { + Self::n_rows_digest_rlc() * N_BYTES_U256 + } + + /// The number of rows to encode the size of each chunk in a batch, in the Blob Data config. + /// chunk_size is u32, we use 4 bytes/rows. + const fn n_rows_chunk_sizes() -> usize { + N_SNARKS * N_ROWS_CHUNK_SIZE + } + + /// The total number of rows in "digest rlc" and "digest bytes" sections. + const fn n_rows_digest() -> usize { + Self::n_rows_digest_rlc() + Self::n_rows_digest_bytes() + } + + /// The number of rows in Blob Data config's layout to represent the "blob metadata" section. + pub const fn n_rows_metadata() -> usize { + N_ROWS_NUM_CHUNKS + Self::n_rows_chunk_sizes() + } + + /// The number of rows in Blob Data config's layout to represent the "chunk data" section. + pub const fn n_rows_data() -> usize { + N_BATCH_BYTES - Self::n_rows_metadata() + } + + /// The total number of rows used in Blob Data config's layout. + pub const fn n_rows() -> usize { + N_BATCH_BYTES + Self::n_rows_digest() + } + + /// Construct BatchData from chunks + pub fn new(num_valid_chunks: usize, chunks_with_padding: &[ChunkInfo]) -> Self { + assert!(num_valid_chunks > 0); + assert!(num_valid_chunks <= N_SNARKS); + + // padded chunk has 0 size, valid chunk's size is the number of bytes consumed by the + // flattened data from signed L2 transactions. + let chunk_sizes: [u32; N_SNARKS] = chunks_with_padding + .iter() + .map(|chunk| { + if chunk.is_padding { + 0 + } else { + chunk.tx_bytes.len() as u32 + } + }) + .collect::>() + .try_into() + .unwrap(); + + if chunk_sizes.iter().sum::() > Self::n_rows_data() as u32 { + panic!( + "invalid chunk_sizes {}, n_rows_data {}", + chunk_sizes.iter().sum::(), + Self::n_rows_data() + ) + } + + // chunk data of the "last valid chunk" is repeated over the padded chunks for simplicity + // in calculating chunk_data_digest for those padded chunks. However, for the "chunk data" + // section rows (self.to_data_rows()) we only consider `num_valid_chunks` chunks. + let chunk_data = chunks_with_padding + .iter() + .map(|chunk| chunk.tx_bytes.to_vec()) + .collect::>>() + .try_into() + .unwrap(); + + Self { + num_valid_chunks: num_valid_chunks as u16, + chunk_sizes, + chunk_data, + } + } + + /// Get the preimage of the challenge digest. + pub(crate) fn get_challenge_digest_preimage(&self, versioned_hash: H256) -> Vec { + let metadata_digest = keccak256(self.to_metadata_bytes()); + let chunk_digests = self.chunk_data.iter().map(keccak256); + + // preimage = + // metadata_digest || + // chunk[0].chunk_data_digest || ... + // chunk[N_SNARKS-1].chunk_data_digest || + // blob_versioned_hash + // + // where chunk_data_digest for a padded chunk is set equal to the "last valid chunk"'s + // chunk_data_digest. + metadata_digest + .into_iter() + .chain(chunk_digests.flatten()) + .chain(versioned_hash.to_fixed_bytes()) + .collect::>() + } + + /// Compute the challenge digest from blob bytes. + pub(crate) fn get_challenge_digest(&self, versioned_hash: H256) -> U256 { + let challenge_digest = keccak256(self.get_challenge_digest_preimage(versioned_hash)); + U256::from_big_endian(&challenge_digest) + } + + /// Get the batch data bytes that will be populated in BatchDataConfig. + pub fn get_batch_data_bytes(&self) -> Vec { + let metadata_bytes = self.to_metadata_bytes(); + metadata_bytes + .iter() + .chain( + self.chunk_data + .iter() + .take(self.num_valid_chunks as usize) + .flatten(), + ) + .cloned() + .collect() + } + + /// Get the list of preimages that need to go through the keccak hashing function, and + /// eventually required to be checked for the consistency of blob's metadata, its chunks' bytes + /// and the final blob preimage. + pub fn preimages(&self, versioned_hash: H256) -> Vec> { + let mut preimages = Vec::with_capacity(2 + N_SNARKS); + + // metadata + preimages.push(self.to_metadata_bytes()); + + // each valid chunk's data + for chunk in self.chunk_data.iter().take(self.num_valid_chunks as usize) { + preimages.push(chunk.to_vec()); + } + + // preimage for challenge digest + preimages.push(self.get_challenge_digest_preimage(versioned_hash)); + + preimages + } + + /// Get the witness rows for assignment to the BlobDataConfig. + fn to_rows( + &self, + versioned_hash: H256, + challenge: Challenges>, + ) -> Vec> { + let metadata_rows = self.to_metadata_rows(challenge); + assert_eq!(metadata_rows.len(), Self::n_rows_metadata()); + + let data_rows = self.to_data_rows(challenge); + assert_eq!(data_rows.len(), Self::n_rows_data()); + + let digest_rows = self.to_digest_rows(versioned_hash, challenge); + assert_eq!(digest_rows.len(), Self::n_rows_digest()); + + metadata_rows + .into_iter() + .chain(data_rows) + .chain(digest_rows) + .collect::>>() + } + + /// Get the blob bytes that encode the batch's metadata. + /// + /// metadata_bytes = + /// be_bytes(num_valid_chunks) || + /// be_bytes(chunks[0].chunk_size) || ... + /// be_bytes(chunks[N_SNARKS-1].chunk_size) + /// + /// where: + /// - chunk_size of a padded chunk is 0 + /// - num_valid_chunks is u16 + /// - each chunk_size is u32 + fn to_metadata_bytes(&self) -> Vec { + self.num_valid_chunks + .to_be_bytes() + .into_iter() + .chain( + self.chunk_sizes + .iter() + .flat_map(|chunk_size| chunk_size.to_be_bytes()), + ) + .collect() + } + + /// Get the witness rows for the "metadata" section of Blob data config. + fn to_metadata_rows(&self, challenge: Challenges>) -> Vec> { + // metadata bytes. + let bytes = self.to_metadata_bytes(); + + // accumulators represent the running linear combination of bytes. + let accumulators_iter = self + .num_valid_chunks + .to_be_bytes() + .into_iter() + .scan(0u64, |acc, x| { + *acc = *acc * 256 + (x as u64); + Some(*acc) + }) + .chain(self.chunk_sizes.into_iter().flat_map(|chunk_size| { + chunk_size.to_be_bytes().into_iter().scan(0u64, |acc, x| { + *acc = *acc * 256 + (x as u64); + Some(*acc) + }) + })); + + // digest_rlc is set only for the last row in the "metadata" section, and it denotes the + // RLC of the metadata_digest bytes. + let digest_rlc_iter = { + let digest = keccak256(&bytes); + let digest_rlc = digest.iter().fold(Value::known(Fr::zero()), |acc, &x| { + acc * challenge.evm_word() + Value::known(Fr::from(x as u64)) + }); + repeat(Value::known(Fr::zero())) + .take(Self::n_rows_metadata() - 1) + .chain(once(digest_rlc)) + }; + + // preimage_rlc is the running RLC over bytes in the "metadata" section. + let preimage_rlc_iter = bytes.iter().scan(Value::known(Fr::zero()), |acc, &x| { + *acc = *acc * challenge.keccak_input() + Value::known(Fr::from(x as u64)); + Some(*acc) + }); + + bytes + .iter() + .zip_eq(accumulators_iter) + .zip_eq(preimage_rlc_iter) + .zip_eq(digest_rlc_iter) + .enumerate() + .map( + |(i, (((&byte, accumulator), preimage_rlc), digest_rlc))| BatchDataRow { + byte, + accumulator, + preimage_rlc, + digest_rlc, + // we set boundary on the last row of the "metadata" section to enable a lookup + // to the keccak table. + is_boundary: i == bytes.len() - 1, + ..Default::default() + }, + ) + .collect() + } + + /// Get the witness rows for the "chunk data" section of Blob data config. + fn to_data_rows(&self, challenge: Challenges>) -> Vec> { + // consider only the `valid` chunks while constructing rows for the "chunk data" section. + self.chunk_data + .iter() + .take(self.num_valid_chunks as usize) + .enumerate() + .flat_map(|(i, bytes)| { + let chunk_idx = (i + 1) as u64; + let chunk_size = bytes.len(); + let chunk_digest = keccak256(bytes); + let digest_rlc = chunk_digest + .iter() + .fold(Value::known(Fr::zero()), |acc, &byte| { + acc * challenge.evm_word() + Value::known(Fr::from(byte as u64)) + }); + bytes.iter().enumerate().scan( + (0u64, Value::known(Fr::zero())), + move |acc, (j, &byte)| { + acc.0 += 1; + acc.1 = + acc.1 * challenge.keccak_input() + Value::known(Fr::from(byte as u64)); + Some(BatchDataRow { + byte, + accumulator: acc.0, + chunk_idx, + is_boundary: j == chunk_size - 1, + is_padding: false, + preimage_rlc: acc.1, + digest_rlc: if j == chunk_size - 1 { + digest_rlc + } else { + Value::known(Fr::zero()) + }, + }) + }, + ) + }) + .chain(repeat(BatchDataRow::padding_row())) + .take(Self::n_rows_data()) + .collect() + } + + /// Get the witness rows for both "digest rlc" and "digest bytes" sections of Blob data config. + fn to_digest_rows( + &self, + versioned_hash: H256, + challenge: Challenges>, + ) -> Vec> { + let zero = Value::known(Fr::zero()); + + // metadata + let metadata_bytes = self.to_metadata_bytes(); + let metadata_digest = keccak256(metadata_bytes); + let metadata_digest_rlc = metadata_digest.iter().fold(zero, |acc, &byte| { + acc * challenge.evm_word() + Value::known(Fr::from(byte as u64)) + }); + + // chunk data + // Note: here we don't restrict to considering only `valid` chunks, as the + // `chunk_data_digest` gets repeated for the padded chunks, copying the last valid chunk's + // `chunk_data_digest`. + let (chunk_digests, chunk_digest_rlcs): (Vec<[u8; 32]>, Vec>) = self + .chunk_data + .iter() + .map(|chunk| { + let digest = keccak256(chunk); + let digest_rlc = digest.iter().fold(zero, |acc, &byte| { + acc * challenge.evm_word() + Value::known(Fr::from(byte as u64)) + }); + (digest, digest_rlc) + }) + .unzip(); + + // challenge digest + let challenge_digest_preimage = self.get_challenge_digest_preimage(versioned_hash); + let challenge_digest_preimage_rlc = + challenge_digest_preimage.iter().fold(zero, |acc, &byte| { + acc * challenge.keccak_input() + Value::known(Fr::from(byte as u64)) + }); + let challenge_digest = keccak256(&challenge_digest_preimage); + let challenge_digest_rlc = challenge_digest.iter().fold(zero, |acc, &byte| { + acc * challenge.evm_word() + Value::known(Fr::from(byte as u64)) + }); + + // blob versioned hash + let versioned_hash_rlc = versioned_hash.as_bytes().iter().fold(zero, |acc, &byte| { + acc * challenge.evm_word() + Value::known(Fr::from(byte as u64)) + }); + + // - metadata digest rlc + // - chunks[i].chunk_data_digest rlc for each chunk + // - versioned hash rlc + // - challenge digest rlc + // - metadata digest bytes + // - chunks[i].chunk_data_digest bytes for each chunk + // - versioned hash bytes + // - challenge digest bytes + once(BatchDataRow { + preimage_rlc: Value::known(Fr::zero()), + digest_rlc: metadata_digest_rlc, + // this is_padding assignment does not matter as we have already crossed the "chunk + // data" section. This assignment to 1 is simply to allow the custom gate to check: + // - padding transitions from 0 -> 1 only once. + is_padding: true, + ..Default::default() + }) + .chain( + chunk_digest_rlcs + .iter() + .zip_eq(self.chunk_sizes.iter()) + .enumerate() + .map(|(i, (&digest_rlc, &chunk_size))| BatchDataRow { + preimage_rlc: Value::known(Fr::zero()), + digest_rlc, + chunk_idx: (i + 1) as u64, + accumulator: chunk_size as u64, + ..Default::default() + }), + ) + // versioned hash RLC + .chain(once(BatchDataRow { + preimage_rlc: Value::known(Fr::zero()), + digest_rlc: versioned_hash_rlc, + ..Default::default() + })) + .chain(once(BatchDataRow { + preimage_rlc: challenge_digest_preimage_rlc, + digest_rlc: challenge_digest_rlc, + accumulator: 32 * (N_SNARKS + 1 + 1) as u64, + is_boundary: true, + ..Default::default() + })) + .chain(metadata_digest.iter().map(|&byte| BatchDataRow { + preimage_rlc: Value::known(Fr::zero()), + digest_rlc: Value::known(Fr::zero()), + byte, + ..Default::default() + })) + .chain(chunk_digests.iter().flat_map(|digest| { + digest.iter().map(|&byte| BatchDataRow { + preimage_rlc: Value::known(Fr::zero()), + digest_rlc: Value::known(Fr::zero()), + byte, + ..Default::default() + }) + })) + // bytes of versioned hash + .chain(versioned_hash.as_bytes().iter().map(|&byte| BatchDataRow { + preimage_rlc: Value::known(Fr::zero()), + digest_rlc: Value::known(Fr::zero()), + byte, + ..Default::default() + })) + .chain(challenge_digest.iter().map(|&byte| BatchDataRow { + preimage_rlc: Value::known(Fr::zero()), + digest_rlc: Value::known(Fr::zero()), + byte, + ..Default::default() + })) + .collect() + } +} + +/// Witness row to the Blob Data Config. +#[derive(Clone, Copy, Default, Debug)] +struct BatchDataRow { + /// Byte value at this row. + pub byte: u8, + /// Multi-purpose accumulator value. + pub accumulator: u64, + /// The chunk index we are currently traversing. + pub chunk_idx: u64, + /// Whether this marks the end of a chunk. + pub is_boundary: bool, + /// Whether the row represents right-padded 0s to the blob data. + pub is_padding: bool, + /// A running accumulator of RLC of preimages. + pub preimage_rlc: Value, + /// RLC of the digest. + pub digest_rlc: Value, +} + +impl BatchDataRow { + fn padding_row() -> Self { + Self { + is_padding: true, + preimage_rlc: Value::known(Fr::zero()), + digest_rlc: Value::known(Fr::zero()), + ..Default::default() + } + } +} diff --git a/aggregator/src/aggregation/circuit.rs b/aggregator/src/aggregation/circuit.rs index 535439a975..b9a3cc7c79 100644 --- a/aggregator/src/aggregation/circuit.rs +++ b/aggregator/src/aggregation/circuit.rs @@ -23,16 +23,15 @@ use std::{env, fs::File, rc::Rc}; use zkevm_circuits::util::Challenges; use crate::{ - aggregation::{decoder::WORKED_EXAMPLE, witgen::process, BatchCircuitConfig}, + aggregation::{decoder::WORKED_EXAMPLE, witgen::process, BatchCircuitConfig, BatchData}, batch::BatchHash, - blob::BatchData, + blob_consistency::BlobConsistencyConfig, constants::{ACC_LEN, DIGEST_LEN}, core::{assign_batch_hashes, extract_proof_and_instances_with_pairing_check}, util::parse_hash_digest_cells, witgen::{zstd_encode, MultiBlockProcessResult}, - AssignedBarycentricEvaluationConfig, ConfigParams, LOG_DEGREE, PI_CHAIN_ID, - PI_CURRENT_BATCH_HASH, PI_CURRENT_STATE_ROOT, PI_CURRENT_WITHDRAW_ROOT, PI_PARENT_BATCH_HASH, - PI_PARENT_STATE_ROOT, + ConfigParams, LOG_DEGREE, PI_CHAIN_ID, PI_CURRENT_BATCH_HASH, PI_CURRENT_STATE_ROOT, + PI_CURRENT_WITHDRAW_ROOT, PI_PARENT_BATCH_HASH, PI_PARENT_STATE_ROOT, }; /// Batch circuit, the chunk aggregation routine below recursion circuit @@ -189,11 +188,7 @@ impl Circuit for BatchCircuit { |region| { if first_pass { first_pass = false; - return Ok(( - vec![], - vec![], - AssignedBarycentricEvaluationConfig::default(), - )); + return Ok(Default::default()); } // stores accumulators for all snarks, including the padded ones @@ -250,12 +245,10 @@ impl Circuit for BatchCircuit { let mut ctx = Rc::into_inner(loader).unwrap().into_ctx(); log::debug!("batching: assigning barycentric"); - let barycentric = config.barycentric.assign( + let barycentric = config.blob_consistency_config.assign_barycentric( &mut ctx, - &self.batch_hash.point_evaluation_assignments.coefficients, - self.batch_hash - .point_evaluation_assignments - .challenge_digest, + &self.batch_hash.blob_bytes, + self.batch_hash.blob_consistency_witness.challenge_digest(), ); ctx.print_stats(&["barycentric"]); @@ -407,11 +400,16 @@ impl Circuit for BatchCircuit { let batch_data = BatchData::from(&self.batch_hash); - let blob_data_exports = config.blob_data_config.assign( + let blob_data_exports = config.blob_consistency_config.assign_blob_data( &mut layouter, challenges, &config.rlc_config, &self.batch_hash.blob_bytes, + )?; + + BlobConsistencyConfig::::link( + &mut layouter, + &blob_data_exports.blob_crts_limbs, barycentric_assignments, )?; @@ -421,7 +419,7 @@ impl Circuit for BatchCircuit { &config.rlc_config, &assigned_batch_hash.chunks_are_padding, &batch_data, - self.batch_hash.versioned_hash, + self.batch_hash.blob_consistency_witness.id(), barycentric_assignments, )?; diff --git a/aggregator/src/aggregation/config.rs b/aggregator/src/aggregation/config.rs index 0d744ec060..e6eab4930d 100644 --- a/aggregator/src/aggregation/config.rs +++ b/aggregator/src/aggregation/config.rs @@ -17,10 +17,10 @@ use zkevm_circuits::{ }; use crate::{ + blob_consistency::BlobConsistencyConfig, constants::{BITS, LIMBS}, param::ConfigParams, - BarycentricEvaluationConfig, BatchDataConfig, BlobDataConfig, DecoderConfig, DecoderConfigArgs, - RlcConfig, + BatchDataConfig, DecoderConfig, DecoderConfigArgs, RlcConfig, }; #[derive(Debug, Clone)] @@ -34,14 +34,12 @@ pub struct BatchCircuitConfig { pub keccak_circuit_config: KeccakCircuitConfig, /// RLC config pub rlc_config: RlcConfig, - /// The blob data's config. - pub blob_data_config: BlobDataConfig, /// The batch data's config. pub batch_data_config: BatchDataConfig, /// The zstd decoder's config. pub decoder_config: DecoderConfig<1024, 512>, - /// Config to do the barycentric evaluation on blob polynomial. - pub barycentric: BarycentricEvaluationConfig, + /// Config check witness blob matches commitment to blob obtained from data availability provider. + pub blob_consistency_config: BlobConsistencyConfig, /// Instance for public input; stores /// - accumulator from aggregation (12 elements) /// - chain id (1 element) @@ -99,9 +97,6 @@ impl BatchCircuitConfig { params.degree as usize, ); - // Barycentric. - let barycentric = BarycentricEvaluationConfig::construct(base_field_config.range.clone()); - let columns = keccak_circuit_config.cell_manager.columns(); log::info!("keccak uses {} columns", columns.len(),); @@ -120,7 +115,6 @@ impl BatchCircuitConfig { let u8_table = U8Table::construct(meta); let range_table = RangeTable::construct(meta); let challenges_expr = challenges.exprs(meta); - let blob_data_config = BlobDataConfig::configure(meta, &challenges_expr, u8_table); let batch_data_config = BatchDataConfig::configure( meta, &challenges_expr, @@ -132,6 +126,14 @@ impl BatchCircuitConfig { // Zstd decoder. let pow_rand_table = PowOfRandTable::construct(meta, &challenges_expr); + // Blob consistency. + let blob_consistency_config = BlobConsistencyConfig::construct( + meta, + &challenges_expr, + u8_table, + base_field_config.range.clone(), + ); + let pow2_table = Pow2Table::construct(meta); let range8 = RangeTable::construct(meta); let range16 = RangeTable::construct(meta); @@ -172,12 +174,11 @@ impl BatchCircuitConfig { Self { base_field_config, rlc_config, - blob_data_config, keccak_circuit_config, instance, - barycentric, batch_data_config, decoder_config, + blob_consistency_config, } } diff --git a/aggregator/src/aggregation/decoder.rs b/aggregator/src/aggregation/decoder.rs index bb9c7661f0..1bf8ac72ea 100644 --- a/aggregator/src/aggregation/decoder.rs +++ b/aggregator/src/aggregation/decoder.rs @@ -5438,6 +5438,53 @@ impl DecoderConfig { } } +/// Given the blob's bytes, take into account the first byte, i.e. enable_encoding? and either spit +/// out the raw bytes or zstd decode them. +pub fn decode_bytes(bytes: &[u8]) -> std::io::Result> { + let enable_encoding = bytes[0].eq(&1); + + // If not encoded, spit out the rest of the bytes, as it is. + if !enable_encoding { + return Ok(bytes[1..].to_vec()); + } + + // The bytes following the first byte represent the zstd-encoded bytes. + let mut encoded_bytes = bytes[1..].to_vec(); + let mut encoded_len = encoded_bytes.len(); + let mut decoded_bytes = Vec::with_capacity(5 * 4096 * 32); + loop { + let mut decoder = zstd_encoder::zstd::stream::read::Decoder::new(encoded_bytes.as_slice())?; + decoder.include_magicbytes(false)?; + decoder.window_log_max(30)?; + + decoded_bytes.clear(); + + if std::io::copy(&mut decoder, &mut decoded_bytes).is_ok() { + break; + } + + // The error above means we need to truncate the suffix 0-byte. + encoded_len -= 1; + encoded_bytes.truncate(encoded_len); + } + + Ok(decoded_bytes) +} + +/// The inverse of decode_bytes. +pub fn encode_bytes(batch_bytes: &[u8]) -> Vec { + let mut blob_bytes = crate::witgen::zstd_encode(batch_bytes); + + // Whether we encode batch -> blob or not. + let enable_encoding = blob_bytes.len() < batch_bytes.len(); + if !enable_encoding { + blob_bytes = batch_bytes.to_vec(); + } + blob_bytes.insert(0, enable_encoding as u8); + + blob_bytes +} + #[cfg(test)] mod tests { use crate::{ diff --git a/aggregator/src/batch.rs b/aggregator/src/batch.rs index b43d788c0b..5e3f5bbe6b 100644 --- a/aggregator/src/batch.rs +++ b/aggregator/src/batch.rs @@ -1,16 +1,12 @@ //! This module implements related functions that aggregates public inputs of many chunks into a //! single one. -use eth_types::{ToBigEndian, H256}; +use eth_types::H256; use ethers_core::utils::keccak256; use gadgets::{util::split_h256, Field}; use serde::{Deserialize, Serialize}; -use crate::{ - blob::{BatchData, PointEvaluationAssignments}, - chunk::ChunkInfo, - eip4844::{get_coefficients, get_versioned_hash}, -}; +use crate::{aggregation::BatchData, blob_consistency::BlobConsistencyWitness, chunk::ChunkInfo}; /// Batch header provides additional fields from the context (within recursion) /// for constructing the preimage of the batch hash. @@ -79,11 +75,7 @@ impl BatchHeader { let batch_data_hash = keccak256(batch_data_hash_preimage); let batch_data = BatchData::::new(number_of_valid_chunks, &chunks_with_padding); - let coeffs = get_coefficients(blob_bytes); - let blob_versioned_hash = get_versioned_hash(&coeffs); - let point_evaluation_assignments = - PointEvaluationAssignments::new(&batch_data, blob_bytes, blob_versioned_hash); - + let blob_consistency_witness = BlobConsistencyWitness::new(blob_bytes, &batch_data); Self { version, batch_index, @@ -92,11 +84,8 @@ impl BatchHeader { parent_batch_hash, last_block_timestamp, data_hash: batch_data_hash.into(), - blob_versioned_hash, - blob_data_proof: [ - H256::from_slice(&point_evaluation_assignments.challenge.to_be_bytes()), - H256::from_slice(&point_evaluation_assignments.evaluation.to_be_bytes()), - ], + blob_versioned_hash: blob_consistency_witness.id(), + blob_data_proof: blob_consistency_witness.blob_data_proof(), } } @@ -163,14 +152,12 @@ pub struct BatchHash { pub(crate) current_batch_hash: H256, /// The number of chunks that contain meaningful data, i.e. not padded chunks. pub(crate) number_of_valid_chunks: usize, - /// 4844 point evaluation check related assignments. - pub(crate) point_evaluation_assignments: PointEvaluationAssignments, - /// The 4844 versioned hash for the blob. - pub(crate) versioned_hash: H256, /// The context batch header pub(crate) batch_header: BatchHeader, /// The blob bytes (may be encoded batch bytes, or may be raw batch bytes). pub(crate) blob_bytes: Vec, + /// Witness data to prove that the blob used as advice in the circuit matches the blob from the data availability layer. + pub blob_consistency_witness: BlobConsistencyWitness, } impl BatchHash { @@ -285,36 +272,16 @@ impl BatchHash { ); let batch_data = BatchData::::new(number_of_valid_chunks, chunks_with_padding); - let coeffs = get_coefficients(blob_bytes); - let versioned_hash = get_versioned_hash(&coeffs); - let point_evaluation_assignments = - PointEvaluationAssignments::new(&batch_data, blob_bytes, versioned_hash); - - assert_eq!( - batch_header.blob_data_proof[0], - H256::from_slice(&point_evaluation_assignments.challenge.to_be_bytes()), - "Expect provided BatchHeader's blob_data_proof field 0 to be correct" - ); - assert_eq!( - batch_header.blob_data_proof[1], - H256::from_slice(&point_evaluation_assignments.evaluation.to_be_bytes()), - "Expect provided BatchHeader's blob_data_proof field 1 to be correct" - ); - - assert_eq!( - batch_header.blob_versioned_hash, versioned_hash, - "Expect provided BatchHeader's blob_versioned_hash field to be correct" - ); - let current_batch_hash = batch_header.batch_hash(); + let blob_consistency_witness = BlobConsistencyWitness::new(blob_bytes, &batch_data); log::info!( "batch hash {:?}, datahash {}, z {}, y {}, versioned hash {:x}", current_batch_hash, hex::encode(batch_data_hash), - hex::encode(point_evaluation_assignments.challenge.to_be_bytes()), - hex::encode(point_evaluation_assignments.evaluation.to_be_bytes()), - versioned_hash, + hex::encode(blob_consistency_witness.challenge().to_bytes()), + hex::encode(blob_consistency_witness.evaluation().to_bytes()), + blob_consistency_witness.id(), ); Self { @@ -326,18 +293,12 @@ impl BatchHash { data_hash: batch_data_hash.into(), current_batch_hash, number_of_valid_chunks, - point_evaluation_assignments, - versioned_hash, batch_header, blob_bytes: blob_bytes.to_vec(), + blob_consistency_witness, } } - /// Return the blob polynomial and its evaluation at challenge - pub fn point_evaluation_assignments(&self) -> PointEvaluationAssignments { - self.point_evaluation_assignments.clone() - } - /// Extract all the hash inputs that will ever be used. /// There are N_SNARKS + 2 hashes. /// @@ -373,19 +334,17 @@ impl BatchHash { .to_be_bytes() .as_ref(), self.data_hash.as_bytes(), - self.versioned_hash.as_bytes(), + self.batch_header.blob_versioned_hash.as_bytes(), self.batch_header.parent_batch_hash.as_bytes(), self.batch_header .last_block_timestamp .to_be_bytes() .as_ref(), - self.point_evaluation_assignments - .challenge - .to_be_bytes() + self.batch_header.blob_data_proof[0] + .to_fixed_bytes() .as_ref(), - self.point_evaluation_assignments - .evaluation - .to_be_bytes() + self.batch_header.blob_data_proof[1] + .to_fixed_bytes() .as_ref(), ] .concat(); @@ -423,7 +382,7 @@ impl BatchHash { // - preimage for each chunk's flattened L2 signed tx data // - preimage for the challenge digest let batch_data = BatchData::from(self); - let dynamic_preimages = batch_data.preimages(self.versioned_hash); + let dynamic_preimages = batch_data.preimages(self.batch_header.blob_versioned_hash); for dynamic_preimage in dynamic_preimages { res.push(dynamic_preimage); } diff --git a/aggregator/src/blob.rs b/aggregator/src/blob.rs deleted file mode 100644 index d52c0c4333..0000000000 --- a/aggregator/src/blob.rs +++ /dev/null @@ -1,818 +0,0 @@ -use crate::{ - aggregation::{interpolate, BLS_MODULUS}, - eip4844::get_coefficients, - BatchHash, ChunkInfo, -}; - -use eth_types::{H256, U256}; -use ethers_core::utils::keccak256; -use halo2_proofs::{ - circuit::Value, - halo2curves::{bls12_381::Scalar, bn256::Fr}, -}; -use itertools::Itertools; -use once_cell::sync::Lazy; -use std::{ - iter::{once, repeat}, - sync::Arc, -}; -use zkevm_circuits::util::Challenges; - -/// The number of coefficients (BLS12-381 scalars) to represent the blob polynomial in evaluation -/// form. -pub const BLOB_WIDTH: usize = 4096; - -/// The number of bytes to represent an unsigned 256 bit number. -pub const N_BYTES_U256: usize = 32; - -/// The number data bytes we pack each BLS12-381 scalar into. The most-significant byte is 0. -pub const N_DATA_BYTES_PER_COEFFICIENT: usize = 31; - -/// The number of rows to encode number of valid chunks (num_valid_snarks) in a batch, in the Blob -/// Data config. Since num_valid_chunks is u16, we use 2 bytes/rows. -pub const N_ROWS_NUM_CHUNKS: usize = 2; - -/// The number of rows to encode chunk size (u32). -pub const N_ROWS_CHUNK_SIZE: usize = 4; - -/// The number of bytes that we can fit in a blob. Note that each coefficient is represented in 32 -/// bytes, however, since those 32 bytes must represent a BLS12-381 scalar in its canonical form, -/// we explicitly set the most-significant byte to 0, effectively utilising only 31 bytes. -pub const N_BLOB_BYTES: usize = BLOB_WIDTH * N_DATA_BYTES_PER_COEFFICIENT; - -/// Allow up to 5x compression via zstd encoding of the batch data. -pub const N_BATCH_BYTES: usize = N_BLOB_BYTES * 5; - -/// KZG trusted setup -pub static KZG_TRUSTED_SETUP: Lazy> = Lazy::new(|| { - Arc::new( - c_kzg::KzgSettings::load_trusted_setup( - &revm_primitives::kzg::G1_POINTS.0, - &revm_primitives::kzg::G2_POINTS.0, - ) - .expect("failed to load trusted setup"), - ) -}); - -/// Helper struct to generate witness for the Batch Data Config. -#[derive(Clone, Debug)] -pub struct BatchData { - /// The number of valid chunks in the batch. This could be any number between: - /// [1, N_SNARKS] - pub num_valid_chunks: u16, - /// The size of each chunk. The chunk size can be zero if: - /// - The chunk is a padded chunk (not a valid chunk). - /// - The chunk has no L2 transactions, but only L1 msg txs. - pub chunk_sizes: [u32; N_SNARKS], - /// Flattened L2 signed transaction data, for each chunk. - /// - /// Note that in BatchData struct, only `num_valid_chunks` number of chunks' bytes are supposed - /// to be read (for witness generation). For simplicity, the last valid chunk's bytes are - /// copied over for the padded chunks. The `chunk_data_digest` for padded chunks is the - /// `chunk_data_digest` of the last valid chunk (from Aggregation Circuit's perspective). - pub chunk_data: [Vec; N_SNARKS], -} - -impl BatchData { - /// For raw batch bytes with metadata, this function segments the byte stream into chunk segments. - /// Metadata will be removed from the result. - pub fn segment_with_metadata(batch_bytes_with_metadata: Vec) -> Vec> { - let n_bytes_metadata = Self::n_rows_metadata(); - let metadata_bytes = batch_bytes_with_metadata - .clone() - .into_iter() - .take(n_bytes_metadata) - .collect::>(); - let batch_bytes = batch_bytes_with_metadata - .clone() - .into_iter() - .skip(n_bytes_metadata) - .collect::>(); - - // Decoded batch bytes require segmentation based on chunk length - let batch_data_len = batch_bytes.len(); - let chunk_lens = metadata_bytes[N_ROWS_NUM_CHUNKS..] - .chunks(N_ROWS_CHUNK_SIZE) - .map(|chunk| { - chunk - .iter() - .fold(0usize, |acc, &d| acc * 256usize + d as usize) - }) - .collect::>(); - - // length segments sanity check - let valid_chunks = metadata_bytes - .iter() - .take(N_ROWS_NUM_CHUNKS) - .fold(0usize, |acc, &d| acc * 256usize + d as usize); - let calculated_len = chunk_lens.iter().take(valid_chunks).sum::(); - assert_eq!( - batch_data_len, calculated_len, - "chunk segmentation len must add up to the correct value" - ); - - // reconstruct segments - let mut segmented_batch_data: Vec> = Vec::new(); - let mut offset: usize = 0; - let mut segment: usize = 0; - while offset < batch_data_len { - segmented_batch_data.push( - batch_bytes - .clone() - .into_iter() - .skip(offset) - .take(chunk_lens[segment]) - .collect::>(), - ); - - offset += chunk_lens[segment]; - segment += 1; - } - - segmented_batch_data - } -} - -impl From<&BatchHash> for BatchData { - fn from(batch_hash: &BatchHash) -> Self { - Self::new( - batch_hash.number_of_valid_chunks, - &batch_hash.chunks_with_padding, - ) - } -} - -// If the chunk data is represented as a vector of u8's this implementation converts data from -// dynamic number of chunks into BatchData. -impl From<&Vec>> for BatchData { - fn from(chunks: &Vec>) -> Self { - let num_valid_chunks = chunks.len(); - assert!(num_valid_chunks > 0); - assert!(num_valid_chunks <= N_SNARKS); - - let chunk_sizes: [u32; N_SNARKS] = chunks - .iter() - .map(|chunk| chunk.len() as u32) - .chain(repeat(0)) - .take(N_SNARKS) - .collect::>() - .try_into() - .expect("we have N_SNARKS chunks"); - assert!(chunk_sizes.iter().sum::() <= Self::n_rows_data().try_into().unwrap()); - - let last_chunk_data = chunks.last().expect("last chunk exists"); - let chunk_data = chunks - .iter() - .chain(repeat(last_chunk_data)) - .take(N_SNARKS) - .cloned() - .collect::>() - .try_into() - .expect("we have N_SNARKS chunks"); - - Self { - num_valid_chunks: num_valid_chunks.try_into().unwrap(), - chunk_sizes, - chunk_data, - } - } -} - -impl Default for BatchData { - fn default() -> Self { - // default value corresponds to a batch with 1 chunk with no transactions - Self::from(&vec![vec![]]) - } -} - -impl BatchData { - /// The number of rows in Blob Data config's layout to represent the "digest rlc" section. - /// - metadata digest RLC (1 row) - /// - chunk_digests RLC for each chunk (MAX_AGG_SNARKS rows) - /// - blob versioned hash RLC (1 row) - /// - challenge digest RLC (1 row) - pub const fn n_rows_digest_rlc() -> usize { - 1 + N_SNARKS + 1 + 1 - } - - /// The number of rows in Blob Data config's layout to represent the "digest bytes" section. - pub const fn n_rows_digest_bytes() -> usize { - Self::n_rows_digest_rlc() * N_BYTES_U256 - } - - /// The number of rows to encode the size of each chunk in a batch, in the Blob Data config. - /// chunk_size is u32, we use 4 bytes/rows. - const fn n_rows_chunk_sizes() -> usize { - N_SNARKS * N_ROWS_CHUNK_SIZE - } - - /// The total number of rows in "digest rlc" and "digest bytes" sections. - const fn n_rows_digest() -> usize { - Self::n_rows_digest_rlc() + Self::n_rows_digest_bytes() - } - - /// The number of rows in Blob Data config's layout to represent the "blob metadata" section. - pub const fn n_rows_metadata() -> usize { - N_ROWS_NUM_CHUNKS + Self::n_rows_chunk_sizes() - } - - /// The number of rows in Blob Data config's layout to represent the "chunk data" section. - pub const fn n_rows_data() -> usize { - N_BATCH_BYTES - Self::n_rows_metadata() - } - - /// The total number of rows used in Blob Data config's layout. - pub const fn n_rows() -> usize { - N_BATCH_BYTES + Self::n_rows_digest() - } - - /// Construct BatchData from chunks - pub fn new(num_valid_chunks: usize, chunks_with_padding: &[ChunkInfo]) -> Self { - assert!(num_valid_chunks > 0); - assert!(num_valid_chunks <= N_SNARKS); - - // padded chunk has 0 size, valid chunk's size is the number of bytes consumed by the - // flattened data from signed L2 transactions. - let chunk_sizes: [u32; N_SNARKS] = chunks_with_padding - .iter() - .map(|chunk| { - if chunk.is_padding { - 0 - } else { - chunk.tx_bytes.len() as u32 - } - }) - .collect::>() - .try_into() - .unwrap(); - - if chunk_sizes.iter().sum::() > Self::n_rows_data() as u32 { - panic!( - "invalid chunk_sizes {}, n_rows_data {}", - chunk_sizes.iter().sum::(), - Self::n_rows_data() - ) - } - - // chunk data of the "last valid chunk" is repeated over the padded chunks for simplicity - // in calculating chunk_data_digest for those padded chunks. However, for the "chunk data" - // section rows (self.to_data_rows()) we only consider `num_valid_chunks` chunks. - let chunk_data = chunks_with_padding - .iter() - .map(|chunk| chunk.tx_bytes.to_vec()) - .collect::>>() - .try_into() - .unwrap(); - - Self { - num_valid_chunks: num_valid_chunks as u16, - chunk_sizes, - chunk_data, - } - } - - /// Get the preimage of the challenge digest. - pub(crate) fn get_challenge_digest_preimage(&self, versioned_hash: H256) -> Vec { - let metadata_digest = keccak256(self.to_metadata_bytes()); - let chunk_digests = self.chunk_data.iter().map(keccak256); - - // preimage = - // metadata_digest || - // chunk[0].chunk_data_digest || ... - // chunk[N_SNARKS-1].chunk_data_digest || - // blob_versioned_hash - // - // where chunk_data_digest for a padded chunk is set equal to the "last valid chunk"'s - // chunk_data_digest. - metadata_digest - .into_iter() - .chain(chunk_digests.flatten()) - .chain(versioned_hash.to_fixed_bytes()) - .collect::>() - } - - /// Compute the challenge digest from blob bytes. - pub(crate) fn get_challenge_digest(&self, versioned_hash: H256) -> U256 { - let challenge_digest = keccak256(self.get_challenge_digest_preimage(versioned_hash)); - U256::from_big_endian(&challenge_digest) - } - - /// Get the batch data bytes that will be populated in BatchDataConfig. - pub fn get_batch_data_bytes(&self) -> Vec { - let metadata_bytes = self.to_metadata_bytes(); - metadata_bytes - .iter() - .chain( - self.chunk_data - .iter() - .take(self.num_valid_chunks as usize) - .flatten(), - ) - .cloned() - .collect() - } - - /// Get the list of preimages that need to go through the keccak hashing function, and - /// eventually required to be checked for the consistency of blob's metadata, its chunks' bytes - /// and the final blob preimage. - pub fn preimages(&self, versioned_hash: H256) -> Vec> { - let mut preimages = Vec::with_capacity(2 + N_SNARKS); - - // metadata - preimages.push(self.to_metadata_bytes()); - - // each valid chunk's data - for chunk in self.chunk_data.iter().take(self.num_valid_chunks as usize) { - preimages.push(chunk.to_vec()); - } - - // preimage for challenge digest - preimages.push(self.get_challenge_digest_preimage(versioned_hash)); - - preimages - } - - /// Get the witness rows for assignment to the BlobDataConfig. - pub(crate) fn to_rows( - &self, - versioned_hash: H256, - challenge: Challenges>, - ) -> Vec> { - let metadata_rows = self.to_metadata_rows(challenge); - assert_eq!(metadata_rows.len(), Self::n_rows_metadata()); - - let data_rows = self.to_data_rows(challenge); - assert_eq!(data_rows.len(), Self::n_rows_data()); - - let digest_rows = self.to_digest_rows(versioned_hash, challenge); - assert_eq!(digest_rows.len(), Self::n_rows_digest()); - - metadata_rows - .into_iter() - .chain(data_rows) - .chain(digest_rows) - .collect::>>() - } - - /// Get the blob bytes that encode the batch's metadata. - /// - /// metadata_bytes = - /// be_bytes(num_valid_chunks) || - /// be_bytes(chunks[0].chunk_size) || ... - /// be_bytes(chunks[N_SNARKS-1].chunk_size) - /// - /// where: - /// - chunk_size of a padded chunk is 0 - /// - num_valid_chunks is u16 - /// - each chunk_size is u32 - fn to_metadata_bytes(&self) -> Vec { - self.num_valid_chunks - .to_be_bytes() - .into_iter() - .chain( - self.chunk_sizes - .iter() - .flat_map(|chunk_size| chunk_size.to_be_bytes()), - ) - .collect() - } - - /// Get the witness rows for the "metadata" section of Blob data config. - fn to_metadata_rows(&self, challenge: Challenges>) -> Vec> { - // metadata bytes. - let bytes = self.to_metadata_bytes(); - - // accumulators represent the running linear combination of bytes. - let accumulators_iter = self - .num_valid_chunks - .to_be_bytes() - .into_iter() - .scan(0u64, |acc, x| { - *acc = *acc * 256 + (x as u64); - Some(*acc) - }) - .chain(self.chunk_sizes.into_iter().flat_map(|chunk_size| { - chunk_size.to_be_bytes().into_iter().scan(0u64, |acc, x| { - *acc = *acc * 256 + (x as u64); - Some(*acc) - }) - })); - - // digest_rlc is set only for the last row in the "metadata" section, and it denotes the - // RLC of the metadata_digest bytes. - let digest_rlc_iter = { - let digest = keccak256(&bytes); - let digest_rlc = digest.iter().fold(Value::known(Fr::zero()), |acc, &x| { - acc * challenge.evm_word() + Value::known(Fr::from(x as u64)) - }); - repeat(Value::known(Fr::zero())) - .take(Self::n_rows_metadata() - 1) - .chain(once(digest_rlc)) - }; - - // preimage_rlc is the running RLC over bytes in the "metadata" section. - let preimage_rlc_iter = bytes.iter().scan(Value::known(Fr::zero()), |acc, &x| { - *acc = *acc * challenge.keccak_input() + Value::known(Fr::from(x as u64)); - Some(*acc) - }); - - bytes - .iter() - .zip_eq(accumulators_iter) - .zip_eq(preimage_rlc_iter) - .zip_eq(digest_rlc_iter) - .enumerate() - .map( - |(i, (((&byte, accumulator), preimage_rlc), digest_rlc))| BatchDataRow { - byte, - accumulator, - preimage_rlc, - digest_rlc, - // we set boundary on the last row of the "metadata" section to enable a lookup - // to the keccak table. - is_boundary: i == bytes.len() - 1, - ..Default::default() - }, - ) - .collect() - } - - /// Get the witness rows for the "chunk data" section of Blob data config. - fn to_data_rows(&self, challenge: Challenges>) -> Vec> { - // consider only the `valid` chunks while constructing rows for the "chunk data" section. - self.chunk_data - .iter() - .take(self.num_valid_chunks as usize) - .enumerate() - .flat_map(|(i, bytes)| { - let chunk_idx = (i + 1) as u64; - let chunk_size = bytes.len(); - let chunk_digest = keccak256(bytes); - let digest_rlc = chunk_digest - .iter() - .fold(Value::known(Fr::zero()), |acc, &byte| { - acc * challenge.evm_word() + Value::known(Fr::from(byte as u64)) - }); - bytes.iter().enumerate().scan( - (0u64, Value::known(Fr::zero())), - move |acc, (j, &byte)| { - acc.0 += 1; - acc.1 = - acc.1 * challenge.keccak_input() + Value::known(Fr::from(byte as u64)); - Some(BatchDataRow { - byte, - accumulator: acc.0, - chunk_idx, - is_boundary: j == chunk_size - 1, - is_padding: false, - preimage_rlc: acc.1, - digest_rlc: if j == chunk_size - 1 { - digest_rlc - } else { - Value::known(Fr::zero()) - }, - }) - }, - ) - }) - .chain(repeat(BatchDataRow::padding_row())) - .take(Self::n_rows_data()) - .collect() - } - - /// Get the witness rows for both "digest rlc" and "digest bytes" sections of Blob data config. - fn to_digest_rows( - &self, - versioned_hash: H256, - challenge: Challenges>, - ) -> Vec> { - let zero = Value::known(Fr::zero()); - - // metadata - let metadata_bytes = self.to_metadata_bytes(); - let metadata_digest = keccak256(metadata_bytes); - let metadata_digest_rlc = metadata_digest.iter().fold(zero, |acc, &byte| { - acc * challenge.evm_word() + Value::known(Fr::from(byte as u64)) - }); - - // chunk data - // Note: here we don't restrict to considering only `valid` chunks, as the - // `chunk_data_digest` gets repeated for the padded chunks, copying the last valid chunk's - // `chunk_data_digest`. - let (chunk_digests, chunk_digest_rlcs): (Vec<[u8; 32]>, Vec>) = self - .chunk_data - .iter() - .map(|chunk| { - let digest = keccak256(chunk); - let digest_rlc = digest.iter().fold(zero, |acc, &byte| { - acc * challenge.evm_word() + Value::known(Fr::from(byte as u64)) - }); - (digest, digest_rlc) - }) - .unzip(); - - // challenge digest - let challenge_digest_preimage = self.get_challenge_digest_preimage(versioned_hash); - let challenge_digest_preimage_rlc = - challenge_digest_preimage.iter().fold(zero, |acc, &byte| { - acc * challenge.keccak_input() + Value::known(Fr::from(byte as u64)) - }); - let challenge_digest = keccak256(&challenge_digest_preimage); - let challenge_digest_rlc = challenge_digest.iter().fold(zero, |acc, &byte| { - acc * challenge.evm_word() + Value::known(Fr::from(byte as u64)) - }); - - // blob versioned hash - let versioned_hash_rlc = versioned_hash.as_bytes().iter().fold(zero, |acc, &byte| { - acc * challenge.evm_word() + Value::known(Fr::from(byte as u64)) - }); - - // - metadata digest rlc - // - chunks[i].chunk_data_digest rlc for each chunk - // - versioned hash rlc - // - challenge digest rlc - // - metadata digest bytes - // - chunks[i].chunk_data_digest bytes for each chunk - // - versioned hash bytes - // - challenge digest bytes - once(BatchDataRow { - preimage_rlc: Value::known(Fr::zero()), - digest_rlc: metadata_digest_rlc, - // this is_padding assignment does not matter as we have already crossed the "chunk - // data" section. This assignment to 1 is simply to allow the custom gate to check: - // - padding transitions from 0 -> 1 only once. - is_padding: true, - ..Default::default() - }) - .chain( - chunk_digest_rlcs - .iter() - .zip_eq(self.chunk_sizes.iter()) - .enumerate() - .map(|(i, (&digest_rlc, &chunk_size))| BatchDataRow { - preimage_rlc: Value::known(Fr::zero()), - digest_rlc, - chunk_idx: (i + 1) as u64, - accumulator: chunk_size as u64, - ..Default::default() - }), - ) - // versioned hash RLC - .chain(once(BatchDataRow { - preimage_rlc: Value::known(Fr::zero()), - digest_rlc: versioned_hash_rlc, - ..Default::default() - })) - .chain(once(BatchDataRow { - preimage_rlc: challenge_digest_preimage_rlc, - digest_rlc: challenge_digest_rlc, - accumulator: 32 * (N_SNARKS + 1 + 1) as u64, - is_boundary: true, - ..Default::default() - })) - .chain(metadata_digest.iter().map(|&byte| BatchDataRow { - preimage_rlc: Value::known(Fr::zero()), - digest_rlc: Value::known(Fr::zero()), - byte, - ..Default::default() - })) - .chain(chunk_digests.iter().flat_map(|digest| { - digest.iter().map(|&byte| BatchDataRow { - preimage_rlc: Value::known(Fr::zero()), - digest_rlc: Value::known(Fr::zero()), - byte, - ..Default::default() - }) - })) - // bytes of versioned hash - .chain(versioned_hash.as_bytes().iter().map(|&byte| BatchDataRow { - preimage_rlc: Value::known(Fr::zero()), - digest_rlc: Value::known(Fr::zero()), - byte, - ..Default::default() - })) - .chain(challenge_digest.iter().map(|&byte| BatchDataRow { - preimage_rlc: Value::known(Fr::zero()), - digest_rlc: Value::known(Fr::zero()), - byte, - ..Default::default() - })) - .collect() - } -} - -#[derive(Clone, Debug)] -pub struct PointEvaluationAssignments { - /// The random challenge scalar z. - pub challenge: U256, - /// The 32-bytes keccak digest for the challenge. We have the relation: - /// - challenge := challenge_digest % BLS_MODULUS. - pub challenge_digest: U256, - /// The evaluation of the blob polynomial at challenge. - pub evaluation: U256, - /// The blob polynomial represented in evaluation form. - pub coefficients: [U256; BLOB_WIDTH], -} - -impl Default for PointEvaluationAssignments { - fn default() -> Self { - Self { - challenge: U256::default(), - challenge_digest: U256::default(), - evaluation: U256::default(), - coefficients: [U256::default(); BLOB_WIDTH], - } - } -} - -impl PointEvaluationAssignments { - /// Construct the point evaluation assignments. - pub fn new( - batch_data: &BatchData, - blob_bytes: &[u8], - versioned_hash: H256, - ) -> Self { - // blob polynomial in evaluation form. - // - // also termed P(x) - let coefficients = get_coefficients(blob_bytes); - let coefficients_as_scalars = coefficients.map(|coeff| Scalar::from_raw(coeff.0)); - - // challenge := challenge_digest % BLS_MODULUS - // - // also termed z - let challenge_digest = batch_data.get_challenge_digest(versioned_hash); - let (_, challenge) = challenge_digest.div_mod(*BLS_MODULUS); - - // y = P(z) - let evaluation = U256::from_little_endian( - &interpolate(Scalar::from_raw(challenge.0), &coefficients_as_scalars).to_bytes(), - ); - - Self { - challenge, - challenge_digest, - evaluation, - coefficients, - } - } -} - -/// Witness row to the Blob Data Config. -#[derive(Clone, Copy, Default, Debug)] -pub struct BatchDataRow { - /// Byte value at this row. - pub byte: u8, - /// Multi-purpose accumulator value. - pub accumulator: u64, - /// The chunk index we are currently traversing. - pub chunk_idx: u64, - /// Whether this marks the end of a chunk. - pub is_boundary: bool, - /// Whether the row represents right-padded 0s to the blob data. - pub is_padding: bool, - /// A running accumulator of RLC of preimages. - pub preimage_rlc: Value, - /// RLC of the digest. - pub digest_rlc: Value, -} - -impl BatchDataRow { - fn padding_row() -> Self { - Self { - is_padding: true, - preimage_rlc: Value::known(Fr::zero()), - digest_rlc: Value::known(Fr::zero()), - ..Default::default() - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::{ - eip4844::{get_blob_bytes, get_versioned_hash}, - MAX_AGG_SNARKS, - }; - - #[test] - #[ignore = "only required for logging challenge digest"] - fn log_challenge() { - let n_rows_data = BatchData::::n_rows_data(); - - for (annotation, tcase) in [ - ("single empty chunk", vec![vec![]]), - ("single non-empty chunk", vec![vec![1, 2, 3]]), - ("multiple empty chunks", vec![vec![], vec![]]), - ( - "multiple non-empty chunks", - vec![vec![1, 2, 3], vec![7, 8, 9]], - ), - ( - "empty chunk followed by non-empty chunk", - vec![vec![], vec![1, 2, 3]], - ), - ( - "non-empty chunk followed by empty chunk", - vec![vec![7, 8, 9], vec![]], - ), - ( - "max number of chunks all empty", - vec![vec![]; MAX_AGG_SNARKS], - ), - ( - "max number of chunks all non-empty", - (0..MAX_AGG_SNARKS) - .map(|i| (10u8..11 + u8::try_from(i).unwrap()).collect()) - .collect(), - ), - ("single chunk blob full", vec![vec![123; n_rows_data]]), - ( - "multiple chunks blob full", - vec![vec![123; 1111], vec![231; n_rows_data - 1111]], - ), - ( - "max number of chunks only last one non-empty not full blob", - repeat(vec![]) - .take(MAX_AGG_SNARKS - 1) - .chain(once(vec![132; n_rows_data - 1111])) - .collect(), - ), - ( - "max number of chunks only last one non-empty full blob", - repeat(vec![]) - .take(MAX_AGG_SNARKS - 1) - .chain(once(vec![132; n_rows_data])) - .collect(), - ), - ( - "max number of chunks but last is empty", - repeat(vec![111; 100]) - .take(MAX_AGG_SNARKS - 1) - .chain(once(vec![])) - .collect(), - ), - ] - .iter() - { - let batch_header = crate::batch::BatchHeader { - version: 3, - batch_index: 6789, - l1_message_popped: 101, - total_l1_message_popped: 10101, - parent_batch_hash: H256::repeat_byte(1), - last_block_timestamp: 192837, - ..Default::default() - }; - let batch_data: BatchData = tcase.into(); - let batch_bytes = batch_data.get_batch_data_bytes(); - let blob_bytes = get_blob_bytes(&batch_bytes); - let coeffs = get_coefficients(&blob_bytes); - let versioned_hash = get_versioned_hash(&coeffs); - let chunks_without_padding = crate::chunk::ChunkInfo::mock_chunk_infos(tcase); - let batch_hash = BatchHash::::construct_with_unpadded( - &chunks_without_padding, - batch_header, - &blob_bytes, - ); - let point_evaluation_assignments = - PointEvaluationAssignments::new(&batch_data, &blob_bytes, versioned_hash); - println!( - "[[ {:60} ]]\nchallenge (z) = {:0>64x}, evaluation (y) = {:0>64x}, versioned hash = {:0>64x}, batch_hash = {:0>64x}\n\n", - annotation, - point_evaluation_assignments.challenge, - point_evaluation_assignments.evaluation, - versioned_hash, - batch_hash.current_batch_hash, - ); - } - } - - #[test] - fn default_batch_data() { - let mut default_metadata = [0u8; BatchData::::n_rows_metadata()]; - default_metadata[1] = 1; - let default_metadata_digest = keccak256(default_metadata); - let default_chunk_digests = [keccak256([]); MAX_AGG_SNARKS]; - - let default_batch = BatchData::::default(); - let batch_bytes = default_batch.get_batch_data_bytes(); - let blob_bytes = get_blob_bytes(&batch_bytes); - let coeffs = get_coefficients(&blob_bytes); - let versioned_hash = get_versioned_hash(&coeffs); - let point_evaluation_assignments = - PointEvaluationAssignments::new(&default_batch, &blob_bytes, versioned_hash); - let versioned_hash = get_versioned_hash(&point_evaluation_assignments.coefficients); - assert_eq!( - default_batch.get_challenge_digest(versioned_hash), - U256::from(keccak256( - default_metadata_digest - .into_iter() - .chain(default_chunk_digests.into_iter().flatten()) - .chain(versioned_hash.to_fixed_bytes()) - .collect::>() - )), - ) - } -} diff --git a/aggregator/src/blob_consistency.rs b/aggregator/src/blob_consistency.rs new file mode 100644 index 0000000000..b49f676011 --- /dev/null +++ b/aggregator/src/blob_consistency.rs @@ -0,0 +1,27 @@ +use cfg_if::cfg_if; + +// enum DataAvailability { +// Eip4844, +// Avail, +// } + +mod blob_data; +use blob_data::{AssignedBlobDataExport, BlobDataConfig}; + +// TODO: remove dead code instead +#[allow(dead_code)] +mod avail; + +// TODO: remove dead code instead +#[allow(dead_code)] +mod eip4844; + +cfg_if! { + if #[cfg(feature = "da-avail")] { + // const DATA_AVAILABILITY: DataAvailability = DataAvailability::Avail; + pub use avail::{BlobConsistencyConfig, BlobConsistencyWitness, BLOB_WIDTH}; + } else { + // const DATA_AVAILABILITY: DatayAvailability = DataAvailability::Eip4844; + pub use eip4844::{BlobConsistencyConfig, BlobConsistencyWitness, BLOB_WIDTH}; + } +} diff --git a/aggregator/src/blob_consistency/avail.rs b/aggregator/src/blob_consistency/avail.rs new file mode 100644 index 0000000000..7c7474c8be --- /dev/null +++ b/aggregator/src/blob_consistency/avail.rs @@ -0,0 +1,100 @@ +use super::AssignedBlobDataExport; +use crate::{BatchData, RlcConfig}; +use eth_types::{H256, U256}; +use halo2_base::{gates::range::RangeConfig, AssignedValue, Context}; +use halo2_ecc::bigint::CRTInteger; +use halo2_proofs::halo2curves::bls12_381::Scalar; +use halo2_proofs::{ + circuit::{AssignedCell, Layouter, Value}, + halo2curves::bn256::Fr, + plonk::{ConstraintSystem, Error, Expression}, +}; +use snark_verifier_sdk::LIMBS; +use zkevm_circuits::{table::U8Table, util::Challenges}; + +pub const BLOB_WIDTH: usize = 4096; + +#[derive(Debug, Clone)] +pub struct BlobConsistencyConfig {} + +impl BlobConsistencyConfig { + pub fn construct( + _meta: &mut ConstraintSystem, + _challenges: &Challenges>, + _u8_table: U8Table, + _: RangeConfig, + ) -> Self { + unimplemented!() + } + + pub fn assign_barycentric( + &self, + _ctx: &mut Context, + _bytes: &[u8], + _challenge: U256, + ) -> AssignedBarycentricEvaluationConfig { + unimplemented!() + } + + pub fn assign_blob_data( + &self, + _layouter: &mut impl Layouter, + _challenge_value: Challenges>, + _rlc_config: &RlcConfig, + _blob_bytes: &[u8], + ) -> Result { + unimplemented!() + } + + pub fn link( + _layouter: &mut impl Layouter, + _blob_crts_limbs: &[[AssignedCell; LIMBS]], + _barycentric_crts: &[CRTInteger], + ) -> Result<(), Error> { + unimplemented!() + } +} + +#[derive(Debug, Clone, Copy, Default)] +pub struct BlobConsistencyWitness { + blob_versioned_hash: H256, + challenge_digest: H256, + evaluation: Scalar, +} + +impl BlobConsistencyWitness { + pub fn new(_bytes: &[u8], _batch_data: &BatchData) -> Self { + unimplemented!() + } + + pub fn id(&self) -> H256 { + unimplemented!() + } + + pub fn challenge_digest(&self) -> U256 { + unimplemented!() + } + + pub fn challenge(&self) -> Scalar { + unimplemented!() + } + + pub fn evaluation(&self) -> Scalar { + unimplemented!() + } + + pub fn blob_data_proof(&self) -> [H256; 2] { + unimplemented!() + } +} + +#[derive(Default)] +pub struct AssignedBarycentricEvaluationConfig { + /// CRTIntegers for the BLOB_WIDTH number of blob polynomial coefficients, followed by a + /// CRTInteger for the challenge digest. + pub(crate) barycentric_assignments: Vec>, + /// 32 Assigned cells representing the LE-bytes of challenge z. + pub(crate) z_le: Vec>, + /// 32 Assigned cells representing the LE-bytes of evaluation y. + pub(crate) y_le: Vec>, +} diff --git a/aggregator/src/aggregation/blob_data.rs b/aggregator/src/blob_consistency/blob_data.rs similarity index 83% rename from aggregator/src/aggregation/blob_data.rs rename to aggregator/src/blob_consistency/blob_data.rs index 4d02c55db9..e604eb34b9 100644 --- a/aggregator/src/aggregation/blob_data.rs +++ b/aggregator/src/blob_consistency/blob_data.rs @@ -1,5 +1,12 @@ +use crate::{ + aggregation::{ + batch_data::{N_BLOB_BYTES, N_DATA_BYTES_PER_COEFFICIENT}, + POWS_OF_256, + }, + blob_consistency::BLOB_WIDTH, + RlcConfig, +}; use gadgets::util::Expr; -use halo2_ecc::bigint::CRTInteger; use halo2_proofs::{ circuit::{AssignedCell, Layouter, Region, Value}, halo2curves::bn256::Fr, @@ -7,14 +14,9 @@ use halo2_proofs::{ poly::Rotation, }; use itertools::Itertools; +use snark_verifier_sdk::{BITS, LIMBS}; use zkevm_circuits::{table::U8Table, util::Challenges}; -use crate::{ - aggregation::rlc::POWS_OF_256, - blob::{BLOB_WIDTH, N_BLOB_BYTES, N_DATA_BYTES_PER_COEFFICIENT}, - RlcConfig, -}; - /// Blob is represented by 4096 BLS12-381 scalar field elements, where each element is represented /// by 32 bytes. The scalar field element is required to be in the canonical form, i.e. its value /// MUST BE less than the BLS_MODULUS. In order to ensure this, we hard-code the most-significant @@ -50,6 +52,7 @@ pub struct AssignedBlobDataExport { pub bytes_rlc: AssignedCell, pub bytes_len: AssignedCell, pub cooked_len: AssignedCell, + pub blob_crts_limbs: [[AssignedCell; LIMBS]; BLOB_WIDTH], } impl BlobDataConfig { @@ -154,7 +157,6 @@ impl BlobDataConfig { challenge_value: Challenges>, rlc_config: &RlcConfig, blob_bytes: &[u8], - barycentric_assignments: &[CRTInteger], ) -> Result { let (assigned_bytes, bytes_rlc, bytes_len, enable_encoding_bool) = layouter.assign_region( || "BlobData bytes", @@ -162,16 +164,10 @@ impl BlobDataConfig { )?; let enable_encoding = assigned_bytes[0].clone(); - let cooked_len = layouter.assign_region( + let (cooked_len, blob_crts_limbs) = layouter.assign_region( || "BlobData internal checks", |mut region| { - self.assign_internal_checks( - &mut region, - rlc_config, - barycentric_assignments, - &assigned_bytes, - &bytes_len, - ) + self.assign_internal_checks(&mut region, rlc_config, &assigned_bytes, &bytes_len) }, )?; @@ -181,6 +177,7 @@ impl BlobDataConfig { bytes_rlc, bytes_len, cooked_len, + blob_crts_limbs, }) } @@ -290,14 +287,20 @@ impl BlobDataConfig { )) } + #[allow(clippy::type_complexity)] pub fn assign_internal_checks( &self, region: &mut Region, rlc_config: &RlcConfig, - barycentric_assignments: &[CRTInteger], assigned_bytes: &[AssignedCell], bytes_len: &AssignedCell, - ) -> Result, Error> { + ) -> Result< + ( + AssignedCell, + [[AssignedCell; LIMBS]; BLOB_WIDTH], + ), + Error, + > { rlc_config.init(region)?; let mut rlc_config_offset = 0; @@ -328,51 +331,56 @@ impl BlobDataConfig { // form of the batch. rlc_config.enforce_binary(region, &assigned_bytes[0], &mut rlc_config_offset)?; - //////////////////////////////////////////////////////////////////////////////// - //////////////////////////////////// LINKING /////////////////////////////////// - //////////////////////////////////////////////////////////////////////////////// - - assert_eq!(barycentric_assignments.len(), BLOB_WIDTH + 1); - let blob_crts = barycentric_assignments - .iter() - .take(BLOB_WIDTH) - .collect::>(); - let mut blob_fields: Vec>> = Vec::with_capacity(BLOB_WIDTH); - for chunk in assigned_bytes.chunks_exact(N_DATA_BYTES_PER_COEFFICIENT) { - // blob bytes are supposed to be deserialised in big-endianness. However, we - // have the export from BarycentricConfig in little-endian bytes. - blob_fields.push(chunk.iter().rev().cloned().collect()); - } - - for (blob_crt, blob_field) in blob_crts.iter().zip_eq(blob_fields.iter()) { - let limb1 = rlc_config.inner_product( - region, - &blob_field[0..11], - &pows_of_256, - &mut rlc_config_offset, - )?; - let limb2 = rlc_config.inner_product( - region, - &blob_field[11..22], - &pows_of_256, - &mut rlc_config_offset, - )?; - let limb3 = rlc_config.inner_product( - region, - &blob_field[22..31], - &pows_of_256[0..9], - &mut rlc_config_offset, - )?; - region.constrain_equal(limb1.cell(), blob_crt.truncation.limbs[0].cell())?; - region.constrain_equal(limb2.cell(), blob_crt.truncation.limbs[1].cell())?; - region.constrain_equal(limb3.cell(), blob_crt.truncation.limbs[2].cell())?; - } + // Compute 88 bit limbs so we can later check to equality to the inputs in the barycentric + // evaluation circuit. + let blob_crts_limbs = blob_crts_limbs( + region, + rlc_config, + assigned_bytes, + &pows_of_256, + &mut rlc_config_offset, + ); // The zstd decoder (DecoderConfig) exports an encoded length that is 1 more than the // actual number of bytes in encoded data. Accordingly we "cook" the actual len(bytes) here // by adding +1 to it before exporting. let cooked_bytes_len = rlc_config.add(region, bytes_len, &one, &mut rlc_config_offset)?; - Ok(cooked_bytes_len) + Ok((cooked_bytes_len, blob_crts_limbs)) } } + +fn blob_crts_limbs( + region: &mut Region, + rlc_config: &RlcConfig, + bytes: &[AssignedCell], + powers_of_256: &[AssignedCell], + offset: &mut usize, +) -> [[AssignedCell; LIMBS]; BLOB_WIDTH] { + bytes + .chunks_exact(N_DATA_BYTES_PER_COEFFICIENT) + .map(|coefficient_bytes| { + coefficient_bytes + .iter() + .rev() // reverse bytes to match endianness of crt limbs + .chunks(BITS / 8) + .into_iter() + .map(|chunk_bytes| { + let chunk_bytes = chunk_bytes.into_iter().cloned().collect_vec(); + rlc_config + .inner_product( + region, + &chunk_bytes, + &powers_of_256[..chunk_bytes.len()], + offset, + ) + .unwrap() + }) + .collect_vec() + .try_into() + .unwrap() + }) + .collect_vec() + .try_into() + .unwrap() +} diff --git a/aggregator/src/blob_consistency/eip4844.rs b/aggregator/src/blob_consistency/eip4844.rs new file mode 100644 index 0000000000..8958f0ffd5 --- /dev/null +++ b/aggregator/src/blob_consistency/eip4844.rs @@ -0,0 +1,231 @@ +/// Config to evaluate blob polynomial at a random challenge. +mod barycentric; +use barycentric::{ + interpolate, AssignedBarycentricEvaluationConfig, BarycentricEvaluationConfig, BLS_MODULUS, +}; + +/// blob struct and constants +mod blob; +use blob::PointEvaluationAssignments; + +#[cfg(test)] +mod tests; + +use super::{AssignedBlobDataExport, BlobDataConfig}; +use crate::{ + aggregation::batch_data::N_DATA_BYTES_PER_COEFFICIENT, constants::N_BYTES_U256, BatchData, + RlcConfig, +}; +use eth_types::{ToBigEndian, ToLittleEndian, H256, U256}; +use ethers_core::k256::sha2::{Digest, Sha256}; +use halo2_base::{gates::range::RangeConfig, Context}; +use halo2_ecc::bigint::CRTInteger; +use halo2_proofs::halo2curves::bls12_381::Scalar; +use halo2_proofs::{ + circuit::{AssignedCell, Layouter, Value}, + halo2curves::bn256::Fr, + plonk::{ConstraintSystem, Error, Expression}, +}; +use itertools::Itertools; +use once_cell::sync::Lazy; +use revm_primitives::VERSIONED_HASH_VERSION_KZG; +use snark_verifier_sdk::LIMBS; +use std::sync::Arc; +use zkevm_circuits::{table::U8Table, util::Challenges}; + +pub const BLOB_WIDTH: usize = 4096; +pub const N_BLOB_BYTES: usize = BLOB_WIDTH * N_DATA_BYTES_PER_COEFFICIENT; + +/// Get the BLOB_WIDTH number of scalar field elements, as 32-bytes unsigned integers. +fn get_coefficients(blob_bytes: &[u8]) -> [U256; BLOB_WIDTH] { + let mut coefficients = [[0u8; N_BYTES_U256]; BLOB_WIDTH]; + + assert!( + blob_bytes.len() <= N_BLOB_BYTES, + "too many bytes in batch data" + ); + + for (i, &byte) in blob_bytes.iter().enumerate() { + coefficients[i / 31][1 + (i % 31)] = byte; + } + + coefficients.map(|coeff| U256::from_big_endian(&coeff)) +} + +/// KZG trusted setup +static KZG_TRUSTED_SETUP: Lazy> = Lazy::new(|| { + Arc::new( + c_kzg::KzgSettings::load_trusted_setup( + &revm_primitives::kzg::G1_POINTS.0, + &revm_primitives::kzg::G2_POINTS.0, + ) + .expect("failed to load trusted setup"), + ) +}); + +/// Get the versioned hash as per EIP-4844. +pub fn get_versioned_hash(coefficients: &[U256; BLOB_WIDTH]) -> H256 { + let blob = c_kzg::Blob::from_bytes( + &coefficients + .iter() + .cloned() + .flat_map(|coeff| coeff.to_be_bytes()) + .collect::>(), + ) + .expect("blob-coefficients to 4844 blob should succeed"); + let c = c_kzg::KzgCommitment::blob_to_kzg_commitment(&blob, &KZG_TRUSTED_SETUP) + .expect("blob to kzg commitment should succeed"); + kzg_to_versioned_hash(&c) +} + +fn kzg_to_versioned_hash(commitment: &c_kzg::KzgCommitment) -> H256 { + let mut res = Sha256::digest(commitment.as_slice()); + res[0] = VERSIONED_HASH_VERSION_KZG; + H256::from_slice(&res[..]) +} + +#[cfg(test)] +/// Get the blob data bytes that will be populated in BlobDataConfig. +pub fn get_blob_bytes(batch_bytes: &[u8]) -> Vec { + let mut blob_bytes = crate::witgen::zstd_encode(batch_bytes); + + // Whether we encode batch -> blob or not. + let enable_encoding = blob_bytes.len() < batch_bytes.len(); + if !enable_encoding { + blob_bytes = batch_bytes.to_vec(); + } + blob_bytes.insert(0, enable_encoding as u8); + + blob_bytes +} + +#[derive(Debug, Clone)] +pub struct BlobConsistencyConfig { + data: BlobDataConfig, + barycentric_evaluation: BarycentricEvaluationConfig, +} + +impl BlobConsistencyConfig { + pub fn construct( + meta: &mut ConstraintSystem, + challenges: &Challenges>, + u8_table: U8Table, + range: RangeConfig, + ) -> Self { + Self { + data: BlobDataConfig::configure(meta, challenges, u8_table), + barycentric_evaluation: BarycentricEvaluationConfig::construct(range), + } + } + + pub fn assign_barycentric( + &self, + ctx: &mut Context, + bytes: &[u8], + challenge: U256, + ) -> AssignedBarycentricEvaluationConfig { + self.barycentric_evaluation.assign(ctx, bytes, challenge) + } + + pub fn assign_blob_data( + &self, + layouter: &mut impl Layouter, + challenge_value: Challenges>, + rlc_config: &RlcConfig, + blob_bytes: &[u8], + ) -> Result { + self.data + .assign(layouter, challenge_value, rlc_config, blob_bytes) + } + + pub fn link( + layouter: &mut impl Layouter, + blob_crts_limbs: &[[AssignedCell; LIMBS]], + barycentric_crts: &[CRTInteger], + ) -> Result<(), Error> { + assert_eq!(blob_crts_limbs.len(), BLOB_WIDTH); + + layouter.assign_region( + || "constrain barycentric inputs to match blob", + |mut region| { + for (blob_crt_limbs, barycentric_crt) in blob_crts_limbs + .iter() + .zip_eq(barycentric_crts.iter().take(BLOB_WIDTH)) + { + for (blob_limb, barycentric_limb) in + blob_crt_limbs.iter().zip_eq(barycentric_crt.limbs()) + { + region.constrain_equal(blob_limb.cell(), barycentric_limb.cell())?; + } + } + Ok(()) + }, + ) + } +} + +#[derive(Debug, Clone, Copy, Default)] +pub struct BlobConsistencyWitness { + blob_versioned_hash: H256, + challenge_digest: H256, + evaluation: Scalar, +} + +impl BlobConsistencyWitness { + pub fn new(bytes: &[u8], batch_data: &BatchData) -> Self { + let coeffs = get_coefficients(bytes); + let blob_versioned_hash = get_versioned_hash(&coeffs); + let point_evaluation_assignments = + PointEvaluationAssignments::new(batch_data, bytes, blob_versioned_hash); + Self { + blob_versioned_hash, + challenge_digest: digest_from_word(point_evaluation_assignments.challenge_digest), + evaluation: scalar_from_word(point_evaluation_assignments.evaluation), + } + } + + pub fn id(&self) -> H256 { + self.blob_versioned_hash + } + + pub fn challenge_digest(&self) -> U256 { + word_from_digest(self.challenge_digest) + } + + pub fn challenge(&self) -> Scalar { + scalar_from_digest(self.challenge_digest) + } + + pub fn evaluation(&self) -> Scalar { + self.evaluation + } + + pub fn blob_data_proof(&self) -> [H256; 2] { + [self.challenge(), self.evaluation].map(digest_from_scalar) + } +} + +fn digest_from_word(x: U256) -> H256 { + H256::from_slice(&x.to_be_bytes()) +} + +fn digest_from_scalar(x: Scalar) -> H256 { + let mut bytes = x.to_bytes(); + bytes.reverse(); + H256::from_slice(&bytes) +} + +fn scalar_from_word(x: U256) -> Scalar { + let (_quotient, remainder) = x.div_mod(*BLS_MODULUS); + Scalar::from_bytes(&remainder.to_le_bytes()).expect("non-canonical bytes") +} + +fn scalar_from_digest(x: H256) -> Scalar { + scalar_from_word(word_from_digest(x)) +} + +fn word_from_digest(x: H256) -> U256 { + U256::from_big_endian(&x.to_fixed_bytes()) +} + +// word_from_scalar would not be used. diff --git a/aggregator/src/aggregation/barycentric.rs b/aggregator/src/blob_consistency/eip4844/barycentric.rs similarity index 98% rename from aggregator/src/aggregation/barycentric.rs rename to aggregator/src/blob_consistency/eip4844/barycentric.rs index f485eac9b8..c46f20c068 100644 --- a/aggregator/src/aggregation/barycentric.rs +++ b/aggregator/src/blob_consistency/eip4844/barycentric.rs @@ -1,3 +1,5 @@ +use super::{blob::BLOB_WIDTH, get_coefficients}; +use crate::constants::{BITS, LIMBS, N_BYTES_U256}; use eth_types::{ToLittleEndian, U256}; use halo2_base::{ gates::{range::RangeConfig, GateInstructions}, @@ -17,11 +19,6 @@ use itertools::Itertools; use num_bigint::{BigInt, Sign}; use std::{iter::successors, sync::LazyLock}; -use crate::{ - blob::{BLOB_WIDTH, N_BYTES_U256}, - constants::{BITS, LIMBS}, -}; - /// Base 2 logarithm of BLOB_WIDTH. const LOG_BLOB_WIDTH: usize = 12; @@ -29,6 +26,7 @@ pub static BLS_MODULUS: LazyLock = LazyLock::new(|| { U256::from_str_radix(Scalar::MODULUS, 16).expect("BLS_MODULUS from bls crate") }); +// does this need to be pub? pub static ROOTS_OF_UNITY: LazyLock> = LazyLock::new(|| { // https://github.com/ethereum/consensus-specs/blob/dev/specs/deneb/polynomial-commitments.md#constants let primitive_root_of_unity = Scalar::from(7); @@ -101,9 +99,11 @@ impl BarycentricEvaluationConfig { pub fn assign( &self, ctx: &mut Context, - blob: &[U256; BLOB_WIDTH], + bytes: &[u8], challenge_digest: U256, ) -> AssignedBarycentricEvaluationConfig { + let blob = get_coefficients(bytes); + // some constants for later use. let one = self.scalar.load_constant(ctx, fe_to_biguint(&Fr::one())); let blob_width = self @@ -354,8 +354,8 @@ pub fn interpolate(z: Scalar, coefficients: &[Scalar; BLOB_WIDTH]) -> Scalar { mod tests { use super::*; use crate::{ - blob::{BatchData, KZG_TRUSTED_SETUP}, - eip4844::{get_blob_bytes, get_coefficients}, + aggregation::BatchData, + blob_consistency::eip4844::{get_blob_bytes, get_coefficients, KZG_TRUSTED_SETUP}, MAX_AGG_SNARKS, }; use c_kzg::{Blob as RethBlob, KzgProof}; diff --git a/aggregator/src/blob_consistency/eip4844/blob.rs b/aggregator/src/blob_consistency/eip4844/blob.rs new file mode 100644 index 0000000000..aa153e2689 --- /dev/null +++ b/aggregator/src/blob_consistency/eip4844/blob.rs @@ -0,0 +1,196 @@ +use super::{get_coefficients, interpolate, BLS_MODULUS}; +use crate::BatchData; +use eth_types::{H256, U256}; +use halo2_proofs::halo2curves::bls12_381::Scalar; + +/// The number of coefficients (BLS12-381 scalars) to represent the blob polynomial in evaluation +/// form. +pub const BLOB_WIDTH: usize = 4096; + +#[derive(Clone, Debug)] +pub struct PointEvaluationAssignments { + /// The random challenge scalar z. + pub challenge: U256, + /// The 32-bytes keccak digest for the challenge. We have the relation: + /// - challenge := challenge_digest % BLS_MODULUS. + pub challenge_digest: U256, + /// The evaluation of the blob polynomial at challenge. + pub evaluation: U256, + /// The blob polynomial represented in evaluation form. + pub coefficients: [U256; BLOB_WIDTH], +} + +impl Default for PointEvaluationAssignments { + fn default() -> Self { + Self { + challenge: U256::default(), + challenge_digest: U256::default(), + evaluation: U256::default(), + coefficients: [U256::default(); BLOB_WIDTH], + } + } +} + +impl PointEvaluationAssignments { + /// Construct the point evaluation assignments. + pub fn new( + batch_data: &BatchData, + blob_bytes: &[u8], + versioned_hash: H256, + ) -> Self { + // blob polynomial in evaluation form. + // + // also termed P(x) + let coefficients = get_coefficients(blob_bytes); + let coefficients_as_scalars = coefficients.map(|coeff| Scalar::from_raw(coeff.0)); + + // challenge := challenge_digest % BLS_MODULUS + // + // also termed z + let challenge_digest = batch_data.get_challenge_digest(versioned_hash); + let (_, challenge) = challenge_digest.div_mod(*BLS_MODULUS); + + // y = P(z) + let evaluation = U256::from_little_endian( + &interpolate(Scalar::from_raw(challenge.0), &coefficients_as_scalars).to_bytes(), + ); + + Self { + challenge, + challenge_digest, + evaluation, + coefficients, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + blob_consistency::eip4844::{get_blob_bytes, get_versioned_hash}, + BatchHash, ChunkInfo, MAX_AGG_SNARKS, + }; + use ethers_core::utils::keccak256; + use std::iter::{once, repeat}; + + #[test] + #[ignore = "only required for logging challenge digest"] + fn log_challenge() { + let n_rows_data = BatchData::::n_rows_data(); + + for (annotation, tcase) in [ + ("single empty chunk", vec![vec![]]), + ("single non-empty chunk", vec![vec![1, 2, 3]]), + ("multiple empty chunks", vec![vec![], vec![]]), + ( + "multiple non-empty chunks", + vec![vec![1, 2, 3], vec![7, 8, 9]], + ), + ( + "empty chunk followed by non-empty chunk", + vec![vec![], vec![1, 2, 3]], + ), + ( + "non-empty chunk followed by empty chunk", + vec![vec![7, 8, 9], vec![]], + ), + ( + "max number of chunks all empty", + vec![vec![]; MAX_AGG_SNARKS], + ), + ( + "max number of chunks all non-empty", + (0..MAX_AGG_SNARKS) + .map(|i| (10u8..11 + u8::try_from(i).unwrap()).collect()) + .collect(), + ), + ("single chunk blob full", vec![vec![123; n_rows_data]]), + ( + "multiple chunks blob full", + vec![vec![123; 1111], vec![231; n_rows_data - 1111]], + ), + ( + "max number of chunks only last one non-empty not full blob", + repeat(vec![]) + .take(MAX_AGG_SNARKS - 1) + .chain(once(vec![132; n_rows_data - 1111])) + .collect(), + ), + ( + "max number of chunks only last one non-empty full blob", + repeat(vec![]) + .take(MAX_AGG_SNARKS - 1) + .chain(once(vec![132; n_rows_data])) + .collect(), + ), + ( + "max number of chunks but last is empty", + repeat(vec![111; 100]) + .take(MAX_AGG_SNARKS - 1) + .chain(once(vec![])) + .collect(), + ), + ] + .iter() + { + let batch_header = crate::batch::BatchHeader { + version: 3, + batch_index: 6789, + l1_message_popped: 101, + total_l1_message_popped: 10101, + parent_batch_hash: H256::repeat_byte(1), + last_block_timestamp: 192837, + ..Default::default() + }; + let batch_data: BatchData = tcase.into(); + let batch_bytes = batch_data.get_batch_data_bytes(); + let blob_bytes = get_blob_bytes(&batch_bytes); + let coeffs = get_coefficients(&blob_bytes); + let versioned_hash = get_versioned_hash(&coeffs); + let chunks_without_padding = ChunkInfo::mock_chunk_infos(tcase); + let batch_hash = BatchHash::::construct_with_unpadded( + &chunks_without_padding, + batch_header, + &blob_bytes, + ); + let point_evaluation_assignments = + PointEvaluationAssignments::new(&batch_data, &blob_bytes, versioned_hash); + println!( + "[[ {:60} ]]\nchallenge (z) = {:0>64x}, evaluation (y) = {:0>64x}, versioned hash = {:0>64x}, batch_hash = {:0>64x}\n\n", + annotation, + point_evaluation_assignments.challenge, + point_evaluation_assignments.evaluation, + versioned_hash, + batch_hash.current_batch_hash, + ); + } + } + + #[test] + fn default_batch_data() { + let mut default_metadata = [0u8; BatchData::::n_rows_metadata()]; + default_metadata[1] = 1; + let default_metadata_digest = keccak256(default_metadata); + let default_chunk_digests = [keccak256([]); MAX_AGG_SNARKS]; + + let default_batch = BatchData::::default(); + let batch_bytes = default_batch.get_batch_data_bytes(); + let blob_bytes = get_blob_bytes(&batch_bytes); + let coeffs = get_coefficients(&blob_bytes); + let versioned_hash = get_versioned_hash(&coeffs); + let point_evaluation_assignments = + PointEvaluationAssignments::new(&default_batch, &blob_bytes, versioned_hash); + let versioned_hash = get_versioned_hash(&point_evaluation_assignments.coefficients); + assert_eq!( + default_batch.get_challenge_digest(versioned_hash), + U256::from(keccak256( + default_metadata_digest + .into_iter() + .chain(default_chunk_digests.into_iter().flatten()) + .chain(versioned_hash.to_fixed_bytes()) + .collect::>() + )), + ) + } +} diff --git a/aggregator/src/tests/blob.rs b/aggregator/src/blob_consistency/eip4844/tests.rs similarity index 94% rename from aggregator/src/tests/blob.rs rename to aggregator/src/blob_consistency/eip4844/tests.rs index b2d98c329f..891b492ec1 100644 --- a/aggregator/src/tests/blob.rs +++ b/aggregator/src/blob_consistency/eip4844/tests.rs @@ -18,12 +18,19 @@ use zkevm_circuits::{ use crate::{ aggregation::{ + batch_data::N_BLOB_BYTES, witgen::{process, MultiBlockProcessResult}, - AssignedBarycentricEvaluationConfig, BarycentricEvaluationConfig, BlobDataConfig, - RlcConfig, + BatchData, RlcConfig, }, - blob::{BatchData, PointEvaluationAssignments, N_BLOB_BYTES, N_BYTES_U256}, - eip4844::{decode_blob, get_blob_bytes, get_coefficients, get_versioned_hash}, + blob_consistency::{ + eip4844::{ + blob::PointEvaluationAssignments, get_blob_bytes, get_coefficients, get_versioned_hash, + AssignedBarycentricEvaluationConfig, BarycentricEvaluationConfig, + }, + BlobDataConfig, + }, + constants::N_BYTES_U256, + decode_bytes, param::ConfigParams, BatchDataConfig, ChunkInfo, MAX_AGG_SNARKS, }; @@ -145,11 +152,9 @@ impl Circuit for BlobCircuit { let point_eval = PointEvaluationAssignments::new(&self.data, &blob_bytes, versioned_hash); - Ok(config.barycentric.assign( - &mut ctx, - &point_eval.coefficients, - point_eval.challenge_digest, - )) + Ok(config + .barycentric + .assign(&mut ctx, &blob_bytes, point_eval.challenge_digest)) }, )?; @@ -176,13 +181,9 @@ impl Circuit for BlobCircuit { config.batch_data_config.load_range_tables(&mut layouter)?; - config.blob_data.assign( - &mut layouter, - challenge_values, - &config.rlc, - &blob_bytes, - &barycentric_assignments.barycentric_assignments, - )?; + config + .blob_data + .assign(&mut layouter, challenge_values, &config.rlc, &blob_bytes)?; layouter.assign_region( || "BatchDataConfig", @@ -623,7 +624,23 @@ fn test_decode_blob() { // case 2: yes encode assert_eq!( - decode_blob(&conditional_encode(batch_bytes.as_slice(), true)).expect("should decode"), + decode_bytes(&conditional_encode(batch_bytes.as_slice(), true)).expect("should decode"), batch_bytes, ); } + +use super::*; + +#[test] +fn test_conversions() { + let scalar = Scalar::one(); + let word = U256::one(); + let mut digest = H256::zero(); + digest.0[31] = 1; + + assert_eq!(digest_from_word(word), digest); + assert_eq!(digest_from_scalar(scalar), digest); + assert_eq!(scalar_from_word(word), scalar); + assert_eq!(scalar_from_digest(digest), scalar); + assert_eq!(word_from_digest(digest), word); +} diff --git a/aggregator/src/constants.rs b/aggregator/src/constants.rs index f7cdc73692..7d5d633172 100644 --- a/aggregator/src/constants.rs +++ b/aggregator/src/constants.rs @@ -85,3 +85,6 @@ pub(crate) const BITS: usize = 88; /// If the input size is less than this, dummy snarks /// will be padded. pub const MAX_AGG_SNARKS: usize = 45; + +// Number of bytes in a u256. +pub const N_BYTES_U256: usize = 32; diff --git a/aggregator/src/eip4844.rs b/aggregator/src/eip4844.rs deleted file mode 100644 index 5f4a29a5dc..0000000000 --- a/aggregator/src/eip4844.rs +++ /dev/null @@ -1,89 +0,0 @@ -use eth_types::{ToBigEndian, H256, U256}; -use ethers_core::k256::sha2::{Digest, Sha256}; -use revm_primitives::VERSIONED_HASH_VERSION_KZG; - -use crate::blob::{BLOB_WIDTH, KZG_TRUSTED_SETUP, N_BLOB_BYTES, N_BYTES_U256}; - -/// Get the BLOB_WIDTH number of scalar field elements, as 32-bytes unsigned integers. -pub(crate) fn get_coefficients(blob_bytes: &[u8]) -> [U256; BLOB_WIDTH] { - let mut coefficients = [[0u8; N_BYTES_U256]; BLOB_WIDTH]; - - assert!( - blob_bytes.len() <= N_BLOB_BYTES, - "too many bytes in batch data" - ); - - for (i, &byte) in blob_bytes.iter().enumerate() { - coefficients[i / 31][1 + (i % 31)] = byte; - } - - coefficients.map(|coeff| U256::from_big_endian(&coeff)) -} - -/// Get the versioned hash as per EIP-4844. -pub(crate) fn get_versioned_hash(coefficients: &[U256; BLOB_WIDTH]) -> H256 { - let blob = c_kzg::Blob::from_bytes( - &coefficients - .iter() - .cloned() - .flat_map(|coeff| coeff.to_be_bytes()) - .collect::>(), - ) - .expect("blob-coefficients to 4844 blob should succeed"); - let c = c_kzg::KzgCommitment::blob_to_kzg_commitment(&blob, &KZG_TRUSTED_SETUP) - .expect("blob to kzg commitment should succeed"); - kzg_to_versioned_hash(&c) -} - -fn kzg_to_versioned_hash(commitment: &c_kzg::KzgCommitment) -> H256 { - let mut res = Sha256::digest(commitment.as_slice()); - res[0] = VERSIONED_HASH_VERSION_KZG; - H256::from_slice(&res[..]) -} - -/// Get the blob data bytes that will be populated in BlobDataConfig. -pub fn get_blob_bytes(batch_bytes: &[u8]) -> Vec { - let mut blob_bytes = crate::witgen::zstd_encode(batch_bytes); - - // Whether we encode batch -> blob or not. - let enable_encoding = blob_bytes.len() < batch_bytes.len(); - if !enable_encoding { - blob_bytes = batch_bytes.to_vec(); - } - blob_bytes.insert(0, enable_encoding as u8); - - blob_bytes -} - -/// Given the blob's bytes, take into account the first byte, i.e. enable_encoding? and either spit -/// out the raw bytes or zstd decode them. -pub fn decode_blob(blob_bytes: &[u8]) -> std::io::Result> { - let enable_encoding = blob_bytes[0].eq(&1); - - // If not encoded, spit out the rest of the bytes, as it is. - if !enable_encoding { - return Ok(blob_bytes[1..].to_vec()); - } - - // The bytes following the first byte represent the zstd-encoded bytes. - let mut encoded_bytes = blob_bytes[1..].to_vec(); - let mut encoded_len = encoded_bytes.len(); - let mut decoded_bytes = Vec::with_capacity(5 * 4096 * 32); - loop { - let mut decoder = zstd_encoder::zstd::stream::read::Decoder::new(encoded_bytes.as_slice())?; - decoder.include_magicbytes(false)?; - decoder.window_log_max(30)?; - - decoded_bytes.clear(); - - if std::io::copy(&mut decoder, &mut decoded_bytes).is_ok() { - break; - } - - // The error above means we need to truncate the suffix 0-byte. - encoded_len -= 1; - encoded_bytes.truncate(encoded_len); - } - - Ok(decoded_bytes) -} diff --git a/aggregator/src/lib.rs b/aggregator/src/lib.rs index 18f8e67017..684a33c973 100644 --- a/aggregator/src/lib.rs +++ b/aggregator/src/lib.rs @@ -3,12 +3,12 @@ mod aggregation; /// This module implements `Batch` related data types. /// A batch is a list of chunk. mod batch; -/// blob struct and constants -mod blob; /// Config to recursive aggregate multiple aggregations mod recursion; // This module implements `Chunk` related data types. // A chunk is a list of blocks. +/// Blob consistency checks +pub mod blob_consistency; mod chunk; /// proof compression mod compression; @@ -16,8 +16,6 @@ mod compression; mod constants; /// Core module for circuit assignment mod core; -/// EIP-4844 related utils. -pub mod eip4844; /// Parameters for compression circuit mod param; /// utilities @@ -29,7 +27,6 @@ mod tests; pub use self::core::extract_proof_and_instances_with_pairing_check; pub use aggregation::*; pub use batch::{BatchHash, BatchHeader}; -pub use blob::BatchData; pub use chunk::ChunkInfo; pub use compression::*; pub use constants::MAX_AGG_SNARKS; diff --git a/aggregator/src/tests.rs b/aggregator/src/tests.rs index deeb6f0ab0..8c98351b61 100644 --- a/aggregator/src/tests.rs +++ b/aggregator/src/tests.rs @@ -1,5 +1,4 @@ mod aggregation; -mod blob; mod compression; mod mock_chunk; mod recursion; diff --git a/aggregator/src/tests/aggregation.rs b/aggregator/src/tests/aggregation.rs index cfac56a830..ec374c420b 100644 --- a/aggregator/src/tests/aggregation.rs +++ b/aggregator/src/tests/aggregation.rs @@ -7,10 +7,10 @@ use snark_verifier::loader::halo2::halo2_ecc::halo2_base::utils::fs::gen_srs; use snark_verifier_sdk::{gen_pk, gen_snark_shplonk, verify_snark_shplonk, CircuitExt}; use crate::{ + aggregation::encode_bytes, aggregation::BatchCircuit, batch::{BatchHash, BatchHeader}, constants::MAX_AGG_SNARKS, - eip4844::get_blob_bytes, layer_0, tests::mock_chunk::MockChunkCircuit, BatchData, ChunkInfo, @@ -180,7 +180,7 @@ fn build_new_batch_circuit( .concat(); let batch_data = BatchData::::new(num_real_chunks, &chunks_with_padding); let batch_bytes = batch_data.get_batch_data_bytes(); - let blob_bytes = get_blob_bytes(&batch_bytes); + let blob_bytes = encode_bytes(&batch_bytes); let batch_header = BatchHeader::construct_from_chunks( batch_proving_task.batch_header.version, batch_proving_task.batch_header.batch_index, @@ -261,7 +261,7 @@ fn build_batch_circuit_skip_encoding() -> BatchCircuit::new(num_chunks, &chunks_with_padding); let batch_bytes = batch_data.get_batch_data_bytes(); - let blob_bytes = get_blob_bytes(&batch_bytes); + let blob_bytes = encode_bytes(&batch_bytes); let corrected_batch_header = BatchHeader::construct_from_chunks( batch_proving_task.batch_header.version, batch_proving_task.batch_header.batch_index, diff --git a/prover/src/aggregator.rs b/prover/src/aggregator.rs index 0a5a63f9e3..177c313c1a 100644 --- a/prover/src/aggregator.rs +++ b/prover/src/aggregator.rs @@ -2,5 +2,5 @@ mod prover; mod verifier; pub use self::prover::{check_chunk_hashes, Prover}; -pub use aggregator::{eip4844, BatchData, BatchHash, BatchHeader, MAX_AGG_SNARKS}; +pub use aggregator::{BatchData, BatchHash, BatchHeader, MAX_AGG_SNARKS}; pub use verifier::Verifier; diff --git a/prover/src/aggregator/prover.rs b/prover/src/aggregator/prover.rs index fcfe4831ee..e4b42295df 100644 --- a/prover/src/aggregator/prover.rs +++ b/prover/src/aggregator/prover.rs @@ -1,8 +1,6 @@ use std::{collections::BTreeMap, env, iter::repeat}; -use aggregator::{ - eip4844::decode_blob, BatchData, BatchHash, BatchHeader, ChunkInfo, MAX_AGG_SNARKS, -}; +use aggregator::{decode_bytes, BatchData, BatchHash, BatchHeader, ChunkInfo, MAX_AGG_SNARKS}; use anyhow::{bail, Result}; use eth_types::H256; use halo2_proofs::{halo2curves::bn256::Bn256, poly::kzg::commitment::ParamsKZG}; @@ -198,17 +196,17 @@ impl<'params> Prover<'params> { assert_eq!( batch_header.blob_data_proof[0], batch.batch_header.blob_data_proof[0], "BatchHeader(sanity) mismatch blob data proof (z) expected={}, got={}", - batch.batch_header.blob_data_proof[0], batch_header.blob_data_proof[0], + batch_header.blob_data_proof[0], batch.batch_header.blob_data_proof[0], ); assert_eq!( batch_header.blob_data_proof[1], batch.batch_header.blob_data_proof[1], "BatchHeader(sanity) mismatch blob data proof (y) expected={}, got={}", - batch.batch_header.blob_data_proof[1], batch_header.blob_data_proof[1], + batch_header.blob_data_proof[1], batch.batch_header.blob_data_proof[1], ); assert_eq!( batch_header.blob_versioned_hash, batch.batch_header.blob_versioned_hash, "BatchHeader(sanity) mismatch blob versioned hash expected={}, got={}", - batch.batch_header.blob_versioned_hash, batch_header.blob_versioned_hash, + batch_header.blob_versioned_hash, batch.batch_header.blob_versioned_hash, ); let batch_hash = batch_header.batch_hash(); @@ -219,7 +217,7 @@ impl<'params> Prover<'params> { // sanity check: // - conditionally decoded blob should match batch data. let batch_bytes = batch_data.get_batch_data_bytes(); - let decoded_blob_bytes = decode_blob(&batch.blob_bytes)?; + let decoded_blob_bytes = decode_bytes(&batch.blob_bytes)?; assert_eq!( batch_bytes, decoded_blob_bytes, "BatchProvingTask(sanity) mismatch batch bytes and decoded blob bytes",