Merge latest master into release (#842)

* fix: createReservation lock (#825)

* fix: createReservation lock

* fix: additional locking places

* fix: acquire lock

* chore: feedback

Co-authored-by: markspanbroek <mark@spanbroek.net>
Signed-off-by: Adam Uhlíř <adam@uhlir.dev>

* feat: withLock template and fixed tests

* fix: use proc for MockReservations constructor

* chore: feedback

Co-authored-by: Eric <5089238+emizzle@users.noreply.github.com>
Signed-off-by: Adam Uhlíř <adam@uhlir.dev>

* chore: feedback implementation

---------

Signed-off-by: Adam Uhlíř <adam@uhlir.dev>
Co-authored-by: markspanbroek <mark@spanbroek.net>
Co-authored-by: Eric <5089238+emizzle@users.noreply.github.com>

* Block deletion with ref count & repostore refactor (#631)

* Fix StoreStream so it doesn't return parity bytes  (#838)

* fix storestream so it doesn\'t return parity bits for protected/verifiable manifests

* use Cid.example instead of creating a mock manually

* Fix verifiable manifest initialization (#839)

* fix verifiable manifest initialization

* fix linearstrategy, use verifiableStrategy to select blocks for slots

* check for both strategies in attribute inheritance test

* ci: add verify_circuit=true to the releases (#840)

* provisional fix so EC errors do not crash the node on download (#841)

---------

Signed-off-by: Adam Uhlíř <adam@uhlir.dev>
Co-authored-by: Adam Uhlíř <adam@uhlir.dev>
Co-authored-by: markspanbroek <mark@spanbroek.net>
Co-authored-by: Eric <5089238+emizzle@users.noreply.github.com>
Co-authored-by: Tomasz Bekas <tomasz.bekas@gmail.com>
Co-authored-by: Giuliano Mega <giuliano.mega@gmail.com>

.github/workflows/release.yml

@@ -11,6 +11,7 @@ env:
   nim_version: v1.6.14
   rust_version: 1.78.0
   binary_base: codex
+  nim_flags: '-d:verify_circuit=true'
   upload_to_codex: false
 
 jobs:
@@ -74,7 +75,7 @@ jobs:
 
       - name: Release - Build
         run: |
-          make NIMFLAGS="--out:${{ env.binary }}"
+          make NIMFLAGS="--out:${{ env.binary }} ${{ env.nim_flags }}"
           
       - name: Release - Upload binaries
         uses: actions/upload-artifact@v4

codex/codex.nim

@@ -258,7 +258,7 @@ proc new*(
       repoDs = repoData,
       metaDs = LevelDbDatastore.new(config.dataDir / CodexMetaNamespace)
         .expect("Should create metadata store!"),
-      quotaMaxBytes = config.storageQuota.uint,
+      quotaMaxBytes = config.storageQuota,
       blockTtl = config.blockTtl)
 
     maintenance = BlockMaintainer.new(

codex/indexingstrategy.nim

@@ -49,8 +49,8 @@ func getLinearIndicies(
   self.checkIteration(iteration)
 
   let
-    first = self.firstIndex + iteration * (self.step + 1)
-    last = min(first + self.step, self.lastIndex)
+    first = self.firstIndex + iteration * self.step
+    last = min(first + self.step - 1, self.lastIndex)
 
   getIter(first, last, 1)
 
@@ -94,4 +94,4 @@ func init*(
     firstIndex: firstIndex,
     lastIndex: lastIndex,
     iterations: iterations,
-    step: divUp((lastIndex - firstIndex), iterations))
+    step: divUp((lastIndex - firstIndex + 1), iterations))

codex/manifest/manifest.nim

@@ -135,13 +135,6 @@ func isManifest*(mc: MultiCodec): ?!bool =
 # Various sizes and verification
 ############################################################
 
-func bytes*(self: Manifest, pad = true): NBytes =
-  ## Compute how many bytes corresponding StoreStream(Manifest, pad) will return
-  if pad or self.protected:
-    self.blocksCount.NBytes * self.blockSize
-  else:
-    self.datasetSize
-
 func rounded*(self: Manifest): int =
   ## Number of data blocks in *protected* manifest including padding at the end
   roundUp(self.originalBlocksCount, self.ecK)
@@ -238,7 +231,7 @@ func new*(
   treeCid: Cid,
   datasetSize: NBytes,
   ecK, ecM: int,
-  strategy: StrategyType): Manifest =
+  strategy = SteppedStrategy): Manifest =
   ## Create an erasure protected dataset from an
   ## unprotected one
   ##
@@ -284,7 +277,7 @@ func new*(
   ecM: int,
   originalTreeCid: Cid,
   originalDatasetSize: NBytes,
-  strategy: StrategyType): Manifest =
+  strategy = SteppedStrategy): Manifest =
 
   Manifest(
     treeCid: treeCid,
@@ -306,7 +299,7 @@ func new*(
   verifyRoot: Cid,
   slotRoots: openArray[Cid],
   cellSize = DefaultCellSize,
-  strategy = SteppedStrategy): ?!Manifest =
+  strategy = LinearStrategy): ?!Manifest =
   ## Create a verifiable dataset from an
   ## protected one
   ##
@@ -331,6 +324,7 @@ func new*(
     ecM: manifest.ecM,
     originalTreeCid: manifest.treeCid,
     originalDatasetSize: manifest.originalDatasetSize,
+    protectedStrategy: manifest.protectedStrategy,
     verifiable: true,
     verifyRoot: verifyRoot,
     slotRoots: @slotRoots,

codex/merkletree/codex/coders.nim

@@ -14,6 +14,8 @@ push: {.upraises: [].}
 import pkg/libp2p
 import pkg/questionable
 import pkg/questionable/results
+import pkg/stew/byteutils
+import pkg/serde/json
 
 import ../../units
 import ../../errors
@@ -100,3 +102,18 @@ proc decode*(_: type CodexProof, data: seq[byte]): ?!CodexProof =
       nodes.add node
 
   CodexProof.init(mcodec, index.int, nleaves.int, nodes)
+
+proc fromJson*(
+  _: type CodexProof,
+  json: JsonNode
+): ?!CodexProof =
+  expectJsonKind(Cid, JString, json)
+  var bytes: seq[byte]
+  try:
+    bytes = hexToSeqByte(json.str)
+  except ValueError as err:
+    return failure(err)
+
+  CodexProof.decode(bytes)
+
+func `%`*(proof: CodexProof): JsonNode = % byteutils.toHex(proof.encode())

codex/node.nim

@@ -240,14 +240,14 @@ proc streamEntireDataset(
   self: CodexNodeRef,
   manifest: Manifest,
   manifestCid: Cid,
-): ?!LPStream =
+): Future[?!LPStream] {.async.} =
   ## Streams the contents of the entire dataset described by the manifest.
   ##
   trace "Retrieving blocks from manifest", manifestCid
 
   if manifest.protected:
     # Retrieve, decode and save to the local store all EС groups
-    proc erasureJob(): Future[void] {.async.} =
+    proc erasureJob(): Future[?!void] {.async.} =
       try:
         # Spawn an erasure decoding job
         let
@@ -258,10 +258,16 @@ proc streamEntireDataset(
             self.taskpool)
         without _ =? (await erasure.decode(manifest)), error:
           trace "Unable to erasure decode manifest", manifestCid, exc = error.msg
-      except CatchableError as exc:
+      # --------------------------------------------------------------------------
+      # FIXME this is a HACK so that the node does not crash during the workshop.
+      #   We should NOT catch Defect.
+      except Exception as exc:
         trace "Exception decoding manifest", manifestCid, exc = exc.msg
+        return failure(exc.msg)
+      # --------------------------------------------------------------------------
 
-    asyncSpawn erasureJob()
+    if err =? (await erasureJob()).errorOption:
+      return failure(err)
 
   # Retrieve all blocks of the dataset sequentially from the local store or network
   trace "Creating store stream for manifest", manifestCid
@@ -283,7 +289,7 @@ proc retrieve*(
 
     return await self.streamSingleBlock(cid)
 
-  self.streamEntireDataset(manifest, cid)
+  await self.streamEntireDataset(manifest, cid)
 
 proc store*(
   self: CodexNodeRef,
@@ -534,7 +540,9 @@ proc onStore(
     trace "Unable to fetch manifest for cid", cid, err = err.msg
     return failure(err)
 
-  without builder =? Poseidon2Builder.new(self.networkStore, manifest), err:
+  without builder =? Poseidon2Builder.new(
+    self.networkStore, manifest, manifest.verifiableStrategy
+  ), err:
     trace "Unable to create slots builder", err = err.msg
     return failure(err)
 
@@ -559,8 +567,8 @@ proc onStore(
 
     return success()
 
-  without indexer =? manifest.protectedStrategy.init(
-    0, manifest.numSlotBlocks() - 1, manifest.numSlots).catch, err:
+  without indexer =? manifest.verifiableStrategy.init(
+    0, manifest.blocksCount - 1, manifest.numSlots).catch, err:
     trace "Unable to create indexing strategy from protected manifest", err = err.msg
     return failure(err)
 

codex/rest/json.nim

@@ -7,6 +7,7 @@ import ../sales
 import ../purchasing
 import ../utils/json
 import ../manifest
+import ../units
 
 export json
 
@@ -65,10 +66,10 @@ type
     id*: NodeId
 
   RestRepoStore* = object
-    totalBlocks* {.serialize.}: uint
-    quotaMaxBytes* {.serialize.}: uint
-    quotaUsedBytes* {.serialize.}: uint
-    quotaReservedBytes* {.serialize.}: uint
+    totalBlocks* {.serialize.}: Natural
+    quotaMaxBytes* {.serialize.}: NBytes
+    quotaUsedBytes* {.serialize.}: NBytes
+    quotaReservedBytes* {.serialize.}: NBytes
 
 proc init*(_: type RestContentList, content: seq[RestContent]): RestContentList =
   RestContentList(