Skip to content

Commit

Permalink
Implement observer with SQL query support and parquet export
Browse files Browse the repository at this point in the history
Implement an F3 observer that:
* listens to F3 gossipsub messages
* collects them as long as they can be decoded as CBOR
* offers a SQL interface to query them over a HTTP
* rotates them into Parquet files
* deletes any rotated files based on a configured maximum age

Fix base image to distroless/cc because of duckdb libstdc++
dependencies.
  • Loading branch information
masih committed Nov 18, 2024
1 parent b1869c1 commit d9a5ac4
Show file tree
Hide file tree
Showing 18 changed files with 1,102 additions and 603 deletions.
1 change: 0 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,3 @@ coverage.txt
/f3
# f3 manifest
/manifest.json
/observer
4 changes: 2 additions & 2 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@ COPY go.mod go.sum ./
RUN go mod download

COPY . .
RUN CGO_ENABLED=0 go build -o /go/bin/f3 ./cmd/f3
RUN go build -o /go/bin/f3 ./cmd/f3

FROM gcr.io/distroless/static-debian11
FROM gcr.io/distroless/cc
COPY --from=build /go/bin/f3 /usr/bin/

ENTRYPOINT ["/usr/bin/f3"]
3 changes: 2 additions & 1 deletion cmd/f3/manifest.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"os"
"time"

"github.com/filecoin-project/go-f3/internal/psutil"
"github.com/filecoin-project/go-f3/manifest"
"github.com/libp2p/go-libp2p"
pubsub "github.com/libp2p/go-libp2p-pubsub"
Expand Down Expand Up @@ -217,7 +218,7 @@ var manifestServeCmd = cli.Command{
pubSub, err := pubsub.NewGossipSub(c.Context, host,
pubsub.WithPeerExchange(true),
pubsub.WithFloodPublish(true),
pubsub.WithPeerScore(PubsubPeerScoreParams, PubsubPeerScoreThresholds),
pubsub.WithPeerScore(psutil.PubsubPeerScoreParams, psutil.PubsubPeerScoreThresholds),
)
if err != nil {
return fmt.Errorf("initialzing pubsub: %w", err)
Expand Down
75 changes: 0 additions & 75 deletions cmd/f3/msgdump/envelope.go

This file was deleted.

91 changes: 0 additions & 91 deletions cmd/f3/msgdump/parquet.go

This file was deleted.

27 changes: 0 additions & 27 deletions cmd/f3/msgdump/parquet_test.go

This file was deleted.

14 changes: 0 additions & 14 deletions cmd/f3/msgdump/writer.go

This file was deleted.

Loading

0 comments on commit d9a5ac4

Please sign in to comment.