Skip to content

Commit

Permalink
multithreaded zstd
Browse files Browse the repository at this point in the history
  • Loading branch information
michaelkirk committed Aug 10, 2024
1 parent 792283f commit b6b8b22
Showing 1 changed file with 5 additions and 5 deletions.
10 changes: 5 additions & 5 deletions Earthfile
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ save-gtfs:
ARG output_prefix=$(basename $transit_feeds .gtfs_feeds.csv)

COPY (+gtfs-build/gtfs --transit_feeds=${transit_feeds} --cache_key=${cache_key}) /gtfs
RUN tar --zstd -cf /gtfs.tar.zst -C /gtfs .
RUN tar --use-compress-program="zstd -T0" -cf /gtfs.tar.zst -C /gtfs .
# This isn't used at runtime, but it might be useful to archive the input
SAVE ARTIFACT /gtfs.tar.zst AS LOCAL ./data/${area}-${output_prefix}-${cache_key}.gtfs.tar.zst

Expand Down Expand Up @@ -135,7 +135,7 @@ save-otp:
--otp_build_config=${otp_build_config} \
) /graph.obj

RUN zstd /graph.obj
RUN zstd -T0 /graph.obj
SAVE ARTIFACT /graph.obj.zst AS LOCAL ./data/${output_name}-${cache_key}.graph.obj.zst

save-mbtiles:
Expand All @@ -148,23 +148,23 @@ save-valhalla:
FROM +save-base
ARG --required area
COPY (+valhalla-build/tiles --area=${area}) /valhalla
RUN tar --zstd -cf /valhalla.tar.zst -C /valhalla .
RUN tar --use-compress-program="zstd -T0" -cf /valhalla.tar.zst -C /valhalla .
SAVE ARTIFACT /valhalla.tar.zst AS LOCAL ./data/${area}.valhalla.tar.zst

save-elasticsearch:
FROM +save-base
ARG --required area
ARG countries
COPY (+pelias-import/elasticsearch --area=${area} --countries=${countries}) /elasticsearch
RUN tar --zstd -cf /elasticsearch.tar.zst -C /elasticsearch .
RUN tar --use-compress-program="zstd -T0" -cf /elasticsearch.tar.zst -C /elasticsearch .
SAVE ARTIFACT /elasticsearch.tar.zst AS LOCAL ./data/${area}.elasticsearch.tar.zst

save-placeholder:
FROM +save-base
ARG --required area
ARG countries
COPY (+pelias-prepare-placeholder/placeholder --countries=${countries}) /placeholder
RUN tar --zstd -cf /placeholder.tar.zst -C /placeholder .
RUN tar --use-compress-program="zstd -T0" -cf /placeholder.tar.zst -C /placeholder .
SAVE ARTIFACT /placeholder.tar.zst AS LOCAL ./data/${area}.placeholder.tar.zst

save-pelias-config:
Expand Down

0 comments on commit b6b8b22

Please sign in to comment.