Skip to content

Commit 41c4c14

Browse files
Mark-SimulacrumMark-Simulacrum
committed
zstd-compress archives
Archives are smaller, faster to compress, and faster to decompress. Installing zstd is pretty easy on most platforms as well.
1 parent 6bf2017 commit 41c4c14

12 files changed

+81
-52
lines changed

‎Cargo.lock

+29
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml

+1
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ docsrs-metadata = { git = "https://github.com/rust-lang/docs.rs/" }
1919
dotenv = "0.15"
2020
failure = "0.1.3"
2121
flate2 = "1"
22+
zstd = "0.13.0"
2223
http = "0.2"
2324
hyper = "0.14"
2425
lazy_static = "1.0"

‎src/report/archives.rs

+23-24
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ use crate::experiments::Experiment;
88
use crate::prelude::*;
99
use crate::report::{compare, Comparison, ReportWriter};
1010
use crate::results::{EncodedLog, EncodingType, ReadResults};
11-
use flate2::{write::GzEncoder, Compression};
1211
use indexmap::IndexMap;
1312
use tar::{Builder as TarBuilder, Header as TarHeader};
1413
use tempfile::tempfile;
@@ -156,7 +155,7 @@ fn write_all_archive<DB: ReadResults, W: ReportWriter>(
156155
// writes to S3 (requiring buffer management etc) while avoiding keeping the blob entirely
157156
// in memory.
158157
let backing = tempfile()?;
159-
let mut all = TarBuilder::new(GzEncoder::new(backing, Compression::default()));
158+
let mut all = TarBuilder::new(zstd::stream::Encoder::new(backing, 0)?);
160159
for entry in iterate(db, ex, crates, config) {
161160
let entry = entry?;
162161
let mut header = entry.header();
@@ -180,9 +179,9 @@ fn write_all_archive<DB: ReadResults, W: ReportWriter>(
180179
view = &buffer[..];
181180
}
182181
match dest.write_bytes(
183-
"logs-archives/all.tar.gz",
182+
"logs-archives/all.tar.zst",
184183
view,
185-
&"application/gzip".parse().unwrap(),
184+
&"application/zstd".parse().unwrap(),
186185
EncodingType::Plain,
187186
) {
188187
Ok(()) => break,
@@ -192,7 +191,7 @@ fn write_all_archive<DB: ReadResults, W: ReportWriter>(
192191
} else {
193192
std::thread::sleep(std::time::Duration::from_secs(2));
194193
warn!(
195-
"retry ({}/{}) writing logs-archives/all.tar.gz ({} bytes) (error: {:?})",
194+
"retry ({}/{}) writing logs-archives/all.tar.zst ({} bytes) (error: {:?})",
196195
i,
197196
RETRIES,
198197
view.len(),
@@ -206,7 +205,7 @@ fn write_all_archive<DB: ReadResults, W: ReportWriter>(
206205

207206
Ok(Archive {
208207
name: "All the crates".to_string(),
209-
path: "logs-archives/all.tar.gz".to_string(),
208+
path: "logs-archives/all.tar.zst".to_string(),
210209
})
211210
}
212211

@@ -229,22 +228,22 @@ pub fn write_logs_archives<DB: ReadResults, W: ReportWriter>(
229228

230229
by_comparison
231230
.entry(entry.comparison)
232-
.or_insert_with(|| TarBuilder::new(GzEncoder::new(Vec::new(), Compression::default())))
231+
.or_insert_with(|| TarBuilder::new(zstd::stream::Encoder::new(Vec::new(), 3).unwrap()))
233232
.append_data(&mut entry.header(), &entry.path, &entry.log_bytes[..])?;
234233
}
235234

236235
for (comparison, archive) in by_comparison.drain(..) {
237236
let data = archive.into_inner()?.finish()?;
238237
dest.write_bytes(
239-
format!("logs-archives/{comparison}.tar.gz"),
238+
format!("logs-archives/{comparison}.tar.zst"),
240239
&data,
241-
&"application/gzip".parse().unwrap(),
240+
&"application/zstd".parse().unwrap(),
242241
EncodingType::Plain,
243242
)?;
244243

245244
archives.push(Archive {
246245
name: format!("{comparison} crates"),
247-
path: format!("logs-archives/{comparison}.tar.gz"),
246+
path: format!("logs-archives/{comparison}.tar.zst"),
248247
});
249248
}
250249

@@ -261,11 +260,11 @@ mod tests {
261260
use crate::prelude::*;
262261
use crate::report::DummyWriter;
263262
use crate::results::{DatabaseDB, EncodingType, FailureReason, TestResult, WriteResults};
264-
use flate2::read::GzDecoder;
265263
use mime::Mime;
266264
use rustwide::logging::LogStorage;
267265
use std::io::Read;
268266
use tar::Archive;
267+
use zstd::stream::Decoder;
269268

270269
#[test]
271270
fn test_logs_archives_generation() {
@@ -355,20 +354,20 @@ mod tests {
355354
assert_eq!(
356355
&archives_paths,
357356
&[
358-
"logs-archives/all.tar.gz",
359-
"logs-archives/regressed.tar.gz",
360-
"logs-archives/test-pass.tar.gz",
357+
"logs-archives/all.tar.zst",
358+
"logs-archives/regressed.tar.zst",
359+
"logs-archives/test-pass.tar.zst",
361360
]
362361
);
363362

364363
// Load the content of all the archives
365-
let mime: Mime = "application/gzip".parse().unwrap();
366-
let all_content = writer.get("logs-archives/all.tar.gz", &mime);
367-
let mut all = Archive::new(GzDecoder::new(all_content.as_slice()));
368-
let regressed_content = writer.get("logs-archives/regressed.tar.gz", &mime);
369-
let mut regressed = Archive::new(GzDecoder::new(regressed_content.as_slice()));
370-
let test_pass_content = writer.get("logs-archives/test-pass.tar.gz", &mime);
371-
let mut test_pass = Archive::new(GzDecoder::new(test_pass_content.as_slice()));
364+
let mime: Mime = "application/zstd".parse().unwrap();
365+
let all_content = writer.get("logs-archives/all.tar.zst", &mime);
366+
let mut all = Archive::new(Decoder::new(all_content.as_slice()).unwrap());
367+
let regressed_content = writer.get("logs-archives/regressed.tar.zst", &mime);
368+
let mut regressed = Archive::new(Decoder::new(regressed_content.as_slice()).unwrap());
369+
let test_pass_content = writer.get("logs-archives/test-pass.tar.zst", &mime);
370+
let mut test_pass = Archive::new(Decoder::new(test_pass_content.as_slice()).unwrap());
372371

373372
macro_rules! check_content {
374373
($archive:ident: { $($file:expr => $match:expr,)* }) => {{
@@ -401,21 +400,21 @@ mod tests {
401400
}}
402401
}
403402

404-
// Check all.tar.gz
403+
// Check all.tar.zst
405404
check_content!(all: {
406405
format!("regressed/{}/{}.txt", crate1.id(), ex.toolchains[0]) => "tc1 crate1",
407406
format!("regressed/{}/{}.txt", crate1.id(), ex.toolchains[1]) => "tc2 crate1",
408407
format!("test-pass/{}/{}.txt", crate2.id(), ex.toolchains[0]) => "tc1 crate2",
409408
format!("test-pass/{}/{}.txt", crate2.id(), ex.toolchains[1]) => "tc2 crate2",
410409
});
411410

412-
// Check regressed.tar.gz
411+
// Check regressed.tar.zst
413412
check_content!(regressed: {
414413
format!("regressed/{}/{}.txt", crate1.id(), ex.toolchains[0]) => "tc1 crate1",
415414
format!("regressed/{}/{}.txt", crate1.id(), ex.toolchains[1]) => "tc2 crate1",
416415
});
417416

418-
// Check test-pass.tar.gz
417+
// Check test-pass.tar.zst
419418
check_content!(test_pass: {
420419
format!("test-pass/{}/{}.txt", crate2.id(), ex.toolchains[0]) => "tc1 crate2",
421420
format!("test-pass/{}/{}.txt", crate2.id(), ex.toolchains[1]) => "tc2 crate2",

‎templates/report/downloads.html

+1-1
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414

1515
<div class="category">
1616
<div class="header header-background toggle" data-toggle="#downloads-archives">
17-
Build logs (tar.gz)
17+
Build logs (tar.zst)
1818
</div>
1919
<div class="crates" id="downloads-archives">
2020
{% for archive in available_archives %}

‎tests/minicrater/blacklist/downloads.html.context.expected.json

+3-3
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,15 @@
22
"available_archives": [
33
{
44
"name": "All the crates",
5-
"path": "logs-archives/all.tar.gz"
5+
"path": "logs-archives/all.tar.zst"
66
},
77
{
88
"name": "test-pass crates",
9-
"path": "logs-archives/test-pass.tar.gz"
9+
"path": "logs-archives/test-pass.tar.zst"
1010
},
1111
{
1212
"name": "test-skipped crates",
13-
"path": "logs-archives/test-skipped.tar.gz"
13+
"path": "logs-archives/test-skipped.tar.zst"
1414
}
1515
],
1616
"crates_count": 3,

‎tests/minicrater/clippy/downloads.html.context.expected.json

+3-3
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,15 @@
22
"available_archives": [
33
{
44
"name": "All the crates",
5-
"path": "logs-archives/all.tar.gz"
5+
"path": "logs-archives/all.tar.zst"
66
},
77
{
88
"name": "test-pass crates",
9-
"path": "logs-archives/test-pass.tar.gz"
9+
"path": "logs-archives/test-pass.tar.zst"
1010
},
1111
{
1212
"name": "regressed crates",
13-
"path": "logs-archives/regressed.tar.gz"
13+
"path": "logs-archives/regressed.tar.zst"
1414
}
1515
],
1616
"crates_count": 2,

‎tests/minicrater/doc/downloads.html.context.expected.json

+3-3
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,15 @@
22
"available_archives": [
33
{
44
"name": "All the crates",
5-
"path": "logs-archives/all.tar.gz"
5+
"path": "logs-archives/all.tar.zst"
66
},
77
{
88
"name": "test-pass crates",
9-
"path": "logs-archives/test-pass.tar.gz"
9+
"path": "logs-archives/test-pass.tar.zst"
1010
},
1111
{
1212
"name": "build-fail crates",
13-
"path": "logs-archives/build-fail.tar.gz"
13+
"path": "logs-archives/build-fail.tar.zst"
1414
}
1515
],
1616
"crates_count": 2,

‎tests/minicrater/full/downloads.html.context.expected.json

+7-7
Original file line numberDiff line numberDiff line change
@@ -2,31 +2,31 @@
22
"available_archives": [
33
{
44
"name": "All the crates",
5-
"path": "logs-archives/all.tar.gz"
5+
"path": "logs-archives/all.tar.zst"
66
},
77
{
88
"name": "regressed crates",
9-
"path": "logs-archives/regressed.tar.gz"
9+
"path": "logs-archives/regressed.tar.zst"
1010
},
1111
{
1212
"name": "fixed crates",
13-
"path": "logs-archives/fixed.tar.gz"
13+
"path": "logs-archives/fixed.tar.zst"
1414
},
1515
{
1616
"name": "broken crates",
17-
"path": "logs-archives/broken.tar.gz"
17+
"path": "logs-archives/broken.tar.zst"
1818
},
1919
{
2020
"name": "build-fail crates",
21-
"path": "logs-archives/build-fail.tar.gz"
21+
"path": "logs-archives/build-fail.tar.zst"
2222
},
2323
{
2424
"name": "test-pass crates",
25-
"path": "logs-archives/test-pass.tar.gz"
25+
"path": "logs-archives/test-pass.tar.zst"
2626
},
2727
{
2828
"name": "test-fail crates",
29-
"path": "logs-archives/test-fail.tar.gz"
29+
"path": "logs-archives/test-fail.tar.zst"
3030
}
3131
],
3232
"crates_count": 17,

‎tests/minicrater/ignore-blacklist/downloads.html.context.expected.json

+3-3
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,15 @@
22
"available_archives": [
33
{
44
"name": "All the crates",
5-
"path": "logs-archives/all.tar.gz"
5+
"path": "logs-archives/all.tar.zst"
66
},
77
{
88
"name": "test-pass crates",
9-
"path": "logs-archives/test-pass.tar.gz"
9+
"path": "logs-archives/test-pass.tar.zst"
1010
},
1111
{
1212
"name": "test-fail crates",
13-
"path": "logs-archives/test-fail.tar.gz"
13+
"path": "logs-archives/test-fail.tar.zst"
1414
}
1515
],
1616
"crates_count": 3,

‎tests/minicrater/missing-repo/downloads.html.context.expected.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,11 @@
22
"available_archives": [
33
{
44
"name": "All the crates",
5-
"path": "logs-archives/all.tar.gz"
5+
"path": "logs-archives/all.tar.zst"
66
},
77
{
88
"name": "broken crates",
9-
"path": "logs-archives/broken.tar.gz"
9+
"path": "logs-archives/broken.tar.zst"
1010
}
1111
],
1212
"crates_count": 1,

‎tests/minicrater/resource-exhaustion/downloads.html.context.expected.json

+3-3
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,15 @@
22
"available_archives": [
33
{
44
"name": "All the crates",
5-
"path": "logs-archives/all.tar.gz"
5+
"path": "logs-archives/all.tar.zst"
66
},
77
{
88
"name": "test-pass crates",
9-
"path": "logs-archives/test-pass.tar.gz"
9+
"path": "logs-archives/test-pass.tar.zst"
1010
},
1111
{
1212
"name": "spurious-fixed crates",
13-
"path": "logs-archives/spurious-fixed.tar.gz"
13+
"path": "logs-archives/spurious-fixed.tar.zst"
1414
}
1515
],
1616
"crates_count": 2,

‎tests/minicrater/small/downloads.html.context.expected.json

+3-3
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,15 @@
22
"available_archives": [
33
{
44
"name": "All the crates",
5-
"path": "logs-archives/all.tar.gz"
5+
"path": "logs-archives/all.tar.zst"
66
},
77
{
88
"name": "regressed crates",
9-
"path": "logs-archives/regressed.tar.gz"
9+
"path": "logs-archives/regressed.tar.zst"
1010
},
1111
{
1212
"name": "test-pass crates",
13-
"path": "logs-archives/test-pass.tar.gz"
13+
"path": "logs-archives/test-pass.tar.zst"
1414
}
1515
],
1616
"crates_count": 2,

0 commit comments

Comments
 (0)