Auto merge of #159628 - Mark-Simulacrum:ci-experiments, r=<try>

CI experiments


try-job: dist-x86_64-linux

Author: bors

Cargo.lock

@@ -871,7 +871,7 @@ version = "3.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34"
 dependencies = [
- "windows-sys 0.59.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -1284,7 +1284,7 @@ dependencies = [
  "libc",
  "option-ext",
  "redox_users",
- "windows-sys 0.59.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -1411,7 +1411,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
 dependencies = [
  "libc",
- "windows-sys 0.59.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -2068,6 +2068,7 @@ dependencies = [
  "tar",
  "walkdir",
  "xz2",
+ "zstd",
 ]
 
 [[package]]
@@ -2198,7 +2199,7 @@ dependencies = [
  "portable-atomic",
  "portable-atomic-util",
  "serde_core",
- "windows-sys 0.59.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -2685,7 +2686,7 @@ version = "0.50.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
 dependencies = [
- "windows-sys 0.59.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -2905,6 +2906,7 @@ dependencies = [
  "tar",
  "tempfile",
  "xz2",
+ "zstd",
 ]
 
 [[package]]
@@ -5113,7 +5115,7 @@ dependencies = [
  "errno",
  "libc",
  "linux-raw-sys",
- "windows-sys 0.59.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -5545,7 +5547,7 @@ dependencies = [
  "getrandom 0.3.3",
  "once_cell",
  "rustix",
- "windows-sys 0.59.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -5564,7 +5566,7 @@ version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d8c27177b12a6399ffc08b98f76f7c9a1f4fe9fc967c784c5a071fa8d93cf7e1"
 dependencies = [
- "windows-sys 0.59.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -6542,7 +6544,7 @@ version = "0.1.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
 dependencies = [
- "windows-sys 0.59.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -7144,3 +7146,31 @@ name = "zlib-rs"
 version = "0.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3be3d40e40a133f9c916ee3f9f4fa2d9d63435b5fbe1bfc6d9dae0aa0ada1513"
+
+[[package]]
+name = "zstd"
+version = "0.13.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a"
+dependencies = [
+ "zstd-safe",
+]
+
+[[package]]
+name = "zstd-safe"
+version = "7.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d"
+dependencies = [
+ "zstd-sys",
+]
+
+[[package]]
+name = "zstd-sys"
+version = "2.0.16+zstd.1.5.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748"
+dependencies = [
+ "cc",
+ "pkg-config",
+]

src/ci/citool/Cargo.lock

@@ -66,9 +66,9 @@ checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04"
 
 [[package]]
 name = "askama"
-version = "0.15.4"
+version = "0.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08e1676b346cadfec169374f949d7490fd80a24193d37d2afce0c047cf695e57"
+checksum = "f1bf825125edd887a019d0a3a837dcc5499a68b0d034cc3eb594070c3e18addc"
 dependencies = [
  "askama_macros",
  "itoa",
@@ -79,12 +79,13 @@ dependencies = [
 
 [[package]]
 name = "askama_derive"
-version = "0.15.4"
+version = "0.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7661ff56517787343f376f75db037426facd7c8d3049cef8911f1e75016f3a37"
+checksum = "e1c7065972a130eafa84215f21352ae15b4a7393da48c1f5e103904490736738"
 dependencies = [
  "askama_parser",
  "basic-toml",
+ "glob",
  "memchr",
  "proc-macro2",
  "quote",
@@ -96,18 +97,18 @@ dependencies = [
 
 [[package]]
 name = "askama_macros"
-version = "0.15.4"
+version = "0.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "713ee4dbfd1eb719c2dab859465b01fa1d21cb566684614a713a6b7a99a4e47b"
+checksum = "0e23b1d2c4bd39a41971f6124cef4cc6fd0540913ecb90919b69ab3bbe44ae1a"
 dependencies = [
  "askama_derive",
 ]
 
 [[package]]
 name = "askama_parser"
-version = "0.15.4"
+version = "0.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d62d674238a526418b30c0def480d5beadb9d8964e7f38d635b03bf639c704c"
+checksum = "7db09fde9143e7ac4513358fb32ee32847125b63b18ea715afd487956da715da"
 dependencies = [
  "rustc-hash",
  "serde",
@@ -380,6 +381,12 @@ dependencies = [
  "wasi",
 ]
 
+[[package]]
+name = "glob"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"
+
 [[package]]
 name = "glob-match"
 version = "0.2.1"
@@ -1105,9 +1112,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
 
 [[package]]
 name = "winnow"
-version = "0.7.6"
+version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "63d3fcd9bba44b03821e7d699eeee959f3126dcc4aa8e4ae18ec617c2a5cea10"
+checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81"
 dependencies = [
  "memchr",
 ]

src/ci/citool/src/main.rs

@@ -97,6 +97,7 @@ fn run_workflow_locally(db: JobDatabase, job_type: JobType, name: String) -> any
     let jobs = match job_type {
         JobType::Auto => &db.auto_jobs,
         JobType::PR => &db.pr_jobs,
+        JobType::Try => &db.try_jobs,
     };
     let job =
         jobs::find_linux_job(jobs, &name).with_context(|| format!("Cannot find job {name}"))?;
@@ -111,6 +112,9 @@ fn run_workflow_locally(db: JobDatabase, job_type: JobType, name: String) -> any
             custom_env.insert("DEPLOY".to_string(), "1".to_string());
         }
     }
+    if name == "dist-x86_64-linux-quick" {
+        custom_env.insert("DIST_TRY_BUILD".to_string(), "1".to_owned());
+    }
     custom_env.extend(job.env.iter().map(|(key, value)| {
         let value = match value {
             Value::Bool(value) => value.to_string(),
@@ -269,6 +273,8 @@ pub enum JobType {
     Auto,
     /// Pull request job
     PR,
+    /// Try jobs.
+    Try,
 }
 
 fn main() -> anyhow::Result<()> {

src/ci/github-actions/jobs.yml

@@ -62,13 +62,7 @@ runners:
   - &job-linux-36c-codebuild
     free_disk: true
     codebuild: true
-    os: codebuild-ubuntu-22-36c-$github.run_id-$github.run_attempt
-    <<: *base-job
-
-  - &job-linux-8c-codebuild
-    free_disk: true
-    codebuild: true
-    os: codebuild-ubuntu-22-8c-$github.run_id-$github.run_attempt
+    os: simulacrum-aws-test-runner-3a47918cf60f4d36fb1c526a04753e0e
     <<: *base-job
 
 envs:

src/ci/run.sh

@@ -78,11 +78,11 @@ RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-units-std=1"
 # generally implies single-threaded compression which results in wasting most
 # of our CPU resources.
 RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set dist.compression-profile=balanced"
+RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --dist-compression-formats=zstd"
 
-# Only produce xz tarballs on CI. gz tarballs will be generated by the release
-# process by recompressing the existing xz ones. This decreases the storage
-# space required for CI artifacts.
-RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --dist-compression-formats=xz"
+# Enable parallel rustc for CI compilation. We trust this enough to not
+# miscompile and it makes CI builds faster.
+RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.parallel-frontend-threads=6"
 
 # Enable the `c` feature for compiler_builtins, but only when the `compiler-rt` source is available
 # (to avoid spending a lot of time cloning llvm)

src/tools/opt-dist/Cargo.toml

@@ -15,6 +15,7 @@ fs_extra = "1"
 camino = "1"
 tar = "0.4.45"
 xz = { version = "0.1", package = "xz2" }
+zstd = "0.13.3"
 serde_json = "1"
 glob = "0.3"
 tempfile = "3.5"

src/tools/opt-dist/src/tests.rs

@@ -19,7 +19,7 @@ pub fn run_tests(env: &Environment) -> anyhow::Result<()> {
     std::fs::create_dir_all(&unpacked_dist_dir)?;
 
     let extract_dist_dir = |name: &str| -> anyhow::Result<Utf8PathBuf> {
-        unpack_archive(&dist_dir.join(format!("{name}.tar.xz")), &unpacked_dist_dir)?;
+        unpack_archive(&dist_dir.join(format!("{name}.tar.zstd")), &unpacked_dist_dir)?;
         let extracted_path = unpacked_dist_dir.join(name);
         assert!(extracted_path.is_dir());
         Ok(extracted_path)
@@ -37,7 +37,7 @@ pub fn run_tests(env: &Environment) -> anyhow::Result<()> {
     let extracted_src_dir = extract_dist_dir(&format!("rust-src-{version}"))?.join("rust-src");
 
     // If we have a Cranelift archive, copy it to the rustc sysroot
-    if let Ok(_) = find_file_in_dir(&dist_dir, "rustc-codegen-cranelift-", ".tar.xz") {
+    if let Ok(_) = find_file_in_dir(&dist_dir, "rustc-codegen-cranelift-", ".tar.zstd") {
         let extracted_codegen_dir =
             extract_dist_dir(&format!("rustc-codegen-cranelift-{version}-{host_triple}"))?
                 .join("rustc-codegen-cranelift-preview");
@@ -160,7 +160,7 @@ where
 /// Tries to find the version of the dist artifacts (either nightly, beta, or 1.XY.Z).
 fn find_dist_version(directory: &Utf8Path) -> anyhow::Result<String> {
     // Lookup a known file with a unique prefix and extract the version from its filename
-    let archive = find_file_in_dir(directory, "reproducible-artifacts-", ".tar.xz")?
+    let archive = find_file_in_dir(directory, "reproducible-artifacts-", ".tar.zstd")?
         .file_name()
         .unwrap()
         .to_string();

src/tools/opt-dist/src/training.rs

@@ -39,7 +39,7 @@ fn init_compiler_benchmarks(
         "--exact-match",
         crates.join(",").as_str(),
         "--jobs",
-        "4",
+        "16",
     ])
     .env("RUSTC", env.rustc_stage_0().as_str())
     .env("RUSTC_BOOTSTRAP", "1")

src/tools/opt-dist/src/utils/io.rs

@@ -47,9 +47,9 @@ pub fn delete_directory(path: &Utf8Path) -> anyhow::Result<()> {
 pub fn unpack_archive(path: &Utf8Path, dest_dir: &Utf8Path) -> anyhow::Result<()> {
     log::info!("Unpacking directory `{path}` into `{dest_dir}`");
 
-    assert!(path.as_str().ends_with(".tar.xz"));
+    assert!(path.as_str().ends_with(".tar.zstd"));
     let file = File::open(path.as_std_path())?;
-    let file = xz::read::XzDecoder::new(file);
+    let file = zstd::Decoder::new(file)?;
     let mut archive = tar::Archive::new(file);
     archive.unpack(dest_dir.as_std_path())?;
     Ok(())

src/tools/rust-installer/Cargo.toml

@@ -16,6 +16,7 @@ rayon = "1.0"
 tar = "0.4.45"
 walkdir = "2"
 xz2 = "0.1.4"
+zstd = { version = "0.13.3", features = ["zstdmt"] }
 
 [dependencies.clap]
 features = ["derive"]