[WIP] try isolating tests and allowing for unique package-specific tests

eng/common/scripts/logging.ps1

@@ -50,8 +50,7 @@ function LogSuccess {
   Write-Host "${green}$args${reset}"
 }
 
-function LogErrorForFile($file, $errorString)
-{
+function LogErrorForFile($file, $errorString) {
   if (Test-SupportsDevOpsLogging) {
     Write-Host ("##vso[task.logissue type=error;sourcepath=$file;linenumber=1;columnnumber=1;]$errorString" -replace "`n", "%0D%0A")
   }

eng/scripts/Language-Settings.ps1

@@ -63,6 +63,10 @@ function Get-AllPackageInfoFromRepo ([string] $ServiceDirectory) {
 
       $package.DirectoryPath = Split-Path $package.manifest_path -Parent
       $package.DependentPackages = @()
+
+      # Collect the crate types available in this package
+      $package.CrateTypes = $package.targets | Select-Object -ExpandProperty crate_types | Select-Object -Unique
+
       $packageManifests[$package.name] = $package
     }
   }
@@ -101,6 +105,7 @@ function Get-AllPackageInfoFromRepo ([string] $ServiceDirectory) {
     $pkgProp = [PackageProps]::new($package.name, $package.version, $package.DirectoryPath, $package.ServiceDirectoryName)
     $pkgProp.IsNewSdk = $true
     $pkgProp.ArtifactName = $package.name
+    $pkgProp.TargetKinds = $package.targets | Select-Object -ExpandProperty kind | Select-Object -Unique
 
     if ($package.name -match "mgmt") {
       $pkgProp.SdkType = "mgmt"
@@ -120,7 +125,7 @@ function Get-AllPackageInfoFromRepo ([string] $ServiceDirectory) {
 function Get-rust-AdditionalValidationPackagesFromPackageSet ($packagesWithChanges, $diff, $allPackageProperties) {
   # if the change was in a service directory, but not in a package directory, test all the packages in the service directory
   [array]$serviceFiles = ($diff.ChangedFiles + $diff.DeletedFiles) | ForEach-Object { $_ -replace '\\', '/' } | Where-Object { $_ -match "^sdk/.+/" }
-  
+
   # remove files that target any specific package
   foreach ($package in $allPackageProperties) {
     $packagePathPattern = "^$( [Regex]::Escape($package.DirectoryPath.Replace('\', '/')) )/"

eng/scripts/Pack-Crates.ps1

@@ -17,11 +17,6 @@ $ErrorActionPreference = 'Stop'
 
 . ([System.IO.Path]::Combine($PSScriptRoot, '..', 'common', 'scripts', 'common.ps1'))
 
-Write-Host @"
-Packing crates with
-    RUSTFLAGS: '${env:RUSTFLAGS}'
-"@
-
 function Get-OutputPackageNames($workspacePackages) {
   $packablePackages = $workspacePackages | Where-Object -Property publish -NE -Value @()
   $packablePackageNames = $packablePackages.name
@@ -73,6 +68,19 @@ function Get-CargoPackages() {
   return $metadata.packages
 }
 
+function Test-PackageCanBePublished($package) {
+  if ($null -ne $package.publish -and $package.publish.Count -eq 0) {
+    return $false
+  }
+
+  foreach ($target in $package.targets) {
+    if ($target.kind -contains 'lib' -or $target.kind -contains 'bin' -or $target.kind -contains 'proc-macro') {
+      return $true
+    }
+  }
+  return $false
+}
+
 function Get-PackagesToBuild() {
   $packages = Get-CargoPackages
   $outputPackageNames = Get-OutputPackageNames $packages
@@ -84,6 +92,10 @@ function Get-PackagesToBuild() {
 
   [array]$packagesToBuild = $packages | Where-Object { $outputPackageNames.Contains($_.name) }
 
+  # Only pack crates with a lib, bin, or proc-macro target, as these are the only crates crates.io accepts.
+  # This means skipping any crate that ONLY contains tests, benchmarks, examples, or native code (cdylib/staticlib)
+  $packagesToBuild = $packagesToBuild | Where-Object { Test-PackageCanBePublished $_ }
+
   if ($Release) {
     return $packagesToBuild
   }
@@ -96,12 +108,14 @@ function Get-PackagesToBuild() {
 
     foreach ($dependency in $package.UnreleasedDependencies) {
       if (!$packagesToBuild.Contains($dependency) -and !$toProcess.Contains($dependency)) {
+        if (-not (Test-PackageCanBePublished $dependency)) {
+          Write-Error "Package '$($dependency.name)' is a dependency of '$($package.name)' but cannot be published to crates.io"
+        }
         $packagesToBuild += $dependency
         $toProcess += $dependency
       }
     }
   }
-
   return $packagesToBuild
 }
 
@@ -124,11 +138,20 @@ try {
   Set-Location $RepoRoot
 
   [array]$packages = Get-PackagesToBuild
+
+  Write-Host "Packing crates:"
   $packageParams = @()
   foreach ($package in $packages) {
     $packageParams += "--package", $package.name
+    Write-Host "    '$($package.name)' version '$($package.version)'"
   }
 
+  Write-Host @"
+with
+    RUSTFLAGS: '${env:RUSTFLAGS}'
+
+"@
+
   if ($NoVerify) {
     $packageParams += "--no-verify"
   }

eng/scripts/Test-Package.ps1

@@ -0,0 +1,63 @@
+#!/usr/bin/env pwsh
+
+#Requires -Version 7.0
+param(
+  [string]$PackageName,
+  [string]$DirectoryPath,
+  [string[]]$CrateTypes
+)
+
+$ErrorActionPreference = 'Stop'
+Set-StrictMode -Version 2.0
+. "$PSScriptRoot/../common/scripts/common.ps1"
+
+$DirectoryPath = ([System.IO.Path]::Combine($RepoRoot, $DirectoryPath))
+
+Write-Host "Testing package: '$PackageName' in directory: '$DirectoryPath'"
+
+Set-Location $DirectoryPath
+
+$env:THEENV = "INNER"
+Write-Host "Current THEENV: '$env:THEENV"
+
+$setupScript = Join-Path $DirectoryPath "Test-Setup.ps1"
+if (Test-Path $setupScript) {
+  Write-Host "`n`nRunning test setup script for package: '$PackageName'`n"
+  Invoke-LoggedCommand $setupScript -GroupOutput
+  if (!$? -ne 0) {
+    Write-Error "Test setup script failed for package: '$PackageName'"
+    exit 1
+  }
+}
+
+Write-Host "`n`nTesting package: '$PackageName'`n"
+
+Invoke-LoggedCommand "cargo build --keep-going" -GroupOutput
+Write-Host "`n`n"
+
+# Doc tests are only applicable for library crates.
+if ($CrateTypes -contains 'lib') {
+  Invoke-LoggedCommand "cargo test --doc --no-fail-fast" -GroupOutput
+  Write-Host "`n`n"
+}
+
+Invoke-LoggedCommand "cargo test --all-targets --no-fail-fast" -GroupOutput
+Write-Host "`n`n"
+
+# If the package has an additional test script, run it now.
+$additionalTestScript = Join-Path $DirectoryPath "Test-Additional.ps1"
+if (Test-Path $additionalTestScript) {
+  Write-Host "`n`nRunning additional test script for package: '$PackageName'`n"
+  Invoke-LoggedCommand $additionalTestScript -GroupOutput
+  if (!$? -ne 0) {
+    Write-Error "Additional test script failed for package: '$PackageName'"
+    exit 1
+  }
+}
+
+$cleanupScript = Join-Path $DirectoryPath "Test-Cleanup.ps1"
+if (Test-Path $cleanupScript) {
+  Write-Host "`n`nRunning test cleanup script for package: '$PackageName'`n"
+  Invoke-LoggedCommand $cleanupScript -GroupOutput
+  # We ignore the exit code of the cleanup script.
+}

eng/scripts/Test-Packages.ps1

@@ -2,7 +2,8 @@
 
 #Requires -Version 7.0
 param(
-  [string]$PackageInfoDirectory
+  [string]$PackageInfoDirectory,
+  [string[]]$PackageNames
 )
 
 $ErrorActionPreference = 'Stop'
@@ -34,45 +35,29 @@ else {
   $packagesToTest = Get-AllPackageInfoFromRepo
 }
 
+if ($PackageNames -and $PackageNames.Length -gt 0) {
+  $packagesToTest = $packagesToTest | Where-Object { $PackageNames -contains $_.Name }
+}
+
 Write-Host "Testing packages:"
 foreach ($package in $packagesToTest) {
   Write-Host "  '$($package.Name)' in '$($package.DirectoryPath)'"
 }
 
 foreach ($package in $packagesToTest) {
-  Push-Location ([System.IO.Path]::Combine($RepoRoot, $package.DirectoryPath))
-  try {
-    $packageDirectory = ([System.IO.Path]::Combine($RepoRoot, $package.DirectoryPath))
-
-    $setupScript = Join-Path $packageDirectory "Test-Setup.ps1"
-    if (Test-Path $setupScript) {
-      Write-Host "`n`nRunning test setup script for package: '$($package.Name)'`n"
-      Invoke-LoggedCommand $setupScript -GroupOutput
-      if (!$? -ne 0) {
-        Write-Error "Test setup script failed for package: '$($package.Name)'"
-        exit 1
-      }
-    }
-
-    Write-Host "`n`nTesting package: '$($package.Name)'`n"
-
-    Invoke-LoggedCommand "cargo build --keep-going" -GroupOutput
-    Write-Host "`n`n"
-
-    Invoke-LoggedCommand "cargo test --doc --no-fail-fast" -GroupOutput
-    Write-Host "`n`n"
-
-    Invoke-LoggedCommand "cargo test --all-targets --no-fail-fast" -GroupOutput
-    Write-Host "`n`n"
-
-    $cleanupScript = Join-Path $packageDirectory "Test-Cleanup.ps1"
-    if (Test-Path $cleanupScript) {
-      Write-Host "`n`nRunning test cleanup script for package: '$($package.Name)'`n"
-      Invoke-LoggedCommand $cleanupScript -GroupOutput
-      # We ignore the exit code of the cleanup script.
-    }
-  }
-  finally {
-    Pop-Location
+  Write-Host "Testing package '$($package.Name)', with crate types $($package.CrateTypes -join ', ') ..."
+
+  # Launch a child process to test the package to isolate environment changes.
+  # NOTE: This means we can only pass simple parameters (strings, arrays) to the child process.
+  $Command = @(
+    Join-Path $PSScriptRoot 'Test-Package.ps1'
+    '-PackageName', $package.Name
+    '-DirectoryPath', $package.DirectoryPath
+    '-CrateTypes', ($package.CrateTypes -join ',')
+  )
+  Start-Process -FilePath pwsh -ArgumentList $Command -NoNewWindow -Wait
+  if ($LASTEXITCODE -ne 0) {
+    Write-Error "Testing package '$($package.Name)' failed."
+    exit $LASTEXITCODE
   }
 }