Run acceptance tests against the locally built sources (#2340)

* Use current version of TP in acceptance tests

Author: nohwnd

.gitignore

@@ -31,6 +31,7 @@ msbuild*.err
 build*.log
 build*.wrn
 build*.err
+*.binlog
 
 # MSTest test Results
 [Tt]est[Rr]esult*/

TestPlatform.sln

@@ -17,7 +17,7 @@
   <ItemGroup>
     <PackageReference Include="MSTest.TestFramework" Version="$(MSTestFrameworkVersion)" />
     <PackageReference Include="MSTest.TestAdapter" Version="$(MSTestAdapterVersion)" />
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="$(NETTestSdkPreviousVersion)" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="$(NETTestSdkVersion)" />
   </ItemGroup>
   <ItemGroup Condition=" '$(TargetFramework)' == 'net451' ">
     <Reference Include="System.Runtime" />

samples/UnitTestProject/UnitTestProject.csproj

@@ -62,15 +62,15 @@ $env:TP_OUT_DIR = Join-Path $env:TP_ROOT_DIR "artifacts"
 $env:TP_TESTARTIFACTS = Join-Path $env:TP_OUT_DIR "testArtifacts"
 $env:TP_PACKAGE_PROJ_DIR = Join-Path $env:TP_ROOT_DIR "src\package"
 
-# Set Version from scripts/build/TestPlatform.Settings.targets
+# Set Version from scripts/build/TestPlatform.Settings.targets, when we are running locally and not providing the version as the parameter 
+# or when the build is done directly in VS
 if([string]::IsNullOrWhiteSpace($Version))
 {
-    $Version = ([xml](Get-Content $env:TP_ROOT_DIR\scripts\build\TestPlatform.Settings.targets)).Project.PropertyGroup.TPVersionPrefix | 
-        Where-Object { $_ } | 
+    $Version = ([xml](Get-Content $env:TP_ROOT_DIR\scripts\build\TestPlatform.Settings.targets)).Project.PropertyGroup[0].TPVersionPrefix | 
         ForEach-Object { $_.Trim() } |
         Select-Object -First 1 
 
-    Write-Verbose "Version was not provided using version '$Version' from TestPlatform.Settings.targets"
+    Write-Verbose "Version was not provided using version '$Version' from TestPlatform.Settings.targets"    
 }
 
 #
@@ -102,7 +102,7 @@ $TPB_TargetRuntime = $TargetRuntime
 $TPB_X64_Runtime = "win7-x64"
 $TPB_X86_Runtime = "win7-x86"
 
-# Version suffix is empty for RTM releases
+# Version suffix is empty for RTM release
 $TPB_Version = if ($VersionSuffix -ne '') { $Version + "-" + $VersionSuffix } else { $Version }
 $TPB_CIBuild = $CIBuild
 $TPB_PublishTests = $PublishTestArtifacts
@@ -116,6 +116,14 @@ $Script:ScriptFailed = $false
 
 Import-Module "$($CurrentScriptDir.FullName)\verify-nupkgs.ps1"
 
+# Update the version in the dependencies props to be the TPB_version version, this is not ideal but because changing how this is resolved would 
+# mean that we need to change the whole build process this is a solution with the least amount of impact, that does not require us to keep track of 
+# the version in multiple places
+$dependenciesPath = "$env:TP_ROOT_DIR\scripts\build\TestPlatform.Dependencies.props"
+$dependencies = Get-Content -Raw -Encoding UTF8 $dependenciesPath
+$updatedDependencies = $dependencies -replace "<NETTestSdkVersion>.*?</NETTestSdkVersion>", "<NETTestSdkVersion>$TPB_Version</NETTestSdkVersion>"
+$updatedDependencies | Set-Content -Encoding UTF8 $dependenciesPath
+
 function Write-Log ([string] $message)
 {
     $currentColor = $Host.UI.RawUI.ForegroundColor
@@ -163,22 +171,42 @@ function Install-DotNetCli
     Write-Log "Install-DotNetCli: Get the latest dotnet cli toolset..."
     $dotnetInstallPath = Join-Path $env:TP_TOOLS_DIR "dotnet"
     New-Item -ItemType directory -Path $dotnetInstallPath -Force | Out-Null
-    & $dotnetInstallScript -Channel "master" -InstallDir $dotnetInstallPath -NoPath -Version $env:DOTNET_CLI_VERSION
+    & $dotnetInstallScript -Channel "master" -InstallDir $dotnetInstallPath -Version $env:DOTNET_CLI_VERSION
 
     # Pull in additional shared frameworks.
     # Get netcoreapp2.1 shared components.
-    if (!(Test-Path "$dotnetInstallPath\shared\Microsoft.NETCore.App\2.1.0")) {
-        & $dotnetInstallScript -InstallDir $dotnetInstallPath -SharedRuntime -Version '2.1.0' -Channel 'release/2.1.0'
-    }
+    
+    & $dotnetInstallScript -InstallDir "$dotnetInstallPath" -Runtime 'dotnet' -Version '2.1.0' -Channel 'release/2.1.0' -Architecture x64 -NoPath
+    $env:DOTNET_ROOT= $dotnetInstallPath
+
+    & $dotnetInstallScript -InstallDir "${dotnetInstallPath}_x86" -Runtime 'dotnet' -Version '2.1.0' -Channel 'release/2.1.0' -Architecture x86 -NoPath
+    ${env:DOTNET_ROOT(x86)} = "${dotnetInstallPath}_x86"
+    
+    $env:DOTNET_MULTILEVEL_LOOKUP=0
+
+    "---- dotnet environment variables"
+    Get-ChildItem "Env:\dotnet_*"
+    
+    "`n`n---- x64 dotnet"
+    & "$env:DOTNET_ROOT\dotnet.exe" --info
+
+    "`n`n---- x86 dotnet"
+    & "${env:DOTNET_ROOT(x86)}\dotnet.exe" --info
 
-    # Get shared components which is compatible with dotnet cli version $env:DOTNET_CLI_VERSION
-    #if (!(Test-Path "$dotnetInstallPath\shared\Microsoft.NETCore.App\$env:DOTNET_RUNTIME_VERSION")) {
-        #& $dotnetInstallScript -InstallDir $dotnetInstallPath -SharedRuntime -Version $env:DOTNET_RUNTIME_VERSION -Channel 'master'
-    #}
 
     Write-Log "Install-DotNetCli: Complete. {$(Get-ElapsedTime($timer))}"
 }
 
+function Clear-Package {
+    # find all microsoft packages that have the same version as we specified
+    # this is cache-busting the nuget packages, so we don't reuse them from cache 
+    # after we built new ones
+    if (Test-Path $env:TP_PACKAGES_DIR) {
+        $devPackages = Get-ChildItem $env:TP_PACKAGES_DIR/microsoft.*/$TPB_Version | Select-Object -ExpandProperty FullName 
+        $devPackages | Remove-Item -Force -Recurse -Confirm:$false
+    }
+}
+
 function Restore-Package
 {
     $timer = Start-Timer
@@ -202,26 +230,51 @@ function Invoke-Build
 
     Write-Log ".. .. Build: Source: $TPB_Solution"
     Write-Verbose "$dotnetExe build $TPB_Solution --configuration $TPB_Configuration -v:minimal -p:Version=$TPB_Version -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild"
-    & $dotnetExe build $TPB_Solution --configuration $TPB_Configuration -v:minimal -p:Version=$TPB_Version -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild
+    & $dotnetExe build $TPB_Solution --configuration $TPB_Configuration -v:minimal -p:Version=$TPB_Version -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -bl:TestPlatform.binlog
     Write-Log ".. .. Build: Complete."
 
     Set-ScriptFailedOnError
 
+    Write-Log "Invoke-Build: Complete. {$(Get-ElapsedTime($timer))}"
+}
+
+function Invoke-TestAssetsBuild
+{
+    $timer = Start-Timer
+    Write-Log "Invoke-TestAssetsBuild: Start test assets build."
+    $dotnetExe = Get-DotNetPath
+
+    
     Write-Log ".. .. Build: Source: $TPB_TestAssets_Solution"
     Write-Verbose "$dotnetExe build $TPB_TestAssets_Solution --configuration $TPB_Configuration -v:minimal -p:Version=$TPB_Version -p:CIBuild=$TPB_CIBuild"
-    & $dotnetExe build $TPB_TestAssets_Solution --configuration $TPB_Configuration -v:minimal -p:Version=$TPB_Version -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild
+    & $dotnetExe build $TPB_TestAssets_Solution --configuration $TPB_Configuration -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -bl:"$($env:TP_ROOT_DIR)\TestAssets.binlog"
     Write-Log ".. .. Build: Complete."
 
     Set-ScriptFailedOnError
 
-    Write-Log "Invoke-Build: Complete. {$(Get-ElapsedTime($timer))}"
+    Write-Log "Invoke-TestAssetsBuild: Complete. {$(Get-ElapsedTime($timer))}"
+}
+
+function Copy-PackageIntoStaticDirectory { 
+    # packages are published into folder based on configuration, but 
+    # nuget does not understand that, and does not support wildcards in paths 
+    # in order to be able to use the produced packages for acceptance tests we
+    # need to put them in folder that is not changing it's name based on config
+    $tpPackagesPath = "$env:TP_OUT_DIR\$TPB_Configuration\packages\"
+    $tpPackagesDestination = "$env:TP_TESTARTIFACTS"
+    Copy-Item $tpPackagesPath $tpPackagesDestination -Force -Filter *.nupkg -Verbose -Recurse
 }
 
 function Publish-PatchedDotnet { 
     Write-Log "Publish-PatchedDotnet: Copy local dotnet installation to testArtifacts"
     $dotnetPath = "$env:TP_TOOLS_DIR\dotnet\"
-    
+
     $dotnetTestArtifactsPath = "$env:TP_TESTARTIFACTS\dotnet\" 
+    
+    if (Test-Path $dotnetTestArtifactsPath) { 
+        Remove-Item -Force -Recurse $dotnetTestArtifactsPath
+    }
+
     $dotnetTestArtifactsSdkPath = "$env:TP_TESTARTIFACTS\dotnet\sdk\$env:DOTNET_CLI_VERSION\"
     Copy-Item $dotnetPath $dotnetTestArtifactsPath -Force -Recurse
 
@@ -922,15 +975,19 @@ Get-ChildItem env: | Where-Object -FilterScript { $_.Name.StartsWith("TP_") } |
 Write-Log "Test platform build variables: "
 Get-Variable | Where-Object -FilterScript { $_.Name.StartsWith("TPB_") } | Format-Table
 Install-DotNetCli
+Clear-Package
 Restore-Package
 Update-LocalizedResources
 Invoke-Build
 Publish-Package
-Publish-PatchedDotnet
 Publish-Tests
 Create-VsixPackage
 Create-NugetPackages
 Generate-Manifest
+Publish-PatchedDotnet
+Copy-PackageIntoStaticDirectory
+Invoke-TestAssetsBuild
+ 
 Write-Log "Build complete. {$(Get-ElapsedTime($timer))}"
 if ($Script:ScriptFailed) { Exit 1 } else { Exit 0 }
 

scripts/build.ps1

@@ -111,7 +111,7 @@ VERSION=$(test -z $VERSION && grep TPVersionPrefix $TP_ROOT_DIR/scripts/build/Te
 export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
 # Dotnet build doesnt support --packages yet. See https://github.com./dotnet/cli/issues/2712
 export NUGET_PACKAGES=$TP_PACKAGES_DIR
-DOTNET_CLI_VERSION="3.1.100-preview2-014569"
+DOTNET_CLI_VERSION="3.1.100"
 #DOTNET_RUNTIME_VERSION="LATEST"
 
 #

scripts/build.sh

@@ -1,13 +1,20 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 <Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <PropertyGroup>
     <VSSdkBuildToolsVersion>15.8.3247</VSSdkBuildToolsVersion>
 
     <!-- Name of the elements must be in sync with test\Microsoft.TestPlatform.TestUtilities\IntegrationTestBase.cs -->
-    <NETTestSdkPreviousVersion>15.5.0</NETTestSdkPreviousVersion>
 
-    <MSTestFrameworkVersion>1.4.0</MSTestFrameworkVersion>
-    <MSTestAdapterVersion>1.4.0</MSTestAdapterVersion>
+    <!-- this version needs to be the same as in TestPlatform.Settings.targets, we might want to unify this into 
+    a single .props file, but the other version is used by multiple build scripts, so I just query it and validate that  
+    they are both the same during build -->
+    <!-- this version also needs to be "statically" readable because the test fixture will inspect this file for the version 
+    and because during the test `dotnet test` will run and re-build some of the test projects and at that time the version 
+    from a build parameter would not be available, so I am writing this version from the build.ps1 script to keep it in sync --> 
+    <NETTestSdkVersion>16.6.0-dev</NETTestSdkVersion>
+
+    <MSTestFrameworkVersion>2.1.0</MSTestFrameworkVersion>
+    <MSTestAdapterVersion>2.1.0</MSTestAdapterVersion>
     <MSTestAssertExtensionVersion>1.0.3-preview</MSTestAssertExtensionVersion>
 
     <XUnitFrameworkVersion>2.3.1</XUnitFrameworkVersion>

scripts/build/TestPlatform.Dependencies.props

@@ -2,7 +2,10 @@
 <Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <PropertyGroup>
     <TestPlatformRoot Condition="$(TestPlatformRoot) == ''">$(MSBuildThisFileDirectory)../../</TestPlatformRoot>
-    <TPVersionPrefix>16.5.0</TPVersionPrefix>
+    <!-- This version is read by vsts-prebuild.ps1 and is a base for the current version, this should be updated
+    at the start of new iteration to the goal number. This is also used to version the local packages. This version needs to be statically
+    readable when we read the file as xml, don't move it to a .props file, unless you change the build server process -->
+    <TPVersionPrefix>16.6.0</TPVersionPrefix>
   </PropertyGroup>
   <PropertyGroup>
     <!-- Versioning is defined from the build script. Use a default dev build if it's not defined.

scripts/build/TestPlatform.Settings.targets

@@ -59,15 +59,32 @@ $env:TP_ROOT_DIR = (Get-Item (Split-Path $MyInvocation.MyCommand.Path)).Parent.F
 $env:TP_TOOLS_DIR = Join-Path $env:TP_ROOT_DIR "tools"
 $env:TP_PACKAGES_DIR = Join-Path $env:TP_ROOT_DIR "packages"
 $env:TP_OUT_DIR = Join-Path $env:TP_ROOT_DIR "artifacts"
+
+Write-Verbose "Setup dotnet configuration."
 # Add latest dotnet.exe directory to environment variable PATH to tests run on latest dotnet.
 $env:PATH = "$(Split-Path $(Get-DotNetPath));$env:PATH"
 
-#
-# Dotnet configuration
-#
+# set the roots to use only the versions of dotnet that we installed
+$env:DOTNET_ROOT = Join-Path $env:TP_TOOLS_DIR "dotnet"
+# set the root for x86 runtimes as well
+${env:DOTNET_ROOT(x86)} = "${env:DOTNET_ROOT}_x86"
+# disable looking up other dotnets in programfiles
+$env:DOTNET_MULTILEVEL_LOOKUP = 0
+
 # Disable first run since we want to control all package sources 
-Write-Verbose "Setup dotnet configuration."
 $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE = 1 
+
+# list what we have set and what is available
+"---- dotnet environment variables"
+Get-ChildItem "Env:\dotnet_*"
+
+"`n`n---- x64 dotnet"
+& "$env:DOTNET_ROOT\dotnet.exe" --info
+
+"`n`n---- x86 dotnet"
+& "${env:DOTNET_ROOT(x86)}\dotnet.exe" --info
+    
+
 # Dotnet build doesn't support --packages yet. See https://github.com./dotnet/cli/issues/2712
 $env:NUGET_PACKAGES = $env:TP_PACKAGES_DIR
 

scripts/test.ps1

@@ -52,12 +52,33 @@ public static IDictionary<string, string> GetArgumentsDictionary(string[] args)
         /// <exception cref="ArgumentException">Thrown if value of an argument is not an integer.</exception>
         public static int GetIntArgFromDict(IDictionary<string, string> argsDictionary, string fullname)
         {
-            string optionValue;
-            return argsDictionary.TryGetValue(fullname, out optionValue) ? int.Parse(optionValue) : 0;
+            var found = TryGetIntArgFromDict(argsDictionary, fullname, out var value);
+            return found ? value : 0;
         }
 
         /// <summary>
-        /// Parse the value of an argument as an integer.
+        /// Try get the argument and parse the value of an argument as an integer.
+        /// </summary>
+        /// <param name="argsDictionary">Dictionary of all arguments Ex: <c>{ "--port":"12312", "--parentprocessid":"2312" }</c></param>
+        /// <param name="fullname">The full name for required argument. Ex: "--port"</param>
+        /// <returns>Value of the argument.</returns>
+        /// <exception cref="ArgumentException">Thrown if value of an argument is not an integer.</exception>
+        public static bool TryGetIntArgFromDict(IDictionary<string, string> argsDictionary, string fullname, out int value)
+        {
+            var found = argsDictionary.TryGetValue(fullname, out var optionValue);
+            if (!found)
+            {
+                value = default;
+                return false; 
+            }
+
+            value = int.Parse(optionValue);
+            return true;
+        }
+
+
+        /// <summary>
+        /// Parse the value of an argument as a string.
         /// </summary>
         /// <param name="argsDictionary">Dictionary of all arguments Ex: <c>{ "--port":"12312", "--parentprocessid":"2312" }</c></param>
         /// <param name="fullname">The full name for required argument. Ex: "--port"</param>

src/Microsoft.TestPlatform.CoreUtilities/Helpers/CommandLineArgumentsHelper.cs

@@ -94,7 +94,16 @@ private void PopulateCacheForTypeAndMethodSymbols(string binaryPath)
                     // At this point, the assembly should be already loaded into the load context. We query for a reference to
                     // find the types and cache the symbol information. Let the loader follow default lookup order instead of
                     // forcing load from a specific path.
-                    var asm = Assembly.Load(new PlatformAssemblyLoadContext().GetAssemblyNameFromPath(binaryPath));
+                    Assembly asm;
+                    try
+                    {
+                        asm = Assembly.Load(new PlatformAssemblyLoadContext().GetAssemblyNameFromPath(binaryPath));
+                    }
+                    catch (FileNotFoundException)
+                    {
+                        // fallback when the assembly is not loaded
+                        asm = Assembly.LoadFile(binaryPath);
+                    }
 
                     foreach (var type in asm.GetTypes())
                     {

src/Microsoft.TestPlatform.ObjectModel/Navigation/PortableSymbolReader.cs

@@ -109,7 +109,7 @@ public void StartShouldStartVanguardProcessWithCollectCommand()
             this.vanguard.Start(this.outputFileName, this.dataCollectionContext);
             cts.Cancel();
 
-            var numOfProcessCreated = numOfProcessCreatedTask.Result;
+            var numOfProcessCreated = numOfProcessCreatedTask.Result.Count;
 
             // TODO find the reason why additional process launched when collecting code coverage.
             Assert.IsTrue(numOfProcessCreated == 1 || numOfProcessCreated == 2, $"Number of process created:{numOfProcessCreated} expected is 1 or 2.");
@@ -172,7 +172,7 @@ public void StopShouldLaunchVarguardWithShutdownCommand()
             this.vanguard.Stop();
             cts.Cancel();
 
-            var numOfProcessCreated = numOfProcessCreatedTask.Result;
+            var numOfProcessCreated = numOfProcessCreatedTask.Result.Count;
 
             // TODO find the reason why additional process launched when collecting code coverage.
             Assert.IsTrue(numOfProcessCreated == 2 || numOfProcessCreated == 4, $"Number of process created:{numOfProcessCreated} expected is 2 or 4.");