Add integration tests for dotnet test

global.json

@@ -1,12 +1,12 @@
 {
   "sdk": {
-    "version": "5.0.100",
+    "version": "6.0.100-alpha.1.21067.8",
     "rollForward": "minor",
     "allowPrerelease": false,
     "architecture": "x64"
   },
   "tools": {
-    "dotnet": "5.0.100"
+    "dotnet": "6.0.100-alpha.1.21067.8"
   },
   "msbuild-sdks": {
     "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.20621.2",

scripts/build.ps1

@@ -88,7 +88,7 @@ $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE = 1
 # Dotnet build doesn't support --packages yet. See https://github.com./dotnet/cli/issues/2712
 $env:NUGET_PACKAGES = $env:TP_PACKAGES_DIR
 $env:NUGET_EXE_Version = "3.4.3"
-$env:DOTNET_CLI_VERSION = "5.0.100"
+$env:DOTNET_CLI_VERSION = "6.0.100-alpha.1.21067.8"
 # $env:DOTNET_RUNTIME_VERSION = "LATEST"
 $env:VSWHERE_VERSION = "2.0.2"
 $env:MSBUILD_VERSION = "15.0"
@@ -196,6 +196,12 @@ function Install-DotNetCli
     & $dotnetInstallScript -InstallDir "${dotnetInstallPath}_x86" -Runtime 'dotnet' -Version '3.1.0' -Channel '3.1.0' -Architecture x86 -NoPath
     ${env:DOTNET_ROOT(x86)} = "${dotnetInstallPath}_x86"
 
+    & $dotnetInstallScript -InstallDir "$dotnetInstallPath" -Runtime 'dotnet' -Version '5.0.1' -Channel '5.0.1' -Architecture x64 -NoPath
+    $env:DOTNET_ROOT= $dotnetInstallPath
+
+    & $dotnetInstallScript -InstallDir "${dotnetInstallPath}_x86" -Runtime 'dotnet' -Version '5.0.1' -Channel '5.0.1' -Architecture x86 -NoPath
+    ${env:DOTNET_ROOT(x86)} = "${dotnetInstallPath}_x86"
+
     $env:DOTNET_MULTILEVEL_LOOKUP=0
 
     "---- dotnet environment variables"

scripts/build.sh

@@ -111,7 +111,7 @@ VERSION=$(test -z $VERSION && grep TPVersionPrefix $TP_ROOT_DIR/scripts/build/Te
 export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
 # Dotnet build doesnt support --packages yet. See https://github.com./dotnet/cli/issues/2712
 export NUGET_PACKAGES=$TP_PACKAGES_DIR
-DOTNET_CLI_VERSION="5.0.100"
+DOTNET_CLI_VERSION="6.0.100-alpha.1.21067.8"
 #DOTNET_RUNTIME_VERSION="LATEST"
 
 #
@@ -187,6 +187,7 @@ function install_cli()
         # Get netcoreapp1.1 shared components
         $install_script  --runtime dotnet --version "2.1.0" --channel "release/2.1.0" --install-dir "$TP_DOTNET_DIR" --no-path --architecture x64
         $install_script  --runtime dotnet --version "3.1.0" --channel "release/3.1.0" --install-dir "$TP_DOTNET_DIR" --no-path --architecture x64
+        $install_script  --runtime dotnet --version "5.0.1" --channel "release/5.0.1" --install-dir "$TP_DOTNET_DIR" --no-path --architecture x64
 
         log "install_cli: Get the latest dotnet cli toolset..."
         $install_script --install-dir "$TP_DOTNET_DIR" --no-path --channel "master" --version $DOTNET_CLI_VERSION

test/Microsoft.TestPlatform.AcceptanceTests/DotnetTestTests.cs

@@ -0,0 +1,79 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace Microsoft.TestPlatform.AcceptanceTests
+{
+    using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+    [TestClass]
+    public class DotnetTestTests : AcceptanceTestBase
+    {
+        [TestMethod]
+        // patched dotnet is not published on non-windows systems
+        [TestCategory("Windows-Review")]
+        [NetCoreTargetFrameworkDataSource]
+        public void RunDotnetTestWithCsproj(RunnerInfo runnerInfo)
+        {
+            AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo);
+
+            var projectName = "SimpleTestProject.csproj";
+            var projectPath = this.GetProjectFullPath(projectName);
+
+            this.InvokeDotnetTest($@"{projectPath} --logger:""Console;Verbosity=normal""");
+
+            // ensure our dev version is used
+            this.StdOutputContains("-dev");
+            this.ValidateSummaryStatus(1, 1, 1);
+            this.ExitCodeEquals(1);
+        }
+
+
+        [TestMethod]
+        // patched dotnet is not published on non-windows systems
+        [TestCategory("Windows-Review")]
+        [NetCoreTargetFrameworkDataSource]
+        public void RunDotnetTestWithDll(RunnerInfo runnerInfo)
+        {
+            AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo);
+
+            var assemblyPath = this.BuildMultipleAssemblyPath("SimpleTestProject.dll").Trim('\"');
+            this.InvokeDotnetTest($@"{assemblyPath} --logger:""Console;Verbosity=normal""");
+
+            // ensure our dev version is used
+            this.StdOutputContains("-dev");
+            this.ValidateSummaryStatus(1, 1, 1);
+            this.ExitCodeEquals(1);
+        }
+
+        [TestMethod]
+        // patched dotnet is not published on non-windows systems
+        [TestCategory("Windows-Review")]
+        [NetCoreTargetFrameworkDataSource]
+        public void PassInlineSettings(RunnerInfo runnerInfo)
+        {
+            AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo);
+
+            var projectName = "ParametrizedTestProject.csproj";
+            var projectPath = this.GetProjectFullPath(projectName);
+            this.InvokeDotnetTest($@"{projectPath} --logger:""Console;Verbosity=normal"" -- TestRunParameters.Parameter(name =\""weburl\"", value=\""http://localhost//def\"")");
+            this.ValidateSummaryStatus(1, 0, 0);
+            this.ExitCodeEquals(0);
+        }
+
+        [TestMethod]
+        // patched dotnet is not published on non-windows systems
+        [TestCategory("Windows-Review")]
+        [NetCoreTargetFrameworkDataSource]
+        public void PassInlineSettingsToDll(RunnerInfo runnerInfo)
+        {
+
+            AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo);
+
+            var assemblyPath = this.BuildMultipleAssemblyPath("ParametrizedTestProject.dll").Trim('\"');
+            this.InvokeDotnetTest($@"{assemblyPath} --logger:""Console;Verbosity=normal"" -- TestRunParameters.Parameter(name=\""weburl\"", value=\""http://localhost//def\"")");
+
+            this.ValidateSummaryStatus(1, 0, 0);
+            this.ExitCodeEquals(0);
+        }
+    }
+}

test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs

@@ -129,8 +129,25 @@ public void InvokeVsTest(string arguments)
         /// <param name="arguments">Arguments provided to <c>vstest.console</c>.exe</param>
         public void InvokeDotnetTest(string arguments)
         {
-            this.ExecutePatchedDotnet("test", arguments, out this.standardTestOutput, out this.standardTestError, out this.runnerExitCode);
-            this.FormatStandardOutCome();
+            var vstestConsolePath = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "artifacts", IntegrationTestEnvironment.BuildConfiguration, "netcoreapp2.1", "vstest.console.dll");
+            var env = "VSTEST_CONSOLE_PATH";
+            var originalVstestConsolePath = Environment.GetEnvironmentVariable(env);
+
+            try
+            {
+                Environment.SetEnvironmentVariable(env, vstestConsolePath);
+                if (arguments.Contains(".csproj"))
+                {                   
+                    arguments = $@"-p:VsTestConsolePath=""{vstestConsolePath}"" " + arguments;
+                }
+
+                this.ExecutePatchedDotnet("test", arguments, out this.standardTestOutput, out this.standardTestError, out this.runnerExitCode);
+                this.FormatStandardOutCome();
+            }
+            finally
+            {
+                Environment.SetEnvironmentVariable(env, originalVstestConsolePath);
+            }
         }
 
         /// <summary>

test/TestAssets/ParametrizedTestProject/UnitTest1.cs

@@ -0,0 +1,24 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace ParametrizedTestProject
+{
+    /// <summary>
+    /// The unit test 1.
+    /// </summary>
+    [TestClass]
+    public class UnitTest1
+    {
+        public TestContext TestContext { get; set; }
+
+        /// <summary>
+        /// The passing test.
+        /// </summary>
+        [TestMethod]
+        public void CheckingParameters()
+        {
+            Assert.AreEqual("http://localhost//def", TestContext.Properties["weburl"]);
+        }
+    }
+}

test/TestAssets/TestAssets.sln

@@ -88,6 +88,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "timeout", "timeout\timeout.
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "problematic-child", "problematic-child\problematic-child.csproj", "{5FB9DFB8-2453-40EA-AAAD-D44677BCDF50}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ParametrizedTestProject", "ParametrizedTestProject\ParametrizedTestProject.csproj", "{CA82D2B3-CCF7-4F68-8840-286BA2D5B23B}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -578,6 +580,18 @@ Global
 		{5FB9DFB8-2453-40EA-AAAD-D44677BCDF50}.Release|x64.Build.0 = Release|Any CPU
 		{5FB9DFB8-2453-40EA-AAAD-D44677BCDF50}.Release|x86.ActiveCfg = Release|Any CPU
 		{5FB9DFB8-2453-40EA-AAAD-D44677BCDF50}.Release|x86.Build.0 = Release|Any CPU
+		{CA82D2B3-CCF7-4F68-8840-286BA2D5B23B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{CA82D2B3-CCF7-4F68-8840-286BA2D5B23B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{CA82D2B3-CCF7-4F68-8840-286BA2D5B23B}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{CA82D2B3-CCF7-4F68-8840-286BA2D5B23B}.Debug|x64.Build.0 = Debug|Any CPU
+		{CA82D2B3-CCF7-4F68-8840-286BA2D5B23B}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{CA82D2B3-CCF7-4F68-8840-286BA2D5B23B}.Debug|x86.Build.0 = Debug|Any CPU
+		{CA82D2B3-CCF7-4F68-8840-286BA2D5B23B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{CA82D2B3-CCF7-4F68-8840-286BA2D5B23B}.Release|Any CPU.Build.0 = Release|Any CPU
+		{CA82D2B3-CCF7-4F68-8840-286BA2D5B23B}.Release|x64.ActiveCfg = Release|Any CPU
+		{CA82D2B3-CCF7-4F68-8840-286BA2D5B23B}.Release|x64.Build.0 = Release|Any CPU
+		{CA82D2B3-CCF7-4F68-8840-286BA2D5B23B}.Release|x86.ActiveCfg = Release|Any CPU
+		{CA82D2B3-CCF7-4F68-8840-286BA2D5B23B}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE