From 7d9232ba350df32d62ffa3c28d09f474b3accb68 Mon Sep 17 00:00:00 2001 From: Oski Kervinen Date: Mon, 30 Jan 2023 12:11:53 +0200 Subject: [PATCH 1/2] Make usable as a NuGet package * End assembly file name in .TestAdapter to make VS recognize it from a nuget package. * Create Catch2TestAdapter.nuspec to configure the nuget package. I tried embedding the nuget metadata in the project, but could not get it to support the exotic combination of having .NET assemblies that are meant to be installed in a native project. See https://github.com/microsoft/vstest/blob/main/docs/RFCs/0004-Adapter-Extensibility.md Command to generate the package: `nuget pack .\Catch2TestAdapter.csproj -Version 1.8.0 -IncludeReferencedProjects -p Configuration=Debug` --- Libraries/Catch2.TestAdapter.nuspec | 30 +++++++++++++++++++ .../Catch2TestAdapter.csproj | 2 +- 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 Libraries/Catch2.TestAdapter.nuspec diff --git a/Libraries/Catch2.TestAdapter.nuspec b/Libraries/Catch2.TestAdapter.nuspec new file mode 100644 index 0000000..c715bc6 --- /dev/null +++ b/Libraries/Catch2.TestAdapter.nuspec @@ -0,0 +1,30 @@ + + + + Catch2.TestAdapter + $version$ + Visual Studio Test Adapter for Catch2 + JohnnyHendriks + JohnnyHendriks + https://github.com/JohnnyHendriks/TestAdapter_Catch2 + false + MIT + + Class library containing the Visual Studio Test Adapter for Catch2 + + + + native catch2 test adapter + + + + + + + + + + diff --git a/Libraries/Catch2TestAdapter/Catch2TestAdapter.csproj b/Libraries/Catch2TestAdapter/Catch2TestAdapter.csproj index 9e932d1..a7a14d6 100644 --- a/Libraries/Catch2TestAdapter/Catch2TestAdapter.csproj +++ b/Libraries/Catch2TestAdapter/Catch2TestAdapter.csproj @@ -12,7 +12,7 @@ Library Properties Catch2TestAdapter - Catch2TestAdapter + Catch2.TestAdapter v4.6 512 From 2258b31ab9627c8237265bc95ae6785afb532a2f Mon Sep 17 00:00:00 2001 From: Oski Kervinen Date: Tue, 31 Jan 2023 14:46:18 +0200 Subject: [PATCH 2/2] Add the capability to run tests inside DLL using a custom wrapper exe When Catch2 tests are embedded in a DLL, they cannot be run by directly running the output itself. Instead some kind of wrapper executable must be used. The way such wrappers invoke the tests in the DLL is a project-specific matter, so such a system requires that the wrapper and its command line parameters are configurable. This MR achieves this with two additional settings: * `DllExecutor`, a path to the executable to run tests in DLL:s with. * `DllExecutorCommandLine`, the command line parameters passed to the executor to run the tests inside some Source DLL. --- Docs/Settings.md | 29 ++++++++ Libraries/Catch2Interface/Constants.cs | 8 +++ Libraries/Catch2Interface/Discoverer.cs | 4 +- Libraries/Catch2Interface/Executor.cs | 28 ++++---- Libraries/Catch2Interface/Settings.cs | 71 +++++++++++++++++++ Libraries/Catch2TestAdapter/TestDiscoverer.cs | 1 + Libraries/Catch2TestAdapter/TestExecutor.cs | 8 +-- .../Discover/Catch2XmlTests.cs | 40 +++++++++++ .../TestExecution/SingleModeTests.cs | 36 ++++++++++ 9 files changed, 205 insertions(+), 20 deletions(-) diff --git a/Docs/Settings.md b/Docs/Settings.md index cc72144..7a31716 100644 --- a/Docs/Settings.md +++ b/Docs/Settings.md @@ -8,6 +8,8 @@ In order for the **Test Adapter for Catch2** to do its job, it requires certain - [``](#combinedtimeout) - [``](#debugbreak) - [``](#discovercommandline) +- [``](#dllexecutor) +- [``](#dllexecutorcommandline) - [``](#discovertimeout) - [``](#environment) - [``](#executionmode) @@ -125,6 +127,33 @@ When you use Catch2 v3, you can set the reporter to xml for improved discovery. > Default value changed in v1.5.0, and v1.8.0 +## DllExecutor + +Default: "" + +Used to support running tests embedded in DLL:s. +When a test source is a DLL, this program is used to run it. The program will need to +know how to invoke Catch2 tests inside your DLL:s, there is no standard for this. +Supports `%ENVIRONMENT VARIABLES%`. Path can be absolute, or relative to the `source` or any of its +parent directories. + +## DllExecutorCommandLine + +Default: "$(Source) $(CatchParameters)" + +The command line parameters passed to the `DllExecutor` when running tests from a DLL. + +The string `$(Source)` will be replaced by the original source executable. +The string `$(CatchParameters)` will be replaced by the regular catch parameters. +Supports `%ENVIRONMENT VARIABLES%`. + +```xml + + my-catch2-wrapper + --source $(Source) $(CatchParameters) + +``` + ## DiscoverTimeout Default: 1000 ms diff --git a/Libraries/Catch2Interface/Constants.cs b/Libraries/Catch2Interface/Constants.cs index 75a590b..0d9ce2c 100644 --- a/Libraries/Catch2Interface/Constants.cs +++ b/Libraries/Catch2Interface/Constants.cs @@ -45,6 +45,8 @@ public static class Constants public const string NodeName_CombinedTimeout = "CombinedTimeout"; public const string NodeName_DebugBreak = "DebugBreak"; public const string NodeName_DiscoverCommanline = "DiscoverCommandLine"; + public const string NodeName_DllExecutor = "DllExecutor"; + public const string NodeName_DllExecutorCommandLine = "DllExecutorCommandLine"; public const string NodeName_DiscoverTimeout = "DiscoverTimeout"; public const string NodeName_Environment = "Environment"; public const string NodeName_ExecutionMode = "ExecutionMode"; @@ -65,6 +67,7 @@ public static class Constants public const bool S_DefaultDebugBreak = false; public const bool S_DefaultDisabled = false; public const string S_DefaultDiscoverCommandline = "--verbosity high --list-tests --reporter xml *"; + public const string S_DefaultDllExecutorCommandLine = "$(Source) $(CatchParameters)"; public const int S_DefaultDiscoverTimeout = 1000; // Time in milliseconds public const string S_DefaultExecutionModeForceSingleTagRgx = @"(?i:tafc_Single)"; public const string S_DefaultFilenameFilter = ""; // By default give invalid value @@ -74,6 +77,11 @@ public static class Constants public const int S_DefaultStackTraceMaxLength = 80; public const string S_DefaultStackTracePointReplacement = ","; + // The string to replace with the source executable in DllExecutorCommandLine. + public const string Tag_Source = "$(Source)"; + // The string to replace with the normal catch parameters in DllExecutorCommandLine. + public const string Tag_CatchParameters = "$(CatchParameters)"; + public const ExecutionModes S_DefaultExecutionMode = ExecutionModes.SingleTestCase; public const LoggingLevels S_DefaultLoggingLevel = LoggingLevels.Normal; public const MessageFormats S_DefaultMessageFormat = MessageFormats.StatsOnly; diff --git a/Libraries/Catch2Interface/Discoverer.cs b/Libraries/Catch2Interface/Discoverer.cs index 375789b..6e4fac7 100644 --- a/Libraries/Catch2Interface/Discoverer.cs +++ b/Libraries/Catch2Interface/Discoverer.cs @@ -157,8 +157,8 @@ private string GetTestCaseInfo(string source) // Retrieve test cases using (var process = new Process()) { - process.StartInfo.FileName = source; - process.StartInfo.Arguments = _settings.DiscoverCommandLine; + process.StartInfo.FileName = _settings.GetExecutable(source); + process.StartInfo.Arguments = _settings.FormatParameters(source, _settings.DiscoverCommandLine); process.StartInfo.CreateNoWindow = true; process.StartInfo.RedirectStandardOutput = true; process.StartInfo.RedirectStandardError = true; diff --git a/Libraries/Catch2Interface/Executor.cs b/Libraries/Catch2Interface/Executor.cs index ec8850b..0e208e8 100644 --- a/Libraries/Catch2Interface/Executor.cs +++ b/Libraries/Catch2Interface/Executor.cs @@ -96,37 +96,37 @@ public void Cancel() } } - public string GenerateCommandlineArguments_Single(string testname, string reportfilename) + public string GenerateCommandlineArguments_Single(string source, string testname, string reportfilename) { - return $"{GenerateTestnameForCommandline(testname)} --reporter xml --durations yes --out {"\""}{reportfilename}{"\""}"; + return _settings.FormatParameters(source, $"{GenerateTestnameForCommandline(testname)} --reporter xml --durations yes --out {"\""}{reportfilename}{"\""}"); } - public string GenerateCommandlineArguments_Single_Dbg(string testname) + public string GenerateCommandlineArguments_Single_Dbg(string source, string testname) { if (_settings.DebugBreak) { - return $"{GenerateTestnameForCommandline(testname)} --reporter xml --durations yes --break"; + return _settings.FormatParameters(source, $"{GenerateTestnameForCommandline(testname)} --reporter xml --durations yes --break"); } else { - return $"{GenerateTestnameForCommandline(testname)} --reporter xml --durations yes"; + return _settings.FormatParameters(source, $"{GenerateTestnameForCommandline(testname)} --reporter xml --durations yes"); } } - public string GenerateCommandlineArguments_Combined(string caselistfilename, string reportfilename) + public string GenerateCommandlineArguments_Combined(string source, string caselistfilename, string reportfilename) { - return $"--reporter xml --durations yes --input-file {"\""}{caselistfilename}{"\""} --out {"\""}{reportfilename}{"\""}"; + return _settings.FormatParameters(source, $"--reporter xml --durations yes --input-file {"\""}{caselistfilename}{"\""} --out {"\""}{reportfilename}{"\""}"); } - public string GenerateCommandlineArguments_Combined_Dbg(string caselistfilename) + public string GenerateCommandlineArguments_Combined_Dbg(string source, string caselistfilename) { if (_settings.DebugBreak) { - return $"--reporter xml --durations yes --break --input-file {"\""}{caselistfilename}{"\""}"; + return _settings.FormatParameters(source, $"--reporter xml --durations yes --break --input-file {"\""}{caselistfilename}{"\""}"); } else { - return $"--reporter xml --durations yes --input-file {"\""}{caselistfilename}{"\""}"; + return _settings.FormatParameters(source, $"--reporter xml --durations yes --input-file {"\""}{caselistfilename}{"\""}"); } } @@ -139,8 +139,8 @@ public TestResult Run(string testname, string source) string reportfilename = MakeReportFilename(source); var process = new Process(); - process.StartInfo.FileName = source; - process.StartInfo.Arguments = GenerateCommandlineArguments_Single(testname, reportfilename); + process.StartInfo.FileName = _settings.GetExecutable(source); + process.StartInfo.Arguments = GenerateCommandlineArguments_Single(source, testname, reportfilename); process.StartInfo.CreateNoWindow = true; process.StartInfo.RedirectStandardOutput = true; process.StartInfo.UseShellExecute = false; @@ -220,8 +220,8 @@ public XmlOutput Run(TestCaseGroup group) // Run tests var process = new Process(); - process.StartInfo.FileName = group.Source; - process.StartInfo.Arguments = GenerateCommandlineArguments_Combined(caselistfilename, reportfilename); + process.StartInfo.FileName = _settings.GetExecutable( group.Source ); + process.StartInfo.Arguments = GenerateCommandlineArguments_Combined(group.Source, caselistfilename, reportfilename); process.StartInfo.CreateNoWindow = true; process.StartInfo.RedirectStandardOutput = true; process.StartInfo.UseShellExecute = false; diff --git a/Libraries/Catch2Interface/Settings.cs b/Libraries/Catch2Interface/Settings.cs index 9044457..6f32f07 100644 --- a/Libraries/Catch2Interface/Settings.cs +++ b/Libraries/Catch2Interface/Settings.cs @@ -11,9 +11,11 @@ ** Basic Info **/ +using System; using System.Collections; using System.Collections.Generic; using System.Collections.Specialized; +using System.IO; using System.Text.RegularExpressions; using System.Xml; @@ -130,6 +132,8 @@ public class Settings public bool DebugBreak { get; set; } = Constants.S_DefaultDebugBreak; public bool Disabled { get; set; } = Constants.S_DefaultDisabled; public string DiscoverCommandLine { get; set; } = Constants.S_DefaultDiscoverCommandline; + public string DllExecutor { get; set; } = string.Empty; + public string DllExecutorCommandLine { get; set; } = Constants.S_DefaultDllExecutorCommandLine; public int DiscoverTimeout { get; set; } = Constants.S_DefaultDiscoverTimeout; public IDictionary Environment { get; set; } public ExecutionModes ExecutionMode { get; set; } = Constants.S_DefaultExecutionMode; @@ -322,6 +326,16 @@ public static Settings Extract(XmlNode node) } } + // TestExecutableOverride + var dllExecutor = node.SelectSingleNode(Constants.NodeName_DllExecutor)?.FirstChild; + if( dllExecutor != null && dllExecutor.NodeType == XmlNodeType.Text ) + settings.DllExecutor = dllExecutor.Value; + + // ExtraParameters + var dllExecutorCommandLine = node.SelectSingleNode(Constants.NodeName_DllExecutorCommandLine)?.FirstChild; + if (dllExecutorCommandLine != null && dllExecutorCommandLine.NodeType == XmlNodeType.Text) + settings.DllExecutorCommandLine = dllExecutorCommandLine.Value; + // WorkingDirectory var workingdir = node.SelectSingleNode(Constants.NodeName_WorkingDirectory)?.FirstChild; if( workingdir != null && workingdir.NodeType == XmlNodeType.Text ) @@ -421,6 +435,63 @@ public IDictionary GetEnviromentVariablesForDebug() return envvars; } + /// + /// Returns the executable that should be run for tests in source. + /// Defaults to the source, but may be overridden. + /// + /// + /// + public string GetExecutable(string source) + { + // If there is no DLL executor, or the source is not a DLL, always return the source binary. + if (!source.ToLower().EndsWith(".dll") || String.IsNullOrEmpty(DllExecutor)) + return source; + + // Expand environment variables. + string expandedExecutor = System.Environment.ExpandEnvironmentVariables(DllExecutor); + + // If the path is absolute, return it as-is. + if (Path.IsPathRooted(expandedExecutor)) + return expandedExecutor; + + // If the file exists relative to the CWD, return from there. + string relativeToCWD = Path.GetFullPath( expandedExecutor ); + if (File.Exists(relativeToCWD)) + return relativeToCWD; + + // Otherwise try to find it relative to the source and its parent directories. + for (DirectoryInfo parent = Directory.GetParent(source); parent != null; parent = parent.Parent) + { + // Formulate the path relative to this folder. + string relativeToParent = Path.Combine(parent.FullName, expandedExecutor); + if (File.Exists(relativeToParent)) + return relativeToParent; + } + + // Report failure. + throw new ArgumentException( $"Could not find file specified by {nameof(DllExecutor)}='{expandedExecutor}'." ); + } + + /// + /// Returns extra parameters formatted ready for inclusion in the command line. + /// (Empty string if there are no parameters, followed by a space if there are.) + /// + /// + public string FormatParameters(string source, string catchParameters) + { + // If the source is a dll, it is executed using the DLL executor. + // Format the parameters with the DLL executor command line. + if (source.ToLower().EndsWith(".dll")) + return System.Environment.ExpandEnvironmentVariables( + DllExecutorCommandLine + .Replace(Constants.Tag_Source, source) + .Replace(Constants.Tag_Source.ToLower(), source) + .Replace(Constants.Tag_CatchParameters, catchParameters) + .Replace(Constants.Tag_CatchParameters.ToLower(), catchParameters) ); + else + return catchParameters; + } + #endregion // Public Methods #region Static Private diff --git a/Libraries/Catch2TestAdapter/TestDiscoverer.cs b/Libraries/Catch2TestAdapter/TestDiscoverer.cs index ea6631e..0f1e1b6 100644 --- a/Libraries/Catch2TestAdapter/TestDiscoverer.cs +++ b/Libraries/Catch2TestAdapter/TestDiscoverer.cs @@ -21,6 +21,7 @@ namespace Catch2TestAdapter { [DefaultExecutorUri("executor://Catch2TestExecutor")] [FileExtension(".exe")] + [FileExtension(".dll")] public class TestDiscoverer : ITestDiscoverer { #region Fields diff --git a/Libraries/Catch2TestAdapter/TestExecutor.cs b/Libraries/Catch2TestAdapter/TestExecutor.cs index 5d522d7..9799732 100644 --- a/Libraries/Catch2TestAdapter/TestExecutor.cs +++ b/Libraries/Catch2TestAdapter/TestExecutor.cs @@ -279,9 +279,9 @@ private void RunTests_Combine(IEnumerable tests) LogVerbose(TestMessageLevel.Informational, "Start debug run."); _frameworkHandle - .LaunchProcessWithDebuggerAttached( testcasegroup.Source + .LaunchProcessWithDebuggerAttached( _settings.GetExecutable(testcasegroup.Source) , _executor.WorkingDirectory(testcasegroup.Source) - , _executor.GenerateCommandlineArguments_Combined_Dbg(caselistfilename) + , _executor.GenerateCommandlineArguments_Combined_Dbg(testcasegroup.Source, caselistfilename) , _settings.GetEnviromentVariablesForDebug()); // Do not process output in Debug mode @@ -389,9 +389,9 @@ private void RunTest(TestCase test) { LogVerbose(TestMessageLevel.Informational, "Start debug run."); _frameworkHandle - .LaunchProcessWithDebuggerAttached( test.Source + .LaunchProcessWithDebuggerAttached( _settings.GetExecutable(test.Source) , _executor.WorkingDirectory(test.Source) - , _executor.GenerateCommandlineArguments_Single_Dbg(test.DisplayName) + , _executor.GenerateCommandlineArguments_Single_Dbg(test.Source, test.DisplayName) , _settings.GetEnviromentVariablesForDebug() ); // Do not process output in Debug mode diff --git a/UnitTests/UT_Catch2Interface/Discover/Catch2XmlTests.cs b/UnitTests/UT_Catch2Interface/Discover/Catch2XmlTests.cs index 0de81b8..9187e59 100644 --- a/UnitTests/UT_Catch2Interface/Discover/Catch2XmlTests.cs +++ b/UnitTests/UT_Catch2Interface/Discover/Catch2XmlTests.cs @@ -320,6 +320,46 @@ public void Names(string versionpath) Assert.AreEqual( 89, tests[12].Line ); } + // Tests that the option "TestExecutableOverride" overrides the executable + // that is used to run the tests. + [DataTestMethod] + [DynamicData( nameof( VersionPaths ), DynamicDataSourceType.Property )] + public void TestDllExecutor(string versionpath) + { + // Find the real source. + var source = Paths.TestExecutable_Hidden( TestContext, versionpath ); + if (string.IsNullOrEmpty(source)) + { + Assert.Fail( $"Missing test executable for {versionpath}." ); + return; + } + + var settings = new Settings(); + settings.DiscoverCommandLine = "--discover [Tag1]"; + settings.FilenameFilter = ".*"; + settings.IncludeHidden = false; + + // The point of the TestExecutableOverride is to wrap the catch + // execution with a different executable, but we don't want to + // create a dummy executable just for this test. + // So instead, we override the test executable with the + // real source exe, and use a dummy source. + settings.DllExecutor = source; + settings.DllExecutorCommandLine = Constants.Tag_CatchParameters; + + var discoverer = new Discoverer( settings ); + + // Pass nonsense as the source. The discoverer checks that this is a + // file that exists, so pass in a path to the currently executing file. + // It will also have the -dll suffix required to trigger the DllExecutor. + string[] sources = { System.Reflection.Assembly.GetExecutingAssembly().Location }; + + // This should work despite the source being invalid, because we told the discoverer to + // use a specific executable, not the source. + var tests = discoverer.GetTests( sources ) as List; + Assert.AreEqual( 2, tests.Count ); + } + #endregion // TestCases #region LongTestCaseNames diff --git a/UnitTests/UT_Catch2Interface/TestExecution/SingleModeTests.cs b/UnitTests/UT_Catch2Interface/TestExecution/SingleModeTests.cs index 12801cc..32c84fb 100644 --- a/UnitTests/UT_Catch2Interface/TestExecution/SingleModeTests.cs +++ b/UnitTests/UT_Catch2Interface/TestExecution/SingleModeTests.cs @@ -56,6 +56,42 @@ public void TestRun_Basic(string versionpath) Assert.AreEqual(3, result.OverallResults.TotalAssertions); } + [DataTestMethod] + [DynamicData( nameof( VersionPaths ), DynamicDataSourceType.Property )] + public void TestRun_TestDllExecutor(string versionpath) + { + var source = Paths.TestExecutable_Execution(TestContext, versionpath); + if (string.IsNullOrEmpty( source )) + { + Assert.Fail( $"Missing test executable for {versionpath}." ); + return; + } + + // TestExecutableOverride is meant for wrapping the test execution + // with a different executable. We don't have a suitable wrapper + // executable, so we test the behaviour with a trick: + // we put the real source as the override, and pass a dummy + // value as the source. + var settings = new Settings(); + settings.ExecutionMode = ExecutionModes.SingleTestCase; + settings.DllExecutor = source; + settings.DllExecutorCommandLine = Constants.Tag_CatchParameters; + + // Use the executing assembly as the dummy value ensure that it + // is an existing file, because the executor checks that. + // The test assembly is also a dll, triggering the DllExecutor. + string dummySource = System.Reflection.Assembly.GetExecutingAssembly().Location; + + var executor = new Executor( settings, _pathSolution, _pathTestRun ); + + // The run should work with the dummy source, because we overrode the + // test executable. + var result = executor.Run( "Names. abcdefghijklmnopqrstuvwxyz", dummySource ); + Assert.AreEqual( TestOutcomes.Passed, result.Outcome ); + Assert.AreEqual( 0, result.OverallResults.Failures ); + Assert.AreEqual( 1, result.OverallResults.Successes ); + } + [DataTestMethod] [DynamicData(nameof(VersionPaths), DynamicDataSourceType.Property)] public void TestRun_TestNames_Pass(string versionpath)