DLL Runner

Docs/Settings.md

@@ -8,6 +8,8 @@ In order for the **Test Adapter for Catch2** to do its job, it requires certain
 - [`<CombinedTimeout>`](#combinedtimeout)
 - [`<DebugBreak>`](#debugbreak)
 - [`<DiscoverCommandLine>`](#discovercommandline)
+- [`<DllExecutor>`](#dllexecutor)
+- [`<DllExecutorCommandLine>`](#dllexecutorcommandline)
 - [`<DiscoverTimeout>`](#discovertimeout)
 - [`<Environment>`](#environment)
 - [`<ExecutionMode>`](#executionmode)
@@ -125,6 +127,33 @@ When you use Catch2 v3, you can set the reporter to xml for improved discovery.
 
 > Default value changed in v1.5.0, and v1.8.0
 
+## DllExecutor
+
+Default: ""
+
+Used to support running tests embedded in DLL:s.
+When a test source is a DLL, this program is used to run it. The program will need to
+know how to invoke Catch2 tests inside your DLL:s, there is no standard for this.
+Supports `%ENVIRONMENT VARIABLES%`. Path can be absolute, or relative to the `source` or any of its
+parent directories.
+
+## DllExecutorCommandLine
+
+Default: "$(Source) $(CatchParameters)"
+
+The command line parameters passed to the `DllExecutor` when running tests from a DLL.
+
+The string `$(Source)` will be replaced by the original source executable.
+The string `$(CatchParameters)` will be replaced by the regular catch parameters.
+Supports `%ENVIRONMENT VARIABLES%`.
+
+```xml
+<Catch2Adapter>
+    <DllExecutor>my-catch2-wrapper</DllExecutor>
+    <DllExecutorCommandLine>--source $(Source) $(CatchParameters)</DllExecutorCommandLine>
+</Catch2Adapter>
+```
+
 ## DiscoverTimeout
 
 Default: 1000 ms

Libraries/Catch2.TestAdapter.nuspec

@@ -0,0 +1,30 @@
+<?xml version="1.0"?>
+<package >
+  <metadata>
+    <id>Catch2.TestAdapter</id>
+    <version>$version$</version>
+    <title>Visual Studio Test Adapter for Catch2</title>
+    <authors>JohnnyHendriks</authors>
+    <owners>JohnnyHendriks</owners>
+    <projectUrl>https://github.com/JohnnyHendriks/TestAdapter_Catch2</projectUrl>
+    <requireLicenseAcceptance>false</requireLicenseAcceptance>
+    <license type="expression">MIT</license>
+    <description>
+        Class library containing the Visual Studio Test Adapter for Catch2
+    </description>
+    <releaseNotes></releaseNotes>
+    <copyright></copyright>
+    <tags>native catch2 test adapter</tags>
+    <dependencies>
+      <group targetFramework="native" />
+    </dependencies>
+  </metadata>
+  <files>
+    <!-- Must copy the files to the folder "native" to make the package installable in native projects.
+         The Visual Studio test window recognizes any file named *.TestAdapter.dll in a nuget package
+         as a potential adapter, not caring about the platform. See
+         https://github.com/microsoft/vstest/blob/main/docs/RFCs/0004-Adapter-Extensibility.md -->
+    <file src="Catch2TestAdapter\bin\$configuration$\Catch2.TestAdapter.dll" target="lib\native\" />
+    <file src="Catch2Interface\bin\$configuration$\Catch2Interface.dll" target="lib\native\" />
+  </files>
+</package>

Libraries/Catch2Interface/Constants.cs

@@ -45,6 +45,8 @@ public static class Constants
         public const string NodeName_CombinedTimeout = "CombinedTimeout";
         public const string NodeName_DebugBreak = "DebugBreak";
         public const string NodeName_DiscoverCommanline = "DiscoverCommandLine";
+        public const string NodeName_DllExecutor = "DllExecutor";
+        public const string NodeName_DllExecutorCommandLine = "DllExecutorCommandLine";
         public const string NodeName_DiscoverTimeout = "DiscoverTimeout";
         public const string NodeName_Environment = "Environment";
         public const string NodeName_ExecutionMode = "ExecutionMode";
@@ -65,6 +67,7 @@ public static class Constants
         public const bool   S_DefaultDebugBreak = false;
         public const bool   S_DefaultDisabled = false;
         public const string S_DefaultDiscoverCommandline = "--verbosity high --list-tests --reporter xml *";
+        public const string S_DefaultDllExecutorCommandLine = "$(Source) $(CatchParameters)";
         public const int    S_DefaultDiscoverTimeout = 1000; // Time in milliseconds
         public const string S_DefaultExecutionModeForceSingleTagRgx = @"(?i:tafc_Single)";
         public const string S_DefaultFilenameFilter = "";    // By default give invalid value
@@ -74,6 +77,11 @@ public static class Constants
         public const int    S_DefaultStackTraceMaxLength = 80;
         public const string S_DefaultStackTracePointReplacement = ",";
 
+        // The string to replace with the source executable in DllExecutorCommandLine.
+        public const string Tag_Source = "$(Source)";
+        // The string to replace with the normal catch parameters in DllExecutorCommandLine.
+        public const string Tag_CatchParameters = "$(CatchParameters)";
+
         public const ExecutionModes        S_DefaultExecutionMode = ExecutionModes.SingleTestCase;
         public const LoggingLevels         S_DefaultLoggingLevel = LoggingLevels.Normal;
         public const MessageFormats        S_DefaultMessageFormat = MessageFormats.StatsOnly;

Libraries/Catch2Interface/Discoverer.cs

@@ -157,8 +157,8 @@ private string GetTestCaseInfo(string source)
             // Retrieve test cases
             using (var process = new Process())
             {
-                process.StartInfo.FileName = source;
-                process.StartInfo.Arguments = _settings.DiscoverCommandLine;
+                process.StartInfo.FileName = _settings.GetExecutable(source);
+                process.StartInfo.Arguments = _settings.FormatParameters(source, _settings.DiscoverCommandLine);
                 process.StartInfo.CreateNoWindow = true;
                 process.StartInfo.RedirectStandardOutput = true;
                 process.StartInfo.RedirectStandardError = true;

Libraries/Catch2Interface/Executor.cs

@@ -96,37 +96,37 @@ public void Cancel()
             }
         }
 
-        public string GenerateCommandlineArguments_Single(string testname, string reportfilename)
+        public string GenerateCommandlineArguments_Single(string source, string testname, string reportfilename)
         {
-            return $"{GenerateTestnameForCommandline(testname)} --reporter xml --durations yes --out {"\""}{reportfilename}{"\""}";
+            return _settings.FormatParameters(source, $"{GenerateTestnameForCommandline(testname)} --reporter xml --durations yes --out {"\""}{reportfilename}{"\""}");
         }
 
-        public string GenerateCommandlineArguments_Single_Dbg(string testname)
+        public string GenerateCommandlineArguments_Single_Dbg(string source, string testname)
         {
             if (_settings.DebugBreak)
             {
-                return $"{GenerateTestnameForCommandline(testname)} --reporter xml --durations yes --break";
+                return _settings.FormatParameters(source, $"{GenerateTestnameForCommandline(testname)} --reporter xml --durations yes --break");
             }
             else
             {
-                return $"{GenerateTestnameForCommandline(testname)} --reporter xml --durations yes";
+                return _settings.FormatParameters(source, $"{GenerateTestnameForCommandline(testname)} --reporter xml --durations yes");
             }
         }
 
-        public string GenerateCommandlineArguments_Combined(string caselistfilename, string reportfilename)
+        public string GenerateCommandlineArguments_Combined(string source, string caselistfilename, string reportfilename)
         {
-            return $"--reporter xml --durations yes --input-file {"\""}{caselistfilename}{"\""} --out {"\""}{reportfilename}{"\""}";
+            return _settings.FormatParameters(source, $"--reporter xml --durations yes --input-file {"\""}{caselistfilename}{"\""} --out {"\""}{reportfilename}{"\""}");
         }
 
-        public string GenerateCommandlineArguments_Combined_Dbg(string caselistfilename)
+        public string GenerateCommandlineArguments_Combined_Dbg(string source, string caselistfilename)
         {
             if (_settings.DebugBreak)
             {
-                return $"--reporter xml --durations yes --break --input-file {"\""}{caselistfilename}{"\""}";
+                return _settings.FormatParameters(source, $"--reporter xml --durations yes --break --input-file {"\""}{caselistfilename}{"\""}");
             }
             else
             {
-                return $"--reporter xml --durations yes --input-file {"\""}{caselistfilename}{"\""}";
+                return _settings.FormatParameters(source, $"--reporter xml --durations yes --input-file {"\""}{caselistfilename}{"\""}");
             }
         }
 
@@ -139,8 +139,8 @@ public TestResult Run(string testname, string source)
             string reportfilename = MakeReportFilename(source);
 
             var process = new Process();
-            process.StartInfo.FileName = source;
-            process.StartInfo.Arguments = GenerateCommandlineArguments_Single(testname, reportfilename);
+            process.StartInfo.FileName = _settings.GetExecutable(source);
+            process.StartInfo.Arguments = GenerateCommandlineArguments_Single(source, testname, reportfilename);
             process.StartInfo.CreateNoWindow = true;
             process.StartInfo.RedirectStandardOutput = true;
             process.StartInfo.UseShellExecute = false;
@@ -220,8 +220,8 @@ public XmlOutput Run(TestCaseGroup group)
 
             // Run tests
             var process = new Process();
-            process.StartInfo.FileName = group.Source;
-            process.StartInfo.Arguments = GenerateCommandlineArguments_Combined(caselistfilename, reportfilename);
+            process.StartInfo.FileName = _settings.GetExecutable( group.Source );
+            process.StartInfo.Arguments = GenerateCommandlineArguments_Combined(group.Source, caselistfilename, reportfilename);
             process.StartInfo.CreateNoWindow = true;
             process.StartInfo.RedirectStandardOutput = true;
             process.StartInfo.UseShellExecute = false;

Libraries/Catch2Interface/Settings.cs

@@ -11,9 +11,11 @@
 
 ** Basic Info **/
 
+using System;
 using System.Collections;
 using System.Collections.Generic;
 using System.Collections.Specialized;
+using System.IO;
 using System.Text.RegularExpressions;
 using System.Xml;
 
@@ -130,6 +132,8 @@ public class Settings
         public bool                       DebugBreak { get; set; }                     = Constants.S_DefaultDebugBreak;
         public bool                       Disabled { get; set; }                       = Constants.S_DefaultDisabled;
         public string                     DiscoverCommandLine { get; set; }            = Constants.S_DefaultDiscoverCommandline;
+        public string                     DllExecutor { get; set; }                    = string.Empty;
+        public string                     DllExecutorCommandLine { get; set; }         = Constants.S_DefaultDllExecutorCommandLine;
         public int                        DiscoverTimeout { get; set; }                = Constants.S_DefaultDiscoverTimeout;
         public IDictionary<string,string> Environment { get; set; }
         public ExecutionModes             ExecutionMode { get; set; }                  = Constants.S_DefaultExecutionMode;
@@ -322,6 +326,16 @@ public static Settings Extract(XmlNode node)
                     }
                 }
 
+                // TestExecutableOverride
+                var dllExecutor = node.SelectSingleNode(Constants.NodeName_DllExecutor)?.FirstChild;
+                if( dllExecutor != null && dllExecutor.NodeType == XmlNodeType.Text )
+                    settings.DllExecutor = dllExecutor.Value;
+
+                // ExtraParameters
+                var dllExecutorCommandLine = node.SelectSingleNode(Constants.NodeName_DllExecutorCommandLine)?.FirstChild;
+                if (dllExecutorCommandLine != null && dllExecutorCommandLine.NodeType == XmlNodeType.Text)
+                    settings.DllExecutorCommandLine = dllExecutorCommandLine.Value;
+
                 // WorkingDirectory
                 var workingdir = node.SelectSingleNode(Constants.NodeName_WorkingDirectory)?.FirstChild;
                 if( workingdir != null && workingdir.NodeType == XmlNodeType.Text )
@@ -421,6 +435,63 @@ public IDictionary<string, string> GetEnviromentVariablesForDebug()
             return envvars;
         }
 
+        /// <summary>
+        /// Returns the executable that should be run for tests in source.
+        /// Defaults to the source, but may be overridden.
+        /// </summary>
+        /// <param name="source"></param>
+        /// <returns></returns>
+        public string GetExecutable(string source)
+        {
+            // If there is no DLL executor, or the source is not a DLL, always return the source binary.
+            if (!source.ToLower().EndsWith(".dll") || String.IsNullOrEmpty(DllExecutor))
+                return source;
+
+            // Expand environment variables.
+            string expandedExecutor = System.Environment.ExpandEnvironmentVariables(DllExecutor);
+
+            // If the path is absolute, return it as-is.
+            if (Path.IsPathRooted(expandedExecutor))
+                return expandedExecutor;
+
+            // If the file exists relative to the CWD, return from there.
+            string relativeToCWD = Path.GetFullPath( expandedExecutor );
+            if (File.Exists(relativeToCWD))
+                return relativeToCWD;
+
+            // Otherwise try to find it relative to the source and its parent directories.
+            for (DirectoryInfo parent = Directory.GetParent(source); parent != null; parent = parent.Parent)
+            {
+                // Formulate the path relative to this folder.
+                string relativeToParent = Path.Combine(parent.FullName, expandedExecutor);
+                if (File.Exists(relativeToParent))
+                    return relativeToParent;
+            }
+
+            // Report failure.
+            throw new ArgumentException( $"Could not find file specified by {nameof(DllExecutor)}='{expandedExecutor}'." );
+        }
+
+        /// <summary>
+        /// Returns extra parameters formatted ready for inclusion in the command line.
+        /// (Empty string if there are no parameters, followed by a space if there are.)
+        /// </summary>
+        /// <returns></returns>
+        public string FormatParameters(string source, string catchParameters)
+        {
+            // If the source is a dll, it is executed using the DLL executor.
+            // Format the parameters with the DLL executor command line.
+            if (source.ToLower().EndsWith(".dll"))
+                return System.Environment.ExpandEnvironmentVariables(
+                    DllExecutorCommandLine
+                        .Replace(Constants.Tag_Source, source)
+                        .Replace(Constants.Tag_Source.ToLower(), source)
+                        .Replace(Constants.Tag_CatchParameters, catchParameters)
+                        .Replace(Constants.Tag_CatchParameters.ToLower(), catchParameters) );
+            else
+                return catchParameters;
+        }
+
         #endregion // Public Methods
 
         #region Static Private

Libraries/Catch2TestAdapter/Catch2TestAdapter.csproj

@@ -12,7 +12,7 @@
     <OutputType>Library</OutputType>
     <AppDesignerFolder>Properties</AppDesignerFolder>
     <RootNamespace>Catch2TestAdapter</RootNamespace>
-    <AssemblyName>Catch2TestAdapter</AssemblyName>
+    <AssemblyName>Catch2.TestAdapter</AssemblyName>
     <TargetFrameworkVersion>v4.6</TargetFrameworkVersion>
     <FileAlignment>512</FileAlignment>
   </PropertyGroup>

Libraries/Catch2TestAdapter/TestDiscoverer.cs

@@ -21,6 +21,7 @@ namespace Catch2TestAdapter
 {
     [DefaultExecutorUri("executor://Catch2TestExecutor")]
     [FileExtension(".exe")]
+    [FileExtension(".dll")]
     public class TestDiscoverer : ITestDiscoverer
     {
         #region Fields

Libraries/Catch2TestAdapter/TestExecutor.cs

@@ -279,9 +279,9 @@ private void RunTests_Combine(IEnumerable<TestCase> tests)
 
                 LogVerbose(TestMessageLevel.Informational, "Start debug run.");
                 _frameworkHandle
-                    .LaunchProcessWithDebuggerAttached( testcasegroup.Source
+                    .LaunchProcessWithDebuggerAttached( _settings.GetExecutable(testcasegroup.Source)
                                                       , _executor.WorkingDirectory(testcasegroup.Source)
-                                                      , _executor.GenerateCommandlineArguments_Combined_Dbg(caselistfilename)
+                                                      , _executor.GenerateCommandlineArguments_Combined_Dbg(testcasegroup.Source, caselistfilename)
                                                       , _settings.GetEnviromentVariablesForDebug());
 
                 // Do not process output in Debug mode
@@ -389,9 +389,9 @@ private void RunTest(TestCase test)
             {
                 LogVerbose(TestMessageLevel.Informational, "Start debug run.");
                 _frameworkHandle
-                    .LaunchProcessWithDebuggerAttached( test.Source
+                    .LaunchProcessWithDebuggerAttached( _settings.GetExecutable(test.Source)
                                                       , _executor.WorkingDirectory(test.Source)
-                                                      , _executor.GenerateCommandlineArguments_Single_Dbg(test.DisplayName)
+                                                      , _executor.GenerateCommandlineArguments_Single_Dbg(test.Source, test.DisplayName)
                                                       , _settings.GetEnviromentVariablesForDebug() );
 
                 // Do not process output in Debug mode

UnitTests/UT_Catch2Interface/Discover/Catch2XmlTests.cs

@@ -320,6 +320,46 @@ public void Names(string versionpath)
             Assert.AreEqual( 89, tests[12].Line );
         }
 
+        // Tests that the option "TestExecutableOverride" overrides the executable
+        // that is used to run the tests.
+        [DataTestMethod]
+        [DynamicData( nameof( VersionPaths ), DynamicDataSourceType.Property )]
+        public void TestDllExecutor(string versionpath)
+        {
+            // Find the real source.
+            var source = Paths.TestExecutable_Hidden( TestContext, versionpath );
+            if (string.IsNullOrEmpty(source))
+            {
+                Assert.Fail( $"Missing test executable for {versionpath}." );
+                return;
+            }
+
+            var settings = new Settings();
+            settings.DiscoverCommandLine = "--discover [Tag1]";
+            settings.FilenameFilter = ".*";
+            settings.IncludeHidden = false;
+
+            // The point of the TestExecutableOverride is to wrap the catch
+            // execution with a different executable, but we don't want to
+            // create a dummy executable just for this test.
+            // So instead, we override the test executable with the
+            // real source exe, and use a dummy source.
+            settings.DllExecutor = source;
+            settings.DllExecutorCommandLine = Constants.Tag_CatchParameters;
+
+            var discoverer = new Discoverer( settings );
+
+            // Pass nonsense as the source. The discoverer checks that this is a
+            // file that exists, so pass in a path to the currently executing file.
+            // It will also have the -dll suffix required to trigger the DllExecutor.
+            string[] sources = { System.Reflection.Assembly.GetExecutingAssembly().Location };
+
+            // This should work despite the source being invalid, because we told the discoverer to
+            // use a specific executable, not the source.
+            var tests = discoverer.GetTests( sources ) as List<TestCase>;
+            Assert.AreEqual( 2, tests.Count );
+        }
+
         #endregion // TestCases
 
         #region LongTestCaseNames