From 2ddb0377e9fa0b12358b9811b408a1984250214f Mon Sep 17 00:00:00 2001
From: Adam Boniecki <20281641+abonie@users.noreply.github.com>
Date: Sun, 12 Jul 2026 12:16:27 +0200
Subject: [PATCH 01/15] Add Code Action Apex tests based on TS
---
VisualFSharp.slnx | 3 +
eng/Versions.props | 2 +
.../CodeFixes/CodeActionTests.cs | 65 +++++
.../CodeFixes/SuggestionCodeFix.cs | 60 +++++
...FSharp.Editor.Apex.IntegrationTests.csproj | 41 ++++
.../FSharpLanguageServiceApexTest.cs | 225 ++++++++++++++++++
.../FSharpLanguageServiceLibrary.cs | 50 ++++
.../TestFramework/ProjectCreationHelper.cs | 49 ++++
.../TestFramework/RetryTestMethodAttribute.cs | 62 +++++
.../TestFramework/SynchronizationHelper.cs | 64 +++++
.../TestFramework/TextDocumentView.cs | 45 ++++
11 files changed, 666 insertions(+)
create mode 100644 vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/CodeFixes/CodeActionTests.cs
create mode 100644 vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/CodeFixes/SuggestionCodeFix.cs
create mode 100644 vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/FSharp.Editor.Apex.IntegrationTests.csproj
create mode 100644 vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/FSharpLanguageServiceApexTest.cs
create mode 100644 vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/FSharpLanguageServiceLibrary.cs
create mode 100644 vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/ProjectCreationHelper.cs
create mode 100644 vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/RetryTestMethodAttribute.cs
create mode 100644 vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/SynchronizationHelper.cs
create mode 100644 vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/TextDocumentView.cs
diff --git a/VisualFSharp.slnx b/VisualFSharp.slnx
index 9f6eed02f50..5f24b321992 100644
--- a/VisualFSharp.slnx
+++ b/VisualFSharp.slnx
@@ -176,6 +176,9 @@
+
+
+
diff --git a/eng/Versions.props b/eng/Versions.props
index 60486fabcaa..dbaeb9ad65c 100644
--- a/eng/Versions.props
+++ b/eng/Versions.props
@@ -130,6 +130,8 @@
$(VisualStudioEditorPackagesVersion)
$(VisualStudioEditorPackagesVersion)
17.14.0
+ 18.10.0-preview-1-12005-078
+ 3.6.4
0.1.800-beta
$(MicrosoftVisualStudioExtensibilityTestingVersion)
diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/CodeFixes/CodeActionTests.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/CodeFixes/CodeActionTests.cs
new file mode 100644
index 00000000000..515cd14883f
--- /dev/null
+++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/CodeFixes/CodeActionTests.cs
@@ -0,0 +1,65 @@
+// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information.
+//
+// Apex counterparts of the xUnit/IdeFact tests in
+// vsintegration/tests/FSharp.Editor.IntegrationTests/CodeActionTests.cs. Each case sets up the same
+// triggering F# source, places the caret where that test does, and verifies the same code/error fix
+// is offered on the light bulb (asserting on the action's display text). The Apex light-bulb model
+// exposes a flat action list, so the CodeFix/ErrorFix category asserted by the source tests is not
+// mirrored here.
+
+using FSharp.Editor.Apex.IntegrationTests.TestFramework;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace FSharp.Editor.Apex.IntegrationTests
+{
+ [TestClass]
+ public class CodeActionTests : FSharpLanguageServiceApexTest
+ {
+ protected override bool AutomaticallyDismissMessageBoxes => false;
+
+ [RetryTestMethod]
+ [Timeout(ShortTimeoutMS)]
+ [Description("Verify the 'Remove unused open declarations' code fix is offered on an unused open.")]
+ [Owner("fsharptools")]
+ [TestCategory("nightly"), TestCategory("Walkthrough")]
+ public void UnusedOpenDeclarations()
+ {
+ var code = @"module Library
+
+open System
+
+let x = 42";
+
+ this.AssertCodeActionOffered(code, "open System", "Remove unused open declarations");
+ }
+
+ [RetryTestMethod]
+ [Timeout(ShortTimeoutMS)]
+ [Description("Verify the 'Add missing 'fun' keyword' error fix is offered on a lambda missing 'fun'.")]
+ [Owner("fsharptools")]
+ [TestCategory("nightly"), TestCategory("Walkthrough")]
+ public void AddMissingFunKeyword()
+ {
+ var code = @"module Library
+
+let original = []
+let transformed = original |> List.map (x -> x)";
+
+ this.AssertCodeActionOffered(code, "->", "Add missing 'fun' keyword");
+ }
+
+ [RetryTestMethod]
+ [Timeout(ShortTimeoutMS)]
+ [Description("Verify the 'Add 'new' keyword' code fix is offered when constructing a disposable.")]
+ [Owner("fsharptools")]
+ [TestCategory("nightly"), TestCategory("Walkthrough")]
+ public void AddNewKeywordToDisposables()
+ {
+ var code = @"module Library
+
+let sr = System.IO.StreamReader("""")";
+
+ this.AssertCodeActionOffered(code, "let sr", "Add 'new' keyword");
+ }
+ }
+}
diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/CodeFixes/SuggestionCodeFix.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/CodeFixes/SuggestionCodeFix.cs
new file mode 100644
index 00000000000..a9a330290fb
--- /dev/null
+++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/CodeFixes/SuggestionCodeFix.cs
@@ -0,0 +1,60 @@
+// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information.
+//
+// Ported verbatim from the TypeScript-VS repository
+// (VS/LanguageService/Tests/Integration/CodeFixes/SuggestionCodeFix.cs) and then adjusted so that
+// the setup is tailored for the F# extension. A single test case is kept, exercising an F# code fix
+// (Wrap expression in parentheses, FS0597). The triggering code is taken from the F# code-fix unit
+// tests: vsintegration/tests/FSharp.Editor.Tests/CodeFixes/WrapExpressionInParenthesesTests.fs.
+
+using System;
+using System.Linq;
+using FSharp.Editor.Apex.IntegrationTests.TestFramework;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace FSharp.Editor.Apex.IntegrationTests
+{
+ [TestClass]
+ public class SuggestionCodeFix : FSharpLanguageServiceApexTest
+ {
+ #region Private members
+ protected override bool AutomaticallyDismissMessageBoxes => false;
+ private const string code = @"module Library
+
+let rng = System.Random()
+
+printfn ""Hello %d"" rng.Next(5)";
+ private const string expectedCode = @"module Library
+
+let rng = System.Random()
+
+printfn ""Hello %d"" (rng.Next(5))";
+ #endregion
+
+ [RetryTestMethod]
+ [Timeout(ShortTimeoutMS)]
+ [Description("Verify code-fix offers suggestion to wrap an expression in parentheses in an F# file.")]
+ [Owner("fsharptools")]
+ [TestCategory("nightly"), TestCategory("Walkthrough")]
+ public void SuggestionCodeFixInFSharpFile()
+ {
+ var document = this.OpenFSharpDocument(code, "rng.Next");
+
+ this.InvokeCodeFix(document, "rng.Next");
+ this.Library.Synchronization.WaitForSolutionCrawler();
+
+ this.Library.Synchronization.WaitFor(() => expectedCode == document.Contents, TimeSpan.FromSeconds(15));
+ }
+
+ #region Helps
+ public void InvokeCodeFix(TextDocumentView document, string expression)
+ {
+ var lightBulb = this.MoveToExpressionAndExpandLightBulb(document, expression);
+ var globalScopeAction = lightBulb.Actions.Single(action =>
+ action.Text.IndexOf("Wrap", StringComparison.OrdinalIgnoreCase) >= 0
+ );
+
+ globalScopeAction.Invoke();
+ }
+ #endregion
+ }
+}
diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/FSharp.Editor.Apex.IntegrationTests.csproj b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/FSharp.Editor.Apex.IntegrationTests.csproj
new file mode 100644
index 00000000000..924a74297f4
--- /dev/null
+++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/FSharp.Editor.Apex.IntegrationTests.csproj
@@ -0,0 +1,41 @@
+
+
+
+
+ net472
+ preview
+ disable
+ Library
+ false
+
+ true
+ $(NoWarn);CS1591;VSTHRD200
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/FSharpLanguageServiceApexTest.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/FSharpLanguageServiceApexTest.cs
new file mode 100644
index 00000000000..b118afcfd9b
--- /dev/null
+++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/FSharpLanguageServiceApexTest.cs
@@ -0,0 +1,225 @@
+//-----------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+//-----------------------------------------------------------------------------
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Web.Script.Serialization;
+using Microsoft.Test.Apex.Editor;
+using Microsoft.Test.Apex.VisualStudio;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace FSharp.Editor.Apex.IntegrationTests.TestFramework
+{
+ ///
+ /// Base test class for F# language service tests built using the Apex framework.
+ /// Compact, self-contained equivalent of the TypeScript-VS LanguageServiceApexTest, tailored for
+ /// the F# extension and without the JavaScript/TypeScript BrowserTools layer.
+ ///
+ [TestClass]
+ public abstract class FSharpLanguageServiceApexTest : VisualStudioHostTest
+ {
+ protected const int ShortTimeoutMS = 1000 * 60 * 12;
+ protected const int LongTimeoutMS = 1000 * 60 * 20;
+
+ // Apex launches the Visual Studio instance pointed to by this environment variable, if set.
+ private const string InstallationUnderTestPathVariable = "VisualStudio.InstallationUnderTest.Path";
+
+ private FSharpLanguageServiceLibrary libraryCache;
+
+ ///
+ /// Gets the F# language service test library.
+ ///
+ public FSharpLanguageServiceLibrary Library
+ => this.libraryCache ??= new FSharpLanguageServiceLibrary(this.VisualStudio, this.Operations);
+
+ ///
+ /// Whether message boxes shown during the test run should be dismissed automatically.
+ ///
+ protected virtual bool AutomaticallyDismissMessageBoxes => true;
+
+ ///
+ /// The registry root suffix (experimental hive) of the Visual Studio instance to launch.
+ /// Defaults to "RoslynDev", where the locally-built F# extension is deployed.
+ ///
+ protected virtual string RootSuffix => "RoslynDev";
+
+ ///
+ /// The product milestone of the installed Visual Studio to launch, as reported by vswhere's
+ /// catalog.productMilestone. Defaults to "Canary" (the IntCanary channel), which selects the
+ /// Canary install rather than Insiders when both are present.
+ ///
+ protected virtual string TargetProductMilestone => "Canary";
+
+ protected override VisualStudioHostConfiguration GetVisualStudioHostConfiguration()
+ {
+ this.EnsureTargetInstallationSelected();
+
+ var config = base.GetVisualStudioHostConfiguration();
+ config.AutomaticallyDismissMessageBoxes = this.AutomaticallyDismissMessageBoxes;
+ config.RootSuffix = this.RootSuffix;
+ return config;
+ }
+
+ ///
+ /// Points Apex at the Visual Studio install matching
+ /// (e.g. Canary). An explicit VisualStudio.InstallationUnderTest.Path override is respected.
+ ///
+ private void EnsureTargetInstallationSelected()
+ {
+ if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable(InstallationUnderTestPathVariable)))
+ {
+ return;
+ }
+
+ string devenvPath = ResolveDevenvByMilestone(this.TargetProductMilestone);
+ if (!string.IsNullOrEmpty(devenvPath))
+ {
+ Environment.SetEnvironmentVariable(InstallationUnderTestPathVariable, devenvPath);
+ }
+ }
+
+ private static string ResolveDevenvByMilestone(string milestone)
+ {
+ string programFilesX86 = Environment.GetFolderPath(Environment.SpecialFolder.ProgramFilesX86);
+ string vswhere = Path.Combine(programFilesX86, "Microsoft Visual Studio", "Installer", "vswhere.exe");
+ if (!File.Exists(vswhere))
+ {
+ return null;
+ }
+
+ var startInfo = new ProcessStartInfo(vswhere, "-all -prerelease -format json -utf8")
+ {
+ RedirectStandardOutput = true,
+ UseShellExecute = false,
+ CreateNoWindow = true,
+ };
+
+ string json;
+ using (var process = Process.Start(startInfo))
+ {
+ json = process.StandardOutput.ReadToEnd();
+ process.WaitForExit();
+ }
+
+ if (string.IsNullOrWhiteSpace(json)
+ || !(new JavaScriptSerializer { MaxJsonLength = int.MaxValue }.DeserializeObject(json) is object[] installations))
+ {
+ return null;
+ }
+
+ foreach (var installation in installations.OfType>())
+ {
+ if (installation.TryGetValue("catalog", out var catalogObject)
+ && catalogObject is Dictionary catalog
+ && catalog.TryGetValue("productMilestone", out var installedMilestone)
+ && string.Equals(installedMilestone as string, milestone, StringComparison.OrdinalIgnoreCase)
+ && installation.TryGetValue("productPath", out var productPath))
+ {
+ return productPath as string;
+ }
+ }
+
+ return null;
+ }
+
+ ///
+ /// Moves the caret onto , triggers the code-fix request and
+ /// expands the resulting light bulb.
+ ///
+ protected ILightBulbTestExtension MoveToExpressionAndExpandLightBulb(TextDocumentView document, string expression)
+ {
+ // Nudging the caret into the diagnostic range triggers a request for code fixes.
+ document.MoveToExpression(expression);
+ document.MoveRight();
+ document.MoveLeft();
+
+ this.Library.Synchronization.WaitForLightBulb();
+
+ Assert.IsTrue(
+ this.Library.Editor.LightBulb.Verify.IsLightBulbPresent(TimeSpan.FromSeconds(60)),
+ $"Expected a light bulb to be present at expression '{expression}'.");
+
+ var lightBulb = this.Library.Editor.LightBulb.GetActiveLightBulb();
+ lightBulb.Expand();
+
+ // Wait for the suggested actions to populate. Poll the already-expanded session directly:
+ // re-querying GetActiveLightBulb() here returns null once the light bulb is expanded into
+ // its flyout (LightBulbBroker no longer reports it as the active session), and the flyout
+ // taking focus can also null out Library.Editor (ActiveDocumentWindowAsTextEditor) — either
+ // of which previously threw a NullReferenceException inside the poll. LightBulbTestExtension
+ // .Actions reads the captured session and returns an empty (never null) set while pending.
+ this.Library.Synchronization.TryWaitForCondition(() => lightBulb.Actions.Any());
+
+ return lightBulb;
+ }
+
+ ///
+ /// Creates a fresh single-project F# solution whose Library.fs contains
+ /// and opens it in the editor. Waits for the project to finish loading before editing (so the
+ /// F# project system does not reload the empty file over the insert) and confirms
+ /// reached the buffer before returning, so downstream caret
+ /// searches are not racing document initialization.
+ ///
+ protected TextDocumentView OpenFSharpDocument(string code, string expectedContent)
+ {
+ var project = this.Library.ProjectCreation.CreateFSharpLibrary();
+ var documentItem = this.Library.ProjectCreation.AddProjectItemFromEmptyFile(project, "Library.fs");
+ var document = this.Library.OpenDocument(documentItem);
+
+ this.Library.Synchronization.WaitForSolutionCrawler();
+
+ document.InsertText(code);
+
+ Assert.IsTrue(
+ this.Library.Synchronization.TryWaitForCondition(
+ () => document.Contents.Contains(expectedContent), TimeSpan.FromSeconds(15)),
+ $"Inserted source did not appear in the document buffer (looking for '{expectedContent}').");
+
+ return document;
+ }
+
+ ///
+ /// Builds the current solution and waits for it to finish. Building performs a NuGet restore and
+ /// resolves references — which the F# language service needs before it can type-check and surface
+ /// SEMANTIC diagnostics (e.g. the unused-open analyzer, or FS0760 for disposables). This is the
+ /// Apex counterpart of the source IdeFact tests' RestoreNuGetPackagesAsync + WaitForProjectSystem.
+ /// Parse-based fixes (FS0597, FS0010) don't need it. Build success is not asserted: some fixture
+ /// sources intentionally contain errors, and a failed compile still restores references.
+ ///
+ protected void BuildSolution()
+ {
+ this.Library.VisualStudio.ObjectModel.Solution.BuildManager.Build(waitForBuildToFinish: true);
+ this.Library.Synchronization.WaitForSolutionCrawler();
+ }
+
+ ///
+ /// Verifies that placing the caret on in the given F# source
+ /// offers exactly one light-bulb action whose text contains .
+ /// Mirrors the display-text assertions in FSharp.Editor.IntegrationTests CodeActionTests (the Apex
+ /// light-bulb model exposes a flat action list, so the code-fix/error-fix category is not checked).
+ ///
+ protected void AssertCodeActionOffered(string code, string caretExpression, string expectedActionText)
+ {
+ var document = this.OpenFSharpDocument(code, caretExpression);
+
+ // Restore/resolve references so the F# language service can type-check and offer semantic
+ // fixes (the unused-open analyzer, FS0760, ...); without this their light bulb never appears.
+ this.BuildSolution();
+
+ var lightBulb = this.MoveToExpressionAndExpandLightBulb(document, caretExpression);
+
+ var matchCount = lightBulb.Actions.Count(action =>
+ action.Text.IndexOf(expectedActionText, StringComparison.OrdinalIgnoreCase) >= 0);
+
+ Assert.AreEqual(
+ 1,
+ matchCount,
+ $"Expected exactly one code action containing '{expectedActionText}'. Offered actions: " +
+ string.Join(", ", lightBulb.Actions.Select(action => $"'{action.Text}'")));
+ }
+ }
+}
diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/FSharpLanguageServiceLibrary.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/FSharpLanguageServiceLibrary.cs
new file mode 100644
index 00000000000..562b5e53926
--- /dev/null
+++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/FSharpLanguageServiceLibrary.cs
@@ -0,0 +1,50 @@
+//-----------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+//-----------------------------------------------------------------------------
+
+using Microsoft.Test.Apex;
+using Microsoft.Test.Apex.Services;
+using Microsoft.Test.Apex.VisualStudio;
+using Microsoft.Test.Apex.VisualStudio.Editor;
+using Microsoft.Test.Apex.VisualStudio.Shell;
+using Microsoft.Test.Apex.VisualStudio.Solution;
+
+namespace FSharp.Editor.Apex.IntegrationTests.TestFramework
+{
+ ///
+ /// Compact test library for the F# Apex integration tests. Provides the small facade over the
+ /// Apex that the code-action tests use (project creation,
+ /// synchronization, the active editor and document opening), tailored for the F# extension.
+ ///
+ public sealed class FSharpLanguageServiceLibrary
+ {
+ public FSharpLanguageServiceLibrary(VisualStudioHost visualStudio, IOperations operations)
+ {
+ this.VisualStudio = visualStudio;
+ this.Synchronization = new SynchronizationHelper(visualStudio, operations.Get());
+ this.ProjectCreation = new ProjectCreationHelper(visualStudio);
+ }
+
+ /// The Apex Visual Studio host.
+ public VisualStudioHost VisualStudio { get; }
+
+ /// Synchronization helpers used to wait for background work.
+ public SynchronizationHelper Synchronization { get; }
+
+ /// F# project and project-item creation helpers.
+ public ProjectCreationHelper ProjectCreation { get; }
+
+ /// The editor of the active document window, or null if none is active.
+ public IVisualStudioTextEditorTestExtension Editor
+ => this.VisualStudio.ObjectModel.WindowManager.ActiveDocumentWindowAsTextEditor?.Editor;
+
+ ///
+ /// Opens the given project item in the text editor and returns a view over it.
+ ///
+ public TextDocumentView OpenDocument(ProjectItemTestExtension documentItem)
+ {
+ var window = documentItem.Open();
+ return new TextDocumentView(window.Editor);
+ }
+ }
+}
diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/ProjectCreationHelper.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/ProjectCreationHelper.cs
new file mode 100644
index 00000000000..1dbe473ec17
--- /dev/null
+++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/ProjectCreationHelper.cs
@@ -0,0 +1,49 @@
+//-----------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+//-----------------------------------------------------------------------------
+
+using System.IO;
+using Microsoft.Test.Apex.VisualStudio;
+using Microsoft.Test.Apex.VisualStudio.Solution;
+
+namespace FSharp.Editor.Apex.IntegrationTests.TestFramework
+{
+ ///
+ /// Creates F# projects and project items for the Apex integration tests.
+ /// Tailored for the F# extension: the TypeScript-VS ProjectCreationHelper created ASP.NET Core
+ /// web apps for JavaScript/TypeScript; this one creates plain F# class libraries.
+ ///
+ public sealed class ProjectCreationHelper
+ {
+ private readonly VisualStudioHost visualStudio;
+ private int projectCounter = 0;
+
+ public ProjectCreationHelper(VisualStudioHost visualStudio)
+ {
+ this.visualStudio = visualStudio;
+ }
+
+ ///
+ /// Creates a new F# class library in a fresh solution.
+ ///
+ public ProjectTestExtension CreateFSharpLibrary()
+ {
+ string projectName = $"FSharpLibrary_{++this.projectCounter}";
+ return this.visualStudio.ObjectModel.Solution.CreateProject(
+ ProjectLanguage.FSharp,
+ ProjectTemplate.ClassLibrary,
+ projectName);
+ }
+
+ ///
+ /// Creates an empty file on disk and adds it to the given project or project item.
+ ///
+ public ProjectItemTestExtension AddProjectItemFromEmptyFile(IProjectItemsContainer container, string fileName)
+ {
+ string containerDirectory = container.IsProject ? container.ProjectDirectory : container.FullPath;
+ string filePath = Path.Combine(containerDirectory, fileName);
+ File.WriteAllText(filePath, string.Empty);
+ return container.AddProjectItemFromFile(filePath);
+ }
+ }
+}
diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/RetryTestMethodAttribute.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/RetryTestMethodAttribute.cs
new file mode 100644
index 00000000000..a5f834042ec
--- /dev/null
+++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/RetryTestMethodAttribute.cs
@@ -0,0 +1,62 @@
+// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information.
+
+using System;
+using System.Diagnostics;
+using System.Linq;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace FSharp.Editor.Apex.IntegrationTests.TestFramework
+{
+ ///
+ /// Drop-in replacement for that re-runs a test up to
+ /// maxAttempts times, passing as soon as one attempt passes. These Apex tests drive a real
+ /// Visual Studio through UI automation and are occasionally flaky (focus, background analysis and
+ /// build/restore timing); a bounded retry keeps them reliable without masking a genuine failure —
+ /// every attempt has to fail before the test is reported as failed.
+ ///
+ /// Each attempt goes through the full per-test pipeline (a fresh test-class instance plus
+ /// TestInitialize/TestCleanup, i.e. a fresh VS session for the Apex host), so a retry starts from a
+ /// clean state rather than reusing the dirtied state of the previous attempt.
+ ///
+ /// MSTest's own [Retry] attribute is only available from 3.8; this repo pins 3.6.4, so the
+ /// classic "derive from TestMethodAttribute and override Execute" idiom is used instead.
+ ///
+ [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
+ public sealed class RetryTestMethodAttribute : TestMethodAttribute
+ {
+ private readonly int maxAttempts;
+
+ public RetryTestMethodAttribute(int maxAttempts = 2)
+ {
+ if (maxAttempts < 1)
+ {
+ throw new ArgumentOutOfRangeException(nameof(maxAttempts), maxAttempts, "At least one attempt is required.");
+ }
+
+ this.maxAttempts = maxAttempts;
+ }
+
+ public override TestResult[] Execute(ITestMethod testMethod)
+ {
+ TestResult[] results = null;
+
+ for (var attempt = 1; attempt <= this.maxAttempts; attempt++)
+ {
+ results = base.Execute(testMethod);
+
+ if (results.All(result => result.Outcome == UnitTestOutcome.Passed))
+ {
+ return results;
+ }
+
+ if (attempt < this.maxAttempts)
+ {
+ Trace.WriteLine(
+ $"[RetryTestMethod] '{testMethod.TestMethodName}' failed on attempt {attempt} of {this.maxAttempts}; retrying.");
+ }
+ }
+
+ return results;
+ }
+ }
+}
diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/SynchronizationHelper.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/SynchronizationHelper.cs
new file mode 100644
index 00000000000..0e37b1d8c1a
--- /dev/null
+++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/SynchronizationHelper.cs
@@ -0,0 +1,64 @@
+//-----------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+//-----------------------------------------------------------------------------
+
+using System;
+using Microsoft.Test.Apex.Services;
+using Microsoft.Test.Apex.VisualStudio;
+
+namespace FSharp.Editor.Apex.IntegrationTests.TestFramework
+{
+ ///
+ /// Synchronization helper for the F# Apex integration tests.
+ /// This is a compact, self-contained equivalent of the TypeScript-VS SynchronizationHelper:
+ /// where that type delegated to the Roslyn "WaitForFeatures" async-operation waiter, this one
+ /// polls via the Apex so the harness has no dependency on
+ /// the JavaScript/TypeScript BrowserTools layer.
+ ///
+ public sealed class SynchronizationHelper
+ {
+ private static readonly TimeSpan DefaultTimeout = TimeSpan.FromSeconds(30);
+ private static readonly TimeSpan DefaultInterval = TimeSpan.FromSeconds(1);
+
+ private readonly VisualStudioHost visualStudio;
+ private readonly ISynchronizationService synchronizationService;
+
+ public SynchronizationHelper(VisualStudioHost visualStudio, ISynchronizationService synchronizationService)
+ {
+ this.visualStudio = visualStudio;
+ this.synchronizationService = synchronizationService;
+ }
+
+ ///
+ /// Waits for the solution to be fully loaded and for background analysis to settle.
+ ///
+ public void WaitForSolutionCrawler()
+ {
+ this.visualStudio.ObjectModel.Solution.WaitForFullyLoaded();
+ this.Settle();
+ }
+
+ ///
+ /// Waits for the light bulb / code-action pipeline to settle before the light bulb is queried.
+ ///
+ public void WaitForLightBulb() => this.Settle();
+
+ ///
+ /// Blocks until returns true or the timeout elapses.
+ ///
+ public void WaitFor(Func condition, TimeSpan timeout)
+ => this.TryWaitForCondition(condition, timeout);
+
+ ///
+ /// Polls until it returns true or the timeout elapses.
+ ///
+ /// True if the condition became true within the timeout; otherwise false.
+ public bool TryWaitForCondition(Func condition, TimeSpan? timeout = null)
+ {
+ return this.synchronizationService.TryWaitFor(timeout ?? DefaultTimeout, () => condition());
+ }
+
+ private void Settle()
+ => this.synchronizationService.TryWaitFor(DefaultInterval, () => false);
+ }
+}
diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/TextDocumentView.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/TextDocumentView.cs
new file mode 100644
index 00000000000..de45a57c447
--- /dev/null
+++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/TextDocumentView.cs
@@ -0,0 +1,45 @@
+//-----------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+//-----------------------------------------------------------------------------
+
+using Microsoft.Test.Apex.VisualStudio.Editor;
+
+namespace FSharp.Editor.Apex.IntegrationTests.TestFramework
+{
+ ///
+ /// Represents the text view of an open VS document. Compact, self-contained equivalent of the
+ /// TypeScript-VS TextDocumentView / TextDocumentViewBase, wrapping the Apex text editor extension
+ /// with the small set of operations the code-action tests need.
+ ///
+ public sealed class TextDocumentView
+ {
+ private readonly IVisualStudioTextEditorTestExtension editor;
+
+ public TextDocumentView(IVisualStudioTextEditorTestExtension editor)
+ {
+ this.editor = editor;
+ }
+
+ /// The full text content of the document.
+ public string Contents => this.editor.Contents;
+
+ ///
+ /// Inserts text verbatim at the current caret position via a direct text-buffer edit.
+ /// Unlike InsertTextWithReturn, this does not simulate per-character typing, so the F#
+ /// editor's brace/quote auto-completion, IntelliSense auto-commit and smart indentation do not
+ /// fire and rewrite the buffer. That keeps the inserted source exactly as given, which the
+ /// code-fix tests rely on when locating expressions afterwards.
+ ///
+ public void InsertText(string text) => this.editor.Edit.InsertTextInBuffer(text);
+
+ /// Moves the caret to the first (or -th) occurrence of an expression.
+ public void MoveToExpression(string expression, int matchIndex = 1)
+ => this.editor.Caret.MoveToExpression(expression, matchIndex: matchIndex);
+
+ /// Moves the caret one character to the left.
+ public void MoveLeft() => this.editor.Caret.MoveLeft();
+
+ /// Moves the caret one character to the right.
+ public void MoveRight() => this.editor.Caret.MoveRight();
+ }
+}
From b6fdb0df6101cc73d1213352075e22cdabf35bbe Mon Sep 17 00:00:00 2001
From: Adam Boniecki <20281641+abonie@users.noreply.github.com>
Date: Wed, 15 Jul 2026 14:35:55 +0200
Subject: [PATCH 02/15] Copy other integration tests to apex
---
.../Build/BuildProjectTests.cs | 81 ++++++++++++++++
.../CreateProject/CreateProjectTests.cs | 82 ++++++++++++++++
.../GoToDefinition/GoToDefinitionTests.cs | 86 ++++++++++++++++
.../FSharpLanguageServiceApexTest.cs | 71 ++++++++++++++
.../FSharpLanguageServiceLibrary.cs | 34 ++++++-
.../TestFramework/ProjectCreationHelper.cs | 65 ++++++++++++-
.../TestFramework/TextDocumentView.cs | 97 ++++++++++++++++++-
7 files changed, 508 insertions(+), 8 deletions(-)
create mode 100644 vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/Build/BuildProjectTests.cs
create mode 100644 vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/CreateProject/CreateProjectTests.cs
create mode 100644 vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/GoToDefinition/GoToDefinitionTests.cs
diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/Build/BuildProjectTests.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/Build/BuildProjectTests.cs
new file mode 100644
index 00000000000..453cac96bb6
--- /dev/null
+++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/Build/BuildProjectTests.cs
@@ -0,0 +1,81 @@
+// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information.
+//
+// Apex counterparts of the xUnit/IdeFact tests in
+// vsintegration/tests/FSharp.Editor.IntegrationTests/BuildProjectTests.cs. The project is created from
+// the .NET SDK class-library template (via `dotnet new`, matching the source tests; Apex's
+// ProjectTemplate.ClassLibrary is the legacy .NET Framework template). The template auto-opens the
+// default source file, so the source is set through the open document and saved to disk (an
+// out-of-band disk write would be superseded by the still-open template buffer, and the compiler would
+// build the valid template instead of the intended code). Rather than parsing the "Build: 1 succeeded,
+// 0 failed ..." output-pane summary string, these assert on the idiomatic Apex build result:
+// BuildManager.Succeeded and BuildManager.Verify.HasFailedWithErrors for the error case.
+
+using System.IO;
+using FSharp.Editor.Apex.IntegrationTests.TestFramework;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace FSharp.Editor.Apex.IntegrationTests
+{
+ [TestClass]
+ public class BuildProjectTests : FSharpLanguageServiceApexTest
+ {
+ protected override bool AutomaticallyDismissMessageBoxes => false;
+
+ [RetryTestMethod]
+ [Timeout(ShortTimeoutMS)]
+ [Description("Verify a well-formed F# project builds successfully.")]
+ [Owner("fsharptools")]
+ [TestCategory("nightly"), TestCategory("Walkthrough")]
+ public void SuccessfulBuild()
+ {
+ var code = @"module Test
+
+let answer = 42";
+
+ this.SetLibraryContent(code);
+
+ Assert.IsTrue(this.BuildSolutionSucceeded(), "Build was expected to succeed.");
+ }
+
+ [RetryTestMethod]
+ [Timeout(ShortTimeoutMS)]
+ [Description("Verify an F# project with an incomplete binding fails to build with FS0010.")]
+ [Owner("fsharptools")]
+ [TestCategory("nightly"), TestCategory("Walkthrough")]
+ public void FailedBuild()
+ {
+ var code = @"module Test
+
+let answer =";
+
+ this.SetLibraryContent(code);
+
+ this.BuildSolution();
+
+ var buildManager = this.Library.VisualStudio.ObjectModel.Solution.BuildManager;
+ Assert.IsFalse(buildManager.Succeeded, "Build was expected to fail.");
+
+ // The Apex error-list verifier matches the error DESCRIPTION (the message text); the code
+ // 'FS0010' lives in a separate Code column, so match on the FS0010 message instead.
+ Assert.IsTrue(
+ buildManager.Verify.HasFailedWithErrors(
+ new[] { "Incomplete structured construct" }, alsoVerifyInOutputWindow: false),
+ "Build was expected to fail with the FS0010 'Incomplete structured construct' error.");
+ }
+
+ ///
+ /// Creates an F# class library and replaces its default Library.fs with
+ /// through the open document (saved to disk), verifying the file on disk matches before building.
+ ///
+ private void SetLibraryContent(string code)
+ {
+ var project = this.Library.ProjectCreation.CreateFSharpProjectFromSdkTemplate("classlib", "Library");
+ this.Library.Synchronization.WaitForSolutionCrawler();
+
+ var document = this.Library.OpenProjectFile(project, "Library.fs");
+ document.ReplaceAllAndSave(code);
+
+ AssertSourceEquals(code, File.ReadAllText(document.FilePath));
+ }
+ }
+}
diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/CreateProject/CreateProjectTests.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/CreateProject/CreateProjectTests.cs
new file mode 100644
index 00000000000..4ce393246a3
--- /dev/null
+++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/CreateProject/CreateProjectTests.cs
@@ -0,0 +1,82 @@
+// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information.
+//
+// Apex counterparts of the xUnit/IdeFact tests in
+// vsintegration/tests/FSharp.Editor.IntegrationTests/CreateProjectTests.cs. Each case creates an F#
+// project from a .NET SDK template (via `dotnet new`, matching the SDK templates the source tests
+// target — Apex's ProjectTemplate enum maps to the legacy .NET Framework F# templates with different
+// filenames and content) and asserts the auto-generated default source.
+
+using FSharp.Editor.Apex.IntegrationTests.TestFramework;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace FSharp.Editor.Apex.IntegrationTests
+{
+ [TestClass]
+ public class CreateProjectTests : FSharpLanguageServiceApexTest
+ {
+ protected override bool AutomaticallyDismissMessageBoxes => false;
+
+ [RetryTestMethod]
+ [Timeout(ShortTimeoutMS)]
+ [Description("Verify the default source of a new F# class library project.")]
+ [Owner("fsharptools")]
+ [TestCategory("nightly"), TestCategory("Walkthrough")]
+ public void ClassLibrary()
+ {
+ var expectedCode = @"namespace Library
+
+module Say =
+ let hello name =
+ printfn ""Hello %s"" name";
+
+ var project = this.Library.ProjectCreation.CreateFSharpProjectFromSdkTemplate("classlib", "Library");
+ this.Library.Synchronization.WaitForSolutionCrawler();
+
+ var document = this.Library.OpenProjectFile(project, "Library.fs");
+
+ AssertSourceEquals(expectedCode, document.Contents);
+ }
+
+ [RetryTestMethod]
+ [Timeout(ShortTimeoutMS)]
+ [Description("Verify the default source of a new F# console application project.")]
+ [Owner("fsharptools")]
+ [TestCategory("nightly"), TestCategory("Walkthrough")]
+ public void ConsoleApp()
+ {
+ var expectedCode = @"// For more information see https://aka.ms/fsharp-console-apps
+printfn ""Hello from F#""";
+
+ var project = this.Library.ProjectCreation.CreateFSharpProjectFromSdkTemplate("console", "ConsoleApp");
+ this.Library.Synchronization.WaitForSolutionCrawler();
+
+ var document = this.Library.OpenProjectFile(project, "Program.fs");
+
+ AssertSourceEquals(expectedCode, document.Contents);
+ }
+
+ [RetryTestMethod]
+ [Timeout(ShortTimeoutMS)]
+ [Description("Verify the default source of a new F# xUnit test project.")]
+ [Owner("fsharptools")]
+ [TestCategory("nightly"), TestCategory("Walkthrough")]
+ public void XUnitTestProject()
+ {
+ var expectedCode = @"module Tests
+
+open System
+open Xunit
+
+[]
+let ``My test`` () =
+ Assert.True(true)";
+
+ var project = this.Library.ProjectCreation.CreateFSharpProjectFromSdkTemplate("xunit", "Tests");
+ this.Library.Synchronization.WaitForSolutionCrawler();
+
+ var document = this.Library.OpenProjectFile(project, "Tests.fs");
+
+ AssertSourceEquals(expectedCode, document.Contents);
+ }
+ }
+}
diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/GoToDefinition/GoToDefinitionTests.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/GoToDefinition/GoToDefinitionTests.cs
new file mode 100644
index 00000000000..5eff72aa107
--- /dev/null
+++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/GoToDefinition/GoToDefinitionTests.cs
@@ -0,0 +1,86 @@
+// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information.
+//
+// Apex counterparts of the xUnit/IdeFact tests in
+// vsintegration/tests/FSharp.Editor.IntegrationTests/GoToDefinitionTests.cs. Go To Definition is
+// invoked idiomatically via the editor caret (IVisualStudioCaretTestExtension.GoToDefinition), and the
+// result is read from the active document (current line + window caption) rather than the async
+// SolutionExplorer/Editor/Shell helpers used by the xUnit harness.
+
+using FSharp.Editor.Apex.IntegrationTests.TestFramework;
+using Microsoft.Test.Apex.VisualStudio.Solution;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace FSharp.Editor.Apex.IntegrationTests
+{
+ [TestClass]
+ public class GoToDefinitionTests : FSharpLanguageServiceApexTest
+ {
+ protected override bool AutomaticallyDismissMessageBoxes => false;
+
+ [RetryTestMethod]
+ [Timeout(ShortTimeoutMS)]
+ [Description("Verify Go To Definition navigates from a use of a binding to its definition.")]
+ [Owner("fsharptools")]
+ [TestCategory("nightly"), TestCategory("Walkthrough")]
+ public void GoesToDefinition()
+ {
+ var code = @"module Test
+
+let add x y = x + y
+
+let increment = add 1";
+
+ var document = this.OpenFSharpDocument(code, "add 1");
+
+ // Resolve references so the file type-checks and Go To Definition can bind the symbol.
+ this.BuildSolution();
+
+ this.GoToDefinitionAndWait(document, "add 1", line => line.Contains("let add x y = x + y"));
+ }
+
+ [RetryTestMethod]
+ [Timeout(ShortTimeoutMS)]
+ [Description("Verify Go To Definition stays within the signature file and the implementation file respectively.")]
+ [Owner("fsharptools")]
+ [TestCategory("nightly"), TestCategory("Walkthrough")]
+ public void FsiAndFsFilesGoToCorrespondentDefinitions()
+ {
+ var fsi = @"module Module
+
+type SomeType =
+| Number of int
+| Letter of char
+
+val id: t: SomeType -> SomeType";
+
+ var fs = @"module Module
+
+type SomeType =
+ | Number of int
+ | Letter of char
+
+let id (t: SomeType) = t";
+
+ var project = this.Library.ProjectCreation.CreateFSharpProject(ProjectTemplate.ClassLibrary, "Library");
+
+ // The signature file must precede the implementation file in F# compile order. When a file is
+ // added, VS inserts it alphabetically, so adding "Module.fsi" then "Module.fs" would place the
+ // implementation first ("Module.fs" sorts before "Module.fsi") and the signature would not
+ // apply. Add the signature under a name that sorts first, then rename it — the same approach as
+ // the source IdeFact test.
+ var fsiItem = this.Library.ProjectCreation.AddProjectItemFromContent(project, "AModule.fsi", fsi);
+ this.Library.ProjectCreation.AddProjectItemFromContent(project, "Module.fs", fs);
+ fsiItem.Rename("Module.fsi");
+ this.Library.Synchronization.WaitForSolutionCrawler();
+ this.BuildSolution();
+
+ // From the signature file, Go To Definition on the type usage stays in Module.fsi.
+ var fsiDocument = this.Library.OpenProjectFile(project, "Module.fsi");
+ this.GoToDefinitionAndWait(fsiDocument, "SomeType ->", line => line.Trim() == "type SomeType =", "Module.fsi");
+
+ // From the implementation file, Go To Definition on the type usage stays in Module.fs.
+ var fsDocument = this.Library.OpenProjectFile(project, "Module.fs");
+ this.GoToDefinitionAndWait(fsDocument, "SomeType)", line => line.Trim() == "type SomeType =", "Module.fs");
+ }
+ }
+}
diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/FSharpLanguageServiceApexTest.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/FSharpLanguageServiceApexTest.cs
index b118afcfd9b..2a3b53a2950 100644
--- a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/FSharpLanguageServiceApexTest.cs
+++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/FSharpLanguageServiceApexTest.cs
@@ -196,6 +196,77 @@ protected void BuildSolution()
this.Library.Synchronization.WaitForSolutionCrawler();
}
+ ///
+ /// Builds the current solution and returns whether it succeeded.
+ ///
+ protected bool BuildSolutionSucceeded()
+ {
+ this.BuildSolution();
+ return this.Library.VisualStudio.ObjectModel.Solution.BuildManager.Succeeded;
+ }
+
+ ///
+ /// Places the caret on , focuses the editor and invokes Go To
+ /// Definition, then waits for navigation to settle. The result is read from the active document
+ /// afterwards (which may be the same or a different file).
+ ///
+ protected void GoToDefinition(TextDocumentView document, string expression)
+ {
+ document.MoveToExpression(expression);
+ document.Focus();
+ document.GoToDefinition();
+ this.Library.Synchronization.WaitForSolutionCrawler();
+ }
+
+ ///
+ /// Invokes Go To Definition on and polls until the active document
+ /// settles on a line matching and, when given, a window caption
+ /// equal to . Go To Definition navigates asynchronously (it may
+ /// open/activate another document and only then move the caret), so — exactly like the light-bulb
+ /// helper waits for its actions — we poll for the end state instead of reading it immediately.
+ /// Reading it immediately is the fire-and-assert race that makes the ported navigation tests flaky.
+ ///
+ protected void GoToDefinitionAndWait(
+ TextDocumentView document,
+ string expression,
+ Func lineMatches,
+ string expectedCaption = null)
+ {
+ this.GoToDefinition(document, expression);
+
+ string lastLine = null;
+ string lastCaption = null;
+ bool landed = this.Library.Synchronization.TryWaitForCondition(
+ () =>
+ {
+ lastLine = this.Library.ActiveDocumentCurrentLineText;
+ lastCaption = this.Library.ActiveDocumentCaption;
+ return lastLine != null
+ && lineMatches(lastLine)
+ && (expectedCaption == null || string.Equals(lastCaption, expectedCaption, StringComparison.Ordinal));
+ },
+ TimeSpan.FromSeconds(30));
+
+ Assert.IsTrue(
+ landed,
+ $"Go To Definition on '{expression}' did not settle on the expected location. "
+ + (expectedCaption != null ? $"Expected caption '{expectedCaption}', actual '{lastCaption}'. " : string.Empty)
+ + $"Actual current line: '{lastLine}'.");
+ }
+
+ ///
+ /// Asserts two pieces of F# source are equal, ignoring line-ending style and any trailing
+ /// newline. Template output and editor buffers differ in those incidental ways across SDKs, so
+ /// normalizing avoids brittle failures while still comparing the meaningful content exactly.
+ ///
+ protected static void AssertSourceEquals(string expected, string actual)
+ {
+ static string Normalize(string source)
+ => (source ?? string.Empty).Replace("\r\n", "\n").Replace("\r", "\n").TrimEnd('\n');
+
+ Assert.AreEqual(Normalize(expected), Normalize(actual));
+ }
+
///
/// Verifies that placing the caret on in the given F# source
/// offers exactly one light-bulb action whose text contains .
diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/FSharpLanguageServiceLibrary.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/FSharpLanguageServiceLibrary.cs
index 562b5e53926..a3b98ab410f 100644
--- a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/FSharpLanguageServiceLibrary.cs
+++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/FSharpLanguageServiceLibrary.cs
@@ -2,6 +2,8 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
//-----------------------------------------------------------------------------
+using System;
+using System.Linq;
using Microsoft.Test.Apex;
using Microsoft.Test.Apex.Services;
using Microsoft.Test.Apex.VisualStudio;
@@ -44,7 +46,37 @@ public IVisualStudioTextEditorTestExtension Editor
public TextDocumentView OpenDocument(ProjectItemTestExtension documentItem)
{
var window = documentItem.Open();
- return new TextDocumentView(window.Editor);
+ return new TextDocumentView(window);
}
+
+ ///
+ /// Finds under , opens it in the text
+ /// editor and returns a view over it. The project's item tree is populated lazily, so the lookup
+ /// is polled (TryFindChild is single-shot); on timeout it throws listing the project's actual
+ /// top-level item names, which makes a template-filename surprise obvious.
+ ///
+ public TextDocumentView OpenProjectFile(ProjectTestExtension project, string fileName)
+ {
+ ProjectItemTestExtension item = null;
+ this.Synchronization.TryWaitForCondition(
+ () => (item = project.TryFindChild(fileName, true)) != null);
+
+ if (item == null)
+ {
+ var actualNames = string.Join(", ", project.ProjectItems.Select(child => $"'{child.Name}'"));
+ throw new InvalidOperationException(
+ $"Project item '{fileName}' was not found in project '{project.Name}'. Items present: {actualNames}.");
+ }
+
+ return this.OpenDocument(item);
+ }
+
+ /// The short caption (file name) of the active document window, or null if none is active.
+ public string ActiveDocumentCaption
+ => this.VisualStudio.ObjectModel.WindowManager.ActiveDocumentWindow?.Caption;
+
+ /// The text of the caret's current line in the active document editor, or null if none is active.
+ public string ActiveDocumentCurrentLineText
+ => this.Editor?.Caret.GetCurrentLineText();
}
}
diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/ProjectCreationHelper.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/ProjectCreationHelper.cs
index 1dbe473ec17..6961cca20c2 100644
--- a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/ProjectCreationHelper.cs
+++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/ProjectCreationHelper.cs
@@ -2,6 +2,8 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
//-----------------------------------------------------------------------------
+using System;
+using System.Diagnostics;
using System.IO;
using Microsoft.Test.Apex.VisualStudio;
using Microsoft.Test.Apex.VisualStudio.Solution;
@@ -29,10 +31,37 @@ public ProjectCreationHelper(VisualStudioHost visualStudio)
public ProjectTestExtension CreateFSharpLibrary()
{
string projectName = $"FSharpLibrary_{++this.projectCounter}";
- return this.visualStudio.ObjectModel.Solution.CreateProject(
+ return this.CreateFSharpProject(ProjectTemplate.ClassLibrary, projectName);
+ }
+
+ ///
+ /// Creates a new F# project of the given built-in template with the given name.
+ ///
+ public ProjectTestExtension CreateFSharpProject(ProjectTemplate template, string projectName)
+ => this.visualStudio.ObjectModel.Solution.CreateProject(
ProjectLanguage.FSharp,
- ProjectTemplate.ClassLibrary,
+ template,
projectName);
+
+ ///
+ /// Creates a fresh single-project solution from an F# SDK template (e.g. "classlib", "console",
+ /// "xunit") scaffolded on disk with dotnet new. Apex's enum
+ /// maps to the legacy .NET Framework F# templates (which produce Library1.fs/Script.fsx), whereas
+ /// the source FSharp.Editor.IntegrationTests target the .NET SDK templates; using dotnet new
+ /// makes the project (filenames and default source) match those. An empty solution is created
+ /// first because AddProject throws when no solution is open.
+ ///
+ public ProjectTestExtension CreateFSharpProjectFromSdkTemplate(string sdkTemplateName, string projectName)
+ {
+ string projectDirectory = Path.Combine(
+ Path.GetTempPath(), "FSharpApexTemplates", Guid.NewGuid().ToString("N"), projectName);
+ Directory.CreateDirectory(projectDirectory);
+
+ RunDotNet($"new {sdkTemplateName} --language \"F#\" --name \"{projectName}\" --output \"{projectDirectory}\"");
+
+ string projectPath = Path.Combine(projectDirectory, projectName + ".fsproj");
+ this.visualStudio.ObjectModel.Solution.CreateEmptySolution();
+ return this.visualStudio.ObjectModel.Solution.AddProject(projectPath);
}
///
@@ -45,5 +74,37 @@ public ProjectItemTestExtension AddProjectItemFromEmptyFile(IProjectItemsContain
File.WriteAllText(filePath, string.Empty);
return container.AddProjectItemFromFile(filePath);
}
+
+ ///
+ /// Creates a file with the given content on disk and adds it to the given project or item.
+ ///
+ public ProjectItemTestExtension AddProjectItemFromContent(IProjectItemsContainer container, string fileName, string content)
+ {
+ string containerDirectory = container.IsProject ? container.ProjectDirectory : container.FullPath;
+ string filePath = Path.Combine(containerDirectory, fileName);
+ File.WriteAllText(filePath, content);
+ return container.AddProjectItemFromFile(filePath);
+ }
+
+ private static void RunDotNet(string arguments)
+ {
+ var startInfo = new ProcessStartInfo("dotnet", arguments)
+ {
+ UseShellExecute = false,
+ CreateNoWindow = true,
+ RedirectStandardOutput = true,
+ RedirectStandardError = true,
+ };
+
+ using var process = Process.Start(startInfo);
+ string standardError = process.StandardError.ReadToEnd();
+ process.StandardOutput.ReadToEnd();
+ process.WaitForExit();
+
+ if (process.ExitCode != 0)
+ {
+ throw new InvalidOperationException($"'dotnet {arguments}' failed with exit code {process.ExitCode}: {standardError}");
+ }
+ }
}
}
diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/TextDocumentView.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/TextDocumentView.cs
index de45a57c447..47e756ea87c 100644
--- a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/TextDocumentView.cs
+++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/TextDocumentView.cs
@@ -2,7 +2,11 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
//-----------------------------------------------------------------------------
+using System;
+using System.Text.RegularExpressions;
+using System.Threading;
using Microsoft.Test.Apex.VisualStudio.Editor;
+using Microsoft.Test.Apex.VisualStudio.Shell;
namespace FSharp.Editor.Apex.IntegrationTests.TestFramework
{
@@ -13,16 +17,21 @@ namespace FSharp.Editor.Apex.IntegrationTests.TestFramework
///
public sealed class TextDocumentView
{
- private readonly IVisualStudioTextEditorTestExtension editor;
+ private readonly TextEditorDocumentWindowTestExtension window;
- public TextDocumentView(IVisualStudioTextEditorTestExtension editor)
+ public TextDocumentView(TextEditorDocumentWindowTestExtension window)
{
- this.editor = editor;
+ this.window = window;
}
+ private IVisualStudioTextEditorTestExtension editor => this.window.Editor;
+
/// The full text content of the document.
public string Contents => this.editor.Contents;
+ /// The path of the file backing this document.
+ public string FilePath => this.window.FilePath;
+
///
/// Inserts text verbatim at the current caret position via a direct text-buffer edit.
/// Unlike InsertTextWithReturn, this does not simulate per-character typing, so the F#
@@ -32,14 +41,92 @@ public TextDocumentView(IVisualStudioTextEditorTestExtension editor)
///
public void InsertText(string text) => this.editor.Edit.InsertTextInBuffer(text);
- /// Moves the caret to the first (or -th) occurrence of an expression.
+ ///
+ /// Moves the caret to the first (or -th) occurrence of an
+ /// expression. Apex's caret search treats the argument as a regular expression, but every caller
+ /// passes a literal source fragment (which may contain regex metacharacters such as ')' in
+ /// "SomeType)"), so the input is escaped to match it literally.
+ ///
public void MoveToExpression(string expression, int matchIndex = 1)
- => this.editor.Caret.MoveToExpression(expression, matchIndex: matchIndex);
+ => this.editor.Caret.MoveToExpression(Regex.Escape(expression), matchIndex: matchIndex);
/// Moves the caret one character to the left.
public void MoveLeft() => this.editor.Caret.MoveLeft();
/// Moves the caret one character to the right.
public void MoveRight() => this.editor.Caret.MoveRight();
+
+ /// Gives this document's editor keyboard focus (needed before caret-driven commands).
+ public void Focus() => this.editor.Focus();
+
+ /// The text of the line the caret is currently on (untrimmed).
+ public string CurrentLineText => this.editor.Caret.GetCurrentLineText();
+
+ /// Invokes Go To Definition from the current caret position.
+ public void GoToDefinition() => this.editor.Caret.GoToDefinition();
+
+ ///
+ /// Replaces the entire document with and saves it to disk. Build tests
+ /// need this: the template auto-opens the default file, so the source must be set through the
+ /// open document (buffer) and saved, otherwise an out-of-band disk write is superseded by the
+ /// still-open (valid template) buffer and the compiler never sees the intended code.
+ ///
+ /// The template content loads into the buffer asynchronously after the document opens, so this
+ /// first waits for that initial content to arrive and then applies the whole-buffer replacement
+ /// with verify-and-retry: without this a replacement done too early is overwritten by the
+ /// late-arriving template content, leaving the original source on disk. The selection is deleted
+ /// before a direct buffer insert (which does not itself replace a selection and, unlike simulated
+ /// typing, avoids brace/quote auto-completion).
+ ///
+ public void ReplaceAllAndSave(string text)
+ {
+ this.editor.Focus();
+
+ if (!WaitUntil(() => !string.IsNullOrEmpty(this.editor.Contents)))
+ {
+ throw new InvalidOperationException("Document did not load its initial content before editing.");
+ }
+
+ var replaced = WaitUntil(() =>
+ {
+ this.editor.Selection.SelectAll();
+ this.editor.Edit.DeleteCharNext(1);
+ this.editor.Edit.InsertTextInBuffer(text);
+ return Normalize(this.editor.Contents) == Normalize(text);
+ });
+
+ if (!replaced)
+ {
+ throw new InvalidOperationException(
+ $"Failed to set document content. Expected:\n{text}\nActual:\n{this.editor.Contents}");
+ }
+
+ this.window.Save();
+
+ if (!this.window.TryWaitForNotDirty(TimeSpan.FromSeconds(15), TimeSpan.FromMilliseconds(100)))
+ {
+ throw new InvalidOperationException("Document remained dirty after saving.");
+ }
+ }
+
+ private static bool WaitUntil(Func condition)
+ {
+ var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(15);
+ do
+ {
+ if (condition())
+ {
+ return true;
+ }
+
+ Thread.Sleep(200);
+ }
+ while (DateTime.UtcNow < deadline);
+
+ return false;
+ }
+
+ private static string Normalize(string source)
+ => (source ?? string.Empty).Replace("\r\n", "\n").Replace("\r", "\n").TrimEnd('\n');
}
}
From 6c4ca07f9894506b2166e92f5863fd5cc7a9322a Mon Sep 17 00:00:00 2001
From: Adam Boniecki <20281641+abonie@users.noreply.github.com>
Date: Wed, 22 Jul 2026 21:39:14 +0200
Subject: [PATCH 03/15] Add orchestration for CI and fix missing assertion
---
azure-pipelines-PR.yml | 58 +++++++++++++++++
eng/Build.ps1 | 63 +++++++++++++++++++
.../CodeFixes/SuggestionCodeFix.cs | 9 ++-
.../FSharpLanguageServiceApexTest.cs | 23 +++++--
.../TestFramework/ProjectCreationHelper.cs | 11 +++-
.../TestFramework/SynchronizationHelper.cs | 6 --
6 files changed, 155 insertions(+), 15 deletions(-)
diff --git a/azure-pipelines-PR.yml b/azure-pipelines-PR.yml
index 827b97f33ac..e19d3824dad 100644
--- a/azure-pipelines-PR.yml
+++ b/azure-pipelines-PR.yml
@@ -509,6 +509,64 @@ stages:
continueOnError: true
condition: failed()
+ # Windows Apex VS integration tests (non-gating — published for signal only)
+ - job: WindowsApexIntegration
+ variables:
+ - name: _configuration
+ value: Release
+ - name: __VSNeverShowWhatsNew
+ value: 1
+ pool:
+ name: $(DncEngPublicBuildPool)
+ demands: ImageOverride -equals $(_WindowsMachineQueueName)
+ timeoutInMinutes: 90
+ steps:
+ - checkout: self
+ clean: true
+
+ - powershell: eng\SetupVSHive.ps1
+ displayName: Setup VS Hive
+
+ # Build + deploy the F# VSIX into the RoslynDev hive, then run the Apex tests. The Apex host
+ # resolves the installed VS via LocateVisualStudio (set as VisualStudio.InstallationUnderTest.Path
+ # by the -testApex path), so it does not depend on the local-only 'Canary' milestone.
+ # continueOnError keeps this leg non-gating: failures surface as signal but do not fail the PR.
+ - script: eng\CIBuildNoPublish.cmd -configuration $(_configuration) -deployExtensions -testApex
+ env:
+ NativeToolsOnMachine: true
+ displayName: Build, deploy extension and run Apex integration tests
+ continueOnError: true
+
+ - task: PublishTestResults@2
+ displayName: Publish Apex Test Results
+ inputs:
+ testResultsFormat: 'VSTest'
+ testRunTitle: WindowsApexIntegration
+ mergeTestResults: true
+ testResultsFiles: '*.trx'
+ searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_configuration)'
+ continueOnError: true
+ condition: succeededOrFailed()
+
+ - task: PublishBuildArtifacts@1
+ displayName: Publish Apex Test Logs
+ inputs:
+ PathtoPublish: '$(Build.SourcesDirectory)\artifacts\TestResults\$(_configuration)'
+ ArtifactName: Windows $(_configuration) Apex test logs
+ publishLocation: Container
+ continueOnError: true
+ condition: always()
+
+ - task: PublishBuildArtifacts@1
+ displayName: Publish Apex Build Logs
+ condition: always()
+ continueOnError: true
+ inputs:
+ PathToPublish: '$(Build.SourcesDirectory)\artifacts\log\$(_configuration)'
+ ArtifactName: Windows $(_configuration) Apex build logs
+ ArtifactType: Container
+ parallel: true
+
# Windows With Compressed Metadata Desktop (split into 3 batches)
- job: WindowsCompressedMetadata_Desktop
strategy:
diff --git a/eng/Build.ps1 b/eng/Build.ps1
index 41a52df6395..aa31b96efb6 100644
--- a/eng/Build.ps1
+++ b/eng/Build.ps1
@@ -60,6 +60,7 @@ param (
[switch]$testIntegration,
[switch]$testScripting,
[switch]$testVs,
+ [switch]$testApex,
[switch]$testAll,
[switch]$testAllButIntegration,
[switch]$testAllButIntegrationAndAot,
@@ -128,6 +129,7 @@ function Print-Usage() {
Write-Host " -testIntegration Run F# integration tests"
Write-Host " -testScripting Run Scripting tests"
Write-Host " -testVs Run F# editor unit tests"
+ Write-Host " -testApex Run F# Apex VS integration tests (requires -deployExtensions)"
Write-Host " -testpack Verify built packages"
Write-Host " -testAOT Run AOT/Trimming tests"
Write-Host " -testEditor Run VS Editor tests"
@@ -402,6 +404,63 @@ function TestUsingMSBuild([string] $testProject, [string] $targetFramework, [str
Exec-Console $dotnetExe $test_args
}
+# Runs the Apex VS integration tests. These are a classic MSTest (VSTest) project, NOT part of the
+# repo's Microsoft.Testing.Platform 'dotnet test' path, and they drive a real VS instance via the Apex
+# UI-automation framework — so they are launched with vstest.console.exe (from the installed VS) against
+# the already-built test assembly. The F# VSIX must be deployed into the RoslynDev hive first
+# (build with -deployExtensions); Apex launches devenv /rootsuffix RoslynDev.
+function TestApexUsingVSTest([string] $targetFramework) {
+ $projectName = "FSharp.Editor.Apex.IntegrationTests"
+ $testAssembly = Join-Path $ArtifactsDir "bin\$projectName\$configuration\$targetFramework\$projectName.dll"
+ if (-not (Test-Path $testAssembly)) {
+ throw "Apex test assembly not found at '$testAssembly'. Build the VS solution first."
+ }
+
+ $vsInfo = LocateVisualStudio
+ if ($null -eq $vsInfo) {
+ throw "Unable to locate a Visual Studio installation to run the Apex tests."
+ }
+ $vsDir = $vsInfo.installationPath.TrimEnd("\")
+ $vstestConsole = @(
+ (Join-Path $vsDir "Common7\IDE\CommonExtensions\Microsoft\TestWindow\vstest.console.exe"),
+ (Join-Path $vsDir "Common7\IDE\Extensions\TestPlatform\vstest.console.exe")
+ ) | Where-Object { Test-Path $_ } | Select-Object -First 1
+ if (-not $vstestConsole) {
+ throw "vstest.console.exe not found under '$vsDir' (looked in CommonExtensions\Microsoft\TestWindow and Extensions\TestPlatform)."
+ }
+
+ # Point Apex at this VS install so it does not rely on the local-only 'Canary' milestone lookup.
+ # This process env var propagates to the child vstest.console -> testhost processes.
+ if (-not ${env:VisualStudio.InstallationUnderTest.Path}) {
+ ${env:VisualStudio.InstallationUnderTest.Path} = Join-Path $vsDir "Common7\IDE\devenv.exe"
+ }
+
+ $testResultsDir = "$ArtifactsDir\TestResults\$configuration"
+ Create-Directory $testResultsDir
+ $jobName = if ($env:SYSTEM_JOBNAME) { $env:SYSTEM_JOBNAME } else { "local" }
+ $trxFileName = "$projectName.$targetFramework.$jobName.trx"
+
+ $vstestArgs = @(
+ $testAssembly,
+ "/Platform:x64",
+ "/Framework:.NETFramework,Version=v4.7.2",
+ "/logger:trx;LogFileName=$trxFileName",
+ "/ResultsDirectory:$testResultsDir"
+ )
+
+ Write-Host "$vstestConsole $($vstestArgs -join ' ')"
+ & $vstestConsole @vstestArgs
+ $vstestExitCode = $LASTEXITCODE
+
+ # This leg is non-gating (signal only): a test failure must NOT fail the build. Surface it as a
+ # warning and let the published TRX / Tests tab carry the signal. Genuine infra errors (missing
+ # assembly / vstest.console) already threw above and remain hard failures.
+ if ($vstestExitCode -ne 0) {
+ Write-Host "##vso[task.logissue type=warning]Apex integration tests reported failures (vstest.console exit code $vstestExitCode). This leg is non-gating; see the published TRX and the Tests tab."
+ Write-Host "Apex tests exit code $vstestExitCode (non-gating; not failing the build)."
+ }
+}
+
function Prepare-TempDir() {
Copy-Item (Join-Path $RepoRoot "tests\Resources\Directory.Build.props") $TempDir
Copy-Item (Join-Path $RepoRoot "tests\Resources\Directory.Build.targets") $TempDir
@@ -669,6 +728,10 @@ try {
TestUsingMSBuild -testProject "$RepoRoot\vsintegration\tests\FSharp.Editor.IntegrationTests\FSharp.Editor.IntegrationTests.csproj" -targetFramework $script:desktopTargetFramework
}
+ if ($testApex) {
+ TestApexUsingVSTest -targetFramework $script:desktopTargetFramework
+ }
+
if ($testAOT) {
Push-Location "$RepoRoot\tests\AheadOfTime"
./check.ps1
diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/CodeFixes/SuggestionCodeFix.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/CodeFixes/SuggestionCodeFix.cs
index a9a330290fb..1aa66153b95 100644
--- a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/CodeFixes/SuggestionCodeFix.cs
+++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/CodeFixes/SuggestionCodeFix.cs
@@ -42,7 +42,14 @@ public void SuggestionCodeFixInFSharpFile()
this.InvokeCodeFix(document, "rng.Next");
this.Library.Synchronization.WaitForSolutionCrawler();
- this.Library.Synchronization.WaitFor(() => expectedCode == document.Contents, TimeSpan.FromSeconds(15));
+ // Poll until the fix rewrites the buffer, then assert it actually applied. Without the
+ // assertion a fix that never fires would leave the original source and the test would still
+ // pass. Compare on normalized source so incidental line-ending differences do not race the poll.
+ this.Library.Synchronization.TryWaitForCondition(
+ () => NormalizeSource(document.Contents) == NormalizeSource(expectedCode),
+ TimeSpan.FromSeconds(15));
+
+ AssertSourceEquals(expectedCode, document.Contents);
}
#region Helps
diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/FSharpLanguageServiceApexTest.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/FSharpLanguageServiceApexTest.cs
index 2a3b53a2950..1670153811d 100644
--- a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/FSharpLanguageServiceApexTest.cs
+++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/FSharpLanguageServiceApexTest.cs
@@ -50,9 +50,16 @@ public FSharpLanguageServiceLibrary Library
///
/// The product milestone of the installed Visual Studio to launch, as reported by vswhere's
/// catalog.productMilestone. Defaults to "Canary" (the IntCanary channel), which selects the
- /// Canary install rather than Insiders when both are present.
+ /// Canary install rather than Insiders when both are present locally. On CI the installed VS is
+ /// a different channel (e.g. "Preview"), so this can be overridden with the
+ /// FSHARP_APEX_VS_MILESTONE environment variable; setting VisualStudio.InstallationUnderTest.Path
+ /// directly takes precedence over milestone resolution entirely.
///
- protected virtual string TargetProductMilestone => "Canary";
+ protected virtual string TargetProductMilestone
+ => Environment.GetEnvironmentVariable("FSHARP_APEX_VS_MILESTONE") is string milestone
+ && !string.IsNullOrEmpty(milestone)
+ ? milestone
+ : "Canary";
protected override VisualStudioHostConfiguration GetVisualStudioHostConfiguration()
{
@@ -261,12 +268,16 @@ protected void GoToDefinitionAndWait(
///
protected static void AssertSourceEquals(string expected, string actual)
{
- static string Normalize(string source)
- => (source ?? string.Empty).Replace("\r\n", "\n").Replace("\r", "\n").TrimEnd('\n');
-
- Assert.AreEqual(Normalize(expected), Normalize(actual));
+ Assert.AreEqual(NormalizeSource(expected), NormalizeSource(actual));
}
+ ///
+ /// Normalizes F# source for comparison by unifying line-ending style and trimming any trailing
+ /// newline, so comparisons ignore those incidental differences.
+ ///
+ protected static string NormalizeSource(string source)
+ => (source ?? string.Empty).Replace("\r\n", "\n").Replace("\r", "\n").TrimEnd('\n');
+
///
/// Verifies that placing the caret on in the given F# source
/// offers exactly one light-bulb action whose text contains .
diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/ProjectCreationHelper.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/ProjectCreationHelper.cs
index 6961cca20c2..11a837bd233 100644
--- a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/ProjectCreationHelper.cs
+++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/ProjectCreationHelper.cs
@@ -5,6 +5,7 @@
using System;
using System.Diagnostics;
using System.IO;
+using System.Threading.Tasks;
using Microsoft.Test.Apex.VisualStudio;
using Microsoft.Test.Apex.VisualStudio.Solution;
@@ -96,9 +97,15 @@ private static void RunDotNet(string arguments)
RedirectStandardError = true,
};
- using var process = Process.Start(startInfo);
- string standardError = process.StandardError.ReadToEnd();
+ using var process = Process.Start(startInfo)
+ ?? throw new InvalidOperationException($"Failed to start 'dotnet {arguments}'.");
+
+ // Drain both redirected streams concurrently. Reading one to completion before the other can
+ // deadlock: if the child fills the stdout pipe buffer while we block on stderr's EOF (which
+ // only arrives at process exit), neither side can make progress.
+ Task standardErrorTask = process.StandardError.ReadToEndAsync();
process.StandardOutput.ReadToEnd();
+ string standardError = standardErrorTask.GetAwaiter().GetResult();
process.WaitForExit();
if (process.ExitCode != 0)
diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/SynchronizationHelper.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/SynchronizationHelper.cs
index 0e37b1d8c1a..80969ffd74c 100644
--- a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/SynchronizationHelper.cs
+++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/SynchronizationHelper.cs
@@ -43,12 +43,6 @@ public void WaitForSolutionCrawler()
///
public void WaitForLightBulb() => this.Settle();
- ///
- /// Blocks until returns true or the timeout elapses.
- ///
- public void WaitFor(Func condition, TimeSpan timeout)
- => this.TryWaitForCondition(condition, timeout);
-
///
/// Polls until it returns true or the timeout elapses.
///
From 0c8b741987f9e61c36ea53979921e9c68269f326 Mon Sep 17 00:00:00 2001
From: Adam Boniecki <20281641+abonie@users.noreply.github.com>
Date: Thu, 23 Jul 2026 16:31:33 +0200
Subject: [PATCH 04/15] Remove Apex tests from slnx and gate on Insiders
---
VisualFSharp.slnx | 3 --
eng/Build.ps1 | 88 ++++++++++++++++++++++++++++++++++++++--------
eng/Versions.props | 2 +-
3 files changed, 75 insertions(+), 18 deletions(-)
diff --git a/VisualFSharp.slnx b/VisualFSharp.slnx
index 5f24b321992..9f6eed02f50 100644
--- a/VisualFSharp.slnx
+++ b/VisualFSharp.slnx
@@ -176,9 +176,6 @@
-
-
-
diff --git a/eng/Build.ps1 b/eng/Build.ps1
index aa31b96efb6..72d6a43aa96 100644
--- a/eng/Build.ps1
+++ b/eng/Build.ps1
@@ -406,21 +406,76 @@ function TestUsingMSBuild([string] $testProject, [string] $targetFramework, [str
# Runs the Apex VS integration tests. These are a classic MSTest (VSTest) project, NOT part of the
# repo's Microsoft.Testing.Platform 'dotnet test' path, and they drive a real VS instance via the Apex
-# UI-automation framework — so they are launched with vstest.console.exe (from the installed VS) against
-# the already-built test assembly. The F# VSIX must be deployed into the RoslynDev hive first
-# (build with -deployExtensions); Apex launches devenv /rootsuffix RoslynDev.
+# UI-automation framework — so they are launched with vstest.console.exe (from the installed VS).
+# The project is intentionally excluded from VisualFSharp.slnx (its dependencies must version-match the
+# VS it drives), so this builds it explicitly. It also must match a specific VS version
+# (MicrosoftTestApexVisualStudioVersion); if the installed VS does not match, the run is skipped rather
+# than producing confusing Apex/VS-mismatch failures. The F# VSIX must be deployed into the RoslynDev
+# hive first (build with -deployExtensions); Apex launches devenv /rootsuffix RoslynDev.
function TestApexUsingVSTest([string] $targetFramework) {
$projectName = "FSharp.Editor.Apex.IntegrationTests"
- $testAssembly = Join-Path $ArtifactsDir "bin\$projectName\$configuration\$targetFramework\$projectName.dll"
- if (-not (Test-Path $testAssembly)) {
- throw "Apex test assembly not found at '$testAssembly'. Build the VS solution first."
+ $apexProject = Join-Path $RepoRoot "vsintegration\tests\$projectName\$projectName.csproj"
+
+ # The Apex package (MicrosoftTestApexVisualStudioVersion) targets this VS major.minor. Keep in sync;
+ # override with FSHARP_APEX_VS_VERSION when a machine's matching VS has a different minor.
+ $expectedVsVersion = if (${env:FSHARP_APEX_VS_VERSION}) { ${env:FSHARP_APEX_VS_VERSION} } else { "18.9" }
+
+ # Determine which VS will run the tests, and gate on its version. An explicit
+ # VisualStudio.InstallationUnderTest.Path override is trusted (gate skipped); otherwise enumerate
+ # ALL installed VS instances (not just the latest) and pick the one whose major.minor matches the
+ # Apex target, skipping the run if none match rather than producing confusing Apex/VS-mismatch errors.
+ $overridePath = ${env:VisualStudio.InstallationUnderTest.Path}
+ if ($overridePath) {
+ Write-Host "Using explicit VisualStudio.InstallationUnderTest.Path=$overridePath (VS version gate skipped)."
+ $vsDir = Split-Path -Parent (Split-Path -Parent (Split-Path -Parent $overridePath))
}
+ else {
+ $vswhere = Join-Path ${env:ProgramFiles(x86)} "Microsoft Visual Studio\Installer\vswhere.exe"
+ # Query the properties separately (parallel arrays in a stable order) instead of parsing
+ # `-format json`: Windows PowerShell 5.1's ConvertFrom-Json mishandles vswhere's multi-instance
+ # JSON array (it collapses the instances into one object with array-valued properties), which
+ # would make every version comparison fail. Build.ps1 runs under Windows PowerShell 5.1.
+ $versions = @()
+ $paths = @()
+ if (Test-Path $vswhere) {
+ $versions = @(& $vswhere -all -prerelease -products * -property installationVersion)
+ $paths = @(& $vswhere -all -prerelease -products * -property installationPath)
+ }
+ $found = $versions -join ', '
+
+ # Among installs whose major.minor matches the Apex target, pick the first that actually has an
+ # IDE (devenv.exe) — excludes Build Tools and catches an incomplete/mid-update install.
+ $vsDir = $null
+ $selectedVersion = $null
+ $versionMatchPath = $null
+ for ($k = 0; $k -lt $versions.Count; $k++) {
+ $mm = ($versions[$k] -split '\.')[0..1] -join '.'
+ if ($mm -eq $expectedVsVersion) {
+ if (-not $versionMatchPath) { $versionMatchPath = $paths[$k] }
+ if (Test-Path (Join-Path $paths[$k] "Common7\IDE\devenv.exe")) {
+ $vsDir = $paths[$k].TrimEnd("\")
+ $selectedVersion = $versions[$k]
+ break
+ }
+ }
+ }
- $vsInfo = LocateVisualStudio
- if ($null -eq $vsInfo) {
- throw "Unable to locate a Visual Studio installation to run the Apex tests."
+ if (-not $vsDir) {
+ if ($versionMatchPath) {
+ Write-Host "##vso[task.logissue type=warning]Found VS $expectedVsVersion ($versionMatchPath) but no devenv.exe under Common7\IDE (incomplete install, Build Tools, or a VS update in progress); skipping Apex integration tests (non-gating)."
+ Write-Host "Skipping Apex tests: VS $expectedVsVersion present but devenv.exe missing at [$versionMatchPath]. If VS was updating, retry once it finishes."
+ }
+ else {
+ Write-Host "##vso[task.logissue type=warning]No installed VS matches the Apex target $expectedVsVersion (found: $found); skipping Apex integration tests (non-gating)."
+ Write-Host "Skipping Apex tests: no VS $expectedVsVersion found among installs [$found]. Set FSHARP_APEX_VS_VERSION or VisualStudio.InstallationUnderTest.Path to run against a different install."
+ }
+ return
+ }
+ Write-Host "Selected VS $selectedVersion at '$vsDir' for the Apex tests (target $expectedVsVersion)."
+ # This process env var propagates to the child vstest.console -> testhost -> Apex processes.
+ ${env:VisualStudio.InstallationUnderTest.Path} = Join-Path $vsDir "Common7\IDE\devenv.exe"
}
- $vsDir = $vsInfo.installationPath.TrimEnd("\")
+
$vstestConsole = @(
(Join-Path $vsDir "Common7\IDE\CommonExtensions\Microsoft\TestWindow\vstest.console.exe"),
(Join-Path $vsDir "Common7\IDE\Extensions\TestPlatform\vstest.console.exe")
@@ -429,10 +484,15 @@ function TestApexUsingVSTest([string] $targetFramework) {
throw "vstest.console.exe not found under '$vsDir' (looked in CommonExtensions\Microsoft\TestWindow and Extensions\TestPlatform)."
}
- # Point Apex at this VS install so it does not rely on the local-only 'Canary' milestone lookup.
- # This process env var propagates to the child vstest.console -> testhost processes.
- if (-not ${env:VisualStudio.InstallationUnderTest.Path}) {
- ${env:VisualStudio.InstallationUnderTest.Path} = Join-Path $vsDir "Common7\IDE\devenv.exe"
+ # Build the Apex test project explicitly (it is intentionally excluded from VisualFSharp.slnx).
+ $dotnetPath = InitializeDotNetCli
+ $dotnetExe = Join-Path $dotnetPath "dotnet.exe"
+ $buildBinLog = "$LogDir\${projectName}_$targetFramework.binlog"
+ Exec-Console $dotnetExe "build `"$apexProject`" -c $configuration /bl:`"$buildBinLog`""
+
+ $testAssembly = Join-Path $ArtifactsDir "bin\$projectName\$configuration\$targetFramework\$projectName.dll"
+ if (-not (Test-Path $testAssembly)) {
+ throw "Apex test assembly not found at '$testAssembly' after build."
}
$testResultsDir = "$ArtifactsDir\TestResults\$configuration"
diff --git a/eng/Versions.props b/eng/Versions.props
index dbaeb9ad65c..1321a5cca61 100644
--- a/eng/Versions.props
+++ b/eng/Versions.props
@@ -130,7 +130,7 @@
$(VisualStudioEditorPackagesVersion)
$(VisualStudioEditorPackagesVersion)
17.14.0
- 18.10.0-preview-1-12005-078
+ 18.9.0-preview-1-11914-018
3.6.4
0.1.800-beta
$(MicrosoftVisualStudioExtensibilityTestingVersion)
From d1757c716977d96511996d3a0efa777ac3c9a935 Mon Sep 17 00:00:00 2001
From: Adam Boniecki <20281641+abonie@users.noreply.github.com>
Date: Thu, 23 Jul 2026 19:41:14 +0200
Subject: [PATCH 05/15] List VS instances
---
eng/Build.ps1 | 22 ++++++++++++++++++++--
1 file changed, 20 insertions(+), 2 deletions(-)
diff --git a/eng/Build.ps1 b/eng/Build.ps1
index 72d6a43aa96..4271e8a21f7 100644
--- a/eng/Build.ps1
+++ b/eng/Build.ps1
@@ -416,6 +416,11 @@ function TestApexUsingVSTest([string] $targetFramework) {
$projectName = "FSharp.Editor.Apex.IntegrationTests"
$apexProject = Join-Path $RepoRoot "vsintegration\tests\$projectName\$projectName.csproj"
+ # Create the results directory up-front so the CI 'publish test logs' step never fails on a missing
+ # path, even when the run is skipped below (a marker file records the skip reason).
+ $testResultsDir = "$ArtifactsDir\TestResults\$configuration"
+ Create-Directory $testResultsDir
+
# The Apex package (MicrosoftTestApexVisualStudioVersion) targets this VS major.minor. Keep in sync;
# override with FSHARP_APEX_VS_VERSION when a machine's matching VS has a different minor.
$expectedVsVersion = if (${env:FSHARP_APEX_VS_VERSION}) { ${env:FSHARP_APEX_VS_VERSION} } else { "18.9" }
@@ -437,12 +442,26 @@ function TestApexUsingVSTest([string] $targetFramework) {
# would make every version comparison fail. Build.ps1 runs under Windows PowerShell 5.1.
$versions = @()
$paths = @()
+ $names = @()
if (Test-Path $vswhere) {
$versions = @(& $vswhere -all -prerelease -products * -property installationVersion)
$paths = @(& $vswhere -all -prerelease -products * -property installationPath)
+ $names = @(& $vswhere -all -prerelease -products * -property displayName)
}
$found = $versions -join ', '
+ # Print the full VS inventory for diagnostics (helps explain a skip on CI). Always logged.
+ Write-Host "Visual Studio instances found by vswhere ($($versions.Count)) [vswhere: $vswhere]:"
+ if ($versions.Count -eq 0) {
+ Write-Host " (none)"
+ }
+ for ($k = 0; $k -lt $versions.Count; $k++) {
+ $mm = ($versions[$k] -split '\.')[0..1] -join '.'
+ $hasIde = Test-Path (Join-Path $paths[$k] "Common7\IDE\devenv.exe")
+ $name = if ($k -lt $names.Count) { $names[$k] } else { "" }
+ Write-Host (" [{0}] version={1} (major.minor {2}) devenv={3} name='{4}' path={5}" -f $k, $versions[$k], $mm, $hasIde, $name, $paths[$k])
+ }
+
# Among installs whose major.minor matches the Apex target, pick the first that actually has an
# IDE (devenv.exe) — excludes Build Tools and catches an incomplete/mid-update install.
$vsDir = $null
@@ -469,6 +488,7 @@ function TestApexUsingVSTest([string] $targetFramework) {
Write-Host "##vso[task.logissue type=warning]No installed VS matches the Apex target $expectedVsVersion (found: $found); skipping Apex integration tests (non-gating)."
Write-Host "Skipping Apex tests: no VS $expectedVsVersion found among installs [$found]. Set FSHARP_APEX_VS_VERSION or VisualStudio.InstallationUnderTest.Path to run against a different install."
}
+ Set-Content -Path (Join-Path $testResultsDir "apex-tests-skipped.txt") -Value "Apex integration tests skipped: no usable VS matching target $expectedVsVersion (installed: $found)."
return
}
Write-Host "Selected VS $selectedVersion at '$vsDir' for the Apex tests (target $expectedVsVersion)."
@@ -495,8 +515,6 @@ function TestApexUsingVSTest([string] $targetFramework) {
throw "Apex test assembly not found at '$testAssembly' after build."
}
- $testResultsDir = "$ArtifactsDir\TestResults\$configuration"
- Create-Directory $testResultsDir
$jobName = if ($env:SYSTEM_JOBNAME) { $env:SYSTEM_JOBNAME } else { "local" }
$trxFileName = "$projectName.$targetFramework.$jobName.trx"
From f4149638a51c8876a3a6a010a4f3ffff326d8a8c Mon Sep 17 00:00:00 2001
From: Adam Boniecki <20281641+abonie@users.noreply.github.com>
Date: Fri, 24 Jul 2026 10:24:34 +0200
Subject: [PATCH 06/15] Pin Apex 18.7
---
eng/Build.ps1 | 39 +++++++++++++++++++++++----------------
eng/Versions.props | 2 +-
2 files changed, 24 insertions(+), 17 deletions(-)
diff --git a/eng/Build.ps1 b/eng/Build.ps1
index 4271e8a21f7..b4ad6c71cfc 100644
--- a/eng/Build.ps1
+++ b/eng/Build.ps1
@@ -421,14 +421,16 @@ function TestApexUsingVSTest([string] $targetFramework) {
$testResultsDir = "$ArtifactsDir\TestResults\$configuration"
Create-Directory $testResultsDir
- # The Apex package (MicrosoftTestApexVisualStudioVersion) targets this VS major.minor. Keep in sync;
- # override with FSHARP_APEX_VS_VERSION when a machine's matching VS has a different minor.
- $expectedVsVersion = if (${env:FSHARP_APEX_VS_VERSION}) { ${env:FSHARP_APEX_VS_VERSION} } else { "18.9" }
+ # The Apex package (MicrosoftTestApexVisualStudioVersion) is pinned to a VS 18.x build. Because the
+ # installed VS differs by machine (CI's image trails a dev's local install and they rarely share the
+ # same minor), gate on the MAJOR version by default so any VS 18.x can run the tests; set
+ # FSHARP_APEX_VS_VERSION to a stricter prefix (e.g. "18.9") to force an exact minor.
+ $expectedVsVersion = if (${env:FSHARP_APEX_VS_VERSION}) { ${env:FSHARP_APEX_VS_VERSION} } else { "18" }
# Determine which VS will run the tests, and gate on its version. An explicit
# VisualStudio.InstallationUnderTest.Path override is trusted (gate skipped); otherwise enumerate
- # ALL installed VS instances (not just the latest) and pick the one whose major.minor matches the
- # Apex target, skipping the run if none match rather than producing confusing Apex/VS-mismatch errors.
+ # ALL installed VS instances (not just the latest) and pick one whose version starts with the target
+ # prefix, skipping the run if none match rather than producing confusing Apex/VS-mismatch errors.
$overridePath = ${env:VisualStudio.InstallationUnderTest.Path}
if ($overridePath) {
Write-Host "Using explicit VisualStudio.InstallationUnderTest.Path=$overridePath (VS version gate skipped)."
@@ -462,20 +464,25 @@ function TestApexUsingVSTest([string] $targetFramework) {
Write-Host (" [{0}] version={1} (major.minor {2}) devenv={3} name='{4}' path={5}" -f $k, $versions[$k], $mm, $hasIde, $name, $paths[$k])
}
- # Among installs whose major.minor matches the Apex target, pick the first that actually has an
- # IDE (devenv.exe) — excludes Build Tools and catches an incomplete/mid-update install.
+ # Among installs whose version starts with the target prefix (major, or an explicit major.minor)
+ # and that actually have an IDE (devenv.exe — excludes Build Tools / incomplete installs), pick
+ # the LOWEST version: it is closest to the pinned Apex minor, and "Apex older than VS" is the
+ # safer cross-minor direction than the reverse.
$vsDir = $null
$selectedVersion = $null
$versionMatchPath = $null
- for ($k = 0; $k -lt $versions.Count; $k++) {
- $mm = ($versions[$k] -split '\.')[0..1] -join '.'
- if ($mm -eq $expectedVsVersion) {
- if (-not $versionMatchPath) { $versionMatchPath = $paths[$k] }
- if (Test-Path (Join-Path $paths[$k] "Common7\IDE\devenv.exe")) {
- $vsDir = $paths[$k].TrimEnd("\")
- $selectedVersion = $versions[$k]
- break
- }
+ $candidates = for ($k = 0; $k -lt $versions.Count; $k++) {
+ if ($versions[$k] -like "$expectedVsVersion.*") {
+ [PSCustomObject]@{ Version = $versions[$k]; Path = $paths[$k] }
+ }
+ }
+ $candidates = @($candidates | Sort-Object { [version]$_.Version })
+ foreach ($c in $candidates) {
+ if (-not $versionMatchPath) { $versionMatchPath = $c.Path }
+ if (Test-Path (Join-Path $c.Path "Common7\IDE\devenv.exe")) {
+ $vsDir = $c.Path.TrimEnd("\")
+ $selectedVersion = $c.Version
+ break
}
}
diff --git a/eng/Versions.props b/eng/Versions.props
index 1321a5cca61..adb4f0006ba 100644
--- a/eng/Versions.props
+++ b/eng/Versions.props
@@ -130,7 +130,7 @@
$(VisualStudioEditorPackagesVersion)
$(VisualStudioEditorPackagesVersion)
17.14.0
- 18.9.0-preview-1-11914-018
+ 18.7.0-preview-1-11715-043
3.6.4
0.1.800-beta
$(MicrosoftVisualStudioExtensibilityTestingVersion)
From 2d4dbe963f1ac9d5c53d1aa11b4f778b739918bf Mon Sep 17 00:00:00 2001
From: Adam Boniecki <20281641+abonie@users.noreply.github.com>
Date: Fri, 24 Jul 2026 16:59:14 +0200
Subject: [PATCH 07/15] Add more VS setup
---
eng/SetupVSHive.ps1 | 18 ++++++++++++++++++
1 file changed, 18 insertions(+)
diff --git a/eng/SetupVSHive.ps1 b/eng/SetupVSHive.ps1
index 50dcf7531d0..f88f3e4a488 100644
--- a/eng/SetupVSHive.ps1
+++ b/eng/SetupVSHive.ps1
@@ -12,6 +12,24 @@ $vsRegEdit = Join-Path (Join-Path (Join-Path $vsDir 'Common7') 'IDE') 'VSRegEdit
$hive = "RoslynDev"
&$vsRegEdit set "$vsDir" $hive HKCU "Roslyn\Internal\OnOff\Features" OOP64Bit dword 0
+# Suppress modal dialogs and online-profile prompts that block UI automation (Apex) and cause the
+# integration tests to hang until they time out. These mirror the settings dotnet/roslyn applies to its
+# RoslynDev hive before running VS integration tests.
+
+# Disable the editor "report exceptions" dialog: fail silently and keep the test running instead of
+# popping a modal that stalls automation.
+&$vsRegEdit set "$vsDir" $hive HKCU "Text Editor" "Report Exceptions" dword 0
+
+# Disable roaming settings so an online user profile / sign-in prompt can't interfere on CI.
+&$vsRegEdit set "$vsDir" $hive HKCU "ApplicationPrivateSettings\Microsoft\VisualStudio" RoamingEnabled string "1*System.Boolean*False"
+
+# Disable background download UI to avoid toasts appearing over the IDE during a run.
+&$vsRegEdit set "$vsDir" $hive HKCU "FeatureFlags\Setup\BackgroundDownload" Value dword 0
+
+# Disable targeted (remote settings) notifications. This one can't be set via VsRegEdit, so write it
+# directly to the user hive (it is not hive-suffix specific).
+reg add hkcu\Software\Microsoft\VisualStudio\RemoteSettings /f /t REG_DWORD /v TurnOffSwitch /d 1 | Out-Null
+
Write-Host "-- VS Info --"
$isolationIni = Join-Path (Join-Path (Join-Path $vsDir 'Common7') 'IDE') 'devenv.isolation.ini'
Get-Content $isolationIni | Write-Host
From d2fd5fdc12d518368ae07da5fd8eb06f3b8f6910 Mon Sep 17 00:00:00 2001
From: Adam Boniecki <20281641+abonie@users.noreply.github.com>
Date: Sat, 25 Jul 2026 12:40:09 +0200
Subject: [PATCH 08/15] Debug logging
---
eng/Build.ps1 | 97 +++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 97 insertions(+)
diff --git a/eng/Build.ps1 b/eng/Build.ps1
index b4ad6c71cfc..6149e223f33 100644
--- a/eng/Build.ps1
+++ b/eng/Build.ps1
@@ -412,6 +412,90 @@ function TestUsingMSBuild([string] $testProject, [string] $targetFramework, [str
# (MicrosoftTestApexVisualStudioVersion); if the installed VS does not match, the run is skipped rather
# than producing confusing Apex/VS-mismatch failures. The F# VSIX must be deployed into the RoslynDev
# hive first (build with -deployExtensions); Apex launches devenv /rootsuffix RoslynDev.
+
+# ===== BEGIN Apex CI diagnostics helpers (Phase 2, TEMPORARY — revert once the CI hang is understood) =====
+# On CI the Apex host failed with "Timed out ... trying get running object": devenv started but never
+# registered its DTE automation object in the ROT. These helpers (a) make sure devenv has an interactive
+# console desktop (the usual reason DTE never registers) and (b) collect the RoslynDev-hive logs so we can
+# tell an interactive-desktop problem apart from an F# VSIX / package load failure.
+function Capture-ApexScreenshot([string] $path) {
+ # Returns $true only if a screenshot could be taken, which implies an interactive desktop is attached.
+ try {
+ Add-Type -AssemblyName System.Windows.Forms -ErrorAction Stop
+ Add-Type -AssemblyName System.Drawing -ErrorAction Stop
+ $bounds = [System.Windows.Forms.SystemInformation]::VirtualScreen
+ $bmp = New-Object System.Drawing.Bitmap $bounds.Width, $bounds.Height
+ $gfx = [System.Drawing.Graphics]::FromImage($bmp)
+ try {
+ $gfx.CopyFromScreen($bounds.Location, [System.Drawing.Point]::Empty, $bounds.Size)
+ $bmp.Save($path, [System.Drawing.Imaging.ImageFormat]::Png)
+ Write-Host "Captured screenshot to '$path'."
+ return $true
+ } finally {
+ $gfx.Dispose(); $bmp.Dispose()
+ }
+ } catch {
+ Write-Host "Screenshot capture failed (no interactive desktop?): $($_.Exception.Message)"
+ return $false
+ }
+}
+
+function Prepare-ApexInteractiveSession([string] $screenshotPath) {
+ # Mirrors dotnet/roslyn's Setup-IntegrationTestRun: if the desktop is not interactive, reconnect the
+ # session to the console so devenv can publish its DTE automation object; then minimize other windows.
+ if (-not (Capture-ApexScreenshot $screenshotPath)) {
+ Write-Host "No interactive desktop detected; attempting to reconnect the session to the console."
+ try {
+ $quserItems = ((quser $env:USERNAME | Select-Object -Skip 1) -split '\s+')
+ $sessionId = $quserItems[2]
+ if ($sessionId -eq 'Disc') { $sessionId = $quserItems[1] }
+ Write-Host "tscon $sessionId /dest:console"
+ tscon $sessionId /dest:console
+ Start-Sleep -Seconds 3
+ [void](Capture-ApexScreenshot $screenshotPath)
+ } catch {
+ Write-Host "##vso[task.logissue type=warning]Failed to reconnect session to console: $($_.Exception.Message)"
+ }
+ }
+
+ try {
+ (New-Object -ComObject "Shell.Application").MinimizeAll()
+ } catch {
+ Write-Host "MinimizeAll failed: $($_.Exception.Message)"
+ }
+}
+
+function Collect-ApexDiagnostics([string] $screenshotPath, [string] $destination) {
+ # After-run screenshot plus the RoslynDev experimental-hive logs. ActivityLog.xml records whether the
+ # F# package failed to load; ComponentModelCache\*.err records MEF composition failures.
+ [void](Capture-ApexScreenshot $screenshotPath)
+
+ $roamingVs = Join-Path $env:USERPROFILE "AppData\Roaming\Microsoft\VisualStudio"
+ $localVs = Join-Path $env:USERPROFILE "AppData\Local\Microsoft\VisualStudio"
+
+ if (Test-Path $roamingVs) {
+ foreach ($hiveDir in @(Get-ChildItem -Path $roamingVs -Directory -Filter "*RoslynDev" -ErrorAction SilentlyContinue)) {
+ foreach ($name in @("ActivityLog.xml", "ActivityLog.xsl")) {
+ $src = Join-Path $hiveDir.FullName $name
+ if (Test-Path $src) {
+ Copy-Item $src (Join-Path $destination "$($hiveDir.Name)-$name") -Force -ErrorAction SilentlyContinue
+ Write-Host "Collected $src"
+ }
+ }
+ }
+ }
+ if (Test-Path $localVs) {
+ foreach ($hiveDir in @(Get-ChildItem -Path $localVs -Directory -Filter "*RoslynDev" -ErrorAction SilentlyContinue)) {
+ $mefErr = Join-Path $hiveDir.FullName "ComponentModelCache\Microsoft.VisualStudio.Default.err"
+ if (Test-Path $mefErr) {
+ Copy-Item $mefErr (Join-Path $destination "$($hiveDir.Name)-Microsoft.VisualStudio.Default.err") -Force -ErrorAction SilentlyContinue
+ Write-Host "Collected $mefErr"
+ }
+ }
+ }
+}
+# ===== END Apex CI diagnostics helpers (Phase 2) =====
+
function TestApexUsingVSTest([string] $targetFramework) {
$projectName = "FSharp.Editor.Apex.IntegrationTests"
$apexProject = Join-Path $RepoRoot "vsintegration\tests\$projectName\$projectName.csproj"
@@ -534,9 +618,22 @@ function TestApexUsingVSTest([string] $targetFramework) {
)
Write-Host "$vstestConsole $($vstestArgs -join ' ')"
+
+ # ===== BEGIN Apex CI diagnostics (Phase 2, TEMPORARY — revert once the CI hang is understood) =====
+ if ($ci) {
+ Prepare-ApexInteractiveSession (Join-Path $testResultsDir "apex-before-run.png")
+ }
+ # ===== END Apex CI diagnostics (Phase 2) =====
+
& $vstestConsole @vstestArgs
$vstestExitCode = $LASTEXITCODE
+ # ===== BEGIN Apex CI diagnostics (Phase 2, TEMPORARY — revert once the CI hang is understood) =====
+ if ($ci) {
+ Collect-ApexDiagnostics (Join-Path $testResultsDir "apex-after-run.png") $testResultsDir
+ }
+ # ===== END Apex CI diagnostics (Phase 2) =====
+
# This leg is non-gating (signal only): a test failure must NOT fail the build. Surface it as a
# warning and let the published TRX / Tests tab carry the signal. Genuine infra errors (missing
# assembly / vstest.console) already threw above and remain hard failures.
From 2a772b0be56dc7f30c9f79bf995eb2c768a81b47 Mon Sep 17 00:00:00 2001
From: Adam Boniecki <20281641+abonie@users.noreply.github.com>
Date: Sun, 26 Jul 2026 23:06:01 +0200
Subject: [PATCH 09/15] Skip VS sign in
---
.../TestFramework/FSharpLanguageServiceApexTest.cs | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/FSharpLanguageServiceApexTest.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/FSharpLanguageServiceApexTest.cs
index 1670153811d..4583a7b321f 100644
--- a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/FSharpLanguageServiceApexTest.cs
+++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/FSharpLanguageServiceApexTest.cs
@@ -68,6 +68,16 @@ protected override VisualStudioHostConfiguration GetVisualStudioHostConfiguratio
var config = base.GetVisualStudioHostConfiguration();
config.AutomaticallyDismissMessageBoxes = this.AutomaticallyDismissMessageBoxes;
config.RootSuffix = this.RootSuffix;
+
+ // On a fresh experimental hive (e.g. CI), devenv shows a full-screen "Sign in to Visual Studio"
+ // first-launch dialog that blocks the UI thread, so it never registers its DTE automation
+ // object and Apex times out waiting for it. The devenv '/NoSigninPrompt' switch (registered by
+ // Microsoft.VisualStudio.Shell.Connected under AppCommandLine) suppresses that dialog.
+ config.CommandLineArguments =
+ string.IsNullOrEmpty(config.CommandLineArguments)
+ ? "/NoSigninPrompt"
+ : config.CommandLineArguments + " /NoSigninPrompt";
+
return config;
}
From fccb0c7b88e30d3fa9d5bdf564990fa814013234 Mon Sep 17 00:00:00 2001
From: Adam Boniecki <20281641+abonie@users.noreply.github.com>
Date: Mon, 27 Jul 2026 10:43:33 +0200
Subject: [PATCH 10/15] Start VS with an empty env
---
eng/SetupVSHive.ps1 | 22 ++++++++++++++++++++++
1 file changed, 22 insertions(+)
diff --git a/eng/SetupVSHive.ps1 b/eng/SetupVSHive.ps1
index f88f3e4a488..b6ec083d97f 100644
--- a/eng/SetupVSHive.ps1
+++ b/eng/SetupVSHive.ps1
@@ -30,6 +30,28 @@ $hive = "RoslynDev"
# directly to the user hive (it is not hive-suffix specific).
reg add hkcu\Software\Microsoft\VisualStudio\RemoteSettings /f /t REG_DWORD /v TurnOffSwitch /d 1 | Out-Null
+# Initialize the experimental hive's first-launch state non-interactively. On a fresh hive devenv shows a
+# full-screen "Sign in to Visual Studio" first-launch experience that blocks the UI thread, so it never
+# registers its DTE automation object and Apex times out waiting for it. Importing settings via
+# /resetsettings completes first launch without that dialog. This mirrors what the
+# Microsoft.VisualStudio.Extensibility.Testing harness does before running VS integration tests on CI.
+
+# Start VS with an empty environment (no start window / start page) on launch.
+&$vsRegEdit set "$vsDir" $hive HKCU "General" OnEnvironmentStartup dword 10
+
+$devenv = Join-Path (Join-Path (Join-Path $vsDir 'Common7') 'IDE') 'devenv.exe'
+$firstLaunchLog = Join-Path $env:TEMP "vs-firstlaunch-$hive.log"
+Write-Host "Initializing '$hive' hive first launch: devenv /rootsuffix $hive /resetsettings General.vssettings /command File.Exit"
+$devenvArgs = @("/rootsuffix", $hive, "/resetsettings", "General.vssettings", "/command", "File.Exit", "/logFile:$firstLaunchLog")
+$proc = Start-Process -FilePath $devenv -ArgumentList $devenvArgs -PassThru
+if (-not $proc.WaitForExit(180000)) {
+ Write-Host "##vso[task.logissue type=warning]devenv first-launch init did not exit within 180s; terminating it."
+ try { Stop-Process -Id $proc.Id -Force } catch {}
+}
+else {
+ Write-Host "devenv first-launch init exited with code $($proc.ExitCode)."
+}
+
Write-Host "-- VS Info --"
$isolationIni = Join-Path (Join-Path (Join-Path $vsDir 'Common7') 'IDE') 'devenv.isolation.ini'
Get-Content $isolationIni | Write-Host
From 19675db3b4ae0843e7747288ef7617d3ed87c65e Mon Sep 17 00:00:00 2001
From: Adam Boniecki <20281641+abonie@users.noreply.github.com>
Date: Mon, 27 Jul 2026 18:41:24 +0200
Subject: [PATCH 11/15] Pin Roslyn packages differently for CI
---
azure-pipelines-PR.yml | 6 ++++++
eng/Versions.props | 28 ++++++++++++++++++++++++++++
2 files changed, 34 insertions(+)
diff --git a/azure-pipelines-PR.yml b/azure-pipelines-PR.yml
index e19d3824dad..3cc556049a4 100644
--- a/azure-pipelines-PR.yml
+++ b/azure-pipelines-PR.yml
@@ -527,6 +527,12 @@ stages:
- powershell: eng\SetupVSHive.ps1
displayName: Setup VS Hive
+ # Detect the installed VS's Roslyn version and set FSHARP_APEX_ROSLYN_VERSION so the VSIX below
+ # is built against it (the repo's forward Roslyn pin is newer than any CI/scout VS, so a VSIX
+ # built against the pin cannot load its Roslyn assemblies in the installed VS). Non-fatal.
+ - powershell: eng\SetApexRoslynVersion.ps1
+ displayName: Detect CI VS Roslyn version for Apex VSIX build
+
# Build + deploy the F# VSIX into the RoslynDev hive, then run the Apex tests. The Apex host
# resolves the installed VS via LocateVisualStudio (set as VisualStudio.InstallationUnderTest.Path
# by the -testApex path), so it does not depend on the local-only 'Canary' milestone.
diff --git a/eng/Versions.props b/eng/Versions.props
index adb4f0006ba..94aa71dc8f0 100644
--- a/eng/Versions.props
+++ b/eng/Versions.props
@@ -167,4 +167,32 @@
+
+
+ $(FSHARP_APEX_ROSLYN_VERSION)
+ $(FSHARP_APEX_ROSLYN_VERSION)
+ $(FSHARP_APEX_ROSLYN_VERSION)
+ $(FSHARP_APEX_ROSLYN_VERSION)
+ $(FSHARP_APEX_ROSLYN_VERSION)
+ $(FSHARP_APEX_ROSLYN_VERSION)
+ $(FSHARP_APEX_ROSLYN_VERSION)
+ $(FSHARP_APEX_ROSLYN_VERSION)
+
+ $(FSHARP_APEX_ROSLYN_VERSION)
+ $(FSHARP_APEX_ROSLYN_VERSION)
+ $(FSHARP_APEX_ROSLYN_VERSION)
+ $(FSHARP_APEX_ROSLYN_VERSION)
+ $(FSHARP_APEX_ROSLYN_VERSION)
+ $(FSHARP_APEX_ROSLYN_VERSION)
+ $(FSHARP_APEX_ROSLYN_VERSION)
+ $(FSHARP_APEX_ROSLYN_VERSION)
+
+
From 1596a31e2de1c5c30c9090e20c924cb70c7b6a38 Mon Sep 17 00:00:00 2001
From: Adam Boniecki <20281641+abonie@users.noreply.github.com>
Date: Mon, 27 Jul 2026 19:46:46 +0200
Subject: [PATCH 12/15] Add missing eng/SetApexRoslynVersion.ps1 for the Apex
Roslyn detection step
The azure-pipelines-PR.yml 'Detect CI VS Roslyn version' step and the
FSHARP_APEX_ROSLYN_VERSION override in eng/Versions.props were committed, but the
script itself was left untracked, so on CI 'powershell: eng\SetApexRoslynVersion.ps1'
could not find the file and failed with 'The module ''eng'' could not be loaded'.
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: fe2b3239-4176-4f13-b6f6-486c8799765a
---
eng/SetApexRoslynVersion.ps1 | 82 ++++++++++++++++++++++++++++++++++++
1 file changed, 82 insertions(+)
create mode 100644 eng/SetApexRoslynVersion.ps1
diff --git a/eng/SetApexRoslynVersion.ps1 b/eng/SetApexRoslynVersion.ps1
new file mode 100644
index 00000000000..e95f42e721f
--- /dev/null
+++ b/eng/SetApexRoslynVersion.ps1
@@ -0,0 +1,82 @@
+# Detects the Roslyn version of the *installed* Visual Studio and exports it as the pipeline variable
+# FSHARP_APEX_ROSLYN_VERSION so the Apex integration-test leg builds the F# VSIX against it.
+#
+# Why: the repo pins Roslyn forward (to the VS being inserted into), which is newer than any VS on the CI
+# scout image. A VSIX built against that pin cannot bind Microsoft.VisualStudio.LanguageServices at
+# runtime in the older CI VS, so FSharpPackage/FSharpProjectPackage fail to load and every Apex test
+# fails. The F# editor references Roslyn compile-only (ExcludeAssets=runtime), so building against the
+# VS's own Roslyn version is sufficient and ships nothing extra. The eng/Versions.props override consumes
+# this variable; the committed pin is unchanged. If detection fails we leave the variable unset (the
+# build falls back to the committed pin) rather than break the build.
+[CmdletBinding()]
+param()
+
+$ErrorActionPreference = 'Stop'
+
+function Write-Skip([string] $message) {
+ Write-Host "##vso[task.logissue type=warning]$message"
+ Write-Host "Apex Roslyn override not set; the VSIX will build against the repo's pinned Roslyn."
+}
+
+$vswhere = Join-Path ${env:ProgramFiles(x86)} 'Microsoft Visual Studio\Installer\vswhere.exe'
+if (-not (Test-Path $vswhere)) {
+ Write-Skip "vswhere.exe not found; cannot detect the installed VS Roslyn version."
+ return
+}
+
+# Select the same install the Apex gate runs against: the lowest matching (default major '18') VS that
+# actually has devenv.exe. On CI there is a single VS, so any selection resolves to it.
+$prefix = if (${env:FSHARP_APEX_VS_VERSION}) { ${env:FSHARP_APEX_VS_VERSION} } else { '18' }
+$versions = @(& $vswhere -all -prerelease -products * -property installationVersion)
+$paths = @(& $vswhere -all -prerelease -products * -property installationPath)
+
+$candidates = for ($i = 0; $i -lt $versions.Count; $i++) {
+ if ($versions[$i] -like "$prefix.*") {
+ [PSCustomObject]@{ Version = $versions[$i]; Path = $paths[$i] }
+ }
+}
+$candidates = @($candidates | Sort-Object { [version]$_.Version })
+
+$vsDir = $null
+foreach ($c in $candidates) {
+ if (Test-Path (Join-Path $c.Path 'Common7\IDE\devenv.exe')) { $vsDir = $c.Path.TrimEnd('\'); break }
+}
+if (-not $vsDir) {
+ Write-Skip "No installed VS $prefix.x with devenv.exe found; cannot detect the Roslyn version."
+ return
+}
+
+$lsDll = Join-Path $vsDir 'Common7\IDE\CommonExtensions\Microsoft\VBCSharp\LanguageServices\Microsoft.VisualStudio.LanguageServices.dll'
+if (-not (Test-Path $lsDll)) {
+ $lsDll = Get-ChildItem -Path $vsDir -Recurse -Filter 'Microsoft.VisualStudio.LanguageServices.dll' -ErrorAction SilentlyContinue |
+ Select-Object -First 1 -ExpandProperty FullName
+}
+if (-not $lsDll -or -not (Test-Path $lsDll)) {
+ Write-Skip "Microsoft.VisualStudio.LanguageServices.dll not found under '$vsDir'."
+ return
+}
+
+$asmVersion = [System.Reflection.AssemblyName]::GetAssemblyName($lsDll).Version
+$majorMinor = "$($asmVersion.Major).$($asmVersion.Minor)"
+Write-Host "Installed VS '$vsDir' ships Roslyn assembly $asmVersion (major.minor $majorMinor)."
+
+# Resolve a matching Microsoft.CodeAnalysis NuGet version. The whole Roslyn family is published together
+# with the same version, and every '..0-*' build has assembly version ..0.0,
+# so any such package binds to the installed VS at runtime; pick the newest for freshness.
+$feed = 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3'
+try {
+ $allVersions = (Invoke-RestMethod -Uri "$feed/flat2/microsoft.codeanalysis.externalaccess.fsharp/index.json" -UseBasicParsing).versions
+} catch {
+ Write-Skip "Failed to query the dotnet-tools feed for Roslyn versions: $($_.Exception.Message)"
+ return
+}
+
+$selected = @($allVersions | Where-Object { $_ -match "^$([regex]::Escape($majorMinor))\.0-" }) |
+ Sort-Object | Select-Object -Last 1
+if (-not $selected) {
+ Write-Skip "No Roslyn package '$majorMinor.0-*' found on the feed to match the installed VS."
+ return
+}
+
+Write-Host "Selected Roslyn NuGet version '$selected' for the Apex VSIX build (matches VS Roslyn $majorMinor)."
+Write-Host "##vso[task.setvariable variable=FSHARP_APEX_ROSLYN_VERSION]$selected"
From c0c3dd70e0a2f2aa2a9613fba1a83e21d71b2b1b Mon Sep 17 00:00:00 2001
From: Adam Boniecki <20281641+abonie@users.noreply.github.com>
Date: Tue, 28 Jul 2026 10:38:19 +0200
Subject: [PATCH 13/15] Roll System.Composition/Immutable back to 10.0.2 for
the Apex leg
main (via darc) bumped the dotnet/runtime System.* packages to 10.0.8 (assembly
10.0.0.8). The F# VSIX then references System.Composition.AttributedModel 10.0.0.8,
which the CI scout VS (18.7, ships 8.0.0.0) cannot bind -- its binding-redirect
ceiling is below 10.0.0.8 -- so every FSharp.Editor MEF part fails to load,
IFSharpWorkspaceService is null, and all Apex tests fail with an NRE in
LanguageService.fs. This is the same forward-pinned-dependency vs older-CI-VS
problem already handled for Roslyn.
Extend the FSHARP_APEX_ROSLYN_VERSION-gated override in eng/Versions.props to also
roll System.Composition and System.Collections.Immutable back to 10.0.2 for the
Apex leg only. Verified: with a simulated 10.0.8 base the override forces 10.0.2,
and FSharp.Editor compiles and references System.Composition.AttributedModel
10.0.0.2. The committed pin and product/main builds are unaffected.
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: fe2b3239-4176-4f13-b6f6-486c8799765a
---
eng/Versions.props | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/eng/Versions.props b/eng/Versions.props
index 94aa71dc8f0..56976083693 100644
--- a/eng/Versions.props
+++ b/eng/Versions.props
@@ -193,6 +193,18 @@
$(FSHARP_APEX_ROSLYN_VERSION)
$(FSHARP_APEX_ROSLYN_VERSION)
$(FSHARP_APEX_ROSLYN_VERSION)
+
+
+ 10.0.2
+ $(FSharpApexSystemPackageVersion)
+ $(FSharpApexSystemPackageVersion)
+ $(FSharpApexSystemPackageVersion)
+ $(FSharpApexSystemPackageVersion)
From 1f2d505f556e88c40a41a9bd8b923dfc73491fa2 Mon Sep 17 00:00:00 2001
From: Adam Boniecki <20281641+abonie@users.noreply.github.com>
Date: Tue, 28 Jul 2026 12:05:24 +0200
Subject: [PATCH 14/15] Revert "Roll System.Composition/Immutable back to
10.0.2 for the Apex leg"
This reverts commit c0c3dd70e0a2f2aa2a9613fba1a83e21d71b2b1b.
---
eng/Versions.props | 12 ------------
1 file changed, 12 deletions(-)
diff --git a/eng/Versions.props b/eng/Versions.props
index 56976083693..94aa71dc8f0 100644
--- a/eng/Versions.props
+++ b/eng/Versions.props
@@ -193,18 +193,6 @@
$(FSHARP_APEX_ROSLYN_VERSION)
$(FSHARP_APEX_ROSLYN_VERSION)
$(FSHARP_APEX_ROSLYN_VERSION)
-
-
- 10.0.2
- $(FSharpApexSystemPackageVersion)
- $(FSharpApexSystemPackageVersion)
- $(FSharpApexSystemPackageVersion)
- $(FSharpApexSystemPackageVersion)
From 07db08b8ed2bb60b13d447667d946914ce8d8799 Mon Sep 17 00:00:00 2001
From: Adam Boniecki <20281641+abonie@users.noreply.github.com>
Date: Tue, 28 Jul 2026 16:05:59 +0200
Subject: [PATCH 15/15] Add DartLab VS-integration pipeline scaffold for the F#
Apex tests
Scaffolds an internal DevDiv DartLab pipeline (modeled on dotnet/roslyn's
azure-pipelines-integration-dartlab.yml) that installs a matching Visual Studio via
the VS bootstrapper on a VS-Platform test machine, deploys the F# VSIX, and runs the
existing Apex tests. This is the robust fix for the version-skew that blocks running
the VSIX on the older pre-baked public scout VS (the VS-SDK floor, e.g. RpcContracts
requiring System.Composition >= 10.0.8, prevents downgrading to match it).
Files (all trigger:none / inert until registered):
- azure-pipelines-integration-dartlab.yml: entry, extends the VS real-sign template.
- eng/pipelines/apex-integration/{stage,integration-job}.yml: provision + install VS +
deploy VSIX + run Apex via the existing -testApex path.
- eng/pipelines/apex-integration/README.md: external prerequisites + a concrete request
to the DartLab/VS team.
- eng/setup-pr-validation.ps1: PR-branch setup on the test machine (adapted from Roslyn).
Every access-dependent value is marked '# TODO(P0):'. Requires DevDiv/DartLab access,
an internal dotnet-fsharp mirror, and a VS-Platform pool before it can be registered
and run; those are outside this repo.
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: fe2b3239-4176-4f13-b6f6-486c8799765a
---
azure-pipelines-integration-dartlab.yml | 78 ++++++++++++
eng/pipelines/apex-integration/README.md | 81 ++++++++++++
.../apex-integration/integration-job.yml | 72 +++++++++++
eng/pipelines/apex-integration/stage.yml | 119 ++++++++++++++++++
eng/setup-pr-validation.ps1 | 70 +++++++++++
5 files changed, 420 insertions(+)
create mode 100644 azure-pipelines-integration-dartlab.yml
create mode 100644 eng/pipelines/apex-integration/README.md
create mode 100644 eng/pipelines/apex-integration/integration-job.yml
create mode 100644 eng/pipelines/apex-integration/stage.yml
create mode 100644 eng/setup-pr-validation.ps1
diff --git a/azure-pipelines-integration-dartlab.yml b/azure-pipelines-integration-dartlab.yml
new file mode 100644
index 00000000000..3c7a09f6c5c
--- /dev/null
+++ b/azure-pipelines-integration-dartlab.yml
@@ -0,0 +1,78 @@
+# F# VS integration (Apex) test gate — runs against a Visual Studio installed via the VS
+# bootstrapper on internal DartLab test machines, so the VS matches what this repo targets.
+#
+# SCAFFOLD — modeled on dotnet/roslyn's azure-pipelines-integration-dartlab.yml.
+# This pipeline is INTERNAL-only and requires DevDiv/DartLab access that is not part of this
+# repo. See eng/pipelines/apex-integration/README.md for the P0 prerequisites and every
+# `# TODO(P0):` below for the real values that must be supplied before it can run.
+
+trigger: none
+pr: none
+
+resources:
+ repositories:
+ - repository: DartLabTemplates
+ type: git
+ name: DevDiv/DartLab.Templates
+ ref: refs/heads/main
+ - repository: VSTemplates
+ type: git
+ name: DevDiv/VS.Templates
+ ref: refs/heads/main
+ # TODO(P0): provision an internal mirror of dotnet/fsharp and a matching service connection.
+ # Roslyn uses `internal/dotnet-roslyn` via the `dnceng-internal-code-access` endpoint.
+ - repository: FSharpMirror
+ endpoint: dnceng-internal-code-access
+ type: git
+ name: internal/dotnet-fsharp
+ ref: $(Build.SourceBranch)
+ # TODO(P0): the VS build under test supplies the bootstrapper to install. Interim: omit this and
+ # hard-code `visualStudioBootstrapperURI` in stage.yml to a pinned int.main Products drop. End
+ # state: reference the VS build pipeline so this runs as a gate on the consuming VS build.
+ # - pipeline: VisualStudioBuildUnderTest
+ # source: ''
+
+parameters:
+ - name: prNumber
+ type: string
+ default: 'None'
+ - name: sha
+ type: string
+ default: 'None'
+ - name: EnforceLatestCommit
+ type: boolean
+ default: true
+ # '(default)' => stage.yml resolves the bootstrapper from the VS-build-under-test drop.
+ - name: visualStudioBootstrapperURI
+ type: string
+ default: '(default)'
+ - name: testCaseFilter
+ type: string
+ default: '(all)'
+ # Set to 'stop' to keep the test machine around for investigation after the run.
+ - name: testMachineCleanUpStrategy
+ displayName: Test Machine Clean Up Strategy
+ type: string
+ default: delete
+ values:
+ - delete
+ - stop
+ # Retention period for the test machine (format "D:H:M:S").
+ - name: testMachineRetention
+ displayName: Test Machine Retention (format "D:H:M:S")
+ type: string
+ default: 1:0:0:0
+
+extends:
+ template: /DartLab/1ES/pipelines/ci/real-sign.yml@VSTemplates
+ parameters:
+ testStages:
+ - template: /eng/pipelines/apex-integration/stage.yml@self
+ parameters:
+ prNumber: ${{ parameters.prNumber }}
+ sha: ${{ parameters.sha }}
+ EnforceLatestCommit: ${{ parameters.EnforceLatestCommit }}
+ visualStudioBootstrapperURI: ${{ parameters.visualStudioBootstrapperURI }}
+ testCaseFilter: ${{ parameters.testCaseFilter }}
+ testMachineCleanUpStrategy: ${{ parameters.testMachineCleanUpStrategy }}
+ testMachineRetention: ${{ parameters.testMachineRetention }}
diff --git a/eng/pipelines/apex-integration/README.md b/eng/pipelines/apex-integration/README.md
new file mode 100644
index 00000000000..42b6e54d92e
--- /dev/null
+++ b/eng/pipelines/apex-integration/README.md
@@ -0,0 +1,81 @@
+# F# Apex VS integration tests — DartLab pipeline (scaffold)
+
+This directory contains an **internal DevDiv DartLab** pipeline that runs the F# Apex VS
+integration tests (`vsintegration/tests/FSharp.Editor.Apex.IntegrationTests`) against a
+Visual Studio build that **matches** what this repo targets, by *installing* that VS on a
+fresh test machine via the VS bootstrapper.
+
+It is modeled directly on dotnet/roslyn's
+[`azure-pipelines-integration-dartlab.yml`](https://github.com/dotnet/roslyn/blob/main/azure-pipelines-integration-dartlab.yml)
+and its `eng/pipelines/test-gates/*` stage templates.
+
+## Why this exists
+
+The F# editor VSIX is pinned (Roslyn, and the dotnet/runtime `System.*` family) to the VS it
+inserts into (F# `main` → VS `main`, ~18.10). Visual Studio has **no binding redirects for our
+assemblies**, so a VSIX built against those pins only loads in a matching (or newer) VS. The
+public PR CI scout image trails that VS (e.g. 18.7), and the mismatch cannot be worked around
+by downgrading dependencies (the VS-SDK imposes a floor — e.g. `Microsoft.VisualStudio.RpcContracts`
+requires `System.Composition >= 10.0.8`, so a downgrade fails restore with NU1605).
+
+The industry-standard fix (used by both NuGet/NuGet.Client and dotnet/roslyn) is to **install a
+matching VS via the bootstrapper on internal DartLab test machines**, not to use a pre-baked older
+public image.
+
+## Prerequisites that must be provisioned before this can run (P0 — external)
+
+These are **not** contained in this repo and require coordination with the DevDiv / DartLab / VS
+team (as Roslyn did). The YAML here uses `# TODO(P0):` markers wherever a real value is required.
+
+1. **DartLab + VS pipeline templates access** — the pipeline `extends`/references
+ `DevDiv/DartLab.Templates` and `DevDiv/VS.Templates` (internal AzDO repos).
+2. **Internal source mirror** — an `internal/dotnet-fsharp` mirror reachable via a
+ `dnceng-internal-code-access` service connection (Roslyn uses `internal/dotnet-roslyn`). The
+ test machine checks out from this mirror, not from GitHub.
+3. **VS-Platform test lab pool** + **1ES** onboarding for a new internal pipeline definition.
+4. **F# DevDiv area path / owner** for `templateContext` (Roslyn uses
+ `mlinfraswat` / `DevDiv\NET Developer Experience\CSharp and VB IDE`).
+5. **VS-build-under-test source** — where the matching VS bootstrapper comes from:
+ - *Interim:* pin a recent VS `int.main` Products drop and hard-code its bootstrapper URI.
+ - *End state:* wire a `VisualStudioBuildUnderTest` pipeline resource and compute the drop name
+ with DartLab's `Get-VisualStudioDropName.ps1` (as Roslyn does), so the tests run as a gate on
+ the VS build that consumes the F# insertion.
+
+## Files
+
+- `../../azure-pipelines-integration-dartlab.yml` — pipeline entry (`trigger: none`), extends the
+ VS real-sign template and wires resources.
+- `stage.yml` — the DartLab VS test stage: provisions a `VS-Platform` machine, installs VS via the
+ bootstrapper with an F#-minimal component set, then deploys + runs.
+- `integration-job.yml` — deploy the F# VSIX + run the Apex tests via `eng/Build.ps1 -testApex`
+ against the freshly installed VS; publish the TRX.
+
+## Once matching-VS runs are green
+
+The interim public-CI workarounds become unnecessary and should be retired (see plan P4):
+- `eng/SetApexRoslynVersion.ps1` and the `FSHARP_APEX_ROSLYN_VERSION` override group in
+ `eng/Versions.props`.
+Keep `eng/SetupVSHive.ps1` (first-launch/registry prep) and the `/NoSigninPrompt` launch argument.
+
+## Concrete request to send to the DartLab / VS test team
+
+Onboarding is the same shape Roslyn used. Ask for / decide:
+
+1. Read access for the F# pipeline's service identity to the internal AzDO repos
+ `DevDiv/DartLab.Templates` and `DevDiv/VS.Templates`.
+2. An internal Git mirror of `dotnet/fsharp` (e.g. `internal/dotnet-fsharp`) and a
+ `dnceng-internal-code-access` service connection so the test machine can check it out.
+3. Use of the `VS-Platform` test-lab pool for the new pipeline, and 1ES registration of
+ `azure-pipelines-integration-dartlab.yml` as an internal pipeline.
+4. The F# `templateContext.owner` and `areaPath` to record against test results
+ (fill into `eng/pipelines/apex-integration/stage.yml`).
+5. The VS build to test against:
+ - *Interim:* a specific VS `int.main` **Products** drop whose
+ `bootstrappers/Enterprise/vs_enterprise.exe` we can pin as `visualStudioBootstrapperURI`.
+ - *End state:* register a `VisualStudioBuildUnderTest` pipeline resource so the F# tests run as
+ a gate on the VS build that consumes the F# insertion (uncomment the resource + the two
+ `preTestMachineConfigurationStepList` steps in `stage.yml`).
+
+After that: fill in every `# TODO(P0):` marker, register the pipeline, and iterate to green. Then do
+plan P4 (retire the version overrides) and P5 (disable the public `WindowsApexIntegration` job).
+
diff --git a/eng/pipelines/apex-integration/integration-job.yml b/eng/pipelines/apex-integration/integration-job.yml
new file mode 100644
index 00000000000..9d8e3c86f95
--- /dev/null
+++ b/eng/pipelines/apex-integration/integration-job.yml
@@ -0,0 +1,72 @@
+# Deploy the F# VSIX and run the Apex VS integration tests against the VS installed on this
+# DartLab machine (at C:\Test\VisualStudio). SCAFFOLD — mirrors dotnet/roslyn's
+# eng/pipelines/test-integration-job.yml, reusing this repo's existing -testApex path.
+#
+# Because the installed VS MATCHES the repo's pins, the interim public-CI version overrides are
+# NOT used here (no eng/SetApexRoslynVersion.ps1 / FSHARP_APEX_ROSLYN_VERSION). Only the
+# first-launch/registry prep and the /NoSigninPrompt launch arg from the public leg are reused.
+
+parameters:
+ - name: configuration
+ type: string
+ default: 'Release'
+ - name: testCaseFilter
+ type: string
+ default: ''
+
+steps:
+ # Checkout is performed by the DartLab deployAndRunTestsStepList, so skip it here.
+
+ # Point Apex at the VS the bootstrapper installed on this machine.
+ - powershell: |
+ $devenv = 'C:\Test\VisualStudio\Common7\IDE\devenv.exe'
+ if (-not (Test-Path $devenv)) { Write-Host "##vso[task.logissue type=error]devenv not found at $devenv"; exit 1 }
+ Write-Host "##vso[task.setvariable variable=VisualStudio.InstallationUnderTest.Path]$devenv"
+ Write-Host "Apex will target: $devenv"
+ displayName: Select installed VS for Apex
+
+ - task: PowerShell@2
+ displayName: Enable Long Paths
+ inputs:
+ targetType: 'inline'
+ script: reg add HKLM\SYSTEM\CurrentControlSet\Control\FileSystem /v LongPathsEnabled /t REG_DWORD /d 1 /f
+
+ # First-launch / experimental-hive registry prep + non-interactive settings import (shared with
+ # the public leg). Suppresses the sign-in ISE so devenv reaches automation-ready state.
+ - powershell: eng\SetupVSHive.ps1
+ displayName: Setup VS Hive
+
+ # Build F#, deploy the VSIX into the RoslynDev hive, and run the Apex tests via -testApex.
+ # The installed VS matches the repo pins, so no Roslyn/System.* override is applied.
+ - script: eng\CIBuildNoPublish.cmd -configuration ${{ parameters.configuration }} -deployExtensions -testApex
+ env:
+ NativeToolsOnMachine: true
+ displayName: Build, deploy extension and run Apex integration tests
+
+ - task: PublishTestResults@2
+ displayName: Publish Apex Test Results
+ inputs:
+ testResultsFormat: 'VSTest'
+ testRunTitle: 'FSharpVSIntegration ${{ parameters.configuration }}'
+ mergeTestResults: true
+ testResultsFiles: '*.trx'
+ searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/${{ parameters.configuration }}'
+ condition: succeededOrFailed()
+
+ - task: PublishBuildArtifacts@1
+ displayName: Publish Apex Test Logs
+ inputs:
+ PathtoPublish: '$(Build.SourcesDirectory)\artifacts\TestResults\${{ parameters.configuration }}'
+ ArtifactName: 'FSharp Apex ${{ parameters.configuration }} test logs $(System.JobAttempt)'
+ publishLocation: Container
+ continueOnError: true
+ condition: always()
+
+ - task: PublishBuildArtifacts@1
+ displayName: Publish Apex Build Logs
+ inputs:
+ PathToPublish: '$(Build.SourcesDirectory)\artifacts\log\${{ parameters.configuration }}'
+ ArtifactName: 'FSharp Apex ${{ parameters.configuration }} build logs $(System.JobAttempt)'
+ publishLocation: Container
+ continueOnError: true
+ condition: always()
diff --git a/eng/pipelines/apex-integration/stage.yml b/eng/pipelines/apex-integration/stage.yml
new file mode 100644
index 00000000000..e523b08d9b6
--- /dev/null
+++ b/eng/pipelines/apex-integration/stage.yml
@@ -0,0 +1,119 @@
+# F# Apex VS integration test stage (DartLab). SCAFFOLD — mirrors dotnet/roslyn's
+# eng/pipelines/test-gates/roslyn-vs-integration/stage.yml, trimmed to the F# scenario.
+# See eng/pipelines/apex-integration/README.md and every `# TODO(P0):` below.
+
+parameters:
+- name: prNumber
+ type: string
+ default: 'None'
+- name: sha
+ type: string
+ default: 'None'
+- name: EnforceLatestCommit
+ type: boolean
+ default: true
+- name: visualStudioBootstrapperURI
+ type: string
+ default: '(default)'
+- name: testCaseFilter
+ type: string
+ default: ''
+- name: testMachineCleanUpStrategy
+ type: string
+ default: delete
+- name: testMachineRetention
+ type: string
+ default: 1:0:0:0
+- name: dartLabTemplatesRepositoryAlias
+ type: string
+ default: DartLabTemplates
+- name: fsharpRepositoryAlias
+ type: string
+ default: self
+
+stages:
+- template: /stages/visual-studio/agent.yml@${{ parameters.dartLabTemplatesRepositoryAlias }}
+ parameters:
+ name: FSharpVSIntegration
+ displayName: F# VS Integration (Apex)
+ testLabPoolName: VS-Platform
+ testMachineCleanUpStrategy: ${{ parameters.testMachineCleanUpStrategy }}
+ testMachineRetention: ${{ parameters.testMachineRetention }}
+ templateContext:
+ # TODO(P0): set the F# DevDiv owner / areaPath (Roslyn uses
+ # `mlinfraswat` / `DevDiv\NET Developer Experience\CSharp and VB IDE`).
+ owner: fsharp
+ areaPath: DevDiv\NET Developer Experience\FSharp
+ # '(default)' => install the VS from the build-under-test Products drop. Interim: replace the
+ # whole expression with a hard-coded bootstrapper URI for a pinned int.main Products drop, e.g.
+ # 'https://vsdrop.corp.microsoft.com/file/v1/Products/DevDiv/VS/int.main/;bootstrappers/Enterprise/vs_enterprise.exe'.
+ visualStudioBootstrapperURI: ${{ iif(eq(parameters.visualStudioBootstrapperURI, '(default)'), 'https://vsdrop.corp.microsoft.com/file/v1/$(VisualStudio.BuildUnderTest.ProductsDropName);bootstrappers/Enterprise/vs_enterprise.exe', parameters.visualStudioBootstrapperURI) }}
+ # F#-minimal component set: core editor + MSBuild + .NET SDK/targeting packs + Roslyn language
+ # services + the F# components + the VS SDK (so the locally-built F# VSIX can be deployed/loaded).
+ # Expand only if a test needs more; every added component increases install time.
+ visualStudioInstallationParameters: >-
+ --add Microsoft.VisualStudio.Component.CoreEditor
+ --add Microsoft.VisualStudio.Workload.CoreEditor
+ --add Microsoft.Component.MSBuild
+ --add Microsoft.NetCore.Component.SDK
+ --add Microsoft.NetCore.Component.Runtime.8.0
+ --add Microsoft.NetCore.Component.DevelopmentTools
+ --add Microsoft.Net.Component.4.7.2.TargetingPack
+ --add Microsoft.Net.Component.4.8.SDK
+ --add Microsoft.VisualStudio.Component.Roslyn.Compiler
+ --add Microsoft.VisualStudio.Component.Roslyn.LanguageServices
+ --add Microsoft.VisualStudio.Component.FSharp
+ --add Microsoft.VisualStudio.Component.FSharp.Desktop
+ --add Microsoft.VisualStudio.Component.ManagedDesktop.Core
+ --add Microsoft.VisualStudio.Component.ManagedDesktop.Prerequisites
+ --add Microsoft.VisualStudio.Workload.ManagedDesktop
+ --add Microsoft.Component.CodeAnalysis.SDK
+ --add Microsoft.VisualStudio.Component.VSSDK
+ --add Microsoft.VisualStudio.ComponentGroup.VisualStudioExtension.Prerequisites
+ --add Microsoft.VisualStudio.Workload.VisualStudioExtension
+ --installPath "C:\Test\VisualStudio" --quiet --norestart --wait
+ testExecutionJobStrategy:
+ matrix:
+ release:
+ _configuration: Release
+ testMachineTotalCount: 1
+ testAgentElevated: true
+ preTestMachineConfigurationStepList:
+ - checkout: none
+ # TODO(P0): only needed for the gate/end-state where the VS bootstrapper comes from a VS build
+ # resource. For the interim pinned-drop approach (hard-coded bootstrapper URI above), delete
+ # these two steps.
+ # - task: DownloadBuildArtifacts@1
+ # displayName: Download Visual Studio build artifacts
+ # inputs:
+ # buildType: specific
+ # project: $(resources.pipeline.VisualStudioBuildUnderTest.projectID)
+ # definition: $(resources.pipeline.VisualStudioBuildUnderTest.pipelineID)
+ # buildVersionToDownload: specific
+ # buildId: $(resources.pipeline.VisualStudioBuildUnderTest.runID)
+ # downloadType: specific
+ # artifactName: BuildArtifacts
+ # downloadPath: $(Pipeline.Workspace)\VisualStudioBuildUnderTest
+ # - task: PowerShell@2
+ # name: SetProductsDropName
+ # displayName: Set 'VisualStudio.BuildUnderTest.ProductsDropName'
+ # inputs:
+ # filePath: $(DartLab.Path)\Scripts\VisualStudio\Build\Get-VisualStudioDropName.ps1
+ # arguments: -DropNamePrefix 'Products' -VstsDropUrlsJson '$(Pipeline.Workspace)\VisualStudioBuildUnderTest\BuildArtifacts\VstsDropUrls.json' -OutVariableName 'VisualStudio.BuildUnderTest.ProductsDropName'
+ deployAndRunTestsStepList:
+ # TODO(P0): check out the internal mirror (see azure-pipelines-integration-dartlab.yml resources).
+ - checkout: FSharpMirror
+ fetchDepth: 1
+ fetchTags: false
+ persistCredentials: true
+ - ${{ if ne(parameters.prNumber, 'None') }}:
+ - task: PowerShell@2
+ displayName: Setup branch for validation
+ inputs:
+ filePath: 'eng\setup-pr-validation.ps1'
+ arguments: "-sourceBranchName $(Build.SourceBranch) -prNumber ${{ parameters.prNumber }} -commitSHA ${{ parameters.sha }} -enforceLatestCommit ${{ iif(parameters.EnforceLatestCommit, '1', '0') }}"
+ condition: succeeded()
+ - template: /eng/pipelines/apex-integration/integration-job.yml@${{ parameters.fsharpRepositoryAlias }}
+ parameters:
+ configuration: $(_configuration)
+ testCaseFilter: ${{ parameters.testCaseFilter }}
diff --git a/eng/setup-pr-validation.ps1 b/eng/setup-pr-validation.ps1
new file mode 100644
index 00000000000..00cae5630af
--- /dev/null
+++ b/eng/setup-pr-validation.ps1
@@ -0,0 +1,70 @@
+# Sets up the working tree on a DartLab test machine to validate a specific GitHub PR: fetches the
+# PR merge commit, optionally enforces that the PR head still matches the reviewed SHA, and checks
+# it out. Referenced by eng/pipelines/apex-integration/stage.yml (only when a PR number is supplied).
+# Adapted from dotnet/roslyn's eng/setup-pr-validation.ps1.
+[CmdletBinding(PositionalBinding=$false)]
+param (
+ [string]$sourceBranchName,
+ [string]$prNumber,
+ [string]$commitSHA,
+ [boolean]$enforceLatestCommit)
+
+try {
+ # name and email are only used for the merge commit; the values do not matter.
+ git config user.name "FSharpValidation"
+ git config user.email "validation@fsharp.net"
+
+ if ($commitSHA.Length -lt 7) {
+ Write-Host "##vso[task.LogIssue type=error;]The PR Commit SHA must be at least 7 characters long."
+ exit 1
+ }
+
+ git remote add gh https://github.com/dotnet/fsharp.git
+
+ Write-Host "Getting the hash of refs/pull/$prNumber/head..."
+ $remoteRef = git ls-remote gh refs/pull/$prNumber/head
+ Write-Host ($remoteRef | Out-String)
+
+ $prHeadSHA = $remoteRef.Split()[0]
+
+ if ($enforceLatestCommit) {
+ Write-Host "Validating the PR head matches the specified commit SHA ($commitSHA)..."
+ if (!$prHeadSHA.StartsWith($commitSHA)) {
+ Write-Host "##vso[task.LogIssue type=error;]The PR's Head SHA ($prHeadSHA) does not begin with the specified commit SHA ($commitSHA). Unreviewed changes may have been pushed to the PR."
+ exit 1
+ }
+ }
+
+ Write-Host "Setting up the build for PR validation by fetching refs/pull/$prNumber/merge..."
+ git fetch gh refs/pull/$prNumber/merge
+ if (!$?) {
+ Write-Host "##vso[task.LogIssue type=error;]Fetching ref refs/pull/$prNumber/merge failed."
+ exit 1
+ }
+
+ git checkout FETCH_HEAD
+ if (!$?) {
+ Write-Host "##vso[task.LogIssue type=error;]Checking out FETCH_HEAD for refs/pull/$prNumber/merge failed."
+ exit 1
+ }
+
+ if (!$enforceLatestCommit) {
+ if ($prHeadSHA.StartsWith($commitSHA)) {
+ Write-Host "PR head SHA ($prHeadSHA) already matches the specified commit SHA ($commitSHA), skipping checkout."
+ }
+ else {
+ Write-Host "Checking out the specified commit SHA ($commitSHA)..."
+ git checkout $commitSHA
+ if (!$?) {
+ Write-Host "##vso[task.LogIssue type=error;]Checking out commit SHA $commitSHA failed."
+ exit 1
+ }
+ }
+ }
+}
+catch {
+ Write-Host $_
+ Write-Host $_.Exception
+ Write-Host $_.ScriptStackTrace
+ exit 1
+}