Port page-goto, set-content and wait-for-selector-2 tests, add content includeShadow

src/PlaywrightNative.Tests/PageGotoParityTests.cs

@@ -19,6 +19,7 @@
 using System.Globalization;
 using System.IO;
 using System.Text;
+using System.Text.Json;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Http;
@@ -40,6 +41,10 @@ namespace PlaywrightNative.Tests
     [NonParallelizable]
     public class PageGotoParityTests : PageTestEx
     {
+        private const string NodeListenerLeakReason =
+            "Node-only: asserts no MaxListenersExceededWarning on process.on('warning'). "
+            + ".NET events have no listener cap or leak warning.";
+
         private static SimpleServer _ownedServer;
         private static SimpleServer _ownedHttps;
         private static string Prefix = TestConstants.ServerUrl;
@@ -461,6 +466,18 @@ await http.Response.WriteAsync(
             Assert.That(response.Request.Failure, Is.Null.Or.Empty);
         }
 
+        [PlaywrightTest("page-goto.spec.ts", "should work with Cross-Origin-Opener-Policy and history state")]
+        [Test]
+        [Timeout(30_000)]
+        public Task ShouldWorkWithCrossOriginOpenerPolicyAndHistoryState()
+            => CrossOriginOpenerPolicyHistoryStateAsync(crossOriginEmbedderPolicy: null);
+
+        [PlaywrightTest("page-goto.spec.ts", "should work with Cross-Origin-Opener-Policy and Cross-Origin-Embedder-Policy and history state")]
+        [Test]
+        [Timeout(30_000)]
+        public Task ShouldWorkWithCrossOriginOpenerPolicyAndCrossOriginEmbedderPolicyAndHistoryState()
+            => CrossOriginOpenerPolicyHistoryStateAsync(crossOriginEmbedderPolicy: "require-corp");
+
         [PlaywrightTest("page-goto.spec.ts", "should work with Cross-Origin-Opener-Policy and interception")]
         [Test]
         [Timeout(30_000)]
@@ -1252,6 +1269,30 @@ public async Task ShouldReturnLastResponseInRedirectChain()
             Assert.That(response.Url, Is.EqualTo(EmptyPage));
         }
 
+        [PlaywrightTest("page-goto.spec.ts", "should not leak listeners during navigation")]
+        [Test]
+        [Timeout(30_000)]
+        public void ShouldNotLeakListenersDuringNavigation()
+        {
+            Assert.Ignore(NodeListenerLeakReason);
+        }
+
+        [PlaywrightTest("page-goto.spec.ts", "should not leak listeners during bad navigation")]
+        [Test]
+        [Timeout(30_000)]
+        public void ShouldNotLeakListenersDuringBadNavigation()
+        {
+            Assert.Ignore(NodeListenerLeakReason);
+        }
+
+        [PlaywrightTest("page-goto.spec.ts", "should not leak listeners during 20 waitForNavigation")]
+        [Test]
+        [Timeout(30_000)]
+        public void ShouldNotLeakListenersDuring20WaitForNavigation()
+        {
+            Assert.Ignore(NodeListenerLeakReason);
+        }
+
         [PlaywrightTest("page-goto.spec.ts", "should navigate to dataURL and not fire dataURL requests")]
         [Test]
         [Timeout(30_000)]
@@ -1693,5 +1734,36 @@ public async Task ShouldReturnUrlWithBasicAuthInfo()
             await page.GoToAsync(url).ConfigureAwait(false);
             Assert.That(page.Url, Is.EqualTo(url));
         }
+
+        private async Task CrossOriginOpenerPolicyHistoryStateAsync(string crossOriginEmbedderPolicy)
+        {
+            // https://github.com/microsoft/playwright/issues/42731
+            EnsureServer();
+            Server.SetRoute("/empty.html", async http =>
+            {
+                http.Response.Headers["Cross-Origin-Opener-Policy"] = "same-origin";
+                if (crossOriginEmbedderPolicy != null)
+                {
+                    http.Response.Headers["Cross-Origin-Embedder-Policy"] = crossOriginEmbedderPolicy;
+                }
+
+                await http.Response.WriteAsync(@"
+        <!doctype html>
+        <h1>hello</h1>
+        <script type=""module"">history.replaceState({ key: 1 }, '', location.href);</script>
+      ").ConfigureAwait(false);
+            });
+
+            await using IBrowser browser = await BrowserLauncher.LaunchAsync().ConfigureAwait(false);
+            await using IBrowserContext context = await browser.NewContextAsync().ConfigureAwait(false);
+            IPage page = await context.NewPageAsync().ConfigureAwait(false);
+
+            IResponse response = await page.GoToAsync(EmptyPage).ConfigureAwait(false);
+            Assert.That(response.Status, Is.EqualTo(200));
+            Assert.That(page.Url, Is.EqualTo(EmptyPage));
+            JsonElement state = await page.EvaluateAsync<JsonElement>("() => history.state").ConfigureAwait(false);
+            Assert.That(state.GetRawText(), Is.EqualTo("{\"key\":1}"));
+            await Assertions.Expect(page.Locator("h1")).ToHaveTextAsync("hello").ConfigureAwait(false);
+        }
     }
 }

src/PlaywrightNative.Tests/PageSetContentTests.cs

@@ -26,7 +26,7 @@ namespace PlaywrightNative.Tests
 {
     /// <summary>
     /// Official <c>page-set-content.spec.ts</c>.
-    /// Skipped (not implemented):
+    /// Skipped (Node-only internals):
     /// <c>should handle timeout properly</c> (needs toImpl utility-world console.debug tamper);
     /// <c>should handle timeout properly 2</c> (needs toImpl document.close infinite loop).
     /// </summary>
@@ -132,6 +132,25 @@ public async Task ShouldWorkWithHtml4Doctype()
             Assert.That(result, Is.EqualTo(doctype + ExpectedHtml));
         }
 
+        [PlaywrightTest("page-set-content.spec.ts", "should include shadow roots")]
+        [Test]
+        [Timeout(30_000)]
+        public async Task ShouldIncludeShadowRoots()
+        {
+            await using IBrowser browser = await BrowserLauncher.LaunchAsync().ConfigureAwait(false);
+            await using IBrowserContext context = await browser.NewContextAsync().ConfigureAwait(false);
+            IPage page = await context.NewPageAsync().ConfigureAwait(false);
+
+            const string html = "<!DOCTYPE html><html lang=\"en\"><head></head><body><div id=\"host\"><template shadowrootmode=\"open\"><div id=\"nested\"><template shadowrootmode=\"open\"><span>nested</span></template><slot></slot></div></template><span>light</span></div><div id=\"closed\"></div></body></html>";
+
+            // Closed shadow roots are not accessible from script and are never serialized.
+            await page.SetContentAsync(html.Replace("<div id=\"closed\">", "<div id=\"closed\"><template shadowrootmode=\"closed\"><span>closed</span></template>", StringComparison.Ordinal)).ConfigureAwait(false);
+            Assert.That(
+                await page.ContentAsync().ConfigureAwait(false),
+                Is.EqualTo("<!DOCTYPE html><html lang=\"en\"><head></head><body><div id=\"host\"><span>light</span></div><div id=\"closed\"></div></body></html>"));
+            Assert.That(await page.ContentAsync(new() { IncludeShadow = true }).ConfigureAwait(false), Is.EqualTo(html));
+        }
+
         [PlaywrightTest("page-set-content.spec.ts", "should respect timeout")]
         [Test]
         [Timeout(30_000)]
@@ -322,5 +341,23 @@ public async Task ShouldReturnEmptyContentThereIsNoIframeSrc()
             string content = await child.ContentAsync().ConfigureAwait(false);
             Assert.That(content, Is.EqualTo(EmptyHtml));
         }
+
+        [PlaywrightTest("page-set-content.spec.ts", "should handle timeout properly")]
+        [Test]
+        [Timeout(30_000)]
+        public void ShouldHandleTimeoutProperly()
+        {
+            Assert.Ignore("Node-only: toImpl(page).mainFrame().evaluateExpression tampers console.debug in the server utility world. "
+                + "SetContent here does not use a persistent utility world or the console.debug tag.");
+        }
+
+        [PlaywrightTest("page-set-content.spec.ts", "should handle timeout properly 2")]
+        [Test]
+        [Timeout(30_000)]
+        public void ShouldHandleTimeoutProperly2()
+        {
+            Assert.Ignore("Node-only: toImpl(page).mainFrame().evaluateExpression replaces document.close in the server utility world. "
+                + "SetContent here does not use a persistent utility world.");
+        }
     }
 }

src/PlaywrightNative.Tests/PageWaitForSelector2ParityTests.cs

@@ -41,6 +41,10 @@ namespace PlaywrightNative.Tests
     [TestFixture]
     public class PageWaitForSelector2ParityTests : PageTestEx
     {
+        private const string NodeAdoptionHookReason =
+            "Node-only: drives the server-side __testHookBeforeAdoptNode between the utility-world match "
+            + "and its adoption into the main world. waitForSelector here polls the main world and has no adoption step.";
+
         private const string AddElement = "tag => document.body.appendChild(document.createElement(tag))";
 
         private static SimpleServer _ownedServer;
@@ -622,6 +626,22 @@ await page.SetContentAsync(@"
             await page.WaitForSelectorAsync("div", WaitForSelectorState.Hidden).ConfigureAwait(false);
         }
 
+        [PlaywrightTest("page-wait-for-selector-2.spec.ts", "should work when navigating before node adoption")]
+        [Test]
+        [Timeout(TestConstants.DefaultTestTimeout)]
+        public void ShouldWorkWhenNavigatingBeforeNodeAdoption()
+        {
+            Assert.Ignore(NodeAdoptionHookReason);
+        }
+
+        [PlaywrightTest("page-wait-for-selector-2.spec.ts", "should fail when navigating while on handle")]
+        [Test]
+        [Timeout(TestConstants.DefaultTestTimeout)]
+        public void ShouldFailWhenNavigatingWhileOnHandle()
+        {
+            Assert.Ignore(NodeAdoptionHookReason);
+        }
+
         [PlaywrightTest("page-wait-for-selector-2.spec.ts", "should fail if element handle was detached while waiting")]
         [Test]
         [Timeout(TestConstants.DefaultTestTimeout)]

src/PlaywrightNative/Chromium/CRPage.cs

@@ -2001,9 +2001,10 @@ internal async Task<CRExecutionContext> GetUtilityWorldAsync(Frame frame)
         /// <summary>
         /// Returns the full HTML of the current document, including doctype and root element.
         /// </summary>
+        /// <param name="includeShadow">Whether to serialize open shadow roots as declarative shadow DOM.</param>
         /// <returns>The serialized document HTML.</returns>
-        internal Task<string> ContentAsync()
-            => PageContent.ReadAsync(() => EvaluateAsync<string>(PageContent.EvaluateExpression));
+        internal Task<string> ContentAsync(bool includeShadow = false)
+            => PageContent.ReadAsync(() => EvaluateAsync<string>(PageContent.Expression(includeShadow)));
 
         /// <summary>
         /// Adds a <c>&lt;script&gt;</c> tag to the page by URL or inline content. For URL

src/PlaywrightNative/Firefox/FFPage.cs

@@ -20,6 +20,7 @@
 using System.Text.Json;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
+using PlaywrightNative.Helpers;
 
 namespace PlaywrightNative.Firefox
 {
@@ -372,19 +373,9 @@ internal IReadOnlyCollection<string> SnapshotLifecycle()
         /// <summary>
         /// Returns the full HTML content of the page.
         /// </summary>
-        internal async Task<string> ContentAsync()
-        {
-            return await EvaluateFunctionAsync<string>(@"() => {
-                let retVal = '';
-                if (document.doctype) {
-                    retVal = new XMLSerializer().serializeToString(document.doctype);
-                }
-                if (document.documentElement) {
-                    retVal += document.documentElement.outerHTML;
-                }
-                return retVal;
-            }").ConfigureAwait(false);
-        }
+        /// <param name="includeShadow">Whether to serialize open shadow roots as declarative shadow DOM.</param>
+        internal Task<string> ContentAsync(bool includeShadow = false)
+            => EvaluateFunctionAsync<string>("() => " + PageContent.Expression(includeShadow));
 
         /// <summary>
         /// Sets the page HTML content.

src/PlaywrightNative/Firefox/FirefoxPage.cs

@@ -984,6 +984,11 @@ public Task DragAndDropAsync(
             throw NotImplementedHelper.ForMethod(nameof(DragAndDropAsync));
         }
 
+        /// <summary>Returns the page HTML, optionally including open shadow roots.</summary>
+        /// <param name="includeShadow">Whether to serialize open shadow roots as declarative shadow DOM.</param>
+        /// <returns>The serialized document HTML.</returns>
+        internal Task<string> ContentAsync(bool includeShadow) => _page.ContentAsync(includeShadow);
+
         private async Task PauseInternalAsync()
         {
             int timeoutMs = TimeoutSettings.TimeoutMs(DefaultTimeout);

src/PlaywrightNative/Helpers/PageContent.cs

@@ -42,13 +42,52 @@ internal static class PageContent
                 return retVal;
             })()";
 
+        /// <summary>
+        /// JavaScript for official <c>page.content({ includeShadow: true })</c>
+        /// (<c>injectedScript.documentContent</c>): open shadow roots are serialized as
+        /// declarative shadow DOM via <c>getHTML({ shadowRoots })</c>.
+        /// </summary>
+        internal const string EvaluateWithShadowExpression =
+            @"(() => {
+                let content = '';
+                if (document.doctype) {
+                    content = new XMLSerializer().serializeToString(document.doctype);
+                }
+                const root = document.documentElement;
+                if (!root) {
+                    return content;
+                }
+                const shadowRoots = [];
+                const collectShadowRoots = node => {
+                    for (const element of node.querySelectorAll('*')) {
+                        if (element.shadowRoot) {
+                            shadowRoots.push(element.shadowRoot);
+                            collectShadowRoots(element.shadowRoot);
+                        }
+                    }
+                };
+                collectShadowRoots(document);
+                // getHTML() serializes children only, wrap them with the root element tags.
+                const emptyRoot = root.cloneNode(false).outerHTML;
+                const endTagIndex = emptyRoot.lastIndexOf('</');
+                return content + emptyRoot.slice(0, endTagIndex) + root.getHTML({ shadowRoots }) + emptyRoot.slice(endTagIndex);
+            })()";
+
         /// <summary>
         /// Official message when <c>content()</c> is evaluated while the document is
         /// being replaced by a navigation.
         /// </summary>
         internal const string NavigationError =
             "Unable to retrieve content because the page is navigating and changing the content.";
 
+        /// <summary>
+        /// Picks the content expression for <paramref name="includeShadow"/>.
+        /// </summary>
+        /// <param name="includeShadow">Whether to serialize open shadow roots.</param>
+        /// <returns>The JavaScript expression.</returns>
+        internal static string Expression(bool includeShadow)
+            => includeShadow ? EvaluateWithShadowExpression : EvaluateExpression;
+
         /// <summary>
         /// Runs <paramref name="evaluateAsync"/> and rewrites retriable evaluation
         /// failures (destroyed context, mid-navigation) to

src/PlaywrightNative/MicrosoftSurfaceCompatExtensions.cs

@@ -117,6 +117,31 @@ public static Task<IResponse> GoBackAsync(this IPage page, WaitUntilState waitUn
             }
         }
 
+        /// <summary>
+        /// Gets the full HTML contents of the page, including the doctype. Official
+        /// <c>page.content({ includeShadow })</c>, which <c>Microsoft.Playwright</c> does not expose yet.
+        /// </summary>
+        /// <param name="page">The page.</param>
+        /// <param name="options">Call options.</param>
+        /// <returns>The serialized document HTML.</returns>
+        public static Task<string> ContentAsync(this IPage page, PageContentOptions options)
+        {
+            bool includeShadow = options?.IncludeShadow ?? false;
+            switch (page)
+            {
+                case Page chromium:
+                    return chromium.ContentAsync(includeShadow);
+                case FirefoxPage firefox:
+                    return firefox.ContentAsync(includeShadow);
+                case WKPage webkit:
+                    return webkit.ContentAsync(includeShadow);
+                default:
+                    return includeShadow
+                        ? throw new NotSupportedException("includeShadow is not supported for " + page?.GetType().Name + ".")
+                        : page.ContentAsync();
+            }
+        }
+
         /// <summary>Legacy drag-and-drop with source/target positions.</summary>
         public static Task DragAndDropAsync(
             this IPage page,

src/PlaywrightNative/Page.cs

@@ -1761,6 +1761,12 @@ public Task<IFileChooser> WaitForFileChooserAsync(float? timeout = default, Canc
         public Task<T> WaitForEventAsync<T>(PlaywrightEvent<T> pageEvent, Func<T, bool> predicate = null, float? timeout = null)
             => PageWaitForEventHelper.WaitAsync(this, pageEvent, predicate, timeout);
 
+        /// <summary>Returns the page HTML, optionally including open shadow roots.</summary>
+        /// <param name="includeShadow">Whether to serialize open shadow roots as declarative shadow DOM.</param>
+        /// <returns>The serialized document HTML.</returns>
+        internal Task<string> ContentAsync(bool includeShadow)
+            => _crPage.ContentAsync(includeShadow);
+
         /// <summary>
         /// Returns the cached instance for <paramref name="frame"/>.
         /// </summary>

src/PlaywrightNative/PageContentOptions.cs

@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2020 Dario Kondratiuk
+ * Modifications copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+namespace PlaywrightNative
+{
+    /// <summary>
+    /// Options for <see cref="MicrosoftSurfaceCompatExtensions.ContentAsync(Microsoft.Playwright.IPage, PageContentOptions)"/>.
+    /// </summary>
+    public class PageContentOptions
+    {
+        /// <summary>
+        /// Gets or sets a value indicating whether the contents of open shadow roots are
+        /// included as declarative shadow DOM, i.e. <c>&lt;template shadowrootmode="open"&gt;</c>
+        /// elements nested inside their host elements. Closed shadow roots are never
+        /// included. Defaults to <c>false</c>.
+        /// </summary>
+        public bool? IncludeShadow { get; set; }
+    }
+}