ci: DH-18428: vscode-extension-tester e2e tests

.dockerignore

@@ -0,0 +1,20 @@
+# Hidden file / directories
+**/.*/
+
+# Add back specific hidden files
+!.eslintrc.json
+!.nvmrc
+!.vscodeignore
+!e2e-testing/.mocharc*.js
+
+vitest.config.mts
+
+*.tsbuildinfo
+*.vsix
+
+**/dist
+**/node_modules/
+**/out
+docs
+releases
+test-reports

.eslintrc.json

@@ -28,6 +28,7 @@
         "project": [
           "./tsconfig.json",
           "./e2e/tsconfig.json",
+          "./e2e-testing/tsconfig.json",
           "./tsconfig.unit.json",
           "./packages/*/tsconfig.json"
         ]
@@ -44,5 +45,11 @@
       }
     }
   ],
-  "ignorePatterns": ["out", "dist", "**/*.d.ts"]
+  "ignorePatterns": [
+    "out",
+    "dist",
+    "**/*.d.ts",
+    ".resources",
+    ".test-extensions"
+  ]
 }

.github/workflows/e2e.yml

@@ -19,10 +19,16 @@ jobs:
       - run: npm ci
       - name: Run end-to-end tests
         run: xvfb-run npm run test:e2e
+      - name: Upload Screenshots as Artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: e2e-screenshots
+          path: e2e-testing/.resources/screenshots
+          retention-days: 1
       - name: Publish Test Summary Results
         if: ${{ always() }}
         run: |
-          npm run report:ctrfmerge
           npm run report:ctrfsummary
           sed -i 's/<h3>Test Summary<\/h3>/<h3>End-to-end Test Summary<\/h3>/' $GITHUB_STEP_SUMMARY
           npm run report:prcomment

.gitignore

@@ -1,9 +1,12 @@
 out
 dist
 node_modules
+/*.vsix
 .vscode-test/
 .DS_Store
 .wdio-vscode-service
 e2e/reports/
+e2e-testing/.resources
+e2e-testing/.test-extensions
 test-reports/
 tsconfig.tsbuildinfo

.vscode/launch.json

@@ -12,6 +12,16 @@
       "args": ["--extensionDevelopmentPath=${workspaceFolder}"],
       "outFiles": ["${workspaceFolder}/out/**/*.js"],
       "preLaunchTask": "${defaultBuildTask}"
+    },
+    {
+      "name": "e2e Tests",
+      "type": "node",
+      "request": "launch",
+      "program": "${workspaceFolder}/e2e-testing/out/runner.mjs",
+      "args": ["--debug"],
+      "console": "integratedTerminal",
+      "internalConsoleOptions": "neverOpen",
+      "preLaunchTask": "${defaultBuildTask}"
     }
   ]
 }

CONTRIBUTING.md

@@ -12,10 +12,10 @@ npm run test
 
 ### End-to-end Testing
 
-End-to-end tests are configured to run via `wdio-vscode-service`. This allows
+End-to-end tests are configured to run via `vscode-extension-tester. This allows
 testing workflows and has better abstractions for ui testing than `@vscode/test-electron`.
 
-See [wdio-vscode-service](https://www.npmjs.com/package/wdio-vscode-service) for more details.
+See [vscode-extension-tester](https://github.com/redhat-developer/vscode-extension-tester) for more details.
 
 To run end-to-end tests:
 
@@ -26,23 +26,10 @@ npm run test:e2e
 To run using `VS Code` debugger:
 
 1. Set a breakpoint in a test
-2. Either
+2. Run the `e2e Tests` launch config in VS Code
+3. You should see the tests start and `VS Code` stop at the breakpoint
 
-   Enable auto attach with flag (`Command palette -> Debug: Toggle Auto Attach -> Only With Flag`).
-
-   > Note you may need to open a new terminal for this to take effect.
-
-   or
-
-   Open `Javascript Debug Terminal` (Click the dropdown arrow beside the + button for adding a new terminal)
-
-3. Run the script:
-
-   ```sh
-   npm run test:e2e
-   ```
-
-4. You should see the tests start and `VS Code` stop at the breakpoint.
+The `vscode-extension-tester` library uses `Mocha` to run tests. If you need to tweak debugging settings such as test timeout, you can do so in [`e2e-testing/src/mocharcDebug.ts`](./e2e-testing/src/mocharcDebug.ts).
 
 ## VSCE
 [vsce](https://github.com/microsoft/vscode-vsce), short for "Visual Studio Code Extensions", is a command-line tool for packaging, publishing and managing `VS Code` extensions. The Deephaven extension calls `vsce` via npm scripts. Note that `vsce package` and `vsce publish` both call the `vscode:prepublish` script.

e2e-testing/docker-compose.yml

@@ -0,0 +1,9 @@
+services:
+  dhc-server:
+    container_name: dhc-server
+    image: ghcr.io/deephaven/server:${DHC_VERSION:-edge}
+    pull_policy: always
+    ports:
+      - 10000:10000
+    environment:
+      - START_OPTS=-Xmx4g -DAuthHandlers=io.deephaven.auth.AnonymousAuthenticationHandler -Ddeephaven.console.type=python

e2e-testing/src/mocharc.ts

@@ -0,0 +1,11 @@
+import { MochaOptions } from 'vscode-extension-tester';
+
+const options: MochaOptions = {
+  timeout: 30000,
+  reporter: 'mocha-ctrf-json-reporter',
+  reporterOptions: {
+    outputDir: 'test-reports',
+  },
+};
+
+module.exports = options;

e2e-testing/src/mocharcDebug.ts

@@ -0,0 +1,13 @@
+import { MochaOptions } from 'vscode-extension-tester';
+
+import * as baseOptions from './mocharc';
+
+const options: MochaOptions = {
+  ...baseOptions,
+  // When debugging tests in VS Code, we want to be able to set breakpoints and
+  // inspect DOM, so we increase Mocha tests timeout. An arbitrary 3 minutes
+  // seems sufficient for most cases but can be tweaked as necessary.
+  timeout: 60000 * 3,
+};
+
+module.exports = options;

e2e-testing/src/pageObjects/EditorViewExtended.ts

@@ -0,0 +1,200 @@
+import {
+  By,
+  EditorView,
+  TextEditor,
+  VSBrowser,
+  type EditorGroup,
+  type WebElement,
+} from 'vscode-extension-tester';
+import {
+  extractErrorType,
+  locators,
+  switchToFrame,
+  type EditorGroupData,
+  type TabData,
+  type WebViewData,
+} from '../util';
+import { WebViewExtended } from './WebViewExtended';
+
+export class EditorViewExtended extends EditorView {
+  /**
+   * Get a serializable data representation for all editor groups, their tabs,
+   * and associated webviews.
+   * @returns Promise resolving to an array of EditorGroupData objects
+   */
+  async getEditorGroupsData(): Promise<EditorGroupData[]> {
+    const groupDataList: EditorGroupData[] = [];
+    let groupIndex = 0;
+    for (const group of await this.getEditorGroups()) {
+      const tabs: TabData[] = [];
+
+      // An editor group may be associated with 0-n webviews
+      let webViewCount = 0;
+
+      for (const tab of await group.getOpenTabs()) {
+        const title = await tab.getTitle();
+        const isSelected = await tab.isSelected();
+        const resourceName = await tab.getAttribute('data-resource-name');
+        const isWebView = resourceName.startsWith('webview-');
+
+        if (isWebView) {
+          webViewCount += 1;
+        }
+
+        tabs.push({
+          title,
+          isSelected,
+          isWebView,
+        });
+      }
+
+      let webViews: WebViewData[] | undefined;
+
+      if (webViewCount > 0) {
+        const driver = this.getDriver();
+        webViews = [];
+
+        const webView = await new WebViewExtended(group).wait();
+        const parentFlowToElementId = await webView.getParentFlowToElementId();
+
+        // Each webview consists of a container div containing an iframe. The
+        // container divs all get appended as siblings near the end of the
+        // document body and are associated with an editor instance via
+        // `parentFlowToElementId`. Only one of the containers will be visible
+        // for each editor group, corresponding to the selected tab / active
+        // editor instance within the group.
+        const iframeContainers = await driver.findElements(
+          locators.webViewContainer(parentFlowToElementId)
+        );
+
+        for (const container of iframeContainers) {
+          const style = await container.getAttribute('style');
+          const isVisible = style.includes('visibility: visible');
+
+          // Grab current context so we can switch back to it
+          const windowHandle = await driver.getWindowHandle();
+
+          const iframeAccessor = async (): Promise<WebElement> =>
+            container.findElement(By.xpath('.//iframe'));
+
+          let hasContent = false;
+          try {
+            // Attempt to switch to nested content frame to see if content has
+            // been loaded. Nested iframes won't exist if not, and this will fail.
+            // WebView.switchToFrame() only switches to the webview container
+            // that is currently visible. We want to inspect all of them, even
+            // the hidden ones, so we use the `switchToFrame` utility function
+            // instead.
+            await switchToFrame(
+              [
+                iframeAccessor,
+                WebViewExtended.activeFrameSelector,
+                WebViewExtended.contentFrameSelector,
+              ],
+              // If tab is visible, we could possibly be loading for the first time,
+              // so wait a little longer. If tab is invisible, the only scenario
+              // where content would be there is if it has already loaded, so we
+              // don't need to wait as long
+              isVisible ? 3000 : 250
+            );
+            hasContent = true;
+          } catch (err) {
+            // We expect TimeoutErrors in standard case where there is no loaded
+            // content. Other errors indicate something unexpected.
+            if (extractErrorType(err) !== 'TimeoutError') {
+              // eslint-disable-next-line no-console
+              console.log('Err:', err);
+            }
+          } finally {
+            // Reset context
+            await driver.switchTo().window(windowHandle);
+          }
+
+          const webViewData: WebViewData = {};
+
+          // Only set isVisible & hasContent if true to make test comparisons
+          // easier to write / read
+          if (isVisible) {
+            webViewData.isVisible = true;
+          }
+          if (hasContent) {
+            webViewData.hasContent = true;
+          }
+          webViews.push(webViewData);
+        }
+      }
+
+      const groupData: EditorGroupData = {
+        groupIndex: groupIndex++,
+        tabs,
+      };
+
+      if (webViews) {
+        groupData.webViews = webViews;
+      }
+
+      groupDataList.push(groupData);
+    }
+
+    return groupDataList;
+  }
+
+  /**
+   * Switch to a TextEditor tab with the given title + optional group index. Throws
+   * if a matching Editor is found that isn't an instance of a TextEditor.
+   * @param title title of the tab
+   * @param groupIndex zero based index for the editor group (0 for the left most group)
+   * @returns Promise resolving to TextEditor object
+   */
+  async openTextEditor(
+    title: string,
+    groupIndex?: number
+  ): Promise<TextEditor> {
+    const editor = await this.openEditor(title, groupIndex);
+
+    if (!(editor instanceof TextEditor)) {
+      throw new Error('Editor is not a text editor');
+    }
+
+    return editor;
+  }
+
+  /**
+   * Switch to a TextEditor tab with the given title + optional group index.
+   * Throws if a matching Editor is found that isn't associated with a WebView.
+   * @param groupIndex zero based index for the editor group
+   * @param title title of the tab
+   * @returns Promise resolving to `WebViewExtended` object
+   */
+  async openWebView(
+    title: string,
+    groupIndex?: number
+  ): Promise<WebViewExtended> {
+    const group = await this.getEditorGroup(groupIndex ?? 0);
+
+    const tab = await group.getTabByTitle(title);
+    const resourceName = await tab.getAttribute('data-resource-name');
+    const isWebView = resourceName.startsWith('webview-');
+
+    if (!isWebView) {
+      throw new Error('Tab is not associated with a WebView');
+    }
+
+    await tab.select();
+
+    return new WebViewExtended(group).wait();
+  }
+
+  /**
+   * Wait for an editor group to be available
+   * @param groupIndex zero based index for the editor group
+   * @returns Promise resolving to EditorGroup object
+   */
+  async waitForEditorGroup(groupIndex: number): Promise<EditorGroup> {
+    await VSBrowser.instance.driver.wait(
+      async () => (await this.getEditorGroups()).length > groupIndex
+    );
+
+    return this.getEditorGroup(groupIndex);
+  }
+}