From 92100d4a073840c3244267f9f976c03336c7399a Mon Sep 17 00:00:00 2001 From: tech-sushant Date: Wed, 7 May 2025 23:57:30 +0530 Subject: [PATCH 01/15] Feat: Integrate MCP Instrumentation tracking across various tools --- src/index.ts | 16 +++++++++++ src/lib/instrumentation.ts | 53 +++++++++++++++++++++++++++++++++++++ src/lib/utils.ts | 19 +++++++++++++ src/tools/accessibility.ts | 2 ++ src/tools/applive.ts | 2 ++ src/tools/automate.ts | 6 ++++- src/tools/bstack-sdk.ts | 2 ++ src/tools/live.ts | 2 ++ src/tools/observability.ts | 2 ++ src/tools/testmanagement.ts | 3 +++ 10 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 src/lib/instrumentation.ts diff --git a/src/index.ts b/src/index.ts index dc5a675..982c4ed 100644 --- a/src/index.ts +++ b/src/index.ts @@ -12,6 +12,7 @@ import addBrowserLiveTools from "./tools/live"; import addAccessibilityTools from "./tools/accessibility"; import addAutomateTools from "./tools/automate"; import addTestManagementTools from "./tools/testmanagement"; +import { createCustomInitializeHandler } from "./lib/utils"; function registerTools(server: McpServer) { addSDKTools(server); @@ -31,6 +32,19 @@ const server: McpServer = new McpServer({ registerTools(server); +let clientName: string | undefined; + +function setClientName(name: string) { + clientName = name; +} + +const origInitializeHandler = + server.server["_requestHandlers"].get("initialize"); +server.server["_requestHandlers"].set( + "initialize", + createCustomInitializeHandler(origInitializeHandler, logger, setClientName), +); + async function main() { logger.info( "Launching BrowserStack MCP server, version %s", @@ -50,3 +64,5 @@ main().catch(console.error); process.on("exit", () => { logger.flush(); }); + +export { clientName }; diff --git a/src/lib/instrumentation.ts b/src/lib/instrumentation.ts new file mode 100644 index 0000000..06771f5 --- /dev/null +++ b/src/lib/instrumentation.ts @@ -0,0 +1,53 @@ +import logger from "../logger"; +import config from "../config"; +import packageJson from "../../package.json"; +import axios from "axios"; +import { clientName } from "../index"; + +interface MCPEventPayload { + event_type: string; + event_properties: { + mcp_version: string; + tool_name: string; + mcp_client: string; + }; +} + +export function trackMCPEvent(toolName: string): void { + const instrumentationEndpoint = "https://api.browserstack.com/sdk/v1/event"; + const mcpClient = clientName || "unknown"; + + const event: MCPEventPayload = { + event_type: "MCPInstrumentation", + event_properties: { + mcp_version: packageJson.version, + tool_name: toolName, + mcp_client: mcpClient, + }, + }; + + axios + .post(instrumentationEndpoint, event, { + headers: { + "Content-Type": "application/json", + Authorization: `Basic ${Buffer.from( + `${config.browserstackUsername}:${config.browserstackAccessKey}`, + ).toString("base64")}`, + }, + timeout: 2000, + }) + .then((response) => { + logger.info("MCP event tracked successfully", { + toolName, + response, + }); + }) + .catch((error: unknown) => { + logger.warn( + `Failed to track MCP event: ${error instanceof Error ? error.message : String(error)}`, + { + toolName, + }, + ); + }); +} diff --git a/src/lib/utils.ts b/src/lib/utils.ts index 72b75a9..6b9cacb 100644 --- a/src/lib/utils.ts +++ b/src/lib/utils.ts @@ -24,3 +24,22 @@ export interface HarEntry { serverIPAddress?: string; time?: number; } + +export function createCustomInitializeHandler( + origHandler: (request: any, extra: any) => Promise, + logger: any, + setClientName: (name: string) => void, +) { + return async function (this: any, request: any, extra: any) { + const clientInfo = request.params.clientInfo; + if (clientInfo && clientInfo.name) { + setClientName(clientInfo.name); + logger.info( + `Client connected: ${clientInfo.name} (version: ${clientInfo.version})`, + ); + } else { + logger.info("Client connected: unknown client"); + } + return origHandler.call(this, request, extra); + }; +} diff --git a/src/tools/accessibility.ts b/src/tools/accessibility.ts index 513d246..d9468bd 100644 --- a/src/tools/accessibility.ts +++ b/src/tools/accessibility.ts @@ -5,6 +5,7 @@ import { startAccessibilityScan, AccessibilityScanResponse, } from "./accessiblity-utils/accessibility"; +import { trackMCPEvent } from "../lib/instrumentation"; async function runAccessibilityScan( name: string, @@ -55,6 +56,7 @@ export default function addAccessibilityTools(server: McpServer) { pageURL: z.string().describe("The URL to scan for accessibility issues"), }, async (args) => { + trackMCPEvent("startAccessibilityScan") return runAccessibilityScan(args.name, args.pageURL); }, ); diff --git a/src/tools/applive.ts b/src/tools/applive.ts index a7db8be..156fb81 100644 --- a/src/tools/applive.ts +++ b/src/tools/applive.ts @@ -4,6 +4,7 @@ import { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; import fs from "fs"; import { startSession } from "./applive-utils/start-session"; import logger from "../logger"; +import { trackMCPEvent } from "../lib/instrumentation"; /** * Launches an App Live Session on BrowserStack. @@ -89,6 +90,7 @@ export default function addAppLiveTools(server: McpServer) { }, async (args) => { try { + trackMCPEvent("runAppLiveSession"); return startAppLiveSession(args); } catch (error) { return { diff --git a/src/tools/automate.ts b/src/tools/automate.ts index 28f02e4..c8ee322 100644 --- a/src/tools/automate.ts +++ b/src/tools/automate.ts @@ -3,6 +3,7 @@ import { z } from "zod"; import { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; import logger from "../logger"; import { retrieveNetworkFailures } from "../lib/api"; +import { trackMCPEvent } from "../lib/instrumentation"; /** * Fetches failed network requests from a BrowserStack Automate session. @@ -57,6 +58,9 @@ export default function addAutomateTools(server: McpServer) { { sessionId: z.string().describe("The Automate session ID."), }, - getNetworkFailures, + async (args) => { + trackMCPEvent("getNetworkFailures"); + return getNetworkFailures(args); + }, ); } diff --git a/src/tools/bstack-sdk.ts b/src/tools/bstack-sdk.ts index 3b3ea15..2fa1cfd 100644 --- a/src/tools/bstack-sdk.ts +++ b/src/tools/bstack-sdk.ts @@ -10,6 +10,7 @@ import { generateBrowserStackYMLInstructions, getInstructionsForProjectConfiguration, } from "./sdk-utils/instructions"; +import { trackMCPEvent } from "../lib/instrumentation"; /** * BrowserStack SDK hooks into your test framework to seamlessly run tests on BrowserStack. @@ -80,6 +81,7 @@ export default function addSDKTools(server: McpServer) { const desiredPlatforms = args.desiredPlatforms; try { + trackMCPEvent("runTestsOnBrowserStack"); return bootstrapProjectWithSDK({ detectedBrowserAutomationFramework, detectedTestingFramework, diff --git a/src/tools/live.ts b/src/tools/live.ts index 2092685..2436713 100644 --- a/src/tools/live.ts +++ b/src/tools/live.ts @@ -4,6 +4,7 @@ import { z } from "zod"; import logger from "../logger"; import { startBrowserSession } from "./live-utils/start-session"; import { PlatformType } from "./live-utils/types"; +import { trackMCPEvent } from "../lib/instrumentation"; // Define the schema shape const LiveArgsShape = { @@ -118,6 +119,7 @@ export default function addBrowserLiveTools(server: McpServer) { LiveArgsShape, async (args) => { try { + trackMCPEvent("runBrowserLiveSession"); const result = await runBrowserSession(args); return result; } catch (error: any) { diff --git a/src/tools/observability.ts b/src/tools/observability.ts index 3fa5fc0..d82553d 100644 --- a/src/tools/observability.ts +++ b/src/tools/observability.ts @@ -2,6 +2,7 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { z } from "zod"; import { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; import { getLatestO11YBuildInfo } from "../lib/api"; +import { trackMCPEvent } from "../lib/instrumentation"; export async function getFailuresInLastRun( buildName: string, @@ -57,6 +58,7 @@ export default function addObservabilityTools(server: McpServer) { }, async (args) => { try { + trackMCPEvent("getFailuresInLastRun"); return getFailuresInLastRun(args.buildName, args.projectName); } catch (error) { return { diff --git a/src/tools/testmanagement.ts b/src/tools/testmanagement.ts index 1dc925c..110a8c9 100644 --- a/src/tools/testmanagement.ts +++ b/src/tools/testmanagement.ts @@ -1,6 +1,7 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { z } from "zod"; import { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; +import { trackMCPEvent } from "../lib/instrumentation"; import { createProjectOrFolder, CreateProjFoldSchema, @@ -19,6 +20,7 @@ export async function createProjectOrFolderTool( args: z.infer, ): Promise { try { + trackMCPEvent("createProjectOrFolder"); return await createProjectOrFolder(args); } catch (err) { return { @@ -45,6 +47,7 @@ export async function createTestCaseTool( // Sanitize input arguments const cleanedArgs = sanitizeArgs(args); try { + trackMCPEvent("createTestCase"); return await createTestCaseAPI(cleanedArgs); } catch (err) { return { From 03ebc05bde15ce688f443ca3a84c73f5363afb4a Mon Sep 17 00:00:00 2001 From: tech-sushant Date: Thu, 8 May 2025 00:40:28 +0530 Subject: [PATCH 02/15] Modified TC to mock trackMCPEvent --- src/tools/accessibility.ts | 2 +- tests/tools/applive.test.ts | 3 +++ tests/tools/automate.test.ts | 3 +++ tests/tools/live.test.ts | 3 +++ tests/tools/observability.test.ts | 4 ++++ tests/tools/testmanagement.test.ts | 3 +++ 6 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/tools/accessibility.ts b/src/tools/accessibility.ts index d9468bd..a4ee1c1 100644 --- a/src/tools/accessibility.ts +++ b/src/tools/accessibility.ts @@ -56,7 +56,7 @@ export default function addAccessibilityTools(server: McpServer) { pageURL: z.string().describe("The URL to scan for accessibility issues"), }, async (args) => { - trackMCPEvent("startAccessibilityScan") + trackMCPEvent("startAccessibilityScan"); return runAccessibilityScan(args.name, args.pageURL); }, ); diff --git a/tests/tools/applive.test.ts b/tests/tools/applive.test.ts index f36d730..471eebc 100644 --- a/tests/tools/applive.test.ts +++ b/tests/tools/applive.test.ts @@ -15,6 +15,9 @@ jest.mock('../../src/tools/applive-utils/start-session', () => ({ jest.mock('../../src/logger', () => ({ error: jest.fn() })); +jest.mock('../../src/lib/instrumentation', () => ({ + trackMCPEvent: jest.fn() +})); describe('startAppLiveSession', () => { beforeEach(() => { diff --git a/tests/tools/automate.test.ts b/tests/tools/automate.test.ts index 8bd4698..e3aa600 100644 --- a/tests/tools/automate.test.ts +++ b/tests/tools/automate.test.ts @@ -4,6 +4,9 @@ import { retrieveNetworkFailures } from '../../src/lib/api'; jest.mock('../../src/lib/api', () => ({ retrieveNetworkFailures: jest.fn() })); +jest.mock('../../src/lib/instrumentation', () => ({ + trackMCPEvent: jest.fn() +})); jest.mock('../../src/logger', () => ({ error: jest.fn(), info: jest.fn() diff --git a/tests/tools/live.test.ts b/tests/tools/live.test.ts index 8486175..6aee36e 100644 --- a/tests/tools/live.test.ts +++ b/tests/tools/live.test.ts @@ -13,6 +13,9 @@ jest.mock('../../src/lib/local', () => ({ jest.mock('../../src/logger', () => ({ error: jest.fn() })); +jest.mock('../../src/lib/instrumentation', () => ({ + trackMCPEvent: jest.fn() +})); describe('startBrowserLiveSession', () => { let serverMock: any; diff --git a/tests/tools/observability.test.ts b/tests/tools/observability.test.ts index 93e434b..52e7426 100644 --- a/tests/tools/observability.test.ts +++ b/tests/tools/observability.test.ts @@ -6,6 +6,10 @@ jest.mock('../../src/lib/api', () => ({ getLatestO11YBuildInfo: jest.fn(), })); +jest.mock('../../src/lib/instrumentation', () => ({ + trackMCPEvent: jest.fn() +})); + describe('getFailuresInLastRun', () => { beforeEach(() => { jest.clearAllMocks(); diff --git a/tests/tools/testmanagement.test.ts b/tests/tools/testmanagement.test.ts index f7fffac..6d74159 100644 --- a/tests/tools/testmanagement.test.ts +++ b/tests/tools/testmanagement.test.ts @@ -22,6 +22,9 @@ jest.mock('../../src/config', () => ({ browserstackAccessKey: 'fake-key', }, })); +jest.mock('../../src/lib/instrumentation', () => ({ + trackMCPEvent: jest.fn(), +})); describe('createTestCaseTool', () => { beforeEach(() => { From 93e431b6b6220d0665f4f86707e795410d11d3b5 Mon Sep 17 00:00:00 2001 From: tech-sushant Date: Thu, 8 May 2025 14:36:56 +0530 Subject: [PATCH 03/15] Feat: Enhance MCP event tracking --- src/index.ts | 14 -------------- src/lib/instrumentation.ts | 16 +++++++++++++--- src/tools/accessibility.ts | 3 ++- src/tools/applive.ts | 3 ++- src/tools/automate.ts | 3 ++- src/tools/bstack-sdk.ts | 3 ++- src/tools/live.ts | 3 ++- src/tools/observability.ts | 3 ++- src/tools/testmanagement.ts | 9 +++++++-- 9 files changed, 32 insertions(+), 25 deletions(-) diff --git a/src/index.ts b/src/index.ts index 982c4ed..6988ce5 100644 --- a/src/index.ts +++ b/src/index.ts @@ -12,7 +12,6 @@ import addBrowserLiveTools from "./tools/live"; import addAccessibilityTools from "./tools/accessibility"; import addAutomateTools from "./tools/automate"; import addTestManagementTools from "./tools/testmanagement"; -import { createCustomInitializeHandler } from "./lib/utils"; function registerTools(server: McpServer) { addSDKTools(server); @@ -32,18 +31,6 @@ const server: McpServer = new McpServer({ registerTools(server); -let clientName: string | undefined; - -function setClientName(name: string) { - clientName = name; -} - -const origInitializeHandler = - server.server["_requestHandlers"].get("initialize"); -server.server["_requestHandlers"].set( - "initialize", - createCustomInitializeHandler(origInitializeHandler, logger, setClientName), -); async function main() { logger.info( @@ -65,4 +52,3 @@ process.on("exit", () => { logger.flush(); }); -export { clientName }; diff --git a/src/lib/instrumentation.ts b/src/lib/instrumentation.ts index 06771f5..da9d761 100644 --- a/src/lib/instrumentation.ts +++ b/src/lib/instrumentation.ts @@ -2,7 +2,6 @@ import logger from "../logger"; import config from "../config"; import packageJson from "../../package.json"; import axios from "axios"; -import { clientName } from "../index"; interface MCPEventPayload { event_type: string; @@ -13,9 +12,20 @@ interface MCPEventPayload { }; } -export function trackMCPEvent(toolName: string): void { +export function trackMCPEvent( + toolName: string, + clientInfo: { name?: string; version?: string }, +): void { const instrumentationEndpoint = "https://api.browserstack.com/sdk/v1/event"; - const mcpClient = clientName || "unknown"; + + const mcpClient = clientInfo?.name || "unknown"; + if (clientInfo?.name) { + logger.info( + `Client connected: ${clientInfo.name} (version: ${clientInfo.version})`, + ); + } else { + logger.info("Client connected: unknown client"); + } const event: MCPEventPayload = { event_type: "MCPInstrumentation", diff --git a/src/tools/accessibility.ts b/src/tools/accessibility.ts index a4ee1c1..dedf41a 100644 --- a/src/tools/accessibility.ts +++ b/src/tools/accessibility.ts @@ -56,7 +56,8 @@ export default function addAccessibilityTools(server: McpServer) { pageURL: z.string().describe("The URL to scan for accessibility issues"), }, async (args) => { - trackMCPEvent("startAccessibilityScan"); + const clientInfo = server.server.getClientVersion(); + trackMCPEvent("startAccessibilityScan", clientInfo!); return runAccessibilityScan(args.name, args.pageURL); }, ); diff --git a/src/tools/applive.ts b/src/tools/applive.ts index 156fb81..622b149 100644 --- a/src/tools/applive.ts +++ b/src/tools/applive.ts @@ -90,7 +90,8 @@ export default function addAppLiveTools(server: McpServer) { }, async (args) => { try { - trackMCPEvent("runAppLiveSession"); + const clientInfo = server.server.getClientVersion(); + trackMCPEvent("runAppLiveSession", clientInfo!); return startAppLiveSession(args); } catch (error) { return { diff --git a/src/tools/automate.ts b/src/tools/automate.ts index c8ee322..d801eee 100644 --- a/src/tools/automate.ts +++ b/src/tools/automate.ts @@ -59,7 +59,8 @@ export default function addAutomateTools(server: McpServer) { sessionId: z.string().describe("The Automate session ID."), }, async (args) => { - trackMCPEvent("getNetworkFailures"); + const clientInfo = server.server.getClientVersion(); + trackMCPEvent("startAccessibilityScan", clientInfo!); return getNetworkFailures(args); }, ); diff --git a/src/tools/bstack-sdk.ts b/src/tools/bstack-sdk.ts index 2fa1cfd..48fa64e 100644 --- a/src/tools/bstack-sdk.ts +++ b/src/tools/bstack-sdk.ts @@ -81,7 +81,8 @@ export default function addSDKTools(server: McpServer) { const desiredPlatforms = args.desiredPlatforms; try { - trackMCPEvent("runTestsOnBrowserStack"); + const clientInfo = server.server.getClientVersion(); + trackMCPEvent("runTestsOnBrowserStack", clientInfo!); return bootstrapProjectWithSDK({ detectedBrowserAutomationFramework, detectedTestingFramework, diff --git a/src/tools/live.ts b/src/tools/live.ts index 2436713..bdf2af6 100644 --- a/src/tools/live.ts +++ b/src/tools/live.ts @@ -119,7 +119,8 @@ export default function addBrowserLiveTools(server: McpServer) { LiveArgsShape, async (args) => { try { - trackMCPEvent("runBrowserLiveSession"); + const clientInfo = server.server.getClientVersion(); + trackMCPEvent("runBrowserLiveSession", clientInfo!); const result = await runBrowserSession(args); return result; } catch (error: any) { diff --git a/src/tools/observability.ts b/src/tools/observability.ts index d82553d..2aa34ae 100644 --- a/src/tools/observability.ts +++ b/src/tools/observability.ts @@ -58,7 +58,8 @@ export default function addObservabilityTools(server: McpServer) { }, async (args) => { try { - trackMCPEvent("getFailuresInLastRun"); + const clientInfo = server.server.getClientVersion(); + trackMCPEvent("getFailuresInLastRun", clientInfo!); return getFailuresInLastRun(args.buildName, args.projectName); } catch (error) { return { diff --git a/src/tools/testmanagement.ts b/src/tools/testmanagement.ts index 110a8c9..d2da9d9 100644 --- a/src/tools/testmanagement.ts +++ b/src/tools/testmanagement.ts @@ -13,6 +13,8 @@ import { CreateTestCaseSchema, } from "./testmanagement-utils/create-testcase"; +let serverInstance: McpServer; + /** * Wrapper to call createProjectOrFolder util. */ @@ -20,7 +22,8 @@ export async function createProjectOrFolderTool( args: z.infer, ): Promise { try { - trackMCPEvent("createProjectOrFolder"); + const clientInfo = serverInstance.server.getClientVersion(); + trackMCPEvent("createProjectOrFolder", clientInfo!); return await createProjectOrFolder(args); } catch (err) { return { @@ -47,7 +50,8 @@ export async function createTestCaseTool( // Sanitize input arguments const cleanedArgs = sanitizeArgs(args); try { - trackMCPEvent("createTestCase"); + const clientInfo = serverInstance.server.getClientVersion(); + trackMCPEvent("createTestCase", clientInfo!); return await createTestCaseAPI(cleanedArgs); } catch (err) { return { @@ -69,6 +73,7 @@ export async function createTestCaseTool( * Registers both project/folder and test-case tools. */ export default function addTestManagementTools(server: McpServer) { + serverInstance = server; server.tool( "createProjectOrFolder", "Create a project and/or folder in BrowserStack Test Management.", From 66e2a0c6df0f797024f50b25eba89faea23d0248 Mon Sep 17 00:00:00 2001 From: tech-sushant Date: Thu, 8 May 2025 14:38:04 +0530 Subject: [PATCH 04/15] Update index.ts --- src/index.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/index.ts b/src/index.ts index 6988ce5..dc5a675 100644 --- a/src/index.ts +++ b/src/index.ts @@ -31,7 +31,6 @@ const server: McpServer = new McpServer({ registerTools(server); - async function main() { logger.info( "Launching BrowserStack MCP server, version %s", @@ -51,4 +50,3 @@ main().catch(console.error); process.on("exit", () => { logger.flush(); }); - From 2cd40993a82566187aac406f1f6d1e6fd121fb39 Mon Sep 17 00:00:00 2001 From: tech-sushant Date: Thu, 8 May 2025 14:39:31 +0530 Subject: [PATCH 05/15] Update utils.ts --- src/lib/utils.ts | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/src/lib/utils.ts b/src/lib/utils.ts index 6b9cacb..72b75a9 100644 --- a/src/lib/utils.ts +++ b/src/lib/utils.ts @@ -24,22 +24,3 @@ export interface HarEntry { serverIPAddress?: string; time?: number; } - -export function createCustomInitializeHandler( - origHandler: (request: any, extra: any) => Promise, - logger: any, - setClientName: (name: string) => void, -) { - return async function (this: any, request: any, extra: any) { - const clientInfo = request.params.clientInfo; - if (clientInfo && clientInfo.name) { - setClientName(clientInfo.name); - logger.info( - `Client connected: ${clientInfo.name} (version: ${clientInfo.version})`, - ); - } else { - logger.info("Client connected: unknown client"); - } - return origHandler.call(this, request, extra); - }; -} From 34a07d96e81bff0a1860ed81be019b640ca96e94 Mon Sep 17 00:00:00 2001 From: tech-sushant Date: Thu, 8 May 2025 14:58:51 +0530 Subject: [PATCH 06/15] Mocking tests for server --- tests/tools/live.test.ts | 3 +++ tests/tools/testmanagement.test.ts | 17 +++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/tests/tools/live.test.ts b/tests/tools/live.test.ts index 6aee36e..2df6769 100644 --- a/tests/tools/live.test.ts +++ b/tests/tools/live.test.ts @@ -26,6 +26,9 @@ describe('startBrowserLiveSession', () => { tool: jest.fn((name, desc, schema, handler) => { serverMock.handler = handler; }), + server: { + getClientVersion: jest.fn().mockReturnValue({ version: '1.0.0' }) + } }; addBrowserLiveTools(serverMock); diff --git a/tests/tools/testmanagement.test.ts b/tests/tools/testmanagement.test.ts index 6d74159..f352a49 100644 --- a/tests/tools/testmanagement.test.ts +++ b/tests/tools/testmanagement.test.ts @@ -2,6 +2,8 @@ import { createProjectOrFolderTool } from '../../src/tools/testmanagement'; import { createProjectOrFolder } from '../../src/tools/testmanagement-utils/create-project-folder'; import { createTestCaseTool } from '../../src/tools/testmanagement'; import { createTestCase, sanitizeArgs, TestCaseCreateRequest } from '../../src/tools/testmanagement-utils/create-testcase'; +import addTestManagementTools from '../../src/tools/testmanagement'; +import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; // Mock dependencies jest.mock('../../src/tools/testmanagement-utils/create-project-folder', () => ({ @@ -14,6 +16,9 @@ jest.mock('../../src/tools/testmanagement-utils/create-project-folder', () => ({ jest.mock('../../src/tools/testmanagement-utils/create-testcase', () => ({ createTestCase: jest.fn(), sanitizeArgs: jest.fn((args) => args), + CreateTestCaseSchema: { + shape: {}, + }, })); jest.mock('../../src/config', () => ({ __esModule: true, @@ -26,6 +31,18 @@ jest.mock('../../src/lib/instrumentation', () => ({ trackMCPEvent: jest.fn(), })); +const mockServer = { + tool: jest.fn(), + server: { + getClientVersion: jest.fn(() => ({ + name: 'jest-client', + version: '1.0.0', + })), + }, +} as unknown as McpServer; + +addTestManagementTools(mockServer); + describe('createTestCaseTool', () => { beforeEach(() => { jest.clearAllMocks(); From 819fe6d6b83d736a1269e55f9990800b34745008 Mon Sep 17 00:00:00 2001 From: tech-sushant Date: Thu, 8 May 2025 16:18:50 +0530 Subject: [PATCH 07/15] Feat: Add failure tracking for MCP events across various tools --- src/lib/instrumentation.ts | 55 ++++++++++++++++++++++++++ src/tools/accessibility.ts | 72 +++++++++++++++++----------------- src/tools/applive.ts | 23 +++++------ src/tools/automate.ts | 78 ++++++++++++++++++------------------- src/tools/bstack-sdk.ts | 26 ++++++------- src/tools/live.ts | 52 +++++++++---------------- src/tools/observability.ts | 10 +++-- src/tools/testmanagement.ts | 13 ++++--- 8 files changed, 184 insertions(+), 145 deletions(-) diff --git a/src/lib/instrumentation.ts b/src/lib/instrumentation.ts index da9d761..419ace6 100644 --- a/src/lib/instrumentation.ts +++ b/src/lib/instrumentation.ts @@ -9,6 +9,9 @@ interface MCPEventPayload { mcp_version: string; tool_name: string; mcp_client: string; + success?: boolean; + error_message?: string; + error_type?: string; }; } @@ -33,6 +36,7 @@ export function trackMCPEvent( mcp_version: packageJson.version, tool_name: toolName, mcp_client: mcpClient, + success: true, }, }; @@ -61,3 +65,54 @@ export function trackMCPEvent( ); }); } + +export function trackMCPFailure( + toolName: string, + error: unknown, + clientInfo: { name?: string; version?: string }, +): void { + const instrumentationEndpoint = "https://api.browserstack.com/sdk/v1/event"; + + const mcpClient = clientInfo?.name || "unknown"; + const errorMessage = error instanceof Error ? error.message : String(error); + const errorType = error instanceof Error ? error.constructor.name : "Unknown"; + + logger.error(`Tool failure: ${toolName} - ${errorMessage}`, { errorType }); + + const event: MCPEventPayload = { + event_type: "MCPInstrumentation", + event_properties: { + mcp_version: packageJson.version, + tool_name: toolName, + mcp_client: mcpClient, + success: false, + error_message: errorMessage, + error_type: errorType, + }, + }; + + axios + .post(instrumentationEndpoint, event, { + headers: { + "Content-Type": "application/json", + Authorization: `Basic ${Buffer.from( + `${config.browserstackUsername}:${config.browserstackAccessKey}`, + ).toString("base64")}`, + }, + timeout: 2000, + }) + .then((response) => { + logger.info("MCP failure event tracked successfully", { + toolName, + response, + }); + }) + .catch((error: unknown) => { + logger.warn( + `Failed to track MCP failure event: ${error instanceof Error ? error.message : String(error)}`, + { + toolName, + }, + ); + }); +} \ No newline at end of file diff --git a/src/tools/accessibility.ts b/src/tools/accessibility.ts index dedf41a..fc0058d 100644 --- a/src/tools/accessibility.ts +++ b/src/tools/accessibility.ts @@ -5,46 +5,33 @@ import { startAccessibilityScan, AccessibilityScanResponse, } from "./accessiblity-utils/accessibility"; -import { trackMCPEvent } from "../lib/instrumentation"; +import { trackMCPEvent, trackMCPFailure } from "../lib/instrumentation"; async function runAccessibilityScan( name: string, pageURL: string, ): Promise { - try { - const response: AccessibilityScanResponse = await startAccessibilityScan( - name, - [pageURL], - ); - const scanId = response.data?.id; - const scanRunId = response.data?.scanRunId; - - if (!scanId || !scanRunId) { - throw new Error( - "Unable to start a accessibility scan, please try again later or open an issue on GitHub if the problem persists", - ); - } + const response: AccessibilityScanResponse = await startAccessibilityScan( + name, + [pageURL], + ); + const scanId = response.data?.id; + const scanRunId = response.data?.scanRunId; - return { - content: [ - { - type: "text", - text: `Successfully queued accessibility scan, you will get a report via email within 5 minutes.`, - }, - ], - }; - } catch (error) { - return { - content: [ - { - type: "text", - text: `Failed to start accessibility scan: ${error instanceof Error ? error.message : "Unknown error"}. Please open an issue on GitHub if the problem persists`, - isError: true, - }, - ], - isError: true, - }; + if (!scanId || !scanRunId) { + throw new Error( + "Unable to start a accessibility scan, please try again later or open an issue on GitHub if the problem persists", + ); } + + return { + content: [ + { + type: "text", + text: `Successfully queued accessibility scan, you will get a report via email within 5 minutes.`, + }, + ], + }; } export default function addAccessibilityTools(server: McpServer) { @@ -56,9 +43,22 @@ export default function addAccessibilityTools(server: McpServer) { pageURL: z.string().describe("The URL to scan for accessibility issues"), }, async (args) => { - const clientInfo = server.server.getClientVersion(); - trackMCPEvent("startAccessibilityScan", clientInfo!); - return runAccessibilityScan(args.name, args.pageURL); + try { + trackMCPEvent("startAccessibilityScan", server.server.getClientVersion()!); + return await runAccessibilityScan(args.name, args.pageURL); + } catch (error) { + trackMCPFailure("startAccessibilityScan", error, server.server.getClientVersion()!); + return { + content: [ + { + type: "text", + text: `Failed to start accessibility scan: ${error instanceof Error ? error.message : "Unknown error"}. Please open an issue on GitHub if the problem persists`, + isError: true, + }, + ], + isError: true, + }; + } }, ); } diff --git a/src/tools/applive.ts b/src/tools/applive.ts index 622b149..d45422a 100644 --- a/src/tools/applive.ts +++ b/src/tools/applive.ts @@ -4,7 +4,7 @@ import { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; import fs from "fs"; import { startSession } from "./applive-utils/start-session"; import logger from "../logger"; -import { trackMCPEvent } from "../lib/instrumentation"; +import { trackMCPEvent, trackMCPFailure } from "../lib/instrumentation"; /** * Launches an App Live Session on BrowserStack. @@ -34,16 +34,12 @@ export async function startAppLiveSession(args: { if (args.desiredPlatform === "ios" && !args.appPath.endsWith(".ipa")) { throw new Error("You must provide a valid iOS app path."); } + // check if the app path exists && is readable - try { - if (!fs.existsSync(args.appPath)) { - throw new Error("The app path does not exist."); - } - fs.accessSync(args.appPath, fs.constants.R_OK); - } catch (error) { - logger.error("The app path does not exist or is not readable: %s", error); - throw new Error("The app path does not exist or is not readable."); + if (!fs.existsSync(args.appPath)) { + throw new Error("The app path does not exist."); } + fs.accessSync(args.appPath, fs.constants.R_OK); const launchUrl = await startSession({ appPath: args.appPath, @@ -90,15 +86,16 @@ export default function addAppLiveTools(server: McpServer) { }, async (args) => { try { - const clientInfo = server.server.getClientVersion(); - trackMCPEvent("runAppLiveSession", clientInfo!); - return startAppLiveSession(args); + trackMCPEvent("runAppLiveSession", server.server.getClientVersion()!); + return await startAppLiveSession(args); } catch (error) { + logger.error("App live session failed: %s", error); + trackMCPFailure("runAppLiveSession", error, server.server.getClientVersion()!); return { content: [ { type: "text", - text: `Failed to start an app live session. Error: ${error}. Please open an issue on GitHub if the problem persists`, + text: `Failed to start app live session: ${error instanceof Error ? error.message : String(error)}`, isError: true, }, ], diff --git a/src/tools/automate.ts b/src/tools/automate.ts index d801eee..feae250 100644 --- a/src/tools/automate.ts +++ b/src/tools/automate.ts @@ -3,7 +3,7 @@ import { z } from "zod"; import { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; import logger from "../logger"; import { retrieveNetworkFailures } from "../lib/api"; -import { trackMCPEvent } from "../lib/instrumentation"; +import { trackMCPEvent, trackMCPFailure } from "../lib/instrumentation"; /** * Fetches failed network requests from a BrowserStack Automate session. @@ -12,43 +12,26 @@ import { trackMCPEvent } from "../lib/instrumentation"; export async function getNetworkFailures(args: { sessionId: string; }): Promise { - try { - const failureLogs = await retrieveNetworkFailures(args.sessionId); - logger.info( - "Successfully fetched failure network logs for session: %s", - args.sessionId, - ); - - // Check if there are any failures - const hasFailures = failureLogs.totalFailures > 0; - const text = hasFailures - ? `${failureLogs.totalFailures} network failure(s) found for session :\n\n${JSON.stringify(failureLogs.failures, null, 2)}` - : `No network failures found for session`; + const failureLogs = await retrieveNetworkFailures(args.sessionId); + logger.info( + "Successfully fetched failure network logs for session: %s", + args.sessionId, + ); - return { - content: [ - { - type: "text", - text, - }, - ], - }; - } catch (error) { - const errorMessage = - error instanceof Error ? error.message : "An unknown error occurred"; - logger.error("Failed to fetch network logs: %s", errorMessage); + // Check if there are any failures + const hasFailures = failureLogs.totalFailures > 0; + const text = hasFailures + ? `${failureLogs.totalFailures} network failure(s) found for session :\n\n${JSON.stringify(failureLogs.failures, null, 2)}` + : `No network failures found for session`; - return { - content: [ - { - type: "text", - text: `Failed to fetch network logs: ${errorMessage}`, - isError: true, - }, - ], - isError: true, - }; - } + return { + content: [ + { + type: "text", + text, + }, + ], + }; } export default function addAutomateTools(server: McpServer) { @@ -59,9 +42,26 @@ export default function addAutomateTools(server: McpServer) { sessionId: z.string().describe("The Automate session ID."), }, async (args) => { - const clientInfo = server.server.getClientVersion(); - trackMCPEvent("startAccessibilityScan", clientInfo!); - return getNetworkFailures(args); + try { + trackMCPEvent("getNetworkFailures", server.server.getClientVersion()!); + return await getNetworkFailures(args); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + logger.error("Failed to fetch network logs: %s", errorMessage); + + trackMCPFailure("getNetworkFailures", error, server.server.getClientVersion()!); + + return { + content: [ + { + type: "text", + text: `Failed to fetch network logs: ${errorMessage}`, + isError: true, + }, + ], + isError: true, + }; + } }, ); } diff --git a/src/tools/bstack-sdk.ts b/src/tools/bstack-sdk.ts index 48fa64e..be15870 100644 --- a/src/tools/bstack-sdk.ts +++ b/src/tools/bstack-sdk.ts @@ -10,7 +10,7 @@ import { generateBrowserStackYMLInstructions, getInstructionsForProjectConfiguration, } from "./sdk-utils/instructions"; -import { trackMCPEvent } from "../lib/instrumentation"; +import { trackMCPEvent, trackMCPFailure } from "../lib/instrumentation"; /** * BrowserStack SDK hooks into your test framework to seamlessly run tests on BrowserStack. @@ -73,23 +73,19 @@ export default function addSDKTools(server: McpServer) { ), }, async (args) => { - const detectedBrowserAutomationFramework = - args.detectedBrowserAutomationFramework as SDKSupportedBrowserAutomationFramework; - const detectedTestingFramework = - args.detectedTestingFramework as SDKSupportedTestingFramework; - const detectedLanguage = args.detectedLanguage as SDKSupportedLanguage; - const desiredPlatforms = args.desiredPlatforms; - try { - const clientInfo = server.server.getClientVersion(); - trackMCPEvent("runTestsOnBrowserStack", clientInfo!); - return bootstrapProjectWithSDK({ - detectedBrowserAutomationFramework, - detectedTestingFramework, - detectedLanguage, - desiredPlatforms, + trackMCPEvent("runTestsOnBrowserStack", server.server.getClientVersion()!); + + return await bootstrapProjectWithSDK({ + detectedBrowserAutomationFramework: + args.detectedBrowserAutomationFramework as SDKSupportedBrowserAutomationFramework, + detectedTestingFramework: + args.detectedTestingFramework as SDKSupportedTestingFramework, + detectedLanguage: args.detectedLanguage as SDKSupportedLanguage, + desiredPlatforms: args.desiredPlatforms, }); } catch (error) { + trackMCPFailure("runTestsOnBrowserStack", error, server.server.getClientVersion()!); return { content: [ { diff --git a/src/tools/live.ts b/src/tools/live.ts index bdf2af6..8b53e08 100644 --- a/src/tools/live.ts +++ b/src/tools/live.ts @@ -4,7 +4,7 @@ import { z } from "zod"; import logger from "../logger"; import { startBrowserSession } from "./live-utils/start-session"; import { PlatformType } from "./live-utils/types"; -import { trackMCPEvent } from "../lib/instrumentation"; +import { trackMCPEvent, trackMCPFailure } from "../lib/instrumentation"; // Define the schema shape const LiveArgsShape = { @@ -82,34 +82,20 @@ async function runBrowserSession(rawArgs: any) { // Validate and narrow const args = LiveArgsSchema.parse(rawArgs); - try { - // Branch desktop vs mobile and delegate - const launchUrl = - args.platformType === PlatformType.DESKTOP - ? await launchDesktopSession(args) - : await launchMobileSession(args); + // Branch desktop vs mobile and delegate + const launchUrl = + args.platformType === PlatformType.DESKTOP + ? await launchDesktopSession(args) + : await launchMobileSession(args); - return { - content: [ - { - type: "text" as const, - text: `✅ Session started. If it didn't open automatically, visit:\n${launchUrl}`, - }, - ], - }; - } catch (err: any) { - logger.error("Live session failed: %s", err); - return { - content: [ - { - type: "text" as const, - text: `❌ Failed to start session: ${err.message || err}`, - isError: true, - }, - ], - isError: true, - }; - } + return { + content: [ + { + type: "text" as const, + text: `✅ Session started. If it didn't open automatically, visit:\n${launchUrl}`, + }, + ], + }; } export default function addBrowserLiveTools(server: McpServer) { @@ -119,11 +105,11 @@ export default function addBrowserLiveTools(server: McpServer) { LiveArgsShape, async (args) => { try { - const clientInfo = server.server.getClientVersion(); - trackMCPEvent("runBrowserLiveSession", clientInfo!); - const result = await runBrowserSession(args); - return result; - } catch (error: any) { + trackMCPEvent("runBrowserLiveSession", server.server.getClientVersion()!); + return await runBrowserSession(args); + } catch (error) { + logger.error("Live session failed: %s", error); + trackMCPFailure("runBrowserLiveSession", error, server.server.getClientVersion()!); return { content: [ { diff --git a/src/tools/observability.ts b/src/tools/observability.ts index 2aa34ae..bc86909 100644 --- a/src/tools/observability.ts +++ b/src/tools/observability.ts @@ -2,7 +2,8 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { z } from "zod"; import { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; import { getLatestO11YBuildInfo } from "../lib/api"; -import { trackMCPEvent } from "../lib/instrumentation"; +import { trackMCPEvent, trackMCPFailure } from "../lib/instrumentation"; +import logger from "../logger"; export async function getFailuresInLastRun( buildName: string, @@ -58,10 +59,11 @@ export default function addObservabilityTools(server: McpServer) { }, async (args) => { try { - const clientInfo = server.server.getClientVersion(); - trackMCPEvent("getFailuresInLastRun", clientInfo!); - return getFailuresInLastRun(args.buildName, args.projectName); + trackMCPEvent("getFailuresInLastRun", server.server.getClientVersion()!); + return await getFailuresInLastRun(args.buildName, args.projectName); } catch (error) { + logger.error("Failed to get failures in the last run: %s", error); + trackMCPFailure("getFailuresInLastRun", error, server.server.getClientVersion()!); return { content: [ { diff --git a/src/tools/testmanagement.ts b/src/tools/testmanagement.ts index d2da9d9..a9e2890 100644 --- a/src/tools/testmanagement.ts +++ b/src/tools/testmanagement.ts @@ -1,7 +1,8 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { z } from "zod"; import { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; -import { trackMCPEvent } from "../lib/instrumentation"; +import { trackMCPEvent, trackMCPFailure } from "../lib/instrumentation"; +import logger from "../logger"; import { createProjectOrFolder, CreateProjFoldSchema, @@ -22,10 +23,11 @@ export async function createProjectOrFolderTool( args: z.infer, ): Promise { try { - const clientInfo = serverInstance.server.getClientVersion(); - trackMCPEvent("createProjectOrFolder", clientInfo!); + trackMCPEvent("createProjectOrFolder", serverInstance.server.getClientVersion()!); return await createProjectOrFolder(args); } catch (err) { + logger.error("Failed to create project/folder: %s", err); + trackMCPFailure("createProjectOrFolder", err, serverInstance.server.getClientVersion()!); return { content: [ { @@ -50,10 +52,11 @@ export async function createTestCaseTool( // Sanitize input arguments const cleanedArgs = sanitizeArgs(args); try { - const clientInfo = serverInstance.server.getClientVersion(); - trackMCPEvent("createTestCase", clientInfo!); + trackMCPEvent("createTestCase", serverInstance.server.getClientVersion()!); return await createTestCaseAPI(cleanedArgs); } catch (err) { + logger.error("Failed to create test case: %s", err); + trackMCPFailure("createTestCase", err, serverInstance.server.getClientVersion()!); return { content: [ { From 1bf372093e58cb93627e16e483e26f01dbb5fb55 Mon Sep 17 00:00:00 2001 From: tech-sushant Date: Thu, 8 May 2025 17:01:08 +0530 Subject: [PATCH 08/15] Fix: Adding MCPFailure tracking mock --- src/tools/applive.ts | 9 +++++++-- tests/tools/applive.test.ts | 3 ++- tests/tools/automate.test.ts | 23 +++++++++++++++++++---- tests/tools/live.test.ts | 5 +++-- tests/tools/observability.test.ts | 3 ++- tests/tools/testmanagement.test.ts | 1 + 6 files changed, 34 insertions(+), 10 deletions(-) diff --git a/src/tools/applive.ts b/src/tools/applive.ts index d45422a..576428e 100644 --- a/src/tools/applive.ts +++ b/src/tools/applive.ts @@ -37,9 +37,14 @@ export async function startAppLiveSession(args: { // check if the app path exists && is readable if (!fs.existsSync(args.appPath)) { - throw new Error("The app path does not exist."); + throw new Error("The app path does not exist"); + } + + try { + fs.accessSync(args.appPath, fs.constants.R_OK); + } catch (error) { + throw new Error("The app path does not exist or is not readable"); } - fs.accessSync(args.appPath, fs.constants.R_OK); const launchUrl = await startSession({ appPath: args.appPath, diff --git a/tests/tools/applive.test.ts b/tests/tools/applive.test.ts index 471eebc..f16ed16 100644 --- a/tests/tools/applive.test.ts +++ b/tests/tools/applive.test.ts @@ -16,7 +16,8 @@ jest.mock('../../src/logger', () => ({ error: jest.fn() })); jest.mock('../../src/lib/instrumentation', () => ({ - trackMCPEvent: jest.fn() + trackMCPEvent: jest.fn(), + trackMCPFailure: jest.fn() })); describe('startAppLiveSession', () => { diff --git a/tests/tools/automate.test.ts b/tests/tools/automate.test.ts index e3aa600..3917390 100644 --- a/tests/tools/automate.test.ts +++ b/tests/tools/automate.test.ts @@ -1,11 +1,13 @@ import { getNetworkFailures } from '../../src/tools/automate'; import { retrieveNetworkFailures } from '../../src/lib/api'; +import addAutomateTools from '../../src/tools/automate'; jest.mock('../../src/lib/api', () => ({ retrieveNetworkFailures: jest.fn() })); jest.mock('../../src/lib/instrumentation', () => ({ - trackMCPEvent: jest.fn() + trackMCPEvent: jest.fn(), + trackMCPFailure: jest.fn() })); jest.mock('../../src/logger', () => ({ error: jest.fn(), @@ -26,10 +28,23 @@ describe('getNetworkFailures', () => { ], totalFailures: 1 }; + + let serverMock: any; beforeEach(() => { jest.clearAllMocks(); (retrieveNetworkFailures as jest.Mock).mockResolvedValue(mockFailures); + + serverMock = { + tool: jest.fn((name, desc, schema, handler) => { + serverMock.handler = handler; + }), + server: { + getClientVersion: jest.fn().mockReturnValue({ name: 'test-client', version: '1.0.0' }) + } + }; + + addAutomateTools(serverMock); }); it('should return failure logs when present', async () => { @@ -44,13 +59,13 @@ describe('getNetworkFailures', () => { (retrieveNetworkFailures as jest.Mock).mockResolvedValue({ failures: [], totalFailures: 0 }); const result = await getNetworkFailures({ sessionId: validSessionId }); expect(retrieveNetworkFailures).toHaveBeenCalledWith(validSessionId); - expect(result.content[0].text).toContain('No network failures found for sessio'); + expect(result.content[0].text).toContain('No network failures found for session'); expect(result.isError).toBeFalsy(); }); it('should handle errors from the API', async () => { (retrieveNetworkFailures as jest.Mock).mockRejectedValue(new Error('Invalid session ID')); - const result = await getNetworkFailures({ sessionId: 'invalid-id' }); + const result = await serverMock.handler({ sessionId: 'invalid-id' }); expect(retrieveNetworkFailures).toHaveBeenCalledWith('invalid-id'); expect(result.content[0].text).toBe('Failed to fetch network logs: Invalid session ID'); expect(result.content[0].isError).toBe(true); @@ -59,7 +74,7 @@ describe('getNetworkFailures', () => { it('should handle empty session ID', async () => { (retrieveNetworkFailures as jest.Mock).mockRejectedValue(new Error('Session ID is required')); - const result = await getNetworkFailures({ sessionId: '' }); + const result = await serverMock.handler({ sessionId: '' }); expect(retrieveNetworkFailures).toHaveBeenCalledWith(''); expect(result.content[0].text).toBe('Failed to fetch network logs: Session ID is required'); expect(result.content[0].isError).toBe(true); diff --git a/tests/tools/live.test.ts b/tests/tools/live.test.ts index 2df6769..03a9e4c 100644 --- a/tests/tools/live.test.ts +++ b/tests/tools/live.test.ts @@ -14,7 +14,8 @@ jest.mock('../../src/logger', () => ({ error: jest.fn() })); jest.mock('../../src/lib/instrumentation', () => ({ - trackMCPEvent: jest.fn() + trackMCPEvent: jest.fn(), + trackMCPFailure: jest.fn() })); describe('startBrowserLiveSession', () => { @@ -73,7 +74,7 @@ describe('startBrowserLiveSession', () => { (startBrowserSession as jest.Mock).mockRejectedValue(new Error('Session start failed')); const result = await serverMock.handler(validDesktopArgs); expect(logger.error).toHaveBeenCalled(); - expect(result.content[0].text).toContain('❌ Failed to start session'); + expect(result.content[0].text).toContain('Failed to start a browser live session'); }); it('should fail on schema validation error (missing desiredBrowser)', async () => { diff --git a/tests/tools/observability.test.ts b/tests/tools/observability.test.ts index 52e7426..d67561d 100644 --- a/tests/tools/observability.test.ts +++ b/tests/tools/observability.test.ts @@ -7,7 +7,8 @@ jest.mock('../../src/lib/api', () => ({ })); jest.mock('../../src/lib/instrumentation', () => ({ - trackMCPEvent: jest.fn() + trackMCPEvent: jest.fn(), + trackMCPFailure: jest.fn() })); describe('getFailuresInLastRun', () => { diff --git a/tests/tools/testmanagement.test.ts b/tests/tools/testmanagement.test.ts index f352a49..434c2cc 100644 --- a/tests/tools/testmanagement.test.ts +++ b/tests/tools/testmanagement.test.ts @@ -29,6 +29,7 @@ jest.mock('../../src/config', () => ({ })); jest.mock('../../src/lib/instrumentation', () => ({ trackMCPEvent: jest.fn(), + trackMCPFailure: jest.fn() })); const mockServer = { From 492319f7b6781bcc9e878e8dc0234d295e8626bc Mon Sep 17 00:00:00 2001 From: tech-sushant Date: Thu, 8 May 2025 17:04:07 +0530 Subject: [PATCH 09/15] Minor Fix --- src/tools/applive.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/tools/applive.ts b/src/tools/applive.ts index 576428e..ba9683a 100644 --- a/src/tools/applive.ts +++ b/src/tools/applive.ts @@ -43,7 +43,8 @@ export async function startAppLiveSession(args: { try { fs.accessSync(args.appPath, fs.constants.R_OK); } catch (error) { - throw new Error("The app path does not exist or is not readable"); + logger.error("The app path does not exist or is not readable: %s", error); + throw new Error("The app path does not exist or is not readable."); } const launchUrl = await startSession({ From 95e5536e5a5bc451956b29d0bf4b617e82efcc4b Mon Sep 17 00:00:00 2001 From: tech-sushant Date: Thu, 8 May 2025 17:26:42 +0530 Subject: [PATCH 10/15] Code formatting --- src/lib/instrumentation.ts | 2 +- src/tools/accessibility.ts | 11 ++++++-- src/tools/applive.ts | 15 ++++++----- src/tools/automate.ts | 54 ++++++++++++++++++++++--------------- src/tools/bstack-sdk.ts | 13 ++++++--- src/tools/live.ts | 11 ++++++-- src/tools/observability.ts | 11 ++++++-- src/tools/testmanagement.ts | 17 +++++++++--- 8 files changed, 93 insertions(+), 41 deletions(-) diff --git a/src/lib/instrumentation.ts b/src/lib/instrumentation.ts index 419ace6..2850c6b 100644 --- a/src/lib/instrumentation.ts +++ b/src/lib/instrumentation.ts @@ -115,4 +115,4 @@ export function trackMCPFailure( }, ); }); -} \ No newline at end of file +} diff --git a/src/tools/accessibility.ts b/src/tools/accessibility.ts index fc0058d..b260810 100644 --- a/src/tools/accessibility.ts +++ b/src/tools/accessibility.ts @@ -44,10 +44,17 @@ export default function addAccessibilityTools(server: McpServer) { }, async (args) => { try { - trackMCPEvent("startAccessibilityScan", server.server.getClientVersion()!); + trackMCPEvent( + "startAccessibilityScan", + server.server.getClientVersion()!, + ); return await runAccessibilityScan(args.name, args.pageURL); } catch (error) { - trackMCPFailure("startAccessibilityScan", error, server.server.getClientVersion()!); + trackMCPFailure( + "startAccessibilityScan", + error, + server.server.getClientVersion()!, + ); return { content: [ { diff --git a/src/tools/applive.ts b/src/tools/applive.ts index ba9683a..9b2e966 100644 --- a/src/tools/applive.ts +++ b/src/tools/applive.ts @@ -34,13 +34,12 @@ export async function startAppLiveSession(args: { if (args.desiredPlatform === "ios" && !args.appPath.endsWith(".ipa")) { throw new Error("You must provide a valid iOS app path."); } - + // check if the app path exists && is readable - if (!fs.existsSync(args.appPath)) { - throw new Error("The app path does not exist"); - } - try { + if (!fs.existsSync(args.appPath)) { + throw new Error("The app path does not exist."); + } fs.accessSync(args.appPath, fs.constants.R_OK); } catch (error) { logger.error("The app path does not exist or is not readable: %s", error); @@ -96,7 +95,11 @@ export default function addAppLiveTools(server: McpServer) { return await startAppLiveSession(args); } catch (error) { logger.error("App live session failed: %s", error); - trackMCPFailure("runAppLiveSession", error, server.server.getClientVersion()!); + trackMCPFailure( + "runAppLiveSession", + error, + server.server.getClientVersion()!, + ); return { content: [ { diff --git a/src/tools/automate.ts b/src/tools/automate.ts index feae250..138edef 100644 --- a/src/tools/automate.ts +++ b/src/tools/automate.ts @@ -12,26 +12,31 @@ import { trackMCPEvent, trackMCPFailure } from "../lib/instrumentation"; export async function getNetworkFailures(args: { sessionId: string; }): Promise { - const failureLogs = await retrieveNetworkFailures(args.sessionId); - logger.info( - "Successfully fetched failure network logs for session: %s", - args.sessionId, - ); + try { + const failureLogs = await retrieveNetworkFailures(args.sessionId); + logger.info( + "Successfully fetched failure network logs for session: %s", + args.sessionId, + ); - // Check if there are any failures - const hasFailures = failureLogs.totalFailures > 0; - const text = hasFailures - ? `${failureLogs.totalFailures} network failure(s) found for session :\n\n${JSON.stringify(failureLogs.failures, null, 2)}` - : `No network failures found for session`; + // Check if there are any failures + const hasFailures = failureLogs.totalFailures > 0; + const text = hasFailures + ? `${failureLogs.totalFailures} network failure(s) found for session :\n\n${JSON.stringify(failureLogs.failures, null, 2)}` + : `No network failures found for session`; - return { - content: [ - { - type: "text", - text, - }, - ], - }; + return { + content: [ + { + type: "text", + text, + }, + ], + }; + } catch (error) { + logger.error("Failed to fetch network logs: %s", error); + throw new Error(error instanceof Error ? error.message : String(error)); + } } export default function addAutomateTools(server: McpServer) { @@ -46,11 +51,16 @@ export default function addAutomateTools(server: McpServer) { trackMCPEvent("getNetworkFailures", server.server.getClientVersion()!); return await getNetworkFailures(args); } catch (error) { - const errorMessage = error instanceof Error ? error.message : String(error); + const errorMessage = + error instanceof Error ? error.message : String(error); logger.error("Failed to fetch network logs: %s", errorMessage); - - trackMCPFailure("getNetworkFailures", error, server.server.getClientVersion()!); - + + trackMCPFailure( + "getNetworkFailures", + error, + server.server.getClientVersion()!, + ); + return { content: [ { diff --git a/src/tools/bstack-sdk.ts b/src/tools/bstack-sdk.ts index be15870..e98c3a1 100644 --- a/src/tools/bstack-sdk.ts +++ b/src/tools/bstack-sdk.ts @@ -74,8 +74,11 @@ export default function addSDKTools(server: McpServer) { }, async (args) => { try { - trackMCPEvent("runTestsOnBrowserStack", server.server.getClientVersion()!); - + trackMCPEvent( + "runTestsOnBrowserStack", + server.server.getClientVersion()!, + ); + return await bootstrapProjectWithSDK({ detectedBrowserAutomationFramework: args.detectedBrowserAutomationFramework as SDKSupportedBrowserAutomationFramework, @@ -85,7 +88,11 @@ export default function addSDKTools(server: McpServer) { desiredPlatforms: args.desiredPlatforms, }); } catch (error) { - trackMCPFailure("runTestsOnBrowserStack", error, server.server.getClientVersion()!); + trackMCPFailure( + "runTestsOnBrowserStack", + error, + server.server.getClientVersion()!, + ); return { content: [ { diff --git a/src/tools/live.ts b/src/tools/live.ts index 8b53e08..b6cee2f 100644 --- a/src/tools/live.ts +++ b/src/tools/live.ts @@ -105,11 +105,18 @@ export default function addBrowserLiveTools(server: McpServer) { LiveArgsShape, async (args) => { try { - trackMCPEvent("runBrowserLiveSession", server.server.getClientVersion()!); + trackMCPEvent( + "runBrowserLiveSession", + server.server.getClientVersion()!, + ); return await runBrowserSession(args); } catch (error) { logger.error("Live session failed: %s", error); - trackMCPFailure("runBrowserLiveSession", error, server.server.getClientVersion()!); + trackMCPFailure( + "runBrowserLiveSession", + error, + server.server.getClientVersion()!, + ); return { content: [ { diff --git a/src/tools/observability.ts b/src/tools/observability.ts index bc86909..33495a0 100644 --- a/src/tools/observability.ts +++ b/src/tools/observability.ts @@ -59,11 +59,18 @@ export default function addObservabilityTools(server: McpServer) { }, async (args) => { try { - trackMCPEvent("getFailuresInLastRun", server.server.getClientVersion()!); + trackMCPEvent( + "getFailuresInLastRun", + server.server.getClientVersion()!, + ); return await getFailuresInLastRun(args.buildName, args.projectName); } catch (error) { logger.error("Failed to get failures in the last run: %s", error); - trackMCPFailure("getFailuresInLastRun", error, server.server.getClientVersion()!); + trackMCPFailure( + "getFailuresInLastRun", + error, + server.server.getClientVersion()!, + ); return { content: [ { diff --git a/src/tools/testmanagement.ts b/src/tools/testmanagement.ts index a9e2890..3b74cf5 100644 --- a/src/tools/testmanagement.ts +++ b/src/tools/testmanagement.ts @@ -23,11 +23,18 @@ export async function createProjectOrFolderTool( args: z.infer, ): Promise { try { - trackMCPEvent("createProjectOrFolder", serverInstance.server.getClientVersion()!); + trackMCPEvent( + "createProjectOrFolder", + serverInstance.server.getClientVersion()!, + ); return await createProjectOrFolder(args); } catch (err) { logger.error("Failed to create project/folder: %s", err); - trackMCPFailure("createProjectOrFolder", err, serverInstance.server.getClientVersion()!); + trackMCPFailure( + "createProjectOrFolder", + err, + serverInstance.server.getClientVersion()!, + ); return { content: [ { @@ -56,7 +63,11 @@ export async function createTestCaseTool( return await createTestCaseAPI(cleanedArgs); } catch (err) { logger.error("Failed to create test case: %s", err); - trackMCPFailure("createTestCase", err, serverInstance.server.getClientVersion()!); + trackMCPFailure( + "createTestCase", + err, + serverInstance.server.getClientVersion()!, + ); return { content: [ { From 89a28f580d10653ef6c53c6a94a919dd6a65b1e4 Mon Sep 17 00:00:00 2001 From: tech-sushant Date: Thu, 8 May 2025 18:06:37 +0530 Subject: [PATCH 11/15] Refactor: Consolidate MCP event tracking functions --- src/lib/instrumentation.ts | 70 ++++++++----------------------------- src/tools/accessibility.ts | 10 +++--- src/tools/applive.ts | 8 ++--- src/tools/automate.ts | 8 ++--- src/tools/bstack-sdk.ts | 10 +++--- src/tools/live.ts | 10 +++--- src/tools/observability.ts | 10 +++--- src/tools/testmanagement.ts | 16 ++++----- 8 files changed, 50 insertions(+), 92 deletions(-) diff --git a/src/lib/instrumentation.ts b/src/lib/instrumentation.ts index 2850c6b..be4fb2a 100644 --- a/src/lib/instrumentation.ts +++ b/src/lib/instrumentation.ts @@ -15,13 +15,16 @@ interface MCPEventPayload { }; } -export function trackMCPEvent( +export function trackMCP( toolName: string, clientInfo: { name?: string; version?: string }, + error?: unknown ): void { const instrumentationEndpoint = "https://api.browserstack.com/sdk/v1/event"; - + const isSuccess = !error; const mcpClient = clientInfo?.name || "unknown"; + + // Log client information if (clientInfo?.name) { logger.info( `Client connected: ${clientInfo.name} (version: ${clientInfo.version})`, @@ -36,60 +39,15 @@ export function trackMCPEvent( mcp_version: packageJson.version, tool_name: toolName, mcp_client: mcpClient, - success: true, + success: isSuccess, }, }; - axios - .post(instrumentationEndpoint, event, { - headers: { - "Content-Type": "application/json", - Authorization: `Basic ${Buffer.from( - `${config.browserstackUsername}:${config.browserstackAccessKey}`, - ).toString("base64")}`, - }, - timeout: 2000, - }) - .then((response) => { - logger.info("MCP event tracked successfully", { - toolName, - response, - }); - }) - .catch((error: unknown) => { - logger.warn( - `Failed to track MCP event: ${error instanceof Error ? error.message : String(error)}`, - { - toolName, - }, - ); - }); -} - -export function trackMCPFailure( - toolName: string, - error: unknown, - clientInfo: { name?: string; version?: string }, -): void { - const instrumentationEndpoint = "https://api.browserstack.com/sdk/v1/event"; - - const mcpClient = clientInfo?.name || "unknown"; - const errorMessage = error instanceof Error ? error.message : String(error); - const errorType = error instanceof Error ? error.constructor.name : "Unknown"; - - logger.error(`Tool failure: ${toolName} - ${errorMessage}`, { errorType }); - - const event: MCPEventPayload = { - event_type: "MCPInstrumentation", - event_properties: { - mcp_version: packageJson.version, - tool_name: toolName, - mcp_client: mcpClient, - success: false, - error_message: errorMessage, - error_type: errorType, - }, - }; + // Add error details if applicable + if (error) { + event.event_properties.error_message = error instanceof Error ? error.message : String(error); + event.event_properties.error_type = error instanceof Error ? error.constructor.name : "Unknown"; + } axios .post(instrumentationEndpoint, event, { @@ -102,17 +60,17 @@ export function trackMCPFailure( timeout: 2000, }) .then((response) => { - logger.info("MCP failure event tracked successfully", { + logger.info(`MCP ${isSuccess ? 'event' : 'failure event'} tracked successfully`, { toolName, response, }); }) .catch((error: unknown) => { logger.warn( - `Failed to track MCP failure event: ${error instanceof Error ? error.message : String(error)}`, + `Failed to track MCP ${isSuccess ? 'event' : 'failure event'}: ${error instanceof Error ? error.message : String(error)}`, { toolName, }, ); }); -} +} \ No newline at end of file diff --git a/src/tools/accessibility.ts b/src/tools/accessibility.ts index b260810..2c5c2d7 100644 --- a/src/tools/accessibility.ts +++ b/src/tools/accessibility.ts @@ -5,7 +5,7 @@ import { startAccessibilityScan, AccessibilityScanResponse, } from "./accessiblity-utils/accessibility"; -import { trackMCPEvent, trackMCPFailure } from "../lib/instrumentation"; +import { trackMCP } from "../lib/instrumentation"; async function runAccessibilityScan( name: string, @@ -44,16 +44,16 @@ export default function addAccessibilityTools(server: McpServer) { }, async (args) => { try { - trackMCPEvent( + trackMCP( "startAccessibilityScan", - server.server.getClientVersion()!, + server.server.getClientVersion()! ); return await runAccessibilityScan(args.name, args.pageURL); } catch (error) { - trackMCPFailure( + trackMCP( "startAccessibilityScan", - error, server.server.getClientVersion()!, + error ); return { content: [ diff --git a/src/tools/applive.ts b/src/tools/applive.ts index 9b2e966..db0acd1 100644 --- a/src/tools/applive.ts +++ b/src/tools/applive.ts @@ -4,7 +4,7 @@ import { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; import fs from "fs"; import { startSession } from "./applive-utils/start-session"; import logger from "../logger"; -import { trackMCPEvent, trackMCPFailure } from "../lib/instrumentation"; +import { trackMCP } from "../lib/instrumentation"; /** * Launches an App Live Session on BrowserStack. @@ -91,14 +91,14 @@ export default function addAppLiveTools(server: McpServer) { }, async (args) => { try { - trackMCPEvent("runAppLiveSession", server.server.getClientVersion()!); + trackMCP("runAppLiveSession", server.server.getClientVersion()!); return await startAppLiveSession(args); } catch (error) { logger.error("App live session failed: %s", error); - trackMCPFailure( + trackMCP( "runAppLiveSession", - error, server.server.getClientVersion()!, + error ); return { content: [ diff --git a/src/tools/automate.ts b/src/tools/automate.ts index 138edef..3f98276 100644 --- a/src/tools/automate.ts +++ b/src/tools/automate.ts @@ -3,7 +3,7 @@ import { z } from "zod"; import { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; import logger from "../logger"; import { retrieveNetworkFailures } from "../lib/api"; -import { trackMCPEvent, trackMCPFailure } from "../lib/instrumentation"; +import { trackMCP } from "../lib/instrumentation"; /** * Fetches failed network requests from a BrowserStack Automate session. @@ -48,17 +48,17 @@ export default function addAutomateTools(server: McpServer) { }, async (args) => { try { - trackMCPEvent("getNetworkFailures", server.server.getClientVersion()!); + trackMCP("getNetworkFailures", server.server.getClientVersion()!); return await getNetworkFailures(args); } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error); logger.error("Failed to fetch network logs: %s", errorMessage); - trackMCPFailure( + trackMCP( "getNetworkFailures", - error, server.server.getClientVersion()!, + error ); return { diff --git a/src/tools/bstack-sdk.ts b/src/tools/bstack-sdk.ts index e98c3a1..24d1358 100644 --- a/src/tools/bstack-sdk.ts +++ b/src/tools/bstack-sdk.ts @@ -10,7 +10,7 @@ import { generateBrowserStackYMLInstructions, getInstructionsForProjectConfiguration, } from "./sdk-utils/instructions"; -import { trackMCPEvent, trackMCPFailure } from "../lib/instrumentation"; +import { trackMCP } from "../lib/instrumentation"; /** * BrowserStack SDK hooks into your test framework to seamlessly run tests on BrowserStack. @@ -74,9 +74,9 @@ export default function addSDKTools(server: McpServer) { }, async (args) => { try { - trackMCPEvent( + trackMCP( "runTestsOnBrowserStack", - server.server.getClientVersion()!, + server.server.getClientVersion()! ); return await bootstrapProjectWithSDK({ @@ -88,10 +88,10 @@ export default function addSDKTools(server: McpServer) { desiredPlatforms: args.desiredPlatforms, }); } catch (error) { - trackMCPFailure( + trackMCP( "runTestsOnBrowserStack", - error, server.server.getClientVersion()!, + error ); return { content: [ diff --git a/src/tools/live.ts b/src/tools/live.ts index b6cee2f..bccc99c 100644 --- a/src/tools/live.ts +++ b/src/tools/live.ts @@ -4,7 +4,7 @@ import { z } from "zod"; import logger from "../logger"; import { startBrowserSession } from "./live-utils/start-session"; import { PlatformType } from "./live-utils/types"; -import { trackMCPEvent, trackMCPFailure } from "../lib/instrumentation"; +import { trackMCP } from "../lib/instrumentation"; // Define the schema shape const LiveArgsShape = { @@ -105,17 +105,17 @@ export default function addBrowserLiveTools(server: McpServer) { LiveArgsShape, async (args) => { try { - trackMCPEvent( + trackMCP( "runBrowserLiveSession", - server.server.getClientVersion()!, + server.server.getClientVersion()! ); return await runBrowserSession(args); } catch (error) { logger.error("Live session failed: %s", error); - trackMCPFailure( + trackMCP( "runBrowserLiveSession", - error, server.server.getClientVersion()!, + error ); return { content: [ diff --git a/src/tools/observability.ts b/src/tools/observability.ts index 33495a0..fabea9e 100644 --- a/src/tools/observability.ts +++ b/src/tools/observability.ts @@ -2,7 +2,7 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { z } from "zod"; import { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; import { getLatestO11YBuildInfo } from "../lib/api"; -import { trackMCPEvent, trackMCPFailure } from "../lib/instrumentation"; +import { trackMCP } from "../lib/instrumentation"; import logger from "../logger"; export async function getFailuresInLastRun( @@ -59,17 +59,17 @@ export default function addObservabilityTools(server: McpServer) { }, async (args) => { try { - trackMCPEvent( + trackMCP( "getFailuresInLastRun", - server.server.getClientVersion()!, + server.server.getClientVersion()! ); return await getFailuresInLastRun(args.buildName, args.projectName); } catch (error) { logger.error("Failed to get failures in the last run: %s", error); - trackMCPFailure( + trackMCP( "getFailuresInLastRun", - error, server.server.getClientVersion()!, + error ); return { content: [ diff --git a/src/tools/testmanagement.ts b/src/tools/testmanagement.ts index 3b74cf5..a032259 100644 --- a/src/tools/testmanagement.ts +++ b/src/tools/testmanagement.ts @@ -1,7 +1,7 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { z } from "zod"; import { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; -import { trackMCPEvent, trackMCPFailure } from "../lib/instrumentation"; +import { trackMCP } from "../lib/instrumentation"; import logger from "../logger"; import { createProjectOrFolder, @@ -23,17 +23,17 @@ export async function createProjectOrFolderTool( args: z.infer, ): Promise { try { - trackMCPEvent( + trackMCP( "createProjectOrFolder", - serverInstance.server.getClientVersion()!, + serverInstance.server.getClientVersion()! ); return await createProjectOrFolder(args); } catch (err) { logger.error("Failed to create project/folder: %s", err); - trackMCPFailure( + trackMCP( "createProjectOrFolder", - err, serverInstance.server.getClientVersion()!, + err ); return { content: [ @@ -59,14 +59,14 @@ export async function createTestCaseTool( // Sanitize input arguments const cleanedArgs = sanitizeArgs(args); try { - trackMCPEvent("createTestCase", serverInstance.server.getClientVersion()!); + trackMCP("createTestCase", serverInstance.server.getClientVersion()!); return await createTestCaseAPI(cleanedArgs); } catch (err) { logger.error("Failed to create test case: %s", err); - trackMCPFailure( + trackMCP( "createTestCase", - err, serverInstance.server.getClientVersion()!, + err ); return { content: [ From 1cd54a7deb8454fb1a5ae4a73f2fb08c6d33fa1f Mon Sep 17 00:00:00 2001 From: tech-sushant Date: Thu, 8 May 2025 18:08:40 +0530 Subject: [PATCH 12/15] Refactor: Improve consistency in MCP event --- src/lib/instrumentation.ts | 25 +++++++++++++++---------- src/tools/accessibility.ts | 7 ++----- src/tools/applive.ts | 6 +----- src/tools/automate.ts | 2 +- src/tools/bstack-sdk.ts | 7 ++----- src/tools/live.ts | 7 ++----- src/tools/observability.ts | 7 ++----- src/tools/testmanagement.ts | 10 +++------- tests/tools/applive.test.ts | 3 +-- tests/tools/automate.test.ts | 3 +-- tests/tools/live.test.ts | 3 +-- tests/tools/observability.test.ts | 3 +-- tests/tools/testmanagement.test.ts | 3 +-- 13 files changed, 33 insertions(+), 53 deletions(-) diff --git a/src/lib/instrumentation.ts b/src/lib/instrumentation.ts index be4fb2a..8c17760 100644 --- a/src/lib/instrumentation.ts +++ b/src/lib/instrumentation.ts @@ -18,12 +18,12 @@ interface MCPEventPayload { export function trackMCP( toolName: string, clientInfo: { name?: string; version?: string }, - error?: unknown + error?: unknown, ): void { const instrumentationEndpoint = "https://api.browserstack.com/sdk/v1/event"; const isSuccess = !error; const mcpClient = clientInfo?.name || "unknown"; - + // Log client information if (clientInfo?.name) { logger.info( @@ -45,8 +45,10 @@ export function trackMCP( // Add error details if applicable if (error) { - event.event_properties.error_message = error instanceof Error ? error.message : String(error); - event.event_properties.error_type = error instanceof Error ? error.constructor.name : "Unknown"; + event.event_properties.error_message = + error instanceof Error ? error.message : String(error); + event.event_properties.error_type = + error instanceof Error ? error.constructor.name : "Unknown"; } axios @@ -60,17 +62,20 @@ export function trackMCP( timeout: 2000, }) .then((response) => { - logger.info(`MCP ${isSuccess ? 'event' : 'failure event'} tracked successfully`, { - toolName, - response, - }); + logger.info( + `MCP ${isSuccess ? "event" : "failure event"} tracked successfully`, + { + toolName, + response, + }, + ); }) .catch((error: unknown) => { logger.warn( - `Failed to track MCP ${isSuccess ? 'event' : 'failure event'}: ${error instanceof Error ? error.message : String(error)}`, + `Failed to track MCP ${isSuccess ? "event" : "failure event"}: ${error instanceof Error ? error.message : String(error)}`, { toolName, }, ); }); -} \ No newline at end of file +} diff --git a/src/tools/accessibility.ts b/src/tools/accessibility.ts index 2c5c2d7..1e28084 100644 --- a/src/tools/accessibility.ts +++ b/src/tools/accessibility.ts @@ -44,16 +44,13 @@ export default function addAccessibilityTools(server: McpServer) { }, async (args) => { try { - trackMCP( - "startAccessibilityScan", - server.server.getClientVersion()! - ); + trackMCP("startAccessibilityScan", server.server.getClientVersion()!); return await runAccessibilityScan(args.name, args.pageURL); } catch (error) { trackMCP( "startAccessibilityScan", server.server.getClientVersion()!, - error + error, ); return { content: [ diff --git a/src/tools/applive.ts b/src/tools/applive.ts index db0acd1..248e3b6 100644 --- a/src/tools/applive.ts +++ b/src/tools/applive.ts @@ -95,11 +95,7 @@ export default function addAppLiveTools(server: McpServer) { return await startAppLiveSession(args); } catch (error) { logger.error("App live session failed: %s", error); - trackMCP( - "runAppLiveSession", - server.server.getClientVersion()!, - error - ); + trackMCP("runAppLiveSession", server.server.getClientVersion()!, error); return { content: [ { diff --git a/src/tools/automate.ts b/src/tools/automate.ts index 3f98276..8b62b22 100644 --- a/src/tools/automate.ts +++ b/src/tools/automate.ts @@ -58,7 +58,7 @@ export default function addAutomateTools(server: McpServer) { trackMCP( "getNetworkFailures", server.server.getClientVersion()!, - error + error, ); return { diff --git a/src/tools/bstack-sdk.ts b/src/tools/bstack-sdk.ts index 24d1358..6c71600 100644 --- a/src/tools/bstack-sdk.ts +++ b/src/tools/bstack-sdk.ts @@ -74,10 +74,7 @@ export default function addSDKTools(server: McpServer) { }, async (args) => { try { - trackMCP( - "runTestsOnBrowserStack", - server.server.getClientVersion()! - ); + trackMCP("runTestsOnBrowserStack", server.server.getClientVersion()!); return await bootstrapProjectWithSDK({ detectedBrowserAutomationFramework: @@ -91,7 +88,7 @@ export default function addSDKTools(server: McpServer) { trackMCP( "runTestsOnBrowserStack", server.server.getClientVersion()!, - error + error, ); return { content: [ diff --git a/src/tools/live.ts b/src/tools/live.ts index bccc99c..cf126f9 100644 --- a/src/tools/live.ts +++ b/src/tools/live.ts @@ -105,17 +105,14 @@ export default function addBrowserLiveTools(server: McpServer) { LiveArgsShape, async (args) => { try { - trackMCP( - "runBrowserLiveSession", - server.server.getClientVersion()! - ); + trackMCP("runBrowserLiveSession", server.server.getClientVersion()!); return await runBrowserSession(args); } catch (error) { logger.error("Live session failed: %s", error); trackMCP( "runBrowserLiveSession", server.server.getClientVersion()!, - error + error, ); return { content: [ diff --git a/src/tools/observability.ts b/src/tools/observability.ts index fabea9e..f0e2e9a 100644 --- a/src/tools/observability.ts +++ b/src/tools/observability.ts @@ -59,17 +59,14 @@ export default function addObservabilityTools(server: McpServer) { }, async (args) => { try { - trackMCP( - "getFailuresInLastRun", - server.server.getClientVersion()! - ); + trackMCP("getFailuresInLastRun", server.server.getClientVersion()!); return await getFailuresInLastRun(args.buildName, args.projectName); } catch (error) { logger.error("Failed to get failures in the last run: %s", error); trackMCP( "getFailuresInLastRun", server.server.getClientVersion()!, - error + error, ); return { content: [ diff --git a/src/tools/testmanagement.ts b/src/tools/testmanagement.ts index a032259..d399045 100644 --- a/src/tools/testmanagement.ts +++ b/src/tools/testmanagement.ts @@ -25,7 +25,7 @@ export async function createProjectOrFolderTool( try { trackMCP( "createProjectOrFolder", - serverInstance.server.getClientVersion()! + serverInstance.server.getClientVersion()!, ); return await createProjectOrFolder(args); } catch (err) { @@ -33,7 +33,7 @@ export async function createProjectOrFolderTool( trackMCP( "createProjectOrFolder", serverInstance.server.getClientVersion()!, - err + err, ); return { content: [ @@ -63,11 +63,7 @@ export async function createTestCaseTool( return await createTestCaseAPI(cleanedArgs); } catch (err) { logger.error("Failed to create test case: %s", err); - trackMCP( - "createTestCase", - serverInstance.server.getClientVersion()!, - err - ); + trackMCP("createTestCase", serverInstance.server.getClientVersion()!, err); return { content: [ { diff --git a/tests/tools/applive.test.ts b/tests/tools/applive.test.ts index f16ed16..88b2440 100644 --- a/tests/tools/applive.test.ts +++ b/tests/tools/applive.test.ts @@ -16,8 +16,7 @@ jest.mock('../../src/logger', () => ({ error: jest.fn() })); jest.mock('../../src/lib/instrumentation', () => ({ - trackMCPEvent: jest.fn(), - trackMCPFailure: jest.fn() + trackMCP: jest.fn() })); describe('startAppLiveSession', () => { diff --git a/tests/tools/automate.test.ts b/tests/tools/automate.test.ts index 3917390..25a468e 100644 --- a/tests/tools/automate.test.ts +++ b/tests/tools/automate.test.ts @@ -6,8 +6,7 @@ jest.mock('../../src/lib/api', () => ({ retrieveNetworkFailures: jest.fn() })); jest.mock('../../src/lib/instrumentation', () => ({ - trackMCPEvent: jest.fn(), - trackMCPFailure: jest.fn() + trackMCP: jest.fn() })); jest.mock('../../src/logger', () => ({ error: jest.fn(), diff --git a/tests/tools/live.test.ts b/tests/tools/live.test.ts index 03a9e4c..4a57814 100644 --- a/tests/tools/live.test.ts +++ b/tests/tools/live.test.ts @@ -14,8 +14,7 @@ jest.mock('../../src/logger', () => ({ error: jest.fn() })); jest.mock('../../src/lib/instrumentation', () => ({ - trackMCPEvent: jest.fn(), - trackMCPFailure: jest.fn() + trackMCP: jest.fn() })); describe('startBrowserLiveSession', () => { diff --git a/tests/tools/observability.test.ts b/tests/tools/observability.test.ts index d67561d..a6b7c33 100644 --- a/tests/tools/observability.test.ts +++ b/tests/tools/observability.test.ts @@ -7,8 +7,7 @@ jest.mock('../../src/lib/api', () => ({ })); jest.mock('../../src/lib/instrumentation', () => ({ - trackMCPEvent: jest.fn(), - trackMCPFailure: jest.fn() + trackMCP: jest.fn() })); describe('getFailuresInLastRun', () => { diff --git a/tests/tools/testmanagement.test.ts b/tests/tools/testmanagement.test.ts index 434c2cc..4792fa3 100644 --- a/tests/tools/testmanagement.test.ts +++ b/tests/tools/testmanagement.test.ts @@ -28,8 +28,7 @@ jest.mock('../../src/config', () => ({ }, })); jest.mock('../../src/lib/instrumentation', () => ({ - trackMCPEvent: jest.fn(), - trackMCPFailure: jest.fn() + trackMCP: jest.fn() })); const mockServer = { From cd924398f86233463dd8787d7870e968baae98ea Mon Sep 17 00:00:00 2001 From: tech-sushant Date: Thu, 8 May 2025 21:30:57 +0530 Subject: [PATCH 13/15] Refactor: Add TODO for moving traceMCP and catch block to parent function --- src/tools/testmanagement.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/tools/testmanagement.ts b/src/tools/testmanagement.ts index 5f32547..390a611 100644 --- a/src/tools/testmanagement.ts +++ b/src/tools/testmanagement.ts @@ -38,6 +38,8 @@ import { } from "./testmanagement-utils/update-testrun"; +//TODO: Moving the traceMCP and catch block to the parent(server) function + /** * Wrapper to call createProjectOrFolder util. */ From 4577110801d8f9d5efbb777f39895e3b1d7c948d Mon Sep 17 00:00:00 2001 From: tech-sushant Date: Fri, 9 May 2025 14:18:28 +0530 Subject: [PATCH 14/15] Introduce DEV_MODE configuration to control MCP tracking behavior --- src/config.ts | 2 ++ src/lib/instrumentation.ts | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/src/config.ts b/src/config.ts index fae14cc..4afbd43 100644 --- a/src/config.ts +++ b/src/config.ts @@ -11,12 +11,14 @@ export class Config { constructor( public readonly browserstackUsername: string, public readonly browserstackAccessKey: string, + public readonly DEV_MODE: boolean, ) {} } const config = new Config( process.env.BROWSERSTACK_USERNAME!, process.env.BROWSERSTACK_ACCESS_KEY!, + process.env.DEV_MODE === "true" ); export default config; diff --git a/src/lib/instrumentation.ts b/src/lib/instrumentation.ts index 8c17760..e6e8062 100644 --- a/src/lib/instrumentation.ts +++ b/src/lib/instrumentation.ts @@ -20,6 +20,12 @@ export function trackMCP( clientInfo: { name?: string; version?: string }, error?: unknown, ): void { + + if (config.DEV_MODE) { + logger.info("Tracking MCP is disabled in dev mode"); + return; + } + const instrumentationEndpoint = "https://api.browserstack.com/sdk/v1/event"; const isSuccess = !error; const mcpClient = clientInfo?.name || "unknown"; From 1907119674334733cfa913fdd8c71890fae3c155 Mon Sep 17 00:00:00 2001 From: tech-sushant Date: Fri, 9 May 2025 15:40:37 +0530 Subject: [PATCH 15/15] Remove unnecessary logging --- src/lib/instrumentation.ts | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/src/lib/instrumentation.ts b/src/lib/instrumentation.ts index e6e8062..120f0c6 100644 --- a/src/lib/instrumentation.ts +++ b/src/lib/instrumentation.ts @@ -67,21 +67,5 @@ export function trackMCP( }, timeout: 2000, }) - .then((response) => { - logger.info( - `MCP ${isSuccess ? "event" : "failure event"} tracked successfully`, - { - toolName, - response, - }, - ); - }) - .catch((error: unknown) => { - logger.warn( - `Failed to track MCP ${isSuccess ? "event" : "failure event"}: ${error instanceof Error ? error.message : String(error)}`, - { - toolName, - }, - ); - }); + .catch(() => {}); }