Mocking Readable Streams

[bliu13]

I am having an issue trying to mock a http response stream. I am trying to write a fetch request to download an excel file, but for whatever reason I am not able to write a unit test. I get the following error:

 FAIL  __tests__/excel.test.js > getReadableStream > should return a readable stream for an HTTP URL
Error: Can't find end of central directory : is this a zip file ? If it is, see https://stuk.github.io/jszip/documentation/howto/read_zip.html
 ❯ ZipEntries.readEndOfCentral node_modules/jszip/lib/zipEntries.js:167:23
 ❯ ZipEntries.load node_modules/jszip/lib/zipEntries.js:255:14
 ❯ node_modules/jszip/lib/load.js:48:24
 ❯ XLSX.load node_modules/exceljs/lib/xlsx/xlsx.js:279:17
 ❯ Module.readExcel helpers/excel.js:51:5
     49| 
     50|     const workbook = new ExcelJS.Workbook()
     51|     await workbook.xlsx.read(stream)
       |     ^
     52| 
     53|     return workbook
 ❯ __tests__/excel.test.js:95:19

excel.test.js

import { PassThrough } from 'node:stream'
import ExcelJS from 'exceljs'
import { describe, it, expect, beforeAll, afterAll, afterEach } from 'vitest'
import { setupServer } from 'msw/node'
import { http, HttpResponse } from 'msw'
import { readExcel, getReadableStream } from '../excel.js'
import data from './__fixtures__/data.json'

const sheetName = 'Sheet1'

// Create a mock server
const server = setupServer()

// Start the server before all tests
beforeAll(() => server.listen())

// Close the server after all tests
afterAll(() => server.close())

// Reset handlers after each test
afterEach(() => server.resetHandlers())

export async function generateReadableStreamFromJson (jsonData) {
  const workbook = new ExcelJS.Workbook()
  const worksheet = workbook.addWorksheet(sheetName)

  if (jsonData.length > 0) {
    // Add header row
    const header = Object.keys(jsonData[0])
    worksheet.addRow(header)

    // Add data rows
    jsonData.forEach((data) => {
      const rowValues = Object.values(data).map(value => value === null ? '' : value)
      worksheet.addRow(rowValues)
    })
  } else {
    // Add header row with empty data
    worksheet.addRow([])
  }

  // Create a PassThrough stream
  const stream = new PassThrough()

  try {
    // Write the workbook to the stream
    await workbook.xlsx.write(stream)
    stream.end()
  } catch (error) {
    stream.emit('error', error)
  }

  return stream
}

function validateExcelDataAgainstJson (excelData, jsonData) {
  const [headerRow, ...dataRows] = excelData[0].data

  // Transform Excel data into key-value objects
  const excelObjects = dataRows.map((row) => {
    const obj = {}
    headerRow.slice(1).forEach((header, colIndex) => {
      obj[header] = row[colIndex + 1]
    })
    return obj
  })

  // Compare the transformed Excel data against the JSON data
  expect(excelObjects).toHaveLength(jsonData.length)
  jsonData.forEach((jsonObj, index) => {
    const excelObj = excelObjects[index]
    Object.keys(jsonObj).forEach((key) => {
      expect(excelObj[key]).toEqual(jsonObj[key] ?? '')
    })
  })
}

describe('getReadableStream', () => {
  it('should return a readable stream for an HTTP URL', async () => {
    const url = 'http://example.com/data.xlsx'
    const readableStream = await generateReadableStreamFromJson(data)

    server.use(
      http.get(url, () => {
        return new HttpResponse(readableStream, {
          headers: {
            'Content-Type': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
          },
        })
      })
    )

    const stream = await getReadableStream(url)

    const excel = await readExcel(stream)
    validateExcelDataAgainstJson(excel, data)
  })
})

excel.js

import { URL } from 'node:url'
import { createReadStream } from 'node:fs'
import path from 'node:path'
import { Readable } from 'node:stream'
import ExcelJS from 'exceljs'

export async function getReadableStream (input) {
  if (typeof input !== 'string' || input.trim() === '') {
    throw new Error('Invalid input: Must be a non-empty string or a readable stream')
  }

  try {
    const parsedUrl = new URL(input)

    switch (parsedUrl.protocol) {
      case 'http:':
      case 'https:': {
        const response = await fetch(input)
        if (!response.ok) {
          throw new Error(`Network request failed for ${input} with status code ${response.status}`)
        }
        if (!response.body || typeof response.body.pipe !== 'function') {
          throw new Error(`Response body is not a readable stream for ${input}`)
        }
        return response.body
      }
      case 'file:':
        return createReadStream(parsedUrl.pathname)
      default:
        throw new Error(`Unsupported protocol for URL: ${input}. Must use HTTP, HTTPS, or file protocol`)
    }
  } catch (e) {
    // If the URL constructor throws an error or the protocol is unsupported, assume it's a local file path
    console.warn(`Failed to parse URL: ${e.message}. Assuming local file path: ${input}`)
    return createReadStream(path.resolve(input))
  }
}

export async function readExcel (input) {
  try {
    let stream
    if (typeof input === 'string') {
      stream = await getReadableStream(input)
    } else if (input instanceof Readable) {
      stream = input
    } else {
      throw new Error('Invalid input: Must be a URL, file path, or readable stream')
    }

    const workbook = new ExcelJS.Workbook()
    await workbook.xlsx.read(stream)

    return workbook
  } catch (e) {
    console.error(`Failed to read Excel file from ${input}`, e)
    throw e
  }
}

package.json

{
  "name": "excel-stream-test",
  "description": "exceljs test",
  "version": "1.0.0",
  "license": "UNLICENSED",
  "main": "index.js",
  "type": "module",
  "scripts": {
    "test": "vitest run --passWithNoTests"
  },
  "engines": {
    "node": ">=22.0.0"
  },
  "dependencies": {
    "exceljs": "4.4.0"
  },
  "devDependencies": {
    "@vitest/coverage-v8": "2.1.5",
    "msw": "2.6.5",
    "vite": "5.4.11",
    "vitest": "2.1.5"
  }
}

data.json

[
  {
    "field_1": "1"
  },
  {
    "field_1": "2"
  }
]

I am able to read the excel stream if I don't make an http request, but I get an error about a zip file if I mock the http response to the stream.

[kettanaito]

Hi.

This likely has to do with the package you are using to create a stream. This part, for example, makes little sense to me:

await workbook.xlsx.write(stream)

You normally don't await writes/stream because that defies the entire purpose of a pending stream where data is being written to continuously.

MSW supports ReadableStream, so if something is amiss I recommend boil it down to using plain PassThrough() and pushing data there, then advance the complexity until you reach the breaking point.

I also don't see you using Web Streams at all in your examples. You use node:stream, which is not the same. You have to convert Readable to ReadableStream before you pass it to MSW. To do that, use node:stream:

import { Readable } from 'node:stream'

const webStream = Readable.toWeb(nodeStream)
new HttpResponse(webStream, init)