diff --git a/CHANGELOG.md b/CHANGELOG.md index b834ee03..f60d9ef4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,7 +43,21 @@ Format: - --> -## [Unreleased] + + +## [5.13.3] - 2025-01-05 + +### Fixed + +- Updates to latest docassemble HTML. See [#960](https://github.com/SuffolkLITLab/ALKiln/issues/960). Changes all appearances of `fieldset` to keep the code consistent. + +## [5.13.2] - 2024-11-23 + +### Fixed + +- Fixes download Step unable to use a partial filename match. The Step needed the whole name, including the extension. See [#725](https://github.com/SuffolkLITLab/ALKiln/issues/725). + +## [5.13.1] - 2024-10-23 ### Changed @@ -68,6 +82,7 @@ Format: - Closes [#659](https://github.com/SuffolkLITLab/ALKiln/issues/659), abstract adding to debug_log - Closes [#925](https://github.com/SuffolkLITLab/ALKiln/issues/925), allow a `Log` to throw an error - Addresses [#461](https://github.com/SuffolkLITLab/ALKiln/issues/461), setup and takedown reports +- Updates pdfjs-dist for node v20 and v22. See [#952](https://github.com/SuffolkLITLab/ALKiln/issues/952). ## [5.13.0] - 2024-07-11 diff --git a/README.md b/README.md index 7f486760..046ce295 100644 --- a/README.md +++ b/README.md @@ -41,25 +41,46 @@ Read about contributing in our [CONTRIBUTING.md document](CONTRIBUTING.md). Here ## Cheat sheet *Once you've already read the contributing documentation, you can use these as quick reminders for running our internal tests.* -To set up for the integration tests, create the project on the server: + +### `setup` before starting development of a feature/fix + +Set up for the cucumber integration tests. + +Add the feature branch name to your `.env` file: + +``` +BRANCH=42_feat_life_the_univers_and_everything +``` + +Then create the project on the server: ```bash npm run setup ``` -Use the syntax below to trigger specific tags: +### Run tests repeatedly + +Run all cucumber tests that are not currently blocked by upstream changes: + +```bash +npm run pass +``` + +Trigger cucumber tests with specific tags: ```bash -npm run cucumber -- "--tags" "@tagname" +npm run cucumber "@tagname" ``` -To run the unit tests in isolation: +Run the unit tests: ```bash npm run unit ``` -If you or someone else changes the interview code in `./docassemble`, you have to clean up the old data on the server before running `setup` again: +### Always run `takedown` before running `setup` again + +If you or someone else changes the interview code in our `./docassemble` directory, you have to delete the code currently on the testing server before running `setup` again. Also do this when you're done with the feature/fix or starting a new feature/fix. ```bash npm run takedown diff --git a/docassemble/ALKilnTests/data/sources/observation_steps.feature b/docassemble/ALKilnTests/data/sources/observation_steps.feature index 6afeba9f..69875ffa 100644 --- a/docassemble/ALKilnTests/data/sources/observation_steps.feature +++ b/docassemble/ALKilnTests/data/sources/observation_steps.feature @@ -239,19 +239,19 @@ Scenario: I compare different PDFs Given the final Scenario status should be "failed" And the Scenario report should include: """ - Could not find the existing PDF at DOES_NOT_EXIST.pdf + ALK0156 """ And the Scenario report should include: """ - The PDFs were not the same. + ALK0157 """ - And the Scenario report should include: + And the Scenario report should include: """ - The new PDF added: + - diff """ And the Scenario report should include: """ - - diff + ALK0093 """ Given I start the interview at "test_pdf" Then the question id should be "proxy vars" @@ -268,6 +268,7 @@ Scenario: I compare different PDFs And I tap to continue # Next page Then the question id should be "2_signature download" - When I download "2_signature.pdf" + # Match a partial name + When I download "2_signatu" And I expect the baseline PDF "DOES_NOT_EXIST.pdf" and the new PDF "2_signature.pdf" to be the same And I expect the baseline PDF "linear_2_signature-Baseline.pdf" and the new PDF "2_signature.pdf" to be the same diff --git a/lib/scope.js b/lib/scope.js index f8538a84..2236c705 100644 --- a/lib/scope.js +++ b/lib/scope.js @@ -518,13 +518,17 @@ module.exports = { }, continue_button_selector: `fieldset.da-field-buttons button[type="submit"].btn-primary`, + continue_button_selector_backup: `.da-fieldset.da-field-buttons button[type="submit"].btn-primary`, signature_selector: `fieldset .dasigsave`, + signature_selector_backup: `.da-fieldset .dasigsave`, continue: async function ( scope ) { /* Presses whatever button it finds that might lead to the next page. */ // Any selectors I find seem somewhat precarious. let elem = await Promise.race([ scope.page.waitForSelector(scope.continue_button_selector), // other pages (this is the most consistent way) + scope.page.waitForSelector(scope.continue_button_selector_backup), scope.page.waitForSelector(scope.signature_selector), // signature page + scope.page.waitForSelector(scope.signature_selector_backup), ]); await elem.evaluate( el => { return el.className }); // Waits for navigation or user error @@ -544,9 +548,11 @@ module.exports = { // // `buttons:` can be used in question blocks as choices let regular = await scope.page.$(scope.continue_button_selector); + let regular_backup = await scope.page.$(scope.continue_button_selector_backup); let signature = await scope.page.$(scope.signature_selector); + let signature_backup = await scope.page.$(scope.signature_selector_backup); - return regular !== null || signature !== null; + return regular !== null || regular_backup !== null || signature !== null || signature_backup !== null; }, // Ends scope.continue_exists() @@ -918,7 +924,7 @@ module.exports = { // All the different types of fields // buttons, canvases, inputs of all kinds, selects (dropdowns), textareas. Are there more? // Will deal with `option` once inside `select` - let all_nodes = $( `#dasigpage, fieldset button, .daquestionactionbutton, fieldset input, .da-form-group input, .da-form-group select, form select, .da-form-group textarea` ); + let all_nodes = $( `#dasigpage, fieldset button, .da-fieldset button, .daquestionactionbutton, fieldset input, da-fieldset input, .da-form-group input, .da-form-group select, form select, .da-form-group textarea` ); for ( let node of all_nodes ) { // Decision: Do not abstract the below. There's too much data to pass around for it to make sense let $node = $( node ); @@ -2628,6 +2634,7 @@ module.exports = { let fullPage = true; let signature_elem = await scope.page.$(scope.signature_selector); + signature_elem = signature_elem || await scope.page.$(scope.signature_selector_backup); if ( signature_elem !== null ) { fullPage = false; } @@ -2774,7 +2781,7 @@ module.exports = { await scope.afterStep(scope); }, // Ends scope.steps.sign() - download: async ( scope, filename ) => { + download: async ( scope, full_or_partial_file_href ) => { /* Taps the link that leads to the given filename to trigger downloading. * and waits till the file has been downloaded before allowing the tests to continue. * WARNING: Cannot download the same file twice in a single scenario. @@ -2784,47 +2791,86 @@ module.exports = { * TODO: Properly wait for download to complete. See notes in * scope.js scope.detectDownloadComplete() */ - let [elem] = await scope.page.$$(`xpath/.//a[contains(@href, "${ filename }")]`); + let [elem] = await scope.page.$$(`xpath/.//a[contains(@href, "${ full_or_partial_file_href }")]`); - let msg = `"${ filename }" seems to be missing. Cannot find a link to that document.`; + let msg = `"${ full_or_partial_file_href }" seems to be missing on the page. Cannot find a link to that document.`; if ( !elem ) { reports.addToReport(scope, { type: `error`, code: `ALK0152`, value: msg }); } expect( elem, msg ).to.exist; let failed_to_download = false; let err_msg = ""; try { - const binaryStr = await scope.page.evaluate(el => { + + const { disposition, binaryStr } = await scope.page.evaluate(async function ( el ) { const url = el.getAttribute("href"); return new Promise(async (resolve, reject) => { const response = await fetch(url, {method: "GET"}); const reader = new FileReader(); reader.readAsBinaryString(await response.blob()); - reader.onload = () => resolve(reader.result); + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Disposition + reader.onload = () => resolve({ + disposition: response.headers.get(`Content-Disposition`), + binaryStr: reader.result + }); reader.onerror = () => reject(`šŸ¤• ALK0153 ERROR: Error occurred on page when downloading ${ url }: ${ reader.error }`); }); }, elem); + + err_msg = `could not get the actual name of the file from the response headers.`; + let actual_filename = await scope.steps.get_response_filename({ + disposition, default_name: full_or_partial_file_href + }); + if (binaryStr !== '') { + err_msg = `binary data for download was empty`; const fileData = Buffer.from(binaryStr, 'binary'); - fs.writeFileSync(`${ scope.paths.scenario }/${ filename }`, fileData); - reports.addToReport(scope, { type: `row info`, code: `ALK0154`, value: `Downloaded ${ filename } (${ fileData.length } bytes) to ${ scope.paths.scenario }`}); + fs.writeFileSync(`${ scope.paths.scenario }/${ actual_filename }`, fileData); + reports.addToReport(scope, { type: `row info`, code: `ALK0154`, value: `Downloaded "${ actual_filename }" (${ fileData.length } bytes) to ${ scope.paths.scenario }`}); } else { failed_to_download = true; - err_msg = `Couldn't download ${ filename }, binary data for download was empty`; } + } catch (error) { failed_to_download = true; err_msg = error; } if (failed_to_download) { - reports.addToReport(scope, { type: `warning`, code: `ALK0155`, value: `Could not download file using fetch (${ err_msg }). ALKiln will now fallback to the click download method.` }); - scope.toDownload = filename; + reports.addToReport(scope, { type: `warning`, code: `ALK0155`, value: `Could not download a file matching the name "${ full_or_partial_file_href }" using fetch (${ err_msg }). ALKiln will now fallback to the click download method.` }); + scope.toDownload = full_or_partial_file_href; // Should this be using `scope.tapElement`? await elem.evaluate( elem => { return elem.click(); }); await scope.detectDownloadComplete( scope ); } }, // Ends scope.steps.download() + get_response_filename: async function({ disposition, default_name=`found_no_file_name.pdf` }) { + /** Given a fetch response headers' Content-Disposition str, return the + * filename of the response's file. + * + * @return { string | null } - Name of fetched file + * */ + let filename = default_name; + if ( disposition ) { + filename = disposition.split(`;`)[1].split(`=`)[1]; + } + + // Handle potential UTF-8 encoded filenames + if ( filename.toLowerCase().startsWith( `utf-8''` )) { + filename = decodeURIComponent( filename.replace( /utf-8''/i, `` )); + } else { + // Replace starting and ending quotes if they exist + filename = filename.replace( /^['"]/, `` ).replace( /['"]$/, `` ); + } + + // TODO: Add debug log here + // console.log(`filename:`, filename); + + // TODO: Add to the report if we had to use the default name + + return filename; + }, // Ends scope.steps.get_response_filename() + compare_pdfs: async function (scope, {existing_pdf_path, new_pdf_path}) { let existing_paths = await scope.findFiles(scope, {to_find_names: [existing_pdf_path]}); if (existing_paths.length == 0) { @@ -2845,7 +2891,7 @@ module.exports = { let removed_str = diffs.filter(part => part.removed).reduce((err_str, part) => { return err_str + `- ${ part.value }\n` }, 'The new PDF removed: \n'); - let msg = `The PDFs were not the same.\n${ added_str }\n${removed_str}\n\n You can see the full PDFs at ${ full_existing_path } and ${ full_new_pdf_path}`; + let msg = `The PDFs were different.\nAdded:\n${ added_str }\nRemoved:\n${removed_str}\n\nThere might be more information if you actually look at the files. You can see the full PDFs at ${ full_existing_path } and ${ full_new_pdf_path}`; reports.addToReport(scope, { type: `error`, code: `ALK0157`, value: msg }); scope.failed_pdf_compares.push(msg); } diff --git a/lib/steps.js b/lib/steps.js index 4dce3f30..5c4c2730 100644 --- a/lib/steps.js +++ b/lib/steps.js @@ -1298,10 +1298,11 @@ After(async function(scenario) { if ( scope.failed_pdf_compares.length > 0) { let msg = scope.failed_pdf_compares.reduce((str, new_msg) => `${ str }\nā€•ā€•ā€•\n${ new_msg }`) changeable_test_status = `FAILED`; + // TODO: This may be redundant and therefore confusing reports.addToReport(scope, { type: `error`, code: `ALK0093`, - value: `PDF comparison failed ${ scope.failed_pdf_compares.length } time(s)\nā€•ā€•ā€•\n${ msg }\nā€•ā€•ā€•\n` + value: `ALKiln ran into an error when it tried to compare ${ scope.failed_pdf_compares.length } PDF(s)\nā€•ā€•ā€•\n${ msg }\nā€•ā€•ā€•\n` }); } diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 30576977..1f8120d1 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -22,7 +22,7 @@ "minimatch": "3.0.5", "minimist": "1.2.8", "mocha": "9.2.2", - "pdfjs-dist": "3.2.146", + "pdfjs-dist": "3.11.174", "puppeteer": "22.15.0", "qs": "6.10.3", "sanitize-filename": "1.6.3", @@ -2908,6 +2908,16 @@ "node": ">=0.10.0" } }, + "node_modules/path2d-polyfill": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/path2d-polyfill/-/path2d-polyfill-2.0.1.tgz", + "integrity": "sha512-ad/3bsalbbWhmBo0D6FZ4RNMwsLsPpL6gnvhuSaU5Vm7b06Kr5ubSltQQ0T7YKsiJQO+g22zJ4dJKNTXIyOXtA==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=8" + } + }, "node_modules/pathval": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.1.tgz", @@ -2917,14 +2927,16 @@ } }, "node_modules/pdfjs-dist": { - "version": "3.2.146", - "resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-3.2.146.tgz", - "integrity": "sha512-wy1OB/v75usRD1LqgxBUWC+ZOiKTmG5J8c2z9XVFrVSSWiVbSuseNojmvFa/TT0pYtcFmkL4zn6KaxvqfPYMRg==", - "dependencies": { - "web-streams-polyfill": "^3.2.1" + "version": "3.11.174", + "resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-3.11.174.tgz", + "integrity": "sha512-TdTZPf1trZ8/UFu5Cx/GXB7GZM30LT+wWUNfsi6Bq8ePLnb+woNKtDymI2mxZYBpMbonNFqKmiz684DIfnd8dA==", + "license": "Apache-2.0", + "engines": { + "node": ">=18" }, "optionalDependencies": { - "canvas": "^2.11.0" + "canvas": "^2.11.2", + "path2d-polyfill": "^2.0.1" } }, "node_modules/pend": { @@ -3782,14 +3794,6 @@ "extsprintf": "^1.2.0" } }, - "node_modules/web-streams-polyfill": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", - "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", - "engines": { - "node": ">= 8" - } - }, "node_modules/webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", diff --git a/package.json b/package.json index f56998df..a08c6057 100644 --- a/package.json +++ b/package.json @@ -47,7 +47,7 @@ "minimatch": "3.0.5", "minimist": "1.2.8", "mocha": "9.2.2", - "pdfjs-dist": "3.2.146", + "pdfjs-dist": "3.11.174", "puppeteer": "22.15.0", "qs": "6.10.3", "sanitize-filename": "1.6.3",