Merge branch 'v5' into 805_verbose

CHANGELOG.md

@@ -43,7 +43,21 @@ Format:
 - 
 -->
 
-## [Unreleased]
+<!-- ## [Unreleased] -->
+
+## [5.13.3] - 2025-01-05
+
+### Fixed
+
+- Updates to latest docassemble HTML. See [#960](https://github.com/SuffolkLITLab/ALKiln/issues/960). Changes all appearances of `fieldset` to keep the code consistent.
+
+## [5.13.2] - 2024-11-23
+
+### Fixed
+
+- Fixes download Step unable to use a partial filename match. The Step needed the whole name, including the extension. See [#725](https://github.com/SuffolkLITLab/ALKiln/issues/725).
+
+## [5.13.1] - 2024-10-23
 
 ### Changed
 
@@ -68,6 +82,7 @@ Format:
 - Closes [#659](https://github.com/SuffolkLITLab/ALKiln/issues/659), abstract adding to debug_log
 - Closes [#925](https://github.com/SuffolkLITLab/ALKiln/issues/925), allow a `Log` to throw an error
 - Addresses [#461](https://github.com/SuffolkLITLab/ALKiln/issues/461), setup and takedown reports
+- Updates pdfjs-dist for node v20 and v22. See [#952](https://github.com/SuffolkLITLab/ALKiln/issues/952).
 
 ## [5.13.0] - 2024-07-11
 

README.md

@@ -41,25 +41,46 @@ Read about contributing in our [CONTRIBUTING.md document](CONTRIBUTING.md). Here
 ## Cheat sheet
 
 *Once you've already read the contributing documentation, you can use these as quick reminders for running our internal tests.*
-To set up for the integration tests, create the project on the server:
+
+### `setup` before starting development of a feature/fix
+
+Set up for the cucumber integration tests.
+
+Add the feature branch name to your `.env` file:
+
+```
+BRANCH=42_feat_life_the_univers_and_everything
+```
+
+Then create the project on the server:
 
 ```bash
 npm run setup
 ```
 
-Use the syntax below to trigger specific tags:
+### Run tests repeatedly
+
+Run all cucumber tests that are not currently blocked by upstream changes:
+
+```bash
+npm run pass
+```
+
+Trigger cucumber tests with specific tags:
 
 ```bash
-npm run cucumber -- "--tags" "@tagname"
+npm run cucumber "@tagname"
 ```
 
-To run the unit tests in isolation:
+Run the unit tests:
 
 ```bash
 npm run unit
 ```
 
-If you or someone else changes the interview code in `./docassemble`, you have to clean up the old data on the server before running `setup` again:
+### Always run `takedown` before running `setup` again
+
+If you or someone else changes the interview code in our `./docassemble` directory, you have to delete the code currently on the testing server before running `setup` again. Also do this when you're done with the feature/fix or starting a new feature/fix.
 
 ```bash
 npm run takedown

docassemble/ALKilnTests/data/sources/observation_steps.feature

@@ -239,19 +239,19 @@ Scenario: I compare different PDFs
   Given the final Scenario status should be "failed"
   And the Scenario report should include:
   """
-  Could not find the existing PDF at DOES_NOT_EXIST.pdf
+  ALK0156
   """
   And the Scenario report should include:
   """
-  The PDFs were not the same.
+  ALK0157
   """
-  And the Scenario report should include: 
+  And the Scenario report should include:
   """
-  The new PDF added:
+  -  diff
   """
   And the Scenario report should include:
   """
-  -  diff
+  ALK0093
   """
   Given I start the interview at "test_pdf"
   Then the question id should be "proxy vars"
@@ -268,6 +268,7 @@ Scenario: I compare different PDFs
   And I tap to continue
   # Next page
   Then the question id should be "2_signature download"
-  When I download "2_signature.pdf"
+  # Match a partial name
+  When I download "2_signatu"
   And I expect the baseline PDF "DOES_NOT_EXIST.pdf" and the new PDF "2_signature.pdf" to be the same
   And I expect the baseline PDF "linear_2_signature-Baseline.pdf" and the new PDF "2_signature.pdf" to be the same

lib/scope.js

@@ -518,13 +518,17 @@ module.exports = {
   },
 
   continue_button_selector: `fieldset.da-field-buttons button[type="submit"].btn-primary`,
+  continue_button_selector_backup: `.da-fieldset.da-field-buttons button[type="submit"].btn-primary`,
   signature_selector: `fieldset .dasigsave`,
+  signature_selector_backup: `.da-fieldset .dasigsave`,
   continue: async function ( scope ) {
     /* Presses whatever button it finds that might lead to the next page. */
     // Any selectors I find seem somewhat precarious.
     let elem = await Promise.race([
       scope.page.waitForSelector(scope.continue_button_selector),  // other pages (this is the most consistent way)
+      scope.page.waitForSelector(scope.continue_button_selector_backup),
       scope.page.waitForSelector(scope.signature_selector),  // signature page
+      scope.page.waitForSelector(scope.signature_selector_backup),
     ]);
     await elem.evaluate( el => { return el.className });
     // Waits for navigation or user error
@@ -544,9 +548,11 @@ module.exports = {
     // <button type="submit" class="btn btn-da btn-danger" name="X211bHRpcGxlX2Nob2ljZQ" value="0">exit</button>
     // `buttons:` can be used in question blocks as choices
     let regular = await scope.page.$(scope.continue_button_selector);
+    let regular_backup = await scope.page.$(scope.continue_button_selector_backup);
     let signature = await scope.page.$(scope.signature_selector);
+    let signature_backup = await scope.page.$(scope.signature_selector_backup);
 
-    return regular !== null || signature !== null;
+    return regular !== null || regular_backup !== null || signature !== null || signature_backup !== null;
 
   },  // Ends scope.continue_exists()
 
@@ -918,7 +924,7 @@ module.exports = {
     // All the different types of fields
     // buttons, canvases, inputs of all kinds, selects (dropdowns), textareas. Are there more?
     // Will deal with `option` once inside `select`
-    let all_nodes = $( `#dasigpage, fieldset button, .daquestionactionbutton, fieldset input, .da-form-group input, .da-form-group select, form select, .da-form-group textarea` );
+    let all_nodes = $( `#dasigpage, fieldset button, .da-fieldset button, .daquestionactionbutton, fieldset input, da-fieldset input, .da-form-group input, .da-form-group select, form select, .da-form-group textarea` );
     for ( let node of all_nodes ) {
       // Decision: Do not abstract the below. There's too much data to pass around for it to make sense
       let $node = $( node );
@@ -2628,6 +2634,7 @@ module.exports = {
 
     let fullPage = true;
     let signature_elem = await scope.page.$(scope.signature_selector);
+    signature_elem = signature_elem || await scope.page.$(scope.signature_selector_backup);
     if ( signature_elem !== null ) {
       fullPage = false;
     }
@@ -2774,7 +2781,7 @@ module.exports = {
       await scope.afterStep(scope);
     },  // Ends scope.steps.sign()
 
-    download: async ( scope, filename ) => {
+    download: async ( scope, full_or_partial_file_href ) => {
       /* Taps the link that leads to the given filename to trigger downloading.
       *    and waits till the file has been downloaded before allowing the tests to continue.
       *    WARNING: Cannot download the same file twice in a single scenario.
@@ -2784,47 +2791,86 @@ module.exports = {
       * TODO: Properly wait for download to complete. See notes in
       * scope.js scope.detectDownloadComplete()
       */
-      let [elem] = await scope.page.$$(`xpath/.//a[contains(@href, "${ filename }")]`);
+      let [elem] = await scope.page.$$(`xpath/.//a[contains(@href, "${ full_or_partial_file_href }")]`);
 
-      let msg = `"${ filename }" seems to be missing. Cannot find a link to that document.`;
+      let msg = `"${ full_or_partial_file_href }" seems to be missing on the page. Cannot find a link to that document.`;
       if ( !elem ) { reports.addToReport(scope, { type: `error`, code: `ALK0152`, value: msg }); }
       expect( elem, msg ).to.exist;
 
       let failed_to_download = false;
       let err_msg = "";
       try {
-        const binaryStr = await scope.page.evaluate(el => {
+
+        const { disposition,  binaryStr } = await scope.page.evaluate(async function ( el ) {
           const url = el.getAttribute("href");
           return new Promise(async (resolve, reject) => {
             const response = await fetch(url, {method: "GET"});
             const reader = new FileReader();
             reader.readAsBinaryString(await response.blob());
-            reader.onload = () => resolve(reader.result);
+            // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Disposition
+            reader.onload = () => resolve({
+              disposition: response.headers.get(`Content-Disposition`),
+              binaryStr: reader.result
+            });
             reader.onerror = () => reject(`🤕 ALK0153 ERROR: Error occurred on page when downloading ${ url }: ${ reader.error }`);
           });
         }, elem);
+
+        err_msg = `could not get the actual name of the file from the response headers.`;
+        let actual_filename = await scope.steps.get_response_filename({
+          disposition, default_name: full_or_partial_file_href
+        });
+
         if (binaryStr !== '') {
+          err_msg = `binary data for download was empty`;
           const fileData = Buffer.from(binaryStr, 'binary');
-          fs.writeFileSync(`${ scope.paths.scenario }/${ filename }`, fileData);
-          reports.addToReport(scope, { type: `row info`, code: `ALK0154`, value: `Downloaded ${ filename } (${ fileData.length } bytes) to ${ scope.paths.scenario }`});
+          fs.writeFileSync(`${ scope.paths.scenario }/${ actual_filename }`, fileData);
+          reports.addToReport(scope, { type: `row info`, code: `ALK0154`, value: `Downloaded "${ actual_filename }" (${ fileData.length } bytes) to ${ scope.paths.scenario }`});
         } else {
           failed_to_download = true;
-          err_msg = `Couldn't download ${ filename }, binary data for download was empty`;
         }
+
       } catch (error) {
         failed_to_download = true;
         err_msg = error;
       }
 
       if (failed_to_download) {
-        reports.addToReport(scope, { type: `warning`, code: `ALK0155`, value: `Could not download file using fetch (${ err_msg }). ALKiln will now fallback to the click download method.` });
-        scope.toDownload = filename;
+        reports.addToReport(scope, { type: `warning`, code: `ALK0155`, value: `Could not download a file matching the name "${ full_or_partial_file_href }" using fetch (${ err_msg }). ALKiln will now fallback to the click download method.` });
+        scope.toDownload = full_or_partial_file_href;
         // Should this be using `scope.tapElement`?
         await elem.evaluate( elem => { return elem.click(); });
         await scope.detectDownloadComplete( scope );
       }
     },  // Ends scope.steps.download()
 
+    get_response_filename: async function({ disposition, default_name=`found_no_file_name.pdf` }) {
+      /** Given a fetch response headers' Content-Disposition str, return the
+       *  filename of the response's file.
+       *
+       * @return { string | null } - Name of fetched file
+       * */
+      let filename = default_name;
+      if ( disposition ) {
+        filename = disposition.split(`;`)[1].split(`=`)[1];
+      }
+
+      // Handle potential UTF-8 encoded filenames
+      if ( filename.toLowerCase().startsWith( `utf-8''` )) {
+        filename = decodeURIComponent( filename.replace( /utf-8''/i, `` ));
+      } else {
+        // Replace starting and ending quotes if they exist
+        filename = filename.replace( /^['"]/, `` ).replace( /['"]$/, `` );
+      }
+
+      // TODO: Add debug log here
+      // console.log(`filename:`, filename);
+
+      // TODO: Add to the report if we had to use the default name
+
+      return filename;
+    },  // Ends scope.steps.get_response_filename()
+
     compare_pdfs: async function (scope, {existing_pdf_path, new_pdf_path}) {
       let existing_paths = await scope.findFiles(scope, {to_find_names: [existing_pdf_path]});
       if (existing_paths.length == 0) {
@@ -2845,7 +2891,7 @@ module.exports = {
         let removed_str = diffs.filter(part => part.removed).reduce((err_str, part) => {
           return err_str + `- ${ part.value }\n`
         }, 'The new PDF removed: \n');
-        let msg = `The PDFs were not the same.\n${ added_str }\n${removed_str}\n\n You can see the full PDFs at ${ full_existing_path } and ${ full_new_pdf_path}`;
+        let msg = `The PDFs were different.\nAdded:\n${ added_str }\nRemoved:\n${removed_str}\n\nThere might be more information if you actually look at the files. You can see the full PDFs at ${ full_existing_path } and ${ full_new_pdf_path}`;
         reports.addToReport(scope, { type: `error`, code: `ALK0157`, value: msg });
         scope.failed_pdf_compares.push(msg);
       }

lib/steps.js

@@ -1298,10 +1298,11 @@ After(async function(scenario) {
   if ( scope.failed_pdf_compares.length > 0) {
     let msg = scope.failed_pdf_compares.reduce((str, new_msg) => `${ str }\n―――\n${ new_msg }`)
     changeable_test_status = `FAILED`;
+    // TODO: This may be redundant and therefore confusing
     reports.addToReport(scope, {
       type: `error`,
       code: `ALK0093`,
-      value: `PDF comparison failed ${ scope.failed_pdf_compares.length } time(s)\n―――\n${ msg }\n―――\n`
+      value: `ALKiln ran into an error when it tried to compare ${ scope.failed_pdf_compares.length } PDF(s)\n―――\n${ msg }\n―――\n`
     });
   }
 

package.json

@@ -47,7 +47,7 @@
     "minimatch": "3.0.5",
     "minimist": "1.2.8",
     "mocha": "9.2.2",
-    "pdfjs-dist": "3.2.146",
+    "pdfjs-dist": "3.11.174",
     "puppeteer": "22.15.0",
     "qs": "6.10.3",
     "sanitize-filename": "1.6.3",