Start to Improve Scenarios as Regression Tests (#275)

* Start to Improve Scenarios as Regression Tests

This patch starts to improve scenarios so that they can pass consistently. We make a few changes:

1. We add a `QUIET_SCENARIOS` flag that pushes all scenario output to temp files. This makes it possible to run scenarios and see the output and not 1,000 pages of validator text.
2. We add an `EventTracker` module that handles tracking events without using global vars (which was probably messing up test cases). Events are hard because they come in from a subscription in PolkadotJS, but our scenarios often ask "Have you seen any Substrate `Lock()` events?" and we have to say something like "Yes, we saw one more recently than the last time you asked." That can be a bit tricky and now that state is encapsulated and doesn't leak from one scenario to the next (this may have been a large bug in running sequential tests).
3. We start to move back away from `--runInBand` which means tests will run in separate processes and concurrently again. We may want to make it easier to not run in band for building tests (it's just a matter of passing the flag, but engineers might forget/not know how to do this, so we should make that clear).
4. We remove `only: true` and start to add back multiple tests so we can measure what is working and not. So far the results are not too excessively shabby.

* Continue to fix integration tests

* Add RPC decoration to preclude warnings

* Make scenarios quiet in CI

* Fix line number errors

* Track sleeps for better teardown procedure

* Try to quiet them down

* Increase timeout

* Try again

* Get more tests passing

* Get last upgrade scen working again

* Finish prices tests

* Add complex scenario for borrow cash interest

* Add features for integration mode

* More cleanup and fixes

* Clean up Gov tests

* Clean up more tests, including prices

* Fix some types issues and merge conflicts

* It was always a String, let's make it so

Author: hayesgm

.github/workflows/test-workflow.yml

@@ -237,14 +237,14 @@ jobs:
 
       - name: Build Gateway
         run: |
-          cargo build
+          cargo build --release --features freeze-time --features runtime-debug
 
       - name: Run Integration Test
-        timeout-minutes: 20
+        timeout-minutes: 40
         run: |
           set -eo pipefail
           cd integration
-          sudo npx jest --ci --reporters=default --reporters=jest-junit
+          sudo PROFILE=release QUIET_SCENARIOS=true npx jest --ci --reporters=default --reporters=jest-junit
 
       - name: Prepublish Integration results
         if: always()

base_types.json

@@ -5,8 +5,8 @@
   "Keys": "SessionKeys",
   "SignedPayload": "Vec<u8>",
   "VersionedAuthorityList": {
-    "authorityList": "AuthorityList",
-    "version": "u8"
+    "version": "u8",
+    "authorityList": "AuthorityList"
   },
   "LookupSource": "MultiAddress",
   "SessionKeys": {

integration/__tests__/borrow_scen.js

@@ -1,6 +1,5 @@
 const {
-  buildScenarios,
-  sleep,
+  buildScenarios
 } = require('../util/scenario');
 const { getNotice } = require('../util/substrate');
 
@@ -22,13 +21,15 @@ async function lockZRX({ ashley, bert, bat, zrx }) {
 buildScenarios('Borrow Scenarios', borrow_scen_info, { beforeEach: lockZRX }, [
   {
     name: "Borrow BAT and Garner Interest",
-    scenario: async ({ ashley, bert, bat, zrx, chain, starport, log, cash }) => {
+    notes: "This test allows arbitrary time passage, and thus has some estimation",
+    scenario: async ({ ashley, bert, bat, zrx, chain, starport, log, cash, sleep }) => {
       let cashBalance0 = await ashley.chainBalance(cash);
       let cashIndex0 = await chain.cashIndex();
       let cash0 = await ashley.cash();
 
       expect(cash0).toEqual(0);
 
+      // Ashley Borrows BAT
       let notice = getNotice(await ashley.extract(1_000_000, bat));
 
       // Check totals
@@ -37,24 +38,32 @@ buildScenarios('Borrow Scenarios', borrow_scen_info, { beforeEach: lockZRX }, [
       expect(await bat.totalChainSupply()).toEqual(2_000_000);
       expect(await bat.totalChainBorrows()).toEqual(1_000_000);
 
+      // Pull BAT from the Starport
       let signatures = await chain.getNoticeSignatures(notice);
       await starport.invoke(notice, signatures);
+
+      // Check totals
       expect(await ashley.tokenBalance(bat)).toEqual(1_000_000);
       expect(await ashley.chainBalance(zrx)).toEqual(10_000_000);
       expect(await ashley.chainBalance(bat)).toEqual(-1_000_000);
+
+      // See that we've had _any_ cash interest accrued (assume it's less than a dime)
       let cashBalance1 = await ashley.chainBalance(cash);
       let cashIndex1 = await chain.cashIndex();
       let cash1 = await ashley.cash();
       expect(cash1).toBeLessThan(0);
-      expect(cash1).toBeGreaterThan(-0.001);
+      expect(cash1).toBeGreaterThan(-0.10);
+
       await sleep(20000);
+
+      // See that we've had _more_ cash interest accrued (assume it's less than a quarter)
       let cashBalance2 = await ashley.chainBalance(cash);
       let cashIndex2 = await chain.cashIndex();
       let cash2 = await ashley.cash();
 
       // Nothing is exact here.
       expect(cash2).toBeLessThan(cash1);
-      expect(cash2).toBeGreaterThan(-0.01);
+      expect(cash2).toBeGreaterThan(-0.25);
     }
   }
 ]);

integration/__tests__/cash_scen.js

@@ -0,0 +1,94 @@
+const { buildScenarios } = require('../util/scenario');
+
+let now = Date.now();
+
+let cash_scen_info = {
+  tokens: [
+    { token: 'usdc', balances: { ashley: 1000 } },
+    { token: 'zrx', balances: { bert: 1000000 } },
+    { token: 'comp' }
+  ],
+  validators: ['alice', 'bob'],
+  freeze_time: now,
+  initial_yield: 300,
+  initial_yield_start_ms: now
+};
+
+buildScenarios('Cash Scenarios', cash_scen_info, [
+  {
+    name: 'Cash Interest',
+    scenario: async ({ ashley, bert, cash, chain, usdc }) => {
+      await ashley.lock(1000, usdc);
+      await ashley.transfer(10, cash, bert);
+      expect(await ashley.chainBalance(usdc)).toEqual(1000);
+      expect(await ashley.chainBalance(cash)).toEqual(-10.01); // $10 + 1¢ transfer fee
+      expect(await bert.chainBalance(cash)).toEqual(10);
+      await chain.accelerateTime({years: 1});
+      expect(await ashley.chainBalance(usdc)).toEqual(1000);
+      expect(await ashley.chainBalance(cash)).toBeCloseTo(-10.314849, 4); // $10.01 @ 3% for 1 Year Continously Compounding
+      expect(await bert.chainBalance(cash)).toBeCloseTo(10.304545, 4); // // $10 @ 3% for 1 Year Continously Compounding
+    }
+  },
+  {
+    name: 'Collateral Borrowed Interest Lump Sum',
+    scenario: async ({ ashley, bert, chuck, cash, chain, usdc, zrx }) => {
+      await chain.setFixedRate(usdc, 500); // 5% APY fixed
+      await bert.lock(1000000, zrx);
+      await bert.transfer(1000, usdc, chuck);
+      expect(await bert.chainBalance(usdc)).toEqual(-1000);
+      expect(await chuck.chainBalance(usdc)).toEqual(1000);
+      expect(await bert.chainBalance(cash)).toEqual(-0.01); // 1¢ transfer fee
+      expect(await chuck.chainBalance(cash)).toEqual(0);
+      await chain.accelerateTime({years: 1});
+      expect(await bert.chainBalance(usdc)).toEqual(-1000);
+      expect(await chuck.chainBalance(usdc)).toEqual(1000);
+      expect(await bert.chainBalance(cash)).toBeCloseTo(-51.53272669767585, 3); // -50 * Math.exp(0.03) - 0.01
+      expect(await chuck.chainBalance(cash)).toBeCloseTo(51.52272669767585, 3); // 50 * Math.exp(0.03)
+    }
+  },
+  {
+    name: 'Collateral Borrowed Interest 12-Month Chunked',
+    scenario: async ({ ashley, bert, chuck, cash, chain, usdc, zrx }) => {
+      await chain.setFixedRate(usdc, 500); // 5% APY fixed
+      await bert.lock(1000000, zrx);
+      await bert.transfer(1000, usdc, chuck);
+      expect(await bert.chainBalance(usdc)).toEqual(-1000);
+      expect(await chuck.chainBalance(usdc)).toEqual(1000);
+      expect(await bert.chainBalance(cash)).toEqual(-0.01); // 1¢ transfer fee
+      expect(await chuck.chainBalance(cash)).toEqual(0);
+      for (const i in [...new Array(12)]) {
+        await chain.accelerateTime({months: 1});
+      }
+      expect(await bert.chainBalance(usdc)).toEqual(-1000);
+      expect(await chuck.chainBalance(usdc)).toEqual(1000);
+      expect(await bert.chainBalance(cash)).toBeCloseTo(-50.79, 1); // ~ -50*(1+0.015) - 0.01
+      expect(await chuck.chainBalance(cash)).toBeCloseTo(50.78, 1); // ~ -50*(1+0.015)
+    }
+  },
+  {
+    name: 'Multi-Collateral and Cash Netting',
+    scenario: async ({ ashley, bert, chuck, cash, chain, comp, usdc, zrx }) => {
+      await chain.setFixedRate(usdc, 500); // 5% APY fixed
+      await chain.setFixedRate(comp, 1000); // 10% APY fixed
+      await bert.lock(1000000, zrx);
+      await bert.transfer(300.01, cash, chuck);
+      await bert.transfer(1000, usdc, chuck);
+      await chuck.transfer(1, comp, bert);
+      // Chuck has +1000 USDC @ 5% [Price=$1] [Util=100%]
+      // Chuck has -1 COMP @ 10% [Price=$229.125] [Util=100%]
+      // Chuck has 300 Cash @ 3% APY
+      expect(await chuck.chainBalance(usdc)).toEqual(1000);
+      expect(await chuck.chainBalance(comp)).toEqual(-1);
+      expect(await chuck.chainBalance(cash)).toEqual(300);
+      await chain.accelerateTime({years: 1});
+      expect(await chuck.chainBalance(usdc)).toEqual(1000);
+      expect(await chuck.chainBalance(comp)).toEqual(-1);
+      /*
+          { Cash }  {  USDC Interest  }   {  Comp Interest  }   {  Cash APY   }
+
+        (   300   +   1000 * 1 * 0.05   -  1 * 229.125 * 0.1 ) * Math.exp(0.03)
+      */
+      expect(await chuck.chainBalance(cash)).toBeCloseTo(337.048797374521, 3);
+    }
+  }
+]);

integration/__tests__/extract_scen.js

@@ -3,6 +3,8 @@ const {
 } = require('../util/scenario');
 const { getNotice } = require('../util/substrate');
 
+let now = Date.now();
+
 let extract_scen_info = {
   tokens: [
     { token: "zrx", balances: { ashley: 1000 } },
@@ -43,7 +45,6 @@ buildScenarios('Extract Scenarios', extract_scen_info, { beforeEach: lockUSDC },
     }
   },
   {
-    only: true,
     name: "Extract Cash",
     scenario: async ({ ashley, zrx, chain, starport, cash, log }) => {
       let notice = getNotice(await ashley.extract(20, cash));
@@ -72,23 +73,28 @@ buildScenarios('Extract Scenarios', extract_scen_info, { beforeEach: lockUSDC },
   {
     name: "Extract Comp Torrey",
     beforeEach: null,
+    info: {
+      freeze_time: now,
+    },
     scenario: async ({ ashley, bert, usdc, comp, chain, starport, cash, log }) => {
-      // User A Supplied 100k USDC
+      // Alice supplies 100K USDC
       await ashley.lock(100_000, usdc);
-      // User B Supplied 200 COMP
+      // Bert supplies 200 COMP
       await bert.lock(200, comp);
-      // User A downloaded 50 COMP
+      // Alice extracts 50 COMP
       let notice = getNotice(await ashley.extract(50, comp));
       let signatures = await chain.getNoticeSignatures(notice);
 
+      await chain.accelerateTime({days: 1});
+
       expect(await ashley.tokenBalance(comp)).toEqual(0);
       let tx = await starport.invoke(notice, signatures);
       let ashleyComp = await ashley.tokenBalance(comp);
       let ashleyCash = await ashley.cash();
       let ashleyLiquidity = await ashley.liquidity();
       expect(ashleyComp).toEqual(50);
-      expect(ashleyCash).toBeCloseTo(-0.0003809713723277, 4);
-      expect(ashleyLiquidity).toBeCloseTo(64724.99962, 2);
+      expect(ashleyCash).toBeCloseTo(-2.743447, 4);
+      expect(ashleyLiquidity).toBeCloseTo(64700, -3);
     }
   },
   {