package update & debug

ScottishCovidResponse · Sep 25, 2020 · bb2326c · bb2326c
1 parent 37404fe
commit bb2326c
Show file tree

Hide file tree

Showing 7 changed files with 908 additions and 1,706 deletions.
diff --git a/data-api/package-lock.json b/data-api/package-lock.json
diff --git a/data-api/package.json b/data-api/package.json
@@ -11,68 +11,68 @@
     "dev:debug": "nodemon --inspect src/server.ts"
   },
   "dependencies": {
-    "@fast-csv/format": "^4.3.0",
-    "@fast-csv/parse": "^4.3.0",
+    "@fast-csv/format": "^4.3.1",
+    "@fast-csv/parse": "^4.3.2",
     "accesscontrol": "^2.2.1",
-    "app-root-path": "^2.2.1",
+    "app-root-path": "^3.0.0",
     "automapper-ts": "^1.9.0",
     "bcryptjs": "^2.4.3",
     "body-parser": "^1.19.0",
-    "class-transformer": "^0.2.3",
-    "class-validator": "^0.11.0",
+    "class-transformer": "^0.3.1",
+    "class-validator": "^0.12.2",
     "compression": "^1.7.4",
-    "config": "^3.2.2",
-    "connect-mongo": "^3.0.0",
-    "convert-csv-to-json": "0.0.15",
+    "config": "^3.3.2",
+    "connect-mongo": "^3.2.0",
+    "convert-csv-to-json": "1.0.0",
     "cors": "^2.8.5",
     "csvtojson": "^2.0.10",
-    "debug": "~4.1.1",
-    "envalid": "^5.0.0",
+    "debug": "~4.2.0",
+    "envalid": "^6.0.2",
     "express": "~4.17.1",
     "express-session": "^1.17.1",
-    "fast-csv": "^4.3.0",
+    "fast-csv": "^4.3.2",
     "inversify": "^5.0.1",
     "inversify-binding-decorators": "^4.0.0",
     "inversify-express-utils": "^6.3.2",
     "inversify-inject-decorators": "^3.1.0",
     "jsonwebtoken": "^8.5.1",
-    "lodash": "^4.17.15",
-    "moment": "^2.24.0",
-    "mongodb": "^3.5.2",
-    "morgan": "^1.9.1",
+    "lodash": "^4.17.20",
+    "moment": "^2.29.0",
+    "mongodb": "^3.6.2",
+    "morgan": "^1.10.0",
     "multer": "^1.4.2",
     "npm-run-all": "^4.1.5",
     "passport": "^0.4.1",
     "passport-github2": "^0.1.12",
     "passport-local": "^1.0.0",
-    "python-shell": "^2.0.1",
+    "python-shell": "^2.0.2",
     "reflect-metadata": "^0.1.13",
     "request-promise": "^4.2.5",
-    "rimraf": "^3.0.0",
+    "rimraf": "^3.0.2",
     "s": "^1.0.0",
-    "underscore": "^1.9.2",
-    "winston": "^3.2.1"
+    "underscore": "^1.11.0",
+    "winston": "^3.3.3"
   },
   "devDependencies": {
     "@types/app-root-path": "^1.2.4",
     "@types/bcryptjs": "^2.4.2",
-    "@types/compression": "1.0.1",
-    "@types/config": "0.0.34",
-    "@types/connect-mongo": "0.0.43",
-    "@types/cors": "^2.8.6",
-    "@types/express": "^4.17.1",
-    "@types/express-session": "^1.15.14",
-    "@types/jsonwebtoken": "^8.3.3",
-    "@types/mongodb": "^3.3.15",
-    "@types/morgan": "^1.7.37",
-    "@types/multer": "^1.4.3",
-    "@types/node": "^12.7.5",
+    "@types/compression": "1.7.0",
+    "@types/config": "0.0.36",
+    "@types/connect-mongo": "3.1.2",
+    "@types/cors": "^2.8.7",
+    "@types/express": "^4.17.8",
+    "@types/express-session": "^1.17.0",
+    "@types/jsonwebtoken": "^8.5.0",
+    "@types/mongodb": "^3.5.27",
+    "@types/morgan": "^1.9.1",
+    "@types/multer": "^1.4.4",
+    "@types/node": "^14.11.2",
     "@types/passport-github2": "^1.2.4",
-    "@types/request-promise": "^4.1.45",
-    "@types/underscore": "^1.9.4",
-    "nodemon": "^1.19.2",
-    "ts-node": "^8.3.0",
-    "tslint": "^5.20.0",
-    "typescript": "^3.6.3"
+    "@types/request-promise": "^4.1.46",
+    "@types/underscore": "^1.10.23",
+    "nodemon": "^2.0.4",
+    "ts-node": "^9.0.0",
+    "tslint": "^5.20.1",
+    "typescript": "^4.0.3"
   }
 }
diff --git a/data-api/src/middleware/dto.validate.ts b/data-api/src/middleware/dto.validate.ts
@@ -21,17 +21,22 @@ export function dtoValidate<T>(type: any, skipMissingProperties = false): Reques
   };
 }
 
+// TODO - remove duplicate code
 export function queryParamValidate<T>(type: any, skipMissingProperties = false): RequestHandler {
+
   return (req, res, next) => {
-    validate(plainToClass(type, req.query), { skipMissingProperties })
-      .then((errors: ValidationError[]) => {
-        if (errors.length > 0) {
-          const message = errors.map((error: ValidationError) => Object.values(error.constraints)).join(', ');
-          next(new InvalidQueryParametersException(message));
-        } else {
-          next();
-        }
-      });
+      validate(plainToClass(type, req.query), {skipMissingProperties})
+          .then((errors: ValidationError[]) => {
+              if (errors.length > 0) {
+                  const message = errors.map((error: ValidationError) => {
+                      const constraints: any[] = getNestedProperties(error);
+                      return constraints.map((constraint: any) => Object.values(constraint));
+                  }).join(', ');
+                  next(new InvalidQueryParametersException(message));
+              } else {
+                  next();
+              }
+          });
   };
 }
 

diff --git a/data-api/src/services/activity.service.ts b/data-api/src/services/activity.service.ts
@@ -47,7 +47,7 @@ export class ActivityService extends DataService<IActivity> {
     }
 
     async removeActivitiesOfAccount(accountId: string): Promise<number> {
-        const result = await this.getCollection().deleteMany({ accountId: accountId } as FilterQuery<IActivity>);
+        const result = await this.getDbCollection().deleteMany({ accountId: accountId } as FilterQuery<IActivity>);
         return result.deletedCount as number;
     }
 

diff --git a/data-api/src/services/bookmark.service.ts b/data-api/src/services/bookmark.service.ts
@@ -46,7 +46,7 @@ export class BookmarkService extends DataService<IBookmark> {
     }
 
     async deleteBookmark(user: IUser, pageId: string): Promise<IBookmark> {
-        const res: FindAndModifyWriteOpResultObject<IBookmark> = await this.getCollection().findOneAndDelete({userId: <string>user._id , pageId: pageId } as FilterQuery<IBookmark>);
+        const res: FindAndModifyWriteOpResultObject<IBookmark> = await this.getDbCollection().findOneAndDelete({userId: <string>user._id , pageId: pageId } as FilterQuery<IBookmark>);
         return automapper.map(MAPPING_TYPES.MongoDbObjectId, MAPPING_TYPES.TsString, res.value);
     }
 

diff --git a/data-api/src/services/data.service.ts b/data-api/src/services/data.service.ts
@@ -1,12 +1,12 @@
-import {inject, unmanaged} from 'inversify';
+import { inject, unmanaged } from 'inversify';
 import { provide } from 'inversify-binding-decorators';
-
 import {
     AggregationCursor,
     Collection,
     FilterQuery,
     FindAndModifyWriteOpResultObject,
-    FindOneAndUpdateOption, FindOneOptions,
+    FindOneAndUpdateOption,
+    FindOneOptions,
     ObjectId,
     UpdateOneOptions,
     UpdateQuery,
@@ -19,106 +19,104 @@ import { TYPES } from './config/types';
 
 @provide(TYPES.DataService)
 export abstract class DataService<T extends { _id: any }> {
-    private readonly db: Collection<T>;
-    private readonly dbName: string;
-    private readonly collectionName: string;
+    private readonly dbCollection: Collection<T>;
+    private dbName: string;
+    private readonly dbCollectionName: string;
 
     protected constructor(
         @inject(TYPES.DbClient) dbClient: DbClient,
         @unmanaged() dbName: string,
         @unmanaged() collectionName: string,
     ) {
         this.dbName = dbName;
-        this.collectionName = collectionName;
-        this.db = dbClient.db(this.dbName).collection<T>(this.collectionName);
+        this.dbCollectionName = collectionName;
+        this.dbCollection = dbClient.db(this.dbName).collection<T>(this.dbCollectionName);
     }
 
-    public getCollection(): Collection<T> {
-        return this.db;
+    public getDbCollection(): Collection<T> {
+        return this.dbCollection;
     }
 
-    public async get(id: string | FilterQuery<T>): Promise<T> {
-
+    public async get(arg: string | FilterQuery<T>): Promise<T> {
         let doc: T;     // typeof _id = ObjectId
-        let result: T;  // typeof _id = string
 
-        if (typeof id === 'string') {
-            doc = await this.db.findOne({_id: new ObjectId(id)} as FilterQuery<T>) as T;
-        } else if (typeof id === 'object') {
-            doc = await this.db.findOne(id as FilterQuery<T>) as T;
+        if (typeof arg === 'string') {
+            doc = await this.dbCollection.findOne({ _id: new ObjectId(arg) } as FilterQuery<T>) as T;
+        } else if (typeof arg === 'object') {
+            doc = await this.dbCollection.findOne(arg as FilterQuery<T>) as T;
         } else {
             return null as any;
         }
 
-        result = automapper.map(MAPPING_TYPES.MongoDbObjectId, MAPPING_TYPES.TsString, doc);
-        return result;
+        return automapper.map(MAPPING_TYPES.MongoDbObjectId, MAPPING_TYPES.TsString, doc);
     }
 
-    public async getAll(query: FilterQuery<T> = {}, options: FindOneOptions = {}): Promise<T[]> {
-        const docs: any[] = await this.db.find(query, options).toArray();
-        const results: T[] = automapper.map(MAPPING_TYPES.MongoDbObjectId, MAPPING_TYPES.TsString, docs);
-        return results;
+    public async getAll(query: FilterQuery<T> = {}, options: FindOneOptions<any> = {}): Promise<T[]> {
+        const docs: any[] = await this.dbCollection.find(query).project(options).toArray();
+        return automapper.map(MAPPING_TYPES.MongoDbObjectId, MAPPING_TYPES.TsString, docs);
     }
 
+    // Get latest numDocs number of document from a collection passing the query
     public async getLatest(query: FilterQuery<T> = {}, numDocs: number = 1): Promise<T> {
-        const docs: any[] = await this.db.find(query).sort({$natural: -1}).limit(numDocs).toArray();
-        const result: T = automapper.map(MAPPING_TYPES.MongoDbObjectId, MAPPING_TYPES.TsString, docs)[0];
-        return result;
+        // Auto created _id field it has a lastPostDate embedded in it (_id: 1) or use natural order ($natural: 1)
+        // 1 will sort ascending (oldest to newest) and -1 will sort descending (newest to oldest.)
+        const docs: any[] = await this.dbCollection.find(query).sort({ $natural: -1 }).limit(numDocs).toArray();
+        return automapper.map(MAPPING_TYPES.MongoDbObjectId, MAPPING_TYPES.TsString, docs)[0];
     }
 
+    // get or create and get object
     public async getOrCreate(query: FilterQuery<T> = {}, entity: T): Promise<T> {
-        const res: FindAndModifyWriteOpResultObject<T> = await this.getCollection().findOneAndUpdate(
+        const res: FindAndModifyWriteOpResultObject<T> = await this.getDbCollection().findOneAndUpdate(
             query,
             { $setOnInsert: entity } as UpdateQuery<T>,
             {
-                upsert: true,
                 returnOriginal: false,  // When false, returns the updated document rather than the original. The default is true.
-            } as FindOneAndUpdateOption,
+                upsert: true,
+            } as FindOneAndUpdateOption<T>,
         );
 
-        const result: T = automapper.map(MAPPING_TYPES.MongoDbObjectId, MAPPING_TYPES.TsString, res.value);
-        return result as T;
+        return automapper.map(MAPPING_TYPES.MongoDbObjectId, MAPPING_TYPES.TsString, res.value);
     }
 
     public async save(entity: T): Promise<T> {
-        const res = await this.db.insertOne(entity as any);
-        const result: T = automapper.map(MAPPING_TYPES.MongoDbObjectId, MAPPING_TYPES.TsString, res.ops[0]);
-        return result;
+        const res = await this.dbCollection.insertOne(entity as any);
+        return automapper.map(MAPPING_TYPES.MongoDbObjectId, MAPPING_TYPES.TsString, res.ops[0]);
     }
 
     public async delete(id: string): Promise<T> {
-        const res: FindAndModifyWriteOpResultObject<T> = await this.db.findOneAndDelete({_id: new ObjectId(id)} as FilterQuery<T>);
-        const result: T = automapper.map(MAPPING_TYPES.MongoDbObjectId, MAPPING_TYPES.TsString, res.value);
-        return result;
+        const res: FindAndModifyWriteOpResultObject<T> = await this.dbCollection.findOneAndDelete({ _id: new ObjectId(id) } as FilterQuery<T>);
+        return automapper.map(MAPPING_TYPES.MongoDbObjectId, MAPPING_TYPES.TsString, res.value);
     }
 
+    // update and get update status
     public async update(id: string, entity: T): Promise<boolean> {
-        const res: UpdateWriteOpResult = await this.db.updateOne(
-            {_id: new ObjectId(id)} as FilterQuery<T>,
-            {$set: entity} as UpdateQuery<T>,
+        const res: UpdateWriteOpResult = await this.dbCollection.updateOne(
+            { _id: new ObjectId(id) } as FilterQuery<T>,
+            { $set: entity } as UpdateQuery<T>,
             {
                 upsert: false, // does not insert a new document when no match is found
             } as UpdateOneOptions,
         );
         return res.result.ok === 1;
     }
 
+    // update and get the updated doc
     public async updateAndGet(id: string, entity: T): Promise<T> {
-        const res: FindAndModifyWriteOpResultObject<T> = await this.getCollection().findOneAndUpdate(
+        const res: FindAndModifyWriteOpResultObject<T> = await this.getDbCollection().findOneAndUpdate(
             { _id: new ObjectId(id) } as FilterQuery<T>,
-            { $set: entity} as UpdateQuery<T>,
+            { $set: entity } as UpdateQuery<T>,
             {
-                upsert: false,
                 returnOriginal: false,  // When false, returns the updated document rather than the original. The default is true.
-            } as FindOneAndUpdateOption,
+                upsert: false,
+            } as FindOneAndUpdateOption<T>,
         );
 
         const result: T = automapper.map(MAPPING_TYPES.MongoDbObjectId, MAPPING_TYPES.TsString, res.value);
         return result as T;
     }
 
     public aggregate(pipeline: object[]): AggregationCursor<T> {
-        return this.db.aggregate(pipeline);
+        return this.dbCollection.aggregate(pipeline);
     }
 
     public async aggregateOne(pipeline: object[]): Promise<T | null> {
@@ -130,17 +128,19 @@ export abstract class DataService<T extends { _id: any }> {
     }
 
     public async createIndex(field: any, expiry: number): Promise<boolean> {
-
         return new Promise<boolean>(async (resolve) => {
             try {
-                await this.getCollection().dropIndex(field);
+                await this.getDbCollection().dropIndex(field);
             } catch (e) {
                 //
             }
 
-            this.getCollection().createIndex(field, {expireAfterSeconds: expiry},
+            this.getDbCollection().createIndex(field,
+                { expireAfterSeconds: expiry },
                 (err, dbResult) => {
-                    if (err) { throw err; }
+                    if (err) {
+                        throw err;
+                    }
                     resolve(true);
                 });
         });
@@ -151,12 +151,12 @@ export abstract class DataService<T extends { _id: any }> {
             try {
                 for (const d of Object.keys(obj)) {
                     console.log('DataService: createTextIndex: d : ', d);
-                    await this.getCollection().dropIndex(d);
+                    await this.getDbCollection().dropIndex(d);
                 }
             } catch (e) {
             }
 
-            this.getCollection().createIndex(obj, (err, dbResult) => {
+            this.getDbCollection().createIndex(obj, (err, dbResult) => {
                     if (err) { throw err; }
                     resolve(true);
                 });

diff --git a/data-api/src/services/search.service.ts b/data-api/src/services/search.service.ts
@@ -30,7 +30,7 @@ export class SearchService extends DataService<IPage> {
             },
             {"$sort": {"score": {"$meta": "textScore"}}}
         ];
-        return this.getCollection().aggregate(pipeline).toArray();
+        return this.getDbCollection().aggregate(pipeline).toArray();
 
     }
 }