Harvest db store (#246)

* Create and use a database based storage for harvest data.
* Remove harvest file storage.
* Add Harvest integration tests.
* Fix code style issues.
* Fix unit tests.
* Fix dkan-dummy-content.
* Fix code style issues.
* Fix typo.

cypress/integration/02_harvest_empty.spec.js

@@ -0,0 +1,37 @@
+context('Harvest', () => {
+
+  let user_credentials = Cypress.env("TEST_USER_CREDENTIALS");
+  let apiUri = Cypress.config().apiUri;
+
+  // Set up.
+  before(() => {
+
+  });
+
+  // Clean up.
+  after(() => {
+
+  });
+
+  context('GET harvest/plans', () => {
+    it('List harvest identifiers', () => {
+      cy.request({
+        url: apiUri + '/harvest/plans',
+        auth: user_credentials
+      }).then((response) => {
+        expect(response.status).eql(200);
+        expect(response.body.length).eql(0);
+      })
+    })
+
+    it('Requires authenticated user', () => {
+      cy.request({
+        url: apiUri + '/harvest/plans',
+        failOnStatusCode: false
+      }).then((response) => {
+        expect(response.status).eql(401)
+      })
+    });
+  });
+
+});

cypress/integration/03_harvest.spec.js

@@ -0,0 +1,165 @@
+context('Harvest', () => {
+
+  let user_credentials = Cypress.env("TEST_USER_CREDENTIALS");
+  let apiUri = Cypress.config().apiUri;
+
+  // Set up.
+  before(() => {
+    cy.request({
+      method: 'POST',
+      url: apiUri + '/harvest/plans',
+      auth: user_credentials,
+      body: {
+        "identifier": "test",
+        "extract": {
+          "type": "\\Harvest\\ETL\\Extract\\DataJson",
+          "uri": "https://dkan-default-content-files.s3.amazonaws.com/data.json"
+        },
+        "load": {
+          "type": "\\Drupal\\dkan_harvest\\Load\\Dataset"
+        }
+      }
+    }).then((response) => {
+      expect(response.status).eql(200)
+    })
+  });
+
+  // Clean up.
+  after(() => {
+    cy.request({
+      method: 'DELETE',
+      url: apiUri + '/harvest/plans/test',
+      auth: user_credentials,
+    }).then((response) => {
+      expect(response.status).eql(200)
+    })
+  });
+
+  context('GET harvest/plans', () => {
+    it('List harvest identifiers', () => {
+      cy.request({
+        url: apiUri + '/harvest/plans',
+        auth: user_credentials
+      }).then((response) => {
+        expect(response.status).eql(200);
+        expect(response.body.length).eql(1);
+        expect(response.body[0]).eql('test');
+        cy.request({
+          method: "POST",
+          url: apiUri + '/harvest/runs',
+          auth: user_credentials,
+          body: {
+            "plan_id": "test"
+          }
+        }).then((response) => {
+          expect(response.status).eql(200);
+          cy.log(response)
+        })
+      })
+    })
+
+    it('Requires authenticated user', () => {
+      cy.request({
+        url: apiUri + '/harvest/plans',
+        failOnStatusCode: false
+      }).then((response) => {
+        expect(response.status).eql(401)
+      })
+    });
+  });
+
+  context('POST harvest/plans', () => {
+    it('Requires authenticated user', () => {
+      cy.request({
+        method: "POST",
+        url: apiUri + '/harvest/plans',
+        failOnStatusCode: false
+      }).then((response) => {
+        expect(response.status).eql(401)
+      })
+    })
+  });
+
+  context('GET harvest/plans/PLAN_ID', () => {
+    it('Get a single harvest plan', () => {
+      cy.request({
+        url: apiUri + '/harvest/plans/test',
+        auth: user_credentials
+      }).then((response) => {
+        expect(response.status).eql(200)
+        expect(response.body.identifier).eql('test')
+      })
+    });
+
+    it('Requires authenticated user', () => {
+      cy.request({
+        url: apiUri + '/harvest/runs',
+        failOnStatusCode: false
+      }).then((response) => {
+        expect(response.status).eql(401)
+      })
+    })
+  });
+
+  context('GET harvest/runs?plan=PLAN_ID', () => {
+    it('Gives list of previous runs for a harvest id', () => {
+      cy.request({
+        url: apiUri + '/harvest/runs?plan=test',
+        auth: user_credentials
+      }).then((response) => {
+        expect(response.status).eql(200)
+        expect(response.body.length).eql(1)
+      })
+    });
+
+    it('Requires authenticated user', () => {
+      cy.request({
+        url: apiUri + '/harvest/runs?plan=PLAN_ID',
+        failOnStatusCode: false
+      }).then((response) => {
+        expect(response.status).eql(401)
+      })
+    })
+  });
+
+  context('POST harvest/runs', () => {
+    it('Requires authenticated user', () => {
+      cy.request({
+        method: "POST",
+        url: apiUri + '/harvest/runs',
+        failOnStatusCode: false
+      }).then((response) => {
+        expect(response.status).eql(401)
+      })
+    })
+  });
+
+  context('GET harvest/runs/{identifier}', () => {
+    it('Gives information about a single previous harvest run', () => {
+      cy.request({
+        url: apiUri + '/harvest/runs?plan=test',
+        auth: user_credentials
+      }).then((response) => {
+        expect(response.status).eql(200)
+        let run_id = response.body[0]
+        cy.request({
+          url: apiUri + '/harvest/runs/' + run_id + '?plan=test',
+          auth: user_credentials
+        }).then((response) => {
+          expect(response.status).eql(200)
+          expect(response.body.status.extract).eql("SUCCESS")
+        })
+      })
+    });
+
+    it('Requires authenticated user', () => {
+      cy.request({
+        url: apiUri + '/harvest/runs',
+        failOnStatusCode: false
+      }).then((response) => {
+        expect(response.status).eql(401)
+      })
+    })
+  });
+
+});

modules/custom/dkan_common/src/Storage/AbstractDatabaseTable.php

@@ -88,9 +88,11 @@ public function retrieveAll(): array {
       return [];
     }
 
-    return array_map(function ($item) {
+    $result = array_map(function ($item) {
       return $item->{$this->primaryKey()};
     }, $result);
+
+    return $result;
   }
 
   /**
@@ -103,11 +105,13 @@ public function store($data, string $id = NULL): string {
 
     $data = $this->prepareData($data, $id);
 
+    $returned_id = NULL;
+
     if (!$existing) {
       $q = $this->connection->insert($this->getTableName());
       $q->fields($this->getNonSerialFields());
       $q->values($data);
-      $id = $q->execute();
+      $returned_id = $q->execute();
     }
     else {
       $q = $this->connection->update($this->getTableName());
@@ -116,7 +120,7 @@ public function store($data, string $id = NULL): string {
         ->execute();
     }
 
-    return "{$id}";
+    return ($returned_id) ? "$returned_id" : "{$id}";
   }
 
   /**

modules/custom/dkan_datastore/src/Storage/JobStore.php

@@ -2,28 +2,27 @@
 
 namespace Drupal\dkan_datastore\Storage;
 
-use Contracts\RetrieverInterface;
 use Drupal\dkan_common\Storage\AbstractDatabaseTable;
 use Procrastinator\Job\Job;
 use Drupal\Core\Database\Connection;
 
 /**
  * Retrieve a serialized job (datastore importer or harvest) from the database.
  */
-class JobStore extends AbstractDatabaseTable implements RetrieverInterface {
+class JobStore extends AbstractDatabaseTable {
 
   private $jobClass;
 
   /**
    * Constructor.
    */
   public function __construct(string $jobClass, Connection $connection) {
-    parent::__construct($connection);
     if (!$this->validateJobClass($jobClass)) {
       throw new \Exception("Invalid jobType provided: $jobClass");
     }
     $this->jobClass = $jobClass;
     $this->setOurSchema();
+    parent::__construct($connection);
   }
 
   /**
@@ -54,10 +53,10 @@ private function setOurSchema() {
       'indexes' => [
         'ref_uuid' => ['ref_uuid'],
       ],
-      'foriegn_keys' => [
+      'foreign keys' => [
         'ref_uuid' => ['table' => 'node', 'columns' => ['uuid' => 'uuid']],
       ],
-      'primary_key' => ['ref_uuid'],
+      'primary key' => ['ref_uuid'],
     ];
 
     $this->setSchema($schema);