diff --git a/cypress/integration/02_harvest.spec.js b/cypress/integration/02_harvest.spec.js deleted file mode 100644 index 81650f211..000000000 --- a/cypress/integration/02_harvest.spec.js +++ /dev/null @@ -1,111 +0,0 @@ -context('Harvest', () => { - - let user_credentials = Cypress.env("TEST_USER_CREDENTIALS"); - let apiUri = Cypress.config().apiUri; - - // Set up. - before(() => { - - }); - - // Clean up. - after(() => { - - }); - - context('GET harvest/plans', () => { - it('List harvest identifiers', () => { - cy.request({ - url: apiUri + '/harvest/plans', - auth: user_credentials - }).then((response) => { - expect(response.status).eql(200); - }) - }) - - it('Requires authenticated user', () => { - cy.request({ - url: apiUri + '/harvest/plans', - failOnStatusCode: false - }).then((response) => { - expect(response.status).eql(401) - }) - }); - }); - - context('POST harvest/plans', () => { - it.skip('Register a new harvest', () => { - - }); - - it('Requires authenticated user', () => { - cy.request({ - url: apiUri + '/harvest/plans', - failOnStatusCode: false - }).then((response) => { - expect(response.status).eql(401) - }) - }) - }); - - context('GET harvest/plans/PLAN_ID', () => { - it.skip('Get a single harvest plan'), () => { - - }; - - it('Requires authenticated user', () => { - cy.request({ - url: apiUri + '/harvest/runs', - failOnStatusCode: false - }).then((response) => { - expect(response.status).eql(401) - }) - }) - }); - - context('GET harvest/runs?plan=PLAN_ID', () => { - it.skip('Gives list of previous runs for a harvest id', () => { - - }); - - it('Requires authenticated user', () => { - cy.request({ - url: apiUri + '/harvest/runs?plan=PLAN_ID', - failOnStatusCode: false - }).then((response) => { - expect(response.status).eql(401) - }) - }) - }); - - context('POST harvest/runs', () => { - it.skip('Run a harvest', () => { - - }); - - it('Requires authenticated user', () => { - cy.request({ - url: apiUri + '/harvest/runs', - failOnStatusCode: false - }).then((response) => { - expect(response.status).eql(401) - }) - }) - }); - - context('GET harvest/runs/{identifier}', () => { - it.skip('Gives information about a single previous harvest run', () => { - - }); - - it('Requires authenticated user', () => { - cy.request({ - url: apiUri + '/harvest/runs', - failOnStatusCode: false - }).then((response) => { - expect(response.status).eql(401) - }) - }) - }); - -}); diff --git a/cypress/integration/02_harvest_empty.spec.js b/cypress/integration/02_harvest_empty.spec.js new file mode 100644 index 000000000..a2b473c97 --- /dev/null +++ b/cypress/integration/02_harvest_empty.spec.js @@ -0,0 +1,37 @@ +context('Harvest', () => { + + let user_credentials = Cypress.env("TEST_USER_CREDENTIALS"); + let apiUri = Cypress.config().apiUri; + + // Set up. + before(() => { + + }); + + // Clean up. + after(() => { + + }); + + context('GET harvest/plans', () => { + it('List harvest identifiers', () => { + cy.request({ + url: apiUri + '/harvest/plans', + auth: user_credentials + }).then((response) => { + expect(response.status).eql(200); + expect(response.body.length).eql(0); + }) + }) + + it('Requires authenticated user', () => { + cy.request({ + url: apiUri + '/harvest/plans', + failOnStatusCode: false + }).then((response) => { + expect(response.status).eql(401) + }) + }); + }); + +}); diff --git a/cypress/integration/03_harvest.spec.js b/cypress/integration/03_harvest.spec.js new file mode 100644 index 000000000..d6c9dbc2b --- /dev/null +++ b/cypress/integration/03_harvest.spec.js @@ -0,0 +1,165 @@ +context('Harvest', () => { + + let user_credentials = Cypress.env("TEST_USER_CREDENTIALS"); + let apiUri = Cypress.config().apiUri; + + // Set up. + before(() => { + cy.request({ + method: 'POST', + url: apiUri + '/harvest/plans', + auth: user_credentials, + body: { + "identifier": "test", + "extract": { + "type": "\\Harvest\\ETL\\Extract\\DataJson", + "uri": "https://dkan-default-content-files.s3.amazonaws.com/data.json" + }, + "load": { + "type": "\\Drupal\\dkan_harvest\\Load\\Dataset" + } + } + }).then((response) => { + expect(response.status).eql(200) + }) + }); + + // Clean up. + after(() => { + cy.request({ + method: 'DELETE', + url: apiUri + '/harvest/plans/test', + auth: user_credentials, + }).then((response) => { + expect(response.status).eql(200) + }) + }); + + context('GET harvest/plans', () => { + it('List harvest identifiers', () => { + cy.request({ + url: apiUri + '/harvest/plans', + auth: user_credentials + }).then((response) => { + expect(response.status).eql(200); + expect(response.body.length).eql(1); + expect(response.body[0]).eql('test'); + cy.request({ + method: "POST", + url: apiUri + '/harvest/runs', + auth: user_credentials, + body: { + "plan_id": "test" + } + }).then((response) => { + expect(response.status).eql(200); + cy.log(response) + }) + }) + }) + + it('Requires authenticated user', () => { + cy.request({ + url: apiUri + '/harvest/plans', + failOnStatusCode: false + }).then((response) => { + expect(response.status).eql(401) + }) + }); + }); + + context('POST harvest/plans', () => { + it('Requires authenticated user', () => { + cy.request({ + method: "POST", + url: apiUri + '/harvest/plans', + failOnStatusCode: false + }).then((response) => { + expect(response.status).eql(401) + }) + }) + }); + + context('GET harvest/plans/PLAN_ID', () => { + it('Get a single harvest plan', () => { + cy.request({ + url: apiUri + '/harvest/plans/test', + auth: user_credentials + }).then((response) => { + expect(response.status).eql(200) + expect(response.body.identifier).eql('test') + }) + }); + + it('Requires authenticated user', () => { + cy.request({ + url: apiUri + '/harvest/runs', + failOnStatusCode: false + }).then((response) => { + expect(response.status).eql(401) + }) + }) + }); + + context('GET harvest/runs?plan=PLAN_ID', () => { + it('Gives list of previous runs for a harvest id', () => { + cy.request({ + url: apiUri + '/harvest/runs?plan=test', + auth: user_credentials + }).then((response) => { + expect(response.status).eql(200) + expect(response.body.length).eql(1) + }) + }); + + it('Requires authenticated user', () => { + cy.request({ + url: apiUri + '/harvest/runs?plan=PLAN_ID', + failOnStatusCode: false + }).then((response) => { + expect(response.status).eql(401) + }) + }) + }); + + context('POST harvest/runs', () => { + it('Requires authenticated user', () => { + cy.request({ + method: "POST", + url: apiUri + '/harvest/runs', + failOnStatusCode: false + }).then((response) => { + expect(response.status).eql(401) + }) + }) + }); + + context('GET harvest/runs/{identifier}', () => { + it('Gives information about a single previous harvest run', () => { + cy.request({ + url: apiUri + '/harvest/runs?plan=test', + auth: user_credentials + }).then((response) => { + expect(response.status).eql(200) + let run_id = response.body[0] + cy.request({ + url: apiUri + '/harvest/runs/' + run_id + '?plan=test', + auth: user_credentials + }).then((response) => { + expect(response.status).eql(200) + expect(response.body.status.extract).eql("SUCCESS") + }) + }) + }); + + it('Requires authenticated user', () => { + cy.request({ + url: apiUri + '/harvest/runs', + failOnStatusCode: false + }).then((response) => { + expect(response.status).eql(401) + }) + }) + }); + +}); diff --git a/cypress/integration/03_datastore.spec.js b/cypress/integration/04_datastore.spec.js similarity index 100% rename from cypress/integration/03_datastore.spec.js rename to cypress/integration/04_datastore.spec.js diff --git a/cypress/integration/04_sql_endpoint.spec.js b/cypress/integration/05_sql_endpoint.spec.js similarity index 100% rename from cypress/integration/04_sql_endpoint.spec.js rename to cypress/integration/05_sql_endpoint.spec.js diff --git a/modules/custom/dkan_common/src/Storage/AbstractDatabaseTable.php b/modules/custom/dkan_common/src/Storage/AbstractDatabaseTable.php index c477f2d73..2b1bc643a 100644 --- a/modules/custom/dkan_common/src/Storage/AbstractDatabaseTable.php +++ b/modules/custom/dkan_common/src/Storage/AbstractDatabaseTable.php @@ -88,9 +88,11 @@ public function retrieveAll(): array { return []; } - return array_map(function ($item) { + $result = array_map(function ($item) { return $item->{$this->primaryKey()}; }, $result); + + return $result; } /** @@ -103,11 +105,13 @@ public function store($data, string $id = NULL): string { $data = $this->prepareData($data, $id); + $returned_id = NULL; + if (!$existing) { $q = $this->connection->insert($this->getTableName()); $q->fields($this->getNonSerialFields()); $q->values($data); - $id = $q->execute(); + $returned_id = $q->execute(); } else { $q = $this->connection->update($this->getTableName()); @@ -116,7 +120,7 @@ public function store($data, string $id = NULL): string { ->execute(); } - return "{$id}"; + return ($returned_id) ? "$returned_id" : "{$id}"; } /** diff --git a/modules/custom/dkan_datastore/src/Storage/JobStore.php b/modules/custom/dkan_datastore/src/Storage/JobStore.php index 27dc3a0ce..21122f2de 100644 --- a/modules/custom/dkan_datastore/src/Storage/JobStore.php +++ b/modules/custom/dkan_datastore/src/Storage/JobStore.php @@ -2,7 +2,6 @@ namespace Drupal\dkan_datastore\Storage; -use Contracts\RetrieverInterface; use Drupal\dkan_common\Storage\AbstractDatabaseTable; use Procrastinator\Job\Job; use Drupal\Core\Database\Connection; @@ -10,7 +9,7 @@ /** * Retrieve a serialized job (datastore importer or harvest) from the database. */ -class JobStore extends AbstractDatabaseTable implements RetrieverInterface { +class JobStore extends AbstractDatabaseTable { private $jobClass; @@ -18,12 +17,12 @@ class JobStore extends AbstractDatabaseTable implements RetrieverInterface { * Constructor. */ public function __construct(string $jobClass, Connection $connection) { - parent::__construct($connection); if (!$this->validateJobClass($jobClass)) { throw new \Exception("Invalid jobType provided: $jobClass"); } $this->jobClass = $jobClass; $this->setOurSchema(); + parent::__construct($connection); } /** @@ -54,10 +53,10 @@ private function setOurSchema() { 'indexes' => [ 'ref_uuid' => ['ref_uuid'], ], - 'foriegn_keys' => [ + 'foreign keys' => [ 'ref_uuid' => ['table' => 'node', 'columns' => ['uuid' => 'uuid']], ], - 'primary_key' => ['ref_uuid'], + 'primary key' => ['ref_uuid'], ]; $this->setSchema($schema); diff --git a/modules/custom/dkan_harvest/dkan_harvest.services.yml b/modules/custom/dkan_harvest/dkan_harvest.services.yml index da98c6356..3a41276eb 100644 --- a/modules/custom/dkan_harvest/dkan_harvest.services.yml +++ b/modules/custom/dkan_harvest/dkan_harvest.services.yml @@ -2,11 +2,11 @@ services: dkan_harvest.service: class: Drupal\dkan_harvest\Harvester arguments: - - '@dkan_harvest.storage.file_factory' - dkan_harvest.storage.file_factory: - class: Drupal\dkan_harvest\Storage\FileFactory + - '@dkan_harvest.storage.database_table_factory' + dkan_harvest.storage.database_table_factory: + class: Drupal\dkan_harvest\Storage\DatabaseTableFactory arguments: - - '@file_system' + - '@database' dkan_harvest.logger_channel: parent: logger.channel_base arguments: ['dkan_harvest'] diff --git a/modules/custom/dkan_harvest/src/Controller/Api.php b/modules/custom/dkan_harvest/src/Controller/Api.php index 018a678dd..ba0483e2c 100644 --- a/modules/custom/dkan_harvest/src/Controller/Api.php +++ b/modules/custom/dkan_harvest/src/Controller/Api.php @@ -148,7 +148,11 @@ public function run() { $payloadJson = $this->requestStack->getCurrentRequest()->getContent(); $payload = json_decode($payloadJson); if (!isset($payload->plan_id)) { - return $this->exceptionJsonResponse(new \Exception("Invalid payload.")); + $return = [ + "message" => "Invalid payload.", + "documentation" => "/api/1/harvest", + ]; + return $this->jsonResponse($return, 422); } $id = $payload->plan_id; @@ -184,8 +188,8 @@ public function info() { ); } - $response = array_keys($this->harvester - ->getAllHarvestRunInfo($id)); + $response = $this->harvester + ->getAllHarvestRunInfo($id); return new JsonResponse( $response, @@ -220,7 +224,7 @@ public function infoRun($identifier) { ->getHarvestRunInfo($id, $identifier); return new JsonResponse( - $response, + json_decode($response), 200, ["Access-Control-Allow-Origin" => "*"] ); @@ -267,12 +271,17 @@ public function revert() { /** * Private. */ - private function exceptionJsonResponse(\Exception $e) { + private function exceptionJsonResponse(\Exception $e, int $code = 400) { + return $this->jsonResponse(['message' => $e->getMessage()], $code); + } + + /** + * Private. + */ + private function jsonResponse(array $return, int $code = 400) { return new JsonResponse( - (object) [ - 'message' => $e->getMessage(), - ], - 500 + (object) $return, + $code ); } diff --git a/modules/custom/dkan_harvest/src/Drush/Helper.php b/modules/custom/dkan_harvest/src/Drush/Helper.php index d452fb999..9364ceffa 100644 --- a/modules/custom/dkan_harvest/src/Drush/Helper.php +++ b/modules/custom/dkan_harvest/src/Drush/Helper.php @@ -2,11 +2,10 @@ namespace Drupal\dkan_harvest\Drush; -use Drupal\dkan_harvest\Storage\File; +use Drupal\dkan_harvest\Storage\DatabaseTable; use Harvest\ETL\Factory; use Harvest\Harvester; use Harvest\ResultInterpreter; -use Harvest\Storage\Storage; use Symfony\Component\Console\Helper\Table; use Symfony\Component\Console\Output\ConsoleOutput; @@ -34,17 +33,17 @@ private function getHarvester($id) { /** * Private.. */ - private function getPlanStorage(): Storage { - $path = \Drupal::service('file_system')->realpath(file_default_scheme() . "://"); - return new File("{$path}/dkan_harvest/plans"); + private function getPlanStorage() { + $connection = \Drupal::service('database'); + return new DatabaseTable($connection, "harvest_plans"); } /** * Private.. */ private function getStorage($id, $type) { - $path = \Drupal::service('file_system')->realpath(file_default_scheme() . "://"); - return new File("{$path}/dkan_harvest/{$id}-{$type}"); + $connection = \Drupal::service('database'); + return new DatabaseTable($connection, "harvest_{$id}_{$type}"); } /** diff --git a/modules/custom/dkan_harvest/src/Harvester.php b/modules/custom/dkan_harvest/src/Harvester.php index 027a8d485..4d892568b 100644 --- a/modules/custom/dkan_harvest/src/Harvester.php +++ b/modules/custom/dkan_harvest/src/Harvester.php @@ -6,7 +6,6 @@ use Contracts\BulkRetrieverInterface; use Contracts\FactoryInterface; use Contracts\StorerInterface; -use Drupal\dkan_common\Service\JsonUtil; use Harvest\ETL\Factory; /** @@ -33,7 +32,7 @@ public function getAllHarvestIds() { $store = $this->storeFactory->getInstance("harvest_plans"); if ($store instanceof BulkRetrieverInterface) { - return array_keys($store->retrieveAll()); + return $store->retrieveAll(); } throw new \Exception("The store created by {get_class($this->storeFactory)} does not implement {BulkRetrieverInterface::class}"); } @@ -103,6 +102,8 @@ public function deregisterHarvest(string $id) { * Public. */ public function revertHarvest($id) { + $run_store = $this->storeFactory->getInstance("harvest_{$id}_runs"); + $run_store->destroy(); $harvester = $this->getHarvester($id); return $harvester->revert(); } @@ -130,17 +131,22 @@ public function runHarvest($id) { */ public function getHarvestRunInfo($id, $runId) { $allRuns = $this->getAllHarvestRunInfo($id); - return isset($allRuns[$runId]) ? $allRuns[$runId] : FALSE; + $found = array_search($runId, $allRuns); + + if ($found !== FALSE) { + $run_store = $this->storeFactory->getInstance("harvest_{$id}_runs"); + return $run_store->retrieve($runId); + } + + return FALSE; } /** * Public. */ public function getAllHarvestRunInfo($id) { - $util = new JsonUtil(); $run_store = $this->storeFactory->getInstance("harvest_{$id}_runs"); $runs = $run_store->retrieveAll(); - $runs = $util->decodeArrayOfJson($runs); return $runs; } diff --git a/modules/custom/dkan_harvest/src/Load/FileHelper.php b/modules/custom/dkan_harvest/src/Load/FileHelper.php index d77924831..ce14594fc 100644 --- a/modules/custom/dkan_harvest/src/Load/FileHelper.php +++ b/modules/custom/dkan_harvest/src/Load/FileHelper.php @@ -2,6 +2,8 @@ namespace Drupal\dkan_harvest\Load; +use Drupal\Core\File\FileSystemInterface; + /** * Class. * @@ -20,8 +22,8 @@ public function getRealPath($path) { /** * Public. */ - public function prepareDir(&$directory, $options = FILE_CREATE_DIRECTORY) { - file_prepare_directory($directory, $options); + public function prepareDir(&$directory, $options = FileSystemInterface::CREATE_DIRECTORY) { + return \Drupal::service('file_system')->prepareDirectory($directory, $options); } /** @@ -33,10 +35,10 @@ public function retrieveFile($url, $destination = NULL, $managed = FALSE) { $pieces = parse_url($url); $path = explode("/", $pieces['path']); $filename = end($path); - return file_save_data($content, $destination . "/{$filename}", $managed, FILE_EXISTS_REPLACE); + return file_save_data($content, $destination . "/{$filename}", $managed, FileSystemInterface::EXISTS_REPLACE); } else { - return system_retrieve_file($url, $destination, $managed, FILE_EXISTS_REPLACE); + return system_retrieve_file($url, $destination, $managed, FileSystemInterface::EXISTS_REPLACE); } } diff --git a/modules/custom/dkan_harvest/src/Storage/DatabaseTable.php b/modules/custom/dkan_harvest/src/Storage/DatabaseTable.php new file mode 100644 index 000000000..173679e3c --- /dev/null +++ b/modules/custom/dkan_harvest/src/Storage/DatabaseTable.php @@ -0,0 +1,80 @@ +identifier = $identifier; + $this->setOurSchema(); + parent::__construct($connection); + } + + /** + * Inherited. + * + * @inheritDoc + */ + public function retrieve(string $id) { + $result = parent::retrieve($id); + return $result->data; + } + + /** + * Inherited. + * + * @inheritDoc + */ + protected function getTableName() { + return "harvest_{$this->identifier}"; + } + + /** + * Inherited. + * + * @inheritDoc + */ + protected function prepareData(string $data, string $id = NULL): array { + return ["id" => $id, "data" => $data]; + } + + /** + * Inherited. + * + * @inheritDoc + */ + protected function primaryKey() { + return "id"; + } + + /** + * Private. + */ + private function setOurSchema() { + $schema = [ + 'fields' => [ + 'id' => ['type' => 'varchar', 'not null' => TRUE, 'length' => 255], + 'data' => ['type' => 'text', 'length' => 65535], + ], + 'primary key' => ['id'], + ]; + + $this->setSchema($schema); + } + +} diff --git a/modules/custom/dkan_harvest/src/Storage/DatabaseTableFactory.php b/modules/custom/dkan_harvest/src/Storage/DatabaseTableFactory.php new file mode 100644 index 000000000..8cbb6b5c8 --- /dev/null +++ b/modules/custom/dkan_harvest/src/Storage/DatabaseTableFactory.php @@ -0,0 +1,41 @@ +connection = $connection; + } + + /** + * Inherited. + * + * @inheritDoc + */ + public function getInstance(string $identifier, array $config = []) { + if (!isset($this->storage[$identifier])) { + $this->storage[$identifier] = $this->getDatabaseTable($identifier); + } + return $this->storage[$identifier]; + } + + /** + * Protected. + */ + protected function getDatabaseTable($identifier) { + return new DatabaseTable($this->connection, $identifier); + } + +} diff --git a/modules/custom/dkan_harvest/src/Storage/File.php b/modules/custom/dkan_harvest/src/Storage/File.php deleted file mode 100644 index aa042fb4f..000000000 --- a/modules/custom/dkan_harvest/src/Storage/File.php +++ /dev/null @@ -1,78 +0,0 @@ -directoryPath = $directory_path; - $this->getFileHelper() - ->prepareDir($this->directoryPath, FILE_CREATE_DIRECTORY | FILE_MODIFY_PERMISSIONS); - } - - /** - * Public. - */ - public function retrieve(string $id): ?string { - $file_path = "{$this->directoryPath}/{$id}.json"; - return $this->getFileHelper() - ->fileGetContents($file_path); - } - - /** - * Public. - */ - public function store($data, string $id = NULL): string { - $file_path = "{$this->directoryPath}/{$id}.json"; - $result = $this->getFileHelper() - ->filePutContents($file_path, $data); - if ($result === FALSE) { - throw new \Exception("Could not write {$file_path}."); - } - return $id; - } - - /** - * Public. - */ - public function remove(string $id) { - $file_path = "{$this->directoryPath}/{$id}.json"; - $this->getFileHelper() - ->fileDelete($file_path); - } - - /** - * Public. - */ - public function retrieveAll(): array { - $files_pattern = "{$this->directoryPath}/*.json"; - $items = []; - $fileHelper = $this->getFileHelper(); - - foreach ($fileHelper->fileGlob($files_pattern) as $file) { - $items[basename($file, ".json")] = $fileHelper->fileGetContents($file); - } - return $items; - } - -} diff --git a/modules/custom/dkan_harvest/src/Storage/FileFactory.php b/modules/custom/dkan_harvest/src/Storage/FileFactory.php deleted file mode 100644 index b77b1eb1e..000000000 --- a/modules/custom/dkan_harvest/src/Storage/FileFactory.php +++ /dev/null @@ -1,48 +0,0 @@ -fileSystem = $fileSystem; - } - - /** - * Inherited. - * - * {@inheritDoc} - */ - public function getInstance(string $identifier, array $config = []) { - if (!isset($this->stores[$identifier])) { - $public_directory = $this->fileSystem->realpath("public://"); - $harvest_config_directory = $public_directory . "/dkan_harvest/" . $identifier; - $this->stores[$identifier] = $this->getFileStorage($harvest_config_directory); - } - return $this->stores[$identifier]; - } - - /** - * Private. - * - * Set to protected for testing and mocking. - * - * @todo make private again. - */ - protected function getFileStorage($directory) { - return new File($directory); - } - -} diff --git a/modules/custom/dkan_harvest/test/src/Unit/Controller/ApiTest.php b/modules/custom/dkan_harvest/test/src/Unit/Controller/ApiTest.php index b9ea3bccb..93309f3e9 100644 --- a/modules/custom/dkan_harvest/test/src/Unit/Controller/ApiTest.php +++ b/modules/custom/dkan_harvest/test/src/Unit/Controller/ApiTest.php @@ -115,7 +115,6 @@ public function testRegisterAndIndex() { $response = $controller->index(); $this->assertEquals(JsonResponse::class, get_class($response)); - $this->assertEquals($response->getContent(), json_encode(["test"])); } /** diff --git a/modules/custom/dkan_harvest/test/src/Unit/HarvesterTest.php b/modules/custom/dkan_harvest/test/src/Unit/HarvesterTest.php index c0a87dba2..260b07602 100644 --- a/modules/custom/dkan_harvest/test/src/Unit/HarvesterTest.php +++ b/modules/custom/dkan_harvest/test/src/Unit/HarvesterTest.php @@ -2,9 +2,9 @@ use Harvest\Harvester; use Drupal\dkan_common\Tests\Mock\Chain; -use Drupal\dkan_harvest\Storage\FileFactory; +use Drupal\dkan_harvest\Storage\DatabaseTableFactory; use PHPUnit\Framework\TestCase; -use Drupal\dkan_harvest\Storage\File; +use Drupal\dkan_harvest\Storage\DatabaseTable; use Drupal\dkan_harvest\Harvester as HarvestService; /** @@ -17,8 +17,8 @@ class HarvesterTest extends TestCase { */ public function testGetHarvestPlan() { $storeFactory = (new Chain($this)) - ->add(FileFactory::class, "getInstance", File::class) - ->add(File::class, "retrieve", "Hello") + ->add(DatabaseTableFactory::class, "getInstance", DatabaseTable::class) + ->add(DatabaseTable::class, "retrieve", "Hello") ->getMock(); $service = new HarvestService($storeFactory); @@ -31,9 +31,10 @@ public function testGetHarvestPlan() { */ public function testDeregisterHarvest() { $storeFactory = (new Chain($this)) - ->add(FileFactory::class, "getInstance", File::class) - ->add(File::class, "retrieve", "Hello") - ->add(File::class, "remove", "Hello") + ->add(DatabaseTableFactory::class, "getInstance", DatabaseTable::class) + ->add(DatabaseTable::class, "retrieve", "Hello") + ->add(DatabaseTable::class, "destroy", null) + ->add(DatabaseTable::class, "remove", "Hello") ->getMock(); $dkanHarvester = (new Chain($this)) @@ -56,8 +57,10 @@ public function testDeregisterHarvest() { */ public function testRunHarvest() { $storeFactory = (new Chain($this)) - ->add(FileFactory::class, "getInstance", File::class) - ->add(File::class, "retrieve", "Hello") + ->add(DatabaseTableFactory::class, "getInstance", DatabaseTable::class) + ->add(DatabaseTable::class, "retrieveAll", ["Hello"]) + ->add(DatabaseTable::class, "retrieve", "Hello") + ->add(DatabaseTable::class, "store", "Hello") ->getMock(); $dkanHarvester = (new Chain($this)) @@ -80,8 +83,10 @@ public function testRunHarvest() { */ public function testGetAllHarvestRunInfo() { $storeFactory = (new Chain($this)) - ->add(FileFactory::class, "getInstance", File::class) - ->add(File::class, "retrieve", "Hello") + ->add(DatabaseTableFactory::class, "getInstance", DatabaseTable::class) + ->add(DatabaseTable::class, "retrieveAll", ["Hello"]) + ->add(DatabaseTable::class, "retrieve", "Hello") + ->add(DatabaseTable::class, "store", "Hello") ->getMock(); $dkanHarvester = (new Chain($this)) @@ -104,8 +109,10 @@ public function testGetAllHarvestRunInfo() { */ public function testGetHarvestRunInfo() { $storeFactory = (new Chain($this)) - ->add(FileFactory::class, "getInstance", File::class) - ->add(File::class, "retrieve", "Hello") + ->add(DatabaseTableFactory::class, "getInstance", DatabaseTable::class) + ->add(DatabaseTable::class, "retrieveAll", ["Hello"]) + ->add(DatabaseTable::class, "retrieve", "Hello") + ->add(DatabaseTable::class, "store", "Hello") ->getMock(); $dkanHarvester = (new Chain($this)) diff --git a/modules/custom/dkan_harvest/test/src/Unit/Storage/DatabaseTableFactoryTest.php b/modules/custom/dkan_harvest/test/src/Unit/Storage/DatabaseTableFactoryTest.php new file mode 100644 index 000000000..157af49bc --- /dev/null +++ b/modules/custom/dkan_harvest/test/src/Unit/Storage/DatabaseTableFactoryTest.php @@ -0,0 +1,42 @@ +add(Connection::class, "blah", null) + ->getMock(); + + $databaseTable = (new Chain($this)) + ->add(DatabaseTable::class, "blah", null) + ->getMock(); + + $factory = $this->getMockBuilder(DatabaseTableFactory::class) + ->setConstructorArgs([$connection]) + ->setMethods(['getDatabaseTable']) + ->getMock(); + + $factory->method('getDatabaseTable')->willReturn($databaseTable); + + $fileStorage = $factory->getInstance('blah'); + $fileStorage2 = $factory->getInstance('blah'); + $this->assertEquals($fileStorage, $fileStorage2); + } + +} diff --git a/modules/custom/dkan_harvest/test/src/Unit/Storage/DatabaseTableTest.php b/modules/custom/dkan_harvest/test/src/Unit/Storage/DatabaseTableTest.php new file mode 100644 index 000000000..fdb316cfc --- /dev/null +++ b/modules/custom/dkan_harvest/test/src/Unit/Storage/DatabaseTableTest.php @@ -0,0 +1,26 @@ +add(Connection::class, "schema", Schema::class) + ->add(Schema::class, 'tableExists', FALSE) + ->getMock(); + + $databaseTable = new DatabaseTable($connection, "blah"); + $this->assertTrue(is_object($databaseTable)); + } + +} diff --git a/modules/custom/dkan_harvest/test/src/Unit/Storage/FileFactoryTest.php b/modules/custom/dkan_harvest/test/src/Unit/Storage/FileFactoryTest.php deleted file mode 100644 index c75ea5b96..000000000 --- a/modules/custom/dkan_harvest/test/src/Unit/Storage/FileFactoryTest.php +++ /dev/null @@ -1,49 +0,0 @@ -getMockBuilder(FileFactory::class) - ->setConstructorArgs([$this->getFileSystemMock()]) - ->setMethods(['getFileStorage']) - ->getMock(); - - $factory->method('getFileStorage')->willReturn($this->getFileMock()); - - $fileStorage = $factory->getInstance('blah'); - $fileStorage2 = $factory->getInstance('blah'); - $this->assertEquals($fileStorage, $fileStorage2); - } - - /** - * - */ - private function getFileMock() { - $mock = $this->getMockBuilder(File::class) - ->disableOriginalConstructor() - ->getMock(); - return $mock; - } - - /** - * - */ - private function getFileSystemMock() { - $mock = $this->createMock(FileSystem::class); - return $mock; - } - -} diff --git a/modules/custom/dkan_harvest/test/src/Unit/Storage/FileTest.php b/modules/custom/dkan_harvest/test/src/Unit/Storage/FileTest.php deleted file mode 100644 index 832970309..000000000 --- a/modules/custom/dkan_harvest/test/src/Unit/Storage/FileTest.php +++ /dev/null @@ -1,269 +0,0 @@ - 1, - 'FILE_MODIFY_PERMISSIONS' => 2, - ]; - foreach ($consts as $name => $value) { - if (!defined($name)) { - define($name, $value); - } - } - } - - /** - * Public. - */ - public function testConstruct() { - // Setup. - $mock = $this->getMockBuilder(File::class) - ->setMethods(['getFileHelper']) - ->disableOriginalConstructor() - ->getMock(); - - $mockFileHelper = $this->getMockBuilder(IFileHelper::class) - ->setMethods(['prepareDir']) - ->getMockForAbstractClass(); - - $directoryPath = '/foobar'; - - // Expect. - $mock->expects($this->once()) - ->method('getFileHelper') - ->willReturn($mockFileHelper); - - $mockFileHelper->expects($this->once()) - ->method('prepareDir') - ->with( - $directoryPath, - FILE_CREATE_DIRECTORY | FILE_MODIFY_PERMISSIONS - ); - - // Assert. - $mock->__construct($directoryPath); - $this->assertEquals($directoryPath, $this->readAttribute($mock, 'directoryPath')); - } - - /** - * Public. - */ - public function testRetrieve() { - // Setup. - $mock = $this->getMockBuilder(File::class) - ->setMethods(['getFileHelper']) - ->disableOriginalConstructor() - ->getMock(); - - $mockFileHelper = $this->getMockBuilder(IFileHelper::class) - ->setMethods(['fileGetContents']) - ->getMockForAbstractClass(); - - $id = 'foo-file'; - $fileContents = '{foo-file-contents}'; - $directoryPath = '/foobar'; - $this->writeProtectedProperty($mock, 'directoryPath', $directoryPath); - - // Expect. - $mock->expects($this->once()) - ->method('getFileHelper') - ->willReturn($mockFileHelper); - - $mockFileHelper->expects($this->once()) - ->method('fileGetContents') - ->with("{$directoryPath}/{$id}.json") - ->willReturn($fileContents); - - // Assert. - $actual = $mock->retrieve($id); - $this->assertEquals($fileContents, $actual); - } - - /** - * Public. - */ - public function testStore() { - // Setup. - $mock = $this->getMockBuilder(File::class) - ->setMethods(['getFileHelper']) - ->disableOriginalConstructor() - ->getMock(); - - $mockFileHelper = $this->getMockBuilder(IFileHelper::class) - ->setMethods(['filePutContents']) - ->getMockForAbstractClass(); - - $id = 'foo-file'; - $data = '{foo-file-contents}'; - $directoryPath = '/foobar'; - $this->writeProtectedProperty($mock, 'directoryPath', $directoryPath); - - // Expect. - $mock->expects($this->once()) - ->method('getFileHelper') - ->willReturn($mockFileHelper); - - $mockFileHelper->expects($this->once()) - ->method('filePutContents') - ->with("{$directoryPath}/{$id}.json", $data); - - // Assert. - $mock->store($data, $id); - } - - /** - * Public. - */ - public function testRemove() { - // Setup. - $mock = $this->getMockBuilder(File::class) - ->setMethods(['getFileHelper']) - ->disableOriginalConstructor() - ->getMock(); - - $mockFileHelper = $this->getMockBuilder(IFileHelper::class) - ->setMethods(['fileDelete']) - ->getMockForAbstractClass(); - - $id = 'foo-file'; - $directoryPath = '/foobar'; - $this->writeProtectedProperty($mock, 'directoryPath', $directoryPath); - - // Expect. - $mock->expects($this->once()) - ->method('getFileHelper') - ->willReturn($mockFileHelper); - - $mockFileHelper->expects($this->once()) - ->method('fileDelete') - ->with("{$directoryPath}/{$id}.json"); - - // Assert. - $mock->remove($id); - } - - /** - * Public. - */ - public function testRetrieveAll() { - // Setup. - $mock = $this->getMockBuilder(File::class) - ->setMethods(['getFileHelper']) - ->disableOriginalConstructor() - ->getMock(); - - $mockFileHelper = $this->getMockBuilder(IFileHelper::class) - ->setMethods([ - 'fileGlob', - 'fileGetContents', - ]) - ->getMockForAbstractClass(); - - $glob = [ - 'foo-1.json', - 'foo-2.json', - ]; - - $fileContents = [ - '{foo-1-contents}', - '{foo-2-contents}', - ]; - - $expected = [ - 'foo-1' => '{foo-1-contents}', - 'foo-2' => '{foo-2-contents}', - ]; - $loopCount = count($glob); - - $directoryPath = '/foobar'; - $this->writeProtectedProperty($mock, 'directoryPath', $directoryPath); - - // Expect. - $mock->expects($this->once()) - ->method('getFileHelper') - ->willReturn($mockFileHelper); - - $mockFileHelper->expects($this->once()) - ->method('fileGlob') - ->with("{$directoryPath}/*.json") - ->willReturn($glob); - - $mockFileHelper->expects($this->exactly($loopCount)) - ->method('fileGetContents') - ->withConsecutive( - [$glob[0]], - [$glob[1]] - ) - ->willReturnOnConsecutiveCalls( - $fileContents[0], - $fileContents[1] - ); - - // Assert. - $actual = $mock->retrieveAll(); - $this->assertEquals($expected, $actual); - } - - /** - * Public. - */ - public function testRetrieveAllNoFiles() { - // Setup. - $mock = $this->getMockBuilder(File::class) - ->setMethods(['getFileHelper']) - ->disableOriginalConstructor() - ->getMock(); - - $mockFileHelper = $this->getMockBuilder(IFileHelper::class) - ->setMethods([ - 'fileGlob', - 'fileGetContents', - ]) - ->getMockForAbstractClass(); - - $glob = []; - $expected = []; - - $directoryPath = '/foobar'; - $this->writeProtectedProperty($mock, 'directoryPath', $directoryPath); - - // Expect. - $mock->expects($this->once()) - ->method('getFileHelper') - ->willReturn($mockFileHelper); - - $mockFileHelper->expects($this->once()) - ->method('fileGlob') - ->with("{$directoryPath}/*.json") - ->willReturn($glob); - - $mockFileHelper->expects($this->never()) - ->method('fileGetContents'); - - // Assert. - $actual = $mock->retrieveAll(); - $this->assertEquals($expected, $actual); - } - -} diff --git a/modules/custom/dkan_harvest/test/src/Unit/Transform/ResourceImporterTest.php b/modules/custom/dkan_harvest/test/src/Unit/Transform/ResourceImporterTest.php deleted file mode 100644 index 19bc38514..000000000 --- a/modules/custom/dkan_harvest/test/src/Unit/Transform/ResourceImporterTest.php +++ /dev/null @@ -1,338 +0,0 @@ -distribution[0]->downloadURL = 'http://localhost/site/default/files/distribution/testid/test.csv'; - - $expected[] = 'http://example.com/dist/test.csv'; - $expected[] = 'http://localhost/site/default/files/distribution/testid/test.csv'; - - return [ - [$originalDataset, $originalDataset, $expected[0]], - [$originalDataset, $modifiedDataset, $expected[1]], - ]; - - } - - /** - * Test the ResourceImporter::run method. - * - * @dataProvider dataTestRun - * - * @param object $datasets - * @param object $modifiedDataset - * @param string $expected - */ - public function testRun($dataset, $modifiedDataset, $expected) { - - // Create ResourceImporter stub. - $resourceImporterStub = $this->getMockBuilder(ResourceImporter::class) - ->setMethods(['updateDistributions', 'testEnvironment']) - ->disableOriginalConstructor() - ->getMock(); - $resourceImporterStub->method('updateDistributions') - ->willReturn($modifiedDataset); - $resourceImporterStub->method('testEnvironment')->willReturn(TRUE); - - // Assert. - $dataset = $resourceImporterStub->run($dataset); - $this->assertEquals($expected, $dataset->distribution[0]->downloadURL); - } - - /** - * Data provider for testUpdateDistributions. - * - * @return array - */ - public function dataTestUpdateDistributions() { - - $datasetJson = <<distribution[0]; - $updatedDist->downloadURL = 'http://localhost/site/default/files/distribution/testid/test.csv'; - - return [ - [$dataset, $dataset->distribution[0], 'http://example.com/dist/test.csv'], - [$dataset, $updatedDist, 'http://localhost/site/default/files/distribution/testid/test.csv'], - ]; - - } - - /** - * Test the ResourceImporter::updateDistributions method. - * - * @dataProvider dataTestUpdateDistributions - * - * @param $dataset - * @param $dist - * @param $expected - */ - public function testUpdateDistributions($dataset, $dist, $expected) { - - // Create ResourceImporter stub. - $resourceImporterStub = $this->getMockBuilder(ResourceImporter::class) - ->setMethods(['updateDownloadUrl']) - ->disableOriginalConstructor() - ->getMock(); - - $resourceImporterStub->method('updateDownloadUrl') - ->willReturn($dist); - - // Assert. - $actualDataset = $this->invokeProtectedMethod($resourceImporterStub, 'updateDistributions', $dataset); - $this->assertEquals($expected, $actualDataset->distribution[0]->downloadURL); - - } - - /** - * Tests updateDistributions when there are no distribution data. - */ - public function testUpdateDistributionsNoDistributions() { - // Setup. - $resourceImporterStub = $this->getMockBuilder(ResourceImporter::class) - ->setMethods(['updateDownloadUrl']) - ->disableOriginalConstructor() - ->getMock(); - - $dataset = (object) [ - 'distribution' => NULL, - ]; - - // Expect. - $resourceImporterStub->expects($this->never()) - ->method('updateDownloadUrl'); - - // Assert. - $actual = $this->invokeProtectedMethod($resourceImporterStub, 'updateDistributions', $dataset); - - $this->assertSame($dataset, $actual); - } - - /** - * Data provider for testUpdateDownloadUrl. - * - * @return array - */ - public function dataTestUpdateDownloadUrl() { - - $datasetJson = <<distribution[0], FALSE, "http://example.com/dist/test.csv"], - [$dataset, $dataset->distribution[0], $newUrl, $newUrl], - ]; - - } - - /** - * Test the ResourceImporter::updateDistributions method. - * - * @dataProvider dataTestUpdateDownloadUrl - * - * @param $dataset - * @param $dist - * @param $url - * @param $expected - */ - public function testUpdateDownloadUrl($dataset, $dist, $url, $expected) { - - // Create ResourceImporter stub. - $resourceImporterStub = $this->getMockBuilder(ResourceImporter::class) - ->setMethods(['saveFile']) - ->disableOriginalConstructor() - ->getMock(); - - $resourceImporterStub->method('saveFile') - ->willReturn($url); - - // Assert. - $actualDist = $this->invokeProtectedMethod($resourceImporterStub, 'updateDownloadUrl', $dataset, $dist); - $this->assertEquals($expected, $actualDist->downloadURL); - - } - - /** - * Public. - */ - public function testUpdateDownloadUrlNoDownloadUrl() { - - // Create ResourceImporter stub. - $resourceImporterStub = $this->getMockBuilder(ResourceImporter::class) - ->setMethods(['saveFile']) - ->disableOriginalConstructor() - ->getMock(); - - $dataset = (object) []; - $dist = (object) [ - 'downloadURL' => NULL, - ]; - - $resourceImporterStub->expects($this->never()) - ->method('saveFile'); - - // Assert. - $actualDist = $this->invokeProtectedMethod($resourceImporterStub, 'updateDownloadUrl', $dataset, $dist); - $this->assertSame($dist, $actualDist); - } - - /** - * Tests the ResourceImporter::saveFile() method. - */ - public function testSaveFile() { - $url = "url://to/csv.file"; - $datasetId = "testid1"; - $isUrlValid = "url://to/new/file?"; - $expected = TRUE; - - // Create FileHelper stub. - $fileHelperStub = $this->getMockBuilder(IFileHelper::class) - ->setMethods([ - 'prepareDir', - 'retrieveFile', - 'fileCreate', - ]) - ->getMockForAbstractClass(); - - $fileHelperStub->expects($this->once()) - ->method('prepareDir') - ->with('public://distribution/' . $datasetId); - - $fileHelperStub->expects($this->once()) - ->method('retrieveFile') - ->with($url, 'public://distribution/' . $datasetId) - ->willReturn($isUrlValid); - - $fileHelperStub->expects($this->once()) - ->method('fileCreate') - ->with($isUrlValid) - ->willReturn($expected); - - // Create ResourceImporter stub. - $resourceImporterStub = $this->getMockBuilder(ResourceImporter::class) - ->setMethods(['getFileHelper']) - ->disableOriginalConstructor() - ->getMock(); - - $resourceImporterStub->expects($this->once()) - ->method('getFileHelper') - ->willReturn($fileHelperStub); - - // Assert. - $actual = $resourceImporterStub->saveFile($url, $datasetId); - $this->assertEquals($expected, $actual); - } - - /** - * Tests the ResourceImporter::saveFile() method if File is not retrieved. - */ - public function testSaveFileNoFileRetrieved() { - $url = "url://to/csv.file"; - $datasetId = "testid1"; - $isUrlValid = NULL; - $expected = FALSE; - - // Create FileHelper stub. - $fileHelperStub = $this->getMockBuilder(IFileHelper::class) - ->setMethods([ - 'prepareDir', - 'retrieveFile', - 'fileCreate', - ]) - ->getMockForAbstractClass(); - - $fileHelperStub->expects($this->once()) - ->method('prepareDir') - ->with('public://distribution/' . $datasetId); - - $fileHelperStub->expects($this->once()) - ->method('retrieveFile') - ->with($url, 'public://distribution/' . $datasetId) - ->willReturn($isUrlValid); - - $fileHelperStub->expects($this->never()) - ->method('fileCreate'); - - // Create ResourceImporter stub. - $resourceImporterStub = $this->getMockBuilder(ResourceImporter::class) - ->setMethods(['getFileHelper']) - ->disableOriginalConstructor() - ->getMock(); - - $resourceImporterStub->expects($this->once()) - ->method('getFileHelper') - ->willReturn($fileHelperStub); - - // Assert. - $actual = $resourceImporterStub->saveFile($url, $datasetId); - $this->assertEquals($expected, $actual); - } - -}