diff --git a/cypress/fixtures/electionDistricts.json b/cypress/fixtures/electionDistricts.json index 2555e4e1f..84855eab0 100644 --- a/cypress/fixtures/electionDistricts.json +++ b/cypress/fixtures/electionDistricts.json @@ -1,6 +1,7 @@ { "uuid": "c9e2d352-e24c-4051-9158-f48127aa5692", "properties" : [ + "record_number", "lon", "lat", "unit_type", diff --git a/cypress/integration/03_datastore.spec.js b/cypress/integration/03_datastore.spec.js index af4cf7295..7ddc6b262 100644 --- a/cypress/integration/03_datastore.spec.js +++ b/cypress/integration/03_datastore.spec.js @@ -127,7 +127,7 @@ context('Datastore API', () => { expect(response.status).eql(200); expect(response.body.columns).eql(expected_columns); expect(response.body.numOfRows).eql(2); - expect(response.body.numOfColumns).eql(5); + expect(response.body.numOfColumns).eql(6); }); // Delete. diff --git a/modules/custom/dkan_common/src/Storage/AbstractDatabaseTable.php b/modules/custom/dkan_common/src/Storage/AbstractDatabaseTable.php index e360a0ea6..c477f2d73 100644 --- a/modules/custom/dkan_common/src/Storage/AbstractDatabaseTable.php +++ b/modules/custom/dkan_common/src/Storage/AbstractDatabaseTable.php @@ -2,17 +2,19 @@ namespace Drupal\dkan_common\Storage; +use Contracts\RemoverInterface; +use Contracts\RetrieverInterface; use Dkan\Datastore\Storage\StorageInterface; use Dkan\Datastore\Storage\Database\SqlStorageTrait; use Drupal\Core\Database\Connection; -use Drupal\Core\Database\Query\Select; use Drupal\dkan_datastore\Storage\Query; /** * AbstractDatabaseTable class. */ -abstract class AbstractDatabaseTable implements StorageInterface { +abstract class AbstractDatabaseTable implements StorageInterface, RetrieverInterface, RemoverInterface { use SqlStorageTrait; + use QueryToQueryHelperTrait; protected $connection; @@ -30,7 +32,7 @@ abstract protected function getTableName(); * Transform the string data given into what should be use by the insert * query. */ - abstract protected function prepareData(string $data): array; + abstract protected function prepareData(string $data, string $id = NULL): array; /** * Get the primary key used in the table. @@ -51,6 +53,23 @@ public function __construct(Connection $connection) { } } + /** + * Inherited. + * + * @inheritDoc + */ + public function retrieve(string $id) { + $this->setTable(); + + $result = $this->connection->select($this->getTableName(), 't') + ->fields('t', array_keys($this->getSchema()['fields'])) + ->condition($this->primaryKey(), $id) + ->execute() + ->fetch(); + + return $result; + } + /** * Inherited. * @@ -79,16 +98,52 @@ public function retrieveAll(): array { */ public function store($data, string $id = NULL): string { $this->setTable(); - $data = $this->prepareData($data); - $q = $this->connection->insert($this->getTableName()); - $q->fields(array_keys($this->schema['fields'])); - $q->values($data); - $id = $q->execute(); + $existing = $this->retrieve($id); + + $data = $this->prepareData($data, $id); + + if (!$existing) { + $q = $this->connection->insert($this->getTableName()); + $q->fields($this->getNonSerialFields()); + $q->values($data); + $id = $q->execute(); + } + else { + $q = $this->connection->update($this->getTableName()); + $q->fields($data) + ->condition($this->primaryKey(), $id) + ->execute(); + } return "{$id}"; } + /** + * Private. + */ + private function getNonSerialFields() { + $fields = []; + foreach ($this->schema['fields'] as $field => $info) { + if ($info['type'] != 'serial') { + $fields[] = $field; + } + } + return $fields; + } + + /** + * Inherited. + * + * @inheritDoc + */ + public function remove(string $id) { + $tableName = $this->getTableName(); + $this->connection->delete($tableName) + ->condition($this->primaryKey(), $id) + ->execute(); + } + /** * Count rows in table. */ @@ -136,45 +191,6 @@ private function setTable() { } } - /** - * Private. - */ - private function setQueryConditions(Select $db_query, Query $query) { - foreach ($query->conditions as $property => $value) { - $db_query->condition($property, $value, "LIKE"); - } - } - - /** - * Private. - */ - private function setQueryOrderBy(Select $db_query, Query $query) { - foreach ($query->sort['ASC'] as $property) { - $db_query->orderBy($property); - } - - foreach ($query->sort['DESC'] as $property) { - $db_query->orderBy($property, 'DESC'); - } - } - - /** - * Private. - */ - private function setQueryLimitAndOffset(Select $db_query, Query $query) { - if ($query->limit) { - if ($query->offset) { - $db_query->range($query->offset, $query->limit); - } - else { - $db_query->range(0, $query->limit); - } - } - elseif ($query->offset) { - $db_query->range($query->limit); - } - } - /** * Destroy. * diff --git a/modules/custom/dkan_common/src/Storage/QueryToQueryHelperTrait.php b/modules/custom/dkan_common/src/Storage/QueryToQueryHelperTrait.php new file mode 100644 index 000000000..8c3af985b --- /dev/null +++ b/modules/custom/dkan_common/src/Storage/QueryToQueryHelperTrait.php @@ -0,0 +1,54 @@ +conditions as $property => $value) { + $db_query->condition($property, $value, "LIKE"); + } + } + + /** + * Private. + */ + private function setQueryOrderBy(Select $db_query, Query $query) { + foreach ($query->sort['ASC'] as $property) { + $db_query->orderBy($property); + } + + foreach ($query->sort['DESC'] as $property) { + $db_query->orderBy($property, 'DESC'); + } + } + + /** + * Private. + */ + private function setQueryLimitAndOffset(Select $db_query, Query $query) { + if ($query->limit) { + if ($query->offset) { + $db_query->range($query->offset, $query->limit); + } + else { + $db_query->range(0, $query->limit); + } + } + elseif ($query->offset) { + $db_query->range($query->limit); + } + } + +} diff --git a/modules/custom/dkan_datastore/src/Storage/DatabaseTable.php b/modules/custom/dkan_datastore/src/Storage/DatabaseTable.php index a15e81cb1..70064823a 100755 --- a/modules/custom/dkan_datastore/src/Storage/DatabaseTable.php +++ b/modules/custom/dkan_datastore/src/Storage/DatabaseTable.php @@ -80,7 +80,7 @@ protected function getTableName() { /** * Protected. */ - protected function prepareData(string $data): array { + protected function prepareData(string $data, string $id = NULL): array { return json_decode($data); } @@ -91,4 +91,24 @@ protected function primaryKey() { return "record_number"; } + /** + * Overriden. + */ + public function setSchema($schema) { + $fields = $schema['fields']; + $new_field = [ + $this->primaryKey() => + [ + 'type' => 'serial', + 'unsigned' => TRUE, + 'not null' => TRUE, + ], + ]; + $fields = array_merge($new_field, $fields); + + $schema['fields'] = $fields; + $schema['primary key'] = [$this->primaryKey()]; + parent::setSchema($schema); + } + } diff --git a/modules/custom/dkan_datastore/src/Storage/JobStore.php b/modules/custom/dkan_datastore/src/Storage/JobStore.php index b780b5ebb..27dc3a0ce 100644 --- a/modules/custom/dkan_datastore/src/Storage/JobStore.php +++ b/modules/custom/dkan_datastore/src/Storage/JobStore.php @@ -3,23 +3,14 @@ namespace Drupal\dkan_datastore\Storage; use Contracts\RetrieverInterface; -use Contracts\StorerInterface; +use Drupal\dkan_common\Storage\AbstractDatabaseTable; use Procrastinator\Job\Job; use Drupal\Core\Database\Connection; /** * Retrieve a serialized job (datastore importer or harvest) from the database. - * - * @todo should probably be a service in its own module. */ -class JobStore implements StorerInterface, RetrieverInterface { - - /** - * The database connection to use for querrying jobstore tables. - * - * @var \Drupal\Core\Database\Connection - */ - private $connection; +class JobStore extends AbstractDatabaseTable implements RetrieverInterface { private $jobClass; @@ -27,140 +18,49 @@ class JobStore implements StorerInterface, RetrieverInterface { * Constructor. */ public function __construct(string $jobClass, Connection $connection) { - $this->jobClass = $jobClass; - $this->connection = $connection; - } - - /** - * Get. - */ - public function retrieve(string $identifier) { - $tableName = $this->getTableName($this->jobClass); - - $this->validateJobClassAndTableExistence($this->jobClass, $tableName); - - $result = $this->connection->select($tableName, 't') - ->fields('t', ['job_data']) - ->condition('ref_uuid', $identifier) - ->execute() - ->fetch(); - - return $result->job_data; - } - - /** - * Store. - */ - public function store($data, string $id = NULL): string { - $tableName = $this->getTableName($this->jobClass); - - if (!$this->tableExists($tableName)) { - $this->createTable($tableName); - } - - $existing_id = $this->connection->select($tableName, 't') - ->fields('t', ['jid']) - ->condition('ref_uuid', $id) - ->execute() - ->fetch(); - - $values = ['ref_uuid' => $id, 'job_data' => $data]; - if (!$existing_id) { - $q = $this->connection->insert($tableName); - $q->fields(array_keys($values)) - ->values(array_values($values)) - ->execute(); - } - else { - $q = $this->connection->update($tableName); - $q->fields($values) - ->condition('jid', $existing_id->jid) - ->execute(); - } - - return $id; - } - - /** - * Retrieve all. - */ - public function retrieveAll(): array { - $tableName = $this->getTableName($this->jobClass); - - $this->validateJobClassAndTableExistence($this->jobClass, $tableName); - - $result = $this->connection->select($tableName, 't') - ->fields('t', ['ref_uuid']) - ->execute() - ->fetchAll(); - - if ($result === FALSE) { - throw new \Exception("No data in table: $tableName"); - } - - return array_map(function ($item) { - return $item->ref_uuid; - }, $result); - } - - /** - * Private. - */ - private function validateJobClassAndTableExistence($jobClass, $tableName) { + parent::__construct($connection); if (!$this->validateJobClass($jobClass)) { throw new \Exception("Invalid jobType provided: $jobClass"); } - - if (!$this->tableExists($tableName)) { - $this->createTable($tableName); - } + $this->jobClass = $jobClass; + $this->setOurSchema(); } /** - * Remove. + * Get. */ - public function remove($uuid) { - $tableName = $this->getTableName($this->jobClass); - $this->connection->delete($tableName) - ->condition('ref_uuid', $uuid) - ->execute(); + public function retrieve(string $id) { + $result = parent::retrieve($id); + return $result->job_data; } /** - * Private. + * Protected. */ - private function getTableName($jobClass) { - $safeClassName = strtolower(preg_replace('/\\\\/', '_', $jobClass)); + protected function getTableName() { + $safeClassName = strtolower(preg_replace('/\\\\/', '_', $this->jobClass)); return 'jobstore_' . $safeClassName; } /** * Private. */ - private function createTable(string $tableName) { + private function setOurSchema() { $schema = [ 'fields' => [ - 'jid' => ['type' => 'serial', 'unsigned' => TRUE, 'not null' => TRUE], - 'ref_uuid' => ['type' => 'varchar', 'length' => 128], + 'ref_uuid' => ['type' => 'varchar', 'length' => 128, 'not null' => TRUE], 'job_data' => ['type' => 'text', 'length' => 65535], ], 'indexes' => [ - 'jid' => ['jid'], + 'ref_uuid' => ['ref_uuid'], ], 'foriegn_keys' => [ 'ref_uuid' => ['table' => 'node', 'columns' => ['uuid' => 'uuid']], ], - 'primary_key' => ['jid'], + 'primary_key' => ['ref_uuid'], ]; - $this->connection->schema()->createTable($tableName, $schema); - } - /** - * Check for existence of a table name. - */ - private function tableExists($tableName) { - $exists = $this->connection->schema()->tableExists($tableName); - return $exists; + $this->setSchema($schema); } /** @@ -173,4 +73,22 @@ private function validateJobClass(string $jobClass): bool { return FALSE; } + /** + * Inherited. + * + * @inheritDoc + */ + protected function prepareData(string $data, string $id = NULL): array { + return ['ref_uuid' => $id, 'job_data' => $data]; + } + + /** + * Inherited. + * + * @inheritDoc + */ + protected function primaryKey() { + return 'ref_uuid'; + } + } diff --git a/modules/custom/dkan_datastore/tests/src/Unit/Storage/DatabaseTableTest.php b/modules/custom/dkan_datastore/tests/src/Unit/Storage/DatabaseTableTest.php index e320f9ef5..fe8e0968c 100644 --- a/modules/custom/dkan_datastore/tests/src/Unit/Storage/DatabaseTableTest.php +++ b/modules/custom/dkan_datastore/tests/src/Unit/Storage/DatabaseTableTest.php @@ -67,13 +67,18 @@ public function testStore() { ->add(Connection::class, 'insert', Insert::class) ->add(Insert::class, 'fields', Insert::class) ->add(Insert::class, 'values', Insert::class) - ->add(Insert::class, 'execute', "1"); + ->add(Insert::class, 'execute', "1") + ->add(Connection::class, 'select', Select::class, 'select_1') + ->add(Select::class, 'fields', Select::class) + ->add(Select::class, 'condition', Select::class) + ->add(Select::class, 'execute', Statement::class) + ->add(Statement::class, 'fetch', NULL); $databaseTable = new DatabaseTable( $connectionChain->getMock(), $this->getResource() ); - $this->assertEquals("1", $databaseTable->store('["Gerardo", "Gonzalez"]')); + $this->assertEquals("1", $databaseTable->store('["Gerardo", "Gonzalez"]', "1")); } /** @@ -111,8 +116,8 @@ public function testGetSummary() { ); $this->assertEquals( new TableSummary( - 2, - ["first_name", "last_name"], + 3, + ["record_number", "first_name", "last_name"], 1 ), $databaseTable->getSummary()); } diff --git a/modules/custom/dkan_datastore/tests/src/Unit/Storage/JobStoreTest.php b/modules/custom/dkan_datastore/tests/src/Unit/Storage/JobStoreTest.php index 073685b6e..a5dea82fe 100644 --- a/modules/custom/dkan_datastore/tests/src/Unit/Storage/JobStoreTest.php +++ b/modules/custom/dkan_datastore/tests/src/Unit/Storage/JobStoreTest.php @@ -10,6 +10,7 @@ use Drupal\Core\Database\Schema; use Drupal\Core\Database\Statement; use Drupal\dkan_common\Tests\Mock\Chain; +use Drupal\dkan_common\Tests\Mock\Sequence; use Drupal\dkan_datastore\Storage\JobStore; use FileFetcher\FileFetcher; use PHPUnit\Framework\TestCase; @@ -24,7 +25,9 @@ class JobStoreTest extends TestCase { */ public function testConstruction() { $chain = (new Chain($this)) - ->add(Connection::class, "blah", "blah"); + ->add(Connection::class, "blah", "blah") + ->add(Connection::class, "schema", Schema::class) + ->add(Schema::class, "tableExists", FALSE); $jobStore = new JobStore(FileFetcher::class, $chain->getMock()); $this->assertTrue(is_object($jobStore)); @@ -39,6 +42,11 @@ public function testRetrieve() { $job->ref_uuid = "1"; $job->job_data = $job_data; + $fieldInfo = [ + (object) ['Field' => "ref_uuid"], + (object) ['Field' => "job_data"], + ]; + $chain = (new Chain($this)) ->add(Connection::class, "schema", Schema::class) ->add(Schema::class, "tableExists", TRUE) @@ -46,7 +54,9 @@ public function testRetrieve() { ->add(Select::class, 'fields', Select::class) ->add(Select::class, 'condition', Select::class) ->add(Select::class, 'execute', Statement::class) - ->add(Statement::class, 'fetch', $job); + ->add(Statement::class, 'fetch', $job) + ->add(Connection::class, 'query', Statement::class) + ->add(Statement::class, 'fetchAll', $fieldInfo); $jobStore = new JobStore(FileFetcher::class, $chain->getMock()); $this->assertEquals($job_data, $jobStore->retrieve("1", FileFetcher::class)); @@ -61,16 +71,26 @@ public function testRetrieveAll() { $job->ref_uuid = "1"; $job->job_data = $job_data; + $fieldInfo = [ + (object) ['Field' => "ref_uuid"], + (object) ['Field' => "job_data"], + ]; + + $sequence = (new Sequence()) + ->add($fieldInfo) + ->add([$job]); + $chain = (new Chain($this)) ->add(Connection::class, "schema", Schema::class) ->add(Schema::class, "tableExists", TRUE) ->add(Connection::class, 'select', Select::class, 'select_1') ->add(Select::class, 'fields', Select::class) ->add(Select::class, 'execute', Statement::class) - ->add(Statement::class, 'fetchAll', [$job]); + ->add(Connection::class, 'query', Statement::class) + ->add(Statement::class, 'fetchAll', $sequence); $jobStore = new JobStore(FileFetcher::class, $chain->getMock()); - $this->assertTrue(is_array($jobStore->retrieveAll(FileFetcher::class))); + $this->assertTrue(is_array($jobStore->retrieveAll())); } /** @@ -85,6 +105,11 @@ public function testStore() { $job->ref_uuid = "1"; $job->job_data = $job_data; + $fieldInfo = [ + (object) ['Field' => "ref_uuid"], + (object) ['Field' => "job_data"], + ]; + $connection = (new Chain($this)) ->add(Connection::class, "schema", Schema::class) ->add(Schema::class, "tableExists", TRUE) @@ -97,6 +122,8 @@ public function testStore() { ->add(Update::class, "fields", Update::class) ->add(Update::class, "condition", Update::class) ->add(Update::class, "execute", NULL) + ->add(Connection::class, 'query', Statement::class) + ->add(Statement::class, 'fetchAll', $fieldInfo) ->getMock(); $jobStore = new JobStore(FileFetcher::class, $connection); @@ -108,12 +135,19 @@ public function testStore() { * */ public function testRemove() { + $fieldInfo = [ + (object) ['Field' => "ref_uuid"], + (object) ['Field' => "job_data"], + ]; + $connection = (new Chain($this)) ->add(Connection::class, "schema", Schema::class) ->add(Schema::class, "tableExists", TRUE) ->add(Connection::class, "delete", Delete::class) ->add(Delete::class, "condition", Delete::class) ->add(Delete::class, "execute", NULL) + ->add(Connection::class, 'query', Statement::class) + ->add(Statement::class, 'fetchAll', $fieldInfo) ->getMock(); $jobStore = new JobStore(FileFetcher::class, $connection);