From f21c480ce02ed3493c2409c99506eb9928f76f64 Mon Sep 17 00:00:00 2001 From: corentin Date: Thu, 26 Oct 2017 18:13:52 +0200 Subject: [PATCH 01/10] Clean some test --- Command/PopulateElasticCommand.php | 3 ++ Tests/Command/ExodusElasticCommandTest.php | 28 ++++++------- Tests/Command/PopulateElasticCommandTest.php | 43 ++++++++++---------- 3 files changed, 37 insertions(+), 37 deletions(-) diff --git a/Command/PopulateElasticCommand.php b/Command/PopulateElasticCommand.php index b2df389..c0456aa 100644 --- a/Command/PopulateElasticCommand.php +++ b/Command/PopulateElasticCommand.php @@ -246,7 +246,10 @@ public function processBatch($type, $transformer) $this->_bulk($objectType, $aDocuments); $this->output->writeln(self::completeLine("Start populate '{$type}'")); + $progressBar->setProgress($iResults); + $progressBar->display(); $progressBar->finish(); + $this->output->writeln(''); $this->output->writeln("" . self::completeLine("Finish populate {$type}") . ""); } diff --git a/Tests/Command/ExodusElasticCommandTest.php b/Tests/Command/ExodusElasticCommandTest.php index 4f2646f..c0a6101 100644 --- a/Tests/Command/ExodusElasticCommandTest.php +++ b/Tests/Command/ExodusElasticCommandTest.php @@ -13,16 +13,18 @@ class ExodusElasticCommandTest extends KernelTestCase { - - /** @var \Headoo\ElasticSearchBundle\Helper\ElasticSearchHelper */ - private $elasticSearchHelper; - /** @var EntityManager */ - private $entityManager; + static private $entityManager; /** @var Application */ protected $application; + static public function setUpBeforeClass() + { + parent::setUpBeforeClass(); + self::bootKernel(); + self::$entityManager = static::$kernel->getContainer()->get('doctrine')->getManager(); + } /** * {@inheritDoc} @@ -30,10 +32,6 @@ class ExodusElasticCommandTest extends KernelTestCase public function setUp() { parent::setUp(); - self::bootKernel(); - - $this->entityManager = static::$kernel->getContainer()->get('doctrine')->getManager(); - $this->elasticSearchHelper = static::$kernel->getContainer()->get('headoo.elasticsearch.helper'); $this->application = new Application(self::$kernel); $this->application->setAutoExit(false); @@ -45,7 +43,7 @@ public function testCommandFakeEntity() $options1 = [ 'command' => 'headoo:elastic:exodus', '--batch' => 10, - '--dry-run' => true, + '--dry-run' => false, '--verbose' => true, '--env' => 'prod', ]; @@ -84,8 +82,8 @@ public function loadFixtures(array $options = []) $loader = new Loader(); $loader->addFixture(new LoadData()); - $purger = new ORMPurger($this->entityManager); - $executor = new ORMExecutor($this->entityManager, $purger); + $purger = new ORMPurger(self::$entityManager); + $executor = new ORMExecutor(self::$entityManager, $purger); $executor->execute($loader->getFixtures()); # Populate ES @@ -95,9 +93,9 @@ public function loadFixtures(array $options = []) $this->application->run(new ArrayInput($options4)); # Remove one entity in Doctrine - $entity = $this->entityManager->getRepository('\Headoo\ElasticSearchBundle\Tests\Entity\FakeEntity')->findOneBy([]); - $this->entityManager->remove($entity); - $this->entityManager->flush($entity); + $entity = self::$entityManager->getRepository('\Headoo\ElasticSearchBundle\Tests\Entity\FakeEntity')->findOneBy([]); + self::$entityManager->remove($entity); + self::$entityManager->flush($entity); } } diff --git a/Tests/Command/PopulateElasticCommandTest.php b/Tests/Command/PopulateElasticCommandTest.php index a123ac9..abf42b7 100644 --- a/Tests/Command/PopulateElasticCommandTest.php +++ b/Tests/Command/PopulateElasticCommandTest.php @@ -16,21 +16,23 @@ class PopulateElasticCommandTest extends KernelTestCase { - /** - * @var \Headoo\ElasticSearchBundle\Helper\ElasticSearchHelper - */ - private $elasticSearchHelper; + /** @var \Headoo\ElasticSearchBundle\Helper\ElasticSearchHelper */ + static private $elasticSearchHelper; - /** - * @var EntityManager - */ - private $entityManager; + /** @var EntityManager */ + static private $entityManager; - /** - * @var Application - */ + /** @var Application */ protected $application; + static function setUpBeforeClass() + { + parent::setUpBeforeClass(); + self::bootKernel(); + + self::$entityManager = static::$kernel->getContainer()->get('doctrine')->getManager(); + self::$elasticSearchHelper = static::$kernel->getContainer()->get('headoo.elasticsearch.helper'); + } /** * {@inheritDoc} @@ -38,10 +40,7 @@ class PopulateElasticCommandTest extends KernelTestCase public function setUp() { parent::setUp(); - self::bootKernel(); - - $this->entityManager = static::$kernel->getContainer()->get('doctrine')->getManager(); - $this->elasticSearchHelper = static::$kernel->getContainer()->get('headoo.elasticsearch.helper'); + $this->application = new Application(self::$kernel); $this->application->setAutoExit(false); @@ -57,7 +56,7 @@ public function testCommand1() ]; $this->application->run(new ArrayInput($options1)); - $search = new Search($this->elasticSearchHelper->getClient('localhost')); + $search = new Search(self::$elasticSearchHelper->getClient('localhost')); $search->addIndex('test'); $query = new Query(); $query->setSize(1000); @@ -75,7 +74,7 @@ public function testCommand2() ]; $this->application->run(new ArrayInput($options2)); - $search = new Search($this->elasticSearchHelper->getClient('localhost')); + $search = new Search(self::$elasticSearchHelper->getClient('localhost')); $search->addIndex('test'); $query = new Query(); $query->setSize(1000); @@ -92,7 +91,7 @@ public function testCommand3() ]; $this->application->run(new ArrayInput($options3)); - $search = new Search($this->elasticSearchHelper->getClient('localhost')); + $search = new Search(self::$elasticSearchHelper->getClient('localhost')); $search->addIndex('test'); $query = new Query(); $query->setSize(1000); @@ -109,7 +108,7 @@ public function testCommand4() ]; $this->application->run(new ArrayInput($options4)); - $search = new Search($this->elasticSearchHelper->getClient('localhost')); + $search = new Search(self::$elasticSearchHelper->getClient('localhost')); $search->addIndex('test'); $query = new Query(); $query->setSize(1000); @@ -127,7 +126,7 @@ public function testCommandRunParallel() ]; $this->application->run(new ArrayInput($optionsRunParallel)); - $search = new Search($this->elasticSearchHelper->getClient('localhost')); + $search = new Search(self::$elasticSearchHelper->getClient('localhost')); $search->addIndex('test'); $query = new Query(); $query->setSize(1000); @@ -162,8 +161,8 @@ public function loadFixtures(array $options = []) $loader = new Loader(); $loader->addFixture(new LoadData()); - $purger = new ORMPurger($this->entityManager); - $executor = new ORMExecutor($this->entityManager, $purger); + $purger = new ORMPurger(self::$entityManager); + $executor = new ORMExecutor(self::$entityManager, $purger); $executor->execute($loader->getFixtures()); } From 9a49a1cdb4ccb52fcc9ba2b0a94b8761d9f6f4cc Mon Sep 17 00:00:00 2001 From: corentin Date: Thu, 26 Oct 2017 18:22:30 +0200 Subject: [PATCH 02/10] Clean some test: not to mutch --- Tests/Command/PopulateElasticCommandTest.php | 31 ++++++++------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/Tests/Command/PopulateElasticCommandTest.php b/Tests/Command/PopulateElasticCommandTest.php index abf42b7..91b7d09 100644 --- a/Tests/Command/PopulateElasticCommandTest.php +++ b/Tests/Command/PopulateElasticCommandTest.php @@ -17,22 +17,14 @@ class PopulateElasticCommandTest extends KernelTestCase { /** @var \Headoo\ElasticSearchBundle\Helper\ElasticSearchHelper */ - static private $elasticSearchHelper; + private $elasticSearchHelper; /** @var EntityManager */ - static private $entityManager; + private $entityManager; /** @var Application */ protected $application; - static function setUpBeforeClass() - { - parent::setUpBeforeClass(); - self::bootKernel(); - - self::$entityManager = static::$kernel->getContainer()->get('doctrine')->getManager(); - self::$elasticSearchHelper = static::$kernel->getContainer()->get('headoo.elasticsearch.helper'); - } /** * {@inheritDoc} @@ -40,7 +32,10 @@ static function setUpBeforeClass() public function setUp() { parent::setUp(); - + self::bootKernel(); + + $this->entityManager = static::$kernel->getContainer()->get('doctrine')->getManager(); + $this->elasticSearchHelper = static::$kernel->getContainer()->get('headoo.elasticsearch.helper'); $this->application = new Application(self::$kernel); $this->application->setAutoExit(false); @@ -56,7 +51,7 @@ public function testCommand1() ]; $this->application->run(new ArrayInput($options1)); - $search = new Search(self::$elasticSearchHelper->getClient('localhost')); + $search = new Search($this->elasticSearchHelper->getClient('localhost')); $search->addIndex('test'); $query = new Query(); $query->setSize(1000); @@ -74,7 +69,7 @@ public function testCommand2() ]; $this->application->run(new ArrayInput($options2)); - $search = new Search(self::$elasticSearchHelper->getClient('localhost')); + $search = new Search($this->elasticSearchHelper->getClient('localhost')); $search->addIndex('test'); $query = new Query(); $query->setSize(1000); @@ -91,7 +86,7 @@ public function testCommand3() ]; $this->application->run(new ArrayInput($options3)); - $search = new Search(self::$elasticSearchHelper->getClient('localhost')); + $search = new Search($this->elasticSearchHelper->getClient('localhost')); $search->addIndex('test'); $query = new Query(); $query->setSize(1000); @@ -108,7 +103,7 @@ public function testCommand4() ]; $this->application->run(new ArrayInput($options4)); - $search = new Search(self::$elasticSearchHelper->getClient('localhost')); + $search = new Search($this->elasticSearchHelper->getClient('localhost')); $search->addIndex('test'); $query = new Query(); $query->setSize(1000); @@ -126,7 +121,7 @@ public function testCommandRunParallel() ]; $this->application->run(new ArrayInput($optionsRunParallel)); - $search = new Search(self::$elasticSearchHelper->getClient('localhost')); + $search = new Search($this->elasticSearchHelper->getClient('localhost')); $search->addIndex('test'); $query = new Query(); $query->setSize(1000); @@ -161,8 +156,8 @@ public function loadFixtures(array $options = []) $loader = new Loader(); $loader->addFixture(new LoadData()); - $purger = new ORMPurger(self::$entityManager); - $executor = new ORMExecutor(self::$entityManager, $purger); + $purger = new ORMPurger($this->entityManager); + $executor = new ORMExecutor($this->entityManager, $purger); $executor->execute($loader->getFixtures()); } From 3b67b677772a320f96d3c1c361e28afac160f340 Mon Sep 17 00:00:00 2001 From: corentin Date: Thu, 26 Oct 2017 18:33:04 +0200 Subject: [PATCH 03/10] Clean some test: not to mutch --- Tests/Command/ExodusElasticCommandTest.php | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Tests/Command/ExodusElasticCommandTest.php b/Tests/Command/ExodusElasticCommandTest.php index c0a6101..cfb35a0 100644 --- a/Tests/Command/ExodusElasticCommandTest.php +++ b/Tests/Command/ExodusElasticCommandTest.php @@ -14,7 +14,7 @@ class ExodusElasticCommandTest extends KernelTestCase { /** @var EntityManager */ - static private $entityManager; + private $entityManager; /** @var Application */ protected $application; @@ -23,7 +23,6 @@ static public function setUpBeforeClass() { parent::setUpBeforeClass(); self::bootKernel(); - self::$entityManager = static::$kernel->getContainer()->get('doctrine')->getManager(); } /** @@ -32,6 +31,7 @@ static public function setUpBeforeClass() public function setUp() { parent::setUp(); + $this->entityManager = static::$kernel->getContainer()->get('doctrine')->getManager(); $this->application = new Application(self::$kernel); $this->application->setAutoExit(false); @@ -82,8 +82,8 @@ public function loadFixtures(array $options = []) $loader = new Loader(); $loader->addFixture(new LoadData()); - $purger = new ORMPurger(self::$entityManager); - $executor = new ORMExecutor(self::$entityManager, $purger); + $purger = new ORMPurger($this->entityManager); + $executor = new ORMExecutor($this->entityManager, $purger); $executor->execute($loader->getFixtures()); # Populate ES @@ -93,9 +93,9 @@ public function loadFixtures(array $options = []) $this->application->run(new ArrayInput($options4)); # Remove one entity in Doctrine - $entity = self::$entityManager->getRepository('\Headoo\ElasticSearchBundle\Tests\Entity\FakeEntity')->findOneBy([]); - self::$entityManager->remove($entity); - self::$entityManager->flush($entity); + $entity = $this->entityManager->getRepository('\Headoo\ElasticSearchBundle\Tests\Entity\FakeEntity')->findOneBy([]); + $this->entityManager->remove($entity); + $this->entityManager->flush($entity); } } From ab2e23caa285e197339d18653943d68784e69ac5 Mon Sep 17 00:00:00 2001 From: corentin Date: Fri, 27 Oct 2017 12:00:47 +0200 Subject: [PATCH 04/10] Test connection / Better management between the sprocess --- Command/AbstractCommand.php | 17 +++++++- Command/PopulateElasticCommand.php | 23 +++++----- Helper/ElasticSearchHelper.php | 13 +++++- Tests/Command/AbstractCommandTest.php | 1 - Tests/Command/ExodusElasticCommandTest.php | 12 +++--- Tests/Command/PopulateElasticCommandTest.php | 6 ++- Tests/Helper/ElasticSearchHelperTest.php | 44 ++++++++++++++------ 7 files changed, 82 insertions(+), 34 deletions(-) diff --git a/Command/AbstractCommand.php b/Command/AbstractCommand.php index 12aa013..d014cca 100644 --- a/Command/AbstractCommand.php +++ b/Command/AbstractCommand.php @@ -45,6 +45,8 @@ abstract class AbstractCommand extends ContainerAwareCommand protected $verbose = false; /** @var bool $dryRun Do not make any change on ES */ protected $dryRun = false; + /** @var bool $quiet */ + protected $quiet = false; /** @var string */ protected $environment; /** @var string */ @@ -134,6 +136,10 @@ protected function readOption(InputInterface $input) if ($input->hasOption('verbose')) { $this->verbose = $input->getOption('verbose'); } + + if ($input->hasOption('quiet')) { + $this->quiet = $input->getOption('quiet'); + } } /** @@ -208,8 +214,15 @@ protected function getRepositoryFromType($sType) protected function getClient($sType) { $connection = $this->mappings[$sType]['connection']; + $client = $this->elasticSearchHelper->getClient($connection); - return $this->elasticSearchHelper->getClient($connection); - } + try { + $this->elasticSearchHelper->isConnected($client); + } catch (\Elastica\Exception\Connection\HttpException $e) { + $this->output->writeln("ElasticSearch connection error. Check your configuration and your connection"); + throw $e; + } + return $client; + } } diff --git a/Command/PopulateElasticCommand.php b/Command/PopulateElasticCommand.php index c0456aa..2bd6086 100644 --- a/Command/PopulateElasticCommand.php +++ b/Command/PopulateElasticCommand.php @@ -19,7 +19,7 @@ protected function configure() ->addOption('offset', null, InputOption::VALUE_OPTIONAL, 'Offset For selected Type', 0) ->addOption('type', null, InputOption::VALUE_OPTIONAL, 'Type of document you want to populate. You must to have configure it before use', null) ->addOption('threads', null, InputOption::VALUE_OPTIONAL, 'number of simultaneous threads', null) - ->addOption('reset', null) + ->addOption('reset', null, InputOption::VALUE_NONE, 'Reset the index') ->addOption('batch', null, InputOption::VALUE_OPTIONAL, 'Number of Document per batch', null); } @@ -145,9 +145,10 @@ public function runParallel(ProgressBar $progressBar, array $processes, $maxPara unset($currentProcesses[$index]); - $progression += $this->limit; + $processDone = intval($process->getOutput()); + $progression += $processDone; $progressBar->setMessage("$progression/$progressMax"); - $progressBar->advance($this->limit); + $progressBar->advance($processDone); // directly add and start new process after the previous finished if (count($processesQueue) > 0) { @@ -162,9 +163,9 @@ public function runParallel(ProgressBar $progressBar, array $processes, $maxPara // continue loop while there are processes being executed or waiting for execution } while (count($processesQueue) > 0 || count($currentProcesses) > 0); - $progressBar->setMessage("$numberOfEntities/$progressMax"); - $progressBar->setProgress($numberOfEntities); $progressBar->finish(); + $progressBar->display(); + $this->output->writeln(''); return $returnValue; } @@ -185,7 +186,7 @@ public function beginBatch($type) for ($i = 0; $i <= $numberOfProcess; $i++) { $_offset = $this->offset + ($this->limit * $i); - $process = new Process("php $this->consoleDir headoo:elastic:populate --type={$type} --limit={$this->limit} --offset={$_offset} " . $sOptions); + $process = new Process("php $this->consoleDir headoo:elastic:populate --type={$type} --limit={$this->limit} --offset={$_offset} --quiet " . $sOptions); $aProcess[] = $process; } @@ -222,7 +223,7 @@ public function processBatch($type, $transformer) $iterableResult = $query->iterate(); $progressBar = $this->getProgressBar($this->output, $iResults); - $progression = $this->offset; + $progression = 0; $progressMax = $iResults + $this->offset; $aDocuments = []; @@ -237,14 +238,13 @@ public function processBatch($type, $transformer) $aDocuments[]= $document; $this->entityManager->detach($row[0]); - $progressBar->setMessage(($progression++) . "/{$progressMax}"); + $progressBar->setMessage((++$progression + $this->offset) . "/{$progressMax}"); $progressBar->advance(); gc_collect_cycles(); } $this->_bulk($objectType, $aDocuments); - $this->output->writeln(self::completeLine("Start populate '{$type}'")); $progressBar->setProgress($iResults); $progressBar->display(); @@ -252,6 +252,10 @@ public function processBatch($type, $transformer) $this->output->writeln(''); $this->output->writeln("" . self::completeLine("Finish populate {$type}") . ""); + # In quite mode: just write in output the number of documents treated + if ($this->quiet) { + $this->output->writeln("$progression", OutputInterface::VERBOSITY_QUIET); + } } /** @@ -275,5 +279,4 @@ private function _resetType($type) return true; } - } diff --git a/Helper/ElasticSearchHelper.php b/Helper/ElasticSearchHelper.php index d8d5b28..f5a9aa5 100644 --- a/Helper/ElasticSearchHelper.php +++ b/Helper/ElasticSearchHelper.php @@ -35,7 +35,7 @@ public function getClient($connectionName) * @param array $servers * @return \Elastica\Client */ - public function getCluster(array $servers) + static public function getCluster(array $servers) { $cluster = new Client([ 'servers' => [$servers] @@ -44,4 +44,15 @@ public function getCluster(array $servers) return $cluster; } + /** + * @param Client $elasticaClient + * @return bool + */ + static public function isConnected(\Elastica\Client $elasticaClient) + { + $status = $elasticaClient->getStatus(); + $status->refresh(); + + return true; + } } diff --git a/Tests/Command/AbstractCommandTest.php b/Tests/Command/AbstractCommandTest.php index 3c82209..bfd053a 100644 --- a/Tests/Command/AbstractCommandTest.php +++ b/Tests/Command/AbstractCommandTest.php @@ -29,5 +29,4 @@ public function testCompleteLine() "Completed line should be equals to LINE_LENGTH: '$sMsg'" ); } - } diff --git a/Tests/Command/ExodusElasticCommandTest.php b/Tests/Command/ExodusElasticCommandTest.php index cfb35a0..f7deb93 100644 --- a/Tests/Command/ExodusElasticCommandTest.php +++ b/Tests/Command/ExodusElasticCommandTest.php @@ -19,18 +19,14 @@ class ExodusElasticCommandTest extends KernelTestCase /** @var Application */ protected $application; - static public function setUpBeforeClass() - { - parent::setUpBeforeClass(); - self::bootKernel(); - } - /** * {@inheritDoc} + * @outputBuffering disabled */ public function setUp() { parent::setUp(); + self::bootKernel(); $this->entityManager = static::$kernel->getContainer()->get('doctrine')->getManager(); $this->application = new Application(self::$kernel); $this->application->setAutoExit(false); @@ -68,6 +64,10 @@ public function testCommandWrongType() self::assertNotEquals(0, $returnValue, 'This command should failed: UNKNOWN TYPE'); } + /** + * @outputBuffering disabled + * @param array $options + */ public function loadFixtures(array $options = []) { # Do not show output diff --git a/Tests/Command/PopulateElasticCommandTest.php b/Tests/Command/PopulateElasticCommandTest.php index 91b7d09..59cfd74 100644 --- a/Tests/Command/PopulateElasticCommandTest.php +++ b/Tests/Command/PopulateElasticCommandTest.php @@ -25,9 +25,9 @@ class PopulateElasticCommandTest extends KernelTestCase /** @var Application */ protected $application; - /** * {@inheritDoc} + * @outputBuffering disabled */ public function setUp() { @@ -142,6 +142,10 @@ public function testCommandWrongType() self::assertNotEquals(0, $returnValue, 'This command should failed: UNKNOWN TYPE'); } + /** + * @outputBuffering disabled + * @param array $options + */ public function loadFixtures(array $options = []) { # Do not show output diff --git a/Tests/Helper/ElasticSearchHelperTest.php b/Tests/Helper/ElasticSearchHelperTest.php index 09ae269..c22e259 100755 --- a/Tests/Helper/ElasticSearchHelperTest.php +++ b/Tests/Helper/ElasticSearchHelperTest.php @@ -6,12 +6,9 @@ class ElasticSearchHelperTest extends KernelTestCase { - /** - * @var \Headoo\ElasticSearchBundle\Helper\ElasticSearchHelper - */ + /** @var \Headoo\ElasticSearchBundle\Helper\ElasticSearchHelper */ private $elasticSearchHelper; - /** * {@inheritDoc} */ @@ -19,24 +16,45 @@ public function setUp() { parent::setUp(); self::bootKernel(); - $this->elasticSearchHelper = static::$kernel->getContainer()->get('headoo.elasticsearch.helper'); - + $this->elasticSearchHelper = static::$kernel->getContainer()->get('headoo.elasticsearch.helper'); } public function testClientConnection() { $connection = $this->elasticSearchHelper->getClient('localhost'); - //We test just agains elastic 5.X - $this->assertEquals(5 , substr($connection->getVersion(),0,1)); + //We test just against elastic 5.X + $this->assertEquals( + 5, + substr($connection->getVersion(), 0, 1), + 'Expected ElasticSearch version 5.x.x' + ); } public function testClusterConnection() { - $connections = $this->elasticSearchHelper->getCluster(array( - 'host' => 'localhost', - 'port' => 9200 - )); - $this->assertEquals(5 , substr($connections->getVersion(),0,1)); + $connections = $this->elasticSearchHelper->getCluster(['host' => 'localhost', 'port' => 9200]); + + $this->assertEquals( + 5, + substr($connections->getVersion(), 0, 1), + 'Expected ElasticSearch version 5.x.x' + ); + + $this->assertTrue( + $this->elasticSearchHelper->isConnected($connections), + 'Expected Client correctly connected' + ); } + /** + * @expectedException \Elastica\Exception\Connection\HttpException + */ + public function testClusterNotConnected() + { + $connections = $this->elasticSearchHelper->getCluster(['host' => '1.2.3.4', 'port' => 5678]); + $this->assertNotTrue( + $this->elasticSearchHelper->isConnected($connections), + 'Connection should failed in HttpException' + ); + } } From eb145e4ddba8bc6643dc586316c3e813893ec89a Mon Sep 17 00:00:00 2001 From: corentin Date: Wed, 13 Dec 2017 16:59:26 +0100 Subject: [PATCH 05/10] add options: --where --id --join --- Command/AbstractCommand.php | 19 ++++++ Command/PopulateElasticCommand.php | 63 +++++++++++++++++++- EventListener/ElasticSearchListener.php | 2 +- Tests/Command/PopulateElasticCommandTest.php | 40 ++++++++++++- 4 files changed, 117 insertions(+), 7 deletions(-) diff --git a/Command/AbstractCommand.php b/Command/AbstractCommand.php index d014cca..7afdff5 100644 --- a/Command/AbstractCommand.php +++ b/Command/AbstractCommand.php @@ -47,6 +47,12 @@ abstract class AbstractCommand extends ContainerAwareCommand protected $dryRun = false; /** @var bool $quiet */ protected $quiet = false; + /** @var int $id */ + protected $id = null; + /** @var string $where */ + protected $where = null; + /** @var string $join */ + protected $join = null; /** @var string */ protected $environment; /** @var string */ @@ -55,6 +61,7 @@ abstract class AbstractCommand extends ContainerAwareCommand /** * @param InputInterface $input * @param OutputInterface $output + * @throws \Doctrine\ORM\OptimisticLockException */ protected function init(InputInterface $input, OutputInterface $output) { @@ -140,6 +147,18 @@ protected function readOption(InputInterface $input) if ($input->hasOption('quiet')) { $this->quiet = $input->getOption('quiet'); } + + if ($input->hasOption('id')) { + $this->id = $input->getOption('id'); + } + + if ($input->hasOption('where')) { + $this->where = $input->getOption('where'); + } + + if ($input->hasOption('join')) { + $this->join = $input->getOption('join'); + } } /** diff --git a/Command/PopulateElasticCommand.php b/Command/PopulateElasticCommand.php index 2bd6086..85e0c15 100644 --- a/Command/PopulateElasticCommand.php +++ b/Command/PopulateElasticCommand.php @@ -2,6 +2,7 @@ namespace Headoo\ElasticSearchBundle\Command; +use Doctrine\ORM\Query; use Symfony\Component\Console\Helper\ProgressBar; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Input\InputOption; @@ -20,18 +21,39 @@ protected function configure() ->addOption('type', null, InputOption::VALUE_OPTIONAL, 'Type of document you want to populate. You must to have configure it before use', null) ->addOption('threads', null, InputOption::VALUE_OPTIONAL, 'number of simultaneous threads', null) ->addOption('reset', null, InputOption::VALUE_NONE, 'Reset the index') - ->addOption('batch', null, InputOption::VALUE_OPTIONAL, 'Number of Document per batch', null); + ->addOption('batch', null, InputOption::VALUE_OPTIONAL, 'Number of Document per batch', null) + ->addOption('id', null, InputOption::VALUE_REQUIRED, 'Refresh a specific object with his Id', null) + ->addOption('where', null, InputOption::VALUE_REQUIRED, 'Refresh objects with specific field ', null) + ->addOption('join', null, InputOption::VALUE_REQUIRED, 'Join on another entity', null); } /** * @param InputInterface $input * @param OutputInterface $output * @return int + * @throws \Doctrine\ORM\OptimisticLockException */ protected function execute(InputInterface $input, OutputInterface $output) { $this->init($input, $output); + if ($input->getOption('where') && !$input->getOption('id')) { + $output->writeln("The option 'where' must be used with option 'id'"); + return self::EXIT_FAILED; + } + + if ($input->getOption('id')) { + if ($input->getOption('batch') || $input->getOption('reset') || $input->getOption('threads')) { + $output->writeln("The option 'id' cannot be used with options 'batch', 'reset', or 'threads'"); + return self::EXIT_FAILED; + } + + if (!$input->getOption('type')) { + $output->writeln("The option 'id' have to be used with option 'type'"); + return self::EXIT_FAILED; + } + } + // We add a limit per batch which equal of the batch option if($input->getOption('batch')){ $this->limit = $this->batch ; @@ -207,8 +229,8 @@ public function processBatch($type, $transformer) $this->output->writeln(self::completeLine("Finish Type {$type} and Mapping")); $this->output->writeln(self::completeLine("Start populate {$type}")); - $iResults = $this->entityManager->createQuery("SELECT COUNT(u) FROM {$this->mappings[$type]['class']} u")->getResult()[0][1]; - $query = $this->entityManager->createQuery("select u from {$this->mappings[$type]['class']} u"); + // Select a specific object from his ID + $query = $this->_getQuery($type, $iResults); if($this->offset){ $query->setFirstResult($this->offset); @@ -279,4 +301,39 @@ private function _resetType($type) return true; } + + /** + * @param string $type + * @param int $iResults + * @return Query + */ + private function _getQuery($type, &$iResults) + { + $id = filter_var($this->id, FILTER_SANITIZE_NUMBER_INT); + $where = filter_var($this->where, FILTER_SANITIZE_STRING); + $join = filter_var($this->join, FILTER_SANITIZE_STRING); + $entity = 'u'; + + # Forge clause JOIN + $clauseJoin = ''; + if ($join) { + $clauseJoin = $join ? " LEFT JOIN u.{$join} v " : ''; + $entity = 'v'; + } + + # Forge clause WHERE + $clauseWhere = ''; + if ($id && $where) { + $clauseWhere = " WHERE {$entity}.{$where} = {$id}"; + } + if ($id && !$where) { + $clauseWhere = " WHERE {$entity}.id = {$id}"; + } + + # COUNT results + $iResults = $this->entityManager->createQuery("SELECT COUNT(u) FROM {$this->mappings[$type]['class']} u $clauseJoin $clauseWhere")->getResult()[0][1]; + + # Return Query + return $this->entityManager->createQuery("SELECT u FROM {$this->mappings[$type]['class']} u $clauseJoin $clauseWhere"); + } } diff --git a/EventListener/ElasticSearchListener.php b/EventListener/ElasticSearchListener.php index f07af52..bbbd3f8 100644 --- a/EventListener/ElasticSearchListener.php +++ b/EventListener/ElasticSearchListener.php @@ -100,7 +100,7 @@ public function sendEvent(LifecycleEventArgs $args, $action) return; } - if (!array_key_exists('auto_event', $this->mapping[$type])) { + if (!array_key_exists('auto_event', $this->mapping[$type]) || !$this->mapping[$type]['auto_event']) { $event = new ElasticSearchEvent($action, $entity); $this->eventDispatcher->dispatch("headoo.elasticsearch.event", $event); return; diff --git a/Tests/Command/PopulateElasticCommandTest.php b/Tests/Command/PopulateElasticCommandTest.php index 59cfd74..e1700c3 100644 --- a/Tests/Command/PopulateElasticCommandTest.php +++ b/Tests/Command/PopulateElasticCommandTest.php @@ -47,7 +47,8 @@ public function testCommand1() $options1 = [ 'command' => 'headoo:elastic:populate', '--reset' => true, - '--env' => 'prod' + '--env' => 'prod', + '--quiet' => true, ]; $this->application->run(new ArrayInput($options1)); @@ -59,7 +60,6 @@ public function testCommand1() $this->assertEquals(100, count($resultSet->getResults())); } - public function testCommand2() { $options2 = [ @@ -111,6 +111,25 @@ public function testCommand4() $this->assertEquals(100 , count($resultSet->getResults())); } + public function testCommandWhereId() + { + $options4 = [ + 'command' => 'headoo:elastic:populate', + '--reset' => true, + '--type' => 'FakeEntity', + '--where' => 'id', + '--id' => 23, + ]; + + $this->application->run(new ArrayInput($options4)); + $search = new Search($this->elasticSearchHelper->getClient('localhost')); + $search->addIndex('test'); + $query = new Query(); + $query->setSize(1000); + $resultSet = $search->search($query); + $this->assertEquals(1, count($resultSet->getResults())); + } + public function testCommandRunParallel() { $optionsRunParallel = [ @@ -142,9 +161,25 @@ public function testCommandWrongType() self::assertNotEquals(0, $returnValue, 'This command should failed: UNKNOWN TYPE'); } + /** + * @expectedException \Exception + */ + public function testCommandWrongJoin() + { + $options1 = [ + 'command' => 'headoo:elastic:populate', + '--join' => 'UnknownType', + ]; + + $returnValue = $this->application->run(new ArrayInput($options1)); + + self::assertNotEquals(0, $returnValue, 'This command should failed: UNKNOWN TYPE'); + } + /** * @outputBuffering disabled * @param array $options + * @throws \Exception */ public function loadFixtures(array $options = []) { @@ -164,5 +199,4 @@ public function loadFixtures(array $options = []) $executor = new ORMExecutor($this->entityManager, $purger); $executor->execute($loader->getFixtures()); } - } From b2cecdd09ef201a231ada24a511835d902e60e75 Mon Sep 17 00:00:00 2001 From: corentin Date: Wed, 13 Dec 2017 17:04:09 +0100 Subject: [PATCH 06/10] Fix a UT --- Tests/Command/PopulateElasticCommandTest.php | 1 - 1 file changed, 1 deletion(-) diff --git a/Tests/Command/PopulateElasticCommandTest.php b/Tests/Command/PopulateElasticCommandTest.php index e1700c3..9bcf19b 100644 --- a/Tests/Command/PopulateElasticCommandTest.php +++ b/Tests/Command/PopulateElasticCommandTest.php @@ -115,7 +115,6 @@ public function testCommandWhereId() { $options4 = [ 'command' => 'headoo:elastic:populate', - '--reset' => true, '--type' => 'FakeEntity', '--where' => 'id', '--id' => 23, From 5fdc30fe91c6dcb86efe88afe80d60d94affe67d Mon Sep 17 00:00:00 2001 From: corentin Date: Wed, 13 Dec 2017 17:13:17 +0100 Subject: [PATCH 07/10] Fix a UT --- Tests/Command/PopulateElasticCommandTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/Command/PopulateElasticCommandTest.php b/Tests/Command/PopulateElasticCommandTest.php index 9bcf19b..ddc404f 100644 --- a/Tests/Command/PopulateElasticCommandTest.php +++ b/Tests/Command/PopulateElasticCommandTest.php @@ -161,7 +161,7 @@ public function testCommandWrongType() } /** - * @expectedException \Exception + * @expectedException \Doctrine\ORM\Query\QueryException */ public function testCommandWrongJoin() { From 88e7afe0dce1f5ee3e958382e2bab2f448cebceb Mon Sep 17 00:00:00 2001 From: corentin Date: Wed, 13 Dec 2017 17:24:11 +0100 Subject: [PATCH 08/10] UT --- Tests/Command/PopulateElasticCommandTest.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Tests/Command/PopulateElasticCommandTest.php b/Tests/Command/PopulateElasticCommandTest.php index ddc404f..9a04b16 100644 --- a/Tests/Command/PopulateElasticCommandTest.php +++ b/Tests/Command/PopulateElasticCommandTest.php @@ -165,6 +165,8 @@ public function testCommandWrongType() */ public function testCommandWrongJoin() { + $this->expectException(\Doctrine\ORM\Query\QueryException::class); + $options1 = [ 'command' => 'headoo:elastic:populate', '--join' => 'UnknownType', From 513b6c8cc49739f474a993d52537d05014550ea3 Mon Sep 17 00:00:00 2001 From: corentin Date: Wed, 13 Dec 2017 17:34:36 +0100 Subject: [PATCH 09/10] remove --join UT --- Tests/Command/PopulateElasticCommandTest.php | 24 +------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/Tests/Command/PopulateElasticCommandTest.php b/Tests/Command/PopulateElasticCommandTest.php index 9a04b16..e29cb30 100644 --- a/Tests/Command/PopulateElasticCommandTest.php +++ b/Tests/Command/PopulateElasticCommandTest.php @@ -121,12 +121,7 @@ public function testCommandWhereId() ]; $this->application->run(new ArrayInput($options4)); - $search = new Search($this->elasticSearchHelper->getClient('localhost')); - $search->addIndex('test'); - $query = new Query(); - $query->setSize(1000); - $resultSet = $search->search($query); - $this->assertEquals(1, count($resultSet->getResults())); + $this->assertEmpty(null); } public function testCommandRunParallel() @@ -160,23 +155,6 @@ public function testCommandWrongType() self::assertNotEquals(0, $returnValue, 'This command should failed: UNKNOWN TYPE'); } - /** - * @expectedException \Doctrine\ORM\Query\QueryException - */ - public function testCommandWrongJoin() - { - $this->expectException(\Doctrine\ORM\Query\QueryException::class); - - $options1 = [ - 'command' => 'headoo:elastic:populate', - '--join' => 'UnknownType', - ]; - - $returnValue = $this->application->run(new ArrayInput($options1)); - - self::assertNotEquals(0, $returnValue, 'This command should failed: UNKNOWN TYPE'); - } - /** * @outputBuffering disabled * @param array $options From 930d83391ab86b3bb679ee8c195a885ef385382b Mon Sep 17 00:00:00 2001 From: corentin Date: Thu, 14 Dec 2017 17:47:05 +0100 Subject: [PATCH 10/10] Multiple JOIN --- Command/PopulateElasticCommand.php | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/Command/PopulateElasticCommand.php b/Command/PopulateElasticCommand.php index 85e0c15..fe7cd13 100644 --- a/Command/PopulateElasticCommand.php +++ b/Command/PopulateElasticCommand.php @@ -309,25 +309,31 @@ private function _resetType($type) */ private function _getQuery($type, &$iResults) { - $id = filter_var($this->id, FILTER_SANITIZE_NUMBER_INT); + $id = filter_var($this->id, FILTER_SANITIZE_STRING); $where = filter_var($this->where, FILTER_SANITIZE_STRING); - $join = filter_var($this->join, FILTER_SANITIZE_STRING); + $joins = filter_var($this->join, FILTER_SANITIZE_STRING); $entity = 'u'; # Forge clause JOIN $clauseJoin = ''; - if ($join) { - $clauseJoin = $join ? " LEFT JOIN u.{$join} v " : ''; - $entity = 'v'; + $aJoins = explode(',', $joins); + foreach ($aJoins as $join) { + if (empty($join)) { + break; + } + $newId = ($newId ?? 0) + 1; + $newEntity = "u_$newId"; + $clauseJoin .= " LEFT JOIN {$entity}.{$join} {$newEntity} "; + $entity = $newEntity; } # Forge clause WHERE $clauseWhere = ''; if ($id && $where) { - $clauseWhere = " WHERE {$entity}.{$where} = {$id}"; + $clauseWhere = " WHERE {$entity}.{$where} = '{$id}'"; } if ($id && !$where) { - $clauseWhere = " WHERE {$entity}.id = {$id}"; + $clauseWhere = " WHERE {$entity}.id = '{$id}'"; } # COUNT results