From e38b369149dc92c6db0a36963c562c2976088540 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Wed, 12 Jul 2023 10:26:22 +0200 Subject: [PATCH 1/6] [cron-job] add a new "lastExecution" data on CronJobExecution entity This column will store the results of the last execution --- .../Entity/CronJobExecution.php | 18 +++++++++- .../migrations/Version20230711152947.php | 33 +++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 src/Bundle/ChillMainBundle/migrations/Version20230711152947.php diff --git a/src/Bundle/ChillMainBundle/Entity/CronJobExecution.php b/src/Bundle/ChillMainBundle/Entity/CronJobExecution.php index 0cacffac9..2883055fc 100644 --- a/src/Bundle/ChillMainBundle/Entity/CronJobExecution.php +++ b/src/Bundle/ChillMainBundle/Entity/CronJobExecution.php @@ -31,7 +31,6 @@ class CronJobExecution private string $key; /** - * @var DateTimeImmutable * @ORM\Column(type="datetime_immutable", nullable=true, options={"default": null}) */ private ?DateTimeImmutable $lastEnd = null; @@ -46,6 +45,11 @@ class CronJobExecution */ private ?int $lastStatus = null; + /** + * @ORM\Column(type="json", options={"default": "'{}'::jsonb", "jsonb": true}) + */ + private array $lastExecutionData = []; + public function __construct(string $key) { $this->key = $key; @@ -92,4 +96,16 @@ class CronJobExecution return $this; } + + public function getLastExecutionData(): array + { + return $this->lastExecutionData; + } + + public function setLastExecutionData(array $lastExecutionData): CronJobExecution + { + $this->lastExecutionData = $lastExecutionData; + + return $this; + } } diff --git a/src/Bundle/ChillMainBundle/migrations/Version20230711152947.php b/src/Bundle/ChillMainBundle/migrations/Version20230711152947.php new file mode 100644 index 000000000..ed804473e --- /dev/null +++ b/src/Bundle/ChillMainBundle/migrations/Version20230711152947.php @@ -0,0 +1,33 @@ +addSql('ALTER TABLE chill_main_cronjob_execution ADD lastExecutionData JSONB DEFAULT \'{}\'::jsonb NOT NULL'); + } + + public function down(Schema $schema): void + { + $this->addSql('ALTER TABLE chill_main_cronjob_execution DROP COLUMN lastExecutionData'); + } +} From 3f66e1a862416658a3afce675dbc1cb29440fb24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Wed, 12 Jul 2023 11:36:26 +0200 Subject: [PATCH 2/6] [cron-job] allow a cronjob to pass data from one execution to another When a cronjob is executed, it may return an array of data. This data will be passed as parameter on the next execution --- .changes/unreleased/DX-20230712-113603.yaml | 6 ++ .../ChillMainBundle/Cron/CronJobInterface.php | 10 ++- .../ChillMainBundle/Cron/CronManager.php | 25 +++++- ...eographicalUnitMaterializedViewCronJob.php | 4 +- .../Cron/CronJobDatabaseInteractionTest.php | 87 +++++++++++++++++++ .../Tests/Cron/CronManagerTest.php | 12 +-- .../AccompanyingPeriodStepChangeCronjob.php | 4 +- 7 files changed, 137 insertions(+), 11 deletions(-) create mode 100644 .changes/unreleased/DX-20230712-113603.yaml create mode 100644 src/Bundle/ChillMainBundle/Tests/Cron/CronJobDatabaseInteractionTest.php diff --git a/.changes/unreleased/DX-20230712-113603.yaml b/.changes/unreleased/DX-20230712-113603.yaml new file mode 100644 index 000000000..518ac3ca9 --- /dev/null +++ b/.changes/unreleased/DX-20230712-113603.yaml @@ -0,0 +1,6 @@ +kind: DX +body: '[cronjob] when a cronjob is executed, it may return an array of data that will + be passed as argument on the next execution' +time: 2023-07-12T11:36:03.813179067+02:00 +custom: + Issue: "" diff --git a/src/Bundle/ChillMainBundle/Cron/CronJobInterface.php b/src/Bundle/ChillMainBundle/Cron/CronJobInterface.php index 4e1ca9ff6..69edf8464 100644 --- a/src/Bundle/ChillMainBundle/Cron/CronJobInterface.php +++ b/src/Bundle/ChillMainBundle/Cron/CronJobInterface.php @@ -19,5 +19,13 @@ interface CronJobInterface public function getKey(): string; - public function run(): void; + /** + * Execute the cronjob + * + * If data is returned, this data is passed as argument on the next execution + * + * @param array $lastExecutionData the data which was returned from the previous execution + * @return array|null optionally return an array with the same data than the previous execution + */ + public function run(array $lastExecutionData): null|array; } diff --git a/src/Bundle/ChillMainBundle/Cron/CronManager.php b/src/Bundle/ChillMainBundle/Cron/CronManager.php index f69dcba76..a3e82a170 100644 --- a/src/Bundle/ChillMainBundle/Cron/CronManager.php +++ b/src/Bundle/ChillMainBundle/Cron/CronManager.php @@ -14,6 +14,7 @@ namespace Chill\MainBundle\Cron; use Chill\MainBundle\Entity\CronJobExecution; use Chill\MainBundle\Repository\CronJobExecutionRepositoryInterface; use DateTimeImmutable; +use Doctrine\DBAL\Types\Types; use Doctrine\ORM\EntityManagerInterface; use Exception; use Psr\Log\LoggerInterface; @@ -46,6 +47,8 @@ class CronManager implements CronManagerInterface private const UPDATE_BEFORE_EXEC = 'UPDATE ' . CronJobExecution::class . ' cr SET cr.lastStart = :now WHERE cr.key = :key'; + private const UPDATE_LAST_EXECUTION_DATA = 'UPDATE ' . CronJobExecution::class . ' cr SET cr.lastExecutionData = :data WHERE cr.key = :key'; + private CronJobExecutionRepositoryInterface $cronJobExecutionRepository; private EntityManagerInterface $entityManager; @@ -85,6 +88,9 @@ class CronManager implements CronManagerInterface foreach ($orderedJobs as $job) { if ($job->canRun($lasts[$job->getKey()] ?? null)) { if (array_key_exists($job->getKey(), $lasts)) { + + $executionData = $lasts[$job->getKey()]->getLastExecutionData(); + $this->entityManager ->createQuery(self::UPDATE_BEFORE_EXEC) ->setParameters([ @@ -96,12 +102,17 @@ class CronManager implements CronManagerInterface $execution = new CronJobExecution($job->getKey()); $this->entityManager->persist($execution); $this->entityManager->flush(); + + $executionData = $execution->getLastExecutionData(); } $this->entityManager->clear(); + // note: at this step, the entity manager does not have any entity CronJobExecution + // into his internal memory + try { $this->logger->info(sprintf('%sWill run job', self::LOG_PREFIX), ['job' => $job->getKey()]); - $job->run(); + $result = $job->run($executionData); $this->entityManager ->createQuery(self::UPDATE_AFTER_EXEC) @@ -112,6 +123,14 @@ class CronManager implements CronManagerInterface ]) ->execute(); + if (null !== $result) { + $this->entityManager + ->createQuery(self::UPDATE_LAST_EXECUTION_DATA) + ->setParameter('data', $result, Types::JSON) + ->setParameter('key', $job->getKey(), Types::STRING) + ->execute(); + } + $this->logger->info(sprintf('%sSuccessfully run job', self::LOG_PREFIX), ['job' => $job->getKey()]); return; @@ -133,7 +152,7 @@ class CronManager implements CronManagerInterface } /** - * @return array<0: CronJobInterface[], 1: array> + * @return array{0: array, 1: array} */ private function getOrderedJobs(): array { @@ -174,7 +193,7 @@ class CronManager implements CronManagerInterface { foreach ($this->jobs as $job) { if ($job->getKey() === $forceJob) { - $job->run(); + $job->run([]); } } } diff --git a/src/Bundle/ChillMainBundle/Service/AddressGeographicalUnit/RefreshAddressToGeographicalUnitMaterializedViewCronJob.php b/src/Bundle/ChillMainBundle/Service/AddressGeographicalUnit/RefreshAddressToGeographicalUnitMaterializedViewCronJob.php index 9dbb38a3f..3a6ff8fb9 100644 --- a/src/Bundle/ChillMainBundle/Service/AddressGeographicalUnit/RefreshAddressToGeographicalUnitMaterializedViewCronJob.php +++ b/src/Bundle/ChillMainBundle/Service/AddressGeographicalUnit/RefreshAddressToGeographicalUnitMaterializedViewCronJob.php @@ -49,8 +49,10 @@ class RefreshAddressToGeographicalUnitMaterializedViewCronJob implements CronJob return 'refresh-materialized-view-address-to-geog-units'; } - public function run(): void + public function run(array $lastExecutionData): null|array { $this->connection->executeQuery('REFRESH MATERIALIZED VIEW view_chill_main_address_geographical_unit'); + + return null; } } diff --git a/src/Bundle/ChillMainBundle/Tests/Cron/CronJobDatabaseInteractionTest.php b/src/Bundle/ChillMainBundle/Tests/Cron/CronJobDatabaseInteractionTest.php new file mode 100644 index 000000000..0b80730fe --- /dev/null +++ b/src/Bundle/ChillMainBundle/Tests/Cron/CronJobDatabaseInteractionTest.php @@ -0,0 +1,87 @@ +entityManager = self::$container->get(EntityManagerInterface::class); + $this->cronJobExecutionRepository = self::$container->get(CronJobExecutionRepository::class); + } + + public function testCompleteLifeCycle(): void + { + $cronjob = $this->prophesize(CronJobInterface::class); + $cronjob->canRun(null)->willReturn(true); + $cronjob->canRun(Argument::type(CronJobExecution::class))->willReturn(true); + $cronjob->getKey()->willReturn('test-with-data'); + $cronjob->run([])->willReturn(['test' => 'execution-0']); + $cronjob->run(['test' => 'execution-0'])->willReturn(['test' => 'execution-1']); + + $cronjob->run([])->shouldBeCalledOnce(); + $cronjob->run(['test' => 'execution-0'])->shouldBeCalledOnce(); + + $manager = new CronManager( + $this->cronJobExecutionRepository, + $this->entityManager, + [$cronjob->reveal()], + new NullLogger() + ); + + // run a first time + $manager->run(); + + // run a second time + $manager->run(); + } + +} + +class JobWithReturn implements CronJobInterface +{ + public function canRun(?CronJobExecution $cronJobExecution): bool + { + return true; + } + + public function getKey(): string + { + return 'with-data'; + } + + public function run(array $lastExecutionData): null|array + { + return ['data' => 'test']; + } +} diff --git a/src/Bundle/ChillMainBundle/Tests/Cron/CronManagerTest.php b/src/Bundle/ChillMainBundle/Tests/Cron/CronManagerTest.php index 4b812ce2b..47c929a52 100644 --- a/src/Bundle/ChillMainBundle/Tests/Cron/CronManagerTest.php +++ b/src/Bundle/ChillMainBundle/Tests/Cron/CronManagerTest.php @@ -40,7 +40,7 @@ final class CronManagerTest extends TestCase $jobToExecute = $this->prophesize(CronJobInterface::class); $jobToExecute->getKey()->willReturn('to-exec'); $jobToExecute->canRun(Argument::type(CronJobExecution::class))->willReturn(true); - $jobToExecute->run()->shouldBeCalled(); + $jobToExecute->run([])->shouldBeCalled(); $executions = [ ['key' => $jobOld1->getKey(), 'lastStart' => new DateTimeImmutable('yesterday'), 'lastEnd' => new DateTimeImmutable('1 hours ago'), 'lastStatus' => CronJobExecution::SUCCESS], @@ -64,7 +64,7 @@ final class CronManagerTest extends TestCase $jobAlreadyExecuted = new JobCanRun('k'); $jobNeverExecuted = $this->prophesize(CronJobInterface::class); $jobNeverExecuted->getKey()->willReturn('never-executed'); - $jobNeverExecuted->run()->shouldBeCalled(); + $jobNeverExecuted->run([])->shouldBeCalled(); $jobNeverExecuted->canRun(null)->willReturn(true); $executions = [ @@ -86,7 +86,7 @@ final class CronManagerTest extends TestCase $jobAlreadyExecuted = new JobCanRun('k'); $jobNeverExecuted = $this->prophesize(CronJobInterface::class); $jobNeverExecuted->getKey()->willReturn('never-executed'); - $jobNeverExecuted->run()->shouldBeCalled(); + $jobNeverExecuted->run([])->shouldBeCalled(); $jobNeverExecuted->canRun(null)->willReturn(true); $executions = [ @@ -178,8 +178,9 @@ class JobCanRun implements CronJobInterface return $this->key; } - public function run(): void + public function run(array $lastExecutionData): null|array { + return null; } } @@ -195,7 +196,8 @@ class JobCannotRun implements CronJobInterface return 'job-b'; } - public function run(): void + public function run(array $lastExecutionData): null|array { + return null; } } diff --git a/src/Bundle/ChillPersonBundle/AccompanyingPeriod/Lifecycle/AccompanyingPeriodStepChangeCronjob.php b/src/Bundle/ChillPersonBundle/AccompanyingPeriod/Lifecycle/AccompanyingPeriodStepChangeCronjob.php index f637e70b9..2ddf3415c 100644 --- a/src/Bundle/ChillPersonBundle/AccompanyingPeriod/Lifecycle/AccompanyingPeriodStepChangeCronjob.php +++ b/src/Bundle/ChillPersonBundle/AccompanyingPeriod/Lifecycle/AccompanyingPeriodStepChangeCronjob.php @@ -39,8 +39,10 @@ readonly class AccompanyingPeriodStepChangeCronjob implements CronJobInterface return 'accompanying-period-step-change'; } - public function run(): void + public function run(array $lastExecutionData): null|array { ($this->requestor)(); + + return null; } } From 1552b3c9d7c426950e356e8592505f24976d8551 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Wed, 12 Jul 2023 17:30:46 +0200 Subject: [PATCH 3/6] [Addresses] create a service to collate addresses with the address reference --- ...ollateAddressWithReferenceOrPostalCode.php | 147 ++++++++++++++++++ ...teAddressWithReferenceOrPostalCodeTest.php | 44 ++++++ 2 files changed, 191 insertions(+) create mode 100644 src/Bundle/ChillMainBundle/Service/AddressGeographicalUnit/CollateAddressWithReferenceOrPostalCode.php create mode 100644 src/Bundle/ChillMainBundle/Tests/Services/AddressGeographicalUnit/CollateAddressWithReferenceOrPostalCodeTest.php diff --git a/src/Bundle/ChillMainBundle/Service/AddressGeographicalUnit/CollateAddressWithReferenceOrPostalCode.php b/src/Bundle/ChillMainBundle/Service/AddressGeographicalUnit/CollateAddressWithReferenceOrPostalCode.php new file mode 100644 index 000000000..656751eeb --- /dev/null +++ b/src/Bundle/ChillMainBundle/Service/AddressGeographicalUnit/CollateAddressWithReferenceOrPostalCode.php @@ -0,0 +1,147 @@ + :since_id -- to set the first id + ) sq + WHERE ranked = 1) + UPDATE chill_main_address SET postcode_id = cmpc_reference_id FROM recollate WHERE recollate.address_id = chill_main_address.id; + SQL; + + /** + * associate the address with the most similar address reference. + * + * This query intentionally ignores the existing addressreference_id, to let fixing the address match the + * most similar address reference. + */ + private const FORCE_MOST_SIMILAR_ADDRESS_REFERENCE = <<<'SQL' + WITH recollate AS ( + SELECT * FROM ( + SELECT cma.id AS address_id, cma.streetnumber, cma.street, cmpc.code, cmpc.label, cmar.id AS address_reference_id, cmar.streetnumber, cmar.street, cmpc_reference.code, cmpc_reference.label, + similarity(cma.street, cmar.street), + RANK() OVER (PARTITION BY cma.id ORDER BY SIMILARITY (cma.street, cmar.street) DESC, SIMILARITY (cma.streetnumber, cmar.streetnumber), cmar.id ASC) AS ranked + FROM + chill_main_address cma + JOIN chill_main_postal_code cmpc on cma.postcode_id = cmpc.id, + chill_main_address_reference cmar JOIN chill_main_postal_code cmpc_reference ON cmar.postcode_id = cmpc_reference.id + WHERE + -- only where the reference is null + -- cma.addressreference_id IS NULL + cma.addressreference_id != cmar.id + -- only if cmpc is a reference (must be matched before executing this query) + AND cma.postcode_id = cmar.postcode_id + -- join cmpc to cma + AND SIMILARITY(LOWER(cma.street), LOWER(cmar.street)) > 0.6 AND LOWER(cma.streetnumber) = LOWER(cmar.streetnumber) + -- only addresses which match the address reference - let the user decide if the reference has changed + AND cma.refstatus = 'match' + -- only the most recent + AND cma.id > :since_id + ) AS sq + WHERE ranked = 1 + ) + UPDATE chill_main_address SET addressreference_id = recollate.address_reference_id FROM recollate WHERE chill_main_address.id = recollate.address_id; + SQL; + + private const UPDATE_POINT = <<<'SQL' + WITH address_geom AS ( + SELECT cma.id AS address_id, COALESCE(cmar.point, cmpc.center) AS point + FROM chill_main_address cma + LEFT JOIN chill_main_address_reference cmar ON cma.addressreference_id = cmar.id + LEFT JOIN chill_main_postal_code cmpc ON cma.postcode_id = cmpc.id + WHERE cma.id > :since_id + ) + UPDATE chill_main_address SET point = address_geom.point FROM address_geom WHERE address_geom.address_id = chill_main_address.id + SQL; + + private const MAX_ADDRESS_ID = <<<'SQL' + SELECT MAX(id) AS max_id FROM chill_main_address; + SQL; + + + public function __construct( + private Connection $connection, + private LoggerInterface $logger, + ) { + } + + /** + * @throws \Throwable + */ + public function __invoke(int $sinceId = 0): int + { + try { + [ + $postCodeSetReferenceFromMostSimilar, + $addressReferenceMatch, + $pointUpdates, + $lastId, + ] = $this->connection->transactional(function () use ($sinceId) { + $postCodeSetReferenceFromMostSimilar = $this->connection->executeQuery(self::FORCE_ORIGINAL_POSTAL_CODE, ['since_id' => $sinceId]); + $addressReferenceMatch = $this->connection->executeQuery(self::FORCE_MOST_SIMILAR_ADDRESS_REFERENCE, ['since_id' => $sinceId]); + $pointUpdates = $this->connection->executeQuery(self::UPDATE_POINT, ['since_id' => $sinceId]); + $lastId = $this->connection->fetchOne(self::MAX_ADDRESS_ID); + + return [ + $postCodeSetReferenceFromMostSimilar, + $addressReferenceMatch, + $pointUpdates, + $lastId, + ]; + }); + } catch (\Throwable $e) { + $this->logger->error(self::LOG_PREFIX . "error while re-collating addresses", [ + 'message' => $e->getMessage(), + 'trace' => $e->getTraceAsString() + ]); + + throw $e; + } + + $this->logger->info(self::LOG_PREFIX . "Collate the addresses with reference", [ + 'set_postcode_from_most_similar' => $postCodeSetReferenceFromMostSimilar, + 'address_reference_match' => $addressReferenceMatch, + 'point_update' => $pointUpdates, + 'since_id' => $sinceId, + 'last_id' => $lastId, + ]); + + return $lastId; + } +} diff --git a/src/Bundle/ChillMainBundle/Tests/Services/AddressGeographicalUnit/CollateAddressWithReferenceOrPostalCodeTest.php b/src/Bundle/ChillMainBundle/Tests/Services/AddressGeographicalUnit/CollateAddressWithReferenceOrPostalCodeTest.php new file mode 100644 index 000000000..61b36e669 --- /dev/null +++ b/src/Bundle/ChillMainBundle/Tests/Services/AddressGeographicalUnit/CollateAddressWithReferenceOrPostalCodeTest.php @@ -0,0 +1,44 @@ +connection = self::$container->get(Connection::class); + } + + public function testRun(): void + { + $collator = new CollateAddressWithReferenceOrPostalCode( + $this->connection, + new NullLogger() + ); + + $result = $collator(0); + + self::assertGreaterThan(0, $result); + } +} From a7842b2597349413dff4b751a037f1db221f1c03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Wed, 12 Jul 2023 18:00:29 +0200 Subject: [PATCH 4/6] [Addresses] add a cronjob to collate addresses with reference --- .../unreleased/Feature-20230712-180023.yaml | 6 ++ ...ollateAddressWithReferenceOrPostalCode.php | 10 ++- ...ddressWithReferenceOrPostalCodeCronJob.php | 46 +++++++++++++ ...ressWithReferenceOrPostalCodeInterface.php | 20 ++++++ ...ssWithReferenceOrPostalCodeCronJobTest.php | 65 +++++++++++++++++++ 5 files changed, 141 insertions(+), 6 deletions(-) create mode 100644 .changes/unreleased/Feature-20230712-180023.yaml create mode 100644 src/Bundle/ChillMainBundle/Service/AddressGeographicalUnit/CollateAddressWithReferenceOrPostalCodeCronJob.php create mode 100644 src/Bundle/ChillMainBundle/Service/AddressGeographicalUnit/CollateAddressWithReferenceOrPostalCodeInterface.php create mode 100644 src/Bundle/ChillMainBundle/Tests/Services/AddressGeographicalUnit/CollateAddressWithReferenceOrPostalCodeCronJobTest.php diff --git a/.changes/unreleased/Feature-20230712-180023.yaml b/.changes/unreleased/Feature-20230712-180023.yaml new file mode 100644 index 000000000..4610dcdbc --- /dev/null +++ b/.changes/unreleased/Feature-20230712-180023.yaml @@ -0,0 +1,6 @@ +kind: Feature +body: | + [addresses] Add a cronjob to re-associate addresses with addresses reference every 6 hours +time: 2023-07-12T18:00:23.037677413+02:00 +custom: + Issue: "112" diff --git a/src/Bundle/ChillMainBundle/Service/AddressGeographicalUnit/CollateAddressWithReferenceOrPostalCode.php b/src/Bundle/ChillMainBundle/Service/AddressGeographicalUnit/CollateAddressWithReferenceOrPostalCode.php index 656751eeb..2a5dda3f7 100644 --- a/src/Bundle/ChillMainBundle/Service/AddressGeographicalUnit/CollateAddressWithReferenceOrPostalCode.php +++ b/src/Bundle/ChillMainBundle/Service/AddressGeographicalUnit/CollateAddressWithReferenceOrPostalCode.php @@ -14,7 +14,7 @@ namespace Chill\MainBundle\Service\AddressGeographicalUnit; use Doctrine\DBAL\Connection; use Psr\Log\LoggerInterface; -final readonly class CollateAddressWithReferenceOrPostalCode +final readonly class CollateAddressWithReferenceOrPostalCode implements CollateAddressWithReferenceOrPostalCodeInterface { private const LOG_PREFIX = '[collate addresses] '; /** @@ -62,8 +62,6 @@ final readonly class CollateAddressWithReferenceOrPostalCode JOIN chill_main_postal_code cmpc on cma.postcode_id = cmpc.id, chill_main_address_reference cmar JOIN chill_main_postal_code cmpc_reference ON cmar.postcode_id = cmpc_reference.id WHERE - -- only where the reference is null - -- cma.addressreference_id IS NULL cma.addressreference_id != cmar.id -- only if cmpc is a reference (must be matched before executing this query) AND cma.postcode_id = cmar.postcode_id @@ -113,9 +111,9 @@ final readonly class CollateAddressWithReferenceOrPostalCode $pointUpdates, $lastId, ] = $this->connection->transactional(function () use ($sinceId) { - $postCodeSetReferenceFromMostSimilar = $this->connection->executeQuery(self::FORCE_ORIGINAL_POSTAL_CODE, ['since_id' => $sinceId]); - $addressReferenceMatch = $this->connection->executeQuery(self::FORCE_MOST_SIMILAR_ADDRESS_REFERENCE, ['since_id' => $sinceId]); - $pointUpdates = $this->connection->executeQuery(self::UPDATE_POINT, ['since_id' => $sinceId]); + $postCodeSetReferenceFromMostSimilar = $this->connection->executeStatement(self::FORCE_ORIGINAL_POSTAL_CODE, ['since_id' => $sinceId]); + $addressReferenceMatch = $this->connection->executeStatement(self::FORCE_MOST_SIMILAR_ADDRESS_REFERENCE, ['since_id' => $sinceId]); + $pointUpdates = $this->connection->executeStatement(self::UPDATE_POINT, ['since_id' => $sinceId]); $lastId = $this->connection->fetchOne(self::MAX_ADDRESS_ID); return [ diff --git a/src/Bundle/ChillMainBundle/Service/AddressGeographicalUnit/CollateAddressWithReferenceOrPostalCodeCronJob.php b/src/Bundle/ChillMainBundle/Service/AddressGeographicalUnit/CollateAddressWithReferenceOrPostalCodeCronJob.php new file mode 100644 index 000000000..b35a687dd --- /dev/null +++ b/src/Bundle/ChillMainBundle/Service/AddressGeographicalUnit/CollateAddressWithReferenceOrPostalCodeCronJob.php @@ -0,0 +1,46 @@ +clock->now(); + + return $now->sub(new \DateInterval('PT6H')) > $cronJobExecution->getLastStart(); + } + + public function getKey(): string + { + return 'collate-address'; + } + + public function run(array $lastExecutionData): null|array + { + $maxId = ($this->collateAddressWithReferenceOrPostalCode)($lastExecutionData[self::LAST_MAX_ID] ?? 0); + + return [self::LAST_MAX_ID => $maxId]; + } +} diff --git a/src/Bundle/ChillMainBundle/Service/AddressGeographicalUnit/CollateAddressWithReferenceOrPostalCodeInterface.php b/src/Bundle/ChillMainBundle/Service/AddressGeographicalUnit/CollateAddressWithReferenceOrPostalCodeInterface.php new file mode 100644 index 000000000..cd3f2606f --- /dev/null +++ b/src/Bundle/ChillMainBundle/Service/AddressGeographicalUnit/CollateAddressWithReferenceOrPostalCodeInterface.php @@ -0,0 +1,20 @@ +setLastStart($lastExecution); + + $clock = new MockClock($now); + $collator = $this->prophesize(CollateAddressWithReferenceOrPostalCodeInterface::class); + + $job = new CollateAddressWithReferenceOrPostalCodeCronJob($clock, $collator->reveal()); + + self::assertEquals($expected, $job->canRun($execution)); + } + + public function testRun(): void + { + $clock = new MockClock(); + $collator = $this->prophesize(CollateAddressWithReferenceOrPostalCodeInterface::class); + $collator->__invoke(0)->shouldBeCalledOnce(); + $collator->__invoke(0)->willReturn(1); + + $job = new CollateAddressWithReferenceOrPostalCodeCronJob($clock, $collator->reveal()); + + $actual = $job->run(['last-max-id' => 0]); + self::assertEquals(['last-max-id' => 1], $actual); + } + + public static function provideDataCanRun(): iterable + { + yield [new \DateTimeImmutable('2023-07-10T12:00:00'), new \DateTimeImmutable('2023-07-10T11:00:00'), false]; + yield [new \DateTimeImmutable('2023-07-10T12:00:00'), new \DateTimeImmutable('2023-07-10T05:00:00'), true]; + yield [new \DateTimeImmutable('2023-07-10T12:00:00'), new \DateTimeImmutable('2023-07-01T12:00:00'), true]; + } + +} From b0fcffea2d1b0a1b5dc3d358e11ec355583f8301 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Thu, 13 Jul 2023 09:03:41 +0200 Subject: [PATCH 5/6] fixed: take into account the first iteration of the cronjob --- .../CollateAddressWithReferenceOrPostalCodeCronJob.php | 4 ++++ ...ollateAddressWithReferenceOrPostalCodeCronJobTest.php | 9 ++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/Bundle/ChillMainBundle/Service/AddressGeographicalUnit/CollateAddressWithReferenceOrPostalCodeCronJob.php b/src/Bundle/ChillMainBundle/Service/AddressGeographicalUnit/CollateAddressWithReferenceOrPostalCodeCronJob.php index b35a687dd..d2c9fc960 100644 --- a/src/Bundle/ChillMainBundle/Service/AddressGeographicalUnit/CollateAddressWithReferenceOrPostalCodeCronJob.php +++ b/src/Bundle/ChillMainBundle/Service/AddressGeographicalUnit/CollateAddressWithReferenceOrPostalCodeCronJob.php @@ -27,6 +27,10 @@ final readonly class CollateAddressWithReferenceOrPostalCodeCronJob implements C public function canRun(?CronJobExecution $cronJobExecution): bool { + if (null === $cronJobExecution) { + return true; + } + $now = $this->clock->now(); return $now->sub(new \DateInterval('PT6H')) > $cronJobExecution->getLastStart(); diff --git a/src/Bundle/ChillMainBundle/Tests/Services/AddressGeographicalUnit/CollateAddressWithReferenceOrPostalCodeCronJobTest.php b/src/Bundle/ChillMainBundle/Tests/Services/AddressGeographicalUnit/CollateAddressWithReferenceOrPostalCodeCronJobTest.php index f8b4a547e..69eeca2da 100644 --- a/src/Bundle/ChillMainBundle/Tests/Services/AddressGeographicalUnit/CollateAddressWithReferenceOrPostalCodeCronJobTest.php +++ b/src/Bundle/ChillMainBundle/Tests/Services/AddressGeographicalUnit/CollateAddressWithReferenceOrPostalCodeCronJobTest.php @@ -29,10 +29,12 @@ class CollateAddressWithReferenceOrPostalCodeCronJobTest extends TestCase /** * @dataProvider provideDataCanRun */ - public function testCanRun(\DateTimeImmutable $now, \DateTimeImmutable $lastExecution, bool $expected): void + public function testCanRun(\DateTimeImmutable $now, ?\DateTimeImmutable $lastExecution, bool $expected): void { - $execution = new CronJobExecution('collate-address'); - $execution ->setLastStart($lastExecution); + $execution = match ($lastExecution) { + null => null, + default => (new CronJobExecution('collate-address'))->setLastStart($lastExecution), + }; $clock = new MockClock($now); $collator = $this->prophesize(CollateAddressWithReferenceOrPostalCodeInterface::class); @@ -60,6 +62,7 @@ class CollateAddressWithReferenceOrPostalCodeCronJobTest extends TestCase yield [new \DateTimeImmutable('2023-07-10T12:00:00'), new \DateTimeImmutable('2023-07-10T11:00:00'), false]; yield [new \DateTimeImmutable('2023-07-10T12:00:00'), new \DateTimeImmutable('2023-07-10T05:00:00'), true]; yield [new \DateTimeImmutable('2023-07-10T12:00:00'), new \DateTimeImmutable('2023-07-01T12:00:00'), true]; + yield [new \DateTimeImmutable('2023-07-10T12:00:00'), null, true]; } } From 643d8f99be43ae85dc32f851c90de3db6fbd59f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Fri, 14 Jul 2023 10:52:33 +0200 Subject: [PATCH 6/6] collate address: force the country to be the same when comparing postal codes --- .../CollateAddressWithReferenceOrPostalCode.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Bundle/ChillMainBundle/Service/AddressGeographicalUnit/CollateAddressWithReferenceOrPostalCode.php b/src/Bundle/ChillMainBundle/Service/AddressGeographicalUnit/CollateAddressWithReferenceOrPostalCode.php index 2a5dda3f7..17e3f5ce2 100644 --- a/src/Bundle/ChillMainBundle/Service/AddressGeographicalUnit/CollateAddressWithReferenceOrPostalCode.php +++ b/src/Bundle/ChillMainBundle/Service/AddressGeographicalUnit/CollateAddressWithReferenceOrPostalCode.php @@ -34,11 +34,11 @@ final readonly class CollateAddressWithReferenceOrPostalCode implements CollateA WHERE -- use only postal code which are reference cmpc_reference.id != cmpc.id AND cmpc_reference.origin = 0 - -- only where the reference is null or the cmpc is created manually - --AND cma.addressreference_id IS NULL + -- only where cmpc is created manually AND cmpc.origin != 0 -- only when postal code match AND TRIM(REPLACE(LOWER(cmpc.code), ' ', '')) = LOWER(cmpc_reference.code) + AND cmpc.country_id = cmpc_reference.country_id AND cma.id > :since_id -- to set the first id ) sq WHERE ranked = 1)