From 4fcfe3f5d25296a94159370f5969f622b9f77143 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Sun, 11 Dec 2022 01:22:39 +0100 Subject: [PATCH 01/13] Feature: [cron] Create a cron manager --- .../ChillMainBundle/Cron/CronJobInterface.php | 23 ++ .../ChillMainBundle/Cron/CronManager.php | 159 ++++++++++++++ .../Entity/CronJobExecution.php | 92 ++++++++ .../Repository/CronJobExecutionRepository.php | 57 +++++ .../CronJobExecutionRepositoryInterface.php | 34 +++ .../Tests/Cron/CronManagerTest.php | 201 ++++++++++++++++++ 6 files changed, 566 insertions(+) create mode 100644 src/Bundle/ChillMainBundle/Cron/CronJobInterface.php create mode 100644 src/Bundle/ChillMainBundle/Cron/CronManager.php create mode 100644 src/Bundle/ChillMainBundle/Entity/CronJobExecution.php create mode 100644 src/Bundle/ChillMainBundle/Repository/CronJobExecutionRepository.php create mode 100644 src/Bundle/ChillMainBundle/Repository/CronJobExecutionRepositoryInterface.php create mode 100644 src/Bundle/ChillMainBundle/Tests/Cron/CronManagerTest.php diff --git a/src/Bundle/ChillMainBundle/Cron/CronJobInterface.php b/src/Bundle/ChillMainBundle/Cron/CronJobInterface.php new file mode 100644 index 000000000..4e1ca9ff6 --- /dev/null +++ b/src/Bundle/ChillMainBundle/Cron/CronJobInterface.php @@ -0,0 +1,23 @@ + + */ + private iterable $jobs; + + private LoggerInterface $logger; + + /** + * @param CronJobInterface[] $jobs + */ + public function __construct( + CronJobExecutionRepositoryInterface $cronJobExecutionRepository, + EntityManagerInterface $entityManager, + iterable $jobs, + LoggerInterface $logger + ) { + $this->cronJobExecutionRepository = $cronJobExecutionRepository; + $this->entityManager = $entityManager; + $this->jobs = $jobs; + $this->logger = $logger; + } + + public function run(?string $forceJob = null): void + { + if (null !== $forceJob) { + $this->runForce($forceJob); + + return; + } + + [$orderedJobs, $lasts] = $this->getOrderedJobs(); + + foreach ($orderedJobs as $job) { + if ($job->canRun($lasts[$job->getKey()] ?? null)) { + if (array_key_exists($job->getKey(), $lasts)) { + $this->entityManager + ->createQuery(self::UPDATE_BEFORE_EXEC) + ->setParameters([ + 'now' => new DateTimeImmutable('now'), + 'key' => $job->getKey(), + ]); + } else { + $execution = new CronJobExecution($job->getKey()); + $this->entityManager->persist($execution); + $this->entityManager->flush(); + } + $this->entityManager->clear(); + + try { + $this->logger->info(sprintf('%sWill run job', self::LOG_PREFIX), ['job' => $job->getKey()]); + $job->run(); + + $this->entityManager + ->createQuery(self::UPDATE_AFTER_EXEC) + ->setParameters([ + 'now' => new DateTimeImmutable('now'), + 'status' => CronJobExecution::SUCCESS, + 'key' => $job->getKey(), + ]) + ->execute(); + + $this->logger->info(sprintf('%sSuccessfully run job', self::LOG_PREFIX), ['job' => $job->getKey()]); + + return; + } catch (Exception $e) { + $this->logger->error(sprintf('%sRunning job failed', self::LOG_PREFIX), ['job' => $job->getKey()]); + $this->entityManager + ->createQuery(self::UPDATE_AFTER_EXEC) + ->setParameters([ + 'now' => new DateTimeImmutable('now'), + 'status' => CronJobExecution::FAILURE, + 'key' => $job->getKey(), + ]) + ->execute(); + + return; + } + } + } + } + + /** + * @return array<0: CronJobInterface[], 1: array> + */ + private function getOrderedJobs(): array + { + /** @var array $lasts */ + $lasts = []; + + foreach ($this->cronJobExecutionRepository->findAll() as $execution) { + $lasts[$execution->getKey()] = $execution; + } + + // order by last, NULL first + $orderedJobs = iterator_to_array($this->jobs); + usort( + $orderedJobs, + static function (CronJobInterface $a, CronJobInterface $b) use ($lasts): int { + if ( + (!array_key_exists($a->getKey(), $lasts) && !array_key_exists($b->getKey(), $lasts)) + ) { + return 0; + } + + if (!array_key_exists($a->getKey(), $lasts) && array_key_exists($b->getKey(), $lasts)) { + return -1; + } + + if (!array_key_exists($b->getKey(), $lasts) && array_key_exists($a->getKey(), $lasts)) { + return 1; + } + + return $lasts[$a->getKey()]->getLastStart() <=> $lasts[$b->getKey()]->getLastStart(); + } + ); + + return [$orderedJobs, $lasts]; + } + + private function runForce(string $forceJob): void + { + foreach ($this->jobs as $job) { + $job->run(); + } + } +} diff --git a/src/Bundle/ChillMainBundle/Entity/CronJobExecution.php b/src/Bundle/ChillMainBundle/Entity/CronJobExecution.php new file mode 100644 index 000000000..e0899e21f --- /dev/null +++ b/src/Bundle/ChillMainBundle/Entity/CronJobExecution.php @@ -0,0 +1,92 @@ +key = $key; + $this->lastStart = new DateTimeImmutable('now'); + } + + public function getKey(): string + { + return $this->key; + } + + public function getLastEnd(): DateTimeImmutable + { + return $this->lastEnd; + } + + public function getLastStart(): DateTimeImmutable + { + return $this->lastStart; + } + + public function getLastStatus(): ?int + { + return $this->lastStatus; + } + + public function setLastEnd(?DateTimeImmutable $lastEnd): CronJobExecution + { + $this->lastEnd = $lastEnd; + + return $this; + } + + public function setLastStart(DateTimeImmutable $lastStart): CronJobExecution + { + $this->lastStart = $lastStart; + + return $this; + } + + public function setLastStatus(?int $lastStatus): CronJobExecution + { + $this->lastStatus = $lastStatus; + + return $this; + } +} diff --git a/src/Bundle/ChillMainBundle/Repository/CronJobExecutionRepository.php b/src/Bundle/ChillMainBundle/Repository/CronJobExecutionRepository.php new file mode 100644 index 000000000..eb0b63724 --- /dev/null +++ b/src/Bundle/ChillMainBundle/Repository/CronJobExecutionRepository.php @@ -0,0 +1,57 @@ +repository = $entityManager->getRepository($this->getClassName()); + } + + public function find($id): ?CronJobExecution + { + return $this->repository->find($id); + } + + /** + * @return array|CronJobExecution[] + */ + public function findAll(): array + { + return $this->repository->findAll(); + } + + /** + * @return array|CronJobExecution[] + */ + public function findBy(array $criteria, ?array $orderBy = null, ?int $limit = null, ?int $offset = null): array + { + return $this->repository->findBy($criteria, $orderBy, $limit, $offset); + } + + public function findOneBy(array $criteria): ?CronJobExecution + { + return $this->repository->findOneBy($criteria); + } + + public function getClassName(): string + { + return CronJobExecutionRepository::class; + } +} diff --git a/src/Bundle/ChillMainBundle/Repository/CronJobExecutionRepositoryInterface.php b/src/Bundle/ChillMainBundle/Repository/CronJobExecutionRepositoryInterface.php new file mode 100644 index 000000000..df894bbfb --- /dev/null +++ b/src/Bundle/ChillMainBundle/Repository/CronJobExecutionRepositoryInterface.php @@ -0,0 +1,34 @@ +prophesize(CronJobInterface::class); + $jobToExecute->getKey()->willReturn('to-exec'); + $jobToExecute->canRun(Argument::type(CronJobExecution::class))->willReturn(true); + $jobToExecute->run()->shouldBeCalled(); + + $executions = [ + ['key' => $jobOld1->getKey(), 'lastStart' => new DateTimeImmutable('yesterday'), 'lastEnd' => new DateTimeImmutable('1 hours ago'), 'lastStatus' => CronJobExecution::SUCCESS], + ['key' => $jobOld2->getKey(), 'lastStart' => new DateTimeImmutable('3 days ago'), 'lastEnd' => new DateTimeImmutable('36 hours ago'), 'lastStatus' => CronJobExecution::SUCCESS], + // this is the oldest one + ['key' => 'to-exec', 'lastStart' => new DateTimeImmutable('1 month ago'), 'lastEnd' => new DateTimeImmutable('10 days ago'), 'lastStatus' => CronJobExecution::SUCCESS], + ]; + + $cronManager = new CronManager( + $this->buildCronJobExecutionRepository($executions), + $this->buildEntityManager([]), + new ArrayObject([$jobOld1, $jobToExecute->reveal(), $jobOld2]), + new NullLogger() + ); + + $cronManager->run(); + } + + public function testSelectNewJobFirstAndNewJobIsFirstInList(): void + { + $jobAlreadyExecuted = new JobCanRun('k'); + $jobNeverExecuted = $this->prophesize(CronJobInterface::class); + $jobNeverExecuted->getKey()->willReturn('never-executed'); + $jobNeverExecuted->run()->shouldBeCalled(); + $jobNeverExecuted->canRun(null)->willReturn(true); + + $executions = [ + ['key' => $jobAlreadyExecuted->getKey(), 'lastStart' => new DateTimeImmutable('yesterday'), 'lastEnd' => new DateTimeImmutable('1 hours ago'), 'lastStatus' => CronJobExecution::SUCCESS], + ]; + + $cronManager = new CronManager( + $this->buildCronJobExecutionRepository($executions), + $this->buildEntityManager([Argument::type(CronJobExecution::class)]), + new ArrayObject([$jobNeverExecuted->reveal(), $jobAlreadyExecuted]), + new NullLogger() + ); + + $cronManager->run(); + } + + public function testSelectNewJobFirstAndNewJobIsLastInList(): void + { + $jobAlreadyExecuted = new JobCanRun('k'); + $jobNeverExecuted = $this->prophesize(CronJobInterface::class); + $jobNeverExecuted->getKey()->willReturn('never-executed'); + $jobNeverExecuted->run()->shouldBeCalled(); + $jobNeverExecuted->canRun(null)->willReturn(true); + + $executions = [ + ['key' => $jobAlreadyExecuted->getKey(), 'lastStart' => new DateTimeImmutable('yesterday'), 'lastEnd' => new DateTimeImmutable('1 hours ago'), 'lastStatus' => CronJobExecution::SUCCESS], + ]; + + $cronManager = new CronManager( + $this->buildCronJobExecutionRepository($executions), + $this->buildEntityManager([Argument::type(CronJobExecution::class)]), + new ArrayObject([$jobAlreadyExecuted, $jobNeverExecuted->reveal()]), + new NullLogger() + ); + + $cronManager->run(); + } + + /** + * @param array $executions + */ + private function buildCronJobExecutionRepository(array $executions): CronJobExecutionRepositoryInterface + { + $repository = $this->prophesize(CronJobExecutionRepositoryInterface::class); + + $repository->findAll()->willReturn( + array_map( + static function (array $exec): CronJobExecution { + $e = new CronJobExecution($exec['key']); + $e->setLastStart($exec['lastStart']); + + if (array_key_exists('lastEnd', $exec)) { + $e->setLastEnd($exec['lastEnd']); + } + + if (array_key_exists('lastStatus', $exec)) { + $e->setLastStatus($exec['lastStatus']); + } + + return $e; + }, + $executions + ) + ); + + return $repository->reveal(); + } + + private function buildEntityManager(array $persistArgsShouldBeCalled = []): EntityManagerInterface + { + $em = $this->prophesize(EntityManagerInterface::class); + + if ([] === $persistArgsShouldBeCalled) { + $em->persist(Argument::any())->shouldNotBeCalled(); + } else { + foreach ($persistArgsShouldBeCalled as $arg) { + $em->persist($arg)->shouldBeCalled(); + } + $em->flush()->shouldBeCalled(); + } + + // other methods + $em->clear()->shouldBeCalled(); + + $query = $this->prophesize(AbstractQuery::class); + $query->setParameters(Argument::type('array'))->willReturn($query->reveal()); + $query->execute()->shouldBeCalled(); + + $em->createQuery(Argument::type('string'))->willReturn($query->reveal()); + + return $em->reveal(); + } +} + +class JobCanRun implements CronJobInterface +{ + private string $key; + + public function __construct(string $key) + { + $this->key = $key; + } + + public function canRun(?CronJobExecution $cronJobExecution): bool + { + return true; + } + + public function getKey(): string + { + return $this->key; + } + + public function run(): void + { + } +} + +class JobCannotRun implements CronJobInterface +{ + public function canRun(?CronJobExecution $cronJobExecution): bool + { + return false; + } + + public function getKey(): string + { + return 'job-b'; + } + + public function run(): void + { + } +} From e0c9e12008754f7349a3a2f6935418d51259453c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Mon, 12 Dec 2022 14:35:18 +0100 Subject: [PATCH 02/13] Feature: [cron] Create a cron job to refresh materialized view address - geographical unit association --- ...eographicalUnitMaterializedViewCronJob.php | 60 +++++++++++++++++++ ...aphicalUnitMaterializedViewCronJobTest.php | 46 ++++++++++++++ .../ChillMainBundle/config/services.yaml | 9 +-- 3 files changed, 108 insertions(+), 7 deletions(-) create mode 100644 src/Bundle/ChillMainBundle/Service/AddressGeographicalUnit/RefreshAddressToGeographicalUnitMaterializedViewCronJob.php create mode 100644 src/Bundle/ChillMainBundle/Tests/Services/AddressGeographicalUnit/RefreshAddressToGeographicalUnitMaterializedViewCronJobTest.php diff --git a/src/Bundle/ChillMainBundle/Service/AddressGeographicalUnit/RefreshAddressToGeographicalUnitMaterializedViewCronJob.php b/src/Bundle/ChillMainBundle/Service/AddressGeographicalUnit/RefreshAddressToGeographicalUnitMaterializedViewCronJob.php new file mode 100644 index 000000000..a141f5100 --- /dev/null +++ b/src/Bundle/ChillMainBundle/Service/AddressGeographicalUnit/RefreshAddressToGeographicalUnitMaterializedViewCronJob.php @@ -0,0 +1,60 @@ +connection = $connection; + } + + public function canRun(?CronJobExecution $cronJobExecution): bool + { + if ($cronJobExecution->getKey() !== $this->getKey()) { + throw new UnexpectedValueException(); + } + + if (null === $cronJobExecution) { + return true; + } + + $now = new DateTimeImmutable('now'); + + return $cronJobExecution->getLastStart() < $now->sub(new DateInterval('P1D')) + && in_array($now->format('H'), self::ACCEPTED_HOURS, true) + // introduce a random component to ensure a roll when multiple instances are hosted on same machines + && mt_rand(0, 5) === 0; + } + + public function getKey(): string + { + return 'refresh-materialized-view-address-to-geog-units'; + } + + public function run(): void + { + $this->connection->executeQuery('REFRESH MATERIALIZED VIEW view_chill_main_address_geographical_unit'); + } +} diff --git a/src/Bundle/ChillMainBundle/Tests/Services/AddressGeographicalUnit/RefreshAddressToGeographicalUnitMaterializedViewCronJobTest.php b/src/Bundle/ChillMainBundle/Tests/Services/AddressGeographicalUnit/RefreshAddressToGeographicalUnitMaterializedViewCronJobTest.php new file mode 100644 index 000000000..f4e4a586e --- /dev/null +++ b/src/Bundle/ChillMainBundle/Tests/Services/AddressGeographicalUnit/RefreshAddressToGeographicalUnitMaterializedViewCronJobTest.php @@ -0,0 +1,46 @@ +connection = self::$container->get(Connection::class); + } + + public function testFullRun(): void + { + $job = new \Chill\MainBundle\Service\AddressGeographicalUnit\RefreshAddressToGeographicalUnitMaterializedViewCronJob( + $this->connection + ); + + $lastExecution = new CronJobExecution($job->getKey()); + $lastExecution->setLastStart(new DateTimeImmutable('2 days ago')); + + $this->assertIsBool($job->canRun($lastExecution)); + + $job->run(); + } +} diff --git a/src/Bundle/ChillMainBundle/config/services.yaml b/src/Bundle/ChillMainBundle/config/services.yaml index f99c80d2c..7bd1b1038 100644 --- a/src/Bundle/ChillMainBundle/config/services.yaml +++ b/src/Bundle/ChillMainBundle/config/services.yaml @@ -102,12 +102,7 @@ services: Chill\MainBundle\Security\Resolver\CenterResolverDispatcherInterface: '@Chill\MainBundle\Security\Resolver\CenterResolverDispatcher' - Chill\MainBundle\Service\Import\: - resource: '../Service/Import/' - autowire: true - autoconfigure: true - - Chill\MainBundle\Service\RollingDate\: - resource: '../Service/RollingDate/' + Chill\MainBundle\Service\: + resource: '../Service/' autowire: true autoconfigure: true From ceee5e7eea925602ec37deb28ffd83464bb815bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Mon, 12 Dec 2022 21:14:33 +0100 Subject: [PATCH 03/13] Feature: [cron] Create an interface for CronManagerInterface --- src/Bundle/ChillMainBundle/Cron/CronManager.php | 2 +- .../Cron/CronManagerInterface.php | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 src/Bundle/ChillMainBundle/Cron/CronManagerInterface.php diff --git a/src/Bundle/ChillMainBundle/Cron/CronManager.php b/src/Bundle/ChillMainBundle/Cron/CronManager.php index 1438f3ec5..8cef52438 100644 --- a/src/Bundle/ChillMainBundle/Cron/CronManager.php +++ b/src/Bundle/ChillMainBundle/Cron/CronManager.php @@ -19,7 +19,7 @@ use Exception; use Psr\Log\LoggerInterface; use function array_key_exists; -class CronManager +class CronManager implements CronManagerInterface { private const LOG_PREFIX = '[cron manager] '; diff --git a/src/Bundle/ChillMainBundle/Cron/CronManagerInterface.php b/src/Bundle/ChillMainBundle/Cron/CronManagerInterface.php new file mode 100644 index 000000000..3b9680918 --- /dev/null +++ b/src/Bundle/ChillMainBundle/Cron/CronManagerInterface.php @@ -0,0 +1,17 @@ + Date: Mon, 12 Dec 2022 21:15:05 +0100 Subject: [PATCH 04/13] Fixes: Fix doctrine annotation and repository for CronJobExecution --- src/Bundle/ChillMainBundle/Entity/CronJobExecution.php | 5 ++++- .../Repository/CronJobExecutionRepository.php | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Bundle/ChillMainBundle/Entity/CronJobExecution.php b/src/Bundle/ChillMainBundle/Entity/CronJobExecution.php index e0899e21f..0cacffac9 100644 --- a/src/Bundle/ChillMainBundle/Entity/CronJobExecution.php +++ b/src/Bundle/ChillMainBundle/Entity/CronJobExecution.php @@ -32,7 +32,7 @@ class CronJobExecution /** * @var DateTimeImmutable - * @ORM\Column(type="datetime_immutable, nullable=true, options={"default"": null}) + * @ORM\Column(type="datetime_immutable", nullable=true, options={"default": null}) */ private ?DateTimeImmutable $lastEnd = null; @@ -41,6 +41,9 @@ class CronJobExecution */ private DateTimeImmutable $lastStart; + /** + * @ORM\Column(type="integer", nullable=true, options={"default": null}) + */ private ?int $lastStatus = null; public function __construct(string $key) diff --git a/src/Bundle/ChillMainBundle/Repository/CronJobExecutionRepository.php b/src/Bundle/ChillMainBundle/Repository/CronJobExecutionRepository.php index eb0b63724..a3c495d7d 100644 --- a/src/Bundle/ChillMainBundle/Repository/CronJobExecutionRepository.php +++ b/src/Bundle/ChillMainBundle/Repository/CronJobExecutionRepository.php @@ -52,6 +52,6 @@ class CronJobExecutionRepository implements CronJobExecutionRepositoryInterface public function getClassName(): string { - return CronJobExecutionRepository::class; + return CronJobExecution::class; } } From 1ebdcd1530546ce990fdea322a836fcc8cdf3aba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Mon, 12 Dec 2022 21:15:32 +0100 Subject: [PATCH 05/13] Feature: create table for CronJobExecution --- .../migrations/Version20221212163734.php | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 src/Bundle/ChillMainBundle/migrations/Version20221212163734.php diff --git a/src/Bundle/ChillMainBundle/migrations/Version20221212163734.php b/src/Bundle/ChillMainBundle/migrations/Version20221212163734.php new file mode 100644 index 000000000..3ec61d38f --- /dev/null +++ b/src/Bundle/ChillMainBundle/migrations/Version20221212163734.php @@ -0,0 +1,36 @@ +addSql('DROP TABLE chill_main_cronjob_execution'); + } + + public function getDescription(): string + { + return 'Table for executed jobs'; + } + + public function up(Schema $schema): void + { + $this->addSql('CREATE TABLE chill_main_cronjob_execution (key TEXT NOT NULL, lastEnd TIMESTAMP(0) WITHOUT TIME ZONE DEFAULT NULL, + lastStart TIMESTAMP(0) WITHOUT TIME ZONE NOT NULL, lastStatus INTEGER DEFAULT NULL, PRIMARY KEY(key))'); + $this->addSql('COMMENT ON COLUMN chill_main_cronjob_execution.lastEnd IS \'(DC2Type:datetime_immutable)\''); + $this->addSql('COMMENT ON COLUMN chill_main_cronjob_execution.lastStart IS \'(DC2Type:datetime_immutable)\''); + } +} From b789250b8d880642ed2bd588fc837ba3856d383a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Mon, 12 Dec 2022 21:15:58 +0100 Subject: [PATCH 06/13] Feature: bootstrap dependency injection for CronJobManager --- src/Bundle/ChillMainBundle/ChillMainBundle.php | 3 +++ src/Bundle/ChillMainBundle/config/services.yaml | 12 ++++++++++++ .../ChillMainBundle/config/services/command.yaml | 6 ++++++ 3 files changed, 21 insertions(+) diff --git a/src/Bundle/ChillMainBundle/ChillMainBundle.php b/src/Bundle/ChillMainBundle/ChillMainBundle.php index 4442bd19b..9fd6a9891 100644 --- a/src/Bundle/ChillMainBundle/ChillMainBundle.php +++ b/src/Bundle/ChillMainBundle/ChillMainBundle.php @@ -11,6 +11,7 @@ declare(strict_types=1); namespace Chill\MainBundle; +use Chill\MainBundle\Cron\CronJobInterface; use Chill\MainBundle\CRUD\CompilerPass\CRUDControllerCompilerPass; use Chill\MainBundle\DependencyInjection\CompilerPass\ACLFlagsCompilerPass; use Chill\MainBundle\DependencyInjection\CompilerPass\ExportsCompilerPass; @@ -59,6 +60,8 @@ class ChillMainBundle extends Bundle ->addTag('chill.count_notification.user'); $container->registerForAutoconfiguration(EntityWorkflowHandlerInterface::class) ->addTag('chill_main.workflow_handler'); + $container->registerForAutoconfiguration(CronJobInterface::class) + ->addTag('chill_main.cron_job'); $container->addCompilerPass(new SearchableServicesCompilerPass()); $container->addCompilerPass(new ConfigConsistencyCompilerPass()); diff --git a/src/Bundle/ChillMainBundle/config/services.yaml b/src/Bundle/ChillMainBundle/config/services.yaml index 7bd1b1038..6248c508e 100644 --- a/src/Bundle/ChillMainBundle/config/services.yaml +++ b/src/Bundle/ChillMainBundle/config/services.yaml @@ -106,3 +106,15 @@ services: resource: '../Service/' autowire: true autoconfigure: true + + Chill\MainBundle\Cron\: + resource: '../Cron' + autowire: true + autoconfigure: true + + Chill\MainBundle\Cron\CronManager: + autoconfigure: true + autowire: true + lazy: true + arguments: + $jobs: !tagged_iterator chill_main.cron_job diff --git a/src/Bundle/ChillMainBundle/config/services/command.yaml b/src/Bundle/ChillMainBundle/config/services/command.yaml index e5f285545..f9a863f10 100644 --- a/src/Bundle/ChillMainBundle/config/services/command.yaml +++ b/src/Bundle/ChillMainBundle/config/services/command.yaml @@ -61,3 +61,9 @@ services: autowire: true tags: - { name: console.command } + + Chill\MainBundle\Command\ExecuteCronJobCommand: + autoconfigure: true + autowire: true + tags: + - {name: console.command } From 204ebd4415fa95db8f84341fb97896b4051c0b10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Mon, 12 Dec 2022 21:16:27 +0100 Subject: [PATCH 07/13] Feature: bootstrap dependency injection for CronJobManager and create a command --- .../Command/ExecuteCronJobCommand.php | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 src/Bundle/ChillMainBundle/Command/ExecuteCronJobCommand.php diff --git a/src/Bundle/ChillMainBundle/Command/ExecuteCronJobCommand.php b/src/Bundle/ChillMainBundle/Command/ExecuteCronJobCommand.php new file mode 100644 index 000000000..9dca2996a --- /dev/null +++ b/src/Bundle/ChillMainBundle/Command/ExecuteCronJobCommand.php @@ -0,0 +1,56 @@ +cronManager = $cronManager; + } + + protected function configure() + { + $this + ->setDescription('Execute the cronjob(s) given as argument, or one cronjob scheduled by system.') + ->setHelp("If no job is specified, the next available cronjob will be executed by system.\nThis command should be execute every 15 minutes (more or less)") + ->addArgument('job', InputArgument::OPTIONAL | InputArgument::IS_ARRAY, 'one or more job to force execute (by default, all jobs are executed)', []) + ->addUsage(''); + } + + protected function execute(InputInterface $input, OutputInterface $output) + { + if ([] === $input->getArgument('job')) { + $this->cronManager->run(); + + return 0; + } + + foreach ($input->getArgument('job') as $jobName) { + $this->cronManager->run($jobName); + } + + return 0; + } +} From 7d469df62ae5e65aa6a4ccf77d463d20b0ef6f1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Mon, 12 Dec 2022 21:17:03 +0100 Subject: [PATCH 08/13] Feature: CronJob for refreshing materialized view association between address and geographical unit --- ...shAddressToGeographicalUnitMaterializedViewCronJob.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Bundle/ChillMainBundle/Service/AddressGeographicalUnit/RefreshAddressToGeographicalUnitMaterializedViewCronJob.php b/src/Bundle/ChillMainBundle/Service/AddressGeographicalUnit/RefreshAddressToGeographicalUnitMaterializedViewCronJob.php index a141f5100..a836f4124 100644 --- a/src/Bundle/ChillMainBundle/Service/AddressGeographicalUnit/RefreshAddressToGeographicalUnitMaterializedViewCronJob.php +++ b/src/Bundle/ChillMainBundle/Service/AddressGeographicalUnit/RefreshAddressToGeographicalUnitMaterializedViewCronJob.php @@ -32,14 +32,14 @@ class RefreshAddressToGeographicalUnitMaterializedViewCronJob implements CronJob public function canRun(?CronJobExecution $cronJobExecution): bool { - if ($cronJobExecution->getKey() !== $this->getKey()) { - throw new UnexpectedValueException(); - } - if (null === $cronJobExecution) { return true; } + if ($cronJobExecution->getKey() !== $this->getKey()) { + throw new UnexpectedValueException(); + } + $now = new DateTimeImmutable('now'); return $cronJobExecution->getLastStart() < $now->sub(new DateInterval('P1D')) From 17461aa21e7376fbc8ef272e7de6987e0aa72825 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Mon, 12 Dec 2022 22:21:46 +0100 Subject: [PATCH 09/13] Fixed: [cronjob manager] Fix execution of one single job --- src/Bundle/ChillMainBundle/Cron/CronManager.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Bundle/ChillMainBundle/Cron/CronManager.php b/src/Bundle/ChillMainBundle/Cron/CronManager.php index 8cef52438..20934fbdb 100644 --- a/src/Bundle/ChillMainBundle/Cron/CronManager.php +++ b/src/Bundle/ChillMainBundle/Cron/CronManager.php @@ -153,7 +153,9 @@ class CronManager implements CronManagerInterface private function runForce(string $forceJob): void { foreach ($this->jobs as $job) { - $job->run(); + if ($job->getKey() === $forceJob) { + $job->run(); + } } } } From 0bfb5c617ee1d3fcb8aeeafcfd270f0b16024855 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Mon, 12 Dec 2022 22:48:57 +0100 Subject: [PATCH 10/13] Doc: [cronjob manager] Doc for implementing cronjob --- docs/source/development/cronjob.rst | 93 +++++++++++++++++++ docs/source/development/index.rst | 1 + docs/source/installation/prod.rst | 29 ++++-- .../ChillMainBundle/Cron/CronManager.php | 19 ++++ .../Cron/CronManagerInterface.php | 6 ++ 5 files changed, 141 insertions(+), 7 deletions(-) create mode 100644 docs/source/development/cronjob.rst diff --git a/docs/source/development/cronjob.rst b/docs/source/development/cronjob.rst new file mode 100644 index 000000000..df72fa922 --- /dev/null +++ b/docs/source/development/cronjob.rst @@ -0,0 +1,93 @@ + +.. Copyright (C) 2014-2023 Champs Libres Cooperative SCRLFS + Permission is granted to copy, distribute and/or modify this document + under the terms of the GNU Free Documentation License, Version 1.3 + or any later version published by the Free Software Foundation; + with no Invariant Sections, no Front-Cover Texts, and no Back-Cover Texts. + A copy of the license is included in the section entitled "GNU + Free Documentation License". + +.. _cronjob: + +Cron jobs +********* + +Some tasks must be executed regularly: refresh some materialized views, remove old data, ... + +For this purpose, one can programmatically implements a "cron job", which will be scheduled by a specific command. + +The command :code:`chill:cron-job:execute` +========================================== + +The command :code:`chill:cron-job:execute` will schedule a task, one by one. In a classical implementation, it should +be executed every 15 minutes (more or less), to ensure that every task can be executed. + +.. warning:: + + This command should not be executed in parallel. The installer should ensure that two job are executed concurrently. + +How to implements a cron job ? +============================== + +Implements a :code:`Chill\MainBundle\Cron\CronJobInterface`. Here is an example: + +.. code-block:: php + + namespace Chill\MainBundle\Service\Something; + + use Chill\MainBundle\Cron\CronJobInterface; + use Chill\MainBundle\Entity\CronJobExecution; + use DateInterval; + use DateTimeImmutable; + + class MyCronJob implements CronJobInterface + { + public function canRun(?CronJobExecution $cronJobExecution): bool + { + // the parameter $cronJobExecution contains data about the last execution of the cronjob + // if it is null, it should be executed immediatly + if (null === $cronJobExecution) { + return true; + } + + if ($cronJobExecution->getKey() !== $this->getKey()) { + throw new UnexpectedValueException(); + } + + // this cron job should be executed if the last execution is greater than one day, but only during the night + + $now = new DateTimeImmutable('now'); + + return $cronJobExecution->getLastStart() < $now->sub(new DateInterval('P1D')) + && in_array($now->format('H'), self::ACCEPTED_HOURS, true) + // introduce a random component to ensure a roll of task execution when multiple instances are hosted on same machines + && mt_rand(0, 5) === 0; + } + + public function getKey(): string + { + return 'arbitrary-and-unique-key'; + } + + public function run(): void + { + // here, we execute the command + } + } + +How are cron job scheduled ? +============================ + +If the command :code:`chill:cron-job:execute` is run with one or more :code:`job` argument, those jobs are run, **without checking that the job can run** (the method :code:`canRun` is not executed). + +If any :code:`job` argument is given, the :code:`CronManager` schedule job with those steps: + +* the tasks are ordered, with: + * a priority is given for tasks that weren't never executed; + * then, the tasks are ordered, the last executed are the first in the list +* then, for each tasks, and in the given order, the first task where :code:`canRun` return :code:`TRUE` will be executed. + +The command :code:`chill:cron-job:execute` execute **only one** task. + + + diff --git a/docs/source/development/index.rst b/docs/source/development/index.rst index f35bc12db..52c541c8e 100644 --- a/docs/source/development/index.rst +++ b/docs/source/development/index.rst @@ -34,6 +34,7 @@ As Chill rely on the `symfony `_ framework, reading the fram Useful snippets manual/index.rst Assets + Cron Jobs Layout and UI ************** diff --git a/docs/source/installation/prod.rst b/docs/source/installation/prod.rst index 4e670b8f8..f51141e8d 100644 --- a/docs/source/installation/prod.rst +++ b/docs/source/installation/prod.rst @@ -1,10 +1,12 @@ .. Copyright (C) 2014-2019 Champs Libres Cooperative SCRLFS -Permission is granted to copy, distribute and/or modify this document -under the terms of the GNU Free Documentation License, Version 1.3 -or any later version published by the Free Software Foundation; -with no Invariant Sections, no Front-Cover Texts, and no Back-Cover Texts. -A copy of the license is included in the section entitled "GNU -Free Documentation License". + Permission is granted to copy, distribute and/or modify this document + under the terms of the GNU Free Documentation License, Version 1.3 + or any later version published by the Free Software Foundation; + with no Invariant Sections, no Front-Cover Texts, and no Back-Cover Texts. + A copy of the license is included in the section entitled "GNU + Free Documentation License". + +.. _prod: Installation for production ########################### @@ -36,12 +38,25 @@ This should be adapted to your needs: * Think about how you will backup your database. Some adminsys find easier to store database outside of docker, which might be easier to administrate or replicate. +Cron jobs +========= + +The command :code:`chill:cron-job:execute` should be executed every 15 minutes (more or less). + +This command should never be executed concurrently. It should be not have more than one process for a single instance. + +Post-install tasks +================== + +- import addresses. See :ref:`addresses`. + + Tweak symfony messenger ======================= Calendar sync is processed using symfony messenger. -You can tweak the configuration +You can tweak the configuration Going further: diff --git a/src/Bundle/ChillMainBundle/Cron/CronManager.php b/src/Bundle/ChillMainBundle/Cron/CronManager.php index 20934fbdb..52cba3d38 100644 --- a/src/Bundle/ChillMainBundle/Cron/CronManager.php +++ b/src/Bundle/ChillMainBundle/Cron/CronManager.php @@ -19,6 +19,25 @@ use Exception; use Psr\Log\LoggerInterface; use function array_key_exists; +/** + * Manage cronjob and execute them. + * + * If any :code:`job` argument is given, the :code:`CronManager` schedule job with those steps: + * + * - the tasks are ordered, with: + * - a priority is given for tasks that weren't never executed; + * - then, the tasks are ordered, the last executed are the first in the list + * + * Then, for each tasks, and in the given order, the first task where :code:`canRun` return :code:`TRUE` will be executed. + * + * The error inside job execution are catched (with the exception of _out of memory error_). + * + * The manager will mark the task as executed even if an error is catched. This will lead as failed job + * will not have priority any more on other tasks. + * + * If a tasks is "forced", there is no test about eligibility of the task (the `canRun` method is not called), + * and the last task execution is not recorded. + */ class CronManager implements CronManagerInterface { private const LOG_PREFIX = '[cron manager] '; diff --git a/src/Bundle/ChillMainBundle/Cron/CronManagerInterface.php b/src/Bundle/ChillMainBundle/Cron/CronManagerInterface.php index 3b9680918..57f96c74b 100644 --- a/src/Bundle/ChillMainBundle/Cron/CronManagerInterface.php +++ b/src/Bundle/ChillMainBundle/Cron/CronManagerInterface.php @@ -13,5 +13,11 @@ namespace Chill\MainBundle\Cron; interface CronManagerInterface { + /** + * Execute one job, with a given priority, or the given job (identified by his key) + * + * @param string|null $forceJob + * @return void + */ public function run(?string $forceJob = null): void; } From acfa3d68490ab22b84db2fabbff33546badd4262 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Mon, 12 Dec 2022 22:49:30 +0100 Subject: [PATCH 11/13] Fixed: [cronjob command] force command name --- src/Bundle/ChillMainBundle/Command/ExecuteCronJobCommand.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Bundle/ChillMainBundle/Command/ExecuteCronJobCommand.php b/src/Bundle/ChillMainBundle/Command/ExecuteCronJobCommand.php index 9dca2996a..0e81177dc 100644 --- a/src/Bundle/ChillMainBundle/Command/ExecuteCronJobCommand.php +++ b/src/Bundle/ChillMainBundle/Command/ExecuteCronJobCommand.php @@ -22,7 +22,6 @@ class ExecuteCronJobCommand extends Command private CronManagerInterface $cronManager; public function __construct( - ?string $name, CronManagerInterface $cronManager ) { parent::__construct('chill:cron-job:execute'); From 23d7d1c8f0c88ddcfc1daf50ef116d06669211e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Mon, 12 Dec 2022 22:49:50 +0100 Subject: [PATCH 12/13] Doc: doc for installation and cronjob, and addresses --- docs/source/installation/index.rst | 3 +- docs/source/installation/load-addresses.rst | 50 +++++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 docs/source/installation/load-addresses.rst diff --git a/docs/source/installation/index.rst b/docs/source/installation/index.rst index 01a8c4aee..29856f2e4 100644 --- a/docs/source/installation/index.rst +++ b/docs/source/installation/index.rst @@ -18,6 +18,7 @@ Installation & Usage :maxdepth: 2 prod.rst + load-addresses.rst prod-calendar-sms-sending.rst msgraph-configure.rst @@ -170,7 +171,7 @@ There are several users available: The password is always ``password``. -Now, read `Operations` below. +Now, read `Operations` below. For running in production, read `prod_`. Operations diff --git a/docs/source/installation/load-addresses.rst b/docs/source/installation/load-addresses.rst new file mode 100644 index 000000000..779032fd0 --- /dev/null +++ b/docs/source/installation/load-addresses.rst @@ -0,0 +1,50 @@ + +.. _addresses: + +Addresses +********* + +Chill can store a list of geolocated address references, which are used to suggest address and ensure that the data is correctly stored. + +Those addresses may be load from a dedicated source. + +In France +========= + +The address are loaded from the `BANO `_. The postal codes are loaded from `the official list of +postal codes `_ + +.. code-block:: bash + + # first, load postal codes + bin/console chill:main:postal-code:load:FR + # then, load all addresses, by departement (multiple departement can be loaded by repeating the departement code + bin/console chill:main:address-ref-from-bano 57 54 51 + +In Belgium +========== + +Addresses are prepared from the `BeST Address data `_. + +Postal code are loaded from this database. There is no need to load postal codes from another source (actually, this is strongly discouraged). + +The data are prepared for Chill (`See this repository `_). +One can select postal code by his first number (:code:`1xxx` for postal codes from 1000 to 1999), or a limited list for development purpose. + +.. code-block:: bash + + # load postal code from 1000 to 3999: + bin/console chill:main:address-ref-from-best-addresse 1xxx 2xxx 3xxx + + # load only an extract (for dev purposes) + bin/console chill:main:address-ref-from-best-addresse extract + + # load full addresses (discouraged) + bin/console chill:main:address-ref-from-best-addresse full + +.. note:: + + There is a possibility to load the full list of addresses is discouraged: the loading is optimized with smaller extracts. + + Once you load the full list, it is not possible to load smaller extract: each extract loaded **after** will not + delete the addresses loaded with the full extract (and some addresses will be present twice). From d41dc7035c773d9e30497c009a4b3891a6e2f6bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Mon, 12 Dec 2022 23:29:03 +0100 Subject: [PATCH 13/13] Doc: doc for CronJobManagerInterface --- src/Bundle/ChillMainBundle/Cron/CronManagerInterface.php | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/Bundle/ChillMainBundle/Cron/CronManagerInterface.php b/src/Bundle/ChillMainBundle/Cron/CronManagerInterface.php index 57f96c74b..d2292d455 100644 --- a/src/Bundle/ChillMainBundle/Cron/CronManagerInterface.php +++ b/src/Bundle/ChillMainBundle/Cron/CronManagerInterface.php @@ -14,10 +14,7 @@ namespace Chill\MainBundle\Cron; interface CronManagerInterface { /** - * Execute one job, with a given priority, or the given job (identified by his key) - * - * @param string|null $forceJob - * @return void + * Execute one job, with a given priority, or the given job (identified by his key). */ public function run(?string $forceJob = null): void; }