Merge branch 'master' into VSR-issues

This commit is contained in:
Mathieu Jaumotte 2022-12-15 15:19:43 +01:00
commit 2deed644b6
24 changed files with 1034 additions and 19 deletions

View File

@ -0,0 +1,93 @@
.. Copyright (C) 2014-2023 Champs Libres Cooperative SCRLFS
Permission is granted to copy, distribute and/or modify this document
under the terms of the GNU Free Documentation License, Version 1.3
or any later version published by the Free Software Foundation;
with no Invariant Sections, no Front-Cover Texts, and no Back-Cover Texts.
A copy of the license is included in the section entitled "GNU
Free Documentation License".
.. _cronjob:
Cron jobs
*********
Some tasks must be executed regularly: refresh some materialized views, remove old data, ...
For this purpose, one can programmatically implements a "cron job", which will be scheduled by a specific command.
The command :code:`chill:cron-job:execute`
==========================================
The command :code:`chill:cron-job:execute` will schedule a task, one by one. In a classical implementation, it should
be executed every 15 minutes (more or less), to ensure that every task can be executed.
.. warning::
This command should not be executed in parallel. The installer should ensure that two job are executed concurrently.
How to implements a cron job ?
==============================
Implements a :code:`Chill\MainBundle\Cron\CronJobInterface`. Here is an example:
.. code-block:: php
namespace Chill\MainBundle\Service\Something;
use Chill\MainBundle\Cron\CronJobInterface;
use Chill\MainBundle\Entity\CronJobExecution;
use DateInterval;
use DateTimeImmutable;
class MyCronJob implements CronJobInterface
{
public function canRun(?CronJobExecution $cronJobExecution): bool
{
// the parameter $cronJobExecution contains data about the last execution of the cronjob
// if it is null, it should be executed immediatly
if (null === $cronJobExecution) {
return true;
}
if ($cronJobExecution->getKey() !== $this->getKey()) {
throw new UnexpectedValueException();
}
// this cron job should be executed if the last execution is greater than one day, but only during the night
$now = new DateTimeImmutable('now');
return $cronJobExecution->getLastStart() < $now->sub(new DateInterval('P1D'))
&& in_array($now->format('H'), self::ACCEPTED_HOURS, true)
// introduce a random component to ensure a roll of task execution when multiple instances are hosted on same machines
&& mt_rand(0, 5) === 0;
}
public function getKey(): string
{
return 'arbitrary-and-unique-key';
}
public function run(): void
{
// here, we execute the command
}
}
How are cron job scheduled ?
============================
If the command :code:`chill:cron-job:execute` is run with one or more :code:`job` argument, those jobs are run, **without checking that the job can run** (the method :code:`canRun` is not executed).
If any :code:`job` argument is given, the :code:`CronManager` schedule job with those steps:
* the tasks are ordered, with:
* a priority is given for tasks that weren't never executed;
* then, the tasks are ordered, the last executed are the first in the list
* then, for each tasks, and in the given order, the first task where :code:`canRun` return :code:`TRUE` will be executed.
The command :code:`chill:cron-job:execute` execute **only one** task.

View File

@ -34,6 +34,7 @@ As Chill rely on the `symfony <http://symfony.com>`_ framework, reading the fram
Useful snippets <useful-snippets.rst> Useful snippets <useful-snippets.rst>
manual/index.rst manual/index.rst
Assets <assets.rst> Assets <assets.rst>
Cron Jobs <cronjob.rst>
Layout and UI Layout and UI
************** **************

View File

@ -18,6 +18,7 @@ Installation & Usage
:maxdepth: 2 :maxdepth: 2
prod.rst prod.rst
load-addresses.rst
prod-calendar-sms-sending.rst prod-calendar-sms-sending.rst
msgraph-configure.rst msgraph-configure.rst
@ -170,7 +171,7 @@ There are several users available:
The password is always ``password``. The password is always ``password``.
Now, read `Operations` below. Now, read `Operations` below. For running in production, read `prod_`.
Operations Operations

View File

@ -0,0 +1,50 @@
.. _addresses:
Addresses
*********
Chill can store a list of geolocated address references, which are used to suggest address and ensure that the data is correctly stored.
Those addresses may be load from a dedicated source.
In France
=========
The address are loaded from the `BANO <https://bano.openstreetmap.fr/>`_. The postal codes are loaded from `the official list of
postal codes <https://datanova.laposte.fr/explore/dataset/laposte_hexasmal/information/>`_
.. code-block:: bash
# first, load postal codes
bin/console chill:main:postal-code:load:FR
# then, load all addresses, by departement (multiple departement can be loaded by repeating the departement code
bin/console chill:main:address-ref-from-bano 57 54 51
In Belgium
==========
Addresses are prepared from the `BeST Address data <https://www.geo.be/catalog/details/ca0fd5c0-8146-11e9-9012-482ae30f98d9>`_.
Postal code are loaded from this database. There is no need to load postal codes from another source (actually, this is strongly discouraged).
The data are prepared for Chill (`See this repository <https://gitea.champs-libres.be/Chill-project/belgian-bestaddresses-transform/releases>`_).
One can select postal code by his first number (:code:`1xxx` for postal codes from 1000 to 1999), or a limited list for development purpose.
.. code-block:: bash
# load postal code from 1000 to 3999:
bin/console chill:main:address-ref-from-best-addresse 1xxx 2xxx 3xxx
# load only an extract (for dev purposes)
bin/console chill:main:address-ref-from-best-addresse extract
# load full addresses (discouraged)
bin/console chill:main:address-ref-from-best-addresse full
.. note::
There is a possibility to load the full list of addresses is discouraged: the loading is optimized with smaller extracts.
Once you load the full list, it is not possible to load smaller extract: each extract loaded **after** will not
delete the addresses loaded with the full extract (and some addresses will be present twice).

View File

@ -1,10 +1,12 @@
.. Copyright (C) 2014-2019 Champs Libres Cooperative SCRLFS .. Copyright (C) 2014-2019 Champs Libres Cooperative SCRLFS
Permission is granted to copy, distribute and/or modify this document Permission is granted to copy, distribute and/or modify this document
under the terms of the GNU Free Documentation License, Version 1.3 under the terms of the GNU Free Documentation License, Version 1.3
or any later version published by the Free Software Foundation; or any later version published by the Free Software Foundation;
with no Invariant Sections, no Front-Cover Texts, and no Back-Cover Texts. with no Invariant Sections, no Front-Cover Texts, and no Back-Cover Texts.
A copy of the license is included in the section entitled "GNU A copy of the license is included in the section entitled "GNU
Free Documentation License". Free Documentation License".
.. _prod:
Installation for production Installation for production
########################### ###########################
@ -36,6 +38,19 @@ This should be adapted to your needs:
* Think about how you will backup your database. Some adminsys find easier to store database outside of docker, which might be easier to administrate or replicate. * Think about how you will backup your database. Some adminsys find easier to store database outside of docker, which might be easier to administrate or replicate.
Cron jobs
=========
The command :code:`chill:cron-job:execute` should be executed every 15 minutes (more or less).
This command should never be executed concurrently. It should be not have more than one process for a single instance.
Post-install tasks
==================
- import addresses. See :ref:`addresses`.
Tweak symfony messenger Tweak symfony messenger
======================= =======================

View File

@ -51,8 +51,13 @@ class CalendarForShortMessageProvider
*/ */
public function getCalendars(DateTimeImmutable $at): iterable public function getCalendars(DateTimeImmutable $at): iterable
{ {
['startDate' => $startDate, 'endDate' => $endDate] = $this->rangeGenerator $range = $this->rangeGenerator->generateRange($at);
->generateRange($at);
if (null === $range) {
return;
}
['startDate' => $startDate, 'endDate' => $endDate] = $range;
$offset = 0; $offset = 0;
$batchSize = 10; $batchSize = 10;

View File

@ -31,14 +31,14 @@ use UnexpectedValueException;
*/ */
class DefaultRangeGenerator implements RangeGeneratorInterface class DefaultRangeGenerator implements RangeGeneratorInterface
{ {
public function generateRange(\DateTimeImmutable $date): array public function generateRange(\DateTimeImmutable $date): ?array
{ {
$onMidnight = DateTimeImmutable::createFromFormat('Y-m-d H:i:s', $date->format('Y-m-d') . ' 00:00:00'); $onMidnight = DateTimeImmutable::createFromFormat('Y-m-d H:i:s', $date->format('Y-m-d') . ' 00:00:00');
switch ($dow = (int) $onMidnight->format('w')) { switch ($dow = (int) $onMidnight->format('w')) {
case 6: // Saturday case 6: // Saturday
case 0: // Sunday case 0: // Sunday
return ['startDate' => null, 'endDate' => null]; return null;
case 1: // Monday case 1: // Monday
// send for Tuesday and Wednesday // send for Tuesday and Wednesday

View File

@ -23,7 +23,7 @@ use DateTimeImmutable;
interface RangeGeneratorInterface interface RangeGeneratorInterface
{ {
/** /**
* @return array<startDate: \DateTimeImmutable, endDate: \DateTimeImmutable> * @return ?array{startDate: DateTimeImmutable, endDate: DateTimeImmutable} when return is null, then no ShortMessage must be send
*/ */
public function generateRange(DateTimeImmutable $date): array; public function generateRange(DateTimeImmutable $date): ?array;
} }

View File

@ -22,6 +22,7 @@ use Chill\CalendarBundle\Entity\Calendar;
use Chill\CalendarBundle\Repository\CalendarRepository; use Chill\CalendarBundle\Repository\CalendarRepository;
use Chill\CalendarBundle\Service\ShortMessageNotification\CalendarForShortMessageProvider; use Chill\CalendarBundle\Service\ShortMessageNotification\CalendarForShortMessageProvider;
use Chill\CalendarBundle\Service\ShortMessageNotification\DefaultRangeGenerator; use Chill\CalendarBundle\Service\ShortMessageNotification\DefaultRangeGenerator;
use Chill\CalendarBundle\Service\ShortMessageNotification\RangeGeneratorInterface;
use DateTimeImmutable; use DateTimeImmutable;
use Doctrine\ORM\EntityManagerInterface; use Doctrine\ORM\EntityManagerInterface;
use PHPUnit\Framework\TestCase; use PHPUnit\Framework\TestCase;
@ -37,6 +38,32 @@ final class CalendarForShortMessageProviderTest extends TestCase
{ {
use ProphecyTrait; use ProphecyTrait;
public function testGenerateRangeIsNull()
{
$calendarRepository = $this->prophesize(CalendarRepository::class);
$calendarRepository->findByNotificationAvailable(
Argument::type(DateTimeImmutable::class),
Argument::type(DateTimeImmutable::class),
Argument::type('int'),
Argument::exact(0)
)->shouldBeCalledTimes(0);
$rangeGenerator = $this->prophesize(RangeGeneratorInterface::class);
$rangeGenerator->generateRange(Argument::type(DateTimeImmutable::class))->willReturn(null);
$em = $this->prophesize(EntityManagerInterface::class);
$em->clear()->shouldNotBeCalled();
$provider = new CalendarForShortMessageProvider(
$calendarRepository->reveal(),
$em->reveal(),
$rangeGenerator->reveal()
);
$calendars = iterator_to_array($provider->getCalendars(new DateTimeImmutable('now')));
$this->assertEquals(0, count($calendars));
}
public function testGetCalendars() public function testGetCalendars()
{ {
$calendarRepository = $this->prophesize(CalendarRepository::class); $calendarRepository = $this->prophesize(CalendarRepository::class);

View File

@ -11,6 +11,7 @@ declare(strict_types=1);
namespace Chill\MainBundle; namespace Chill\MainBundle;
use Chill\MainBundle\Cron\CronJobInterface;
use Chill\MainBundle\CRUD\CompilerPass\CRUDControllerCompilerPass; use Chill\MainBundle\CRUD\CompilerPass\CRUDControllerCompilerPass;
use Chill\MainBundle\DependencyInjection\CompilerPass\ACLFlagsCompilerPass; use Chill\MainBundle\DependencyInjection\CompilerPass\ACLFlagsCompilerPass;
use Chill\MainBundle\DependencyInjection\CompilerPass\ExportsCompilerPass; use Chill\MainBundle\DependencyInjection\CompilerPass\ExportsCompilerPass;
@ -59,6 +60,8 @@ class ChillMainBundle extends Bundle
->addTag('chill.count_notification.user'); ->addTag('chill.count_notification.user');
$container->registerForAutoconfiguration(EntityWorkflowHandlerInterface::class) $container->registerForAutoconfiguration(EntityWorkflowHandlerInterface::class)
->addTag('chill_main.workflow_handler'); ->addTag('chill_main.workflow_handler');
$container->registerForAutoconfiguration(CronJobInterface::class)
->addTag('chill_main.cron_job');
$container->addCompilerPass(new SearchableServicesCompilerPass()); $container->addCompilerPass(new SearchableServicesCompilerPass());
$container->addCompilerPass(new ConfigConsistencyCompilerPass()); $container->addCompilerPass(new ConfigConsistencyCompilerPass());

View File

@ -0,0 +1,55 @@
<?php
declare(strict_types=1);
/*
* Chill is a software for social workers
*
* For the full copyright and license information, please view
* the LICENSE file that was distributed with this source code.
*/
namespace Chill\MainBundle\Command;
use Chill\MainBundle\Cron\CronManagerInterface;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
class ExecuteCronJobCommand extends Command
{
private CronManagerInterface $cronManager;
public function __construct(
CronManagerInterface $cronManager
) {
parent::__construct('chill:cron-job:execute');
$this->cronManager = $cronManager;
}
protected function configure()
{
$this
->setDescription('Execute the cronjob(s) given as argument, or one cronjob scheduled by system.')
->setHelp("If no job is specified, the next available cronjob will be executed by system.\nThis command should be execute every 15 minutes (more or less)")
->addArgument('job', InputArgument::OPTIONAL | InputArgument::IS_ARRAY, 'one or more job to force execute (by default, all jobs are executed)', [])
->addUsage('');
}
protected function execute(InputInterface $input, OutputInterface $output)
{
if ([] === $input->getArgument('job')) {
$this->cronManager->run();
return 0;
}
foreach ($input->getArgument('job') as $jobName) {
$this->cronManager->run($jobName);
}
return 0;
}
}

View File

@ -0,0 +1,23 @@
<?php
declare(strict_types=1);
/*
* Chill is a software for social workers
*
* For the full copyright and license information, please view
* the LICENSE file that was distributed with this source code.
*/
namespace Chill\MainBundle\Cron;
use Chill\MainBundle\Entity\CronJobExecution;
interface CronJobInterface
{
public function canRun(?CronJobExecution $cronJobExecution): bool;
public function getKey(): string;
public function run(): void;
}

View File

@ -0,0 +1,180 @@
<?php
declare(strict_types=1);
/*
* Chill is a software for social workers
*
* For the full copyright and license information, please view
* the LICENSE file that was distributed with this source code.
*/
namespace Chill\MainBundle\Cron;
use Chill\MainBundle\Entity\CronJobExecution;
use Chill\MainBundle\Repository\CronJobExecutionRepositoryInterface;
use DateTimeImmutable;
use Doctrine\ORM\EntityManagerInterface;
use Exception;
use Psr\Log\LoggerInterface;
use function array_key_exists;
/**
* Manage cronjob and execute them.
*
* If any :code:`job` argument is given, the :code:`CronManager` schedule job with those steps:
*
* - the tasks are ordered, with:
* - a priority is given for tasks that weren't never executed;
* - then, the tasks are ordered, the last executed are the first in the list
*
* Then, for each tasks, and in the given order, the first task where :code:`canRun` return :code:`TRUE` will be executed.
*
* The error inside job execution are catched (with the exception of _out of memory error_).
*
* The manager will mark the task as executed even if an error is catched. This will lead as failed job
* will not have priority any more on other tasks.
*
* If a tasks is "forced", there is no test about eligibility of the task (the `canRun` method is not called),
* and the last task execution is not recorded.
*/
class CronManager implements CronManagerInterface
{
private const LOG_PREFIX = '[cron manager] ';
private const UPDATE_AFTER_EXEC = 'UPDATE ' . CronJobExecution::class . ' cr SET cr.lastEnd = :now, cr.lastStatus = :status WHERE cr.key = :key';
private const UPDATE_BEFORE_EXEC = 'UPDATE ' . CronJobExecution::class . ' cr SET cr.lastExecution = :now WHERE cr.key = :key';
private CronJobExecutionRepositoryInterface $cronJobExecutionRepository;
private EntityManagerInterface $entityManager;
/**
* @var iterable<CronJobInterface>
*/
private iterable $jobs;
private LoggerInterface $logger;
/**
* @param CronJobInterface[] $jobs
*/
public function __construct(
CronJobExecutionRepositoryInterface $cronJobExecutionRepository,
EntityManagerInterface $entityManager,
iterable $jobs,
LoggerInterface $logger
) {
$this->cronJobExecutionRepository = $cronJobExecutionRepository;
$this->entityManager = $entityManager;
$this->jobs = $jobs;
$this->logger = $logger;
}
public function run(?string $forceJob = null): void
{
if (null !== $forceJob) {
$this->runForce($forceJob);
return;
}
[$orderedJobs, $lasts] = $this->getOrderedJobs();
foreach ($orderedJobs as $job) {
if ($job->canRun($lasts[$job->getKey()] ?? null)) {
if (array_key_exists($job->getKey(), $lasts)) {
$this->entityManager
->createQuery(self::UPDATE_BEFORE_EXEC)
->setParameters([
'now' => new DateTimeImmutable('now'),
'key' => $job->getKey(),
]);
} else {
$execution = new CronJobExecution($job->getKey());
$this->entityManager->persist($execution);
$this->entityManager->flush();
}
$this->entityManager->clear();
try {
$this->logger->info(sprintf('%sWill run job', self::LOG_PREFIX), ['job' => $job->getKey()]);
$job->run();
$this->entityManager
->createQuery(self::UPDATE_AFTER_EXEC)
->setParameters([
'now' => new DateTimeImmutable('now'),
'status' => CronJobExecution::SUCCESS,
'key' => $job->getKey(),
])
->execute();
$this->logger->info(sprintf('%sSuccessfully run job', self::LOG_PREFIX), ['job' => $job->getKey()]);
return;
} catch (Exception $e) {
$this->logger->error(sprintf('%sRunning job failed', self::LOG_PREFIX), ['job' => $job->getKey()]);
$this->entityManager
->createQuery(self::UPDATE_AFTER_EXEC)
->setParameters([
'now' => new DateTimeImmutable('now'),
'status' => CronJobExecution::FAILURE,
'key' => $job->getKey(),
])
->execute();
return;
}
}
}
}
/**
* @return array<0: CronJobInterface[], 1: array<string, CronJobExecution>>
*/
private function getOrderedJobs(): array
{
/** @var array<string, CronJobExecution> $lasts */
$lasts = [];
foreach ($this->cronJobExecutionRepository->findAll() as $execution) {
$lasts[$execution->getKey()] = $execution;
}
// order by last, NULL first
$orderedJobs = iterator_to_array($this->jobs);
usort(
$orderedJobs,
static function (CronJobInterface $a, CronJobInterface $b) use ($lasts): int {
if (
(!array_key_exists($a->getKey(), $lasts) && !array_key_exists($b->getKey(), $lasts))
) {
return 0;
}
if (!array_key_exists($a->getKey(), $lasts) && array_key_exists($b->getKey(), $lasts)) {
return -1;
}
if (!array_key_exists($b->getKey(), $lasts) && array_key_exists($a->getKey(), $lasts)) {
return 1;
}
return $lasts[$a->getKey()]->getLastStart() <=> $lasts[$b->getKey()]->getLastStart();
}
);
return [$orderedJobs, $lasts];
}
private function runForce(string $forceJob): void
{
foreach ($this->jobs as $job) {
if ($job->getKey() === $forceJob) {
$job->run();
}
}
}
}

View File

@ -0,0 +1,20 @@
<?php
declare(strict_types=1);
/*
* Chill is a software for social workers
*
* For the full copyright and license information, please view
* the LICENSE file that was distributed with this source code.
*/
namespace Chill\MainBundle\Cron;
interface CronManagerInterface
{
/**
* Execute one job, with a given priority, or the given job (identified by his key).
*/
public function run(?string $forceJob = null): void;
}

View File

@ -0,0 +1,95 @@
<?php
declare(strict_types=1);
/*
* Chill is a software for social workers
*
* For the full copyright and license information, please view
* the LICENSE file that was distributed with this source code.
*/
namespace Chill\MainBundle\Entity;
use DateTimeImmutable;
use Doctrine\ORM\Mapping as ORM;
/**
* @ORM\Entity
* @ORM\Table(name="chill_main_cronjob_execution")
*/
class CronJobExecution
{
public const FAILURE = 100;
public const SUCCESS = 1;
/**
* @ORM\Column(type="text", nullable=false)
* @ORM\Id
*/
private string $key;
/**
* @var DateTimeImmutable
* @ORM\Column(type="datetime_immutable", nullable=true, options={"default": null})
*/
private ?DateTimeImmutable $lastEnd = null;
/**
* @ORM\Column(type="datetime_immutable", nullable=false)
*/
private DateTimeImmutable $lastStart;
/**
* @ORM\Column(type="integer", nullable=true, options={"default": null})
*/
private ?int $lastStatus = null;
public function __construct(string $key)
{
$this->key = $key;
$this->lastStart = new DateTimeImmutable('now');
}
public function getKey(): string
{
return $this->key;
}
public function getLastEnd(): DateTimeImmutable
{
return $this->lastEnd;
}
public function getLastStart(): DateTimeImmutable
{
return $this->lastStart;
}
public function getLastStatus(): ?int
{
return $this->lastStatus;
}
public function setLastEnd(?DateTimeImmutable $lastEnd): CronJobExecution
{
$this->lastEnd = $lastEnd;
return $this;
}
public function setLastStart(DateTimeImmutable $lastStart): CronJobExecution
{
$this->lastStart = $lastStart;
return $this;
}
public function setLastStatus(?int $lastStatus): CronJobExecution
{
$this->lastStatus = $lastStatus;
return $this;
}
}

View File

@ -0,0 +1,57 @@
<?php
declare(strict_types=1);
/*
* Chill is a software for social workers
*
* For the full copyright and license information, please view
* the LICENSE file that was distributed with this source code.
*/
namespace Chill\MainBundle\Repository;
use Chill\MainBundle\Entity\CronJobExecution;
use Doctrine\ORM\EntityManagerInterface;
use Doctrine\ORM\EntityRepository;
class CronJobExecutionRepository implements CronJobExecutionRepositoryInterface
{
private EntityRepository $repository;
public function __construct(EntityManagerInterface $entityManager)
{
$this->repository = $entityManager->getRepository($this->getClassName());
}
public function find($id): ?CronJobExecution
{
return $this->repository->find($id);
}
/**
* @return array|CronJobExecution[]
*/
public function findAll(): array
{
return $this->repository->findAll();
}
/**
* @return array|CronJobExecution[]
*/
public function findBy(array $criteria, ?array $orderBy = null, ?int $limit = null, ?int $offset = null): array
{
return $this->repository->findBy($criteria, $orderBy, $limit, $offset);
}
public function findOneBy(array $criteria): ?CronJobExecution
{
return $this->repository->findOneBy($criteria);
}
public function getClassName(): string
{
return CronJobExecution::class;
}
}

View File

@ -0,0 +1,34 @@
<?php
declare(strict_types=1);
/*
* Chill is a software for social workers
*
* For the full copyright and license information, please view
* the LICENSE file that was distributed with this source code.
*/
namespace Chill\MainBundle\Repository;
use Chill\MainBundle\Entity\CronJobExecution;
use Doctrine\Persistence\ObjectRepository;
interface CronJobExecutionRepositoryInterface extends ObjectRepository
{
public function find($id): ?CronJobExecution;
/**
* @return array|CronJobExecution[]
*/
public function findAll(): array;
/**
* @return array|CronJobExecution[]
*/
public function findBy(array $criteria, ?array $orderBy = null, ?int $limit = null, ?int $offset = null): array;
public function findOneBy(array $criteria): ?CronJobExecution;
public function getClassName(): string;
}

View File

@ -0,0 +1,60 @@
<?php
declare(strict_types=1);
/*
* Chill is a software for social workers
*
* For the full copyright and license information, please view
* the LICENSE file that was distributed with this source code.
*/
namespace Chill\MainBundle\Service\AddressGeographicalUnit;
use Chill\MainBundle\Cron\CronJobInterface;
use Chill\MainBundle\Entity\CronJobExecution;
use DateInterval;
use DateTimeImmutable;
use Doctrine\DBAL\Connection;
use UnexpectedValueException;
use function in_array;
class RefreshAddressToGeographicalUnitMaterializedViewCronJob implements CronJobInterface
{
private const ACCEPTED_HOURS = ['0', '1', '2', '3', '4', '5'];
private Connection $connection;
public function __construct(Connection $connection)
{
$this->connection = $connection;
}
public function canRun(?CronJobExecution $cronJobExecution): bool
{
if (null === $cronJobExecution) {
return true;
}
if ($cronJobExecution->getKey() !== $this->getKey()) {
throw new UnexpectedValueException();
}
$now = new DateTimeImmutable('now');
return $cronJobExecution->getLastStart() < $now->sub(new DateInterval('P1D'))
&& in_array($now->format('H'), self::ACCEPTED_HOURS, true)
// introduce a random component to ensure a roll when multiple instances are hosted on same machines
&& mt_rand(0, 5) === 0;
}
public function getKey(): string
{
return 'refresh-materialized-view-address-to-geog-units';
}
public function run(): void
{
$this->connection->executeQuery('REFRESH MATERIALIZED VIEW view_chill_main_address_geographical_unit');
}
}

View File

@ -0,0 +1,201 @@
<?php
declare(strict_types=1);
/*
* Chill is a software for social workers
*
* For the full copyright and license information, please view
* the LICENSE file that was distributed with this source code.
*/
namespace Chill\MainBundle\Tests\Cron;
use ArrayObject;
use Chill\MainBundle\Cron\CronJobInterface;
use Chill\MainBundle\Cron\CronManager;
use Chill\MainBundle\Entity\CronJobExecution;
use Chill\MainBundle\Repository\CronJobExecutionRepositoryInterface;
use DateTimeImmutable;
use Doctrine\ORM\AbstractQuery;
use Doctrine\ORM\EntityManagerInterface;
use PHPUnit\Framework\TestCase;
use Prophecy\Argument;
use Prophecy\PhpUnit\ProphecyTrait;
use Psr\Log\NullLogger;
use function array_key_exists;
/**
* @internal
* @coversNothing
*/
final class CronManagerTest extends TestCase
{
use ProphecyTrait;
public function testScheduleMostOldExecutedJob()
{
$jobOld1 = new JobCanRun('k');
$jobOld2 = new JobCanRun('l');
$jobToExecute = $this->prophesize(CronJobInterface::class);
$jobToExecute->getKey()->willReturn('to-exec');
$jobToExecute->canRun(Argument::type(CronJobExecution::class))->willReturn(true);
$jobToExecute->run()->shouldBeCalled();
$executions = [
['key' => $jobOld1->getKey(), 'lastStart' => new DateTimeImmutable('yesterday'), 'lastEnd' => new DateTimeImmutable('1 hours ago'), 'lastStatus' => CronJobExecution::SUCCESS],
['key' => $jobOld2->getKey(), 'lastStart' => new DateTimeImmutable('3 days ago'), 'lastEnd' => new DateTimeImmutable('36 hours ago'), 'lastStatus' => CronJobExecution::SUCCESS],
// this is the oldest one
['key' => 'to-exec', 'lastStart' => new DateTimeImmutable('1 month ago'), 'lastEnd' => new DateTimeImmutable('10 days ago'), 'lastStatus' => CronJobExecution::SUCCESS],
];
$cronManager = new CronManager(
$this->buildCronJobExecutionRepository($executions),
$this->buildEntityManager([]),
new ArrayObject([$jobOld1, $jobToExecute->reveal(), $jobOld2]),
new NullLogger()
);
$cronManager->run();
}
public function testSelectNewJobFirstAndNewJobIsFirstInList(): void
{
$jobAlreadyExecuted = new JobCanRun('k');
$jobNeverExecuted = $this->prophesize(CronJobInterface::class);
$jobNeverExecuted->getKey()->willReturn('never-executed');
$jobNeverExecuted->run()->shouldBeCalled();
$jobNeverExecuted->canRun(null)->willReturn(true);
$executions = [
['key' => $jobAlreadyExecuted->getKey(), 'lastStart' => new DateTimeImmutable('yesterday'), 'lastEnd' => new DateTimeImmutable('1 hours ago'), 'lastStatus' => CronJobExecution::SUCCESS],
];
$cronManager = new CronManager(
$this->buildCronJobExecutionRepository($executions),
$this->buildEntityManager([Argument::type(CronJobExecution::class)]),
new ArrayObject([$jobNeverExecuted->reveal(), $jobAlreadyExecuted]),
new NullLogger()
);
$cronManager->run();
}
public function testSelectNewJobFirstAndNewJobIsLastInList(): void
{
$jobAlreadyExecuted = new JobCanRun('k');
$jobNeverExecuted = $this->prophesize(CronJobInterface::class);
$jobNeverExecuted->getKey()->willReturn('never-executed');
$jobNeverExecuted->run()->shouldBeCalled();
$jobNeverExecuted->canRun(null)->willReturn(true);
$executions = [
['key' => $jobAlreadyExecuted->getKey(), 'lastStart' => new DateTimeImmutable('yesterday'), 'lastEnd' => new DateTimeImmutable('1 hours ago'), 'lastStatus' => CronJobExecution::SUCCESS],
];
$cronManager = new CronManager(
$this->buildCronJobExecutionRepository($executions),
$this->buildEntityManager([Argument::type(CronJobExecution::class)]),
new ArrayObject([$jobAlreadyExecuted, $jobNeverExecuted->reveal()]),
new NullLogger()
);
$cronManager->run();
}
/**
* @param array<array{key: string, ?lastEnd: ?DateTimeImmutable, lastStart: DateTimeImmutable, ?lastStatus: ?int}> $executions
*/
private function buildCronJobExecutionRepository(array $executions): CronJobExecutionRepositoryInterface
{
$repository = $this->prophesize(CronJobExecutionRepositoryInterface::class);
$repository->findAll()->willReturn(
array_map(
static function (array $exec): CronJobExecution {
$e = new CronJobExecution($exec['key']);
$e->setLastStart($exec['lastStart']);
if (array_key_exists('lastEnd', $exec)) {
$e->setLastEnd($exec['lastEnd']);
}
if (array_key_exists('lastStatus', $exec)) {
$e->setLastStatus($exec['lastStatus']);
}
return $e;
},
$executions
)
);
return $repository->reveal();
}
private function buildEntityManager(array $persistArgsShouldBeCalled = []): EntityManagerInterface
{
$em = $this->prophesize(EntityManagerInterface::class);
if ([] === $persistArgsShouldBeCalled) {
$em->persist(Argument::any())->shouldNotBeCalled();
} else {
foreach ($persistArgsShouldBeCalled as $arg) {
$em->persist($arg)->shouldBeCalled();
}
$em->flush()->shouldBeCalled();
}
// other methods
$em->clear()->shouldBeCalled();
$query = $this->prophesize(AbstractQuery::class);
$query->setParameters(Argument::type('array'))->willReturn($query->reveal());
$query->execute()->shouldBeCalled();
$em->createQuery(Argument::type('string'))->willReturn($query->reveal());
return $em->reveal();
}
}
class JobCanRun implements CronJobInterface
{
private string $key;
public function __construct(string $key)
{
$this->key = $key;
}
public function canRun(?CronJobExecution $cronJobExecution): bool
{
return true;
}
public function getKey(): string
{
return $this->key;
}
public function run(): void
{
}
}
class JobCannotRun implements CronJobInterface
{
public function canRun(?CronJobExecution $cronJobExecution): bool
{
return false;
}
public function getKey(): string
{
return 'job-b';
}
public function run(): void
{
}
}

View File

@ -0,0 +1,46 @@
<?php
declare(strict_types=1);
/*
* Chill is a software for social workers
*
* For the full copyright and license information, please view
* the LICENSE file that was distributed with this source code.
*/
namespace Chill\MainBundle\Tests\Services\AddressGeographicalUnit;
use Chill\MainBundle\Entity\CronJobExecution;
use DateTimeImmutable;
use Doctrine\DBAL\Connection;
use Symfony\Bundle\FrameworkBundle\Test\KernelTestCase;
/**
* @internal
* @coversNothing
*/
final class RefreshAddressToGeographicalUnitMaterializedViewCronJobTest extends KernelTestCase
{
private Connection $connection;
protected function setUp(): void
{
parent::bootKernel();
$this->connection = self::$container->get(Connection::class);
}
public function testFullRun(): void
{
$job = new \Chill\MainBundle\Service\AddressGeographicalUnit\RefreshAddressToGeographicalUnitMaterializedViewCronJob(
$this->connection
);
$lastExecution = new CronJobExecution($job->getKey());
$lastExecution->setLastStart(new DateTimeImmutable('2 days ago'));
$this->assertIsBool($job->canRun($lastExecution));
$job->run();
}
}

View File

@ -102,12 +102,19 @@ services:
Chill\MainBundle\Security\Resolver\CenterResolverDispatcherInterface: '@Chill\MainBundle\Security\Resolver\CenterResolverDispatcher' Chill\MainBundle\Security\Resolver\CenterResolverDispatcherInterface: '@Chill\MainBundle\Security\Resolver\CenterResolverDispatcher'
Chill\MainBundle\Service\Import\: Chill\MainBundle\Service\:
resource: '../Service/Import/' resource: '../Service/'
autowire: true autowire: true
autoconfigure: true autoconfigure: true
Chill\MainBundle\Service\RollingDate\: Chill\MainBundle\Cron\:
resource: '../Service/RollingDate/' resource: '../Cron'
autowire: true autowire: true
autoconfigure: true autoconfigure: true
Chill\MainBundle\Cron\CronManager:
autoconfigure: true
autowire: true
lazy: true
arguments:
$jobs: !tagged_iterator chill_main.cron_job

View File

@ -61,3 +61,9 @@ services:
autowire: true autowire: true
tags: tags:
- { name: console.command } - { name: console.command }
Chill\MainBundle\Command\ExecuteCronJobCommand:
autoconfigure: true
autowire: true
tags:
- {name: console.command }

View File

@ -0,0 +1,36 @@
<?php
declare(strict_types=1);
/*
* Chill is a software for social workers
*
* For the full copyright and license information, please view
* the LICENSE file that was distributed with this source code.
*/
namespace Chill\Migrations\Main;
use Doctrine\DBAL\Schema\Schema;
use Doctrine\Migrations\AbstractMigration;
final class Version20221212163734 extends AbstractMigration
{
public function down(Schema $schema): void
{
$this->addSql('DROP TABLE chill_main_cronjob_execution');
}
public function getDescription(): string
{
return 'Table for executed jobs';
}
public function up(Schema $schema): void
{
$this->addSql('CREATE TABLE chill_main_cronjob_execution (key TEXT NOT NULL, lastEnd TIMESTAMP(0) WITHOUT TIME ZONE DEFAULT NULL,
lastStart TIMESTAMP(0) WITHOUT TIME ZONE NOT NULL, lastStatus INTEGER DEFAULT NULL, PRIMARY KEY(key))');
$this->addSql('COMMENT ON COLUMN chill_main_cronjob_execution.lastEnd IS \'(DC2Type:datetime_immutable)\'');
$this->addSql('COMMENT ON COLUMN chill_main_cronjob_execution.lastStart IS \'(DC2Type:datetime_immutable)\'');
}
}

View File

@ -24,7 +24,7 @@
<td> <td>
<ul class="record_actions"> <ul class="record_actions">
<li> <li>
<a href="{{ chill_path_add_return_path('chill_crud_person_household_position_edit', { 'id': entity.id }) }}" class="btn btn-edit"></a> <a href="{{ chill_path_add_return_path('chill_crud_person_household_composition_type_edit', { 'id': entity.id }) }}" class="btn btn-edit"></a>
</li> </li>
</ul> </ul>
</td> </td>