Merge remote-tracking branch 'origin/master' into upgrade-sf5

This commit is contained in:
2024-04-04 18:45:01 +02:00
162 changed files with 3849 additions and 713 deletions

View File

@@ -17,52 +17,88 @@ use Chill\DocGeneratorBundle\Entity\DocGeneratorTemplate;
use Chill\DocGeneratorBundle\GeneratorDriver\DriverInterface;
use Chill\DocGeneratorBundle\GeneratorDriver\Exception\TemplateException;
use Chill\DocStoreBundle\Entity\StoredObject;
use Chill\DocStoreBundle\Exception\StoredObjectManagerException;
use Chill\DocStoreBundle\Service\StoredObjectManagerInterface;
use Chill\MainBundle\Entity\User;
use Doctrine\ORM\EntityManagerInterface;
use Doctrine\Persistence\ManagerRegistry;
use Psr\Log\LoggerInterface;
use Symfony\Component\HttpFoundation\File\File;
use Symfony\Component\Yaml\Yaml;
class Generator implements GeneratorInterface
{
private const LOG_PREFIX = '[docgen generator] ';
public function __construct(private readonly ContextManagerInterface $contextManager, private readonly DriverInterface $driver, private readonly EntityManagerInterface $entityManager, private readonly LoggerInterface $logger, private readonly StoredObjectManagerInterface $storedObjectManager) {}
public function __construct(
private readonly ContextManagerInterface $contextManager,
private readonly DriverInterface $driver,
private readonly ManagerRegistry $objectManagerRegistry,
private readonly LoggerInterface $logger,
private readonly StoredObjectManagerInterface $storedObjectManager
) {
}
public function generateDataDump(
DocGeneratorTemplate $template,
int $entityId,
array $contextGenerationDataNormalized,
StoredObject $destinationStoredObject,
User $creator,
bool $clearEntityManagerDuringProcess = true,
): StoredObject {
return $this->generateFromTemplate(
$template,
$entityId,
$contextGenerationDataNormalized,
$destinationStoredObject,
$creator,
$clearEntityManagerDuringProcess,
true,
);
}
/**
* @template T of File|null
* @template B of bool
*
* @param B $isTest
* @param (B is true ? T : null) $testFile
*
* @psalm-return (B is true ? string : null)
*
* @throws \Symfony\Component\Serializer\Exception\ExceptionInterface|\Throwable
*/
public function generateDocFromTemplate(
DocGeneratorTemplate $template,
int $entityId,
array $contextGenerationDataNormalized,
?StoredObject $destinationStoredObject = null,
bool $isTest = false,
?File $testFile = null,
?User $creator = null
): ?string {
if ($destinationStoredObject instanceof StoredObject && StoredObject::STATUS_PENDING !== $destinationStoredObject->getStatus()) {
StoredObject $destinationStoredObject,
User $creator,
bool $clearEntityManagerDuringProcess = true,
): StoredObject {
return $this->generateFromTemplate(
$template,
$entityId,
$contextGenerationDataNormalized,
$destinationStoredObject,
$creator,
$clearEntityManagerDuringProcess,
false,
);
}
private function generateFromTemplate(
DocGeneratorTemplate $template,
int $entityId,
array $contextGenerationDataNormalized,
StoredObject $destinationStoredObject,
User $creator,
bool $clearEntityManagerDuringProcess = true,
bool $generateDumpOnly = false,
): StoredObject {
if (StoredObject::STATUS_PENDING !== $destinationStoredObject->getStatus()) {
$this->logger->info(self::LOG_PREFIX.'Aborting generation of an already generated document');
throw new ObjectReadyException();
}
$this->logger->info(self::LOG_PREFIX.'Starting generation of a document', [
'entity_id' => $entityId,
'destination_stored_object' => null === $destinationStoredObject ? null : $destinationStoredObject->getId(),
'destination_stored_object' => $destinationStoredObject->getId(),
]);
$context = $this->contextManager->getContextByDocGeneratorTemplate($template);
$entity = $this
->entityManager
->objectManagerRegistry
->getManagerForClass($context->getEntityClass())
->find($context->getEntityClass(), $entityId)
;
@@ -80,17 +116,47 @@ class Generator implements GeneratorInterface
$data = $context->getData($template, $entity, $contextGenerationDataNormalized);
$destinationStoredObjectId = $destinationStoredObject instanceof StoredObject ? $destinationStoredObject->getId() : null;
$this->entityManager->clear();
gc_collect_cycles();
if (null !== $destinationStoredObjectId) {
$destinationStoredObject = $this->entityManager->find(StoredObject::class, $destinationStoredObjectId);
$destinationStoredObjectId = $destinationStoredObject->getId();
if ($clearEntityManagerDuringProcess) {
// we clean the entity manager
$this->objectManagerRegistry->getManagerForClass($context->getEntityClass())?->clear();
// this will force php to clean the memory
gc_collect_cycles();
}
if ($isTest && ($testFile instanceof File)) {
$templateDecrypted = file_get_contents($testFile->getPathname());
} else {
// as we potentially deleted the storedObject from memory, we have to restore it
$destinationStoredObject = $this->objectManagerRegistry
->getManagerForClass(StoredObject::class)
->find(StoredObject::class, $destinationStoredObjectId);
if ($generateDumpOnly) {
$content = Yaml::dump($data, 6);
/* @var StoredObject $destinationStoredObject */
$destinationStoredObject
->setType('application/yaml')
->setFilename(sprintf('%s_yaml', uniqid('doc_', true)))
->setStatus(StoredObject::STATUS_READY)
;
try {
$this->storedObjectManager->write($destinationStoredObject, $content);
} catch (StoredObjectManagerException $e) {
$destinationStoredObject->addGenerationErrors($e->getMessage());
throw new GeneratorException([$e->getMessage()], $e);
}
return $destinationStoredObject;
}
try {
$templateDecrypted = $this->storedObjectManager->read($template->getFile());
} catch (StoredObjectManagerException $e) {
$destinationStoredObject->addGenerationErrors($e->getMessage());
throw new GeneratorException([$e->getMessage()], $e);
}
try {
@@ -103,19 +169,10 @@ class Generator implements GeneratorInterface
$template->getFile()->getFilename()
);
} catch (TemplateException $e) {
$destinationStoredObject->addGenerationErrors(implode("\n", $e->getErrors()));
throw new GeneratorException($e->getErrors(), $e);
}
if (true === $isTest) {
$this->logger->info(self::LOG_PREFIX.'Finished generation of a document', [
'is_test' => true,
'entity_id' => $entityId,
'destination_stored_object' => null === $destinationStoredObject ? null : $destinationStoredObject->getId(),
]);
return $generatedResource;
}
/* @var StoredObject $destinationStoredObject */
$destinationStoredObject
->setType($template->getFile()->getType())
@@ -123,15 +180,19 @@ class Generator implements GeneratorInterface
->setStatus(StoredObject::STATUS_READY)
;
$this->storedObjectManager->write($destinationStoredObject, $generatedResource);
try {
$this->storedObjectManager->write($destinationStoredObject, $generatedResource);
} catch (StoredObjectManagerException $e) {
$destinationStoredObject->addGenerationErrors($e->getMessage());
$this->entityManager->flush();
throw new GeneratorException([$e->getMessage()], $e);
}
$this->logger->info(self::LOG_PREFIX.'Finished generation of a document', [
'entity_id' => $entityId,
'destination_stored_object' => $destinationStoredObject->getId(),
]);
return null;
return $destinationStoredObject;
}
}

View File

@@ -13,29 +13,48 @@ namespace Chill\DocGeneratorBundle\Service\Generator;
use Chill\DocGeneratorBundle\Entity\DocGeneratorTemplate;
use Chill\DocStoreBundle\Entity\StoredObject;
use Chill\DocStoreBundle\Exception\StoredObjectManagerException;
use Chill\MainBundle\Entity\User;
use Symfony\Component\HttpFoundation\File\File;
interface GeneratorInterface
{
/**
* @template T of File|null
* @template B of bool
* Generate a document and store the document on disk.
*
* @param B $isTest
* @param (B is true ? T : null) $testFile
* The given $destinationStoredObject will be updated with filename, status, and eventually errors will be stored
* into the object. The number of generation trial will also be incremented.
*
* @psalm-return (B is true ? string : null)
* This process requires a huge amount of data. For this reason, the entity manager will be cleaned during the process,
* unless the paarameter `$clearEntityManagerDuringProcess` is set on false.
*
* @throws \Symfony\Component\Serializer\Exception\ExceptionInterface|\Throwable
* As the entity manager might be cleaned, the new instance of the stored object will be returned by this method.
*
* Ensure to store change in the database after each generation trial (call `EntityManagerInterface::flush`).
*
* @phpstan-impure
*
* @param StoredObject $destinationStoredObject will be update with filename, status and incremented of generation trials
*
* @throws StoredObjectManagerException if unable to decrypt the template or store the document
*/
public function generateDocFromTemplate(
DocGeneratorTemplate $template,
int $entityId,
array $contextGenerationDataNormalized,
?StoredObject $destinationStoredObject = null,
bool $isTest = false,
?File $testFile = null,
?User $creator = null
): ?string;
StoredObject $destinationStoredObject,
User $creator,
bool $clearEntityManagerDuringProcess = true,
): StoredObject;
/**
* Generate a data dump, and store it within the `$destinationStoredObject`.
*/
public function generateDataDump(
DocGeneratorTemplate $template,
int $entityId,
array $contextGenerationDataNormalized,
StoredObject $destinationStoredObject,
User $creator,
bool $clearEntityManagerDuringProcess = true,
): StoredObject;
}

View File

@@ -0,0 +1,64 @@
<?php
declare(strict_types=1);
/*
* Chill is a software for social workers
*
* For the full copyright and license information, please view
* the LICENSE file that was distributed with this source code.
*/
namespace Chill\DocGeneratorBundle\Service\Messenger;
use Chill\DocStoreBundle\Service\StoredObjectManagerInterface;
use Psr\Log\LoggerInterface;
use Symfony\Component\EventDispatcher\EventSubscriberInterface;
use Symfony\Component\Messenger\Event\WorkerMessageFailedEvent;
use Symfony\Component\Messenger\Event\WorkerMessageHandledEvent;
/**
* The OnAfterMessageHandledClearStoredObjectCache class is an event subscriber that clears the stored object cache
* after a specific message is handled or fails.
*/
final readonly class OnAfterMessageHandledClearStoredObjectCache implements EventSubscriberInterface
{
public function __construct(
private StoredObjectManagerInterface $storedObjectManager,
private LoggerInterface $logger,
) {
}
public static function getSubscribedEvents()
{
return [
WorkerMessageHandledEvent::class => [
['afterHandling', 0],
],
WorkerMessageFailedEvent::class => [
['afterFails', 0],
],
];
}
public function afterHandling(WorkerMessageHandledEvent $event): void
{
if ($event->getEnvelope()->getMessage() instanceof RequestGenerationMessage) {
$this->clearStoredObjectCache();
}
}
public function afterFails(WorkerMessageFailedEvent $event): void
{
if ($event->getEnvelope()->getMessage() instanceof RequestGenerationMessage) {
$this->clearStoredObjectCache();
}
}
private function clearStoredObjectCache(): void
{
$this->logger->debug('clear the cache after generation of a document');
$this->storedObjectManager->clearCache();
}
}

View File

@@ -11,10 +11,11 @@ declare(strict_types=1);
namespace Chill\DocGeneratorBundle\Service\Messenger;
use Chill\DocGeneratorBundle\Repository\DocGeneratorTemplateRepository;
use Chill\DocGeneratorBundle\Repository\DocGeneratorTemplateRepositoryInterface;
use Chill\DocGeneratorBundle\Service\Generator\GeneratorException;
use Chill\DocGeneratorBundle\tests\Service\Messenger\OnGenerationFailsTest;
use Chill\DocStoreBundle\Entity\StoredObject;
use Chill\DocStoreBundle\Repository\StoredObjectRepository;
use Chill\DocStoreBundle\Repository\StoredObjectRepositoryInterface;
use Chill\MainBundle\Repository\UserRepositoryInterface;
use Doctrine\ORM\EntityManagerInterface;
use Psr\Log\LoggerInterface;
@@ -24,11 +25,23 @@ use Symfony\Component\Mailer\MailerInterface;
use Symfony\Component\Messenger\Event\WorkerMessageFailedEvent;
use Symfony\Contracts\Translation\TranslatorInterface;
/**
* @see OnGenerationFailsTest for test suite
*/
final readonly class OnGenerationFails implements EventSubscriberInterface
{
public const LOG_PREFIX = '[docgen failed] ';
public function __construct(private DocGeneratorTemplateRepository $docGeneratorTemplateRepository, private EntityManagerInterface $entityManager, private LoggerInterface $logger, private MailerInterface $mailer, private StoredObjectRepository $storedObjectRepository, private TranslatorInterface $translator, private UserRepositoryInterface $userRepository) {}
public function __construct(
private DocGeneratorTemplateRepositoryInterface $docGeneratorTemplateRepository,
private EntityManagerInterface $entityManager,
private LoggerInterface $logger,
private MailerInterface $mailer,
private StoredObjectRepositoryInterface $storedObjectRepository,
private TranslatorInterface $translator,
private UserRepositoryInterface $userRepository
) {
}
public static function getSubscribedEvents()
{
@@ -43,13 +56,12 @@ final readonly class OnGenerationFails implements EventSubscriberInterface
return;
}
if (!$event->getEnvelope()->getMessage() instanceof RequestGenerationMessage) {
$message = $event->getEnvelope()->getMessage();
if (!$message instanceof RequestGenerationMessage) {
return;
}
/** @var RequestGenerationMessage $message */
$message = $event->getEnvelope()->getMessage();
$this->logger->error(self::LOG_PREFIX.'Docgen failed', [
'stored_object_id' => $message->getDestinationStoredObjectId(),
'entity_id' => $message->getEntityId(),
@@ -77,16 +89,8 @@ final readonly class OnGenerationFails implements EventSubscriberInterface
private function warnCreator(RequestGenerationMessage $message, WorkerMessageFailedEvent $event): void
{
$creatorId = $message->getCreatorId();
if (null === $creator = $this->userRepository->find($creatorId)) {
$this->logger->error(self::LOG_PREFIX.'Creator not found with given id', ['creator_id', $creatorId]);
return;
}
if (null === $creator->getEmail() || '' === $creator->getEmail()) {
$this->logger->info(self::LOG_PREFIX.'Creator does not have any email', ['user' => $creator->getUsernameCanonical()]);
if (null === $message->getSendResultToEmail() || '' === $message->getSendResultToEmail()) {
$this->logger->info(self::LOG_PREFIX.'No email associated with this request generation');
return;
}
@@ -94,7 +98,7 @@ final readonly class OnGenerationFails implements EventSubscriberInterface
// if the exception is not a GeneratorException, we try the previous one...
$throwable = $event->getThrowable();
if (!$throwable instanceof GeneratorException) {
$throwable = $throwable->getPrevious();
$throwable = $throwable->getPrevious() ?? $throwable;
}
if ($throwable instanceof GeneratorException) {
@@ -109,8 +113,14 @@ final readonly class OnGenerationFails implements EventSubscriberInterface
return;
}
if (null === $creator = $this->userRepository->find($message->getCreatorId())) {
$this->logger->error(self::LOG_PREFIX.'Creator not found');
return;
}
$email = (new TemplatedEmail())
->to($creator->getEmail())
->to($message->getSendResultToEmail())
->subject($this->translator->trans('docgen.failure_email.The generation of a document failed'))
->textTemplate('@ChillDocGenerator/Email/on_generation_failed_email.txt.twig')
->context([

View File

@@ -11,15 +11,21 @@ declare(strict_types=1);
namespace Chill\DocGeneratorBundle\Service\Messenger;
use ChampsLibres\AsyncUploaderBundle\TempUrl\TempUrlGeneratorInterface;
use Chill\DocGeneratorBundle\Repository\DocGeneratorTemplateRepository;
use Chill\DocGeneratorBundle\Service\Generator\Generator;
use Chill\DocGeneratorBundle\Service\Generator\GeneratorException;
use Chill\DocStoreBundle\Entity\StoredObject;
use Chill\DocStoreBundle\Exception\StoredObjectManagerException;
use Chill\DocStoreBundle\Repository\StoredObjectRepository;
use Chill\MainBundle\Repository\UserRepositoryInterface;
use Doctrine\ORM\EntityManagerInterface;
use Psr\Log\LoggerInterface;
use Symfony\Bridge\Twig\Mime\TemplatedEmail;
use Symfony\Component\Mailer\MailerInterface;
use Symfony\Component\Messenger\Exception\UnrecoverableMessageHandlingException;
use Symfony\Component\Messenger\Handler\MessageHandlerInterface;
use Symfony\Contracts\Translation\TranslatorInterface;
/**
* Handle the request of document generation.
@@ -30,7 +36,18 @@ class RequestGenerationHandler implements MessageHandlerInterface
private const LOG_PREFIX = '[docgen message handler] ';
public function __construct(private readonly DocGeneratorTemplateRepository $docGeneratorTemplateRepository, private readonly EntityManagerInterface $entityManager, private readonly Generator $generator, private readonly LoggerInterface $logger, private readonly StoredObjectRepository $storedObjectRepository, private readonly UserRepositoryInterface $userRepository) {}
public function __construct(
private readonly DocGeneratorTemplateRepository $docGeneratorTemplateRepository,
private readonly EntityManagerInterface $entityManager,
private readonly Generator $generator,
private readonly LoggerInterface $logger,
private readonly StoredObjectRepository $storedObjectRepository,
private readonly UserRepositoryInterface $userRepository,
private readonly MailerInterface $mailer,
private readonly TempUrlGeneratorInterface $tempUrlGenerator,
private readonly TranslatorInterface $translator,
) {
}
public function __invoke(RequestGenerationMessage $message)
{
@@ -43,25 +60,59 @@ class RequestGenerationHandler implements MessageHandlerInterface
}
if ($destinationStoredObject->getGenerationTrialsCounter() >= self::AUTHORIZED_TRIALS) {
$this->logger->error(self::LOG_PREFIX.'Request generation abandoned: maximum number of retry reached', [
'template_id' => $message->getTemplateId(),
'destination_stored_object' => $message->getDestinationStoredObjectId(),
'trial' => $destinationStoredObject->getGenerationTrialsCounter(),
]);
throw new UnrecoverableMessageHandlingException('maximum number of retry reached');
}
$creator = $this->userRepository->find($message->getCreatorId());
// we increase the number of generation trial in the object, and, in the same time, update the counter
// on the database side. This ensure that, if the script fails for any reason (memory limit reached), the
// counter is inscreased
$destinationStoredObject->addGenerationTrial();
$this->entityManager->createQuery('UPDATE '.StoredObject::class.' s SET s.generationTrialsCounter = s.generationTrialsCounter + 1 WHERE s.id = :id')
->setParameter('id', $destinationStoredObject->getId())
->execute();
$this->generator->generateDocFromTemplate(
$template,
$message->getEntityId(),
$message->getContextGenerationData(),
$destinationStoredObject,
false,
null,
$creator
);
try {
if ($message->isDumpOnly()) {
$destinationStoredObject = $this->generator->generateDataDump(
$template,
$message->getEntityId(),
$message->getContextGenerationData(),
$destinationStoredObject,
$creator
);
$this->sendDataDump($destinationStoredObject, $message);
} else {
$destinationStoredObject = $this->generator->generateDocFromTemplate(
$template,
$message->getEntityId(),
$message->getContextGenerationData(),
$destinationStoredObject,
$creator
);
}
} catch (StoredObjectManagerException|GeneratorException $e) {
$this->entityManager->flush();
$this->logger->error(self::LOG_PREFIX.'Request generation failed', [
'template_id' => $message->getTemplateId(),
'destination_stored_object' => $message->getDestinationStoredObjectId(),
'trial' => $destinationStoredObject->getGenerationTrialsCounter(),
'error' => $e->getTraceAsString(),
]);
throw $e;
}
$this->entityManager->flush();
$this->logger->info(self::LOG_PREFIX.'Request generation finished', [
'template_id' => $message->getTemplateId(),
@@ -69,4 +120,23 @@ class RequestGenerationHandler implements MessageHandlerInterface
'duration_int' => (new \DateTimeImmutable('now'))->getTimestamp() - $message->getCreatedAt()->getTimestamp(),
]);
}
private function sendDataDump(StoredObject $destinationStoredObject, RequestGenerationMessage $message): void
{
$url = $this->tempUrlGenerator->generate('GET', $destinationStoredObject->getFilename(), 3600);
$parts = [];
parse_str(parse_url((string) $url->url)['query'], $parts);
$validity = \DateTimeImmutable::createFromFormat('U', $parts['temp_url_expires']);
$email = (new TemplatedEmail())
->to($message->getSendResultToEmail())
->textTemplate('@ChillDocGenerator/Email/send_data_dump_to_admin.txt.twig')
->context([
'link' => $url->url,
'validity' => $validity,
])
->subject($this->translator->trans('docgen.data_dump_email.subject'));
$this->mailer->send($email);
}
}

View File

@@ -15,27 +15,33 @@ use Chill\DocGeneratorBundle\Entity\DocGeneratorTemplate;
use Chill\DocStoreBundle\Entity\StoredObject;
use Chill\MainBundle\Entity\User;
class RequestGenerationMessage
final readonly class RequestGenerationMessage
{
private readonly int $creatorId;
private int $creatorId;
private readonly int $templateId;
private int $templateId;
private readonly int $destinationStoredObjectId;
private int $destinationStoredObjectId;
private readonly \DateTimeImmutable $createdAt;
private \DateTimeImmutable $createdAt;
private ?string $sendResultToEmail;
public function __construct(
User $creator,
DocGeneratorTemplate $template,
private readonly int $entityId,
private int $entityId,
StoredObject $destinationStoredObject,
private readonly array $contextGenerationData
private array $contextGenerationData,
private bool $isTest = false,
?string $sendResultToEmail = null,
private bool $dumpOnly = false,
) {
$this->creatorId = $creator->getId();
$this->templateId = $template->getId();
$this->destinationStoredObjectId = $destinationStoredObject->getId();
$this->createdAt = new \DateTimeImmutable('now');
$this->sendResultToEmail = $sendResultToEmail ?? $creator->getEmail();
}
public function getCreatorId(): int
@@ -67,4 +73,19 @@ class RequestGenerationMessage
{
return $this->createdAt;
}
public function isTest(): bool
{
return $this->isTest;
}
public function getSendResultToEmail(): ?string
{
return $this->sendResultToEmail;
}
public function isDumpOnly(): bool
{
return $this->dumpOnly;
}
}