Feature: [docgen] generate documents in an async queue

The documents are now generated in a queue, using symfony messenger. This queue should be configured: ```yaml # app/config/messenger.yaml framework: messenger: # reset services after consuming messages # reset_on_message: true failure_transport: failed transports: # https://symfony.com/doc/current/messenger.html#transport-configuration async: '%env(MESSENGER_TRANSPORT_DSN)%' priority: dsn: '%env(MESSENGER_TRANSPORT_DSN)%' failed: 'doctrine://default?queue_name=failed' routing: # ... other messages 'Chill\DocGeneratorBundle\Service\Messenger\RequestGenerationMessage': priority ``` `StoredObject`s now have additionnal properties: * status (pending, failure, ready (by default) ), which explain if the document is generated; * a generationTrialCounter, which is incremented on each generation trial, which prevent each generation more than 5 times; The generator computation is moved from the `DocGenTemplateController` to a `Generator` (implementing `GeneratorInterface`. There are new methods to `Context` which allow to normalize/denormalize context data to/from a messenger's `Message`.
2025-09-02 04:53:49 +00:00 · 2023-02-28 15:25:47 +00:00
parent 27f13e0dd1
commit a16244a3f5
40 changed files with 1050 additions and 177 deletions
--- a/src/Bundle/ChillDocGeneratorBundle/Service/Generator/Generator.php
+++ b/src/Bundle/ChillDocGeneratorBundle/Service/Generator/Generator.php
@@ -3,6 +3,7 @@
 namespace Chill\DocGeneratorBundle\Service\Generator;

 use Chill\DocGeneratorBundle\Context\ContextManagerInterface;
+use Chill\DocGeneratorBundle\Context\DocGeneratorContextWithPublicFormInterface;
 use Chill\DocGeneratorBundle\Entity\DocGeneratorTemplate;
 use Chill\DocGeneratorBundle\GeneratorDriver\DriverInterface;
 use Chill\DocGeneratorBundle\GeneratorDriver\Exception\TemplateException;
@@ -12,7 +13,7 @@ use Doctrine\ORM\EntityManagerInterface;
 use Psr\Log\LoggerInterface;
 use Symfony\Component\HttpFoundation\File\File;

-class Generator
+class Generator implements GeneratorInterface
 {
    private ContextManagerInterface $contextManager;

@@ -24,6 +25,8 @@ class Generator

    private StoredObjectManagerInterface $storedObjectManager;

+    private const LOG_PREFIX = '[docgen generator] ';
+
    public function __construct(
        ContextManagerInterface $contextManager,
        DriverInterface $driver,
@@ -48,18 +51,23 @@ class Generator
     */
    public function generateDocFromTemplate(
        DocGeneratorTemplate $template,
-        string               $entityClassName,
        int                  $entityId,
+        array                $contextGenerationDataNormalized,
        ?StoredObject        $destinationStoredObject = null,
        bool                 $isTest = false,
        ?File                $testFile = null
    ): ?string {
        if ($destinationStoredObject instanceof StoredObject && StoredObject::STATUS_PENDING !== $destinationStoredObject->getStatus()) {
+            $this->logger->info(self::LOG_PREFIX.'Aborting generation of an already generated document');
            throw new ObjectReadyException();
        }

+        $this->logger->info(self::LOG_PREFIX.'Starting generation of a document', [
+            'entity_id' => $entityId,
+            'destination_stored_object' => $destinationStoredObject === null ? null : $destinationStoredObject->getId()
+        ]);
+
        $context = $this->contextManager->getContextByDocGeneratorTemplate($template);
-        $contextGenerationData = ['test_file' => $testFile];

        $entity = $this
            ->entityManager
@@ -67,22 +75,38 @@ class Generator
            ;

        if (null === $entity) {
-            throw new RelatedEntityNotFoundException($entityClassName, $entityId);
+            throw new RelatedEntityNotFoundException($template->getEntity(), $entityId);
+        }
+
+        $contextGenerationDataNormalized = array_merge(
+            $contextGenerationDataNormalized,
+                $context instanceof DocGeneratorContextWithPublicFormInterface ?
+                    $context->contextGenerationDataDenormalize($template, $entity, $contextGenerationDataNormalized)
+                    : []
+        );
+
+        $data = $context->getData($template, $entity, $contextGenerationDataNormalized);
+
+        $destinationStoredObjectId = $destinationStoredObject instanceof StoredObject ? $destinationStoredObject->getId() : null;
+        $this->entityManager->clear();
+        gc_collect_cycles();
+        if (null !== $destinationStoredObjectId) {
+            $destinationStoredObject = $this->entityManager->find(StoredObject::class, $destinationStoredObjectId);
        }

        if ($isTest && ($testFile instanceof File)) {
-            $dataDecrypted = file_get_contents($testFile->getPathname());
+            $templateDecrypted = file_get_contents($testFile->getPathname());
        } else {
-            $dataDecrypted = $this->storedObjectManager->read($template->getFile());
+            $templateDecrypted = $this->storedObjectManager->read($template->getFile());
        }

        try {
            $generatedResource = $this
                ->driver
                ->generateFromString(
-                    $dataDecrypted,
+                    $templateDecrypted,
                    $template->getFile()->getType(),
-                    $context->getData($template, $entity, $contextGenerationData),
+                    $data,
                    $template->getFile()->getFilename()
                );
        } catch (TemplateException $e) {
@@ -90,6 +114,11 @@ class Generator
        }

        if ($isTest) {
+            $this->logger->info(self::LOG_PREFIX.'Finished generation of a document', [
+                'is_test' => true,
+                'entity_id' => $entityId,
+                'destination_stored_object' => $destinationStoredObject === null ? null : $destinationStoredObject->getId()
+            ]);
            return $generatedResource;
        }

@@ -102,31 +131,13 @@ class Generator

        $this->storedObjectManager->write($destinationStoredObject, $generatedResource);

-        try {
-            $context
-                ->storeGenerated(
-                    $template,
-                    $destinationStoredObject,
-                    $entity,
-                    $contextGenerationData
-                );
-        } catch (\Exception $e) {
-            $this
-                ->logger
-                ->error(
-                    'Unable to store the associated document to entity',
-                    [
-                        'entityClassName' => $entityClassName,
-                        'entityId' => $entityId,
-                        'contextKey' => $context->getName(),
-                    ]
-                );
-
-            throw $e;
-        }
-
        $this->entityManager->flush();

+        $this->logger->info(self::LOG_PREFIX.'Finished generation of a document', [
+            'entity_id' => $entityId,
+            'destination_stored_object' => $destinationStoredObject->getId(),
+        ]);
+
        return null;
    }
 }
--- a/src/Bundle/ChillDocGeneratorBundle/Service/Generator/GeneratorException.php
+++ b/src/Bundle/ChillDocGeneratorBundle/Service/Generator/GeneratorException.php
@@ -15,4 +15,12 @@ class GeneratorException extends \RuntimeException
        parent::__construct("Could not generate the document", 15252,
            $previous);
    }
+
+    /**
+     * @return array
+     */
+    public function getErrors(): array
+    {
+        return $this->errors;
+    }
 }
--- a/src/Bundle/ChillDocGeneratorBundle/Service/Generator/GeneratorInterface.php
+++ b/src/Bundle/ChillDocGeneratorBundle/Service/Generator/GeneratorInterface.php
@@ -0,0 +1,27 @@
+<?php
+
+namespace Chill\DocGeneratorBundle\Service\Generator;
+
+use Chill\DocGeneratorBundle\Entity\DocGeneratorTemplate;
+use Chill\DocStoreBundle\Entity\StoredObject;
+use Symfony\Component\HttpFoundation\File\File;
+
+interface GeneratorInterface
+{
+    /**
+     * @template T of File|null
+     * @template B of bool
+     * @param B $isTest
+     * @param (B is true ? T : null) $testFile
+     * @psalm-return (B is true ? string : null)
+     * @throws \Symfony\Component\Serializer\Exception\ExceptionInterface|\Throwable
+     */
+    public function generateDocFromTemplate(
+        DocGeneratorTemplate $template,
+        int                  $entityId,
+        array                $contextGenerationDataNormalized,
+        ?StoredObject        $destinationStoredObject = null,
+        bool                 $isTest = false,
+        ?File                $testFile = null
+    ): ?string;
+}
--- a/src/Bundle/ChillDocGeneratorBundle/Service/Messenger/OnGenerationFails.php
+++ b/src/Bundle/ChillDocGeneratorBundle/Service/Messenger/OnGenerationFails.php
@@ -0,0 +1,156 @@
+<?php
+
+namespace Chill\DocGeneratorBundle\Service\Messenger;
+
+use Chill\DocGeneratorBundle\Repository\DocGeneratorTemplateRepository;
+use Chill\DocGeneratorBundle\Service\Generator\GeneratorException;
+use Chill\DocStoreBundle\Entity\StoredObject;
+use Chill\DocStoreBundle\Repository\StoredObjectRepository;
+use Chill\MainBundle\Repository\UserRepositoryInterface;
+use Doctrine\ORM\EntityManagerInterface;
+use Psr\Log\LoggerInterface;
+use Symfony\Bridge\Twig\Mime\TemplatedEmail;
+use Symfony\Component\EventDispatcher\EventSubscriberInterface;
+use Symfony\Component\Mailer\MailerInterface;
+use Symfony\Component\Messenger\Event\WorkerMessageFailedEvent;
+use Symfony\Contracts\Translation\TranslatorInterface;
+
+final class OnGenerationFails implements EventSubscriberInterface
+{
+    private DocGeneratorTemplateRepository $docGeneratorTemplateRepository;
+
+    private EntityManagerInterface $entityManager;
+
+    private LoggerInterface $logger;
+
+    private MailerInterface $mailer;
+
+    private StoredObjectRepository $storedObjectRepository;
+
+    private TranslatorInterface $translator;
+
+    private UserRepositoryInterface $userRepository;
+
+    const LOG_PREFIX = '[docgen failed] ';
+
+    /**
+     * @param DocGeneratorTemplateRepository $docGeneratorTemplateRepository
+     * @param EntityManagerInterface $entityManager
+     * @param LoggerInterface $logger
+     * @param MailerInterface $mailer
+     * @param StoredObjectRepository $storedObjectRepository
+     * @param TranslatorInterface $translator
+     * @param UserRepositoryInterface $userRepository
+     */
+    public function __construct(
+        DocGeneratorTemplateRepository $docGeneratorTemplateRepository,
+        EntityManagerInterface $entityManager,
+        LoggerInterface $logger,
+        MailerInterface $mailer,
+        StoredObjectRepository $storedObjectRepository,
+        TranslatorInterface $translator,
+        UserRepositoryInterface $userRepository
+    ) {
+        $this->docGeneratorTemplateRepository = $docGeneratorTemplateRepository;
+        $this->entityManager = $entityManager;
+        $this->logger = $logger;
+        $this->mailer = $mailer;
+        $this->storedObjectRepository = $storedObjectRepository;
+        $this->translator = $translator;
+        $this->userRepository = $userRepository;
+    }
+
+
+    public static function getSubscribedEvents()
+    {
+        return [
+            WorkerMessageFailedEvent::class => 'onMessageFailed'
+        ];
+    }
+
+    public function onMessageFailed(WorkerMessageFailedEvent $event): void
+    {
+        if ($event->willRetry()) {
+            return;
+        }
+
+        if (!$event->getEnvelope()->getMessage() instanceof RequestGenerationMessage) {
+            return;
+        }
+
+        /** @var \Chill\DocGeneratorBundle\Service\Messenger\RequestGenerationMessage $message */
+        $message = $event->getEnvelope()->getMessage();
+
+        $this->logger->error(self::LOG_PREFIX.'Docgen failed', [
+            'stored_object_id' => $message->getDestinationStoredObjectId(),
+            'entity_id' => $message->getEntityId(),
+            'template_id' => $message->getTemplateId(),
+            'creator_id' => $message->getCreatorId(),
+            'throwable_class' => get_class($event->getThrowable()),
+        ]);
+
+        $this->markObjectAsFailed($message);
+        $this->warnCreator($message, $event);
+    }
+
+    private function markObjectAsFailed(RequestGenerationMessage $message): void
+    {
+        $object = $this->storedObjectRepository->find($message->getDestinationStoredObjectId());
+
+        if (null === $object) {
+            $this->logger->error(self::LOG_PREFIX.'Stored object not found', ['stored_object_id', $message->getDestinationStoredObjectId()]);
+        }
+
+        $object->setStatus(StoredObject::STATUS_FAILURE);
+
+        $this->entityManager->flush();
+    }
+
+    private function warnCreator(RequestGenerationMessage $message, WorkerMessageFailedEvent $event): void
+    {
+        if (null === $creatorId = $message->getCreatorId()) {
+            $this->logger->info(self::LOG_PREFIX.'creator id is null');
+            return;
+        }
+
+        if (null === $creator = $this->userRepository->find($creatorId)) {
+            $this->logger->error(self::LOG_PREFIX.'Creator not found with given id', ['creator_id', $creatorId]);
+            return;
+        }
+
+        if (null === $creator->getEmail() || '' === $creator->getEmail()) {
+            $this->logger->info(self::LOG_PREFIX.'Creator does not have any email', ['user' => $creator->getUsernameCanonical()]);
+            return;
+        }
+
+        // if the exception is not a GeneratorException, we try the previous one...
+        $throwable = $event->getThrowable();
+        if (!$throwable instanceof GeneratorException) {
+            $throwable = $throwable->getPrevious();
+        }
+
+        if ($throwable instanceof GeneratorException) {
+            $errors = $throwable->getErrors();
+        } else {
+            $errors = [$throwable->getTraceAsString()];
+        }
+
+        if (null === $template = $this->docGeneratorTemplateRepository->find($message->getTemplateId())) {
+            $this->logger->info(self::LOG_PREFIX.'Template not found', ['template_id' => $message->getTemplateId()]);
+            return;
+        }
+
+        $email = (new TemplatedEmail())
+            ->to($creator->getEmail())
+            ->subject($this->translator->trans('docgen.failure_email.The generation of a document failed'))
+            ->textTemplate('@ChillDocGenerator/Email/on_generation_failed_email.txt.twig')
+            ->context([
+                'errors' => $errors,
+                'template' => $template,
+                'creator' => $creator,
+                'stored_object_id' => $message->getDestinationStoredObjectId(),
+            ]);
+
+        $this->mailer->send($email);
+    }
+}
--- a/src/Bundle/ChillDocGeneratorBundle/Service/Messenger/RequestGenerationHandler.php
+++ b/src/Bundle/ChillDocGeneratorBundle/Service/Messenger/RequestGenerationHandler.php
@@ -0,0 +1,66 @@
+<?php
+
+namespace Chill\DocGeneratorBundle\Service\Messenger;
+
+use Chill\DocGeneratorBundle\Repository\DocGeneratorTemplateRepository;
+use Chill\DocGeneratorBundle\Service\Generator\Generator;
+use Chill\DocStoreBundle\Entity\StoredObject;
+use Chill\DocStoreBundle\Repository\StoredObjectRepository;
+use Doctrine\ORM\EntityManagerInterface;
+use Symfony\Component\Messenger\Exception\UnrecoverableMessageHandlingException;
+use Symfony\Component\Messenger\Handler\MessageHandlerInterface;
+
+/**
+ * Handle the request of document generation
+ */
+class RequestGenerationHandler implements MessageHandlerInterface
+{
+    private StoredObjectRepository $storedObjectRepository;
+
+    private DocGeneratorTemplateRepository $docGeneratorTemplateRepository;
+
+    private EntityManagerInterface $entityManager;
+
+    private Generator $generator;
+
+    public const AUTHORIZED_TRIALS = 5;
+
+    public function __construct(
+        DocGeneratorTemplateRepository $docGeneratorTemplateRepository,
+        EntityManagerInterface $entityManager,
+        Generator $generator,
+        StoredObjectRepository $storedObjectRepository,
+    ) {
+        $this->docGeneratorTemplateRepository = $docGeneratorTemplateRepository;
+        $this->entityManager = $entityManager;
+        $this->generator = $generator;
+        $this->storedObjectRepository = $storedObjectRepository;
+    }
+
+    public function __invoke(RequestGenerationMessage $message)
+    {
+        if (null === $template = $this->docGeneratorTemplateRepository->find($message->getTemplateId())) {
+            throw new \RuntimeException('template not found: ' . $message->getTemplateId());
+        }
+
+        if (null === $destinationStoredObject = $this->storedObjectRepository->find($message->getDestinationStoredObjectId())) {
+            throw new \RuntimeException('destination stored object not found : ' . $message->getDestinationStoredObjectId());
+        }
+
+        if ($destinationStoredObject->getGenerationTrialsCounter() >= self::AUTHORIZED_TRIALS) {
+            throw new UnrecoverableMessageHandlingException('maximum number of retry reached');
+        }
+
+        $destinationStoredObject->addGenerationTrial();
+        $this->entityManager->createQuery('UPDATE '.StoredObject::class.' s SET s.generationTrialsCounter = s.generationTrialsCounter + 1 WHERE s.id = :id')
+            ->setParameter('id', $destinationStoredObject->getId())
+            ->execute();
+
+        $this->generator->generateDocFromTemplate(
+            $template,
+            $message->getEntityId(),
+            $message->getContextGenerationData(),
+            $destinationStoredObject
+        );
+    }
+}
--- a/src/Bundle/ChillDocGeneratorBundle/Service/Messenger/RequestGenerationMessage.php
+++ b/src/Bundle/ChillDocGeneratorBundle/Service/Messenger/RequestGenerationMessage.php
@@ -3,6 +3,7 @@
 namespace Chill\DocGeneratorBundle\Service\Messenger;

 use Chill\DocGeneratorBundle\Entity\DocGeneratorTemplate;
+use Chill\DocStoreBundle\Entity\StoredObject;
 use Chill\MainBundle\Entity\User;

 class RequestGenerationMessage
@@ -13,14 +14,22 @@ class RequestGenerationMessage

   private int $entityId;

-   private string $entityClassName;
+   private int $destinationStoredObjectId;

-    public function __construct(User $creator, DocGeneratorTemplate $template, int $entityId, string $entityClassName)
-    {
+   private array $contextGenerationData;
+
+    public function __construct(
+        User $creator,
+        DocGeneratorTemplate $template,
+        int $entityId,
+        StoredObject $destinationStoredObject,
+        array $contextGenerationData
+    ) {
        $this->creatorId = $creator->getId();
        $this->templateId = $template->getId();
        $this->entityId = $entityId;
-        $this->entityClassName = $entityClassName;
+        $this->destinationStoredObjectId = $destinationStoredObject->getId();
+        $this->contextGenerationData = $contextGenerationData;
    }

    public function getCreatorId(): int
@@ -28,6 +37,11 @@ class RequestGenerationMessage
        return $this->creatorId;
    }

+    public function getDestinationStoredObjectId(): int
+    {
+        return $this->destinationStoredObjectId;
+    }
+
    public function getTemplateId(): int
    {
        return $this->templateId;
@@ -38,8 +52,8 @@ class RequestGenerationMessage
        return $this->entityId;
    }

-    public function getEntityClassName(): string
+    public function getContextGenerationData(): array
    {
-        return $this->entityClassName;
+        return $this->contextGenerationData;
    }
 }