diff --git a/config/packages/messenger.yaml b/config/packages/messenger.yaml index b460fd60c..78fcfb26f 100644 --- a/config/packages/messenger.yaml +++ b/config/packages/messenger.yaml @@ -5,6 +5,10 @@ framework: # Uncomment this (and the failed transport below) to send failed messages to this transport for later handling. failure_transport: failed + buses: + messenger.bus.default: + middleware: + - 'Chill\MainBundle\Messenger\Middleware\AuthenticationMiddleware' transports: # those transports are added by chill-bundles recipes @@ -19,7 +23,9 @@ framework: async: ~ auto_setup: true - priority: '%env(MESSENGER_TRANSPORT_DSN)%/priority' + priority: + dsn: '%env(MESSENGER_TRANSPORT_DSN)%/priority' + # end of transports added by chill-bundles recipes # https://symfony.com/doc/current/messenger.html#transport-configuration failed: 'doctrine://default?queue_name=failed' @@ -61,6 +67,7 @@ framework: 'Chill\MainBundle\Workflow\Messenger\PostSignatureStateChangeMessage': priority 'Chill\MainBundle\Workflow\Messenger\PostPublicViewMessage': async 'Chill\MainBundle\Service\Workflow\CancelStaleWorkflowMessage': async + 'Chill\MainBundle\Export\Messenger\ExportRequestGenerationMessage': priority # end of routes added by chill-bundles recipes # Route your messages to the transports # 'App\Message\YourMessage': async diff --git a/docs/source/development/export-sequence.puml b/docs/source/development/export-sequence.puml new file mode 100644 index 000000000..0d0c77c14 --- /dev/null +++ b/docs/source/development/export-sequence.puml @@ -0,0 +1,84 @@ +@startuml +'https://plantuml.com/sequence-diagram + +autonumber + +User -> ExportController: configure export using form +activate ExportController +ExportController -> ExportForm: build form +activate ExportForm + +loop for every ExportElement (Filter, Aggregator) + ExportForm -> ExportElement: `buildForm` + activate ExportElement + ExportElement -> ExportForm: add form to builders + deactivate ExportElement +end + +ExportForm -> ExportController +deactivate ExportForm + +ExportController -> User: show form +deactivate ExportController + +note left of User: Configure the export:\ncheck filters, aggregators, … + +User -> ExportController: post configuration of the export +activate ExportController + +ExportController -> ExportForm: `getData` +activate ExportForm +ExportForm -> ExportController: return data: list of entities, etc. +deactivate ExportForm + +loop for every ExportElement (Filter, Aggregator) + ExportController -> ExportElement: serializeData (data) + activate ExportElement + ExportElement -> ExportController: return serializedData (simple array with string, int, …) + deactivate ExportElement +end + +ExportController -> Database: `INSERT INTO RequestGeneration_table` (insert new entity) +ExportController -> MessageQueue: warn about a new request +activate MessageQueue +ExportController -> User: "ok, generation is in process" +deactivate ExportController + +note left of User: The user see a waiting screen + +MessageQueue -> MessengerConsumer: forward the message to the MessengerConsumer +deactivate MessageQueue +activate MessengerConsumer +MessengerConsumer -> Database: `SELECT * FROM RequestGeneration_table WHERE id = %s` +activate Database +Database -> MessengerConsumer: return RequestGeneration with serializedData +deactivate Database + +loop for every ExportElement (Filter, Aggregator) + MessengerConsumer -> ExportElement: deserializeData + activate ExportElement + ExportElement -> MessengerConsumer: return data (list of entities, etc.) from the serialized array + deactivate ExportElement + MessengerConsumer -> ExportElement: alter the sql query (`ExportElement::alterQuery`) + activate ExportElement + ExportElement -> MessengerConsumer: return the query with WHERE and GROUP BY clauses + deactivate ExportElement +end + +MessengerConsumer -> MessengerConsumer: prepare the export +MessengerConsumer -> MessengerConsumer: save the export as a stored object +MessengerConsumer -> Database: `UPDATE RequestGeneration_table SET ready = true` +deactivate MessengerConsumer + +User -> ExportController: pull every 5s to know if the export is generated +activate ExportController +ExportController -> User: warn the export is generated +deactivate ExportController + +User -> ExportController: download the export from object storage + + + + + +@enduml diff --git a/package.json b/package.json index a24c4ee8f..e2b8d8ba3 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "@hotwired/stimulus": "^3.0.0", "@luminateone/eslint-baseline": "^1.0.9", "@symfony/stimulus-bridge": "^3.2.0", + "@symfony/ux-translator": "file:vendor/symfony/ux-translator/assets", "@symfony/webpack-encore": "^4.1.0", "@tsconfig/node20": "^20.1.4", "@types/dompurify": "^3.0.5", diff --git a/src/Bundle/ChillMainBundle/Controller/ExportController.php b/src/Bundle/ChillMainBundle/Controller/ExportController.php index 08b89e4bb..28f2bbe1a 100644 --- a/src/Bundle/ChillMainBundle/Controller/ExportController.php +++ b/src/Bundle/ChillMainBundle/Controller/ExportController.php @@ -11,22 +11,26 @@ declare(strict_types=1); namespace Chill\MainBundle\Controller; +use Chill\MainBundle\Entity\ExportGeneration; use Chill\MainBundle\Entity\SavedExport; use Chill\MainBundle\Entity\User; use Chill\MainBundle\Export\DirectExportInterface; use Chill\MainBundle\Export\ExportFormHelper; use Chill\MainBundle\Export\ExportInterface; use Chill\MainBundle\Export\ExportManager; +use Chill\MainBundle\Export\Messenger\ExportRequestGenerationMessage; use Chill\MainBundle\Form\SavedExportType; use Chill\MainBundle\Form\Type\Export\ExportType; use Chill\MainBundle\Form\Type\Export\FormatterType; use Chill\MainBundle\Form\Type\Export\PickCenterType; +use Chill\MainBundle\Messenger\Stamp\AuthenticationStamp; use Chill\MainBundle\Redis\ChillRedis; use Chill\MainBundle\Repository\SavedExportRepositoryInterface; use Chill\MainBundle\Security\Authorization\SavedExportVoter; use Doctrine\ORM\EntityManagerInterface; use Psr\Log\LoggerInterface; use Symfony\Bundle\FrameworkBundle\Controller\AbstractController; +use Symfony\Component\Clock\ClockInterface; use Symfony\Component\DependencyInjection\ParameterBag\ParameterBagInterface; use Symfony\Component\Form\Extension\Core\Type\FormType; use Symfony\Component\Form\Extension\Core\Type\SubmitType; @@ -37,6 +41,8 @@ use Symfony\Component\HttpFoundation\Request; use Symfony\Component\HttpFoundation\Response; use Symfony\Component\HttpFoundation\Session\SessionInterface; use Symfony\Component\HttpKernel\Exception\AccessDeniedHttpException; +use Symfony\Component\Messenger\Envelope; +use Symfony\Component\Messenger\MessageBusInterface; use Symfony\Component\Routing\Annotation\Route; use Symfony\Component\Security\Core\Security; use Symfony\Contracts\Translation\TranslatorInterface; @@ -61,6 +67,8 @@ class ExportController extends AbstractController private readonly SavedExportRepositoryInterface $savedExportRepository, private readonly Security $security, ParameterBagInterface $parameterBag, + private readonly MessageBusInterface $messageBus, + private readonly ClockInterface $clock, ) { $this->filterStatsByCenters = $parameterBag->get('chill_main')['acl']['filter_stats_by_center']; } @@ -128,22 +136,21 @@ class ExportController extends AbstractController * @throws \RedisException */ #[Route(path: '/{_locale}/exports/generate-from-saved/{id}', name: 'chill_main_export_generate_from_saved')] - public function generateFromSavedExport(SavedExport $savedExport): RedirectResponse + public function generateFromSavedExport(SavedExport $savedExport): Response { $this->denyAccessUnlessGranted(SavedExportVoter::GENERATE, $savedExport); - $key = md5(uniqid((string) random_int(0, mt_getrandmax()), false)); + $exportGeneration = ExportGeneration::fromSavedExport($savedExport, $this->clock->now()->add(new \DateInterval('P3M'))); + $this->entityManager->persist($exportGeneration); + $this->entityManager->flush(); - $this->redis->setEx($key, 3600, \serialize($savedExport->getOptions())); + $this->messageBus->dispatch( + new Envelope( + new ExportRequestGenerationMessage($exportGeneration), + [new AuthenticationStamp($this->security->getUser())] + )); - return $this->redirectToRoute( - 'chill_main_export_download', - [ - 'alias' => $savedExport->getExportAlias(), - 'key' => $key, 'prevent_save' => true, - 'returnPath' => $this->generateUrl('chill_main_export_saved_list_my'), - ] - ); + return new Response('Ok: '.$exportGeneration->getId()->toString()); } /** diff --git a/src/Bundle/ChillMainBundle/Export/DirectExportInterface.php b/src/Bundle/ChillMainBundle/Export/DirectExportInterface.php index 0949b0be0..b06944eb8 100644 --- a/src/Bundle/ChillMainBundle/Export/DirectExportInterface.php +++ b/src/Bundle/ChillMainBundle/Export/DirectExportInterface.php @@ -28,8 +28,10 @@ interface DirectExportInterface extends ExportElementInterface /** * Generate the export. + * + * @return FormattedExportGeneration */ - public function generate(array $acl, array $data = []): Response; + public function generate(array $acl, array $data = []): Response|FormattedExportGeneration; /** * get a description, which will be used in UI (and translated). diff --git a/src/Bundle/ChillMainBundle/Export/ExportFormHelper.php b/src/Bundle/ChillMainBundle/Export/ExportFormHelper.php index ce43c230a..b76e91dfd 100644 --- a/src/Bundle/ChillMainBundle/Export/ExportFormHelper.php +++ b/src/Bundle/ChillMainBundle/Export/ExportFormHelper.php @@ -11,7 +11,7 @@ declare(strict_types=1); namespace Chill\MainBundle\Export; -use Chill\MainBundle\Entity\SavedExport; +use Chill\MainBundle\Entity\ExportGeneration; use Chill\MainBundle\Form\Type\Export\ExportType; use Chill\MainBundle\Form\Type\Export\FilterType; use Chill\MainBundle\Form\Type\Export\FormatterType; @@ -91,7 +91,7 @@ final readonly class ExportFormHelper } public function savedExportDataToFormData( - SavedExport $savedExport, + ExportGeneration $savedExport, string $step, array $formOptions = [], ): array { @@ -104,7 +104,7 @@ final readonly class ExportFormHelper } private function savedExportDataToFormDataStepCenter( - SavedExport $savedExport, + ExportGeneration $savedExport, ): array { $builder = $this->formFactory ->createBuilder( @@ -125,7 +125,7 @@ final readonly class ExportFormHelper } private function savedExportDataToFormDataStepExport( - SavedExport $savedExport, + ExportGeneration $savedExport, array $formOptions, ): array { $builder = $this->formFactory @@ -147,7 +147,7 @@ final readonly class ExportFormHelper } private function savedExportDataToFormDataStepFormatter( - SavedExport $savedExport, + ExportGeneration $savedExport, array $formOptions, ): array { $builder = $this->formFactory diff --git a/src/Bundle/ChillMainBundle/Export/ExportGenerationContext.php b/src/Bundle/ChillMainBundle/Export/ExportGenerationContext.php new file mode 100644 index 000000000..0ffb29c20 --- /dev/null +++ b/src/Bundle/ChillMainBundle/Export/ExportGenerationContext.php @@ -0,0 +1,21 @@ +entityManager->wrapInTransaction(function () use ($exportGeneration) { + $object = $exportGeneration->getStoredObject(); + $this->entityManager->refresh($exportGeneration, LockMode::PESSIMISTIC_WRITE); + $this->entityManager->refresh($object, LockMode::PESSIMISTIC_WRITE); + + if (StoredObject::STATUS_PENDING !== $object->getStatus()) { + return; + } + + $generation = $this->exportManager->generateExport( + $exportGeneration->getExportAlias(), + $centers = $this->exportFormHelper->savedExportDataToFormData($exportGeneration, 'centers'), + $this->exportFormHelper->savedExportDataToFormData($exportGeneration, 'export', ['picked_centers' => $centers]), + $this->exportFormHelper->savedExportDataToFormData($exportGeneration, 'formatter', ['picked_centers' => $centers]), + $user, + ); + + $this->storedObjectManager->write($exportGeneration->getStoredObject(), $generation->content, $generation->contentType); + }); + } +} diff --git a/src/Bundle/ChillMainBundle/Export/ExportManager.php b/src/Bundle/ChillMainBundle/Export/ExportManager.php index 949523eb0..103441408 100644 --- a/src/Bundle/ChillMainBundle/Export/ExportManager.php +++ b/src/Bundle/ChillMainBundle/Export/ExportManager.php @@ -165,25 +165,31 @@ class ExportManager $this->formatters[$alias] = $formatter; } - /** - * Generate a response which contains the requested data. - */ - public function generate(string $exportAlias, array $pickedCentersData, array $data, array $formatterData): Response + public function generateExport(string $exportAlias, array $pickedCentersData, array $data, array $formatterData, User $byUser): FormattedExportGeneration { $export = $this->getExport($exportAlias); $centers = $this->getPickedCenters($pickedCentersData); + $context = new ExportGenerationContext($byUser); if ($export instanceof DirectExportInterface) { - return $export->generate( + $generatedExport = $export->generate( $this->buildCenterReachableScopes($centers, $export), - $data[ExportType::EXPORT_KEY] + $data[ExportType::EXPORT_KEY], ); + + if ($generatedExport instanceof Response) { + trigger_deprecation('chill-project/chill-bundles', '3.10', 'DirectExportInterface should not return a %s instance, but a %s instance', Response::class, FormattedExportGeneration::class); + + return new FormattedExportGeneration($generatedExport->getContent(), $generatedExport->headers->get('Content-Type')); + } + + return $generatedExport; } $query = $export->initiateQuery( $this->retrieveUsedModifiers($data), $this->buildCenterReachableScopes($centers, $export), - $data[ExportType::EXPORT_KEY] + $export->denormalizeFormData($data[ExportType::EXPORT_KEY], $context), ); if ($query instanceof \Doctrine\ORM\NativeQuery) { @@ -194,10 +200,10 @@ class ExportManager } } elseif ($query instanceof QueryBuilder) { // handle filters - $this->handleFilters($export, $query, $data[ExportType::FILTER_KEY], $centers); + $this->handleFilters($export, $query, $data[ExportType::FILTER_KEY], $centers, $context); // handle aggregators - $this->handleAggregators($export, $query, $data[ExportType::AGGREGATOR_KEY], $centers); + $this->handleAggregators($export, $query, $data[ExportType::AGGREGATOR_KEY], $centers, $context); $this->logger->notice('[export] will execute this qb in export', [ 'dql' => $query->getDQL(), @@ -206,7 +212,7 @@ class ExportManager throw new \UnexpectedValueException('The method `intiateQuery` should return a `\Doctrine\ORM\NativeQuery` or a `Doctrine\ORM\QueryBuilder` object.'); } - $result = $export->getResult($query, $data[ExportType::EXPORT_KEY]); + $result = $export->getResult($query, $export->denormalizeFormData($data[ExportType::EXPORT_KEY], $context)); if (!is_iterable($result)) { throw new \UnexpectedValueException(sprintf('The result of the export should be an iterable, %s given', \gettype($result))); @@ -231,14 +237,44 @@ class ExportManager $filtersData[$alias] = $data[ExportType::FILTER_KEY][$alias]['form']; } - return $formatter->getResponse( + if (method_exists($formatter, 'generate')) { + return $formatter->generate( + $result, + $formatterData, + $exportAlias, + $data[ExportType::EXPORT_KEY], + $filtersData, + $aggregatorsData, + ); + } + + trigger_deprecation('chill-project/chill-bundles', '3.10', '%s should implements the "generate" method', FormatterInterface::class); + + $generatedExport = $formatter->getResponse( $result, $formatterData, $exportAlias, $data[ExportType::EXPORT_KEY], $filtersData, - $aggregatorsData + $aggregatorsData, ); + + return new FormattedExportGeneration($generatedExport->getContent(), $generatedExport->headers->get('content-type')); + } + + /** + * Generate a response which contains the requested data. + */ + public function generate(string $exportAlias, array $pickedCentersData, array $data, array $formatterData): Response + { + $generated = $this->generateExport( + $exportAlias, + $pickedCentersData, + $data, + $formatterData, + ); + + return new Response($generated->content, headers: ['Content-Type' => $generated->contentType]); } /** @@ -453,6 +489,7 @@ class ExportManager DirectExportInterface|ExportInterface|null $export = null, ?array $centers = null, ): bool { + dump(__METHOD__, $this->tokenStorage->getToken()->getUser()); if ($element instanceof ExportInterface || $element instanceof DirectExportInterface) { $role = $element->requiredRole(); } else { @@ -473,7 +510,7 @@ class ExportManager $role ); } - +dump($centers); foreach ($centers as $center) { if (false === $this->authorizationChecker->isGranted($role, $center)) { // debugging @@ -534,16 +571,13 @@ class ExportManager QueryBuilder $qb, array $data, array $center, + ExportGenerationContext $context, ) { $aggregators = $this->retrieveUsedAggregators($data); foreach ($aggregators as $alias => $aggregator) { - if (false === $this->isGrantedForElement($aggregator, $export, $center)) { - throw new UnauthorizedHttpException('You are not authorized to use the aggregator'.$aggregator->getTitle()); - } - $formData = $data[$alias]; - $aggregator->alterQuery($qb, $formData['form']); + $aggregator->alterQuery($qb, $aggregator->denormalizeFormData($formData['form'], $context)); } } @@ -561,20 +595,17 @@ class ExportManager QueryBuilder $qb, mixed $data, array $centers, + ExportGenerationContext $context, ) { $filters = $this->retrieveUsedFilters($data); foreach ($filters as $alias => $filter) { - if (false === $this->isGrantedForElement($filter, $export, $centers)) { - throw new UnauthorizedHttpException('You are not authorized to use the filter '.$filter->getTitle()); - } - $formData = $data[$alias]; $this->logger->debug('alter query by filter '.$alias, [ 'class' => self::class, 'function' => __FUNCTION__, ]); - $filter->alterQuery($qb, $formData['form']); + $filter->alterQuery($qb, $filter->denormalizeFormData($formData['form'], $context)); } } diff --git a/src/Bundle/ChillMainBundle/Export/FormattedExportGeneration.php b/src/Bundle/ChillMainBundle/Export/FormattedExportGeneration.php new file mode 100644 index 000000000..678bb8f31 --- /dev/null +++ b/src/Bundle/ChillMainBundle/Export/FormattedExportGeneration.php @@ -0,0 +1,20 @@ +id = $exportGeneration->getId(); + $this->userId = $user->getId(); + } +} diff --git a/src/Bundle/ChillMainBundle/Export/Messenger/ExportRequestGenerationMessageHandler.php b/src/Bundle/ChillMainBundle/Export/Messenger/ExportRequestGenerationMessageHandler.php new file mode 100644 index 000000000..60079c982 --- /dev/null +++ b/src/Bundle/ChillMainBundle/Export/Messenger/ExportRequestGenerationMessageHandler.php @@ -0,0 +1,41 @@ +repository->find($exportRequestGenerationMessage->id)) { + throw new \UnexpectedValueException('ExportRequestGenerationMessage not found'); + } + + if (null === $user = $this->userRepository->find($exportRequestGenerationMessage->userId)) { + throw new \UnexpectedValueException('User not found'); + } + + $this->exportGenerator->generate($exportGeneration, $user); + } +} diff --git a/src/Bundle/ChillMainBundle/Messenger/Authentication/AuthenticatedMessengerToken.php b/src/Bundle/ChillMainBundle/Messenger/Authentication/AuthenticatedMessengerToken.php new file mode 100644 index 000000000..0cfb4d17f --- /dev/null +++ b/src/Bundle/ChillMainBundle/Messenger/Authentication/AuthenticatedMessengerToken.php @@ -0,0 +1,21 @@ +setUser($user); + $this->setAuthenticated(true); + } + + public function getCredentials(): null + { + return null; + } +} diff --git a/src/Bundle/ChillMainBundle/Messenger/Middleware/AuthenticationMiddleware.php b/src/Bundle/ChillMainBundle/Messenger/Middleware/AuthenticationMiddleware.php new file mode 100644 index 000000000..411713f56 --- /dev/null +++ b/src/Bundle/ChillMainBundle/Messenger/Middleware/AuthenticationMiddleware.php @@ -0,0 +1,47 @@ +last(AuthenticationStamp::class)) { + return; + /** @var AuthenticationStamp $authenticationStamp */ + dump("authenticate user", $authenticationStamp->getUserId()); + if (null !== $this->tokenStorage->getToken()) { + dump("token already present"); + + } else { + $user = $this->userProvider->loadUserByUsername($authenticationStamp->getUserId()); + $this->tokenStorage->setToken(new AuthenticatedMessengerToken($user, [...$user->getRoles(), 'IS_AUTHENTICATED_FULLY'])); + } + } + + return $stack->next()->handle($envelope, $stack); + } +} diff --git a/src/Bundle/ChillMainBundle/Messenger/Stamp/AuthenticationStamp.php b/src/Bundle/ChillMainBundle/Messenger/Stamp/AuthenticationStamp.php new file mode 100644 index 000000000..a290f3d36 --- /dev/null +++ b/src/Bundle/ChillMainBundle/Messenger/Stamp/AuthenticationStamp.php @@ -0,0 +1,33 @@ +userId = $user->getUserIdentifier(); + } + + public function getUserId(): string + { + return $this->userId; + } +} diff --git a/src/Bundle/ChillMainBundle/Repository/ExportGenerationRepository.php b/src/Bundle/ChillMainBundle/Repository/ExportGenerationRepository.php new file mode 100644 index 000000000..e23407e9a --- /dev/null +++ b/src/Bundle/ChillMainBundle/Repository/ExportGenerationRepository.php @@ -0,0 +1,27 @@ + + */ +class ExportGenerationRepository extends ServiceEntityRepository +{ + public function __construct(ManagerRegistry $registry) + { + parent::__construct($registry, ExportGeneration::class); + } +} diff --git a/src/Bundle/ChillMainBundle/config/services.yaml b/src/Bundle/ChillMainBundle/config/services.yaml index a9829f99d..1db9d25ef 100644 --- a/src/Bundle/ChillMainBundle/config/services.yaml +++ b/src/Bundle/ChillMainBundle/config/services.yaml @@ -101,6 +101,11 @@ services: autowire: true autoconfigure: true + Chill\MainBundle\Messenger\: + resource: '../Messenger/' + autowire: true + autoconfigure: true + Chill\MainBundle\Cron\: resource: '../Cron' autowire: true diff --git a/src/Bundle/ChillMainBundle/config/services/export.yaml b/src/Bundle/ChillMainBundle/config/services/export.yaml index ece7ae902..c1728f916 100644 --- a/src/Bundle/ChillMainBundle/config/services/export.yaml +++ b/src/Bundle/ChillMainBundle/config/services/export.yaml @@ -6,8 +6,12 @@ services: Chill\MainBundle\Export\Helper\: resource: '../../Export/Helper' + Chill\MainBundle\Export\Messenger\ExportRequestGenerationMessageHandler: ~ + Chill\MainBundle\Export\ExportFormHelper: ~ + Chill\MainBundle\Export\ExportGenerator: ~ + chill.main.export_element_validator: class: Chill\MainBundle\Validator\Constraints\Export\ExportElementConstraintValidator tags: diff --git a/src/Bundle/ChillPersonBundle/Export/Export/CountAccompanyingCourse.php b/src/Bundle/ChillPersonBundle/Export/Export/CountAccompanyingCourse.php index 1118b2cc4..93a00af5e 100644 --- a/src/Bundle/ChillPersonBundle/Export/Export/CountAccompanyingCourse.php +++ b/src/Bundle/ChillPersonBundle/Export/Export/CountAccompanyingCourse.php @@ -12,6 +12,7 @@ declare(strict_types=1); namespace Chill\PersonBundle\Export\Export; use Chill\MainBundle\Export\AccompanyingCourseExportHelper; +use Chill\MainBundle\Export\ExportGenerationContext; use Chill\MainBundle\Export\ExportInterface; use Chill\MainBundle\Export\FormatterInterface; use Chill\MainBundle\Export\GroupedExportInterface; @@ -49,6 +50,16 @@ class CountAccompanyingCourse implements ExportInterface, GroupedExportInterface return []; } + public function normalizeFormData(array $formData): array + { + return $formData; + } + + public function denormalizeFormData(array $formData, ExportGenerationContext $context): array + { + return $formData; + } + public function getAllowedFormattersTypes(): array { return [FormatterInterface::TYPE_TABULAR];