From ab311eaecbb12f950346c6bacd45339709688c2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Thu, 9 Jan 2025 12:21:10 +0100 Subject: [PATCH] Add email reporting for unimported addresses in import commands Enhanced address import commands to optionally send a recap of unimported addresses via email. Updated import logic to handle cases where postal codes are missing, log issues, and generate compressed CSV reports with failed entries. --- .../unreleased/Feature-20250109-121757.yaml | 5 + composer.json | 1 + .../LoadAddressesBEFromBestAddressCommand.php | 10 +- .../LoadAddressesFRFromBANOCommand.php | 6 +- .../LoadAddressesLUFromBDAddressCommand.php | 9 +- .../AddressReferenceBEFromBestAddress.php | 8 +- .../Import/AddressReferenceBaseImporter.php | 91 ++++++++++++++++--- .../Import/AddressReferenceFromBano.php | 4 +- .../Service/Import/AddressReferenceLU.php | 8 +- 9 files changed, 113 insertions(+), 29 deletions(-) create mode 100644 .changes/unreleased/Feature-20250109-121757.yaml diff --git a/.changes/unreleased/Feature-20250109-121757.yaml b/.changes/unreleased/Feature-20250109-121757.yaml new file mode 100644 index 000000000..87d9b29f8 --- /dev/null +++ b/.changes/unreleased/Feature-20250109-121757.yaml @@ -0,0 +1,5 @@ +kind: Feature +body: Importer for addresses does not fails when the postal code is not found with some addresses, and compute a recap list of all addresses that could not be imported. This recap list can be send by email. +time: 2025-01-09T12:17:57.981181677+01:00 +custom: + Issue: "" diff --git a/composer.json b/composer.json index 34426a7b8..00a89c4c7 100644 --- a/composer.json +++ b/composer.json @@ -13,6 +13,7 @@ "ext-json": "*", "ext-openssl": "*", "ext-redis": "*", + "ext-zlib": "*", "champs-libres/wopi-bundle": "dev-master@dev", "champs-libres/wopi-lib": "dev-master@dev", "doctrine/doctrine-bundle": "^2.1", diff --git a/src/Bundle/ChillMainBundle/Command/LoadAddressesBEFromBestAddressCommand.php b/src/Bundle/ChillMainBundle/Command/LoadAddressesBEFromBestAddressCommand.php index dfe485f04..db852c650 100644 --- a/src/Bundle/ChillMainBundle/Command/LoadAddressesBEFromBestAddressCommand.php +++ b/src/Bundle/ChillMainBundle/Command/LoadAddressesBEFromBestAddressCommand.php @@ -16,6 +16,7 @@ use Chill\MainBundle\Service\Import\PostalCodeBEFromBestAddress; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Output\OutputInterface; class LoadAddressesBEFromBestAddressCommand extends Command @@ -34,14 +35,19 @@ class LoadAddressesBEFromBestAddressCommand extends Command $this ->setName('chill:main:address-ref-from-best-addresses') ->addArgument('lang', InputArgument::REQUIRED, "Language code, for example 'fr'") - ->addArgument('list', InputArgument::IS_ARRAY, "The list to add, for example 'full', or 'extract' (dev) or '1xxx' (brussel CP)"); + ->addArgument('list', InputArgument::IS_ARRAY, "The list to add, for example 'full', or 'extract' (dev) or '1xxx' (brussel CP)") + ->addOption('send-report-email', 's', InputOption::VALUE_REQUIRED, 'Email address where a list of unimported addresses can be send'); } protected function execute(InputInterface $input, OutputInterface $output): int { $this->postalCodeBEFromBestAddressImporter->import(); - $this->addressImporter->import($input->getArgument('lang'), $input->getArgument('list')); + $this->addressImporter->import( + $input->getArgument('lang'), + $input->getArgument('list'), + $input->hasOption('send-report-email') ? $input->getOption('send-report-email') : null + ); return Command::SUCCESS; } diff --git a/src/Bundle/ChillMainBundle/Command/LoadAddressesFRFromBANOCommand.php b/src/Bundle/ChillMainBundle/Command/LoadAddressesFRFromBANOCommand.php index 2bfc3be5c..a5471f2aa 100644 --- a/src/Bundle/ChillMainBundle/Command/LoadAddressesFRFromBANOCommand.php +++ b/src/Bundle/ChillMainBundle/Command/LoadAddressesFRFromBANOCommand.php @@ -15,6 +15,7 @@ use Chill\MainBundle\Service\Import\AddressReferenceFromBano; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Output\OutputInterface; class LoadAddressesFRFromBANOCommand extends Command @@ -29,7 +30,8 @@ class LoadAddressesFRFromBANOCommand extends Command protected function configure() { $this->setName('chill:main:address-ref-from-bano') - ->addArgument('departementNo', InputArgument::REQUIRED | InputArgument::IS_ARRAY, 'a list of departement numbers'); + ->addArgument('departementNo', InputArgument::REQUIRED | InputArgument::IS_ARRAY, 'a list of departement numbers') + ->addOption('send-report-email', 's', InputOption::VALUE_REQUIRED, 'Email address where a list of unimported addresses can be send'); } protected function execute(InputInterface $input, OutputInterface $output): int @@ -37,7 +39,7 @@ class LoadAddressesFRFromBANOCommand extends Command foreach ($input->getArgument('departementNo') as $departementNo) { $output->writeln('Import addresses for '.$departementNo); - $this->addressReferenceFromBano->import($departementNo); + $this->addressReferenceFromBano->import($departementNo, $input->hasOption('send-report-email') ? $input->getOption('send-report-email') : null); } return Command::SUCCESS; diff --git a/src/Bundle/ChillMainBundle/Command/LoadAddressesLUFromBDAddressCommand.php b/src/Bundle/ChillMainBundle/Command/LoadAddressesLUFromBDAddressCommand.php index 2477a486d..10e2383a3 100644 --- a/src/Bundle/ChillMainBundle/Command/LoadAddressesLUFromBDAddressCommand.php +++ b/src/Bundle/ChillMainBundle/Command/LoadAddressesLUFromBDAddressCommand.php @@ -14,6 +14,7 @@ namespace Chill\MainBundle\Command; use Chill\MainBundle\Service\Import\AddressReferenceLU; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Output\OutputInterface; class LoadAddressesLUFromBDAddressCommand extends Command @@ -28,12 +29,16 @@ class LoadAddressesLUFromBDAddressCommand extends Command protected function configure() { - $this->setName('chill:main:address-ref-lux'); + $this + ->setName('chill:main:address-ref-lux') + ->addOption('send-report-email', 's', InputOption::VALUE_REQUIRED, 'Email address where a list of unimported addresses can be send'); } protected function execute(InputInterface $input, OutputInterface $output): int { - $this->addressImporter->import(); + $this->addressImporter->import( + $input->hasOption('send-report-email') ? $input->getOption('send-report-email') : null, + ); return Command::SUCCESS; } diff --git a/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceBEFromBestAddress.php b/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceBEFromBestAddress.php index f383bd799..ea6575e43 100644 --- a/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceBEFromBestAddress.php +++ b/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceBEFromBestAddress.php @@ -22,10 +22,10 @@ class AddressReferenceBEFromBestAddress public function __construct(private readonly HttpClientInterface $client, private readonly AddressReferenceBaseImporter $baseImporter, private readonly AddressToReferenceMatcher $addressToReferenceMatcher) {} - public function import(string $lang, array $lists): void + public function import(string $lang, array $lists, ?string $sendAddressReportToEmail = null): void { foreach ($lists as $list) { - $this->importList($lang, $list); + $this->importList($lang, $list, $sendAddressReportToEmail); } } @@ -43,7 +43,7 @@ class AddressReferenceBEFromBestAddress return array_values($asset)[0]['browser_download_url']; } - private function importList(string $lang, string $list): void + private function importList(string $lang, string $list, ?string $sendAddressReportToEmail = null): void { $downloadUrl = $this->getDownloadUrl($lang, $list); @@ -85,7 +85,7 @@ class AddressReferenceBEFromBestAddress ); } - $this->baseImporter->finalize(); + $this->baseImporter->finalize(sendAddressReportToEmail: $sendAddressReportToEmail); $this->addressToReferenceMatcher->checkAddressesMatchingReferences(); diff --git a/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceBaseImporter.php b/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceBaseImporter.php index f570796f1..3f4c0aca1 100644 --- a/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceBaseImporter.php +++ b/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceBaseImporter.php @@ -13,7 +13,12 @@ namespace Chill\MainBundle\Service\Import; use Doctrine\DBAL\Connection; use Doctrine\DBAL\Statement; +use League\Csv\Writer; use Psr\Log\LoggerInterface; +use Symfony\Component\Filesystem\Path; +use Symfony\Component\Mailer\Exception\TransportExceptionInterface; +use Symfony\Component\Mailer\MailerInterface; +use Symfony\Component\Mime\Email; /** * Import addresses into the database. @@ -25,15 +30,15 @@ final class AddressReferenceBaseImporter { private const INSERT = <<<'SQL' INSERT INTO reference_address_temp - (postcode_id, refid, street, streetnumber, municipalitycode, source, point) + (postcode_id, postalcode, refid, street, streetnumber, municipalitycode, source, point) SELECT - cmpc.id, i.refid, i.street, i.streetnumber, i.refpostalcode, i.source, + cmpc.id, i.postalcode, i.refid, i.street, i.streetnumber, i.refpostalcode, i.source, CASE WHEN (i.lon::float != 0.0 AND i.lat::float != 0.0) THEN ST_Transform(ST_setSrid(ST_point(i.lon::float, i.lat::float), i.srid::int), 4326) ELSE NULL END FROM (VALUES {{ values }} ) AS i (refid, refpostalcode, postalcode, street, streetnumber, source, lat, lon, srid) - JOIN chill_main_postal_code cmpc ON cmpc.refpostalcodeid = i.refpostalcode and cmpc.code = i.postalcode + LEFT JOIN chill_main_postal_code cmpc ON cmpc.refpostalcodeid = i.refpostalcode and cmpc.code = i.postalcode SQL; private const LOG_PREFIX = '[AddressReferenceImporter] '; @@ -51,7 +56,11 @@ final class AddressReferenceBaseImporter private array $waitingForInsert = []; - public function __construct(private readonly Connection $defaultConnection, private readonly LoggerInterface $logger) {} + public function __construct( + private readonly Connection $defaultConnection, + private readonly LoggerInterface $logger, + private readonly MailerInterface $mailer, + ) {} /** * Finalize the import process and make reconciliation with addresses. @@ -60,11 +69,11 @@ final class AddressReferenceBaseImporter * * @throws \Exception */ - public function finalize(bool $allowRemoveDoubleRefId = false): void + public function finalize(bool $allowRemoveDoubleRefId = false, ?string $sendAddressReportToEmail = null): void { $this->doInsertPending(); - $this->updateAddressReferenceTable($allowRemoveDoubleRefId); + $this->updateAddressReferenceTable($allowRemoveDoubleRefId, $sendAddressReportToEmail); $this->deleteTemporaryTable(); @@ -116,7 +125,8 @@ final class AddressReferenceBaseImporter private function createTemporaryTable(): void { $this->defaultConnection->executeStatement('CREATE TEMPORARY TABLE reference_address_temp ( - postcode_id INT, + postcode_id INT DEFAULT NULL, + postalcode TEXT DEFAULT \'\', refid VARCHAR(255), street VARCHAR(255), streetnumber VARCHAR(255), @@ -185,15 +195,15 @@ final class AddressReferenceBaseImporter $this->isInitialized = true; } - private function updateAddressReferenceTable(bool $allowRemoveDoubleRefId): void + private function updateAddressReferenceTable(bool $allowRemoveDoubleRefId, ?string $sendAddressReportToEmail = null): void { $this->defaultConnection->executeStatement( - 'CREATE INDEX idx_ref_add_temp ON reference_address_temp (refid)' + 'CREATE INDEX idx_ref_add_temp ON reference_address_temp (refid) WHERE postcode_id IS NOT NULL' ); // 0) detect for doublon in current temporary table $results = $this->defaultConnection->executeQuery( - 'SELECT COUNT(*) AS nb_appearance, refid FROM reference_address_temp GROUP BY refid HAVING count(*) > 1' + 'SELECT COUNT(*) AS nb_appearance, refid FROM reference_address_temp WHERE postcode_id IS NOT NULL GROUP BY refid HAVING count(*) > 1' ); $hasDouble = false; @@ -210,7 +220,7 @@ final class AddressReferenceBaseImporter WITH ordering AS ( SELECT gid, rank() over (PARTITION BY refid ORDER BY gid DESC) AS ranking FROM reference_address_temp - WHERE refid IN (SELECT refid FROM reference_address_temp group by refid having count(*) > 1) + WHERE postcode_id IS NOT NULL AND refid IN (SELECT refid FROM reference_address_temp WHERE postcode_id IS NOT NULL group by refid having count(*) > 1) ), keep_last AS ( SELECT gid, ranking FROM ordering where ranking > 1 @@ -240,7 +250,7 @@ final class AddressReferenceBaseImporter NOW(), null, NOW() - FROM reference_address_temp + FROM reference_address_temp WHERE postcode_id IS NOT NULL ON CONFLICT (refid, source) DO UPDATE SET postcode_id = excluded.postcode_id, refid = excluded.refid, street = excluded.street, streetnumber = excluded.streetnumber, municipalitycode = excluded.municipalitycode, source = excluded.source, point = excluded.point, updatedat = NOW(), deletedAt = NULL "); @@ -251,10 +261,65 @@ final class AddressReferenceBaseImporter $affected = $connection->executeStatement('UPDATE chill_main_address_reference SET deletedat = NOW() WHERE - chill_main_address_reference.refid NOT IN (SELECT refid FROM reference_address_temp WHERE source LIKE ?) + chill_main_address_reference.refid NOT IN (SELECT refid FROM reference_address_temp WHERE source LIKE ? AND postcode_id IS NOT NULL) AND chill_main_address_reference.source LIKE ? ', [$this->currentSource, $this->currentSource]); $this->logger->info(self::LOG_PREFIX.'addresses deleted', ['deleted' => $affected]); }); + + + // Create a list of addresses without any postal code + $results = $this->defaultConnection->executeQuery('SELECT + postalcode, + refid, + street, + streetnumber, + municipalitycode, + source, + ST_AsText(point) + FROM reference_address_temp + WHERE postcode_id IS NULL + '); + $count = $results->rowCount(); + + if ($count > 0) { + $this->logger->warning(self::LOG_PREFIX.'There are addresses that could not be associated with a postal code', ['nb' => $count]); + + $filename = sprintf('%s-%s.csv', (new \DateTimeImmutable())->format('Ymd-His'), uniqid()); + $path = Path::normalize(sprintf('%s%s%s', sys_get_temp_dir(), DIRECTORY_SEPARATOR, $filename)); + $writer = Writer::createFromPath($path, 'w+'); + // insert headers + $writer->insertOne([ + 'postalcode', + 'refid', + 'street', + 'streetnumber', + 'municipalitycode', + 'source', + 'point', + ]); + + $writer->insertAll($results->iterateAssociative()); + $this->logger->info(sprintf(self::LOG_PREFIX.'The addresses that could not be inserted within the database are registered at path %s', $path)); + + if (null !== $sendAddressReportToEmail) { + // first, we compress the existing file which can be quite big + $attachment = gzopen($attachmentPath = sprintf('%s.gz', $path), 'w9'); + gzwrite($attachment, file_get_contents($path)); + gzclose($attachment); + + $email = (new Email()) + ->addTo($sendAddressReportToEmail) + ->subject('Addresses that could not be imported') + ->attachFromPath($attachmentPath); + + try { + $this->mailer->send($email); + } catch (TransportExceptionInterface $e) { + $this->logger->error(self::LOG_PREFIX.'Could not send an email with addresses that could not be registered', ['exception' => $e->getTraceAsString()]); + } + unlink($attachmentPath); + } + } } } diff --git a/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceFromBano.php b/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceFromBano.php index fd17f2cd2..5ee3f6964 100644 --- a/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceFromBano.php +++ b/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceFromBano.php @@ -19,7 +19,7 @@ class AddressReferenceFromBano { public function __construct(private readonly HttpClientInterface $client, private readonly AddressReferenceBaseImporter $baseImporter, private readonly AddressToReferenceMatcher $addressToReferenceMatcher) {} - public function import(string $departementNo): void + public function import(string $departementNo, ?string $sendAddressReportToEmail = null): void { if (!is_numeric($departementNo) || !\is_int((int) $departementNo)) { throw new \UnexpectedValueException('Could not parse this department number'); @@ -69,7 +69,7 @@ class AddressReferenceFromBano ); } - $this->baseImporter->finalize(); + $this->baseImporter->finalize(sendAddressReportToEmail: $sendAddressReportToEmail); $this->addressToReferenceMatcher->checkAddressesMatchingReferences(); diff --git a/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceLU.php b/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceLU.php index 50ee16401..86c1b61b8 100644 --- a/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceLU.php +++ b/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceLU.php @@ -21,7 +21,7 @@ class AddressReferenceLU public function __construct(private readonly HttpClientInterface $client, private readonly AddressReferenceBaseImporter $addressBaseImporter, private readonly PostalCodeBaseImporter $postalCodeBaseImporter, private readonly AddressToReferenceMatcher $addressToReferenceMatcher) {} - public function import(): void + public function import(?string $sendAddressReportToEmail = null): void { $downloadUrl = self::RELEASE; @@ -45,14 +45,14 @@ class AddressReferenceLU $this->process_postal_code($csv); - $this->process_address($csv); + $this->process_address($csv, $sendAddressReportToEmail); $this->addressToReferenceMatcher->checkAddressesMatchingReferences(); fclose($file); } - private function process_address(Reader $csv): void + private function process_address(Reader $csv, ?string $sendAddressReportToEmail = null): void { $stmt = Statement::create()->process($csv); foreach ($stmt as $record) { @@ -69,7 +69,7 @@ class AddressReferenceLU ); } - $this->addressBaseImporter->finalize(); + $this->addressBaseImporter->finalize(sendAddressReportToEmail: $sendAddressReportToEmail); } private function process_postal_code(Reader $csv): void