Merge branch 'import-addresses-handle-no-postcode' into 'master'

Allow addresses without postal code to be imported without failure, and add email reporting for unimported addresses in import commands

See merge request Chill-Projet/chill-bundles!780

.changes/unreleased/Feature-20250109-121757.yaml

@@ -0,0 +1,5 @@
+kind: Feature
+body: Importer for addresses does not fails when the postal code is not found with some addresses, and compute a recap list of all addresses that could not be imported. This recap list can be send by email.
+time: 2025-01-09T12:17:57.981181677+01:00
+custom:
+    Issue: ""

composer.json

@@ -13,6 +13,7 @@
         "ext-json": "*",
         "ext-openssl": "*",
         "ext-redis": "*",
+        "ext-zlib": "*",
         "champs-libres/wopi-bundle": "dev-master@dev",
         "champs-libres/wopi-lib": "dev-master@dev",
         "doctrine/doctrine-bundle": "^2.1",

src/Bundle/ChillMainBundle/Command/LoadAddressesBEFromBestAddressCommand.php

@@ -16,6 +16,7 @@ use Chill\MainBundle\Service\Import\PostalCodeBEFromBestAddress;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Input\InputArgument;
 use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
 
 class LoadAddressesBEFromBestAddressCommand extends Command
@@ -34,14 +35,19 @@ class LoadAddressesBEFromBestAddressCommand extends Command
         $this
             ->setName('chill:main:address-ref-from-best-addresses')
             ->addArgument('lang', InputArgument::REQUIRED, "Language code, for example 'fr'")
-            ->addArgument('list', InputArgument::IS_ARRAY, "The list to add, for example 'full', or 'extract' (dev) or '1xxx' (brussel CP)");
+            ->addArgument('list', InputArgument::IS_ARRAY, "The list to add, for example 'full', or 'extract' (dev) or '1xxx' (brussel CP)")
+            ->addOption('send-report-email', 's', InputOption::VALUE_REQUIRED, 'Email address where a list of unimported addresses can be send');
     }
 
     protected function execute(InputInterface $input, OutputInterface $output): int
     {
         $this->postalCodeBEFromBestAddressImporter->import();
 
-        $this->addressImporter->import($input->getArgument('lang'), $input->getArgument('list'));
+        $this->addressImporter->import(
+            $input->getArgument('lang'),
+            $input->getArgument('list'),
+            $input->hasOption('send-report-email') ? $input->getOption('send-report-email') : null
+        );
 
         return Command::SUCCESS;
     }

src/Bundle/ChillMainBundle/Command/LoadAddressesFRFromBANOCommand.php

@@ -15,6 +15,7 @@ use Chill\MainBundle\Service\Import\AddressReferenceFromBano;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Input\InputArgument;
 use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
 
 class LoadAddressesFRFromBANOCommand extends Command
@@ -29,7 +30,8 @@ class LoadAddressesFRFromBANOCommand extends Command
     protected function configure()
     {
         $this->setName('chill:main:address-ref-from-bano')
-            ->addArgument('departementNo', InputArgument::REQUIRED | InputArgument::IS_ARRAY, 'a list of departement numbers');
+            ->addArgument('departementNo', InputArgument::REQUIRED | InputArgument::IS_ARRAY, 'a list of departement numbers')
+            ->addOption('send-report-email', 's', InputOption::VALUE_REQUIRED, 'Email address where a list of unimported addresses can be send');
     }
 
     protected function execute(InputInterface $input, OutputInterface $output): int
@@ -37,7 +39,7 @@ class LoadAddressesFRFromBANOCommand extends Command
         foreach ($input->getArgument('departementNo') as $departementNo) {
             $output->writeln('Import addresses for '.$departementNo);
 
-            $this->addressReferenceFromBano->import($departementNo);
+            $this->addressReferenceFromBano->import($departementNo, $input->hasOption('send-report-email') ? $input->getOption('send-report-email') : null);
         }
 
         return Command::SUCCESS;

src/Bundle/ChillMainBundle/Command/LoadAddressesLUFromBDAddressCommand.php

@@ -14,6 +14,7 @@ namespace Chill\MainBundle\Command;
 use Chill\MainBundle\Service\Import\AddressReferenceLU;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
 
 class LoadAddressesLUFromBDAddressCommand extends Command
@@ -28,12 +29,16 @@ class LoadAddressesLUFromBDAddressCommand extends Command
 
     protected function configure()
     {
-        $this->setName('chill:main:address-ref-lux');
+        $this
+            ->setName('chill:main:address-ref-lux')
+            ->addOption('send-report-email', 's', InputOption::VALUE_REQUIRED, 'Email address where a list of unimported addresses can be send');
     }
 
     protected function execute(InputInterface $input, OutputInterface $output): int
     {
-        $this->addressImporter->import();
+        $this->addressImporter->import(
+            $input->hasOption('send-report-email') ? $input->getOption('send-report-email') : null,
+        );
 
         return Command::SUCCESS;
     }

src/Bundle/ChillMainBundle/Service/Import/AddressReferenceBEFromBestAddress.php

@@ -22,10 +22,10 @@ class AddressReferenceBEFromBestAddress
 
     public function __construct(private readonly HttpClientInterface $client, private readonly AddressReferenceBaseImporter $baseImporter, private readonly AddressToReferenceMatcher $addressToReferenceMatcher) {}
 
-    public function import(string $lang, array $lists): void
+    public function import(string $lang, array $lists, ?string $sendAddressReportToEmail = null): void
     {
         foreach ($lists as $list) {
-            $this->importList($lang, $list);
+            $this->importList($lang, $list, $sendAddressReportToEmail);
         }
     }
 
@@ -43,7 +43,7 @@ class AddressReferenceBEFromBestAddress
         return array_values($asset)[0]['browser_download_url'];
     }
 
-    private function importList(string $lang, string $list): void
+    private function importList(string $lang, string $list, ?string $sendAddressReportToEmail = null): void
     {
         $downloadUrl = $this->getDownloadUrl($lang, $list);
 
@@ -85,7 +85,7 @@ class AddressReferenceBEFromBestAddress
             );
         }
 
-        $this->baseImporter->finalize();
+        $this->baseImporter->finalize(sendAddressReportToEmail: $sendAddressReportToEmail);
 
         $this->addressToReferenceMatcher->checkAddressesMatchingReferences();
 

src/Bundle/ChillMainBundle/Service/Import/AddressReferenceBaseImporter.php

@@ -13,7 +13,12 @@ namespace Chill\MainBundle\Service\Import;
 
 use Doctrine\DBAL\Connection;
 use Doctrine\DBAL\Statement;
+use League\Csv\Writer;
 use Psr\Log\LoggerInterface;
+use Symfony\Component\Filesystem\Path;
+use Symfony\Component\Mailer\Exception\TransportExceptionInterface;
+use Symfony\Component\Mailer\MailerInterface;
+use Symfony\Component\Mime\Email;
 
 /**
  * Import addresses into the database.
@@ -25,15 +30,15 @@ final class AddressReferenceBaseImporter
 {
     private const INSERT = <<<'SQL'
         INSERT INTO reference_address_temp
-            (postcode_id, refid, street, streetnumber, municipalitycode, source, point)
+            (postcode_id, postalcode, refid, street, streetnumber, municipalitycode, source, point)
         SELECT
-            cmpc.id, i.refid, i.street, i.streetnumber, i.refpostalcode, i.source,
+            cmpc.id, i.postalcode, i.refid, i.street, i.streetnumber, i.refpostalcode, i.source,
             CASE WHEN (i.lon::float != 0.0 AND i.lat::float != 0.0) THEN ST_Transform(ST_setSrid(ST_point(i.lon::float, i.lat::float), i.srid::int), 4326) ELSE NULL END
         FROM
             (VALUES
                 {{ values }}
              ) AS i (refid, refpostalcode, postalcode, street, streetnumber, source, lat, lon, srid)
-        JOIN chill_main_postal_code cmpc ON cmpc.refpostalcodeid = i.refpostalcode and cmpc.code = i.postalcode
+        LEFT JOIN chill_main_postal_code cmpc ON cmpc.refpostalcodeid = i.refpostalcode and cmpc.code = i.postalcode
         SQL;
 
     private const LOG_PREFIX = '[AddressReferenceImporter] ';
@@ -51,7 +56,11 @@ final class AddressReferenceBaseImporter
 
     private array $waitingForInsert = [];
 
-    public function __construct(private readonly Connection $defaultConnection, private readonly LoggerInterface $logger) {}
+    public function __construct(
+        private readonly Connection $defaultConnection,
+        private readonly LoggerInterface $logger,
+        private readonly MailerInterface $mailer,
+    ) {}
 
     /**
      * Finalize the import process and make reconciliation with addresses.
@@ -60,11 +69,11 @@ final class AddressReferenceBaseImporter
      *
      * @throws \Exception
      */
-    public function finalize(bool $allowRemoveDoubleRefId = false): void
+    public function finalize(bool $allowRemoveDoubleRefId = false, ?string $sendAddressReportToEmail = null): void
     {
         $this->doInsertPending();
 
-        $this->updateAddressReferenceTable($allowRemoveDoubleRefId);
+        $this->updateAddressReferenceTable($allowRemoveDoubleRefId, $sendAddressReportToEmail);
 
         $this->deleteTemporaryTable();
 
@@ -116,7 +125,8 @@ final class AddressReferenceBaseImporter
     private function createTemporaryTable(): void
     {
         $this->defaultConnection->executeStatement('CREATE TEMPORARY TABLE reference_address_temp (
-            postcode_id INT,
+            postcode_id INT DEFAULT NULL,
+            postalcode TEXT DEFAULT \'\',
             refid VARCHAR(255),
             street VARCHAR(255),
             streetnumber VARCHAR(255),
@@ -185,15 +195,15 @@ final class AddressReferenceBaseImporter
         $this->isInitialized = true;
     }
 
-    private function updateAddressReferenceTable(bool $allowRemoveDoubleRefId): void
+    private function updateAddressReferenceTable(bool $allowRemoveDoubleRefId, ?string $sendAddressReportToEmail = null): void
     {
         $this->defaultConnection->executeStatement(
-            'CREATE INDEX idx_ref_add_temp ON reference_address_temp (refid)'
+            'CREATE INDEX idx_ref_add_temp ON reference_address_temp (refid) WHERE postcode_id IS NOT NULL'
         );
 
         // 0) detect for doublon in current temporary table
         $results = $this->defaultConnection->executeQuery(
-            'SELECT COUNT(*) AS nb_appearance, refid FROM reference_address_temp GROUP BY refid HAVING count(*) > 1'
+            'SELECT COUNT(*) AS nb_appearance, refid FROM reference_address_temp WHERE postcode_id IS NOT NULL GROUP BY refid HAVING count(*) > 1'
         );
 
         $hasDouble = false;
@@ -210,7 +220,7 @@ final class AddressReferenceBaseImporter
                 WITH ordering AS (
                     SELECT gid, rank() over (PARTITION BY refid ORDER BY gid DESC) AS ranking
                     FROM reference_address_temp
-                    WHERE refid IN (SELECT refid FROM reference_address_temp group by refid having count(*) > 1)
+                    WHERE postcode_id IS NOT NULL AND refid IN (SELECT refid FROM reference_address_temp WHERE postcode_id IS NOT NULL group by refid having count(*) > 1)
                     ),
                 keep_last AS (
                     SELECT gid, ranking FROM ordering where ranking > 1
@@ -240,7 +250,7 @@ final class AddressReferenceBaseImporter
                 NOW(),
                 null,
                 NOW()
-            FROM reference_address_temp
+            FROM reference_address_temp WHERE postcode_id IS NOT NULL
             ON CONFLICT (refid, source) DO UPDATE
             SET postcode_id = excluded.postcode_id, refid = excluded.refid, street = excluded.street, streetnumber = excluded.streetnumber, municipalitycode = excluded.municipalitycode, source = excluded.source, point = excluded.point, updatedat = NOW(), deletedAt = NULL
         ");
@@ -251,10 +261,65 @@ final class AddressReferenceBaseImporter
             $affected = $connection->executeStatement('UPDATE chill_main_address_reference
             SET deletedat = NOW()
             WHERE
-                chill_main_address_reference.refid NOT IN (SELECT refid FROM reference_address_temp WHERE source LIKE ?)
+                chill_main_address_reference.refid NOT IN (SELECT refid FROM reference_address_temp WHERE source LIKE ? AND postcode_id IS NOT NULL)
                 AND chill_main_address_reference.source LIKE ?
         ', [$this->currentSource, $this->currentSource]);
             $this->logger->info(self::LOG_PREFIX.'addresses deleted', ['deleted' => $affected]);
         });
+
+
+        // Create a list of addresses without any postal code
+        $results = $this->defaultConnection->executeQuery('SELECT
+            postalcode,
+            refid,
+            street,
+            streetnumber,
+            municipalitycode,
+            source,
+            ST_AsText(point)
+            FROM reference_address_temp
+            WHERE postcode_id IS NULL
+        ');
+        $count = $results->rowCount();
+
+        if ($count > 0) {
+            $this->logger->warning(self::LOG_PREFIX.'There are addresses that could not be associated with a postal code', ['nb' => $count]);
+
+            $filename = sprintf('%s-%s.csv', (new \DateTimeImmutable())->format('Ymd-His'), uniqid());
+            $path = Path::normalize(sprintf('%s%s%s', sys_get_temp_dir(), DIRECTORY_SEPARATOR, $filename));
+            $writer = Writer::createFromPath($path, 'w+');
+            // insert headers
+            $writer->insertOne([
+                'postalcode',
+                'refid',
+                'street',
+                'streetnumber',
+                'municipalitycode',
+                'source',
+                'point',
+            ]);
+
+            $writer->insertAll($results->iterateAssociative());
+            $this->logger->info(sprintf(self::LOG_PREFIX.'The addresses that could not be inserted within the database are registered at path %s', $path));
+
+            if (null !== $sendAddressReportToEmail) {
+                // first, we compress the existing file which can be quite big
+                $attachment = gzopen($attachmentPath = sprintf('%s.gz', $path), 'w9');
+                gzwrite($attachment, file_get_contents($path));
+                gzclose($attachment);
+
+                $email = (new Email())
+                    ->addTo($sendAddressReportToEmail)
+                    ->subject('Addresses that could not be imported')
+                    ->attachFromPath($attachmentPath);
+
+                try {
+                    $this->mailer->send($email);
+                } catch (TransportExceptionInterface $e) {
+                    $this->logger->error(self::LOG_PREFIX.'Could not send an email with addresses that could not be registered', ['exception' => $e->getTraceAsString()]);
+                }
+                unlink($attachmentPath);
+            }
+        }
     }
 }

src/Bundle/ChillMainBundle/Service/Import/AddressReferenceFromBano.php

@@ -19,7 +19,7 @@ class AddressReferenceFromBano
 {
     public function __construct(private readonly HttpClientInterface $client, private readonly AddressReferenceBaseImporter $baseImporter, private readonly AddressToReferenceMatcher $addressToReferenceMatcher) {}
 
-    public function import(string $departementNo): void
+    public function import(string $departementNo, ?string $sendAddressReportToEmail = null): void
     {
         if (!is_numeric($departementNo) || !\is_int((int) $departementNo)) {
             throw new \UnexpectedValueException('Could not parse this department number');
@@ -69,7 +69,7 @@ class AddressReferenceFromBano
             );
         }
 
-        $this->baseImporter->finalize();
+        $this->baseImporter->finalize(sendAddressReportToEmail: $sendAddressReportToEmail);
 
         $this->addressToReferenceMatcher->checkAddressesMatchingReferences();
 

src/Bundle/ChillMainBundle/Service/Import/AddressReferenceLU.php

@@ -21,7 +21,7 @@ class AddressReferenceLU
 
     public function __construct(private readonly HttpClientInterface $client, private readonly AddressReferenceBaseImporter $addressBaseImporter, private readonly PostalCodeBaseImporter $postalCodeBaseImporter, private readonly AddressToReferenceMatcher $addressToReferenceMatcher) {}
 
-    public function import(): void
+    public function import(?string $sendAddressReportToEmail = null): void
     {
         $downloadUrl = self::RELEASE;
 
@@ -45,14 +45,14 @@ class AddressReferenceLU
 
         $this->process_postal_code($csv);
 
-        $this->process_address($csv);
+        $this->process_address($csv, $sendAddressReportToEmail);
 
         $this->addressToReferenceMatcher->checkAddressesMatchingReferences();
 
         fclose($file);
     }
 
-    private function process_address(Reader $csv): void
+    private function process_address(Reader $csv, ?string $sendAddressReportToEmail = null): void
     {
         $stmt = Statement::create()->process($csv);
         foreach ($stmt as $record) {
@@ -69,7 +69,7 @@ class AddressReferenceLU
             );
         }
 
-        $this->addressBaseImporter->finalize();
+        $this->addressBaseImporter->finalize(sendAddressReportToEmail: $sendAddressReportToEmail);
     }
 
     private function process_postal_code(Reader $csv): void