From 58ddf9038d625866da3c18f5e5089260cdf5f27c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Julien=20Fastr=C3=A9?= <julien.fastre@champs-libres.coop>
Date: Sat, 30 Jul 2022 02:01:42 +0200
Subject: [PATCH] Feature: load french postal code from laposte hexasmal open
 data

---
 .../Command/LoadPostalCodeFR.php              |  42 ++++++
 .../Service/Import/PostalCodeBaseImporter.php | 123 ++++++++++++++++++
 .../Import/PostalCodeFRFromOpenData.php       | 105 +++++++++++++++
 .../ChillMainBundle/config/services.yaml      |   4 +
 .../config/services/command.yaml              |   6 +
 5 files changed, 280 insertions(+)
 create mode 100644 src/Bundle/ChillMainBundle/Command/LoadPostalCodeFR.php
 create mode 100644 src/Bundle/ChillMainBundle/Service/Import/PostalCodeBaseImporter.php
 create mode 100644 src/Bundle/ChillMainBundle/Service/Import/PostalCodeFRFromOpenData.php

diff --git a/src/Bundle/ChillMainBundle/Command/LoadPostalCodeFR.php b/src/Bundle/ChillMainBundle/Command/LoadPostalCodeFR.php
new file mode 100644
index 000000000..1c7f9cbc5
--- /dev/null
+++ b/src/Bundle/ChillMainBundle/Command/LoadPostalCodeFR.php
@@ -0,0 +1,42 @@
+<?php
+
+/**
+ * Chill is a software for social workers
+ *
+ * For the full copyright and license information, please view
+ * the LICENSE file that was distributed with this source code.
+ */
+
+declare(strict_types=1);
+
+namespace Chill\MainBundle\Command;
+
+use Chill\MainBundle\Service\Import\PostalCodeFRFromOpenData;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+
+class LoadPostalCodeFR extends Command
+{
+    private PostalCodeFRFromOpenData $loader;
+
+    public function __construct(PostalCodeFRFromOpenData $loader)
+    {
+        $this->loader = $loader;
+
+        parent::__construct();
+    }
+
+    public function configure(): void
+    {
+        $this->setName('chill:main:postal-code:load:FR')
+            ->setDescription('Load France\'s postal code from online open data');
+    }
+
+    public function execute(InputInterface $input, OutputInterface $output): int
+    {
+        $this->loader->import();
+
+        return 0;
+    }
+}
diff --git a/src/Bundle/ChillMainBundle/Service/Import/PostalCodeBaseImporter.php b/src/Bundle/ChillMainBundle/Service/Import/PostalCodeBaseImporter.php
new file mode 100644
index 000000000..a7e8c8e70
--- /dev/null
+++ b/src/Bundle/ChillMainBundle/Service/Import/PostalCodeBaseImporter.php
@@ -0,0 +1,123 @@
+<?php
+
+/**
+ * Chill is a software for social workers
+ *
+ * For the full copyright and license information, please view
+ * the LICENSE file that was distributed with this source code.
+ */
+
+declare(strict_types=1);
+
+namespace Chill\MainBundle\Service\Import;
+
+use Doctrine\DBAL\Connection;
+use Doctrine\DBAL\Statement;
+use function array_key_exists;
+use function count;
+
+/**
+ * Optimized way to load postal code into database.
+ */
+class PostalCodeBaseImporter
+{
+    private const QUERY = <<<'SQL'
+        WITH g AS (
+            SELECT DISTINCT
+                country.id AS country_id,
+                g.*
+            FROM (VALUES
+                      {{ values }}
+                 ) AS g (countrycode, label, code, refpostalcodeid, postalcodeSource, lon, lat, srid)
+            JOIN country ON country.countrycode = g.countrycode
+        )
+        INSERT INTO chill_main_postal_code (id, country_id, label, code, origin, refpostalcodeid, postalcodeSource, center, createdAt, updatedAt)
+               SELECT
+                   nextval('chill_main_postal_code_id_seq'),
+                   g.country_id,
+                   g.label AS glabel,
+                   g.code,
+                   0,
+                   g.refpostalcodeid,
+                   g.postalcodeSource,
+                   CASE WHEN (g.lon::float != 0.0 AND g.lat::float != 0.0) THEN ST_setSrid(ST_point(g.lon::float, g.lat::float), g.srid::int) ELSE NULL END,
+                   NOW(),
+                   NOW()
+               FROM g
+        ON CONFLICT (code, refpostalcodeid, postalcodeSource) WHERE refpostalcodeid IS NOT NULL DO UPDATE SET label = excluded.label, center = excluded.center, updatedAt = NOW()
+        SQL;
+
+    private const VALUE = '(?, ?, ?, ?, ?, ?, ?, ?)';
+
+    /**
+     * @var array<int, Statement>
+     */
+    private array $cachingStatements = [];
+
+    private Connection $defaultConnection;
+
+    private array $waitingForInsert = [];
+
+    public function __construct(
+        Connection $defaultConnection
+    ) {
+        $this->defaultConnection = $defaultConnection;
+    }
+
+    public function finalize(): void
+    {
+        $this->doInsertPending();
+    }
+
+    public function importCode(
+        string $countryCode,
+        string $label,
+        string $code,
+        string $refPostalCodeId,
+        string $refPostalCodeSource,
+        float $centerLat,
+        float $centerLon,
+        int $centerSRID
+    ): void {
+        $this->waitingForInsert[] = [
+            $countryCode,
+            $label,
+            $code,
+            $refPostalCodeId,
+            $refPostalCodeSource,
+            $centerLon,
+            $centerLat,
+            $centerSRID,
+        ];
+
+        if (100 <= count($this->waitingForInsert)) {
+            $this->doInsertPending();
+        }
+    }
+
+    private function doInsertPending(): void
+    {
+        if (!array_key_exists($forNumber = count($this->waitingForInsert), $this->cachingStatements)) {
+            $sql = strtr(self::QUERY, [
+                '{{ values }}' => implode(
+                    ', ',
+                    array_fill(0, $forNumber, self::VALUE)
+                ),
+            ]);
+
+            $this->cachingStatements[$forNumber] = $this->defaultConnection->prepare($sql);
+        }
+
+        $statement = $this->cachingStatements[$forNumber];
+
+        try {
+            $statement->executeStatement(array_merge(...$this->waitingForInsert));
+        } catch (\Exception $e) {
+            // in some case, we can add debug code here
+            //dump($this->waitingForInsert);
+            throw $e;
+        } finally {
+            $this->waitingForInsert = [];
+        }
+    }
+}
diff --git a/src/Bundle/ChillMainBundle/Service/Import/PostalCodeFRFromOpenData.php b/src/Bundle/ChillMainBundle/Service/Import/PostalCodeFRFromOpenData.php
new file mode 100644
index 000000000..a49bb10a4
--- /dev/null
+++ b/src/Bundle/ChillMainBundle/Service/Import/PostalCodeFRFromOpenData.php
@@ -0,0 +1,105 @@
+<?php
+
+/**
+ * Chill is a software for social workers
+ *
+ * For the full copyright and license information, please view
+ * the LICENSE file that was distributed with this source code.
+ */
+
+declare(strict_types=1);
+
+namespace Chill\MainBundle\Service\Import;
+
+use League\Csv\Reader;
+use Psr\Log\LoggerInterface;
+use RuntimeException;
+use Symfony\Contracts\HttpClient\HttpClientInterface;
+
+/**
+ * Load French's postal codes from opendata.
+ *
+ * Currently, the source is datanova / la poste:
+ * https://datanova.legroupe.laposte.fr/explore/dataset/laposte_hexasmal/information/
+ */
+class PostalCodeFRFromOpenData
+{
+    private const CSV = 'https://datanova.legroupe.laposte.fr/explore/dataset/laposte_hexasmal/download/?format=csv&timezone=Europe/Berlin&lang=fr&use_labels_for_header=true&csv_separator=%3B';
+
+    private PostalCodeBaseImporter $baseImporter;
+
+    private HttpClientInterface $client;
+
+    private LoggerInterface $logger;
+
+    public function __construct(
+        PostalCodeBaseImporter $baseImporter,
+        HttpClientInterface $client,
+        LoggerInterface $logger
+    ) {
+        $this->baseImporter = $baseImporter;
+        $this->client = $client;
+        $this->logger = $logger;
+    }
+
+    public function import(): void
+    {
+        $response = $this->client->request('GET', self::CSV);
+
+        if (200 !== $response->getStatusCode()) {
+            throw new RuntimeException('could not download CSV');
+        }
+
+        $tmpfile = tmpfile();
+
+        if (false === $tmpfile) {
+            throw new RuntimeException('could not create temporary file');
+        }
+
+        foreach ($this->client->stream($response) as $chunk) {
+            fwrite($tmpfile, $chunk->getContent());
+        }
+
+        fseek($tmpfile, 0);
+
+        $csv = Reader::createFromStream($tmpfile);
+        $csv->setDelimiter(';');
+        $csv->setHeaderOffset(0);
+
+        foreach ($csv as $offset => $record) {
+            $this->handleRecord($record);
+        }
+
+        $this->baseImporter->finalize();
+        fclose($tmpfile);
+
+        $this->logger->info(__CLASS__ . ' postal code fetched', ['offset' => $offset]);
+    }
+
+    private function handleRecord(array $record): void
+    {
+        if ('' !== trim($record['coordonnees_gps'])) {
+            [$lat, $lon] = array_map(static fn ($el) => (float) trim($el), explode(',', $record['coordonnees_gps']));
+        } else {
+            $lat = $lon = 0.0;
+        }
+
+        $ref = trim($record['Code_commune_INSEE']);
+
+        if ('987' === substr($ref, 0, 3)) {
+            // some differences in French Polynesia
+            $ref .= '.' . trim($record['Libellé_d_acheminement']);
+        }
+
+        $this->baseImporter->importCode(
+            'FR',
+            trim($record['Libellé_d_acheminement']),
+            trim($record['Code_postal']),
+            $ref,
+            'INSEE',
+            $lat,
+            $lon,
+            4326
+        );
+    }
+}
diff --git a/src/Bundle/ChillMainBundle/config/services.yaml b/src/Bundle/ChillMainBundle/config/services.yaml
index 6d55532a6..d3475752b 100644
--- a/src/Bundle/ChillMainBundle/config/services.yaml
+++ b/src/Bundle/ChillMainBundle/config/services.yaml
@@ -97,3 +97,7 @@ services:
 
     Chill\MainBundle\Security\Resolver\CenterResolverDispatcherInterface: '@Chill\MainBundle\Security\Resolver\CenterResolverDispatcher'
 
+    Chill\MainBundle\Service\Import\:
+        resource: '../Service/Import/'
+        autowire: true
+        autoconfigure: true
diff --git a/src/Bundle/ChillMainBundle/config/services/command.yaml b/src/Bundle/ChillMainBundle/config/services/command.yaml
index 87220bc1f..07b3c1721 100644
--- a/src/Bundle/ChillMainBundle/config/services/command.yaml
+++ b/src/Bundle/ChillMainBundle/config/services/command.yaml
@@ -43,3 +43,9 @@ services:
             $entityManager: '@doctrine.orm.entity_manager'
         tags:
             - { name: console.command }
+
+    Chill\MainBundle\Command\LoadPostalCodeFR:
+        autoconfigure: true
+        autowire: true
+        tags:
+            - { name: console.command }