import addresses and postal codes from bestaddress

src/Bundle/ChillMainBundle/Command/LoadAddressesBEFromBestAddressCommand.php

@@ -0,0 +1,52 @@
+<?php
+
+/**
+ * Chill is a software for social workers
+ *
+ * For the full copyright and license information, please view
+ * the LICENSE file that was distributed with this source code.
+ */
+
+declare(strict_types=1);
+
+namespace Chill\MainBundle\Command;
+
+use Chill\MainBundle\Service\Import\AddressReferenceBEFromBestAddress;
+use Chill\MainBundle\Service\Import\PostalCodeBEFromBestAddress;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+
+class LoadAddressesBEFromBestAddressCommand extends Command
+{
+    private AddressReferenceBEFromBestAddress $addressImporter;
+
+    private PostalCodeBEFromBestAddress $postalCodeBEFromBestAddressImporter;
+
+    public function __construct(
+        AddressReferenceBEFromBestAddress $addressImporter,
+        PostalCodeBEFromBestAddress $postalCodeBEFromBestAddressImporter
+    ) {
+        parent::__construct();
+        $this->addressImporter = $addressImporter;
+        $this->postalCodeBEFromBestAddressImporter = $postalCodeBEFromBestAddressImporter;
+    }
+
+    protected function configure()
+    {
+        $this
+            ->setName('chill:main:address-ref-from-best-addresses')
+            ->addArgument('lang', InputArgument::REQUIRED)
+            ->addArgument('list', InputArgument::IS_ARRAY, 'The list to add');
+    }
+
+    protected function execute(InputInterface $input, OutputInterface $output): int
+    {
+        $this->postalCodeBEFromBestAddressImporter->import();
+
+        $this->addressImporter->import($input->getArgument('lang'), $input->getArgument('list'));
+
+        return 0;
+    }
+}

src/Bundle/ChillMainBundle/Service/Import/AddressReferenceBEFromBestAddress.php

@@ -0,0 +1,103 @@
+<?php
+
+/**
+ * Chill is a software for social workers
+ *
+ * For the full copyright and license information, please view
+ * the LICENSE file that was distributed with this source code.
+ */
+
+declare(strict_types=1);
+
+namespace Chill\MainBundle\Service\Import;
+
+use League\Csv\Reader;
+use League\Csv\Statement;
+use RuntimeException;
+use Symfony\Contracts\HttpClient\Exception\TransportExceptionInterface;
+use Symfony\Contracts\HttpClient\HttpClientInterface;
+
+class AddressReferenceBEFromBestAddress
+{
+    private const RELEASE = 'https://gitea.champs-libres.be/api/v1/repos/Chill-project/belgian-bestaddresses-transform/releases/tags/v1.0.0';
+
+    private AddressReferenceBaseImporter $baseImporter;
+
+    private HttpClientInterface $client;
+
+    public function __construct(HttpClientInterface $client, AddressReferenceBaseImporter $baseImporter)
+    {
+        $this->client = $client;
+        $this->baseImporter = $baseImporter;
+    }
+
+    public function import(string $lang, array $lists): void
+    {
+        foreach ($lists as $list) {
+            $this->importList($lang, $list);
+        }
+    }
+
+    private function getDownloadUrl(string $lang, string $list): string
+    {
+        try {
+            $release = $this->client->request('GET', self::RELEASE)
+                ->toArray();
+        } catch (TransportExceptionInterface $e) {
+            throw new RuntimeException('could not get the release definition', 0, $e);
+        }
+
+        $asset = array_filter($release['assets'], static function (array $item) use ($lang, $list) {
+            return 'addresses-' . $list . '.' . $lang . '.csv.gz' === $item['name'];
+        });
+
+        return array_values($asset)[0]['browser_download_url'];
+    }
+
+    private function importList(string $lang, string $list): void
+    {
+        $downloadUrl = $this->getDownloadUrl($lang, $list);
+
+        $response = $this->client->request('GET', $downloadUrl);
+
+        if (200 !== $response->getStatusCode()) {
+            throw new Exception('Could not download CSV: ' . $response->getStatusCode());
+        }
+
+        $tmpname = tempnam(sys_get_temp_dir(), 'php-add-' . $list . $lang);
+        $file = fopen($tmpname, 'r+b');
+
+        foreach ($this->client->stream($response) as $chunk) {
+            fwrite($file, $chunk->getContent());
+        }
+
+        fclose($file);
+
+        $uncompressedStream = gzopen($tmpname, 'r');
+
+        $csv = Reader::createFromStream($uncompressedStream);
+        $csv->setDelimiter(',');
+        $csv->setHeaderOffset(0);
+
+        $stmt = Statement::create()
+            ->process($csv);
+
+        foreach ($stmt as $record) {
+            $this->baseImporter->importAddress(
+                $record['best_id'],
+                $record['municipality_objectid'],
+                $record['postal_info_objectid'],
+                $record['streetname'],
+                $record['housenumber'] . $record['boxnumber'],
+                'bestaddress.' . $list,
+                (float) $record['X'],
+                (float) $record['Y'],
+                3812
+            );
+        }
+
+        $this->baseImporter->finalize();
+
+        gzclose($uncompressedStream);
+    }
+}

src/Bundle/ChillMainBundle/Service/Import/AddressReferenceBaseImporter.php

@@ -26,7 +26,7 @@ final class AddressReferenceBaseImporter
             (postcode_id, refid, street, streetnumber, municipalitycode, source, point)
         SELECT
             cmpc.id, i.refid, i.street, i.streetnumber, i.refpostalcode, i.source,
-            CASE WHEN (i.lon::float != 0.0 AND i.lat::float != 0.0) THEN ST_setSrid(ST_point(i.lon::float, i.lat::float), i.srid::int) ELSE NULL END
+            CASE WHEN (i.lon::float != 0.0 AND i.lat::float != 0.0) THEN ST_Transform(ST_setSrid(ST_point(i.lon::float, i.lat::float), i.srid::int), 4326) ELSE NULL END
         FROM
             (VALUES
                 {{ values }}
@@ -137,9 +137,18 @@ final class AddressReferenceBaseImporter
                 ),
             ]);
 
+            $this->logger->debug(self::LOG_PREFIX . ' generated sql for insert', [
+                'sql' => $sql,
+                'forNumber' => $forNumber,
+            ]);
+
             $this->cachingStatements[$forNumber] = $this->defaultConnection->prepare($sql);
         }
 
+        if (0 === $forNumber) {
+            return;
+        }
+
         $this->logger->debug(self::LOG_PREFIX . ' inserting pending addresses', [
             'number' => $forNumber,
             'first' => $this->waitingForInsert[0] ?? null,
@@ -188,7 +197,7 @@ final class AddressReferenceBaseImporter
                 NOW(),
                 null,
                 NOW()
-            FROM reference_address_temp 
+            FROM reference_address_temp
             ON CONFLICT (refid, source) DO UPDATE
             SET postcode_id = excluded.postcode_id, refid = excluded.refid, street = excluded.street, streetnumber = excluded.streetnumber, municipalitycode = excluded.municipalitycode, source = excluded.source, point = excluded.point, updatedat = NOW(), deletedAt = NULL
         ");

src/Bundle/ChillMainBundle/Service/Import/PostalCodeBEFromBestAddress.php

@@ -0,0 +1,105 @@
+<?php
+
+/**
+ * Chill is a software for social workers
+ *
+ * For the full copyright and license information, please view
+ * the LICENSE file that was distributed with this source code.
+ */
+
+declare(strict_types=1);
+
+namespace Chill\MainBundle\Service\Import;
+
+use League\Csv\Reader;
+use Psr\Log\LoggerInterface;
+use RuntimeException;
+use Symfony\Contracts\HttpClient\Exception\TransportExceptionInterface;
+use Symfony\Contracts\HttpClient\HttpClientInterface;
+
+class PostalCodeBEFromBestAddress
+{
+    private const RELEASE = 'https://gitea.champs-libres.be/api/v1/repos/Chill-project/belgian-bestaddresses-transform/releases/tags/v1.0.0';
+
+    private PostalCodeBaseImporter $baseImporter;
+
+    private HttpClientInterface $client;
+
+    private LoggerInterface $logger;
+
+    public function __construct(PostalCodeBaseImporter $baseImporter, HttpClientInterface $client, LoggerInterface $logger)
+    {
+        $this->baseImporter = $baseImporter;
+        $this->client = $client;
+        $this->logger = $logger;
+    }
+
+    public function import(string $lang = 'fr'): void
+    {
+        $fileDownloadUrl = $this->getFileDownloadUrl($lang);
+
+        $response = $this->client->request('GET', $fileDownloadUrl);
+
+        $tmpname = tempnam(sys_get_temp_dir(), 'postalcodes');
+        $tmpfile = fopen($tmpname, 'r+b');
+
+        if (false === $tmpfile) {
+            throw new RuntimeException('could not create temporary file');
+        }
+
+        foreach ($this->client->stream($response) as $chunk) {
+            fwrite($tmpfile, $chunk->getContent());
+        }
+
+        fclose($tmpfile);
+
+        $uncompressedStream = gzopen($tmpname, 'r');
+
+        $csv = Reader::createFromStream($uncompressedStream);
+        $csv->setDelimiter(',');
+        $csv->setHeaderOffset(0);
+
+        foreach ($csv as $offset => $record) {
+            $this->handleRecord($record);
+        }
+
+        gzclose($uncompressedStream);
+        unlink($tmpname);
+
+        $this->logger->info(__CLASS__ . ' list of postal code downloaded');
+
+        $this->baseImporter->finalize();
+
+        $this->logger->info(__CLASS__ . ' postal code fetched', ['offset' => $offset ?? 0]);
+    }
+
+    private function getFileDownloadUrl(string $lang): string
+    {
+        try {
+            $release = $this->client->request('GET', self::RELEASE)
+                ->toArray();
+        } catch (TransportExceptionInterface $e) {
+            throw new RuntimeException('could not get the release definition', 0, $e);
+        }
+
+        $postals = array_filter($release['assets'], static function (array $item) use ($lang) {
+            return 'postals.' . $lang . '.csv.gz' === $item['name'];
+        });
+
+        return array_values($postals)[0]['browser_download_url'];
+    }
+
+    private function handleRecord(array $record): void
+    {
+        $this->baseImporter->importCode(
+            'BE',
+            trim($record['municipality_name']),
+            trim($record['postal_info_objectid']),
+            $record['municipality_objectid'],
+            'bestaddress',
+            $record['Y'],
+            $record['X'],
+            3812
+        );
+    }
+}

src/Bundle/ChillMainBundle/Service/Import/PostalCodeBaseImporter.php

@@ -13,6 +13,7 @@ namespace Chill\MainBundle\Service\Import;
 
 use Doctrine\DBAL\Connection;
 use Doctrine\DBAL\Statement;
+use Exception;
 use function array_key_exists;
 use function count;
 
@@ -40,7 +41,7 @@ class PostalCodeBaseImporter
                    0,
                    g.refpostalcodeid,
                    g.postalcodeSource,
-                   CASE WHEN (g.lon::float != 0.0 AND g.lat::float != 0.0) THEN ST_setSrid(ST_point(g.lon::float, g.lat::float), g.srid::int) ELSE NULL END,
+                   CASE WHEN (g.lon::float != 0.0 AND g.lat::float != 0.0) THEN ST_Transform(ST_setSrid(ST_point(g.lon::float, g.lat::float), g.srid::int), 4326) ELSE NULL END,
                    NOW(),
                    NOW()
                FROM g
@@ -112,7 +113,7 @@ class PostalCodeBaseImporter
 
         try {
             $statement->executeStatement(array_merge(...$this->waitingForInsert));
-        } catch (\Exception $e) {
+        } catch (Exception $e) {
             // in some case, we can add debug code here
             //dump($this->waitingForInsert);
             throw $e;

src/Bundle/ChillMainBundle/config/services/command.yaml

@@ -50,6 +50,12 @@ services:
         tags:
             - { name: console.command }
 
+    Chill\MainBundle\Command\LoadAddressesBEFromBestAddressCommand:
+        autoconfigure: true
+        autowire: true
+        tags:
+            - { name: console.command }
+
     Chill\MainBundle\Command\LoadPostalCodeFR:
         autoconfigure: true
         autowire: true