diff --git a/src/Bundle/ChillMainBundle/Command/LoadAddressesBEFromBestAddressCommand.php b/src/Bundle/ChillMainBundle/Command/LoadAddressesBEFromBestAddressCommand.php new file mode 100644 index 000000000..abd4e294a --- /dev/null +++ b/src/Bundle/ChillMainBundle/Command/LoadAddressesBEFromBestAddressCommand.php @@ -0,0 +1,52 @@ +addressImporter = $addressImporter; + $this->postalCodeBEFromBestAddressImporter = $postalCodeBEFromBestAddressImporter; + } + + protected function configure() + { + $this + ->setName('chill:main:address-ref-from-best-addresses') + ->addArgument('lang', InputArgument::REQUIRED) + ->addArgument('list', InputArgument::IS_ARRAY, 'The list to add'); + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + $this->postalCodeBEFromBestAddressImporter->import(); + + $this->addressImporter->import($input->getArgument('lang'), $input->getArgument('list')); + + return 0; + } +} diff --git a/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceBEFromBestAddress.php b/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceBEFromBestAddress.php new file mode 100644 index 000000000..b034dbca7 --- /dev/null +++ b/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceBEFromBestAddress.php @@ -0,0 +1,103 @@ +client = $client; + $this->baseImporter = $baseImporter; + } + + public function import(string $lang, array $lists): void + { + foreach ($lists as $list) { + $this->importList($lang, $list); + } + } + + private function getDownloadUrl(string $lang, string $list): string + { + try { + $release = $this->client->request('GET', self::RELEASE) + ->toArray(); + } catch (TransportExceptionInterface $e) { + throw new RuntimeException('could not get the release definition', 0, $e); + } + + $asset = array_filter($release['assets'], static function (array $item) use ($lang, $list) { + return 'addresses-' . $list . '.' . $lang . '.csv.gz' === $item['name']; + }); + + return array_values($asset)[0]['browser_download_url']; + } + + private function importList(string $lang, string $list): void + { + $downloadUrl = $this->getDownloadUrl($lang, $list); + + $response = $this->client->request('GET', $downloadUrl); + + if (200 !== $response->getStatusCode()) { + throw new Exception('Could not download CSV: ' . $response->getStatusCode()); + } + + $tmpname = tempnam(sys_get_temp_dir(), 'php-add-' . $list . $lang); + $file = fopen($tmpname, 'r+b'); + + foreach ($this->client->stream($response) as $chunk) { + fwrite($file, $chunk->getContent()); + } + + fclose($file); + + $uncompressedStream = gzopen($tmpname, 'r'); + + $csv = Reader::createFromStream($uncompressedStream); + $csv->setDelimiter(','); + $csv->setHeaderOffset(0); + + $stmt = Statement::create() + ->process($csv); + + foreach ($stmt as $record) { + $this->baseImporter->importAddress( + $record['best_id'], + $record['municipality_objectid'], + $record['postal_info_objectid'], + $record['streetname'], + $record['housenumber'] . $record['boxnumber'], + 'bestaddress.' . $list, + (float) $record['X'], + (float) $record['Y'], + 3812 + ); + } + + $this->baseImporter->finalize(); + + gzclose($uncompressedStream); + } +} diff --git a/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceBaseImporter.php b/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceBaseImporter.php index 87659e65d..0ca17b5de 100644 --- a/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceBaseImporter.php +++ b/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceBaseImporter.php @@ -26,7 +26,7 @@ final class AddressReferenceBaseImporter (postcode_id, refid, street, streetnumber, municipalitycode, source, point) SELECT cmpc.id, i.refid, i.street, i.streetnumber, i.refpostalcode, i.source, - CASE WHEN (i.lon::float != 0.0 AND i.lat::float != 0.0) THEN ST_setSrid(ST_point(i.lon::float, i.lat::float), i.srid::int) ELSE NULL END + CASE WHEN (i.lon::float != 0.0 AND i.lat::float != 0.0) THEN ST_Transform(ST_setSrid(ST_point(i.lon::float, i.lat::float), i.srid::int), 4326) ELSE NULL END FROM (VALUES {{ values }} @@ -137,9 +137,18 @@ final class AddressReferenceBaseImporter ), ]); + $this->logger->debug(self::LOG_PREFIX . ' generated sql for insert', [ + 'sql' => $sql, + 'forNumber' => $forNumber, + ]); + $this->cachingStatements[$forNumber] = $this->defaultConnection->prepare($sql); } + if (0 === $forNumber) { + return; + } + $this->logger->debug(self::LOG_PREFIX . ' inserting pending addresses', [ 'number' => $forNumber, 'first' => $this->waitingForInsert[0] ?? null, @@ -188,7 +197,7 @@ final class AddressReferenceBaseImporter NOW(), null, NOW() - FROM reference_address_temp + FROM reference_address_temp ON CONFLICT (refid, source) DO UPDATE SET postcode_id = excluded.postcode_id, refid = excluded.refid, street = excluded.street, streetnumber = excluded.streetnumber, municipalitycode = excluded.municipalitycode, source = excluded.source, point = excluded.point, updatedat = NOW(), deletedAt = NULL "); diff --git a/src/Bundle/ChillMainBundle/Service/Import/PostalCodeBEFromBestAddress.php b/src/Bundle/ChillMainBundle/Service/Import/PostalCodeBEFromBestAddress.php new file mode 100644 index 000000000..a5b8b216b --- /dev/null +++ b/src/Bundle/ChillMainBundle/Service/Import/PostalCodeBEFromBestAddress.php @@ -0,0 +1,105 @@ +baseImporter = $baseImporter; + $this->client = $client; + $this->logger = $logger; + } + + public function import(string $lang = 'fr'): void + { + $fileDownloadUrl = $this->getFileDownloadUrl($lang); + + $response = $this->client->request('GET', $fileDownloadUrl); + + $tmpname = tempnam(sys_get_temp_dir(), 'postalcodes'); + $tmpfile = fopen($tmpname, 'r+b'); + + if (false === $tmpfile) { + throw new RuntimeException('could not create temporary file'); + } + + foreach ($this->client->stream($response) as $chunk) { + fwrite($tmpfile, $chunk->getContent()); + } + + fclose($tmpfile); + + $uncompressedStream = gzopen($tmpname, 'r'); + + $csv = Reader::createFromStream($uncompressedStream); + $csv->setDelimiter(','); + $csv->setHeaderOffset(0); + + foreach ($csv as $offset => $record) { + $this->handleRecord($record); + } + + gzclose($uncompressedStream); + unlink($tmpname); + + $this->logger->info(__CLASS__ . ' list of postal code downloaded'); + + $this->baseImporter->finalize(); + + $this->logger->info(__CLASS__ . ' postal code fetched', ['offset' => $offset ?? 0]); + } + + private function getFileDownloadUrl(string $lang): string + { + try { + $release = $this->client->request('GET', self::RELEASE) + ->toArray(); + } catch (TransportExceptionInterface $e) { + throw new RuntimeException('could not get the release definition', 0, $e); + } + + $postals = array_filter($release['assets'], static function (array $item) use ($lang) { + return 'postals.' . $lang . '.csv.gz' === $item['name']; + }); + + return array_values($postals)[0]['browser_download_url']; + } + + private function handleRecord(array $record): void + { + $this->baseImporter->importCode( + 'BE', + trim($record['municipality_name']), + trim($record['postal_info_objectid']), + $record['municipality_objectid'], + 'bestaddress', + $record['Y'], + $record['X'], + 3812 + ); + } +} diff --git a/src/Bundle/ChillMainBundle/Service/Import/PostalCodeBaseImporter.php b/src/Bundle/ChillMainBundle/Service/Import/PostalCodeBaseImporter.php index a7e8c8e70..20b41420f 100644 --- a/src/Bundle/ChillMainBundle/Service/Import/PostalCodeBaseImporter.php +++ b/src/Bundle/ChillMainBundle/Service/Import/PostalCodeBaseImporter.php @@ -13,6 +13,7 @@ namespace Chill\MainBundle\Service\Import; use Doctrine\DBAL\Connection; use Doctrine\DBAL\Statement; +use Exception; use function array_key_exists; use function count; @@ -40,7 +41,7 @@ class PostalCodeBaseImporter 0, g.refpostalcodeid, g.postalcodeSource, - CASE WHEN (g.lon::float != 0.0 AND g.lat::float != 0.0) THEN ST_setSrid(ST_point(g.lon::float, g.lat::float), g.srid::int) ELSE NULL END, + CASE WHEN (g.lon::float != 0.0 AND g.lat::float != 0.0) THEN ST_Transform(ST_setSrid(ST_point(g.lon::float, g.lat::float), g.srid::int), 4326) ELSE NULL END, NOW(), NOW() FROM g @@ -112,7 +113,7 @@ class PostalCodeBaseImporter try { $statement->executeStatement(array_merge(...$this->waitingForInsert)); - } catch (\Exception $e) { + } catch (Exception $e) { // in some case, we can add debug code here //dump($this->waitingForInsert); throw $e; diff --git a/src/Bundle/ChillMainBundle/config/services/command.yaml b/src/Bundle/ChillMainBundle/config/services/command.yaml index 8c1d9b94e..e5f285545 100644 --- a/src/Bundle/ChillMainBundle/config/services/command.yaml +++ b/src/Bundle/ChillMainBundle/config/services/command.yaml @@ -50,6 +50,12 @@ services: tags: - { name: console.command } + Chill\MainBundle\Command\LoadAddressesBEFromBestAddressCommand: + autoconfigure: true + autowire: true + tags: + - { name: console.command } + Chill\MainBundle\Command\LoadPostalCodeFR: autoconfigure: true autowire: true