From ba25c181f5c228419af4811ce9d0bdb7f5c39be7 Mon Sep 17 00:00:00 2001 From: nobohan Date: Thu, 27 Jun 2024 10:07:24 +0200 Subject: [PATCH 1/5] DX import Luxembourg address command --- .changes/unreleased/DX-20240627-100653.yaml | 6 ++ .../LoadAddressesLUFromBDAddressCommand.php | 40 +++++++ .../Service/Import/AddressReferenceLU.php | 102 ++++++++++++++++++ 3 files changed, 148 insertions(+) create mode 100644 .changes/unreleased/DX-20240627-100653.yaml create mode 100644 src/Bundle/ChillMainBundle/Command/LoadAddressesLUFromBDAddressCommand.php create mode 100644 src/Bundle/ChillMainBundle/Service/Import/AddressReferenceLU.php diff --git a/.changes/unreleased/DX-20240627-100653.yaml b/.changes/unreleased/DX-20240627-100653.yaml new file mode 100644 index 000000000..538cf2697 --- /dev/null +++ b/.changes/unreleased/DX-20240627-100653.yaml @@ -0,0 +1,6 @@ +kind: DX +body: Add a command for reading official address DB from Luxembourg and update chill + addresses +time: 2024-06-27T10:06:53.098022939+02:00 +custom: + Issue: "" diff --git a/src/Bundle/ChillMainBundle/Command/LoadAddressesLUFromBDAddressCommand.php b/src/Bundle/ChillMainBundle/Command/LoadAddressesLUFromBDAddressCommand.php new file mode 100644 index 000000000..2477a486d --- /dev/null +++ b/src/Bundle/ChillMainBundle/Command/LoadAddressesLUFromBDAddressCommand.php @@ -0,0 +1,40 @@ +setName('chill:main:address-ref-lux'); + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + $this->addressImporter->import(); + + return Command::SUCCESS; + } +} diff --git a/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceLU.php b/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceLU.php new file mode 100644 index 000000000..c29921847 --- /dev/null +++ b/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceLU.php @@ -0,0 +1,102 @@ +client->request('GET', $downloadUrl); + + if (200 !== $response->getStatusCode()) { + throw new \Exception('Could not download CSV: '.$response->getStatusCode()); + } + + $tmpname = tempnam(sys_get_temp_dir(), 'php-add-'); + $file = fopen($tmpname, 'r+b'); + + foreach ($this->client->stream($response) as $chunk) { + fwrite($file, $chunk->getContent()); + } + + fclose($file); + + $uncompressedStream = gzopen($tmpname, 'r'); + + $csv = Reader::createFromStream($uncompressedStream); + $csv->setDelimiter(';'); + $csv->setHeaderOffset(0); + + $this->process_postal_code($csv); + + $this->process_address($csv); + + $this->addressToReferenceMatcher->checkAddressesMatchingReferences(); + + gzclose($uncompressedStream); + } + + private function process_address(Reader $csv): void + { + $stmt = Statement::create()->process($csv); + foreach ($stmt as $record) { + $this->addressBaseImporter->importAddress( + $record['id_geoportail'], + $record['code_postal'], + $record['code_postal'], + $record['rue'], + $record['numero'], + 'bd addresses', + (float) $record['lat_wgs84'], + (float) $record['lon_wgs84'], + 4326 + ); + } + + $this->addressBaseImporter->finalize(); + } + + private function process_postal_code(Reader $csv): void + { + $stmt = Statement::create()->process($csv); + $arr_postal_codes = []; + foreach ($stmt as $record) { + if (false === \in_array($record['code_postal'], $arr_postal_codes, true)) { + $this->postalCodeBaseImporter->importCode( + 'LU', + trim((string) $record['localite']), + trim((string) $record['code_postal']), + trim((string) $record['code_postal']), + 'bd addresses', + (float) $record['lat_wgs84'], + (float) $record['lon_wgs84'], + 4326 + ); + \array_push($arr_postal_codes, $record['code_postal']); + } + } + } +} + + + From 25ccb1630899f48895c32e24e271fff996e92694 Mon Sep 17 00:00:00 2001 From: nobohan Date: Thu, 27 Jun 2024 10:17:08 +0200 Subject: [PATCH 2/5] DX import Luxembourg address command - csfixer --- .../ChillMainBundle/Service/Import/AddressReferenceLU.php | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceLU.php b/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceLU.php index c29921847..3e45fc565 100644 --- a/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceLU.php +++ b/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceLU.php @@ -13,14 +13,15 @@ namespace Chill\MainBundle\Service\Import; use League\Csv\Reader; use League\Csv\Statement; -use Symfony\Contracts\HttpClient\Exception\TransportExceptionInterface; use Symfony\Contracts\HttpClient\HttpClientInterface; class AddressReferenceLU { private const RELEASE = 'https://data.public.lu/fr/datasets/r/5cadc5b8-6a7d-4283-87bc-f9e58dd771f7'; - public function __construct(private readonly HttpClientInterface $client, private readonly AddressReferenceBaseImporter $addressBaseImporter, private readonly PostalCodeBaseImporter $postalCodeBaseImporter, private readonly AddressToReferenceMatcher $addressToReferenceMatcher) {} + public function __construct(private readonly HttpClientInterface $client, private readonly AddressReferenceBaseImporter $addressBaseImporter, private readonly PostalCodeBaseImporter $postalCodeBaseImporter, private readonly AddressToReferenceMatcher $addressToReferenceMatcher) + { + } public function import(): void { @@ -97,6 +98,3 @@ class AddressReferenceLU } } } - - - From d9c50cffb7952b43796eef6d9574413921237e84 Mon Sep 17 00:00:00 2001 From: nobohan Date: Thu, 27 Jun 2024 10:34:41 +0200 Subject: [PATCH 3/5] DX import Luxembourg address command - phpstan --- .../Command/LoadAddressesLUFromBDAddressCommand.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Bundle/ChillMainBundle/Command/LoadAddressesLUFromBDAddressCommand.php b/src/Bundle/ChillMainBundle/Command/LoadAddressesLUFromBDAddressCommand.php index 2477a486d..499751c7c 100644 --- a/src/Bundle/ChillMainBundle/Command/LoadAddressesLUFromBDAddressCommand.php +++ b/src/Bundle/ChillMainBundle/Command/LoadAddressesLUFromBDAddressCommand.php @@ -35,6 +35,6 @@ class LoadAddressesLUFromBDAddressCommand extends Command { $this->addressImporter->import(); - return Command::SUCCESS; + return 0; } } From c5a24e8ac54135c5c7de24413ddc6ae7646330e0 Mon Sep 17 00:00:00 2001 From: nobohan Date: Fri, 28 Jun 2024 09:27:45 +0200 Subject: [PATCH 4/5] DX: Improve Lux address import command - WIP --- .../Service/Import/AddressReferenceLU.php | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceLU.php b/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceLU.php index 3e45fc565..f5cb03f19 100644 --- a/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceLU.php +++ b/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceLU.php @@ -42,9 +42,9 @@ class AddressReferenceLU fclose($file); - $uncompressedStream = gzopen($tmpname, 'r'); + //$uncompressedStream = gzopen($tmpname, 'r'); - $csv = Reader::createFromStream($uncompressedStream); + $csv = Reader::createFromStream($file); $csv->setDelimiter(';'); $csv->setHeaderOffset(0); @@ -54,7 +54,7 @@ class AddressReferenceLU $this->addressToReferenceMatcher->checkAddressesMatchingReferences(); - gzclose($uncompressedStream); + //gzclose($uncompressedStream); } private function process_address(Reader $csv): void @@ -67,7 +67,7 @@ class AddressReferenceLU $record['code_postal'], $record['rue'], $record['numero'], - 'bd addresses', + 'bd-addresses.lux', (float) $record['lat_wgs84'], (float) $record['lon_wgs84'], 4326 @@ -82,7 +82,8 @@ class AddressReferenceLU $stmt = Statement::create()->process($csv); $arr_postal_codes = []; foreach ($stmt as $record) { - if (false === \in_array($record['code_postal'], $arr_postal_codes, true)) { + //if (false === \in_array($record['code_postal'], $arr_postal_codes, true)) { + if (false === \array_key_exists($record['code_postal'], $arr_postal_codes, true)) { $this->postalCodeBaseImporter->importCode( 'LU', trim((string) $record['localite']), @@ -93,7 +94,7 @@ class AddressReferenceLU (float) $record['lon_wgs84'], 4326 ); - \array_push($arr_postal_codes, $record['code_postal']); + $arr_postal_codes[$record['code_postal']] = 1; } } } From 9c28df25a1b32499eec56acb4200ac9eb160effe Mon Sep 17 00:00:00 2001 From: nobohan Date: Fri, 28 Jun 2024 10:45:58 +0200 Subject: [PATCH 5/5] DX: Improve Lux adress import command + register in config --- .../Service/Import/AddressReferenceLU.php | 14 +++++--------- .../ChillMainBundle/config/services/command.yaml | 6 ++++++ 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceLU.php b/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceLU.php index f5cb03f19..d70b5085d 100644 --- a/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceLU.php +++ b/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceLU.php @@ -33,16 +33,13 @@ class AddressReferenceLU throw new \Exception('Could not download CSV: '.$response->getStatusCode()); } - $tmpname = tempnam(sys_get_temp_dir(), 'php-add-'); - $file = fopen($tmpname, 'r+b'); + $file = tmpfile(); foreach ($this->client->stream($response) as $chunk) { fwrite($file, $chunk->getContent()); } - fclose($file); - - //$uncompressedStream = gzopen($tmpname, 'r'); + fseek($file, 0); $csv = Reader::createFromStream($file); $csv->setDelimiter(';'); @@ -54,7 +51,7 @@ class AddressReferenceLU $this->addressToReferenceMatcher->checkAddressesMatchingReferences(); - //gzclose($uncompressedStream); + fclose($file); } private function process_address(Reader $csv): void @@ -82,14 +79,13 @@ class AddressReferenceLU $stmt = Statement::create()->process($csv); $arr_postal_codes = []; foreach ($stmt as $record) { - //if (false === \in_array($record['code_postal'], $arr_postal_codes, true)) { - if (false === \array_key_exists($record['code_postal'], $arr_postal_codes, true)) { + if (false === \array_key_exists($record['code_postal'], $arr_postal_codes)) { $this->postalCodeBaseImporter->importCode( 'LU', trim((string) $record['localite']), trim((string) $record['code_postal']), trim((string) $record['code_postal']), - 'bd addresses', + 'bd-addresses.lux', (float) $record['lat_wgs84'], (float) $record['lon_wgs84'], 4326 diff --git a/src/Bundle/ChillMainBundle/config/services/command.yaml b/src/Bundle/ChillMainBundle/config/services/command.yaml index 8a2327c0b..94cb2cf97 100644 --- a/src/Bundle/ChillMainBundle/config/services/command.yaml +++ b/src/Bundle/ChillMainBundle/config/services/command.yaml @@ -59,6 +59,12 @@ services: tags: - { name: console.command } + Chill\MainBundle\Command\LoadAddressesLUFromBDAddressCommand: + autoconfigure: true + autowire: true + tags: + - { name: console.command } + Chill\MainBundle\Command\ExecuteCronJobCommand: autoconfigure: true autowire: true