From 62ff4998a0a4c777b9b54e80c0af49b27d93c551 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Fri, 29 Jul 2022 22:53:40 +0200 Subject: [PATCH 1/8] Fixed: annotation schema for ManyToMany relationship between Evaluation and SocialAction Before this commit, the owning side of the relationship between Evaluation and SocialAction was declared twice. --- src/Bundle/ChillPersonBundle/Entity/SocialWork/Evaluation.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Bundle/ChillPersonBundle/Entity/SocialWork/Evaluation.php b/src/Bundle/ChillPersonBundle/Entity/SocialWork/Evaluation.php index e9c7496ff..131cc4bac 100644 --- a/src/Bundle/ChillPersonBundle/Entity/SocialWork/Evaluation.php +++ b/src/Bundle/ChillPersonBundle/Entity/SocialWork/Evaluation.php @@ -49,9 +49,8 @@ class Evaluation /** * @ORM\ManyToMany( * targetEntity=SocialAction::class, - * inversedBy="evaluations" + * mappedBy="evaluations" * ) - * @ORM\JoinTable(name="chill_person_social_work_evaluation_action") */ private Collection $socialActions; From a9b354a6f5aee2889bab2e3aa612823bce946e67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Sat, 30 Jul 2022 01:59:32 +0200 Subject: [PATCH 2/8] Feature: add constraint to ensure postal code uniqueness and track creation and update of postal code --- .../ChillMainBundle/Entity/PostalCode.php | 20 ++++++- .../migrations/Version20220729205416.php | 53 +++++++++++++++++++ 2 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 src/Bundle/ChillMainBundle/migrations/Version20220729205416.php diff --git a/src/Bundle/ChillMainBundle/Entity/PostalCode.php b/src/Bundle/ChillMainBundle/Entity/PostalCode.php index 4b79f58e8..769f6dfd7 100644 --- a/src/Bundle/ChillMainBundle/Entity/PostalCode.php +++ b/src/Bundle/ChillMainBundle/Entity/PostalCode.php @@ -12,6 +12,11 @@ declare(strict_types=1); namespace Chill\MainBundle\Entity; use Chill\MainBundle\Doctrine\Model\Point; +use Chill\MainBundle\Doctrine\Model\TrackCreationInterface; +use Chill\MainBundle\Doctrine\Model\TrackCreationTrait; +use Chill\MainBundle\Doctrine\Model\TrackUpdateInterface; +use Chill\MainBundle\Doctrine\Model\TrackUpdateTrait; +use DateTimeImmutable; use Doctrine\ORM\Mapping as ORM; use Symfony\Component\Serializer\Annotation\Groups; @@ -21,6 +26,10 @@ use Symfony\Component\Serializer\Annotation\Groups; * @ORM\Entity * @ORM\Table( * name="chill_main_postal_code", + * uniqueConstraints={ + * @ORM\UniqueConstraint(name="postal_code_import_unicity", columns={"code", "refpostalcodeid", "postalcodesource"}, + * options={"where": "refpostalcodeid is not null"}) + * }, * indexes={ * @ORM\Index(name="search_name_code", columns={"code", "label"}), * @ORM\Index(name="search_by_reference_code", columns={"code", "refpostalcodeid"}) @@ -28,8 +37,12 @@ use Symfony\Component\Serializer\Annotation\Groups; * * @ORM\HasLifecycleCallbacks */ -class PostalCode +class PostalCode implements TrackUpdateInterface, TrackCreationInterface { + use TrackCreationTrait; + + use TrackUpdateTrait; + /** * This is an internal column which is populated by database. * @@ -63,6 +76,11 @@ class PostalCode */ private $country; + /** + * @ORM\Column(type="datetime_immutable", nullable=true, options={"default": null}) + */ + private ?DateTimeImmutable $deletedAt = null; + /** * @var int * diff --git a/src/Bundle/ChillMainBundle/migrations/Version20220729205416.php b/src/Bundle/ChillMainBundle/migrations/Version20220729205416.php new file mode 100644 index 000000000..69ffd856f --- /dev/null +++ b/src/Bundle/ChillMainBundle/migrations/Version20220729205416.php @@ -0,0 +1,53 @@ +addSql('DROP INDEX postal_code_import_unicity'); + $this->addSql('ALTER TABLE chill_main_postal_code DROP deletedAt'); + $this->addSql('ALTER TABLE chill_main_postal_code DROP updatedAt'); + $this->addSql('ALTER TABLE chill_main_postal_code DROP createdAt'); + $this->addSql('ALTER TABLE chill_main_postal_code DROP updatedBy_id'); + $this->addSql('ALTER TABLE chill_main_postal_code DROP createdBy_id'); + $this->addSql('ALTER TABLE chill_main_postal_code DROP CONSTRAINT chill_internal_postal_code_import_unicity'); + } + + public function getDescription(): string + { + return 'postal code: add columns to track creation, update and deletion'; + } + + public function up(Schema $schema): void + { + $this->addSql('ALTER TABLE chill_main_postal_code ADD deletedAt TIMESTAMP(0) WITHOUT TIME ZONE DEFAULT NULL'); + $this->addSql('ALTER TABLE chill_main_postal_code ADD updatedAt TIMESTAMP(0) WITHOUT TIME ZONE DEFAULT NULL'); + $this->addSql('ALTER TABLE chill_main_postal_code ADD createdAt TIMESTAMP(0) WITHOUT TIME ZONE DEFAULT NULL'); + $this->addSql('ALTER TABLE chill_main_postal_code ADD updatedBy_id INT DEFAULT NULL'); + $this->addSql('ALTER TABLE chill_main_postal_code ADD createdBy_id INT DEFAULT NULL'); + $this->addSql('COMMENT ON COLUMN chill_main_postal_code.deletedAt IS \'(DC2Type:datetime_immutable)\''); + $this->addSql('COMMENT ON COLUMN chill_main_postal_code.updatedAt IS \'(DC2Type:datetime_immutable)\''); + $this->addSql('COMMENT ON COLUMN chill_main_postal_code.createdAt IS \'(DC2Type:datetime_immutable)\''); + $this->addSql('ALTER TABLE chill_main_postal_code ADD CONSTRAINT FK_6CA145FA65FF1AEC FOREIGN KEY (updatedBy_id) REFERENCES users (id) NOT DEFERRABLE INITIALLY IMMEDIATE'); + $this->addSql('ALTER TABLE chill_main_postal_code ADD CONSTRAINT FK_6CA145FA3174800F FOREIGN KEY (createdBy_id) REFERENCES users (id) NOT DEFERRABLE INITIALLY IMMEDIATE'); + $this->addSql('CREATE INDEX IDX_6CA145FA65FF1AEC ON chill_main_postal_code (updatedBy_id)'); + $this->addSql('CREATE INDEX IDX_6CA145FA3174800F ON chill_main_postal_code (createdBy_id)'); + $this->addSql('CREATE UNIQUE INDEX postal_code_import_unicity ON chill_main_postal_code (code, refpostalcodeid, postalcodesource) WHERE refpostalcodeid is not null'); + //$this->addSql('ALTER TABLE chill_main_postal_code ADD CONSTRAINT chill_internal_postal_code_import_unicity '. + // 'EXCLUDE (code WITH =, refpostalcodeid WITH =, postalcodesource WITH =) WHERE (refpostalcodeid IS NOT NULL)'); + } +} From 58ddf9038d625866da3c18f5e5089260cdf5f27c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Sat, 30 Jul 2022 02:01:42 +0200 Subject: [PATCH 3/8] Feature: load french postal code from laposte hexasmal open data --- .../Command/LoadPostalCodeFR.php | 42 ++++++ .../Service/Import/PostalCodeBaseImporter.php | 123 ++++++++++++++++++ .../Import/PostalCodeFRFromOpenData.php | 105 +++++++++++++++ .../ChillMainBundle/config/services.yaml | 4 + .../config/services/command.yaml | 6 + 5 files changed, 280 insertions(+) create mode 100644 src/Bundle/ChillMainBundle/Command/LoadPostalCodeFR.php create mode 100644 src/Bundle/ChillMainBundle/Service/Import/PostalCodeBaseImporter.php create mode 100644 src/Bundle/ChillMainBundle/Service/Import/PostalCodeFRFromOpenData.php diff --git a/src/Bundle/ChillMainBundle/Command/LoadPostalCodeFR.php b/src/Bundle/ChillMainBundle/Command/LoadPostalCodeFR.php new file mode 100644 index 000000000..1c7f9cbc5 --- /dev/null +++ b/src/Bundle/ChillMainBundle/Command/LoadPostalCodeFR.php @@ -0,0 +1,42 @@ +loader = $loader; + + parent::__construct(); + } + + public function configure(): void + { + $this->setName('chill:main:postal-code:load:FR') + ->setDescription('Load France\'s postal code from online open data'); + } + + public function execute(InputInterface $input, OutputInterface $output): int + { + $this->loader->import(); + + return 0; + } +} diff --git a/src/Bundle/ChillMainBundle/Service/Import/PostalCodeBaseImporter.php b/src/Bundle/ChillMainBundle/Service/Import/PostalCodeBaseImporter.php new file mode 100644 index 000000000..a7e8c8e70 --- /dev/null +++ b/src/Bundle/ChillMainBundle/Service/Import/PostalCodeBaseImporter.php @@ -0,0 +1,123 @@ + + */ + private array $cachingStatements = []; + + private Connection $defaultConnection; + + private array $waitingForInsert = []; + + public function __construct( + Connection $defaultConnection + ) { + $this->defaultConnection = $defaultConnection; + } + + public function finalize(): void + { + $this->doInsertPending(); + } + + public function importCode( + string $countryCode, + string $label, + string $code, + string $refPostalCodeId, + string $refPostalCodeSource, + float $centerLat, + float $centerLon, + int $centerSRID + ): void { + $this->waitingForInsert[] = [ + $countryCode, + $label, + $code, + $refPostalCodeId, + $refPostalCodeSource, + $centerLon, + $centerLat, + $centerSRID, + ]; + + if (100 <= count($this->waitingForInsert)) { + $this->doInsertPending(); + } + } + + private function doInsertPending(): void + { + if (!array_key_exists($forNumber = count($this->waitingForInsert), $this->cachingStatements)) { + $sql = strtr(self::QUERY, [ + '{{ values }}' => implode( + ', ', + array_fill(0, $forNumber, self::VALUE) + ), + ]); + + $this->cachingStatements[$forNumber] = $this->defaultConnection->prepare($sql); + } + + $statement = $this->cachingStatements[$forNumber]; + + try { + $statement->executeStatement(array_merge(...$this->waitingForInsert)); + } catch (\Exception $e) { + // in some case, we can add debug code here + //dump($this->waitingForInsert); + throw $e; + } finally { + $this->waitingForInsert = []; + } + } +} diff --git a/src/Bundle/ChillMainBundle/Service/Import/PostalCodeFRFromOpenData.php b/src/Bundle/ChillMainBundle/Service/Import/PostalCodeFRFromOpenData.php new file mode 100644 index 000000000..a49bb10a4 --- /dev/null +++ b/src/Bundle/ChillMainBundle/Service/Import/PostalCodeFRFromOpenData.php @@ -0,0 +1,105 @@ +baseImporter = $baseImporter; + $this->client = $client; + $this->logger = $logger; + } + + public function import(): void + { + $response = $this->client->request('GET', self::CSV); + + if (200 !== $response->getStatusCode()) { + throw new RuntimeException('could not download CSV'); + } + + $tmpfile = tmpfile(); + + if (false === $tmpfile) { + throw new RuntimeException('could not create temporary file'); + } + + foreach ($this->client->stream($response) as $chunk) { + fwrite($tmpfile, $chunk->getContent()); + } + + fseek($tmpfile, 0); + + $csv = Reader::createFromStream($tmpfile); + $csv->setDelimiter(';'); + $csv->setHeaderOffset(0); + + foreach ($csv as $offset => $record) { + $this->handleRecord($record); + } + + $this->baseImporter->finalize(); + fclose($tmpfile); + + $this->logger->info(__CLASS__ . ' postal code fetched', ['offset' => $offset]); + } + + private function handleRecord(array $record): void + { + if ('' !== trim($record['coordonnees_gps'])) { + [$lat, $lon] = array_map(static fn ($el) => (float) trim($el), explode(',', $record['coordonnees_gps'])); + } else { + $lat = $lon = 0.0; + } + + $ref = trim($record['Code_commune_INSEE']); + + if ('987' === substr($ref, 0, 3)) { + // some differences in French Polynesia + $ref .= '.' . trim($record['Libellé_d_acheminement']); + } + + $this->baseImporter->importCode( + 'FR', + trim($record['Libellé_d_acheminement']), + trim($record['Code_postal']), + $ref, + 'INSEE', + $lat, + $lon, + 4326 + ); + } +} diff --git a/src/Bundle/ChillMainBundle/config/services.yaml b/src/Bundle/ChillMainBundle/config/services.yaml index 6d55532a6..d3475752b 100644 --- a/src/Bundle/ChillMainBundle/config/services.yaml +++ b/src/Bundle/ChillMainBundle/config/services.yaml @@ -97,3 +97,7 @@ services: Chill\MainBundle\Security\Resolver\CenterResolverDispatcherInterface: '@Chill\MainBundle\Security\Resolver\CenterResolverDispatcher' + Chill\MainBundle\Service\Import\: + resource: '../Service/Import/' + autowire: true + autoconfigure: true diff --git a/src/Bundle/ChillMainBundle/config/services/command.yaml b/src/Bundle/ChillMainBundle/config/services/command.yaml index 87220bc1f..07b3c1721 100644 --- a/src/Bundle/ChillMainBundle/config/services/command.yaml +++ b/src/Bundle/ChillMainBundle/config/services/command.yaml @@ -43,3 +43,9 @@ services: $entityManager: '@doctrine.orm.entity_manager' tags: - { name: console.command } + + Chill\MainBundle\Command\LoadPostalCodeFR: + autoconfigure: true + autowire: true + tags: + - { name: console.command } From 9b3b9f2552a67f170591c353020b2449b7409113 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Sat, 30 Jul 2022 23:07:51 +0200 Subject: [PATCH 4/8] Fixed: possible unexisting variable --- .../ChillMainBundle/Service/Import/PostalCodeFRFromOpenData.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Bundle/ChillMainBundle/Service/Import/PostalCodeFRFromOpenData.php b/src/Bundle/ChillMainBundle/Service/Import/PostalCodeFRFromOpenData.php index a49bb10a4..26a52c840 100644 --- a/src/Bundle/ChillMainBundle/Service/Import/PostalCodeFRFromOpenData.php +++ b/src/Bundle/ChillMainBundle/Service/Import/PostalCodeFRFromOpenData.php @@ -73,7 +73,7 @@ class PostalCodeFRFromOpenData $this->baseImporter->finalize(); fclose($tmpfile); - $this->logger->info(__CLASS__ . ' postal code fetched', ['offset' => $offset]); + $this->logger->info(__CLASS__ . ' postal code fetched', ['offset' => $offset ?? 0]); } private function handleRecord(array $record): void From 84cda8845dd207e42f513327f4bb9c7a98b6e966 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Sat, 30 Jul 2022 23:10:19 +0200 Subject: [PATCH 5/8] Feature: command to load addresses from France from bano.openstreetmap.fr --- .../LoadAddressesFRFromBANOCommand.php | 47 ++++ .../Entity/AddressReference.php | 3 + .../Import/AddressReferenceBaseImporter.php | 207 ++++++++++++++++++ .../Import/AddressReferenceFromBano.php | 87 ++++++++ .../config/services/command.yaml | 6 + .../migrations/Version20220730204216.php | 26 +++ 6 files changed, 376 insertions(+) create mode 100644 src/Bundle/ChillMainBundle/Command/LoadAddressesFRFromBANOCommand.php create mode 100644 src/Bundle/ChillMainBundle/Service/Import/AddressReferenceBaseImporter.php create mode 100644 src/Bundle/ChillMainBundle/Service/Import/AddressReferenceFromBano.php create mode 100644 src/Bundle/ChillMainBundle/migrations/Version20220730204216.php diff --git a/src/Bundle/ChillMainBundle/Command/LoadAddressesFRFromBANOCommand.php b/src/Bundle/ChillMainBundle/Command/LoadAddressesFRFromBANOCommand.php new file mode 100644 index 000000000..96e7023fb --- /dev/null +++ b/src/Bundle/ChillMainBundle/Command/LoadAddressesFRFromBANOCommand.php @@ -0,0 +1,47 @@ +addressReferenceFromBano = $addressReferenceFromBano; + } + + protected function configure() + { + $this->setName('chill:main:address-ref-from-bano') + ->addArgument('departementNo', InputArgument::REQUIRED | InputArgument::IS_ARRAY, 'a list of departement numbers') + ->setDescription('Import addresses from bano (see https://bano.openstreetmap.fr'); + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + foreach ($input->getArgument('departementNo') as $departementNo) { + $output->writeln('Import addresses for ' . $departementNo); + + $this->addressReferenceFromBano->import($departementNo); + } + + return 0; + } +} diff --git a/src/Bundle/ChillMainBundle/Entity/AddressReference.php b/src/Bundle/ChillMainBundle/Entity/AddressReference.php index fc4339fe0..5a6b176c2 100644 --- a/src/Bundle/ChillMainBundle/Entity/AddressReference.php +++ b/src/Bundle/ChillMainBundle/Entity/AddressReference.php @@ -20,6 +20,9 @@ use Symfony\Component\Serializer\Annotation\Groups; * @ORM\Entity * @ORM\Table(name="chill_main_address_reference", indexes={ * @ORM\Index(name="address_refid", columns={"refId"}) + * }, + * uniqueConstraints={ + * @ORM\UniqueConstraint(name="chill_main_address_reference_unicity", columns={"refId", "source"}) * }) * @ORM\HasLifecycleCallbacks */ diff --git a/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceBaseImporter.php b/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceBaseImporter.php new file mode 100644 index 000000000..87659e65d --- /dev/null +++ b/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceBaseImporter.php @@ -0,0 +1,207 @@ + + */ + private array $cachingStatements = []; + + private ?string $currentSource = null; + + private Connection $defaultConnection; + + private bool $isInitialized = false; + + private LoggerInterface $logger; + + private array $waitingForInsert = []; + + public function __construct(Connection $defaultConnection, LoggerInterface $logger) + { + $this->defaultConnection = $defaultConnection; + $this->logger = $logger; + } + + public function finalize(): void + { + $this->doInsertPending(); + + $this->updateAddressReferenceTable(); + + $this->deleteTemporaryTable(); + + $this->currentSource = null; + $this->isInitialized = false; + } + + public function importAddress( + string $refAddress, + string $refPostalCode, + string $postalCode, + string $street, + string $streetNumber, + string $source, + ?float $lat = null, + ?float $lon = null, + ?int $srid = null + ): void { + if (!$this->isInitialized) { + $this->initialize($source); + } + + if ($this->currentSource !== $source) { + throw new LogicException('Cannot store addresses from different sources during same import. Execute finalize to commit inserts before changing the source'); + } + + $this->waitingForInsert[] = [ + $refAddress, + $refPostalCode, + $postalCode, + $street, + $streetNumber, + $source, + $lat, + $lon, + $srid, + ]; + + if (100 <= count($this->waitingForInsert)) { + $this->doInsertPending(); + } + } + + private function createTemporaryTable(): void + { + $this->defaultConnection->executeStatement('CREATE TEMPORARY TABLE reference_address_temp ( + postcode_id INT, + refid VARCHAR(255), + street VARCHAR(255), + streetnumber VARCHAR(255), + municipalitycode VARCHAR(255), + source VARCHAR(255), + point GEOMETRY + ); + '); + $this->defaultConnection->executeStatement('SET work_mem TO \'50MB\''); + } + + private function deleteTemporaryTable(): void + { + $this->defaultConnection->executeStatement('DROP TABLE IF EXISTS reference_address_temp'); + } + + private function doInsertPending(): void + { + if (!array_key_exists($forNumber = count($this->waitingForInsert), $this->cachingStatements)) { + $sql = strtr(self::INSERT, [ + '{{ values }}' => implode( + ', ', + array_fill(0, $forNumber, self::VALUE) + ), + ]); + + $this->cachingStatements[$forNumber] = $this->defaultConnection->prepare($sql); + } + + $this->logger->debug(self::LOG_PREFIX . ' inserting pending addresses', [ + 'number' => $forNumber, + 'first' => $this->waitingForInsert[0] ?? null, + ]); + + $statement = $this->cachingStatements[$forNumber]; + + try { + $statement->executeStatement(array_merge(...$this->waitingForInsert)); + } catch (Exception $e) { + // in some case, we can add debug code here + //dump($this->waitingForInsert); + throw $e; + } finally { + $this->waitingForInsert = []; + } + } + + private function initialize(string $source): void + { + $this->currentSource = $source; + $this->deleteTemporaryTable(); + $this->createTemporaryTable(); + $this->isInitialized = true; + } + + private function updateAddressReferenceTable(): void + { + $this->defaultConnection->executeStatement( + 'CREATE INDEX idx_ref_add_temp ON reference_address_temp (refid)' + ); + + //1) Add new addresses + $this->logger->info(self::LOG_PREFIX . 'upsert new addresses'); + $affected = $this->defaultConnection->executeStatement("INSERT INTO chill_main_address_reference + (id, postcode_id, refid, street, streetnumber, municipalitycode, source, point, createdat, deletedat, updatedat) + SELECT + nextval('chill_main_address_reference_id_seq'), + postcode_id, + refid, + street, + streetnumber, + municipalitycode, + source, + point, + NOW(), + null, + NOW() + FROM reference_address_temp + ON CONFLICT (refid, source) DO UPDATE + SET postcode_id = excluded.postcode_id, refid = excluded.refid, street = excluded.street, streetnumber = excluded.streetnumber, municipalitycode = excluded.municipalitycode, source = excluded.source, point = excluded.point, updatedat = NOW(), deletedAt = NULL + "); + $this->logger->info(self::LOG_PREFIX . 'addresses upserted', ['upserted' => $affected]); + + //3) Delete addresses + $this->logger->info(self::LOG_PREFIX . 'soft delete adresses'); + $affected = $this->defaultConnection->executeStatement('UPDATE chill_main_address_reference + SET deletedat = NOW() + WHERE + chill_main_address_reference.refid NOT IN (SELECT refid FROM reference_address_temp WHERE source LIKE ?) + AND chill_main_address_reference.source LIKE ? + ', [$this->currentSource, $this->currentSource]); + $this->logger->info(self::LOG_PREFIX . 'addresses deleted', ['deleted' => $affected]); + } +} diff --git a/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceFromBano.php b/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceFromBano.php new file mode 100644 index 000000000..c30b7c4f2 --- /dev/null +++ b/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceFromBano.php @@ -0,0 +1,87 @@ +client = $client; + $this->baseImporter = $baseImporter; + } + + public function import(string $departementNo): void + { + if (!is_numeric($departementNo) || !is_int((int) $departementNo)) { + throw new UnexpectedValueException('Could not parse this department number'); + } + + $url = "https://bano.openstreetmap.fr/data/bano-{$departementNo}.csv"; + + $response = $this->client->request('GET', $url); + + if (200 !== $response->getStatusCode()) { + throw new Exception('Could not download CSV: ' . $response->getStatusCode()); + } + + $file = tmpfile(); + + foreach ($this->client->stream($response) as $chunk) { + fwrite($file, $chunk->getContent()); + } + + fseek($file, 0); + + $csv = Reader::createFromStream($file); + $csv->setDelimiter(','); + $stmt = Statement::create() + ->process($csv, [ + 'refId', + 'streetNumber', + 'street', + 'postcode', + 'city', + '_o', + 'lat', + 'lon', + ]); + + foreach ($stmt as $record) { + $this->baseImporter->importAddress( + $record['refId'], + substr($record['refId'], 0, 5), // extract insee from reference + $record['postcode'], + $record['street'], + $record['streetNumber'], + 'BANO.' . $departementNo, + (float) $record['lat'], + (float) $record['lon'], + 4326 + ); + } + + $this->baseImporter->finalize(); + + fclose($file); + } +} diff --git a/src/Bundle/ChillMainBundle/config/services/command.yaml b/src/Bundle/ChillMainBundle/config/services/command.yaml index 07b3c1721..8c1d9b94e 100644 --- a/src/Bundle/ChillMainBundle/config/services/command.yaml +++ b/src/Bundle/ChillMainBundle/config/services/command.yaml @@ -44,6 +44,12 @@ services: tags: - { name: console.command } + Chill\MainBundle\Command\LoadAddressesFRFromBANOCommand: + autoconfigure: true + autowire: true + tags: + - { name: console.command } + Chill\MainBundle\Command\LoadPostalCodeFR: autoconfigure: true autowire: true diff --git a/src/Bundle/ChillMainBundle/migrations/Version20220730204216.php b/src/Bundle/ChillMainBundle/migrations/Version20220730204216.php new file mode 100644 index 000000000..4ed1b2918 --- /dev/null +++ b/src/Bundle/ChillMainBundle/migrations/Version20220730204216.php @@ -0,0 +1,26 @@ +addSql('CREATE UNIQUE INDEX chill_main_address_reference_unicity ON chill_main_address_reference (refId, source)'); + } + + public function down(Schema $schema): void + { + $this->addSql('DROP INDEX chill_main_address_reference_unicity'); + } +} From 0f63548d5a78611301eeb1e1f7eb765d6fd9608a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Sat, 3 Sep 2022 00:40:21 +0200 Subject: [PATCH 6/8] import addresses and postal codes from bestaddress --- .../LoadAddressesBEFromBestAddressCommand.php | 52 +++++++++ .../AddressReferenceBEFromBestAddress.php | 103 +++++++++++++++++ .../Import/AddressReferenceBaseImporter.php | 13 ++- .../Import/PostalCodeBEFromBestAddress.php | 105 ++++++++++++++++++ .../Service/Import/PostalCodeBaseImporter.php | 5 +- .../config/services/command.yaml | 6 + 6 files changed, 280 insertions(+), 4 deletions(-) create mode 100644 src/Bundle/ChillMainBundle/Command/LoadAddressesBEFromBestAddressCommand.php create mode 100644 src/Bundle/ChillMainBundle/Service/Import/AddressReferenceBEFromBestAddress.php create mode 100644 src/Bundle/ChillMainBundle/Service/Import/PostalCodeBEFromBestAddress.php diff --git a/src/Bundle/ChillMainBundle/Command/LoadAddressesBEFromBestAddressCommand.php b/src/Bundle/ChillMainBundle/Command/LoadAddressesBEFromBestAddressCommand.php new file mode 100644 index 000000000..abd4e294a --- /dev/null +++ b/src/Bundle/ChillMainBundle/Command/LoadAddressesBEFromBestAddressCommand.php @@ -0,0 +1,52 @@ +addressImporter = $addressImporter; + $this->postalCodeBEFromBestAddressImporter = $postalCodeBEFromBestAddressImporter; + } + + protected function configure() + { + $this + ->setName('chill:main:address-ref-from-best-addresses') + ->addArgument('lang', InputArgument::REQUIRED) + ->addArgument('list', InputArgument::IS_ARRAY, 'The list to add'); + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + $this->postalCodeBEFromBestAddressImporter->import(); + + $this->addressImporter->import($input->getArgument('lang'), $input->getArgument('list')); + + return 0; + } +} diff --git a/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceBEFromBestAddress.php b/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceBEFromBestAddress.php new file mode 100644 index 000000000..b034dbca7 --- /dev/null +++ b/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceBEFromBestAddress.php @@ -0,0 +1,103 @@ +client = $client; + $this->baseImporter = $baseImporter; + } + + public function import(string $lang, array $lists): void + { + foreach ($lists as $list) { + $this->importList($lang, $list); + } + } + + private function getDownloadUrl(string $lang, string $list): string + { + try { + $release = $this->client->request('GET', self::RELEASE) + ->toArray(); + } catch (TransportExceptionInterface $e) { + throw new RuntimeException('could not get the release definition', 0, $e); + } + + $asset = array_filter($release['assets'], static function (array $item) use ($lang, $list) { + return 'addresses-' . $list . '.' . $lang . '.csv.gz' === $item['name']; + }); + + return array_values($asset)[0]['browser_download_url']; + } + + private function importList(string $lang, string $list): void + { + $downloadUrl = $this->getDownloadUrl($lang, $list); + + $response = $this->client->request('GET', $downloadUrl); + + if (200 !== $response->getStatusCode()) { + throw new Exception('Could not download CSV: ' . $response->getStatusCode()); + } + + $tmpname = tempnam(sys_get_temp_dir(), 'php-add-' . $list . $lang); + $file = fopen($tmpname, 'r+b'); + + foreach ($this->client->stream($response) as $chunk) { + fwrite($file, $chunk->getContent()); + } + + fclose($file); + + $uncompressedStream = gzopen($tmpname, 'r'); + + $csv = Reader::createFromStream($uncompressedStream); + $csv->setDelimiter(','); + $csv->setHeaderOffset(0); + + $stmt = Statement::create() + ->process($csv); + + foreach ($stmt as $record) { + $this->baseImporter->importAddress( + $record['best_id'], + $record['municipality_objectid'], + $record['postal_info_objectid'], + $record['streetname'], + $record['housenumber'] . $record['boxnumber'], + 'bestaddress.' . $list, + (float) $record['X'], + (float) $record['Y'], + 3812 + ); + } + + $this->baseImporter->finalize(); + + gzclose($uncompressedStream); + } +} diff --git a/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceBaseImporter.php b/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceBaseImporter.php index 87659e65d..0ca17b5de 100644 --- a/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceBaseImporter.php +++ b/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceBaseImporter.php @@ -26,7 +26,7 @@ final class AddressReferenceBaseImporter (postcode_id, refid, street, streetnumber, municipalitycode, source, point) SELECT cmpc.id, i.refid, i.street, i.streetnumber, i.refpostalcode, i.source, - CASE WHEN (i.lon::float != 0.0 AND i.lat::float != 0.0) THEN ST_setSrid(ST_point(i.lon::float, i.lat::float), i.srid::int) ELSE NULL END + CASE WHEN (i.lon::float != 0.0 AND i.lat::float != 0.0) THEN ST_Transform(ST_setSrid(ST_point(i.lon::float, i.lat::float), i.srid::int), 4326) ELSE NULL END FROM (VALUES {{ values }} @@ -137,9 +137,18 @@ final class AddressReferenceBaseImporter ), ]); + $this->logger->debug(self::LOG_PREFIX . ' generated sql for insert', [ + 'sql' => $sql, + 'forNumber' => $forNumber, + ]); + $this->cachingStatements[$forNumber] = $this->defaultConnection->prepare($sql); } + if (0 === $forNumber) { + return; + } + $this->logger->debug(self::LOG_PREFIX . ' inserting pending addresses', [ 'number' => $forNumber, 'first' => $this->waitingForInsert[0] ?? null, @@ -188,7 +197,7 @@ final class AddressReferenceBaseImporter NOW(), null, NOW() - FROM reference_address_temp + FROM reference_address_temp ON CONFLICT (refid, source) DO UPDATE SET postcode_id = excluded.postcode_id, refid = excluded.refid, street = excluded.street, streetnumber = excluded.streetnumber, municipalitycode = excluded.municipalitycode, source = excluded.source, point = excluded.point, updatedat = NOW(), deletedAt = NULL "); diff --git a/src/Bundle/ChillMainBundle/Service/Import/PostalCodeBEFromBestAddress.php b/src/Bundle/ChillMainBundle/Service/Import/PostalCodeBEFromBestAddress.php new file mode 100644 index 000000000..a5b8b216b --- /dev/null +++ b/src/Bundle/ChillMainBundle/Service/Import/PostalCodeBEFromBestAddress.php @@ -0,0 +1,105 @@ +baseImporter = $baseImporter; + $this->client = $client; + $this->logger = $logger; + } + + public function import(string $lang = 'fr'): void + { + $fileDownloadUrl = $this->getFileDownloadUrl($lang); + + $response = $this->client->request('GET', $fileDownloadUrl); + + $tmpname = tempnam(sys_get_temp_dir(), 'postalcodes'); + $tmpfile = fopen($tmpname, 'r+b'); + + if (false === $tmpfile) { + throw new RuntimeException('could not create temporary file'); + } + + foreach ($this->client->stream($response) as $chunk) { + fwrite($tmpfile, $chunk->getContent()); + } + + fclose($tmpfile); + + $uncompressedStream = gzopen($tmpname, 'r'); + + $csv = Reader::createFromStream($uncompressedStream); + $csv->setDelimiter(','); + $csv->setHeaderOffset(0); + + foreach ($csv as $offset => $record) { + $this->handleRecord($record); + } + + gzclose($uncompressedStream); + unlink($tmpname); + + $this->logger->info(__CLASS__ . ' list of postal code downloaded'); + + $this->baseImporter->finalize(); + + $this->logger->info(__CLASS__ . ' postal code fetched', ['offset' => $offset ?? 0]); + } + + private function getFileDownloadUrl(string $lang): string + { + try { + $release = $this->client->request('GET', self::RELEASE) + ->toArray(); + } catch (TransportExceptionInterface $e) { + throw new RuntimeException('could not get the release definition', 0, $e); + } + + $postals = array_filter($release['assets'], static function (array $item) use ($lang) { + return 'postals.' . $lang . '.csv.gz' === $item['name']; + }); + + return array_values($postals)[0]['browser_download_url']; + } + + private function handleRecord(array $record): void + { + $this->baseImporter->importCode( + 'BE', + trim($record['municipality_name']), + trim($record['postal_info_objectid']), + $record['municipality_objectid'], + 'bestaddress', + $record['Y'], + $record['X'], + 3812 + ); + } +} diff --git a/src/Bundle/ChillMainBundle/Service/Import/PostalCodeBaseImporter.php b/src/Bundle/ChillMainBundle/Service/Import/PostalCodeBaseImporter.php index a7e8c8e70..20b41420f 100644 --- a/src/Bundle/ChillMainBundle/Service/Import/PostalCodeBaseImporter.php +++ b/src/Bundle/ChillMainBundle/Service/Import/PostalCodeBaseImporter.php @@ -13,6 +13,7 @@ namespace Chill\MainBundle\Service\Import; use Doctrine\DBAL\Connection; use Doctrine\DBAL\Statement; +use Exception; use function array_key_exists; use function count; @@ -40,7 +41,7 @@ class PostalCodeBaseImporter 0, g.refpostalcodeid, g.postalcodeSource, - CASE WHEN (g.lon::float != 0.0 AND g.lat::float != 0.0) THEN ST_setSrid(ST_point(g.lon::float, g.lat::float), g.srid::int) ELSE NULL END, + CASE WHEN (g.lon::float != 0.0 AND g.lat::float != 0.0) THEN ST_Transform(ST_setSrid(ST_point(g.lon::float, g.lat::float), g.srid::int), 4326) ELSE NULL END, NOW(), NOW() FROM g @@ -112,7 +113,7 @@ class PostalCodeBaseImporter try { $statement->executeStatement(array_merge(...$this->waitingForInsert)); - } catch (\Exception $e) { + } catch (Exception $e) { // in some case, we can add debug code here //dump($this->waitingForInsert); throw $e; diff --git a/src/Bundle/ChillMainBundle/config/services/command.yaml b/src/Bundle/ChillMainBundle/config/services/command.yaml index 8c1d9b94e..e5f285545 100644 --- a/src/Bundle/ChillMainBundle/config/services/command.yaml +++ b/src/Bundle/ChillMainBundle/config/services/command.yaml @@ -50,6 +50,12 @@ services: tags: - { name: console.command } + Chill\MainBundle\Command\LoadAddressesBEFromBestAddressCommand: + autoconfigure: true + autowire: true + tags: + - { name: console.command } + Chill\MainBundle\Command\LoadPostalCodeFR: autoconfigure: true autowire: true From 658e846120cd9a5b75214688050defdafcbd7f3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Sat, 17 Sep 2022 10:44:26 +0200 Subject: [PATCH 7/8] add test for AddressReferenceBaseImporter --- .../Import/AddressReferenceBaseImporter.php | 8 +- .../AddressReferenceBaseImporterTest.php | 87 +++++++++++++++++++ 2 files changed, 93 insertions(+), 2 deletions(-) create mode 100644 src/Bundle/ChillMainBundle/Tests/Services/Import/AddressReferenceBaseImporterTest.php diff --git a/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceBaseImporter.php b/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceBaseImporter.php index 0ca17b5de..6f1d218a9 100644 --- a/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceBaseImporter.php +++ b/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceBaseImporter.php @@ -73,7 +73,7 @@ final class AddressReferenceBaseImporter public function importAddress( string $refAddress, - string $refPostalCode, + ?string $refPostalCode, string $postalCode, string $street, string $streetNumber, @@ -157,7 +157,11 @@ final class AddressReferenceBaseImporter $statement = $this->cachingStatements[$forNumber]; try { - $statement->executeStatement(array_merge(...$this->waitingForInsert)); + $affected = $statement->executeStatement(array_merge(...$this->waitingForInsert)); + + if ($affected === 0) { + throw new \RuntimeException('no row affected'); + } } catch (Exception $e) { // in some case, we can add debug code here //dump($this->waitingForInsert); diff --git a/src/Bundle/ChillMainBundle/Tests/Services/Import/AddressReferenceBaseImporterTest.php b/src/Bundle/ChillMainBundle/Tests/Services/Import/AddressReferenceBaseImporterTest.php new file mode 100644 index 000000000..d4302413e --- /dev/null +++ b/src/Bundle/ChillMainBundle/Tests/Services/Import/AddressReferenceBaseImporterTest.php @@ -0,0 +1,87 @@ +importer = self::$container->get(AddressReferenceBaseImporter::class); + $this->addressReferenceRepository = self::$container->get(AddressReferenceRepository::class); + $this->entityManager = self::$container->get(EntityManagerInterface::class); + $this->postalCodeRepository = self::$container->get(PostalCodeRepository::class); + } + + public function testImportAddress(): void + { + $postalCode = (new PostalCode()) + ->setRefPostalCodeId($postalCodeId = '1234'.uniqid()) + ->setPostalCodeSource('testing') + ->setCode('TEST456') + ->setName('testing'); + + $this->entityManager->persist($postalCode); + $this->entityManager->flush(); + + $this->importer->importAddress( + '0000', + $postalCodeId, + 'TEST456', + 'Rue test abccc-guessed', + '-1', + 'unit-test', + 50.0, + 5.0, + 4326 + ); + + $this->importer->finalize(); + + $addresses = $this->addressReferenceRepository->findByPostalCodePattern( + $postalCode, + 'Rue test abcc guessed'); + + $this->assertCount(1, $addresses); + $this->assertEquals('Rue test abccc-guessed', $addresses[0]->getStreet()); + + $this->entityManager->clear(); + + $this->importer->importAddress( + '0000', + $postalCodeId, + 'TEST456', + 'Rue test abccc guessed fixed', + '-1', + 'unit-test', + 50.0, + 5.0, + 4326 + ); + + $this->importer->finalize(); + + $addresses = $this->addressReferenceRepository->findByPostalCodePattern( + $postalCode, + 'abcc guessed fixed'); + + $this->assertCount('1', $addresses); + $this->assertEquals( 'Rue test abccc guessed fixed', $addresses[0]->getStreet()); + } + + +} \ No newline at end of file From 8129739d27b76497e87d9c6e4e7932ed9d56b771 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Sat, 17 Sep 2022 10:59:04 +0200 Subject: [PATCH 8/8] test for postal code base importer --- .../AddressReferenceBaseImporterTest.php | 3 + .../Import/PostalCodeBaseImporterTest.php | 85 +++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100644 src/Bundle/ChillMainBundle/Tests/Services/Import/PostalCodeBaseImporterTest.php diff --git a/src/Bundle/ChillMainBundle/Tests/Services/Import/AddressReferenceBaseImporterTest.php b/src/Bundle/ChillMainBundle/Tests/Services/Import/AddressReferenceBaseImporterTest.php index d4302413e..cb23634f1 100644 --- a/src/Bundle/ChillMainBundle/Tests/Services/Import/AddressReferenceBaseImporterTest.php +++ b/src/Bundle/ChillMainBundle/Tests/Services/Import/AddressReferenceBaseImporterTest.php @@ -59,6 +59,8 @@ class AddressReferenceBaseImporterTest extends KernelTestCase $this->assertCount(1, $addresses); $this->assertEquals('Rue test abccc-guessed', $addresses[0]->getStreet()); + $previousAddressId = $addresses[0]->getId(); + $this->entityManager->clear(); $this->importer->importAddress( @@ -81,6 +83,7 @@ class AddressReferenceBaseImporterTest extends KernelTestCase $this->assertCount('1', $addresses); $this->assertEquals( 'Rue test abccc guessed fixed', $addresses[0]->getStreet()); + $this->assertEquals($previousAddressId, $addresses[0]->getId()); } diff --git a/src/Bundle/ChillMainBundle/Tests/Services/Import/PostalCodeBaseImporterTest.php b/src/Bundle/ChillMainBundle/Tests/Services/Import/PostalCodeBaseImporterTest.php new file mode 100644 index 000000000..826e581ac --- /dev/null +++ b/src/Bundle/ChillMainBundle/Tests/Services/Import/PostalCodeBaseImporterTest.php @@ -0,0 +1,85 @@ +entityManager = self::$container->get(EntityManagerInterface::class); + $this->importer = self::$container->get(PostalCodeBaseImporter::class); + $this->postalCodeRepository = self::$container->get(PostalCodeRepository::class); + $this->countryRepository = self::$container->get(CountryRepository::class); + } + + public function testImportPostalCode(): void + { + $this->importer->importCode( + 'BE', + 'tested with pattern '. ($uniqid = uniqid()), + '12345', + $refPostalCodeId = 'test'.uniqid(), + 'test', + 50.0, + 5.0, + 4326 + ); + + $this->importer->finalize(); + + $postalCodes = $this->postalCodeRepository->findByPattern( + 'with pattern '.$uniqid, + $this->countryRepository->findOneBy(['countryCode' => 'BE']) + ); + + $this->assertCount(1, $postalCodes); + $this->assertStringStartsWith('tested with pattern', $postalCodes[0]->getName()); + + $previousId = $postalCodes[0]->getId(); + + $this->entityManager->clear(); + + $this->importer->importCode( + 'BE', + 'tested with adapted pattern '. ($uniqid = uniqid()), + '12345', + $refPostalCodeId, + 'test', + 50.0, + 5.0, + 4326 + ); + + $this->importer->finalize(); + + $postalCodes = $this->postalCodeRepository->findByPattern( + 'with pattern '.$uniqid, + $this->countryRepository->findOneBy(['countryCode' => 'BE']) + ); + + $this->assertCount(1, $postalCodes); + $this->assertStringStartsWith('tested with adapted pattern', $postalCodes[0]->getName()); + $this->assertEquals($previousId, $postalCodes[0]->getId()); + } + + + +} \ No newline at end of file