From c56ae08faefaa02e5488dbc32d7fac40779a6baf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Tue, 7 Mar 2023 22:16:16 +0100 Subject: [PATCH] Feature: [address reference] write logic to match existing addresses with updates Feature: [address reference] match addresses with reference after import fix sql query for address reference matcher --- .../AddressReferenceBEFromBestAddress.php | 16 ++- .../Import/AddressReferenceFromBano.php | 7 +- .../Import/AddressToReferenceMatcher.php | 87 ++++++++++++ .../Import/AddressToReferenceMatcherTest.php | 127 ++++++++++++++++++ 4 files changed, 232 insertions(+), 5 deletions(-) create mode 100644 src/Bundle/ChillMainBundle/Service/Import/AddressToReferenceMatcher.php create mode 100644 src/Bundle/ChillMainBundle/Tests/Services/Import/AddressToReferenceMatcherTest.php diff --git a/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceBEFromBestAddress.php b/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceBEFromBestAddress.php index 8dfc84bc3..88a9cb4c5 100644 --- a/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceBEFromBestAddress.php +++ b/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceBEFromBestAddress.php @@ -24,12 +24,18 @@ class AddressReferenceBEFromBestAddress private AddressReferenceBaseImporter $baseImporter; + private AddressToReferenceMatcher $addressToReferenceMatcher; + private HttpClientInterface $client; - public function __construct(HttpClientInterface $client, AddressReferenceBaseImporter $baseImporter) - { + public function __construct( + HttpClientInterface $client, + AddressReferenceBaseImporter $baseImporter, + AddressToReferenceMatcher $addressToReferenceMatcher + ) { $this->client = $client; $this->baseImporter = $baseImporter; + $this->addressToReferenceMatcher = $addressToReferenceMatcher; } public function import(string $lang, array $lists): void @@ -89,16 +95,18 @@ class AddressReferenceBEFromBestAddress $record['municipality_objectid'], $record['postal_info_objectid'], $record['streetname'], - $record['housenumber'] . $record['boxnumber'], + $record['housenumber'] .($record['boxnumber'] !== '' ? ' bte '. $record['boxnumber'] : ''), 'bestaddress.' . $list, - (float) $record['X'], (float) $record['Y'], + (float) $record['X'], 3812 ); } $this->baseImporter->finalize(); + $this->addressToReferenceMatcher->checkAddressesMatchingReferences(); + gzclose($uncompressedStream); } } diff --git a/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceFromBano.php b/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceFromBano.php index 7a9659884..d02a846ec 100644 --- a/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceFromBano.php +++ b/src/Bundle/ChillMainBundle/Service/Import/AddressReferenceFromBano.php @@ -22,12 +22,15 @@ class AddressReferenceFromBano { private AddressReferenceBaseImporter $baseImporter; + private AddressToReferenceMatcher $addressToReferenceMatcher; + private HttpClientInterface $client; - public function __construct(HttpClientInterface $client, AddressReferenceBaseImporter $baseImporter) + public function __construct(HttpClientInterface $client, AddressReferenceBaseImporter $baseImporter, AddressToReferenceMatcher $addressToReferenceMatcher) { $this->client = $client; $this->baseImporter = $baseImporter; + $this->addressToReferenceMatcher = $addressToReferenceMatcher; } public function import(string $departementNo): void @@ -82,6 +85,8 @@ class AddressReferenceFromBano $this->baseImporter->finalize(); + $this->addressToReferenceMatcher->checkAddressesMatchingReferences(); + fclose($file); } } diff --git a/src/Bundle/ChillMainBundle/Service/Import/AddressToReferenceMatcher.php b/src/Bundle/ChillMainBundle/Service/Import/AddressToReferenceMatcher.php new file mode 100644 index 000000000..de37cfc15 --- /dev/null +++ b/src/Bundle/ChillMainBundle/Service/Import/AddressToReferenceMatcher.php @@ -0,0 +1,87 @@ += NOW()) + -- only addresses that are marked matching or "to review", but before the update + AND + (a.refstatus LIKE '{{ matching }}' + OR (a.refstatus LIKE '{{ reviewed }}' AND a.refstatuslastupdate < ar.updatedat)) + AND ( + a.postcode_id != ar.postcode_id + OR a.street != ar.street + OR a.streetnumber != ar.streetnumber + OR ROUND(ST_X(a.point) * 1000000) <> ROUND(ST_X(ar.point) * 1000000) + OR ROUND(ST_Y(a.point) * 1000000) <> ROUND(ST_Y(ar.point) * 1000000) + ) + SQL; + + private const SQL_MARK_MATCHING_ADDRESSES_REVIEWED_OR_TO_REVIEW = <<= NOW()) + AND a.refstatus IN ('{{ to_review }}', '{{ reviewed }}') + AND a.postcode_id = ar.postcode_id + AND a.street = ar.street + AND a.streetnumber = ar.streetnumber + AND ROUND(ST_X(a.point) * 1000000) = ROUND(ST_X(ar.point) * 1000000) + AND ROUND(ST_Y(a.point) * 1000000) = ROUND(ST_Y(ar.point) * 1000000) + SQL; + + private const SUBSTITUTES = [ + '{{ to_review }}' => Address::ADDR_REFERENCE_STATUS_TO_REVIEW, + '{{ matching }}' => Address::ADDR_REFERENCE_STATUS_MATCH, + '{{ reviewed }}' => Address::ADDR_REFERENCE_STATUS_REVIEWED + ]; + + public function __construct(Connection $connection, LoggerInterface $logger) + { + $this->connection = $connection; + $this->logger = $logger; + } + + public function checkAddressesMatchingReferences(): void + { + $this->logger->notice(self::LOG_PREFIX.'Starting addresses matching'); + + $this->connection->transactional(function () { + $markedAsMatching = $this->connection->executeStatement( + strtr(self::SQL_MARK_MATCHING_ADDRESSES_REVIEWED_OR_TO_REVIEW, self::SUBSTITUTES) + ); + + $markedAsToReview = $this->connection->executeStatement( + strtr(self::SQL_MARK_TO_REVIEW_ADDRESS_UNMATCHING, self::SUBSTITUTES) + ); + + $this->logger->info(self::LOG_PREFIX.'Executed address matching', [ + 'marked_as_matching' => $markedAsMatching, + 'marked_as_to_review' => $markedAsToReview, + ]); + }); + } +} diff --git a/src/Bundle/ChillMainBundle/Tests/Services/Import/AddressToReferenceMatcherTest.php b/src/Bundle/ChillMainBundle/Tests/Services/Import/AddressToReferenceMatcherTest.php new file mode 100644 index 000000000..ad2324012 --- /dev/null +++ b/src/Bundle/ChillMainBundle/Tests/Services/Import/AddressToReferenceMatcherTest.php @@ -0,0 +1,127 @@ +addressToReferenceMatcher = self::$container->get(AddressToReferenceMatcher::class); + $this->addressReferenceRepository = self::$container->get(AddressReferenceRepository::class); + $this->countryRepository = self::$container->get(CountryRepository::class); + $this->addressReferenceBaseImporter = self::$container->get(AddressReferenceBaseImporter::class); + $this->entityManager = self::$container->get(EntityManagerInterface::class); + } + + public function testCheckAddressesMatchingReferences(): void + { + if (null === $belgium = $this->countryRepository->findOneBy(['countryCode' => 'BE'])) { + throw new \RuntimeException('Belgium not found'); + } + + $postalCode = (new PostalCode()) + ->setName('test for matcher') + ->setPostalCodeSource('test for matcher') + ->setCode('78910') + ->setRefPostalCodeId($refPostalCodeId = '78910'.uniqid()) + ->setCountry($belgium) + ; + $this->entityManager->persist($postalCode); + $this->entityManager->flush(); + + $this->addressReferenceBaseImporter->importAddress( + $refAddress = '010203_test'.uniqid(), + $refPostalCodeId, + '78910', + $street = 'Rue Poulet', + $streetNumber = '14', + 'test_matcher', + $lat = 50.0123456789, + $lon = 5.0123456789, + 4326 + ); + $this->addressReferenceBaseImporter->finalize(); + + if (null === $addressReference = $this->addressReferenceRepository->findOneBy(['refId' => $refAddress])) { + throw new \RuntimeException('address reference not created'); + } + + $address = Address::createFromAddressReference($addressReference); + + $this->assertEquals('Rue Poulet', $address->getStreet()); + + $this->entityManager->persist($address); + $this->entityManager->flush(); + + // we update the address + $this->addressReferenceBaseImporter->importAddress( + $refAddress, + $refPostalCodeId, + '78910', + 'Rue Poulet aux amandes', + '14', + 'test_matcher', + 50.01234456789, + 5.0123456789, + 4326 + ); + $this->addressReferenceBaseImporter->finalize(); + + $this->entityManager->flush(); + + $this->addressToReferenceMatcher->checkAddressesMatchingReferences(); + + $this->entityManager->refresh($address); + + $this->assertEquals('to_review', $address->getRefStatus()); + + // we update the address + $this->addressReferenceBaseImporter->importAddress( + $refAddress, + $refPostalCodeId, + '78910', + $street, + $streetNumber, + 'test_matcher', + $lat, + $lon, + 4326 + ); + $this->addressReferenceBaseImporter->finalize(); + + $this->addressToReferenceMatcher->checkAddressesMatchingReferences(); + + $this->entityManager->refresh($address); + + $this->assertEquals('Rue Poulet', $address->getStreet()); + $this->assertEquals('match', $address->getRefStatus()); + } +}