From 030553a4de2a32d0ef484748c37499735b3ce49f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Mon, 23 Feb 2026 20:05:04 +0000 Subject: [PATCH] =?UTF-8?q?Resolve=20"Lors=20de=20l'import=20de=20code=20p?= =?UTF-8?q?ostaux,=20les=20codes=20absents=20de=20l'import=20depuis=20la?= =?UTF-8?q?=20m=C3=AAme=20source=20ne=20sont=20pas=20supprim=C3=A9s"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../unreleased/Fixed-20260223-182212.yaml | 7 ++ .../ChillMainBundle/Entity/PostalCode.php | 10 ++ .../Service/Import/PostalCodeBaseImporter.php | 104 ++++++++++++++++-- .../Import/PostalCodeBaseImporterTest.php | 76 +++++++++++++ 4 files changed, 185 insertions(+), 12 deletions(-) create mode 100644 .changes/unreleased/Fixed-20260223-182212.yaml diff --git a/.changes/unreleased/Fixed-20260223-182212.yaml b/.changes/unreleased/Fixed-20260223-182212.yaml new file mode 100644 index 000000000..ec6a4423c --- /dev/null +++ b/.changes/unreleased/Fixed-20260223-182212.yaml @@ -0,0 +1,7 @@ +kind: Fixed +body: 'Fix import of postal code: mark postal code as deleted if they are not present in the import any more' +time: 2026-02-23T18:22:12.92214987+01:00 +custom: + Issue: "502" + MR: "968" + SchemaChange: No schema change diff --git a/src/Bundle/ChillMainBundle/Entity/PostalCode.php b/src/Bundle/ChillMainBundle/Entity/PostalCode.php index ecab19a5f..d6ffe763f 100644 --- a/src/Bundle/ChillMainBundle/Entity/PostalCode.php +++ b/src/Bundle/ChillMainBundle/Entity/PostalCode.php @@ -215,4 +215,14 @@ class PostalCode implements TrackUpdateInterface, TrackCreationInterface return $this; } + + public function isDeleted(): bool + { + return null !== $this->deletedAt; + } + + public function getDeletedAt(): ?\DateTimeImmutable + { + return $this->deletedAt; + } } diff --git a/src/Bundle/ChillMainBundle/Service/Import/PostalCodeBaseImporter.php b/src/Bundle/ChillMainBundle/Service/Import/PostalCodeBaseImporter.php index b1ace3875..a10579b23 100644 --- a/src/Bundle/ChillMainBundle/Service/Import/PostalCodeBaseImporter.php +++ b/src/Bundle/ChillMainBundle/Service/Import/PostalCodeBaseImporter.php @@ -19,31 +19,66 @@ use Doctrine\DBAL\Statement; */ class PostalCodeBaseImporter { - private const QUERY = <<<'SQL' + private const CREATE_TEMP_TABLE = <<<'SQL' + CREATE TEMPORARY TABLE chill_main_postal_code_temp ( + countrycode VARCHAR(10), + label VARCHAR(255), + code VARCHAR(100), + refpostalcodeid VARCHAR(255), + postalcodeSource VARCHAR(255), + lon FLOAT, + lat FLOAT, + srid INT + ) + SQL; + + private const INSERT_TEMP = <<<'SQL' + INSERT INTO chill_main_postal_code_temp + (countrycode, label, code, refpostalcodeid, postalcodeSource, lon, lat, srid) + VALUES + {{ values }} + SQL; + + private const UPSERT = <<<'SQL' WITH g AS ( SELECT DISTINCT country.id AS country_id, - g.* - FROM (VALUES - {{ values }} - ) AS g (countrycode, label, code, refpostalcodeid, postalcodeSource, lon, lat, srid) - JOIN country ON country.countrycode = g.countrycode + temp.* + FROM chill_main_postal_code_temp temp + JOIN country ON country.countrycode = temp.countrycode ) - INSERT INTO chill_main_postal_code (id, country_id, label, code, origin, refpostalcodeid, postalcodeSource, center, createdAt, updatedAt) + INSERT INTO chill_main_postal_code (id, country_id, label, code, origin, refpostalcodeid, postalcodeSource, center, createdAt, updatedAt, deletedAt) SELECT nextval('chill_main_postal_code_id_seq'), g.country_id, - g.label AS glabel, + g.label, g.code, 0, g.refpostalcodeid, g.postalcodeSource, - CASE WHEN (g.lon::float != 0.0 AND g.lat::float != 0.0) THEN ST_Transform(ST_setSrid(ST_point(g.lon::float, g.lat::float), g.srid::int), 4326) ELSE NULL END, + CASE WHEN (g.lon != 0.0 AND g.lat != 0.0) THEN ST_Transform(ST_setSrid(ST_point(g.lon, g.lat), g.srid), 4326) ELSE NULL END, NOW(), - NOW() + NOW(), + NULL FROM g ON CONFLICT (code, refpostalcodeid, postalcodeSource) WHERE refpostalcodeid IS NOT NULL DO UPDATE - SET label = excluded.label, center = excluded.center, updatedAt = CASE WHEN NOT st_equals(excluded.center, chill_main_postal_code.center) OR excluded.label != chill_main_postal_code.label THEN NOW() ELSE chill_main_postal_code.updatedAt END + SET label = excluded.label, + center = excluded.center, + deletedAt = NULL, + updatedAt = CASE WHEN NOT st_equals(excluded.center, chill_main_postal_code.center) OR excluded.label != chill_main_postal_code.label OR chill_main_postal_code.deletedAt IS NOT NULL THEN NOW() ELSE chill_main_postal_code.updatedAt END + SQL; + + private const DELETE_MISSING = <<<'SQL' + UPDATE chill_main_postal_code + SET deletedAt = NOW(), updatedAt = NOW() + WHERE postalcodeSource = ? + AND deletedAt IS NULL + AND NOT EXISTS ( + SELECT 1 FROM chill_main_postal_code_temp temp + WHERE temp.code = chill_main_postal_code.code + AND temp.refpostalcodeid = chill_main_postal_code.refpostalcodeid + AND temp.postalcodeSource = chill_main_postal_code.postalcodeSource + ) SQL; private const VALUE = '(?, ?, ?, ?, ?, ?, ?, ?)'; @@ -55,11 +90,26 @@ class PostalCodeBaseImporter private array $waitingForInsert = []; + private bool $isInitialized = false; + + private ?string $currentSource = null; + public function __construct(private readonly Connection $defaultConnection) {} public function finalize(): void { $this->doInsertPending(); + + if ($this->isInitialized && null !== $this->currentSource) { + $this->defaultConnection->transactional(function (Connection $connection): void { + $connection->executeStatement(self::UPSERT); + $connection->executeStatement(self::DELETE_MISSING, [$this->currentSource]); + }); + $this->deleteTemporaryTable(); + } + + $this->isInitialized = false; + $this->currentSource = null; } public function importCode( @@ -72,6 +122,14 @@ class PostalCodeBaseImporter float $centerLon, int $centerSRID, ): void { + if (!$this->isInitialized) { + $this->initialize($refPostalCodeSource); + } + + if ($this->currentSource !== $refPostalCodeSource) { + throw new \LogicException('Cannot store postal codes from different sources during same import. Execute finalize to commit inserts before changing the source'); + } + $this->waitingForInsert[] = [ $countryCode, $label, @@ -88,10 +146,32 @@ class PostalCodeBaseImporter } } + private function initialize(string $source): void + { + $this->currentSource = $source; + $this->deleteTemporaryTable(); + $this->createTemporaryTable(); + $this->isInitialized = true; + } + + private function createTemporaryTable(): void + { + $this->defaultConnection->executeStatement(self::CREATE_TEMP_TABLE); + } + + private function deleteTemporaryTable(): void + { + $this->defaultConnection->executeStatement('DROP TABLE IF EXISTS chill_main_postal_code_temp'); + } + private function doInsertPending(): void { + if ([] == $this->waitingForInsert) { + return; + } + if (!\array_key_exists($forNumber = \count($this->waitingForInsert), $this->cachingStatements)) { - $sql = strtr(self::QUERY, [ + $sql = strtr(self::INSERT_TEMP, [ '{{ values }}' => implode( ', ', array_fill(0, $forNumber, self::VALUE) diff --git a/src/Bundle/ChillMainBundle/Tests/Services/Import/PostalCodeBaseImporterTest.php b/src/Bundle/ChillMainBundle/Tests/Services/Import/PostalCodeBaseImporterTest.php index d7447e7a0..eb313ba0d 100644 --- a/src/Bundle/ChillMainBundle/Tests/Services/Import/PostalCodeBaseImporterTest.php +++ b/src/Bundle/ChillMainBundle/Tests/Services/Import/PostalCodeBaseImporterTest.php @@ -93,4 +93,80 @@ final class PostalCodeBaseImporterTest extends KernelTestCase $this->assertStringStartsWith('tested with adapted pattern', $postalCodes[0]->getName()); $this->assertEquals($previousId, $postalCodes[0]->getId()); } + + public function testPostalCodeRemoval(): void + { + $source = 'removal_test_'.uniqid(); + $refId1 = 'ref1_'.uniqid(); + $refId2 = 'ref2_'.uniqid(); + + // 1. Import two postal codes + $this->importer->importCode('BE', 'Label 1', '1000', $refId1, $source, 50.0, 5.0, 4326); + $this->importer->importCode('BE', 'Label 2', '2000', $refId2, $source, 50.0, 5.0, 4326); + $this->importer->finalize(); + + $pc1 = $this->postalCodeRepository->findOneBy(['refPostalCodeId' => $refId1, 'postalCodeSource' => $source]); + $pc2 = $this->postalCodeRepository->findOneBy(['refPostalCodeId' => $refId2, 'postalCodeSource' => $source]); + + $this->assertNotNull($pc1); + $this->assertNotNull($pc2); + + // 2. Import only the first one + $this->importer->importCode('BE', 'Label 1 updated', '1000', $refId1, $source, 50.0, 5.0, 4326); + $this->importer->finalize(); + + $this->entityManager->clear(); + + $pc1 = $this->postalCodeRepository->findOneBy(['refPostalCodeId' => $refId1, 'postalCodeSource' => $source]); + $pc2 = $this->postalCodeRepository->findOneBy(['refPostalCodeId' => $refId2, 'postalCodeSource' => $source]); + + $this->assertNotNull($pc1); + $this->assertEquals('Label 1 updated', $pc1->getName()); + + $this->assertFalse($pc1->isDeleted(), 'pc1 should NOT be marked as deleted'); + + // pc2 should be marked as deleted. Note: findOneBy might still find it if it doesn't filter by deletedAt + $this->assertNotNull($pc2); + + $this->assertTrue($pc2->isDeleted(), 'Postal code should be marked as deleted (deletedAt is not null)'); + + // 3. Reactivate pc2 by re-importing it + $this->importer->importCode('BE', 'Label 2 restored', '2000', $refId2, $source, 50.0, 5.0, 4326); + $this->importer->finalize(); + + $this->entityManager->clear(); + $pc2 = $this->postalCodeRepository->findOneBy(['refPostalCodeId' => $refId2, 'postalCodeSource' => $source]); + $this->assertFalse($pc2->isDeleted(), 'Postal code should NOT be marked as deleted after restoration'); + $this->assertEquals('Label 2 restored', $pc2->getName()); + } + + public function testNoInterferenceBetweenSources(): void + { + $source1 = 'source1_'.uniqid(); + $source2 = 'source2_'.uniqid(); + $refId1 = 'ref1_'.uniqid(); + $refId2 = 'ref2_'.uniqid(); + + // 1. Import from source1 + $this->importer->importCode('BE', 'Label 1', '1000', $refId1, $source1, 50.0, 5.0, 4326); + $this->importer->finalize(); + + $pc1 = $this->postalCodeRepository->findOneBy(['refPostalCodeId' => $refId1, 'postalCodeSource' => $source1]); + $this->assertNotNull($pc1); + $this->assertFalse($pc1->isDeleted()); + + // 2. Import from source2 + $this->importer->importCode('BE', 'Label 2', '2000', $refId2, $source2, 50.0, 5.0, 4326); + $this->importer->finalize(); + + $this->entityManager->clear(); + + $pc1 = $this->postalCodeRepository->findOneBy(['refPostalCodeId' => $refId1, 'postalCodeSource' => $source1]); + $pc2 = $this->postalCodeRepository->findOneBy(['refPostalCodeId' => $refId2, 'postalCodeSource' => $source2]); + + $this->assertNotNull($pc1); + $this->assertNotNull($pc2); + $this->assertFalse($pc1->isDeleted(), 'pc1 from source1 should NOT be deleted after import from source2'); + $this->assertFalse($pc2->isDeleted(), 'pc2 from source2 should NOT be deleted'); + } }