Merge branch '502-fix-import-postal-code-removed' into 'master'

Resolve "Lors de l'import de code postaux, les codes absents de l'import depuis la même source ne sont pas supprimés"

Closes #502

See merge request Chill-Projet/chill-bundles!968

.changes/unreleased/Fixed-20260223-182212.yaml

@@ -0,0 +1,7 @@
+kind: Fixed
+body: 'Fix import of postal code: mark postal code as deleted if they are not present in the import any more'
+time: 2026-02-23T18:22:12.92214987+01:00
+custom:
+    Issue: "502"
+    MR: "968"
+    SchemaChange: No schema change

src/Bundle/ChillMainBundle/Entity/PostalCode.php

@@ -215,4 +215,14 @@ class PostalCode implements TrackUpdateInterface, TrackCreationInterface
 
         return $this;
     }
+
+    public function isDeleted(): bool
+    {
+        return null !== $this->deletedAt;
+    }
+
+    public function getDeletedAt(): ?\DateTimeImmutable
+    {
+        return $this->deletedAt;
+    }
 }

src/Bundle/ChillMainBundle/Service/Import/PostalCodeBaseImporter.php

@@ -19,31 +19,66 @@ use Doctrine\DBAL\Statement;
  */
 class PostalCodeBaseImporter
 {
-    private const QUERY = <<<'SQL'
+    private const CREATE_TEMP_TABLE = <<<'SQL'
+        CREATE TEMPORARY TABLE chill_main_postal_code_temp (
+            countrycode VARCHAR(10),
+            label VARCHAR(255),
+            code VARCHAR(100),
+            refpostalcodeid VARCHAR(255),
+            postalcodeSource VARCHAR(255),
+            lon FLOAT,
+            lat FLOAT,
+            srid INT
+        )
+        SQL;
+
+    private const INSERT_TEMP = <<<'SQL'
+        INSERT INTO chill_main_postal_code_temp
+            (countrycode, label, code, refpostalcodeid, postalcodeSource, lon, lat, srid)
+        VALUES
+            {{ values }}
+        SQL;
+
+    private const UPSERT = <<<'SQL'
         WITH g AS (
             SELECT DISTINCT
                 country.id AS country_id,
-                g.*
-            FROM (VALUES
-                      {{ values }}
-                 ) AS g (countrycode, label, code, refpostalcodeid, postalcodeSource, lon, lat, srid)
-            JOIN country ON country.countrycode = g.countrycode
+                temp.*
+            FROM chill_main_postal_code_temp temp
+            JOIN country ON country.countrycode = temp.countrycode
         )
-        INSERT INTO chill_main_postal_code (id, country_id, label, code, origin, refpostalcodeid, postalcodeSource, center, createdAt, updatedAt)
+        INSERT INTO chill_main_postal_code (id, country_id, label, code, origin, refpostalcodeid, postalcodeSource, center, createdAt, updatedAt, deletedAt)
                SELECT
                    nextval('chill_main_postal_code_id_seq'),
                    g.country_id,
-                   g.label AS glabel,
+                   g.label,
                    g.code,
                    0,
                    g.refpostalcodeid,
                    g.postalcodeSource,
-                   CASE WHEN (g.lon::float != 0.0 AND g.lat::float != 0.0) THEN ST_Transform(ST_setSrid(ST_point(g.lon::float, g.lat::float), g.srid::int), 4326) ELSE NULL END,
+                   CASE WHEN (g.lon != 0.0 AND g.lat != 0.0) THEN ST_Transform(ST_setSrid(ST_point(g.lon, g.lat), g.srid), 4326) ELSE NULL END,
                    NOW(),
-                   NOW()
+                   NOW(),
+                   NULL
                FROM g
         ON CONFLICT (code, refpostalcodeid, postalcodeSource) WHERE refpostalcodeid IS NOT NULL DO UPDATE
-            SET label = excluded.label, center = excluded.center, updatedAt = CASE WHEN NOT st_equals(excluded.center, chill_main_postal_code.center) OR excluded.label != chill_main_postal_code.label THEN NOW() ELSE chill_main_postal_code.updatedAt END
+            SET label = excluded.label,
+                center = excluded.center,
+                deletedAt = NULL,
+                updatedAt = CASE WHEN NOT st_equals(excluded.center, chill_main_postal_code.center) OR excluded.label != chill_main_postal_code.label OR chill_main_postal_code.deletedAt IS NOT NULL THEN NOW() ELSE chill_main_postal_code.updatedAt END
+        SQL;
+
+    private const DELETE_MISSING = <<<'SQL'
+        UPDATE chill_main_postal_code
+        SET deletedAt = NOW(), updatedAt = NOW()
+        WHERE postalcodeSource = ?
+          AND deletedAt IS NULL
+          AND NOT EXISTS (
+              SELECT 1 FROM chill_main_postal_code_temp temp
+              WHERE temp.code = chill_main_postal_code.code
+                AND temp.refpostalcodeid = chill_main_postal_code.refpostalcodeid
+                AND temp.postalcodeSource = chill_main_postal_code.postalcodeSource
+          )
         SQL;
 
     private const VALUE = '(?, ?, ?, ?, ?, ?, ?, ?)';
@@ -55,11 +90,26 @@ class PostalCodeBaseImporter
 
     private array $waitingForInsert = [];
 
+    private bool $isInitialized = false;
+
+    private ?string $currentSource = null;
+
     public function __construct(private readonly Connection $defaultConnection) {}
 
     public function finalize(): void
     {
         $this->doInsertPending();
+
+        if ($this->isInitialized && null !== $this->currentSource) {
+            $this->defaultConnection->transactional(function (Connection $connection): void {
+                $connection->executeStatement(self::UPSERT);
+                $connection->executeStatement(self::DELETE_MISSING, [$this->currentSource]);
+            });
+            $this->deleteTemporaryTable();
+        }
+
+        $this->isInitialized = false;
+        $this->currentSource = null;
     }
 
     public function importCode(
@@ -72,6 +122,14 @@ class PostalCodeBaseImporter
         float $centerLon,
         int $centerSRID,
     ): void {
+        if (!$this->isInitialized) {
+            $this->initialize($refPostalCodeSource);
+        }
+
+        if ($this->currentSource !== $refPostalCodeSource) {
+            throw new \LogicException('Cannot store postal codes from different sources during same import. Execute finalize to commit inserts before changing the source');
+        }
+
         $this->waitingForInsert[] = [
             $countryCode,
             $label,
@@ -88,10 +146,32 @@ class PostalCodeBaseImporter
         }
     }
 
+    private function initialize(string $source): void
+    {
+        $this->currentSource = $source;
+        $this->deleteTemporaryTable();
+        $this->createTemporaryTable();
+        $this->isInitialized = true;
+    }
+
+    private function createTemporaryTable(): void
+    {
+        $this->defaultConnection->executeStatement(self::CREATE_TEMP_TABLE);
+    }
+
+    private function deleteTemporaryTable(): void
+    {
+        $this->defaultConnection->executeStatement('DROP TABLE IF EXISTS chill_main_postal_code_temp');
+    }
+
     private function doInsertPending(): void
     {
+        if ([] == $this->waitingForInsert) {
+            return;
+        }
+
         if (!\array_key_exists($forNumber = \count($this->waitingForInsert), $this->cachingStatements)) {
-            $sql = strtr(self::QUERY, [
+            $sql = strtr(self::INSERT_TEMP, [
                 '{{ values }}' => implode(
                     ', ',
                     array_fill(0, $forNumber, self::VALUE)

src/Bundle/ChillMainBundle/Tests/Services/Import/PostalCodeBaseImporterTest.php

@@ -93,4 +93,80 @@ final class PostalCodeBaseImporterTest extends KernelTestCase
         $this->assertStringStartsWith('tested with adapted pattern', $postalCodes[0]->getName());
         $this->assertEquals($previousId, $postalCodes[0]->getId());
     }
+
+    public function testPostalCodeRemoval(): void
+    {
+        $source = 'removal_test_'.uniqid();
+        $refId1 = 'ref1_'.uniqid();
+        $refId2 = 'ref2_'.uniqid();
+
+        // 1. Import two postal codes
+        $this->importer->importCode('BE', 'Label 1', '1000', $refId1, $source, 50.0, 5.0, 4326);
+        $this->importer->importCode('BE', 'Label 2', '2000', $refId2, $source, 50.0, 5.0, 4326);
+        $this->importer->finalize();
+
+        $pc1 = $this->postalCodeRepository->findOneBy(['refPostalCodeId' => $refId1, 'postalCodeSource' => $source]);
+        $pc2 = $this->postalCodeRepository->findOneBy(['refPostalCodeId' => $refId2, 'postalCodeSource' => $source]);
+
+        $this->assertNotNull($pc1);
+        $this->assertNotNull($pc2);
+
+        // 2. Import only the first one
+        $this->importer->importCode('BE', 'Label 1 updated', '1000', $refId1, $source, 50.0, 5.0, 4326);
+        $this->importer->finalize();
+
+        $this->entityManager->clear();
+
+        $pc1 = $this->postalCodeRepository->findOneBy(['refPostalCodeId' => $refId1, 'postalCodeSource' => $source]);
+        $pc2 = $this->postalCodeRepository->findOneBy(['refPostalCodeId' => $refId2, 'postalCodeSource' => $source]);
+
+        $this->assertNotNull($pc1);
+        $this->assertEquals('Label 1 updated', $pc1->getName());
+
+        $this->assertFalse($pc1->isDeleted(), 'pc1 should NOT be marked as deleted');
+
+        // pc2 should be marked as deleted. Note: findOneBy might still find it if it doesn't filter by deletedAt
+        $this->assertNotNull($pc2);
+
+        $this->assertTrue($pc2->isDeleted(), 'Postal code should be marked as deleted (deletedAt is not null)');
+
+        // 3. Reactivate pc2 by re-importing it
+        $this->importer->importCode('BE', 'Label 2 restored', '2000', $refId2, $source, 50.0, 5.0, 4326);
+        $this->importer->finalize();
+
+        $this->entityManager->clear();
+        $pc2 = $this->postalCodeRepository->findOneBy(['refPostalCodeId' => $refId2, 'postalCodeSource' => $source]);
+        $this->assertFalse($pc2->isDeleted(), 'Postal code should NOT be marked as deleted after restoration');
+        $this->assertEquals('Label 2 restored', $pc2->getName());
+    }
+
+    public function testNoInterferenceBetweenSources(): void
+    {
+        $source1 = 'source1_'.uniqid();
+        $source2 = 'source2_'.uniqid();
+        $refId1 = 'ref1_'.uniqid();
+        $refId2 = 'ref2_'.uniqid();
+
+        // 1. Import from source1
+        $this->importer->importCode('BE', 'Label 1', '1000', $refId1, $source1, 50.0, 5.0, 4326);
+        $this->importer->finalize();
+
+        $pc1 = $this->postalCodeRepository->findOneBy(['refPostalCodeId' => $refId1, 'postalCodeSource' => $source1]);
+        $this->assertNotNull($pc1);
+        $this->assertFalse($pc1->isDeleted());
+
+        // 2. Import from source2
+        $this->importer->importCode('BE', 'Label 2', '2000', $refId2, $source2, 50.0, 5.0, 4326);
+        $this->importer->finalize();
+
+        $this->entityManager->clear();
+
+        $pc1 = $this->postalCodeRepository->findOneBy(['refPostalCodeId' => $refId1, 'postalCodeSource' => $source1]);
+        $pc2 = $this->postalCodeRepository->findOneBy(['refPostalCodeId' => $refId2, 'postalCodeSource' => $source2]);
+
+        $this->assertNotNull($pc1);
+        $this->assertNotNull($pc2);
+        $this->assertFalse($pc1->isDeleted(), 'pc1 from source1 should NOT be deleted after import from source2');
+        $this->assertFalse($pc2->isDeleted(), 'pc2 from source2 should NOT be deleted');
+    }
 }