From 841eb57144a88426c20c1f5b4c14c5584b3deb33 Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Tue, 13 Sep 2022 15:10:12 +0200 Subject: [PATCH 1/9] GeographicalUnit Filter: add join clauses --- .../GeographicalUnitStatFilter.php | 18 +++++++++++++- .../services/exports_accompanying_course.yaml | 24 +++++++++---------- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/src/Bundle/ChillPersonBundle/Export/Filter/AccompanyingCourseFilters/GeographicalUnitStatFilter.php b/src/Bundle/ChillPersonBundle/Export/Filter/AccompanyingCourseFilters/GeographicalUnitStatFilter.php index 8f6748b70..e29dfd658 100644 --- a/src/Bundle/ChillPersonBundle/Export/Filter/AccompanyingCourseFilters/GeographicalUnitStatFilter.php +++ b/src/Bundle/ChillPersonBundle/Export/Filter/AccompanyingCourseFilters/GeographicalUnitStatFilter.php @@ -17,6 +17,7 @@ use Chill\MainBundle\Form\Type\ChillDateType; use Chill\PersonBundle\Export\Declarations; use DateTime; use Doctrine\DBAL\Types\Types; +use Doctrine\ORM\Query\Expr; use Doctrine\ORM\Query\Expr\Andx; use Doctrine\ORM\QueryBuilder; use Symfony\Bridge\Doctrine\Form\Type\EntityType; @@ -41,8 +42,23 @@ class GeographicalUnitStatFilter implements FilterInterface public function alterQuery(QueryBuilder $qb, $data) { + if (!in_array('location', $qb->getAllAliases(), true)) { + $qb->join('acp.administrativeLocation', 'location'); + } + + if (!in_array('address', $qb->getAllAliases(), true)) { + $qb->join('location.address', 'address'); + } + + if (!in_array('geounit', $qb->getAllAliases(), true)) { + $qb->join(GeographicalUnit::class, 'geounit', Expr\Join::WITH, 'ST_CONTAINS(address.point, geounit.geom) = TRUE'); + } + $where = $qb->getDQLPart('where'); - $clause = $qb->expr()->eq(1, 1); + $clause = $qb->expr()->andX( + $qb->expr()->eq($x, ':date'), + $qb->expr()->in($x, ':loctype') + ); if ($where instanceof Andx) { $where->add($clause); diff --git a/src/Bundle/ChillPersonBundle/config/services/exports_accompanying_course.yaml b/src/Bundle/ChillPersonBundle/config/services/exports_accompanying_course.yaml index f19db0649..8a8b20cbe 100644 --- a/src/Bundle/ChillPersonBundle/config/services/exports_accompanying_course.yaml +++ b/src/Bundle/ChillPersonBundle/config/services/exports_accompanying_course.yaml @@ -44,12 +44,12 @@ services: tags: - { name: chill.export_filter, alias: accompanyingcourse_step_filter } - #chill.person.export.filter_geographicalunitstat: - # class: Chill\PersonBundle\Export\Filter\AccompanyingCourseFilters\GeographicalUnitStatFilter - # autowire: true - # autoconfigure: true - # tags: - # - { name: chill.export_filter, alias: accompanyingcourse_geographicalunitstat_filter } + chill.person.export.filter_geographicalunitstat: + class: Chill\PersonBundle\Export\Filter\AccompanyingCourseFilters\GeographicalUnitStatFilter + autowire: true + autoconfigure: true + tags: + - { name: chill.export_filter, alias: accompanyingcourse_geographicalunitstat_filter } chill.person.export.filter_socialaction: class: Chill\PersonBundle\Export\Filter\AccompanyingCourseFilters\SocialActionFilter @@ -171,12 +171,12 @@ services: tags: - { name: chill.export_aggregator, alias: accompanyingcourse_step_aggregator } - #chill.person.export.aggregator_geographicalunitstat: - # class: Chill\PersonBundle\Export\Aggregator\AccompanyingCourseAggregators\GeographicalUnitStatAggregator - # autowire: true - # autoconfigure: true - # tags: - # - { name: chill.export_aggregator, alias: accompanyingcourse_geographicalunitstat_aggregator } + chill.person.export.aggregator_geographicalunitstat: + class: Chill\PersonBundle\Export\Aggregator\AccompanyingCourseAggregators\GeographicalUnitStatAggregator + autowire: true + autoconfigure: true + tags: + - { name: chill.export_aggregator, alias: accompanyingcourse_geographicalunitstat_aggregator } chill.person.export.aggregator_socialaction: class: Chill\PersonBundle\Export\Aggregator\AccompanyingCourseAggregators\SocialActionAggregator From b13686bbf196b74843a636feff1ea026c72972ea Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Tue, 13 Sep 2022 20:49:24 +0200 Subject: [PATCH 2/9] fix geom type in db: multipolygon --- .../migrations/Version20220913174922.php | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 src/Bundle/ChillMainBundle/migrations/Version20220913174922.php diff --git a/src/Bundle/ChillMainBundle/migrations/Version20220913174922.php b/src/Bundle/ChillMainBundle/migrations/Version20220913174922.php new file mode 100644 index 000000000..cabc7f1f8 --- /dev/null +++ b/src/Bundle/ChillMainBundle/migrations/Version20220913174922.php @@ -0,0 +1,29 @@ +addSql('ALTER TABLE chill_main_geographical_unit ALTER COLUMN geom SET DATA TYPE GEOMETRY(MULTIPOLYGON, 4326)'); + } + + public function down(Schema $schema): void + { + $this->addSql('ALTER TABLE chill_main_geographical_unit ALTER COLUMN geom SET DATA TYPE TEXT'); + } +} From 7ef84b9fd07ed0a78e12c14a608d20fff6fcc57e Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Wed, 21 Sep 2022 13:27:26 +0200 Subject: [PATCH 3/9] exports: create new aggregator test (untested) --- .../GeographicalUnitStatAggregatorTest.php | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 src/Bundle/ChillPersonBundle/Tests/Export/Aggregator/AccompanyingCourseAggregators/GeographicalUnitStatAggregatorTest.php diff --git a/src/Bundle/ChillPersonBundle/Tests/Export/Aggregator/AccompanyingCourseAggregators/GeographicalUnitStatAggregatorTest.php b/src/Bundle/ChillPersonBundle/Tests/Export/Aggregator/AccompanyingCourseAggregators/GeographicalUnitStatAggregatorTest.php new file mode 100644 index 000000000..84204e629 --- /dev/null +++ b/src/Bundle/ChillPersonBundle/Tests/Export/Aggregator/AccompanyingCourseAggregators/GeographicalUnitStatAggregatorTest.php @@ -0,0 +1,57 @@ +aggregator = self::$container->get('chill.person.export.aggregator_geographicalunitstat'); + } + + public function getAggregator() + { + return $this->aggregator; + } + + public function getFormData(): array + { + return [ + [], + ]; + } + + public function getQueryBuilders(): array + { + if (null === self::$kernel) { + self::bootKernel(); + } + + $em = self::$container->get(EntityManagerInterface::class); + + return [ + $em->createQueryBuilder() + ->select('count(acp.id)') + ->from(AccompanyingPeriod::class, 'acp') + , + ]; + } +} From 994160f28a91b637b634caf277393903acd92b7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Mon, 3 Oct 2022 13:50:10 +0200 Subject: [PATCH 4/9] Feature: add a proper entity for geographical unit layer --- .../Entity/GeographicalUnit.php | 17 ++++-- .../Entity/GeographicalUnitLayer.php | 56 ++++++++++++++++++ .../migrations/Version20221003112151.php | 58 +++++++++++++++++++ 3 files changed, 125 insertions(+), 6 deletions(-) create mode 100644 src/Bundle/ChillMainBundle/Entity/GeographicalUnitLayer.php create mode 100644 src/Bundle/ChillMainBundle/migrations/Version20221003112151.php diff --git a/src/Bundle/ChillMainBundle/Entity/GeographicalUnit.php b/src/Bundle/ChillMainBundle/Entity/GeographicalUnit.php index 9e119e30d..fac5d9127 100644 --- a/src/Bundle/ChillMainBundle/Entity/GeographicalUnit.php +++ b/src/Bundle/ChillMainBundle/Entity/GeographicalUnit.php @@ -15,14 +15,14 @@ use Doctrine\ORM\Mapping as ORM; /** * @ORM\Table(name="chill_main_geographical_unit") - * @ORM\Entity + * @ORM\Entity(readOnly=true) */ class GeographicalUnit { /** * @ORM\Column(type="text", nullable=true) */ - private $geom; + private string $geom; /** * @ORM\Id @@ -32,14 +32,19 @@ class GeographicalUnit private ?int $id = null; /** - * @ORM\Column(type="string", length=255, nullable=true) + * @ORM\Column(type="text", nullable=false, options={"default": ""}) */ - private $layerName; + private string $unitName; /** - * @ORM\Column(type="string", length=255, nullable=true) + * @ORM\Column(type="text", nullable=false, options={"default": ""}) */ - private $unitName; + private string $unitRefId; + + /** + * @ORM\ManyToOne(targetEntity=GeographicalUnitLayer::class) + */ + private ?GeographicalUnitLayer $layer; public function getId(): ?int { diff --git a/src/Bundle/ChillMainBundle/Entity/GeographicalUnitLayer.php b/src/Bundle/ChillMainBundle/Entity/GeographicalUnitLayer.php new file mode 100644 index 000000000..76ac06e9c --- /dev/null +++ b/src/Bundle/ChillMainBundle/Entity/GeographicalUnitLayer.php @@ -0,0 +1,56 @@ +id; + } + + /** + * @return array + */ + public function getName(): array + { + return $this->name; + } + + /** + * @param array $name + * @return GeographicalUnitLayer + */ + public function setName(array $name): GeographicalUnitLayer + { + $this->name = $name; + return $this; + } +} \ No newline at end of file diff --git a/src/Bundle/ChillMainBundle/migrations/Version20221003112151.php b/src/Bundle/ChillMainBundle/migrations/Version20221003112151.php new file mode 100644 index 000000000..36b1834ea --- /dev/null +++ b/src/Bundle/ChillMainBundle/migrations/Version20221003112151.php @@ -0,0 +1,58 @@ +addSql('CREATE SEQUENCE chill_main_geographical_unit_layer_id_seq INCREMENT BY 1 MINVALUE 1 START 1'); + $this->addSql('CREATE TABLE chill_main_geographical_unit_layer (id INT NOT NULL, name JSONB DEFAULT \'[]\'::jsonb NOT NULL, refid TEXT DEFAULT \'\' NOT NULL, PRIMARY KEY(id))'); + $this->addSql("COMMENT ON COLUMN chill_main_geographical_unit_layer.name IS '(DC2Type:json)';"); + + $this->addSql('INSERT INTO chill_main_geographical_unit_layer (id, name, refid) + SELECT DISTINCT nextval(\'chill_main_geographical_unit_layer_id_seq\'), jsonb_build_object(\'fr\', layername), layername FROM chill_main_geographical_unit'); + + $this->addSql('ALTER TABLE chill_main_geographical_unit ADD layer_id INT DEFAULT NULL'); + + $this->addSql('UPDATE chill_main_geographical_unit SET layer_id = layer.id FROM chill_main_geographical_unit_layer AS layer WHERE layer.refid = chill_main_geographical_unit.layername'); + + $this->addSql('ALTER TABLE chill_main_geographical_unit ADD unitRefId TEXT DEFAULT \'\' NOT NULL'); + $this->addSql('ALTER TABLE chill_main_geographical_unit DROP layername'); + $this->addSql("COMMENT ON COLUMN chill_main_geographical_unit.geom IS '(DC2Type:text)';"); + $this->addSql('ALTER TABLE chill_main_geographical_unit ALTER unitname TYPE TEXT'); + $this->addSql('ALTER TABLE chill_main_geographical_unit ALTER unitname SET DEFAULT \'\''); + $this->addSql('ALTER TABLE chill_main_geographical_unit ALTER unitname SET NOT NULL'); + $this->addSql('ALTER TABLE chill_main_geographical_unit ADD CONSTRAINT FK_360A2B2FEA6EFDCD FOREIGN KEY (layer_id) REFERENCES chill_main_geographical_unit_layer (id) NOT DEFERRABLE INITIALLY IMMEDIATE'); + $this->addSql('CREATE INDEX IDX_360A2B2FEA6EFDCD ON chill_main_geographical_unit (layer_id)'); + + } + + public function down(Schema $schema): void + { + $this->throwIrreversibleMigrationException(); + + /* for memory + $this->addSql('ALTER TABLE chill_main_geographical_unit DROP CONSTRAINT FK_360A2B2FEA6EFDCD'); + $this->addSql('DROP SEQUENCE chill_main_geographical_unit_layer_id_seq CASCADE'); + $this->addSql('DROP TABLE chill_main_geographical_unit_layer'); + $this->addSql('ALTER TABLE chill_main_geographical_unit ADD layername VARCHAR(255) DEFAULT NULL'); + $this->addSql('ALTER TABLE chill_main_geographical_unit DROP layer_id'); + $this->addSql('ALTER TABLE chill_main_geographical_unit DROP unitRefId'); + $this->addSql('ALTER TABLE chill_main_geographical_unit ALTER geom TYPE VARCHAR(255)'); + $this->addSql('ALTER TABLE chill_main_geographical_unit ALTER unitName TYPE VARCHAR(255)'); + $this->addSql('ALTER TABLE chill_main_geographical_unit ALTER unitName DROP DEFAULT'); + $this->addSql('ALTER TABLE chill_main_geographical_unit ALTER unitName DROP NOT NULL'); + */ + } +} From 9c3ac72426be727352277b37c072de907f802b24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Mon, 3 Oct 2022 15:45:42 +0200 Subject: [PATCH 5/9] Feature: Create a base importer for geographical units and add index --- .../Entity/GeographicalUnit.php | 4 +- .../Entity/GeographicalUnitLayer.php | 5 +- .../Import/GeographicalUnitBaseImporter.php | 234 ++++++++++++++++++ .../GeographicalUnitBaseImporterTest.php | 89 +++++++ .../migrations/Version20221003132620.php | 30 +++ 5 files changed, 359 insertions(+), 3 deletions(-) create mode 100644 src/Bundle/ChillMainBundle/Service/Import/GeographicalUnitBaseImporter.php create mode 100644 src/Bundle/ChillMainBundle/Tests/Services/Import/GeographicalUnitBaseImporterTest.php create mode 100644 src/Bundle/ChillMainBundle/migrations/Version20221003132620.php diff --git a/src/Bundle/ChillMainBundle/Entity/GeographicalUnit.php b/src/Bundle/ChillMainBundle/Entity/GeographicalUnit.php index fac5d9127..45a7fc54e 100644 --- a/src/Bundle/ChillMainBundle/Entity/GeographicalUnit.php +++ b/src/Bundle/ChillMainBundle/Entity/GeographicalUnit.php @@ -14,7 +14,9 @@ namespace Chill\MainBundle\Entity; use Doctrine\ORM\Mapping as ORM; /** - * @ORM\Table(name="chill_main_geographical_unit") + * @ORM\Table(name="chill_main_geographical_unit", uniqueConstraints={ + * @ORM\UniqueConstraint(name="geographical_unit_refid", columns={"unitRefId"}) + * }) * @ORM\Entity(readOnly=true) */ class GeographicalUnit diff --git a/src/Bundle/ChillMainBundle/Entity/GeographicalUnitLayer.php b/src/Bundle/ChillMainBundle/Entity/GeographicalUnitLayer.php index 76ac06e9c..54bde03ec 100644 --- a/src/Bundle/ChillMainBundle/Entity/GeographicalUnitLayer.php +++ b/src/Bundle/ChillMainBundle/Entity/GeographicalUnitLayer.php @@ -5,12 +5,13 @@ namespace Chill\MainBundle\Entity; use Doctrine\ORM\Mapping as ORM; /** - * @ORM\Table(name="chill_main_geographical_unit_layer") + * @ORM\Table(name="chill_main_geographical_unit_layer", uniqueConstraints={ + * @ORM\UniqueConstraint(name="geographical_unit_layer_refid", columns={"refId"}) + * }) * @ORM\Entity */ class GeographicalUnitLayer { - /** * @ORM\Id * @ORM\GeneratedValue diff --git a/src/Bundle/ChillMainBundle/Service/Import/GeographicalUnitBaseImporter.php b/src/Bundle/ChillMainBundle/Service/Import/GeographicalUnitBaseImporter.php new file mode 100644 index 000000000..9df865a28 --- /dev/null +++ b/src/Bundle/ChillMainBundle/Service/Import/GeographicalUnitBaseImporter.php @@ -0,0 +1,234 @@ + + */ + private array $cachingStatements = []; + + private Connection $defaultConnection; + + private bool $isInitialized = false; + + private LoggerInterface $logger; + + private array $waitingForInsert = []; + + public function __construct(Connection $defaultConnection, LoggerInterface $logger) + { + $this->defaultConnection = $defaultConnection; + $this->logger = $logger; + } + + public function finalize(): void + { + $this->doInsertPending(); + + $this->prepareForFinalize(); + + $this->updateGeographicalUnitTable(); + + $this->deleteTemporaryTable(); + + $this->isInitialized = false; + } + + public function importUnit( + string $layerKey, + array $layerName, + string $unitName, + string $unitKey, + string $geomAsWKT, + int $srid = null + ): void { + $this->initialize(); + + $this->waitingForInsert[] = [ + 'layerKey' => $layerKey, + 'layerName' => $layerName, + 'unitName' => $unitName, + 'unitKey' => $unitKey, + 'geomAsWKT' => $geomAsWKT, + 'srid' => $srid + ]; + + if (100 <= count($this->waitingForInsert)) { + $this->doInsertPending(); + } + } + + private function createTemporaryTable(): void + { + $this->defaultConnection->executeStatement("CREATE TEMPORARY TABLE geographical_unit_temp ( + layerKey TEXT DEFAULT '' NOT NULL, + layerName JSONB DEFAULT '[]'::jsonb NOT NULL, + unitName TEXT default '' NOT NULL, + unitKey TEXT default '' NOT NULL, + geom GEOMETRY(MULTIPOLYGON, 4326) + )"); + + $this->defaultConnection->executeStatement('SET work_mem TO \'50MB\''); + } + + private function deleteTemporaryTable(): void + { + $this->defaultConnection->executeStatement('DROP TABLE IF EXISTS geographical_unit_temp'); + } + + private function doInsertPending(): void + { + $forNumber = count($this->waitingForInsert); + + if (0 === $forNumber) { + return; + } + + if (!array_key_exists($forNumber, $this->cachingStatements)) { + $sql = strtr(self::INSERT, [ + '{{ values }}' => implode( + ', ', + array_fill(0, $forNumber, self::VALUE) + ), + ]); + + $this->logger->debug(self::LOG_PREFIX . ' generated sql for insert', [ + 'sql' => $sql, + 'forNumber' => $forNumber, + ]); + + $this->cachingStatements[$forNumber] = $this->defaultConnection->prepare($sql); + } + + $this->logger->debug(self::LOG_PREFIX . ' inserting pending addresses', [ + 'number' => $forNumber, + 'first' => $this->waitingForInsert[0] ?? null, + ]); + + $statement = $this->cachingStatements[$forNumber]; + try { + $i = 0; + foreach ($this->waitingForInsert as $insert) { + $statement->bindValue(++$i, $insert['layerKey'], Types::STRING); + $statement->bindValue(++$i, $insert['layerName'], Types::JSON); + $statement->bindValue(++$i, $insert['unitName'], Types::STRING); + $statement->bindValue(++$i, $insert['unitKey'], Types::STRING); + $statement->bindValue(++$i, $insert['geomAsWKT'], Types::STRING); + $statement->bindValue(++$i, $insert['srid'], Types::INTEGER); + } + + $affected = $statement->executeStatement(); + + if ($affected === 0) { + throw new \RuntimeException('no row affected'); + } + } catch (Exception $e) { + throw $e; + } finally { + $this->waitingForInsert = []; + } + } + + private function initialize(): void + { + if ($this->isInitialized) { + return; + } + + $this->deleteTemporaryTable(); + $this->createTemporaryTable(); + $this->isInitialized = true; + } + + private function prepareForFinalize(): void + { + $this->defaultConnection->executeStatement( + 'CREATE INDEX idx_ref_add_temp ON geographical_unit_temp (unitKey)' + ); + } + + private function updateGeographicalUnitTable(): void + { + $this->defaultConnection->transactional( + function() { + // 0) create new layers + $this->defaultConnection->executeStatement( + " + WITH unique_layers AS ( + SELECT DISTINCT layerKey, layerName FROM geographical_unit_temp + ) + INSERT INTO chill_main_geographical_unit_layer (id, name, refid) + SELECT + nextval('chill_main_geographical_unit_layer_id_seq'), + layerName, + layerKey + FROM unique_layers + ON CONFLICT (refid) + DO UPDATE SET name=EXCLUDED.name + "); + + //1) Add new units + $this->logger->info(self::LOG_PREFIX . 'upsert new units'); + $affected = $this->defaultConnection->executeStatement("INSERT INTO chill_main_geographical_unit + (id, geom, unitname, layer_id, unitrefid) + SELECT + nextval('chill_main_geographical_unit_id_seq'), + geom, + unitName, + layer.id, + unitKey + FROM geographical_unit_temp JOIN chill_main_geographical_unit_layer AS layer ON layer.refid = layerKey + ON CONFLICT (unitrefid) + DO UPDATE + SET geom = EXCLUDED.geom, unitname = EXCLUDED.unitname + "); + $this->logger->info(self::LOG_PREFIX . 'units upserted', ['upserted' => $affected]); + + //3) Delete units + $this->logger->info(self::LOG_PREFIX . 'soft delete adresses'); + $affected = $this->defaultConnection->executeStatement('DELETE FROM chill_main_geographical_unit + WHERE + unitrefid NOT IN (SELECT distinct unitKey FROM geographical_unit_temp) + '); + $this->logger->info(self::LOG_PREFIX . 'addresses deleted', ['deleted' => $affected]); + } + ); + } +} diff --git a/src/Bundle/ChillMainBundle/Tests/Services/Import/GeographicalUnitBaseImporterTest.php b/src/Bundle/ChillMainBundle/Tests/Services/Import/GeographicalUnitBaseImporterTest.php new file mode 100644 index 000000000..bc66c3691 --- /dev/null +++ b/src/Bundle/ChillMainBundle/Tests/Services/Import/GeographicalUnitBaseImporterTest.php @@ -0,0 +1,89 @@ +connection = self::$container->get(Connection::class); + $this->entityManager = self::$container->get(EntityManagerInterface::class); + } + + public function testImportUnit(): void + { + $importer = new GeographicalUnitBaseImporter( + $this->connection, + new NullLogger() + ); + + $importer->importUnit( + 'test', + ['fr' => 'Test Layer'], + 'Layer one', + 'layer_one', + 'MULTIPOLYGON (((30 20, 45 40, 10 40, 30 20)),((15 5, 40 10, 10 20, 5 10, 15 5)))', + 3812 + ); + + $importer->finalize(); + + $unit = $this->connection->executeQuery(" + SELECT unitname, unitrefid, cmgul.refid AS layerrefid, cmgul.name AS layername, ST_AsText(ST_snapToGrid(ST_Transform(u.geom, 3812), 1)) AS geom + FROM chill_main_geographical_unit u JOIN chill_main_geographical_unit_layer cmgul on u.layer_id = cmgul.id + WHERE u.unitrefid = ?", ['layer_one']); + + $results = $unit->fetchAssociative(); + + $this->assertEquals($results['unitrefid'], 'layer_one'); + $this->assertEquals($results['unitname'], 'Layer one'); + $this->assertEquals(json_decode($results['layername'], true), ['fr' => 'Test Layer']); + $this->assertEquals($results['layerrefid'], 'test'); + $this->assertEquals($results['geom'], 'MULTIPOLYGON(((30 20,45 40,10 40,30 20)),((15 5,40 10,10 20,5 10,15 5)))'); + + $importer = new GeographicalUnitBaseImporter( + $this->connection, + new NullLogger() + ); + + $importer->importUnit( + 'test', + ['fr' => 'Test Layer fixed'], + 'Layer one fixed', + 'layer_one', + 'MULTIPOLYGON (((130 120, 45 40, 10 40, 130 120)),((0 0, 15 5, 40 10, 10 20, 0 0)))', + 3812 + ); + + $importer->finalize(); + + $unit = $this->connection->executeQuery(" + SELECT unitname, unitrefid, cmgul.refid AS layerrefid, cmgul.name AS layername, ST_AsText(ST_snapToGrid(ST_Transform(u.geom, 3812), 1)) AS geom + FROM chill_main_geographical_unit u JOIN chill_main_geographical_unit_layer cmgul on u.layer_id = cmgul.id + WHERE u.unitrefid = ?", ['layer_one']); + + $results = $unit->fetchAssociative(); + + $this->assertEquals($results['unitrefid'], 'layer_one'); + $this->assertEquals($results['unitname'], 'Layer one fixed'); + $this->assertEquals(json_decode($results['layername'], true), ['fr' => 'Test Layer fixed']); + $this->assertEquals($results['layerrefid'], 'test'); + $this->assertEquals($results['geom'], 'MULTIPOLYGON(((130 120,45 40,10 40,130 120)),((0 0,15 5,40 10,10 20,0 0)))'); + + } + +} \ No newline at end of file diff --git a/src/Bundle/ChillMainBundle/migrations/Version20221003132620.php b/src/Bundle/ChillMainBundle/migrations/Version20221003132620.php new file mode 100644 index 000000000..39b01a0bc --- /dev/null +++ b/src/Bundle/ChillMainBundle/migrations/Version20221003132620.php @@ -0,0 +1,30 @@ +addSql('CREATE UNIQUE INDEX geographical_unit_layer_refid ON chill_main_geographical_unit_layer (refId)'); + $this->addSql('CREATE UNIQUE INDEX geographical_unit_refid ON chill_main_geographical_unit (unitRefId)'); + $this->addSql('CREATE INDEX chill_internal_geographical_unit_layer_geom_idx ON chill_main_geographical_unit USING GIST (geom)'); + } + + public function down(Schema $schema): void + { + $this->addSql('DROP INDEX geographical_unit_layer_refid'); + $this->addSql('DROP INDEX geographical_unit_refid'); + $this->addSql('DROP INDEX chill_internal_geographical_unit_layer_geom_idx'); + } +} From 65f6712a15d516d59304cb03e7e912ffd330303e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Mon, 3 Oct 2022 17:19:11 +0200 Subject: [PATCH 6/9] Fixed: take layer into account for unicity of geographical unit's keys --- .../Entity/GeographicalUnit.php | 2 +- .../Import/GeographicalUnitBaseImporter.php | 20 +++++++++---------- .../migrations/Version20221003132620.php | 4 ++-- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/Bundle/ChillMainBundle/Entity/GeographicalUnit.php b/src/Bundle/ChillMainBundle/Entity/GeographicalUnit.php index 45a7fc54e..b4c4388b6 100644 --- a/src/Bundle/ChillMainBundle/Entity/GeographicalUnit.php +++ b/src/Bundle/ChillMainBundle/Entity/GeographicalUnit.php @@ -15,7 +15,7 @@ use Doctrine\ORM\Mapping as ORM; /** * @ORM\Table(name="chill_main_geographical_unit", uniqueConstraints={ - * @ORM\UniqueConstraint(name="geographical_unit_refid", columns={"unitRefId"}) + * @ORM\UniqueConstraint(name="geographical_unit_refid", columns={"layer_id", "unitRefId"}) * }) * @ORM\Entity(readOnly=true) */ diff --git a/src/Bundle/ChillMainBundle/Service/Import/GeographicalUnitBaseImporter.php b/src/Bundle/ChillMainBundle/Service/Import/GeographicalUnitBaseImporter.php index 9df865a28..539e69028 100644 --- a/src/Bundle/ChillMainBundle/Service/Import/GeographicalUnitBaseImporter.php +++ b/src/Bundle/ChillMainBundle/Service/Import/GeographicalUnitBaseImporter.php @@ -137,11 +137,6 @@ final class GeographicalUnitBaseImporter $this->cachingStatements[$forNumber] = $this->defaultConnection->prepare($sql); } - $this->logger->debug(self::LOG_PREFIX . ' inserting pending addresses', [ - 'number' => $forNumber, - 'first' => $this->waitingForInsert[0] ?? null, - ]); - $statement = $this->cachingStatements[$forNumber]; try { $i = 0; @@ -215,7 +210,7 @@ final class GeographicalUnitBaseImporter layer.id, unitKey FROM geographical_unit_temp JOIN chill_main_geographical_unit_layer AS layer ON layer.refid = layerKey - ON CONFLICT (unitrefid) + ON CONFLICT (layer_id, unitrefid) DO UPDATE SET geom = EXCLUDED.geom, unitname = EXCLUDED.unitname "); @@ -223,10 +218,15 @@ final class GeographicalUnitBaseImporter //3) Delete units $this->logger->info(self::LOG_PREFIX . 'soft delete adresses'); - $affected = $this->defaultConnection->executeStatement('DELETE FROM chill_main_geographical_unit - WHERE - unitrefid NOT IN (SELECT distinct unitKey FROM geographical_unit_temp) - '); + $affected = $this->defaultConnection->executeStatement('WITH to_delete AS ( + SELECT cmgu.id + FROM chill_main_geographical_unit AS cmgu + JOIN chill_main_geographical_unit_layer AS cmgul ON cmgul.id = cmgu.layer_id + JOIN geographical_unit_temp AS gut ON cmgul.refid = gut.layerKey AND cmgu.unitrefid = gut.unitKey + ) + DELETE FROM chill_main_geographical_unit + WHERE id NOT IN (SELECT id FROM to_delete) + '); $this->logger->info(self::LOG_PREFIX . 'addresses deleted', ['deleted' => $affected]); } ); diff --git a/src/Bundle/ChillMainBundle/migrations/Version20221003132620.php b/src/Bundle/ChillMainBundle/migrations/Version20221003132620.php index 39b01a0bc..e07885dce 100644 --- a/src/Bundle/ChillMainBundle/migrations/Version20221003132620.php +++ b/src/Bundle/ChillMainBundle/migrations/Version20221003132620.php @@ -17,8 +17,8 @@ final class Version20221003132620 extends AbstractMigration public function up(Schema $schema): void { $this->addSql('CREATE UNIQUE INDEX geographical_unit_layer_refid ON chill_main_geographical_unit_layer (refId)'); - $this->addSql('CREATE UNIQUE INDEX geographical_unit_refid ON chill_main_geographical_unit (unitRefId)'); - $this->addSql('CREATE INDEX chill_internal_geographical_unit_layer_geom_idx ON chill_main_geographical_unit USING GIST (geom)'); + $this->addSql('CREATE UNIQUE INDEX geographical_unit_refid ON chill_main_geographical_unit (layer_id, unitRefId)'); + $this->addSql('CREATE INDEX chill_internal_geographical_unit_layer_geom_idx ON chill_main_geographical_unit USING GIST (layer_id, geom)'); } public function down(Schema $schema): void From 52435f5331660efe8fa09a1f3476e4c8cfee1ecb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Mon, 3 Oct 2022 18:20:13 +0200 Subject: [PATCH 7/9] Feature: aggregator for accompanying course by geographical level --- .../Entity/GeographicalUnit.php | 2 +- .../Entity/GeographicalUnitLayer.php | 28 ++++ .../GeographicalUnitLayerLayerRepository.php | 59 ++++++++ ...ographicalUnitLayerRepositoryInterface.php | 14 ++ .../GeographicalUnitStatAggregator.php | 143 +++++++++++++++--- .../translations/messages.fr.yml | 5 + 6 files changed, 228 insertions(+), 23 deletions(-) create mode 100644 src/Bundle/ChillMainBundle/Repository/GeographicalUnitLayerLayerRepository.php create mode 100644 src/Bundle/ChillMainBundle/Repository/GeographicalUnitLayerRepositoryInterface.php diff --git a/src/Bundle/ChillMainBundle/Entity/GeographicalUnit.php b/src/Bundle/ChillMainBundle/Entity/GeographicalUnit.php index b4c4388b6..e601ab147 100644 --- a/src/Bundle/ChillMainBundle/Entity/GeographicalUnit.php +++ b/src/Bundle/ChillMainBundle/Entity/GeographicalUnit.php @@ -44,7 +44,7 @@ class GeographicalUnit private string $unitRefId; /** - * @ORM\ManyToOne(targetEntity=GeographicalUnitLayer::class) + * @ORM\ManyToOne(targetEntity=GeographicalUnitLayer::class, inversedBy="units") */ private ?GeographicalUnitLayer $layer; diff --git a/src/Bundle/ChillMainBundle/Entity/GeographicalUnitLayer.php b/src/Bundle/ChillMainBundle/Entity/GeographicalUnitLayer.php index 54bde03ec..162d6ccf4 100644 --- a/src/Bundle/ChillMainBundle/Entity/GeographicalUnitLayer.php +++ b/src/Bundle/ChillMainBundle/Entity/GeographicalUnitLayer.php @@ -2,6 +2,8 @@ namespace Chill\MainBundle\Entity; +use Doctrine\Common\Collections\ArrayCollection; +use Doctrine\Common\Collections\Collection; use Doctrine\ORM\Mapping as ORM; /** @@ -29,6 +31,16 @@ class GeographicalUnitLayer */ private string $refId = ''; + /** + * @ORM\OneToMany(targetEntity=GeographicalUnit::class, mappedBy="layer") + */ + private Collection $units; + + public function __construct() + { + $this->units = new ArrayCollection(); + } + /** * @return int|null */ @@ -54,4 +66,20 @@ class GeographicalUnitLayer $this->name = $name; return $this; } + + /** + * @return string + */ + public function getRefId(): string + { + return $this->refId; + } + + /** + * @return Collection + */ + public function getUnits(): Collection + { + return $this->units; + } } \ No newline at end of file diff --git a/src/Bundle/ChillMainBundle/Repository/GeographicalUnitLayerLayerRepository.php b/src/Bundle/ChillMainBundle/Repository/GeographicalUnitLayerLayerRepository.php new file mode 100644 index 000000000..7ef99ebac --- /dev/null +++ b/src/Bundle/ChillMainBundle/Repository/GeographicalUnitLayerLayerRepository.php @@ -0,0 +1,59 @@ +repository = $em->getRepository($this->getClassName()); + } + + public function find($id): ?GeographicalUnitLayer + { + return $this->repository->find($id); + } + + /** + * @return array|GeographicalUnitLayer[] + */ + public function findAll(): array + { + return $this->repository->findAll(); + } + + /** + * @return array|GeographicalUnitLayer[] + */ + public function findBy(array $criteria, ?array $orderBy = null, ?int $limit = null, ?int $offset = null): array + { + return $this->repository->findBy($criteria, $orderBy, $limit, $offset); + } + + public function findOneBy(array $criteria): ?GeographicalUnitLayer + { + return $this->repository->findOneBy($criteria); + } + + public function getClassName(): string + { + return GeographicalUnitLayer::class; + } + + public function findAllHavingUnits(): array + { + $qb = $this->repository->createQueryBuilder('l'); + + return $qb->where($qb->expr()->gt('SIZE(l.units)', 0)) + ->getQuery() + ->getResult(); + } +} \ No newline at end of file diff --git a/src/Bundle/ChillMainBundle/Repository/GeographicalUnitLayerRepositoryInterface.php b/src/Bundle/ChillMainBundle/Repository/GeographicalUnitLayerRepositoryInterface.php new file mode 100644 index 000000000..e7c5abdd9 --- /dev/null +++ b/src/Bundle/ChillMainBundle/Repository/GeographicalUnitLayerRepositoryInterface.php @@ -0,0 +1,14 @@ +repository = $em->getRepository(...::class); + $this->geographicalUnitLayerRepository = $geographicalUnitLayerRepository; + $this->translatableStringHelper = $translatableStringHelper; } - */ /** * @inheritDoc */ public function getLabels($key, array $values, $data) { - return function ($value): string { - if ('_header' === $value) { - return 'Geographical unit'; - } + switch ($key) { + case 'acp_geog_agg_unitname': + return function ($value): string { + if ('_header' === $value) { + return 'acp_geog_agg_unitname'; + } - $g = $this->repository->find($value); + if (null === $value) { + return ''; + } - return $g; //... - }; + return $value; + }; + case 'acp_geog_agg_unitrefid': + return function ($value): string { + if ('_header' === $value) { + return 'acp_geog_agg_unitrefid'; + } + + if (null === $value) { + return ''; + } + + return $value; + }; + default: + throw new \UnexpectedValueException('this value should not happens'); + } } /** @@ -42,7 +72,7 @@ final class GeographicalUnitStatAggregator implements AggregatorInterface */ public function getQueryKeys($data): array { - return ['geographicalunitstat_aggregator']; + return ['acp_geog_agg_unitname', 'acp_geog_agg_unitrefid']; } /** @@ -50,7 +80,22 @@ final class GeographicalUnitStatAggregator implements AggregatorInterface */ public function buildForm(FormBuilderInterface $builder) { - // TODO: Implement buildForm() method. + $builder + ->add('date_calc', ChillDateType::class, [ + 'label' => 'Compute geographical location at date', + 'required' => true, + 'data' => new \DateTimeImmutable('today'), + 'input' => 'datetime_immutable', + ]) + ->add('level', EntityType::class, [ + 'label' => 'Geographical layer', + 'placeholder' => 'Select a geographical layer', + 'class' => GeographicalUnitLayer::class, + 'choices' => $this->geographicalUnitLayerRepository->findAllHavingUnits(), + 'choice_label' => function(GeographicalUnitLayer $item) { + return $this->translatableStringHelper->localize($item->getName()); + }, + ]); } /** @@ -74,16 +119,70 @@ final class GeographicalUnitStatAggregator implements AggregatorInterface */ public function alterQuery(QueryBuilder $qb, $data) { + if (!in_array('acp_geog_agg_location_history', $qb->getAllAliases(), true)) { + $qb->leftJoin('acp.locationHistories', 'acp_geog_agg_location_history'); - //$qb->addSelect('... AS geographicalunitstat_aggregator'); + $qb->andWhere( + $qb->expr()->andX( + 'acp_geog_agg_location_history.startDate <= :acp_geog_aggregator_date', + $qb->expr()->orX( + 'acp_geog_agg_location_history.endDate IS NULL', + 'acp_geog_agg_location_history.endDate > :acp_geog_aggregator_date' + ) + ) + ); - $groupby = $qb->getDQLPart('groupBy'); - - if (!empty($groupBy)) { - $qb->addGroupBy('geographicalunitstat_aggregator'); - } else { - $qb->groupBy('geographicalunitstat_aggregator'); + $qb->setParameter('acp_geog_aggregator_date', $data['date_calc']); } + + // link between location history and person + if (!in_array('acp_geog_agg_address_person_location', $qb->getAllAliases(), true)) { + $qb->leftJoin( + PersonHouseholdAddress::class, + 'acp_geog_agg_address_person_location', + Join::WITH, + $qb->expr()->andX( + 'IDENTITY(acp_geog_agg_address_person_location.person) = IDENTITY(acp_geog_agg_location_history.personLocation)', + 'acp_geog_agg_address_person_location.validFrom < :acp_geog_aggregator_date', + $qb->expr()->orX( + 'acp_geog_agg_address_person_location.validTo > :acp_geog_aggregator_date', + $qb->expr()->isNull('acp_geog_agg_address_person_location.validTo') + ) + ) + ); + + $qb->setParameter('acp_geog_aggregator_date', $data['date_calc']); + } + + // we finally find an address + if (!in_array('acp_geog_agg_address', $qb->getAllAliases(), true)) { + $qb->leftJoin( + Address::class, + 'acp_geog_agg_address', + Join::WITH, + 'COALESCE(IDENTITY(acp_geog_agg_address_person_location.address), IDENTITY(acp_geog_agg_location_history.addressLocation)) = acp_geog_agg_address.id' + ); + } + + // and we do a join with units + $qb->leftJoin( + GeographicalUnit::class, + 'acp_geog_units', + Join::WITH, + 'ST_CONTAINS(acp_geog_units.geom, acp_geog_agg_address.point) = TRUE' + ); + + $qb->andWhere($qb->expr()->eq('acp_geog_units.layer', ':acp_geog_unit_layer')); + + $qb->setParameter('acp_geog_unit_layer', $data['level']); + + // we add group by + $qb + ->addSelect('acp_geog_units.unitName AS acp_geog_agg_unitname') + ->addSelect('acp_geog_units.unitRefId AS acp_geog_agg_unitrefid') + ->addGroupBy('acp_geog_agg_unitname') + ->addGroupBy('acp_geog_agg_unitrefid') + ; } /** @@ -93,4 +192,4 @@ final class GeographicalUnitStatAggregator implements AggregatorInterface { return Declarations::ACP_TYPE; } -} \ No newline at end of file +} diff --git a/src/Bundle/ChillPersonBundle/translations/messages.fr.yml b/src/Bundle/ChillPersonBundle/translations/messages.fr.yml index c94b4963d..9e3ddf6a5 100644 --- a/src/Bundle/ChillPersonBundle/translations/messages.fr.yml +++ b/src/Bundle/ChillPersonBundle/translations/messages.fr.yml @@ -456,7 +456,12 @@ Group by step: Grouper les parcours par statut du parcours Filter by geographical unit: Filtrer les parcours par zone géographique Group by geographical unit: Grouper les parcours par zone géographique +Compute geographical location at date: Date de calcul de la localisation géographique Geographical unit: Zone géographique +acp_geog_agg_unitname: Zone géographique +acp_geog_agg_unitrefid: Clé de la zone géographique +Geographical layer: Couche géographique +Select a geographical layer: Choisir une couche géographique Filter by socialaction: Filtrer les parcours par action d'accompagnement Accepted socialactions: Actions d'accompagnement From fc567868c16c2d08df76a36f61e82bc0d73e157b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Tue, 4 Oct 2022 15:07:07 +0200 Subject: [PATCH 8/9] [export][person] Feature: allow to filter accompanying period by geographical unit --- .../Entity/GeographicalUnit.php | 30 +++-- .../Repository/GeographicalUnitRepository.php | 57 +++++++++ .../GeographicalUnitRepositoryInterface.php | 10 ++ .../GeographicalUnitStatFilter.php | 116 +++++++++++------- 4 files changed, 164 insertions(+), 49 deletions(-) create mode 100644 src/Bundle/ChillMainBundle/Repository/GeographicalUnitRepository.php create mode 100644 src/Bundle/ChillMainBundle/Repository/GeographicalUnitRepositoryInterface.php diff --git a/src/Bundle/ChillMainBundle/Entity/GeographicalUnit.php b/src/Bundle/ChillMainBundle/Entity/GeographicalUnit.php index e601ab147..8c704a166 100644 --- a/src/Bundle/ChillMainBundle/Entity/GeographicalUnit.php +++ b/src/Bundle/ChillMainBundle/Entity/GeographicalUnit.php @@ -53,20 +53,36 @@ class GeographicalUnit return $this->id; } - public function getLayerName(): ?string - { - return $this->layerName; - } - public function getUnitName(): ?string { return $this->unitName; } - public function setLayerName(?string $layerName): self + /** + * @return GeographicalUnitLayer|null + */ + public function getLayer(): ?GeographicalUnitLayer { - $this->layerName = $layerName; + return $this->layer; + } + /** + * @param string $unitRefId + * @return GeographicalUnit + */ + public function setUnitRefId(string $unitRefId): GeographicalUnit + { + $this->unitRefId = $unitRefId; + return $this; + } + + /** + * @param GeographicalUnitLayer|null $layer + * @return GeographicalUnit + */ + public function setLayer(?GeographicalUnitLayer $layer): GeographicalUnit + { + $this->layer = $layer; return $this; } diff --git a/src/Bundle/ChillMainBundle/Repository/GeographicalUnitRepository.php b/src/Bundle/ChillMainBundle/Repository/GeographicalUnitRepository.php new file mode 100644 index 000000000..ed259840b --- /dev/null +++ b/src/Bundle/ChillMainBundle/Repository/GeographicalUnitRepository.php @@ -0,0 +1,57 @@ +repository = $em->getRepository($this->getClassName()); + $this->em = $em; + } + + + public function find($id): ?GeographicalUnit + { + return $this->repository->find($id); + } + + /** + * Will return only partial object, where the @link{GeographicalUnit::geom} property is not loaded + * + * @return array|GeographicalUnit[] + */ + public function findAll(): array + { + return $this->repository + ->createQueryBuilder('gu') + ->addSelect('PARTIAL gu.{id,unitName,unitRefId,layer}') + ->addOrderBy('IDENTITY(gu.layer)') + ->addOrderBy(('gu.unitName')) + ->getQuery() + ->getResult(); + } + + public function findBy(array $criteria, ?array $orderBy = null, ?int $limit = null, ?int $offset = null): ?GeographicalUnit + { + return $this->repository->findBy($criteria, $orderBy, $limit, $offset); + } + + public function findOneBy(array $criteria): ?GeographicalUnit + { + return $this->repository->findOneBy($criteria); + } + + public function getClassName(): string + { + return GeographicalUnit::class; + } +} \ No newline at end of file diff --git a/src/Bundle/ChillMainBundle/Repository/GeographicalUnitRepositoryInterface.php b/src/Bundle/ChillMainBundle/Repository/GeographicalUnitRepositoryInterface.php new file mode 100644 index 000000000..10d07893b --- /dev/null +++ b/src/Bundle/ChillMainBundle/Repository/GeographicalUnitRepositoryInterface.php @@ -0,0 +1,10 @@ +em = $em; + $this->geographicalUnitRepository = $geographicalUnitRepository; + $this->translatableStringHelper = $translatableStringHelper; + } + public function addRole(): ?string { return null; @@ -42,33 +58,32 @@ class GeographicalUnitStatFilter implements FilterInterface public function alterQuery(QueryBuilder $qb, $data) { - if (!in_array('location', $qb->getAllAliases(), true)) { - $qb->join('acp.administrativeLocation', 'location'); - } + $subQueryDql = + 'SELECT + 1 + FROM '.AccompanyingPeriod\AccompanyingPeriodLocationHistory::class.' acp_geog_filter_location_history + LEFT JOIN '.PersonHouseholdAddress::class.' acp_geog_filter_address_person_location + WITH IDENTITY(acp_geog_filter_location_history.personLocation) = IDENTITY(acp_geog_filter_address_person_location.person) + LEFT JOIN '.Address::class.' acp_geog_filter_address + WITH COALESCE(IDENTITY(acp_geog_filter_address_person_location.address), IDENTITY(acp_geog_filter_location_history.addressLocation)) = acp_geog_filter_address.id + LEFT JOIN '.GeographicalUnit::class.' acp_geog_filter_units WITH ST_CONTAINS(acp_geog_units.geom, acp_geog_filter_address.point) = TRUE + WHERE + (acp_geog_filter_location_history.startDate <= :acp_geog_filter_date AND ( + acp_geog_filter_location_history.endDate IS NULL OR acp_geog_filter_location_history.endDate < :acp_geog_filter_date + )) + AND + (acp_geog_filter_address_person_location.validFrom < :acp_geog_filter_date AND ( + acp_geog_filter_address_person_location.validTo IS NULL OR acp_geog_filter_address_person_location.validTo < :acp_geog_filter_date + )) + AND acp_geog_filter_units IN (:acp_geog_filter_units) + AND acp_geog_filter_location_history.period = acp.id + '; - if (!in_array('address', $qb->getAllAliases(), true)) { - $qb->join('location.address', 'address'); - } - - if (!in_array('geounit', $qb->getAllAliases(), true)) { - $qb->join(GeographicalUnit::class, 'geounit', Expr\Join::WITH, 'ST_CONTAINS(address.point, geounit.geom) = TRUE'); - } - - $where = $qb->getDQLPart('where'); - $clause = $qb->expr()->andX( - $qb->expr()->eq($x, ':date'), - $qb->expr()->in($x, ':loctype') - ); - - if ($where instanceof Andx) { - $where->add($clause); - } else { - $where = $qb->expr()->andX($clause); - } - - $qb->add('where', $where); - $qb->setParameter('date', $data['date'], Types::DATE_MUTABLE); - $qb->setParameter('loctype', $data['accepted_loctype']); + $qb + ->andWhere($qb->expr()->exists($subQueryDql)) + ->setParameter('acp_geog_filter_date', $data['date_calc']) + ->setParameter('acp_geog_filter_units', $data['units']) + ; } public function applyOn(): string @@ -79,23 +94,40 @@ class GeographicalUnitStatFilter implements FilterInterface public function buildForm(FormBuilderInterface $builder) { $builder - ->add('date', ChillDateType::class, [ - 'data' => new DateTime(), + ->add('date_calc', ChillDateType::class, [ + 'label' => 'Compute geographical location at date', + 'required' => true, + 'data' => new \DateTimeImmutable('today'), + 'input' => 'datetime_immutable', ]) - ->add('accepted_loctype', EntityType::class, [ + ->add('units', EntityType::class, [ + 'label' => 'Geographical unit', + 'placeholder' => 'Select a geographical unit', 'class' => GeographicalUnit::class, - 'choice_label' => static function (GeographicalUnit $u) { - return $u->getUnitName(); + 'choices' => $this->geographicalUnitRepository->findAll(), + 'choice_label' => function(GeographicalUnit $item) { + return $this->translatableStringHelper->localize($item->getLayer()->getName()) . ' > ' . $item->getUnitName(); }, + 'attr' => [ + 'class' => 'select2', + ], 'multiple' => true, - 'expanded' => true, ]); } public function describeAction($data, $format = 'string'): array { - return ['Filtered by geographic unit: only %date%', [ - '%date%' => $data['date']->format('d-m-Y'), + return ['Filtered by geographic unit: computed at %date%, only in %units%', [ + '%date%' => $data['date_calc']->format('d-m-Y'), + '%units' => implode( + ', ', + array_map( + function(GeographicalUnit $item) { + return $this->translatableStringHelper->localize($item->getLayer()->getName()) . ' > ' . $item->getUnitName(); + }, + $data['units'] + ) + ) ]]; } From 432acc0ace569cc249313700f16c02ff4b957940 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Tue, 4 Oct 2022 22:17:16 +0200 Subject: [PATCH 9/9] [export][person] Feature: add filter and aggregator by geographical unit on person --- .../Entity/GeographicalUnit.php | 7 + .../Repository/GeographicalUnitRepository.php | 4 +- .../GeographicalUnitAggregator.php | 163 ++++++++++++++++++ .../PersonFilters/GeographicalUnitFilter.php | 134 ++++++++++++++ .../config/services/exports_person.yaml | 13 ++ .../translations/messages+intl-icu.fr.yaml | 5 + .../translations/messages.fr.yml | 2 + 7 files changed, 327 insertions(+), 1 deletion(-) create mode 100644 src/Bundle/ChillPersonBundle/Export/Aggregator/PersonAggregators/GeographicalUnitAggregator.php create mode 100644 src/Bundle/ChillPersonBundle/Export/Filter/PersonFilters/GeographicalUnitFilter.php diff --git a/src/Bundle/ChillMainBundle/Entity/GeographicalUnit.php b/src/Bundle/ChillMainBundle/Entity/GeographicalUnit.php index 8c704a166..49414db27 100644 --- a/src/Bundle/ChillMainBundle/Entity/GeographicalUnit.php +++ b/src/Bundle/ChillMainBundle/Entity/GeographicalUnit.php @@ -53,6 +53,13 @@ class GeographicalUnit return $this->id; } + protected function setId(int $id): self + { + $this->id = $id; + + return $this; + } + public function getUnitName(): ?string { return $this->unitName; diff --git a/src/Bundle/ChillMainBundle/Repository/GeographicalUnitRepository.php b/src/Bundle/ChillMainBundle/Repository/GeographicalUnitRepository.php index ed259840b..3f6c09ee6 100644 --- a/src/Bundle/ChillMainBundle/Repository/GeographicalUnitRepository.php +++ b/src/Bundle/ChillMainBundle/Repository/GeographicalUnitRepository.php @@ -3,6 +3,8 @@ namespace Chill\MainBundle\Repository; use Chill\MainBundle\Entity\GeographicalUnit; +use Chill\MainBundle\Entity\GeographicalUnitDTO; +use Chill\MainBundle\Entity\GeographicalUnitLayer; use Doctrine\ORM\EntityManagerInterface; use Doctrine\ORM\EntityRepository; @@ -33,7 +35,7 @@ class GeographicalUnitRepository implements GeographicalUnitRepositoryInterface { return $this->repository ->createQueryBuilder('gu') - ->addSelect('PARTIAL gu.{id,unitName,unitRefId,layer}') + ->select('PARTIAL gu.{id,unitName,unitRefId,layer}') ->addOrderBy('IDENTITY(gu.layer)') ->addOrderBy(('gu.unitName')) ->getQuery() diff --git a/src/Bundle/ChillPersonBundle/Export/Aggregator/PersonAggregators/GeographicalUnitAggregator.php b/src/Bundle/ChillPersonBundle/Export/Aggregator/PersonAggregators/GeographicalUnitAggregator.php new file mode 100644 index 000000000..816666e2f --- /dev/null +++ b/src/Bundle/ChillPersonBundle/Export/Aggregator/PersonAggregators/GeographicalUnitAggregator.php @@ -0,0 +1,163 @@ +geographicalUnitLayerRepository = $geographicalUnitLayerRepository; + $this->translatableStringHelper = $translatableStringHelper; + } + + /** + * @inheritDoc + */ + public function getLabels($key, array $values, $data) + { + switch ($key) { + case 'geog_unit_name': + return function ($value): string { + if ('_header' === $value) { + return 'acp_geog_agg_unitname'; + } + + if (null === $value) { + return ''; + } + + return $value; + }; + + case 'geog_unit_key': + return function ($value): string { + if ('_header' === $value) { + return 'acp_geog_agg_unitrefid'; + } + + if (null === $value) { + return ''; + } + + return $value; + }; + + default: + throw new \LogicException('key not supported'); + } + } + + /** + * @inheritDoc + */ + public function getQueryKeys($data) + { + return ['geog_unit_name', 'geog_unit_key']; + } + + /** + * @inheritDoc + */ + public function buildForm(FormBuilderInterface $builder) + { + $builder + ->add('date_calc', ChillDateType::class, [ + 'label' => 'Address valid at this date', + 'required' => true, + 'data' => new \DateTimeImmutable('today'), + 'input' => 'datetime_immutable', + ]) + ->add('level', EntityType::class, [ + 'label' => 'Geographical layer', + 'placeholder' => 'Select a geographical layer', + 'class' => GeographicalUnitLayer::class, + 'choices' => $this->geographicalUnitLayerRepository->findAllHavingUnits(), + 'choice_label' => function(GeographicalUnitLayer $item) { + return $this->translatableStringHelper->localize($item->getName()); + }, + ]); + } + + /** + * @inheritDoc + */ + public function getTitle() + { + return 'Group people by geographical unit based on his address'; + } + + /** + * @inheritDoc + */ + public function addRole(): ?string + { + return null; + } + + /** + * @inheritDoc + */ + public function alterQuery(QueryBuilder $qb, $data): void + { + $qb + ->leftJoin('person.householdAddresses', 'person_geog_agg_current_household_address') + ->leftJoin('person_geog_agg_current_household_address.address', 'person_geog_agg_address') + ->leftJoin(GeographicalUnit::class, 'person_geog_agg_geog_unit', Join::WITH, 'ST_CONTAINS(person_geog_agg_geog_unit.geom, person_geog_agg_address.point) = TRUE') + ->andWhere( + $qb->expr()->orX( + $qb->expr()->isNull('person_geog_agg_current_household_address'), + $qb->expr()->andX( + $qb->expr()->lte('person_geog_agg_current_household_address.validFrom', ':person_geog_agg_date'), + $qb->expr()->orX( + $qb->expr()->isNull('person_geog_agg_current_household_address.validTo'), + $qb->expr()->gt('person_geog_agg_current_household_address.validTo', ':person_geog_agg_date') + ) + ) + ) + ) + ->andWhere( + $qb->expr()->orX( + $qb->expr()->isNull('person_geog_agg_geog_unit'), + $qb->expr()->in('person_geog_agg_geog_unit.layer', ':person_geog_agg_layers') + ) + ) + ->setParameter('person_geog_agg_date', $data['date_calc']) + ->setParameter('person_geog_agg_layers', $data['level']) + ->addSelect('person_geog_agg_geog_unit.unitName AS geog_unit_name') + ->addSelect('person_geog_agg_geog_unit.unitRefId AS geog_unit_key') + ->addGroupBy('geog_unit_name') + ->addGroupBy('geog_unit_key') + ; + } + + /** + * @inheritDoc + */ + public function applyOn() + { + return Declarations::PERSON_TYPE; + } + + public static function getDefaultAlias(): string + { + return 'person_geog_agg'; + } +} diff --git a/src/Bundle/ChillPersonBundle/Export/Filter/PersonFilters/GeographicalUnitFilter.php b/src/Bundle/ChillPersonBundle/Export/Filter/PersonFilters/GeographicalUnitFilter.php new file mode 100644 index 000000000..e95aff798 --- /dev/null +++ b/src/Bundle/ChillPersonBundle/Export/Filter/PersonFilters/GeographicalUnitFilter.php @@ -0,0 +1,134 @@ +geographicalUnitRepository = $geographicalUnitRepository; + $this->translatableStringHelper = $translatableStringHelper; + } + + /** + * @inheritDoc + */ + public function buildForm(FormBuilderInterface $builder) + { + $builder + ->add('date_calc', ChillDateType::class, [ + 'label' => 'Compute geographical location at date', + 'required' => true, + 'data' => new \DateTimeImmutable('today'), + 'input' => 'datetime_immutable', + ]) + ->add('units', EntityType::class, [ + 'label' => 'Geographical unit', + 'placeholder' => 'Select a geographical unit', + 'class' => GeographicalUnit::class, + 'choices' => $this->geographicalUnitRepository->findAll(), + 'choice_label' => function(GeographicalUnit $item) { + return $this->translatableStringHelper->localize($item->getLayer()->getName()) . ' > ' . $item->getUnitName(); + }, + 'attr' => [ + 'class' => 'select2', + ], + 'multiple' => true, + ]); + } + + /** + * @inheritDoc + */ + public function getTitle(): string + { + return 'Filter by person\'s geographical unit (based on address)'; + } + + /** + * @inheritDoc + */ + public function describeAction($data, $format = 'string') + { + return [ + 'exports.by_person.Filtered by person\'s geographical unit (based on address) computed at datecalc, only units', + [ + 'datecalc' => $data['date_calc']->format('Y-m-d'), + 'units' => implode( + ', ', + array_map( + function (GeographicalUnit $item) { + return $this->translatableStringHelper->localize($item->getLayer()->getName()) . ' > ' . $item->getUnitName(); + }, + $data['units']->toArray() + ) + ) + ] + ]; + } + + /** + * @inheritDoc + */ + public function addRole(): ?string + { + return null; + } + + /** + * @inheritDoc + */ + public function alterQuery(QueryBuilder $qb, $data) + { + $subQuery = + 'SELECT 1 + FROM '.PersonHouseholdAddress::class.' person_filter_geog_person_household_address + JOIN person_filter_geog_person_household_address.address person_filter_geog_address + JOIN '.GeographicalUnit::class.' person_filter_geog_unit + WITH ST_CONTAINS(person_filter_geog_unit.geom, person_filter_geog_address.point) = TRUE + WHERE + person_filter_geog_person_household_address.validFrom <= :person_filter_geog_date + AND + (person_filter_geog_person_household_address.validTo IS NULL + OR person_filter_geog_person_household_address.validTo > :person_filter_geog_date) + AND + person_filter_geog_unit IN (:person_filter_geog_units) + AND + person_filter_geog_person_household_address.person = person + '; + + $qb + ->andWhere( + $qb->expr()->exists($subQuery) + ) + ->setParameter('person_filter_geog_date', $data['date_calc']) + ->setParameter('person_filter_geog_units', $data['units']) + ; + } + + /** + * @inheritDoc + */ + public function applyOn() + { + return Declarations::PERSON_TYPE; + } +} diff --git a/src/Bundle/ChillPersonBundle/config/services/exports_person.yaml b/src/Bundle/ChillPersonBundle/config/services/exports_person.yaml index fa1c8df8d..ffffaf175 100644 --- a/src/Bundle/ChillPersonBundle/config/services/exports_person.yaml +++ b/src/Bundle/ChillPersonBundle/config/services/exports_person.yaml @@ -90,6 +90,12 @@ services: tags: - { name: chill.export_filter, alias: person_marital_status_filter } + Chill\PersonBundle\Export\Filter\PersonFilters\GeographicalUnitFilter: + autowire: true + autoconfigure: true + tags: + - { name: chill.export_filter, alias: person_geog_filter } + ## Aggregators chill.person.export.aggregator_nationality: class: Chill\PersonBundle\Export\Aggregator\PersonAggregators\NationalityAggregator @@ -132,3 +138,10 @@ services: autoconfigure: true tags: - { name: chill.export_aggregator, alias: person_household_position_aggregator } + + Chill\PersonBundle\Export\Aggregator\PersonAggregators\GeographicalUnitAggregator: + autowire: true + autoconfigure: true + tags: + - { name: chill.export_aggregator, alias: person_geog_aggregator } + diff --git a/src/Bundle/ChillPersonBundle/translations/messages+intl-icu.fr.yaml b/src/Bundle/ChillPersonBundle/translations/messages+intl-icu.fr.yaml index 50cb9f0e6..091b9232f 100644 --- a/src/Bundle/ChillPersonBundle/translations/messages+intl-icu.fr.yaml +++ b/src/Bundle/ChillPersonBundle/translations/messages+intl-icu.fr.yaml @@ -130,3 +130,8 @@ periods: many {Masquer # parcours clôturés ou anciens parcours} other {Masquer # parcours clôturés ou anciens parcours} } + +exports: + by_person: + Filtered by person\'s geographical unit (based on address) computed at date, only units: + "Filtré par zone géographique sur base de l'adresse, calculé à {datecalc, date, short}, seulement les zones suivantes: {units}" diff --git a/src/Bundle/ChillPersonBundle/translations/messages.fr.yml b/src/Bundle/ChillPersonBundle/translations/messages.fr.yml index 9e3ddf6a5..12a193ac2 100644 --- a/src/Bundle/ChillPersonBundle/translations/messages.fr.yml +++ b/src/Bundle/ChillPersonBundle/translations/messages.fr.yml @@ -462,6 +462,8 @@ acp_geog_agg_unitname: Zone géographique acp_geog_agg_unitrefid: Clé de la zone géographique Geographical layer: Couche géographique Select a geographical layer: Choisir une couche géographique +Group people by geographical unit based on his address: Grouper les personnes par zone géographique (sur base de l'adresse) +Filter by person's geographical unit (based on address): Filter les personnes par zone géographique (sur base de l'adresse) Filter by socialaction: Filtrer les parcours par action d'accompagnement Accepted socialactions: Actions d'accompagnement