Merge branch '111_exports_suite' into testing

This commit is contained in:
2022-10-05 10:43:35 +02:00
20 changed files with 1295 additions and 72 deletions

View File

@@ -14,15 +14,17 @@ namespace Chill\MainBundle\Entity;
use Doctrine\ORM\Mapping as ORM;
/**
* @ORM\Table(name="chill_main_geographical_unit")
* @ORM\Entity
* @ORM\Table(name="chill_main_geographical_unit", uniqueConstraints={
* @ORM\UniqueConstraint(name="geographical_unit_refid", columns={"layer_id", "unitRefId"})
* })
* @ORM\Entity(readOnly=true)
*/
class GeographicalUnit
{
/**
* @ORM\Column(type="text", nullable=true)
*/
private $geom;
private string $geom;
/**
* @ORM\Id
@@ -32,23 +34,30 @@ class GeographicalUnit
private ?int $id = null;
/**
* @ORM\Column(type="string", length=255, nullable=true)
* @ORM\Column(type="text", nullable=false, options={"default": ""})
*/
private $layerName;
private string $unitName;
/**
* @ORM\Column(type="string", length=255, nullable=true)
* @ORM\Column(type="text", nullable=false, options={"default": ""})
*/
private $unitName;
private string $unitRefId;
/**
* @ORM\ManyToOne(targetEntity=GeographicalUnitLayer::class, inversedBy="units")
*/
private ?GeographicalUnitLayer $layer;
public function getId(): ?int
{
return $this->id;
}
public function getLayerName(): ?string
protected function setId(int $id): self
{
return $this->layerName;
$this->id = $id;
return $this;
}
public function getUnitName(): ?string
@@ -56,10 +65,31 @@ class GeographicalUnit
return $this->unitName;
}
public function setLayerName(?string $layerName): self
/**
* @return GeographicalUnitLayer|null
*/
public function getLayer(): ?GeographicalUnitLayer
{
$this->layerName = $layerName;
return $this->layer;
}
/**
* @param string $unitRefId
* @return GeographicalUnit
*/
public function setUnitRefId(string $unitRefId): GeographicalUnit
{
$this->unitRefId = $unitRefId;
return $this;
}
/**
* @param GeographicalUnitLayer|null $layer
* @return GeographicalUnit
*/
public function setLayer(?GeographicalUnitLayer $layer): GeographicalUnit
{
$this->layer = $layer;
return $this;
}

View File

@@ -0,0 +1,85 @@
<?php
namespace Chill\MainBundle\Entity;
use Doctrine\Common\Collections\ArrayCollection;
use Doctrine\Common\Collections\Collection;
use Doctrine\ORM\Mapping as ORM;
/**
* @ORM\Table(name="chill_main_geographical_unit_layer", uniqueConstraints={
* @ORM\UniqueConstraint(name="geographical_unit_layer_refid", columns={"refId"})
* })
* @ORM\Entity
*/
class GeographicalUnitLayer
{
/**
* @ORM\Id
* @ORM\GeneratedValue
* @ORM\Column(type="integer")
*/
private ?int $id = null;
/**
* @ORM\Column(type="json", nullable=false, options={"default": "[]"})
*/
private array $name = [];
/**
* @ORM\Column(type="text", nullable=false, options={"default": ""})
*/
private string $refId = '';
/**
* @ORM\OneToMany(targetEntity=GeographicalUnit::class, mappedBy="layer")
*/
private Collection $units;
public function __construct()
{
$this->units = new ArrayCollection();
}
/**
* @return int|null
*/
public function getId(): ?int
{
return $this->id;
}
/**
* @return array
*/
public function getName(): array
{
return $this->name;
}
/**
* @param array $name
* @return GeographicalUnitLayer
*/
public function setName(array $name): GeographicalUnitLayer
{
$this->name = $name;
return $this;
}
/**
* @return string
*/
public function getRefId(): string
{
return $this->refId;
}
/**
* @return Collection
*/
public function getUnits(): Collection
{
return $this->units;
}
}

View File

@@ -0,0 +1,59 @@
<?php
namespace Chill\MainBundle\Repository;
use Chill\MainBundle\Entity\GeographicalUnit;
use Chill\MainBundle\Entity\GeographicalUnitLayer;
use Doctrine\ORM\EntityManagerInterface;
use Doctrine\ORM\EntityRepository;
use UnexpectedValueException;
final class GeographicalUnitLayerLayerRepository implements GeographicalUnitLayerRepositoryInterface
{
private EntityRepository $repository;
public function __construct(EntityManagerInterface $em)
{
$this->repository = $em->getRepository($this->getClassName());
}
public function find($id): ?GeographicalUnitLayer
{
return $this->repository->find($id);
}
/**
* @return array|GeographicalUnitLayer[]
*/
public function findAll(): array
{
return $this->repository->findAll();
}
/**
* @return array|GeographicalUnitLayer[]
*/
public function findBy(array $criteria, ?array $orderBy = null, ?int $limit = null, ?int $offset = null): array
{
return $this->repository->findBy($criteria, $orderBy, $limit, $offset);
}
public function findOneBy(array $criteria): ?GeographicalUnitLayer
{
return $this->repository->findOneBy($criteria);
}
public function getClassName(): string
{
return GeographicalUnitLayer::class;
}
public function findAllHavingUnits(): array
{
$qb = $this->repository->createQueryBuilder('l');
return $qb->where($qb->expr()->gt('SIZE(l.units)', 0))
->getQuery()
->getResult();
}
}

View File

@@ -0,0 +1,14 @@
<?php
namespace Chill\MainBundle\Repository;
use Chill\MainBundle\Entity\GeographicalUnitLayer;
use Doctrine\Persistence\ObjectRepository;
interface GeographicalUnitLayerRepositoryInterface extends ObjectRepository
{
/**
* @return array|GeographicalUnitLayer[]
*/
public function findAllHavingUnits(): array;
}

View File

@@ -0,0 +1,59 @@
<?php
namespace Chill\MainBundle\Repository;
use Chill\MainBundle\Entity\GeographicalUnit;
use Chill\MainBundle\Entity\GeographicalUnitDTO;
use Chill\MainBundle\Entity\GeographicalUnitLayer;
use Doctrine\ORM\EntityManagerInterface;
use Doctrine\ORM\EntityRepository;
class GeographicalUnitRepository implements GeographicalUnitRepositoryInterface
{
private EntityRepository $repository;
private EntityManagerInterface $em;
public function __construct(EntityManagerInterface $em)
{
$this->repository = $em->getRepository($this->getClassName());
$this->em = $em;
}
public function find($id): ?GeographicalUnit
{
return $this->repository->find($id);
}
/**
* Will return only partial object, where the @link{GeographicalUnit::geom} property is not loaded
*
* @return array|GeographicalUnit[]
*/
public function findAll(): array
{
return $this->repository
->createQueryBuilder('gu')
->select('PARTIAL gu.{id,unitName,unitRefId,layer}')
->addOrderBy('IDENTITY(gu.layer)')
->addOrderBy(('gu.unitName'))
->getQuery()
->getResult();
}
public function findBy(array $criteria, ?array $orderBy = null, ?int $limit = null, ?int $offset = null): ?GeographicalUnit
{
return $this->repository->findBy($criteria, $orderBy, $limit, $offset);
}
public function findOneBy(array $criteria): ?GeographicalUnit
{
return $this->repository->findOneBy($criteria);
}
public function getClassName(): string
{
return GeographicalUnit::class;
}
}

View File

@@ -0,0 +1,10 @@
<?php
namespace Chill\MainBundle\Repository;
use Doctrine\Persistence\ObjectRepository;
interface GeographicalUnitRepositoryInterface extends ObjectRepository
{
}

View File

@@ -0,0 +1,234 @@
<?php
/**
* Chill is a software for social workers
*
* For the full copyright and license information, please view
* the LICENSE file that was distributed with this source code.
*/
declare(strict_types=1);
namespace Chill\MainBundle\Service\Import;
use Doctrine\DBAL\Connection;
use Doctrine\DBAL\Statement;
use Doctrine\DBAL\Types\Type;
use Doctrine\DBAL\Types\Types;
use Exception;
use LogicException;
use Psr\Log\LoggerInterface;
use function array_key_exists;
use function count;
final class GeographicalUnitBaseImporter
{
private const INSERT = <<<'SQL'
INSERT INTO geographical_unit_temp
(layerKey, layerName, unitName, unitKey, geom)
SELECT
i.layerKey, i.layerName, i.unitName, i.unitKey,
ST_Transform(ST_setSrid(ST_GeomFromText(i.wkt), i.srid), 4326)
FROM
(VALUES
{{ values }}
) AS i (layerKey, layerName, unitName, unitKey, wkt, srid)
SQL;
private const LOG_PREFIX = '[GeographicalUnitBAseImporter] ';
private const VALUE = '(?, ?::jsonb, ?, ?, ?, ?::int)';
/**
* @var array<int, Statement>
*/
private array $cachingStatements = [];
private Connection $defaultConnection;
private bool $isInitialized = false;
private LoggerInterface $logger;
private array $waitingForInsert = [];
public function __construct(Connection $defaultConnection, LoggerInterface $logger)
{
$this->defaultConnection = $defaultConnection;
$this->logger = $logger;
}
public function finalize(): void
{
$this->doInsertPending();
$this->prepareForFinalize();
$this->updateGeographicalUnitTable();
$this->deleteTemporaryTable();
$this->isInitialized = false;
}
public function importUnit(
string $layerKey,
array $layerName,
string $unitName,
string $unitKey,
string $geomAsWKT,
int $srid = null
): void {
$this->initialize();
$this->waitingForInsert[] = [
'layerKey' => $layerKey,
'layerName' => $layerName,
'unitName' => $unitName,
'unitKey' => $unitKey,
'geomAsWKT' => $geomAsWKT,
'srid' => $srid
];
if (100 <= count($this->waitingForInsert)) {
$this->doInsertPending();
}
}
private function createTemporaryTable(): void
{
$this->defaultConnection->executeStatement("CREATE TEMPORARY TABLE geographical_unit_temp (
layerKey TEXT DEFAULT '' NOT NULL,
layerName JSONB DEFAULT '[]'::jsonb NOT NULL,
unitName TEXT default '' NOT NULL,
unitKey TEXT default '' NOT NULL,
geom GEOMETRY(MULTIPOLYGON, 4326)
)");
$this->defaultConnection->executeStatement('SET work_mem TO \'50MB\'');
}
private function deleteTemporaryTable(): void
{
$this->defaultConnection->executeStatement('DROP TABLE IF EXISTS geographical_unit_temp');
}
private function doInsertPending(): void
{
$forNumber = count($this->waitingForInsert);
if (0 === $forNumber) {
return;
}
if (!array_key_exists($forNumber, $this->cachingStatements)) {
$sql = strtr(self::INSERT, [
'{{ values }}' => implode(
', ',
array_fill(0, $forNumber, self::VALUE)
),
]);
$this->logger->debug(self::LOG_PREFIX . ' generated sql for insert', [
'sql' => $sql,
'forNumber' => $forNumber,
]);
$this->cachingStatements[$forNumber] = $this->defaultConnection->prepare($sql);
}
$statement = $this->cachingStatements[$forNumber];
try {
$i = 0;
foreach ($this->waitingForInsert as $insert) {
$statement->bindValue(++$i, $insert['layerKey'], Types::STRING);
$statement->bindValue(++$i, $insert['layerName'], Types::JSON);
$statement->bindValue(++$i, $insert['unitName'], Types::STRING);
$statement->bindValue(++$i, $insert['unitKey'], Types::STRING);
$statement->bindValue(++$i, $insert['geomAsWKT'], Types::STRING);
$statement->bindValue(++$i, $insert['srid'], Types::INTEGER);
}
$affected = $statement->executeStatement();
if ($affected === 0) {
throw new \RuntimeException('no row affected');
}
} catch (Exception $e) {
throw $e;
} finally {
$this->waitingForInsert = [];
}
}
private function initialize(): void
{
if ($this->isInitialized) {
return;
}
$this->deleteTemporaryTable();
$this->createTemporaryTable();
$this->isInitialized = true;
}
private function prepareForFinalize(): void
{
$this->defaultConnection->executeStatement(
'CREATE INDEX idx_ref_add_temp ON geographical_unit_temp (unitKey)'
);
}
private function updateGeographicalUnitTable(): void
{
$this->defaultConnection->transactional(
function() {
// 0) create new layers
$this->defaultConnection->executeStatement(
"
WITH unique_layers AS (
SELECT DISTINCT layerKey, layerName FROM geographical_unit_temp
)
INSERT INTO chill_main_geographical_unit_layer (id, name, refid)
SELECT
nextval('chill_main_geographical_unit_layer_id_seq'),
layerName,
layerKey
FROM unique_layers
ON CONFLICT (refid)
DO UPDATE SET name=EXCLUDED.name
");
//1) Add new units
$this->logger->info(self::LOG_PREFIX . 'upsert new units');
$affected = $this->defaultConnection->executeStatement("INSERT INTO chill_main_geographical_unit
(id, geom, unitname, layer_id, unitrefid)
SELECT
nextval('chill_main_geographical_unit_id_seq'),
geom,
unitName,
layer.id,
unitKey
FROM geographical_unit_temp JOIN chill_main_geographical_unit_layer AS layer ON layer.refid = layerKey
ON CONFLICT (layer_id, unitrefid)
DO UPDATE
SET geom = EXCLUDED.geom, unitname = EXCLUDED.unitname
");
$this->logger->info(self::LOG_PREFIX . 'units upserted', ['upserted' => $affected]);
//3) Delete units
$this->logger->info(self::LOG_PREFIX . 'soft delete adresses');
$affected = $this->defaultConnection->executeStatement('WITH to_delete AS (
SELECT cmgu.id
FROM chill_main_geographical_unit AS cmgu
JOIN chill_main_geographical_unit_layer AS cmgul ON cmgul.id = cmgu.layer_id
JOIN geographical_unit_temp AS gut ON cmgul.refid = gut.layerKey AND cmgu.unitrefid = gut.unitKey
)
DELETE FROM chill_main_geographical_unit
WHERE id NOT IN (SELECT id FROM to_delete)
');
$this->logger->info(self::LOG_PREFIX . 'addresses deleted', ['deleted' => $affected]);
}
);
}
}

View File

@@ -0,0 +1,89 @@
<?php
namespace Services\Import;
use Chill\MainBundle\Service\Import\GeographicalUnitBaseImporter;
use Doctrine\DBAL\Connection;
use Doctrine\ORM\EntityManagerInterface;
use Psr\Log\NullLogger;
use Symfony\Bundle\FrameworkBundle\Test\KernelTestCase;
class GeographicalUnitBaseImporterTest extends KernelTestCase
{
private Connection $connection;
private EntityManagerInterface $entityManager;
protected function setUp(): void
{
parent::setUp();
self::bootKernel();
$this->connection = self::$container->get(Connection::class);
$this->entityManager = self::$container->get(EntityManagerInterface::class);
}
public function testImportUnit(): void
{
$importer = new GeographicalUnitBaseImporter(
$this->connection,
new NullLogger()
);
$importer->importUnit(
'test',
['fr' => 'Test Layer'],
'Layer one',
'layer_one',
'MULTIPOLYGON (((30 20, 45 40, 10 40, 30 20)),((15 5, 40 10, 10 20, 5 10, 15 5)))',
3812
);
$importer->finalize();
$unit = $this->connection->executeQuery("
SELECT unitname, unitrefid, cmgul.refid AS layerrefid, cmgul.name AS layername, ST_AsText(ST_snapToGrid(ST_Transform(u.geom, 3812), 1)) AS geom
FROM chill_main_geographical_unit u JOIN chill_main_geographical_unit_layer cmgul on u.layer_id = cmgul.id
WHERE u.unitrefid = ?", ['layer_one']);
$results = $unit->fetchAssociative();
$this->assertEquals($results['unitrefid'], 'layer_one');
$this->assertEquals($results['unitname'], 'Layer one');
$this->assertEquals(json_decode($results['layername'], true), ['fr' => 'Test Layer']);
$this->assertEquals($results['layerrefid'], 'test');
$this->assertEquals($results['geom'], 'MULTIPOLYGON(((30 20,45 40,10 40,30 20)),((15 5,40 10,10 20,5 10,15 5)))');
$importer = new GeographicalUnitBaseImporter(
$this->connection,
new NullLogger()
);
$importer->importUnit(
'test',
['fr' => 'Test Layer fixed'],
'Layer one fixed',
'layer_one',
'MULTIPOLYGON (((130 120, 45 40, 10 40, 130 120)),((0 0, 15 5, 40 10, 10 20, 0 0)))',
3812
);
$importer->finalize();
$unit = $this->connection->executeQuery("
SELECT unitname, unitrefid, cmgul.refid AS layerrefid, cmgul.name AS layername, ST_AsText(ST_snapToGrid(ST_Transform(u.geom, 3812), 1)) AS geom
FROM chill_main_geographical_unit u JOIN chill_main_geographical_unit_layer cmgul on u.layer_id = cmgul.id
WHERE u.unitrefid = ?", ['layer_one']);
$results = $unit->fetchAssociative();
$this->assertEquals($results['unitrefid'], 'layer_one');
$this->assertEquals($results['unitname'], 'Layer one fixed');
$this->assertEquals(json_decode($results['layername'], true), ['fr' => 'Test Layer fixed']);
$this->assertEquals($results['layerrefid'], 'test');
$this->assertEquals($results['geom'], 'MULTIPOLYGON(((130 120,45 40,10 40,130 120)),((0 0,15 5,40 10,10 20,0 0)))');
}
}

View File

@@ -0,0 +1,29 @@
<?php
declare(strict_types=1);
namespace Chill\Migrations\Main;
use Doctrine\DBAL\Schema\Schema;
use Doctrine\Migrations\AbstractMigration;
/**
* Auto-generated Migration: Please modify to your needs!
*/
final class Version20220913174922 extends AbstractMigration
{
public function getDescription(): string
{
return 'Geographical Unit correction';
}
public function up(Schema $schema): void
{
$this->addSql('ALTER TABLE chill_main_geographical_unit ALTER COLUMN geom SET DATA TYPE GEOMETRY(MULTIPOLYGON, 4326)');
}
public function down(Schema $schema): void
{
$this->addSql('ALTER TABLE chill_main_geographical_unit ALTER COLUMN geom SET DATA TYPE TEXT');
}
}

View File

@@ -0,0 +1,58 @@
<?php
declare(strict_types=1);
namespace Chill\Migrations\Main;
use Doctrine\DBAL\Schema\Schema;
use Doctrine\Migrations\AbstractMigration;
final class Version20221003112151 extends AbstractMigration
{
public function getDescription(): string
{
return 'Add a proper entity for GeographicalUnitLayer';
}
public function up(Schema $schema): void
{
$this->addSql('CREATE SEQUENCE chill_main_geographical_unit_layer_id_seq INCREMENT BY 1 MINVALUE 1 START 1');
$this->addSql('CREATE TABLE chill_main_geographical_unit_layer (id INT NOT NULL, name JSONB DEFAULT \'[]\'::jsonb NOT NULL, refid TEXT DEFAULT \'\' NOT NULL, PRIMARY KEY(id))');
$this->addSql("COMMENT ON COLUMN chill_main_geographical_unit_layer.name IS '(DC2Type:json)';");
$this->addSql('INSERT INTO chill_main_geographical_unit_layer (id, name, refid)
SELECT DISTINCT nextval(\'chill_main_geographical_unit_layer_id_seq\'), jsonb_build_object(\'fr\', layername), layername FROM chill_main_geographical_unit');
$this->addSql('ALTER TABLE chill_main_geographical_unit ADD layer_id INT DEFAULT NULL');
$this->addSql('UPDATE chill_main_geographical_unit SET layer_id = layer.id FROM chill_main_geographical_unit_layer AS layer WHERE layer.refid = chill_main_geographical_unit.layername');
$this->addSql('ALTER TABLE chill_main_geographical_unit ADD unitRefId TEXT DEFAULT \'\' NOT NULL');
$this->addSql('ALTER TABLE chill_main_geographical_unit DROP layername');
$this->addSql("COMMENT ON COLUMN chill_main_geographical_unit.geom IS '(DC2Type:text)';");
$this->addSql('ALTER TABLE chill_main_geographical_unit ALTER unitname TYPE TEXT');
$this->addSql('ALTER TABLE chill_main_geographical_unit ALTER unitname SET DEFAULT \'\'');
$this->addSql('ALTER TABLE chill_main_geographical_unit ALTER unitname SET NOT NULL');
$this->addSql('ALTER TABLE chill_main_geographical_unit ADD CONSTRAINT FK_360A2B2FEA6EFDCD FOREIGN KEY (layer_id) REFERENCES chill_main_geographical_unit_layer (id) NOT DEFERRABLE INITIALLY IMMEDIATE');
$this->addSql('CREATE INDEX IDX_360A2B2FEA6EFDCD ON chill_main_geographical_unit (layer_id)');
}
public function down(Schema $schema): void
{
$this->throwIrreversibleMigrationException();
/* for memory
$this->addSql('ALTER TABLE chill_main_geographical_unit DROP CONSTRAINT FK_360A2B2FEA6EFDCD');
$this->addSql('DROP SEQUENCE chill_main_geographical_unit_layer_id_seq CASCADE');
$this->addSql('DROP TABLE chill_main_geographical_unit_layer');
$this->addSql('ALTER TABLE chill_main_geographical_unit ADD layername VARCHAR(255) DEFAULT NULL');
$this->addSql('ALTER TABLE chill_main_geographical_unit DROP layer_id');
$this->addSql('ALTER TABLE chill_main_geographical_unit DROP unitRefId');
$this->addSql('ALTER TABLE chill_main_geographical_unit ALTER geom TYPE VARCHAR(255)');
$this->addSql('ALTER TABLE chill_main_geographical_unit ALTER unitName TYPE VARCHAR(255)');
$this->addSql('ALTER TABLE chill_main_geographical_unit ALTER unitName DROP DEFAULT');
$this->addSql('ALTER TABLE chill_main_geographical_unit ALTER unitName DROP NOT NULL');
*/
}
}

View File

@@ -0,0 +1,30 @@
<?php
declare(strict_types=1);
namespace Chill\Migrations\Main;
use Doctrine\DBAL\Schema\Schema;
use Doctrine\Migrations\AbstractMigration;
final class Version20221003132620 extends AbstractMigration
{
public function getDescription(): string
{
return 'Create indexes and unique constraints on geographical unit entities';
}
public function up(Schema $schema): void
{
$this->addSql('CREATE UNIQUE INDEX geographical_unit_layer_refid ON chill_main_geographical_unit_layer (refId)');
$this->addSql('CREATE UNIQUE INDEX geographical_unit_refid ON chill_main_geographical_unit (layer_id, unitRefId)');
$this->addSql('CREATE INDEX chill_internal_geographical_unit_layer_geom_idx ON chill_main_geographical_unit USING GIST (layer_id, geom)');
}
public function down(Schema $schema): void
{
$this->addSql('DROP INDEX geographical_unit_layer_refid');
$this->addSql('DROP INDEX geographical_unit_refid');
$this->addSql('DROP INDEX chill_internal_geographical_unit_layer_geom_idx');
}
}