Merge branch 'feature/load-address-bano' into 'master'

Load addresses from bano and best address

See merge request Chill-Projet/chill-bundles!448
This commit is contained in:
Julien Fastré 2022-09-19 11:51:21 +00:00
commit 5b148f967f
18 changed files with 1184 additions and 3 deletions

View File

@ -0,0 +1,52 @@
<?php
/**
* Chill is a software for social workers
*
* For the full copyright and license information, please view
* the LICENSE file that was distributed with this source code.
*/
declare(strict_types=1);
namespace Chill\MainBundle\Command;
use Chill\MainBundle\Service\Import\AddressReferenceBEFromBestAddress;
use Chill\MainBundle\Service\Import\PostalCodeBEFromBestAddress;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
class LoadAddressesBEFromBestAddressCommand extends Command
{
private AddressReferenceBEFromBestAddress $addressImporter;
private PostalCodeBEFromBestAddress $postalCodeBEFromBestAddressImporter;
public function __construct(
AddressReferenceBEFromBestAddress $addressImporter,
PostalCodeBEFromBestAddress $postalCodeBEFromBestAddressImporter
) {
parent::__construct();
$this->addressImporter = $addressImporter;
$this->postalCodeBEFromBestAddressImporter = $postalCodeBEFromBestAddressImporter;
}
protected function configure()
{
$this
->setName('chill:main:address-ref-from-best-addresses')
->addArgument('lang', InputArgument::REQUIRED)
->addArgument('list', InputArgument::IS_ARRAY, 'The list to add');
}
protected function execute(InputInterface $input, OutputInterface $output): int
{
$this->postalCodeBEFromBestAddressImporter->import();
$this->addressImporter->import($input->getArgument('lang'), $input->getArgument('list'));
return 0;
}
}

View File

@ -0,0 +1,47 @@
<?php
/**
* Chill is a software for social workers
*
* For the full copyright and license information, please view
* the LICENSE file that was distributed with this source code.
*/
declare(strict_types=1);
namespace Chill\MainBundle\Command;
use Chill\MainBundle\Service\Import\AddressReferenceFromBano;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
class LoadAddressesFRFromBANOCommand extends Command
{
private AddressReferenceFromBano $addressReferenceFromBano;
public function __construct(AddressReferenceFromBano $addressReferenceFromBano)
{
parent::__construct();
$this->addressReferenceFromBano = $addressReferenceFromBano;
}
protected function configure()
{
$this->setName('chill:main:address-ref-from-bano')
->addArgument('departementNo', InputArgument::REQUIRED | InputArgument::IS_ARRAY, 'a list of departement numbers')
->setDescription('Import addresses from bano (see https://bano.openstreetmap.fr');
}
protected function execute(InputInterface $input, OutputInterface $output): int
{
foreach ($input->getArgument('departementNo') as $departementNo) {
$output->writeln('Import addresses for ' . $departementNo);
$this->addressReferenceFromBano->import($departementNo);
}
return 0;
}
}

View File

@ -0,0 +1,42 @@
<?php
/**
* Chill is a software for social workers
*
* For the full copyright and license information, please view
* the LICENSE file that was distributed with this source code.
*/
declare(strict_types=1);
namespace Chill\MainBundle\Command;
use Chill\MainBundle\Service\Import\PostalCodeFRFromOpenData;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
class LoadPostalCodeFR extends Command
{
private PostalCodeFRFromOpenData $loader;
public function __construct(PostalCodeFRFromOpenData $loader)
{
$this->loader = $loader;
parent::__construct();
}
public function configure(): void
{
$this->setName('chill:main:postal-code:load:FR')
->setDescription('Load France\'s postal code from online open data');
}
public function execute(InputInterface $input, OutputInterface $output): int
{
$this->loader->import();
return 0;
}
}

View File

@ -20,6 +20,9 @@ use Symfony\Component\Serializer\Annotation\Groups;
* @ORM\Entity
* @ORM\Table(name="chill_main_address_reference", indexes={
* @ORM\Index(name="address_refid", columns={"refId"})
* },
* uniqueConstraints={
* @ORM\UniqueConstraint(name="chill_main_address_reference_unicity", columns={"refId", "source"})
* })
* @ORM\HasLifecycleCallbacks
*/

View File

@ -12,6 +12,11 @@ declare(strict_types=1);
namespace Chill\MainBundle\Entity;
use Chill\MainBundle\Doctrine\Model\Point;
use Chill\MainBundle\Doctrine\Model\TrackCreationInterface;
use Chill\MainBundle\Doctrine\Model\TrackCreationTrait;
use Chill\MainBundle\Doctrine\Model\TrackUpdateInterface;
use Chill\MainBundle\Doctrine\Model\TrackUpdateTrait;
use DateTimeImmutable;
use Doctrine\ORM\Mapping as ORM;
use Symfony\Component\Serializer\Annotation\Groups;
@ -21,6 +26,10 @@ use Symfony\Component\Serializer\Annotation\Groups;
* @ORM\Entity
* @ORM\Table(
* name="chill_main_postal_code",
* uniqueConstraints={
* @ORM\UniqueConstraint(name="postal_code_import_unicity", columns={"code", "refpostalcodeid", "postalcodesource"},
* options={"where": "refpostalcodeid is not null"})
* },
* indexes={
* @ORM\Index(name="search_name_code", columns={"code", "label"}),
* @ORM\Index(name="search_by_reference_code", columns={"code", "refpostalcodeid"})
@ -28,8 +37,12 @@ use Symfony\Component\Serializer\Annotation\Groups;
*
* @ORM\HasLifecycleCallbacks
*/
class PostalCode
class PostalCode implements TrackUpdateInterface, TrackCreationInterface
{
use TrackCreationTrait;
use TrackUpdateTrait;
/**
* This is an internal column which is populated by database.
*
@ -63,6 +76,11 @@ class PostalCode
*/
private $country;
/**
* @ORM\Column(type="datetime_immutable", nullable=true, options={"default": null})
*/
private ?DateTimeImmutable $deletedAt = null;
/**
* @var int
*

View File

@ -0,0 +1,103 @@
<?php
/**
* Chill is a software for social workers
*
* For the full copyright and license information, please view
* the LICENSE file that was distributed with this source code.
*/
declare(strict_types=1);
namespace Chill\MainBundle\Service\Import;
use League\Csv\Reader;
use League\Csv\Statement;
use RuntimeException;
use Symfony\Contracts\HttpClient\Exception\TransportExceptionInterface;
use Symfony\Contracts\HttpClient\HttpClientInterface;
class AddressReferenceBEFromBestAddress
{
private const RELEASE = 'https://gitea.champs-libres.be/api/v1/repos/Chill-project/belgian-bestaddresses-transform/releases/tags/v1.0.0';
private AddressReferenceBaseImporter $baseImporter;
private HttpClientInterface $client;
public function __construct(HttpClientInterface $client, AddressReferenceBaseImporter $baseImporter)
{
$this->client = $client;
$this->baseImporter = $baseImporter;
}
public function import(string $lang, array $lists): void
{
foreach ($lists as $list) {
$this->importList($lang, $list);
}
}
private function getDownloadUrl(string $lang, string $list): string
{
try {
$release = $this->client->request('GET', self::RELEASE)
->toArray();
} catch (TransportExceptionInterface $e) {
throw new RuntimeException('could not get the release definition', 0, $e);
}
$asset = array_filter($release['assets'], static function (array $item) use ($lang, $list) {
return 'addresses-' . $list . '.' . $lang . '.csv.gz' === $item['name'];
});
return array_values($asset)[0]['browser_download_url'];
}
private function importList(string $lang, string $list): void
{
$downloadUrl = $this->getDownloadUrl($lang, $list);
$response = $this->client->request('GET', $downloadUrl);
if (200 !== $response->getStatusCode()) {
throw new Exception('Could not download CSV: ' . $response->getStatusCode());
}
$tmpname = tempnam(sys_get_temp_dir(), 'php-add-' . $list . $lang);
$file = fopen($tmpname, 'r+b');
foreach ($this->client->stream($response) as $chunk) {
fwrite($file, $chunk->getContent());
}
fclose($file);
$uncompressedStream = gzopen($tmpname, 'r');
$csv = Reader::createFromStream($uncompressedStream);
$csv->setDelimiter(',');
$csv->setHeaderOffset(0);
$stmt = Statement::create()
->process($csv);
foreach ($stmt as $record) {
$this->baseImporter->importAddress(
$record['best_id'],
$record['municipality_objectid'],
$record['postal_info_objectid'],
$record['streetname'],
$record['housenumber'] . $record['boxnumber'],
'bestaddress.' . $list,
(float) $record['X'],
(float) $record['Y'],
3812
);
}
$this->baseImporter->finalize();
gzclose($uncompressedStream);
}
}

View File

@ -0,0 +1,220 @@
<?php
/**
* Chill is a software for social workers
*
* For the full copyright and license information, please view
* the LICENSE file that was distributed with this source code.
*/
declare(strict_types=1);
namespace Chill\MainBundle\Service\Import;
use Doctrine\DBAL\Connection;
use Doctrine\DBAL\Statement;
use Exception;
use LogicException;
use Psr\Log\LoggerInterface;
use function array_key_exists;
use function count;
final class AddressReferenceBaseImporter
{
private const INSERT = <<<'SQL'
INSERT INTO reference_address_temp
(postcode_id, refid, street, streetnumber, municipalitycode, source, point)
SELECT
cmpc.id, i.refid, i.street, i.streetnumber, i.refpostalcode, i.source,
CASE WHEN (i.lon::float != 0.0 AND i.lat::float != 0.0) THEN ST_Transform(ST_setSrid(ST_point(i.lon::float, i.lat::float), i.srid::int), 4326) ELSE NULL END
FROM
(VALUES
{{ values }}
) AS i (refid, refpostalcode, postalcode, street, streetnumber, source, lat, lon, srid)
JOIN chill_main_postal_code cmpc ON cmpc.refpostalcodeid = i.refpostalcode and cmpc.code = i.postalcode
SQL;
private const LOG_PREFIX = '[AddressReferenceImporter] ';
private const VALUE = '(?, ?, ?, ?, ?, ?, ?, ?, ?)';
/**
* @var array<int, Statement>
*/
private array $cachingStatements = [];
private ?string $currentSource = null;
private Connection $defaultConnection;
private bool $isInitialized = false;
private LoggerInterface $logger;
private array $waitingForInsert = [];
public function __construct(Connection $defaultConnection, LoggerInterface $logger)
{
$this->defaultConnection = $defaultConnection;
$this->logger = $logger;
}
public function finalize(): void
{
$this->doInsertPending();
$this->updateAddressReferenceTable();
$this->deleteTemporaryTable();
$this->currentSource = null;
$this->isInitialized = false;
}
public function importAddress(
string $refAddress,
?string $refPostalCode,
string $postalCode,
string $street,
string $streetNumber,
string $source,
?float $lat = null,
?float $lon = null,
?int $srid = null
): void {
if (!$this->isInitialized) {
$this->initialize($source);
}
if ($this->currentSource !== $source) {
throw new LogicException('Cannot store addresses from different sources during same import. Execute finalize to commit inserts before changing the source');
}
$this->waitingForInsert[] = [
$refAddress,
$refPostalCode,
$postalCode,
$street,
$streetNumber,
$source,
$lat,
$lon,
$srid,
];
if (100 <= count($this->waitingForInsert)) {
$this->doInsertPending();
}
}
private function createTemporaryTable(): void
{
$this->defaultConnection->executeStatement('CREATE TEMPORARY TABLE reference_address_temp (
postcode_id INT,
refid VARCHAR(255),
street VARCHAR(255),
streetnumber VARCHAR(255),
municipalitycode VARCHAR(255),
source VARCHAR(255),
point GEOMETRY
);
');
$this->defaultConnection->executeStatement('SET work_mem TO \'50MB\'');
}
private function deleteTemporaryTable(): void
{
$this->defaultConnection->executeStatement('DROP TABLE IF EXISTS reference_address_temp');
}
private function doInsertPending(): void
{
if (!array_key_exists($forNumber = count($this->waitingForInsert), $this->cachingStatements)) {
$sql = strtr(self::INSERT, [
'{{ values }}' => implode(
', ',
array_fill(0, $forNumber, self::VALUE)
),
]);
$this->logger->debug(self::LOG_PREFIX . ' generated sql for insert', [
'sql' => $sql,
'forNumber' => $forNumber,
]);
$this->cachingStatements[$forNumber] = $this->defaultConnection->prepare($sql);
}
if (0 === $forNumber) {
return;
}
$this->logger->debug(self::LOG_PREFIX . ' inserting pending addresses', [
'number' => $forNumber,
'first' => $this->waitingForInsert[0] ?? null,
]);
$statement = $this->cachingStatements[$forNumber];
try {
$affected = $statement->executeStatement(array_merge(...$this->waitingForInsert));
if ($affected === 0) {
throw new \RuntimeException('no row affected');
}
} catch (Exception $e) {
// in some case, we can add debug code here
//dump($this->waitingForInsert);
throw $e;
} finally {
$this->waitingForInsert = [];
}
}
private function initialize(string $source): void
{
$this->currentSource = $source;
$this->deleteTemporaryTable();
$this->createTemporaryTable();
$this->isInitialized = true;
}
private function updateAddressReferenceTable(): void
{
$this->defaultConnection->executeStatement(
'CREATE INDEX idx_ref_add_temp ON reference_address_temp (refid)'
);
//1) Add new addresses
$this->logger->info(self::LOG_PREFIX . 'upsert new addresses');
$affected = $this->defaultConnection->executeStatement("INSERT INTO chill_main_address_reference
(id, postcode_id, refid, street, streetnumber, municipalitycode, source, point, createdat, deletedat, updatedat)
SELECT
nextval('chill_main_address_reference_id_seq'),
postcode_id,
refid,
street,
streetnumber,
municipalitycode,
source,
point,
NOW(),
null,
NOW()
FROM reference_address_temp
ON CONFLICT (refid, source) DO UPDATE
SET postcode_id = excluded.postcode_id, refid = excluded.refid, street = excluded.street, streetnumber = excluded.streetnumber, municipalitycode = excluded.municipalitycode, source = excluded.source, point = excluded.point, updatedat = NOW(), deletedAt = NULL
");
$this->logger->info(self::LOG_PREFIX . 'addresses upserted', ['upserted' => $affected]);
//3) Delete addresses
$this->logger->info(self::LOG_PREFIX . 'soft delete adresses');
$affected = $this->defaultConnection->executeStatement('UPDATE chill_main_address_reference
SET deletedat = NOW()
WHERE
chill_main_address_reference.refid NOT IN (SELECT refid FROM reference_address_temp WHERE source LIKE ?)
AND chill_main_address_reference.source LIKE ?
', [$this->currentSource, $this->currentSource]);
$this->logger->info(self::LOG_PREFIX . 'addresses deleted', ['deleted' => $affected]);
}
}

View File

@ -0,0 +1,87 @@
<?php
/**
* Chill is a software for social workers
*
* For the full copyright and license information, please view
* the LICENSE file that was distributed with this source code.
*/
declare(strict_types=1);
namespace Chill\MainBundle\Service\Import;
use Exception;
use League\Csv\Reader;
use League\Csv\Statement;
use Symfony\Contracts\HttpClient\HttpClientInterface;
use UnexpectedValueException;
use function is_int;
class AddressReferenceFromBano
{
private AddressReferenceBaseImporter $baseImporter;
private HttpClientInterface $client;
public function __construct(HttpClientInterface $client, AddressReferenceBaseImporter $baseImporter)
{
$this->client = $client;
$this->baseImporter = $baseImporter;
}
public function import(string $departementNo): void
{
if (!is_numeric($departementNo) || !is_int((int) $departementNo)) {
throw new UnexpectedValueException('Could not parse this department number');
}
$url = "https://bano.openstreetmap.fr/data/bano-{$departementNo}.csv";
$response = $this->client->request('GET', $url);
if (200 !== $response->getStatusCode()) {
throw new Exception('Could not download CSV: ' . $response->getStatusCode());
}
$file = tmpfile();
foreach ($this->client->stream($response) as $chunk) {
fwrite($file, $chunk->getContent());
}
fseek($file, 0);
$csv = Reader::createFromStream($file);
$csv->setDelimiter(',');
$stmt = Statement::create()
->process($csv, [
'refId',
'streetNumber',
'street',
'postcode',
'city',
'_o',
'lat',
'lon',
]);
foreach ($stmt as $record) {
$this->baseImporter->importAddress(
$record['refId'],
substr($record['refId'], 0, 5), // extract insee from reference
$record['postcode'],
$record['street'],
$record['streetNumber'],
'BANO.' . $departementNo,
(float) $record['lat'],
(float) $record['lon'],
4326
);
}
$this->baseImporter->finalize();
fclose($file);
}
}

View File

@ -0,0 +1,105 @@
<?php
/**
* Chill is a software for social workers
*
* For the full copyright and license information, please view
* the LICENSE file that was distributed with this source code.
*/
declare(strict_types=1);
namespace Chill\MainBundle\Service\Import;
use League\Csv\Reader;
use Psr\Log\LoggerInterface;
use RuntimeException;
use Symfony\Contracts\HttpClient\Exception\TransportExceptionInterface;
use Symfony\Contracts\HttpClient\HttpClientInterface;
class PostalCodeBEFromBestAddress
{
private const RELEASE = 'https://gitea.champs-libres.be/api/v1/repos/Chill-project/belgian-bestaddresses-transform/releases/tags/v1.0.0';
private PostalCodeBaseImporter $baseImporter;
private HttpClientInterface $client;
private LoggerInterface $logger;
public function __construct(PostalCodeBaseImporter $baseImporter, HttpClientInterface $client, LoggerInterface $logger)
{
$this->baseImporter = $baseImporter;
$this->client = $client;
$this->logger = $logger;
}
public function import(string $lang = 'fr'): void
{
$fileDownloadUrl = $this->getFileDownloadUrl($lang);
$response = $this->client->request('GET', $fileDownloadUrl);
$tmpname = tempnam(sys_get_temp_dir(), 'postalcodes');
$tmpfile = fopen($tmpname, 'r+b');
if (false === $tmpfile) {
throw new RuntimeException('could not create temporary file');
}
foreach ($this->client->stream($response) as $chunk) {
fwrite($tmpfile, $chunk->getContent());
}
fclose($tmpfile);
$uncompressedStream = gzopen($tmpname, 'r');
$csv = Reader::createFromStream($uncompressedStream);
$csv->setDelimiter(',');
$csv->setHeaderOffset(0);
foreach ($csv as $offset => $record) {
$this->handleRecord($record);
}
gzclose($uncompressedStream);
unlink($tmpname);
$this->logger->info(__CLASS__ . ' list of postal code downloaded');
$this->baseImporter->finalize();
$this->logger->info(__CLASS__ . ' postal code fetched', ['offset' => $offset ?? 0]);
}
private function getFileDownloadUrl(string $lang): string
{
try {
$release = $this->client->request('GET', self::RELEASE)
->toArray();
} catch (TransportExceptionInterface $e) {
throw new RuntimeException('could not get the release definition', 0, $e);
}
$postals = array_filter($release['assets'], static function (array $item) use ($lang) {
return 'postals.' . $lang . '.csv.gz' === $item['name'];
});
return array_values($postals)[0]['browser_download_url'];
}
private function handleRecord(array $record): void
{
$this->baseImporter->importCode(
'BE',
trim($record['municipality_name']),
trim($record['postal_info_objectid']),
$record['municipality_objectid'],
'bestaddress',
$record['Y'],
$record['X'],
3812
);
}
}

View File

@ -0,0 +1,124 @@
<?php
/**
* Chill is a software for social workers
*
* For the full copyright and license information, please view
* the LICENSE file that was distributed with this source code.
*/
declare(strict_types=1);
namespace Chill\MainBundle\Service\Import;
use Doctrine\DBAL\Connection;
use Doctrine\DBAL\Statement;
use Exception;
use function array_key_exists;
use function count;
/**
* Optimized way to load postal code into database.
*/
class PostalCodeBaseImporter
{
private const QUERY = <<<'SQL'
WITH g AS (
SELECT DISTINCT
country.id AS country_id,
g.*
FROM (VALUES
{{ values }}
) AS g (countrycode, label, code, refpostalcodeid, postalcodeSource, lon, lat, srid)
JOIN country ON country.countrycode = g.countrycode
)
INSERT INTO chill_main_postal_code (id, country_id, label, code, origin, refpostalcodeid, postalcodeSource, center, createdAt, updatedAt)
SELECT
nextval('chill_main_postal_code_id_seq'),
g.country_id,
g.label AS glabel,
g.code,
0,
g.refpostalcodeid,
g.postalcodeSource,
CASE WHEN (g.lon::float != 0.0 AND g.lat::float != 0.0) THEN ST_Transform(ST_setSrid(ST_point(g.lon::float, g.lat::float), g.srid::int), 4326) ELSE NULL END,
NOW(),
NOW()
FROM g
ON CONFLICT (code, refpostalcodeid, postalcodeSource) WHERE refpostalcodeid IS NOT NULL DO UPDATE SET label = excluded.label, center = excluded.center, updatedAt = NOW()
SQL;
private const VALUE = '(?, ?, ?, ?, ?, ?, ?, ?)';
/**
* @var array<int, Statement>
*/
private array $cachingStatements = [];
private Connection $defaultConnection;
private array $waitingForInsert = [];
public function __construct(
Connection $defaultConnection
) {
$this->defaultConnection = $defaultConnection;
}
public function finalize(): void
{
$this->doInsertPending();
}
public function importCode(
string $countryCode,
string $label,
string $code,
string $refPostalCodeId,
string $refPostalCodeSource,
float $centerLat,
float $centerLon,
int $centerSRID
): void {
$this->waitingForInsert[] = [
$countryCode,
$label,
$code,
$refPostalCodeId,
$refPostalCodeSource,
$centerLon,
$centerLat,
$centerSRID,
];
if (100 <= count($this->waitingForInsert)) {
$this->doInsertPending();
}
}
private function doInsertPending(): void
{
if (!array_key_exists($forNumber = count($this->waitingForInsert), $this->cachingStatements)) {
$sql = strtr(self::QUERY, [
'{{ values }}' => implode(
', ',
array_fill(0, $forNumber, self::VALUE)
),
]);
$this->cachingStatements[$forNumber] = $this->defaultConnection->prepare($sql);
}
$statement = $this->cachingStatements[$forNumber];
try {
$statement->executeStatement(array_merge(...$this->waitingForInsert));
} catch (Exception $e) {
// in some case, we can add debug code here
//dump($this->waitingForInsert);
throw $e;
} finally {
$this->waitingForInsert = [];
}
}
}

View File

@ -0,0 +1,105 @@
<?php
/**
* Chill is a software for social workers
*
* For the full copyright and license information, please view
* the LICENSE file that was distributed with this source code.
*/
declare(strict_types=1);
namespace Chill\MainBundle\Service\Import;
use League\Csv\Reader;
use Psr\Log\LoggerInterface;
use RuntimeException;
use Symfony\Contracts\HttpClient\HttpClientInterface;
/**
* Load French's postal codes from opendata.
*
* Currently, the source is datanova / la poste:
* https://datanova.legroupe.laposte.fr/explore/dataset/laposte_hexasmal/information/
*/
class PostalCodeFRFromOpenData
{
private const CSV = 'https://datanova.legroupe.laposte.fr/explore/dataset/laposte_hexasmal/download/?format=csv&timezone=Europe/Berlin&lang=fr&use_labels_for_header=true&csv_separator=%3B';
private PostalCodeBaseImporter $baseImporter;
private HttpClientInterface $client;
private LoggerInterface $logger;
public function __construct(
PostalCodeBaseImporter $baseImporter,
HttpClientInterface $client,
LoggerInterface $logger
) {
$this->baseImporter = $baseImporter;
$this->client = $client;
$this->logger = $logger;
}
public function import(): void
{
$response = $this->client->request('GET', self::CSV);
if (200 !== $response->getStatusCode()) {
throw new RuntimeException('could not download CSV');
}
$tmpfile = tmpfile();
if (false === $tmpfile) {
throw new RuntimeException('could not create temporary file');
}
foreach ($this->client->stream($response) as $chunk) {
fwrite($tmpfile, $chunk->getContent());
}
fseek($tmpfile, 0);
$csv = Reader::createFromStream($tmpfile);
$csv->setDelimiter(';');
$csv->setHeaderOffset(0);
foreach ($csv as $offset => $record) {
$this->handleRecord($record);
}
$this->baseImporter->finalize();
fclose($tmpfile);
$this->logger->info(__CLASS__ . ' postal code fetched', ['offset' => $offset ?? 0]);
}
private function handleRecord(array $record): void
{
if ('' !== trim($record['coordonnees_gps'])) {
[$lat, $lon] = array_map(static fn ($el) => (float) trim($el), explode(',', $record['coordonnees_gps']));
} else {
$lat = $lon = 0.0;
}
$ref = trim($record['Code_commune_INSEE']);
if ('987' === substr($ref, 0, 3)) {
// some differences in French Polynesia
$ref .= '.' . trim($record['Libellé_d_acheminement']);
}
$this->baseImporter->importCode(
'FR',
trim($record['Libellé_d_acheminement']),
trim($record['Code_postal']),
$ref,
'INSEE',
$lat,
$lon,
4326
);
}
}

View File

@ -0,0 +1,90 @@
<?php
namespace Services\Import;
use Chill\MainBundle\Entity\PostalCode;
use Chill\MainBundle\Repository\AddressReferenceRepository;
use Chill\MainBundle\Repository\PostalCodeRepository;
use Chill\MainBundle\Service\Import\AddressReferenceBaseImporter;
use Doctrine\ORM\EntityManagerInterface;
use Symfony\Bundle\FrameworkBundle\Test\KernelTestCase;
class AddressReferenceBaseImporterTest extends KernelTestCase
{
private AddressReferenceBaseImporter $importer;
private AddressReferenceRepository $addressReferenceRepository;
private EntityManagerInterface $entityManager;
private PostalCodeRepository $postalCodeRepository;
protected function setUp(): void
{
parent::setUp();
self::bootKernel();
$this->importer = self::$container->get(AddressReferenceBaseImporter::class);
$this->addressReferenceRepository = self::$container->get(AddressReferenceRepository::class);
$this->entityManager = self::$container->get(EntityManagerInterface::class);
$this->postalCodeRepository = self::$container->get(PostalCodeRepository::class);
}
public function testImportAddress(): void
{
$postalCode = (new PostalCode())
->setRefPostalCodeId($postalCodeId = '1234'.uniqid())
->setPostalCodeSource('testing')
->setCode('TEST456')
->setName('testing');
$this->entityManager->persist($postalCode);
$this->entityManager->flush();
$this->importer->importAddress(
'0000',
$postalCodeId,
'TEST456',
'Rue test abccc-guessed',
'-1',
'unit-test',
50.0,
5.0,
4326
);
$this->importer->finalize();
$addresses = $this->addressReferenceRepository->findByPostalCodePattern(
$postalCode,
'Rue test abcc guessed');
$this->assertCount(1, $addresses);
$this->assertEquals('Rue test abccc-guessed', $addresses[0]->getStreet());
$previousAddressId = $addresses[0]->getId();
$this->entityManager->clear();
$this->importer->importAddress(
'0000',
$postalCodeId,
'TEST456',
'Rue test abccc guessed fixed',
'-1',
'unit-test',
50.0,
5.0,
4326
);
$this->importer->finalize();
$addresses = $this->addressReferenceRepository->findByPostalCodePattern(
$postalCode,
'abcc guessed fixed');
$this->assertCount('1', $addresses);
$this->assertEquals( 'Rue test abccc guessed fixed', $addresses[0]->getStreet());
$this->assertEquals($previousAddressId, $addresses[0]->getId());
}
}

View File

@ -0,0 +1,85 @@
<?php
namespace Services\Import;
use Chill\MainBundle\Repository\CountryRepository;
use Chill\MainBundle\Repository\PostalCodeRepository;
use Chill\MainBundle\Service\Import\PostalCodeBaseImporter;
use Doctrine\ORM\EntityManagerInterface;
use Symfony\Bundle\FrameworkBundle\Test\KernelTestCase;
class PostalCodeBaseImporterTest extends KernelTestCase
{
private EntityManagerInterface $entityManager;
private PostalCodeBaseImporter $importer;
private PostalCodeRepository $postalCodeRepository;
private CountryRepository $countryRepository;
protected function setUp(): void
{
parent::setUp();
self::bootKernel();
$this->entityManager = self::$container->get(EntityManagerInterface::class);
$this->importer = self::$container->get(PostalCodeBaseImporter::class);
$this->postalCodeRepository = self::$container->get(PostalCodeRepository::class);
$this->countryRepository = self::$container->get(CountryRepository::class);
}
public function testImportPostalCode(): void
{
$this->importer->importCode(
'BE',
'tested with pattern '. ($uniqid = uniqid()),
'12345',
$refPostalCodeId = 'test'.uniqid(),
'test',
50.0,
5.0,
4326
);
$this->importer->finalize();
$postalCodes = $this->postalCodeRepository->findByPattern(
'with pattern '.$uniqid,
$this->countryRepository->findOneBy(['countryCode' => 'BE'])
);
$this->assertCount(1, $postalCodes);
$this->assertStringStartsWith('tested with pattern', $postalCodes[0]->getName());
$previousId = $postalCodes[0]->getId();
$this->entityManager->clear();
$this->importer->importCode(
'BE',
'tested with adapted pattern '. ($uniqid = uniqid()),
'12345',
$refPostalCodeId,
'test',
50.0,
5.0,
4326
);
$this->importer->finalize();
$postalCodes = $this->postalCodeRepository->findByPattern(
'with pattern '.$uniqid,
$this->countryRepository->findOneBy(['countryCode' => 'BE'])
);
$this->assertCount(1, $postalCodes);
$this->assertStringStartsWith('tested with adapted pattern', $postalCodes[0]->getName());
$this->assertEquals($previousId, $postalCodes[0]->getId());
}
}

View File

@ -97,3 +97,7 @@ services:
Chill\MainBundle\Security\Resolver\CenterResolverDispatcherInterface: '@Chill\MainBundle\Security\Resolver\CenterResolverDispatcher'
Chill\MainBundle\Service\Import\:
resource: '../Service/Import/'
autowire: true
autoconfigure: true

View File

@ -43,3 +43,21 @@ services:
$entityManager: '@doctrine.orm.entity_manager'
tags:
- { name: console.command }
Chill\MainBundle\Command\LoadAddressesFRFromBANOCommand:
autoconfigure: true
autowire: true
tags:
- { name: console.command }
Chill\MainBundle\Command\LoadAddressesBEFromBestAddressCommand:
autoconfigure: true
autowire: true
tags:
- { name: console.command }
Chill\MainBundle\Command\LoadPostalCodeFR:
autoconfigure: true
autowire: true
tags:
- { name: console.command }

View File

@ -0,0 +1,53 @@
<?php
/**
* Chill is a software for social workers
*
* For the full copyright and license information, please view
* the LICENSE file that was distributed with this source code.
*/
declare(strict_types=1);
namespace Chill\Migrations\Main;
use Doctrine\DBAL\Schema\Schema;
use Doctrine\Migrations\AbstractMigration;
final class Version20220729205416 extends AbstractMigration
{
public function down(Schema $schema): void
{
$this->addSql('DROP INDEX postal_code_import_unicity');
$this->addSql('ALTER TABLE chill_main_postal_code DROP deletedAt');
$this->addSql('ALTER TABLE chill_main_postal_code DROP updatedAt');
$this->addSql('ALTER TABLE chill_main_postal_code DROP createdAt');
$this->addSql('ALTER TABLE chill_main_postal_code DROP updatedBy_id');
$this->addSql('ALTER TABLE chill_main_postal_code DROP createdBy_id');
$this->addSql('ALTER TABLE chill_main_postal_code DROP CONSTRAINT chill_internal_postal_code_import_unicity');
}
public function getDescription(): string
{
return 'postal code: add columns to track creation, update and deletion';
}
public function up(Schema $schema): void
{
$this->addSql('ALTER TABLE chill_main_postal_code ADD deletedAt TIMESTAMP(0) WITHOUT TIME ZONE DEFAULT NULL');
$this->addSql('ALTER TABLE chill_main_postal_code ADD updatedAt TIMESTAMP(0) WITHOUT TIME ZONE DEFAULT NULL');
$this->addSql('ALTER TABLE chill_main_postal_code ADD createdAt TIMESTAMP(0) WITHOUT TIME ZONE DEFAULT NULL');
$this->addSql('ALTER TABLE chill_main_postal_code ADD updatedBy_id INT DEFAULT NULL');
$this->addSql('ALTER TABLE chill_main_postal_code ADD createdBy_id INT DEFAULT NULL');
$this->addSql('COMMENT ON COLUMN chill_main_postal_code.deletedAt IS \'(DC2Type:datetime_immutable)\'');
$this->addSql('COMMENT ON COLUMN chill_main_postal_code.updatedAt IS \'(DC2Type:datetime_immutable)\'');
$this->addSql('COMMENT ON COLUMN chill_main_postal_code.createdAt IS \'(DC2Type:datetime_immutable)\'');
$this->addSql('ALTER TABLE chill_main_postal_code ADD CONSTRAINT FK_6CA145FA65FF1AEC FOREIGN KEY (updatedBy_id) REFERENCES users (id) NOT DEFERRABLE INITIALLY IMMEDIATE');
$this->addSql('ALTER TABLE chill_main_postal_code ADD CONSTRAINT FK_6CA145FA3174800F FOREIGN KEY (createdBy_id) REFERENCES users (id) NOT DEFERRABLE INITIALLY IMMEDIATE');
$this->addSql('CREATE INDEX IDX_6CA145FA65FF1AEC ON chill_main_postal_code (updatedBy_id)');
$this->addSql('CREATE INDEX IDX_6CA145FA3174800F ON chill_main_postal_code (createdBy_id)');
$this->addSql('CREATE UNIQUE INDEX postal_code_import_unicity ON chill_main_postal_code (code, refpostalcodeid, postalcodesource) WHERE refpostalcodeid is not null');
//$this->addSql('ALTER TABLE chill_main_postal_code ADD CONSTRAINT chill_internal_postal_code_import_unicity '.
// 'EXCLUDE (code WITH =, refpostalcodeid WITH =, postalcodesource WITH =) WHERE (refpostalcodeid IS NOT NULL)');
}
}

View File

@ -0,0 +1,26 @@
<?php
declare(strict_types=1);
namespace Chill\Migrations\Main;
use Doctrine\DBAL\Schema\Schema;
use Doctrine\Migrations\AbstractMigration;
final class Version20220730204216 extends AbstractMigration
{
public function getDescription(): string
{
return 'Add an unique constraint on addresses references';
}
public function up(Schema $schema): void
{
$this->addSql('CREATE UNIQUE INDEX chill_main_address_reference_unicity ON chill_main_address_reference (refId, source)');
}
public function down(Schema $schema): void
{
$this->addSql('DROP INDEX chill_main_address_reference_unicity');
}
}

View File

@ -49,9 +49,8 @@ class Evaluation
/**
* @ORM\ManyToMany(
* targetEntity=SocialAction::class,
* inversedBy="evaluations"
* mappedBy="evaluations"
* )
* @ORM\JoinTable(name="chill_person_social_work_evaluation_action")
*/
private Collection $socialActions;