From 2f28cb16ef8afaf1dd2ab0cd785183a4f65c8d15 Mon Sep 17 00:00:00 2001
From: nobohan <juminet@gmail.com>
Date: Tue, 18 May 2021 18:02:59 +0200
Subject: [PATCH] addresses: import address reference: add pagination and error
 handling

---
 .../Command/LoadAddressReferenceCommand.php   | 67 +++++++++++++++----
 1 file changed, 55 insertions(+), 12 deletions(-)

diff --git a/src/Bundle/ChillMainBundle/Command/LoadAddressReferenceCommand.php b/src/Bundle/ChillMainBundle/Command/LoadAddressReferenceCommand.php
index a488685bf..650bc6c66 100644
--- a/src/Bundle/ChillMainBundle/Command/LoadAddressReferenceCommand.php
+++ b/src/Bundle/ChillMainBundle/Command/LoadAddressReferenceCommand.php
@@ -27,6 +27,9 @@ class LoadAddressReferenceCommand extends Command
      */
     private $client;
 
+
+    private $wfsBaseUrl = 'https://carto.vendee.fr/arcgis/services/Applications/CHILL/MapServer/WFSServer?SERVICE=WFS&REQUEST=GetFeature&version=2.0.0&TYPENAME=CHILL:Adresses';
+
     /**
      * LoadAddressReferenceCommand constructor.
      */
@@ -56,16 +59,30 @@ class LoadAddressReferenceCommand extends Command
     protected function execute(InputInterface $input, OutputInterface $output): int
     {
 
-        $addressReferenceArray = $this->fetchWFS(1, 2, $output);
+        $maxCount = $this->countWFSFeatures($this->wfsBaseUrl, $output);
+        $totalCount = 0;
+        $batchSize = 100;
+
         $em = $this->entityManager;
 
-        foreach($addressReferenceArray as $a) {
-            if (static::isAddressReferenceNew($a)) {
-                $em->persist($a);
-            }
-        }
+        while($totalCount < $maxCount) {
 
-        $em->flush();
+            try {
+                $addressReferenceArray = $this->fetchWFS($totalCount, $batchSize, $output);
+            } catch (\Exception $ex) {
+                $output->writeln('<warning>Cannot fetch WFS. Error message is: '.$ex->getMessage().'</warning>');
+            }
+
+            foreach($addressReferenceArray as $a) {
+                if (static::isAddressReferenceNew($a)) {
+                    $em->persist($a);
+                }
+            }
+
+            $em->flush();
+
+            $totalCount = $totalCount + $batchSize;
+        }
 
         return 0;
     }
@@ -79,6 +96,24 @@ class LoadAddressReferenceCommand extends Command
     }
 
 
+    private function countWFSFeatures(string $wfsBaseUrl, OutputInterface $output): int
+    {
+        $maxCountWfsUrl = $wfsBaseUrl . '&resultType=hits';
+        try {
+            $response = $this->client->request('GET', $maxCountWfsUrl);
+        } catch (\Exception $ex) {
+            $output->writeln('<warning>Cannot get WFS request working with '.$maxCountWfsUrl.' : '.$ex->getMessage().'</warning>');
+        }
+        if ($response->getStatusCode() === 200) {
+            $content = $response->getContent();
+            $count = explode('"', explode('numberMatched="', $content)[1])[0];
+        } else {
+            $output->writeln('<error>Error getting WFS at ' . $maxCountWfsUrl . '. Status code is ' . $response->getStatusCode() . '.</error>');
+        }
+
+        return $count;
+    }
+
     /*
      * //TODO move this function in App and make the command generic by accepting a fetcher as an argument
      * @see \Symfony\Component\Console\Command\Command::execute()
@@ -86,15 +121,18 @@ class LoadAddressReferenceCommand extends Command
     private function fetchWFS(int $startIndex, int $count, OutputInterface $output): array
     {
 
-        $dataSource = 'BAL GeoVendee';
         $countryCode = 'FR';
 
         $addressReferenceArray = array();
 
-        $wfsBaseUrl = 'https://carto.vendee.fr/arcgis/services/Applications/CHILL/MapServer/WFSServer?SERVICE=WFS&REQUEST=GetFeature&version=2.0.0';
-        $wfsUrl = $wfsBaseUrl . '&TYPENAME=CHILL:Adresses&startIndex=' . $startIndex . '&count=' . $count . '&OUTPUTFORMAT=geojson';
+        $wfsBaseUrl = $this->wfsBaseUrl;
+        $wfsUrl = $wfsBaseUrl . '&startIndex=' . $startIndex . '&count=' . $count . '&OUTPUTFORMAT=geojson';
 
-        $response = $this->client->request('GET', $wfsUrl);
+        try {
+            $response = $this->client->request('GET', $wfsUrl);
+        } catch (\Exception $ex) {
+            $output->writeln('<warning>Cannot get WFS request working with '.$wfsUrl.' : '.$ex->getMessage().'</warning>');
+        }
 
         if ($response->getStatusCode() === 200) {
 
@@ -102,6 +140,11 @@ class LoadAddressReferenceCommand extends Command
 
             foreach ($content['features'] as $key => $value) {
 
+                if ($output->getVerbosity() === OutputInterface::VERBOSITY_VERY_VERBOSE) {
+                    $output->writeln('The address is ...');
+                    $output->write($value);
+                }
+
                 $a = new AddressReference();
 
                 $x = $value['geometry']['coordinates'][0];
@@ -128,7 +171,7 @@ class LoadAddressReferenceCommand extends Command
                 $a->setPoint($point);
                 $a->setPostcode($postcode);
                 $a->setMunicipalityCode($value['properties']['CODE_INSEE']);
-                $a->setSource($dataSource);
+                $a->setSource($value['properties']['SOURCE']);
 
                 $addressReferenceArray[] = $a;
             }