Improve date management for not specified dates

- use real import date for person createdat field
- use year first day for not specified startdate, comment date, etc.

sql/import.sql

@@ -209,19 +209,32 @@ UPDATE import.periodes SET acp_scopes1=json_build_object('fr', trim(acp_scopes))
 ALTER TABLE import.periodes ADD COLUMN intensity1 text;
 UPDATE import.periodes SET intensity1='occasional'; UPDATE import.periodes SET intensity1='regular' WHERE intensity='regular';
 
--- 33. Format mobile numbers
+-- 33-34. Format phones numbers
 ALTER TABLE import.personnes ADD column mobilenumber1 text;
-UPDATE import.personnes SET mobilenumber1=NULLIF(regexp_replace(mobilenumber, '[^0-9]', '', 'g'), ''); -- remove all NaN chars
-UPDATE import.personnes SET mobilenumber1=regexp_replace(mobilenumber1, '^0', '') WHERE mobilenumber1 like '0%'; -- remove first 0 prefix
-UPDATE import.personnes SET mobilenumber1=regexp_replace(mobilenumber1, '(.*)', '+32\1'); -- add belgium intl prefix
-
--- 34. Format phone numbers
 ALTER TABLE import.personnes ADD column phonenumber1 text;
-UPDATE import.personnes SET phonenumber1=NULLIF(regexp_replace(phonenumber, '[^0-9]', '', 'g'), ''); -- remove all NaN chars
-UPDATE import.personnes SET phonenumber1=regexp_replace(phonenumber1, '^0', '') WHERE phonenumber1 like '0%'; -- remove first 0 prefix
-UPDATE import.personnes SET phonenumber1=regexp_replace(phonenumber1, '(.*)', '+32\1'); -- add belgium intl prefix
--- SELECT mobilenumber, mobilenumber1, phonenumber, phonenumber1 FROM import.personnes;
+DO $$
+    DECLARE
+        intl text := 32;
+    BEGIN
+        -- 33. Format mobile numbers
+        UPDATE import.personnes SET mobilenumber1=NULLIF(regexp_replace(mobilenumber, '[^0-9]', '', 'g'), ''); -- remove all NaN chars
+        UPDATE import.personnes SET mobilenumber1 = CASE -- remove intl prefix
+           WHEN LEFT(mobilenumber1, 2) = '00' THEN substr(mobilenumber1, 5, length(mobilenumber1) - 4)
+           WHEN LEFT(mobilenumber1, 2) = intl THEN substr(mobilenumber1, 3, length(mobilenumber1) - 2)
+           ELSE regexp_replace(mobilenumber1, '^0', '')  -- remove first 0 prefix
+        END;
+        UPDATE import.personnes SET mobilenumber1=regexp_replace(mobilenumber1, '(.*)', '+' || intl || '\1'); -- add intl prefix
 
+        -- 34. Format phone numbers
+        UPDATE import.personnes SET phonenumber1=NULLIF(regexp_replace(phonenumber, '[^0-9]', '', 'g'), ''); -- remove all NaN chars
+        UPDATE import.personnes SET phonenumber1 = CASE -- remove intl prefix
+            WHEN LEFT(phonenumber1, 2) = '00' THEN substr(phonenumber1, 5, length(phonenumber1) - 4)
+            WHEN LEFT(phonenumber1, 2) = intl THEN substr(phonenumber1, 3, length(phonenumber1) - 2)
+            ELSE regexp_replace(phonenumber1, '^0', '')  -- remove first 0 prefix
+        END;
+        UPDATE import.personnes SET phonenumber1=regexp_replace(phonenumber1, '(.*)', '+' || intl || '\1'); -- add intl prefix
+        --SELECT mobilenumber, mobilenumber1, phonenumber, phonenumber1 FROM import.personnes ORDER BY id;
+END $$;
 
 -- 35. Prepare required default dates
 -- address_validfrom1 | household_startdate1
@@ -241,10 +254,10 @@ INSERT INTO chill_person_person (
     , firstname, lastname, birthdate, place_of_birth, memo, email, contactinfo, phonenumber, mobilenumber, numberofchildren, gender, deathdate, proxyaccompanyingperiodopenstate
     , createdat, updatedat, createdby_id, updatedby_id
     , center_id
+    , gendercomment_comment, gendercomment_userid, gendercomment_date
     /*
     , maritalstatusdate
     , maritalstatuscomment_comment, maritalstatuscomment_userid, maritalstatuscomment_date
-    , gendercomment_comment, gendercomment_userid, gendercomment_date
     , acceptsms, acceptemail
     */
 ) SELECT
@@ -254,13 +267,13 @@ INSERT INTO chill_person_person (
     , (SELECT c.id FROM chill_main_civility c WHERE c.name::jsonb->>'fr' = ip.civility1::jsonb->>'fr' AND ip.civility1 IS NOT NULL ) -- civility_id
     , (SELECT ms.id FROM chill_person_marital_status ms WHERE ms.name::jsonb->>'fr' = ip.maritalstatus1::jsonb->>'fr' AND ip.maritalstatus1 IS NOT NULL ) -- maritalstatus_id,
     , TRIM(firstname), UPPER(TRIM(lastname)), birthdate1, TRIM(place_of_birth), TRIM(memo), TRIM(email), TRIM(contactinfo), phonenumber1, mobilenumber1, numberofchildren1, gender1, deathdate1, false
-    , CURRENT_DATE, CURRENT_DATE -- createdat, updatedat
+    , CURRENT_DATE, CURRENT_DATE -- createdat, updatedat (= import date)
     , (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM users), (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM users) -- createdby_id, updatedby_id
     , (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM centers) -- center_id
+    , TRIM(gendercomment), (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM users), date(date_trunc('year', CURRENT_DATE)) --gendercomment_<comment|userid|date>
     /*
     , -- maritalstatusdate
     , -- maritalstatuscomment_comment, maritalstatuscomment_userid, maritalstatuscomment_date
-    , -- gendercomment_comment, gendercomment_userid, gendercomment_date
     , -- acceptsms, acceptemail
     */
 FROM import.personnes ip;
@@ -269,7 +282,7 @@ FROM import.personnes ip;
 INSERT INTO chill_person_person_center_history (id, person_id, center_id, startdate) SELECT
     nextval('chill_person_person_center_history_id_seq'), person_id,
     (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM centers) , -- center_id
-    CURRENT_DATE
+    date(date_trunc('year', CURRENT_DATE))
     FROM import.personnes ip;
 -- SELECT ip.id, ip.lastname, ip.firstname, ip.person_id, hist.id, hist.person_id, hist.center_id, hist.startdate, p.id, p.fullnamecanonical, p.email FROM chill_person_person p RIGHT JOIN import.personnes ip ON ip.person_id = p.id FULL JOIN chill_person_person_center_history hist on p.id = hist.person_id ORDER BY ip.person_id;
 
@@ -279,7 +292,7 @@ UPDATE import.personnes SET address_id = nextval('chill_main_address_id_seq');
 ALTER TABLE import.personnes ADD column postcode_id BIGINT;
 UPDATE import.personnes SET postcode_id = cmpc.id FROM chill_main_postal_code cmpc WHERE cmpc.code = postcode AND cmpc.origin = 0;
 INSERT INTO chill_main_address (id, postcode_id, street, streetnumber, extra, validFrom) SELECT
-    address_id, postcode_id, street, streetnumber, extra, CURRENT_DATE
+    address_id, postcode_id, street, streetnumber, extra, COALESCE(validfrom1 , date(date_trunc('year', CURRENT_DATE)))
     FROM import.personnes WHERE postcode_id IS NOT NULL;
 SELECT ip.person_id, ip.address_id FROM import.personnes ip ;
 -- SELECT ip.id, ip.person_id, ip.firstname, ip.lastname, addr.* FROM chill_main_address addr RIGHT JOIN import.personnes ip ON addr.id = ip.address_id ORDER BY ip.id;
@@ -288,7 +301,7 @@ SELECT ip.person_id, ip.address_id FROM import.personnes ip ;
 INSERT INTO chill_person_household (id) SELECT person_id from import.personnes;
 SELECT setval('chill_person_household_id_seq', (SELECT max(id) FROM chill_person_household));
 INSERT INTO chill_person_household_members (id, person_id, household_id, startdate, sharedhousehold, position_id, holder) SELECT
-    nextval('chill_person_household_members_id_seq'), person_id, person_id, COALESCE(household_startdate1, CURRENT_DATE),
+    nextval('chill_person_household_members_id_seq'), person_id, person_id, COALESCE(household_startdate1, date(date_trunc('year', CURRENT_DATE))),
     true, (SELECT id FROM chill_person_household_position pos WHERE household_position1::jsonb->>'fr' = pos.label::jsonb->>'fr'), false
     FROM import.personnes;
 -- SELECT ip.id, ip.person_id, ip.firstname, ip.lastname, hh.* FROM chill_person_household hh FULL JOIN import.personnes ip ON hh.id = ip.person_id ORDER BY hh.id;