Improve date management for not specified dates

- use real import date for person createdat field
- use year first day for not specified startdate, comment date, etc.

sql/import.sql

@@ -150,7 +150,7 @@ UPDATE import.personnes SET civility1=json_build_object('fr', trim(civility)) WH
 -- 21. Prepare personnes gender
 ALTER TABLE import.personnes ADD COLUMN gender1 VARCHAR;
 UPDATE import.personnes SET gender1 = CASE
-    WHEN trim(gender) IN ('Femme', 'femme', 'Woman', 'woman', 'Female', 'female')  THEN 'woman'
+    WHEN trim(gender) IN ('Femme', 'femme', 'Woman', 'woman', 'Female', 'female') THEN 'woman'
     WHEN trim(gender) IN ('Homme', 'homme', 'Man', 'man', 'Male', 'male') THEN 'man'
     WHEN trim(gender) IN ('', 'Inconnu', 'inconnu', 'Unknown', 'unknown') THEN 'unknown'
 ELSE 'both' END;
@@ -209,19 +209,32 @@ UPDATE import.periodes SET acp_scopes1=json_build_object('fr', trim(acp_scopes))
 ALTER TABLE import.periodes ADD COLUMN intensity1 text;
 UPDATE import.periodes SET intensity1='occasional'; UPDATE import.periodes SET intensity1='regular' WHERE intensity='regular';
 
--- 33. Format mobile numbers
+-- 33-34. Format phones numbers
 ALTER TABLE import.personnes ADD column mobilenumber1 text;
-UPDATE import.personnes SET mobilenumber1=NULLIF(regexp_replace(mobilenumber, '[^0-9]', '', 'g'), ''); -- remove all NaN chars
-UPDATE import.personnes SET mobilenumber1=regexp_replace(mobilenumber1, '^0', '') WHERE mobilenumber1 like '0%'; -- remove first 0 prefix
-UPDATE import.personnes SET mobilenumber1=regexp_replace(mobilenumber1, '(.*)', '+32\1'); -- add belgium intl prefix
-
--- 34. Format phone numbers
 ALTER TABLE import.personnes ADD column phonenumber1 text;
-UPDATE import.personnes SET phonenumber1=NULLIF(regexp_replace(phonenumber, '[^0-9]', '', 'g'), ''); -- remove all NaN chars
+DO $$
-UPDATE import.personnes SET phonenumber1=regexp_replace(phonenumber1, '^0', '') WHERE phonenumber1 like '0%'; -- remove first 0 prefix
+    DECLARE
-UPDATE import.personnes SET phonenumber1=regexp_replace(phonenumber1, '(.*)', '+32\1'); -- add belgium intl prefix
+        intl text := 32;
--- SELECT mobilenumber, mobilenumber1, phonenumber, phonenumber1 FROM import.personnes;
+    BEGIN
+        -- 33. Format mobile numbers
+        UPDATE import.personnes SET mobilenumber1=NULLIF(regexp_replace(mobilenumber, '[^0-9]', '', 'g'), ''); -- remove all NaN chars
+        UPDATE import.personnes SET mobilenumber1 = CASE -- remove intl prefix
+           WHEN LEFT(mobilenumber1, 2) = '00' THEN substr(mobilenumber1, 5, length(mobilenumber1) - 4)
+           WHEN LEFT(mobilenumber1, 2) = intl THEN substr(mobilenumber1, 3, length(mobilenumber1) - 2)
+           ELSE regexp_replace(mobilenumber1, '^0', '')  -- remove first 0 prefix
+        END;
+        UPDATE import.personnes SET mobilenumber1=regexp_replace(mobilenumber1, '(.*)', '+' || intl || '\1'); -- add intl prefix
 
+        -- 34. Format phone numbers
+        UPDATE import.personnes SET phonenumber1=NULLIF(regexp_replace(phonenumber, '[^0-9]', '', 'g'), ''); -- remove all NaN chars
+        UPDATE import.personnes SET phonenumber1 = CASE -- remove intl prefix
+            WHEN LEFT(phonenumber1, 2) = '00' THEN substr(phonenumber1, 5, length(phonenumber1) - 4)
+            WHEN LEFT(phonenumber1, 2) = intl THEN substr(phonenumber1, 3, length(phonenumber1) - 2)
+            ELSE regexp_replace(phonenumber1, '^0', '')  -- remove first 0 prefix
+        END;
+        UPDATE import.personnes SET phonenumber1=regexp_replace(phonenumber1, '(.*)', '+' || intl || '\1'); -- add intl prefix
+        --SELECT mobilenumber, mobilenumber1, phonenumber, phonenumber1 FROM import.personnes ORDER BY id;
+END $$;
 
 -- 35. Prepare required default dates
 -- address_validfrom1 | household_startdate1
@@ -241,10 +254,10 @@ INSERT INTO chill_person_person (
     , firstname, lastname, birthdate, place_of_birth, memo, email, contactinfo, phonenumber, mobilenumber, numberofchildren, gender, deathdate, proxyaccompanyingperiodopenstate
     , createdat, updatedat, createdby_id, updatedby_id
     , center_id
+    , gendercomment_comment, gendercomment_userid, gendercomment_date
     /*
     , maritalstatusdate
     , maritalstatuscomment_comment, maritalstatuscomment_userid, maritalstatuscomment_date
-    , gendercomment_comment, gendercomment_userid, gendercomment_date
     , acceptsms, acceptemail
     */
 ) SELECT
@@ -254,13 +267,13 @@ INSERT INTO chill_person_person (
     , (SELECT c.id FROM chill_main_civility c WHERE c.name::jsonb->>'fr' = ip.civility1::jsonb->>'fr' AND ip.civility1 IS NOT NULL ) -- civility_id
     , (SELECT ms.id FROM chill_person_marital_status ms WHERE ms.name::jsonb->>'fr' = ip.maritalstatus1::jsonb->>'fr' AND ip.maritalstatus1 IS NOT NULL ) -- maritalstatus_id,
     , TRIM(firstname), UPPER(TRIM(lastname)), birthdate1, TRIM(place_of_birth), TRIM(memo), TRIM(email), TRIM(contactinfo), phonenumber1, mobilenumber1, numberofchildren1, gender1, deathdate1, false
-    , CURRENT_DATE, CURRENT_DATE -- createdat, updatedat
+    , CURRENT_DATE, CURRENT_DATE -- createdat, updatedat (= import date)
     , (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM users), (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM users) -- createdby_id, updatedby_id
     , (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM centers) -- center_id
+    , TRIM(gendercomment), (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM users), date(date_trunc('year', CURRENT_DATE)) --gendercomment_<comment|userid|date>
     /*
     , -- maritalstatusdate
     , -- maritalstatuscomment_comment, maritalstatuscomment_userid, maritalstatuscomment_date
-    , -- gendercomment_comment, gendercomment_userid, gendercomment_date
     , -- acceptsms, acceptemail
     */
 FROM import.personnes ip;
@@ -269,7 +282,7 @@ FROM import.personnes ip;
 INSERT INTO chill_person_person_center_history (id, person_id, center_id, startdate) SELECT
     nextval('chill_person_person_center_history_id_seq'), person_id,
     (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM centers) , -- center_id
-    CURRENT_DATE
+    date(date_trunc('year', CURRENT_DATE))
     FROM import.personnes ip;
 -- SELECT ip.id, ip.lastname, ip.firstname, ip.person_id, hist.id, hist.person_id, hist.center_id, hist.startdate, p.id, p.fullnamecanonical, p.email FROM chill_person_person p RIGHT JOIN import.personnes ip ON ip.person_id = p.id FULL JOIN chill_person_person_center_history hist on p.id = hist.person_id ORDER BY ip.person_id;
 
@@ -279,7 +292,7 @@ UPDATE import.personnes SET address_id = nextval('chill_main_address_id_seq');
 ALTER TABLE import.personnes ADD column postcode_id BIGINT;
 UPDATE import.personnes SET postcode_id = cmpc.id FROM chill_main_postal_code cmpc WHERE cmpc.code = postcode AND cmpc.origin = 0;
 INSERT INTO chill_main_address (id, postcode_id, street, streetnumber, extra, validFrom) SELECT
-    address_id, postcode_id, street, streetnumber, extra, CURRENT_DATE
+    address_id, postcode_id, street, streetnumber, extra, COALESCE(validfrom1 , date(date_trunc('year', CURRENT_DATE)))
     FROM import.personnes WHERE postcode_id IS NOT NULL;
 SELECT ip.person_id, ip.address_id FROM import.personnes ip ;
 -- SELECT ip.id, ip.person_id, ip.firstname, ip.lastname, addr.* FROM chill_main_address addr RIGHT JOIN import.personnes ip ON addr.id = ip.address_id ORDER BY ip.id;
@@ -288,7 +301,7 @@ SELECT ip.person_id, ip.address_id FROM import.personnes ip ;
 INSERT INTO chill_person_household (id) SELECT person_id from import.personnes;
 SELECT setval('chill_person_household_id_seq', (SELECT max(id) FROM chill_person_household));
 INSERT INTO chill_person_household_members (id, person_id, household_id, startdate, sharedhousehold, position_id, holder) SELECT
-    nextval('chill_person_household_members_id_seq'), person_id, person_id, COALESCE(household_startdate1, CURRENT_DATE),
+    nextval('chill_person_household_members_id_seq'), person_id, person_id, COALESCE(household_startdate1, date(date_trunc('year', CURRENT_DATE))),
     true, (SELECT id FROM chill_person_household_position pos WHERE household_position1::jsonb->>'fr' = pos.label::jsonb->>'fr'), false
     FROM import.personnes;
 -- SELECT ip.id, ip.person_id, ip.firstname, ip.lastname, hh.* FROM chill_person_household hh FULL JOIN import.personnes ip ON hh.id = ip.person_id ORDER BY hh.id;