From f018a5d115a44aca993ba2254ff4d10838051da7 Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Wed, 15 Feb 2023 17:54:26 +0100 Subject: [PATCH 01/48] 1. cast column id --- sql/import.sql | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/sql/import.sql b/sql/import.sql index 5365c0e..136d2c8 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -1,13 +1,16 @@ -- -- CHILL REPRISE DE DONNEES --- version v0.1 +-- version v0.6 (== version canevas) -- --- /!\ +-- /!\ IMPORTANT -- * Avant de migrer (UP), il faut d'abord importer les codes postaux !!! --- * On essaie de matcher le canvas csv et le script -- +-- 1. Adjust Type cast for id columns +ALTER TABLE import.personnes ALTER COLUMN id TYPE INTEGER USING (id::integer); +ALTER TABLE import.periodes ALTER COLUMN id TYPE INTEGER USING (id::integer); + -- -- UP -- From 06b0aab2379ecaa89293ba72d1dc5de8573a5c6d Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Wed, 15 Feb 2023 18:08:06 +0100 Subject: [PATCH 02/48] 2. prepare date columns --- sql/import.sql | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/sql/import.sql b/sql/import.sql index 136d2c8..a8f3961 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -11,6 +11,13 @@ ALTER TABLE import.personnes ALTER COLUMN id TYPE INTEGER USING (id::integer); ALTER TABLE import.periodes ALTER COLUMN id TYPE INTEGER USING (id::integer); +-- 2. Copy date columns with right date type +ALTER TABLE import.personnes ADD column birthdate1 DATE; UPDATE import.personnes SET birthdate1 = to_date(birthdate,'DD/MM/YYYY') WHERE birthdate != ''; +ALTER TABLE import.personnes ADD column deathdate1 DATE; UPDATE import.personnes SET deathdate1 = to_date(deathdate,'DD/MM/YYYY') WHERE deathdate != ''; +ALTER TABLE import.personnes ADD column validfrom1 DATE; UPDATE import.personnes SET validfrom1 = to_date(validfrom,'DD/MM/YYYY') WHERE validfrom != ''; +ALTER TABLE import.personnes ADD column household_startdate1 DATE; UPDATE import.personnes SET household_startdate1 = to_date(household_startdate,'DD/MM/YYYY') WHERE household_startdate != ''; +ALTER TABLE import.periodes ADD column openingdate1 DATE; UPDATE import.periodes SET openingdate1 = to_date(openingdate::text,'DD/MM/YYYY') WHERE openingdate != ''; +ALTER TABLE import.periodes ADD column closingdate1 DATE; UPDATE import.periodes SET closingdate1 = to_date(closingdate,'DD/MM/YYYY') WHERE closingdate != ''; -- -- UP -- @@ -20,6 +27,14 @@ ALTER TABLE import.periodes ALTER COLUMN id TYPE INTEGER USING (id::integer); -- DOWN -- +-- Undo 2. +ALTER TABLE import.personnes DROP column birthdate1; +ALTER TABLE import.personnes DROP column deathdate1; +ALTER TABLE import.personnes DROP column validfrom1; +ALTER TABLE import.personnes DROP column household_startdate1; +ALTER TABLE import.periodes DROP column openingdate1; +ALTER TABLE import.periodes DROP column closingdate1; + -- -- SELECT From 9cd1f19f89630a14ff0c025ff0dde10c4af70573 Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Wed, 15 Feb 2023 20:21:26 +0100 Subject: [PATCH 03/48] 3. complete table chill_main_civility --- sql/import.sql | 47 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/sql/import.sql b/sql/import.sql index a8f3961..3c5b84e 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -6,27 +6,49 @@ -- * Avant de migrer (UP), il faut d'abord importer les codes postaux !!! -- - -- 1. Adjust Type cast for id columns ALTER TABLE import.personnes ALTER COLUMN id TYPE INTEGER USING (id::integer); ALTER TABLE import.periodes ALTER COLUMN id TYPE INTEGER USING (id::integer); --- 2. Copy date columns with right date type -ALTER TABLE import.personnes ADD column birthdate1 DATE; UPDATE import.personnes SET birthdate1 = to_date(birthdate,'DD/MM/YYYY') WHERE birthdate != ''; -ALTER TABLE import.personnes ADD column deathdate1 DATE; UPDATE import.personnes SET deathdate1 = to_date(deathdate,'DD/MM/YYYY') WHERE deathdate != ''; -ALTER TABLE import.personnes ADD column validfrom1 DATE; UPDATE import.personnes SET validfrom1 = to_date(validfrom,'DD/MM/YYYY') WHERE validfrom != ''; -ALTER TABLE import.personnes ADD column household_startdate1 DATE; UPDATE import.personnes SET household_startdate1 = to_date(household_startdate,'DD/MM/YYYY') WHERE household_startdate != ''; -ALTER TABLE import.periodes ADD column openingdate1 DATE; UPDATE import.periodes SET openingdate1 = to_date(openingdate::text,'DD/MM/YYYY') WHERE openingdate != ''; -ALTER TABLE import.periodes ADD column closingdate1 DATE; UPDATE import.periodes SET closingdate1 = to_date(closingdate,'DD/MM/YYYY') WHERE closingdate != ''; -- -- UP -- +-- 2. Copy date columns with right date type +ALTER TABLE import.personnes ADD COLUMN birthdate1 DATE; UPDATE import.personnes SET birthdate1 = to_date(birthdate,'DD/MM/YYYY') WHERE birthdate != ''; +ALTER TABLE import.personnes ADD COLUMN deathdate1 DATE; UPDATE import.personnes SET deathdate1 = to_date(deathdate,'DD/MM/YYYY') WHERE deathdate != ''; +ALTER TABLE import.personnes ADD COLUMN validfrom1 DATE; UPDATE import.personnes SET validfrom1 = to_date(validfrom,'DD/MM/YYYY') WHERE validfrom != ''; +ALTER TABLE import.personnes ADD COLUMN household_startdate1 DATE; UPDATE import.personnes SET household_startdate1 = to_date(household_startdate,'DD/MM/YYYY') WHERE household_startdate != ''; +ALTER TABLE import.periodes ADD COLUMN openingdate1 DATE; UPDATE import.periodes SET openingdate1 = to_date(openingdate::text,'DD/MM/YYYY') WHERE openingdate != ''; +ALTER TABLE import.periodes ADD COLUMN closingdate1 DATE; UPDATE import.periodes SET closingdate1 = to_date(closingdate,'DD/MM/YYYY') WHERE closingdate != ''; + +-- 3. Complete table Civility +ALTER TABLE import.choix_personnes ADD COLUMN civility1 JSONB; +UPDATE import.choix_personnes SET civility1=json_build_object('fr', trim(civility)) WHERE civility!=''; +WITH max_ordering AS ( SELECT MAX(ordering) as max_ordering FROM chill_main_civility ) +INSERT INTO chill_main_civility (id, name, abbreviation, active, ordering) + SELECT nextval('chill_main_civility_id_seq'), + t.civility1, t.civility1, true, + max_ordering.max_ordering + row_number() OVER () as ordering + FROM ( SELECT DISTINCT ON (civility) civility1 FROM import.choix_personnes WHERE civility1 IS NOT NULL ) t + CROSS JOIN max_ordering + WHERE NOT EXISTS ( SELECT 1 FROM chill_main_civility WHERE name::jsonb = t.civility1 ); + + + + + + -- -- DOWN -- +-- Undo 3. +DELETE FROM chill_main_civility cmc USING import.choix_personnes ip WHERE cmc.name::jsonb = ip.civility1::jsonb; +SELECT setval('chill_main_civility_id_seq', (SELECT max(id) FROM chill_main_civility)); +ALTER TABLE import.choix_personnes DROP column civility1; + -- Undo 2. ALTER TABLE import.personnes DROP column birthdate1; ALTER TABLE import.personnes DROP column deathdate1; @@ -36,6 +58,15 @@ ALTER TABLE import.periodes DROP column openingdate1; ALTER TABLE import.periodes DROP column closingdate1; +-- ------------- + +-- personnes choices_list: civility gender nationality countryofbirth country maritalstatus household_composition_type household_position +-- periodes choices_list: intensity referrer job acp_scopes acp_socialissues work_socialaction +-- tiers choices_list: civility kind profession category country + -- -- SELECT -- +SELECT DISTINCT civility FROM import.personnes; +SELECT * FROM import.periodes; +SELECT DISTINCT ON (civility) civility1 FROM import.personnes WHERE civility1 IS NOT NULL; From 40ebabf3f6cacb5c9ede7d228dbcf8e9400d047a Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Wed, 15 Feb 2023 20:56:50 +0100 Subject: [PATCH 04/48] 4. complete country table --- sql/import.sql | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/sql/import.sql b/sql/import.sql index 3c5b84e..0e09ad1 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -34,7 +34,13 @@ INSERT INTO chill_main_civility (id, name, abbreviation, active, ordering) CROSS JOIN max_ordering WHERE NOT EXISTS ( SELECT 1 FROM chill_main_civility WHERE name::jsonb = t.civility1 ); - +-- 4. Complete table Country +ALTER TABLE import.choix_personnes ADD COLUMN country1 JSONB; +UPDATE import.choix_personnes SET country1=json_build_object('fr', trim(country)) WHERE country!=''; +INSERT INTO country (id, name, countrycode) +SELECT nextval('country_id_seq'), t.country1, 'ZZZ' + FROM ( SELECT DISTINCT ON (country) country1 FROM import.choix_personnes WHERE country1 IS NOT NULL ) t + WHERE NOT EXISTS ( SELECT 1 FROM country WHERE name::jsonb = t.country1 ); @@ -44,6 +50,11 @@ INSERT INTO chill_main_civility (id, name, abbreviation, active, ordering) -- DOWN -- +-- Undo 4. +DELETE FROM country USING import.choix_personnes icp WHERE country.name::jsonb = icp.country1::jsonb AND country.id > 249; -- 249 existing countries +SELECT setval('country_id_seq', (SELECT max(id) FROM country)); +ALTER TABLE import.choix_personnes DROP column country1; + -- Undo 3. DELETE FROM chill_main_civility cmc USING import.choix_personnes ip WHERE cmc.name::jsonb = ip.civility1::jsonb; SELECT setval('chill_main_civility_id_seq', (SELECT max(id) FROM chill_main_civility)); @@ -60,9 +71,9 @@ ALTER TABLE import.periodes DROP column closingdate1; -- ------------- --- personnes choices_list: civility gender nationality countryofbirth country maritalstatus household_composition_type household_position +-- personnes choices_list: gender maritalstatus household_composition_type household_position -- periodes choices_list: intensity referrer job acp_scopes acp_socialissues work_socialaction --- tiers choices_list: civility kind profession category country +-- tiers choices_list: civility kind profession category -- -- SELECT From 27ef1034be4067676dc39f9bda5c679650fffe24 Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Wed, 15 Feb 2023 21:28:13 +0100 Subject: [PATCH 05/48] 5. complete chill_person_marital_status table --- sql/import.sql | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/sql/import.sql b/sql/import.sql index 0e09ad1..40b5485 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -38,18 +38,27 @@ INSERT INTO chill_main_civility (id, name, abbreviation, active, ordering) ALTER TABLE import.choix_personnes ADD COLUMN country1 JSONB; UPDATE import.choix_personnes SET country1=json_build_object('fr', trim(country)) WHERE country!=''; INSERT INTO country (id, name, countrycode) -SELECT nextval('country_id_seq'), t.country1, 'ZZZ' + SELECT nextval('country_id_seq'), t.country1, 'ZZZ' FROM ( SELECT DISTINCT ON (country) country1 FROM import.choix_personnes WHERE country1 IS NOT NULL ) t WHERE NOT EXISTS ( SELECT 1 FROM country WHERE name::jsonb = t.country1 ); - - +-- 5. Complete table maritalstatus +ALTER TABLE import.choix_personnes ADD COLUMN maritalstatus1 JSONB; +UPDATE import.choix_personnes SET maritalstatus1=json_build_object('fr', trim(maritalstatus)) WHERE maritalstatus!=''; +INSERT INTO chill_person_marital_status (id, name) + SELECT substr(md5(random()::text), 1, 7), t.maritalstatus1 + FROM ( SELECT DISTINCT ON (maritalstatus) maritalstatus1 FROM import.choix_personnes WHERE maritalstatus1 IS NOT NULL ) t + WHERE NOT EXISTS ( SELECT 1 FROM chill_person_marital_status WHERE name::jsonb = t.maritalstatus1 ); -- -- DOWN -- +-- Undo 5. +DELETE FROM chill_person_marital_status cpms USING import.choix_personnes icp WHERE cpms.name::jsonb = icp.maritalstatus1::jsonb AND cpms.id <> 'célibat.' AND cpms.id <> 'marié'; -- existing marital status +ALTER TABLE import.choix_personnes DROP column maritalstatus1; + -- Undo 4. DELETE FROM country USING import.choix_personnes icp WHERE country.name::jsonb = icp.country1::jsonb AND country.id > 249; -- 249 existing countries SELECT setval('country_id_seq', (SELECT max(id) FROM country)); @@ -71,7 +80,7 @@ ALTER TABLE import.periodes DROP column closingdate1; -- ------------- --- personnes choices_list: gender maritalstatus household_composition_type household_position +-- personnes choices_list: gender household_composition_type household_position -- periodes choices_list: intensity referrer job acp_scopes acp_socialissues work_socialaction -- tiers choices_list: civility kind profession category From de8450267d0a7339cffb50520d8aa49426150820 Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Wed, 15 Feb 2023 21:40:40 +0100 Subject: [PATCH 06/48] 6. complete chill_person_household_composition_type table --- sql/import.sql | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/sql/import.sql b/sql/import.sql index 40b5485..56e2764 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -50,37 +50,50 @@ INSERT INTO chill_person_marital_status (id, name) FROM ( SELECT DISTINCT ON (maritalstatus) maritalstatus1 FROM import.choix_personnes WHERE maritalstatus1 IS NOT NULL ) t WHERE NOT EXISTS ( SELECT 1 FROM chill_person_marital_status WHERE name::jsonb = t.maritalstatus1 ); +-- 6. Complete table household_composition_type +ALTER TABLE import.choix_personnes ADD COLUMN household_composition_type1 JSONB; +UPDATE import.choix_personnes SET household_composition_type1=json_build_object('fr', trim(household_composition_type)) WHERE household_composition_type!=''; +INSERT INTO chill_person_household_composition_type (id, active, label) + SELECT nextval('chill_person_household_composition_type_id_seq'), true, t.household_composition_type1 + FROM ( SELECT DISTINCT ON (household_composition_type) household_composition_type1 FROM import.choix_personnes WHERE household_composition_type1 IS NOT NULL ) t + WHERE NOT EXISTS ( SELECT 1 FROM chill_person_household_composition_type WHERE label::jsonb = t.household_composition_type1 ); + -- -- DOWN -- +-- Undo 6. +DELETE FROM chill_person_household_composition_type cphct USING import.choix_personnes icp WHERE cphct.label::jsonb = icp.household_composition_type1::jsonb AND cphct.id > 6; +SELECT setval('chill_person_household_composition_type_id_seq', (SELECT max(id) FROM chill_person_household_composition_type)); +ALTER TABLE import.choix_personnes DROP COLUMN household_composition_type1; + -- Undo 5. DELETE FROM chill_person_marital_status cpms USING import.choix_personnes icp WHERE cpms.name::jsonb = icp.maritalstatus1::jsonb AND cpms.id <> 'célibat.' AND cpms.id <> 'marié'; -- existing marital status -ALTER TABLE import.choix_personnes DROP column maritalstatus1; +ALTER TABLE import.choix_personnes DROP COLUMN maritalstatus1; -- Undo 4. DELETE FROM country USING import.choix_personnes icp WHERE country.name::jsonb = icp.country1::jsonb AND country.id > 249; -- 249 existing countries SELECT setval('country_id_seq', (SELECT max(id) FROM country)); -ALTER TABLE import.choix_personnes DROP column country1; +ALTER TABLE import.choix_personnes DROP COLUMN country1; -- Undo 3. DELETE FROM chill_main_civility cmc USING import.choix_personnes ip WHERE cmc.name::jsonb = ip.civility1::jsonb; SELECT setval('chill_main_civility_id_seq', (SELECT max(id) FROM chill_main_civility)); -ALTER TABLE import.choix_personnes DROP column civility1; +ALTER TABLE import.choix_personnes DROP COLUMN civility1; -- Undo 2. -ALTER TABLE import.personnes DROP column birthdate1; -ALTER TABLE import.personnes DROP column deathdate1; -ALTER TABLE import.personnes DROP column validfrom1; -ALTER TABLE import.personnes DROP column household_startdate1; -ALTER TABLE import.periodes DROP column openingdate1; -ALTER TABLE import.periodes DROP column closingdate1; +ALTER TABLE import.personnes DROP COLUMN birthdate1; +ALTER TABLE import.personnes DROP COLUMN deathdate1; +ALTER TABLE import.personnes DROP COLUMN validfrom1; +ALTER TABLE import.personnes DROP COLUMN household_startdate1; +ALTER TABLE import.periodes DROP COLUMN openingdate1; +ALTER TABLE import.periodes DROP COLUMN closingdate1; -- ------------- --- personnes choices_list: gender household_composition_type household_position +-- personnes choices_list: gender household_position -- periodes choices_list: intensity referrer job acp_scopes acp_socialissues work_socialaction -- tiers choices_list: civility kind profession category @@ -89,4 +102,3 @@ ALTER TABLE import.periodes DROP column closingdate1; -- SELECT DISTINCT civility FROM import.personnes; SELECT * FROM import.periodes; -SELECT DISTINCT ON (civility) civility1 FROM import.personnes WHERE civility1 IS NOT NULL; From 32da5540fee33aba7d97ccc05583bd041a7c406f Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Wed, 15 Feb 2023 21:57:36 +0100 Subject: [PATCH 07/48] 7. complete chill_person_household_position table --- sql/import.sql | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/sql/import.sql b/sql/import.sql index 56e2764..bb34096 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -58,11 +58,27 @@ INSERT INTO chill_person_household_composition_type (id, active, label) FROM ( SELECT DISTINCT ON (household_composition_type) household_composition_type1 FROM import.choix_personnes WHERE household_composition_type1 IS NOT NULL ) t WHERE NOT EXISTS ( SELECT 1 FROM chill_person_household_composition_type WHERE label::jsonb = t.household_composition_type1 ); +-- 7. Complete table household_position +ALTER TABLE import.choix_personnes ADD COLUMN household_position1 JSONB; +UPDATE import.choix_personnes SET household_position1=json_build_object('fr', trim(household_position)) WHERE household_position!=''; +WITH max_ordering AS ( SELECT MAX(ordering) as max_ordering FROM chill_person_household_position ) +INSERT INTO chill_person_household_position (id, label, sharehousehold, allowholder, ordering) + SELECT + nextval('chill_person_household_position_id_seq'), t.household_position1, true, false, + max_ordering.max_ordering + row_number() OVER () as ordering + FROM ( SELECT DISTINCT ON (household_position) household_position1 FROM import.choix_personnes WHERE household_position1 IS NOT NULL ) t + CROSS JOIN max_ordering + WHERE NOT EXISTS ( SELECT 1 FROM chill_person_household_position WHERE label::jsonb = t.household_position1 ); -- -- DOWN -- +-- Undo 7. +DELETE FROM chill_person_household_position cphp USING import.choix_personnes icp WHERE cphp.label::jsonb = icp.household_position1::jsonb; +SELECT setval('chill_person_household_position_id_seq', (SELECT max(id) FROM chill_person_household_position)); +ALTER TABLE import.choix_personnes DROP COLUMN household_position1; + -- Undo 6. DELETE FROM chill_person_household_composition_type cphct USING import.choix_personnes icp WHERE cphct.label::jsonb = icp.household_composition_type1::jsonb AND cphct.id > 6; SELECT setval('chill_person_household_composition_type_id_seq', (SELECT max(id) FROM chill_person_household_composition_type)); @@ -93,7 +109,7 @@ ALTER TABLE import.periodes DROP COLUMN closingdate1; -- ------------- --- personnes choices_list: gender household_position +-- personnes choices_list: gender -- periodes choices_list: intensity referrer job acp_scopes acp_socialissues work_socialaction -- tiers choices_list: civility kind profession category From 44d24155a1a8841bcdc7e73e0e223107a80ecc70 Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Thu, 16 Feb 2023 18:28:36 +0100 Subject: [PATCH 08/48] 8. complete chill_person_accompanying_period_closingmotive table --- sql/import.sql | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/sql/import.sql b/sql/import.sql index bb34096..f7285bc 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -70,10 +70,29 @@ INSERT INTO chill_person_household_position (id, label, sharehousehold, allowhol CROSS JOIN max_ordering WHERE NOT EXISTS ( SELECT 1 FROM chill_person_household_position WHERE label::jsonb = t.household_position1 ); +-- 8. Complete closingmotive +ALTER TABLE import.choix_periodes ADD COLUMN closingmotive1 JSONB; +UPDATE import.choix_periodes SET closingmotive1=json_build_object('fr', trim(closingmotive)) WHERE closingmotive!=''; +WITH max_ordering AS ( SELECT MAX(ordering) as max_ordering FROM chill_person_accompanying_period_closingmotive ) +INSERT INTO chill_person_accompanying_period_closingmotive (id, name, active, parent_id, ordering) + SELECT nextval('chill_person_accompanying_period_closingmotive_id_seq'), + t.closingmotive1, true, null, + max_ordering.max_ordering + row_number() OVER () as ordering + FROM ( SELECT DISTINCT ON (closingmotive) closingmotive1 FROM import.choix_periodes WHERE closingmotive1 IS NOT NULL ) t + CROSS JOIN max_ordering + WHERE NOT EXISTS ( SELECT 1 FROM chill_person_accompanying_period_closingmotive WHERE name::jsonb = t.closingmotive1 ); + + -- -- DOWN -- + +-- Undo 8. +DELETE FROM chill_person_accompanying_period_closingmotive cpapcm USING import.choix_periodes icp WHERE cpapcm.name::jsonb = icp.closingmotive1::jsonb; +SELECT setval('chill_person_accompanying_period_closingmotive_id_seq', (SELECT max(id) FROM chill_person_accompanying_period_closingmotive)); +ALTER TABLE import.choix_periodes DROP COLUMN closingmotive1; + -- Undo 7. DELETE FROM chill_person_household_position cphp USING import.choix_personnes icp WHERE cphp.label::jsonb = icp.household_position1::jsonb; SELECT setval('chill_person_household_position_id_seq', (SELECT max(id) FROM chill_person_household_position)); From 396444790f448e65381c711095482006f91dee67 Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Fri, 17 Feb 2023 09:43:37 +0100 Subject: [PATCH 09/48] 9. complete chill_person_accompanying_period_origin table --- sql/import.sql | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/sql/import.sql b/sql/import.sql index f7285bc..fc6cdf3 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -82,11 +82,22 @@ INSERT INTO chill_person_accompanying_period_closingmotive (id, name, active, pa CROSS JOIN max_ordering WHERE NOT EXISTS ( SELECT 1 FROM chill_person_accompanying_period_closingmotive WHERE name::jsonb = t.closingmotive1 ); +-- 9. Complete origin +ALTER TABLE import.choix_periodes ADD COLUMN origin1 JSONB; +UPDATE import.choix_periodes SET origin1=json_build_object('fr', trim(origin)) WHERE origin!=''; +INSERT INTO chill_person_accompanying_period_origin (id, label, noactiveafter) + SELECT nextval('chill_person_accompanying_period_origin_id_seq'), t.origin1, null + FROM ( SELECT DISTINCT ON (origin) origin1 FROM import.choix_periodes WHERE origin1 IS NOT NULL ) t + WHERE NOT EXISTS ( SELECT 1 FROM chill_person_accompanying_period_origin WHERE label::jsonb = t.origin1 ); -- -- DOWN -- +-- Undo 9. +DELETE FROM chill_person_accompanying_period_origin cpapo USING import.choix_periodes icp WHERE cpapo.label::jsonb = icp.origin1::jsonb; +SELECT setval('chill_person_accompanying_period_origin_id_seq', (SELECT max(id) FROM chill_person_accompanying_period_origin)); +ALTER TABLE import.choix_periodes DROP COLUMN origin1; -- Undo 8. DELETE FROM chill_person_accompanying_period_closingmotive cpapcm USING import.choix_periodes icp WHERE cpapcm.name::jsonb = icp.closingmotive1::jsonb; From 6d64060965d744ff91f5e27c6cb0046cd3e54428 Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Thu, 16 Feb 2023 19:42:34 +0100 Subject: [PATCH 10/48] 10. complete chill_main_user_job table --- sql/import.sql | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/sql/import.sql b/sql/import.sql index fc6cdf3..3720c8f 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -38,7 +38,7 @@ INSERT INTO chill_main_civility (id, name, abbreviation, active, ordering) ALTER TABLE import.choix_personnes ADD COLUMN country1 JSONB; UPDATE import.choix_personnes SET country1=json_build_object('fr', trim(country)) WHERE country!=''; INSERT INTO country (id, name, countrycode) - SELECT nextval('country_id_seq'), t.country1, 'ZZZ' + SELECT nextval('country_id_seq'), t.country1, 'ZZ' FROM ( SELECT DISTINCT ON (country) country1 FROM import.choix_personnes WHERE country1 IS NOT NULL ) t WHERE NOT EXISTS ( SELECT 1 FROM country WHERE name::jsonb = t.country1 ); @@ -90,10 +90,28 @@ INSERT INTO chill_person_accompanying_period_origin (id, label, noactiveafter) FROM ( SELECT DISTINCT ON (origin) origin1 FROM import.choix_periodes WHERE origin1 IS NOT NULL ) t WHERE NOT EXISTS ( SELECT 1 FROM chill_person_accompanying_period_origin WHERE label::jsonb = t.origin1 ); +-- 10. Complete user_job +ALTER TABLE import.choix_periodes ADD COLUMN job1 JSONB; +UPDATE import.choix_periodes SET job1=json_build_object('fr', trim(job)) WHERE job!=''; +INSERT INTO chill_main_user_job (id, label, active) + SELECT nextval('chill_main_user_job_id_seq'), t.job1, true + FROM ( SELECT DISTINCT ON (job) job1 FROM import.choix_periodes WHERE job1 IS NOT NULL ) t + WHERE NOT EXISTS( SELECT 1 FROM chill_main_user_job WHERE label::jsonb = t.job1 ); + + + + + -- -- DOWN -- + +-- Undo 10. +DELETE FROM chill_main_user_job cmuj USING import.choix_periodes icp WHERE cmuj.label::jsonb = icp.job1::jsonb; +SELECT setval('chill_main_user_job_id_seq', (SELECT max(id) FROM chill_main_user_job)); +ALTER TABLE import.choix_periodes DROP COLUMN job1; + -- Undo 9. DELETE FROM chill_person_accompanying_period_origin cpapo USING import.choix_periodes icp WHERE cpapo.label::jsonb = icp.origin1::jsonb; SELECT setval('chill_person_accompanying_period_origin_id_seq', (SELECT max(id) FROM chill_person_accompanying_period_origin)); From 7b4f04ef401e03c086b48b75a7f49ede58f002ee Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Fri, 17 Feb 2023 09:49:16 +0100 Subject: [PATCH 11/48] 11. complete scopes table --- sql/import.sql | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/sql/import.sql b/sql/import.sql index 3720c8f..e852b44 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -98,6 +98,13 @@ INSERT INTO chill_main_user_job (id, label, active) FROM ( SELECT DISTINCT ON (job) job1 FROM import.choix_periodes WHERE job1 IS NOT NULL ) t WHERE NOT EXISTS( SELECT 1 FROM chill_main_user_job WHERE label::jsonb = t.job1 ); +-- 11. Complete acp_scopes +ALTER TABLE import.choix_periodes ADD COLUMN acp_scopes1 JSONB; +UPDATE import.choix_periodes SET acp_scopes1=json_build_object('fr', trim(acp_scopes)) WHERE acp_scopes!=''; +INSERT INTO scopes (id, name, active) + SELECT nextval('scopes_id_seq'), t.acp_scopes1, true + FROM ( SELECT DISTINCT ON (acp_scopes) acp_scopes1 FROM import.choix_periodes WHERE acp_scopes1 IS NOT NULL ) t + WHERE NOT EXISTS ( SELECT 1 FROM scopes WHERE name::jsonb = t.acp_scopes1 ); @@ -106,6 +113,10 @@ INSERT INTO chill_main_user_job (id, label, active) -- DOWN -- +-- Undo 11. +DELETE FROM scopes USING import.choix_periodes icp WHERE scopes.name::jsonb = icp.acp_scopes1::jsonb; +SELECT setval('scopes_id_seq', (SELECT max(id) FROM scopes)); +ALTER TABLE import.choix_periodes DROP COLUMN acp_scopes1; -- Undo 10. DELETE FROM chill_main_user_job cmuj USING import.choix_periodes icp WHERE cmuj.label::jsonb = icp.job1::jsonb; From e14f6896e3abcfaf15c03b57db4e5980fdf40268 Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Fri, 17 Feb 2023 17:53:54 +0100 Subject: [PATCH 12/48] 12. complete users table (referrer) --- sql/import.sql | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/sql/import.sql b/sql/import.sql index e852b44..1be471d 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -106,13 +106,21 @@ INSERT INTO scopes (id, name, active) FROM ( SELECT DISTINCT ON (acp_scopes) acp_scopes1 FROM import.choix_periodes WHERE acp_scopes1 IS NOT NULL ) t WHERE NOT EXISTS ( SELECT 1 FROM scopes WHERE name::jsonb = t.acp_scopes1 ); - +-- 12. Complete referrers +INSERT INTO users (id, username, password, enabled, locked, attributes, label) + SELECT nextval('users_id_seq'), t.referrer, '', false, false, '[]', t.referrer + FROM ( SELECT DISTINCT ON (referrer) referrer FROM import.choix_periodes WHERE referrer <> '') t + WHERE NOT EXISTS ( SELECT 1 FROM users WHERE username::text = t.referrer ); -- -- DOWN -- +-- Undo 12. +DELETE FROM users USING import.choix_periodes icp WHERE users.username::text = icp.referrer::text AND users.password = '' AND users.enabled = false; +SELECT setval('users_id_seq', (SELECT COALESCE(max(id),1) FROM users)); + -- Undo 11. DELETE FROM scopes USING import.choix_periodes icp WHERE scopes.name::jsonb = icp.acp_scopes1::jsonb; SELECT setval('scopes_id_seq', (SELECT max(id) FROM scopes)); From cb12f3000bd3f8f46986f0f3543c019ba6d28658 Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Fri, 17 Feb 2023 19:15:55 +0100 Subject: [PATCH 13/48] 13. Complete chill_person_social_issue table --- sql/import.sql | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/sql/import.sql b/sql/import.sql index 1be471d..b33c8f5 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -112,11 +112,43 @@ INSERT INTO users (id, username, password, enabled, locked, attributes, label) FROM ( SELECT DISTINCT ON (referrer) referrer FROM import.choix_periodes WHERE referrer <> '') t WHERE NOT EXISTS ( SELECT 1 FROM users WHERE username::text = t.referrer ); +-- 13. Complete Social Issues +ALTER TABLE import.choix_periodes ADD COLUMN parent1 jsonb; +UPDATE import.choix_periodes SET parent1=json_build_object('fr', upper(trim(parent))) WHERE parent !=''; +ALTER TABLE import.choix_periodes ADD COLUMN enfant1 jsonb; +UPDATE import.choix_periodes SET enfant1=json_build_object('fr', upper(trim(enfant))) WHERE enfant !=''; +WITH max_ordering AS ( SELECT MAX(ordering) as max_ordering FROM chill_person_social_issue ) +INSERT INTO chill_person_social_issue (id, parent_id, title, ordering) + SELECT nextval('chill_person_social_issue_id_seq'), + null, t.parent1, + max_ordering.max_ordering + row_number() OVER () as ordering + FROM ( SELECT DISTINCT ON (parent) parent1 FROM import.choix_periodes WHERE parent1 IS NOT NULL ) t + CROSS JOIN max_ordering + WHERE NOT EXISTS ( SELECT 1 FROM chill_person_social_issue WHERE title::jsonb = t.parent1 ); +WITH max_ordering AS ( SELECT MAX(ordering) as max_ordering FROM chill_person_social_issue ) +INSERT INTO chill_person_social_issue (id, parent_id, title, ordering) + SELECT nextval('chill_person_social_issue_id_seq'), + ( SELECT id FROM chill_person_social_issue WHERE parent_id IS NULL AND title = t.parent1 ), t.enfant1, + max_ordering.max_ordering + row_number() OVER () as ordering + FROM ( SELECT DISTINCT ON (acp_socialissues) parent1, enfant1 FROM import.choix_periodes WHERE enfant1 IS NOT NULL ) t + CROSS JOIN max_ordering + WHERE NOT EXISTS ( SELECT 1 FROM chill_person_social_issue + WHERE title::jsonb = t.enfant1 + AND parent_id = (SELECT id FROM chill_person_social_issue WHERE title::jsonb = t.parent1)); + -- -- DOWN -- +-- Undo 13. +DELETE FROM chill_person_social_issue USING import.choix_periodes i + WHERE parent_id IN ( SELECT id FROM chill_person_social_issue cpsi WHERE cpsi.title::jsonb = i.parent1::jsonb AND cpsi.parent_id IS NULL ); +DELETE FROM chill_person_social_issue cpsi USING import.choix_periodes icp WHERE cpsi.title::jsonb = icp.parent1::jsonb AND cpsi.parent_id IS NULL ; +SELECT setval('chill_person_social_issue_id_seq', (SELECT COALESCE(max(id),1) FROM chill_person_social_issue)); +ALTER TABLE import.choix_periodes DROP COLUMN enfant1; +ALTER TABLE import.choix_periodes DROP COLUMN parent1; + -- Undo 12. DELETE FROM users USING import.choix_periodes icp WHERE users.username::text = icp.referrer::text AND users.password = '' AND users.enabled = false; SELECT setval('users_id_seq', (SELECT COALESCE(max(id),1) FROM users)); From 4dc09c559a2bada4480a320068ad43a208989239 Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Fri, 17 Feb 2023 17:29:12 +0100 Subject: [PATCH 14/48] fix setval with coalesce (reset id too if no rows) --- sql/import.sql | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/sql/import.sql b/sql/import.sql index b33c8f5..91428e5 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -155,32 +155,32 @@ SELECT setval('users_id_seq', (SELECT COALESCE(max(id),1) FROM users)); -- Undo 11. DELETE FROM scopes USING import.choix_periodes icp WHERE scopes.name::jsonb = icp.acp_scopes1::jsonb; -SELECT setval('scopes_id_seq', (SELECT max(id) FROM scopes)); +SELECT setval('scopes_id_seq', (SELECT COALESCE(max(id),1) FROM scopes)); ALTER TABLE import.choix_periodes DROP COLUMN acp_scopes1; -- Undo 10. DELETE FROM chill_main_user_job cmuj USING import.choix_periodes icp WHERE cmuj.label::jsonb = icp.job1::jsonb; -SELECT setval('chill_main_user_job_id_seq', (SELECT max(id) FROM chill_main_user_job)); +SELECT setval('chill_main_user_job_id_seq', (SELECT COALESCE(max(id),1) FROM chill_main_user_job)); ALTER TABLE import.choix_periodes DROP COLUMN job1; -- Undo 9. DELETE FROM chill_person_accompanying_period_origin cpapo USING import.choix_periodes icp WHERE cpapo.label::jsonb = icp.origin1::jsonb; -SELECT setval('chill_person_accompanying_period_origin_id_seq', (SELECT max(id) FROM chill_person_accompanying_period_origin)); +SELECT setval('chill_person_accompanying_period_origin_id_seq', (SELECT COALESCE(max(id),1) FROM chill_person_accompanying_period_origin)); ALTER TABLE import.choix_periodes DROP COLUMN origin1; -- Undo 8. DELETE FROM chill_person_accompanying_period_closingmotive cpapcm USING import.choix_periodes icp WHERE cpapcm.name::jsonb = icp.closingmotive1::jsonb; -SELECT setval('chill_person_accompanying_period_closingmotive_id_seq', (SELECT max(id) FROM chill_person_accompanying_period_closingmotive)); +SELECT setval('chill_person_accompanying_period_closingmotive_id_seq', (SELECT COALESCE(max(id),1) FROM chill_person_accompanying_period_closingmotive)); ALTER TABLE import.choix_periodes DROP COLUMN closingmotive1; -- Undo 7. DELETE FROM chill_person_household_position cphp USING import.choix_personnes icp WHERE cphp.label::jsonb = icp.household_position1::jsonb; -SELECT setval('chill_person_household_position_id_seq', (SELECT max(id) FROM chill_person_household_position)); +SELECT setval('chill_person_household_position_id_seq', (SELECT COALESCE(max(id),1) FROM chill_person_household_position)); ALTER TABLE import.choix_personnes DROP COLUMN household_position1; -- Undo 6. DELETE FROM chill_person_household_composition_type cphct USING import.choix_personnes icp WHERE cphct.label::jsonb = icp.household_composition_type1::jsonb AND cphct.id > 6; -SELECT setval('chill_person_household_composition_type_id_seq', (SELECT max(id) FROM chill_person_household_composition_type)); +SELECT setval('chill_person_household_composition_type_id_seq', (SELECT COALESCE(max(id),1) FROM chill_person_household_composition_type)); ALTER TABLE import.choix_personnes DROP COLUMN household_composition_type1; -- Undo 5. @@ -189,12 +189,12 @@ ALTER TABLE import.choix_personnes DROP COLUMN maritalstatus1; -- Undo 4. DELETE FROM country USING import.choix_personnes icp WHERE country.name::jsonb = icp.country1::jsonb AND country.id > 249; -- 249 existing countries -SELECT setval('country_id_seq', (SELECT max(id) FROM country)); +SELECT setval('country_id_seq', (SELECT COALESCE(max(id),1) FROM country)); ALTER TABLE import.choix_personnes DROP COLUMN country1; -- Undo 3. DELETE FROM chill_main_civility cmc USING import.choix_personnes ip WHERE cmc.name::jsonb = ip.civility1::jsonb; -SELECT setval('chill_main_civility_id_seq', (SELECT max(id) FROM chill_main_civility)); +SELECT setval('chill_main_civility_id_seq', (SELECT COALESCE(max(id),1) FROM chill_main_civility)); ALTER TABLE import.choix_personnes DROP COLUMN civility1; -- Undo 2. From dd89aa6ff1f762c394309708e3fc4d64d6e895e5 Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Fri, 17 Feb 2023 19:42:14 +0100 Subject: [PATCH 15/48] improve title comment block --- sql/import.sql | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/sql/import.sql b/sql/import.sql index 91428e5..1f2440a 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -42,7 +42,7 @@ INSERT INTO country (id, name, countrycode) FROM ( SELECT DISTINCT ON (country) country1 FROM import.choix_personnes WHERE country1 IS NOT NULL ) t WHERE NOT EXISTS ( SELECT 1 FROM country WHERE name::jsonb = t.country1 ); --- 5. Complete table maritalstatus +-- 5. Complete table marital_status ALTER TABLE import.choix_personnes ADD COLUMN maritalstatus1 JSONB; UPDATE import.choix_personnes SET maritalstatus1=json_build_object('fr', trim(maritalstatus)) WHERE maritalstatus!=''; INSERT INTO chill_person_marital_status (id, name) @@ -70,7 +70,7 @@ INSERT INTO chill_person_household_position (id, label, sharehousehold, allowhol CROSS JOIN max_ordering WHERE NOT EXISTS ( SELECT 1 FROM chill_person_household_position WHERE label::jsonb = t.household_position1 ); --- 8. Complete closingmotive +-- 8. Complete table accompanying_period_closingmotive ALTER TABLE import.choix_periodes ADD COLUMN closingmotive1 JSONB; UPDATE import.choix_periodes SET closingmotive1=json_build_object('fr', trim(closingmotive)) WHERE closingmotive!=''; WITH max_ordering AS ( SELECT MAX(ordering) as max_ordering FROM chill_person_accompanying_period_closingmotive ) @@ -82,7 +82,7 @@ INSERT INTO chill_person_accompanying_period_closingmotive (id, name, active, pa CROSS JOIN max_ordering WHERE NOT EXISTS ( SELECT 1 FROM chill_person_accompanying_period_closingmotive WHERE name::jsonb = t.closingmotive1 ); --- 9. Complete origin +-- 9. Complete table accompanying_period_origin ALTER TABLE import.choix_periodes ADD COLUMN origin1 JSONB; UPDATE import.choix_periodes SET origin1=json_build_object('fr', trim(origin)) WHERE origin!=''; INSERT INTO chill_person_accompanying_period_origin (id, label, noactiveafter) @@ -90,7 +90,7 @@ INSERT INTO chill_person_accompanying_period_origin (id, label, noactiveafter) FROM ( SELECT DISTINCT ON (origin) origin1 FROM import.choix_periodes WHERE origin1 IS NOT NULL ) t WHERE NOT EXISTS ( SELECT 1 FROM chill_person_accompanying_period_origin WHERE label::jsonb = t.origin1 ); --- 10. Complete user_job +-- 10. Complete table user_job ALTER TABLE import.choix_periodes ADD COLUMN job1 JSONB; UPDATE import.choix_periodes SET job1=json_build_object('fr', trim(job)) WHERE job!=''; INSERT INTO chill_main_user_job (id, label, active) @@ -98,7 +98,7 @@ INSERT INTO chill_main_user_job (id, label, active) FROM ( SELECT DISTINCT ON (job) job1 FROM import.choix_periodes WHERE job1 IS NOT NULL ) t WHERE NOT EXISTS( SELECT 1 FROM chill_main_user_job WHERE label::jsonb = t.job1 ); --- 11. Complete acp_scopes +-- 11. Complete table Scopes ALTER TABLE import.choix_periodes ADD COLUMN acp_scopes1 JSONB; UPDATE import.choix_periodes SET acp_scopes1=json_build_object('fr', trim(acp_scopes)) WHERE acp_scopes!=''; INSERT INTO scopes (id, name, active) @@ -106,13 +106,13 @@ INSERT INTO scopes (id, name, active) FROM ( SELECT DISTINCT ON (acp_scopes) acp_scopes1 FROM import.choix_periodes WHERE acp_scopes1 IS NOT NULL ) t WHERE NOT EXISTS ( SELECT 1 FROM scopes WHERE name::jsonb = t.acp_scopes1 ); --- 12. Complete referrers +-- 12. Complete table Users (acp referrer) INSERT INTO users (id, username, password, enabled, locked, attributes, label) SELECT nextval('users_id_seq'), t.referrer, '', false, false, '[]', t.referrer FROM ( SELECT DISTINCT ON (referrer) referrer FROM import.choix_periodes WHERE referrer <> '') t WHERE NOT EXISTS ( SELECT 1 FROM users WHERE username::text = t.referrer ); --- 13. Complete Social Issues +-- 13. Complete table SocialIssues ALTER TABLE import.choix_periodes ADD COLUMN parent1 jsonb; UPDATE import.choix_periodes SET parent1=json_build_object('fr', upper(trim(parent))) WHERE parent !=''; ALTER TABLE import.choix_periodes ADD COLUMN enfant1 jsonb; @@ -136,11 +136,15 @@ INSERT INTO chill_person_social_issue (id, parent_id, title, ordering) WHERE title::jsonb = t.enfant1 AND parent_id = (SELECT id FROM chill_person_social_issue WHERE title::jsonb = t.parent1)); +-- 14. Complete table WorkSocialActions + -- -- DOWN -- +-- Undo 14. + -- Undo 13. DELETE FROM chill_person_social_issue USING import.choix_periodes i WHERE parent_id IN ( SELECT id FROM chill_person_social_issue cpsi WHERE cpsi.title::jsonb = i.parent1::jsonb AND cpsi.parent_id IS NULL ); @@ -209,7 +213,7 @@ ALTER TABLE import.periodes DROP COLUMN closingdate1; -- ------------- -- personnes choices_list: gender --- periodes choices_list: intensity referrer job acp_scopes acp_socialissues work_socialaction +-- periodes choices_list: intensity -- tiers choices_list: civility kind profession category -- From b6f26900a74bee336dd43ddcad07e4e152f98697 Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Sun, 19 Feb 2023 14:58:11 +0100 Subject: [PATCH 16/48] 20-32. Prepare import personnes and periodes tables --- sql/import.sql | 188 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 180 insertions(+), 8 deletions(-) diff --git a/sql/import.sql b/sql/import.sql index 1f2440a..b5a8cb4 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -139,10 +139,189 @@ INSERT INTO chill_person_social_issue (id, parent_id, title, ordering) -- 14. Complete table WorkSocialActions +-- 20. Prepare personnes civility +ALTER TABLE import.personnes ADD COLUMN civility1 jsonb; +UPDATE import.personnes SET civility1=json_build_object('fr', trim(civility)) WHERE civility!=''; +-- SELECT i.civility, c.name as civility_name, c.id as civility_id FROM import.personnes i JOIN chill_main_civility c ON i.civility1::jsonb = c.name::jsonb; + +-- 21. Prepare personnes gender +ALTER TABLE import.personnes ADD COLUMN gender1 VARCHAR; +UPDATE import.personnes SET gender1 = CASE + WHEN trim(gender) IN ('Femme', 'femme', 'Woman', 'woman', 'Female', 'female') THEN 'woman' + WHEN trim(gender) IN ('Homme', 'homme', 'Man', 'man', 'Male', 'male') THEN 'man' + WHEN trim(gender) IN ('', 'Inconnu', 'inconnu', 'Unknown', 'unknown') THEN 'unknown' +ELSE 'both' END; + +-- 22. Prepare personnes nationality +ALTER TABLE import.personnes ADD COLUMN nationality1 jsonb; +UPDATE import.personnes SET nationality1=json_build_object('fr', trim(nationality)) WHERE nationality!=''; +-- SELECT i.nationality, c.name as country_name, c.id as country_id FROM import.personnes i JOIN country c ON i.nationality1::jsonb = c.name::jsonb; + +-- 23. Prepare personnes countryofbirth +ALTER TABLE import.personnes ADD COLUMN countryofbirth1 jsonb; +UPDATE import.personnes SET countryofbirth1=json_build_object('fr', trim(countryofbirth)) WHERE countryofbirth!=''; +-- SELECT i.countryofbirth, c.name as country_name, c.id as country_id FROM import.personnes i JOIN country c ON i.countryofbirth1::jsonb = c.name::jsonb; + +-- 24. Prepare personnes maritalstatus +ALTER TABLE import.personnes ADD COLUMN maritalstatus1 jsonb; +UPDATE import.personnes SET maritalstatus1=json_build_object('fr', trim(maritalstatus)) WHERE maritalstatus!=''; +-- SELECT i.maritalstatus, ms.name as maritalstatus_name, ms.id as maritalstatus_id FROM import.personnes i JOIN chill_person_marital_status ms ON i.maritalstatus1::jsonb = ms.name::jsonb; + +-- 25. Prepare personnes numberofchildren +ALTER TABLE import.personnes ADD COLUMN numberofchildren1 integer; +UPDATE import.personnes SET numberofchildren1=to_number(trim(numberofchildren::text) ) WHERE numberofchildren!=''; -- <== case BOUM + + +-- 26. Prepare personnes household_composition_type +ALTER TABLE import.personnes ADD COLUMN household_composition_type1 JSONB; +UPDATE import.personnes SET household_composition_type1=json_build_object('fr', trim(household_composition_type)) WHERE household_composition_type!=''; +-- SELECT i.household_composition_type, cphct.id as household_compo_type_id, cphct.label as household_compo_type_label FROM import.personnes i JOIN chill_person_household_composition_type cphct ON i.household_composition_type1::jsonb = cphct.label::jsonb; + +-- 27. Prepare personnes household_position +ALTER TABLE import.personnes ADD COLUMN household_position1 jsonb; +UPDATE import.personnes SET household_position1=json_build_object('fr', trim(household_position)) WHERE household_position!=''; +-- SELECT i.household_position, hp.label as household_position_label, hp.id as household_position_id FROM import.personnes i JOIN chill_person_household_position hp ON i.household_position1::jsonb = hp.label::jsonb; + +-- 28. Prepare periodes closingmotive +ALTER TABLE import.periodes ADD COLUMN closingmotive1 JSONB; +UPDATE import.periodes SET closingmotive1=json_build_object('fr', trim(closingmotive)) WHERE closingmotive!=''; +-- SELECT i.closingmotive1, cm.id as closingmotive_id, cm.name as closingmotive_name FROM import.periodes i JOIN chill_person_accompanying_period_closingmotive cm ON i.closingmotive1::jsonb = cm.name::jsonb; + +-- 29. Prepare periodes origin +ALTER TABLE import.periodes ADD COLUMN origin1 jsonb; +UPDATE import.periodes SET origin1=json_build_object('fr', trim(origin)) WHERE origin!=''; +-- SELECT i.origin1, o.id as origin_id, o.label as origin_label FROM import.periodes i JOIN chill_person_accompanying_period_origin o ON i.origin1::jsonb = o.label::jsonb; + +-- 30. Prepare periodes job +ALTER TABLE import.periodes ADD COLUMN job1 jsonb; +UPDATE import.periodes SET job1=json_build_object('fr', trim(job)) WHERE job!=''; +-- SELECT i.job1, o.id as job_id, o.label as job_label FROM import.periodes i JOIN chill_main_user_job o ON i.job1::jsonb = o.label::jsonb; + +-- 31. Prepare periodes acp_scopes +ALTER TABLE import.periodes ADD COLUMN acp_scopes1 jsonb; +UPDATE import.periodes SET acp_scopes1=json_build_object('fr', trim(acp_scopes)) WHERE acp_scopes!=''; +-- SELECT i.acp_scopes1, s.id as scopes_id, s.name as scopes_name FROM import.periodes i JOIN scopes s ON i.acp_scopes1::jsonb = s.name::jsonb; + +-- 32. Prepare periodes intensity +ALTER TABLE import.periodes ADD COLUMN intensity1 text; +UPDATE import.periodes SET intensity1='occasional'; UPDATE import.periodes SET intensity1='regular' WHERE intensity='regular'; + + +-- 33. Prepare required default dates +-- address_validfrom1 | household_startdate1 +-- acp_openingdate1 + +-- 34. Format phone numbers +-- phonenumber | mobilenumber + +-- 40. insert in chill_person_person +INSERT INTO chill_person_person ( + id + , nationality_id, countryofbirth_id, civility_id, maritalstatus_id + , firstname, lastname + , birthdate, place_of_birth + , memo, email, contactinfo + , phonenumber, mobilenumber + , numberofchildren + , gender + , deathdate + /* + maritalstatusdate + acceptsms + acceptemail + gendercomment_comment + gendercomment_userid + gendercomment_date + maritalstatuscomment_comment + maritalstatuscomment_userid + maritalstatuscomment_date + createdat + updatedat + createdby_id + updatedby_id + center_id + */ +) SELECT + id -- id + , (SELECT c.id as country_id FROM import.personnes i JOIN country c ON i.nationality1::jsonb = c.name::jsonb) -- nationality_id, + , (SELECT c.id as country_id FROM import.personnes i JOIN country c ON i.countryofbirth1::jsonb = c.name::jsonb) -- countryofbirth_id, + , (SELECT c.id as civility_id FROM import.personnes i JOIN chill_main_civility c ON i.civility1::jsonb = c.name::jsonb) -- civility_id + , (SELECT ms.id as maritalstatus_id FROM import.personnes i JOIN chill_person_marital_status ms ON i.maritalstatus1::jsonb = ms.name::jsonb) -- maritalstatus_id, + , firstname, lastname -- firstname, -- lastname, + , birthdate1, place_of_birth -- birthdate, -- place_of_birth, + , memo, email, contactinfo -- memo, -- email, -- contactinfo, + , phonenumber , mobilenumber -- phonenumber, -- mobilenumber, + , numberofchildren1 -- numberofchildren, + , gender1 -- gender, + , deathdate1 -- deathdate, + FROM import.personnes; + /* + -- maritalstatusdate, + -- acceptsms, + -- acceptemail, + -- gendercomment_comment, + -- gendercomment_userid, + -- gendercomment_date, + -- maritalstatuscomment_comment, + -- maritalstatuscomment_userid, + -- maritalstatuscomment_date, + -- createdat, + -- updatedat, + -- createdby_id, + -- updatedby_id, + -- center_id, + */ + +-- lier civility, +-- country, adresse, +-- ajouter dans chill_person_accompanying_course + + + + -- -- DOWN -- +-- Undo 32. +ALTER TABLE import.periodes DROP COLUMN intensity1; + +-- Undo 31. +ALTER TABLE import.periodes DROP COLUMN acp_scopes1; + +-- Undo 30. +ALTER TABLE import.periodes DROP COLUMN job1; + +-- Undo 29. +ALTER TABLE import.periodes DROP COLUMN origin1; + +-- Undo 28. +ALTER TABLE import.periodes DROP COLUMN closingmotive1; + +-- Undo 27. +ALTER TABLE import.personnes DROP COLUMN household_position1; + +-- Undo 26. +ALTER TABLE import.personnes DROP COLUMN household_composition_type1; + +-- Undo 25. +ALTER TABLE import.personnes DROP COLUMN numberofchildren1; + +-- Undo 24. +ALTER TABLE import.personnes DROP COLUMN maritalstatus1; + +-- Undo 23. +ALTER TABLE import.personnes DROP COLUMN countryofbirth1; + +-- Undo 22. +ALTER TABLE import.personnes DROP COLUMN nationality1; + +-- Undo 21. +ALTER TABLE import.personnes DROP COLUMN gender1; + +-- Undo 20. +ALTER TABLE import.personnes DROP COLUMN civility1; + -- Undo 14. -- Undo 13. @@ -211,13 +390,6 @@ ALTER TABLE import.periodes DROP COLUMN closingdate1; -- ------------- - --- personnes choices_list: gender --- periodes choices_list: intensity -- tiers choices_list: civility kind profession category --- --- SELECT --- -SELECT DISTINCT civility FROM import.personnes; -SELECT * FROM import.periodes; + From a36ee22be6190908c573418b9f0e5ca5be32f6a0 Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Wed, 1 Mar 2023 17:34:47 +0100 Subject: [PATCH 17/48] fix numberofchildren cast --- sql/import.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sql/import.sql b/sql/import.sql index b5a8cb4..7aea248 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -169,7 +169,8 @@ UPDATE import.personnes SET maritalstatus1=json_build_object('fr', trim(maritals -- 25. Prepare personnes numberofchildren ALTER TABLE import.personnes ADD COLUMN numberofchildren1 integer; -UPDATE import.personnes SET numberofchildren1=to_number(trim(numberofchildren::text) ) WHERE numberofchildren!=''; -- <== case BOUM +UPDATE import.personnes SET numberofchildren1=NULLIF(numberofchildren, '')::int; +-- SELECT numberofchildren, numberofchildren1 FROM import.personnes; -- 26. Prepare personnes household_composition_type From 25510c1efe243ed823db4a159e500582c5a0b12a Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Wed, 1 Mar 2023 19:17:09 +0100 Subject: [PATCH 18/48] fix query for person insertion --- sql/import.sql | 88 +++++++++++++++++++++----------------------------- 1 file changed, 37 insertions(+), 51 deletions(-) diff --git a/sql/import.sql b/sql/import.sql index 7aea248..ce8b604 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -172,7 +172,6 @@ ALTER TABLE import.personnes ADD COLUMN numberofchildren1 integer; UPDATE import.personnes SET numberofchildren1=NULLIF(numberofchildren, '')::int; -- SELECT numberofchildren, numberofchildren1 FROM import.personnes; - -- 26. Prepare personnes household_composition_type ALTER TABLE import.personnes ADD COLUMN household_composition_type1 JSONB; UPDATE import.personnes SET household_composition_type1=json_build_object('fr', trim(household_composition_type)) WHERE household_composition_type!=''; @@ -215,65 +214,45 @@ UPDATE import.periodes SET intensity1='occasional'; UPDATE import.periodes SET i -- 34. Format phone numbers -- phonenumber | mobilenumber --- 40. insert in chill_person_person +-- 40. Prepare id mapping before insertion +ALTER TABLE import.personnes ADD column person_id BIGINT; +UPDATE import.personnes SET person_id=personid + FROM (SELECT id, nextval('chill_person_person_id_seq') AS personid FROM import.personnes ORDER BY id) AS t + WHERE import.personnes.id = t.id; + +-- 41. Insert in chill_person_person INSERT INTO chill_person_person ( id , nationality_id, countryofbirth_id, civility_id, maritalstatus_id - , firstname, lastname - , birthdate, place_of_birth - , memo, email, contactinfo - , phonenumber, mobilenumber - , numberofchildren - , gender - , deathdate + , firstname, lastname, birthdate, place_of_birth, memo, email, contactinfo, phonenumber, mobilenumber, numberofchildren, gender, deathdate, proxyaccompanyingperiodopenstate + , createdat, updatedat, createdby_id, updatedby_id + , center_id /* - maritalstatusdate - acceptsms - acceptemail - gendercomment_comment - gendercomment_userid - gendercomment_date - maritalstatuscomment_comment - maritalstatuscomment_userid - maritalstatuscomment_date - createdat - updatedat - createdby_id - updatedby_id - center_id + , maritalstatusdate + , maritalstatuscomment_comment, maritalstatuscomment_userid, maritalstatuscomment_date + , gendercomment_comment, gendercomment_userid, gendercomment_date + , acceptsms, acceptemail */ ) SELECT - id -- id - , (SELECT c.id as country_id FROM import.personnes i JOIN country c ON i.nationality1::jsonb = c.name::jsonb) -- nationality_id, - , (SELECT c.id as country_id FROM import.personnes i JOIN country c ON i.countryofbirth1::jsonb = c.name::jsonb) -- countryofbirth_id, - , (SELECT c.id as civility_id FROM import.personnes i JOIN chill_main_civility c ON i.civility1::jsonb = c.name::jsonb) -- civility_id - , (SELECT ms.id as maritalstatus_id FROM import.personnes i JOIN chill_person_marital_status ms ON i.maritalstatus1::jsonb = ms.name::jsonb) -- maritalstatus_id, - , firstname, lastname -- firstname, -- lastname, - , birthdate1, place_of_birth -- birthdate, -- place_of_birth, - , memo, email, contactinfo -- memo, -- email, -- contactinfo, - , phonenumber , mobilenumber -- phonenumber, -- mobilenumber, - , numberofchildren1 -- numberofchildren, - , gender1 -- gender, - , deathdate1 -- deathdate, - FROM import.personnes; + person_id -- id + , (SELECT c.id FROM country c WHERE c.name::jsonb = ip.nationality1::jsonb AND ip.nationality1 IS NOT NULL) -- nationality_id, + , (SELECT c.id FROM country c WHERE c.name::jsonb = ip.countryofbirth1::jsonb AND ip.countryofbirth1 IS NOT NULL ) -- countryofbirth_id, + , (SELECT c.id FROM chill_main_civility c WHERE c.name::jsonb = ip.civility1::jsonb AND ip.civility1 IS NOT NULL ) -- civility_id + , (SELECT ms.id FROM chill_person_marital_status ms WHERE ms.name::jsonb = ip.maritalstatus1::jsonb AND ip.maritalstatus1 IS NOT NULL ) -- maritalstatus_id, + , TRIM(firstname), UPPER(TRIM(lastname)), birthdate1, TRIM(place_of_birth), TRIM(memo), TRIM(email), TRIM(contactinfo), phonenumber1, mobilenumber1, numberofchildren1, gender1, deathdate1, false + , CURRENT_DATE, CURRENT_DATE -- createdat, updatedat + , (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM users), (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM users) -- createdby_id, updatedby_id + , (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM centers) -- center_id /* - -- maritalstatusdate, - -- acceptsms, - -- acceptemail, - -- gendercomment_comment, - -- gendercomment_userid, - -- gendercomment_date, - -- maritalstatuscomment_comment, - -- maritalstatuscomment_userid, - -- maritalstatuscomment_date, - -- createdat, - -- updatedat, - -- createdby_id, - -- updatedby_id, - -- center_id, + , -- maritalstatusdate + , -- maritalstatuscomment_comment, maritalstatuscomment_userid, maritalstatuscomment_date + , -- gendercomment_comment, gendercomment_userid, gendercomment_date + , -- acceptsms, acceptemail */ +FROM import.personnes ip; + + --- lier civility, -- country, adresse, -- ajouter dans chill_person_accompanying_course @@ -284,6 +263,13 @@ INSERT INTO chill_person_person ( -- DOWN -- +-- Undo 41. +DELETE FROM chill_person_person cpp USING import.personnes ip WHERE cpp.id = ip.person_id; +SELECT setval('chill_person_person_id_seq', (SELECT max(id) FROM chill_person_person)); + +-- Undo 40. +ALTER TABLE import.personnes DROP COLUMN person_id; + -- Undo 32. ALTER TABLE import.periodes DROP COLUMN intensity1; From 139dd6209a13eee1a1c8c9b9550b3560c3ea15b0 Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Wed, 1 Mar 2023 20:57:25 +0100 Subject: [PATCH 19/48] 33-34. Format phone numbers --- sql/import.sql | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/sql/import.sql b/sql/import.sql index ce8b604..7bacc94 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -6,6 +6,7 @@ -- * Avant de migrer (UP), il faut d'abord importer les codes postaux !!! -- + -- 1. Adjust Type cast for id columns ALTER TABLE import.personnes ALTER COLUMN id TYPE INTEGER USING (id::integer); ALTER TABLE import.periodes ALTER COLUMN id TYPE INTEGER USING (id::integer); @@ -206,13 +207,24 @@ UPDATE import.periodes SET acp_scopes1=json_build_object('fr', trim(acp_scopes)) ALTER TABLE import.periodes ADD COLUMN intensity1 text; UPDATE import.periodes SET intensity1='occasional'; UPDATE import.periodes SET intensity1='regular' WHERE intensity='regular'; +-- 33. Format mobile numbers +ALTER TABLE import.personnes ADD column mobilenumber1 text; +UPDATE import.personnes SET mobilenumber1=NULLIF(regexp_replace(mobilenumber, '[^0-9]', '', 'g'), ''); -- remove all NaN chars +UPDATE import.personnes SET mobilenumber1=regexp_replace(mobilenumber1, '^0', '') WHERE mobilenumber1 like '0%'; -- remove first 0 prefix +-- UPDATE import.personnes SET mobilenumber1=regexp_replace(mobilenumber1, '(.*)', '+32\1'); -- add belgium intl prefix --- 33. Prepare required default dates +-- 34. Format phone numbers +ALTER TABLE import.personnes ADD column phonenumber1 text; +UPDATE import.personnes SET phonenumber1=NULLIF(regexp_replace(phonenumber, '[^0-9]', '', 'g'), ''); -- remove all NaN chars +UPDATE import.personnes SET phonenumber1=regexp_replace(phonenumber1, '^0', '') WHERE phonenumber1 like '0%'; -- remove first 0 prefix +-- UPDATE import.personnes SET phonenumber1=regexp_replace(phonenumber1, '(.*)', '+32\1'); -- add belgium intl prefix +-- SELECT mobilenumber, mobilenumber1, phonenumber, phonenumber1 FROM import.personnes; + + +-- 35. Prepare required default dates -- address_validfrom1 | household_startdate1 -- acp_openingdate1 --- 34. Format phone numbers --- phonenumber | mobilenumber -- 40. Prepare id mapping before insertion ALTER TABLE import.personnes ADD column person_id BIGINT; @@ -270,6 +282,12 @@ SELECT setval('chill_person_person_id_seq', (SELECT max(id) FROM chill_person_pe -- Undo 40. ALTER TABLE import.personnes DROP COLUMN person_id; +-- Undo 34. +ALTER TABLE import.personnes DROP column phonenumber1; + +-- Undo 33. +ALTER TABLE import.personnes DROP column mobilenumber1; + -- Undo 32. ALTER TABLE import.periodes DROP COLUMN intensity1; From 1a6f3953caa7c9e0b352cd1999745fc07c3fcbea Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Wed, 1 Mar 2023 21:20:19 +0100 Subject: [PATCH 20/48] 42. Add address, and link it to person --- sql/import.sql | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/sql/import.sql b/sql/import.sql index 7bacc94..8dda47e 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -263,9 +263,17 @@ INSERT INTO chill_person_person ( */ FROM import.personnes ip; +-- 42. Add address, and link it to person +ALTER TABLE import.personnes ADD column address_id BIGINT; +UPDATE import.personnes SET address_id = nextval('chill_main_address_id_seq'); +ALTER TABLE import.personnes ADD column postcode_id BIGINT; +UPDATE import.personnes SET postcode_id = cmpc.id FROM chill_main_postal_code cmpc WHERE cmpc.code = postcode AND cmpc.origin = 0; +INSERT INTO chill_main_address (id, postcode_id, street, streetnumber, extra, validFrom) SELECT + address_id, postcode_id, street, streetnumber, extra, CURRENT_DATE +FROM import.personnes WHERE postcode_id IS NOT NULL; + --- country, adresse, -- ajouter dans chill_person_accompanying_course @@ -275,6 +283,12 @@ FROM import.personnes ip; -- DOWN -- +-- Undo 42. +DELETE FROM chill_main_address cma USING import.personnes ip WHERE cma.id = ip.address_id; +SELECT setval('chill_main_address_id_seq', (SELECT max(id) FROM chill_main_address)); +ALTER TABLE import.personnes DROP column postcode_id; +ALTER TABLE import.personnes DROP column address_id; + -- Undo 41. DELETE FROM chill_person_person cpp USING import.personnes ip WHERE cpp.id = ip.person_id; SELECT setval('chill_person_person_id_seq', (SELECT max(id) FROM chill_person_person)); From f3170d0c8733a723cad86daaa2300146bfbd1eb2 Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Wed, 8 Mar 2023 17:49:13 +0100 Subject: [PATCH 21/48] cleaning --- sql/import.sql | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/sql/import.sql b/sql/import.sql index 8dda47e..aec7a20 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -3,7 +3,7 @@ -- version v0.6 (== version canevas) -- -- /!\ IMPORTANT --- * Avant de migrer (UP), il faut d'abord importer les codes postaux !!! +-- Avant de migrer (UP), il faut d'abord avoir importé les codes postaux !!! -- @@ -138,7 +138,7 @@ INSERT INTO chill_person_social_issue (id, parent_id, title, ordering) AND parent_id = (SELECT id FROM chill_person_social_issue WHERE title::jsonb = t.parent1)); -- 14. Complete table WorkSocialActions - +-- (not yet implemented in canvas) -- 20. Prepare personnes civility ALTER TABLE import.personnes ADD COLUMN civility1 jsonb; @@ -270,14 +270,13 @@ ALTER TABLE import.personnes ADD column postcode_id BIGINT; UPDATE import.personnes SET postcode_id = cmpc.id FROM chill_main_postal_code cmpc WHERE cmpc.code = postcode AND cmpc.origin = 0; INSERT INTO chill_main_address (id, postcode_id, street, streetnumber, extra, validFrom) SELECT address_id, postcode_id, street, streetnumber, extra, CURRENT_DATE -FROM import.personnes WHERE postcode_id IS NOT NULL; - - - --- ajouter dans chill_person_accompanying_course + FROM import.personnes WHERE postcode_id IS NOT NULL; +SELECT ip.person_id, ip.address_id FROM import.personnes ip ; +-- SELECT ip.id, ip.person_id, ip.firstname, ip.lastname, addr.* FROM chill_main_address addr RIGHT JOIN import.personnes ip ON addr.id = ip.address_id ORDER BY ip.id; +-- ========================================================================================= -- -- -- DOWN From 838bcb78ff162a911d43b5af8d6cb373af55a53b Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Wed, 8 Mar 2023 21:04:09 +0100 Subject: [PATCH 22/48] 33-34 Fix phone format --- sql/import.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/import.sql b/sql/import.sql index aec7a20..0b51b74 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -211,13 +211,13 @@ UPDATE import.periodes SET intensity1='occasional'; UPDATE import.periodes SET i ALTER TABLE import.personnes ADD column mobilenumber1 text; UPDATE import.personnes SET mobilenumber1=NULLIF(regexp_replace(mobilenumber, '[^0-9]', '', 'g'), ''); -- remove all NaN chars UPDATE import.personnes SET mobilenumber1=regexp_replace(mobilenumber1, '^0', '') WHERE mobilenumber1 like '0%'; -- remove first 0 prefix --- UPDATE import.personnes SET mobilenumber1=regexp_replace(mobilenumber1, '(.*)', '+32\1'); -- add belgium intl prefix +UPDATE import.personnes SET mobilenumber1=regexp_replace(mobilenumber1, '(.*)', '+32\1'); -- add belgium intl prefix -- 34. Format phone numbers ALTER TABLE import.personnes ADD column phonenumber1 text; UPDATE import.personnes SET phonenumber1=NULLIF(regexp_replace(phonenumber, '[^0-9]', '', 'g'), ''); -- remove all NaN chars UPDATE import.personnes SET phonenumber1=regexp_replace(phonenumber1, '^0', '') WHERE phonenumber1 like '0%'; -- remove first 0 prefix --- UPDATE import.personnes SET phonenumber1=regexp_replace(phonenumber1, '(.*)', '+32\1'); -- add belgium intl prefix +UPDATE import.personnes SET phonenumber1=regexp_replace(phonenumber1, '(.*)', '+32\1'); -- add belgium intl prefix -- SELECT mobilenumber, mobilenumber1, phonenumber, phonenumber1 FROM import.personnes; From 3978ee8b1a19a9272b8379e3adf5447b905e2280 Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Wed, 8 Mar 2023 21:05:06 +0100 Subject: [PATCH 23/48] Fix WHERE jsonb comparison --- sql/import.sql | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/sql/import.sql b/sql/import.sql index 0b51b74..4a68df9 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -33,7 +33,7 @@ INSERT INTO chill_main_civility (id, name, abbreviation, active, ordering) max_ordering.max_ordering + row_number() OVER () as ordering FROM ( SELECT DISTINCT ON (civility) civility1 FROM import.choix_personnes WHERE civility1 IS NOT NULL ) t CROSS JOIN max_ordering - WHERE NOT EXISTS ( SELECT 1 FROM chill_main_civility WHERE name::jsonb = t.civility1 ); + WHERE NOT EXISTS ( SELECT 1 FROM chill_main_civility WHERE name::jsonb->>'fr' = t.civility1::jsonb->>'fr' ); -- 4. Complete table Country ALTER TABLE import.choix_personnes ADD COLUMN country1 JSONB; @@ -41,7 +41,7 @@ UPDATE import.choix_personnes SET country1=json_build_object('fr', trim(country) INSERT INTO country (id, name, countrycode) SELECT nextval('country_id_seq'), t.country1, 'ZZ' FROM ( SELECT DISTINCT ON (country) country1 FROM import.choix_personnes WHERE country1 IS NOT NULL ) t - WHERE NOT EXISTS ( SELECT 1 FROM country WHERE name::jsonb = t.country1 ); + WHERE NOT EXISTS ( SELECT 1 FROM country WHERE name::jsonb->>'fr' = t.country1::jsonb->>'fr' ); -- 5. Complete table marital_status ALTER TABLE import.choix_personnes ADD COLUMN maritalstatus1 JSONB; @@ -49,7 +49,7 @@ UPDATE import.choix_personnes SET maritalstatus1=json_build_object('fr', trim(ma INSERT INTO chill_person_marital_status (id, name) SELECT substr(md5(random()::text), 1, 7), t.maritalstatus1 FROM ( SELECT DISTINCT ON (maritalstatus) maritalstatus1 FROM import.choix_personnes WHERE maritalstatus1 IS NOT NULL ) t - WHERE NOT EXISTS ( SELECT 1 FROM chill_person_marital_status WHERE name::jsonb = t.maritalstatus1 ); + WHERE NOT EXISTS ( SELECT 1 FROM chill_person_marital_status WHERE name::jsonb->>'fr' = t.maritalstatus1::jsonb->>'fr' ); -- 6. Complete table household_composition_type ALTER TABLE import.choix_personnes ADD COLUMN household_composition_type1 JSONB; @@ -57,7 +57,7 @@ UPDATE import.choix_personnes SET household_composition_type1=json_build_object( INSERT INTO chill_person_household_composition_type (id, active, label) SELECT nextval('chill_person_household_composition_type_id_seq'), true, t.household_composition_type1 FROM ( SELECT DISTINCT ON (household_composition_type) household_composition_type1 FROM import.choix_personnes WHERE household_composition_type1 IS NOT NULL ) t - WHERE NOT EXISTS ( SELECT 1 FROM chill_person_household_composition_type WHERE label::jsonb = t.household_composition_type1 ); + WHERE NOT EXISTS ( SELECT 1 FROM chill_person_household_composition_type WHERE label::jsonb->>'fr' = t.household_composition_type1::jsonb->>'fr' ); -- 7. Complete table household_position ALTER TABLE import.choix_personnes ADD COLUMN household_position1 JSONB; @@ -69,7 +69,7 @@ INSERT INTO chill_person_household_position (id, label, sharehousehold, allowhol max_ordering.max_ordering + row_number() OVER () as ordering FROM ( SELECT DISTINCT ON (household_position) household_position1 FROM import.choix_personnes WHERE household_position1 IS NOT NULL ) t CROSS JOIN max_ordering - WHERE NOT EXISTS ( SELECT 1 FROM chill_person_household_position WHERE label::jsonb = t.household_position1 ); + WHERE NOT EXISTS ( SELECT 1 FROM chill_person_household_position WHERE label::jsonb->>'fr' = t.household_position1::jsonb->>'fr' ); -- 8. Complete table accompanying_period_closingmotive ALTER TABLE import.choix_periodes ADD COLUMN closingmotive1 JSONB; @@ -81,7 +81,7 @@ INSERT INTO chill_person_accompanying_period_closingmotive (id, name, active, pa max_ordering.max_ordering + row_number() OVER () as ordering FROM ( SELECT DISTINCT ON (closingmotive) closingmotive1 FROM import.choix_periodes WHERE closingmotive1 IS NOT NULL ) t CROSS JOIN max_ordering - WHERE NOT EXISTS ( SELECT 1 FROM chill_person_accompanying_period_closingmotive WHERE name::jsonb = t.closingmotive1 ); + WHERE NOT EXISTS ( SELECT 1 FROM chill_person_accompanying_period_closingmotive WHERE name::jsonb->>'fr' = t.closingmotive1::jsonb->>'fr' ); -- 9. Complete table accompanying_period_origin ALTER TABLE import.choix_periodes ADD COLUMN origin1 JSONB; @@ -89,7 +89,7 @@ UPDATE import.choix_periodes SET origin1=json_build_object('fr', trim(origin)) W INSERT INTO chill_person_accompanying_period_origin (id, label, noactiveafter) SELECT nextval('chill_person_accompanying_period_origin_id_seq'), t.origin1, null FROM ( SELECT DISTINCT ON (origin) origin1 FROM import.choix_periodes WHERE origin1 IS NOT NULL ) t - WHERE NOT EXISTS ( SELECT 1 FROM chill_person_accompanying_period_origin WHERE label::jsonb = t.origin1 ); + WHERE NOT EXISTS ( SELECT 1 FROM chill_person_accompanying_period_origin WHERE label::jsonb->>'fr' = t.origin1::jsonb->>'fr' ); -- 10. Complete table user_job ALTER TABLE import.choix_periodes ADD COLUMN job1 JSONB; @@ -97,7 +97,7 @@ UPDATE import.choix_periodes SET job1=json_build_object('fr', trim(job)) WHERE j INSERT INTO chill_main_user_job (id, label, active) SELECT nextval('chill_main_user_job_id_seq'), t.job1, true FROM ( SELECT DISTINCT ON (job) job1 FROM import.choix_periodes WHERE job1 IS NOT NULL ) t - WHERE NOT EXISTS( SELECT 1 FROM chill_main_user_job WHERE label::jsonb = t.job1 ); + WHERE NOT EXISTS( SELECT 1 FROM chill_main_user_job WHERE label::jsonb->>'fr' = t.job1::jsonb->>'fr' ); -- 11. Complete table Scopes ALTER TABLE import.choix_periodes ADD COLUMN acp_scopes1 JSONB; @@ -105,13 +105,15 @@ UPDATE import.choix_periodes SET acp_scopes1=json_build_object('fr', trim(acp_sc INSERT INTO scopes (id, name, active) SELECT nextval('scopes_id_seq'), t.acp_scopes1, true FROM ( SELECT DISTINCT ON (acp_scopes) acp_scopes1 FROM import.choix_periodes WHERE acp_scopes1 IS NOT NULL ) t - WHERE NOT EXISTS ( SELECT 1 FROM scopes WHERE name::jsonb = t.acp_scopes1 ); + WHERE NOT EXISTS ( SELECT 1 FROM scopes WHERE name::jsonb->>'fr' = t.acp_scopes1::jsonb->>'fr' ); +-- SELECT i.acp_scopes1::jsonb, s.name FROM scopes s LEFT JOIN import.choix_periodes i ON s.name::jsonb = i.acp_scopes1::jsonb WHERE s.name IS NOT NULL or i.acp_scopes1 IS NOT NULL ; -- 12. Complete table Users (acp referrer) INSERT INTO users (id, username, password, enabled, locked, attributes, label) SELECT nextval('users_id_seq'), t.referrer, '', false, false, '[]', t.referrer FROM ( SELECT DISTINCT ON (referrer) referrer FROM import.choix_periodes WHERE referrer <> '') t WHERE NOT EXISTS ( SELECT 1 FROM users WHERE username::text = t.referrer ); +-- SELECT * FROM users; -- 13. Complete table SocialIssues ALTER TABLE import.choix_periodes ADD COLUMN parent1 jsonb; @@ -125,17 +127,17 @@ INSERT INTO chill_person_social_issue (id, parent_id, title, ordering) max_ordering.max_ordering + row_number() OVER () as ordering FROM ( SELECT DISTINCT ON (parent) parent1 FROM import.choix_periodes WHERE parent1 IS NOT NULL ) t CROSS JOIN max_ordering - WHERE NOT EXISTS ( SELECT 1 FROM chill_person_social_issue WHERE title::jsonb = t.parent1 ); + WHERE NOT EXISTS ( SELECT 1 FROM chill_person_social_issue WHERE title::jsonb->>'fr' = t.parent1::jsonb->>'fr' ); WITH max_ordering AS ( SELECT MAX(ordering) as max_ordering FROM chill_person_social_issue ) INSERT INTO chill_person_social_issue (id, parent_id, title, ordering) SELECT nextval('chill_person_social_issue_id_seq'), - ( SELECT id FROM chill_person_social_issue WHERE parent_id IS NULL AND title = t.parent1 ), t.enfant1, + ( SELECT id FROM chill_person_social_issue WHERE parent_id IS NULL AND title::jsonb->>'fr' = t.parent1::jsonb->>'fr' ), t.enfant1, max_ordering.max_ordering + row_number() OVER () as ordering FROM ( SELECT DISTINCT ON (acp_socialissues) parent1, enfant1 FROM import.choix_periodes WHERE enfant1 IS NOT NULL ) t CROSS JOIN max_ordering WHERE NOT EXISTS ( SELECT 1 FROM chill_person_social_issue - WHERE title::jsonb = t.enfant1 - AND parent_id = (SELECT id FROM chill_person_social_issue WHERE title::jsonb = t.parent1)); + WHERE title::jsonb->>'fr' = t.enfant1::jsonb->>'fr' + AND parent_id = (SELECT id FROM chill_person_social_issue WHERE title::jsonb->>'fr' = t.parent1::jsonb->>'fr')); -- 14. Complete table WorkSocialActions -- (not yet implemented in canvas) @@ -247,10 +249,10 @@ INSERT INTO chill_person_person ( */ ) SELECT person_id -- id - , (SELECT c.id FROM country c WHERE c.name::jsonb = ip.nationality1::jsonb AND ip.nationality1 IS NOT NULL) -- nationality_id, - , (SELECT c.id FROM country c WHERE c.name::jsonb = ip.countryofbirth1::jsonb AND ip.countryofbirth1 IS NOT NULL ) -- countryofbirth_id, - , (SELECT c.id FROM chill_main_civility c WHERE c.name::jsonb = ip.civility1::jsonb AND ip.civility1 IS NOT NULL ) -- civility_id - , (SELECT ms.id FROM chill_person_marital_status ms WHERE ms.name::jsonb = ip.maritalstatus1::jsonb AND ip.maritalstatus1 IS NOT NULL ) -- maritalstatus_id, + , (SELECT c.id FROM country c WHERE c.name::jsonb->>'fr' = ip.nationality1::jsonb->>'fr' AND ip.nationality1 IS NOT NULL) -- nationality_id, + , (SELECT c.id FROM country c WHERE c.name::jsonb->>'fr' = ip.countryofbirth1::jsonb->>'fr' AND ip.countryofbirth1 IS NOT NULL ) -- countryofbirth_id, + , (SELECT c.id FROM chill_main_civility c WHERE c.name::jsonb->>'fr' = ip.civility1::jsonb->>'fr' AND ip.civility1 IS NOT NULL ) -- civility_id + , (SELECT ms.id FROM chill_person_marital_status ms WHERE ms.name::jsonb->>'fr' = ip.maritalstatus1::jsonb->>'fr' AND ip.maritalstatus1 IS NOT NULL ) -- maritalstatus_id, , TRIM(firstname), UPPER(TRIM(lastname)), birthdate1, TRIM(place_of_birth), TRIM(memo), TRIM(email), TRIM(contactinfo), phonenumber1, mobilenumber1, numberofchildren1, gender1, deathdate1, false , CURRENT_DATE, CURRENT_DATE -- createdat, updatedat , (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM users), (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM users) -- createdby_id, updatedby_id From 5a6daae5a44b7e0250a2b6c27386a3c7413dca12 Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Wed, 8 Mar 2023 21:06:33 +0100 Subject: [PATCH 24/48] 43-44. Insert household and link to person, address, position --- sql/import.sql | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/sql/import.sql b/sql/import.sql index 4a68df9..3c36358 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -276,6 +276,22 @@ INSERT INTO chill_main_address (id, postcode_id, street, streetnumber, extra, va SELECT ip.person_id, ip.address_id FROM import.personnes ip ; -- SELECT ip.id, ip.person_id, ip.firstname, ip.lastname, addr.* FROM chill_main_address addr RIGHT JOIN import.personnes ip ON addr.id = ip.address_id ORDER BY ip.id; +-- 43. Add houshold and household member with position +INSERT INTO chill_person_household (id) SELECT person_id from import.personnes; +SELECT setval('chill_person_household_id_seq', (SELECT max(id) FROM chill_person_household)); +INSERT INTO chill_person_household_members (id, person_id, household_id, startdate, sharedhousehold, position_id, holder) SELECT + nextval('chill_person_household_members_id_seq'), person_id, person_id, COALESCE(household_startdate1, CURRENT_DATE), + true, (SELECT id FROM chill_person_household_position pos WHERE household_position1::jsonb->>'fr' = pos.label::jsonb->>'fr'), false + FROM import.personnes; +-- SELECT ip.id, ip.person_id, ip.firstname, ip.lastname, hh.* FROM chill_person_household hh FULL JOIN import.personnes ip ON hh.id = ip.person_id ORDER BY hh.id; +-- SELECT ip.id, ip.person_id, ip.firstname, ip.lastname, memb.* FROM chill_person_household_members memb JOIN import.personnes ip ON memb.household_id = ip.person_id ORDER BY memb.household_id; + +-- 44. Add address to household +INSERT INTO chill_person_household_to_addresses (household_id, address_id) SELECT person_id, address_id + FROM import.personnes WHERE postcode_id IS NOT NULL; -- cfr (*) adresse insérées seulement si postcode + +-- 45. Add household composition +-- (not yet implemented in canvas) -- ========================================================================================= -- @@ -284,6 +300,15 @@ SELECT ip.person_id, ip.address_id FROM import.personnes ip ; -- DOWN -- +-- Undo 44. +DELETE FROM chill_person_household_to_addresses cphta USING import.personnes ic WHERE cphta.household_id = ic.person_id ; + +-- Undo 43. +DELETE FROM chill_person_household_members cphm USING import.personnes ic WHERE cphm.person_id = ic.person_id; +SELECT setval('chill_person_household_members_id_seq', (SELECT max(id) FROM chill_person_household_members)); +DELETE FROM chill_person_household cph USING import.personnes ip WHERE cph.id = ip.person_id; +SELECT setval('chill_person_household_id_seq', (SELECT max(id) FROM chill_person_household)); + -- Undo 42. DELETE FROM chill_main_address cma USING import.personnes ip WHERE cma.id = ip.address_id; SELECT setval('chill_main_address_id_seq', (SELECT max(id) FROM chill_main_address)); @@ -412,4 +437,8 @@ ALTER TABLE import.periodes DROP COLUMN closingdate1; -- ------------- -- tiers choices_list: civility kind profession category - +-- ============= +-- QUESTIONS +-- +-- définir par défaut: quel user, quel centre ? +-- From 5f78276762661b308b25b1bec145332cccd24ead Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Wed, 8 Mar 2023 23:43:47 +0100 Subject: [PATCH 25/48] Rename aliases in undo queries --- sql/import.sql | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sql/import.sql b/sql/import.sql index 3c36358..85ddbb8 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -301,22 +301,22 @@ INSERT INTO chill_person_household_to_addresses (household_id, address_id) SELEC -- -- Undo 44. -DELETE FROM chill_person_household_to_addresses cphta USING import.personnes ic WHERE cphta.household_id = ic.person_id ; +DELETE FROM chill_person_household_to_addresses hhaddr USING import.personnes ip WHERE hhaddr.household_id = ip.person_id ; -- Undo 43. -DELETE FROM chill_person_household_members cphm USING import.personnes ic WHERE cphm.person_id = ic.person_id; +DELETE FROM chill_person_household_members memb USING import.personnes ip WHERE memb.person_id = ip.person_id; SELECT setval('chill_person_household_members_id_seq', (SELECT max(id) FROM chill_person_household_members)); -DELETE FROM chill_person_household cph USING import.personnes ip WHERE cph.id = ip.person_id; +DELETE FROM chill_person_household hh USING import.personnes ip WHERE hh.id = ip.person_id; SELECT setval('chill_person_household_id_seq', (SELECT max(id) FROM chill_person_household)); -- Undo 42. -DELETE FROM chill_main_address cma USING import.personnes ip WHERE cma.id = ip.address_id; +DELETE FROM chill_main_address addr USING import.personnes ip WHERE addr.id = ip.address_id; SELECT setval('chill_main_address_id_seq', (SELECT max(id) FROM chill_main_address)); ALTER TABLE import.personnes DROP column postcode_id; ALTER TABLE import.personnes DROP column address_id; -- Undo 41. -DELETE FROM chill_person_person cpp USING import.personnes ip WHERE cpp.id = ip.person_id; +DELETE FROM chill_person_person p USING import.personnes ip WHERE p.id = ip.person_id; SELECT setval('chill_person_person_id_seq', (SELECT max(id) FROM chill_person_person)); -- Undo 40. From 0c7ea76178057ba5d6c762d1e9795ef61e00c492 Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Wed, 8 Mar 2023 23:44:39 +0100 Subject: [PATCH 26/48] 42. Insert query bloc for center history insertion --- sql/import.sql | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/sql/import.sql b/sql/import.sql index 85ddbb8..5839a97 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -265,7 +265,15 @@ INSERT INTO chill_person_person ( */ FROM import.personnes ip; --- 42. Add address, and link it to person +-- 42. Fill Person center history +INSERT INTO chill_person_person_center_history (id, person_id, center_id, startdate) SELECT + nextval('chill_person_person_center_history_id_seq'), person_id, + (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM centers) , -- center_id + CURRENT_DATE + FROM import.personnes ip; +-- SELECT ip.id, ip.lastname, ip.firstname, ip.person_id, hist.id, hist.person_id, hist.center_id, hist.startdate, p.id, p.fullnamecanonical, p.email FROM chill_person_person p RIGHT JOIN import.personnes ip ON ip.person_id = p.id FULL JOIN chill_person_person_center_history hist on p.id = hist.person_id ORDER BY ip.person_id; + +-- 43. Add address, and link it to person ALTER TABLE import.personnes ADD column address_id BIGINT; UPDATE import.personnes SET address_id = nextval('chill_main_address_id_seq'); ALTER TABLE import.personnes ADD column postcode_id BIGINT; @@ -276,7 +284,7 @@ INSERT INTO chill_main_address (id, postcode_id, street, streetnumber, extra, va SELECT ip.person_id, ip.address_id FROM import.personnes ip ; -- SELECT ip.id, ip.person_id, ip.firstname, ip.lastname, addr.* FROM chill_main_address addr RIGHT JOIN import.personnes ip ON addr.id = ip.address_id ORDER BY ip.id; --- 43. Add houshold and household member with position +-- 44. Add houshold and household member with position INSERT INTO chill_person_household (id) SELECT person_id from import.personnes; SELECT setval('chill_person_household_id_seq', (SELECT max(id) FROM chill_person_household)); INSERT INTO chill_person_household_members (id, person_id, household_id, startdate, sharedhousehold, position_id, holder) SELECT @@ -286,11 +294,11 @@ INSERT INTO chill_person_household_members (id, person_id, household_id, startda -- SELECT ip.id, ip.person_id, ip.firstname, ip.lastname, hh.* FROM chill_person_household hh FULL JOIN import.personnes ip ON hh.id = ip.person_id ORDER BY hh.id; -- SELECT ip.id, ip.person_id, ip.firstname, ip.lastname, memb.* FROM chill_person_household_members memb JOIN import.personnes ip ON memb.household_id = ip.person_id ORDER BY memb.household_id; --- 44. Add address to household +-- 45. Add address to household INSERT INTO chill_person_household_to_addresses (household_id, address_id) SELECT person_id, address_id FROM import.personnes WHERE postcode_id IS NOT NULL; -- cfr (*) adresse insérées seulement si postcode --- 45. Add household composition +-- 46. Add household composition -- (not yet implemented in canvas) @@ -300,21 +308,25 @@ INSERT INTO chill_person_household_to_addresses (household_id, address_id) SELEC -- DOWN -- --- Undo 44. +-- Undo 45. DELETE FROM chill_person_household_to_addresses hhaddr USING import.personnes ip WHERE hhaddr.household_id = ip.person_id ; --- Undo 43. +-- Undo 44. DELETE FROM chill_person_household_members memb USING import.personnes ip WHERE memb.person_id = ip.person_id; SELECT setval('chill_person_household_members_id_seq', (SELECT max(id) FROM chill_person_household_members)); DELETE FROM chill_person_household hh USING import.personnes ip WHERE hh.id = ip.person_id; SELECT setval('chill_person_household_id_seq', (SELECT max(id) FROM chill_person_household)); --- Undo 42. +-- Undo 43. DELETE FROM chill_main_address addr USING import.personnes ip WHERE addr.id = ip.address_id; SELECT setval('chill_main_address_id_seq', (SELECT max(id) FROM chill_main_address)); ALTER TABLE import.personnes DROP column postcode_id; ALTER TABLE import.personnes DROP column address_id; +-- Undo 42. +DELETE FROM chill_person_person_center_history hist USING import.personnes ip WHERE hist.person_id = ip.person_id; +SELECT setval('chill_person_person_center_history_id_seq', (SELECT max(id) FROM chill_person_person_center_history)); + -- Undo 41. DELETE FROM chill_person_person p USING import.personnes ip WHERE p.id = ip.person_id; SELECT setval('chill_person_person_id_seq', (SELECT max(id) FROM chill_person_person)); From 9ac2553eb0c8f31a00161d8e6d53fe6c331e7640 Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Fri, 10 Mar 2023 12:16:51 +0100 Subject: [PATCH 27/48] Fix phone numbers format --- sql/import.sql | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/sql/import.sql b/sql/import.sql index 5839a97..0a1744d 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -150,7 +150,7 @@ UPDATE import.personnes SET civility1=json_build_object('fr', trim(civility)) WH -- 21. Prepare personnes gender ALTER TABLE import.personnes ADD COLUMN gender1 VARCHAR; UPDATE import.personnes SET gender1 = CASE - WHEN trim(gender) IN ('Femme', 'femme', 'Woman', 'woman', 'Female', 'female') THEN 'woman' + WHEN trim(gender) IN ('Femme', 'femme', 'Woman', 'woman', 'Female', 'female') THEN 'woman' WHEN trim(gender) IN ('Homme', 'homme', 'Man', 'man', 'Male', 'male') THEN 'man' WHEN trim(gender) IN ('', 'Inconnu', 'inconnu', 'Unknown', 'unknown') THEN 'unknown' ELSE 'both' END; @@ -209,19 +209,32 @@ UPDATE import.periodes SET acp_scopes1=json_build_object('fr', trim(acp_scopes)) ALTER TABLE import.periodes ADD COLUMN intensity1 text; UPDATE import.periodes SET intensity1='occasional'; UPDATE import.periodes SET intensity1='regular' WHERE intensity='regular'; --- 33. Format mobile numbers +-- 33-34. Format phones numbers ALTER TABLE import.personnes ADD column mobilenumber1 text; -UPDATE import.personnes SET mobilenumber1=NULLIF(regexp_replace(mobilenumber, '[^0-9]', '', 'g'), ''); -- remove all NaN chars -UPDATE import.personnes SET mobilenumber1=regexp_replace(mobilenumber1, '^0', '') WHERE mobilenumber1 like '0%'; -- remove first 0 prefix -UPDATE import.personnes SET mobilenumber1=regexp_replace(mobilenumber1, '(.*)', '+32\1'); -- add belgium intl prefix - --- 34. Format phone numbers ALTER TABLE import.personnes ADD column phonenumber1 text; -UPDATE import.personnes SET phonenumber1=NULLIF(regexp_replace(phonenumber, '[^0-9]', '', 'g'), ''); -- remove all NaN chars -UPDATE import.personnes SET phonenumber1=regexp_replace(phonenumber1, '^0', '') WHERE phonenumber1 like '0%'; -- remove first 0 prefix -UPDATE import.personnes SET phonenumber1=regexp_replace(phonenumber1, '(.*)', '+32\1'); -- add belgium intl prefix --- SELECT mobilenumber, mobilenumber1, phonenumber, phonenumber1 FROM import.personnes; +DO $$ + DECLARE + intl text := 32; + BEGIN + -- 33. Format mobile numbers + UPDATE import.personnes SET mobilenumber1=NULLIF(regexp_replace(mobilenumber, '[^0-9]', '', 'g'), ''); -- remove all NaN chars + UPDATE import.personnes SET mobilenumber1 = CASE -- remove intl prefix + WHEN LEFT(mobilenumber1, 2) = '00' THEN substr(mobilenumber1, 5, length(mobilenumber1) - 4) + WHEN LEFT(mobilenumber1, 2) = intl THEN substr(mobilenumber1, 3, length(mobilenumber1) - 2) + ELSE regexp_replace(mobilenumber1, '^0', '') -- remove first 0 prefix + END; + UPDATE import.personnes SET mobilenumber1=regexp_replace(mobilenumber1, '(.*)', '+' || intl || '\1'); -- add intl prefix + -- 34. Format phone numbers + UPDATE import.personnes SET phonenumber1=NULLIF(regexp_replace(phonenumber, '[^0-9]', '', 'g'), ''); -- remove all NaN chars + UPDATE import.personnes SET phonenumber1 = CASE -- remove intl prefix + WHEN LEFT(phonenumber1, 2) = '00' THEN substr(phonenumber1, 5, length(phonenumber1) - 4) + WHEN LEFT(phonenumber1, 2) = intl THEN substr(phonenumber1, 3, length(phonenumber1) - 2) + ELSE regexp_replace(phonenumber1, '^0', '') -- remove first 0 prefix + END; + UPDATE import.personnes SET phonenumber1=regexp_replace(phonenumber1, '(.*)', '+' || intl || '\1'); -- add intl prefix + --SELECT mobilenumber, mobilenumber1, phonenumber, phonenumber1 FROM import.personnes ORDER BY id; +END $$; -- 35. Prepare required default dates -- address_validfrom1 | household_startdate1 From 1cc1b83435c486feffa6db76076ae0ea57a07c8f Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Fri, 10 Mar 2023 12:45:23 +0100 Subject: [PATCH 28/48] add gendercomment in person insertion --- sql/import.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/import.sql b/sql/import.sql index 0a1744d..0b67939 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -254,10 +254,10 @@ INSERT INTO chill_person_person ( , firstname, lastname, birthdate, place_of_birth, memo, email, contactinfo, phonenumber, mobilenumber, numberofchildren, gender, deathdate, proxyaccompanyingperiodopenstate , createdat, updatedat, createdby_id, updatedby_id , center_id + , gendercomment_comment, gendercomment_userid, gendercomment_date /* , maritalstatusdate , maritalstatuscomment_comment, maritalstatuscomment_userid, maritalstatuscomment_date - , gendercomment_comment, gendercomment_userid, gendercomment_date , acceptsms, acceptemail */ ) SELECT @@ -270,10 +270,10 @@ INSERT INTO chill_person_person ( , CURRENT_DATE, CURRENT_DATE -- createdat, updatedat , (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM users), (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM users) -- createdby_id, updatedby_id , (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM centers) -- center_id + , TRIM(gendercomment), (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM users), CURRENT_DATE /* , -- maritalstatusdate , -- maritalstatuscomment_comment, maritalstatuscomment_userid, maritalstatuscomment_date - , -- gendercomment_comment, gendercomment_userid, gendercomment_date , -- acceptsms, acceptemail */ FROM import.personnes ip; From dd3769e2a09bda39353ea9c2a3f8392207ce8fab Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Fri, 10 Mar 2023 12:55:53 +0100 Subject: [PATCH 29/48] use validfrom in address insertion --- sql/import.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/import.sql b/sql/import.sql index 0b67939..4bf3be6 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -292,7 +292,7 @@ UPDATE import.personnes SET address_id = nextval('chill_main_address_id_seq'); ALTER TABLE import.personnes ADD column postcode_id BIGINT; UPDATE import.personnes SET postcode_id = cmpc.id FROM chill_main_postal_code cmpc WHERE cmpc.code = postcode AND cmpc.origin = 0; INSERT INTO chill_main_address (id, postcode_id, street, streetnumber, extra, validFrom) SELECT - address_id, postcode_id, street, streetnumber, extra, CURRENT_DATE + address_id, postcode_id, street, streetnumber, extra, COALESCE(validfrom1 , CURRENT_DATE) FROM import.personnes WHERE postcode_id IS NOT NULL; SELECT ip.person_id, ip.address_id FROM import.personnes ip ; -- SELECT ip.id, ip.person_id, ip.firstname, ip.lastname, addr.* FROM chill_main_address addr RIGHT JOIN import.personnes ip ON addr.id = ip.address_id ORDER BY ip.id; From cadcddddb5b2c15a2d7013a13ba445d9ae2baf68 Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Fri, 10 Mar 2023 13:08:41 +0100 Subject: [PATCH 30/48] Improve date management for not specified dates - use real import date for person createdat field - use year first day for not specified startdate, comment date, etc. --- sql/import.sql | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sql/import.sql b/sql/import.sql index 4bf3be6..6615ba5 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -267,10 +267,10 @@ INSERT INTO chill_person_person ( , (SELECT c.id FROM chill_main_civility c WHERE c.name::jsonb->>'fr' = ip.civility1::jsonb->>'fr' AND ip.civility1 IS NOT NULL ) -- civility_id , (SELECT ms.id FROM chill_person_marital_status ms WHERE ms.name::jsonb->>'fr' = ip.maritalstatus1::jsonb->>'fr' AND ip.maritalstatus1 IS NOT NULL ) -- maritalstatus_id, , TRIM(firstname), UPPER(TRIM(lastname)), birthdate1, TRIM(place_of_birth), TRIM(memo), TRIM(email), TRIM(contactinfo), phonenumber1, mobilenumber1, numberofchildren1, gender1, deathdate1, false - , CURRENT_DATE, CURRENT_DATE -- createdat, updatedat + , CURRENT_DATE, CURRENT_DATE -- createdat, updatedat (= import date) , (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM users), (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM users) -- createdby_id, updatedby_id , (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM centers) -- center_id - , TRIM(gendercomment), (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM users), CURRENT_DATE + , TRIM(gendercomment), (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM users), date(date_trunc('year', CURRENT_DATE)) --gendercomment_ /* , -- maritalstatusdate , -- maritalstatuscomment_comment, maritalstatuscomment_userid, maritalstatuscomment_date @@ -282,7 +282,7 @@ FROM import.personnes ip; INSERT INTO chill_person_person_center_history (id, person_id, center_id, startdate) SELECT nextval('chill_person_person_center_history_id_seq'), person_id, (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM centers) , -- center_id - CURRENT_DATE + date(date_trunc('year', CURRENT_DATE)) FROM import.personnes ip; -- SELECT ip.id, ip.lastname, ip.firstname, ip.person_id, hist.id, hist.person_id, hist.center_id, hist.startdate, p.id, p.fullnamecanonical, p.email FROM chill_person_person p RIGHT JOIN import.personnes ip ON ip.person_id = p.id FULL JOIN chill_person_person_center_history hist on p.id = hist.person_id ORDER BY ip.person_id; @@ -292,7 +292,7 @@ UPDATE import.personnes SET address_id = nextval('chill_main_address_id_seq'); ALTER TABLE import.personnes ADD column postcode_id BIGINT; UPDATE import.personnes SET postcode_id = cmpc.id FROM chill_main_postal_code cmpc WHERE cmpc.code = postcode AND cmpc.origin = 0; INSERT INTO chill_main_address (id, postcode_id, street, streetnumber, extra, validFrom) SELECT - address_id, postcode_id, street, streetnumber, extra, COALESCE(validfrom1 , CURRENT_DATE) + address_id, postcode_id, street, streetnumber, extra, COALESCE(validfrom1 , date(date_trunc('year', CURRENT_DATE))) FROM import.personnes WHERE postcode_id IS NOT NULL; SELECT ip.person_id, ip.address_id FROM import.personnes ip ; -- SELECT ip.id, ip.person_id, ip.firstname, ip.lastname, addr.* FROM chill_main_address addr RIGHT JOIN import.personnes ip ON addr.id = ip.address_id ORDER BY ip.id; @@ -301,7 +301,7 @@ SELECT ip.person_id, ip.address_id FROM import.personnes ip ; INSERT INTO chill_person_household (id) SELECT person_id from import.personnes; SELECT setval('chill_person_household_id_seq', (SELECT max(id) FROM chill_person_household)); INSERT INTO chill_person_household_members (id, person_id, household_id, startdate, sharedhousehold, position_id, holder) SELECT - nextval('chill_person_household_members_id_seq'), person_id, person_id, COALESCE(household_startdate1, CURRENT_DATE), + nextval('chill_person_household_members_id_seq'), person_id, person_id, COALESCE(household_startdate1, date(date_trunc('year', CURRENT_DATE))), true, (SELECT id FROM chill_person_household_position pos WHERE household_position1::jsonb->>'fr' = pos.label::jsonb->>'fr'), false FROM import.personnes; -- SELECT ip.id, ip.person_id, ip.firstname, ip.lastname, hh.* FROM chill_person_household hh FULL JOIN import.personnes ip ON hh.id = ip.person_id ORDER BY hh.id; From 081ce76085f527ee429f78167f5c981ffeca1e15 Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Fri, 10 Mar 2023 16:16:27 +0100 Subject: [PATCH 31/48] renaming alias --- sql/import.sql | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/sql/import.sql b/sql/import.sql index 6615ba5..308dce7 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -236,11 +236,6 @@ DO $$ --SELECT mobilenumber, mobilenumber1, phonenumber, phonenumber1 FROM import.personnes ORDER BY id; END $$; --- 35. Prepare required default dates --- address_validfrom1 | household_startdate1 --- acp_openingdate1 - - -- 40. Prepare id mapping before insertion ALTER TABLE import.personnes ADD column person_id BIGINT; UPDATE import.personnes SET person_id=personid @@ -290,11 +285,11 @@ INSERT INTO chill_person_person_center_history (id, person_id, center_id, startd ALTER TABLE import.personnes ADD column address_id BIGINT; UPDATE import.personnes SET address_id = nextval('chill_main_address_id_seq'); ALTER TABLE import.personnes ADD column postcode_id BIGINT; -UPDATE import.personnes SET postcode_id = cmpc.id FROM chill_main_postal_code cmpc WHERE cmpc.code = postcode AND cmpc.origin = 0; +UPDATE import.personnes SET postcode_id = pc.id FROM chill_main_postal_code pc WHERE pc.code = postcode AND pc.origin = 0; INSERT INTO chill_main_address (id, postcode_id, street, streetnumber, extra, validFrom) SELECT address_id, postcode_id, street, streetnumber, extra, COALESCE(validfrom1 , date(date_trunc('year', CURRENT_DATE))) FROM import.personnes WHERE postcode_id IS NOT NULL; -SELECT ip.person_id, ip.address_id FROM import.personnes ip ; +-- SELECT ip.person_id, ip.address_id FROM import.personnes ip ; -- SELECT ip.id, ip.person_id, ip.firstname, ip.lastname, addr.* FROM chill_main_address addr RIGHT JOIN import.personnes ip ON addr.id = ip.address_id ORDER BY ip.id; -- 44. Add houshold and household member with position From 307cfcbe74ef135fb3a32da800d7b1aafcbcb252 Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Fri, 10 Mar 2023 16:17:27 +0100 Subject: [PATCH 32/48] 46. Add household composition --- sql/import.sql | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/sql/import.sql b/sql/import.sql index 308dce7..96e13e0 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -307,7 +307,11 @@ INSERT INTO chill_person_household_to_addresses (household_id, address_id) SELEC FROM import.personnes WHERE postcode_id IS NOT NULL; -- cfr (*) adresse insérées seulement si postcode -- 46. Add household composition --- (not yet implemented in canvas) +INSERT INTO chill_person_household_composition (id, household_id, startdate, householdcompositiontype_id) SELECT + nextval('chill_person_household_composition_id_seq'), person_id AS household_id, date(date_trunc('year', CURRENT_DATE)) + , (SELECT id FROM chill_person_household_composition_type ct WHERE household_composition_type1::jsonb->>'fr' = ct.label::jsonb->>'fr') +FROM import.personnes +WHERE household_composition_type1 IS NOT NULL ; -- ========================================================================================= -- @@ -316,6 +320,10 @@ INSERT INTO chill_person_household_to_addresses (household_id, address_id) SELEC -- DOWN -- +-- Undo 46. +DELETE FROM chill_person_household_composition c USING import.personnes ip WHERE c.household_id = ip.person_id; +SELECT setval('chill_person_household_composition_id_seq', (SELECT max(id) FROM chill_person_household_composition)); + -- Undo 45. DELETE FROM chill_person_household_to_addresses hhaddr USING import.personnes ip WHERE hhaddr.household_id = ip.person_id ; From 0ab77b77edd6199fc390d370d385415e83f366e8 Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Fri, 10 Mar 2023 16:18:51 +0100 Subject: [PATCH 33/48] improve setval id if remove all rows --- sql/import.sql | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/sql/import.sql b/sql/import.sql index 96e13e0..213ca4e 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -322,30 +322,30 @@ WHERE household_composition_type1 IS NOT NULL ; -- Undo 46. DELETE FROM chill_person_household_composition c USING import.personnes ip WHERE c.household_id = ip.person_id; -SELECT setval('chill_person_household_composition_id_seq', (SELECT max(id) FROM chill_person_household_composition)); +SELECT setval('chill_person_household_composition_id_seq', (SELECT COALESCE(max(id),1) FROM chill_person_household_composition)); -- Undo 45. -DELETE FROM chill_person_household_to_addresses hhaddr USING import.personnes ip WHERE hhaddr.household_id = ip.person_id ; +DELETE FROM chill_person_household_to_addresses hhaddr USING import.personnes ip WHERE hhaddr.household_id = ip.person_id; -- Undo 44. DELETE FROM chill_person_household_members memb USING import.personnes ip WHERE memb.person_id = ip.person_id; -SELECT setval('chill_person_household_members_id_seq', (SELECT max(id) FROM chill_person_household_members)); +SELECT setval('chill_person_household_members_id_seq', (SELECT COALESCE(max(id),1) FROM chill_person_household_members)); DELETE FROM chill_person_household hh USING import.personnes ip WHERE hh.id = ip.person_id; -SELECT setval('chill_person_household_id_seq', (SELECT max(id) FROM chill_person_household)); +SELECT setval('chill_person_household_id_seq', (SELECT COALESCE(max(id),1) FROM chill_person_household)); -- Undo 43. DELETE FROM chill_main_address addr USING import.personnes ip WHERE addr.id = ip.address_id; -SELECT setval('chill_main_address_id_seq', (SELECT max(id) FROM chill_main_address)); +SELECT setval('chill_main_address_id_seq', (SELECT COALESCE(max(id),1) FROM chill_main_address)); ALTER TABLE import.personnes DROP column postcode_id; ALTER TABLE import.personnes DROP column address_id; -- Undo 42. DELETE FROM chill_person_person_center_history hist USING import.personnes ip WHERE hist.person_id = ip.person_id; -SELECT setval('chill_person_person_center_history_id_seq', (SELECT max(id) FROM chill_person_person_center_history)); +SELECT setval('chill_person_person_center_history_id_seq', (SELECT COALESCE(max(id),1) FROM chill_person_person_center_history)); -- Undo 41. DELETE FROM chill_person_person p USING import.personnes ip WHERE p.id = ip.person_id; -SELECT setval('chill_person_person_id_seq', (SELECT max(id) FROM chill_person_person)); +SELECT setval('chill_person_person_id_seq', (SELECT COALESCE(max(id),1) FROM chill_person_person)); -- Undo 40. ALTER TABLE import.personnes DROP COLUMN person_id; From e569d77bd24b05c16940869c295a4cb5a2b7c758 Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Fri, 10 Mar 2023 16:20:58 +0100 Subject: [PATCH 34/48] Fix WHERE jsonb comparison --- sql/import.sql | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/sql/import.sql b/sql/import.sql index 213ca4e..6737850 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -399,8 +399,8 @@ ALTER TABLE import.personnes DROP COLUMN civility1; -- Undo 13. DELETE FROM chill_person_social_issue USING import.choix_periodes i - WHERE parent_id IN ( SELECT id FROM chill_person_social_issue cpsi WHERE cpsi.title::jsonb = i.parent1::jsonb AND cpsi.parent_id IS NULL ); -DELETE FROM chill_person_social_issue cpsi USING import.choix_periodes icp WHERE cpsi.title::jsonb = icp.parent1::jsonb AND cpsi.parent_id IS NULL ; + WHERE parent_id IN ( SELECT id FROM chill_person_social_issue cpsi WHERE cpsi.title::jsonb->>'fr' = i.parent1::jsonb->>'fr' AND cpsi.parent_id IS NULL ); +DELETE FROM chill_person_social_issue cpsi USING import.choix_periodes icp WHERE cpsi.title::jsonb->>'fr' = icp.parent1::jsonb->>'fr' AND cpsi.parent_id IS NULL ; SELECT setval('chill_person_social_issue_id_seq', (SELECT COALESCE(max(id),1) FROM chill_person_social_issue)); ALTER TABLE import.choix_periodes DROP COLUMN enfant1; ALTER TABLE import.choix_periodes DROP COLUMN parent1; @@ -410,46 +410,46 @@ DELETE FROM users USING import.choix_periodes icp WHERE users.username::text = i SELECT setval('users_id_seq', (SELECT COALESCE(max(id),1) FROM users)); -- Undo 11. -DELETE FROM scopes USING import.choix_periodes icp WHERE scopes.name::jsonb = icp.acp_scopes1::jsonb; +DELETE FROM scopes USING import.choix_periodes icp WHERE scopes.name::jsonb->>'fr' = icp.acp_scopes1::jsonb->>'fr'; SELECT setval('scopes_id_seq', (SELECT COALESCE(max(id),1) FROM scopes)); ALTER TABLE import.choix_periodes DROP COLUMN acp_scopes1; -- Undo 10. -DELETE FROM chill_main_user_job cmuj USING import.choix_periodes icp WHERE cmuj.label::jsonb = icp.job1::jsonb; +DELETE FROM chill_main_user_job cmuj USING import.choix_periodes icp WHERE cmuj.label::jsonb->>'fr' = icp.job1::jsonb->>'fr'; SELECT setval('chill_main_user_job_id_seq', (SELECT COALESCE(max(id),1) FROM chill_main_user_job)); ALTER TABLE import.choix_periodes DROP COLUMN job1; -- Undo 9. -DELETE FROM chill_person_accompanying_period_origin cpapo USING import.choix_periodes icp WHERE cpapo.label::jsonb = icp.origin1::jsonb; +DELETE FROM chill_person_accompanying_period_origin cpapo USING import.choix_periodes icp WHERE cpapo.label::jsonb->>'fr' = icp.origin1::jsonb->>'fr'; SELECT setval('chill_person_accompanying_period_origin_id_seq', (SELECT COALESCE(max(id),1) FROM chill_person_accompanying_period_origin)); ALTER TABLE import.choix_periodes DROP COLUMN origin1; -- Undo 8. -DELETE FROM chill_person_accompanying_period_closingmotive cpapcm USING import.choix_periodes icp WHERE cpapcm.name::jsonb = icp.closingmotive1::jsonb; +DELETE FROM chill_person_accompanying_period_closingmotive cpapcm USING import.choix_periodes icp WHERE cpapcm.name::jsonb->>'fr' = icp.closingmotive1::jsonb->>'fr'; SELECT setval('chill_person_accompanying_period_closingmotive_id_seq', (SELECT COALESCE(max(id),1) FROM chill_person_accompanying_period_closingmotive)); ALTER TABLE import.choix_periodes DROP COLUMN closingmotive1; -- Undo 7. -DELETE FROM chill_person_household_position cphp USING import.choix_personnes icp WHERE cphp.label::jsonb = icp.household_position1::jsonb; +DELETE FROM chill_person_household_position cphp USING import.choix_personnes icp WHERE cphp.label::jsonb->>'fr' = icp.household_position1::jsonb->>'fr'; SELECT setval('chill_person_household_position_id_seq', (SELECT COALESCE(max(id),1) FROM chill_person_household_position)); ALTER TABLE import.choix_personnes DROP COLUMN household_position1; -- Undo 6. -DELETE FROM chill_person_household_composition_type cphct USING import.choix_personnes icp WHERE cphct.label::jsonb = icp.household_composition_type1::jsonb AND cphct.id > 6; +DELETE FROM chill_person_household_composition_type cphct USING import.choix_personnes icp WHERE cphct.label::jsonb->>'fr' = icp.household_composition_type1::jsonb->>'fr' AND cphct.id > 6; SELECT setval('chill_person_household_composition_type_id_seq', (SELECT COALESCE(max(id),1) FROM chill_person_household_composition_type)); ALTER TABLE import.choix_personnes DROP COLUMN household_composition_type1; -- Undo 5. -DELETE FROM chill_person_marital_status cpms USING import.choix_personnes icp WHERE cpms.name::jsonb = icp.maritalstatus1::jsonb AND cpms.id <> 'célibat.' AND cpms.id <> 'marié'; -- existing marital status +DELETE FROM chill_person_marital_status cpms USING import.choix_personnes icp WHERE cpms.name::jsonb->>'fr' = icp.maritalstatus1::jsonb->>'fr' AND cpms.id <> 'célibat.' AND cpms.id <> 'marié'; -- existing marital status ALTER TABLE import.choix_personnes DROP COLUMN maritalstatus1; -- Undo 4. -DELETE FROM country USING import.choix_personnes icp WHERE country.name::jsonb = icp.country1::jsonb AND country.id > 249; -- 249 existing countries +DELETE FROM country USING import.choix_personnes icp WHERE country.name::jsonb->>'fr' = icp.country1::jsonb->>'fr' AND country.id > 249; -- 249 existing countries SELECT setval('country_id_seq', (SELECT COALESCE(max(id),1) FROM country)); ALTER TABLE import.choix_personnes DROP COLUMN country1; -- Undo 3. -DELETE FROM chill_main_civility cmc USING import.choix_personnes ip WHERE cmc.name::jsonb = ip.civility1::jsonb; +DELETE FROM chill_main_civility cmc USING import.choix_personnes ip WHERE cmc.name::jsonb->>'fr' = ip.civility1::jsonb->>'fr'; SELECT setval('chill_main_civility_id_seq', (SELECT COALESCE(max(id),1) FROM chill_main_civility)); ALTER TABLE import.choix_personnes DROP COLUMN civility1; From 477565f6e18c9aae02f80ca705cd44b740b61ea7 Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Fri, 10 Mar 2023 16:49:12 +0100 Subject: [PATCH 35/48] Improve undo delete clause to avoid removing still referenced rows (foreign key constraint violation) --- sql/import.sql | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/sql/import.sql b/sql/import.sql index 6737850..75b35d0 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -410,37 +410,44 @@ DELETE FROM users USING import.choix_periodes icp WHERE users.username::text = i SELECT setval('users_id_seq', (SELECT COALESCE(max(id),1) FROM users)); -- Undo 11. -DELETE FROM scopes USING import.choix_periodes icp WHERE scopes.name::jsonb->>'fr' = icp.acp_scopes1::jsonb->>'fr'; +DELETE FROM scopes USING import.choix_periodes icp WHERE scopes.name::jsonb->>'fr' = icp.acp_scopes1::jsonb->>'fr' + AND NOT EXISTS (SELECT 1 FROM import.choix_periodes WHERE scopes.name::jsonb->>'fr' = icp.acp_scopes1::jsonb->>'fr'); SELECT setval('scopes_id_seq', (SELECT COALESCE(max(id),1) FROM scopes)); ALTER TABLE import.choix_periodes DROP COLUMN acp_scopes1; -- Undo 10. -DELETE FROM chill_main_user_job cmuj USING import.choix_periodes icp WHERE cmuj.label::jsonb->>'fr' = icp.job1::jsonb->>'fr'; +DELETE FROM chill_main_user_job cmuj USING import.choix_periodes icp WHERE cmuj.label::jsonb->>'fr' = icp.job1::jsonb->>'fr' + AND NOT EXISTS (SELECT 1 FROM import.choix_periodes WHERE cmuj.label::jsonb->>'fr' = icp.job1::jsonb->>'fr'); SELECT setval('chill_main_user_job_id_seq', (SELECT COALESCE(max(id),1) FROM chill_main_user_job)); ALTER TABLE import.choix_periodes DROP COLUMN job1; -- Undo 9. -DELETE FROM chill_person_accompanying_period_origin cpapo USING import.choix_periodes icp WHERE cpapo.label::jsonb->>'fr' = icp.origin1::jsonb->>'fr'; +DELETE FROM chill_person_accompanying_period_origin cpapo USING import.choix_periodes icp WHERE cpapo.label::jsonb->>'fr' = icp.origin1::jsonb->>'fr' + AND NOT EXISTS (SELECT 1 FROM import.choix_periodes WHERE cpapo.label::jsonb->>'fr' = icp.origin1::jsonb->>'fr'); SELECT setval('chill_person_accompanying_period_origin_id_seq', (SELECT COALESCE(max(id),1) FROM chill_person_accompanying_period_origin)); ALTER TABLE import.choix_periodes DROP COLUMN origin1; -- Undo 8. -DELETE FROM chill_person_accompanying_period_closingmotive cpapcm USING import.choix_periodes icp WHERE cpapcm.name::jsonb->>'fr' = icp.closingmotive1::jsonb->>'fr'; +DELETE FROM chill_person_accompanying_period_closingmotive cpapcm USING import.choix_periodes icp WHERE cpapcm.name::jsonb->>'fr' = icp.closingmotive1::jsonb->>'fr' + AND NOT EXISTS (SELECT 1 FROM import.choix_periodes WHERE cpapcm.name::jsonb->>'fr' = icp.closingmotive1::jsonb->>'fr'); SELECT setval('chill_person_accompanying_period_closingmotive_id_seq', (SELECT COALESCE(max(id),1) FROM chill_person_accompanying_period_closingmotive)); ALTER TABLE import.choix_periodes DROP COLUMN closingmotive1; -- Undo 7. -DELETE FROM chill_person_household_position cphp USING import.choix_personnes icp WHERE cphp.label::jsonb->>'fr' = icp.household_position1::jsonb->>'fr'; +DELETE FROM chill_person_household_position cphp USING import.choix_personnes icp WHERE cphp.label::jsonb->>'fr' = icp.household_position1::jsonb->>'fr' + AND NOT EXISTS (SELECT 1 FROM import.choix_personnes WHERE cphp.label::jsonb->>'fr' = icp.household_position1::jsonb->>'fr'); SELECT setval('chill_person_household_position_id_seq', (SELECT COALESCE(max(id),1) FROM chill_person_household_position)); ALTER TABLE import.choix_personnes DROP COLUMN household_position1; -- Undo 6. -DELETE FROM chill_person_household_composition_type cphct USING import.choix_personnes icp WHERE cphct.label::jsonb->>'fr' = icp.household_composition_type1::jsonb->>'fr' AND cphct.id > 6; +DELETE FROM chill_person_household_composition_type cphct USING import.choix_personnes icp WHERE cphct.label::jsonb->>'fr' = icp.household_composition_type1::jsonb->>'fr' + AND NOT EXISTS (SELECT 1 FROM import.choix_personnes WHERE cphct.label::jsonb->>'fr' = icp.household_composition_type1::jsonb->>'fr'); SELECT setval('chill_person_household_composition_type_id_seq', (SELECT COALESCE(max(id),1) FROM chill_person_household_composition_type)); ALTER TABLE import.choix_personnes DROP COLUMN household_composition_type1; -- Undo 5. -DELETE FROM chill_person_marital_status cpms USING import.choix_personnes icp WHERE cpms.name::jsonb->>'fr' = icp.maritalstatus1::jsonb->>'fr' AND cpms.id <> 'célibat.' AND cpms.id <> 'marié'; -- existing marital status +DELETE FROM chill_person_marital_status cpms USING import.choix_personnes icp WHERE cpms.name::jsonb->>'fr' = icp.maritalstatus1::jsonb->>'fr' + AND NOT EXISTS (SELECT 1 FROM import.choix_personnes WHERE cpms.name::jsonb->>'fr' = icp.maritalstatus1::jsonb->>'fr'); ALTER TABLE import.choix_personnes DROP COLUMN maritalstatus1; -- Undo 4. @@ -449,7 +456,8 @@ SELECT setval('country_id_seq', (SELECT COALESCE(max(id),1) FROM country)); ALTER TABLE import.choix_personnes DROP COLUMN country1; -- Undo 3. -DELETE FROM chill_main_civility cmc USING import.choix_personnes ip WHERE cmc.name::jsonb->>'fr' = ip.civility1::jsonb->>'fr'; +DELETE FROM chill_main_civility cmc USING import.choix_personnes ip WHERE cmc.name::jsonb->>'fr' = ip.civility1::jsonb->>'fr' + AND NOT EXISTS (SELECT 1 FROM import.choix_personnes WHERE cmc.name::jsonb->>'fr' = ip.civility1::jsonb->>'fr'); SELECT setval('chill_main_civility_id_seq', (SELECT COALESCE(max(id),1) FROM chill_main_civility)); ALTER TABLE import.choix_personnes DROP COLUMN civility1; From 67d020d322b5f8dc7238e96dcda75c9f5a81e5a1 Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Fri, 10 Mar 2023 17:51:58 +0100 Subject: [PATCH 36/48] 50-51. Insert in accompanyingPeriods table --- sql/import.sql | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/sql/import.sql b/sql/import.sql index 75b35d0..e275030 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -313,6 +313,19 @@ INSERT INTO chill_person_household_composition (id, household_id, startdate, hou FROM import.personnes WHERE household_composition_type1 IS NOT NULL ; +-- 50. Prepare id mapping before insertion +ALTER TABLE import.periodes ADD column period_id BIGINT; +UPDATE import.periodes SET period_id = periodid + FROM (SELECT id, nextval('chill_person_accompanying_period_id_seq') AS periodid FROM import.periodes ORDER BY id) AS t + WHERE import.periodes.id = t.id; +--SELECT ipe.id, ipe.fullname, ipe.period_id, ip.id, ip.person_id FROM import.periodes ipe JOIN import.personnes ip ON ipe.id = ip.id ORDER BY ipe.id; + +-- 51. Insert in chill_person_accompanying_period +INSERT INTO chill_person_accompanying_period (id, openingdate, step, remark, intensity, createdby_id, createdat, updatedby_id, updatedat) SELECT + period_id, date(date_trunc('year', CURRENT_DATE)), 'CONFIRMED', COALESCE(TRIM(remark), ''), intensity1, + (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM users), CURRENT_DATE, + (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM users), CURRENT_DATE +FROM import.periodes; -- ========================================================================================= -- @@ -320,6 +333,13 @@ WHERE household_composition_type1 IS NOT NULL ; -- DOWN -- +-- Undo 51. +DELETE FROM chill_person_accompanying_period acp USING import.periodes ip WHERE acp.id = ip.period_id; +SELECT setval('chill_person_accompanying_period_id_seq', (SELECT COALESCE(max(id), 1) FROM chill_person_accompanying_period)); + +-- Undo 50. +ALTER TABLE import.periodes DROP column period_id; + -- Undo 46. DELETE FROM chill_person_household_composition c USING import.personnes ip WHERE c.household_id = ip.person_id; SELECT setval('chill_person_household_composition_id_seq', (SELECT COALESCE(max(id),1) FROM chill_person_household_composition)); From f513dd1b36a602890fc57b78af833a03eafd2921 Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Fri, 10 Mar 2023 17:53:05 +0100 Subject: [PATCH 37/48] Add a setval() query after INSERT INTO when id is not nextval() --- sql/import.sql | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sql/import.sql b/sql/import.sql index e275030..b8e8cee 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -272,6 +272,7 @@ INSERT INTO chill_person_person ( , -- acceptsms, acceptemail */ FROM import.personnes ip; +SELECT setval('chill_person_person_id_seq', (SELECT COALESCE(max(id)) FROM chill_person_person)); -- 42. Fill Person center history INSERT INTO chill_person_person_center_history (id, person_id, center_id, startdate) SELECT @@ -289,6 +290,7 @@ UPDATE import.personnes SET postcode_id = pc.id FROM chill_main_postal_code pc W INSERT INTO chill_main_address (id, postcode_id, street, streetnumber, extra, validFrom) SELECT address_id, postcode_id, street, streetnumber, extra, COALESCE(validfrom1 , date(date_trunc('year', CURRENT_DATE))) FROM import.personnes WHERE postcode_id IS NOT NULL; +SELECT setval('chill_main_address_id_seq', (SELECT COALESCE(max(id)) FROM chill_main_address)); -- SELECT ip.person_id, ip.address_id FROM import.personnes ip ; -- SELECT ip.id, ip.person_id, ip.firstname, ip.lastname, addr.* FROM chill_main_address addr RIGHT JOIN import.personnes ip ON addr.id = ip.address_id ORDER BY ip.id; @@ -302,7 +304,7 @@ INSERT INTO chill_person_household_members (id, person_id, household_id, startda -- SELECT ip.id, ip.person_id, ip.firstname, ip.lastname, hh.* FROM chill_person_household hh FULL JOIN import.personnes ip ON hh.id = ip.person_id ORDER BY hh.id; -- SELECT ip.id, ip.person_id, ip.firstname, ip.lastname, memb.* FROM chill_person_household_members memb JOIN import.personnes ip ON memb.household_id = ip.person_id ORDER BY memb.household_id; --- 45. Add address to household +-- 45. Link address to household INSERT INTO chill_person_household_to_addresses (household_id, address_id) SELECT person_id, address_id FROM import.personnes WHERE postcode_id IS NOT NULL; -- cfr (*) adresse insérées seulement si postcode @@ -326,6 +328,7 @@ INSERT INTO chill_person_accompanying_period (id, openingdate, step, remark, int (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM users), CURRENT_DATE, (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM users), CURRENT_DATE FROM import.periodes; +SELECT setval('chill_person_accompanying_period_id_seq', (SELECT max(id) FROM chill_person_accompanying_period)); -- ========================================================================================= -- From 6900d66206092b7197e266be421c2272e3a3bca8 Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Fri, 10 Mar 2023 18:32:09 +0100 Subject: [PATCH 38/48] 52. Link participations to periods --- sql/import.sql | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/sql/import.sql b/sql/import.sql index b8e8cee..22ba61d 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -330,12 +330,21 @@ INSERT INTO chill_person_accompanying_period (id, openingdate, step, remark, int FROM import.periodes; SELECT setval('chill_person_accompanying_period_id_seq', (SELECT max(id) FROM chill_person_accompanying_period)); +-- 52. Link participations to periods +INSERT INTO chill_person_accompanying_period_participation (id, person_id, accompanyingperiod_id, startdate) SELECT + nextval('chill_person_accompanying_period_participation_id_seq'), t.person_id, t.period_id, date(date_trunc('year', CURRENT_DATE)) + FROM (SELECT person_id, period_id FROM import.periodes ip JOIN import.personnes p ON ip.id = p.id ORDER BY person_id) AS t; + -- ========================================================================================= -- -- -- DOWN -- +-- Undo 52. +DELETE FROM chill_person_accompanying_period_participation part USING import.periodes ip WHERE part.accompanyingperiod_id = ip.period_id; +SELECT setval('chill_person_accompanying_period_participation_id_seq', (SELECT COALESCE(max(id), 1) FROM chill_person_accompanying_period_participation)); + -- Undo 51. DELETE FROM chill_person_accompanying_period acp USING import.periodes ip WHERE acp.id = ip.period_id; SELECT setval('chill_person_accompanying_period_id_seq', (SELECT COALESCE(max(id), 1) FROM chill_person_accompanying_period)); From 59daee47905c6f1d429d2c8bd1d483cc2f07c466 Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Fri, 10 Mar 2023 19:27:31 +0100 Subject: [PATCH 39/48] 53-54. Link period to person or temporary address location --- sql/import.sql | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/sql/import.sql b/sql/import.sql index 22ba61d..ef0a2df 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -335,12 +335,40 @@ INSERT INTO chill_person_accompanying_period_participation (id, person_id, accom nextval('chill_person_accompanying_period_participation_id_seq'), t.person_id, t.period_id, date(date_trunc('year', CURRENT_DATE)) FROM (SELECT person_id, period_id FROM import.periodes ip JOIN import.personnes p ON ip.id = p.id ORDER BY person_id) AS t; +-- 53. Prepare temporary address location +INSERT INTO chill_main_address (id, postcode_id, street, streetnumber, validFrom) SELECT + nextval('chill_main_address_id_seq'), + (SELECT pc.id FROM chill_main_postal_code pc WHERE pc.code = postcode), + street, streetnumber, date(date_trunc('year', CURRENT_DATE)) + FROM import.choix_periodes WHERE street != ''; +ALTER TABLE import.choix_periodes ADD COLUMN address_location_id BIGINT; +UPDATE import.choix_periodes SET address_location_id = (SELECT max(id) FROM chill_main_address) WHERE street != ''; + +-- 54. Link period to person or temporary address location +UPDATE chill_person_accompanying_period acp + SET addresslocation_id = (SELECT address_location_id FROM import.choix_periodes WHERE address_location_id IS NOT NULL LIMIT 1) + FROM import.personnes pson JOIN import.periodes piod ON pson.id = piod.id + WHERE piod.period_id = acp.id AND piod.addresslocation = 'oui'; +UPDATE chill_person_accompanying_period acp + SET personlocation_id = pson.person_id + FROM import.personnes pson JOIN import.periodes piod ON pson.id = piod.id + WHERE piod.period_id = acp.id AND piod.personlocation = 'oui'; + + -- ========================================================================================= -- -- -- DOWN -- +-- Undo 54. +UPDATE chill_person_accompanying_period acp SET addresslocation_id = NULL, personlocation_id = NULL FROM import.periodes piod WHERE piod.period_id = acp.id; + +-- Undo 53. +DELETE FROM chill_main_address addr USING import.choix_periodes ic WHERE addr.id = ic.address_location_id; +SELECT setval('chill_main_address_id_seq', (SELECT COALESCE(max(id), 1) FROM chill_main_address)); +ALTER TABLE import.choix_periodes DROP COLUMN address_location_id; + -- Undo 52. DELETE FROM chill_person_accompanying_period_participation part USING import.periodes ip WHERE part.accompanyingperiod_id = ip.period_id; SELECT setval('chill_person_accompanying_period_participation_id_seq', (SELECT COALESCE(max(id), 1) FROM chill_person_accompanying_period_participation)); From 314e625c917d0391c3c7a3b03b28146542f86301 Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Fri, 10 Mar 2023 19:58:25 +0100 Subject: [PATCH 40/48] 55. Copy period in period_location_history --- sql/import.sql | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/sql/import.sql b/sql/import.sql index ef0a2df..fec705a 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -323,8 +323,9 @@ UPDATE import.periodes SET period_id = periodid --SELECT ipe.id, ipe.fullname, ipe.period_id, ip.id, ip.person_id FROM import.periodes ipe JOIN import.personnes ip ON ipe.id = ip.id ORDER BY ipe.id; -- 51. Insert in chill_person_accompanying_period -INSERT INTO chill_person_accompanying_period (id, openingdate, step, remark, intensity, createdby_id, createdat, updatedby_id, updatedat) SELECT - period_id, date(date_trunc('year', CURRENT_DATE)), 'CONFIRMED', COALESCE(TRIM(remark), ''), intensity1, +INSERT INTO chill_person_accompanying_period (id, openingdate, closingdate, step, remark, intensity, createdby_id, createdat, updatedby_id, updatedat) SELECT + period_id, COALESCE(openingdate1, date(date_trunc('year', CURRENT_DATE))), closingdate1, + 'CONFIRMED', COALESCE(TRIM(remark), ''), intensity1, (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM users), CURRENT_DATE, (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM users), CURRENT_DATE FROM import.periodes; @@ -354,6 +355,14 @@ UPDATE chill_person_accompanying_period acp FROM import.personnes pson JOIN import.periodes piod ON pson.id = piod.id WHERE piod.period_id = acp.id AND piod.personlocation = 'oui'; +-- 55. Copy period in period_location_history +INSERT INTO chill_person_accompanying_period_location_history (id, period_id, startdate, enddate, personlocation_id, addresslocation_id, createdat, createdby_id) + SELECT + nextval('chill_person_accompanying_period_location_history_id_seq'), id, acp.openingdate, acp.closingdate, + acp.personlocation_id, acp.addresslocation_id, acp.createdat, acp.createdby_id + FROM chill_person_accompanying_period acp + WHERE id NOT IN (SELECT period_id FROM chill_person_accompanying_period_location_history) AND step LIKE 'CONFIRMED' ORDER BY id; + -- ========================================================================================= -- @@ -361,6 +370,10 @@ UPDATE chill_person_accompanying_period acp -- DOWN -- +-- Undo 55. +DELETE FROM chill_person_accompanying_period_location_history history USING import.periodes ip WHERE history.period_id = ip.period_id; +SELECT setval('chill_person_accompanying_period_location_history_id_seq', (SELECT COALESCE(max(id), 1) FROM chill_person_accompanying_period_location_history)); + -- Undo 54. UPDATE chill_person_accompanying_period acp SET addresslocation_id = NULL, personlocation_id = NULL FROM import.periodes piod WHERE piod.period_id = acp.id; From 1adf47e63e6f6169b71ab34baf3aa9018d98a2c3 Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Sat, 11 Mar 2023 10:49:57 +0100 Subject: [PATCH 41/48] 56. Link socialissues to periods --- sql/import.sql | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/sql/import.sql b/sql/import.sql index fec705a..66aaef8 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -315,6 +315,7 @@ INSERT INTO chill_person_household_composition (id, household_id, startdate, hou FROM import.personnes WHERE household_composition_type1 IS NOT NULL ; + -- 50. Prepare id mapping before insertion ALTER TABLE import.periodes ADD column period_id BIGINT; UPDATE import.periodes SET period_id = periodid @@ -324,7 +325,8 @@ UPDATE import.periodes SET period_id = periodid -- 51. Insert in chill_person_accompanying_period INSERT INTO chill_person_accompanying_period (id, openingdate, closingdate, step, remark, intensity, createdby_id, createdat, updatedby_id, updatedat) SELECT - period_id, COALESCE(openingdate1, date(date_trunc('year', CURRENT_DATE))), closingdate1, + period_id, + COALESCE(openingdate1, date(date_trunc('year', CURRENT_DATE))), closingdate1, 'CONFIRMED', COALESCE(TRIM(remark), ''), intensity1, (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM users), CURRENT_DATE, (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM users), CURRENT_DATE @@ -363,6 +365,24 @@ INSERT INTO chill_person_accompanying_period_location_history (id, period_id, st FROM chill_person_accompanying_period acp WHERE id NOT IN (SELECT period_id FROM chill_person_accompanying_period_location_history) AND step LIKE 'CONFIRMED' ORDER BY id; +-- 56. Link socialIssues to periods +INSERT INTO chill_person_accompanying_period_social_issues (accompanyingperiod_id, socialissue_id) +SELECT + DISTINCT ON (t.period_id) t.period_id, + COALESCE(t.enfant_id, t.parent_id, 1) AS socialissue_id -- quid si pas de socialissue ? +FROM ( + SELECT p.period_id, + (SELECT id FROM chill_person_social_issue WHERE title::jsonb->>'fr' = icp.parent1::jsonb->>'fr' AND parent_id IS NULL) AS parent_id, icp.parent1, + (SELECT id FROM chill_person_social_issue WHERE title::jsonb->>'fr' = icp.enfant1::jsonb->>'fr' AND parent_id = + (SELECT id FROM chill_person_social_issue WHERE title::jsonb->>'fr' = icp.parent1::jsonb->>'fr' AND parent_id IS NULL)) AS enfant_id, icp.enfant1 + FROM import.periodes p + JOIN import.choix_periodes icp ON p.acp_socialissues = icp.acp_socialissues + ORDER BY id) AS t; + + +-- 57. +-- 58. +-- 59. -- ========================================================================================= -- @@ -370,6 +390,13 @@ INSERT INTO chill_person_accompanying_period_location_history (id, period_id, st -- DOWN -- +-- Undo 59. +-- Undo 58. +-- Undo 57. + +-- Undo 56. +DELETE FROM chill_person_accompanying_period_social_issues asi USING import.periodes ip WHERE asi.accompanyingperiod_id = ip.period_id; + -- Undo 55. DELETE FROM chill_person_accompanying_period_location_history history USING import.periodes ip WHERE history.period_id = ip.period_id; SELECT setval('chill_person_accompanying_period_location_history_id_seq', (SELECT COALESCE(max(id), 1) FROM chill_person_accompanying_period_location_history)); From 7893eb3c81a4071e9dcb950760aa840eba704cb1 Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Sat, 11 Mar 2023 13:39:42 +0100 Subject: [PATCH 42/48] 57. Link referrer to periods --- sql/import.sql | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sql/import.sql b/sql/import.sql index 66aaef8..ddca6e3 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -379,8 +379,12 @@ FROM ( JOIN import.choix_periodes icp ON p.acp_socialissues = icp.acp_socialissues ORDER BY id) AS t; +-- 57. Link referrer to periods +UPDATE chill_person_accompanying_period acp + SET user_id = (SELECT id FROM users WHERE users.username = ip.referrer) + FROM import.periodes ip WHERE acp.id = ip.period_id; +--SELECT ip.id, (SELECT id FROM users WHERE users.username = ip.referrer) AS referrer_id, ip.referrer, acp.id as period_id, acp.user_id FROM chill_person_accompanying_period acp JOIN import.periodes ip ON ip.period_id = acp.id ORDER BY ip.id; --- 57. -- 58. -- 59. @@ -393,6 +397,7 @@ FROM ( -- Undo 59. -- Undo 58. -- Undo 57. +UPDATE chill_person_accompanying_period acp SET user_id = null FROM import.periodes ip WHERE ip.period_id = acp.id; -- Undo 56. DELETE FROM chill_person_accompanying_period_social_issues asi USING import.periodes ip WHERE asi.accompanyingperiod_id = ip.period_id; From 264fdb58888a8949f1004f42a201e9ea672ea147 Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Sat, 11 Mar 2023 15:11:47 +0100 Subject: [PATCH 43/48] 58. Link scopes to periods --- sql/import.sql | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/sql/import.sql b/sql/import.sql index ddca6e3..6aeb3dd 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -385,8 +385,20 @@ UPDATE chill_person_accompanying_period acp FROM import.periodes ip WHERE acp.id = ip.period_id; --SELECT ip.id, (SELECT id FROM users WHERE users.username = ip.referrer) AS referrer_id, ip.referrer, acp.id as period_id, acp.user_id FROM chill_person_accompanying_period acp JOIN import.periodes ip ON ip.period_id = acp.id ORDER BY ip.id; --- 58. --- 59. +-- 58. Link scopes to periods +INSERT INTO accompanying_periods_scopes (accompanying_period_id, scope_id) + SELECT ip.period_id, COALESCE( + (SELECT id FROM scopes s WHERE ip.acp_scopes1::jsonb->>'fr' = s.name::jsonb->>'fr'), + (SELECT id from scopes s WHERE s.name::jsonb->>'fr' = 'tous') -- default 'tous' + ) + FROM import.periodes ip; + +-- 59. Link origin to periods +UPDATE chill_person_accompanying_period acp SET origin_id = + (SELECT id FROM chill_person_accompanying_period_origin o WHERE o.label::jsonb->>'fr' = ip.origin1::jsonb->>'fr' ) +FROM import.periodes ip WHERE acp.id = ip.period_id; +--SELECT ip.id, ip.origin1, acp.id as period_id, acp.origin_id FROM chill_person_accompanying_period acp JOIN import.periodes ip ON ip.period_id = acp.id ORDER BY ip.id; + -- ========================================================================================= -- @@ -396,6 +408,8 @@ UPDATE chill_person_accompanying_period acp -- Undo 59. -- Undo 58. +DELETE FROM accompanying_periods_scopes acs USING import.periodes ip WHERE acs.accompanying_period_id = ip.period_id; + -- Undo 57. UPDATE chill_person_accompanying_period acp SET user_id = null FROM import.periodes ip WHERE ip.period_id = acp.id; From 12f0ec0ae48de9321a397cdc7fbc0a91b52e8860 Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Sat, 11 Mar 2023 14:08:56 +0100 Subject: [PATCH 44/48] 59. Link origin to periods --- sql/import.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sql/import.sql b/sql/import.sql index 6aeb3dd..871ef03 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -407,6 +407,8 @@ FROM import.periodes ip WHERE acp.id = ip.period_id; -- -- Undo 59. +UPDATE chill_person_accompanying_period acp SET origin_id = null FROM import.periodes ip WHERE ip.period_id = acp.id; + -- Undo 58. DELETE FROM accompanying_periods_scopes acs USING import.periodes ip WHERE acs.accompanying_period_id = ip.period_id; From d9152fc090cc6f3ace4ad7161916e7ce3beb2f25 Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Sat, 11 Mar 2023 15:27:56 +0100 Subject: [PATCH 45/48] 60. Link jobs to periods --- sql/import.sql | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/sql/import.sql b/sql/import.sql index 871ef03..95a7314 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -347,7 +347,7 @@ INSERT INTO chill_main_address (id, postcode_id, street, streetnumber, validFrom ALTER TABLE import.choix_periodes ADD COLUMN address_location_id BIGINT; UPDATE import.choix_periodes SET address_location_id = (SELECT max(id) FROM chill_main_address) WHERE street != ''; --- 54. Link period to person or temporary address location +-- 54. Link person or temporary address location to periods UPDATE chill_person_accompanying_period acp SET addresslocation_id = (SELECT address_location_id FROM import.choix_periodes WHERE address_location_id IS NOT NULL LIMIT 1) FROM import.personnes pson JOIN import.periodes piod ON pson.id = piod.id @@ -395,10 +395,15 @@ INSERT INTO accompanying_periods_scopes (accompanying_period_id, scope_id) -- 59. Link origin to periods UPDATE chill_person_accompanying_period acp SET origin_id = - (SELECT id FROM chill_person_accompanying_period_origin o WHERE o.label::jsonb->>'fr' = ip.origin1::jsonb->>'fr' ) + (SELECT id FROM chill_person_accompanying_period_origin o WHERE o.label::jsonb->>'fr' = ip.origin1::jsonb->>'fr') FROM import.periodes ip WHERE acp.id = ip.period_id; --SELECT ip.id, ip.origin1, acp.id as period_id, acp.origin_id FROM chill_person_accompanying_period acp JOIN import.periodes ip ON ip.period_id = acp.id ORDER BY ip.id; +-- 60. Link jobs to periods +UPDATE chill_person_accompanying_period acp SET job_id = + (SELECT id FROM chill_main_user_job j WHERE j.label::jsonb->>'fr' = ip.job1::jsonb->>'fr') +FROM import.periodes ip WHERE acp.id = ip.period_id; + -- ========================================================================================= -- @@ -406,6 +411,9 @@ FROM import.periodes ip WHERE acp.id = ip.period_id; -- DOWN -- +-- Undo 60. +UPDATE chill_person_accompanying_period acp SET job_id = null FROM import.periodes ip WHERE acp.id = ip.period_id; + -- Undo 59. UPDATE chill_person_accompanying_period acp SET origin_id = null FROM import.periodes ip WHERE ip.period_id = acp.id; From 07df6c4292e63e44fa40fe909dfe489ca75adf8d Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Sat, 11 Mar 2023 16:50:11 +0100 Subject: [PATCH 46/48] default value for socialissue_id, user_id, scope_id --- sql/import.sql | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/sql/import.sql b/sql/import.sql index 95a7314..2d04063 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -369,7 +369,11 @@ INSERT INTO chill_person_accompanying_period_location_history (id, period_id, st INSERT INTO chill_person_accompanying_period_social_issues (accompanyingperiod_id, socialissue_id) SELECT DISTINCT ON (t.period_id) t.period_id, - COALESCE(t.enfant_id, t.parent_id, 1) AS socialissue_id -- quid si pas de socialissue ? + COALESCE( + t.enfant_id, + t.parent_id, + 1 -- default value ? + ) AS socialissue_id FROM ( SELECT p.period_id, (SELECT id FROM chill_person_social_issue WHERE title::jsonb->>'fr' = icp.parent1::jsonb->>'fr' AND parent_id IS NULL) AS parent_id, icp.parent1, @@ -381,7 +385,10 @@ FROM ( -- 57. Link referrer to periods UPDATE chill_person_accompanying_period acp - SET user_id = (SELECT id FROM users WHERE users.username = ip.referrer) + SET user_id = COALESCE( + (SELECT id FROM users WHERE users.username = ip.referrer), + 1 -- default value ? + ) FROM import.periodes ip WHERE acp.id = ip.period_id; --SELECT ip.id, (SELECT id FROM users WHERE users.username = ip.referrer) AS referrer_id, ip.referrer, acp.id as period_id, acp.user_id FROM chill_person_accompanying_period acp JOIN import.periodes ip ON ip.period_id = acp.id ORDER BY ip.id; @@ -389,7 +396,7 @@ UPDATE chill_person_accompanying_period acp INSERT INTO accompanying_periods_scopes (accompanying_period_id, scope_id) SELECT ip.period_id, COALESCE( (SELECT id FROM scopes s WHERE ip.acp_scopes1::jsonb->>'fr' = s.name::jsonb->>'fr'), - (SELECT id from scopes s WHERE s.name::jsonb->>'fr' = 'tous') -- default 'tous' + (SELECT id from scopes s WHERE s.name::jsonb->>'fr' = 'tous') -- default value 'tous' ) FROM import.periodes ip; @@ -599,8 +606,6 @@ ALTER TABLE import.periodes DROP COLUMN openingdate1; ALTER TABLE import.periodes DROP COLUMN closingdate1; --- ------------- --- tiers choices_list: civility kind profession category -- ============= -- QUESTIONS From b68e0399ccd8f8a4b64e561be90f84b7f043dda9 Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Sat, 11 Mar 2023 16:51:43 +0100 Subject: [PATCH 47/48] 62-63. Add and link comments to periods, pin last comment --- sql/import.sql | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/sql/import.sql b/sql/import.sql index 2d04063..3ed315f 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -411,6 +411,50 @@ UPDATE chill_person_accompanying_period acp SET job_id = (SELECT id FROM chill_main_user_job j WHERE j.label::jsonb->>'fr' = ip.job1::jsonb->>'fr') FROM import.periodes ip WHERE acp.id = ip.period_id; +-- 61. Link administrative Location +-- (to be add in csv) + +-- 62. Add and link comments +INSERT INTO chill_person_accompanying_period_comment (id, accompanyingperiod_id, content, creator_id, createdat, updatedby_id, updatedat) + SELECT nextval('chill_person_accompanying_period_comment_id_seq'), period_id, comment1_content, + (SELECT user_id FROM chill_person_accompanying_period acp WHERE acp.id = ip.period_id), CURRENT_TIMESTAMP, + (SELECT user_id FROM chill_person_accompanying_period acp WHERE acp.id = ip.period_id), CURRENT_TIMESTAMP + FROM import.periodes ip WHERE ip.comment1_content != ''; +INSERT INTO chill_person_accompanying_period_comment (id, accompanyingperiod_id, content, creator_id, createdat, updatedby_id, updatedat) + SELECT nextval('chill_person_accompanying_period_comment_id_seq'), period_id, comment2_content, + (SELECT user_id FROM chill_person_accompanying_period acp WHERE acp.id = ip.period_id), CURRENT_TIMESTAMP, + (SELECT user_id FROM chill_person_accompanying_period acp WHERE acp.id = ip.period_id), CURRENT_TIMESTAMP + FROM import.periodes ip WHERE ip.comment2_content != ''; +INSERT INTO chill_person_accompanying_period_comment (id, accompanyingperiod_id, content, creator_id, createdat, updatedby_id, updatedat) + SELECT nextval('chill_person_accompanying_period_comment_id_seq'), period_id, comment3_content, + (SELECT user_id FROM chill_person_accompanying_period acp WHERE acp.id = ip.period_id), CURRENT_TIMESTAMP, + (SELECT user_id FROM chill_person_accompanying_period acp WHERE acp.id = ip.period_id), CURRENT_TIMESTAMP + FROM import.periodes ip WHERE ip.comment3_content != ''; +INSERT INTO chill_person_accompanying_period_comment (id, accompanyingperiod_id, content, creator_id, createdat, updatedby_id, updatedat) + SELECT nextval('chill_person_accompanying_period_comment_id_seq'), period_id, comment4_content, + (SELECT user_id FROM chill_person_accompanying_period acp WHERE acp.id = ip.period_id), CURRENT_TIMESTAMP, + (SELECT user_id FROM chill_person_accompanying_period acp WHERE acp.id = ip.period_id), CURRENT_TIMESTAMP + FROM import.periodes ip WHERE ip.comment4_content != ''; +INSERT INTO chill_person_accompanying_period_comment (id, accompanyingperiod_id, content, creator_id, createdat, updatedby_id, updatedat) + SELECT nextval('chill_person_accompanying_period_comment_id_seq'), period_id, comment5_content, + (SELECT user_id FROM chill_person_accompanying_period acp WHERE acp.id = ip.period_id), CURRENT_TIMESTAMP, + (SELECT user_id FROM chill_person_accompanying_period acp WHERE acp.id = ip.period_id), CURRENT_TIMESTAMP + FROM import.periodes ip WHERE ip.comment5_content != ''; + +-- 63. Link pinned comment to period +UPDATE import.periodes SET comment1_content = null WHERE comment1_content = ''; +UPDATE import.periodes SET comment2_content = null WHERE comment2_content = ''; +UPDATE import.periodes SET comment3_content = null WHERE comment3_content = ''; +UPDATE import.periodes SET comment4_content = null WHERE comment4_content = ''; +UPDATE import.periodes SET comment5_content = null WHERE comment5_content = ''; +UPDATE chill_person_accompanying_period acp SET pinnedcomment_id = + (SELECT id FROM chill_person_accompanying_period_comment com WHERE com.accompanyingperiod_id = acp.id + AND com.content = COALESCE(ip.comment5_content, ip.comment4_content, ip.comment3_content, ip.comment2_content, ip.comment1_content) + LIMIT 1) +FROM import.periodes ip WHERE acp.id = ip.period_id; + +-- ~~Link closingmotive~~ (to be removed from csv) + -- ========================================================================================= -- @@ -418,6 +462,20 @@ FROM import.periodes ip WHERE acp.id = ip.period_id; -- DOWN -- +-- Undo 63. +UPDATE chill_person_accompanying_period acp SET pinnedcomment_id = null FROM import.periodes ip WHERE acp.id = ip.period_id; +UPDATE import.periodes SET comment1_content = '' WHERE comment1_content IS NULL; +UPDATE import.periodes SET comment2_content = '' WHERE comment2_content IS NULL; +UPDATE import.periodes SET comment3_content = '' WHERE comment3_content IS NULL; +UPDATE import.periodes SET comment4_content = '' WHERE comment4_content IS NULL; +UPDATE import.periodes SET comment5_content = '' WHERE comment5_content IS NULL; + +-- Undo 62. +DELETE FROM chill_person_accompanying_period_comment com USING import.periodes ip WHERE com.accompanyingperiod_id = ip.period_id; +SELECT setval('chill_person_accompanying_period_comment_id_seq', (SELECT COALESCE(max(id), 1) FROM chill_person_accompanying_period_comment)); + +-- Undo 61. + -- Undo 60. UPDATE chill_person_accompanying_period acp SET job_id = null FROM import.periodes ip WHERE acp.id = ip.period_id; From 290a9b917f4a6236d4b6fc05053ecb8f6fdc4f01 Mon Sep 17 00:00:00 2001 From: Mathieu Jaumotte Date: Tue, 14 Mar 2023 12:05:52 +0100 Subject: [PATCH 48/48] Things to think when running script --- sql/import.sql | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sql/import.sql b/sql/import.sql index 3ed315f..3a4845e 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -3,7 +3,12 @@ -- version v0.6 (== version canevas) -- -- /!\ IMPORTANT --- Avant de migrer (UP), il faut d'abord avoir importé les codes postaux !!! +-- * Avant de migrer (UP), il faut d'abord avoir importé les codes postaux !!! +-- * Adapter les valeurs par défaut +-- * centres: cfr. 41 et 42 +-- * socialIssues: cfr. 56 +-- * referrer: cfr. 57 +-- * scopes: cfr. 58 --