From 3651b1a09be561266249c98e0d83649486e43c67 Mon Sep 17 00:00:00 2001 From: nobohan Date: Thu, 19 Sep 2024 17:59:44 +0200 Subject: [PATCH] Improve/correct import SQL queries --- README.md | 24 +++++++++++++++++++++++- sql/import.sql | 22 +++++++++++++++------- sql/prepare-import.sql | 4 +++- 3 files changed, 41 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 04c7ed8..65f6d16 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ On peut aussi utiliser la fonction `\copy` de psql pour charger un fichier csv d Tout d'abord, les tables du schéma "import" doivent être préparées avec le script `prepare-import.sql`. -Puis on importe les 4 fichiers csv. +Puis on importe les 4 fichiers csv: ```bash psql chill-import @@ -77,6 +77,12 @@ chill-import=# \copy "import".choix_periodes FROM 'choix_periodes.csv' DELIMITER chill-import=# \copy "import".periodes FROM 'periodes.csv' DELIMITER ',' CSV HEADER ``` +Enfin, on exporte la base de données en sql: + +```bash +pg_dump chill-import --no-owner > -data.sql +``` + ## 3. Import du schéma 'import' sur le serveur (safran) @@ -92,6 +98,14 @@ debian@safran:~/bin$ ls -l dump/ | tail -1 -rw-r--r-- 1 postgres postgres 234954230 Mar 15 10:40 20230315-104003_cycloprod.sql ``` +ou bien simplement: + +```bash +sudo su postgres +cd +pg_dump -p 5436 laplateformereunionprod > laplateformereunionprod.sql +``` + - importer le fichier sql sur la base cible: `$ sudo su postgres -c 'psql -p5436'` ```sql postgres=# \c cycloprod @@ -117,6 +131,14 @@ cycloprod=# \dt import.* (4 rows) ``` +ou bien simplement: + +```bash +sudo su postgres +psql -p 5436 laplateformereunionprod < /tmp/chill-import-lpreunion.sql +``` + + ## 4. Exécution du script de migration Se fait dans la console postgresql, en tant que user postgres, en étant connecté à la base de donnée cible. diff --git a/sql/import.sql b/sql/import.sql index 5c1bdc3..d5c8361 100644 --- a/sql/import.sql +++ b/sql/import.sql @@ -139,7 +139,7 @@ INSERT INTO chill_person_social_issue (id, parent_id, title, ordering) SELECT nextval('chill_person_social_issue_id_seq'), ( SELECT id FROM chill_person_social_issue WHERE parent_id IS NULL AND title::jsonb->>'fr' = t.parent1::jsonb->>'fr' ), t.enfant1, max_ordering.max_ordering + row_number() OVER () as ordering - FROM ( SELECT DISTINCT ON (acp_socialissues) parent1, enfant1 FROM import.choix_periodes WHERE enfant1 IS NOT NULL ) t + FROM ( SELECT DISTINCT ON (acp_social_issues) parent1, enfant1 FROM import.choix_periodes WHERE enfant1 IS NOT NULL ) t CROSS JOIN max_ordering WHERE NOT EXISTS ( SELECT 1 FROM chill_person_social_issue WHERE title::jsonb->>'fr' = t.enfant1::jsonb->>'fr' @@ -178,9 +178,17 @@ UPDATE import.personnes SET maritalstatus1=json_build_object('fr', trim(maritals -- 25. Prepare personnes numberofchildren ALTER TABLE import.personnes ADD COLUMN numberofchildren1 integer; -UPDATE import.personnes SET numberofchildren1=NULLIF(numberofchildren, '')::int; +UPDATE import.personnes SET numberofchildren1=NULLIF(numberofchildren, 0)::int; -- SELECT numberofchildren, numberofchildren1 FROM import.personnes; +-- 25bis. Prepare personnes place_of_birth +ALTER TABLE import.personnes ADD COLUMN placeofbirth1 varchar; +UPDATE import.personnes SET placeofbirth1=CASE WHEN place_of_birth IS NOT NULL THEN place_of_birth ELSE '' END; + +-- 25ter. Prepare personnes memo +ALTER TABLE import.personnes ADD COLUMN memo1 varchar; +UPDATE import.personnes SET memo1=CASE WHEN memo IS NOT NULL THEN memo ELSE '' END; + -- 26. Prepare personnes household_composition_type ALTER TABLE import.personnes ADD COLUMN household_composition_type1 JSONB; UPDATE import.personnes SET household_composition_type1=json_build_object('fr', trim(household_composition_type)) WHERE household_composition_type!=''; @@ -267,7 +275,7 @@ INSERT INTO chill_person_person ( , (SELECT c.id FROM country c WHERE c.name::jsonb->>'fr' = ip.countryofbirth1::jsonb->>'fr' AND ip.countryofbirth1 IS NOT NULL ) -- countryofbirth_id, , (SELECT c.id FROM chill_main_civility c WHERE c.name::jsonb->>'fr' = ip.civility1::jsonb->>'fr' AND ip.civility1 IS NOT NULL ) -- civility_id , (SELECT ms.id FROM chill_person_marital_status ms WHERE ms.name::jsonb->>'fr' = ip.maritalstatus1::jsonb->>'fr' AND ip.maritalstatus1 IS NOT NULL ) -- maritalstatus_id, - , TRIM(firstname), UPPER(TRIM(lastname)), birthdate1, TRIM(place_of_birth), TRIM(memo), TRIM(email), TRIM(contactinfo), phonenumber1, mobilenumber1, numberofchildren1, gender1, deathdate1, false + , TRIM(firstname), UPPER(TRIM(lastname)), birthdate1, TRIM(placeofbirth1), TRIM(memo1), TRIM(email), TRIM(contactinfo), phonenumber1, mobilenumber1, numberofchildren1, gender1, deathdate1, false , CURRENT_DATE, CURRENT_DATE -- createdat, updatedat (= import date) , (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM users), (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM users) -- createdby_id, updatedby_id , (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM centers) -- center_id @@ -292,7 +300,7 @@ INSERT INTO chill_person_person_center_history (id, person_id, center_id, startd ALTER TABLE import.personnes ADD column address_id BIGINT; UPDATE import.personnes SET address_id = nextval('chill_main_address_id_seq'); ALTER TABLE import.personnes ADD column postcode_id BIGINT; -UPDATE import.personnes SET postcode_id = pc.id FROM chill_main_postal_code pc WHERE pc.code::int = postcode AND pc.origin = 0; +UPDATE import.personnes SET postcode_id = pc.id FROM chill_main_postal_code pc WHERE pc.code::int = postcode::int AND pc.origin = 0; INSERT INTO chill_main_address (id, postcode_id, street, streetnumber, extra, validFrom) SELECT address_id, postcode_id, street, streetnumber, extra, COALESCE(validfrom1 , date(date_trunc('year', CURRENT_DATE))) FROM import.personnes WHERE postcode_id IS NOT NULL; @@ -347,7 +355,7 @@ INSERT INTO chill_person_accompanying_period_participation (id, person_id, accom -- 53. Prepare temporary address location INSERT INTO chill_main_address (id, postcode_id, street, streetnumber, validFrom) SELECT nextval('chill_main_address_id_seq'), - (SELECT pc.id FROM chill_main_postal_code pc WHERE pc.code = postcode), + (SELECT pc.id FROM chill_main_postal_code pc WHERE pc.code::int = postcode::int), street, streetnumber, date(date_trunc('year', CURRENT_DATE)) FROM import.choix_periodes WHERE street != ''; ALTER TABLE import.choix_periodes ADD COLUMN address_location_id BIGINT; @@ -386,7 +394,7 @@ FROM ( (SELECT id FROM chill_person_social_issue WHERE title::jsonb->>'fr' = icp.enfant1::jsonb->>'fr' AND parent_id = (SELECT id FROM chill_person_social_issue WHERE title::jsonb->>'fr' = icp.parent1::jsonb->>'fr' AND parent_id IS NULL)) AS enfant_id, icp.enfant1 FROM import.periodes p - JOIN import.choix_periodes icp ON p.acp_socialissues = icp.acp_socialissues + JOIN import.choix_periodes icp ON p.acp_socialissues = icp.acp_social_issues ORDER BY id) AS t; -- 57. Link referrer to periods @@ -402,7 +410,7 @@ UPDATE chill_person_accompanying_period acp INSERT INTO accompanying_periods_scopes (accompanying_period_id, scope_id) SELECT ip.period_id, COALESCE( (SELECT id FROM scopes s WHERE ip.acp_scopes1::jsonb->>'fr' = s.name::jsonb->>'fr'), - (SELECT id from scopes s WHERE s.name::jsonb->>'fr' = 'tous') -- default value 'tous' + (SELECT id from scopes s WHERE s.name::jsonb->>'fr' = 'Principal') -- default value 'Principal' ) FROM import.periodes ip; diff --git a/sql/prepare-import.sql b/sql/prepare-import.sql index cdd62de..1f67e8f 100644 --- a/sql/prepare-import.sql +++ b/sql/prepare-import.sql @@ -8,6 +8,7 @@ CREATE TABLE "import".choix_personnes ( ); CREATE TABLE "import".personnes ( + id varchar(50) NULL, civility varchar(50) NULL, lastname varchar(50) NULL, firstname varchar(50) NULL, @@ -42,7 +43,7 @@ CREATE TABLE "import".choix_periodes ( origin varchar(50) NULL, acp_scopes varchar(50) NULL, job varchar(50) NULL, - "user" varchar(50) NULL, + referrer varchar(50) NULL, parent varchar(50) NULL, enfant varchar(50) NULL, acp_social_issues varchar(50) NULL, @@ -56,6 +57,7 @@ CREATE TABLE "import".choix_periodes ( CREATE TABLE "import".periodes ( + id varchar(50) NULL, nom varchar(50) NULL, openingdate varchar(50) NULL, closingdate varchar(50) NULL,