champs-libres/chill-import-from-canvas
5
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Improve/correct import SQL queries

Browse Source
This commit is contained in:
nobohan 2024-09-19 17:59:44 +02:00
parent db52c72d22
commit 3651b1a09b
3 changed files with 41 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
README.md
View File

@ -67,7 +67,7 @@ On peut aussi utiliser la fonction `\copy` de psql pour charger un fichier csv d
Tout d'abord, les tables du schéma "import" doivent être préparées avec le script `prepare-import.sql`.
Puis on importe les 4 fichiers csv.
Puis on importe les 4 fichiers csv:
```bash
psql chill-import
@ -77,6 +77,12 @@ chill-import=# \copy "import".choix_periodes FROM 'choix_periodes.csv' DELIMITER
chill-import=# \copy "import".periodes FROM 'periodes.csv' DELIMITER ',' CSV HEADER
```
Enfin, on exporte la base de données en sql:
```bash
pg_dump chill-import --no-owner > <client>-data.sql
```
## 3. Import du schéma 'import' sur le serveur (safran)
@ -92,6 +98,14 @@ debian@safran:~/bin$ ls -l dump/ | tail -1
-rw-r--r-- 1 postgres postgres 234954230 Mar 15 10:40 20230315-104003_cycloprod.sql
```
ou bien simplement:
```bash
sudo su postgres
cd
pg_dump -p 5436 laplateformereunionprod > laplateformereunionprod.sql
```
- importer le fichier sql sur la base cible: `$ sudo su postgres -c 'psql -p5436'`
```sql
postgres=# \c cycloprod
@ -117,6 +131,14 @@ cycloprod=# \dt import.*
(4 rows)
```
ou bien simplement:
```bash
sudo su postgres
psql -p 5436 laplateformereunionprod < /tmp/chill-import-lpreunion.sql
```
## 4. Exécution du script de migration
Se fait dans la console postgresql, en tant que user postgres, en étant connecté à la base de donnée cible.

22
sql/import.sql
View File

@ -139,7 +139,7 @@ INSERT INTO chill_person_social_issue (id, parent_id, title, ordering)
SELECT nextval('chill_person_social_issue_id_seq'),
( SELECT id FROM chill_person_social_issue WHERE parent_id IS NULL AND title::jsonb->>'fr' = t.parent1::jsonb->>'fr' ), t.enfant1,
max_ordering.max_ordering + row_number() OVER () as ordering
FROM ( SELECT DISTINCT ON (acp_socialissues) parent1, enfant1 FROM import.choix_periodes WHERE enfant1 IS NOT NULL ) t
FROM ( SELECT DISTINCT ON (acp_social_issues) parent1, enfant1 FROM import.choix_periodes WHERE enfant1 IS NOT NULL ) t
CROSS JOIN max_ordering
WHERE NOT EXISTS ( SELECT 1 FROM chill_person_social_issue
WHERE title::jsonb->>'fr' = t.enfant1::jsonb->>'fr'
@ -178,9 +178,17 @@ UPDATE import.personnes SET maritalstatus1=json_build_object('fr', trim(maritals
-- 25. Prepare personnes numberofchildren
ALTER TABLE import.personnes ADD COLUMN numberofchildren1 integer;
UPDATE import.personnes SET numberofchildren1=NULLIF(numberofchildren, '')::int;
UPDATE import.personnes SET numberofchildren1=NULLIF(numberofchildren, 0)::int;
-- SELECT numberofchildren, numberofchildren1 FROM import.personnes;
-- 25bis. Prepare personnes place_of_birth
ALTER TABLE import.personnes ADD COLUMN placeofbirth1 varchar;
UPDATE import.personnes SET placeofbirth1=CASE WHEN place_of_birth IS NOT NULL THEN place_of_birth ELSE '' END;
-- 25ter. Prepare personnes memo
ALTER TABLE import.personnes ADD COLUMN memo1 varchar;
UPDATE import.personnes SET memo1=CASE WHEN memo IS NOT NULL THEN memo ELSE '' END;
-- 26. Prepare personnes household_composition_type
ALTER TABLE import.personnes ADD COLUMN household_composition_type1 JSONB;
UPDATE import.personnes SET household_composition_type1=json_build_object('fr', trim(household_composition_type)) WHERE household_composition_type!='';
@ -267,7 +275,7 @@ INSERT INTO chill_person_person (
, (SELECT c.id FROM country c WHERE c.name::jsonb->>'fr' = ip.countryofbirth1::jsonb->>'fr' AND ip.countryofbirth1 IS NOT NULL ) -- countryofbirth_id,
, (SELECT c.id FROM chill_main_civility c WHERE c.name::jsonb->>'fr' = ip.civility1::jsonb->>'fr' AND ip.civility1 IS NOT NULL ) -- civility_id
, (SELECT ms.id FROM chill_person_marital_status ms WHERE ms.name::jsonb->>'fr' = ip.maritalstatus1::jsonb->>'fr' AND ip.maritalstatus1 IS NOT NULL ) -- maritalstatus_id,
, TRIM(firstname), UPPER(TRIM(lastname)), birthdate1, TRIM(place_of_birth), TRIM(memo), TRIM(email), TRIM(contactinfo), phonenumber1, mobilenumber1, numberofchildren1, gender1, deathdate1, false
, TRIM(firstname), UPPER(TRIM(lastname)), birthdate1, TRIM(placeofbirth1), TRIM(memo1), TRIM(email), TRIM(contactinfo), phonenumber1, mobilenumber1, numberofchildren1, gender1, deathdate1, false
, CURRENT_DATE, CURRENT_DATE -- createdat, updatedat (= import date)
, (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM users), (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM users) -- createdby_id, updatedby_id
, (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM centers) -- center_id
@ -292,7 +300,7 @@ INSERT INTO chill_person_person_center_history (id, person_id, center_id, startd
ALTER TABLE import.personnes ADD column address_id BIGINT;
UPDATE import.personnes SET address_id = nextval('chill_main_address_id_seq');
ALTER TABLE import.personnes ADD column postcode_id BIGINT;
UPDATE import.personnes SET postcode_id = pc.id FROM chill_main_postal_code pc WHERE pc.code::int = postcode AND pc.origin = 0;
UPDATE import.personnes SET postcode_id = pc.id FROM chill_main_postal_code pc WHERE pc.code::int = postcode::int AND pc.origin = 0;
INSERT INTO chill_main_address (id, postcode_id, street, streetnumber, extra, validFrom) SELECT
address_id, postcode_id, street, streetnumber, extra, COALESCE(validfrom1 , date(date_trunc('year', CURRENT_DATE)))
FROM import.personnes WHERE postcode_id IS NOT NULL;
@ -347,7 +355,7 @@ INSERT INTO chill_person_accompanying_period_participation (id, person_id, accom
-- 53. Prepare temporary address location
INSERT INTO chill_main_address (id, postcode_id, street, streetnumber, validFrom) SELECT
nextval('chill_main_address_id_seq'),
(SELECT pc.id FROM chill_main_postal_code pc WHERE pc.code = postcode),
(SELECT pc.id FROM chill_main_postal_code pc WHERE pc.code::int = postcode::int),
street, streetnumber, date(date_trunc('year', CURRENT_DATE))
FROM import.choix_periodes WHERE street != '';
ALTER TABLE import.choix_periodes ADD COLUMN address_location_id BIGINT;
@ -386,7 +394,7 @@ FROM (
(SELECT id FROM chill_person_social_issue WHERE title::jsonb->>'fr' = icp.enfant1::jsonb->>'fr' AND parent_id =
(SELECT id FROM chill_person_social_issue WHERE title::jsonb->>'fr' = icp.parent1::jsonb->>'fr' AND parent_id IS NULL)) AS enfant_id, icp.enfant1
FROM import.periodes p
JOIN import.choix_periodes icp ON p.acp_socialissues = icp.acp_socialissues
JOIN import.choix_periodes icp ON p.acp_socialissues = icp.acp_social_issues
ORDER BY id) AS t;
-- 57. Link referrer to periods
@ -402,7 +410,7 @@ UPDATE chill_person_accompanying_period acp
INSERT INTO accompanying_periods_scopes (accompanying_period_id, scope_id)
SELECT ip.period_id, COALESCE(
(SELECT id FROM scopes s WHERE ip.acp_scopes1::jsonb->>'fr' = s.name::jsonb->>'fr'),
(SELECT id from scopes s WHERE s.name::jsonb->>'fr' = 'tous') -- default value 'tous'
(SELECT id from scopes s WHERE s.name::jsonb->>'fr' = 'Principal') -- default value 'Principal'
)
FROM import.periodes ip;

4
sql/prepare-import.sql
View File

@ -8,6 +8,7 @@ CREATE TABLE "import".choix_personnes (
);
CREATE TABLE "import".personnes (
id varchar(50) NULL,
civility varchar(50) NULL,
lastname varchar(50) NULL,
firstname varchar(50) NULL,
@ -42,7 +43,7 @@ CREATE TABLE "import".choix_periodes (
origin varchar(50) NULL,
acp_scopes varchar(50) NULL,
job varchar(50) NULL,
"user" varchar(50) NULL,
referrer varchar(50) NULL,
parent varchar(50) NULL,
enfant varchar(50) NULL,
acp_social_issues varchar(50) NULL,
@ -56,6 +57,7 @@ CREATE TABLE "import".choix_periodes (
CREATE TABLE "import".periodes (
id varchar(50) NULL,
nom varchar(50) NULL,
openingdate varchar(50) NULL,
closingdate varchar(50) NULL,