update readme

(foreign key constraint violation)

README.md

@@ -0,0 +1,112 @@
+CHILL - Reprise de données
+==========================
+
+Ce dépôt contient un script d'import qui s'applique à un canevas excel présenté au client. Le client remplit le fichier excel, puis le script insère les données dans la base de donnée. 
+
+L'opération est semi-automatique et réduit considérablement le temps dédié à l'import en structurant le format des données en entrée. Par contre il y a toujours une série de manipulations, pour préparer et insérer les données correctement. 
+
+Ces manipulations sont décrites ici.
+ 
+Le client a rempli le canevas. Une relecture du fichier est toujours nécessaire afin de repérer les éventuelles irrégularités.
+
+## 1. Préparer les fichiers csv
+
+Le fichier se compose de plusieurs feuilles, chacune doit être sauvée au format csv.
+
+Pour préparer les fichiers on va:
+- nettoyer le fichier pour ne laisser en étiquette que les noms de colonnes en anglais.
+- ajouter une colonne de contôle en fin de ligne, par sécurité. Par exemple une colonne 'endcol' qui contient pour chaque cellule 'endrow'.
+- ajouter les doublequote lors de la sauvegarde du csv, 
+- enlever tous les line breaks et caractères spéciaux.
+
+```bash
+# Exemple de remplacements exécutés sur les fichiers csv pour un import spécifique :
+$ sed -e :1 -e '$q' -e "/$CR\$/b" -e 'N;s/\n//;b1' < file.2.csv > file.3.csv 
+$ sed -e 's#"end"#"end"\n#g' < file.3.csv > file.4.csv
+
+# Exemple pour un autre import:  
+$ cat file2.csv | sed -e 'N; s#_x000D_##g; s#\n##g; s/$CR//g' | tr "\n" " " > file3.csv 
+$ sed -e 's#"endcol"#"endcol"\n#g; s#"endrow"#"endrow"\n#g' < file3.csv > file4.csv
+$ sed -e 's#^,##g; s#^ ##g' < file4.csv > file5.csv
+```
+
+## 2. Insérer les csv dans la base de donnée
+
+On va insérer chaque feuille csv comme table à part entière d'un nouveau schéma `import`. On aura:
+- import.choix_personnes
+- import.personnes
+- import.choix_periodes
+- import.periodes
+
+Pour réaliser cet import, on peut utiliser des outils tels que `pgfutter`, mais celui-ci peut s'avérer capricieux selon le fichier. 
+La meilleure méthode pour moi est de réaliser cette étape en local avec phpstorm, puis d'exporter le schéma `import` avec pg_dump avant de le transférer sur le serveur.
+
+### 2.a Manipulations dans phpstorm
+
+- S'il n'existe pas, créer le schéma `import`; s'il existe, s'assurer qu'il ne contient pas de tables ni de données.
+#### Importer le csv dans la db
+- ouvre le fichier csv > passe en onglet text > edit as table > set options: 
+  - cocher 'first row is header'
+  - 'null value text': undefined (pas de champs null dans la table, mais un texte vide)
+  - then > open table
+- import to database > set options:
+  - régler target/schema: import
+  - et table: même nom que le csv
+  - DDL: TEXT pour tous les champs
+  - then > import
+#### Exporter en sql
+- créer un fichier `<client>-data.sql` vide
+- depuis chaque table du schéma `import`: 
+  - copier le DDL de la table dans le fichier (s'assurer d'ajouter le préfixe `import.` sur chaque requête)
+  - export data > extractor: SQL-insert-multirow > copy to clipboard 
+- coller les données dans `<client>-data.sql`
+
+## 3. Import du schéma 'import' sur le serveur (safran)
+
+- transférer le fichier `<client>-data.sql` sur le serveur (avec scp): 
+```bash
+$ scp cyclo-data.sql debian@safran:~/data/tmp/
+```
+
+- faire une sauvegarde de la base sur laquelle on va réaliser l'insertion
+```bash
+debian@safran:~/bin$ bash backup_now_db.sh 5436 cycloprod
+debian@safran:~/bin$ ls -l dump/ | tail -1
+-rw-r--r-- 1 postgres postgres 234954230 Mar 15 10:40 20230315-104003_cycloprod.sql
+```
+
+- importer le fichier sql sur la base cible: `$ sudo su postgres -c 'psql -p5436'`
+```sql
+postgres=# \c cycloprod
+You are now connected to database "cycloprod" as user "postgres".
+
+cycloprod=# \dt import.*
+Did not find any relation named "import.*".
+
+cycloprod=# CREATE SCHEMA import;
+
+-- insertion
+cycloprod=# \i '/home/debian/data/tmp/cyclo-data.sql'
+
+-- vérifier que le schéma import est en place
+cycloprod=# \dt import.*
+              List of relations
+ Schema |      Name       | Type  |  Owner   
+--------+-----------------+-------+----------
+ import | choix_periodes  | table | postgres
+ import | choix_personnes | table | postgres
+ import | periodes        | table | postgres
+ import | personnes       | table | postgres
+(4 rows)
+```
+
+## 4. Exécution du script de migration 
+
+Se fait dans la console postgresql, en tant que user postgres, en étant connecté à la base de donnée cible.
+
+On joue pas-à-pas les blocs de la section 'Up' du script `sql/import.sql`
+
+## Tips
+
+- Dans phpstorm, si on veut renommer le schéma pour ne pas tout mélanger, il vaut mieux faire 'Modify schema', car 'Rename' va faire des remplacements partout
+

sql/import.sql

@@ -3,7 +3,12 @@
 -- version v0.6 (== version canevas)
 --
 -- /!\ IMPORTANT
--- Avant de migrer (UP), il faut d'abord avoir importé les codes postaux !!!
+-- * Avant de migrer (UP), il faut d'abord avoir importé les codes postaux !!!
+-- * Adapter les valeurs par défaut
+--   * centres: cfr. 41 et 42
+--   * socialIssues: cfr. 56
+--   * referrer: cfr. 57
+--   * scopes: cfr. 58
 --
 
 
@@ -236,11 +241,6 @@ DO $$
         --SELECT mobilenumber, mobilenumber1, phonenumber, phonenumber1 FROM import.personnes ORDER BY id;
 END $$;
 
--- 35. Prepare required default dates
--- address_validfrom1 | household_startdate1
--- acp_openingdate1
-
-
 -- 40. Prepare id mapping before insertion
 ALTER TABLE import.personnes ADD column person_id BIGINT;
 UPDATE import.personnes SET person_id=personid
@@ -277,6 +277,7 @@ INSERT INTO chill_person_person (
     , -- acceptsms, acceptemail
     */
 FROM import.personnes ip;
+SELECT setval('chill_person_person_id_seq', (SELECT COALESCE(max(id)) FROM chill_person_person));
 
 -- 42. Fill Person center history
 INSERT INTO chill_person_person_center_history (id, person_id, center_id, startdate) SELECT
@@ -290,11 +291,12 @@ INSERT INTO chill_person_person_center_history (id, person_id, center_id, startd
 ALTER TABLE import.personnes ADD column address_id BIGINT;
 UPDATE import.personnes SET address_id = nextval('chill_main_address_id_seq');
 ALTER TABLE import.personnes ADD column postcode_id BIGINT;
-UPDATE import.personnes SET postcode_id = cmpc.id FROM chill_main_postal_code cmpc WHERE cmpc.code = postcode AND cmpc.origin = 0;
+UPDATE import.personnes SET postcode_id = pc.id FROM chill_main_postal_code pc WHERE pc.code = postcode AND pc.origin = 0;
 INSERT INTO chill_main_address (id, postcode_id, street, streetnumber, extra, validFrom) SELECT
     address_id, postcode_id, street, streetnumber, extra, COALESCE(validfrom1 , date(date_trunc('year', CURRENT_DATE)))
     FROM import.personnes WHERE postcode_id IS NOT NULL;
-SELECT ip.person_id, ip.address_id FROM import.personnes ip ;
+SELECT setval('chill_main_address_id_seq', (SELECT COALESCE(max(id)) FROM chill_main_address));
+-- SELECT ip.person_id, ip.address_id FROM import.personnes ip ;
 -- SELECT ip.id, ip.person_id, ip.firstname, ip.lastname, addr.* FROM chill_main_address addr RIGHT JOIN import.personnes ip ON addr.id = ip.address_id ORDER BY ip.id;
 
 -- 44. Add houshold and household member with position
@@ -307,12 +309,156 @@ INSERT INTO chill_person_household_members (id, person_id, household_id, startda
 -- SELECT ip.id, ip.person_id, ip.firstname, ip.lastname, hh.* FROM chill_person_household hh FULL JOIN import.personnes ip ON hh.id = ip.person_id ORDER BY hh.id;
 -- SELECT ip.id, ip.person_id, ip.firstname, ip.lastname, memb.* FROM chill_person_household_members memb JOIN import.personnes ip ON memb.household_id = ip.person_id ORDER BY memb.household_id;
 
--- 45. Add address to household
+-- 45. Link address to household
 INSERT INTO chill_person_household_to_addresses (household_id, address_id) SELECT person_id, address_id
     FROM import.personnes WHERE postcode_id IS NOT NULL; -- cfr (*) adresse insérées seulement si postcode
 
 -- 46. Add household composition
--- (not yet implemented in canvas)
+INSERT INTO chill_person_household_composition (id, household_id, startdate, householdcompositiontype_id) SELECT
+    nextval('chill_person_household_composition_id_seq'), person_id AS household_id, date(date_trunc('year', CURRENT_DATE))
+    , (SELECT id FROM chill_person_household_composition_type ct WHERE household_composition_type1::jsonb->>'fr' = ct.label::jsonb->>'fr')
+FROM import.personnes
+WHERE household_composition_type1 IS NOT NULL ;
+
+
+-- 50. Prepare id mapping before insertion
+ALTER TABLE import.periodes ADD column period_id BIGINT;
+UPDATE import.periodes SET period_id = periodid
+    FROM (SELECT id, nextval('chill_person_accompanying_period_id_seq') AS periodid FROM import.periodes ORDER BY id) AS t
+    WHERE import.periodes.id = t.id;
+--SELECT ipe.id, ipe.fullname, ipe.period_id, ip.id, ip.person_id FROM import.periodes ipe JOIN import.personnes ip ON ipe.id = ip.id ORDER BY ipe.id;
+
+-- 51. Insert in chill_person_accompanying_period
+INSERT INTO chill_person_accompanying_period (id, openingdate, closingdate, step, remark, intensity, createdby_id, createdat, updatedby_id, updatedat) SELECT
+    period_id,
+    COALESCE(openingdate1, date(date_trunc('year', CURRENT_DATE))), closingdate1,
+    'CONFIRMED', COALESCE(TRIM(remark), ''), intensity1,
+    (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM users), CURRENT_DATE,
+    (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM users), CURRENT_DATE
+FROM import.periodes;
+SELECT setval('chill_person_accompanying_period_id_seq', (SELECT max(id) FROM chill_person_accompanying_period));
+
+-- 52. Link participations to periods
+INSERT INTO chill_person_accompanying_period_participation (id, person_id, accompanyingperiod_id, startdate) SELECT
+    nextval('chill_person_accompanying_period_participation_id_seq'), t.person_id, t.period_id, date(date_trunc('year', CURRENT_DATE))
+    FROM (SELECT person_id, period_id FROM import.periodes ip JOIN import.personnes p ON ip.id = p.id ORDER BY person_id) AS t;
+
+-- 53. Prepare temporary address location
+INSERT INTO chill_main_address (id, postcode_id, street, streetnumber, validFrom) SELECT
+    nextval('chill_main_address_id_seq'),
+    (SELECT pc.id FROM chill_main_postal_code pc WHERE pc.code = postcode),
+    street, streetnumber, date(date_trunc('year', CURRENT_DATE))
+    FROM import.choix_periodes WHERE street != '';
+ALTER TABLE import.choix_periodes ADD COLUMN address_location_id BIGINT;
+UPDATE import.choix_periodes SET address_location_id = (SELECT max(id) FROM chill_main_address) WHERE street != '';
+
+-- 54. Link person or temporary address location to periods
+UPDATE chill_person_accompanying_period acp
+    SET addresslocation_id = (SELECT address_location_id FROM import.choix_periodes WHERE address_location_id IS NOT NULL LIMIT 1)
+    FROM import.personnes pson JOIN import.periodes piod ON pson.id = piod.id
+    WHERE piod.period_id = acp.id AND piod.addresslocation = 'oui';
+UPDATE chill_person_accompanying_period acp
+    SET personlocation_id = pson.person_id
+    FROM import.personnes pson JOIN import.periodes piod ON pson.id = piod.id
+    WHERE piod.period_id = acp.id AND piod.personlocation = 'oui';
+
+-- 55. Copy period in period_location_history
+INSERT INTO chill_person_accompanying_period_location_history (id, period_id, startdate, enddate, personlocation_id, addresslocation_id, createdat, createdby_id)
+    SELECT
+        nextval('chill_person_accompanying_period_location_history_id_seq'), id, acp.openingdate, acp.closingdate,
+        acp.personlocation_id, acp.addresslocation_id, acp.createdat, acp.createdby_id
+    FROM chill_person_accompanying_period acp
+    WHERE id NOT IN (SELECT period_id FROM chill_person_accompanying_period_location_history) AND step LIKE 'CONFIRMED' ORDER BY id;
+
+-- 56. Link socialIssues to periods
+INSERT INTO chill_person_accompanying_period_social_issues (accompanyingperiod_id, socialissue_id)
+SELECT
+   DISTINCT ON (t.period_id) t.period_id,
+   COALESCE(
+       t.enfant_id,
+       t.parent_id,
+       1  -- default value ?
+   ) AS socialissue_id
+FROM (
+    SELECT p.period_id,
+        (SELECT id FROM chill_person_social_issue WHERE title::jsonb->>'fr' = icp.parent1::jsonb->>'fr' AND parent_id IS NULL) AS parent_id, icp.parent1,
+        (SELECT id FROM chill_person_social_issue WHERE title::jsonb->>'fr' = icp.enfant1::jsonb->>'fr' AND parent_id =
+            (SELECT id FROM chill_person_social_issue WHERE title::jsonb->>'fr' = icp.parent1::jsonb->>'fr' AND parent_id IS NULL)) AS enfant_id, icp.enfant1
+    FROM import.periodes p
+        JOIN import.choix_periodes icp ON p.acp_socialissues = icp.acp_socialissues
+    ORDER BY id) AS t;
+
+-- 57. Link referrer to periods
+UPDATE chill_person_accompanying_period acp
+    SET user_id = COALESCE(
+        (SELECT id FROM users WHERE users.username = ip.referrer),
+        1 -- default value ?
+    )
+    FROM import.periodes ip WHERE acp.id = ip.period_id;
+--SELECT ip.id, (SELECT id FROM users WHERE users.username = ip.referrer) AS referrer_id, ip.referrer, acp.id as period_id, acp.user_id FROM chill_person_accompanying_period acp JOIN import.periodes ip ON ip.period_id = acp.id ORDER BY ip.id;
+
+-- 58. Link scopes to periods
+INSERT INTO accompanying_periods_scopes (accompanying_period_id, scope_id)
+    SELECT ip.period_id, COALESCE(
+        (SELECT id FROM scopes s WHERE ip.acp_scopes1::jsonb->>'fr' = s.name::jsonb->>'fr'),
+        (SELECT id from scopes s WHERE s.name::jsonb->>'fr' = 'tous') -- default value 'tous'
+    )
+    FROM import.periodes ip;
+
+-- 59. Link origin to periods
+UPDATE chill_person_accompanying_period acp SET origin_id =
+    (SELECT id FROM chill_person_accompanying_period_origin o WHERE o.label::jsonb->>'fr' = ip.origin1::jsonb->>'fr')
+FROM import.periodes ip WHERE acp.id = ip.period_id;
+--SELECT ip.id, ip.origin1, acp.id as period_id, acp.origin_id FROM chill_person_accompanying_period acp JOIN import.periodes ip ON ip.period_id = acp.id ORDER BY ip.id;
+
+-- 60. Link jobs to periods
+UPDATE chill_person_accompanying_period acp SET job_id =
+    (SELECT id FROM chill_main_user_job j WHERE j.label::jsonb->>'fr' = ip.job1::jsonb->>'fr')
+FROM import.periodes ip WHERE acp.id = ip.period_id;
+
+-- 61. Link administrative Location
+-- (to be add in csv)
+
+-- 62. Add and link comments
+INSERT INTO chill_person_accompanying_period_comment (id, accompanyingperiod_id, content, creator_id, createdat, updatedby_id, updatedat)
+    SELECT nextval('chill_person_accompanying_period_comment_id_seq'), period_id, comment1_content,
+        (SELECT user_id FROM chill_person_accompanying_period acp WHERE acp.id = ip.period_id), CURRENT_TIMESTAMP,
+        (SELECT user_id FROM chill_person_accompanying_period acp WHERE acp.id = ip.period_id), CURRENT_TIMESTAMP
+    FROM import.periodes ip WHERE ip.comment1_content != '';
+INSERT INTO chill_person_accompanying_period_comment (id, accompanyingperiod_id, content, creator_id, createdat, updatedby_id, updatedat)
+    SELECT nextval('chill_person_accompanying_period_comment_id_seq'), period_id, comment2_content,
+           (SELECT user_id FROM chill_person_accompanying_period acp WHERE acp.id = ip.period_id), CURRENT_TIMESTAMP,
+           (SELECT user_id FROM chill_person_accompanying_period acp WHERE acp.id = ip.period_id), CURRENT_TIMESTAMP
+    FROM import.periodes ip WHERE ip.comment2_content != '';
+INSERT INTO chill_person_accompanying_period_comment (id, accompanyingperiod_id, content, creator_id, createdat, updatedby_id, updatedat)
+    SELECT nextval('chill_person_accompanying_period_comment_id_seq'), period_id, comment3_content,
+           (SELECT user_id FROM chill_person_accompanying_period acp WHERE acp.id = ip.period_id), CURRENT_TIMESTAMP,
+           (SELECT user_id FROM chill_person_accompanying_period acp WHERE acp.id = ip.period_id), CURRENT_TIMESTAMP
+    FROM import.periodes ip WHERE ip.comment3_content != '';
+INSERT INTO chill_person_accompanying_period_comment (id, accompanyingperiod_id, content, creator_id, createdat, updatedby_id, updatedat)
+    SELECT nextval('chill_person_accompanying_period_comment_id_seq'), period_id, comment4_content,
+           (SELECT user_id FROM chill_person_accompanying_period acp WHERE acp.id = ip.period_id), CURRENT_TIMESTAMP,
+           (SELECT user_id FROM chill_person_accompanying_period acp WHERE acp.id = ip.period_id), CURRENT_TIMESTAMP
+    FROM import.periodes ip WHERE ip.comment4_content != '';
+INSERT INTO chill_person_accompanying_period_comment (id, accompanyingperiod_id, content, creator_id, createdat, updatedby_id, updatedat)
+    SELECT nextval('chill_person_accompanying_period_comment_id_seq'), period_id, comment5_content,
+           (SELECT user_id FROM chill_person_accompanying_period acp WHERE acp.id = ip.period_id), CURRENT_TIMESTAMP,
+           (SELECT user_id FROM chill_person_accompanying_period acp WHERE acp.id = ip.period_id), CURRENT_TIMESTAMP
+    FROM import.periodes ip WHERE ip.comment5_content != '';
+
+-- 63. Link pinned comment to period
+UPDATE import.periodes SET comment1_content = null WHERE comment1_content = '';
+UPDATE import.periodes SET comment2_content = null WHERE comment2_content = '';
+UPDATE import.periodes SET comment3_content = null WHERE comment3_content = '';
+UPDATE import.periodes SET comment4_content = null WHERE comment4_content = '';
+UPDATE import.periodes SET comment5_content = null WHERE comment5_content = '';
+UPDATE chill_person_accompanying_period acp SET pinnedcomment_id =
+    (SELECT id FROM chill_person_accompanying_period_comment com WHERE com.accompanyingperiod_id = acp.id
+        AND com.content = COALESCE(ip.comment5_content, ip.comment4_content, ip.comment3_content, ip.comment2_content, ip.comment1_content)
+     LIMIT 1)
+FROM import.periodes ip WHERE acp.id = ip.period_id;
+
+-- ~~Link closingmotive~~ (to be removed from csv)
 
 
 -- ========================================================================================= --
@@ -321,28 +467,84 @@ INSERT INTO chill_person_household_to_addresses (household_id, address_id) SELEC
 -- DOWN
 --
 
+-- Undo 63.
+UPDATE chill_person_accompanying_period acp SET pinnedcomment_id = null FROM import.periodes ip WHERE acp.id = ip.period_id;
+UPDATE import.periodes SET comment1_content = '' WHERE comment1_content IS NULL;
+UPDATE import.periodes SET comment2_content = '' WHERE comment2_content IS NULL;
+UPDATE import.periodes SET comment3_content = '' WHERE comment3_content IS NULL;
+UPDATE import.periodes SET comment4_content = '' WHERE comment4_content IS NULL;
+UPDATE import.periodes SET comment5_content = '' WHERE comment5_content IS NULL;
+
+-- Undo 62.
+DELETE FROM chill_person_accompanying_period_comment com USING import.periodes ip WHERE com.accompanyingperiod_id = ip.period_id;
+SELECT setval('chill_person_accompanying_period_comment_id_seq', (SELECT COALESCE(max(id), 1) FROM chill_person_accompanying_period_comment));
+
+-- Undo 61.
+
+-- Undo 60.
+UPDATE chill_person_accompanying_period acp SET job_id = null FROM import.periodes ip WHERE acp.id = ip.period_id;
+
+-- Undo 59.
+UPDATE chill_person_accompanying_period acp SET origin_id = null FROM import.periodes ip WHERE ip.period_id = acp.id;
+
+-- Undo 58.
+DELETE FROM accompanying_periods_scopes acs USING import.periodes ip WHERE acs.accompanying_period_id = ip.period_id;
+
+-- Undo 57.
+UPDATE chill_person_accompanying_period acp SET user_id = null FROM import.periodes ip WHERE ip.period_id = acp.id;
+
+-- Undo 56.
+DELETE FROM chill_person_accompanying_period_social_issues asi USING import.periodes ip WHERE asi.accompanyingperiod_id = ip.period_id;
+
+-- Undo 55.
+DELETE FROM chill_person_accompanying_period_location_history history USING import.periodes ip WHERE history.period_id = ip.period_id;
+SELECT setval('chill_person_accompanying_period_location_history_id_seq', (SELECT COALESCE(max(id), 1) FROM chill_person_accompanying_period_location_history));
+
+-- Undo 54.
+UPDATE chill_person_accompanying_period acp SET addresslocation_id = NULL, personlocation_id = NULL FROM import.periodes piod WHERE piod.period_id = acp.id;
+
+-- Undo 53.
+DELETE FROM chill_main_address addr USING import.choix_periodes ic WHERE addr.id = ic.address_location_id;
+SELECT setval('chill_main_address_id_seq', (SELECT COALESCE(max(id), 1) FROM chill_main_address));
+ALTER TABLE import.choix_periodes DROP COLUMN address_location_id;
+
+-- Undo 52.
+DELETE FROM chill_person_accompanying_period_participation part USING import.periodes ip WHERE part.accompanyingperiod_id = ip.period_id;
+SELECT setval('chill_person_accompanying_period_participation_id_seq', (SELECT COALESCE(max(id), 1) FROM chill_person_accompanying_period_participation));
+
+-- Undo 51.
+DELETE FROM chill_person_accompanying_period acp USING import.periodes ip WHERE acp.id = ip.period_id;
+SELECT setval('chill_person_accompanying_period_id_seq', (SELECT COALESCE(max(id), 1) FROM chill_person_accompanying_period));
+
+-- Undo 50.
+ALTER TABLE import.periodes DROP column period_id;
+
+-- Undo 46.
+DELETE FROM chill_person_household_composition c USING import.personnes ip WHERE c.household_id = ip.person_id;
+SELECT setval('chill_person_household_composition_id_seq', (SELECT COALESCE(max(id),1) FROM chill_person_household_composition));
+
 -- Undo 45.
-DELETE FROM chill_person_household_to_addresses hhaddr USING import.personnes ip WHERE hhaddr.household_id = ip.person_id ;
+DELETE FROM chill_person_household_to_addresses hhaddr USING import.personnes ip WHERE hhaddr.household_id = ip.person_id;
 
 -- Undo 44.
 DELETE FROM chill_person_household_members memb USING import.personnes ip WHERE memb.person_id = ip.person_id;
-SELECT setval('chill_person_household_members_id_seq', (SELECT max(id) FROM chill_person_household_members));
+SELECT setval('chill_person_household_members_id_seq', (SELECT COALESCE(max(id),1) FROM chill_person_household_members));
 DELETE FROM chill_person_household hh USING import.personnes ip WHERE hh.id = ip.person_id;
-SELECT setval('chill_person_household_id_seq', (SELECT max(id) FROM chill_person_household));
+SELECT setval('chill_person_household_id_seq', (SELECT COALESCE(max(id),1) FROM chill_person_household));
 
 -- Undo 43.
 DELETE FROM chill_main_address addr USING import.personnes ip WHERE addr.id = ip.address_id;
-SELECT setval('chill_main_address_id_seq', (SELECT max(id) FROM chill_main_address));
+SELECT setval('chill_main_address_id_seq', (SELECT COALESCE(max(id),1) FROM chill_main_address));
 ALTER TABLE import.personnes DROP column postcode_id;
 ALTER TABLE import.personnes DROP column address_id;
 
 -- Undo 42.
 DELETE FROM chill_person_person_center_history hist USING import.personnes ip WHERE hist.person_id = ip.person_id;
-SELECT setval('chill_person_person_center_history_id_seq', (SELECT max(id) FROM chill_person_person_center_history));
+SELECT setval('chill_person_person_center_history_id_seq', (SELECT COALESCE(max(id),1) FROM chill_person_person_center_history));
 
 -- Undo 41.
 DELETE FROM chill_person_person p USING import.personnes ip WHERE p.id = ip.person_id;
-SELECT setval('chill_person_person_id_seq', (SELECT max(id) FROM chill_person_person));
+SELECT setval('chill_person_person_id_seq', (SELECT COALESCE(max(id),1) FROM chill_person_person));
 
 -- Undo 40.
 ALTER TABLE import.personnes DROP COLUMN person_id;
@@ -396,8 +598,8 @@ ALTER TABLE import.personnes DROP COLUMN civility1;
 
 -- Undo 13.
 DELETE FROM chill_person_social_issue USING import.choix_periodes i
-    WHERE parent_id IN ( SELECT id FROM chill_person_social_issue cpsi WHERE cpsi.title::jsonb = i.parent1::jsonb AND cpsi.parent_id IS NULL );
+    WHERE parent_id IN ( SELECT id FROM chill_person_social_issue cpsi WHERE cpsi.title::jsonb->>'fr' = i.parent1::jsonb->>'fr' AND cpsi.parent_id IS NULL );
-DELETE FROM chill_person_social_issue cpsi USING import.choix_periodes icp WHERE cpsi.title::jsonb = icp.parent1::jsonb AND cpsi.parent_id IS NULL ;
+DELETE FROM chill_person_social_issue cpsi USING import.choix_periodes icp WHERE cpsi.title::jsonb->>'fr' = icp.parent1::jsonb->>'fr' AND cpsi.parent_id IS NULL ;
 SELECT setval('chill_person_social_issue_id_seq', (SELECT COALESCE(max(id),1) FROM chill_person_social_issue));
 ALTER TABLE import.choix_periodes DROP COLUMN enfant1;
 ALTER TABLE import.choix_periodes DROP COLUMN parent1;
@@ -407,46 +609,54 @@ DELETE FROM users USING import.choix_periodes icp WHERE users.username::text = i
 SELECT setval('users_id_seq', (SELECT COALESCE(max(id),1) FROM users));
 
 -- Undo 11.
-DELETE FROM scopes USING import.choix_periodes icp WHERE scopes.name::jsonb = icp.acp_scopes1::jsonb;
+DELETE FROM scopes USING import.choix_periodes icp WHERE scopes.name::jsonb->>'fr' = icp.acp_scopes1::jsonb->>'fr'
+    AND NOT EXISTS (SELECT 1 FROM import.choix_periodes WHERE scopes.name::jsonb->>'fr' = icp.acp_scopes1::jsonb->>'fr');
 SELECT setval('scopes_id_seq', (SELECT COALESCE(max(id),1) FROM scopes));
 ALTER TABLE import.choix_periodes DROP COLUMN acp_scopes1;
 
 -- Undo 10.
-DELETE FROM chill_main_user_job cmuj USING import.choix_periodes icp WHERE cmuj.label::jsonb = icp.job1::jsonb;
+DELETE FROM chill_main_user_job cmuj USING import.choix_periodes icp WHERE cmuj.label::jsonb->>'fr' = icp.job1::jsonb->>'fr'
+    AND NOT EXISTS (SELECT 1 FROM import.choix_periodes WHERE cmuj.label::jsonb->>'fr' = icp.job1::jsonb->>'fr');
 SELECT setval('chill_main_user_job_id_seq', (SELECT COALESCE(max(id),1) FROM chill_main_user_job));
 ALTER TABLE import.choix_periodes DROP COLUMN job1;
 
 -- Undo 9.
-DELETE FROM chill_person_accompanying_period_origin cpapo USING import.choix_periodes icp WHERE cpapo.label::jsonb = icp.origin1::jsonb;
+DELETE FROM chill_person_accompanying_period_origin cpapo USING import.choix_periodes icp WHERE cpapo.label::jsonb->>'fr' = icp.origin1::jsonb->>'fr'
+    AND NOT EXISTS (SELECT 1 FROM import.choix_periodes WHERE cpapo.label::jsonb->>'fr' = icp.origin1::jsonb->>'fr');
 SELECT setval('chill_person_accompanying_period_origin_id_seq', (SELECT COALESCE(max(id),1) FROM chill_person_accompanying_period_origin));
 ALTER TABLE import.choix_periodes DROP COLUMN origin1;
 
 -- Undo 8.
-DELETE FROM chill_person_accompanying_period_closingmotive cpapcm USING import.choix_periodes icp WHERE cpapcm.name::jsonb = icp.closingmotive1::jsonb;
+DELETE FROM chill_person_accompanying_period_closingmotive cpapcm USING import.choix_periodes icp WHERE cpapcm.name::jsonb->>'fr' = icp.closingmotive1::jsonb->>'fr'
+    AND NOT EXISTS (SELECT 1 FROM import.choix_periodes WHERE cpapcm.name::jsonb->>'fr' = icp.closingmotive1::jsonb->>'fr');
 SELECT setval('chill_person_accompanying_period_closingmotive_id_seq', (SELECT COALESCE(max(id),1) FROM chill_person_accompanying_period_closingmotive));
 ALTER TABLE import.choix_periodes DROP COLUMN closingmotive1;
 
 -- Undo 7.
-DELETE FROM chill_person_household_position cphp USING import.choix_personnes icp WHERE cphp.label::jsonb = icp.household_position1::jsonb;
+DELETE FROM chill_person_household_position cphp USING import.choix_personnes icp WHERE cphp.label::jsonb->>'fr' = icp.household_position1::jsonb->>'fr'
+    AND NOT EXISTS (SELECT 1 FROM import.choix_personnes WHERE cphp.label::jsonb->>'fr' = icp.household_position1::jsonb->>'fr');
 SELECT setval('chill_person_household_position_id_seq', (SELECT COALESCE(max(id),1) FROM chill_person_household_position));
 ALTER TABLE import.choix_personnes DROP COLUMN household_position1;
 
 -- Undo 6.
-DELETE FROM chill_person_household_composition_type cphct USING import.choix_personnes icp WHERE cphct.label::jsonb = icp.household_composition_type1::jsonb AND cphct.id > 6;
+DELETE FROM chill_person_household_composition_type cphct USING import.choix_personnes icp WHERE cphct.label::jsonb->>'fr' = icp.household_composition_type1::jsonb->>'fr'
+    AND NOT EXISTS (SELECT 1 FROM import.choix_personnes WHERE cphct.label::jsonb->>'fr' = icp.household_composition_type1::jsonb->>'fr');
 SELECT setval('chill_person_household_composition_type_id_seq', (SELECT COALESCE(max(id),1) FROM chill_person_household_composition_type));
 ALTER TABLE import.choix_personnes DROP COLUMN household_composition_type1;
 
 -- Undo 5.
-DELETE FROM chill_person_marital_status cpms USING import.choix_personnes icp WHERE cpms.name::jsonb = icp.maritalstatus1::jsonb AND cpms.id <> 'célibat.' AND cpms.id <> 'marié'; -- existing marital status
+DELETE FROM chill_person_marital_status cpms USING import.choix_personnes icp WHERE cpms.name::jsonb->>'fr' = icp.maritalstatus1::jsonb->>'fr'
+    AND NOT EXISTS (SELECT 1 FROM import.choix_personnes WHERE cpms.name::jsonb->>'fr' = icp.maritalstatus1::jsonb->>'fr');
 ALTER TABLE import.choix_personnes DROP COLUMN maritalstatus1;
 
 -- Undo 4.
-DELETE FROM country USING import.choix_personnes icp WHERE country.name::jsonb = icp.country1::jsonb AND country.id > 249; -- 249 existing countries
+DELETE FROM country USING import.choix_personnes icp WHERE country.name::jsonb->>'fr' = icp.country1::jsonb->>'fr' AND country.id > 249; -- 249 existing countries
 SELECT setval('country_id_seq', (SELECT COALESCE(max(id),1) FROM country));
 ALTER TABLE import.choix_personnes DROP COLUMN country1;
 
 -- Undo 3.
-DELETE FROM chill_main_civility cmc USING import.choix_personnes ip WHERE cmc.name::jsonb = ip.civility1::jsonb;
+DELETE FROM chill_main_civility cmc USING import.choix_personnes ip WHERE cmc.name::jsonb->>'fr' = ip.civility1::jsonb->>'fr'
+    AND NOT EXISTS (SELECT 1 FROM import.choix_personnes WHERE cmc.name::jsonb->>'fr' = ip.civility1::jsonb->>'fr');
 SELECT setval('chill_main_civility_id_seq', (SELECT COALESCE(max(id),1) FROM chill_main_civility));
 ALTER TABLE import.choix_personnes DROP COLUMN civility1;
 
@@ -459,8 +669,6 @@ ALTER TABLE import.periodes DROP COLUMN openingdate1;
 ALTER TABLE import.periodes DROP COLUMN closingdate1;
 
 
--- -------------
--- tiers     choices_list:  civility kind profession category
 
 -- =============
 -- QUESTIONS