Ajout de la gestion des actions sociales dans l'importation des données et correction des colonnes dans les fichiers CSV

csv/choix_periodes.dist.csv

@@ -1 +1 @@
-closingmotive,origin,acp_scopes,job,referrer,parent,enfant,acp_socialissues,work_socialactions,street,extra,streetnumber,postcode,country,address_text,endcol
+closingmotive,origin,acp_scopes,job,referrer,parent,enfant,acp_social_issues,work_social_action,street,extra,streetnumber,postcode,country

import_all_csv.sh

@@ -0,0 +1,112 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+DB_HOST="${PGHOST:-localhost}"
+DB_PORT="${PGPORT:-5432}"
+DB_USER="${PGUSER:-postgres}"
+DB_NAME="${PGDATABASE:-chill-import}"
+
+PSQL=(psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME")
+
+required_tables=(
+  "import.personnes"
+  "import.choix_periodes"
+  "import.choix_localisations"
+  "import.tiers"
+)
+
+required_files=(
+  "$ROOT_DIR/csv/choix_usagers.csv"
+  "$ROOT_DIR/csv/choix_periodes.csv"
+  "$ROOT_DIR/csv/choix_localisations.csv"
+  "$ROOT_DIR/csv/choix_tiers.csv"
+)
+
+for table in "${required_tables[@]}"; do
+  exists="$("${PSQL[@]}" -tAc "SELECT to_regclass('${table}') IS NOT NULL;")"
+  if [[ "$exists" != "t" ]]; then
+    echo "Table manquante: ${table}" >&2
+    echo "Lance d'abord: psql -h ${DB_HOST} -p ${DB_PORT} -U ${DB_USER} -d ${DB_NAME} -f sql/prepare-import.sql" >&2
+    exit 1
+  fi
+done
+
+for csv_file in "${required_files[@]}"; do
+  if [[ ! -f "$csv_file" ]]; then
+    echo "Fichier CSV manquant: ${csv_file#$ROOT_DIR/}" >&2
+    exit 1
+  fi
+done
+
+echo "[0/4] Assouplissement des types texte des tables import..."
+"${PSQL[@]}" <<'SQL'
+DO $$
+DECLARE r record;
+BEGIN
+  FOR r IN
+    SELECT table_schema, table_name, column_name
+    FROM information_schema.columns
+    WHERE table_schema = 'import'
+      AND table_name IN ('personnes', 'choix_periodes', 'choix_localisations', 'tiers')
+      AND data_type IN ('character varying', 'character')
+  LOOP
+    EXECUTE format(
+      'ALTER TABLE %I.%I ALTER COLUMN %I TYPE text',
+      r.table_schema,
+      r.table_name,
+      r.column_name
+    );
+  END LOOP;
+END $$;
+SQL
+
+echo "[1/4] Vidage des tables d'import..."
+"${PSQL[@]}" <<'SQL'
+TRUNCATE TABLE
+  import.personnes,
+  import.choix_periodes,
+  import.choix_localisations,
+  import.tiers
+RESTART IDENTITY;
+SQL
+
+echo "[2/4] Import des CSV du dossier csv/..."
+
+copy_table_from_file() {
+  local table="$1"
+  local file_path="$2"
+  local columns="${3:-}"
+  local target="$table"
+  if [[ -n "$columns" ]]; then
+    target="${table}(${columns})"
+  fi
+  echo "  - import ${table} <= ${file_path#$ROOT_DIR/}"
+  "${PSQL[@]}" -c "\\copy ${target} FROM STDIN WITH (FORMAT csv, HEADER true, DELIMITER ',')" < "$file_path"
+}
+
+copy_table_from_file "import.personnes" "$ROOT_DIR/csv/choix_usagers.csv" "civility,lastname,firstname,gender,gendercomment,nationality,memo,birthdate,place_of_birth,countryofbirth,deathdate,email,phonenumber,mobilenumber,contactinfo,street,extra,streetnumber,postcode,country"
+copy_table_from_file "import.choix_periodes" "$ROOT_DIR/csv/choix_periodes.csv" "closingmotive,origin,acp_scopes,job,referrer,parent,enfant,acp_social_issues,work_social_action,street,extra,streetnumber,postcode,country"
+copy_table_from_file "import.choix_localisations" "$ROOT_DIR/csv/choix_localisations.csv" "title,addressRequired,availableForUsers,contactData,defaultFor,editableByUsers"
+
+echo "[3/4] Import de tiers via table staging..."
+"${PSQL[@]}" -c "\copy import.tiers_stage(coordonnees,nom,categorie) FROM STDIN WITH (FORMAT csv, HEADER true, DELIMITER ',')" < "$ROOT_DIR/csv/choix_tiers.csv"
+
+"${PSQL[@]}" <<'SQL'
+INSERT INTO import.tiers (id, coordonnees, nom, categorie)
+SELECT
+  row_number() OVER (ORDER BY nom) AS id,
+  NULLIF(trim(coordonnees), ''),
+  NULLIF(trim(nom), ''),
+  NULLIF(trim(categorie), '')
+FROM import.tiers_stage
+WHERE COALESCE(trim(nom), '') <> '';
+
+DROP TABLE import.tiers_stage;
+SQL
+
+echo "[4/4] Verification des volumes importes..."
+"${PSQL[@]}" -c "SELECT 'personnes' AS table_name, count(*) AS rows FROM import.personnes UNION ALL SELECT 'choix_periodes', count(*) FROM import.choix_periodes UNION ALL SELECT 'choix_localisations', count(*) FROM import.choix_localisations UNION ALL SELECT 'tiers', count(*) FROM import.tiers;"
+
+echo "Import termine sans doublons (tables videes au debut)."

sql/import.sql

@@ -137,16 +137,69 @@ INSERT INTO chill_person_social_issue (id, parent_id, title, ordering)
 WITH max_ordering AS ( SELECT MAX(ordering) as max_ordering FROM chill_person_social_issue )
 INSERT INTO chill_person_social_issue (id, parent_id, title, ordering)
     SELECT nextval('chill_person_social_issue_id_seq'),
-        ( SELECT id FROM chill_person_social_issue WHERE parent_id IS NULL AND title::jsonb->>'fr' = t.parent1::jsonb->>'fr' ), t.enfant1,
+        ( SELECT MIN(id) FROM chill_person_social_issue WHERE parent_id IS NULL AND title::jsonb->>'fr' = t.parent1::jsonb->>'fr' ), t.enfant1,
         max_ordering.max_ordering + row_number() OVER () as ordering
     FROM ( SELECT DISTINCT ON (acp_social_issues) parent1, enfant1 FROM import.choix_periodes WHERE enfant1 IS NOT NULL ) t
     CROSS JOIN max_ordering
     WHERE NOT EXISTS ( SELECT 1 FROM chill_person_social_issue
         WHERE title::jsonb->>'fr' = t.enfant1::jsonb->>'fr'
-        AND parent_id = (SELECT id FROM chill_person_social_issue WHERE title::jsonb->>'fr' = t.parent1::jsonb->>'fr'));
+        AND parent_id = (SELECT MIN(id) FROM chill_person_social_issue WHERE parent_id IS NULL AND title::jsonb->>'fr' = t.parent1::jsonb->>'fr'));
 
 -- 14. Complete table WorkSocialActions
--- (not yet implemented in canvas)
+ALTER TABLE import.choix_periodes ADD COLUMN work_social_action1 JSONB;
+UPDATE import.choix_periodes SET work_social_action1=json_build_object('fr', trim(work_social_action)) WHERE work_social_action!='';
+ALTER TABLE import.choix_periodes ADD COLUMN work_social_issue1 JSONB;
+WITH grouped AS (
+    SELECT ctid,
+        SUM(CASE WHEN NULLIF(trim(acp_social_issues), '') IS NOT NULL THEN 1 ELSE 0 END)
+            OVER (ORDER BY ctid) AS grp
+    FROM import.choix_periodes
+), resolved AS (
+    SELECT cp.ctid,
+        COALESCE(
+            MAX(NULLIF(upper(trim(cp.enfant)), '')) OVER (PARTITION BY g.grp),
+            MAX(NULLIF(upper(trim(cp.parent)), '')) OVER (PARTITION BY g.grp)
+        ) AS issue_title
+    FROM import.choix_periodes cp
+    JOIN grouped g ON cp.ctid = g.ctid
+)
+UPDATE import.choix_periodes cp
+SET work_social_issue1 = json_build_object('fr', r.issue_title)
+FROM resolved r
+WHERE cp.ctid = r.ctid
+    AND r.issue_title IS NOT NULL
+    AND trim(COALESCE(cp.work_social_action, '')) != '';
+WITH max_ordering AS ( SELECT COALESCE(MAX(ordering), 0) as max_ordering FROM chill_person_social_action )
+INSERT INTO chill_person_social_action (id, issue_id, parent_id, desactivationdate, defaultnotificationdelay, title, ordering)
+    SELECT nextval('chill_person_social_action_id_seq'),
+        issue.id,
+        null,
+        null,
+        null,
+        t.work_social_action1,
+        max_ordering.max_ordering + row_number() OVER () as ordering
+    FROM (
+        SELECT DISTINCT ON (work_social_action1, work_social_issue1) work_social_action1, work_social_issue1
+        FROM import.choix_periodes
+        WHERE work_social_action1 IS NOT NULL
+    ) t
+    LEFT JOIN LATERAL (
+        SELECT si.id
+        FROM chill_person_social_issue si
+        WHERE si.title::jsonb->>'fr' = t.work_social_issue1::jsonb->>'fr'
+        ORDER BY (si.parent_id IS NULL), si.id
+        LIMIT 1
+    ) issue ON true
+    CROSS JOIN max_ordering
+    WHERE NOT EXISTS (
+        SELECT 1
+        FROM chill_person_social_action sa
+        WHERE sa.title::jsonb->>'fr' = t.work_social_action1::jsonb->>'fr'
+            AND (
+                (sa.issue_id IS NULL AND issue.id IS NULL)
+                OR sa.issue_id = issue.id
+            )
+    );
 
 -- 20. Prepare personnes civility
 ALTER TABLE import.personnes ADD COLUMN civility1 jsonb;
@@ -452,6 +505,56 @@ UPDATE chill_person_accompanying_period acp
     FROM import.periodes ip WHERE acp.id = ip.period_id;
 --SELECT ip.id, (SELECT id FROM users WHERE users.username = ip.referrer) AS referrer_id, ip.referrer, acp.id as period_id, acp.user_id FROM chill_person_accompanying_period acp JOIN import.periodes ip ON ip.period_id = acp.id ORDER BY ip.id;
 
+-- 57bis. Link work social actions to periods
+INSERT INTO chill_person_accompanying_period_work (
+    id,
+    note,
+    createdat,
+    startdate,
+    enddate,
+    createdautomatically,
+    createdautomaticallyreason,
+    accompanyingperiod_id,
+    socialaction_id,
+    createdby_id,
+    handlingthierparty_id,
+    updatedat,
+    updatedby_id,
+    privatecomment_comments,
+    version
+)
+    SELECT
+        nextval('chill_person_accompanying_period_work_id_seq'),
+        '',
+        CURRENT_TIMESTAMP,
+        COALESCE(ip.openingdate1, date(date_trunc('year', CURRENT_DATE))),
+        ip.closingdate1,
+        true,
+        'Imported from canvas',
+        ip.period_id,
+        sa.id,
+        COALESCE(acp.user_id, (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM users)),
+        null,
+        CURRENT_TIMESTAMP,
+        COALESCE(acp.user_id, (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM users)),
+        '{}'::json,
+        1
+    FROM import.periodes ip
+    JOIN chill_person_accompanying_period acp ON acp.id = ip.period_id
+    JOIN LATERAL (
+        SELECT MIN(id) AS id
+        FROM chill_person_social_action s
+        WHERE s.title::jsonb->>'fr' = trim(ip.work_socialaction)
+    ) sa ON sa.id IS NOT NULL
+    WHERE trim(COALESCE(ip.work_socialaction, '')) != ''
+      AND NOT EXISTS (
+          SELECT 1
+          FROM chill_person_accompanying_period_work w
+          WHERE w.accompanyingperiod_id = ip.period_id
+            AND w.socialaction_id = sa.id
+      );
+SELECT setval('chill_person_accompanying_period_work_id_seq', (SELECT COALESCE(max(id), 1) FROM chill_person_accompanying_period_work));
+
 -- 58. Link scopes to periods
 INSERT INTO accompanying_periods_scopes (accompanying_period_id, scope_id)
     SELECT ip.period_id, COALESCE(
@@ -770,7 +873,7 @@ INSERT INTO chill_3party.third_party
     NULL, -- parent
     (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM users),
     NULL, -- service/dpt
-    NULL,
+    t.acronym,
     CURRENT_DATE,
     CURRENT_DATE,
     (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM users),
@@ -877,6 +980,10 @@ DELETE FROM accompanying_periods_scopes acs USING import.periodes ip WHERE acs.a
 -- Undo 57.
 UPDATE chill_person_accompanying_period acp SET user_id = null FROM import.periodes ip WHERE ip.period_id = acp.id;
 
+-- Undo 57bis.
+DELETE FROM chill_person_accompanying_period_work w USING import.periodes ip WHERE w.accompanyingperiod_id = ip.period_id;
+SELECT setval('chill_person_accompanying_period_work_id_seq', (SELECT COALESCE(max(id), 1) FROM chill_person_accompanying_period_work));
+
 -- Undo 56.
 DELETE FROM chill_person_accompanying_period_social_issues asi USING import.periodes ip WHERE asi.accompanyingperiod_id = ip.period_id;
 
@@ -979,6 +1086,23 @@ ALTER TABLE import.personnes DROP COLUMN gender1;
 ALTER TABLE import.personnes DROP COLUMN civility1;
 
 -- Undo 14.
+DELETE FROM chill_person_social_action sa
+USING import.choix_periodes icp
+LEFT JOIN LATERAL (
+    SELECT si.id
+    FROM chill_person_social_issue si
+    WHERE si.title::jsonb->>'fr' = icp.work_social_issue1::jsonb->>'fr'
+    ORDER BY (si.parent_id IS NULL), si.id
+    LIMIT 1
+) issue ON true
+WHERE sa.title::jsonb->>'fr' = icp.work_social_action1::jsonb->>'fr'
+    AND (
+        (sa.issue_id IS NULL AND issue.id IS NULL)
+        OR sa.issue_id = issue.id
+    );
+SELECT setval('chill_person_social_action_id_seq', (SELECT COALESCE(max(id),1) FROM chill_person_social_action));
+ALTER TABLE import.choix_periodes DROP COLUMN work_social_issue1;
+ALTER TABLE import.choix_periodes DROP COLUMN work_social_action1;
 
 -- Undo 13.
 DELETE FROM chill_person_social_issue USING import.choix_periodes i

sql/prepare-import.sql

@@ -119,11 +119,12 @@ CREATE TABLE "import".users (
 );
 
 CREATE TABLE  "import".tiers (
-    ID INT PRIMARY KEY,
+    ID varchar(50) NULL,
     CATEGORIE VARCHAR(255),
     SECTEUR_AS VARCHAR(255),
     COMMUNE VARCHAR(255),
     NOM VARCHAR(255),
+    ACRONYM VARCHAR(64),
     PHONENUMBER VARCHAR(20),
     PHONENUMBER_2 VARCHAR(20),
     EMAIL VARCHAR(255),