diff --git a/csv/choix_periodes.dist.csv b/csv/choix_periodes.dist.csv old mode 100755 new mode 100644 index a593da1..8859d78 --- a/csv/choix_periodes.dist.csv +++ b/csv/choix_periodes.dist.csv @@ -1 +1 @@ -closingmotive,origin,acp_scopes,job,referrer,parent,enfant,acp_socialissues,work_socialactions,street,extra,streetnumber,postcode,country,address_text,endcol +closingmotive,origin,acp_scopes,job,referrer,parent,enfant,acp_social_issues,work_social_action,street,extra,streetnumber,postcode,country diff --git a/import_all_csv.sh b/import_all_csv.sh new file mode 100755 index 0000000..7f0e940 --- /dev/null +++ b/import_all_csv.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +DB_HOST="${PGHOST:-localhost}" +DB_PORT="${PGPORT:-5432}" +DB_USER="${PGUSER:-postgres}" +DB_NAME="${PGDATABASE:-chill-import}" + +PSQL=(psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME") + +required_tables=( + "import.personnes" + "import.choix_periodes" + "import.choix_localisations" + "import.tiers" +) + +required_files=( + "$ROOT_DIR/csv/choix_usagers.csv" + "$ROOT_DIR/csv/choix_periodes.csv" + "$ROOT_DIR/csv/choix_localisations.csv" + "$ROOT_DIR/csv/choix_tiers.csv" +) + +for table in "${required_tables[@]}"; do + exists="$("${PSQL[@]}" -tAc "SELECT to_regclass('${table}') IS NOT NULL;")" + if [[ "$exists" != "t" ]]; then + echo "Table manquante: ${table}" >&2 + echo "Lance d'abord: psql -h ${DB_HOST} -p ${DB_PORT} -U ${DB_USER} -d ${DB_NAME} -f sql/prepare-import.sql" >&2 + exit 1 + fi +done + +for csv_file in "${required_files[@]}"; do + if [[ ! -f "$csv_file" ]]; then + echo "Fichier CSV manquant: ${csv_file#$ROOT_DIR/}" >&2 + exit 1 + fi +done + +echo "[0/4] Assouplissement des types texte des tables import..." +"${PSQL[@]}" <<'SQL' +DO $$ +DECLARE r record; +BEGIN + FOR r IN + SELECT table_schema, table_name, column_name + FROM information_schema.columns + WHERE table_schema = 'import' + AND table_name IN ('personnes', 'choix_periodes', 'choix_localisations', 'tiers') + AND data_type IN ('character varying', 'character') + LOOP + EXECUTE format( + 'ALTER TABLE %I.%I ALTER COLUMN %I TYPE text', + r.table_schema, + r.table_name, + r.column_name + ); + END LOOP; +END $$; +SQL + +echo "[1/4] Vidage des tables d'import..." +"${PSQL[@]}" <<'SQL' +TRUNCATE TABLE + import.personnes, + import.choix_periodes, + import.choix_localisations, + import.tiers +RESTART IDENTITY; +SQL + +echo "[2/4] Import des CSV du dossier csv/..." + +copy_table_from_file() { + local table="$1" + local file_path="$2" + local columns="${3:-}" + local target="$table" + if [[ -n "$columns" ]]; then + target="${table}(${columns})" + fi + echo " - import ${table} <= ${file_path#$ROOT_DIR/}" + "${PSQL[@]}" -c "\\copy ${target} FROM STDIN WITH (FORMAT csv, HEADER true, DELIMITER ',')" < "$file_path" +} + +copy_table_from_file "import.personnes" "$ROOT_DIR/csv/choix_usagers.csv" "civility,lastname,firstname,gender,gendercomment,nationality,memo,birthdate,place_of_birth,countryofbirth,deathdate,email,phonenumber,mobilenumber,contactinfo,street,extra,streetnumber,postcode,country" +copy_table_from_file "import.choix_periodes" "$ROOT_DIR/csv/choix_periodes.csv" "closingmotive,origin,acp_scopes,job,referrer,parent,enfant,acp_social_issues,work_social_action,street,extra,streetnumber,postcode,country" +copy_table_from_file "import.choix_localisations" "$ROOT_DIR/csv/choix_localisations.csv" "title,addressRequired,availableForUsers,contactData,defaultFor,editableByUsers" + +echo "[3/4] Import de tiers via table staging..." +"${PSQL[@]}" -c "\copy import.tiers_stage(coordonnees,nom,categorie) FROM STDIN WITH (FORMAT csv, HEADER true, DELIMITER ',')" < "$ROOT_DIR/csv/choix_tiers.csv" + +"${PSQL[@]}" <<'SQL' +INSERT INTO import.tiers (id, coordonnees, nom, categorie) +SELECT + row_number() OVER (ORDER BY nom) AS id, + NULLIF(trim(coordonnees), ''), + NULLIF(trim(nom), ''), + NULLIF(trim(categorie), '') +FROM import.tiers_stage +WHERE COALESCE(trim(nom), '') <> ''; + +DROP TABLE import.tiers_stage; +SQL + +echo "[4/4] Verification des volumes importes..." +"${PSQL[@]}" -c "SELECT 'personnes' AS table_name, count(*) AS rows FROM import.personnes UNION ALL SELECT 'choix_periodes', count(*) FROM import.choix_periodes UNION ALL SELECT 'choix_localisations', count(*) FROM import.choix_localisations UNION ALL SELECT 'tiers', count(*) FROM import.tiers;" + +echo "Import termine sans doublons (tables videes au debut)." diff --git a/sql/import.sql b/sql/import.sql index fafc620..927c584 100755 --- a/sql/import.sql +++ b/sql/import.sql @@ -137,16 +137,69 @@ INSERT INTO chill_person_social_issue (id, parent_id, title, ordering) WITH max_ordering AS ( SELECT MAX(ordering) as max_ordering FROM chill_person_social_issue ) INSERT INTO chill_person_social_issue (id, parent_id, title, ordering) SELECT nextval('chill_person_social_issue_id_seq'), - ( SELECT id FROM chill_person_social_issue WHERE parent_id IS NULL AND title::jsonb->>'fr' = t.parent1::jsonb->>'fr' ), t.enfant1, + ( SELECT MIN(id) FROM chill_person_social_issue WHERE parent_id IS NULL AND title::jsonb->>'fr' = t.parent1::jsonb->>'fr' ), t.enfant1, max_ordering.max_ordering + row_number() OVER () as ordering FROM ( SELECT DISTINCT ON (acp_social_issues) parent1, enfant1 FROM import.choix_periodes WHERE enfant1 IS NOT NULL ) t CROSS JOIN max_ordering WHERE NOT EXISTS ( SELECT 1 FROM chill_person_social_issue WHERE title::jsonb->>'fr' = t.enfant1::jsonb->>'fr' - AND parent_id = (SELECT id FROM chill_person_social_issue WHERE title::jsonb->>'fr' = t.parent1::jsonb->>'fr')); + AND parent_id = (SELECT MIN(id) FROM chill_person_social_issue WHERE parent_id IS NULL AND title::jsonb->>'fr' = t.parent1::jsonb->>'fr')); -- 14. Complete table WorkSocialActions --- (not yet implemented in canvas) +ALTER TABLE import.choix_periodes ADD COLUMN work_social_action1 JSONB; +UPDATE import.choix_periodes SET work_social_action1=json_build_object('fr', trim(work_social_action)) WHERE work_social_action!=''; +ALTER TABLE import.choix_periodes ADD COLUMN work_social_issue1 JSONB; +WITH grouped AS ( + SELECT ctid, + SUM(CASE WHEN NULLIF(trim(acp_social_issues), '') IS NOT NULL THEN 1 ELSE 0 END) + OVER (ORDER BY ctid) AS grp + FROM import.choix_periodes +), resolved AS ( + SELECT cp.ctid, + COALESCE( + MAX(NULLIF(upper(trim(cp.enfant)), '')) OVER (PARTITION BY g.grp), + MAX(NULLIF(upper(trim(cp.parent)), '')) OVER (PARTITION BY g.grp) + ) AS issue_title + FROM import.choix_periodes cp + JOIN grouped g ON cp.ctid = g.ctid +) +UPDATE import.choix_periodes cp +SET work_social_issue1 = json_build_object('fr', r.issue_title) +FROM resolved r +WHERE cp.ctid = r.ctid + AND r.issue_title IS NOT NULL + AND trim(COALESCE(cp.work_social_action, '')) != ''; +WITH max_ordering AS ( SELECT COALESCE(MAX(ordering), 0) as max_ordering FROM chill_person_social_action ) +INSERT INTO chill_person_social_action (id, issue_id, parent_id, desactivationdate, defaultnotificationdelay, title, ordering) + SELECT nextval('chill_person_social_action_id_seq'), + issue.id, + null, + null, + null, + t.work_social_action1, + max_ordering.max_ordering + row_number() OVER () as ordering + FROM ( + SELECT DISTINCT ON (work_social_action1, work_social_issue1) work_social_action1, work_social_issue1 + FROM import.choix_periodes + WHERE work_social_action1 IS NOT NULL + ) t + LEFT JOIN LATERAL ( + SELECT si.id + FROM chill_person_social_issue si + WHERE si.title::jsonb->>'fr' = t.work_social_issue1::jsonb->>'fr' + ORDER BY (si.parent_id IS NULL), si.id + LIMIT 1 + ) issue ON true + CROSS JOIN max_ordering + WHERE NOT EXISTS ( + SELECT 1 + FROM chill_person_social_action sa + WHERE sa.title::jsonb->>'fr' = t.work_social_action1::jsonb->>'fr' + AND ( + (sa.issue_id IS NULL AND issue.id IS NULL) + OR sa.issue_id = issue.id + ) + ); -- 20. Prepare personnes civility ALTER TABLE import.personnes ADD COLUMN civility1 jsonb; @@ -452,6 +505,56 @@ UPDATE chill_person_accompanying_period acp FROM import.periodes ip WHERE acp.id = ip.period_id; --SELECT ip.id, (SELECT id FROM users WHERE users.username = ip.referrer) AS referrer_id, ip.referrer, acp.id as period_id, acp.user_id FROM chill_person_accompanying_period acp JOIN import.periodes ip ON ip.period_id = acp.id ORDER BY ip.id; +-- 57bis. Link work social actions to periods +INSERT INTO chill_person_accompanying_period_work ( + id, + note, + createdat, + startdate, + enddate, + createdautomatically, + createdautomaticallyreason, + accompanyingperiod_id, + socialaction_id, + createdby_id, + handlingthierparty_id, + updatedat, + updatedby_id, + privatecomment_comments, + version +) + SELECT + nextval('chill_person_accompanying_period_work_id_seq'), + '', + CURRENT_TIMESTAMP, + COALESCE(ip.openingdate1, date(date_trunc('year', CURRENT_DATE))), + ip.closingdate1, + true, + 'Imported from canvas', + ip.period_id, + sa.id, + COALESCE(acp.user_id, (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM users)), + null, + CURRENT_TIMESTAMP, + COALESCE(acp.user_id, (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM users)), + '{}'::json, + 1 + FROM import.periodes ip + JOIN chill_person_accompanying_period acp ON acp.id = ip.period_id + JOIN LATERAL ( + SELECT MIN(id) AS id + FROM chill_person_social_action s + WHERE s.title::jsonb->>'fr' = trim(ip.work_socialaction) + ) sa ON sa.id IS NOT NULL + WHERE trim(COALESCE(ip.work_socialaction, '')) != '' + AND NOT EXISTS ( + SELECT 1 + FROM chill_person_accompanying_period_work w + WHERE w.accompanyingperiod_id = ip.period_id + AND w.socialaction_id = sa.id + ); +SELECT setval('chill_person_accompanying_period_work_id_seq', (SELECT COALESCE(max(id), 1) FROM chill_person_accompanying_period_work)); + -- 58. Link scopes to periods INSERT INTO accompanying_periods_scopes (accompanying_period_id, scope_id) SELECT ip.period_id, COALESCE( @@ -770,7 +873,7 @@ INSERT INTO chill_3party.third_party NULL, -- parent (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM users), NULL, -- service/dpt - NULL, + t.acronym, CURRENT_DATE, CURRENT_DATE, (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM users), @@ -877,6 +980,10 @@ DELETE FROM accompanying_periods_scopes acs USING import.periodes ip WHERE acs.a -- Undo 57. UPDATE chill_person_accompanying_period acp SET user_id = null FROM import.periodes ip WHERE ip.period_id = acp.id; +-- Undo 57bis. +DELETE FROM chill_person_accompanying_period_work w USING import.periodes ip WHERE w.accompanyingperiod_id = ip.period_id; +SELECT setval('chill_person_accompanying_period_work_id_seq', (SELECT COALESCE(max(id), 1) FROM chill_person_accompanying_period_work)); + -- Undo 56. DELETE FROM chill_person_accompanying_period_social_issues asi USING import.periodes ip WHERE asi.accompanyingperiod_id = ip.period_id; @@ -979,6 +1086,23 @@ ALTER TABLE import.personnes DROP COLUMN gender1; ALTER TABLE import.personnes DROP COLUMN civility1; -- Undo 14. +DELETE FROM chill_person_social_action sa +USING import.choix_periodes icp +LEFT JOIN LATERAL ( + SELECT si.id + FROM chill_person_social_issue si + WHERE si.title::jsonb->>'fr' = icp.work_social_issue1::jsonb->>'fr' + ORDER BY (si.parent_id IS NULL), si.id + LIMIT 1 +) issue ON true +WHERE sa.title::jsonb->>'fr' = icp.work_social_action1::jsonb->>'fr' + AND ( + (sa.issue_id IS NULL AND issue.id IS NULL) + OR sa.issue_id = issue.id + ); +SELECT setval('chill_person_social_action_id_seq', (SELECT COALESCE(max(id),1) FROM chill_person_social_action)); +ALTER TABLE import.choix_periodes DROP COLUMN work_social_issue1; +ALTER TABLE import.choix_periodes DROP COLUMN work_social_action1; -- Undo 13. DELETE FROM chill_person_social_issue USING import.choix_periodes i diff --git a/sql/prepare-import.sql b/sql/prepare-import.sql index 9ce5731..6a2df66 100755 --- a/sql/prepare-import.sql +++ b/sql/prepare-import.sql @@ -119,11 +119,12 @@ CREATE TABLE "import".users ( ); CREATE TABLE "import".tiers ( - ID INT PRIMARY KEY, + ID varchar(50) NULL, CATEGORIE VARCHAR(255), SECTEUR_AS VARCHAR(255), COMMUNE VARCHAR(255), NOM VARCHAR(255), + ACRONYM VARCHAR(64), PHONENUMBER VARCHAR(20), PHONENUMBER_2 VARCHAR(20), EMAIL VARCHAR(255),