Ajout de la gestion des actions sociales dans l'importation des données et correction des colonnes dans les fichiers CSV
This commit is contained in:
Boris Waaub
Executable → Regular
+1
-1
@@ -1 +1 @@
|
||||
closingmotive,origin,acp_scopes,job,referrer,parent,enfant,acp_socialissues,work_socialactions,street,extra,streetnumber,postcode,country,address_text,endcol
|
||||
closingmotive,origin,acp_scopes,job,referrer,parent,enfant,acp_social_issues,work_social_action,street,extra,streetnumber,postcode,country
|
||||
|
||||
|
Executable
+112
@@ -0,0 +1,112 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
DB_HOST="${PGHOST:-localhost}"
|
||||
DB_PORT="${PGPORT:-5432}"
|
||||
DB_USER="${PGUSER:-postgres}"
|
||||
DB_NAME="${PGDATABASE:-chill-import}"
|
||||
|
||||
PSQL=(psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME")
|
||||
|
||||
required_tables=(
|
||||
"import.personnes"
|
||||
"import.choix_periodes"
|
||||
"import.choix_localisations"
|
||||
"import.tiers"
|
||||
)
|
||||
|
||||
required_files=(
|
||||
"$ROOT_DIR/csv/choix_usagers.csv"
|
||||
"$ROOT_DIR/csv/choix_periodes.csv"
|
||||
"$ROOT_DIR/csv/choix_localisations.csv"
|
||||
"$ROOT_DIR/csv/choix_tiers.csv"
|
||||
)
|
||||
|
||||
for table in "${required_tables[@]}"; do
|
||||
exists="$("${PSQL[@]}" -tAc "SELECT to_regclass('${table}') IS NOT NULL;")"
|
||||
if [[ "$exists" != "t" ]]; then
|
||||
echo "Table manquante: ${table}" >&2
|
||||
echo "Lance d'abord: psql -h ${DB_HOST} -p ${DB_PORT} -U ${DB_USER} -d ${DB_NAME} -f sql/prepare-import.sql" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
for csv_file in "${required_files[@]}"; do
|
||||
if [[ ! -f "$csv_file" ]]; then
|
||||
echo "Fichier CSV manquant: ${csv_file#$ROOT_DIR/}" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
echo "[0/4] Assouplissement des types texte des tables import..."
|
||||
"${PSQL[@]}" <<'SQL'
|
||||
DO $$
|
||||
DECLARE r record;
|
||||
BEGIN
|
||||
FOR r IN
|
||||
SELECT table_schema, table_name, column_name
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = 'import'
|
||||
AND table_name IN ('personnes', 'choix_periodes', 'choix_localisations', 'tiers')
|
||||
AND data_type IN ('character varying', 'character')
|
||||
LOOP
|
||||
EXECUTE format(
|
||||
'ALTER TABLE %I.%I ALTER COLUMN %I TYPE text',
|
||||
r.table_schema,
|
||||
r.table_name,
|
||||
r.column_name
|
||||
);
|
||||
END LOOP;
|
||||
END $$;
|
||||
SQL
|
||||
|
||||
echo "[1/4] Vidage des tables d'import..."
|
||||
"${PSQL[@]}" <<'SQL'
|
||||
TRUNCATE TABLE
|
||||
import.personnes,
|
||||
import.choix_periodes,
|
||||
import.choix_localisations,
|
||||
import.tiers
|
||||
RESTART IDENTITY;
|
||||
SQL
|
||||
|
||||
echo "[2/4] Import des CSV du dossier csv/..."
|
||||
|
||||
copy_table_from_file() {
|
||||
local table="$1"
|
||||
local file_path="$2"
|
||||
local columns="${3:-}"
|
||||
local target="$table"
|
||||
if [[ -n "$columns" ]]; then
|
||||
target="${table}(${columns})"
|
||||
fi
|
||||
echo " - import ${table} <= ${file_path#$ROOT_DIR/}"
|
||||
"${PSQL[@]}" -c "\\copy ${target} FROM STDIN WITH (FORMAT csv, HEADER true, DELIMITER ',')" < "$file_path"
|
||||
}
|
||||
|
||||
copy_table_from_file "import.personnes" "$ROOT_DIR/csv/choix_usagers.csv" "civility,lastname,firstname,gender,gendercomment,nationality,memo,birthdate,place_of_birth,countryofbirth,deathdate,email,phonenumber,mobilenumber,contactinfo,street,extra,streetnumber,postcode,country"
|
||||
copy_table_from_file "import.choix_periodes" "$ROOT_DIR/csv/choix_periodes.csv" "closingmotive,origin,acp_scopes,job,referrer,parent,enfant,acp_social_issues,work_social_action,street,extra,streetnumber,postcode,country"
|
||||
copy_table_from_file "import.choix_localisations" "$ROOT_DIR/csv/choix_localisations.csv" "title,addressRequired,availableForUsers,contactData,defaultFor,editableByUsers"
|
||||
|
||||
echo "[3/4] Import de tiers via table staging..."
|
||||
"${PSQL[@]}" -c "\copy import.tiers_stage(coordonnees,nom,categorie) FROM STDIN WITH (FORMAT csv, HEADER true, DELIMITER ',')" < "$ROOT_DIR/csv/choix_tiers.csv"
|
||||
|
||||
"${PSQL[@]}" <<'SQL'
|
||||
INSERT INTO import.tiers (id, coordonnees, nom, categorie)
|
||||
SELECT
|
||||
row_number() OVER (ORDER BY nom) AS id,
|
||||
NULLIF(trim(coordonnees), ''),
|
||||
NULLIF(trim(nom), ''),
|
||||
NULLIF(trim(categorie), '')
|
||||
FROM import.tiers_stage
|
||||
WHERE COALESCE(trim(nom), '') <> '';
|
||||
|
||||
DROP TABLE import.tiers_stage;
|
||||
SQL
|
||||
|
||||
echo "[4/4] Verification des volumes importes..."
|
||||
"${PSQL[@]}" -c "SELECT 'personnes' AS table_name, count(*) AS rows FROM import.personnes UNION ALL SELECT 'choix_periodes', count(*) FROM import.choix_periodes UNION ALL SELECT 'choix_localisations', count(*) FROM import.choix_localisations UNION ALL SELECT 'tiers', count(*) FROM import.tiers;"
|
||||
|
||||
echo "Import termine sans doublons (tables videes au debut)."
|
||||
+128
-4
@@ -137,16 +137,69 @@ INSERT INTO chill_person_social_issue (id, parent_id, title, ordering)
|
||||
WITH max_ordering AS ( SELECT MAX(ordering) as max_ordering FROM chill_person_social_issue )
|
||||
INSERT INTO chill_person_social_issue (id, parent_id, title, ordering)
|
||||
SELECT nextval('chill_person_social_issue_id_seq'),
|
||||
( SELECT id FROM chill_person_social_issue WHERE parent_id IS NULL AND title::jsonb->>'fr' = t.parent1::jsonb->>'fr' ), t.enfant1,
|
||||
( SELECT MIN(id) FROM chill_person_social_issue WHERE parent_id IS NULL AND title::jsonb->>'fr' = t.parent1::jsonb->>'fr' ), t.enfant1,
|
||||
max_ordering.max_ordering + row_number() OVER () as ordering
|
||||
FROM ( SELECT DISTINCT ON (acp_social_issues) parent1, enfant1 FROM import.choix_periodes WHERE enfant1 IS NOT NULL ) t
|
||||
CROSS JOIN max_ordering
|
||||
WHERE NOT EXISTS ( SELECT 1 FROM chill_person_social_issue
|
||||
WHERE title::jsonb->>'fr' = t.enfant1::jsonb->>'fr'
|
||||
AND parent_id = (SELECT id FROM chill_person_social_issue WHERE title::jsonb->>'fr' = t.parent1::jsonb->>'fr'));
|
||||
AND parent_id = (SELECT MIN(id) FROM chill_person_social_issue WHERE parent_id IS NULL AND title::jsonb->>'fr' = t.parent1::jsonb->>'fr'));
|
||||
|
||||
-- 14. Complete table WorkSocialActions
|
||||
-- (not yet implemented in canvas)
|
||||
ALTER TABLE import.choix_periodes ADD COLUMN work_social_action1 JSONB;
|
||||
UPDATE import.choix_periodes SET work_social_action1=json_build_object('fr', trim(work_social_action)) WHERE work_social_action!='';
|
||||
ALTER TABLE import.choix_periodes ADD COLUMN work_social_issue1 JSONB;
|
||||
WITH grouped AS (
|
||||
SELECT ctid,
|
||||
SUM(CASE WHEN NULLIF(trim(acp_social_issues), '') IS NOT NULL THEN 1 ELSE 0 END)
|
||||
OVER (ORDER BY ctid) AS grp
|
||||
FROM import.choix_periodes
|
||||
), resolved AS (
|
||||
SELECT cp.ctid,
|
||||
COALESCE(
|
||||
MAX(NULLIF(upper(trim(cp.enfant)), '')) OVER (PARTITION BY g.grp),
|
||||
MAX(NULLIF(upper(trim(cp.parent)), '')) OVER (PARTITION BY g.grp)
|
||||
) AS issue_title
|
||||
FROM import.choix_periodes cp
|
||||
JOIN grouped g ON cp.ctid = g.ctid
|
||||
)
|
||||
UPDATE import.choix_periodes cp
|
||||
SET work_social_issue1 = json_build_object('fr', r.issue_title)
|
||||
FROM resolved r
|
||||
WHERE cp.ctid = r.ctid
|
||||
AND r.issue_title IS NOT NULL
|
||||
AND trim(COALESCE(cp.work_social_action, '')) != '';
|
||||
WITH max_ordering AS ( SELECT COALESCE(MAX(ordering), 0) as max_ordering FROM chill_person_social_action )
|
||||
INSERT INTO chill_person_social_action (id, issue_id, parent_id, desactivationdate, defaultnotificationdelay, title, ordering)
|
||||
SELECT nextval('chill_person_social_action_id_seq'),
|
||||
issue.id,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
t.work_social_action1,
|
||||
max_ordering.max_ordering + row_number() OVER () as ordering
|
||||
FROM (
|
||||
SELECT DISTINCT ON (work_social_action1, work_social_issue1) work_social_action1, work_social_issue1
|
||||
FROM import.choix_periodes
|
||||
WHERE work_social_action1 IS NOT NULL
|
||||
) t
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT si.id
|
||||
FROM chill_person_social_issue si
|
||||
WHERE si.title::jsonb->>'fr' = t.work_social_issue1::jsonb->>'fr'
|
||||
ORDER BY (si.parent_id IS NULL), si.id
|
||||
LIMIT 1
|
||||
) issue ON true
|
||||
CROSS JOIN max_ordering
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM chill_person_social_action sa
|
||||
WHERE sa.title::jsonb->>'fr' = t.work_social_action1::jsonb->>'fr'
|
||||
AND (
|
||||
(sa.issue_id IS NULL AND issue.id IS NULL)
|
||||
OR sa.issue_id = issue.id
|
||||
)
|
||||
);
|
||||
|
||||
-- 20. Prepare personnes civility
|
||||
ALTER TABLE import.personnes ADD COLUMN civility1 jsonb;
|
||||
@@ -452,6 +505,56 @@ UPDATE chill_person_accompanying_period acp
|
||||
FROM import.periodes ip WHERE acp.id = ip.period_id;
|
||||
--SELECT ip.id, (SELECT id FROM users WHERE users.username = ip.referrer) AS referrer_id, ip.referrer, acp.id as period_id, acp.user_id FROM chill_person_accompanying_period acp JOIN import.periodes ip ON ip.period_id = acp.id ORDER BY ip.id;
|
||||
|
||||
-- 57bis. Link work social actions to periods
|
||||
INSERT INTO chill_person_accompanying_period_work (
|
||||
id,
|
||||
note,
|
||||
createdat,
|
||||
startdate,
|
||||
enddate,
|
||||
createdautomatically,
|
||||
createdautomaticallyreason,
|
||||
accompanyingperiod_id,
|
||||
socialaction_id,
|
||||
createdby_id,
|
||||
handlingthierparty_id,
|
||||
updatedat,
|
||||
updatedby_id,
|
||||
privatecomment_comments,
|
||||
version
|
||||
)
|
||||
SELECT
|
||||
nextval('chill_person_accompanying_period_work_id_seq'),
|
||||
'',
|
||||
CURRENT_TIMESTAMP,
|
||||
COALESCE(ip.openingdate1, date(date_trunc('year', CURRENT_DATE))),
|
||||
ip.closingdate1,
|
||||
true,
|
||||
'Imported from canvas',
|
||||
ip.period_id,
|
||||
sa.id,
|
||||
COALESCE(acp.user_id, (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM users)),
|
||||
null,
|
||||
CURRENT_TIMESTAMP,
|
||||
COALESCE(acp.user_id, (SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM users)),
|
||||
'{}'::json,
|
||||
1
|
||||
FROM import.periodes ip
|
||||
JOIN chill_person_accompanying_period acp ON acp.id = ip.period_id
|
||||
JOIN LATERAL (
|
||||
SELECT MIN(id) AS id
|
||||
FROM chill_person_social_action s
|
||||
WHERE s.title::jsonb->>'fr' = trim(ip.work_socialaction)
|
||||
) sa ON sa.id IS NOT NULL
|
||||
WHERE trim(COALESCE(ip.work_socialaction, '')) != ''
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM chill_person_accompanying_period_work w
|
||||
WHERE w.accompanyingperiod_id = ip.period_id
|
||||
AND w.socialaction_id = sa.id
|
||||
);
|
||||
SELECT setval('chill_person_accompanying_period_work_id_seq', (SELECT COALESCE(max(id), 1) FROM chill_person_accompanying_period_work));
|
||||
|
||||
-- 58. Link scopes to periods
|
||||
INSERT INTO accompanying_periods_scopes (accompanying_period_id, scope_id)
|
||||
SELECT ip.period_id, COALESCE(
|
||||
@@ -770,7 +873,7 @@ INSERT INTO chill_3party.third_party
|
||||
NULL, -- parent
|
||||
(SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM users),
|
||||
NULL, -- service/dpt
|
||||
NULL,
|
||||
t.acronym,
|
||||
CURRENT_DATE,
|
||||
CURRENT_DATE,
|
||||
(SELECT distinct(first_value(id) OVER(ORDER BY id)) FROM users),
|
||||
@@ -877,6 +980,10 @@ DELETE FROM accompanying_periods_scopes acs USING import.periodes ip WHERE acs.a
|
||||
-- Undo 57.
|
||||
UPDATE chill_person_accompanying_period acp SET user_id = null FROM import.periodes ip WHERE ip.period_id = acp.id;
|
||||
|
||||
-- Undo 57bis.
|
||||
DELETE FROM chill_person_accompanying_period_work w USING import.periodes ip WHERE w.accompanyingperiod_id = ip.period_id;
|
||||
SELECT setval('chill_person_accompanying_period_work_id_seq', (SELECT COALESCE(max(id), 1) FROM chill_person_accompanying_period_work));
|
||||
|
||||
-- Undo 56.
|
||||
DELETE FROM chill_person_accompanying_period_social_issues asi USING import.periodes ip WHERE asi.accompanyingperiod_id = ip.period_id;
|
||||
|
||||
@@ -979,6 +1086,23 @@ ALTER TABLE import.personnes DROP COLUMN gender1;
|
||||
ALTER TABLE import.personnes DROP COLUMN civility1;
|
||||
|
||||
-- Undo 14.
|
||||
DELETE FROM chill_person_social_action sa
|
||||
USING import.choix_periodes icp
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT si.id
|
||||
FROM chill_person_social_issue si
|
||||
WHERE si.title::jsonb->>'fr' = icp.work_social_issue1::jsonb->>'fr'
|
||||
ORDER BY (si.parent_id IS NULL), si.id
|
||||
LIMIT 1
|
||||
) issue ON true
|
||||
WHERE sa.title::jsonb->>'fr' = icp.work_social_action1::jsonb->>'fr'
|
||||
AND (
|
||||
(sa.issue_id IS NULL AND issue.id IS NULL)
|
||||
OR sa.issue_id = issue.id
|
||||
);
|
||||
SELECT setval('chill_person_social_action_id_seq', (SELECT COALESCE(max(id),1) FROM chill_person_social_action));
|
||||
ALTER TABLE import.choix_periodes DROP COLUMN work_social_issue1;
|
||||
ALTER TABLE import.choix_periodes DROP COLUMN work_social_action1;
|
||||
|
||||
-- Undo 13.
|
||||
DELETE FROM chill_person_social_issue USING import.choix_periodes i
|
||||
|
||||
@@ -119,11 +119,12 @@ CREATE TABLE "import".users (
|
||||
);
|
||||
|
||||
CREATE TABLE "import".tiers (
|
||||
ID INT PRIMARY KEY,
|
||||
ID varchar(50) NULL,
|
||||
CATEGORIE VARCHAR(255),
|
||||
SECTEUR_AS VARCHAR(255),
|
||||
COMMUNE VARCHAR(255),
|
||||
NOM VARCHAR(255),
|
||||
ACRONYM VARCHAR(64),
|
||||
PHONENUMBER VARCHAR(20),
|
||||
PHONENUMBER_2 VARCHAR(20),
|
||||
EMAIL VARCHAR(255),
|
||||
|
||||
Reference in New Issue
Block a user