c388967ebd
- Added support for importing 'periodes' data and adjusted related SQL scripts. - Introduced a new SQL preparation script to set up the import schema. - Updated the import_all_csv.sh script to handle new CSV files and tables. - Implemented checks for required SQL files and improved error handling. - Modified data types and structures in the import process for better consistency. - Created a new remove-import-data.sql script to clean up imported data effectively. - Enhanced the handling of postal codes and addresses in the import logic. - Added comments and structured the SQL scripts for better readability and maintainability.
417 lines
18 KiB
PL/PgSQL
417 lines
18 KiB
PL/PgSQL
BEGIN;
|
|
|
|
-- Nettoyage des donnees importees a partir des tables du schema import.
|
|
-- Ce script suppose que sql/import.sql a ete execute au moins partiellement.
|
|
|
|
-- 1. Commentaires et liaisons des periodes
|
|
DO $$
|
|
BEGIN
|
|
IF EXISTS (
|
|
SELECT 1 FROM information_schema.columns
|
|
WHERE table_schema = 'import' AND table_name = 'periodes' AND column_name = 'period_id'
|
|
) THEN
|
|
UPDATE chill_person_accompanying_period acp
|
|
SET pinnedcomment_id = NULL
|
|
FROM import.periodes ip
|
|
WHERE acp.id = ip.period_id;
|
|
|
|
DELETE FROM chill_person_accompanying_period_comment com
|
|
USING import.periodes ip
|
|
WHERE com.accompanyingperiod_id = ip.period_id;
|
|
|
|
DELETE FROM chill_person_accompanying_period_work work
|
|
USING import.periodes ip
|
|
WHERE work.accompanyingperiod_id = ip.period_id;
|
|
|
|
DELETE FROM chill_person_accompanying_period_social_issues asi
|
|
USING import.periodes ip
|
|
WHERE asi.accompanyingperiod_id = ip.period_id;
|
|
|
|
DELETE FROM chill_person_accompanying_period_location_history hist
|
|
USING import.periodes ip
|
|
WHERE hist.period_id = ip.period_id;
|
|
|
|
DELETE FROM accompanying_periods_scopes scopes_link
|
|
USING import.periodes ip
|
|
WHERE scopes_link.accompanying_period_id = ip.period_id;
|
|
|
|
DELETE FROM chill_person_accompanying_period_participation part
|
|
USING import.periodes ip
|
|
WHERE part.accompanyingperiod_id = ip.period_id;
|
|
|
|
UPDATE chill_person_accompanying_period acp
|
|
SET addresslocation_id = NULL,
|
|
personlocation_id = NULL,
|
|
user_id = NULL,
|
|
origin_id = NULL,
|
|
job_id = NULL
|
|
FROM import.periodes ip
|
|
WHERE acp.id = ip.period_id;
|
|
|
|
DELETE FROM chill_person_accompanying_period acp
|
|
USING import.periodes ip
|
|
WHERE acp.id = ip.period_id;
|
|
END IF;
|
|
END $$;
|
|
|
|
-- 2. Menages, adresses et personnes
|
|
DO $$
|
|
BEGIN
|
|
IF EXISTS (
|
|
SELECT 1 FROM information_schema.columns
|
|
WHERE table_schema = 'import' AND table_name = 'personnes' AND column_name = 'person_id'
|
|
) THEN
|
|
DELETE FROM chill_person_household_composition comp
|
|
USING import.personnes ip
|
|
WHERE comp.household_id = ip.person_id;
|
|
|
|
DELETE FROM chill_person_household_to_addresses hh_addr
|
|
USING import.personnes ip
|
|
WHERE hh_addr.household_id = ip.person_id;
|
|
|
|
DELETE FROM chill_person_household_members member
|
|
USING import.personnes ip
|
|
WHERE member.person_id = ip.person_id;
|
|
|
|
DELETE FROM chill_person_household hh
|
|
USING import.personnes ip
|
|
WHERE hh.id = ip.person_id;
|
|
|
|
DELETE FROM chill_person_person_center_history hist
|
|
USING import.personnes ip
|
|
WHERE hist.person_id = ip.person_id;
|
|
|
|
DELETE FROM chill_person_person person
|
|
USING import.personnes ip
|
|
WHERE person.id = ip.person_id;
|
|
END IF;
|
|
|
|
IF EXISTS (
|
|
SELECT 1 FROM information_schema.columns
|
|
WHERE table_schema = 'import' AND table_name = 'personnes' AND column_name = 'address_id'
|
|
) THEN
|
|
DELETE FROM chill_main_address addr
|
|
USING import.personnes ip
|
|
WHERE addr.id = ip.address_id;
|
|
END IF;
|
|
END $$;
|
|
|
|
-- 3. Adresses temporaires de periodes et localisations
|
|
DO $$
|
|
BEGIN
|
|
IF EXISTS (
|
|
SELECT 1 FROM information_schema.columns
|
|
WHERE table_schema = 'import' AND table_name = 'choix_periodes' AND column_name = 'address_location_id'
|
|
) THEN
|
|
DELETE FROM chill_main_address addr
|
|
USING import.choix_periodes icp
|
|
WHERE addr.id = icp.address_location_id;
|
|
END IF;
|
|
|
|
IF EXISTS (
|
|
SELECT 1 FROM information_schema.columns
|
|
WHERE table_schema = 'import' AND table_name = 'localisations' AND column_name = 'address_id'
|
|
) THEN
|
|
DELETE FROM chill_main_address addr
|
|
USING import.localisations loc
|
|
WHERE addr.id = loc.address_id;
|
|
END IF;
|
|
END $$;
|
|
|
|
-- 4. Localisations et types de localisation
|
|
DELETE FROM chill_main_location loc
|
|
USING import.localisations il
|
|
WHERE loc.name = trim(il.locname);
|
|
|
|
DELETE FROM chill_main_location_type lt
|
|
USING import.choix_localisations icl
|
|
WHERE lt.title::jsonb->>'fr' = trim(icl.title)
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM chill_main_location loc
|
|
WHERE loc.locationtype_id = lt.id
|
|
);
|
|
|
|
-- 5. Tiers et categories
|
|
DELETE FROM chill_3party.thirdparty_category tpc
|
|
USING chill_3party.third_party tp, import.tiers it
|
|
WHERE tpc.thirdparty_id = tp.id
|
|
AND (
|
|
tp.name = trim(it.nom)
|
|
OR tp.name = trim(it.personne_nom)
|
|
);
|
|
|
|
DELETE FROM chill_3party.third_party tp
|
|
USING import.tiers it
|
|
WHERE tp.name = trim(it.personne_nom)
|
|
OR tp.name = trim(it.nom);
|
|
|
|
DELETE FROM chill_3party.party_category pc
|
|
USING import.tiers it
|
|
WHERE pc.name::jsonb->>'fr' = trim(it.categorie)
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM chill_3party.thirdparty_category tpc
|
|
WHERE tpc.category_id = pc.id
|
|
);
|
|
|
|
-- 6. Utilisateurs importes et historique des metiers
|
|
DELETE FROM chill_main_user_job_history hist
|
|
USING import.users iu
|
|
WHERE hist.user_id = (
|
|
SELECT id FROM users WHERE username = split_part(iu.email, '@', 1)
|
|
);
|
|
|
|
DELETE FROM users u
|
|
USING import.users iu
|
|
WHERE u.username = split_part(iu.email, '@', 1);
|
|
|
|
DELETE FROM chill_main_user_job job
|
|
USING import.users iu
|
|
WHERE job.label::jsonb->>'fr' = trim(iu.metier)
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM chill_main_user_job_history hist
|
|
WHERE hist.job_id = job.id
|
|
);
|
|
|
|
-- 7. Tables de reference ajoutees pendant l'import principal
|
|
DO $$
|
|
BEGIN
|
|
IF EXISTS (
|
|
SELECT 1 FROM information_schema.columns
|
|
WHERE table_schema = 'import' AND table_name = 'choix_periodes' AND column_name = 'work_social_action1'
|
|
) AND EXISTS (
|
|
SELECT 1 FROM information_schema.columns
|
|
WHERE table_schema = 'import' AND table_name = 'choix_periodes' AND column_name = 'work_social_issue1'
|
|
) THEN
|
|
DELETE FROM chill_person_social_action sa
|
|
USING import.choix_periodes icp
|
|
LEFT JOIN LATERAL (
|
|
SELECT si.id
|
|
FROM chill_person_social_issue si
|
|
WHERE si.title::jsonb->>'fr' = icp.work_social_issue1::jsonb->>'fr'
|
|
ORDER BY (si.parent_id IS NULL), si.id
|
|
LIMIT 1
|
|
) issue ON true
|
|
WHERE sa.title::jsonb->>'fr' = icp.work_social_action1::jsonb->>'fr'
|
|
AND (
|
|
(sa.issue_id IS NULL AND issue.id IS NULL)
|
|
OR sa.issue_id = issue.id
|
|
);
|
|
END IF;
|
|
END $$;
|
|
|
|
DO $$
|
|
BEGIN
|
|
IF EXISTS (
|
|
SELECT 1 FROM information_schema.columns
|
|
WHERE table_schema = 'import' AND table_name = 'choix_periodes' AND column_name = 'enfant1'
|
|
) THEN
|
|
DELETE FROM chill_person_social_issue child_issue
|
|
USING import.choix_periodes icp
|
|
WHERE child_issue.title::jsonb->>'fr' = icp.enfant1::jsonb->>'fr';
|
|
END IF;
|
|
|
|
IF EXISTS (
|
|
SELECT 1 FROM information_schema.columns
|
|
WHERE table_schema = 'import' AND table_name = 'choix_periodes' AND column_name = 'parent1'
|
|
) THEN
|
|
DELETE FROM chill_person_social_issue parent_issue
|
|
USING import.choix_periodes icp
|
|
WHERE parent_issue.title::jsonb->>'fr' = icp.parent1::jsonb->>'fr'
|
|
AND parent_issue.parent_id IS NULL;
|
|
END IF;
|
|
END $$;
|
|
|
|
DELETE FROM users u
|
|
USING import.choix_periodes icp
|
|
WHERE u.username::text = icp.referrer::text
|
|
AND u.password = ''
|
|
AND u.enabled = false;
|
|
|
|
DO $$
|
|
BEGIN
|
|
IF EXISTS (
|
|
SELECT 1 FROM information_schema.columns
|
|
WHERE table_schema = 'import' AND table_name = 'choix_periodes' AND column_name = 'acp_scopes1'
|
|
) THEN
|
|
DELETE FROM scopes s
|
|
USING import.choix_periodes icp
|
|
WHERE s.name::jsonb->>'fr' = icp.acp_scopes1::jsonb->>'fr'
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM accompanying_periods_scopes aps
|
|
WHERE aps.scope_id = s.id
|
|
);
|
|
END IF;
|
|
|
|
IF EXISTS (
|
|
SELECT 1 FROM information_schema.columns
|
|
WHERE table_schema = 'import' AND table_name = 'choix_periodes' AND column_name = 'job1'
|
|
) THEN
|
|
DELETE FROM chill_main_user_job job
|
|
USING import.choix_periodes icp
|
|
WHERE job.label::jsonb->>'fr' = icp.job1::jsonb->>'fr'
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM chill_person_accompanying_period acp
|
|
WHERE acp.job_id = job.id
|
|
);
|
|
END IF;
|
|
|
|
IF EXISTS (
|
|
SELECT 1 FROM information_schema.columns
|
|
WHERE table_schema = 'import' AND table_name = 'choix_periodes' AND column_name = 'origin1'
|
|
) THEN
|
|
DELETE FROM chill_person_accompanying_period_origin origin
|
|
USING import.choix_periodes icp
|
|
WHERE origin.label::jsonb->>'fr' = icp.origin1::jsonb->>'fr'
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM chill_person_accompanying_period acp
|
|
WHERE acp.origin_id = origin.id
|
|
);
|
|
END IF;
|
|
|
|
IF EXISTS (
|
|
SELECT 1 FROM information_schema.columns
|
|
WHERE table_schema = 'import' AND table_name = 'choix_periodes' AND column_name = 'closingmotive1'
|
|
) THEN
|
|
DELETE FROM chill_person_accompanying_period_closingmotive motive
|
|
USING import.choix_periodes icp
|
|
WHERE motive.name::jsonb->>'fr' = icp.closingmotive1::jsonb->>'fr';
|
|
END IF;
|
|
END $$;
|
|
|
|
DO $$
|
|
BEGIN
|
|
IF EXISTS (
|
|
SELECT 1 FROM information_schema.columns
|
|
WHERE table_schema = 'import' AND table_name = 'choix_personnes' AND column_name = 'household_position1'
|
|
) THEN
|
|
DELETE FROM chill_person_household_position pos
|
|
USING import.choix_personnes icp
|
|
WHERE pos.label::jsonb->>'fr' = icp.household_position1::jsonb->>'fr';
|
|
END IF;
|
|
|
|
IF EXISTS (
|
|
SELECT 1 FROM information_schema.columns
|
|
WHERE table_schema = 'import' AND table_name = 'choix_personnes' AND column_name = 'household_composition_type1'
|
|
) THEN
|
|
DELETE FROM chill_person_household_composition_type comp_type
|
|
USING import.choix_personnes icp
|
|
WHERE comp_type.label::jsonb->>'fr' = icp.household_composition_type1::jsonb->>'fr';
|
|
END IF;
|
|
|
|
IF EXISTS (
|
|
SELECT 1 FROM information_schema.columns
|
|
WHERE table_schema = 'import' AND table_name = 'choix_personnes' AND column_name = 'maritalstatus1'
|
|
) THEN
|
|
DELETE FROM chill_person_marital_status ms
|
|
USING import.choix_personnes icp
|
|
WHERE ms.name::jsonb->>'fr' = icp.maritalstatus1::jsonb->>'fr';
|
|
END IF;
|
|
|
|
IF EXISTS (
|
|
SELECT 1 FROM information_schema.columns
|
|
WHERE table_schema = 'import' AND table_name = 'choix_personnes' AND column_name = 'country1'
|
|
) THEN
|
|
DELETE FROM country c
|
|
USING import.choix_personnes icp
|
|
WHERE c.name::jsonb->>'fr' = icp.country1::jsonb->>'fr'
|
|
AND c.id > 249;
|
|
END IF;
|
|
|
|
IF EXISTS (
|
|
SELECT 1 FROM information_schema.columns
|
|
WHERE table_schema = 'import' AND table_name = 'choix_personnes' AND column_name = 'civility1'
|
|
) THEN
|
|
DELETE FROM chill_main_civility civ
|
|
USING import.choix_personnes icp
|
|
WHERE civ.name::jsonb->>'fr' = icp.civility1::jsonb->>'fr';
|
|
END IF;
|
|
END $$;
|
|
|
|
-- 8. Suppression des colonnes techniques ajoutees par sql/import.sql
|
|
ALTER TABLE import.personnes DROP COLUMN IF EXISTS person_id;
|
|
ALTER TABLE import.personnes DROP COLUMN IF EXISTS address_id;
|
|
ALTER TABLE import.personnes DROP COLUMN IF EXISTS address_ref_id;
|
|
ALTER TABLE import.personnes DROP COLUMN IF EXISTS postcode_id;
|
|
ALTER TABLE import.personnes DROP COLUMN IF EXISTS postcode_arr;
|
|
ALTER TABLE import.personnes DROP COLUMN IF EXISTS postcode1;
|
|
ALTER TABLE import.personnes DROP COLUMN IF EXISTS birthdate1;
|
|
ALTER TABLE import.personnes DROP COLUMN IF EXISTS deathdate1;
|
|
ALTER TABLE import.personnes DROP COLUMN IF EXISTS validfrom1;
|
|
ALTER TABLE import.personnes DROP COLUMN IF EXISTS household_startdate1;
|
|
ALTER TABLE import.personnes DROP COLUMN IF EXISTS civility1;
|
|
ALTER TABLE import.personnes DROP COLUMN IF EXISTS gender1;
|
|
ALTER TABLE import.personnes DROP COLUMN IF EXISTS nationality1;
|
|
ALTER TABLE import.personnes DROP COLUMN IF EXISTS countryofbirth1;
|
|
ALTER TABLE import.personnes DROP COLUMN IF EXISTS maritalstatus1;
|
|
ALTER TABLE import.personnes DROP COLUMN IF EXISTS numberofchildren1;
|
|
ALTER TABLE import.personnes DROP COLUMN IF EXISTS placeofbirth1;
|
|
ALTER TABLE import.personnes DROP COLUMN IF EXISTS memo1;
|
|
ALTER TABLE import.personnes DROP COLUMN IF EXISTS household_composition_type1;
|
|
ALTER TABLE import.personnes DROP COLUMN IF EXISTS household_position1;
|
|
ALTER TABLE import.personnes DROP COLUMN IF EXISTS mobilenumber1;
|
|
ALTER TABLE import.personnes DROP COLUMN IF EXISTS phonenumber1;
|
|
|
|
ALTER TABLE import.periodes DROP COLUMN IF EXISTS period_id;
|
|
ALTER TABLE import.periodes DROP COLUMN IF EXISTS openingdate1;
|
|
ALTER TABLE import.periodes DROP COLUMN IF EXISTS closingdate1;
|
|
ALTER TABLE import.periodes DROP COLUMN IF EXISTS closingmotive1;
|
|
ALTER TABLE import.periodes DROP COLUMN IF EXISTS origin1;
|
|
ALTER TABLE import.periodes DROP COLUMN IF EXISTS job1;
|
|
ALTER TABLE import.periodes DROP COLUMN IF EXISTS acp_scopes1;
|
|
ALTER TABLE import.periodes DROP COLUMN IF EXISTS intensity1;
|
|
|
|
ALTER TABLE import.choix_periodes DROP COLUMN IF EXISTS closingmotive1;
|
|
ALTER TABLE import.choix_periodes DROP COLUMN IF EXISTS origin1;
|
|
ALTER TABLE import.choix_periodes DROP COLUMN IF EXISTS job1;
|
|
ALTER TABLE import.choix_periodes DROP COLUMN IF EXISTS acp_scopes1;
|
|
ALTER TABLE import.choix_periodes DROP COLUMN IF EXISTS parent1;
|
|
ALTER TABLE import.choix_periodes DROP COLUMN IF EXISTS enfant1;
|
|
ALTER TABLE import.choix_periodes DROP COLUMN IF EXISTS work_social_action1;
|
|
ALTER TABLE import.choix_periodes DROP COLUMN IF EXISTS work_social_issue1;
|
|
ALTER TABLE import.choix_periodes DROP COLUMN IF EXISTS address_location_id;
|
|
|
|
ALTER TABLE import.choix_personnes DROP COLUMN IF EXISTS civility1;
|
|
ALTER TABLE import.choix_personnes DROP COLUMN IF EXISTS country1;
|
|
ALTER TABLE import.choix_personnes DROP COLUMN IF EXISTS maritalstatus1;
|
|
ALTER TABLE import.choix_personnes DROP COLUMN IF EXISTS household_composition_type1;
|
|
ALTER TABLE import.choix_personnes DROP COLUMN IF EXISTS household_position1;
|
|
|
|
ALTER TABLE import.choix_localisations DROP COLUMN IF EXISTS title1;
|
|
ALTER TABLE import.localisations DROP COLUMN IF EXISTS postcode_arr;
|
|
ALTER TABLE import.localisations DROP COLUMN IF EXISTS address_id;
|
|
ALTER TABLE import.localisations DROP COLUMN IF EXISTS address_ref_id;
|
|
|
|
ALTER TABLE import.tiers DROP COLUMN IF EXISTS postcode_arr;
|
|
ALTER TABLE import.tiers DROP COLUMN IF EXISTS address_id;
|
|
ALTER TABLE import.tiers DROP COLUMN IF EXISTS address_ref_id;
|
|
|
|
-- 9. Recalage des sequences principales
|
|
SELECT setval('chill_person_accompanying_period_comment_id_seq', (SELECT COALESCE(max(id), 1) FROM chill_person_accompanying_period_comment));
|
|
SELECT setval('chill_person_accompanying_period_work_id_seq', (SELECT COALESCE(max(id), 1) FROM chill_person_accompanying_period_work));
|
|
SELECT setval('chill_person_accompanying_period_location_history_id_seq', (SELECT COALESCE(max(id), 1) FROM chill_person_accompanying_period_location_history));
|
|
SELECT setval('chill_person_accompanying_period_participation_id_seq', (SELECT COALESCE(max(id), 1) FROM chill_person_accompanying_period_participation));
|
|
SELECT setval('chill_person_accompanying_period_id_seq', (SELECT COALESCE(max(id), 1) FROM chill_person_accompanying_period));
|
|
SELECT setval('chill_person_household_composition_id_seq', (SELECT COALESCE(max(id), 1) FROM chill_person_household_composition));
|
|
SELECT setval('chill_person_household_members_id_seq', (SELECT COALESCE(max(id), 1) FROM chill_person_household_members));
|
|
SELECT setval('chill_person_household_id_seq', (SELECT COALESCE(max(id), 1) FROM chill_person_household));
|
|
SELECT setval('chill_person_person_center_history_id_seq', (SELECT COALESCE(max(id), 1) FROM chill_person_person_center_history));
|
|
SELECT setval('chill_person_person_id_seq', (SELECT COALESCE(max(id), 1) FROM chill_person_person));
|
|
SELECT setval('chill_person_social_action_id_seq', (SELECT COALESCE(max(id), 1) FROM chill_person_social_action));
|
|
SELECT setval('chill_person_social_issue_id_seq', (SELECT COALESCE(max(id), 1) FROM chill_person_social_issue));
|
|
SELECT setval('chill_person_accompanying_period_origin_id_seq', (SELECT COALESCE(max(id), 1) FROM chill_person_accompanying_period_origin));
|
|
SELECT setval('chill_person_accompanying_period_closingmotive_id_seq', (SELECT COALESCE(max(id), 1) FROM chill_person_accompanying_period_closingmotive));
|
|
SELECT setval('chill_person_household_position_id_seq', (SELECT COALESCE(max(id), 1) FROM chill_person_household_position));
|
|
SELECT setval('chill_person_household_composition_type_id_seq', (SELECT COALESCE(max(id), 1) FROM chill_person_household_composition_type));
|
|
SELECT setval('chill_main_civility_id_seq', (SELECT COALESCE(max(id), 1) FROM chill_main_civility));
|
|
SELECT setval('country_id_seq', (SELECT COALESCE(max(id), 1) FROM country));
|
|
SELECT setval('users_id_seq', (SELECT COALESCE(max(id), 1) FROM users));
|
|
SELECT setval('scopes_id_seq', (SELECT COALESCE(max(id), 1) FROM scopes));
|
|
SELECT setval('chill_main_user_job_id_seq', (SELECT COALESCE(max(id), 1) FROM chill_main_user_job));
|
|
SELECT setval('chill_main_address_id_seq', (SELECT COALESCE(max(id), 1) FROM chill_main_address));
|
|
SELECT setval('chill_main_location_id_seq', (SELECT COALESCE(max(id), 1) FROM chill_main_location));
|
|
SELECT setval('chill_main_location_type_id_seq', (SELECT COALESCE(max(id), 1) FROM chill_main_location_type));
|
|
SELECT setval('chill_main_user_job_history_id_seq', (SELECT COALESCE(max(id), 1) FROM chill_main_user_job_history));
|
|
SELECT setval('chill_3party.party_category_id_seq', (SELECT COALESCE(max(id), 1) FROM chill_3party.party_category));
|
|
SELECT setval('chill_3party.third_party_id_seq', (SELECT COALESCE(max(id), 1) FROM chill_3party.third_party));
|
|
|
|
COMMIT; |