c388967ebd
- Added support for importing 'periodes' data and adjusted related SQL scripts. - Introduced a new SQL preparation script to set up the import schema. - Updated the import_all_csv.sh script to handle new CSV files and tables. - Implemented checks for required SQL files and improved error handling. - Modified data types and structures in the import process for better consistency. - Created a new remove-import-data.sql script to clean up imported data effectively. - Enhanced the handling of postal codes and addresses in the import logic. - Added comments and structured the SQL scripts for better readability and maintainability.
126 lines
4.4 KiB
Bash
Executable File
126 lines
4.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
DB_HOST="${PGHOST:-localhost}"
|
|
DB_PORT="${PGPORT:-5432}"
|
|
DB_USER="${PGUSER:-postgres}"
|
|
DB_NAME="${PGDATABASE:-chill-import}"
|
|
|
|
PSQL=(psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME")
|
|
PREPARE_SQL="$ROOT_DIR/sql/prepare-import.sql"
|
|
|
|
required_tables=(
|
|
"import.personnes"
|
|
"import.periodes"
|
|
"import.choix_periodes"
|
|
"import.choix_localisations"
|
|
"import.tiers"
|
|
)
|
|
|
|
required_files=(
|
|
"$ROOT_DIR/csv/choix_usagers.csv"
|
|
"$ROOT_DIR/csv/periodes.csv"
|
|
"$ROOT_DIR/csv/choix_periodes.csv"
|
|
"$ROOT_DIR/csv/choix_localisations.csv"
|
|
"$ROOT_DIR/csv/choix_tiers.csv"
|
|
)
|
|
|
|
if [[ ! -f "$PREPARE_SQL" ]]; then
|
|
echo "Fichier SQL manquant: ${PREPARE_SQL#$ROOT_DIR/}" >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo "[-1/3] Recreation du schema import..."
|
|
"${PSQL[@]}" -c "DROP SCHEMA IF EXISTS import CASCADE;"
|
|
"${PSQL[@]}" -f "$PREPARE_SQL"
|
|
|
|
for table in "${required_tables[@]}"; do
|
|
exists="$("${PSQL[@]}" -tAc "SELECT to_regclass('${table}') IS NOT NULL;")"
|
|
if [[ "$exists" != "t" ]]; then
|
|
echo "Table manquante: ${table}" >&2
|
|
echo "Lance d'abord: psql -h ${DB_HOST} -p ${DB_PORT} -U ${DB_USER} -d ${DB_NAME} -f sql/prepare-import.sql" >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
for csv_file in "${required_files[@]}"; do
|
|
if [[ ! -f "$csv_file" ]]; then
|
|
echo "Fichier CSV manquant: ${csv_file#$ROOT_DIR/}" >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
echo "[0/3] Assouplissement des types texte des tables import..."
|
|
"${PSQL[@]}" <<'SQL'
|
|
DO $$
|
|
DECLARE r record;
|
|
BEGIN
|
|
FOR r IN
|
|
SELECT table_schema, table_name, column_name
|
|
FROM information_schema.columns
|
|
WHERE table_schema = 'import'
|
|
AND table_name IN ('personnes', 'periodes', 'choix_periodes', 'choix_localisations', 'tiers')
|
|
AND data_type IN ('character varying', 'character')
|
|
LOOP
|
|
EXECUTE format(
|
|
'ALTER TABLE %I.%I ALTER COLUMN %I TYPE text',
|
|
r.table_schema,
|
|
r.table_name,
|
|
r.column_name
|
|
);
|
|
END LOOP;
|
|
END $$;
|
|
SQL
|
|
|
|
echo "[1/3] Vidage des tables d'import..."
|
|
"${PSQL[@]}" <<'SQL'
|
|
TRUNCATE TABLE
|
|
import.personnes,
|
|
import.periodes,
|
|
import.choix_periodes,
|
|
import.choix_localisations,
|
|
import.tiers
|
|
RESTART IDENTITY;
|
|
SQL
|
|
|
|
echo "[2/3] Import des CSV du dossier csv/..."
|
|
|
|
copy_table_from_file() {
|
|
local table="$1"
|
|
local file_path="$2"
|
|
local columns="${3:-}"
|
|
local target="$table"
|
|
if [[ -n "$columns" ]]; then
|
|
target="${table}(${columns})"
|
|
fi
|
|
echo " - import ${table} <= ${file_path#$ROOT_DIR/}"
|
|
"${PSQL[@]}" -c "\\copy ${target} FROM STDIN WITH (FORMAT csv, HEADER true, DELIMITER ',')" < "$file_path"
|
|
}
|
|
|
|
copy_table_from_file "import.personnes" "$ROOT_DIR/csv/choix_usagers.csv" "civility,lastname,firstname,gender,gendercomment,nationality,memo,birthdate,place_of_birth,countryofbirth,deathdate,email,phonenumber,mobilenumber,contactinfo,street,extra,streetnumber,postcode,country"
|
|
|
|
echo " - generation des ids manquants dans import.personnes"
|
|
"${PSQL[@]}" <<'SQL'
|
|
WITH numbered AS (
|
|
SELECT ctid, row_number() OVER (ORDER BY ctid) AS new_id
|
|
FROM import.personnes
|
|
WHERE id IS NULL OR NULLIF(id::text, '') IS NULL
|
|
)
|
|
UPDATE import.personnes p
|
|
SET id = numbered.new_id
|
|
FROM numbered
|
|
WHERE p.ctid = numbered.ctid;
|
|
SQL
|
|
|
|
copy_table_from_file "import.periodes" "$ROOT_DIR/csv/periodes.csv" "id,nom,openingdate,closingdate,closingmotive,origin,remark,intensity,referrer,job,acp_scopes,address,personlocation,addresslocation,acp_socialissues,work_socialaction,comment1_content,comment2_content,comment3_content,comment4_content,comment5_content"
|
|
copy_table_from_file "import.choix_periodes" "$ROOT_DIR/csv/choix_periodes.csv" "closingmotive,origin,acp_scopes,job,referrer,parent,enfant,acp_social_issues,work_social_action,street,extra,streetnumber,postcode,country"
|
|
copy_table_from_file "import.choix_localisations" "$ROOT_DIR/csv/choix_localisations.csv" "title,\"addressRequired\",\"availableForUsers\",\"contactData\",\"defaultFor\",\"editableByUsers\""
|
|
copy_table_from_file "import.tiers" "$ROOT_DIR/csv/choix_tiers.csv" "acronym,nom,categorie"
|
|
|
|
echo "[3/3] Verification des volumes importes..."
|
|
"${PSQL[@]}" -c "SELECT 'personnes' AS table_name, count(*) AS rows FROM import.personnes UNION ALL SELECT 'periodes', count(*) FROM import.periodes UNION ALL SELECT 'choix_periodes', count(*) FROM import.choix_periodes UNION ALL SELECT 'choix_localisations', count(*) FROM import.choix_localisations UNION ALL SELECT 'tiers', count(*) FROM import.tiers;"
|
|
|
|
echo "Import termine sans doublons (tables videes au debut)."
|