logger = $logger; $this->helper = $helper; $this->em = $em; $this->customFieldProvider = $customFieldProvider; $this->eventDispatcher = $eventDispatcher; $this->formFactory = $formFactory; parent::__construct('chill:person:import'); } /** * This function is a shortcut to addOption. * * @param string $name * @param string $description * @param string $default * * @return ImportPeopleFromCSVCommand */ protected function addOptionShortcut($name, $description, $default) { $this->addOption($name, null, InputOption::VALUE_OPTIONAL, $description, $default); return $this; } protected function configure() { $this ->addArgument('csv_file', InputArgument::REQUIRED, 'The CSV file to import') ->setDescription('Import people from a csv file') ->setHelp( <<<'EOF' Import people from a csv file. The first row must contains the header column and will determines where the value will be matched. Date format: the possible date format may be separatedby an |. The possible format will be tryed from the first to the last. The format should be explained as http://php.net/manual/en/function.strftime.php php app/console chill:person:import /tmp/hepc.csv fr_FR.utf8 \ --firstname="Prénom" --lastname="Nom" \ --birthdate="D.N." --birthdate_format="%d/%m/%Y" \ --opening_date_format="%B %Y|%Y" --closing_date="der.contact" \ --closing_date_format="%Y" --custom-field="3=code" -vvv EOF ) ->addArgument( 'locale', InputArgument::REQUIRED, 'The locale to use in displaying translatable strings from entities' ) ->addOption( 'force-center', null, InputOption::VALUE_REQUIRED, 'The id of the center' ) ->addOption( 'force', null, InputOption::VALUE_NONE, 'Persist people in the database (default is not to persist people)' ) ->addOption( 'delimiter', 'd', InputOption::VALUE_OPTIONAL, 'The delimiter character of the csv file', ',' ) ->addOption( 'enclosure', null, InputOption::VALUE_OPTIONAL, 'The enclosure character of the csv file', '"' ) ->addOption( 'escape', null, InputOption::VALUE_OPTIONAL, 'The escape character of the csv file', '\\' ) ->addOption( 'length', null, InputOption::VALUE_OPTIONAL, 'The length of line to read. 0 means unlimited.', 0 ) ->addOption( 'dump-choice-matching', null, InputOption::VALUE_REQUIRED, 'The path of the file to dump the matching between label in CSV and answers' ) ->addOption( 'load-choice-matching', null, InputOption::VALUE_OPTIONAL, 'The path of the file to load the matching between label in CSV and answers' ); // mapping columns foreach (self::$mapping as $m) { $this->addOptionShortcut($m[0], $m[1], $m[2]); } // other information $this->addOptionShortcut( 'birthdate_format', 'Format preference for ' . 'birthdate. See help for date formats preferences.', self::$defaultDateInterpreter ); $this->addOptionShortcut( 'opening_date_format', 'Format preference for ' . 'opening date. See help for date formats preferences.', self::$defaultDateInterpreter ); $this->addOptionShortcut( 'closing_date_format', 'Format preference for ' . 'closing date. See help for date formats preferences.', self::$defaultDateInterpreter ); // mapping column to custom fields $this->addOption( 'custom-field', null, InputOption::VALUE_REQUIRED | InputOption::VALUE_IS_ARRAY, 'Mapping a column to a custom fields key. Example: 1=cf_slug' ); $this->addOption( 'skip-interactive-field-mapping', null, InputOption::VALUE_NONE, 'Do not ask for interactive mapping' ); } /** * @param array $headers the processed header : an array as prepared by self::processingHeaders * * @throws Exception */ protected function createPerson(array $row, array $headers): Person { // trying to get the opening date $openingDateString = trim($row[array_search('opening_date', $headers, true)]); $openingDate = $this->processDate($openingDateString, $this->input->getOption('opening_date_format')); // @TODO: Fix the constructor parameter, $openingDate does not exists. $person = $openingDate instanceof DateTime ? new Person($openingDate) : new Person(); // add the center $center = $this->getCenter($row, $headers); if (null === $center) { throw new Exception('center not found'); } $person->setCenter($center); foreach ($headers as $column => $info) { $value = trim($row[$column]); switch ($info) { case 'firstname': $person->setFirstName($value); break; case 'lastname': $person->setLastName($value); break; case 'birthdate': $this->processBirthdate($person, $value); break; case 'gender': $person->setGender($value); break; case 'opening_date': // we have processed this when creating the person object, skipping; break; case 'closing_date': $this->processClosingDate($person, $value); break; case 'memo': $person->setMemo($value); break; case 'email': $person->setEmail($value); break; case 'phonenumber': $person->setPhonenumber($value); break; case 'mobilenumber': $person->setMobilenumber($value); break; // we just keep the column number for those data case 'postalcode': $postalCodeValue = $value; break; case 'street1': $street1Value = $value; break; case 'locality': $localityValue = $value; break; } } // handle address if (in_array('postalcode', $headers, true)) { if (!empty($postalCodeValue)) { $address = new Address(); $postalCode = $this->guessPostalCode($postalCodeValue, $localityValue ?? ''); if (null === $postalCode) { throw new Exception('The locality is not found'); } $address->setPostcode($postalCode); if (in_array('street1', $headers, true)) { $address->setStreetAddress1($street1Value); } $address->setValidFrom(new DateTime('today')); $person->addAddress($address); } } return $person; } protected function dumpAnswerMatching() { if ($this->input->hasOption('dump-choice-matching') && !empty($this->input->getOption('dump-choice-matching'))) { $this->logger->debug('Dump the matching between answer and choices'); $str = json_encode($this->cacheAnswersMapping, JSON_PRETTY_PRINT); $fs = new Filesystem(); $filename = $this->input->getOption('dump-choice-matching'); $fs->dumpFile($filename, $str); } } /** * @throws Exception * * @return int|void|null */ protected function execute(InputInterface $input, OutputInterface $output) { $headers = $rawHeaders = []; $this->input = $input; $this->output = $output; $this->logger->debug('Setting locale to ' . $input->getArgument('locale')); setlocale(LC_TIME, $input->getArgument('locale')); // opening csv as resource $csv = $this->openCSV(); $num = 0; $line = $this->line = 1; try { while (false !== ($row = fgetcsv( $csv, $input->getOption('length'), $input->getOption('delimiter'), $input->getOption('enclosure'), $input->getOption('escape') ))) { $this->logger->debug('Processing line ' . $this->line); if (1 === $line) { $this->logger->debug('Processing line 1, headers'); $rawHeaders = $row; $headers = $this->processingHeaders($row); } else { $person = $this->createPerson($row, $headers); if (count($this->customFieldMapping) > 0) { $this->processingCustomFields($person, $row); } $event = new Event(); $event->person = $person; $event->rawHeaders = $rawHeaders; $event->row = $row; $event->headers = $headers; $event->skipPerson = false; $event->force = $this->input->getOption('force'); $event->input = $this->input; $event->output = $this->output; $event->helperSet = $this->getHelperSet(); $this->eventDispatcher->dispatch('chill_person.person_import', $event); if ($this->input->getOption('force') === true && false === $event->skipPerson) { $this->em->persist($person); } ++$num; } ++$line; ++$this->line; } if ($this->input->getOption('force') === true) { $this->logger->debug('persisting entitites'); $this->em->flush(); } } finally { $this->logger->debug('closing csv', ['method' => __METHOD__]); fclose($csv); // dump the matching between answer and choices $this->dumpAnswerMatching(); } } /** * @return Center|mixed|object|null */ protected function getCenter(array $row, array $headers) { if ($this->input->hasOption('force-center') && !empty($this->input->getOption('force-center'))) { return $this->em->getRepository(Center::class)->find($this->input->getOption('force-center')); } $columnCenter = array_search('center', $headers, true); $centerName = trim($row[$columnCenter]); try { return $this ->em ->createQuery('SELECT c FROM ChillMainBundle:Center c WHERE c.name = :center_name') ->setParameter('center_name', $centerName) ->getSingleResult(); } catch (NonUniqueResultException $e) { return $this->guessCenter($centerName); } catch (NoResultException $e) { return $this->guessCenter($centerName); } } /** * @param $centerName * * @return Center|mixed|object|null */ protected function guessCenter($centerName) { if (!array_key_exists('_center_picked', $this->cacheAnswersMapping)) { $this->cacheAnswersMapping['_center_picked'] = []; } if (array_key_exists($centerName, $this->cacheAnswersMapping['_center_picked'])) { $id = $this->cacheAnswersMapping['_center_picked'][$centerName]; return $this->em->getRepository(Center::class) ->find($id); } $centers = $this->em->createQuery('SELECT c FROM ChillMainBundle:Center c ' . 'ORDER BY SIMILARITY(c.name, :center_name) DESC') ->setParameter('center_name', $centerName) ->setMaxResults(10) ->getResult(); if (count($centers) > 1) { if (strtolower($centers[0]->getName()) === strtolower($centerName)) { return $centers[0]; } } $centersByName = []; $names = array_map(static function (Center $c) use (&$centersByName) { $n = $c->getName(); $centersByName[$n] = $c; return $n; }, $centers); $names[] = 'none of them'; $helper = $this->getHelper('question'); $question = new ChoiceQuestion( sprintf('Which center match the name "%s" ? (default to "%s")', $centerName, $names[0]), $names, 0 ); $answer = $helper->ask($this->input, $this->output, $question); if ('none of them' === $answer) { $questionCreate = new ConfirmationQuestion('Would you like to create it ?', false); $create = $helper->ask($this->input, $this->output, $questionCreate); if ($create) { $center = (new Center()) ->setName($centerName); if ($this->input->getOption('force') === true) { $this->em->persist($center); $this->em->flush(); } return $center; } } $center = $centersByName[$answer]; $this->cacheAnswersMapping['_center_picked'][$centerName] = $center->getId(); return $center; } /** * @param $postalCode * @param $locality * * @return mixed|null */ protected function guessPostalCode($postalCode, $locality) { if (!array_key_exists('_postal_code_picked', $this->cacheAnswersMapping)) { $this->cacheAnswersMapping['_postal_code_picked'] = []; } if (array_key_exists($postalCode, $this->cacheAnswersMapping['_postal_code_picked'])) { if (array_key_exists($locality, $this->cacheAnswersMapping['_postal_code_picked'][$postalCode])) { $id = $this->cacheAnswersMapping['_postal_code_picked'][$postalCode][$locality]; return $this->em->getRepository(PostalCode::class)->find($id); } } $postalCodes = $this->em->createQuery( 'SELECT pc FROM ' . PostalCode::class . ' pc ' . 'WHERE pc.code = :postal_code ' . 'ORDER BY SIMILARITY(pc.name, :locality) DESC ' ) ->setMaxResults(10) ->setParameter('postal_code', $postalCode) ->setParameter('locality', $locality) ->getResult(); if (count($postalCodes) >= 1) { if ($postalCodes[0]->getCode() === $postalCode && $postalCodes[0]->getName() === $locality) { return $postalCodes[0]; } } if (count($postalCodes) === 0) { return null; } $postalCodeByName = []; $names = array_map(static function (PostalCode $pc) use (&$postalCodeByName) { $n = $pc->getName(); $postalCodeByName[$n] = $pc; return $n; }, $postalCodes); $names[] = 'none of them'; $helper = $this->getHelper('question'); $question = new ChoiceQuestion( sprintf( 'Which postal code match the ' . 'name "%s" with postal code "%s" ? (default to "%s")', $locality, $postalCode, $names[0] ), $names, 0 ); $answer = $helper->ask($this->input, $this->output, $question); if ('none of them' === $answer) { return null; } $pc = $postalCodeByName[$answer]; $this->cacheAnswersMapping['_postal_code_picked'][$postalCode][$locality] = $pc->getId(); return $pc; } protected function interact(InputInterface $input, OutputInterface $output) { // preparing the basic $this->input = $input; $this->output = $output; $this->logger = new ConsoleLogger($output); $csv = $this->openCSV(); // getting the first row if (false !== ($row = fgetcsv( $csv, $input->getOption('length'), $input->getOption('delimiter'), $input->getOption('enclosure'), $input->getOption('escape') ))) { try { $this->matchColumnToCustomField($row); } finally { $this->logger->debug('closing csv', ['method' => __METHOD__]); fclose($csv); } } // load the matching between csv and label $this->loadAnswerMatching(); } /** * Load the mapping between answer in CSV and value in choices from a json file. */ protected function loadAnswerMatching() { if ($this->input->hasOption('load-choice-matching')) { $fs = new Filesystem(); $filename = $this->input->getOption('load-choice-matching'); if (!$fs->exists($filename)) { $this->logger->warning("The file {$filename} is not found. Choice matching not loaded"); } else { $this->logger->debug("Loading {$filename} as choice matching"); $this->cacheAnswersMapping = json_decode(file_get_contents($filename), true); } } } /** * @param $row */ protected function matchColumnToCustomField($row) { $cfMappingsOptions = $this->input->getOption('custom-field'); /** @var \Doctrine\Persistence\ObjectManager $em */ $em = $this->em; foreach ($cfMappingsOptions as $cfMappingStringOption) { [$rowNumber, $cfSlug] = preg_split('|=|', $cfMappingStringOption); // check that the column exists, getting the column name $column = $row[$rowNumber]; if (empty($column)) { $message = "The column with row {$rowNumber} is empty."; $this->logger->error($message); throw new RuntimeException($message); } // check a custom field exists try { $customField = $em->createQuery('SELECT cf ' . 'FROM ChillCustomFieldsBundle:CustomField cf ' . 'JOIN cf.customFieldGroup g ' . 'WHERE cf.slug = :slug ' . 'AND g.entity = :entity') ->setParameters([ 'slug' => $cfSlug, 'entity' => Person::class, ]) ->getSingleResult(); } catch (\Doctrine\ORM\NoResultException $e) { $message = sprintf( "The customfield with slug '%s' does not exists. It was associated with column number %d", $cfSlug, $rowNumber ); $this->logger->error($message); throw new RuntimeException($message); } // skip if custom field does not exists if (null === $customField) { $this->logger->error("The custom field with slug {$cfSlug} could not be found. " . 'Stopping this command.'); throw new RuntimeException("The custom field with slug {$cfSlug} could not be found. " . 'Stopping this command.'); } $this->logger->notice(sprintf( "Matched custom field %s (question : '%s') on column %d (displayed in the file as '%s')", $customField->getSlug(), $this->helper->localize($customField->getName()), $rowNumber, $column )); $this->customFieldMapping[$rowNumber] = $customField; } } /** * @throws RuntimeException * * @return resource */ protected function openCSV() { $fs = new Filesystem(); $filename = $this->input->getArgument('csv_file'); if (!$fs->exists($filename)) { throw new RuntimeException('The file does not exists or you do not ' . 'have the right to read it.'); } $resource = fopen($filename, 'rb'); if (false === $resource) { throw new RuntimeException("The file '{$filename}' could not be opened."); } return $resource; } /** * @param $value * * @throws Exception */ protected function processBirthdate(Person $person, $value) { if (empty($value)) { return; } $date = $this->processDate($value, $this->input->getOption('birthdate_format')); if ($date instanceof DateTime) { // we correct birthdate if the date is in the future // the most common error is to set date 100 years to late (ex. 2063 instead of 1963) if ($date > new DateTime('yesterday')) { $date = $date->sub(new DateInterval('P100Y')); } $person->setBirthdate($date); return; } // if we arrive here, we could not process the date $this->logger->warning(sprintf( 'Line %d : the birthdate could not be interpreted. Was %s.', $this->line, $value )); } /** * Process a custom field choice. * * The method try to guess if the result exists amongst the text of the possible * choices. If the texts exists, then this is picked. Else, ask the user. * * @param string $value * * @throws Exception * * @return string */ protected function processChoiceType( $value, \Symfony\Component\Form\FormInterface $form, \Chill\CustomFieldsBundle\Entity\CustomField $cf ) { // getting the possible answer and their value : $view = $form->get($cf->getSlug())->createView(); $answers = $this->collectChoicesAnswers($view->vars['choices']); // if we do not have any answer on the question, throw an error. if (count($answers) === 0) { $message = sprintf( "The question '%s' with slug '%s' does not count any answer.", $this->helper->localize($cf->getName()), $cf->getSlug() ); $this->logger->error($message, [ 'method' => __METHOD__, 'slug' => $cf->getSlug(), 'question' => $this->helper->localize($cf->getName()), ]); throw new RuntimeException($message); } if (false === $view->vars['required']) { $answers[null] = '** no answer'; } // the answer does not exists in cache. Try to find it, or asks the user if (!isset($this->cacheAnswersMapping[$cf->getSlug()][$value])) { // try to find the answer (with array_keys and a search value $values = array_keys( array_map(static function ($label) { return trim(strtolower($label)); }, $answers), trim(strtolower($value)), true ); if (count($values) === 1) { // we could guess an answer ! $this->logger->info('This question accept multiple answers'); $this->cacheAnswersMapping[$cf->getSlug()][$value] = false === $view->vars['multiple'] ? $values[0] : [$values[0]]; $this->logger->info(sprintf( "Guessed that value '%s' match with key '%s' " . 'because the CSV and the label are equals.', $value, $values[0] )); } else { // we could nog guess an answer. Asking the user. $this->output->writeln('I do not know the answer to this question : '); $this->output->writeln($this->helper->localize($cf->getName())); // printing the possible answers /** @var \Symfony\Component\Console\Helper\Table $table */ $table = new Table($this->output); $table->setHeaders(['#', 'label', 'value']); $i = 0; $matchingTableRowAnswer = []; foreach ($answers as $key => $answer) { $table->addRow([ $i, $answer, $key, ]); $matchingTableRowAnswer[$i] = $key; ++$i; } $table->render($this->output); $question = new ChoiceQuestion( sprintf('Please pick your choice for the value "%s"', $value), array_keys($matchingTableRowAnswer) ); $question->setErrorMessage('This choice is not possible'); if ($view->vars['multiple']) { $this->logger->debug('this question is multiple'); $question->setMultiselect(true); } $selected = $this->getHelper('question')->ask($this->input, $this->output, $question); $this->output->writeln( sprintf( 'You have selected "%s"', is_array($answers[$matchingTableRowAnswer[$selected]]) ? implode(',', $answers[$matchingTableRowAnswer[$selected]]) : $answers[$matchingTableRowAnswer[$selected]] ) ); // recording value in cache $this->cacheAnswersMapping[$cf->getSlug()][$value] = $matchingTableRowAnswer[$selected]; $this->logger->debug(sprintf( "Setting the value '%s' in cache for customfield '%s' and answer '%s'", is_array($this->cacheAnswersMapping[$cf->getSlug()][$value]) ? implode(', ', $this->cacheAnswersMapping[$cf->getSlug()][$value]) : $this->cacheAnswersMapping[$cf->getSlug()][$value], $cf->getSlug(), $value )); } } $form->submit([$cf->getSlug() => $this->cacheAnswersMapping[$cf->getSlug()][$value]]); $value = $form->getData()[$cf->getSlug()]; $this->logger->debug( sprintf( "Found value : %s for custom field with question '%s'", is_array($value) ? implode(',', $value) : $value, $this->helper->localize($cf->getName()) ) ); return $value; } /** * @param $value * * @throws Exception */ protected function processClosingDate(Person $person, $value) { if (empty($value)) { return; } // we skip if the opening date is now (or after yesterday) /** @var \Chill\PersonBundle\Entity\AccompanyingPeriod $period */ $period = $person->getCurrentAccompanyingPeriod(); if ($period->getOpeningDate() > new DateTime('yesterday')) { $this->logger->debug(sprintf( 'skipping a closing date because opening date is after yesterday (%s)', $period->getOpeningDate()->format('Y-m-d') )); return; } $date = $this->processDate($value, $this->input->getOption('closing_date_format')); if ($date instanceof DateTime) { // we correct birthdate if the date is in the future // the most common error is to set date 100 years to late (ex. 2063 instead of 1963) if ($date > new DateTime('yesterday')) { $date = $date->sub(new DateInterval('P100Y')); } $period->setClosingDate($date); $person->close(); return; } // if we arrive here, we could not process the date $this->logger->warning(sprintf( 'Line %d : the closing date could not be interpreted. Was %s.', $this->line, $value )); } /** * @param $value * @param $formats * * @return bool|DateTime */ protected function processDate($value, $formats) { $possibleFormats = explode('|', $formats); foreach ($possibleFormats as $format) { $this->logger->debug("Trying format {$format}", [__METHOD__]); $dateR = strptime($value, $format); if (is_array($dateR) && '' === $dateR['unparsed']) { $string = sprintf( '%04d-%02d-%02d %02d:%02d:%02d', ($dateR['tm_year'] + 1900), ($dateR['tm_mon'] + 1), ($dateR['tm_mday']), ($dateR['tm_hour']), ($dateR['tm_min']), ($dateR['tm_sec']) ); $date = DateTime::createFromFormat('Y-m-d H:i:s', $string); $this->logger->debug(sprintf('Interpreting %s as date %s', $value, $date->format('Y-m-d H:i:s'))); return $date; } } // if we arrive here, we could not process the date $this->logger->debug(sprintf( 'Line %d : a date could not be interpreted. Was %s.', $this->line, $value )); return false; } /** * @param $row * * @throws Exception */ protected function processingCustomFields(Person $person, $row) { /** @var \Chill\CustomFieldsBundle\Service\CustomFieldProvider $cfProvider */ $cfProvider = $this->customFieldProvider; $cfData = []; /** @var \Chill\CustomFieldsBundle\Entity\CustomField $$customField */ foreach ($this->customFieldMapping as $rowNumber => $customField) { $builder = $this->formFactory->createBuilder(); $cfProvider->getCustomFieldByType($customField->getType()) ->buildForm($builder, $customField); $form = $builder->getForm(); // get the type of the form $type = get_class($form->get($customField->getSlug()) ->getConfig()->getType()->getInnerType()); $this->logger->debug(sprintf( 'Processing a form of type %s', $type )); switch ($type) { case \Symfony\Component\Form\Extension\Core\Type\TextType::class: $cfData[$customField->getSlug()] = $this->processTextType($row[$rowNumber], $form, $customField); break; case \Symfony\Component\Form\Extension\Core\Type\ChoiceType::class: case \Chill\MainBundle\Form\Type\Select2ChoiceType::class: $cfData[$customField->getSlug()] = $this->processChoiceType($row[$rowNumber], $form, $customField); } } $person->setCFData($cfData); } /** * @return array where keys are column number, and value is information mapped */ protected function processingHeaders(array $firstRow): array { $availableOptions = array_map( static fn (array $m) => $m[0], self::$mapping ); $matchedColumnHeaders = $headers = []; foreach ($availableOptions as $option) { $matchedColumnHeaders[$option] = $this->input->getOption($option); } foreach ($firstRow as $key => $content) { $content = trim($content); if (in_array($content, $matchedColumnHeaders, true)) { $information = array_search($content, $matchedColumnHeaders, true); $headers[$key] = $information; $this->logger->notice("Matched {$information} on column {$key} (displayed in the file as '{$content}')"); } else { $this->logger->notice("Column with content '{$content}' is ignored"); } } return $headers; } /** * Process a text type on a custom field. * * @param type $value * * @return type */ protected function processTextType( $value, \Symfony\Component\Form\FormInterface $form, \Chill\CustomFieldsBundle\Entity\CustomField $cf ) { $form->submit([$cf->getSlug() => $value]); $value = $form->getData()[$cf->getSlug()]; $this->logger->debug(sprintf('Found value : %s for custom field with question ' . "'%s'", $value, $this->helper->localize($cf->getName()))); return $value; } /** * Recursive method to collect the possibles answer from a ChoiceType (or * its inherited types). * * @param mixed $choices * * @throws Exception * * @return array where */ private function collectChoicesAnswers($choices) { $answers = []; /** @var \Symfony\Component\Form\ChoiceList\View\ChoiceView $choice */ foreach ($choices as $choice) { if ($choice instanceof \Symfony\Component\Form\ChoiceList\View\ChoiceView) { $answers[$choice->value] = $choice->label; } elseif ($choice instanceof \Symfony\Component\Form\ChoiceList\View\ChoiceGroupView) { $answers = $answers + $this->collectChoicesAnswers($choice->choices); } else { throw new Exception(sprintf( "The choice type is not know. Expected '%s' or '%s', get '%s'", \Symfony\Component\Form\ChoiceList\View\ChoiceView::class, \Symfony\Component\Form\ChoiceList\View\ChoiceGroupView::class, get_class($choice) )); } } return $answers; } }