From e845d9ba9054b377e4413fcc1b8d374b8f63ec8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Mon, 28 Jun 2021 22:01:33 +0200 Subject: [PATCH 1/3] replace search api by a first workin implementation --- .../Controller/SearchController.php | 10 +- .../ChillMainBundle/Search/SearchApi.php | 175 +++++++++++++----- .../ChillMainBundle/Search/SearchApiQuery.php | 85 +++++++++ .../Search/SearchApiResult.php | 40 ++++ .../config/services/search.yaml | 5 +- .../Repository/PersonRepository.php | 5 + .../Search/SearchPersonApiProvider.php | 52 ++++++ .../config/services/search.yaml | 4 + 8 files changed, 326 insertions(+), 50 deletions(-) create mode 100644 src/Bundle/ChillMainBundle/Search/SearchApiQuery.php create mode 100644 src/Bundle/ChillMainBundle/Search/SearchApiResult.php create mode 100644 src/Bundle/ChillPersonBundle/Search/SearchPersonApiProvider.php diff --git a/src/Bundle/ChillMainBundle/Controller/SearchController.php b/src/Bundle/ChillMainBundle/Controller/SearchController.php index 45390ceca..7c88d24fe 100644 --- a/src/Bundle/ChillMainBundle/Controller/SearchController.php +++ b/src/Bundle/ChillMainBundle/Controller/SearchController.php @@ -36,6 +36,7 @@ use Chill\MainBundle\Search\SearchProvider; use Symfony\Contracts\Translation\TranslatorInterface; use Chill\MainBundle\Pagination\PaginatorFactory; use Chill\MainBundle\Search\SearchApi; +use Symfony\Component\HttpFoundation\Exception\BadRequestException; /** * Class SearchController @@ -151,11 +152,14 @@ class SearchController extends AbstractController { //TODO this is an incomplete implementation $query = $request->query->get('q', ''); + $types = $request->query->get('type', []); - $results = $this->searchApi->getResults($query, 0, 150); - $paginator = $this->paginatorFactory->create(count($results)); + if (count($types) === 0) { + throw new BadRequestException("The request must contains at " + ." one type"); + } - $collection = new Collection($results, $paginator); + $collection = $this->searchApi->getResults($query, $types, []); return $this->json($collection); } diff --git a/src/Bundle/ChillMainBundle/Search/SearchApi.php b/src/Bundle/ChillMainBundle/Search/SearchApi.php index 518edd31b..4894c0f9a 100644 --- a/src/Bundle/ChillMainBundle/Search/SearchApi.php +++ b/src/Bundle/ChillMainBundle/Search/SearchApi.php @@ -2,88 +2,175 @@ namespace Chill\MainBundle\Search; +use Chill\MainBundle\Serializer\Model\Collection; +use Chill\MainBundle\Pagination\PaginatorFactory; use Chill\PersonBundle\Entity\Person; +use Chill\PersonBundle\Search\SearchPersonApiProvider; use Chill\ThirdPartyBundle\Entity\ThirdParty; +use Doctrine\DBAL\Types\Types; use Doctrine\ORM\EntityManagerInterface; +use Doctrine\ORM\Query\ResultSetMappingBuilder; use Chill\MainBundle\Search\SearchProvider; use Symfony\Component\VarDumper\Resources\functions\dump; /** - * ***Warning*** This is an incomplete implementation ***Warning*** */ class SearchApi { private EntityManagerInterface $em; private SearchProvider $search; + private PaginatorFactory $paginator; - public function __construct(EntityManagerInterface $em, SearchProvider $search) + private array $providers = []; + + public function __construct( + EntityManagerInterface $em, + SearchProvider $search, + SearchPersonApiProvider $searchPerson, + PaginatorFactory $paginator + ) { $this->em = $em; $this->search = $search; + $this->providers[] = $searchPerson; + $this->paginator = $paginator; } /** * @return Model/Result[] */ - public function getResults(string $query, int $offset, int $maxResult): array + public function getResults(string $pattern, array $types, array $parameters): Collection { - // **warning again**: this is an incomplete implementation - $results = []; + $queries = $this->findQueries($pattern, $types, $parameters); - foreach ($this->getPersons($query) as $p) { - $results[] = new Model\Result((float)\rand(0, 100) / 100, $p); - } - foreach ($this->getThirdParties($query) as $t) { - $results[] = new Model\Result((float)\rand(0, 100) / 100, $t); - } + $total = $this->countItems($queries, $types, $parameters); + $paginator = $this->paginator->create($total); - \usort($results, function(Model\Result $a, Model\Result $b) { - return ($a->getRelevance() <=> $b->getRelevance()) * -1; - }); + $rawResults = $this->fetchRawResult($queries, $types, $parameters, $paginator); + dump($rawResults); - return $results; + $this->prepareProviders($rawResults); + $results = $this->buildResults($rawResults); + dump($results); + + $collection = new Collection($results, $paginator); + + return $collection; } - public function countResults(string $query): int + private function findQueries($pattern, array $types, array $parameters): array { - return 0; + return \array_map( + fn($p) => $p->provideQuery($pattern, $parameters), + $this->findProviders($pattern, $types, $parameters), + ); } - private function getThirdParties(string $query) + private function findProviders(string $pattern, array $types, array $parameters): array { - $thirdPartiesIds = $this->em->createQuery('SELECT t.id FROM '.ThirdParty::class.' t') - ->getScalarResult(); - $nbResults = rand(0, 15); - - if ($nbResults === 1) { - $nbResults++; - } elseif ($nbResults === 0) { - return []; - } - $ids = \array_map(function ($e) use ($thirdPartiesIds) { return $thirdPartiesIds[$e]['id'];}, - \array_rand($thirdPartiesIds, $nbResults)); - - $a = $this->em->getRepository(ThirdParty::class) - ->findById($ids); - return $a; + return \array_filter( + $this->providers, + fn($p) => $p->supportsTypes($pattern, $types, $parameters) + ); } - private function getPersons(string $query) + private function countItems($providers, $types, $parameters): int { - $params = [ - SearchInterface::SEARCH_PREVIEW_OPTION => false - ]; - $search = $this->search->getResultByName($query, 'person_regular', 0, 50, $params, 'json'); - $ids = \array_map(function($r) { return $r['id']; }, $search['results']); + list($countQuery, $parameters) = $this->buildCountQuery($providers, $types, $parameters); + $rsmCount = new ResultSetMappingBuilder($this->em); + $rsmCount->addScalarResult('count', 'count'); + $countNq = $this->em->createNativeQuery($countQuery, $rsmCount); + $countNq->setParameters($parameters); + + return $countNq->getSingleScalarResult(); + } + private function buildCountQuery(array $queries, $types, $parameters) + { + $query = "SELECT COUNT(sq.key) AS count FROM ({union_unordered}) AS sq"; + $unions = []; + $parameters = []; - if (count($ids) === 0) { - return []; + foreach ($queries as $q) { + $unions[] = $q->buildQuery(); + $parameters = \array_merge($parameters, $q->buildParameters()); } - return $this->em->getRepository(Person::class) - ->findById($ids) + $unionUnordered = \implode(" UNION ", $unions); + + return [ + \strtr($query, [ '{union_unordered}' => $unionUnordered ]), + $parameters + ]; + } + + private function buildUnionQuery(array $queries, $types, $parameters) + { + $query = "{unions} ORDER BY pertinence DESC"; + $unions = []; + $parameters = []; + + foreach ($queries as $q) { + $unions[] = $q->buildQuery(); + $parameters = \array_merge($parameters, $q->buildParameters()); + } + + $union = \implode(" UNION ", $unions); + + return [ + \strtr($query, [ '{unions}' => $union]), + $parameters + ]; + } + + private function fetchRawResult($queries, $types, $parameters, $paginator): array + { + list($union, $parameters) = $this->buildUnionQuery($queries, $types, $parameters, $paginator); + $rsm = new ResultSetMappingBuilder($this->em); + $rsm->addScalarResult('key', 'key', Types::STRING) + ->addScalarResult('metadata', 'metadata', Types::JSON) + ->addScalarResult('pertinence', 'pertinence', Types::FLOAT) ; + + $nq = $this->em->createNativeQuery($union, $rsm); + $nq->setParameters($parameters); + + return $nq->getResult(); + } + + private function prepareProviders($rawResults) + { + $metadatas = []; + foreach ($rawResults as $r) { + foreach ($this->providers as $k => $p) { + if ($p->supportsResult($r['key'], $r['metadata'])) { + $metadatas[$k][] = $r['metadata']; + break; + } + } + } + + foreach ($metadatas as $k => $m) { + $this->providers[$k]->prepare($m); + } + } + + private function buildResults($rawResults) + { + foreach ($rawResults as $r) { + foreach ($this->providers as $k => $p) { + if ($p->supportsResult($r['key'], $r['metadata'])) { + $items[] = (new SearchApiResult($r['pertinence'])) + ->setResult( + $p->getResult($r['key'], $r['metadata'], $r['pertinence']) + ); + break; + } + } + } + + dump($items); + + return $items ?? []; } - } diff --git a/src/Bundle/ChillMainBundle/Search/SearchApiQuery.php b/src/Bundle/ChillMainBundle/Search/SearchApiQuery.php new file mode 100644 index 000000000..7a57d09ae --- /dev/null +++ b/src/Bundle/ChillMainBundle/Search/SearchApiQuery.php @@ -0,0 +1,85 @@ +selectKey = $selectKey; + $this->selectKeyParams = $params; + + return $this; + } + + public function setSelectJsonbMetadata(string $jsonbMetadata, array $params = []): self + { + $this->jsonbMetadata = $jsonbMetadata; + $this->jsonbMetadataParams = $params; + + return $this; + } + + public function setSelectPertinence(string $pertinence, array $params = []): self + { + $this->pertinence = $pertinence; + $this->pertinenceParams = $params; + + return $this; + } + + public function setFromClause(string $fromClause, array $params = []): self + { + $this->fromClause = $fromClause; + $this->fromClauseParams = $params; + + return $this; + } + + public function setWhereClause(string $whereClause, array $params = []): self + { + $this->whereClause = $whereClause; + $this->whereClauseParams = $params; + + return $this; + } + + public function buildQuery(): string + { + return \strtr("SELECT + {key} AS key, + {metadata} AS metadata, + {pertinence} AS pertinence + FROM {from} + WHERE {where} + ", [ + '{key}' => $this->selectKey, + '{metadata}' => $this->jsonbMetadata, + '{pertinence}' => $this->pertinence, + '{from}' => $this->fromClause, + '{where}' => $this->whereClause + ]); + } + + public function buildParameters(): array + { + return \array_merge( + $this->selectKeyParams, + $this->jsonbMetadataParams, + $this->pertinenceParams, + $this->fromClauseParams, + $this->whereClauseParams + ); + } +} diff --git a/src/Bundle/ChillMainBundle/Search/SearchApiResult.php b/src/Bundle/ChillMainBundle/Search/SearchApiResult.php new file mode 100644 index 000000000..bbc66877f --- /dev/null +++ b/src/Bundle/ChillMainBundle/Search/SearchApiResult.php @@ -0,0 +1,40 @@ +relevance = $relevance; + } + + public function setResult($result): self + { + $this->result = $result; + + return $this; + } + + /** + * @Serializer\Groups({"read"}) + */ + public function getResult() + { + return $this->result; + } + + /** + * @Serializer\Groups({"read"}) + */ + public function getRelevance(): float + { + return $this->relevance; + } +} diff --git a/src/Bundle/ChillMainBundle/config/services/search.yaml b/src/Bundle/ChillMainBundle/config/services/search.yaml index b7a1656b3..672c956c5 100644 --- a/src/Bundle/ChillMainBundle/config/services/search.yaml +++ b/src/Bundle/ChillMainBundle/config/services/search.yaml @@ -5,6 +5,5 @@ services: Chill\MainBundle\Search\SearchProvider: '@chill_main.search_provider' Chill\MainBundle\Search\SearchApi: - arguments: - $em: '@Doctrine\ORM\EntityManagerInterface' - $search: '@Chill\MainBundle\Search\SearchProvider' + autowire: true + autoconfigure: true diff --git a/src/Bundle/ChillPersonBundle/Repository/PersonRepository.php b/src/Bundle/ChillPersonBundle/Repository/PersonRepository.php index fdd0f054e..d99ab590e 100644 --- a/src/Bundle/ChillPersonBundle/Repository/PersonRepository.php +++ b/src/Bundle/ChillPersonBundle/Repository/PersonRepository.php @@ -37,6 +37,11 @@ final class PersonRepository return $this->repository->find($id, $lockMode, $lockVersion); } + public function findByIds($ids): array + { + return $this->repository->findBy(['id' => $ids]); + } + /** * @param $centers * @param $firstResult diff --git a/src/Bundle/ChillPersonBundle/Search/SearchPersonApiProvider.php b/src/Bundle/ChillPersonBundle/Search/SearchPersonApiProvider.php new file mode 100644 index 000000000..6b1c5a3e0 --- /dev/null +++ b/src/Bundle/ChillPersonBundle/Search/SearchPersonApiProvider.php @@ -0,0 +1,52 @@ +personRepository = $personRepository; + } + + public function provideQuery(string $pattern, array $parameters): SearchApiQuery + { + $query = new SearchApiQuery(); + $query + ->setSelectKey("'person'") + ->setSelectJsonbMetadata("jsonb_build_object('id', person.id)") + ->setSelectPertinence("SIMILARITY(LOWER(UNACCENT(?)), person.fullnamecanonical)", [ $pattern ]) + ->setFromClause("chill_person_person AS person") + ->setWhereClause("SIMILARITY(LOWER(UNACCENT(?)), person.fullnamecanonical) > 0.20", [ $pattern ]) + ; + + return $query; + } + + public function supportsTypes(string $pattern, array $types, array $parameters): bool + { + return \in_array('person', $types); + } + + public function prepare(array $metadatas): void + { + $ids = \array_map(fn($m) => $m['id'], $metadatas); + + $this->personRepository->findByIds($ids); + } + + public function supportsResult(string $key, array $metadatas): bool + { + return $key === 'person'; + } + + public function getResult(string $key, array $metadata, float $pertinence) + { + return $this->personRepository->find($metadata['id']); + } +} diff --git a/src/Bundle/ChillPersonBundle/config/services/search.yaml b/src/Bundle/ChillPersonBundle/config/services/search.yaml index 8b4db1373..ca6591246 100644 --- a/src/Bundle/ChillPersonBundle/config/services/search.yaml +++ b/src/Bundle/ChillPersonBundle/config/services/search.yaml @@ -28,3 +28,7 @@ services: $em: '@Doctrine\ORM\EntityManagerInterface' $tokenStorage: '@Symfony\Component\Security\Core\Authentication\Token\Storage\TokenStorageInterface' $authorizationHelper: '@Chill\MainBundle\Security\Authorization\AuthorizationHelper' + + Chill\PersonBundle\Search\SearchPersonApiProvider: + autowire: true + autoconfigure: true From 06406318210f3ab1eb6f53a5213f1775b15e4ccf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Mon, 28 Jun 2021 22:44:42 +0200 Subject: [PATCH 2/3] implementation on 3party + quote keys in SearchApiQuery --- .../ChillMainBundle/Search/SearchApi.php | 12 ++--- .../Search/SearchApiInterface.php | 18 +++++++ .../ChillMainBundle/Search/SearchApiQuery.php | 2 +- .../Search/SearchPersonApiProvider.php | 5 +- .../Search/ThirdPartyApiSearch.php | 48 +++++++++++++++++++ .../config/services/search.yaml | 4 ++ 6 files changed, 77 insertions(+), 12 deletions(-) create mode 100644 src/Bundle/ChillMainBundle/Search/SearchApiInterface.php create mode 100644 src/Bundle/ChillThirdPartyBundle/Search/ThirdPartyApiSearch.php diff --git a/src/Bundle/ChillMainBundle/Search/SearchApi.php b/src/Bundle/ChillMainBundle/Search/SearchApi.php index 4894c0f9a..677286f1f 100644 --- a/src/Bundle/ChillMainBundle/Search/SearchApi.php +++ b/src/Bundle/ChillMainBundle/Search/SearchApi.php @@ -4,9 +4,8 @@ namespace Chill\MainBundle\Search; use Chill\MainBundle\Serializer\Model\Collection; use Chill\MainBundle\Pagination\PaginatorFactory; -use Chill\PersonBundle\Entity\Person; use Chill\PersonBundle\Search\SearchPersonApiProvider; -use Chill\ThirdPartyBundle\Entity\ThirdParty; +use Chill\ThirdPartyBundle\Search\ThirdPartyApiSearch; use Doctrine\DBAL\Types\Types; use Doctrine\ORM\EntityManagerInterface; use Doctrine\ORM\Query\ResultSetMappingBuilder; @@ -18,21 +17,20 @@ use Symfony\Component\VarDumper\Resources\functions\dump; class SearchApi { private EntityManagerInterface $em; - private SearchProvider $search; private PaginatorFactory $paginator; private array $providers = []; public function __construct( EntityManagerInterface $em, - SearchProvider $search, SearchPersonApiProvider $searchPerson, + ThirdPartyApiSearch $thirdPartyApiSearch, PaginatorFactory $paginator ) { $this->em = $em; - $this->search = $search; $this->providers[] = $searchPerson; + $this->providers[] = $thirdPartyApiSearch; $this->paginator = $paginator; } @@ -47,11 +45,9 @@ class SearchApi $paginator = $this->paginator->create($total); $rawResults = $this->fetchRawResult($queries, $types, $parameters, $paginator); - dump($rawResults); $this->prepareProviders($rawResults); $results = $this->buildResults($rawResults); - dump($results); $collection = new Collection($results, $paginator); @@ -169,8 +165,6 @@ class SearchApi } } - dump($items); - return $items ?? []; } } diff --git a/src/Bundle/ChillMainBundle/Search/SearchApiInterface.php b/src/Bundle/ChillMainBundle/Search/SearchApiInterface.php new file mode 100644 index 000000000..54269c946 --- /dev/null +++ b/src/Bundle/ChillMainBundle/Search/SearchApiInterface.php @@ -0,0 +1,18 @@ +setSelectKey("'person'") + ->setSelectKey("person") ->setSelectJsonbMetadata("jsonb_build_object('id', person.id)") ->setSelectPertinence("SIMILARITY(LOWER(UNACCENT(?)), person.fullnamecanonical)", [ $pattern ]) ->setFromClause("chill_person_person AS person") diff --git a/src/Bundle/ChillThirdPartyBundle/Search/ThirdPartyApiSearch.php b/src/Bundle/ChillThirdPartyBundle/Search/ThirdPartyApiSearch.php new file mode 100644 index 000000000..29a341b1b --- /dev/null +++ b/src/Bundle/ChillThirdPartyBundle/Search/ThirdPartyApiSearch.php @@ -0,0 +1,48 @@ +thirdPartyRepository = $thirdPartyRepository; + } + + public function provideQuery(string $pattern, array $parameters): SearchApiQuery + { + return (new SearchApiQuery) + ->setSelectKey('tparty') + ->setSelectJsonbMetadata("jsonb_build_object('id', tparty.id)") + ->setSelectPertinence("SIMILARITY(?, LOWER(UNACCENT(tparty.name)))", [ $pattern ]) + ->setFromClause('chill_3party.third_party AS tparty') + ->setWhereClause('SIMILARITY(LOWER(UNACCENT(?)), LOWER(UNACCENT(tparty.name))) > 0.20', [ $pattern ]) + ; + } + + public function supportsTypes(string $pattern, array $types, array $parameters): bool + { + return \in_array('thirdparty', $types); + } + + public function prepare(array $metadatas): void + { + + } + + public function supportsResult(string $key, array $metadatas): bool + { + return $key === 'tparty'; + } + + public function getResult(string $key, array $metadata, float $pertinence) + { + return $this->thirdPartyRepository->find($metadata['id']); + } +} diff --git a/src/Bundle/ChillThirdPartyBundle/config/services/search.yaml b/src/Bundle/ChillThirdPartyBundle/config/services/search.yaml index f4ff72db9..62b5dd205 100644 --- a/src/Bundle/ChillThirdPartyBundle/config/services/search.yaml +++ b/src/Bundle/ChillThirdPartyBundle/config/services/search.yaml @@ -7,3 +7,7 @@ services: $paginatorFactory: '@Chill\MainBundle\Pagination\PaginatorFactory' tags: - { name: 'chill.search', alias: '3party' } + + Chill\ThirdPartyBundle\Search\ThirdPartyApiSearch: + autowire: true + autoconfigure: true From dd991e3572552da63b72e7bbfb24eeaaa3f37366 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Fastr=C3=A9?= Date: Mon, 28 Jun 2021 23:02:15 +0200 Subject: [PATCH 3/3] add test --- .../Controller/SearchApiControllerTest.php | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 src/Bundle/ChillMainBundle/Tests/Controller/SearchApiControllerTest.php diff --git a/src/Bundle/ChillMainBundle/Tests/Controller/SearchApiControllerTest.php b/src/Bundle/ChillMainBundle/Tests/Controller/SearchApiControllerTest.php new file mode 100644 index 000000000..1c76d8975 --- /dev/null +++ b/src/Bundle/ChillMainBundle/Tests/Controller/SearchApiControllerTest.php @@ -0,0 +1,36 @@ +getClientAuthenticated(); + + $client->request( + Request::METHOD_GET, + '/api/1.0/search.json', + [ 'q' => $pattern, 'type' => $types ] + ); + + $this->assertResponseIsSuccessful(); + } + + public function generateSearchData() + { + yield ['per', ['person', 'thirdparty'] ]; + yield ['per', ['thirdparty'] ]; + yield ['per', ['person'] ]; + yield ['fjklmeqjfkdqjklrmefdqjklm', ['person', 'thirdparty'] ]; + } +}