diff --git a/web/modules/custom/materio_sapi/src/Controller/Base.php b/web/modules/custom/materio_sapi/src/Controller/Base.php index 89e547dc..2cd210fd 100644 --- a/web/modules/custom/materio_sapi/src/Controller/Base.php +++ b/web/modules/custom/materio_sapi/src/Controller/Base.php @@ -7,6 +7,7 @@ use Drupal\Core\Controller\ControllerBase; use Symfony\Component\HttpFoundation\JsonResponse; use Symfony\Component\HttpFoundation\Request; use Drupal\Component\Utility\Unicode; +use Drupal\materio_sapi\KeywordText; use Drupal\search_api\Entity\Index; // https://drupal.stackexchange.com/questions/225008/programatically-use-search-api @@ -52,11 +53,11 @@ class Base extends ControllerBase { protected $phrase_query, $and_query, $or_query, $query, $count_query, $count, $exactematch_count; /** - * Filter and term values. + * Term values (autocomplete choice) and items. * * @var mixed */ - protected $filters, $terms, $items; + protected $terms, $items; private $limit = 15; private $offset = 0; @@ -75,144 +76,77 @@ class Base extends ControllerBase { */ protected $phrase = ''; + /** + * Most meaningful words for which the phrase and AND queries run (H2, a). + * + * Beyond, they find nothing (no node holds every word of a long phrase) and + * the AND query may exceed maxClauseCount: only the OR query runs. + */ + public const SHORT_QUERY_WORDS = 6; + + /** + * Runs the keyword queries and merges their results, exact matches first. + * + * The keys are the phrase without stop words (H2, b): the custom types of + * materio_sapi keep them, « de » alone matched 2 236 nodes. + */ private function sapiQuery(){ - // https://www.drupal.org/docs/8/modules/search-api/developer-documentation/executing-a-search-in-code - // https://www.hashbangcode.com/article/drupal-8-date-search-boosting-search-api-and-solr-search - // https://kgaut.net/blog/2018/drupal-8-search-api-effectuer-une-requete-dans-le-code.html - - - // Otherwise, if you use Solr, you can also set the parse mode to “Direct query” - // and use something like 'tm_name:alex AND tm_surname:john' as the keywords (field names may vary). - // That will also ensure scoring works for the keywords (the other way just adds filters, which don’t affect scoring). - // https://www.drupal.org/project/search_api/issues/3049097 - - - $lang = \Drupal::languageManager()->getCurrentLanguage()->getId(); - $this->index = Index::load('database'); + $words = KeywordText::words($this->phrase, $lang); + // Only stop words: search the phrase as typed, as before. + $keys = $words ? implode(' ', $words) : implode(' ', $this->keys); + + $exact = []; + if (count($words) <= self::SHORT_QUERY_WORDS) { + // Exact material names (like "wood-skin"), then all words. + $exact = $this->phraseQuery($keys, $lang); + $exact += $this->andQuery($keys, $lang); + } + $this->exactematch_count = count($exact); + $all = $exact + $this->orQuery($keys, $lang); + $this->results = [ - 'uuids' => [], - 'nids' => [] + 'uuids' => array_column($all, 1), + 'nids' => array_column($all, 0), ]; - - - - - - - - - // ,---.| ,---. - // |---'|---.,---.,---.,---.,---. | |. .,---.,---., . - // | | || ,---|`---.|---' | || ||---'| | | - // ` ` '` `---^`---'`---' `---\`---'`---'` `---| - // `---' - // (to match exact materials names (like "wood-skin")) - $this->phrase_query = $this->index->query(['offset'=>0,'limit'=>10000]); - // set parse mode and conjunction - $parse_mode = \Drupal::service('plugin.manager.search_api.parse_mode') - ->createInstance('phrase'); - $parse_mode->setConjunction('AND'); - $this->phrase_query->setParseMode($parse_mode); - // Set fulltext search keywords and fields. - if ($this->keys) { - $this->phrase_query->keys(implode(' ', $this->keys)); - } + } + + /** + * The whole keys in the title. + * + * @return array + * [nid, uuid] keyed by nid, in result order. + */ + private function phraseQuery(string $keys, string $lang): array { + $this->phrase_query = $this->keywordQuery('phrase', 'AND', $keys, $lang, 'materio_sapi_search_phrase_query'); $this->phrase_query->setFulltextFields(['title']); - - // Restrict the search to specific languages. - $this->phrase_query->setLanguages([$lang]); + return $this->collect($this->phrase_query); + } - // Do paging. - // $this->and_query->range($this->offset, $this->limit); - // retrieve all results - // $this->and_query->range(0, -1); + /** + * Every word in the thesaurus and tag fields, or the chosen terms. + * + * @return array + * [nid, uuid] keyed by nid, in result order. + */ + private function andQuery(string $keys, string $lang): array { + $this->and_query = $this->keywordQuery('terms', 'AND', $keys, $lang, 'materio_sapi_search_and_query'); - // Add sorting. - $this->phrase_query->sort('search_api_relevance', 'DESC'); - // Tiebreaker: among equally relevant matches, show the most recently - // modified first (Solr otherwise falls back to internal doc order, i.e. - // indexation order, which puts the oldest materiaux first). - $this->phrase_query->sort('changed', 'DESC'); - - // Set one or more tags for the query. - // @see hook_search_api_query_TAG_alter() - // @see hook_search_api_results_TAG_alter() - $this->phrase_query->addTag('materio_sapi_search_phrase_query'); - - $phrase_results = $this->phrase_query->execute(); - - foreach ($phrase_results as $result) { - $this->results['uuids'][] = $result->getField('uuid')->getValues()[0]; - $this->results['nids'][] = $result->getField('nid')->getValues()[0]; - } - - - - // ,---. | ,---. - // |---|,---.,---| | |. .,---.,---., . - // | || || | | || ||---'| | | - // ` '` '`---' `---\`---'`---'` `---| - // `---' - $this->and_query = $this->index->query(['offset'=>0,'limit'=>10000]); - // set parse mode and conjunction - $parse_mode = \Drupal::service('plugin.manager.search_api.parse_mode') - ->createInstance('terms'); - $parse_mode->setConjunction('AND'); - $this->and_query->setParseMode($parse_mode); - // Set fulltext search keywords and fields. - if ($this->keys) { - $this->and_query->keys(implode(' ', $this->keys)); - } - - // in case of term id provided restrict the keys to taxo fields + // A term chosen in the autocomplete restricts to the taxonomy fields. if ($this->terms && is_array($this->terms) && count($this->terms)) { $term_conditions = $this->and_query->createConditionGroup('OR'); - // $term = (int) $this->term; foreach ($this->terms as $term) { - $tid = $term->value; foreach (['tag_tid', 'thesaurus_tid'] as $field) { - $term_conditions->addCondition($field, (int) $tid); + $term_conditions->addCondition($field, (int) $term->value); } } $this->and_query->addConditionGroup($term_conditions); - - // INSTEAD TRY TO BOOST THE TAG AND THESAURUS FIELDS - // foreach ($taxoSolrFieldsName as $fname) { - // // $solarium_query->addParam('bf', "recip(abs(ms(NOW,{$solrField})),3.16e-11,10,0.1)"); - // $bfparam = "if(gt(termfreq({$fname},{$this->term}),0),^21,0)"; - // $this->or_query->addParam('bf', $bfparam); - // } - // look @ materio_sapi_search_api_solr_query_alter in materio_sapi.module - // $this->or_query->setOption('termid', $this->term); } - - if ($this->filters) { - // FILTERS - $filters_conditions = $this->and_query->createConditionGroup('AND'); - foreach ($this->filters as $filter) { - $filter = (int) $filter; - foreach (['thesaurus_tid'] as $field) { // 'tag_tid', - $filters_conditions->addCondition($field, $filter); - } - } - $this->and_query->addConditionGroup($filters_conditions); - if(!$this->keys) { - // if no keys but filters switch query to direct and add wildcard solr keys *:* - $direct_and_parse_mode = \Drupal::service('plugin.manager.search_api.parse_mode') - ->createInstance('direct'); - $direct_and_parse_mode->setConjunction('AND'); - $this->and_query->setParseMode($direct_and_parse_mode); - // $this->and_query->keys('*:*'); - } - } - // else{ - $fulltextFields = []; - // Recherche uniquement sur les champ thésaurus et tag - $fulltextFields += [ + // Thesaurus and tag fields only. + $this->and_query->setFulltextFields([ 'thesaurus_name_0', 'thesaurus_synonyms_0', 'thesaurus_name_1', @@ -226,144 +160,68 @@ class Base extends ControllerBase { 'thesaurus_name', 'thesaurus_synonyms', 'tag_name', - 'tag_synonyms' - ]; - if(count($fulltextFields)){ - $this->and_query->setFulltextFields($fulltextFields); - } - // } + 'tag_synonyms', + ]); + return $this->collect($this->and_query); + } - // Restrict the search to specific languages. - $this->and_query->setLanguages([$lang]); + /** + * Any word, in every fulltext field; references like W0117 matched exactly. + * + * @return array + * [nid, uuid] keyed by nid, in result order. + */ + private function orQuery(string $keys, string $lang): array { + $this->or_query = $this->keywordQuery('direct', 'OR', $keys, $lang, 'materio_sapi_search_or_query'); - // Do paging. - // $this->and_query->range($this->offset, $this->limit); - // retrieve all results - // $this->and_query->range(0, -1); - - // Add sorting. - $this->and_query->sort('search_api_relevance', 'DESC'); - // Tiebreaker: among equally relevant matches, show the most recently - // modified first (Solr otherwise falls back to internal doc order, i.e. - // indexation order, which puts the oldest materiaux first). - $this->and_query->sort('changed', 'DESC'); - - // Set one or more tags for the query. - // @see hook_search_api_query_TAG_alter() - // @see hook_search_api_results_TAG_alter() - $this->and_query->addTag('materio_sapi_search_and_query'); - - $and_results = $this->and_query->execute(); - - foreach ($and_results as $result) { - // !! have to remove duplicates from phrase query - $nid = $result->getField('nid')->getValues()[0]; - if ( !in_array($nid, $this->results['nids']) ) { - $this->results['uuids'][] = $result->getField('uuid')->getValues()[0]; - $this->results['nids'][] = $result->getField('nid')->getValues()[0]; - } - } - - - $this->exactematch_count = count($this->results['nids']); - - - - // ,---. ,---. - // | |,---. | |. .,---.,---., . - // | || | || ||---'| | | - // `---'` `---\`---'`---'` `---| - // `---' - $this->or_query = $this->index->query(['offset'=>0,'limit'=>10000]); - - // Change the parse mode for the search. - // Les différentes possibilités sont - // - « direct » => Requête directe - // - « terms » => Multiple words - // - « phrase » => Single phrase - // - " edismax " => ??? - $or_parse_mode = \Drupal::service('plugin.manager.search_api.parse_mode') - ->createInstance('direct'); - $or_parse_mode->setConjunction('OR'); - $this->or_query->setParseMode($or_parse_mode); - - // Set fulltext search keywords and fields. - if ($this->keys) { - $this->or_query->keys(implode(' ', $this->keys)); - } - - // // exclude results from previous and_query - // !! trigering solr "too many boolean clauses" error - // $exclude_and_results_conditions = $this->or_query->createConditionGroup('AND'); - // foreach ($this->results['nids'] as $nid) { - // $exclude_and_results_conditions->addCondition('nid', $nid, '<>'); - // } - // $this->or_query->addConditionGroup($exclude_and_results_conditions); - - if (preg_match_all('/[WTRPCMFGSO]\d{4}/i', implode(' ', $this->keys), $matches)) { - // in case we search for material references like W0117 + if (preg_match_all('/[WTRPCMFGSO]\d{4}/i', $keys, $matches)) { $ref_conditions = $this->or_query->createConditionGroup('OR'); - foreach ($matches[0] as $key => $value) { + foreach ($matches[0] as $value) { $ref_conditions->addCondition('field_reference', $value); } $this->or_query->addConditionGroup($ref_conditions); } + return $this->collect($this->or_query); + } - if ($this->filters) { - // FILTERS - $or_filters_conditions = $this->or_query->createConditionGroup('OR'); - foreach ($this->filters as $filter) { - $filter = (int) $filter; - foreach (['thesaurus_tid'] as $field) { // 'tag_tid', - $or_filters_conditions->addCondition($field, $filter); - } - } - $this->or_query->addConditionGroup($or_filters_conditions); - - if(!$this->keys) { - // if no keys but filters switch query to direct and add wildcard solr keys *:* - $direct_or_parse_mode = \Drupal::service('plugin.manager.search_api.parse_mode') - ->createInstance('direct'); - $direct_or_parse_mode->setConjunction('OR'); - $this->or_query->setParseMode($direct_or_parse_mode); - // $this->or_query->keys('*:*'); - } - + /** + * A keyword query on the database index, not executed yet. + * + * @return \Drupal\search_api\Query\QueryInterface + * The query. + */ + private function keywordQuery(string $parse_mode_id, string $conjunction, string $keys, string $lang, string $tag) { + $query = $this->index->query(['offset' => 0, 'limit' => 10000]); + $parse_mode = \Drupal::service('plugin.manager.search_api.parse_mode')->createInstance($parse_mode_id); + $parse_mode->setConjunction($conjunction); + $query->setParseMode($parse_mode); + if ($keys !== '') { + $query->keys($keys); } - - // Restrict the search to specific languages. - $this->or_query->setLanguages([$lang]); - - // Do paging. - // $this->or_query->range($this->offset, $this->limit); - // retrieve all results - // $this->or_query->range(0, -1); - - // Add sorting. - $this->or_query->sort('search_api_relevance', 'DESC'); + $query->setLanguages([$lang]); + $query->sort('search_api_relevance', 'DESC'); // Tiebreaker: among equally relevant matches, show the most recently // modified first (Solr otherwise falls back to internal doc order, i.e. // indexation order, which puts the oldest materiaux first). - $this->or_query->sort('changed', 'DESC'); - - // Set one or more tags for the query. + $query->sort('changed', 'DESC'); // @see hook_search_api_query_TAG_alter() - // @see hook_search_api_results_TAG_alter() - $this->or_query->addTag('materio_sapi_search_or_query'); + $query->addTag($tag); + return $query; + } - $or_results = $this->or_query->execute(); - - foreach ($or_results as $result) { + /** + * Executes a query and lists its nodes once each, in result order. + * + * @return array + * [nid, uuid] keyed by nid. + */ + private function collect($query): array { + $list = []; + foreach ($query->execute() as $result) { $nid = $result->getField('nid')->getValues()[0]; - // !! have to remove duplicates instead of $exclude_and_results_conditions (solr too many boolean clauses) - if ( !in_array($nid, $this->results['nids']) ) { - $this->results['uuids'][] = $result->getField('uuid')->getValues()[0]; - $this->results['nids'][] = $nid; - } + $list[$nid] ??= [$nid, $result->getField('uuid')->getValues()[0]]; } - - // todo you may like / more like this - + return $list; } private function defaultQuery(){ @@ -402,9 +260,7 @@ class Base extends ControllerBase { // $this->terms = strlen($t) ? explode(',', $t) : null; $this->terms = $t && strlen($t) ? json_decode($t) : null; - // Filters of the assisted search are gone (H1, lot F): the parameter of - // old URLs is ignored. - $this->filters = NULL; + // The filters parameter of the assisted search (old URLs) is ignored. // $this->allparams = $request->query->all(); // $request->attributes->get('_raw_variables')->get('filters') // diff --git a/web/modules/custom/materio_sapi/src/KeywordText.php b/web/modules/custom/materio_sapi/src/KeywordText.php new file mode 100644 index 00000000..992f910c --- /dev/null +++ b/web/modules/custom/materio_sapi/src/KeywordText.php @@ -0,0 +1,87 @@ + ['cherche', 'recherche', 'cherchons', 'recherchons', 'besoin', 'possible', 'svp', 'merci'], + // The English list of search_api_solr is short (no « i », « am », « my »…). + 'en' => ['i', 'am', "i'm", 'we', 'my', 'me', 'looking', 'look', 'searching', 'search', 'need', 'want', 'possible', 'something', 'please', 'thanks'], + ]; + + /** + * Leading elisions of French, removed before the stop word check. + */ + private const ELISION = '/^(?:[cdjlmnst]|qu)[\'’]/u'; + + /** + * Stop words by language, loaded once. + * + * @var array> + */ + private static array $stopWords = []; + + /** + * Returns the words of a phrase that carry meaning, in order. + * + * @return string[] + * Lowercased words; empty when the phrase has only stop words. + */ + public static function words(string $phrase, string $langcode): array { + $skip = self::stopWords($langcode); + $words = []; + foreach (preg_split('/\s+/u', mb_strtolower($phrase), -1, PREG_SPLIT_NO_EMPTY) as $token) { + $token = trim($token, ",;:!?.()[]{}\"«»'’"); + $word = preg_replace(self::ELISION, '', $token) ?? $token; + if ($word === '' || isset($skip[$word]) || isset($skip[$token])) { + continue; + } + $words[] = $word; + } + return $words; + } + + /** + * Stop words and request words of a language. + * + * @return array + * Words as keys. + */ + private static function stopWords(string $langcode): array { + if (!isset(self::$stopWords[$langcode])) { + $list = self::REQUEST_WORDS[$langcode] ?? []; + $storage = \Drupal::entityTypeManager()->getStorage('solr_field_type'); + foreach (["text_{$langcode}_7_0_0", "text_{$langcode}_6_0_0"] as $id) { + /** @var \Drupal\search_api_solr\SolrFieldTypeInterface|null $type */ + if ($type = $storage->load($id)) { + $text = $type->getTextFiles()['stopwords'] ?? ''; + foreach (preg_split('/\R/u', $text) as $line) { + $line = trim(preg_replace('/#.*/u', '', $line) ?? ''); + if ($line !== '') { + $list[] = mb_strtolower($line); + } + } + break; + } + } + self::$stopWords[$langcode] = array_fill_keys($list, TRUE); + } + return self::$stopWords[$langcode]; + } + +}