materio_sapi: split sapiQuery, stop words removed, short-input exact queries (H2)
phraseQuery, andQuery and orQuery return [nid, uuid] lists built by keywordQuery (duplicates removed by key, no more in_array over 10 000 rows); filters code removed. KeywordText strips the stop words of the Solr field type of the language, French elisions and request words: the custom types of materio_sapi keep stop words (« de » alone matched 2 236 nodes). The phrase and AND queries run for 6 meaningful words at most: beyond they find nothing and the AND query could exceed maxClauseCount. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01ShDspHaZGwHoRZCsPBJzsM
This commit is contained in:
@@ -7,6 +7,7 @@ use Drupal\Core\Controller\ControllerBase;
|
||||
use Symfony\Component\HttpFoundation\JsonResponse;
|
||||
use Symfony\Component\HttpFoundation\Request;
|
||||
use Drupal\Component\Utility\Unicode;
|
||||
use Drupal\materio_sapi\KeywordText;
|
||||
use Drupal\search_api\Entity\Index;
|
||||
|
||||
// https://drupal.stackexchange.com/questions/225008/programatically-use-search-api
|
||||
@@ -52,11 +53,11 @@ class Base extends ControllerBase {
|
||||
protected $phrase_query, $and_query, $or_query, $query, $count_query, $count, $exactematch_count;
|
||||
|
||||
/**
|
||||
* Filter and term values.
|
||||
* Term values (autocomplete choice) and items.
|
||||
*
|
||||
* @var mixed
|
||||
*/
|
||||
protected $filters, $terms, $items;
|
||||
protected $terms, $items;
|
||||
|
||||
private $limit = 15;
|
||||
private $offset = 0;
|
||||
@@ -75,144 +76,77 @@ class Base extends ControllerBase {
|
||||
*/
|
||||
protected $phrase = '';
|
||||
|
||||
/**
|
||||
* Most meaningful words for which the phrase and AND queries run (H2, a).
|
||||
*
|
||||
* Beyond, they find nothing (no node holds every word of a long phrase) and
|
||||
* the AND query may exceed maxClauseCount: only the OR query runs.
|
||||
*/
|
||||
public const SHORT_QUERY_WORDS = 6;
|
||||
|
||||
/**
|
||||
* Runs the keyword queries and merges their results, exact matches first.
|
||||
*
|
||||
* The keys are the phrase without stop words (H2, b): the custom types of
|
||||
* materio_sapi keep them, « de » alone matched 2 236 nodes.
|
||||
*/
|
||||
private function sapiQuery(){
|
||||
// https://www.drupal.org/docs/8/modules/search-api/developer-documentation/executing-a-search-in-code
|
||||
// https://www.hashbangcode.com/article/drupal-8-date-search-boosting-search-api-and-solr-search
|
||||
// https://kgaut.net/blog/2018/drupal-8-search-api-effectuer-une-requete-dans-le-code.html
|
||||
|
||||
|
||||
// Otherwise, if you use Solr, you can also set the parse mode to “Direct query”
|
||||
// and use something like 'tm_name:alex AND tm_surname:john' as the keywords (field names may vary).
|
||||
// That will also ensure scoring works for the keywords (the other way just adds filters, which don’t affect scoring).
|
||||
// https://www.drupal.org/project/search_api/issues/3049097
|
||||
|
||||
|
||||
|
||||
$lang = \Drupal::languageManager()->getCurrentLanguage()->getId();
|
||||
|
||||
$this->index = Index::load('database');
|
||||
|
||||
$words = KeywordText::words($this->phrase, $lang);
|
||||
// Only stop words: search the phrase as typed, as before.
|
||||
$keys = $words ? implode(' ', $words) : implode(' ', $this->keys);
|
||||
|
||||
$exact = [];
|
||||
if (count($words) <= self::SHORT_QUERY_WORDS) {
|
||||
// Exact material names (like "wood-skin"), then all words.
|
||||
$exact = $this->phraseQuery($keys, $lang);
|
||||
$exact += $this->andQuery($keys, $lang);
|
||||
}
|
||||
$this->exactematch_count = count($exact);
|
||||
$all = $exact + $this->orQuery($keys, $lang);
|
||||
|
||||
$this->results = [
|
||||
'uuids' => [],
|
||||
'nids' => []
|
||||
'uuids' => array_column($all, 1),
|
||||
'nids' => array_column($all, 0),
|
||||
];
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
// ,---.| ,---.
|
||||
// |---'|---.,---.,---.,---.,---. | |. .,---.,---., .
|
||||
// | | || ,---|`---.|---' | || ||---'| | |
|
||||
// ` ` '` `---^`---'`---' `---\`---'`---'` `---|
|
||||
// `---'
|
||||
// (to match exact materials names (like "wood-skin"))
|
||||
$this->phrase_query = $this->index->query(['offset'=>0,'limit'=>10000]);
|
||||
// set parse mode and conjunction
|
||||
$parse_mode = \Drupal::service('plugin.manager.search_api.parse_mode')
|
||||
->createInstance('phrase');
|
||||
$parse_mode->setConjunction('AND');
|
||||
$this->phrase_query->setParseMode($parse_mode);
|
||||
// Set fulltext search keywords and fields.
|
||||
if ($this->keys) {
|
||||
$this->phrase_query->keys(implode(' ', $this->keys));
|
||||
}
|
||||
|
||||
/**
|
||||
* The whole keys in the title.
|
||||
*
|
||||
* @return array<int, array{0: mixed, 1: string}>
|
||||
* [nid, uuid] keyed by nid, in result order.
|
||||
*/
|
||||
private function phraseQuery(string $keys, string $lang): array {
|
||||
$this->phrase_query = $this->keywordQuery('phrase', 'AND', $keys, $lang, 'materio_sapi_search_phrase_query');
|
||||
$this->phrase_query->setFulltextFields(['title']);
|
||||
|
||||
// Restrict the search to specific languages.
|
||||
$this->phrase_query->setLanguages([$lang]);
|
||||
|
||||
// Do paging.
|
||||
// $this->and_query->range($this->offset, $this->limit);
|
||||
// retrieve all results
|
||||
// $this->and_query->range(0, -1);
|
||||
|
||||
// Add sorting.
|
||||
$this->phrase_query->sort('search_api_relevance', 'DESC');
|
||||
// Tiebreaker: among equally relevant matches, show the most recently
|
||||
// modified first (Solr otherwise falls back to internal doc order, i.e.
|
||||
// indexation order, which puts the oldest materiaux first).
|
||||
$this->phrase_query->sort('changed', 'DESC');
|
||||
|
||||
// Set one or more tags for the query.
|
||||
// @see hook_search_api_query_TAG_alter()
|
||||
// @see hook_search_api_results_TAG_alter()
|
||||
$this->phrase_query->addTag('materio_sapi_search_phrase_query');
|
||||
|
||||
$phrase_results = $this->phrase_query->execute();
|
||||
|
||||
foreach ($phrase_results as $result) {
|
||||
$this->results['uuids'][] = $result->getField('uuid')->getValues()[0];
|
||||
$this->results['nids'][] = $result->getField('nid')->getValues()[0];
|
||||
return $this->collect($this->phrase_query);
|
||||
}
|
||||
|
||||
/**
|
||||
* Every word in the thesaurus and tag fields, or the chosen terms.
|
||||
*
|
||||
* @return array<int, array{0: mixed, 1: string}>
|
||||
* [nid, uuid] keyed by nid, in result order.
|
||||
*/
|
||||
private function andQuery(string $keys, string $lang): array {
|
||||
$this->and_query = $this->keywordQuery('terms', 'AND', $keys, $lang, 'materio_sapi_search_and_query');
|
||||
|
||||
|
||||
// ,---. | ,---.
|
||||
// |---|,---.,---| | |. .,---.,---., .
|
||||
// | || || | | || ||---'| | |
|
||||
// ` '` '`---' `---\`---'`---'` `---|
|
||||
// `---'
|
||||
$this->and_query = $this->index->query(['offset'=>0,'limit'=>10000]);
|
||||
// set parse mode and conjunction
|
||||
$parse_mode = \Drupal::service('plugin.manager.search_api.parse_mode')
|
||||
->createInstance('terms');
|
||||
$parse_mode->setConjunction('AND');
|
||||
$this->and_query->setParseMode($parse_mode);
|
||||
// Set fulltext search keywords and fields.
|
||||
if ($this->keys) {
|
||||
$this->and_query->keys(implode(' ', $this->keys));
|
||||
}
|
||||
|
||||
// in case of term id provided restrict the keys to taxo fields
|
||||
// A term chosen in the autocomplete restricts to the taxonomy fields.
|
||||
if ($this->terms && is_array($this->terms) && count($this->terms)) {
|
||||
$term_conditions = $this->and_query->createConditionGroup('OR');
|
||||
// $term = (int) $this->term;
|
||||
foreach ($this->terms as $term) {
|
||||
$tid = $term->value;
|
||||
foreach (['tag_tid', 'thesaurus_tid'] as $field) {
|
||||
$term_conditions->addCondition($field, (int) $tid);
|
||||
$term_conditions->addCondition($field, (int) $term->value);
|
||||
}
|
||||
}
|
||||
$this->and_query->addConditionGroup($term_conditions);
|
||||
|
||||
// INSTEAD TRY TO BOOST THE TAG AND THESAURUS FIELDS
|
||||
// foreach ($taxoSolrFieldsName as $fname) {
|
||||
// // $solarium_query->addParam('bf', "recip(abs(ms(NOW,{$solrField})),3.16e-11,10,0.1)");
|
||||
// $bfparam = "if(gt(termfreq({$fname},{$this->term}),0),^21,0)";
|
||||
// $this->or_query->addParam('bf', $bfparam);
|
||||
// }
|
||||
// look @ materio_sapi_search_api_solr_query_alter in materio_sapi.module
|
||||
// $this->or_query->setOption('termid', $this->term);
|
||||
}
|
||||
|
||||
if ($this->filters) {
|
||||
// FILTERS
|
||||
$filters_conditions = $this->and_query->createConditionGroup('AND');
|
||||
foreach ($this->filters as $filter) {
|
||||
$filter = (int) $filter;
|
||||
foreach (['thesaurus_tid'] as $field) { // 'tag_tid',
|
||||
$filters_conditions->addCondition($field, $filter);
|
||||
}
|
||||
}
|
||||
$this->and_query->addConditionGroup($filters_conditions);
|
||||
|
||||
if(!$this->keys) {
|
||||
// if no keys but filters switch query to direct and add wildcard solr keys *:*
|
||||
$direct_and_parse_mode = \Drupal::service('plugin.manager.search_api.parse_mode')
|
||||
->createInstance('direct');
|
||||
$direct_and_parse_mode->setConjunction('AND');
|
||||
$this->and_query->setParseMode($direct_and_parse_mode);
|
||||
// $this->and_query->keys('*:*');
|
||||
}
|
||||
}
|
||||
// else{
|
||||
$fulltextFields = [];
|
||||
// Recherche uniquement sur les champ thésaurus et tag
|
||||
$fulltextFields += [
|
||||
// Thesaurus and tag fields only.
|
||||
$this->and_query->setFulltextFields([
|
||||
'thesaurus_name_0',
|
||||
'thesaurus_synonyms_0',
|
||||
'thesaurus_name_1',
|
||||
@@ -226,144 +160,68 @@ class Base extends ControllerBase {
|
||||
'thesaurus_name',
|
||||
'thesaurus_synonyms',
|
||||
'tag_name',
|
||||
'tag_synonyms'
|
||||
];
|
||||
if(count($fulltextFields)){
|
||||
$this->and_query->setFulltextFields($fulltextFields);
|
||||
}
|
||||
// }
|
||||
|
||||
// Restrict the search to specific languages.
|
||||
$this->and_query->setLanguages([$lang]);
|
||||
|
||||
// Do paging.
|
||||
// $this->and_query->range($this->offset, $this->limit);
|
||||
// retrieve all results
|
||||
// $this->and_query->range(0, -1);
|
||||
|
||||
// Add sorting.
|
||||
$this->and_query->sort('search_api_relevance', 'DESC');
|
||||
// Tiebreaker: among equally relevant matches, show the most recently
|
||||
// modified first (Solr otherwise falls back to internal doc order, i.e.
|
||||
// indexation order, which puts the oldest materiaux first).
|
||||
$this->and_query->sort('changed', 'DESC');
|
||||
|
||||
// Set one or more tags for the query.
|
||||
// @see hook_search_api_query_TAG_alter()
|
||||
// @see hook_search_api_results_TAG_alter()
|
||||
$this->and_query->addTag('materio_sapi_search_and_query');
|
||||
|
||||
$and_results = $this->and_query->execute();
|
||||
|
||||
foreach ($and_results as $result) {
|
||||
// !! have to remove duplicates from phrase query
|
||||
$nid = $result->getField('nid')->getValues()[0];
|
||||
if ( !in_array($nid, $this->results['nids']) ) {
|
||||
$this->results['uuids'][] = $result->getField('uuid')->getValues()[0];
|
||||
$this->results['nids'][] = $result->getField('nid')->getValues()[0];
|
||||
}
|
||||
'tag_synonyms',
|
||||
]);
|
||||
return $this->collect($this->and_query);
|
||||
}
|
||||
|
||||
/**
|
||||
* Any word, in every fulltext field; references like W0117 matched exactly.
|
||||
*
|
||||
* @return array<int, array{0: mixed, 1: string}>
|
||||
* [nid, uuid] keyed by nid, in result order.
|
||||
*/
|
||||
private function orQuery(string $keys, string $lang): array {
|
||||
$this->or_query = $this->keywordQuery('direct', 'OR', $keys, $lang, 'materio_sapi_search_or_query');
|
||||
|
||||
$this->exactematch_count = count($this->results['nids']);
|
||||
|
||||
|
||||
|
||||
// ,---. ,---.
|
||||
// | |,---. | |. .,---.,---., .
|
||||
// | || | || ||---'| | |
|
||||
// `---'` `---\`---'`---'` `---|
|
||||
// `---'
|
||||
$this->or_query = $this->index->query(['offset'=>0,'limit'=>10000]);
|
||||
|
||||
// Change the parse mode for the search.
|
||||
// Les différentes possibilités sont
|
||||
// - « direct » => Requête directe
|
||||
// - « terms » => Multiple words
|
||||
// - « phrase » => Single phrase
|
||||
// - " edismax " => ???
|
||||
$or_parse_mode = \Drupal::service('plugin.manager.search_api.parse_mode')
|
||||
->createInstance('direct');
|
||||
$or_parse_mode->setConjunction('OR');
|
||||
$this->or_query->setParseMode($or_parse_mode);
|
||||
|
||||
// Set fulltext search keywords and fields.
|
||||
if ($this->keys) {
|
||||
$this->or_query->keys(implode(' ', $this->keys));
|
||||
}
|
||||
|
||||
// // exclude results from previous and_query
|
||||
// !! trigering solr "too many boolean clauses" error
|
||||
// $exclude_and_results_conditions = $this->or_query->createConditionGroup('AND');
|
||||
// foreach ($this->results['nids'] as $nid) {
|
||||
// $exclude_and_results_conditions->addCondition('nid', $nid, '<>');
|
||||
// }
|
||||
// $this->or_query->addConditionGroup($exclude_and_results_conditions);
|
||||
|
||||
if (preg_match_all('/[WTRPCMFGSO]\d{4}/i', implode(' ', $this->keys), $matches)) {
|
||||
// in case we search for material references like W0117
|
||||
if (preg_match_all('/[WTRPCMFGSO]\d{4}/i', $keys, $matches)) {
|
||||
$ref_conditions = $this->or_query->createConditionGroup('OR');
|
||||
foreach ($matches[0] as $key => $value) {
|
||||
foreach ($matches[0] as $value) {
|
||||
$ref_conditions->addCondition('field_reference', $value);
|
||||
}
|
||||
$this->or_query->addConditionGroup($ref_conditions);
|
||||
}
|
||||
|
||||
if ($this->filters) {
|
||||
// FILTERS
|
||||
$or_filters_conditions = $this->or_query->createConditionGroup('OR');
|
||||
foreach ($this->filters as $filter) {
|
||||
$filter = (int) $filter;
|
||||
foreach (['thesaurus_tid'] as $field) { // 'tag_tid',
|
||||
$or_filters_conditions->addCondition($field, $filter);
|
||||
}
|
||||
}
|
||||
$this->or_query->addConditionGroup($or_filters_conditions);
|
||||
|
||||
if(!$this->keys) {
|
||||
// if no keys but filters switch query to direct and add wildcard solr keys *:*
|
||||
$direct_or_parse_mode = \Drupal::service('plugin.manager.search_api.parse_mode')
|
||||
->createInstance('direct');
|
||||
$direct_or_parse_mode->setConjunction('OR');
|
||||
$this->or_query->setParseMode($direct_or_parse_mode);
|
||||
// $this->or_query->keys('*:*');
|
||||
return $this->collect($this->or_query);
|
||||
}
|
||||
|
||||
/**
|
||||
* A keyword query on the database index, not executed yet.
|
||||
*
|
||||
* @return \Drupal\search_api\Query\QueryInterface
|
||||
* The query.
|
||||
*/
|
||||
private function keywordQuery(string $parse_mode_id, string $conjunction, string $keys, string $lang, string $tag) {
|
||||
$query = $this->index->query(['offset' => 0, 'limit' => 10000]);
|
||||
$parse_mode = \Drupal::service('plugin.manager.search_api.parse_mode')->createInstance($parse_mode_id);
|
||||
$parse_mode->setConjunction($conjunction);
|
||||
$query->setParseMode($parse_mode);
|
||||
if ($keys !== '') {
|
||||
$query->keys($keys);
|
||||
}
|
||||
|
||||
// Restrict the search to specific languages.
|
||||
$this->or_query->setLanguages([$lang]);
|
||||
|
||||
// Do paging.
|
||||
// $this->or_query->range($this->offset, $this->limit);
|
||||
// retrieve all results
|
||||
// $this->or_query->range(0, -1);
|
||||
|
||||
// Add sorting.
|
||||
$this->or_query->sort('search_api_relevance', 'DESC');
|
||||
$query->setLanguages([$lang]);
|
||||
$query->sort('search_api_relevance', 'DESC');
|
||||
// Tiebreaker: among equally relevant matches, show the most recently
|
||||
// modified first (Solr otherwise falls back to internal doc order, i.e.
|
||||
// indexation order, which puts the oldest materiaux first).
|
||||
$this->or_query->sort('changed', 'DESC');
|
||||
|
||||
// Set one or more tags for the query.
|
||||
$query->sort('changed', 'DESC');
|
||||
// @see hook_search_api_query_TAG_alter()
|
||||
// @see hook_search_api_results_TAG_alter()
|
||||
$this->or_query->addTag('materio_sapi_search_or_query');
|
||||
$query->addTag($tag);
|
||||
return $query;
|
||||
}
|
||||
|
||||
$or_results = $this->or_query->execute();
|
||||
|
||||
foreach ($or_results as $result) {
|
||||
/**
|
||||
* Executes a query and lists its nodes once each, in result order.
|
||||
*
|
||||
* @return array<int, array{0: mixed, 1: string}>
|
||||
* [nid, uuid] keyed by nid.
|
||||
*/
|
||||
private function collect($query): array {
|
||||
$list = [];
|
||||
foreach ($query->execute() as $result) {
|
||||
$nid = $result->getField('nid')->getValues()[0];
|
||||
// !! have to remove duplicates instead of $exclude_and_results_conditions (solr too many boolean clauses)
|
||||
if ( !in_array($nid, $this->results['nids']) ) {
|
||||
$this->results['uuids'][] = $result->getField('uuid')->getValues()[0];
|
||||
$this->results['nids'][] = $nid;
|
||||
$list[$nid] ??= [$nid, $result->getField('uuid')->getValues()[0]];
|
||||
}
|
||||
}
|
||||
|
||||
// todo you may like / more like this
|
||||
|
||||
return $list;
|
||||
}
|
||||
|
||||
private function defaultQuery(){
|
||||
@@ -402,9 +260,7 @@ class Base extends ControllerBase {
|
||||
// $this->terms = strlen($t) ? explode(',', $t) : null;
|
||||
$this->terms = $t && strlen($t) ? json_decode($t) : null;
|
||||
|
||||
// Filters of the assisted search are gone (H1, lot F): the parameter of
|
||||
// old URLs is ignored.
|
||||
$this->filters = NULL;
|
||||
// The filters parameter of the assisted search (old URLs) is ignored.
|
||||
// $this->allparams = $request->query->all();
|
||||
// $request->attributes->get('_raw_variables')->get('filters')
|
||||
//
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Drupal\materio_sapi;
|
||||
|
||||
/**
|
||||
* Keyword text of a phrase: stop words and request formulas removed (H2).
|
||||
*
|
||||
* The stop words are those of the Solr field type of the language
|
||||
* (search_api_solr text_LANG_*, the source of stopwords_LANG.txt in the
|
||||
* configset): the keyword queries also search the custom types of
|
||||
* materio_sapi (tags, thesaurus, countries, n-grams), which keep them. The
|
||||
* whole phrase still goes to the model.
|
||||
*/
|
||||
final class KeywordText {
|
||||
|
||||
/**
|
||||
* Words that only ask, and say nothing of the material.
|
||||
*/
|
||||
private const REQUEST_WORDS = [
|
||||
'fr' => ['cherche', 'recherche', 'cherchons', 'recherchons', 'besoin', 'possible', 'svp', 'merci'],
|
||||
// The English list of search_api_solr is short (no « i », « am », « my »…).
|
||||
'en' => ['i', 'am', "i'm", 'we', 'my', 'me', 'looking', 'look', 'searching', 'search', 'need', 'want', 'possible', 'something', 'please', 'thanks'],
|
||||
];
|
||||
|
||||
/**
|
||||
* Leading elisions of French, removed before the stop word check.
|
||||
*/
|
||||
private const ELISION = '/^(?:[cdjlmnst]|qu)[\'’]/u';
|
||||
|
||||
/**
|
||||
* Stop words by language, loaded once.
|
||||
*
|
||||
* @var array<string, array<string, true>>
|
||||
*/
|
||||
private static array $stopWords = [];
|
||||
|
||||
/**
|
||||
* Returns the words of a phrase that carry meaning, in order.
|
||||
*
|
||||
* @return string[]
|
||||
* Lowercased words; empty when the phrase has only stop words.
|
||||
*/
|
||||
public static function words(string $phrase, string $langcode): array {
|
||||
$skip = self::stopWords($langcode);
|
||||
$words = [];
|
||||
foreach (preg_split('/\s+/u', mb_strtolower($phrase), -1, PREG_SPLIT_NO_EMPTY) as $token) {
|
||||
$token = trim($token, ",;:!?.()[]{}\"«»'’");
|
||||
$word = preg_replace(self::ELISION, '', $token) ?? $token;
|
||||
if ($word === '' || isset($skip[$word]) || isset($skip[$token])) {
|
||||
continue;
|
||||
}
|
||||
$words[] = $word;
|
||||
}
|
||||
return $words;
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop words and request words of a language.
|
||||
*
|
||||
* @return array<string, true>
|
||||
* Words as keys.
|
||||
*/
|
||||
private static function stopWords(string $langcode): array {
|
||||
if (!isset(self::$stopWords[$langcode])) {
|
||||
$list = self::REQUEST_WORDS[$langcode] ?? [];
|
||||
$storage = \Drupal::entityTypeManager()->getStorage('solr_field_type');
|
||||
foreach (["text_{$langcode}_7_0_0", "text_{$langcode}_6_0_0"] as $id) {
|
||||
/** @var \Drupal\search_api_solr\SolrFieldTypeInterface|null $type */
|
||||
if ($type = $storage->load($id)) {
|
||||
$text = $type->getTextFiles()['stopwords'] ?? '';
|
||||
foreach (preg_split('/\R/u', $text) as $line) {
|
||||
$line = trim(preg_replace('/#.*/u', '', $line) ?? '');
|
||||
if ($line !== '') {
|
||||
$list[] = mb_strtolower($line);
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
self::$stopWords[$langcode] = array_fill_keys($list, TRUE);
|
||||
}
|
||||
return self::$stopWords[$langcode];
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user