materio_sapi: split sapiQuery, stop words removed, short-input exact queries (H2)

phraseQuery, andQuery and orQuery return [nid, uuid] lists built by
keywordQuery (duplicates removed by key, no more in_array over 10 000 rows);
filters code removed. KeywordText strips the stop words of the Solr field
type of the language, French elisions and request words: the custom types of
materio_sapi keep stop words (« de » alone matched 2 236 nodes). The phrase
and AND queries run for 6 meaningful words at most: beyond they find nothing
and the AND query could exceed maxClauseCount.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ShDspHaZGwHoRZCsPBJzsM
This commit is contained in:
2026-11-17 11:53:13 +01:00
co-authored by Claude Opus 5.5
parent 7db02c8b30
commit ae69b5c03b
2 changed files with 190 additions and 247 deletions
@@ -7,6 +7,7 @@ use Drupal\Core\Controller\ControllerBase;
use Symfony\Component\HttpFoundation\JsonResponse;
use Symfony\Component\HttpFoundation\Request;
use Drupal\Component\Utility\Unicode;
use Drupal\materio_sapi\KeywordText;
use Drupal\search_api\Entity\Index;
// https://drupal.stackexchange.com/questions/225008/programatically-use-search-api
@@ -52,11 +53,11 @@ class Base extends ControllerBase {
protected $phrase_query, $and_query, $or_query, $query, $count_query, $count, $exactematch_count;
/**
* Filter and term values.
* Term values (autocomplete choice) and items.
*
* @var mixed
*/
protected $filters, $terms, $items;
protected $terms, $items;
private $limit = 15;
private $offset = 0;
@@ -75,144 +76,77 @@ class Base extends ControllerBase {
*/
protected $phrase = '';
/**
* Most meaningful words for which the phrase and AND queries run (H2, a).
*
* Beyond, they find nothing (no node holds every word of a long phrase) and
* the AND query may exceed maxClauseCount: only the OR query runs.
*/
public const SHORT_QUERY_WORDS = 6;
/**
* Runs the keyword queries and merges their results, exact matches first.
*
* The keys are the phrase without stop words (H2, b): the custom types of
* materio_sapi keep them, « de » alone matched 2 236 nodes.
*/
private function sapiQuery(){
// https://www.drupal.org/docs/8/modules/search-api/developer-documentation/executing-a-search-in-code
// https://www.hashbangcode.com/article/drupal-8-date-search-boosting-search-api-and-solr-search
// https://kgaut.net/blog/2018/drupal-8-search-api-effectuer-une-requete-dans-le-code.html
// Otherwise, if you use Solr, you can also set the parse mode to “Direct query”
// and use something like 'tm_name:alex AND tm_surname:john' as the keywords (field names may vary).
// That will also ensure scoring works for the keywords (the other way just adds filters, which don’t affect scoring).
// https://www.drupal.org/project/search_api/issues/3049097
$lang = \Drupal::languageManager()->getCurrentLanguage()->getId();
$this->index = Index::load('database');
$words = KeywordText::words($this->phrase, $lang);
// Only stop words: search the phrase as typed, as before.
$keys = $words ? implode(' ', $words) : implode(' ', $this->keys);
$exact = [];
if (count($words) <= self::SHORT_QUERY_WORDS) {
// Exact material names (like "wood-skin"), then all words.
$exact = $this->phraseQuery($keys, $lang);
$exact += $this->andQuery($keys, $lang);
}
$this->exactematch_count = count($exact);
$all = $exact + $this->orQuery($keys, $lang);
$this->results = [
'uuids' => [],
'nids' => []
'uuids' => array_column($all, 1),
'nids' => array_column($all, 0),
];
}
// ,---.| ,---.
// |---'|---.,---.,---.,---.,---. | |. .,---.,---., .
// | | || ,---|`---.|---' | || ||---'| | |
// ` ` '` `---^`---'`---' `---\`---'`---'` `---|
// `---'
// (to match exact materials names (like "wood-skin"))
$this->phrase_query = $this->index->query(['offset'=>0,'limit'=>10000]);
// set parse mode and conjunction
$parse_mode = \Drupal::service('plugin.manager.search_api.parse_mode')
->createInstance('phrase');
$parse_mode->setConjunction('AND');
$this->phrase_query->setParseMode($parse_mode);
// Set fulltext search keywords and fields.
if ($this->keys) {
$this->phrase_query->keys(implode(' ', $this->keys));
}
/**
* The whole keys in the title.
*
* @return array<int, array{0: mixed, 1: string}>
* [nid, uuid] keyed by nid, in result order.
*/
private function phraseQuery(string $keys, string $lang): array {
$this->phrase_query = $this->keywordQuery('phrase', 'AND', $keys, $lang, 'materio_sapi_search_phrase_query');
$this->phrase_query->setFulltextFields(['title']);
return $this->collect($this->phrase_query);
}
// Restrict the search to specific languages.
$this->phrase_query->setLanguages([$lang]);
/**
* Every word in the thesaurus and tag fields, or the chosen terms.
*
* @return array<int, array{0: mixed, 1: string}>
* [nid, uuid] keyed by nid, in result order.
*/
private function andQuery(string $keys, string $lang): array {
$this->and_query = $this->keywordQuery('terms', 'AND', $keys, $lang, 'materio_sapi_search_and_query');
// Do paging.
// $this->and_query->range($this->offset, $this->limit);
// retrieve all results
// $this->and_query->range(0, -1);
// Add sorting.
$this->phrase_query->sort('search_api_relevance', 'DESC');
// Tiebreaker: among equally relevant matches, show the most recently
// modified first (Solr otherwise falls back to internal doc order, i.e.
// indexation order, which puts the oldest materiaux first).
$this->phrase_query->sort('changed', 'DESC');
// Set one or more tags for the query.
// @see hook_search_api_query_TAG_alter()
// @see hook_search_api_results_TAG_alter()
$this->phrase_query->addTag('materio_sapi_search_phrase_query');
$phrase_results = $this->phrase_query->execute();
foreach ($phrase_results as $result) {
$this->results['uuids'][] = $result->getField('uuid')->getValues()[0];
$this->results['nids'][] = $result->getField('nid')->getValues()[0];
}
// ,---. | ,---.
// |---|,---.,---| | |. .,---.,---., .
// | || || | | || ||---'| | |
// ` '` '`---' `---\`---'`---'` `---|
// `---'
$this->and_query = $this->index->query(['offset'=>0,'limit'=>10000]);
// set parse mode and conjunction
$parse_mode = \Drupal::service('plugin.manager.search_api.parse_mode')
->createInstance('terms');
$parse_mode->setConjunction('AND');
$this->and_query->setParseMode($parse_mode);
// Set fulltext search keywords and fields.
if ($this->keys) {
$this->and_query->keys(implode(' ', $this->keys));
}
// in case of term id provided restrict the keys to taxo fields
// A term chosen in the autocomplete restricts to the taxonomy fields.
if ($this->terms && is_array($this->terms) && count($this->terms)) {
$term_conditions = $this->and_query->createConditionGroup('OR');
// $term = (int) $this->term;
foreach ($this->terms as $term) {
$tid = $term->value;
foreach (['tag_tid', 'thesaurus_tid'] as $field) {
$term_conditions->addCondition($field, (int) $tid);
$term_conditions->addCondition($field, (int) $term->value);
}
}
$this->and_query->addConditionGroup($term_conditions);
// INSTEAD TRY TO BOOST THE TAG AND THESAURUS FIELDS
// foreach ($taxoSolrFieldsName as $fname) {
// // $solarium_query->addParam('bf', "recip(abs(ms(NOW,{$solrField})),3.16e-11,10,0.1)");
// $bfparam = "if(gt(termfreq({$fname},{$this->term}),0),^21,0)";
// $this->or_query->addParam('bf', $bfparam);
// }
// look @ materio_sapi_search_api_solr_query_alter in materio_sapi.module
// $this->or_query->setOption('termid', $this->term);
}
if ($this->filters) {
// FILTERS
$filters_conditions = $this->and_query->createConditionGroup('AND');
foreach ($this->filters as $filter) {
$filter = (int) $filter;
foreach (['thesaurus_tid'] as $field) { // 'tag_tid',
$filters_conditions->addCondition($field, $filter);
}
}
$this->and_query->addConditionGroup($filters_conditions);
if(!$this->keys) {
// if no keys but filters switch query to direct and add wildcard solr keys *:*
$direct_and_parse_mode = \Drupal::service('plugin.manager.search_api.parse_mode')
->createInstance('direct');
$direct_and_parse_mode->setConjunction('AND');
$this->and_query->setParseMode($direct_and_parse_mode);
// $this->and_query->keys('*:*');
}
}
// else{
$fulltextFields = [];
// Recherche uniquement sur les champ thésaurus et tag
$fulltextFields += [
// Thesaurus and tag fields only.
$this->and_query->setFulltextFields([
'thesaurus_name_0',
'thesaurus_synonyms_0',
'thesaurus_name_1',
@@ -226,144 +160,68 @@ class Base extends ControllerBase {
'thesaurus_name',
'thesaurus_synonyms',
'tag_name',
'tag_synonyms'
];
if(count($fulltextFields)){
$this->and_query->setFulltextFields($fulltextFields);
}
// }
'tag_synonyms',
]);
return $this->collect($this->and_query);
}
// Restrict the search to specific languages.
$this->and_query->setLanguages([$lang]);
/**
* Any word, in every fulltext field; references like W0117 matched exactly.
*
* @return array<int, array{0: mixed, 1: string}>
* [nid, uuid] keyed by nid, in result order.
*/
private function orQuery(string $keys, string $lang): array {
$this->or_query = $this->keywordQuery('direct', 'OR', $keys, $lang, 'materio_sapi_search_or_query');
// Do paging.
// $this->and_query->range($this->offset, $this->limit);
// retrieve all results
// $this->and_query->range(0, -1);
// Add sorting.
$this->and_query->sort('search_api_relevance', 'DESC');
// Tiebreaker: among equally relevant matches, show the most recently
// modified first (Solr otherwise falls back to internal doc order, i.e.
// indexation order, which puts the oldest materiaux first).
$this->and_query->sort('changed', 'DESC');
// Set one or more tags for the query.
// @see hook_search_api_query_TAG_alter()
// @see hook_search_api_results_TAG_alter()
$this->and_query->addTag('materio_sapi_search_and_query');
$and_results = $this->and_query->execute();
foreach ($and_results as $result) {
// !! have to remove duplicates from phrase query
$nid = $result->getField('nid')->getValues()[0];
if ( !in_array($nid, $this->results['nids']) ) {
$this->results['uuids'][] = $result->getField('uuid')->getValues()[0];
$this->results['nids'][] = $result->getField('nid')->getValues()[0];
}
}
$this->exactematch_count = count($this->results['nids']);
// ,---. ,---.
// | |,---. | |. .,---.,---., .
// | || | || ||---'| | |
// `---'` `---\`---'`---'` `---|
// `---'
$this->or_query = $this->index->query(['offset'=>0,'limit'=>10000]);
// Change the parse mode for the search.
// Les différentes possibilités sont
// - « direct » => Requête directe
// - « terms » => Multiple words
// - « phrase » => Single phrase
// - " edismax " => ???
$or_parse_mode = \Drupal::service('plugin.manager.search_api.parse_mode')
->createInstance('direct');
$or_parse_mode->setConjunction('OR');
$this->or_query->setParseMode($or_parse_mode);
// Set fulltext search keywords and fields.
if ($this->keys) {
$this->or_query->keys(implode(' ', $this->keys));
}
// // exclude results from previous and_query
// !! trigering solr "too many boolean clauses" error
// $exclude_and_results_conditions = $this->or_query->createConditionGroup('AND');
// foreach ($this->results['nids'] as $nid) {
// $exclude_and_results_conditions->addCondition('nid', $nid, '<>');
// }
// $this->or_query->addConditionGroup($exclude_and_results_conditions);
if (preg_match_all('/[WTRPCMFGSO]\d{4}/i', implode(' ', $this->keys), $matches)) {
// in case we search for material references like W0117
if (preg_match_all('/[WTRPCMFGSO]\d{4}/i', $keys, $matches)) {
$ref_conditions = $this->or_query->createConditionGroup('OR');
foreach ($matches[0] as $key => $value) {
foreach ($matches[0] as $value) {
$ref_conditions->addCondition('field_reference', $value);
}
$this->or_query->addConditionGroup($ref_conditions);
}
return $this->collect($this->or_query);
}
if ($this->filters) {
// FILTERS
$or_filters_conditions = $this->or_query->createConditionGroup('OR');
foreach ($this->filters as $filter) {
$filter = (int) $filter;
foreach (['thesaurus_tid'] as $field) { // 'tag_tid',
$or_filters_conditions->addCondition($field, $filter);
}
}
$this->or_query->addConditionGroup($or_filters_conditions);
if(!$this->keys) {
// if no keys but filters switch query to direct and add wildcard solr keys *:*
$direct_or_parse_mode = \Drupal::service('plugin.manager.search_api.parse_mode')
->createInstance('direct');
$direct_or_parse_mode->setConjunction('OR');
$this->or_query->setParseMode($direct_or_parse_mode);
// $this->or_query->keys('*:*');
}
/**
* A keyword query on the database index, not executed yet.
*
* @return \Drupal\search_api\Query\QueryInterface
* The query.
*/
private function keywordQuery(string $parse_mode_id, string $conjunction, string $keys, string $lang, string $tag) {
$query = $this->index->query(['offset' => 0, 'limit' => 10000]);
$parse_mode = \Drupal::service('plugin.manager.search_api.parse_mode')->createInstance($parse_mode_id);
$parse_mode->setConjunction($conjunction);
$query->setParseMode($parse_mode);
if ($keys !== '') {
$query->keys($keys);
}
// Restrict the search to specific languages.
$this->or_query->setLanguages([$lang]);
// Do paging.
// $this->or_query->range($this->offset, $this->limit);
// retrieve all results
// $this->or_query->range(0, -1);
// Add sorting.
$this->or_query->sort('search_api_relevance', 'DESC');
$query->setLanguages([$lang]);
$query->sort('search_api_relevance', 'DESC');
// Tiebreaker: among equally relevant matches, show the most recently
// modified first (Solr otherwise falls back to internal doc order, i.e.
// indexation order, which puts the oldest materiaux first).
$this->or_query->sort('changed', 'DESC');
// Set one or more tags for the query.
$query->sort('changed', 'DESC');
// @see hook_search_api_query_TAG_alter()
// @see hook_search_api_results_TAG_alter()
$this->or_query->addTag('materio_sapi_search_or_query');
$query->addTag($tag);
return $query;
}
$or_results = $this->or_query->execute();
foreach ($or_results as $result) {
/**
* Executes a query and lists its nodes once each, in result order.
*
* @return array<int, array{0: mixed, 1: string}>
* [nid, uuid] keyed by nid.
*/
private function collect($query): array {
$list = [];
foreach ($query->execute() as $result) {
$nid = $result->getField('nid')->getValues()[0];
// !! have to remove duplicates instead of $exclude_and_results_conditions (solr too many boolean clauses)
if ( !in_array($nid, $this->results['nids']) ) {
$this->results['uuids'][] = $result->getField('uuid')->getValues()[0];
$this->results['nids'][] = $nid;
}
$list[$nid] ??= [$nid, $result->getField('uuid')->getValues()[0]];
}
// todo you may like / more like this
return $list;
}
private function defaultQuery(){
@@ -402,9 +260,7 @@ class Base extends ControllerBase {
// $this->terms = strlen($t) ? explode(',', $t) : null;
$this->terms = $t && strlen($t) ? json_decode($t) : null;
// Filters of the assisted search are gone (H1, lot F): the parameter of
// old URLs is ignored.
$this->filters = NULL;
// The filters parameter of the assisted search (old URLs) is ignored.
// $this->allparams = $request->query->all();
// $request->attributes->get('_raw_variables')->get('filters')
//
@@ -0,0 +1,87 @@
<?php
declare(strict_types=1);
namespace Drupal\materio_sapi;
/**
* Keyword text of a phrase: stop words and request formulas removed (H2).
*
* The stop words are those of the Solr field type of the language
* (search_api_solr text_LANG_*, the source of stopwords_LANG.txt in the
* configset): the keyword queries also search the custom types of
* materio_sapi (tags, thesaurus, countries, n-grams), which keep them. The
* whole phrase still goes to the model.
*/
final class KeywordText {
/**
* Words that only ask, and say nothing of the material.
*/
private const REQUEST_WORDS = [
'fr' => ['cherche', 'recherche', 'cherchons', 'recherchons', 'besoin', 'possible', 'svp', 'merci'],
// The English list of search_api_solr is short (no « i », « am », « my »…).
'en' => ['i', 'am', "i'm", 'we', 'my', 'me', 'looking', 'look', 'searching', 'search', 'need', 'want', 'possible', 'something', 'please', 'thanks'],
];
/**
* Leading elisions of French, removed before the stop word check.
*/
private const ELISION = '/^(?:[cdjlmnst]|qu)[\'’]/u';
/**
* Stop words by language, loaded once.
*
* @var array<string, array<string, true>>
*/
private static array $stopWords = [];
/**
* Returns the words of a phrase that carry meaning, in order.
*
* @return string[]
* Lowercased words; empty when the phrase has only stop words.
*/
public static function words(string $phrase, string $langcode): array {
$skip = self::stopWords($langcode);
$words = [];
foreach (preg_split('/\s+/u', mb_strtolower($phrase), -1, PREG_SPLIT_NO_EMPTY) as $token) {
$token = trim($token, ",;:!?.()[]{}\"«»'’");
$word = preg_replace(self::ELISION, '', $token) ?? $token;
if ($word === '' || isset($skip[$word]) || isset($skip[$token])) {
continue;
}
$words[] = $word;
}
return $words;
}
/**
* Stop words and request words of a language.
*
* @return array<string, true>
* Words as keys.
*/
private static function stopWords(string $langcode): array {
if (!isset(self::$stopWords[$langcode])) {
$list = self::REQUEST_WORDS[$langcode] ?? [];
$storage = \Drupal::entityTypeManager()->getStorage('solr_field_type');
foreach (["text_{$langcode}_7_0_0", "text_{$langcode}_6_0_0"] as $id) {
/** @var \Drupal\search_api_solr\SolrFieldTypeInterface|null $type */
if ($type = $storage->load($id)) {
$text = $type->getTextFiles()['stopwords'] ?? '';
foreach (preg_split('/\R/u', $text) as $line) {
$line = trim(preg_replace('/#.*/u', '', $line) ?? '');
if ($line !== '') {
$list[] = mb_strtolower($line);
}
}
break;
}
}
self::$stopWords[$langcode] = array_fill_keys($list, TRUE);
}
return self::$stopWords[$langcode];
}
}