The OR query no longer sends the input raw (direct mode): « fibre de verre \ » broke Solr, « -bois » returned every other node as exact, « verre~ » ran a fuzzy search. KeywordText turns syntax between words into a separator, drops leading + - ! and symbol-only words (« || » broke the payload boost). Hyphens are split for the OR query only, as the direct mode did. Same counts and exact matches on S4, about twice as fast. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01ShDspHaZGwHoRZCsPBJzsM
95 lines
3.4 KiB
PHP
95 lines
3.4 KiB
PHP
<?php
|
||
|
||
declare(strict_types=1);
|
||
|
||
namespace Drupal\materio_sapi;
|
||
|
||
/**
|
||
* Keyword text of a phrase: stop words and request formulas removed (H2).
|
||
*
|
||
* The stop words are those of the Solr field type of the language
|
||
* (search_api_solr text_LANG_*, the source of stopwords_LANG.txt in the
|
||
* configset): the keyword queries also search the custom types of
|
||
* materio_sapi (tags, thesaurus, countries, n-grams), which keep them. The
|
||
* whole phrase still goes to the model.
|
||
*/
|
||
final class KeywordText {
|
||
|
||
/**
|
||
* Words that only ask, and say nothing of the material.
|
||
*/
|
||
private const REQUEST_WORDS = [
|
||
'fr' => ['cherche', 'recherche', 'cherchons', 'recherchons', 'besoin', 'possible', 'svp', 'merci'],
|
||
// The English list of search_api_solr is short (no « i », « am », « my »…).
|
||
'en' => ['i', 'am', "i'm", 'we', 'my', 'me', 'looking', 'look', 'searching', 'search', 'need', 'want', 'possible', 'something', 'please', 'thanks'],
|
||
];
|
||
|
||
/**
|
||
* Leading elisions of French, removed before the stop word check.
|
||
*/
|
||
private const ELISION = '/^(?:[cdjlmnst]|qu)[\'’]/u';
|
||
|
||
/**
|
||
* Stop words by language, loaded once.
|
||
*
|
||
* @var array<string, array<string, true>>
|
||
*/
|
||
private static array $stopWords = [];
|
||
|
||
/**
|
||
* Returns the words of a phrase that carry meaning, in order.
|
||
*
|
||
* @return string[]
|
||
* Lowercased words; empty when the phrase has only stop words.
|
||
*/
|
||
public static function words(string $phrase, string $langcode): array {
|
||
$skip = self::stopWords($langcode);
|
||
$words = [];
|
||
// Solr and Search API syntax between words is a separator (« bois:verre »,
|
||
// « bois/métal »); hyphens, apostrophes, and . , in numbers stay.
|
||
$phrase = preg_replace('#[:;/\\\\^&|()\[\]{}<>"«»~*?!=+\#@$]+#u', ' ', $phrase) ?? $phrase;
|
||
foreach (preg_split('/\s+/u', mb_strtolower($phrase), -1, PREG_SPLIT_NO_EMPTY) as $token) {
|
||
// Edge punctuation, and leading + - ! that Search API reads as operators
|
||
// (« -bois » excluded wood and returned every other node, H3).
|
||
$token = trim($token, ",;:!?.()[]{}\"«»'’<>");
|
||
$token = ltrim($token, '+-!');
|
||
$word = preg_replace(self::ELISION, '', $token) ?? $token;
|
||
// No letter or digit (« || », « & »): breaks the payload boost of Solr.
|
||
if (!preg_match('/[\p{L}\p{N}]/u', $word) || isset($skip[$word]) || isset($skip[$token])) {
|
||
continue;
|
||
}
|
||
$words[] = $word;
|
||
}
|
||
return $words;
|
||
}
|
||
|
||
/**
|
||
* Stop words and request words of a language.
|
||
*
|
||
* @return array<string, true>
|
||
* Words as keys.
|
||
*/
|
||
private static function stopWords(string $langcode): array {
|
||
if (!isset(self::$stopWords[$langcode])) {
|
||
$list = self::REQUEST_WORDS[$langcode] ?? [];
|
||
$storage = \Drupal::entityTypeManager()->getStorage('solr_field_type');
|
||
foreach (["text_{$langcode}_7_0_0", "text_{$langcode}_6_0_0"] as $id) {
|
||
/** @var \Drupal\search_api_solr\SolrFieldTypeInterface|null $type */
|
||
if ($type = $storage->load($id)) {
|
||
$text = $type->getTextFiles()['stopwords'] ?? '';
|
||
foreach (preg_split('/\R/u', $text) as $line) {
|
||
$line = trim(preg_replace('/#.*/u', '', $line) ?? '');
|
||
if ($line !== '') {
|
||
$list[] = mb_strtolower($line);
|
||
}
|
||
}
|
||
break;
|
||
}
|
||
}
|
||
self::$stopWords[$langcode] = array_fill_keys($list, TRUE);
|
||
}
|
||
return self::$stopWords[$langcode];
|
||
}
|
||
|
||
}
|