Files
materio-d9/web/modules/custom/materio_sapi/src/Controller/Base.php
T
bachirandClaude Opus 5.5 797a5714f1 materio_sapi: hybrid search, keyword and kNN results merged by RRF (H5)
RankFusion::fuse(): exact matches (references, phrase, all words) first, in
their order; then the OR query and the kNN neighbours merged by reciprocal
rank fusion (k = 60, ranks only). Base::neighbours() asks materio_ai for 200
neighbours of the phrase without references; no model call for a chosen
term, references alone, a missing service, or an error (logged, keyword
results only). Unit test RankFusionTest.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ShDspHaZGwHoRZCsPBJzsM
2026-11-17 12:43:04 +01:00

528 lines
17 KiB
PHP

<?php
// https://www.drupal.org/docs/8/modules/search-api/developer-documentation/executing-a-search-in-code
namespace Drupal\materio_sapi\Controller;
use Drupal\Core\Controller\ControllerBase;
use Symfony\Component\HttpFoundation\JsonResponse;
use Symfony\Component\HttpFoundation\Request;
use Drupal\Component\Utility\Unicode;
use Drupal\materio_sapi\KeywordText;
use Drupal\materio_sapi\RankFusion;
use Drupal\search_api\Entity\Index;
// https://drupal.stackexchange.com/questions/225008/programatically-use-search-api
/**
* Defines a route controller for materio sapi search (regular and ajax).
*/
class Base extends ControllerBase {
/**
* The search index.
*
* @var \Drupal\search_api\IndexInterface|null
*/
protected $index;
/**
* The request parameters.
*
* @var array
*/
protected $allparams;
/**
* The search keys.
*
* @var string
*/
protected $keys;
/**
* The search results.
*
* @var array
*/
protected $results;
/**
* The search queries and counters.
*
* @var mixed
*/
protected $phrase_query, $and_query, $or_query, $query, $count_query, $count, $exactematch_count;
/**
* Term values (autocomplete choice) and items.
*
* @var mixed
*/
protected $terms, $items;
private $limit = 15;
private $offset = 0;
/**
* Longest phrase kept, in characters (H1, 150 decided on 17/11). The longest
* test phrase (S5) has 89 characters; beyond the limit the phrase is cut,
* not refused.
*/
public const MAX_PHRASE_LENGTH = 150;
/**
* The phrase typed by the user, whole and case kept, for the model (H5).
*
* @var string
*/
protected $phrase = '';
/**
* Material references found in the phrase (W0117…), uppercase (H4).
*
* @var string[]
*/
protected $references = [];
/**
* Pattern of a material reference: family letter and four digits.
*/
public const REFERENCE_PATTERN = '/\\b[WTRPCMFGSO]\\d{4}\\b/iu';
/**
* Most neighbours asked to Solr (kNN topK), H5. Tail cut in H6.
*/
public const KNN_LIMIT = 200;
/**
* Number of neighbours found, for the response tuning of H6.
*
* @var int
*/
protected $knn_count = 0;
/**
* The reference query, run before the others.
*
* @var \Drupal\search_api\Query\QueryInterface|null
*/
protected $reference_query;
/**
* Most meaningful words for which the phrase and AND queries run (H2, a).
*
* Beyond, they find nothing (no node holds every word of a long phrase) and
* the AND query may exceed maxClauseCount: only the OR query runs.
*/
public const SHORT_QUERY_WORDS = 6;
/**
* Runs the keyword queries and merges their results, exact matches first.
*
* The keys are the phrase without stop words (H2, b): the custom types of
* materio_sapi keep them, « de » alone matched 2 236 nodes.
*/
private function sapiQuery(){
$lang = \Drupal::languageManager()->getCurrentLanguage()->getId();
$this->index = Index::load('database');
// References first, as exact matches, whatever their case (H4, B2).
preg_match_all(self::REFERENCE_PATTERN, $this->phrase, $matches);
$this->references = array_values(array_unique(array_map('mb_strtoupper', $matches[0])));
$exact = $this->references ? $this->referenceQuery($this->references, $lang) : [];
if ($this->isReferenceSearch()) {
$this->exactematch_count = count($exact);
$this->results = ['uuids' => array_column($exact, 1), 'nids' => array_column($exact, 0)];
return;
}
// Keys without the references (« bois R0208 » searches « bois »).
$words = KeywordText::words(preg_replace(self::REFERENCE_PATTERN, ' ', $this->phrase) ?? $this->phrase, $lang);
// Only stop words: search the phrase as typed, as before.
$keys = $words ? implode(' ', $words) : implode(' ', $this->keys);
if (count($words) <= self::SHORT_QUERY_WORDS) {
// Exact material names (like "wood-skin"), then all words.
$exact += $this->phraseQuery($keys, $lang);
$exact += $this->andQuery($keys, $lang);
}
$this->exactematch_count = count($exact);
$or = $this->orQuery($keys, $lang);
// Exact matches first, then the OR query and the neighbours merged (H5).
$neighbours = $this->neighbours($lang);
$this->knn_count = count($neighbours);
$nids = RankFusion::fuse(array_keys($exact), [array_keys($or), array_keys($neighbours)]);
$uuids = array_column($exact + $or, 1, 0) + $this->uuids(array_diff(array_keys($neighbours), array_keys($exact + $or)));
$this->results = [
'uuids' => array_map(static fn ($nid) => $uuids[$nid], $nids),
'nids' => $nids,
];
}
/**
* Nodes closest in meaning to the phrase (materio_ai, Solr kNN).
*
* Not for a term chosen in the autocomplete (direct search) nor without
* the materio_ai service. On a model or Solr error, the keyword results
* stand alone (delay and cache in H8).
*
* @return array<int, float>
* Scores keyed by nid, best first.
*/
private function neighbours(string $lang): array {
if ($this->terms || !\Drupal::hasService('materio_ai.neighbour_search')) {
return [];
}
// References are keywords: they mean nothing to the model.
$phrase = trim(preg_replace(self::REFERENCE_PATTERN, ' ', $this->phrase) ?? '');
if ($phrase === '') {
return [];
}
try {
return \Drupal::service('materio_ai.neighbour_search')->search($phrase, $lang, self::KNN_LIMIT);
}
catch (\Throwable $e) {
\Drupal::logger('materio_sapi')->warning('Neighbour search failed, keyword results only: @message', ['@message' => $e->getMessage()]);
return [];
}
}
/**
* UUIDs of nodes found only by the neighbour search.
*
* @param int[] $nids
* Node IDs.
*
* @return array<int, string>
* UUIDs keyed by nid.
*/
private function uuids(array $nids): array {
if (!$nids) {
return [];
}
return \Drupal::database()->select('node', 'n')
->fields('n', ['nid', 'uuid'])
->condition('nid', array_values($nids), 'IN')
->execute()
->fetchAllKeyed();
}
/**
* Nodes whose reference is one of the given ones.
*
* field_reference is case-sensitive: references are uppercased (B2).
*
* @param string[] $references
* Uppercase references.
*
* @return array<int, array{0: mixed, 1: string}>
* [nid, uuid] keyed by nid, in result order.
*/
private function referenceQuery(array $references, string $lang): array {
$this->reference_query = $this->keywordQuery('terms', 'OR', '', $lang, 'materio_sapi_search_reference_query');
$conditions = $this->reference_query->createConditionGroup('OR');
foreach ($references as $reference) {
$conditions->addCondition('field_reference', $reference);
}
$this->reference_query->addConditionGroup($conditions);
return $this->collect($this->reference_query);
}
/**
* Whether the phrase is only references: no need for the model (H4, H5).
*/
public function isReferenceSearch(): bool {
return $this->references && trim(preg_replace(self::REFERENCE_PATTERN, '', $this->phrase) ?? '', " \t,;") === '';
}
/**
* The whole keys in the title.
*
* @return array<int, array{0: mixed, 1: string}>
* [nid, uuid] keyed by nid, in result order.
*/
private function phraseQuery(string $keys, string $lang): array {
$this->phrase_query = $this->keywordQuery('phrase', 'AND', $keys, $lang, 'materio_sapi_search_phrase_query');
$this->phrase_query->setFulltextFields(['title']);
return $this->collect($this->phrase_query);
}
/**
* Every word in the thesaurus and tag fields, or the chosen terms.
*
* @return array<int, array{0: mixed, 1: string}>
* [nid, uuid] keyed by nid, in result order.
*/
private function andQuery(string $keys, string $lang): array {
$this->and_query = $this->keywordQuery('terms', 'AND', $keys, $lang, 'materio_sapi_search_and_query');
// A term chosen in the autocomplete restricts to the taxonomy fields.
if ($this->terms && is_array($this->terms) && count($this->terms)) {
$term_conditions = $this->and_query->createConditionGroup('OR');
foreach ($this->terms as $term) {
foreach (['tag_tid', 'thesaurus_tid'] as $field) {
$term_conditions->addCondition($field, (int) $term->value);
}
}
$this->and_query->addConditionGroup($term_conditions);
}
// Thesaurus and tag fields only.
$this->and_query->setFulltextFields([
'thesaurus_name_0',
'thesaurus_synonyms_0',
'thesaurus_name_1',
'thesaurus_synonyms_1',
'thesaurus_name_2',
'thesaurus_synonyms_2',
'thesaurus_name_3',
'thesaurus_synonyms_3',
'thesaurus_name_4',
'thesaurus_synonyms_4',
'thesaurus_name',
'thesaurus_synonyms',
'tag_name',
'tag_synonyms',
]);
return $this->collect($this->and_query);
}
/**
* Any word, in every fulltext field.
*
* References are no longer a condition here (H4): it restricted the OR
* results to the referenced nodes (« bois R0208 » found nothing).
*
* @return array<int, array{0: mixed, 1: string}>
* [nid, uuid] keyed by nid, in result order.
*/
private function orQuery(string $keys, string $lang): array {
// Terms mode, not direct: Search API escapes the words, so punctuation
// and Solr syntax typed in a phrase (: ( " / \\ ~) are plain text (H3).
// Hyphenated words as separate words, as the direct mode did: in terms mode
// « bio-based » would be a quoted phrase (the phrase query keeps it whole).
$this->or_query = $this->keywordQuery('terms', 'OR', preg_replace('/(?<=\p{L})-(?=\p{L})/u', ' ', $keys), $lang, 'materio_sapi_search_or_query');
return $this->collect($this->or_query);
}
/**
* A keyword query on the database index, not executed yet.
*
* @return \Drupal\search_api\Query\QueryInterface
* The query.
*/
private function keywordQuery(string $parse_mode_id, string $conjunction, string $keys, string $lang, string $tag) {
$query = $this->index->query(['offset' => 0, 'limit' => 10000]);
$parse_mode = \Drupal::service('plugin.manager.search_api.parse_mode')->createInstance($parse_mode_id);
$parse_mode->setConjunction($conjunction);
$query->setParseMode($parse_mode);
if ($keys !== '') {
$query->keys($keys);
}
$query->setLanguages([$lang]);
$query->sort('search_api_relevance', 'DESC');
// Tiebreaker: among equally relevant matches, show the most recently
// modified first (Solr otherwise falls back to internal doc order, i.e.
// indexation order, which puts the oldest materiaux first).
$query->sort('changed', 'DESC');
// @see hook_search_api_query_TAG_alter()
$query->addTag($tag);
return $query;
}
/**
* Executes a query and lists its nodes once each, in result order.
*
* @return array<int, array{0: mixed, 1: string}>
* [nid, uuid] keyed by nid.
*/
private function collect($query): array {
$list = [];
foreach ($query->execute() as $result) {
$nid = $result->getField('nid')->getValues()[0];
$list[$nid] ??= [$nid, $result->getField('uuid')->getValues()[0]];
}
return $list;
}
private function defaultQuery(){
$lang = \Drupal::languageManager()->getCurrentLanguage()->getId();
$entity_storage = \Drupal::entityTypeManager()->getStorage('node');
$this->query = $entity_storage->getQuery()
->condition('type', ['materiau', 'thematique'], 'IN')
->condition('status', '1')
->condition('langcode', $lang)
->range($this->offset, $this->limit)
->accessCheck(TRUE)
->sort('created', 'DESC');
// ->condition('field_example', 'test_value')
$this->results = $this->query->execute();
$this->count_query = $entity_storage->getQuery()
->condition('type', ['materiau', 'thematique'], 'IN')
->condition('langcode', $lang)
->accessCheck(TRUE)
->condition('status', '1')
->count();
$this->count = $this->count_query->execute();
}
/**
* get params from request
*/
private function parseRequest(Request $request){
// One phrase, commas included: no more splitting into tags (H1).
// keys stays an array of one lowercased element for the keyword queries.
$this->phrase = self::cleanPhrase($request->query->get('keys'));
$this->keys = $this->phrase !== '' ? [mb_strtolower($this->phrase)] : NULL;
// get the exacte term id in case of autocomplete
// $this->terms = $request->query->get('terms');
$t = $request->query->get('terms');
// $this->terms = strlen($t) ? explode(',', $t) : null;
$this->terms = $t && strlen($t) ? json_decode($t) : null;
// The filters parameter of the assisted search (old URLs) is ignored.
// $this->allparams = $request->query->all();
// $request->attributes->get('_raw_variables')->get('filters')
//
$this->offset = $request->query->get('offset') ?? $this->offset;
$this->limit = $request->query->get('limit') ?? $this->limit;
}
/**
* Normalizes the typed phrase: spaces collapsed, length limited.
*/
public static function cleanPhrase($keys): string {
if (!is_string($keys)) {
return '';
}
$phrase = trim(preg_replace('/[\s\p{Cc}]+/u', ' ', $keys) ?? '');
return mb_substr($phrase, 0, self::MAX_PHRASE_LENGTH);
}
/**
* Handler for ajax search.
*/
public function getResults(Request $request) {
$this->parseRequest($request);
$resp = [
'range' => array(
'offset' => $this->offset,
'limit' => $this->limit
),
];
if ($this->keys) {
$lang = \Drupal::languageManager()->getCurrentLanguage()->getId();
$this->sapiQuery();
$resp['keys'] = json_encode($this->keys);
$resp['terms'] = json_encode($this->terms);
// $resp['count'] = $this->results->getResultCount();
$resp['count'] = count($this->results['nids']);
$resp['exactematch_count'] = $this->exactematch_count;
$resp['infos'] = t('The search found @exactmatchcount exact match result(s) for @count total result(s) with', array(
"@exactmatchcount" => $resp['exactematch_count'],
"@count" => $resp['count']
));
$resp['infos'] .= t(' keywords @keys', array(
"@keys" => $this->phrase
));
// $resp['options'] = $this->query->getOptions();
// $uuids = [];
// $nids = [];
// foreach ($this->results as $result) {
// $uuids[] = $result->getField('uuid')->getValues()[0];
// $nids[] = $result->getField('nid')->getValues()[0];
// }
$resp['nids'] = array_slice($this->results['nids'], $this->offset, $this->limit);
$resp['uuids'] = array_slice($this->results['uuids'], $this->offset, $this->limit);
} else {
// no keys or terms to search for
// display the default base page
$this->defaultQuery();
// $uuids = [];
$nids = [];
// Using entityTypeManager
// Get a node storage object.
// $node_storage = \Drupal::entityTypeManager()->getStorage('node');
foreach ($this->results as $result) {
// $lang = \Drupal::languageManager()->getCurrentLanguage()->getId();
// Load a single node.
// $node = $node_storage->load($result);
// check if has translation
// i used to filter like bellow because of a graphql probleme
// if ($node->hasTranslation($lang)) {
$nids[] = $result;
// }
}
// $resp['uuids'] = $uuids;
$resp['nids'] = $nids;
$resp['count'] = $this->count;
$resp['infos'] = t('Please use the search form to search from our @count materials.', array(
"@count" => $resp['count']
));
}
return new JsonResponse($resp);
}
/**
* Handler for regular page search.
*/
function pageHandler(Request $request){
// \Drupal::logger('materio_sapi')->notice(print_r($request, true));
$this->parseRequest($request);
$resp = [
"#title" => 'Base'
];
if ($this->keys) {
$resp['#title'] = $this->phrase;
// $this->sapiQuery();
// $node_storage = \Drupal::entityTypeManager()->getStorage('node');
// $node_view_builder = \Drupal::entityTypeManager()->getViewBuilder('node');
// // $items = $this->results->getResultItems();
// $nids = $this->results['nids'];
// $this->items = [];
// foreach ($nids as $nid) {
// // \Drupal::logger('materio_sapi')->notice(print_r($nid, true));
// try {
// /** @var \Drupal\Core\Entity\EntityInterface $entity */
// // $entity = $item->getOriginalObject()->getValue();
// $entity = $node_storage->load($nid);
// }
// catch (SearchApiException $e) {
// continue;
// }
// if (!$entity) {
// continue;
// }
// // TODO: define dynamicly viewmode
// $this->items[] = $node_view_builder->view($entity, 'teaser');
// }
// $resp['items'] = $this->items;
$resp['items'] = [];
}else{
$resp['#markup'] = t("no keys to search for");
}
return $resp;
}
}