RankFusion::fuse(): exact matches (references, phrase, all words) first, in their order; then the OR query and the kNN neighbours merged by reciprocal rank fusion (k = 60, ranks only). Base::neighbours() asks materio_ai for 200 neighbours of the phrase without references; no model call for a chosen term, references alone, a missing service, or an error (logged, keyword results only). Unit test RankFusionTest. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01ShDspHaZGwHoRZCsPBJzsM
528 lines
17 KiB
PHP
528 lines
17 KiB
PHP
<?php
|
|
// https://www.drupal.org/docs/8/modules/search-api/developer-documentation/executing-a-search-in-code
|
|
|
|
namespace Drupal\materio_sapi\Controller;
|
|
|
|
use Drupal\Core\Controller\ControllerBase;
|
|
use Symfony\Component\HttpFoundation\JsonResponse;
|
|
use Symfony\Component\HttpFoundation\Request;
|
|
use Drupal\Component\Utility\Unicode;
|
|
use Drupal\materio_sapi\KeywordText;
|
|
use Drupal\materio_sapi\RankFusion;
|
|
use Drupal\search_api\Entity\Index;
|
|
|
|
// https://drupal.stackexchange.com/questions/225008/programatically-use-search-api
|
|
|
|
/**
|
|
* Defines a route controller for materio sapi search (regular and ajax).
|
|
*/
|
|
class Base extends ControllerBase {
|
|
|
|
/**
|
|
* The search index.
|
|
*
|
|
* @var \Drupal\search_api\IndexInterface|null
|
|
*/
|
|
protected $index;
|
|
|
|
/**
|
|
* The request parameters.
|
|
*
|
|
* @var array
|
|
*/
|
|
protected $allparams;
|
|
|
|
/**
|
|
* The search keys.
|
|
*
|
|
* @var string
|
|
*/
|
|
protected $keys;
|
|
|
|
/**
|
|
* The search results.
|
|
*
|
|
* @var array
|
|
*/
|
|
protected $results;
|
|
|
|
/**
|
|
* The search queries and counters.
|
|
*
|
|
* @var mixed
|
|
*/
|
|
protected $phrase_query, $and_query, $or_query, $query, $count_query, $count, $exactematch_count;
|
|
|
|
/**
|
|
* Term values (autocomplete choice) and items.
|
|
*
|
|
* @var mixed
|
|
*/
|
|
protected $terms, $items;
|
|
|
|
private $limit = 15;
|
|
private $offset = 0;
|
|
|
|
/**
|
|
* Longest phrase kept, in characters (H1, 150 decided on 17/11). The longest
|
|
* test phrase (S5) has 89 characters; beyond the limit the phrase is cut,
|
|
* not refused.
|
|
*/
|
|
public const MAX_PHRASE_LENGTH = 150;
|
|
|
|
/**
|
|
* The phrase typed by the user, whole and case kept, for the model (H5).
|
|
*
|
|
* @var string
|
|
*/
|
|
protected $phrase = '';
|
|
|
|
/**
|
|
* Material references found in the phrase (W0117…), uppercase (H4).
|
|
*
|
|
* @var string[]
|
|
*/
|
|
protected $references = [];
|
|
|
|
/**
|
|
* Pattern of a material reference: family letter and four digits.
|
|
*/
|
|
public const REFERENCE_PATTERN = '/\\b[WTRPCMFGSO]\\d{4}\\b/iu';
|
|
|
|
/**
|
|
* Most neighbours asked to Solr (kNN topK), H5. Tail cut in H6.
|
|
*/
|
|
public const KNN_LIMIT = 200;
|
|
|
|
/**
|
|
* Number of neighbours found, for the response tuning of H6.
|
|
*
|
|
* @var int
|
|
*/
|
|
protected $knn_count = 0;
|
|
|
|
/**
|
|
* The reference query, run before the others.
|
|
*
|
|
* @var \Drupal\search_api\Query\QueryInterface|null
|
|
*/
|
|
protected $reference_query;
|
|
|
|
/**
|
|
* Most meaningful words for which the phrase and AND queries run (H2, a).
|
|
*
|
|
* Beyond, they find nothing (no node holds every word of a long phrase) and
|
|
* the AND query may exceed maxClauseCount: only the OR query runs.
|
|
*/
|
|
public const SHORT_QUERY_WORDS = 6;
|
|
|
|
/**
|
|
* Runs the keyword queries and merges their results, exact matches first.
|
|
*
|
|
* The keys are the phrase without stop words (H2, b): the custom types of
|
|
* materio_sapi keep them, « de » alone matched 2 236 nodes.
|
|
*/
|
|
private function sapiQuery(){
|
|
$lang = \Drupal::languageManager()->getCurrentLanguage()->getId();
|
|
$this->index = Index::load('database');
|
|
|
|
// References first, as exact matches, whatever their case (H4, B2).
|
|
preg_match_all(self::REFERENCE_PATTERN, $this->phrase, $matches);
|
|
$this->references = array_values(array_unique(array_map('mb_strtoupper', $matches[0])));
|
|
$exact = $this->references ? $this->referenceQuery($this->references, $lang) : [];
|
|
if ($this->isReferenceSearch()) {
|
|
$this->exactematch_count = count($exact);
|
|
$this->results = ['uuids' => array_column($exact, 1), 'nids' => array_column($exact, 0)];
|
|
return;
|
|
}
|
|
|
|
// Keys without the references (« bois R0208 » searches « bois »).
|
|
$words = KeywordText::words(preg_replace(self::REFERENCE_PATTERN, ' ', $this->phrase) ?? $this->phrase, $lang);
|
|
// Only stop words: search the phrase as typed, as before.
|
|
$keys = $words ? implode(' ', $words) : implode(' ', $this->keys);
|
|
|
|
if (count($words) <= self::SHORT_QUERY_WORDS) {
|
|
// Exact material names (like "wood-skin"), then all words.
|
|
$exact += $this->phraseQuery($keys, $lang);
|
|
$exact += $this->andQuery($keys, $lang);
|
|
}
|
|
$this->exactematch_count = count($exact);
|
|
$or = $this->orQuery($keys, $lang);
|
|
|
|
// Exact matches first, then the OR query and the neighbours merged (H5).
|
|
$neighbours = $this->neighbours($lang);
|
|
$this->knn_count = count($neighbours);
|
|
$nids = RankFusion::fuse(array_keys($exact), [array_keys($or), array_keys($neighbours)]);
|
|
|
|
$uuids = array_column($exact + $or, 1, 0) + $this->uuids(array_diff(array_keys($neighbours), array_keys($exact + $or)));
|
|
$this->results = [
|
|
'uuids' => array_map(static fn ($nid) => $uuids[$nid], $nids),
|
|
'nids' => $nids,
|
|
];
|
|
}
|
|
|
|
/**
|
|
* Nodes closest in meaning to the phrase (materio_ai, Solr kNN).
|
|
*
|
|
* Not for a term chosen in the autocomplete (direct search) nor without
|
|
* the materio_ai service. On a model or Solr error, the keyword results
|
|
* stand alone (delay and cache in H8).
|
|
*
|
|
* @return array<int, float>
|
|
* Scores keyed by nid, best first.
|
|
*/
|
|
private function neighbours(string $lang): array {
|
|
if ($this->terms || !\Drupal::hasService('materio_ai.neighbour_search')) {
|
|
return [];
|
|
}
|
|
// References are keywords: they mean nothing to the model.
|
|
$phrase = trim(preg_replace(self::REFERENCE_PATTERN, ' ', $this->phrase) ?? '');
|
|
if ($phrase === '') {
|
|
return [];
|
|
}
|
|
try {
|
|
return \Drupal::service('materio_ai.neighbour_search')->search($phrase, $lang, self::KNN_LIMIT);
|
|
}
|
|
catch (\Throwable $e) {
|
|
\Drupal::logger('materio_sapi')->warning('Neighbour search failed, keyword results only: @message', ['@message' => $e->getMessage()]);
|
|
return [];
|
|
}
|
|
}
|
|
|
|
/**
|
|
* UUIDs of nodes found only by the neighbour search.
|
|
*
|
|
* @param int[] $nids
|
|
* Node IDs.
|
|
*
|
|
* @return array<int, string>
|
|
* UUIDs keyed by nid.
|
|
*/
|
|
private function uuids(array $nids): array {
|
|
if (!$nids) {
|
|
return [];
|
|
}
|
|
return \Drupal::database()->select('node', 'n')
|
|
->fields('n', ['nid', 'uuid'])
|
|
->condition('nid', array_values($nids), 'IN')
|
|
->execute()
|
|
->fetchAllKeyed();
|
|
}
|
|
|
|
/**
|
|
* Nodes whose reference is one of the given ones.
|
|
*
|
|
* field_reference is case-sensitive: references are uppercased (B2).
|
|
*
|
|
* @param string[] $references
|
|
* Uppercase references.
|
|
*
|
|
* @return array<int, array{0: mixed, 1: string}>
|
|
* [nid, uuid] keyed by nid, in result order.
|
|
*/
|
|
private function referenceQuery(array $references, string $lang): array {
|
|
$this->reference_query = $this->keywordQuery('terms', 'OR', '', $lang, 'materio_sapi_search_reference_query');
|
|
$conditions = $this->reference_query->createConditionGroup('OR');
|
|
foreach ($references as $reference) {
|
|
$conditions->addCondition('field_reference', $reference);
|
|
}
|
|
$this->reference_query->addConditionGroup($conditions);
|
|
return $this->collect($this->reference_query);
|
|
}
|
|
|
|
/**
|
|
* Whether the phrase is only references: no need for the model (H4, H5).
|
|
*/
|
|
public function isReferenceSearch(): bool {
|
|
return $this->references && trim(preg_replace(self::REFERENCE_PATTERN, '', $this->phrase) ?? '', " \t,;") === '';
|
|
}
|
|
|
|
/**
|
|
* The whole keys in the title.
|
|
*
|
|
* @return array<int, array{0: mixed, 1: string}>
|
|
* [nid, uuid] keyed by nid, in result order.
|
|
*/
|
|
private function phraseQuery(string $keys, string $lang): array {
|
|
$this->phrase_query = $this->keywordQuery('phrase', 'AND', $keys, $lang, 'materio_sapi_search_phrase_query');
|
|
$this->phrase_query->setFulltextFields(['title']);
|
|
return $this->collect($this->phrase_query);
|
|
}
|
|
|
|
/**
|
|
* Every word in the thesaurus and tag fields, or the chosen terms.
|
|
*
|
|
* @return array<int, array{0: mixed, 1: string}>
|
|
* [nid, uuid] keyed by nid, in result order.
|
|
*/
|
|
private function andQuery(string $keys, string $lang): array {
|
|
$this->and_query = $this->keywordQuery('terms', 'AND', $keys, $lang, 'materio_sapi_search_and_query');
|
|
|
|
// A term chosen in the autocomplete restricts to the taxonomy fields.
|
|
if ($this->terms && is_array($this->terms) && count($this->terms)) {
|
|
$term_conditions = $this->and_query->createConditionGroup('OR');
|
|
foreach ($this->terms as $term) {
|
|
foreach (['tag_tid', 'thesaurus_tid'] as $field) {
|
|
$term_conditions->addCondition($field, (int) $term->value);
|
|
}
|
|
}
|
|
$this->and_query->addConditionGroup($term_conditions);
|
|
}
|
|
|
|
// Thesaurus and tag fields only.
|
|
$this->and_query->setFulltextFields([
|
|
'thesaurus_name_0',
|
|
'thesaurus_synonyms_0',
|
|
'thesaurus_name_1',
|
|
'thesaurus_synonyms_1',
|
|
'thesaurus_name_2',
|
|
'thesaurus_synonyms_2',
|
|
'thesaurus_name_3',
|
|
'thesaurus_synonyms_3',
|
|
'thesaurus_name_4',
|
|
'thesaurus_synonyms_4',
|
|
'thesaurus_name',
|
|
'thesaurus_synonyms',
|
|
'tag_name',
|
|
'tag_synonyms',
|
|
]);
|
|
return $this->collect($this->and_query);
|
|
}
|
|
|
|
/**
|
|
* Any word, in every fulltext field.
|
|
*
|
|
* References are no longer a condition here (H4): it restricted the OR
|
|
* results to the referenced nodes (« bois R0208 » found nothing).
|
|
*
|
|
* @return array<int, array{0: mixed, 1: string}>
|
|
* [nid, uuid] keyed by nid, in result order.
|
|
*/
|
|
private function orQuery(string $keys, string $lang): array {
|
|
// Terms mode, not direct: Search API escapes the words, so punctuation
|
|
// and Solr syntax typed in a phrase (: ( " / \\ ~) are plain text (H3).
|
|
// Hyphenated words as separate words, as the direct mode did: in terms mode
|
|
// « bio-based » would be a quoted phrase (the phrase query keeps it whole).
|
|
$this->or_query = $this->keywordQuery('terms', 'OR', preg_replace('/(?<=\p{L})-(?=\p{L})/u', ' ', $keys), $lang, 'materio_sapi_search_or_query');
|
|
return $this->collect($this->or_query);
|
|
}
|
|
|
|
/**
|
|
* A keyword query on the database index, not executed yet.
|
|
*
|
|
* @return \Drupal\search_api\Query\QueryInterface
|
|
* The query.
|
|
*/
|
|
private function keywordQuery(string $parse_mode_id, string $conjunction, string $keys, string $lang, string $tag) {
|
|
$query = $this->index->query(['offset' => 0, 'limit' => 10000]);
|
|
$parse_mode = \Drupal::service('plugin.manager.search_api.parse_mode')->createInstance($parse_mode_id);
|
|
$parse_mode->setConjunction($conjunction);
|
|
$query->setParseMode($parse_mode);
|
|
if ($keys !== '') {
|
|
$query->keys($keys);
|
|
}
|
|
$query->setLanguages([$lang]);
|
|
$query->sort('search_api_relevance', 'DESC');
|
|
// Tiebreaker: among equally relevant matches, show the most recently
|
|
// modified first (Solr otherwise falls back to internal doc order, i.e.
|
|
// indexation order, which puts the oldest materiaux first).
|
|
$query->sort('changed', 'DESC');
|
|
// @see hook_search_api_query_TAG_alter()
|
|
$query->addTag($tag);
|
|
return $query;
|
|
}
|
|
|
|
/**
|
|
* Executes a query and lists its nodes once each, in result order.
|
|
*
|
|
* @return array<int, array{0: mixed, 1: string}>
|
|
* [nid, uuid] keyed by nid.
|
|
*/
|
|
private function collect($query): array {
|
|
$list = [];
|
|
foreach ($query->execute() as $result) {
|
|
$nid = $result->getField('nid')->getValues()[0];
|
|
$list[$nid] ??= [$nid, $result->getField('uuid')->getValues()[0]];
|
|
}
|
|
return $list;
|
|
}
|
|
|
|
private function defaultQuery(){
|
|
$lang = \Drupal::languageManager()->getCurrentLanguage()->getId();
|
|
$entity_storage = \Drupal::entityTypeManager()->getStorage('node');
|
|
$this->query = $entity_storage->getQuery()
|
|
->condition('type', ['materiau', 'thematique'], 'IN')
|
|
->condition('status', '1')
|
|
->condition('langcode', $lang)
|
|
->range($this->offset, $this->limit)
|
|
->accessCheck(TRUE)
|
|
->sort('created', 'DESC');
|
|
// ->condition('field_example', 'test_value')
|
|
$this->results = $this->query->execute();
|
|
|
|
$this->count_query = $entity_storage->getQuery()
|
|
->condition('type', ['materiau', 'thematique'], 'IN')
|
|
->condition('langcode', $lang)
|
|
->accessCheck(TRUE)
|
|
->condition('status', '1')
|
|
->count();
|
|
$this->count = $this->count_query->execute();
|
|
}
|
|
|
|
/**
|
|
* get params from request
|
|
*/
|
|
private function parseRequest(Request $request){
|
|
// One phrase, commas included: no more splitting into tags (H1).
|
|
// keys stays an array of one lowercased element for the keyword queries.
|
|
$this->phrase = self::cleanPhrase($request->query->get('keys'));
|
|
$this->keys = $this->phrase !== '' ? [mb_strtolower($this->phrase)] : NULL;
|
|
// get the exacte term id in case of autocomplete
|
|
// $this->terms = $request->query->get('terms');
|
|
$t = $request->query->get('terms');
|
|
// $this->terms = strlen($t) ? explode(',', $t) : null;
|
|
$this->terms = $t && strlen($t) ? json_decode($t) : null;
|
|
|
|
// The filters parameter of the assisted search (old URLs) is ignored.
|
|
// $this->allparams = $request->query->all();
|
|
// $request->attributes->get('_raw_variables')->get('filters')
|
|
//
|
|
$this->offset = $request->query->get('offset') ?? $this->offset;
|
|
$this->limit = $request->query->get('limit') ?? $this->limit;
|
|
}
|
|
|
|
/**
|
|
* Normalizes the typed phrase: spaces collapsed, length limited.
|
|
*/
|
|
public static function cleanPhrase($keys): string {
|
|
if (!is_string($keys)) {
|
|
return '';
|
|
}
|
|
$phrase = trim(preg_replace('/[\s\p{Cc}]+/u', ' ', $keys) ?? '');
|
|
return mb_substr($phrase, 0, self::MAX_PHRASE_LENGTH);
|
|
}
|
|
|
|
/**
|
|
* Handler for ajax search.
|
|
*/
|
|
public function getResults(Request $request) {
|
|
$this->parseRequest($request);
|
|
|
|
$resp = [
|
|
'range' => array(
|
|
'offset' => $this->offset,
|
|
'limit' => $this->limit
|
|
),
|
|
];
|
|
|
|
if ($this->keys) {
|
|
$lang = \Drupal::languageManager()->getCurrentLanguage()->getId();
|
|
|
|
$this->sapiQuery();
|
|
|
|
$resp['keys'] = json_encode($this->keys);
|
|
$resp['terms'] = json_encode($this->terms);
|
|
// $resp['count'] = $this->results->getResultCount();
|
|
$resp['count'] = count($this->results['nids']);
|
|
$resp['exactematch_count'] = $this->exactematch_count;
|
|
$resp['infos'] = t('The search found @exactmatchcount exact match result(s) for @count total result(s) with', array(
|
|
"@exactmatchcount" => $resp['exactematch_count'],
|
|
"@count" => $resp['count']
|
|
));
|
|
$resp['infos'] .= t(' keywords @keys', array(
|
|
"@keys" => $this->phrase
|
|
));
|
|
// $resp['options'] = $this->query->getOptions();
|
|
|
|
// $uuids = [];
|
|
// $nids = [];
|
|
// foreach ($this->results as $result) {
|
|
// $uuids[] = $result->getField('uuid')->getValues()[0];
|
|
// $nids[] = $result->getField('nid')->getValues()[0];
|
|
// }
|
|
$resp['nids'] = array_slice($this->results['nids'], $this->offset, $this->limit);
|
|
$resp['uuids'] = array_slice($this->results['uuids'], $this->offset, $this->limit);
|
|
} else {
|
|
// no keys or terms to search for
|
|
// display the default base page
|
|
$this->defaultQuery();
|
|
// $uuids = [];
|
|
$nids = [];
|
|
// Using entityTypeManager
|
|
// Get a node storage object.
|
|
// $node_storage = \Drupal::entityTypeManager()->getStorage('node');
|
|
|
|
foreach ($this->results as $result) {
|
|
// $lang = \Drupal::languageManager()->getCurrentLanguage()->getId();
|
|
// Load a single node.
|
|
// $node = $node_storage->load($result);
|
|
// check if has translation
|
|
// i used to filter like bellow because of a graphql probleme
|
|
// if ($node->hasTranslation($lang)) {
|
|
$nids[] = $result;
|
|
// }
|
|
|
|
}
|
|
// $resp['uuids'] = $uuids;
|
|
$resp['nids'] = $nids;
|
|
$resp['count'] = $this->count;
|
|
$resp['infos'] = t('Please use the search form to search from our @count materials.', array(
|
|
"@count" => $resp['count']
|
|
));
|
|
}
|
|
|
|
return new JsonResponse($resp);
|
|
}
|
|
|
|
|
|
/**
|
|
* Handler for regular page search.
|
|
*/
|
|
function pageHandler(Request $request){
|
|
// \Drupal::logger('materio_sapi')->notice(print_r($request, true));
|
|
|
|
$this->parseRequest($request);
|
|
|
|
$resp = [
|
|
"#title" => 'Base'
|
|
];
|
|
|
|
if ($this->keys) {
|
|
$resp['#title'] = $this->phrase;
|
|
|
|
// $this->sapiQuery();
|
|
|
|
// $node_storage = \Drupal::entityTypeManager()->getStorage('node');
|
|
// $node_view_builder = \Drupal::entityTypeManager()->getViewBuilder('node');
|
|
|
|
// // $items = $this->results->getResultItems();
|
|
// $nids = $this->results['nids'];
|
|
// $this->items = [];
|
|
// foreach ($nids as $nid) {
|
|
// // \Drupal::logger('materio_sapi')->notice(print_r($nid, true));
|
|
// try {
|
|
// /** @var \Drupal\Core\Entity\EntityInterface $entity */
|
|
// // $entity = $item->getOriginalObject()->getValue();
|
|
// $entity = $node_storage->load($nid);
|
|
// }
|
|
// catch (SearchApiException $e) {
|
|
// continue;
|
|
// }
|
|
// if (!$entity) {
|
|
// continue;
|
|
// }
|
|
// // TODO: define dynamicly viewmode
|
|
// $this->items[] = $node_view_builder->view($entity, 'teaser');
|
|
|
|
// }
|
|
|
|
// $resp['items'] = $this->items;
|
|
$resp['items'] = [];
|
|
}else{
|
|
$resp['#markup'] = t("no keys to search for");
|
|
}
|
|
|
|
return $resp;
|
|
}
|
|
}
|