1 Commits
Author SHA1 Message Date
bachirandClaude Opus 5.5 797a5714f1 materio_sapi: hybrid search, keyword and kNN results merged by RRF (H5)
RankFusion::fuse(): exact matches (references, phrase, all words) first, in
their order; then the OR query and the kNN neighbours merged by reciprocal
rank fusion (k = 60, ranks only). Base::neighbours() asks materio_ai for 200
neighbours of the phrase without references; no model call for a chosen
term, references alone, a missing service, or an error (logged, keyword
results only). Unit test RankFusionTest.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ShDspHaZGwHoRZCsPBJzsM
2026-11-17 12:43:04 +01:00
3 changed files with 204 additions and 3 deletions
@@ -8,6 +8,7 @@ use Symfony\Component\HttpFoundation\JsonResponse;
use Symfony\Component\HttpFoundation\Request; use Symfony\Component\HttpFoundation\Request;
use Drupal\Component\Utility\Unicode; use Drupal\Component\Utility\Unicode;
use Drupal\materio_sapi\KeywordText; use Drupal\materio_sapi\KeywordText;
use Drupal\materio_sapi\RankFusion;
use Drupal\search_api\Entity\Index; use Drupal\search_api\Entity\Index;
// https://drupal.stackexchange.com/questions/225008/programatically-use-search-api // https://drupal.stackexchange.com/questions/225008/programatically-use-search-api
@@ -88,6 +89,18 @@ class Base extends ControllerBase {
*/ */
public const REFERENCE_PATTERN = '/\\b[WTRPCMFGSO]\\d{4}\\b/iu'; public const REFERENCE_PATTERN = '/\\b[WTRPCMFGSO]\\d{4}\\b/iu';
/**
* Most neighbours asked to Solr (kNN topK), H5. Tail cut in H6.
*/
public const KNN_LIMIT = 200;
/**
* Number of neighbours found, for the response tuning of H6.
*
* @var int
*/
protected $knn_count = 0;
/** /**
* The reference query, run before the others. * The reference query, run before the others.
* *
@@ -134,14 +147,68 @@ class Base extends ControllerBase {
$exact += $this->andQuery($keys, $lang); $exact += $this->andQuery($keys, $lang);
} }
$this->exactematch_count = count($exact); $this->exactematch_count = count($exact);
$all = $exact + $this->orQuery($keys, $lang); $or = $this->orQuery($keys, $lang);
// Exact matches first, then the OR query and the neighbours merged (H5).
$neighbours = $this->neighbours($lang);
$this->knn_count = count($neighbours);
$nids = RankFusion::fuse(array_keys($exact), [array_keys($or), array_keys($neighbours)]);
$uuids = array_column($exact + $or, 1, 0) + $this->uuids(array_diff(array_keys($neighbours), array_keys($exact + $or)));
$this->results = [ $this->results = [
'uuids' => array_column($all, 1), 'uuids' => array_map(static fn ($nid) => $uuids[$nid], $nids),
'nids' => array_column($all, 0), 'nids' => $nids,
]; ];
} }
/**
* Nodes closest in meaning to the phrase (materio_ai, Solr kNN).
*
* Not for a term chosen in the autocomplete (direct search) nor without
* the materio_ai service. On a model or Solr error, the keyword results
* stand alone (delay and cache in H8).
*
* @return array<int, float>
* Scores keyed by nid, best first.
*/
private function neighbours(string $lang): array {
if ($this->terms || !\Drupal::hasService('materio_ai.neighbour_search')) {
return [];
}
// References are keywords: they mean nothing to the model.
$phrase = trim(preg_replace(self::REFERENCE_PATTERN, ' ', $this->phrase) ?? '');
if ($phrase === '') {
return [];
}
try {
return \Drupal::service('materio_ai.neighbour_search')->search($phrase, $lang, self::KNN_LIMIT);
}
catch (\Throwable $e) {
\Drupal::logger('materio_sapi')->warning('Neighbour search failed, keyword results only: @message', ['@message' => $e->getMessage()]);
return [];
}
}
/**
* UUIDs of nodes found only by the neighbour search.
*
* @param int[] $nids
* Node IDs.
*
* @return array<int, string>
* UUIDs keyed by nid.
*/
private function uuids(array $nids): array {
if (!$nids) {
return [];
}
return \Drupal::database()->select('node', 'n')
->fields('n', ['nid', 'uuid'])
->condition('nid', array_values($nids), 'IN')
->execute()
->fetchAllKeyed();
}
/** /**
* Nodes whose reference is one of the given ones. * Nodes whose reference is one of the given ones.
* *
@@ -0,0 +1,66 @@
<?php
declare(strict_types=1);
namespace Drupal\materio_sapi;
/**
* Merges ranked result lists: exact matches first, the rest by RRF (H5).
*
* Reciprocal rank fusion: a node scores the sum of 1 / (k + rank) over the
* lists that hold it (rank from 1). Only ranks count, not the scores of each
* engine, which are not comparable (BM25 against cosine). Equal scores keep
* the order of first appearance, list by list: the keyword lists are already
* sorted by relevance then by date (fallback sort kept).
*/
final class RankFusion {
/**
* Usual constant of RRF: dampens the weight of the first ranks.
*/
public const K = 60;
/**
* Returns the merged order.
*
* @param array<int|string> $exact
* Node IDs of the exact matches (references, phrase, all words), in order.
* Always first, whatever the other lists say.
* @param array<array<int|string>> $rankings
* Ranked lists of node IDs (keyword OR query, kNN neighbours…).
* @param int $k
* RRF constant.
*
* @return array<int|string>
* Node IDs, each once.
*/
public static function fuse(array $exact, array $rankings, int $k = self::K): array {
$merged = [];
foreach ($exact as $nid) {
$merged[$nid] ??= $nid;
}
$scores = [];
$first = [];
$position = 0;
foreach ($rankings as $ranking) {
$rank = 0;
foreach ($ranking as $nid) {
$rank++;
if (isset($merged[$nid])) {
continue;
}
if (!isset($scores[$nid])) {
$scores[$nid] = 0.0;
$first[$nid] = [$position++, $nid];
}
$scores[$nid] += 1 / ($k + $rank);
}
}
uksort($scores, static fn ($a, $b): int => [$scores[$b], $first[$a][0]] <=> [$scores[$a], $first[$b][0]]);
foreach (array_keys($scores) as $nid) {
$merged[$nid] = $first[$nid][1];
}
return array_values($merged);
}
}
@@ -0,0 +1,68 @@
<?php
declare(strict_types=1);
namespace Drupal\Tests\materio_sapi\Unit;
use Drupal\materio_sapi\RankFusion;
use PHPUnit\Framework\TestCase;
// No Drupal bootstrap needed: RankFusion is plain PHP.
require_once __DIR__ . '/../../../src/RankFusion.php';
/**
* Tests the merge of the keyword and kNN rankings (H5).
*
* @coversDefaultClass \Drupal\materio_sapi\RankFusion
* @group materio_sapi
*/
final class RankFusionTest extends TestCase {
/**
* Exact matches stay first, in their order, even when ranked low elsewhere.
*/
public function testExactFirst(): void {
$order = RankFusion::fuse([30, 10], [[1, 2, 10], [2, 1, 30]]);
$this->assertSame([30, 10], array_slice($order, 0, 2));
}
/**
* Each node appears once, exact matches included.
*/
public function testNoDuplicate(): void {
$order = RankFusion::fuse([5], [[5, 1, 2], [2, 5, 3]]);
$this->assertSame([5, 2, 1, 3], $order);
}
/**
* A node found by both lists beats a node first in one list only.
*/
public function testBothListsWin(): void {
// 2 : 1/62 + 1/61 ; 1 : 1/61 ; 9 : 1/61.
$this->assertSame([2, 1, 9], RankFusion::fuse([], [[1, 2], [9, 2]]));
}
/**
* Equal scores keep the order of first appearance (keyword list first).
*/
public function testTiesKeepFirstAppearance(): void {
$this->assertSame([7, 8, 3, 4], RankFusion::fuse([], [[7, 3], [8, 4]]));
}
/**
* Empty lists: the exact matches alone, or nothing.
*/
public function testEmpty(): void {
$this->assertSame([4], RankFusion::fuse([4], [[], []]));
$this->assertSame([], RankFusion::fuse([], []));
}
/**
* Numeric strings and integers are the same node, the first type is kept.
*/
public function testIdTypes(): void {
$this->assertSame([12, 13], RankFusion::fuse([], [[12, 13], ['12']]));
$this->assertSame(['8'], RankFusion::fuse(['8'], [[8]]));
}
}