materio_sapi: hybrid search, keyword and kNN results merged by RRF (H5)
RankFusion::fuse(): exact matches (references, phrase, all words) first, in their order; then the OR query and the kNN neighbours merged by reciprocal rank fusion (k = 60, ranks only). Base::neighbours() asks materio_ai for 200 neighbours of the phrase without references; no model call for a chosen term, references alone, a missing service, or an error (logged, keyword results only). Unit test RankFusionTest. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01ShDspHaZGwHoRZCsPBJzsM
This commit is contained in:
@@ -8,6 +8,7 @@ use Symfony\Component\HttpFoundation\JsonResponse;
|
|||||||
use Symfony\Component\HttpFoundation\Request;
|
use Symfony\Component\HttpFoundation\Request;
|
||||||
use Drupal\Component\Utility\Unicode;
|
use Drupal\Component\Utility\Unicode;
|
||||||
use Drupal\materio_sapi\KeywordText;
|
use Drupal\materio_sapi\KeywordText;
|
||||||
|
use Drupal\materio_sapi\RankFusion;
|
||||||
use Drupal\search_api\Entity\Index;
|
use Drupal\search_api\Entity\Index;
|
||||||
|
|
||||||
// https://drupal.stackexchange.com/questions/225008/programatically-use-search-api
|
// https://drupal.stackexchange.com/questions/225008/programatically-use-search-api
|
||||||
@@ -88,6 +89,18 @@ class Base extends ControllerBase {
|
|||||||
*/
|
*/
|
||||||
public const REFERENCE_PATTERN = '/\\b[WTRPCMFGSO]\\d{4}\\b/iu';
|
public const REFERENCE_PATTERN = '/\\b[WTRPCMFGSO]\\d{4}\\b/iu';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Most neighbours asked to Solr (kNN topK), H5. Tail cut in H6.
|
||||||
|
*/
|
||||||
|
public const KNN_LIMIT = 200;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Number of neighbours found, for the response tuning of H6.
|
||||||
|
*
|
||||||
|
* @var int
|
||||||
|
*/
|
||||||
|
protected $knn_count = 0;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The reference query, run before the others.
|
* The reference query, run before the others.
|
||||||
*
|
*
|
||||||
@@ -134,14 +147,68 @@ class Base extends ControllerBase {
|
|||||||
$exact += $this->andQuery($keys, $lang);
|
$exact += $this->andQuery($keys, $lang);
|
||||||
}
|
}
|
||||||
$this->exactematch_count = count($exact);
|
$this->exactematch_count = count($exact);
|
||||||
$all = $exact + $this->orQuery($keys, $lang);
|
$or = $this->orQuery($keys, $lang);
|
||||||
|
|
||||||
|
// Exact matches first, then the OR query and the neighbours merged (H5).
|
||||||
|
$neighbours = $this->neighbours($lang);
|
||||||
|
$this->knn_count = count($neighbours);
|
||||||
|
$nids = RankFusion::fuse(array_keys($exact), [array_keys($or), array_keys($neighbours)]);
|
||||||
|
|
||||||
|
$uuids = array_column($exact + $or, 1, 0) + $this->uuids(array_diff(array_keys($neighbours), array_keys($exact + $or)));
|
||||||
$this->results = [
|
$this->results = [
|
||||||
'uuids' => array_column($all, 1),
|
'uuids' => array_map(static fn ($nid) => $uuids[$nid], $nids),
|
||||||
'nids' => array_column($all, 0),
|
'nids' => $nids,
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Nodes closest in meaning to the phrase (materio_ai, Solr kNN).
|
||||||
|
*
|
||||||
|
* Not for a term chosen in the autocomplete (direct search) nor without
|
||||||
|
* the materio_ai service. On a model or Solr error, the keyword results
|
||||||
|
* stand alone (delay and cache in H8).
|
||||||
|
*
|
||||||
|
* @return array<int, float>
|
||||||
|
* Scores keyed by nid, best first.
|
||||||
|
*/
|
||||||
|
private function neighbours(string $lang): array {
|
||||||
|
if ($this->terms || !\Drupal::hasService('materio_ai.neighbour_search')) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
// References are keywords: they mean nothing to the model.
|
||||||
|
$phrase = trim(preg_replace(self::REFERENCE_PATTERN, ' ', $this->phrase) ?? '');
|
||||||
|
if ($phrase === '') {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return \Drupal::service('materio_ai.neighbour_search')->search($phrase, $lang, self::KNN_LIMIT);
|
||||||
|
}
|
||||||
|
catch (\Throwable $e) {
|
||||||
|
\Drupal::logger('materio_sapi')->warning('Neighbour search failed, keyword results only: @message', ['@message' => $e->getMessage()]);
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* UUIDs of nodes found only by the neighbour search.
|
||||||
|
*
|
||||||
|
* @param int[] $nids
|
||||||
|
* Node IDs.
|
||||||
|
*
|
||||||
|
* @return array<int, string>
|
||||||
|
* UUIDs keyed by nid.
|
||||||
|
*/
|
||||||
|
private function uuids(array $nids): array {
|
||||||
|
if (!$nids) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
return \Drupal::database()->select('node', 'n')
|
||||||
|
->fields('n', ['nid', 'uuid'])
|
||||||
|
->condition('nid', array_values($nids), 'IN')
|
||||||
|
->execute()
|
||||||
|
->fetchAllKeyed();
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Nodes whose reference is one of the given ones.
|
* Nodes whose reference is one of the given ones.
|
||||||
*
|
*
|
||||||
|
|||||||
@@ -0,0 +1,66 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
namespace Drupal\materio_sapi;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Merges ranked result lists: exact matches first, the rest by RRF (H5).
|
||||||
|
*
|
||||||
|
* Reciprocal rank fusion: a node scores the sum of 1 / (k + rank) over the
|
||||||
|
* lists that hold it (rank from 1). Only ranks count, not the scores of each
|
||||||
|
* engine, which are not comparable (BM25 against cosine). Equal scores keep
|
||||||
|
* the order of first appearance, list by list: the keyword lists are already
|
||||||
|
* sorted by relevance then by date (fallback sort kept).
|
||||||
|
*/
|
||||||
|
final class RankFusion {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Usual constant of RRF: dampens the weight of the first ranks.
|
||||||
|
*/
|
||||||
|
public const K = 60;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the merged order.
|
||||||
|
*
|
||||||
|
* @param array<int|string> $exact
|
||||||
|
* Node IDs of the exact matches (references, phrase, all words), in order.
|
||||||
|
* Always first, whatever the other lists say.
|
||||||
|
* @param array<array<int|string>> $rankings
|
||||||
|
* Ranked lists of node IDs (keyword OR query, kNN neighbours…).
|
||||||
|
* @param int $k
|
||||||
|
* RRF constant.
|
||||||
|
*
|
||||||
|
* @return array<int|string>
|
||||||
|
* Node IDs, each once.
|
||||||
|
*/
|
||||||
|
public static function fuse(array $exact, array $rankings, int $k = self::K): array {
|
||||||
|
$merged = [];
|
||||||
|
foreach ($exact as $nid) {
|
||||||
|
$merged[$nid] ??= $nid;
|
||||||
|
}
|
||||||
|
$scores = [];
|
||||||
|
$first = [];
|
||||||
|
$position = 0;
|
||||||
|
foreach ($rankings as $ranking) {
|
||||||
|
$rank = 0;
|
||||||
|
foreach ($ranking as $nid) {
|
||||||
|
$rank++;
|
||||||
|
if (isset($merged[$nid])) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!isset($scores[$nid])) {
|
||||||
|
$scores[$nid] = 0.0;
|
||||||
|
$first[$nid] = [$position++, $nid];
|
||||||
|
}
|
||||||
|
$scores[$nid] += 1 / ($k + $rank);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
uksort($scores, static fn ($a, $b): int => [$scores[$b], $first[$a][0]] <=> [$scores[$a], $first[$b][0]]);
|
||||||
|
foreach (array_keys($scores) as $nid) {
|
||||||
|
$merged[$nid] = $first[$nid][1];
|
||||||
|
}
|
||||||
|
return array_values($merged);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,68 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
namespace Drupal\Tests\materio_sapi\Unit;
|
||||||
|
|
||||||
|
use Drupal\materio_sapi\RankFusion;
|
||||||
|
use PHPUnit\Framework\TestCase;
|
||||||
|
|
||||||
|
// No Drupal bootstrap needed: RankFusion is plain PHP.
|
||||||
|
require_once __DIR__ . '/../../../src/RankFusion.php';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Tests the merge of the keyword and kNN rankings (H5).
|
||||||
|
*
|
||||||
|
* @coversDefaultClass \Drupal\materio_sapi\RankFusion
|
||||||
|
* @group materio_sapi
|
||||||
|
*/
|
||||||
|
final class RankFusionTest extends TestCase {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Exact matches stay first, in their order, even when ranked low elsewhere.
|
||||||
|
*/
|
||||||
|
public function testExactFirst(): void {
|
||||||
|
$order = RankFusion::fuse([30, 10], [[1, 2, 10], [2, 1, 30]]);
|
||||||
|
$this->assertSame([30, 10], array_slice($order, 0, 2));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Each node appears once, exact matches included.
|
||||||
|
*/
|
||||||
|
public function testNoDuplicate(): void {
|
||||||
|
$order = RankFusion::fuse([5], [[5, 1, 2], [2, 5, 3]]);
|
||||||
|
$this->assertSame([5, 2, 1, 3], $order);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A node found by both lists beats a node first in one list only.
|
||||||
|
*/
|
||||||
|
public function testBothListsWin(): void {
|
||||||
|
// 2 : 1/62 + 1/61 ; 1 : 1/61 ; 9 : 1/61.
|
||||||
|
$this->assertSame([2, 1, 9], RankFusion::fuse([], [[1, 2], [9, 2]]));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Equal scores keep the order of first appearance (keyword list first).
|
||||||
|
*/
|
||||||
|
public function testTiesKeepFirstAppearance(): void {
|
||||||
|
$this->assertSame([7, 8, 3, 4], RankFusion::fuse([], [[7, 3], [8, 4]]));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Empty lists: the exact matches alone, or nothing.
|
||||||
|
*/
|
||||||
|
public function testEmpty(): void {
|
||||||
|
$this->assertSame([4], RankFusion::fuse([4], [[], []]));
|
||||||
|
$this->assertSame([], RankFusion::fuse([], []));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Numeric strings and integers are the same node, the first type is kept.
|
||||||
|
*/
|
||||||
|
public function testIdTypes(): void {
|
||||||
|
$this->assertSame([12, 13], RankFusion::fuse([], [[12, 13], ['12']]));
|
||||||
|
$this->assertSame(['8'], RankFusion::fuse(['8'], [[8]]));
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user