Compare commits
1
Commits
8229a80c57
...
797a5714f1
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
797a5714f1 |
@@ -8,6 +8,7 @@ use Symfony\Component\HttpFoundation\JsonResponse;
|
||||
use Symfony\Component\HttpFoundation\Request;
|
||||
use Drupal\Component\Utility\Unicode;
|
||||
use Drupal\materio_sapi\KeywordText;
|
||||
use Drupal\materio_sapi\RankFusion;
|
||||
use Drupal\search_api\Entity\Index;
|
||||
|
||||
// https://drupal.stackexchange.com/questions/225008/programatically-use-search-api
|
||||
@@ -88,6 +89,18 @@ class Base extends ControllerBase {
|
||||
*/
|
||||
public const REFERENCE_PATTERN = '/\\b[WTRPCMFGSO]\\d{4}\\b/iu';
|
||||
|
||||
/**
|
||||
* Most neighbours asked to Solr (kNN topK), H5. Tail cut in H6.
|
||||
*/
|
||||
public const KNN_LIMIT = 200;
|
||||
|
||||
/**
|
||||
* Number of neighbours found, for the response tuning of H6.
|
||||
*
|
||||
* @var int
|
||||
*/
|
||||
protected $knn_count = 0;
|
||||
|
||||
/**
|
||||
* The reference query, run before the others.
|
||||
*
|
||||
@@ -134,14 +147,68 @@ class Base extends ControllerBase {
|
||||
$exact += $this->andQuery($keys, $lang);
|
||||
}
|
||||
$this->exactematch_count = count($exact);
|
||||
$all = $exact + $this->orQuery($keys, $lang);
|
||||
$or = $this->orQuery($keys, $lang);
|
||||
|
||||
// Exact matches first, then the OR query and the neighbours merged (H5).
|
||||
$neighbours = $this->neighbours($lang);
|
||||
$this->knn_count = count($neighbours);
|
||||
$nids = RankFusion::fuse(array_keys($exact), [array_keys($or), array_keys($neighbours)]);
|
||||
|
||||
$uuids = array_column($exact + $or, 1, 0) + $this->uuids(array_diff(array_keys($neighbours), array_keys($exact + $or)));
|
||||
$this->results = [
|
||||
'uuids' => array_column($all, 1),
|
||||
'nids' => array_column($all, 0),
|
||||
'uuids' => array_map(static fn ($nid) => $uuids[$nid], $nids),
|
||||
'nids' => $nids,
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Nodes closest in meaning to the phrase (materio_ai, Solr kNN).
|
||||
*
|
||||
* Not for a term chosen in the autocomplete (direct search) nor without
|
||||
* the materio_ai service. On a model or Solr error, the keyword results
|
||||
* stand alone (delay and cache in H8).
|
||||
*
|
||||
* @return array<int, float>
|
||||
* Scores keyed by nid, best first.
|
||||
*/
|
||||
private function neighbours(string $lang): array {
|
||||
if ($this->terms || !\Drupal::hasService('materio_ai.neighbour_search')) {
|
||||
return [];
|
||||
}
|
||||
// References are keywords: they mean nothing to the model.
|
||||
$phrase = trim(preg_replace(self::REFERENCE_PATTERN, ' ', $this->phrase) ?? '');
|
||||
if ($phrase === '') {
|
||||
return [];
|
||||
}
|
||||
try {
|
||||
return \Drupal::service('materio_ai.neighbour_search')->search($phrase, $lang, self::KNN_LIMIT);
|
||||
}
|
||||
catch (\Throwable $e) {
|
||||
\Drupal::logger('materio_sapi')->warning('Neighbour search failed, keyword results only: @message', ['@message' => $e->getMessage()]);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* UUIDs of nodes found only by the neighbour search.
|
||||
*
|
||||
* @param int[] $nids
|
||||
* Node IDs.
|
||||
*
|
||||
* @return array<int, string>
|
||||
* UUIDs keyed by nid.
|
||||
*/
|
||||
private function uuids(array $nids): array {
|
||||
if (!$nids) {
|
||||
return [];
|
||||
}
|
||||
return \Drupal::database()->select('node', 'n')
|
||||
->fields('n', ['nid', 'uuid'])
|
||||
->condition('nid', array_values($nids), 'IN')
|
||||
->execute()
|
||||
->fetchAllKeyed();
|
||||
}
|
||||
|
||||
/**
|
||||
* Nodes whose reference is one of the given ones.
|
||||
*
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Drupal\materio_sapi;
|
||||
|
||||
/**
|
||||
* Merges ranked result lists: exact matches first, the rest by RRF (H5).
|
||||
*
|
||||
* Reciprocal rank fusion: a node scores the sum of 1 / (k + rank) over the
|
||||
* lists that hold it (rank from 1). Only ranks count, not the scores of each
|
||||
* engine, which are not comparable (BM25 against cosine). Equal scores keep
|
||||
* the order of first appearance, list by list: the keyword lists are already
|
||||
* sorted by relevance then by date (fallback sort kept).
|
||||
*/
|
||||
final class RankFusion {
|
||||
|
||||
/**
|
||||
* Usual constant of RRF: dampens the weight of the first ranks.
|
||||
*/
|
||||
public const K = 60;
|
||||
|
||||
/**
|
||||
* Returns the merged order.
|
||||
*
|
||||
* @param array<int|string> $exact
|
||||
* Node IDs of the exact matches (references, phrase, all words), in order.
|
||||
* Always first, whatever the other lists say.
|
||||
* @param array<array<int|string>> $rankings
|
||||
* Ranked lists of node IDs (keyword OR query, kNN neighbours…).
|
||||
* @param int $k
|
||||
* RRF constant.
|
||||
*
|
||||
* @return array<int|string>
|
||||
* Node IDs, each once.
|
||||
*/
|
||||
public static function fuse(array $exact, array $rankings, int $k = self::K): array {
|
||||
$merged = [];
|
||||
foreach ($exact as $nid) {
|
||||
$merged[$nid] ??= $nid;
|
||||
}
|
||||
$scores = [];
|
||||
$first = [];
|
||||
$position = 0;
|
||||
foreach ($rankings as $ranking) {
|
||||
$rank = 0;
|
||||
foreach ($ranking as $nid) {
|
||||
$rank++;
|
||||
if (isset($merged[$nid])) {
|
||||
continue;
|
||||
}
|
||||
if (!isset($scores[$nid])) {
|
||||
$scores[$nid] = 0.0;
|
||||
$first[$nid] = [$position++, $nid];
|
||||
}
|
||||
$scores[$nid] += 1 / ($k + $rank);
|
||||
}
|
||||
}
|
||||
uksort($scores, static fn ($a, $b): int => [$scores[$b], $first[$a][0]] <=> [$scores[$a], $first[$b][0]]);
|
||||
foreach (array_keys($scores) as $nid) {
|
||||
$merged[$nid] = $first[$nid][1];
|
||||
}
|
||||
return array_values($merged);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Drupal\Tests\materio_sapi\Unit;
|
||||
|
||||
use Drupal\materio_sapi\RankFusion;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
|
||||
// No Drupal bootstrap needed: RankFusion is plain PHP.
|
||||
require_once __DIR__ . '/../../../src/RankFusion.php';
|
||||
|
||||
/**
|
||||
* Tests the merge of the keyword and kNN rankings (H5).
|
||||
*
|
||||
* @coversDefaultClass \Drupal\materio_sapi\RankFusion
|
||||
* @group materio_sapi
|
||||
*/
|
||||
final class RankFusionTest extends TestCase {
|
||||
|
||||
/**
|
||||
* Exact matches stay first, in their order, even when ranked low elsewhere.
|
||||
*/
|
||||
public function testExactFirst(): void {
|
||||
$order = RankFusion::fuse([30, 10], [[1, 2, 10], [2, 1, 30]]);
|
||||
$this->assertSame([30, 10], array_slice($order, 0, 2));
|
||||
}
|
||||
|
||||
/**
|
||||
* Each node appears once, exact matches included.
|
||||
*/
|
||||
public function testNoDuplicate(): void {
|
||||
$order = RankFusion::fuse([5], [[5, 1, 2], [2, 5, 3]]);
|
||||
$this->assertSame([5, 2, 1, 3], $order);
|
||||
}
|
||||
|
||||
/**
|
||||
* A node found by both lists beats a node first in one list only.
|
||||
*/
|
||||
public function testBothListsWin(): void {
|
||||
// 2 : 1/62 + 1/61 ; 1 : 1/61 ; 9 : 1/61.
|
||||
$this->assertSame([2, 1, 9], RankFusion::fuse([], [[1, 2], [9, 2]]));
|
||||
}
|
||||
|
||||
/**
|
||||
* Equal scores keep the order of first appearance (keyword list first).
|
||||
*/
|
||||
public function testTiesKeepFirstAppearance(): void {
|
||||
$this->assertSame([7, 8, 3, 4], RankFusion::fuse([], [[7, 3], [8, 4]]));
|
||||
}
|
||||
|
||||
/**
|
||||
* Empty lists: the exact matches alone, or nothing.
|
||||
*/
|
||||
public function testEmpty(): void {
|
||||
$this->assertSame([4], RankFusion::fuse([4], [[], []]));
|
||||
$this->assertSame([], RankFusion::fuse([], []));
|
||||
}
|
||||
|
||||
/**
|
||||
* Numeric strings and integers are the same node, the first type is kept.
|
||||
*/
|
||||
public function testIdTypes(): void {
|
||||
$this->assertSame([12, 13], RankFusion::fuse([], [[12, 13], ['12']]));
|
||||
$this->assertSame(['8'], RankFusion::fuse(['8'], [[8]]));
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user