diff --git a/web/modules/custom/materio_sapi/src/Controller/Base.php b/web/modules/custom/materio_sapi/src/Controller/Base.php index 9a55f1b1..151157b7 100644 --- a/web/modules/custom/materio_sapi/src/Controller/Base.php +++ b/web/modules/custom/materio_sapi/src/Controller/Base.php @@ -8,6 +8,7 @@ use Symfony\Component\HttpFoundation\JsonResponse; use Symfony\Component\HttpFoundation\Request; use Drupal\Component\Utility\Unicode; use Drupal\materio_sapi\KeywordText; +use Drupal\materio_sapi\RankFusion; use Drupal\search_api\Entity\Index; // https://drupal.stackexchange.com/questions/225008/programatically-use-search-api @@ -88,6 +89,18 @@ class Base extends ControllerBase { */ public const REFERENCE_PATTERN = '/\\b[WTRPCMFGSO]\\d{4}\\b/iu'; + /** + * Most neighbours asked to Solr (kNN topK), H5. Tail cut in H6. + */ + public const KNN_LIMIT = 200; + + /** + * Number of neighbours found, for the response tuning of H6. + * + * @var int + */ + protected $knn_count = 0; + /** * The reference query, run before the others. * @@ -134,14 +147,68 @@ class Base extends ControllerBase { $exact += $this->andQuery($keys, $lang); } $this->exactematch_count = count($exact); - $all = $exact + $this->orQuery($keys, $lang); + $or = $this->orQuery($keys, $lang); + // Exact matches first, then the OR query and the neighbours merged (H5). + $neighbours = $this->neighbours($lang); + $this->knn_count = count($neighbours); + $nids = RankFusion::fuse(array_keys($exact), [array_keys($or), array_keys($neighbours)]); + + $uuids = array_column($exact + $or, 1, 0) + $this->uuids(array_diff(array_keys($neighbours), array_keys($exact + $or))); $this->results = [ - 'uuids' => array_column($all, 1), - 'nids' => array_column($all, 0), + 'uuids' => array_map(static fn ($nid) => $uuids[$nid], $nids), + 'nids' => $nids, ]; } + /** + * Nodes closest in meaning to the phrase (materio_ai, Solr kNN). + * + * Not for a term chosen in the autocomplete (direct search) nor without + * the materio_ai service. On a model or Solr error, the keyword results + * stand alone (delay and cache in H8). + * + * @return array + * Scores keyed by nid, best first. + */ + private function neighbours(string $lang): array { + if ($this->terms || !\Drupal::hasService('materio_ai.neighbour_search')) { + return []; + } + // References are keywords: they mean nothing to the model. + $phrase = trim(preg_replace(self::REFERENCE_PATTERN, ' ', $this->phrase) ?? ''); + if ($phrase === '') { + return []; + } + try { + return \Drupal::service('materio_ai.neighbour_search')->search($phrase, $lang, self::KNN_LIMIT); + } + catch (\Throwable $e) { + \Drupal::logger('materio_sapi')->warning('Neighbour search failed, keyword results only: @message', ['@message' => $e->getMessage()]); + return []; + } + } + + /** + * UUIDs of nodes found only by the neighbour search. + * + * @param int[] $nids + * Node IDs. + * + * @return array + * UUIDs keyed by nid. + */ + private function uuids(array $nids): array { + if (!$nids) { + return []; + } + return \Drupal::database()->select('node', 'n') + ->fields('n', ['nid', 'uuid']) + ->condition('nid', array_values($nids), 'IN') + ->execute() + ->fetchAllKeyed(); + } + /** * Nodes whose reference is one of the given ones. * diff --git a/web/modules/custom/materio_sapi/src/RankFusion.php b/web/modules/custom/materio_sapi/src/RankFusion.php new file mode 100644 index 00000000..3f311aea --- /dev/null +++ b/web/modules/custom/materio_sapi/src/RankFusion.php @@ -0,0 +1,66 @@ + $exact + * Node IDs of the exact matches (references, phrase, all words), in order. + * Always first, whatever the other lists say. + * @param array> $rankings + * Ranked lists of node IDs (keyword OR query, kNN neighbours…). + * @param int $k + * RRF constant. + * + * @return array + * Node IDs, each once. + */ + public static function fuse(array $exact, array $rankings, int $k = self::K): array { + $merged = []; + foreach ($exact as $nid) { + $merged[$nid] ??= $nid; + } + $scores = []; + $first = []; + $position = 0; + foreach ($rankings as $ranking) { + $rank = 0; + foreach ($ranking as $nid) { + $rank++; + if (isset($merged[$nid])) { + continue; + } + if (!isset($scores[$nid])) { + $scores[$nid] = 0.0; + $first[$nid] = [$position++, $nid]; + } + $scores[$nid] += 1 / ($k + $rank); + } + } + uksort($scores, static fn ($a, $b): int => [$scores[$b], $first[$a][0]] <=> [$scores[$a], $first[$b][0]]); + foreach (array_keys($scores) as $nid) { + $merged[$nid] = $first[$nid][1]; + } + return array_values($merged); + } + +} diff --git a/web/modules/custom/materio_sapi/tests/src/Unit/RankFusionTest.php b/web/modules/custom/materio_sapi/tests/src/Unit/RankFusionTest.php new file mode 100644 index 00000000..7cdf523f --- /dev/null +++ b/web/modules/custom/materio_sapi/tests/src/Unit/RankFusionTest.php @@ -0,0 +1,68 @@ +assertSame([30, 10], array_slice($order, 0, 2)); + } + + /** + * Each node appears once, exact matches included. + */ + public function testNoDuplicate(): void { + $order = RankFusion::fuse([5], [[5, 1, 2], [2, 5, 3]]); + $this->assertSame([5, 2, 1, 3], $order); + } + + /** + * A node found by both lists beats a node first in one list only. + */ + public function testBothListsWin(): void { + // 2 : 1/62 + 1/61 ; 1 : 1/61 ; 9 : 1/61. + $this->assertSame([2, 1, 9], RankFusion::fuse([], [[1, 2], [9, 2]])); + } + + /** + * Equal scores keep the order of first appearance (keyword list first). + */ + public function testTiesKeepFirstAppearance(): void { + $this->assertSame([7, 8, 3, 4], RankFusion::fuse([], [[7, 3], [8, 4]])); + } + + /** + * Empty lists: the exact matches alone, or nothing. + */ + public function testEmpty(): void { + $this->assertSame([4], RankFusion::fuse([4], [[], []])); + $this->assertSame([], RankFusion::fuse([], [])); + } + + /** + * Numeric strings and integers are the same node, the first type is kept. + */ + public function testIdTypes(): void { + $this->assertSame([12, 13], RankFusion::fuse([], [[12, 13], ['12']])); + $this->assertSame(['8'], RankFusion::fuse(['8'], [[8]])); + } + +}