From 7786c2cf713910f312d7157658d1d6be326420cf Mon Sep 17 00:00:00 2001 From: bach Date: Tue, 17 Nov 2026 19:56:53 +0100 Subject: [PATCH] materio_sapi: neighbours cut relative to the search, OR query as fallback (H6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ScoreCut: the kNN search scores every visible node; keep the ones above the mean of this search by more than 2 standard deviations. A fixed threshold meant nothing (at 0.7, « bois » 664 nodes, « wood » 4 671); with k = 2, 373 and 382. Exact matches stay first. - OR query off (OR_WITH_KNN), kept as the fallback when there are no neighbours: model stopped, slow or switched off, term chosen in the autocomplete. - S4: 355 results (1 104), same exact matches; S5: 248 (3 338), better top results, 0.27 s instead of 1.3 s. - Message « X exact result(s) out of N relevant result(s) for "…" », French text added by materio_sapi_update_11001 (translations live in the DB). - ScoreCutTest. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ShDspHaZGwHoRZCsPBJzsM --- .../custom/materio_sapi/materio_sapi.install | 44 ++++++++++++ .../materio_sapi/src/Controller/Base.php | 68 +++++++++++++------ .../custom/materio_sapi/src/ScoreCut.php | 44 ++++++++++++ .../tests/src/Unit/ScoreCutTest.php | 47 +++++++++++++ 4 files changed, 181 insertions(+), 22 deletions(-) create mode 100644 web/modules/custom/materio_sapi/materio_sapi.install create mode 100644 web/modules/custom/materio_sapi/src/ScoreCut.php create mode 100644 web/modules/custom/materio_sapi/tests/src/Unit/ScoreCutTest.php diff --git a/web/modules/custom/materio_sapi/materio_sapi.install b/web/modules/custom/materio_sapi/materio_sapi.install new file mode 100644 index 00000000..1162d7ce --- /dev/null +++ b/web/modules/custom/materio_sapi/materio_sapi.install @@ -0,0 +1,44 @@ + '@exactmatchcount exact result(s) out of @count relevant result(s) for “@keys”', + 'fr' => '@exactmatchcount résultat(s) exact(s) sur @count résultat(s) pertinent(s) pour « @keys »', + ]; +} + +/** + * Adds the French translation of the new search message (H6). + */ +function materio_sapi_update_11001(): string { + if (!\Drupal::moduleHandler()->moduleExists('locale')) { + return 'locale is off: nothing to translate.'; + } + $strings = materio_sapi_search_infos_strings(); + $storage = \Drupal::service('locale.storage'); + $source = $storage->findString(['source' => $strings['source'], 'context' => '']); + if (!$source) { + $source = $storage->createString(['source' => $strings['source'], 'context' => ''])->save(); + } + $storage->createTranslation([ + 'lid' => $source->lid, + 'language' => 'fr', + 'translation' => $strings['fr'], + 'customized' => LOCALE_CUSTOMIZED, + ])->save(); + _locale_refresh_translations(['fr'], [$source->lid]); + return 'French translation of the search message added.'; +} diff --git a/web/modules/custom/materio_sapi/src/Controller/Base.php b/web/modules/custom/materio_sapi/src/Controller/Base.php index 6465b20b..fc376615 100644 --- a/web/modules/custom/materio_sapi/src/Controller/Base.php +++ b/web/modules/custom/materio_sapi/src/Controller/Base.php @@ -10,6 +10,7 @@ use Drupal\Component\Utility\Unicode; use Drupal\materio_ai\Exception\ModelPausedException; use Drupal\materio_sapi\KeywordText; use Drupal\materio_sapi\RankFusion; +use Drupal\materio_sapi\ScoreCut; use Drupal\search_api\Entity\Index; // https://drupal.stackexchange.com/questions/225008/programatically-use-search-api @@ -91,9 +92,27 @@ class Base extends ControllerBase { public const REFERENCE_PATTERN = '/\\b[WTRPCMFGSO]\\d{4}\\b/iu'; /** - * Most neighbours asked to Solr (kNN topK), H5. Tail cut in H6. + * Neighbours asked to Solr (kNN topK): every visible node (8 073 for an + * anonymous visitor in 2026), so that ScoreCut sees the whole distribution + * of the scores (H6). 0.3 to 0.6 s measured locally. */ - public const KNN_LIMIT = 200; + public const KNN_LIMIT = 20000; + + /** + * Standard deviations above the mean kept by ScoreCut (H6, option f, k = 2 + * on trial since 17/11). + */ + public const KNN_SIGMA = 2.0; + + /** + * Whether the OR query is merged with the neighbours (H3, H5). + * + * Off on trial since 17/11 (H6): on a phrase it matched about 40 % of the + * catalogue. Kept: it still runs when there are no neighbours (model + * stopped, slow or switched off, term chosen in the autocomplete), so that + * the search answers as before. + */ + public const OR_WITH_KNN = FALSE; /** * Number of neighbours found, for the response tuning of H6. @@ -178,12 +197,14 @@ class Base extends ControllerBase { $exact += $this->andQuery($keys, $lang); } $this->exactematch_count = count($exact); - $or = $this->orQuery($keys, $lang); - // Exact matches first, then the OR query and the neighbours merged (H5). + // Exact matches first, then the neighbours that stand out (H6), merged + // with the OR query when it is on (H5). No neighbours: the OR query alone. $neighbours = $this->neighbours($lang); + $or = ($neighbours === NULL || self::OR_WITH_KNN) ? $this->orQuery($keys, $lang) : []; + $neighbours = $neighbours === NULL ? [] : ScoreCut::aboveMean($neighbours, self::KNN_SIGMA); $this->knn_count = count($neighbours); - $nids = RankFusion::fuse(array_keys($exact), [array_keys($or), array_keys($neighbours)]); + $nids = RankFusion::fuse(array_keys($exact), array_filter([array_keys($or), array_keys($neighbours)])); $uuids = array_column($exact + $or, 1, 0) + $this->uuids(array_diff(array_keys($neighbours), array_keys($exact + $or))); $this->results = [ @@ -196,31 +217,33 @@ class Base extends ControllerBase { * Nodes closest in meaning to the phrase (materio_ai, Solr kNN). * * Not for a term chosen in the autocomplete (direct search) nor without - * the materio_ai service. On a model or Solr error, or past the delay of - * QueryEmbedder (1 s), the keyword results stand alone, with no message: - * the search answers as before H5. + * the materio_ai service. On a model or Solr error, past the delay of + * QueryEmbedder (1 s), or with materio_ai switched off, the keyword + * results stand alone, with no message: the search answers as before H5. * - * @return array - * Scores keyed by nid, best first. + * @return array|null + * Scores of every visible node keyed by nid, best first; NULL when there + * are no neighbours to use. */ - private function neighbours(string $lang): array { + private function neighbours(string $lang): ?array { if ($this->terms || !\Drupal::hasService('materio_ai.neighbour_search')) { - return []; + return NULL; } // References are keywords: they mean nothing to the model. $phrase = trim(preg_replace(self::REFERENCE_PATTERN, ' ', $this->phrase) ?? ''); if ($phrase === '') { - return []; + return NULL; } try { - return \Drupal::service('materio_ai.neighbour_search')->search($phrase, $lang, self::KNN_LIMIT); + // Empty when materio_ai is switched off. + return \Drupal::service('materio_ai.neighbour_search')->search($phrase, $lang, self::KNN_LIMIT) ?: NULL; } catch (\Throwable $e) { // Paused model: its failure was logged when it happened. if (!$e instanceof ModelPausedException) { \Drupal::logger('materio_sapi')->warning('Neighbour search failed, keyword results only: @message', ['@message' => $e->getMessage()]); } - return []; + return NULL; } } @@ -460,13 +483,14 @@ class Base extends ControllerBase { // $resp['count'] = $this->results->getResultCount(); $resp['count'] = count($this->results['nids']); $resp['exactematch_count'] = $this->exactematch_count; - $resp['infos'] = t('The search found @exactmatchcount exact match result(s) for @count total result(s) with', array( - "@exactmatchcount" => $resp['exactematch_count'], - "@count" => $resp['count'] - )); - $resp['infos'] .= t(' keywords @keys', array( - "@keys" => $this->phrase - )); + // Relevant: exact matches and neighbours that stand out (H6); the + // count no longer depends on an arbitrary limit. French text added by + // materio_sapi_update_11001(). + $resp['infos'] = t('@exactmatchcount exact result(s) out of @count relevant result(s) for “@keys”', [ + '@exactmatchcount' => $resp['exactematch_count'], + '@count' => $resp['count'], + '@keys' => $this->phrase, + ]); // $resp['options'] = $this->query->getOptions(); // $uuids = []; diff --git a/web/modules/custom/materio_sapi/src/ScoreCut.php b/web/modules/custom/materio_sapi/src/ScoreCut.php new file mode 100644 index 00000000..09b11c1b --- /dev/null +++ b/web/modules/custom/materio_sapi/src/ScoreCut.php @@ -0,0 +1,44 @@ + $scores + * Scores keyed by nid, best first, ideally of every visible node. + * @param float $k + * Number of standard deviations. + * + * @return array + * The kept scores, keyed by nid. + */ + public static function aboveMean(array $scores, float $k): array { + $n = count($scores); + if ($n < 2) { + return $scores; + } + $mean = array_sum($scores) / $n; + $variance = 0.0; + foreach ($scores as $score) { + $variance += ($score - $mean) ** 2; + } + $limit = $mean + $k * sqrt($variance / $n); + return array_filter($scores, static fn ($score) => $score > $limit); + } + +} diff --git a/web/modules/custom/materio_sapi/tests/src/Unit/ScoreCutTest.php b/web/modules/custom/materio_sapi/tests/src/Unit/ScoreCutTest.php new file mode 100644 index 00000000..516f4659 --- /dev/null +++ b/web/modules/custom/materio_sapi/tests/src/Unit/ScoreCutTest.php @@ -0,0 +1,47 @@ + 0.9, 3 => 0.8] + array_fill_keys(range(100, 197), 0.6); + $this->assertSame([7 => 0.9, 3 => 0.8], ScoreCut::aboveMean($scores, 2.0)); + $this->assertSame([7 => 0.9], ScoreCut::aboveMean($scores, 7.0)); + } + + /** + * The same distribution shifted keeps the same nodes: no fixed threshold. + */ + public function testShiftedScale(): void { + $scores = [7 => 0.9, 3 => 0.8] + array_fill_keys(range(100, 197), 0.6); + $shifted = array_map(static fn ($s) => $s + 0.15, $scores); + $this->assertSame(array_keys(ScoreCut::aboveMean($scores, 2.0)), array_keys(ScoreCut::aboveMean($shifted, 2.0))); + } + + /** + * Equal scores stand out from nothing; fewer than two scores are kept. + */ + public function testEdgeCases(): void { + $this->assertSame([], ScoreCut::aboveMean([1 => 0.7, 2 => 0.7, 3 => 0.7], 2.0)); + $this->assertSame([5 => 0.8], ScoreCut::aboveMean([5 => 0.8], 2.0)); + $this->assertSame([], ScoreCut::aboveMean([], 2.0)); + } + +}