Compare commits
1
Commits
88ecfb7199
...
7786c2cf71
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7786c2cf71 |
@@ -0,0 +1,44 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @file
|
||||||
|
* Install, update and uninstall functions for materio_sapi.
|
||||||
|
*/
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Source of the search message (H6) and its French translation.
|
||||||
|
*
|
||||||
|
* Interface translations live in the database, not in the repository: the
|
||||||
|
* update brings the French text to every site.
|
||||||
|
*/
|
||||||
|
function materio_sapi_search_infos_strings(): array {
|
||||||
|
return [
|
||||||
|
'source' => '@exactmatchcount exact result(s) out of @count relevant result(s) for “@keys”',
|
||||||
|
'fr' => '@exactmatchcount résultat(s) exact(s) sur @count résultat(s) pertinent(s) pour « @keys »',
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Adds the French translation of the new search message (H6).
|
||||||
|
*/
|
||||||
|
function materio_sapi_update_11001(): string {
|
||||||
|
if (!\Drupal::moduleHandler()->moduleExists('locale')) {
|
||||||
|
return 'locale is off: nothing to translate.';
|
||||||
|
}
|
||||||
|
$strings = materio_sapi_search_infos_strings();
|
||||||
|
$storage = \Drupal::service('locale.storage');
|
||||||
|
$source = $storage->findString(['source' => $strings['source'], 'context' => '']);
|
||||||
|
if (!$source) {
|
||||||
|
$source = $storage->createString(['source' => $strings['source'], 'context' => ''])->save();
|
||||||
|
}
|
||||||
|
$storage->createTranslation([
|
||||||
|
'lid' => $source->lid,
|
||||||
|
'language' => 'fr',
|
||||||
|
'translation' => $strings['fr'],
|
||||||
|
'customized' => LOCALE_CUSTOMIZED,
|
||||||
|
])->save();
|
||||||
|
_locale_refresh_translations(['fr'], [$source->lid]);
|
||||||
|
return 'French translation of the search message added.';
|
||||||
|
}
|
||||||
@@ -10,6 +10,7 @@ use Drupal\Component\Utility\Unicode;
|
|||||||
use Drupal\materio_ai\Exception\ModelPausedException;
|
use Drupal\materio_ai\Exception\ModelPausedException;
|
||||||
use Drupal\materio_sapi\KeywordText;
|
use Drupal\materio_sapi\KeywordText;
|
||||||
use Drupal\materio_sapi\RankFusion;
|
use Drupal\materio_sapi\RankFusion;
|
||||||
|
use Drupal\materio_sapi\ScoreCut;
|
||||||
use Drupal\search_api\Entity\Index;
|
use Drupal\search_api\Entity\Index;
|
||||||
|
|
||||||
// https://drupal.stackexchange.com/questions/225008/programatically-use-search-api
|
// https://drupal.stackexchange.com/questions/225008/programatically-use-search-api
|
||||||
@@ -91,9 +92,27 @@ class Base extends ControllerBase {
|
|||||||
public const REFERENCE_PATTERN = '/\\b[WTRPCMFGSO]\\d{4}\\b/iu';
|
public const REFERENCE_PATTERN = '/\\b[WTRPCMFGSO]\\d{4}\\b/iu';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Most neighbours asked to Solr (kNN topK), H5. Tail cut in H6.
|
* Neighbours asked to Solr (kNN topK): every visible node (8 073 for an
|
||||||
|
* anonymous visitor in 2026), so that ScoreCut sees the whole distribution
|
||||||
|
* of the scores (H6). 0.3 to 0.6 s measured locally.
|
||||||
*/
|
*/
|
||||||
public const KNN_LIMIT = 200;
|
public const KNN_LIMIT = 20000;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Standard deviations above the mean kept by ScoreCut (H6, option f, k = 2
|
||||||
|
* on trial since 17/11).
|
||||||
|
*/
|
||||||
|
public const KNN_SIGMA = 2.0;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Whether the OR query is merged with the neighbours (H3, H5).
|
||||||
|
*
|
||||||
|
* Off on trial since 17/11 (H6): on a phrase it matched about 40 % of the
|
||||||
|
* catalogue. Kept: it still runs when there are no neighbours (model
|
||||||
|
* stopped, slow or switched off, term chosen in the autocomplete), so that
|
||||||
|
* the search answers as before.
|
||||||
|
*/
|
||||||
|
public const OR_WITH_KNN = FALSE;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Number of neighbours found, for the response tuning of H6.
|
* Number of neighbours found, for the response tuning of H6.
|
||||||
@@ -178,12 +197,14 @@ class Base extends ControllerBase {
|
|||||||
$exact += $this->andQuery($keys, $lang);
|
$exact += $this->andQuery($keys, $lang);
|
||||||
}
|
}
|
||||||
$this->exactematch_count = count($exact);
|
$this->exactematch_count = count($exact);
|
||||||
$or = $this->orQuery($keys, $lang);
|
|
||||||
|
|
||||||
// Exact matches first, then the OR query and the neighbours merged (H5).
|
// Exact matches first, then the neighbours that stand out (H6), merged
|
||||||
|
// with the OR query when it is on (H5). No neighbours: the OR query alone.
|
||||||
$neighbours = $this->neighbours($lang);
|
$neighbours = $this->neighbours($lang);
|
||||||
|
$or = ($neighbours === NULL || self::OR_WITH_KNN) ? $this->orQuery($keys, $lang) : [];
|
||||||
|
$neighbours = $neighbours === NULL ? [] : ScoreCut::aboveMean($neighbours, self::KNN_SIGMA);
|
||||||
$this->knn_count = count($neighbours);
|
$this->knn_count = count($neighbours);
|
||||||
$nids = RankFusion::fuse(array_keys($exact), [array_keys($or), array_keys($neighbours)]);
|
$nids = RankFusion::fuse(array_keys($exact), array_filter([array_keys($or), array_keys($neighbours)]));
|
||||||
|
|
||||||
$uuids = array_column($exact + $or, 1, 0) + $this->uuids(array_diff(array_keys($neighbours), array_keys($exact + $or)));
|
$uuids = array_column($exact + $or, 1, 0) + $this->uuids(array_diff(array_keys($neighbours), array_keys($exact + $or)));
|
||||||
$this->results = [
|
$this->results = [
|
||||||
@@ -196,31 +217,33 @@ class Base extends ControllerBase {
|
|||||||
* Nodes closest in meaning to the phrase (materio_ai, Solr kNN).
|
* Nodes closest in meaning to the phrase (materio_ai, Solr kNN).
|
||||||
*
|
*
|
||||||
* Not for a term chosen in the autocomplete (direct search) nor without
|
* Not for a term chosen in the autocomplete (direct search) nor without
|
||||||
* the materio_ai service. On a model or Solr error, or past the delay of
|
* the materio_ai service. On a model or Solr error, past the delay of
|
||||||
* QueryEmbedder (1 s), the keyword results stand alone, with no message:
|
* QueryEmbedder (1 s), or with materio_ai switched off, the keyword
|
||||||
* the search answers as before H5.
|
* results stand alone, with no message: the search answers as before H5.
|
||||||
*
|
*
|
||||||
* @return array<int, float>
|
* @return array<int, float>|null
|
||||||
* Scores keyed by nid, best first.
|
* Scores of every visible node keyed by nid, best first; NULL when there
|
||||||
|
* are no neighbours to use.
|
||||||
*/
|
*/
|
||||||
private function neighbours(string $lang): array {
|
private function neighbours(string $lang): ?array {
|
||||||
if ($this->terms || !\Drupal::hasService('materio_ai.neighbour_search')) {
|
if ($this->terms || !\Drupal::hasService('materio_ai.neighbour_search')) {
|
||||||
return [];
|
return NULL;
|
||||||
}
|
}
|
||||||
// References are keywords: they mean nothing to the model.
|
// References are keywords: they mean nothing to the model.
|
||||||
$phrase = trim(preg_replace(self::REFERENCE_PATTERN, ' ', $this->phrase) ?? '');
|
$phrase = trim(preg_replace(self::REFERENCE_PATTERN, ' ', $this->phrase) ?? '');
|
||||||
if ($phrase === '') {
|
if ($phrase === '') {
|
||||||
return [];
|
return NULL;
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
return \Drupal::service('materio_ai.neighbour_search')->search($phrase, $lang, self::KNN_LIMIT);
|
// Empty when materio_ai is switched off.
|
||||||
|
return \Drupal::service('materio_ai.neighbour_search')->search($phrase, $lang, self::KNN_LIMIT) ?: NULL;
|
||||||
}
|
}
|
||||||
catch (\Throwable $e) {
|
catch (\Throwable $e) {
|
||||||
// Paused model: its failure was logged when it happened.
|
// Paused model: its failure was logged when it happened.
|
||||||
if (!$e instanceof ModelPausedException) {
|
if (!$e instanceof ModelPausedException) {
|
||||||
\Drupal::logger('materio_sapi')->warning('Neighbour search failed, keyword results only: @message', ['@message' => $e->getMessage()]);
|
\Drupal::logger('materio_sapi')->warning('Neighbour search failed, keyword results only: @message', ['@message' => $e->getMessage()]);
|
||||||
}
|
}
|
||||||
return [];
|
return NULL;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -460,13 +483,14 @@ class Base extends ControllerBase {
|
|||||||
// $resp['count'] = $this->results->getResultCount();
|
// $resp['count'] = $this->results->getResultCount();
|
||||||
$resp['count'] = count($this->results['nids']);
|
$resp['count'] = count($this->results['nids']);
|
||||||
$resp['exactematch_count'] = $this->exactematch_count;
|
$resp['exactematch_count'] = $this->exactematch_count;
|
||||||
$resp['infos'] = t('The search found @exactmatchcount exact match result(s) for @count total result(s) with', array(
|
// Relevant: exact matches and neighbours that stand out (H6); the
|
||||||
"@exactmatchcount" => $resp['exactematch_count'],
|
// count no longer depends on an arbitrary limit. French text added by
|
||||||
"@count" => $resp['count']
|
// materio_sapi_update_11001().
|
||||||
));
|
$resp['infos'] = t('@exactmatchcount exact result(s) out of @count relevant result(s) for “@keys”', [
|
||||||
$resp['infos'] .= t(' keywords @keys', array(
|
'@exactmatchcount' => $resp['exactematch_count'],
|
||||||
"@keys" => $this->phrase
|
'@count' => $resp['count'],
|
||||||
));
|
'@keys' => $this->phrase,
|
||||||
|
]);
|
||||||
// $resp['options'] = $this->query->getOptions();
|
// $resp['options'] = $this->query->getOptions();
|
||||||
|
|
||||||
// $uuids = [];
|
// $uuids = [];
|
||||||
|
|||||||
@@ -0,0 +1,44 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
namespace Drupal\materio_sapi;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Keeps the neighbours that stand out for this search (H6, option f).
|
||||||
|
*
|
||||||
|
* The kNN search scores every visible node, so each search has its own score
|
||||||
|
* distribution. A fixed threshold means nothing: the scale moves with the
|
||||||
|
* search and the language (at 0.7, « bois » kept 664 nodes and « wood »
|
||||||
|
* 4 671). A node is kept when its score is above the mean of this search by
|
||||||
|
* more than K standard deviations: with K = 2, « bois » 373 and « wood » 382,
|
||||||
|
* 150 to 500 nodes for one word, about 250 for a phrase (measured 17/11).
|
||||||
|
*/
|
||||||
|
final class ScoreCut {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Scores above mean + $k standard deviations, in their order.
|
||||||
|
*
|
||||||
|
* @param array<int, float> $scores
|
||||||
|
* Scores keyed by nid, best first, ideally of every visible node.
|
||||||
|
* @param float $k
|
||||||
|
* Number of standard deviations.
|
||||||
|
*
|
||||||
|
* @return array<int, float>
|
||||||
|
* The kept scores, keyed by nid.
|
||||||
|
*/
|
||||||
|
public static function aboveMean(array $scores, float $k): array {
|
||||||
|
$n = count($scores);
|
||||||
|
if ($n < 2) {
|
||||||
|
return $scores;
|
||||||
|
}
|
||||||
|
$mean = array_sum($scores) / $n;
|
||||||
|
$variance = 0.0;
|
||||||
|
foreach ($scores as $score) {
|
||||||
|
$variance += ($score - $mean) ** 2;
|
||||||
|
}
|
||||||
|
$limit = $mean + $k * sqrt($variance / $n);
|
||||||
|
return array_filter($scores, static fn ($score) => $score > $limit);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,47 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
namespace Drupal\Tests\materio_sapi\Unit;
|
||||||
|
|
||||||
|
use Drupal\materio_sapi\ScoreCut;
|
||||||
|
use Drupal\Tests\UnitTestCase;
|
||||||
|
use PHPUnit\Framework\Attributes\CoversClass;
|
||||||
|
use PHPUnit\Framework\Attributes\Group;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Tests the cut of the neighbours relative to the search (H6).
|
||||||
|
*/
|
||||||
|
#[CoversClass(ScoreCut::class)]
|
||||||
|
#[Group('materio_sapi')]
|
||||||
|
final class ScoreCutTest extends UnitTestCase {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Only the scores above mean + k standard deviations stay, in order.
|
||||||
|
*/
|
||||||
|
public function testAboveMean(): void {
|
||||||
|
// 98 scores at 0.6, then 0.9 and 0.8: mean 0.6045, deviation ≈ 0.0337.
|
||||||
|
$scores = [7 => 0.9, 3 => 0.8] + array_fill_keys(range(100, 197), 0.6);
|
||||||
|
$this->assertSame([7 => 0.9, 3 => 0.8], ScoreCut::aboveMean($scores, 2.0));
|
||||||
|
$this->assertSame([7 => 0.9], ScoreCut::aboveMean($scores, 7.0));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The same distribution shifted keeps the same nodes: no fixed threshold.
|
||||||
|
*/
|
||||||
|
public function testShiftedScale(): void {
|
||||||
|
$scores = [7 => 0.9, 3 => 0.8] + array_fill_keys(range(100, 197), 0.6);
|
||||||
|
$shifted = array_map(static fn ($s) => $s + 0.15, $scores);
|
||||||
|
$this->assertSame(array_keys(ScoreCut::aboveMean($scores, 2.0)), array_keys(ScoreCut::aboveMean($shifted, 2.0)));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Equal scores stand out from nothing; fewer than two scores are kept.
|
||||||
|
*/
|
||||||
|
public function testEdgeCases(): void {
|
||||||
|
$this->assertSame([], ScoreCut::aboveMean([1 => 0.7, 2 => 0.7, 3 => 0.7], 2.0));
|
||||||
|
$this->assertSame([5 => 0.8], ScoreCut::aboveMean([5 => 0.8], 2.0));
|
||||||
|
$this->assertSame([], ScoreCut::aboveMean([], 2.0));
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user