materio_ai: dense vector field in the generated Solr configset (I1)
SolrConfigSubscriber listens to PostConfigFilesGenerationEvent (search_api_solr 4.x no longer calls hook_search_api_solr_config_files_alter) and appends the knn_vector_1024 DenseVectorField (cosine, HNSW) and the knn1024_* dynamic field (not stored). Solr 9.1 at least. Dependency on search_api_solr added. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01ShDspHaZGwHoRZCsPBJzsM
This commit is contained in:
@@ -8,4 +8,5 @@ dependencies:
|
||||
- address:address
|
||||
- drupal:node
|
||||
- drupal:taxonomy
|
||||
- search_api_solr:search_api_solr
|
||||
- workflow:workflow
|
||||
|
||||
@@ -22,3 +22,9 @@ services:
|
||||
class: Drupal\materio_ai\Vectorizer
|
||||
arguments: ['@entity_type.manager', '@config.factory', '@queue', '@materio_ai.corpus_builder', '@materio_ai.embedding_client', '@materio_ai.embedding_storage']
|
||||
Drupal\materio_ai\Vectorizer: '@materio_ai.vectorizer'
|
||||
|
||||
materio_ai.solr_config_subscriber:
|
||||
class: Drupal\materio_ai\EventSubscriber\SolrConfigSubscriber
|
||||
arguments: ['@config.factory']
|
||||
tags:
|
||||
- { name: event_subscriber }
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Drupal\materio_ai\EventSubscriber;
|
||||
|
||||
use Drupal\Core\Config\ConfigFactoryInterface;
|
||||
use Drupal\materio_ai\Solr\VectorField;
|
||||
use Drupal\search_api_solr\Event\PostConfigFilesGenerationEvent;
|
||||
use Drupal\search_api_solr\Event\SearchApiSolrEvents;
|
||||
use Symfony\Component\EventDispatcher\EventSubscriberInterface;
|
||||
|
||||
/**
|
||||
* Adds the dense vector field to the generated Solr configset.
|
||||
*
|
||||
* Event of search_api_solr 4.x, which replaces
|
||||
* hook_search_api_solr_config_files_alter(). Solr 9.1 at least: pre-filters
|
||||
* (fq) apply to the kNN search only from this version.
|
||||
*/
|
||||
final class SolrConfigSubscriber implements EventSubscriberInterface {
|
||||
|
||||
public function __construct(
|
||||
private readonly ConfigFactoryInterface $configFactory,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* {@inheritdoc}
|
||||
*/
|
||||
public static function getSubscribedEvents(): array {
|
||||
if (!class_exists(SearchApiSolrEvents::class)) {
|
||||
return [];
|
||||
}
|
||||
return [SearchApiSolrEvents::POST_CONFIG_FILES_GENERATION => 'addVectorField'];
|
||||
}
|
||||
|
||||
/**
|
||||
* Appends the field type and the dynamic field to the schema_extra files.
|
||||
*/
|
||||
public function addVectorField(PostConfigFilesGenerationEvent $event): void {
|
||||
if (version_compare($event->getLuceneMatchVersion(), '9.1', '<')) {
|
||||
return;
|
||||
}
|
||||
$dimension = (int) $this->configFactory->get('materio_ai.settings')->get('dimension');
|
||||
if ($dimension <= 0) {
|
||||
return;
|
||||
}
|
||||
$files = $event->getConfigFiles();
|
||||
foreach ([
|
||||
'schema_extra_types.xml' => VectorField::typeXml($dimension),
|
||||
'schema_extra_fields.xml' => VectorField::dynamicFieldXml($dimension),
|
||||
] as $file => $xml) {
|
||||
if (isset($files[$file]) && !str_contains($files[$file], $xml)) {
|
||||
$files[$file] .= $xml;
|
||||
}
|
||||
}
|
||||
$event->setConfigFiles($files);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Drupal\materio_ai\Solr;
|
||||
|
||||
/**
|
||||
* Names and schema snippets of the dense vector field in Solr.
|
||||
*
|
||||
* One field type and one dynamic field per dimension: a model of another
|
||||
* dimension gets its own field, without touching the existing one.
|
||||
*/
|
||||
final class VectorField {
|
||||
|
||||
/**
|
||||
* Field type name for a dimension, e.g. knn_vector_1024.
|
||||
*/
|
||||
public static function typeName(int $dimension): string {
|
||||
return 'knn_vector_' . $dimension;
|
||||
}
|
||||
|
||||
/**
|
||||
* Solr field written to each document, e.g. knn1024_vector.
|
||||
*/
|
||||
public static function fieldName(int $dimension): string {
|
||||
return 'knn' . $dimension . '_vector';
|
||||
}
|
||||
|
||||
/**
|
||||
* <fieldType> for schema_extra_types.xml.
|
||||
*
|
||||
* Cosine: no assumption on the norm of the vectors sent by the model.
|
||||
*/
|
||||
public static function typeXml(int $dimension): string {
|
||||
return sprintf(
|
||||
"<fieldType name=\"%s\" class=\"solr.DenseVectorField\" vectorDimension=\"%d\" similarityFunction=\"cosine\" knnAlgorithm=\"hnsw\" />\n",
|
||||
self::typeName($dimension),
|
||||
$dimension,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* <dynamicField> for schema_extra_fields.xml.
|
||||
*
|
||||
* Not stored: the vectors live in the materio_ai_embedding table.
|
||||
*/
|
||||
public static function dynamicFieldXml(int $dimension): string {
|
||||
return sprintf(
|
||||
"<dynamicField name=\"knn%d_*\" type=\"%s\" indexed=\"true\" stored=\"false\" multiValued=\"false\" />\n",
|
||||
$dimension,
|
||||
self::typeName($dimension),
|
||||
);
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user