materio_ai: dense vector field in the generated Solr configset (I1)
SolrConfigSubscriber listens to PostConfigFilesGenerationEvent (search_api_solr 4.x no longer calls hook_search_api_solr_config_files_alter) and appends the knn_vector_1024 DenseVectorField (cosine, HNSW) and the knn1024_* dynamic field (not stored). Solr 9.1 at least. Dependency on search_api_solr added. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01ShDspHaZGwHoRZCsPBJzsM
This commit is contained in:
@@ -8,4 +8,5 @@ dependencies:
|
|||||||
- address:address
|
- address:address
|
||||||
- drupal:node
|
- drupal:node
|
||||||
- drupal:taxonomy
|
- drupal:taxonomy
|
||||||
|
- search_api_solr:search_api_solr
|
||||||
- workflow:workflow
|
- workflow:workflow
|
||||||
|
|||||||
@@ -22,3 +22,9 @@ services:
|
|||||||
class: Drupal\materio_ai\Vectorizer
|
class: Drupal\materio_ai\Vectorizer
|
||||||
arguments: ['@entity_type.manager', '@config.factory', '@queue', '@materio_ai.corpus_builder', '@materio_ai.embedding_client', '@materio_ai.embedding_storage']
|
arguments: ['@entity_type.manager', '@config.factory', '@queue', '@materio_ai.corpus_builder', '@materio_ai.embedding_client', '@materio_ai.embedding_storage']
|
||||||
Drupal\materio_ai\Vectorizer: '@materio_ai.vectorizer'
|
Drupal\materio_ai\Vectorizer: '@materio_ai.vectorizer'
|
||||||
|
|
||||||
|
materio_ai.solr_config_subscriber:
|
||||||
|
class: Drupal\materio_ai\EventSubscriber\SolrConfigSubscriber
|
||||||
|
arguments: ['@config.factory']
|
||||||
|
tags:
|
||||||
|
- { name: event_subscriber }
|
||||||
|
|||||||
@@ -0,0 +1,59 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
namespace Drupal\materio_ai\EventSubscriber;
|
||||||
|
|
||||||
|
use Drupal\Core\Config\ConfigFactoryInterface;
|
||||||
|
use Drupal\materio_ai\Solr\VectorField;
|
||||||
|
use Drupal\search_api_solr\Event\PostConfigFilesGenerationEvent;
|
||||||
|
use Drupal\search_api_solr\Event\SearchApiSolrEvents;
|
||||||
|
use Symfony\Component\EventDispatcher\EventSubscriberInterface;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Adds the dense vector field to the generated Solr configset.
|
||||||
|
*
|
||||||
|
* Event of search_api_solr 4.x, which replaces
|
||||||
|
* hook_search_api_solr_config_files_alter(). Solr 9.1 at least: pre-filters
|
||||||
|
* (fq) apply to the kNN search only from this version.
|
||||||
|
*/
|
||||||
|
final class SolrConfigSubscriber implements EventSubscriberInterface {
|
||||||
|
|
||||||
|
public function __construct(
|
||||||
|
private readonly ConfigFactoryInterface $configFactory,
|
||||||
|
) {}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* {@inheritdoc}
|
||||||
|
*/
|
||||||
|
public static function getSubscribedEvents(): array {
|
||||||
|
if (!class_exists(SearchApiSolrEvents::class)) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
return [SearchApiSolrEvents::POST_CONFIG_FILES_GENERATION => 'addVectorField'];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Appends the field type and the dynamic field to the schema_extra files.
|
||||||
|
*/
|
||||||
|
public function addVectorField(PostConfigFilesGenerationEvent $event): void {
|
||||||
|
if (version_compare($event->getLuceneMatchVersion(), '9.1', '<')) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
$dimension = (int) $this->configFactory->get('materio_ai.settings')->get('dimension');
|
||||||
|
if ($dimension <= 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
$files = $event->getConfigFiles();
|
||||||
|
foreach ([
|
||||||
|
'schema_extra_types.xml' => VectorField::typeXml($dimension),
|
||||||
|
'schema_extra_fields.xml' => VectorField::dynamicFieldXml($dimension),
|
||||||
|
] as $file => $xml) {
|
||||||
|
if (isset($files[$file]) && !str_contains($files[$file], $xml)) {
|
||||||
|
$files[$file] .= $xml;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
$event->setConfigFiles($files);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,55 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
namespace Drupal\materio_ai\Solr;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Names and schema snippets of the dense vector field in Solr.
|
||||||
|
*
|
||||||
|
* One field type and one dynamic field per dimension: a model of another
|
||||||
|
* dimension gets its own field, without touching the existing one.
|
||||||
|
*/
|
||||||
|
final class VectorField {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Field type name for a dimension, e.g. knn_vector_1024.
|
||||||
|
*/
|
||||||
|
public static function typeName(int $dimension): string {
|
||||||
|
return 'knn_vector_' . $dimension;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Solr field written to each document, e.g. knn1024_vector.
|
||||||
|
*/
|
||||||
|
public static function fieldName(int $dimension): string {
|
||||||
|
return 'knn' . $dimension . '_vector';
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* <fieldType> for schema_extra_types.xml.
|
||||||
|
*
|
||||||
|
* Cosine: no assumption on the norm of the vectors sent by the model.
|
||||||
|
*/
|
||||||
|
public static function typeXml(int $dimension): string {
|
||||||
|
return sprintf(
|
||||||
|
"<fieldType name=\"%s\" class=\"solr.DenseVectorField\" vectorDimension=\"%d\" similarityFunction=\"cosine\" knnAlgorithm=\"hnsw\" />\n",
|
||||||
|
self::typeName($dimension),
|
||||||
|
$dimension,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* <dynamicField> for schema_extra_fields.xml.
|
||||||
|
*
|
||||||
|
* Not stored: the vectors live in the materio_ai_embedding table.
|
||||||
|
*/
|
||||||
|
public static function dynamicFieldXml(int $dimension): string {
|
||||||
|
return sprintf(
|
||||||
|
"<dynamicField name=\"knn%d_*\" type=\"%s\" indexed=\"true\" stored=\"false\" multiValued=\"false\" />\n",
|
||||||
|
$dimension,
|
||||||
|
self::typeName($dimension),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user