feat: implement document loader & transformer for store indexing (#343)

Author: chr-hertel

examples/store/document-splitting.php

@@ -0,0 +1,23 @@
+<?php
+
+/*
+ * This file is part of the Symfony package.
+ *
+ * (c) Fabien Potencier <[email protected]>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+use Symfony\AI\Store\Document\Loader\TextFileLoader;
+use Symfony\AI\Store\Document\Transformer\TextSplitTransformer;
+
+require_once dirname(__DIR__, 2).'/vendor/autoload.php';
+
+$loader = new TextFileLoader();
+$splitter = new TextSplitTransformer();
+$source = dirname(__DIR__, 2).'/tests/Fixture/lorem.txt';
+
+$documents = iterator_to_array($splitter($loader($source)));
+
+dump($documents);

examples/store/document-vectorizing.php

@@ -0,0 +1,40 @@
+<?php
+
+/*
+ * This file is part of the Symfony package.
+ *
+ * (c) Fabien Potencier <[email protected]>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+use Symfony\AI\Platform\Bridge\OpenAI\Embeddings;
+use Symfony\AI\Platform\Bridge\OpenAI\PlatformFactory;
+use Symfony\AI\Store\Document\TextDocument;
+use Symfony\AI\Store\Document\VectorDocument;
+use Symfony\AI\Store\Document\Vectorizer;
+use Symfony\Component\Dotenv\Dotenv;
+use Symfony\Component\Uid\Uuid;
+
+require_once dirname(__DIR__, 2).'/vendor/autoload.php';
+(new Dotenv())->loadEnv(dirname(__DIR__, 2).'/.env');
+
+if (empty($_ENV['OPENAI_API_KEY'])) {
+    echo 'Please set the OPENAI_API_KEY environment variable.'.\PHP_EOL;
+    exit(1);
+}
+
+$platform = PlatformFactory::create($_ENV['OPENAI_API_KEY']);
+$embeddings = new Embeddings(Embeddings::TEXT_3_LARGE);
+
+$textDocuments = [
+    new TextDocument(Uuid::v4(), 'Hello World'),
+    new TextDocument(Uuid::v4(), 'Lorem ipsum dolor sit amet'),
+    new TextDocument(Uuid::v4(), 'PHP Hypertext Preprocessor'),
+];
+
+$vectorizer = new Vectorizer($platform, $embeddings);
+$vectorDocuments = $vectorizer->vectorizeDocuments($textDocuments);
+
+dump(array_map(fn (VectorDocument $document) => $document->vector->getDimensions(), $vectorDocuments));

examples/store/mariadb-similarity-search-gemini.php

@@ -0,0 +1,87 @@
+<?php
+
+/*
+ * This file is part of the Symfony package.
+ *
+ * (c) Fabien Potencier <[email protected]>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+use Doctrine\DBAL\DriverManager;
+use Doctrine\DBAL\Tools\DsnParser;
+use Symfony\AI\Agent\Agent;
+use Symfony\AI\Agent\Toolbox\AgentProcessor;
+use Symfony\AI\Agent\Toolbox\Tool\SimilaritySearch;
+use Symfony\AI\Agent\Toolbox\Toolbox;
+use Symfony\AI\Platform\Bridge\Google\Embeddings;
+use Symfony\AI\Platform\Bridge\Google\Embeddings\TaskType;
+use Symfony\AI\Platform\Bridge\Google\Gemini;
+use Symfony\AI\Platform\Bridge\Google\PlatformFactory;
+use Symfony\AI\Platform\Message\Message;
+use Symfony\AI\Platform\Message\MessageBag;
+use Symfony\AI\Store\Bridge\MariaDB\Store;
+use Symfony\AI\Store\Document\Metadata;
+use Symfony\AI\Store\Document\TextDocument;
+use Symfony\AI\Store\Document\Vectorizer;
+use Symfony\AI\Store\Indexer;
+use Symfony\Component\Dotenv\Dotenv;
+use Symfony\Component\Uid\Uuid;
+
+require_once dirname(__DIR__, 2).'/vendor/autoload.php';
+(new Dotenv())->loadEnv(dirname(__DIR__, 2).'/.env');
+
+if (empty($_ENV['GOOGLE_API_KEY']) || empty($_ENV['MARIADB_URI'])) {
+    echo 'Please set GOOGLE_API_KEY and MARIADB_URI environment variables.'.\PHP_EOL;
+    exit(1);
+}
+
+// initialize the store
+$store = Store::fromDbal(
+    connection: DriverManager::getConnection((new DsnParser())->parse($_ENV['MARIADB_URI'])),
+    tableName: 'my_table',
+    indexName: 'my_index',
+    vectorFieldName: 'embedding',
+);
+
+// our data
+$movies = [
+    ['title' => 'Inception', 'description' => 'A skilled thief is given a chance at redemption if he can successfully perform inception, the act of planting an idea in someone\'s subconscious.', 'director' => 'Christopher Nolan'],
+    ['title' => 'The Matrix', 'description' => 'A hacker discovers the world he lives in is a simulated reality and joins a rebellion to overthrow its controllers.', 'director' => 'The Wachowskis'],
+    ['title' => 'The Godfather', 'description' => 'The aging patriarch of an organized crime dynasty transfers control of his empire to his reluctant son.', 'director' => 'Francis Ford Coppola'],
+];
+
+// create embeddings and documents
+foreach ($movies as $i => $movie) {
+    $documents[] = new TextDocument(
+        id: Uuid::v4(),
+        content: 'Title: '.$movie['title'].\PHP_EOL.'Director: '.$movie['director'].\PHP_EOL.'Description: '.$movie['description'],
+        metadata: new Metadata($movie),
+    );
+}
+
+// initialize the table
+$store->initialize(['dimensions' => 768]);
+
+// create embeddings for documents
+$platform = PlatformFactory::create($_ENV['GOOGLE_API_KEY']);
+$embeddings = new Embeddings(options: ['dimensions' => 768, 'task_type' => TaskType::SemanticSimilarity]);
+$vectorizer = new Vectorizer($platform, $embeddings);
+$indexer = new Indexer($vectorizer, $store);
+$indexer->index($documents);
+
+$model = new Gemini(Gemini::GEMINI_2_FLASH_LITE);
+
+$similaritySearch = new SimilaritySearch($platform, $embeddings, $store);
+$toolbox = Toolbox::create($similaritySearch);
+$processor = new AgentProcessor($toolbox);
+$agent = new Agent($platform, $model, [$processor], [$processor]);
+
+$messages = new MessageBag(
+    Message::forSystem('Please answer all user questions only using SimilaritySearch function.'),
+    Message::ofUser('Which movie fits the theme of the mafia?')
+);
+$response = $agent->call($messages);
+
+echo $response->getContent().\PHP_EOL;

examples/store/mariadb-similarity-search.php

@@ -21,6 +21,7 @@
 use Symfony\AI\Store\Bridge\MariaDB\Store;
 use Symfony\AI\Store\Document\Metadata;
 use Symfony\AI\Store\Document\TextDocument;
+use Symfony\AI\Store\Document\Vectorizer;
 use Symfony\AI\Store\Indexer;
 use Symfony\Component\Dotenv\Dotenv;
 use Symfony\Component\Uid\Uuid;
@@ -62,7 +63,8 @@
 
 // create embeddings for documents
 $platform = PlatformFactory::create($_ENV['OPENAI_API_KEY']);
-$indexer = new Indexer($platform, $embeddings = new Embeddings(), $store);
+$vectorizer = new Vectorizer($platform, $embeddings = new Embeddings());
+$indexer = new Indexer($vectorizer, $store);
 $indexer->index($documents);
 
 $model = new GPT(GPT::GPT_4O_MINI);

examples/store/mongodb-similarity-search.php

@@ -22,6 +22,7 @@
 use Symfony\AI\Store\Bridge\MongoDB\Store;
 use Symfony\AI\Store\Document\Metadata;
 use Symfony\AI\Store\Document\TextDocument;
+use Symfony\AI\Store\Document\Vectorizer;
 use Symfony\AI\Store\Indexer;
 use Symfony\Component\Dotenv\Dotenv;
 use Symfony\Component\Uid\Uuid;
@@ -61,7 +62,8 @@
 
 // create embeddings for documents
 $platform = PlatformFactory::create($_ENV['OPENAI_API_KEY']);
-$indexer = new Indexer($platform, $embeddings = new Embeddings(), $store);
+$vectorizer = new Vectorizer($platform, $embeddings = new Embeddings());
+$indexer = new Indexer($vectorizer, $store);
 $indexer->index($documents);
 
 // initialize the index

examples/store/pinecone-similarity-search.php

@@ -9,6 +9,7 @@
  * file that was distributed with this source code.
  */
 
+use PhpLlm\LlmChain\Store\Document\Vectorizer;
 use Probots\Pinecone\Pinecone;
 use Symfony\AI\Agent\Agent;
 use Symfony\AI\Agent\Toolbox\AgentProcessor;
@@ -55,7 +56,8 @@
 
 // create embeddings for documents
 $platform = PlatformFactory::create($_ENV['OPENAI_API_KEY']);
-$indexer = new Indexer($platform, $embeddings = new Embeddings(), $store);
+$vectorizer = new Vectorizer($platform, $embeddings = new Embeddings());
+$indexer = new Indexer($vectorizer, $store);
 $indexer->index($documents);
 
 $model = new GPT(GPT::GPT_4O_MINI);

fixtures/lorem.txt

@@ -0,0 +1,15 @@
+Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Aenean commodo ligula eget dolor. Aenean massa.
+Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Donec quam felis,
+ultricies nec, pellentesque eu, pretium quis, sem. Nulla consequat massa quis enim. Donec pede justo,
+fringilla vel, aliquet nec, vulputate eget, arcu. In enim justo, rhoncus ut, imperdiet a, venenatis vitae,
+justo. Nullam dictum felis eu pede mollis pretium. Integer tincidunt. Cras dapibus. Vivamus elementum semper
+nisi. Aenean vulputate eleifend tellus. Aenean leo ligula, porttitor eu, consequat vitae, eleifend ac, enim.
+Aliquam lorem ante, dapibus in, viverra quis, feugiat a, tellus. Phasellus viverra nulla ut metus varius
+laoreet. Quisque rutrum. Aenean imperdiet. Etiam ultricies nisi vel augue. Curabitur ullamcorper ultricies
+nisi. Nam eget dui. Etiam rhoncus. Maecenas tempus, tellus eget condimentum rhoncus, sem quam semper libero,
+sit amet adipiscing sem neque sed ipsum. Nam quam nunc, blandit vel, luctus pulvinar, hendrerit id, lorem.
+Maecenas nec odio et ante tincidunt tempus. Donec vitae sapien ut libero venenatis faucibus. Nullam quis
+ante. Etiam sit amet orci eget eros faucibus tincidunt. Duis leo. Sed fringilla mauris sit amet nibh. Donec
+sodales sagittis magna. Sed consequat, leo eget bibendum sodales, augue velit cursus nunc, quis gravida
+magna mi a libero. Fusce vulputate eleifend sapien. Vestibulum purus quam, scelerisque ut, mollis sed,
+nonummy id, met

src/mcp-sdk/tests/Fixtures/InMemoryTransport.php

@@ -1,7 +1,5 @@
 <?php
 
-
-
 /*
  * This file is part of the Symfony package.
  *

src/store/src/Document/Loader/TextFileLoader.php

@@ -0,0 +1,41 @@
+<?php
+
+/*
+ * This file is part of the Symfony package.
+ *
+ * (c) Fabien Potencier <[email protected]>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Symfony\AI\Store\Document\Loader;
+
+use Symfony\AI\Store\Document\LoaderInterface;
+use Symfony\AI\Store\Document\Metadata;
+use Symfony\AI\Store\Document\TextDocument;
+use Symfony\AI\Store\Exception\RuntimeException;
+use Symfony\Component\Uid\Uuid;
+
+/**
+ * @author Christopher Hertel <[email protected]>
+ */
+final readonly class TextFileLoader implements LoaderInterface
+{
+    public function __invoke(string $source, array $options = []): iterable
+    {
+        if (!is_file($source)) {
+            throw new RuntimeException(\sprintf('File "%s" does not exist.', $source));
+        }
+
+        $content = file_get_contents($source);
+
+        if (false === $content) {
+            throw new RuntimeException(\sprintf('Unable to read file "%s"', $source));
+        }
+
+        yield new TextDocument(Uuid::v4(), trim($content), new Metadata([
+            'source' => $source,
+        ]));
+    }
+}

src/store/src/Document/LoaderInterface.php

@@ -0,0 +1,26 @@
+<?php
+
+/*
+ * This file is part of the Symfony package.
+ *
+ * (c) Fabien Potencier <[email protected]>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Symfony\AI\Store\Document;
+
+/**
+ * @author Christopher Hertel <[email protected]>
+ */
+interface LoaderInterface
+{
+    /**
+     * @param string               $source  Identifier for the loader to load the documents from, e.g. file path, folder, or URL.
+     * @param array<string, mixed> $options loader specific set of options to control the loading process
+     *
+     * @return iterable<TextDocument> iterable of TextDocuments loaded from the source
+     */
+    public function __invoke(string $source, array $options = []): iterable;
+}