feat: add audio & document input support for Gemini (#339)

Author: valtzu

examples/google/audio-input.php

@@ -0,0 +1,31 @@
+<?php
+
+use PhpLlm\LlmChain\Chain\Chain;
+use PhpLlm\LlmChain\Platform\Bridge\Google\Gemini;
+use PhpLlm\LlmChain\Platform\Bridge\Google\PlatformFactory;
+use PhpLlm\LlmChain\Platform\Message\Content\Audio;
+use PhpLlm\LlmChain\Platform\Message\Message;
+use PhpLlm\LlmChain\Platform\Message\MessageBag;
+use Symfony\Component\Dotenv\Dotenv;
+
+require_once dirname(__DIR__, 2).'/vendor/autoload.php';
+(new Dotenv())->loadEnv(dirname(__DIR__, 2).'/.env');
+
+if (empty($_ENV['GOOGLE_API_KEY'])) {
+    echo 'Please set the GOOGLE_API_KEY environment variable.'.\PHP_EOL;
+    exit(1);
+}
+
+$platform = PlatformFactory::create($_ENV['GOOGLE_API_KEY']);
+$model = new Gemini(Gemini::GEMINI_1_5_FLASH);
+
+$chain = new Chain($platform, $model);
+$messages = new MessageBag(
+    Message::ofUser(
+        'What is this recording about?',
+        Audio::fromFile(dirname(__DIR__, 2).'/tests/Fixture/audio.mp3'),
+    ),
+);
+$response = $chain->call($messages);
+
+echo $response->getContent().\PHP_EOL;

examples/google/pdf-input-binary.php

@@ -0,0 +1,31 @@
+<?php
+
+use PhpLlm\LlmChain\Chain\Chain;
+use PhpLlm\LlmChain\Platform\Bridge\Google\Gemini;
+use PhpLlm\LlmChain\Platform\Bridge\Google\PlatformFactory;
+use PhpLlm\LlmChain\Platform\Message\Content\Document;
+use PhpLlm\LlmChain\Platform\Message\Message;
+use PhpLlm\LlmChain\Platform\Message\MessageBag;
+use Symfony\Component\Dotenv\Dotenv;
+
+require_once dirname(__DIR__, 2).'/vendor/autoload.php';
+(new Dotenv())->loadEnv(dirname(__DIR__, 2).'/.env');
+
+if (empty($_ENV['GOOGLE_API_KEY'])) {
+    echo 'Please set the GOOGLE_API_KEY environment variable.'.\PHP_EOL;
+    exit(1);
+}
+
+$platform = PlatformFactory::create($_ENV['GOOGLE_API_KEY']);
+$model = new Gemini(Gemini::GEMINI_1_5_FLASH);
+
+$chain = new Chain($platform, $model);
+$messages = new MessageBag(
+    Message::ofUser(
+        Document::fromFile(dirname(__DIR__, 2).'/tests/Fixture/document.pdf'),
+        'What is this document about?',
+    ),
+);
+$response = $chain->call($messages);
+
+echo $response->getContent().\PHP_EOL;

src/Platform/Bridge/Google/Contract/UserMessageNormalizer.php

@@ -6,7 +6,7 @@
 
 use PhpLlm\LlmChain\Platform\Bridge\Google\Gemini;
 use PhpLlm\LlmChain\Platform\Contract\Normalizer\ModelContractNormalizer;
-use PhpLlm\LlmChain\Platform\Message\Content\Image;
+use PhpLlm\LlmChain\Platform\Message\Content\File;
 use PhpLlm\LlmChain\Platform\Message\Content\Text;
 use PhpLlm\LlmChain\Platform\Message\UserMessage;
 use PhpLlm\LlmChain\Platform\Model;
@@ -38,7 +38,7 @@ public function normalize(mixed $data, ?string $format = null, array $context =
             if ($content instanceof Text) {
                 $parts[] = ['text' => $content->text];
             }
-            if ($content instanceof Image) {
+            if ($content instanceof File) {
                 $parts[] = ['inline_data' => [
                     'mime_type' => $content->getFormat(),
                     'data' => $content->asBase64(),

src/Platform/Bridge/Google/Gemini.php

@@ -26,6 +26,8 @@ public function __construct(string $name = self::GEMINI_2_PRO, array $options =
         $capabilities = [
             Capability::INPUT_MESSAGES,
             Capability::INPUT_IMAGE,
+            Capability::INPUT_AUDIO,
+            Capability::INPUT_PDF,
             Capability::OUTPUT_STREAMING,
             Capability::TOOL_CALLING,
         ];

tests/Platform/Bridge/Google/Contract/UserMessageNormalizerTest.php

@@ -7,11 +7,14 @@
 use PhpLlm\LlmChain\Platform\Bridge\Google\Contract\UserMessageNormalizer;
 use PhpLlm\LlmChain\Platform\Bridge\Google\Gemini;
 use PhpLlm\LlmChain\Platform\Contract;
+use PhpLlm\LlmChain\Platform\Message\Content\Audio;
+use PhpLlm\LlmChain\Platform\Message\Content\Document;
 use PhpLlm\LlmChain\Platform\Message\Content\File;
 use PhpLlm\LlmChain\Platform\Message\Content\Image;
 use PhpLlm\LlmChain\Platform\Message\Content\Text;
 use PhpLlm\LlmChain\Platform\Message\UserMessage;
 use PHPUnit\Framework\Attributes\CoversClass;
+use PHPUnit\Framework\Attributes\DataProvider;
 use PHPUnit\Framework\Attributes\Small;
 use PHPUnit\Framework\Attributes\Test;
 use PHPUnit\Framework\Attributes\UsesClass;
@@ -23,6 +26,9 @@
 #[UsesClass(UserMessage::class)]
 #[UsesClass(Text::class)]
 #[UsesClass(File::class)]
+#[UsesClass(Image::class)]
+#[UsesClass(Document::class)]
+#[UsesClass(Audio::class)]
 final class UserMessageNormalizerTest extends TestCase
 {
     #[Test]
@@ -55,22 +61,32 @@ public function normalizeTextContent(): void
         self::assertSame([['text' => 'Write a story about a magic backpack.']], $normalized);
     }
 
+    #[DataProvider('binaryContentProvider')]
     #[Test]
-    public function normalizeImageContent(): void
+    public function normalizeBinaryContent(File $content, string $expectedMimeType, string $expectedPrefix): void
     {
         $normalizer = new UserMessageNormalizer();
-        $imageContent = Image::fromFile(\dirname(__DIR__, 4).'/Fixture/image.jpg');
-        $message = new UserMessage(new Text('Tell me about this instrument'), $imageContent);
+        $message = new UserMessage(new Text('Tell me about this instrument'), $content);
 
         $normalized = $normalizer->normalize($message);
 
         self::assertCount(2, $normalized);
         self::assertSame(['text' => 'Tell me about this instrument'], $normalized[0]);
         self::assertArrayHasKey('inline_data', $normalized[1]);
-        self::assertSame('image/jpeg', $normalized[1]['inline_data']['mime_type']);
+        self::assertSame($expectedMimeType, $normalized[1]['inline_data']['mime_type']);
         self::assertNotEmpty($normalized[1]['inline_data']['data']);
 
-        // Verify that the base64 data string starts correctly for a JPEG
-        self::assertStringStartsWith('/9j/', $normalized[1]['inline_data']['data']);
+        // Verify that the base64 data string starts correctly
+        self::assertStringStartsWith($expectedPrefix, $normalized[1]['inline_data']['data']);
+    }
+
+    /**
+     * @return iterable<string, array{0: File, 1: string, 2: string}>
+     */
+    public static function binaryContentProvider(): iterable
+    {
+        yield 'image' => [Image::fromFile(\dirname(__DIR__, 4).'/Fixture/image.jpg'), 'image/jpeg', '/9j/'];
+        yield 'document' => [Document::fromFile(\dirname(__DIR__, 4).'/Fixture/document.pdf'), 'application/pdf', 'JVBE'];
+        yield 'audio' => [Audio::fromFile(\dirname(__DIR__, 4).'/Fixture/audio.mp3'), 'audio/mpeg', 'SUQz'];
     }
 }