Multi-Modal Agent

1
AI_GATEWAY_API_KEY=your_api_key_here

1
import { streamText, convertToModelMessages, type UIMessage } from 'ai';
2

3
// Allow streaming responses up to 30 seconds
4
export const maxDuration = 30;
5

6
export async function POST(req: Request) {
7
  const { messages }: { messages: UIMessage[] } = await req.json();
8

9
  const result = streamText({
10
    model: 'openai/gpt-4o',
11
    messages: await convertToModelMessages(messages),
12
  });
13

14
  return result.toUIMessageStreamResponse();
15
}

1
'use client';
2

3
import { useChat } from '@ai-sdk/react';
4
import { DefaultChatTransport } from 'ai';
5
import { useState } from 'react';
6

7
export default function Chat() {
8
  const [input, setInput] = useState('');
9

10
  const { messages, sendMessage } = useChat({
11
    transport: new DefaultChatTransport({
12
      api: '/api/chat',
13
    }),
14
  });
15

16
  return (
17
    <div className="flex flex-col w-full max-w-md py-24 mx-auto stretch">
18
      {messages.map(m => (
19
        <div key={m.id} className="whitespace-pre-wrap">
20
          {m.role === 'user' ? 'User: ' : 'AI: '}
21
          {m.parts.map((part, index) => {
22
            if (part.type === 'text') {
23
              return <span key={`${m.id}-text-${index}`}>{part.text}</span>;
24
            }
25
            return null;
26
          })}
27
        </div>
28
      ))}
29

30
      <form
31
        onSubmit={async event => {
32
          event.preventDefault();
33
          sendMessage({
34
            role: 'user',
35
            parts: [{ type: 'text', text: input }],
36
          });
37
          setInput('');
38
        }}
39
        className="fixed bottom-0 w-full max-w-md mb-8 border border-gray-300 rounded shadow-xl"
40
      >
41
        <input
42
          className="w-full p-2"
43
          value={input}
44
          placeholder="Say something..."
45
          onChange={e => setInput(e.target.value)}
46
        />
47
      </form>
48
    </div>
49
  );
50
}

1
'use client';
2

3
import { useChat } from '@ai-sdk/react';
4
import { DefaultChatTransport } from 'ai';
5
import { useRef, useState } from 'react';
6
import Image from 'next/image';
7

8
async function convertFilesToDataURLs(files: FileList) {
9
  return Promise.all(
10
    Array.from(files).map(
11
      file =>
12
        new Promise<{
13
          type: 'file';
14
          mediaType: string;
15
          url: string;
16
        }>((resolve, reject) => {
17
          const reader = new FileReader();
18
          reader.onload = () => {
19
            resolve({
20
              type: 'file',
21
              mediaType: file.type,
22
              url: reader.result as string,
23
            });
24
          };
25
          reader.onerror = reject;
26
          reader.readAsDataURL(file);
27
        }),
28
    ),
29
  );
30
}
31

32
export default function Chat() {
33
  const [input, setInput] = useState('');
34
  const [files, setFiles] = useState<FileList | undefined>(undefined);
35
  const fileInputRef = useRef<HTMLInputElement>(null);
36

37
  const { messages, sendMessage } = useChat({
38
    transport: new DefaultChatTransport({
39
      api: '/api/chat',
40
    }),
41
  });
42

43
  return (
44
    <div className="flex flex-col w-full max-w-md py-24 mx-auto stretch">
45
      {messages.map(m => (
46
        <div key={m.id} className="whitespace-pre-wrap">
47
          {m.role === 'user' ? 'User: ' : 'AI: '}
48
          {m.parts.map((part, index) => {
49
            if (part.type === 'text') {
50
              return <span key={`${m.id}-text-${index}`}>{part.text}</span>;
51
            }
52
            if (part.type === 'file' && part.mediaType?.startsWith('image/')) {
53
              return (
54
                <Image
55
                  key={`${m.id}-image-${index}`}
56
                  src={part.url}
57
                  width={500}
58
                  height={500}
59
                  alt={`attachment-${index}`}
60
                />
61
              );
62
            }
63
            if (part.type === 'file' && part.mediaType === 'application/pdf') {
64
              return (
65
                <iframe
66
                  key={`${m.id}-pdf-${index}`}
67
                  src={part.url}
68
                  width={500}
69
                  height={600}
70
                  title={`pdf-${index}`}
71
                />
72
              );
73
            }
74
            return null;
75
          })}
76
        </div>
77
      ))}
78

79
      <form
80
        className="fixed bottom-0 w-full max-w-md p-2 mb-8 border border-gray-300 rounded shadow-xl space-y-2"
81
        onSubmit={async event => {
82
          event.preventDefault();
83

84
          const fileParts =
85
            files && files.length > 0
86
              ? await convertFilesToDataURLs(files)
87
              : [];
88

89
          sendMessage({
90
            role: 'user',
91
            parts: [{ type: 'text', text: input }, ...fileParts],
92
          });
93

94
          setInput('');
95
          setFiles(undefined);
96

97
          if (fileInputRef.current) {
98
            fileInputRef.current.value = '';
99
          }
100
        }}
101
      >
102
        <input
103
          type="file"
104
          accept="image/*,application/pdf"
105
          className=""
106
          onChange={event => {
107
            if (event.target.files) {
108
              setFiles(event.target.files);
109
            }
110
          }}
111
          multiple
112
          ref={fileInputRef}
113
        />
114
        <input
115
          className="w-full p-2"
116
          value={input}
117
          placeholder="Say something..."
118
          onChange={e => setInput(e.target.value)}
119
        />
120
      </form>
121
    </div>
122
  );
123
}

1
// Using Anthropic
2
const result = streamText({
3
  model: 'anthropic/claude-sonnet-4-20250514',
4
  messages: await convertToModelMessages(messages),
5
});
6

7
// Using Google
8
const result = streamText({
9
  model: 'google/gemini-2.5-flash',
10
  messages: await convertToModelMessages(messages),
11
});