bugfix: fix passing Image type in messages for chat (#390)

---------

Co-authored-by: Aarni Koskela <akx@iki.fi>

README.md

@@ -37,9 +37,6 @@ See [_types.py](ollama/_types.py) for more information on the response types.
 
 Response streaming can be enabled by setting `stream=True`.
 
-> [!NOTE]
-> Streaming Tool/Function calling is not yet supported.
-
 ```python
 from ollama import chat
 

examples/README.md

@@ -30,6 +30,12 @@ python3 examples/<example>.py
 - [multimodal_generate.py](multimodal_generate.py)
 
 
+### Structured Outputs - Generate structured outputs with a model
+- [structured-outputs.py](structured-outputs.py)
+- [async-structured-outputs.py](async-structured-outputs.py)
+- [structured-outputs-image.py](structured-outputs-image.py)
+
+
 ### Ollama List - List all downloaded models and their properties
 - [list.py](list.py)
 

examples/async-structured-outputs.py

@@ -0,0 +1,32 @@
+from pydantic import BaseModel
+from ollama import AsyncClient
+import asyncio
+
+
+# Define the schema for the response
+class FriendInfo(BaseModel):
+  name: str
+  age: int
+  is_available: bool
+
+
+class FriendList(BaseModel):
+  friends: list[FriendInfo]
+
+
+async def main():
+  client = AsyncClient()
+  response = await client.chat(
+    model='llama3.1:8b',
+    messages=[{'role': 'user', 'content': 'I have two friends. The first is Ollama 22 years old busy saving the world, and the second is Alonso 23 years old and wants to hang out. Return a list of friends in JSON format'}],
+    format=FriendList.model_json_schema(),  # Use Pydantic to generate the schema
+    options={'temperature': 0},  # Make responses more deterministic
+  )
+
+  # Use Pydantic to validate the response
+  friends_response = FriendList.model_validate_json(response.message.content)
+  print(friends_response)
+
+
+if __name__ == '__main__':
+  asyncio.run(main())

examples/async-tools.py

@@ -41,21 +41,21 @@ subtract_two_numbers_tool = {
   },
 }
 
+messages = [{'role': 'user', 'content': 'What is three plus one?'}]
+print('Prompt:', messages[0]['content'])
+
+available_functions = {
+  'add_two_numbers': add_two_numbers,
+  'subtract_two_numbers': subtract_two_numbers,
+}
+
 
 async def main():
   client = ollama.AsyncClient()
 
-  prompt = 'What is three plus one?'
-  print('Prompt:', prompt)
-
-  available_functions = {
-    'add_two_numbers': add_two_numbers,
-    'subtract_two_numbers': subtract_two_numbers,
-  }
-
   response: ChatResponse = await client.chat(
     'llama3.1',
-    messages=[{'role': 'user', 'content': prompt}],
+    messages=messages,
     tools=[add_two_numbers, subtract_two_numbers_tool],
   )
 
@@ -66,10 +66,24 @@ async def main():
       if function_to_call := available_functions.get(tool.function.name):
         print('Calling function:', tool.function.name)
         print('Arguments:', tool.function.arguments)
-        print('Function output:', function_to_call(**tool.function.arguments))
+        output = function_to_call(**tool.function.arguments)
+        print('Function output:', output)
       else:
         print('Function', tool.function.name, 'not found')
 
+  # Only needed to chat with the model using the tool call results
+  if response.message.tool_calls:
+    # Add the function response to messages for the model to use
+    messages.append(response.message)
+    messages.append({'role': 'tool', 'content': str(output), 'name': tool.function.name})
+
+    # Get final response from model with function outputs
+    final_response = await client.chat('llama3.1', messages=messages)
+    print('Final response:', final_response.message.content)
+
+  else:
+    print('No tool calls returned from model')
+
 
 if __name__ == '__main__':
   try:

examples/structured-outputs-image.py

@@ -0,0 +1,49 @@
+from pathlib import Path
+from pydantic import BaseModel
+from typing import Literal
+from ollama import chat
+
+
+# Define the schema for image objects
+class Object(BaseModel):
+  name: str
+  confidence: float
+  attributes: str
+
+
+class ImageDescription(BaseModel):
+  summary: str
+  objects: list[Object]
+  scene: str
+  colors: list[str]
+  time_of_day: Literal['Morning', 'Afternoon', 'Evening', 'Night']
+  setting: Literal['Indoor', 'Outdoor', 'Unknown']
+  text_content: str | None = None
+
+
+# Get path from user input
+path = input('Enter the path to your image: ')
+path = Path(path)
+
+# Verify the file exists
+if not path.exists():
+  raise FileNotFoundError(f'Image not found at: {path}')
+
+# Set up chat as usual
+response = chat(
+  model='llama3.2-vision',
+  format=ImageDescription.model_json_schema(),  # Pass in the schema for the response
+  messages=[
+    {
+      'role': 'user',
+      'content': 'Analyze this image and return a detailed JSON description including objects, scene, colors and any text detected. If you cannot determine certain details, leave those fields empty.',
+      'images': [path],
+    },
+  ],
+  options={'temperature': 0},  # Set temperature to 0 for more deterministic output
+)
+
+
+# Convert received content to the schema
+image_analysis = ImageDescription.model_validate_json(response.message.content)
+print(image_analysis)

examples/structured-outputs.py

@@ -0,0 +1,26 @@
+from ollama import chat
+from pydantic import BaseModel
+
+
+# Define the schema for the response
+class FriendInfo(BaseModel):
+  name: str
+  age: int
+  is_available: bool
+
+
+class FriendList(BaseModel):
+  friends: list[FriendInfo]
+
+
+# schema = {'type': 'object', 'properties': {'friends': {'type': 'array', 'items': {'type': 'object', 'properties': {'name': {'type': 'string'}, 'age': {'type': 'integer'}, 'is_available': {'type': 'boolean'}}, 'required': ['name', 'age', 'is_available']}}}, 'required': ['friends']}
+response = chat(
+  model='llama3.1:8b',
+  messages=[{'role': 'user', 'content': 'I have two friends. The first is Ollama 22 years old busy saving the world, and the second is Alonso 23 years old and wants to hang out. Return a list of friends in JSON format'}],
+  format=FriendList.model_json_schema(),  # Use Pydantic to generate the schema or format=schema
+  options={'temperature': 0},  # Make responses more deterministic
+)
+
+# Use Pydantic to validate the response
+friends_response = FriendList.model_validate_json(response.message.content)
+print(friends_response)

examples/tools.py

@@ -40,8 +40,8 @@ subtract_two_numbers_tool = {
   },
 }
 
-prompt = 'What is three plus one?'
-print('Prompt:', prompt)
+messages = [{'role': 'user', 'content': 'What is three plus one?'}]
+print('Prompt:', messages[0]['content'])
 
 available_functions = {
   'add_two_numbers': add_two_numbers,
@@ -50,7 +50,7 @@ available_functions = {
 
 response: ChatResponse = chat(
   'llama3.1',
-  messages=[{'role': 'user', 'content': prompt}],
+  messages=messages,
   tools=[add_two_numbers, subtract_two_numbers_tool],
 )
 
@@ -61,6 +61,20 @@ if response.message.tool_calls:
     if function_to_call := available_functions.get(tool.function.name):
       print('Calling function:', tool.function.name)
       print('Arguments:', tool.function.arguments)
-      print('Function output:', function_to_call(**tool.function.arguments))
+      output = function_to_call(**tool.function.arguments)
+      print('Function output:', output)
     else:
       print('Function', tool.function.name, 'not found')
+
+# Only needed to chat with the model using the tool call results
+if response.message.tool_calls:
+  # Add the function response to messages for the model to use
+  messages.append(response.message)
+  messages.append({'role': 'tool', 'content': str(output), 'name': tool.function.name})
+
+  # Get final response from model with function outputs
+  final_response = chat('llama3.1', messages=messages)
+  print('Final response:', final_response.message.content)
+
+else:
+  print('No tool calls returned from model')

ollama/__init__.py

@@ -2,6 +2,7 @@ from ollama._client import Client, AsyncClient
 from ollama._types import (
   Options,
   Message,
+  Image,
   Tool,
   GenerateResponse,
   ChatResponse,
@@ -21,6 +22,7 @@ __all__ = [
   'AsyncClient',
   'Options',
   'Message',
+  'Image',
   'Tool',
   'GenerateResponse',
   'ChatResponse',

ollama/_client.py

@@ -23,6 +23,8 @@ from typing import (
 
 import sys
 
+from pydantic.json_schema import JsonSchemaValue
+
 
 from ollama._utils import convert_function_to_tool
 
@@ -186,7 +188,7 @@ class Client(BaseClient):
     context: Optional[Sequence[int]] = None,
     stream: Literal[False] = False,
     raw: bool = False,
-    format: Optional[Literal['', 'json']] = None,
+    format: Optional[Union[Literal['', 'json'], JsonSchemaValue]] = None,
     images: Optional[Sequence[Union[str, bytes]]] = None,
     options: Optional[Union[Mapping[str, Any], Options]] = None,
     keep_alive: Optional[Union[float, str]] = None,
@@ -204,7 +206,7 @@ class Client(BaseClient):
     context: Optional[Sequence[int]] = None,
     stream: Literal[True] = True,
     raw: bool = False,
-    format: Optional[Literal['', 'json']] = None,
+    format: Optional[Union[Literal['', 'json'], JsonSchemaValue]] = None,
     images: Optional[Sequence[Union[str, bytes]]] = None,
     options: Optional[Union[Mapping[str, Any], Options]] = None,
     keep_alive: Optional[Union[float, str]] = None,
@@ -221,7 +223,7 @@ class Client(BaseClient):
     context: Optional[Sequence[int]] = None,
     stream: bool = False,
     raw: Optional[bool] = None,
-    format: Optional[Literal['', 'json']] = None,
+    format: Optional[Union[Literal['', 'json'], JsonSchemaValue]] = None,
     images: Optional[Sequence[Union[str, bytes]]] = None,
     options: Optional[Union[Mapping[str, Any], Options]] = None,
     keep_alive: Optional[Union[float, str]] = None,
@@ -250,7 +252,7 @@ class Client(BaseClient):
         stream=stream,
         raw=raw,
         format=format,
-        images=[Image(value=image) for image in images] if images else None,
+        images=[image for image in _copy_images(images)] if images else None,
         options=options,
         keep_alive=keep_alive,
       ).model_dump(exclude_none=True),
@@ -265,7 +267,7 @@ class Client(BaseClient):
     *,
     tools: Optional[Sequence[Union[Mapping[str, Any], Tool, Callable]]] = None,
     stream: Literal[False] = False,
-    format: Optional[Literal['', 'json']] = None,
+    format: Optional[Union[Literal['', 'json'], JsonSchemaValue]] = None,
     options: Optional[Union[Mapping[str, Any], Options]] = None,
     keep_alive: Optional[Union[float, str]] = None,
   ) -> ChatResponse: ...
@@ -278,7 +280,7 @@ class Client(BaseClient):
     *,
     tools: Optional[Sequence[Union[Mapping[str, Any], Tool, Callable]]] = None,
     stream: Literal[True] = True,
-    format: Optional[Literal['', 'json']] = None,
+    format: Optional[Union[Literal['', 'json'], JsonSchemaValue]] = None,
     options: Optional[Union[Mapping[str, Any], Options]] = None,
     keep_alive: Optional[Union[float, str]] = None,
   ) -> Iterator[ChatResponse]: ...
@@ -290,7 +292,7 @@ class Client(BaseClient):
     *,
     tools: Optional[Sequence[Union[Mapping[str, Any], Tool, Callable]]] = None,
     stream: bool = False,
-    format: Optional[Literal['', 'json']] = None,
+    format: Optional[Union[Literal['', 'json'], JsonSchemaValue]] = None,
     options: Optional[Union[Mapping[str, Any], Options]] = None,
     keep_alive: Optional[Union[float, str]] = None,
   ) -> Union[ChatResponse, Iterator[ChatResponse]]:
@@ -327,7 +329,6 @@ class Client(BaseClient):
 
     Returns `ChatResponse` if `stream` is `False`, otherwise returns a `ChatResponse` generator.
     """
-
     return self._request(
       ChatResponse,
       'POST',
@@ -689,7 +690,7 @@ class AsyncClient(BaseClient):
     context: Optional[Sequence[int]] = None,
     stream: Literal[False] = False,
     raw: bool = False,
-    format: Optional[Literal['', 'json']] = None,
+    format: Optional[Union[Literal['', 'json'], JsonSchemaValue]] = None,
     images: Optional[Sequence[Union[str, bytes]]] = None,
     options: Optional[Union[Mapping[str, Any], Options]] = None,
     keep_alive: Optional[Union[float, str]] = None,
@@ -707,7 +708,7 @@ class AsyncClient(BaseClient):
     context: Optional[Sequence[int]] = None,
     stream: Literal[True] = True,
     raw: bool = False,
-    format: Optional[Literal['', 'json']] = None,
+    format: Optional[Union[Literal['', 'json'], JsonSchemaValue]] = None,
     images: Optional[Sequence[Union[str, bytes]]] = None,
     options: Optional[Union[Mapping[str, Any], Options]] = None,
     keep_alive: Optional[Union[float, str]] = None,
@@ -724,7 +725,7 @@ class AsyncClient(BaseClient):
     context: Optional[Sequence[int]] = None,
     stream: bool = False,
     raw: Optional[bool] = None,
-    format: Optional[Literal['', 'json']] = None,
+    format: Optional[Union[Literal['', 'json'], JsonSchemaValue]] = None,
     images: Optional[Sequence[Union[str, bytes]]] = None,
     options: Optional[Union[Mapping[str, Any], Options]] = None,
     keep_alive: Optional[Union[float, str]] = None,
@@ -752,7 +753,7 @@ class AsyncClient(BaseClient):
         stream=stream,
         raw=raw,
         format=format,
-        images=[Image(value=image) for image in images] if images else None,
+        images=[image for image in _copy_images(images)] if images else None,
         options=options,
         keep_alive=keep_alive,
       ).model_dump(exclude_none=True),
@@ -767,7 +768,7 @@ class AsyncClient(BaseClient):
     *,
     tools: Optional[Sequence[Union[Mapping[str, Any], Tool, Callable]]] = None,
     stream: Literal[False] = False,
-    format: Optional[Literal['', 'json']] = None,
+    format: Optional[Union[Literal['', 'json'], JsonSchemaValue]] = None,
     options: Optional[Union[Mapping[str, Any], Options]] = None,
     keep_alive: Optional[Union[float, str]] = None,
   ) -> ChatResponse: ...
@@ -780,7 +781,7 @@ class AsyncClient(BaseClient):
     *,
     tools: Optional[Sequence[Union[Mapping[str, Any], Tool, Callable]]] = None,
     stream: Literal[True] = True,
-    format: Optional[Literal['', 'json']] = None,
+    format: Optional[Union[Literal['', 'json'], JsonSchemaValue]] = None,
     options: Optional[Union[Mapping[str, Any], Options]] = None,
     keep_alive: Optional[Union[float, str]] = None,
   ) -> AsyncIterator[ChatResponse]: ...
@@ -792,7 +793,7 @@ class AsyncClient(BaseClient):
     *,
     tools: Optional[Sequence[Union[Mapping[str, Any], Tool, Callable]]] = None,
     stream: bool = False,
-    format: Optional[Literal['', 'json']] = None,
+    format: Optional[Union[Literal['', 'json'], JsonSchemaValue]] = None,
     options: Optional[Union[Mapping[str, Any], Options]] = None,
     keep_alive: Optional[Union[float, str]] = None,
   ) -> Union[ChatResponse, AsyncIterator[ChatResponse]]:
@@ -1120,10 +1121,15 @@ class AsyncClient(BaseClient):
     )
 
 
+def _copy_images(images: Optional[Sequence[Union[Image, Any]]]) -> Iterator[Image]:
+  for image in images or []:
+    yield image if isinstance(image, Image) else Image(value=image)
+
+
 def _copy_messages(messages: Optional[Sequence[Union[Mapping[str, Any], Message]]]) -> Iterator[Message]:
   for message in messages or []:
     yield Message.model_validate(
-      {k: [Image(value=image) for image in v] if k == 'images' else v for k, v in dict(message).items() if v},
+      {k: [image for image in _copy_images(v)] if k == 'images' else v for k, v in dict(message).items() if v},
     )
 
 

ollama/_types.py

@@ -4,6 +4,7 @@ from pathlib import Path
 from datetime import datetime
 from typing import Any, Mapping, Optional, Union, Sequence
 
+from pydantic.json_schema import JsonSchemaValue
 from typing_extensions import Annotated, Literal
 
 from pydantic import (
@@ -150,7 +151,7 @@ class BaseGenerateRequest(BaseStreamableRequest):
   options: Optional[Union[Mapping[str, Any], Options]] = None
   'Options to use for the request.'
 
-  format: Optional[Literal['', 'json']] = None
+  format: Optional[Union[Literal['', 'json'], JsonSchemaValue]] = None
   'Format of the response.'
 
   keep_alive: Optional[Union[float, str]] = None

poetry.lock

@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.8.5 and should not be changed by hand.
 
 [[package]]
 name = "annotated-types"
@@ -321,103 +321,6 @@ files = [
     {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"},
 ]
 
-[[package]]
-name = "pillow"
-version = "10.4.0"
-description = "Python Imaging Library (Fork)"
-optional = false
-python-versions = ">=3.8"
-files = [
-    {file = "pillow-10.4.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:4d9667937cfa347525b319ae34375c37b9ee6b525440f3ef48542fcf66f2731e"},
-    {file = "pillow-10.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:543f3dc61c18dafb755773efc89aae60d06b6596a63914107f75459cf984164d"},
-    {file = "pillow-10.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7928ecbf1ece13956b95d9cbcfc77137652b02763ba384d9ab508099a2eca856"},
-    {file = "pillow-10.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4d49b85c4348ea0b31ea63bc75a9f3857869174e2bf17e7aba02945cd218e6f"},
-    {file = "pillow-10.4.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:6c762a5b0997f5659a5ef2266abc1d8851ad7749ad9a6a5506eb23d314e4f46b"},
-    {file = "pillow-10.4.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:a985e028fc183bf12a77a8bbf36318db4238a3ded7fa9df1b9a133f1cb79f8fc"},
-    {file = "pillow-10.4.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:812f7342b0eee081eaec84d91423d1b4650bb9828eb53d8511bcef8ce5aecf1e"},
-    {file = "pillow-10.4.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ac1452d2fbe4978c2eec89fb5a23b8387aba707ac72810d9490118817d9c0b46"},
-    {file = "pillow-10.4.0-cp310-cp310-win32.whl", hash = "sha256:bcd5e41a859bf2e84fdc42f4edb7d9aba0a13d29a2abadccafad99de3feff984"},
-    {file = "pillow-10.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:ecd85a8d3e79cd7158dec1c9e5808e821feea088e2f69a974db5edf84dc53141"},
-    {file = "pillow-10.4.0-cp310-cp310-win_arm64.whl", hash = "sha256:ff337c552345e95702c5fde3158acb0625111017d0e5f24bf3acdb9cc16b90d1"},
-    {file = "pillow-10.4.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:0a9ec697746f268507404647e531e92889890a087e03681a3606d9b920fbee3c"},
-    {file = "pillow-10.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:dfe91cb65544a1321e631e696759491ae04a2ea11d36715eca01ce07284738be"},
-    {file = "pillow-10.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5dc6761a6efc781e6a1544206f22c80c3af4c8cf461206d46a1e6006e4429ff3"},
-    {file = "pillow-10.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e84b6cc6a4a3d76c153a6b19270b3526a5a8ed6b09501d3af891daa2a9de7d6"},
-    {file = "pillow-10.4.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:bbc527b519bd3aa9d7f429d152fea69f9ad37c95f0b02aebddff592688998abe"},
-    {file = "pillow-10.4.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:76a911dfe51a36041f2e756b00f96ed84677cdeb75d25c767f296c1c1eda1319"},
-    {file = "pillow-10.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:59291fb29317122398786c2d44427bbd1a6d7ff54017075b22be9d21aa59bd8d"},
-    {file = "pillow-10.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:416d3a5d0e8cfe4f27f574362435bc9bae57f679a7158e0096ad2beb427b8696"},
-    {file = "pillow-10.4.0-cp311-cp311-win32.whl", hash = "sha256:7086cc1d5eebb91ad24ded9f58bec6c688e9f0ed7eb3dbbf1e4800280a896496"},
-    {file = "pillow-10.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:cbed61494057c0f83b83eb3a310f0bf774b09513307c434d4366ed64f4128a91"},
-    {file = "pillow-10.4.0-cp311-cp311-win_arm64.whl", hash = "sha256:f5f0c3e969c8f12dd2bb7e0b15d5c468b51e5017e01e2e867335c81903046a22"},
-    {file = "pillow-10.4.0-cp312-cp312-macosx_10_10_x86_64.whl", hash = "sha256:673655af3eadf4df6b5457033f086e90299fdd7a47983a13827acf7459c15d94"},
-    {file = "pillow-10.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:866b6942a92f56300012f5fbac71f2d610312ee65e22f1aa2609e491284e5597"},
-    {file = "pillow-10.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29dbdc4207642ea6aad70fbde1a9338753d33fb23ed6956e706936706f52dd80"},
-    {file = "pillow-10.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf2342ac639c4cf38799a44950bbc2dfcb685f052b9e262f446482afaf4bffca"},
-    {file = "pillow-10.4.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:f5b92f4d70791b4a67157321c4e8225d60b119c5cc9aee8ecf153aace4aad4ef"},
-    {file = "pillow-10.4.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:86dcb5a1eb778d8b25659d5e4341269e8590ad6b4e8b44d9f4b07f8d136c414a"},
-    {file = "pillow-10.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:780c072c2e11c9b2c7ca37f9a2ee8ba66f44367ac3e5c7832afcfe5104fd6d1b"},
-    {file = "pillow-10.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:37fb69d905be665f68f28a8bba3c6d3223c8efe1edf14cc4cfa06c241f8c81d9"},
-    {file = "pillow-10.4.0-cp312-cp312-win32.whl", hash = "sha256:7dfecdbad5c301d7b5bde160150b4db4c659cee2b69589705b6f8a0c509d9f42"},
-    {file = "pillow-10.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:1d846aea995ad352d4bdcc847535bd56e0fd88d36829d2c90be880ef1ee4668a"},
-    {file = "pillow-10.4.0-cp312-cp312-win_arm64.whl", hash = "sha256:e553cad5179a66ba15bb18b353a19020e73a7921296a7979c4a2b7f6a5cd57f9"},
-    {file = "pillow-10.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8bc1a764ed8c957a2e9cacf97c8b2b053b70307cf2996aafd70e91a082e70df3"},
-    {file = "pillow-10.4.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6209bb41dc692ddfee4942517c19ee81b86c864b626dbfca272ec0f7cff5d9fb"},
-    {file = "pillow-10.4.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bee197b30783295d2eb680b311af15a20a8b24024a19c3a26431ff83eb8d1f70"},
-    {file = "pillow-10.4.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ef61f5dd14c300786318482456481463b9d6b91ebe5ef12f405afbba77ed0be"},
-    {file = "pillow-10.4.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:297e388da6e248c98bc4a02e018966af0c5f92dfacf5a5ca22fa01cb3179bca0"},
-    {file = "pillow-10.4.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:e4db64794ccdf6cb83a59d73405f63adbe2a1887012e308828596100a0b2f6cc"},
-    {file = "pillow-10.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bd2880a07482090a3bcb01f4265f1936a903d70bc740bfcb1fd4e8a2ffe5cf5a"},
-    {file = "pillow-10.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4b35b21b819ac1dbd1233317adeecd63495f6babf21b7b2512d244ff6c6ce309"},
-    {file = "pillow-10.4.0-cp313-cp313-win32.whl", hash = "sha256:551d3fd6e9dc15e4c1eb6fc4ba2b39c0c7933fa113b220057a34f4bb3268a060"},
-    {file = "pillow-10.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:030abdbe43ee02e0de642aee345efa443740aa4d828bfe8e2eb11922ea6a21ea"},
-    {file = "pillow-10.4.0-cp313-cp313-win_arm64.whl", hash = "sha256:5b001114dd152cfd6b23befeb28d7aee43553e2402c9f159807bf55f33af8a8d"},
-    {file = "pillow-10.4.0-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:8d4d5063501b6dd4024b8ac2f04962d661222d120381272deea52e3fc52d3736"},
-    {file = "pillow-10.4.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7c1ee6f42250df403c5f103cbd2768a28fe1a0ea1f0f03fe151c8741e1469c8b"},
-    {file = "pillow-10.4.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b15e02e9bb4c21e39876698abf233c8c579127986f8207200bc8a8f6bb27acf2"},
-    {file = "pillow-10.4.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a8d4bade9952ea9a77d0c3e49cbd8b2890a399422258a77f357b9cc9be8d680"},
-    {file = "pillow-10.4.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:43efea75eb06b95d1631cb784aa40156177bf9dd5b4b03ff38979e048258bc6b"},
-    {file = "pillow-10.4.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:950be4d8ba92aca4b2bb0741285a46bfae3ca699ef913ec8416c1b78eadd64cd"},
-    {file = "pillow-10.4.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:d7480af14364494365e89d6fddc510a13e5a2c3584cb19ef65415ca57252fb84"},
-    {file = "pillow-10.4.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:73664fe514b34c8f02452ffb73b7a92c6774e39a647087f83d67f010eb9a0cf0"},
-    {file = "pillow-10.4.0-cp38-cp38-win32.whl", hash = "sha256:e88d5e6ad0d026fba7bdab8c3f225a69f063f116462c49892b0149e21b6c0a0e"},
-    {file = "pillow-10.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:5161eef006d335e46895297f642341111945e2c1c899eb406882a6c61a4357ab"},
-    {file = "pillow-10.4.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:0ae24a547e8b711ccaaf99c9ae3cd975470e1a30caa80a6aaee9a2f19c05701d"},
-    {file = "pillow-10.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:298478fe4f77a4408895605f3482b6cc6222c018b2ce565c2b6b9c354ac3229b"},
-    {file = "pillow-10.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:134ace6dc392116566980ee7436477d844520a26a4b1bd4053f6f47d096997fd"},
-    {file = "pillow-10.4.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:930044bb7679ab003b14023138b50181899da3f25de50e9dbee23b61b4de2126"},
-    {file = "pillow-10.4.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:c76e5786951e72ed3686e122d14c5d7012f16c8303a674d18cdcd6d89557fc5b"},
-    {file = "pillow-10.4.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:b2724fdb354a868ddf9a880cb84d102da914e99119211ef7ecbdc613b8c96b3c"},
-    {file = "pillow-10.4.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:dbc6ae66518ab3c5847659e9988c3b60dc94ffb48ef9168656e0019a93dbf8a1"},
-    {file = "pillow-10.4.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:06b2f7898047ae93fad74467ec3d28fe84f7831370e3c258afa533f81ef7f3df"},
-    {file = "pillow-10.4.0-cp39-cp39-win32.whl", hash = "sha256:7970285ab628a3779aecc35823296a7869f889b8329c16ad5a71e4901a3dc4ef"},
-    {file = "pillow-10.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:961a7293b2457b405967af9c77dcaa43cc1a8cd50d23c532e62d48ab6cdd56f5"},
-    {file = "pillow-10.4.0-cp39-cp39-win_arm64.whl", hash = "sha256:32cda9e3d601a52baccb2856b8ea1fc213c90b340c542dcef77140dfa3278a9e"},
-    {file = "pillow-10.4.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:5b4815f2e65b30f5fbae9dfffa8636d992d49705723fe86a3661806e069352d4"},
-    {file = "pillow-10.4.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:8f0aef4ef59694b12cadee839e2ba6afeab89c0f39a3adc02ed51d109117b8da"},
-    {file = "pillow-10.4.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9f4727572e2918acaa9077c919cbbeb73bd2b3ebcfe033b72f858fc9fbef0026"},
-    {file = "pillow-10.4.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ff25afb18123cea58a591ea0244b92eb1e61a1fd497bf6d6384f09bc3262ec3e"},
-    {file = "pillow-10.4.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:dc3e2db6ba09ffd7d02ae9141cfa0ae23393ee7687248d46a7507b75d610f4f5"},
-    {file = "pillow-10.4.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:02a2be69f9c9b8c1e97cf2713e789d4e398c751ecfd9967c18d0ce304efbf885"},
-    {file = "pillow-10.4.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:0755ffd4a0c6f267cccbae2e9903d95477ca2f77c4fcf3a3a09570001856c8a5"},
-    {file = "pillow-10.4.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:a02364621fe369e06200d4a16558e056fe2805d3468350df3aef21e00d26214b"},
-    {file = "pillow-10.4.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:1b5dea9831a90e9d0721ec417a80d4cbd7022093ac38a568db2dd78363b00908"},
-    {file = "pillow-10.4.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9b885f89040bb8c4a1573566bbb2f44f5c505ef6e74cec7ab9068c900047f04b"},
-    {file = "pillow-10.4.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87dd88ded2e6d74d31e1e0a99a726a6765cda32d00ba72dc37f0651f306daaa8"},
-    {file = "pillow-10.4.0-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:2db98790afc70118bd0255c2eeb465e9767ecf1f3c25f9a1abb8ffc8cfd1fe0a"},
-    {file = "pillow-10.4.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:f7baece4ce06bade126fb84b8af1c33439a76d8a6fd818970215e0560ca28c27"},
-    {file = "pillow-10.4.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:cfdd747216947628af7b259d274771d84db2268ca062dd5faf373639d00113a3"},
-    {file = "pillow-10.4.0.tar.gz", hash = "sha256:166c1cd4d24309b30d61f79f4a9114b7b2313d7450912277855ff5dfd7cd4a06"},
-]
-
-[package.extras]
-docs = ["furo", "olefile", "sphinx (>=7.3)", "sphinx-copybutton", "sphinx-inline-tabs", "sphinxext-opengraph"]
-fpx = ["olefile"]
-mic = ["olefile"]
-tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout"]
-typing = ["typing-extensions"]
-xmp = ["defusedxml"]
-
 [[package]]
 name = "pluggy"
 version = "1.5.0"
@@ -559,13 +462,13 @@ typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0"
 
 [[package]]
 name = "pytest"
-version = "8.3.3"
+version = "8.3.4"
 description = "pytest: simple powerful testing with Python"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "pytest-8.3.3-py3-none-any.whl", hash = "sha256:a6853c7375b2663155079443d2e45de913a911a11d669df02a50814944db57b2"},
-    {file = "pytest-8.3.3.tar.gz", hash = "sha256:70b98107bd648308a7952b06e6ca9a50bc660be218d53c257cc1fc94fda10181"},
+    {file = "pytest-8.3.4-py3-none-any.whl", hash = "sha256:50e16d954148559c9a74109af1eaf0c945ba2d8f30f0a3d3335edde19788b6f6"},
+    {file = "pytest-8.3.4.tar.gz", hash = "sha256:965370d062bce11e73868e0335abac31b4d3de0e82f4007408d242b4f8610761"},
 ]
 
 [package.dependencies]
@@ -709,4 +612,4 @@ watchdog = ["watchdog (>=2.3)"]
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.8"
-content-hash = "61443e0ce98d3e24a45da6f9c890699fc44fe98cd191b0eb38e6b59093e8149d"
+content-hash = "8e93767305535b0a02f0d724edf1249fd928ff1021644eb9dc26dbfa191f6971"

pyproject.toml

@@ -18,7 +18,6 @@ pytest = ">=7.4.3,<9.0.0"
 pytest-asyncio = ">=0.23.2,<0.25.0"
 pytest-cov = ">=4.1,<6.0"
 pytest-httpserver = "^1.0.8"
-pillow = "^10.2.0"
 ruff = ">=0.1.8,<0.8.0"
 
 [build-system]

tests/test_client.py

@@ -1,16 +1,18 @@
+import base64
 import os
-import io
 import json
-from pydantic import ValidationError
+from pydantic import ValidationError, BaseModel
 import pytest
 import tempfile
 from pathlib import Path
 from pytest_httpserver import HTTPServer, URIPattern
 from werkzeug.wrappers import Request, Response
-from PIL import Image
 
 from ollama._client import Client, AsyncClient, _copy_tools
 
+PNG_BASE64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGNgYGAAAAAEAAH2FzhVAAAAAElFTkSuQmCC'
+PNG_BYTES = base64.b64decode(PNG_BASE64)
+
 
 class PrefixPattern(URIPattern):
   def __init__(self, prefix: str):
@@ -86,7 +88,11 @@ def test_client_chat_stream(httpserver: HTTPServer):
     assert part['message']['content'] == next(it)
 
 
-def test_client_chat_images(httpserver: HTTPServer):
+@pytest.mark.parametrize('message_format', ('dict', 'pydantic_model'))
+@pytest.mark.parametrize('file_style', ('path', 'bytes'))
+def test_client_chat_images(httpserver: HTTPServer, message_format: str, file_style: str, tmp_path):
+  from ollama._types import Message, Image
+
   httpserver.expect_ordered_request(
     '/api/chat',
     method='POST',
@@ -96,7 +102,7 @@ def test_client_chat_images(httpserver: HTTPServer):
         {
           'role': 'user',
           'content': 'Why is the sky blue?',
-          'images': ['iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGNgYGAAAAAEAAH2FzhVAAAAAElFTkSuQmCC'],
+          'images': [PNG_BASE64],
         },
       ],
       'tools': [],
@@ -114,12 +120,146 @@ def test_client_chat_images(httpserver: HTTPServer):
 
   client = Client(httpserver.url_for('/'))
 
-  with io.BytesIO() as b:
-    Image.new('RGB', (1, 1)).save(b, 'PNG')
-    response = client.chat('dummy', messages=[{'role': 'user', 'content': 'Why is the sky blue?', 'images': [b.getvalue()]}])
-    assert response['model'] == 'dummy'
-    assert response['message']['role'] == 'assistant'
-    assert response['message']['content'] == "I don't know."
+  if file_style == 'bytes':
+    image_content = PNG_BYTES
+  elif file_style == 'path':
+    image_path = tmp_path / 'transparent.png'
+    image_path.write_bytes(PNG_BYTES)
+    image_content = str(image_path)
+
+  if message_format == 'pydantic_model':
+    messages = [Message(role='user', content='Why is the sky blue?', images=[Image(value=image_content)])]
+  elif message_format == 'dict':
+    messages = [{'role': 'user', 'content': 'Why is the sky blue?', 'images': [image_content]}]
+  else:
+    raise ValueError(f'Invalid message format: {message_format}')
+
+  response = client.chat('dummy', messages=messages)
+  assert response['model'] == 'dummy'
+  assert response['message']['role'] == 'assistant'
+  assert response['message']['content'] == "I don't know."
+
+
+def test_client_chat_format_json(httpserver: HTTPServer):
+  httpserver.expect_ordered_request(
+    '/api/chat',
+    method='POST',
+    json={
+      'model': 'dummy',
+      'messages': [{'role': 'user', 'content': 'Why is the sky blue?'}],
+      'tools': [],
+      'format': 'json',
+      'stream': False,
+    },
+  ).respond_with_json(
+    {
+      'model': 'dummy',
+      'message': {
+        'role': 'assistant',
+        'content': '{"answer": "Because of Rayleigh scattering"}',
+      },
+    }
+  )
+
+  client = Client(httpserver.url_for('/'))
+  response = client.chat('dummy', messages=[{'role': 'user', 'content': 'Why is the sky blue?'}], format='json')
+  assert response['model'] == 'dummy'
+  assert response['message']['role'] == 'assistant'
+  assert response['message']['content'] == '{"answer": "Because of Rayleigh scattering"}'
+
+
+def test_client_chat_format_pydantic(httpserver: HTTPServer):
+  class ResponseFormat(BaseModel):
+    answer: str
+    confidence: float
+
+  httpserver.expect_ordered_request(
+    '/api/chat',
+    method='POST',
+    json={
+      'model': 'dummy',
+      'messages': [{'role': 'user', 'content': 'Why is the sky blue?'}],
+      'tools': [],
+      'format': {'title': 'ResponseFormat', 'type': 'object', 'properties': {'answer': {'title': 'Answer', 'type': 'string'}, 'confidence': {'title': 'Confidence', 'type': 'number'}}, 'required': ['answer', 'confidence']},
+      'stream': False,
+    },
+  ).respond_with_json(
+    {
+      'model': 'dummy',
+      'message': {
+        'role': 'assistant',
+        'content': '{"answer": "Because of Rayleigh scattering", "confidence": 0.95}',
+      },
+    }
+  )
+
+  client = Client(httpserver.url_for('/'))
+  response = client.chat('dummy', messages=[{'role': 'user', 'content': 'Why is the sky blue?'}], format=ResponseFormat.model_json_schema())
+  assert response['model'] == 'dummy'
+  assert response['message']['role'] == 'assistant'
+  assert response['message']['content'] == '{"answer": "Because of Rayleigh scattering", "confidence": 0.95}'
+
+
+@pytest.mark.asyncio
+async def test_async_client_chat_format_json(httpserver: HTTPServer):
+  httpserver.expect_ordered_request(
+    '/api/chat',
+    method='POST',
+    json={
+      'model': 'dummy',
+      'messages': [{'role': 'user', 'content': 'Why is the sky blue?'}],
+      'tools': [],
+      'format': 'json',
+      'stream': False,
+    },
+  ).respond_with_json(
+    {
+      'model': 'dummy',
+      'message': {
+        'role': 'assistant',
+        'content': '{"answer": "Because of Rayleigh scattering"}',
+      },
+    }
+  )
+
+  client = AsyncClient(httpserver.url_for('/'))
+  response = await client.chat('dummy', messages=[{'role': 'user', 'content': 'Why is the sky blue?'}], format='json')
+  assert response['model'] == 'dummy'
+  assert response['message']['role'] == 'assistant'
+  assert response['message']['content'] == '{"answer": "Because of Rayleigh scattering"}'
+
+
+@pytest.mark.asyncio
+async def test_async_client_chat_format_pydantic(httpserver: HTTPServer):
+  class ResponseFormat(BaseModel):
+    answer: str
+    confidence: float
+
+  httpserver.expect_ordered_request(
+    '/api/chat',
+    method='POST',
+    json={
+      'model': 'dummy',
+      'messages': [{'role': 'user', 'content': 'Why is the sky blue?'}],
+      'tools': [],
+      'format': {'title': 'ResponseFormat', 'type': 'object', 'properties': {'answer': {'title': 'Answer', 'type': 'string'}, 'confidence': {'title': 'Confidence', 'type': 'number'}}, 'required': ['answer', 'confidence']},
+      'stream': False,
+    },
+  ).respond_with_json(
+    {
+      'model': 'dummy',
+      'message': {
+        'role': 'assistant',
+        'content': '{"answer": "Because of Rayleigh scattering", "confidence": 0.95}',
+      },
+    }
+  )
+
+  client = AsyncClient(httpserver.url_for('/'))
+  response = await client.chat('dummy', messages=[{'role': 'user', 'content': 'Why is the sky blue?'}], format=ResponseFormat.model_json_schema())
+  assert response['model'] == 'dummy'
+  assert response['message']['role'] == 'assistant'
+  assert response['message']['content'] == '{"answer": "Because of Rayleigh scattering", "confidence": 0.95}'
 
 
 def test_client_generate(httpserver: HTTPServer):
@@ -187,7 +327,7 @@ def test_client_generate_images(httpserver: HTTPServer):
       'model': 'dummy',
       'prompt': 'Why is the sky blue?',
       'stream': False,
-      'images': ['iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGNgYGAAAAAEAAH2FzhVAAAAAElFTkSuQmCC'],
+      'images': [PNG_BASE64],
     },
   ).respond_with_json(
     {
@@ -199,12 +339,115 @@ def test_client_generate_images(httpserver: HTTPServer):
   client = Client(httpserver.url_for('/'))
 
   with tempfile.NamedTemporaryFile() as temp:
-    Image.new('RGB', (1, 1)).save(temp, 'PNG')
+    temp.write(PNG_BYTES)
+    temp.flush()
     response = client.generate('dummy', 'Why is the sky blue?', images=[temp.name])
     assert response['model'] == 'dummy'
     assert response['response'] == 'Because it is.'
 
 
+def test_client_generate_format_json(httpserver: HTTPServer):
+  httpserver.expect_ordered_request(
+    '/api/generate',
+    method='POST',
+    json={
+      'model': 'dummy',
+      'prompt': 'Why is the sky blue?',
+      'format': 'json',
+      'stream': False,
+    },
+  ).respond_with_json(
+    {
+      'model': 'dummy',
+      'response': '{"answer": "Because of Rayleigh scattering"}',
+    }
+  )
+
+  client = Client(httpserver.url_for('/'))
+  response = client.generate('dummy', 'Why is the sky blue?', format='json')
+  assert response['model'] == 'dummy'
+  assert response['response'] == '{"answer": "Because of Rayleigh scattering"}'
+
+
+def test_client_generate_format_pydantic(httpserver: HTTPServer):
+  class ResponseFormat(BaseModel):
+    answer: str
+    confidence: float
+
+  httpserver.expect_ordered_request(
+    '/api/generate',
+    method='POST',
+    json={
+      'model': 'dummy',
+      'prompt': 'Why is the sky blue?',
+      'format': {'title': 'ResponseFormat', 'type': 'object', 'properties': {'answer': {'title': 'Answer', 'type': 'string'}, 'confidence': {'title': 'Confidence', 'type': 'number'}}, 'required': ['answer', 'confidence']},
+      'stream': False,
+    },
+  ).respond_with_json(
+    {
+      'model': 'dummy',
+      'response': '{"answer": "Because of Rayleigh scattering", "confidence": 0.95}',
+    }
+  )
+
+  client = Client(httpserver.url_for('/'))
+  response = client.generate('dummy', 'Why is the sky blue?', format=ResponseFormat.model_json_schema())
+  assert response['model'] == 'dummy'
+  assert response['response'] == '{"answer": "Because of Rayleigh scattering", "confidence": 0.95}'
+
+
+@pytest.mark.asyncio
+async def test_async_client_generate_format_json(httpserver: HTTPServer):
+  httpserver.expect_ordered_request(
+    '/api/generate',
+    method='POST',
+    json={
+      'model': 'dummy',
+      'prompt': 'Why is the sky blue?',
+      'format': 'json',
+      'stream': False,
+    },
+  ).respond_with_json(
+    {
+      'model': 'dummy',
+      'response': '{"answer": "Because of Rayleigh scattering"}',
+    }
+  )
+
+  client = AsyncClient(httpserver.url_for('/'))
+  response = await client.generate('dummy', 'Why is the sky blue?', format='json')
+  assert response['model'] == 'dummy'
+  assert response['response'] == '{"answer": "Because of Rayleigh scattering"}'
+
+
+@pytest.mark.asyncio
+async def test_async_client_generate_format_pydantic(httpserver: HTTPServer):
+  class ResponseFormat(BaseModel):
+    answer: str
+    confidence: float
+
+  httpserver.expect_ordered_request(
+    '/api/generate',
+    method='POST',
+    json={
+      'model': 'dummy',
+      'prompt': 'Why is the sky blue?',
+      'format': {'title': 'ResponseFormat', 'type': 'object', 'properties': {'answer': {'title': 'Answer', 'type': 'string'}, 'confidence': {'title': 'Confidence', 'type': 'number'}}, 'required': ['answer', 'confidence']},
+      'stream': False,
+    },
+  ).respond_with_json(
+    {
+      'model': 'dummy',
+      'response': '{"answer": "Because of Rayleigh scattering", "confidence": 0.95}',
+    }
+  )
+
+  client = AsyncClient(httpserver.url_for('/'))
+  response = await client.generate('dummy', 'Why is the sky blue?', format=ResponseFormat.model_json_schema())
+  assert response['model'] == 'dummy'
+  assert response['response'] == '{"answer": "Because of Rayleigh scattering", "confidence": 0.95}'
+
+
 def test_client_pull(httpserver: HTTPServer):
   httpserver.expect_ordered_request(
     '/api/pull',
@@ -568,7 +811,7 @@ async def test_async_client_chat_images(httpserver: HTTPServer):
         {
           'role': 'user',
           'content': 'Why is the sky blue?',
-          'images': ['iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGNgYGAAAAAEAAH2FzhVAAAAAElFTkSuQmCC'],
+          'images': [PNG_BASE64],
         },
       ],
       'tools': [],
@@ -586,12 +829,10 @@ async def test_async_client_chat_images(httpserver: HTTPServer):
 
   client = AsyncClient(httpserver.url_for('/'))
 
-  with io.BytesIO() as b:
-    Image.new('RGB', (1, 1)).save(b, 'PNG')
-    response = await client.chat('dummy', messages=[{'role': 'user', 'content': 'Why is the sky blue?', 'images': [b.getvalue()]}])
-    assert response['model'] == 'dummy'
-    assert response['message']['role'] == 'assistant'
-    assert response['message']['content'] == "I don't know."
+  response = await client.chat('dummy', messages=[{'role': 'user', 'content': 'Why is the sky blue?', 'images': [PNG_BYTES]}])
+  assert response['model'] == 'dummy'
+  assert response['message']['role'] == 'assistant'
+  assert response['message']['content'] == "I don't know."
 
 
 @pytest.mark.asyncio
@@ -662,7 +903,7 @@ async def test_async_client_generate_images(httpserver: HTTPServer):
       'model': 'dummy',
       'prompt': 'Why is the sky blue?',
       'stream': False,
-      'images': ['iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGNgYGAAAAAEAAH2FzhVAAAAAElFTkSuQmCC'],
+      'images': [PNG_BASE64],
     },
   ).respond_with_json(
     {
@@ -674,7 +915,8 @@ async def test_async_client_generate_images(httpserver: HTTPServer):
   client = AsyncClient(httpserver.url_for('/'))
 
   with tempfile.NamedTemporaryFile() as temp:
-    Image.new('RGB', (1, 1)).save(temp, 'PNG')
+    temp.write(PNG_BYTES)
+    temp.flush()
     response = await client.generate('dummy', 'Why is the sky blue?', images=[temp.name])
     assert response['model'] == 'dummy'
     assert response['response'] == 'Because it is.'