From 238f142a5c36f73f1a7e2547b34e6f9760892795 Mon Sep 17 00:00:00 2001 From: ParthSareen Date: Tue, 19 Nov 2024 11:35:40 -0800 Subject: [PATCH] Examples and readme updates --- README.md | 5 ++ examples/README.md | 61 +++++++++++++++++ examples/async-chat-stream/README.md | 3 - examples/async-chat-stream/main.py | 59 ---------------- examples/async-chat.py | 35 ++++++++++ examples/async-generate.py | 15 +++++ examples/async-list.py | 37 ++++++++++ examples/async-ps.py | 40 +++++++++++ examples/{tools/main.py => async-tools.py} | 38 ++++------- .../{chat-stream/main.py => chat-stream.py} | 3 +- examples/{chat/main.py => chat.py} | 3 +- examples/{create/main.py => create.py} | 0 .../main.py => fill-in-middle.py} | 0 .../main.py => generate-stream.py} | 2 +- examples/{generate/main.py => generate.py} | 2 +- examples/list.py | 14 ++++ .../{multimodal/main.py => multimodal.py} | 1 - examples/{ps/main.py => ps.py} | 4 +- .../main.py => pull-progress.py} | 2 +- examples/pull-progress/README.md | 9 --- examples/pull-progress/requirements.txt | 1 - examples/tools.py | 67 +++++++++++++++++++ examples/tools/README.md | 3 - 23 files changed, 294 insertions(+), 110 deletions(-) create mode 100644 examples/README.md delete mode 100644 examples/async-chat-stream/README.md delete mode 100644 examples/async-chat-stream/main.py create mode 100644 examples/async-chat.py create mode 100644 examples/async-generate.py create mode 100644 examples/async-list.py create mode 100644 examples/async-ps.py rename examples/{tools/main.py => async-tools.py} (75%) rename examples/{chat-stream/main.py => chat-stream.py} (68%) rename examples/{chat/main.py => chat.py} (75%) rename examples/{create/main.py => create.py} (100%) rename examples/{fill-in-middle/main.py => fill-in-middle.py} (100%) rename examples/{generate-stream/main.py => generate-stream.py} (51%) rename examples/{generate/main.py => generate.py} (50%) create mode 100644 examples/list.py rename examples/{multimodal/main.py => multimodal.py} (99%) rename examples/{ps/main.py => ps.py} (84%) rename examples/{pull-progress/main.py => pull-progress.py} (92%) delete mode 100644 examples/pull-progress/README.md delete mode 100644 examples/pull-progress/requirements.txt create mode 100644 examples/tools.py delete mode 100644 examples/tools/README.md diff --git a/README.md b/README.md index e03ea00..6690d9e 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,11 @@ The Ollama Python library provides the easiest way to integrate Python 3.8+ projects with [Ollama](https://github.com/ollama/ollama). +## Prerequisites + +- Install [Ollama](https://ollama.com/download) +- Pull a model: `ollama pull ` See [Ollama models](https://ollama.com/models) + ## Install ```sh diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..de69eca --- /dev/null +++ b/examples/README.md @@ -0,0 +1,61 @@ +# Running Examples + +Run the examples in this directory with: +```sh +# Navigate to examples directory +cd examples/ + +# Run example +python3 .py +``` + +### Chat +- [chat.py](chat.py) - Basic chat with model +- [chat-stream.py](chat-stream.py) - Stream chat with model +- [async-chat.py](async-chat.py) - Async chat with model + +### Generate +- [generate.py](generate.py) - Generate text with model +- [generate-stream.py](generate-stream.py) - Stream generate text with model +- [async-generate.py](async-generate.py) - Async generate text with model + +### List +- [list.py](list.py) - List all downloaded models and their properties +- [async-list.py](async-list.py) - Async list all downloaded models and their properties + +### Fill in the middle +- [fill-in-middle.py](fill-in-middle.py) - Fill in the middle with model + + +### Multimodal +- [multimodal.py](multimodal.py) - Multimodal chat with model + +### Pull Progress +Requirement: `pip install tqdm` + +- [pull-progress.py](pull-progress.py) - Pull progress with model + +### Ollama create (create a model) +- [create.py](create.py) - Create a model + +### Ollama ps (show model status - cpu/gpu usage) +- [ollama-ps.py](ollama-ps.py) - Ollama ps + +### Tools/Function Calling +- [tools.py](tools.py) - Simple example of Tools/Function Calling +- [async-tools.py](async-tools.py) - Async example of Tools/Function Calling + +## Configuring Clients +Custom parameters can be passed to the client when initializing: +```python +import ollama +client = ollama.Client( + host='http://localhost:11434', + timeout=10.0, # Default: None + follow_redirects=True, # Default: True + headers={'x-some-header': 'some-value'} +) +``` + +Similarly, the `AsyncClient` class can be configured with the same parameters. + diff --git a/examples/async-chat-stream/README.md b/examples/async-chat-stream/README.md deleted file mode 100644 index 611295a..0000000 --- a/examples/async-chat-stream/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# async-chat-stream - -This example demonstrates how to create a conversation history using an asynchronous Ollama client and the chat endpoint. The streaming response is outputted to `stdout` as well as a TTS if enabled with `--speak` and available. Supported TTS are `say` on macOS and `espeak` on Linux. diff --git a/examples/async-chat-stream/main.py b/examples/async-chat-stream/main.py deleted file mode 100644 index 6504776..0000000 --- a/examples/async-chat-stream/main.py +++ /dev/null @@ -1,59 +0,0 @@ -import shutil -import asyncio -import argparse - -import ollama - - -async def speak(speaker, content): - if speaker: - p = await asyncio.create_subprocess_exec(speaker, content) - await p.communicate() - - -async def main(): - parser = argparse.ArgumentParser() - parser.add_argument('--speak', default=False, action='store_true') - args = parser.parse_args() - - speaker = None - if not args.speak: - ... - elif say := shutil.which('say'): - speaker = say - elif (espeak := shutil.which('espeak')) or (espeak := shutil.which('espeak-ng')): - speaker = espeak - - client = ollama.AsyncClient() - - messages = [] - - while True: - if content_in := input('>>> '): - messages.append({'role': 'user', 'content': content_in}) - - content_out = '' - message = {'role': 'assistant', 'content': ''} - async for response in await client.chat(model='mistral', messages=messages, stream=True): - if response['done']: - messages.append(message) - - content = response['message']['content'] - print(content, end='', flush=True) - - content_out += content - if content in ['.', '!', '?', '\n']: - await speak(speaker, content_out) - content_out = '' - - message['content'] += content - - if content_out: - await speak(speaker, content_out) - print() - - -try: - asyncio.run(main()) -except (KeyboardInterrupt, EOFError): - ... diff --git a/examples/async-chat.py b/examples/async-chat.py new file mode 100644 index 0000000..d4f8c4d --- /dev/null +++ b/examples/async-chat.py @@ -0,0 +1,35 @@ +import asyncio +import ollama + + +async def main(): + client = ollama.AsyncClient() + messages = [] + + print("Chat with the model (type 'exit' to quit)") + while True: + # Get user input + user_input = input('You: ') + if user_input.lower() == 'exit': + break + + # Add user message to history + messages.append({'role': 'user', 'content': user_input}) + + # Stream the response + print('Assistant: ', end='', flush=True) + async for chunk in await client.chat(model='llama3.1', messages=messages, stream=True): + # Print the response chunk + print(chunk['message']['content'], end='', flush=True) + + # Update message history with complete response + if chunk['done']: + messages.append(chunk['message']) + print('\n') + + +if __name__ == '__main__': + try: + asyncio.run(main()) + except KeyboardInterrupt: + print('\nGoodbye!') diff --git a/examples/async-generate.py b/examples/async-generate.py new file mode 100644 index 0000000..16598b9 --- /dev/null +++ b/examples/async-generate.py @@ -0,0 +1,15 @@ +import asyncio +import ollama + + +async def main(): + client = ollama.AsyncClient() + response = await client.generate('llama3.1', 'Why is the sky blue?') + print(response['response']) + + +if __name__ == '__main__': + try: + asyncio.run(main()) + except KeyboardInterrupt: + print('\nGoodbye!') diff --git a/examples/async-list.py b/examples/async-list.py new file mode 100644 index 0000000..82d2fcd --- /dev/null +++ b/examples/async-list.py @@ -0,0 +1,37 @@ +import asyncio +import ollama + + +async def main(): + client = ollama.AsyncClient() + + response = await client.list() + models = response['models'] + models_data = [] + for model in models: + if model.get('details'): # Check if details exist + models_data.append( + ( + model.get('name', 'N/A'), + model.get('size', 0) / 1024 / 1024, # Convert to MB + model.get('details', {}).get('format', 'N/A'), + model.get('details', {}).get('family', 'N/A'), + model.get('details', {}).get('parameter_size', 'N/A'), + model.get('details', {}).get('quantization_level', 'N/A'), + ) + ) + + print(f'\n{len(models)} models found!') + print('\nDetailed model information:') + for model in models_data: + print(f'Name: {model[0]}') + print(f'Size (MB): {model[1]:.2f}') + print(f'Format: {model[2]}') + print(f'Family: {model[3]}') + print(f'Parameter Size: {model[4]}') + print(f'Quantization Level: {model[5]}') + print('-' * 50) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/examples/async-ps.py b/examples/async-ps.py new file mode 100644 index 0000000..843ae34 --- /dev/null +++ b/examples/async-ps.py @@ -0,0 +1,40 @@ +import asyncio +from ollama import AsyncClient + + +async def main(): + client = AsyncClient() + + response = await client.pull('llama3.1', stream=True) + progress_states = set() + async for progress in response: + if progress.get('status') in progress_states: + continue + progress_states.add(progress.get('status')) + print(progress.get('status')) + + print('\n') + + response = await client.chat('llama3.1', messages=[{'role': 'user', 'content': 'Hello!'}]) + print(response['message']['content']) + + print('\n') + + response = await client.ps() + + name = response['models'][0]['name'] + size = response['models'][0]['size'] + size_vram = response['models'][0]['size_vram'] + + if size == size_vram: + print(f'{name}: 100% GPU') + elif not size_vram: + print(f'{name}: 100% CPU') + else: + size_cpu = size - size_vram + cpu_percent = round(size_cpu / size * 100) + print(f'{name}: {cpu_percent}% CPU/{100 - cpu_percent}% GPU') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/examples/tools/main.py b/examples/async-tools.py similarity index 75% rename from examples/tools/main.py rename to examples/async-tools.py index 133b238..b38dc31 100644 --- a/examples/tools/main.py +++ b/examples/async-tools.py @@ -6,6 +6,16 @@ import asyncio # Simulates an API call to get flight times # In a real application, this would fetch data from a live database or API def get_flight_times(departure: str, arrival: str) -> str: + """ + Get the flight times between two cities + + Args: + departure (str): The departure city (airport code) + arrival (str): The arrival city (airport code) + + Returns: + str: The flight times between the two cities + """ flights = { 'NYC-LAX': {'departure': '08:00 AM', 'arrival': '11:30 AM', 'duration': '5h 30m'}, 'LAX-NYC': {'departure': '02:00 PM', 'arrival': '10:30 PM', 'duration': '5h 30m'}, @@ -28,29 +38,7 @@ async def run(model: str): response = await client.chat( model=model, messages=messages, - tools=[ - { - 'type': 'function', - 'function': { - 'name': 'get_flight_times', - 'description': 'Get the flight times between two cities', - 'parameters': { - 'type': 'object', - 'properties': { - 'departure': { - 'type': 'string', - 'description': 'The departure city (airport code)', - }, - 'arrival': { - 'type': 'string', - 'description': 'The arrival city (airport code)', - }, - }, - 'required': ['departure', 'arrival'], - }, - }, - }, - ], + tools=[get_flight_times], ) # Add the model's response to the conversation history @@ -80,8 +68,8 @@ async def run(model: str): # Second API call: Get final response from the model final_response = await client.chat(model=model, messages=messages) - print(final_response['message']['content']) + print(final_response.message.content) # Run the async function -asyncio.run(run('mistral')) +asyncio.run(run('llama3.1')) diff --git a/examples/chat-stream/main.py b/examples/chat-stream.py similarity index 68% rename from examples/chat-stream/main.py rename to examples/chat-stream.py index 2a57346..efc9d8b 100644 --- a/examples/chat-stream/main.py +++ b/examples/chat-stream.py @@ -8,8 +8,7 @@ messages = [ }, ] -for part in chat('mistral', messages=messages, stream=True): +for part in chat('llama3.1', messages=messages, stream=True): print(part['message']['content'], end='', flush=True) -# end with a newline print() diff --git a/examples/chat/main.py b/examples/chat.py similarity index 75% rename from examples/chat/main.py rename to examples/chat.py index 90c5f90..f9f6ae7 100644 --- a/examples/chat/main.py +++ b/examples/chat.py @@ -1,6 +1,5 @@ from ollama import chat - messages = [ { 'role': 'user', @@ -8,5 +7,5 @@ messages = [ }, ] -response = chat('mistral', messages=messages) +response = chat('llama3.1', messages=messages) print(response['message']['content']) diff --git a/examples/create/main.py b/examples/create.py similarity index 100% rename from examples/create/main.py rename to examples/create.py diff --git a/examples/fill-in-middle/main.py b/examples/fill-in-middle.py similarity index 100% rename from examples/fill-in-middle/main.py rename to examples/fill-in-middle.py diff --git a/examples/generate-stream/main.py b/examples/generate-stream.py similarity index 51% rename from examples/generate-stream/main.py rename to examples/generate-stream.py index a24b410..2a3e3bb 100644 --- a/examples/generate-stream/main.py +++ b/examples/generate-stream.py @@ -1,5 +1,5 @@ from ollama import generate -for part in generate('mistral', 'Why is the sky blue?', stream=True): +for part in generate('llama3.1', 'Why is the sky blue?', stream=True): print(part['response'], end='', flush=True) diff --git a/examples/generate/main.py b/examples/generate.py similarity index 50% rename from examples/generate/main.py rename to examples/generate.py index e39e295..745822f 100644 --- a/examples/generate/main.py +++ b/examples/generate.py @@ -1,5 +1,5 @@ from ollama import generate -response = generate('mistral', 'Why is the sky blue?') +response = generate('llama3.1', 'Why is the sky blue?') print(response['response']) diff --git a/examples/list.py b/examples/list.py new file mode 100644 index 0000000..0e3115a --- /dev/null +++ b/examples/list.py @@ -0,0 +1,14 @@ +from ollama import list +from ollama._types import ListResponse + +response: ListResponse = list() + +for model in response.models: + if model.details: + print(f'Name: {model.model}') + print(f'Size (MB): {(model.size.real / 1024 / 1024):.2f}') + print(f'Format: {model.details.format}') + print(f'Family: {model.details.family}') + print(f'Parameter Size: {model.details.parameter_size}') + print(f'Quantization Level: {model.details.quantization_level}') + print('-' * 50) diff --git a/examples/multimodal/main.py b/examples/multimodal.py similarity index 99% rename from examples/multimodal/main.py rename to examples/multimodal.py index 44b3716..ad70139 100644 --- a/examples/multimodal/main.py +++ b/examples/multimodal.py @@ -4,7 +4,6 @@ import httpx from ollama import generate - latest = httpx.get('https://xkcd.com/info.0.json') latest.raise_for_status() diff --git a/examples/ps/main.py b/examples/ps.py similarity index 84% rename from examples/ps/main.py rename to examples/ps.py index 822d09a..ed99201 100644 --- a/examples/ps/main.py +++ b/examples/ps.py @@ -1,6 +1,6 @@ from ollama import ps, pull, chat -response = pull('mistral', stream=True) +response = pull('llama3.1', stream=True) progress_states = set() for progress in response: if progress.get('status') in progress_states: @@ -10,7 +10,7 @@ for progress in response: print('\n') -response = chat('mistral', messages=[{'role': 'user', 'content': 'Hello!'}]) +response = chat('llama3.1', messages=[{'role': 'user', 'content': 'Hello!'}]) print(response['message']['content']) print('\n') diff --git a/examples/pull-progress/main.py b/examples/pull-progress.py similarity index 92% rename from examples/pull-progress/main.py rename to examples/pull-progress.py index 89b2f3a..d3eb414 100644 --- a/examples/pull-progress/main.py +++ b/examples/pull-progress.py @@ -3,7 +3,7 @@ from ollama import pull current_digest, bars = '', {} -for progress in pull('mistral', stream=True): +for progress in pull('llama3.1', stream=True): digest = progress.get('digest', '') if digest != current_digest and current_digest in bars: bars[current_digest].close() diff --git a/examples/pull-progress/README.md b/examples/pull-progress/README.md deleted file mode 100644 index 8a44f60..0000000 --- a/examples/pull-progress/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# pull-progress - -This example emulates `ollama pull` using the Python library and [`tqdm`](https://tqdm.github.io/). - -## Setup - -```shell -pip install -r requirements.txt -``` diff --git a/examples/pull-progress/requirements.txt b/examples/pull-progress/requirements.txt deleted file mode 100644 index ae3df91..0000000 --- a/examples/pull-progress/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -tqdm==4.66.1 diff --git a/examples/tools.py b/examples/tools.py new file mode 100644 index 0000000..b434cd0 --- /dev/null +++ b/examples/tools.py @@ -0,0 +1,67 @@ +from ollama import chat +from ollama._types import ChatResponse + + +def add_two_numbers(a: int, b: int) -> int: + """ + Add two numbers + + Args: + a (int): The first number + b (int): The second number + + Returns: + int: The sum of the two numbers + """ + return a + b + + +def subtract_two_numbers(a: int, b: int) -> int: + """ + Subtract two numbers + """ + return a - b + + +# Tools can still be manually defined and passed into chat +subtract_two_numbers_tool = { + 'type': 'function', + 'function': { + 'name': 'subtract_two_numbers', + 'description': 'Subtract two numbers', + 'parameters': { + 'type': 'object', + 'required': ['a', 'b'], + 'properties': { + 'a': {'type': 'integer', 'description': 'The first number'}, + 'b': {'type': 'integer', 'description': 'The second number'}, + }, + }, + }, +} + +prompt = 'What is three minus one?' +print(f'Prompt: {prompt}') + +response: ChatResponse = chat( + 'llama3.1', + messages=[{'role': 'user', 'content': prompt}], + tools=[add_two_numbers, subtract_two_numbers_tool], +) + +available_functions = { + 'add_two_numbers': add_two_numbers, + 'subtract_two_numbers': subtract_two_numbers, +} + +if response.message.tool_calls: + # There may be multiple tool calls in the response + for tool in response.message.tool_calls: + # Ensure the function is available, and then call it + if tool.function.name in available_functions: + print(f'Calling function {tool.function.name}') + print(f'Arguments: {tool.function.arguments}') + function_to_call = available_functions[tool.function.name] + print(f'Function output: {function_to_call(**tool.function.arguments)}') + else: + print(f'Function {tool.function.name} not found') diff --git a/examples/tools/README.md b/examples/tools/README.md deleted file mode 100644 index 85ca5dd..0000000 --- a/examples/tools/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# tools - -This example demonstrates how to utilize tool calls with an asynchronous Ollama client and the chat endpoint.