diff --git a/examples/README.md b/examples/README.md index b55fd4e..b864672 100644 --- a/examples/README.md +++ b/examples/README.md @@ -36,6 +36,10 @@ python3 examples/.py - [structured-outputs-image.py](structured-outputs-image.py) +### Tokenization - Tokenize and detokenize text with a model +- [tokenization.py](tokenization.py) + + ### Ollama List - List all downloaded models and their properties - [list.py](list.py) diff --git a/examples/tokenization.py b/examples/tokenization.py index 68a033d..da1c4cb 100644 --- a/examples/tokenization.py +++ b/examples/tokenization.py @@ -1,10 +1,10 @@ import ollama # Get tokens from a model -response = ollama.tokenize(model='llama3.2', text='Hello world!') +response = ollama.tokenize(model='llama3.2', text='Why the sky is blue?') tokens = response.tokens -print('tokens from model', tokens) +print('Tokens from model', tokens) # Convert tokens back to text response = ollama.detokenize(model='llama3.2', tokens=tokens) -print('text from tokens', response.text) # Prints: Hello world! +print('Text from tokens', response.text) # Prints: Why the sky is blue? diff --git a/tests/test_client.py b/tests/test_client.py index aab2f2e..6f5c644 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -1260,3 +1260,27 @@ def test_tool_validation(): with pytest.raises(ValidationError): invalid_tool = {'type': 'invalid_type', 'function': {'name': 'test'}} list(_copy_tools([invalid_tool])) + + +def test_client_tokenize(httpserver: HTTPServer): + httpserver.expect_ordered_request( + '/api/tokenize', + method='POST', + json={'model': 'dummy', 'text': 'Hello world!'}, + ).respond_with_json({'tokens': [1, 2, 3]}) + + client = Client(httpserver.url_for('/')) + response = client.tokenize('dummy', 'Hello world!') + assert response.tokens == [1, 2, 3] + + +def test_client_detokenize(httpserver: HTTPServer): + httpserver.expect_ordered_request( + '/api/detokenize', + method='POST', + json={'model': 'dummy', 'tokens': [1, 2, 3]}, + ).respond_with_json({'text': 'Hello world!'}) + + client = Client(httpserver.url_for('/')) + response = client.detokenize('dummy', [1, 2, 3]) + assert response.text == 'Hello world!'