Add tests + cleanup

examples/README.md

@@ -36,6 +36,10 @@ python3 examples/<example>.py
 - [structured-outputs-image.py](structured-outputs-image.py)
 
 
+### Tokenization - Tokenize and detokenize text with a model
+- [tokenization.py](tokenization.py)
+
+
 ### Ollama List - List all downloaded models and their properties
 - [list.py](list.py)
 

examples/tokenization.py

@@ -1,10 +1,10 @@
 import ollama
 
 # Get tokens from a model
-response = ollama.tokenize(model='llama3.2', text='Hello world!')
+response = ollama.tokenize(model='llama3.2', text='Why the sky is blue?')
 tokens = response.tokens
-print('tokens from model', tokens)
+print('Tokens from model', tokens)
 
 # Convert tokens back to text
 response = ollama.detokenize(model='llama3.2', tokens=tokens)
-print('text from tokens', response.text)  # Prints: Hello world!
+print('Text from tokens', response.text)  # Prints: Why the sky is blue?

tests/test_client.py

@@ -1260,3 +1260,27 @@ def test_tool_validation():
   with pytest.raises(ValidationError):
     invalid_tool = {'type': 'invalid_type', 'function': {'name': 'test'}}
     list(_copy_tools([invalid_tool]))
+
+
+def test_client_tokenize(httpserver: HTTPServer):
+  httpserver.expect_ordered_request(
+    '/api/tokenize',
+    method='POST',
+    json={'model': 'dummy', 'text': 'Hello world!'},
+  ).respond_with_json({'tokens': [1, 2, 3]})
+
+  client = Client(httpserver.url_for('/'))
+  response = client.tokenize('dummy', 'Hello world!')
+  assert response.tokens == [1, 2, 3]
+
+
+def test_client_detokenize(httpserver: HTTPServer):
+  httpserver.expect_ordered_request(
+    '/api/detokenize',
+    method='POST',
+    json={'model': 'dummy', 'tokens': [1, 2, 3]},
+  ).respond_with_json({'text': 'Hello world!'})
+
+  client = Client(httpserver.url_for('/'))
+  response = client.detokenize('dummy', [1, 2, 3])
+  assert response.text == 'Hello world!'