Merge pull request #150 from ollama/mxyng/quantization

add quantization to create requests

README.md

@@ -1,6 +1,24 @@
 # Ollama Python Library
 
-The Ollama Python library provides the easiest way to integrate Python 3.8+ projects with [Ollama](https://github.com/jmorganca/ollama).
+The Ollama Python library provides the easiest way to integrate Python 3.8+ projects with [Ollama](https://github.com/ollama/ollama).
+
+## Prerequisites
+
+You need to have a local ollama server running to be able to continue. To do this:
+
+- Download: https://ollama.com/
+- Run an LLM: https://ollama.com/library
+    - Example: `ollama run llama2`
+    - Example: `ollama run llama2:70b`
+
+Then:
+
+```sh
+curl https://ollama.ai/install.sh | sh
+ollama serve
+```
+
+Next you can go ahead with `ollama-python`.
 
 ## Install
 
@@ -12,7 +30,7 @@ pip install ollama
 
 ```python
 import ollama
-response = ollama.chat(model='llama2', messages=[
+response = ollama.chat(model='llama3', messages=[
   {
     'role': 'user',
     'content': 'Why is the sky blue?',
@@ -29,7 +47,7 @@ Response streaming can be enabled by setting `stream=True`, modifying function c
 import ollama
 
 stream = ollama.chat(
-    model='llama2',
+    model='llama3',
     messages=[{'role': 'user', 'content': 'Why is the sky blue?'}],
     stream=True,
 )
@@ -40,18 +58,18 @@ for chunk in stream:
 
 ## API
 
-The Ollama Python library's API is designed around the [Ollama REST API](https://github.com/jmorganca/ollama/blob/main/docs/api.md)
+The Ollama Python library's API is designed around the [Ollama REST API](https://github.com/ollama/ollama/blob/main/docs/api.md)
 
 ### Chat
 
 ```python
-ollama.chat(model='llama2', messages=[{'role': 'user', 'content': 'Why is the sky blue?'}])
+ollama.chat(model='llama3', messages=[{'role': 'user', 'content': 'Why is the sky blue?'}])
 ```
 
 ### Generate
 
 ```python
-ollama.generate(model='llama2', prompt='Why is the sky blue?')
+ollama.generate(model='llama3', prompt='Why is the sky blue?')
 ```
 
 ### List
@@ -63,14 +81,14 @@ ollama.list()
 ### Show
 
 ```python
-ollama.show('llama2')
+ollama.show('llama3')
 ```
 
 ### Create
 
 ```python
 modelfile='''
-FROM llama2
+FROM llama3
 SYSTEM You are mario from super mario bros.
 '''
 
@@ -80,31 +98,31 @@ ollama.create(model='example', modelfile=modelfile)
 ### Copy
 
 ```python
-ollama.copy('llama2', 'user/llama2')
+ollama.copy('llama3', 'user/llama3')
 ```
 
 ### Delete
 
 ```python
-ollama.delete('llama2')
+ollama.delete('llama3')
 ```
 
 ### Pull
 
 ```python
-ollama.pull('llama2')
+ollama.pull('llama3')
 ```
 
 ### Push
 
 ```python
-ollama.push('user/llama2')
+ollama.push('user/llama3')
 ```
 
 ### Embeddings
 
 ```python
-ollama.embeddings(model='llama2', prompt='The sky is blue because of rayleigh scattering')
+ollama.embeddings(model='llama3', prompt='The sky is blue because of rayleigh scattering')
 ```
 
 ## Custom client
@@ -117,7 +135,7 @@ A custom client can be created with the following fields:
 ```python
 from ollama import Client
 client = Client(host='http://localhost:11434')
-response = client.chat(model='llama2', messages=[
+response = client.chat(model='llama3', messages=[
   {
     'role': 'user',
     'content': 'Why is the sky blue?',
@@ -133,7 +151,7 @@ from ollama import AsyncClient
 
 async def chat():
   message = {'role': 'user', 'content': 'Why is the sky blue?'}
-  response = await AsyncClient().chat(model='llama2', messages=[message])
+  response = await AsyncClient().chat(model='llama3', messages=[message])
 
 asyncio.run(chat())
 ```
@@ -146,7 +164,7 @@ from ollama import AsyncClient
 
 async def chat():
   message = {'role': 'user', 'content': 'Why is the sky blue?'}
-  async for part in await AsyncClient().chat(model='llama2', messages=[message], stream=True):
+  async for part in await AsyncClient().chat(model='llama3', messages=[message], stream=True):
     print(part['message']['content'], end='', flush=True)
 
 asyncio.run(chat())

ollama/_client.py

@@ -255,6 +255,7 @@ class Client(BaseClient):
     model: str,
     path: Optional[Union[str, PathLike]] = None,
     modelfile: Optional[str] = None,
+    quantize: Optional[str] = None,
     stream: bool = False,
   ) -> Union[Mapping[str, Any], Iterator[Mapping[str, Any]]]:
     """
@@ -276,6 +277,7 @@ class Client(BaseClient):
         'name': model,
         'modelfile': modelfile,
         'stream': stream,
+        'quantize': quantize,
       },
       stream=stream,
     )
@@ -474,7 +476,7 @@ class AsyncClient(BaseClient):
     prompt: str = '',
     options: Optional[Options] = None,
     keep_alive: Optional[Union[float, str]] = None,
-  ) -> Sequence[float]:
+  ) -> Mapping[str, Sequence[float]]:
     response = await self._request(
       'POST',
       '/api/embeddings',
@@ -537,6 +539,7 @@ class AsyncClient(BaseClient):
     model: str,
     path: Optional[Union[str, PathLike]] = None,
     modelfile: Optional[str] = None,
+    quantize: Optional[str] = None,
     stream: bool = False,
   ) -> Union[Mapping[str, Any], AsyncIterator[Mapping[str, Any]]]:
     """
@@ -558,6 +561,7 @@ class AsyncClient(BaseClient):
         'name': model,
         'modelfile': modelfile,
         'stream': stream,
+        'quantize': quantize,
       },
       stream=stream,
     )

ollama/_types.py

@@ -95,7 +95,6 @@ class Options(TypedDict, total=False):
   numa: bool
   num_ctx: int
   num_batch: int
-  num_gqa: int
   num_gpu: int
   main_gpu: int
   low_vram: bool
@@ -105,8 +104,6 @@ class Options(TypedDict, total=False):
   use_mmap: bool
   use_mlock: bool
   embedding_only: bool
-  rope_frequency_base: float
-  rope_frequency_scale: float
   num_thread: int
 
   # runtime options

poetry.lock

@@ -371,13 +371,13 @@ xmp = ["defusedxml"]
 
 [[package]]
 name = "pluggy"
-version = "1.4.0"
+version = "1.5.0"
 description = "plugin and hook calling mechanisms for python"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"},
-    {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"},
+    {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"},
+    {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"},
 ]
 
 [package.extras]
@@ -386,13 +386,13 @@ testing = ["pytest", "pytest-benchmark"]
 
 [[package]]
 name = "pytest"
-version = "8.1.1"
+version = "8.2.0"
 description = "pytest: simple powerful testing with Python"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "pytest-8.1.1-py3-none-any.whl", hash = "sha256:2a8386cfc11fa9d2c50ee7b2a57e7d898ef90470a7a34c4b949ff59662bb78b7"},
-    {file = "pytest-8.1.1.tar.gz", hash = "sha256:ac978141a75948948817d360297b7aae0fcb9d6ff6bc9ec6d514b85d5a65c044"},
+    {file = "pytest-8.2.0-py3-none-any.whl", hash = "sha256:1733f0620f6cda4095bbf0d9ff8022486e91892245bb9e7d5542c018f612f233"},
+    {file = "pytest-8.2.0.tar.gz", hash = "sha256:d507d4482197eac0ba2bae2e9babf0672eb333017bcedaa5fb1a3d42c1174b3f"},
 ]
 
 [package.dependencies]
@@ -400,11 +400,11 @@ colorama = {version = "*", markers = "sys_platform == \"win32\""}
 exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""}
 iniconfig = "*"
 packaging = "*"
-pluggy = ">=1.4,<2.0"
+pluggy = ">=1.5,<2.0"
 tomli = {version = ">=1", markers = "python_version < \"3.11\""}
 
 [package.extras]
-testing = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"]
+dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"]
 
 [[package]]
 name = "pytest-asyncio"
@@ -458,28 +458,28 @@ Werkzeug = ">=2.0.0"
 
 [[package]]
 name = "ruff"
-version = "0.4.1"
+version = "0.4.3"
 description = "An extremely fast Python linter and code formatter, written in Rust."
 optional = false
 python-versions = ">=3.7"
 files = [
-    {file = "ruff-0.4.1-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:2d9ef6231e3fbdc0b8c72404a1a0c46fd0dcea84efca83beb4681c318ea6a953"},
-    {file = "ruff-0.4.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:9485f54a7189e6f7433e0058cf8581bee45c31a25cd69009d2a040d1bd4bfaef"},
-    {file = "ruff-0.4.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d2921ac03ce1383e360e8a95442ffb0d757a6a7ddd9a5be68561a671e0e5807e"},
-    {file = "ruff-0.4.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:eec8d185fe193ad053eda3a6be23069e0c8ba8c5d20bc5ace6e3b9e37d246d3f"},
-    {file = "ruff-0.4.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:baa27d9d72a94574d250f42b7640b3bd2edc4c58ac8ac2778a8c82374bb27984"},
-    {file = "ruff-0.4.1-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:f1ee41580bff1a651339eb3337c20c12f4037f6110a36ae4a2d864c52e5ef954"},
-    {file = "ruff-0.4.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0926cefb57fc5fced629603fbd1a23d458b25418681d96823992ba975f050c2b"},
-    {file = "ruff-0.4.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2c6e37f2e3cd74496a74af9a4fa67b547ab3ca137688c484749189bf3a686ceb"},
-    {file = "ruff-0.4.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efd703a5975ac1998c2cc5e9494e13b28f31e66c616b0a76e206de2562e0843c"},
-    {file = "ruff-0.4.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:b92f03b4aa9fa23e1799b40f15f8b95cdc418782a567d6c43def65e1bbb7f1cf"},
-    {file = "ruff-0.4.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:1c859f294f8633889e7d77de228b203eb0e9a03071b72b5989d89a0cf98ee262"},
-    {file = "ruff-0.4.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:b34510141e393519a47f2d7b8216fec747ea1f2c81e85f076e9f2910588d4b64"},
-    {file = "ruff-0.4.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:6e68d248ed688b9d69fd4d18737edcbb79c98b251bba5a2b031ce2470224bdf9"},
-    {file = "ruff-0.4.1-py3-none-win32.whl", hash = "sha256:b90506f3d6d1f41f43f9b7b5ff845aeefabed6d2494307bc7b178360a8805252"},
-    {file = "ruff-0.4.1-py3-none-win_amd64.whl", hash = "sha256:c7d391e5936af5c9e252743d767c564670dc3889aff460d35c518ee76e4b26d7"},
-    {file = "ruff-0.4.1-py3-none-win_arm64.whl", hash = "sha256:a1eaf03d87e6a7cd5e661d36d8c6e874693cb9bc3049d110bc9a97b350680c43"},
-    {file = "ruff-0.4.1.tar.gz", hash = "sha256:d592116cdbb65f8b1b7e2a2b48297eb865f6bdc20641879aa9d7b9c11d86db79"},
+    {file = "ruff-0.4.3-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:b70800c290f14ae6fcbb41bbe201cf62dfca024d124a1f373e76371a007454ce"},
+    {file = "ruff-0.4.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:08a0d6a22918ab2552ace96adeaca308833873a4d7d1d587bb1d37bae8728eb3"},
+    {file = "ruff-0.4.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eba1f14df3c758dd7de5b55fbae7e1c8af238597961e5fb628f3de446c3c40c5"},
+    {file = "ruff-0.4.3-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:819fb06d535cc76dfddbfe8d3068ff602ddeb40e3eacbc90e0d1272bb8d97113"},
+    {file = "ruff-0.4.3-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0bfc9e955e6dc6359eb6f82ea150c4f4e82b660e5b58d9a20a0e42ec3bb6342b"},
+    {file = "ruff-0.4.3-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:510a67d232d2ebe983fddea324dbf9d69b71c4d2dfeb8a862f4a127536dd4cfb"},
+    {file = "ruff-0.4.3-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc9ff11cd9a092ee7680a56d21f302bdda14327772cd870d806610a3503d001f"},
+    {file = "ruff-0.4.3-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:29efff25bf9ee685c2c8390563a5b5c006a3fee5230d28ea39f4f75f9d0b6f2f"},
+    {file = "ruff-0.4.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:18b00e0bcccf0fc8d7186ed21e311dffd19761cb632241a6e4fe4477cc80ef6e"},
+    {file = "ruff-0.4.3-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:262f5635e2c74d80b7507fbc2fac28fe0d4fef26373bbc62039526f7722bca1b"},
+    {file = "ruff-0.4.3-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:7363691198719c26459e08cc17c6a3dac6f592e9ea3d2fa772f4e561b5fe82a3"},
+    {file = "ruff-0.4.3-py3-none-musllinux_1_2_i686.whl", hash = "sha256:eeb039f8428fcb6725bb63cbae92ad67b0559e68b5d80f840f11914afd8ddf7f"},
+    {file = "ruff-0.4.3-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:927b11c1e4d0727ce1a729eace61cee88a334623ec424c0b1c8fe3e5f9d3c865"},
+    {file = "ruff-0.4.3-py3-none-win32.whl", hash = "sha256:25cacda2155778beb0d064e0ec5a3944dcca9c12715f7c4634fd9d93ac33fd30"},
+    {file = "ruff-0.4.3-py3-none-win_amd64.whl", hash = "sha256:7a1c3a450bc6539ef00da6c819fb1b76b6b065dec585f91456e7c0d6a0bbc725"},
+    {file = "ruff-0.4.3-py3-none-win_arm64.whl", hash = "sha256:71ca5f8ccf1121b95a59649482470c5601c60a416bf189d553955b0338e34614"},
+    {file = "ruff-0.4.3.tar.gz", hash = "sha256:ff0a3ef2e3c4b6d133fbedcf9586abfbe38d076041f2dc18ffb2c7e0485d5a07"},
 ]
 
 [[package]]

tests/test_client.py

@@ -334,6 +334,7 @@ def test_client_create_path(httpserver: HTTPServer):
       'name': 'dummy',
       'modelfile': 'FROM @sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n',
       'stream': False,
+      'quantize': None,
     },
   ).respond_with_json({})
 
@@ -357,6 +358,7 @@ def test_client_create_path_relative(httpserver: HTTPServer):
       'name': 'dummy',
       'modelfile': 'FROM @sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n',
       'stream': False,
+      'quantize': None,
     },
   ).respond_with_json({})
 
@@ -389,6 +391,7 @@ def test_client_create_path_user_home(httpserver: HTTPServer, userhomedir):
       'name': 'dummy',
       'modelfile': 'FROM @sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n',
       'stream': False,
+      'quantize': None,
     },
   ).respond_with_json({})
 
@@ -412,6 +415,7 @@ def test_client_create_modelfile(httpserver: HTTPServer):
       'name': 'dummy',
       'modelfile': 'FROM @sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n',
       'stream': False,
+      'quantize': None,
     },
   ).respond_with_json({})
 
@@ -442,6 +446,7 @@ PARAMETER stop [/INST]
 PARAMETER stop <<SYS>>
 PARAMETER stop <</SYS>>''',
       'stream': False,
+      'quantize': None,
     },
   ).respond_with_json({})
 
@@ -478,6 +483,7 @@ def test_client_create_from_library(httpserver: HTTPServer):
       'name': 'dummy',
       'modelfile': 'FROM llama2',
       'stream': False,
+      'quantize': None,
     },
   ).respond_with_json({})
 
@@ -798,6 +804,7 @@ async def test_async_client_create_path(httpserver: HTTPServer):
       'name': 'dummy',
       'modelfile': 'FROM @sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n',
       'stream': False,
+      'quantize': None,
     },
   ).respond_with_json({})
 
@@ -822,6 +829,7 @@ async def test_async_client_create_path_relative(httpserver: HTTPServer):
       'name': 'dummy',
       'modelfile': 'FROM @sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n',
       'stream': False,
+      'quantize': None,
     },
   ).respond_with_json({})
 
@@ -846,6 +854,7 @@ async def test_async_client_create_path_user_home(httpserver: HTTPServer, userho
       'name': 'dummy',
       'modelfile': 'FROM @sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n',
       'stream': False,
+      'quantize': None,
     },
   ).respond_with_json({})
 
@@ -870,6 +879,7 @@ async def test_async_client_create_modelfile(httpserver: HTTPServer):
       'name': 'dummy',
       'modelfile': 'FROM @sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n',
       'stream': False,
+      'quantize': None,
     },
   ).respond_with_json({})
 
@@ -901,6 +911,7 @@ PARAMETER stop [/INST]
 PARAMETER stop <<SYS>>
 PARAMETER stop <</SYS>>''',
       'stream': False,
+      'quantize': None,
     },
   ).respond_with_json({})
 
@@ -938,6 +949,7 @@ async def test_async_client_create_from_library(httpserver: HTTPServer):
       'name': 'dummy',
       'modelfile': 'FROM llama2',
       'stream': False,
+      'quantize': None,
     },
   ).respond_with_json({})