model: TinyLlama/TinyLlama-1.1B-Chat-v1.0
hostname: localhost
port: 8000
backend: "pytorch"
free_gpu_memory_fraction: 0.15
context_servers:
  num_instances: 2
  router:
    type: load_balancing
    use_tokens: True
  max_batch_size: 1
  max_num_tokens: 3000
  max_seq_len: 4096
  tensor_parallel_size: 1
  pipeline_parallel_size: 1
  kv_cache_config:
    free_gpu_memory_fraction: 0.15
    enable_partial_reuse: False
  use_cuda_graph: False
  disable_overlap_scheduler: True
  urls:
      - "localhost:8001"
      - "localhost:8002"
generation_servers:
  num_instances: 2
  router:
    type: load_balancing
    use_tokens: False
  max_batch_size: 256
  max_num_tokens: 4096
  max_seq_len: 4096
  tensor_parallel_size: 1
  pipeline_parallel_size: 1
  kv_cache_config:
    free_gpu_memory_fraction: 0.15
    enable_partial_reuse: False
  use_cuda_graph: False
  disable_overlap_scheduler: False
  urls:
      - "localhost:8003"
      - "localhost:8004"
