pip install llama-api-server

# to run wth pyllama
pip install llama-api-server[pyllama]

cat > config.yml << EOF
models:
  completions:
    # completions and chat_completions use same model
    text-ada-002:
      type: llama_cpp
      params:
        path: /absolute/path/to/your/7B/ggml-model-q4_0.bin
    text-davinci-002:
      type: pyllama_quant
      params:
        path: /absolute/path/to/your/pyllama-7B4b.pt
    text-davinci-003:
      type: pyllama
      params:
        ckpt_dir: /absolute/path/to/your/7B/
        tokenizer_path: /absolute/path/to/your/tokenizer.model
      # keep to 1 instance to speed up loading of model
  embeddings:
    text-embedding-davinci-002:
      type: pyllama_quant
      params:
        path: /absolute/path/to/your/pyllama-7B4b.pt
      min_instance: 1
      max_instance: 1
      idle_timeout: 3600
    text-embedding-ada-002:
      type: llama_cpp
      params:
        path: /absolute/path/to/your/7B/ggml-model-q4_0.bin
EOF

echo "SOME_TOKEN" > tokens.txt

# start web server
python -m llama_api_server
# or visible across the network
python -m llama_api_server --host=0.0.0.0

export OPENAI_API_KEY=SOME_TOKEN
export OPENAI_API_BASE=http://127.0.0.1:5000/v1

openai api completions.create -e text-ada-002 -p "hello?"
# or using chat
openai api chat_completions.create -e text-ada-002 -g user "hello?"
# or calling embedding
curl -X POST http://127.0.0.1:5000/v1/embeddings -H 'Content-Type: application/json' -d '{"model":"text-embedding-ada-002", "input":"It is good."}'  -H "Authorization: Bearer SOME_TOKEN"