cd sandbox
git clone https://huggingface.co/openlm-research/open_llama_7b_preview_300bt

model_path ="/home/user/sandbox/open_llama_7b_preview_300bt/open_llama_7b_preview_300bt_transformers_weights"

conda update -n base -c defaults conda --yes
conda create --yes -n env-op-llama
conda activate env-op-llama
conda install python=3.10.10 --yes
python -m pip install --upgrade pip

pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu117
pip install accelerate transformers
pip install SentencePiece

python main.py

params = {
        "prompt": prompt,.
        "temperature": 0.5, # The closer to 0, the sharper the probability distribution and the less variation
        "max_new_tokens": 256, # Number of tokens to generate in one turn.
        "context_len": 1024, # Length of the context (conversation continuity) when in chat mode.
        "use_top_k_sampling": True, # True: use TopK sampling. This reduces repetition of sentences.
        "use_bos_for_input": True, # Add BOS to input token prefix; recommended by open_llama.
        "force_set_bos_token_id": 1, # As of 4/5/2023, open_llama's tokeniser config is not set, so set manually
        "force_set_eos_token_id": 2, # As of 5/4/2023, open_llama's tokeniser config is not set, so set manually.
        "stop_strs": stop_str, # Stop string for hanging up sentences when in chat mode.
    }