cornell-relaxml / quip-sharp Goto Github PK

#!/bin/bash
eval "$(conda shell.bash hook)"
conda activate quip
MODEL_NAME=Xwin-Math-7B-V1.0
#MODEL_NAME=ShareGPT4V-7B
BASE_MODEL_DIR=/media/hangyu5/Home/Documents/Hugging-Face/$MODEL_NAME/
SAVE_MODEL_DIR=/media/hangyu5/Home/Documents/Hugging-Face/$MODEL_NAME-QUIP/
TMP_MODEL_DIR=./$MODEL_NAME
BATCH_SIZE=2
CTX_LEN=4096

cd repo/quip-sharp/

if [! -d "$TMP_MODEL_DIR" ]; then
    mkdir -p $TMP_MODEL_DIR
fi

TRANSFORMERS_VERBOSITY=debug CUDA_VISIBLE_DEVICES=0 python ./hessian_offline_llama.py \
    --seed 34 \
    --base_model $BASE_MODEL_DIR \
    --save_path $TMP_MODEL_DIR/Hessian/ \
    --ctx_size $CTX_LEN \
    --batch_size $BATCH_SIZE \
    | tee $TMP_MODEL_DIR/Hessian-quip.log

TRANSFORMERS_VERBOSITY=debug CUDA_VISIBLE_DEVICES=0 python ./quantize_llama.py \
    --seed 34 \
    --base_model $BASE_MODEL_DIR \
    --hessian_path $TMP_MODEL_DIR/Hessian \
    --save_path $TMP_MODEL_DIR/Ckpt \
    --ctx_size $CTX_LEN \
    --batch_size $BATCH_SIZE \
    --codebook E8P12 \
    --scale_override 0.9 \
     | tee $TMP_MODEL_DIR/Ckpt-quip.log

TRANSFORMERS_VERBOSITY=debug CUDA_VISIBLE_DEVICES=0 python hfize_llama.py \
    --quantized_path $TMP_MODEL_DIR/Ckpt \
    --hf_output_path $SAVE_MODEL_DIR \
    | tee $TMP_MODEL_DIR/HFize-quip.log

TRANSFORMERS_VERBOSITY=debug CUDA_VISIBLE_DEVICES=0 python eval_ppl.py \
    --seed 34 \
    --hf_path $BASE_MODEL_DIR \
    --seqlen $CTX_LEN \
    | tee $TMP_MODEL_DIR/PPl-quip.log

cd ../../

from langchain import HuggingFacePipeline
from transformers import AutoTokenizer, AutoModelForCausalLM
import transformers
import torch
from langchain.prompts import PromptTemplate, FewShotPromptTemplate
from langchain import LLMChain


model_name="relaxml/Mistral-7b-E8PRVQ-4Bit"
hf_access_token = "hf_XXXX" # Replace with your HF access token

tokenizer = AutoTokenizer.from_pretrained(
    model_name, use_auth_token=hf_access_token #, use_fast=True
)

model = transformers.AutoModelForCausalLM.from_pretrained(
    model_name,
    trust_remote_code=True,
    device_map="auto",
    use_auth_token=hf_access_token,
)

pipeline = transformers.pipeline(
    "text-generation",  # task
    model=model,
    tokenizer=tokenizer,
    torch_dtype=torch.bfloat16,
    use_fast=True,
    trust_remote_code=True,
    device_map="auto",
    max_length=4000,
    do_sample=True,
    top_k=3,
    num_return_sequences=1,
    pad_token_id=tokenizer.eos_token_id,
    eos_token_id=tokenizer.eos_token_id,
)

llm = HuggingFacePipeline(pipeline=pipeline, model_kwargs={"temperature": 0.0})

text_gen_intent_examples = [
{
"input_text": "please write a text to john saying I am running late",
"generated_text": "Hi John, I wanted to let you know that I'm running a bit late. I apologize for any inconvenience this may cause. I'll be there as soon as I can. Thanks for understanding,"
},
{
"input_text": "write text to Jessica saying the we have a meeting tomorrow",
"generated_text": "Hi Jessica, Just a reminder that we have a meeting scheduled for tomorrow. Please let me know if there's anything specific you'd like to discuss or prepare ahead of time. Looking forward to it!"
},
{
"input_text": "write a message to Patrick to send the presentation slides",
"generated_text": "Hi Patrick, Hope you're doing well. Could you please send over the presentation slides when you get a chance?"
}
]

text_gen_intent_prefix = """
[INST]
Generate a brief and concise text message or email that a person could quickly send while driving.
- It should be polite but direct, without any unnecessary details.
- your response must not be in chat format as shown below:
[/INST]
"""

prompt_template = """
<s>[INST]
input_text: {input_text}
generated_text: {generated_text}
</s>[/INST]
"""

prompt = PromptTemplate(
    template=prompt_template, input_variables=["input_text", "generated_text"]
)


few_shot_template = FewShotPromptTemplate(
    examples=text_gen_intent_examples,
    example_prompt=prompt,
    prefix=text_gen_intent_prefix,
    suffix="input_text: {input_text} \n generated_text:",
    input_variables=["input_text"],
    example_separator="\n\n",
)

llm_chain = LLMChain(prompt=few_shot_template, llm=llm)

text = """
       write an email to Mark stating I am running late for the meeting and won't be able to join. Also, to send the minutes of the
       meeting.
       """
output_lst = []

print(few_shot_template.format(input_text=text.strip()))
print("-" * 30)
time_list = []
iter_num = 5
import time
for i in range(iter_num):
    st_time = time.time()
    print(llm_chain.run(text.strip()))
    end_time = time.time()
    time_taken = end_time - st_time
    print(f' {time_taken=}')
    time_list.append(time_taken)
    print("-" * 30)

total_time_taken = sum(time_list)
avg_time = total_time_taken / len(time_list)
print(f"Average time taken = {round(avg_time, 2)}")

utz cleohl cleufohlarrowarrowarrow orient appar cle indul appar cleailleöt士ötötightssndbugbugบohlufijioenohlohlohlufesser («ohlmulticol Barceluf Plus negarrow Negarrow Inf Infarrowarrow Infarrow Bedarrowaille Dead febrohlohlaille郎郎郎郎arrowiji sealbuglijbuglijlij PlusMSMbugbugbug Nutbugsndbugbugbugbug bleedingportal Negouthbugštštštštiekiekatoniek Contin Contin bleeding Continiekiekiekiekiek Brigiek бы Negwick бы Barcel original seal Negesseresseroke Barcelmulticolmulticolwick Barcel retro retro retro retroFF retro retro retroervicesieriiasmiasmiasmervicesunch plainiasmiasmervicesosoosoiasmiasmiasmwickINEwickbugbugwickwickót Barcel Barcelwickötwick Barcelwickšt Barcelwickwick Barcelwick notenwick Barcel org Barcel Barcel Barcel CL Barcel Barcel Barcel wides Barcelieriieritee absolutteeervicesieriieriieriulumieri Barcel absolutungsungswickwickwickwickwickwickvíervicesieriwickwickieriwickieriieriwickšt济œuverviceservices济erviceservicesoki CHAPTERitsch CHAPTERitschwickwick济

transformers
torch
accelerate
bitsandbytes
xformers
langchain
scipy

Some weights of the model checkpoint at relaxml/Llama-2-7b-E8P-2Bit were not used when initializing LlamaForCausalLM: ['model.layers.27.self_attn.qkv_proj.codebook_id', 'model.layers.19.mlp.down_proj.Qidxs', 'model.layers.26.self_attn.qkv_proj.codebook_id', 'model.layers.15.self_attn.qkv_proj.SV', 'model.layers.16.self_attn.o_proj.Wscale', 'model.layers.23.mlp.down_proj.SU', 'model.layers.8.mlp.upgate_proj.SU', 'model.layers.6.self_attn.qkv_proj.Qidxs', 'model.layers.24.self_attn.o_proj.SU', 'model.layers.17.mlp.upgate_proj.SU', 'model.layers.23.self_attn.o_proj.SU', 'model.layers.9.mlp.down_proj.SU', 'model.layers.24.mlp.down_proj.codebook_id', 'model.layers.28.self_attn.o_proj.Qidxs', 'model.layers.2.self_attn.o_proj.codebook_id', 'model.layers.26.self_attn.o_proj.Qidxs', 'model.layers.10.mlp.down_proj.SV', 'model.layers.25.self_attn.o_proj.SU', 'model.layers.2.mlp.upgate_proj.fuse_scales', 'model.layers.21.mlp.upgate_proj.codebook_id', 'model.layers.31.self_attn.o_proj.codebook_id', 'model.layers.29.self_attn.qkv_proj.fuse_scales', 'model.layers.0.self_attn.qkv_proj.Qidxs', 'model.layers.31.self_attn.o_proj.Wscale', 'model.layers.0.self_attn.qkv_proj.Wscale', 'model.layers.18.mlp.down_proj.Qidxs', 'model.layers.23.mlp.upgate_proj.SU', 'model.layers.21.mlp.down_proj.Wscale', 'model.layers.24.self_attn.o_proj.Qidxs', 'model.layers.2.self_attn.qkv_proj.SV', 'model.layers.27.mlp.down_proj.codebook_id', 'model.layers.18.mlp.down_proj.codebook_id', 'model.layers.3.self_attn.o_proj.Wscale', 'model.layers.19.mlp.upgate_proj.SU', 'model.layers.3.self_attn.o_proj.SV', 'model.layers.20.mlp.upgate_proj.codebook_id', 'model.layers.8.self_attn.qkv_proj.fuse_scales', 'model.layers.20.self_attn.qkv_proj.Wscale', 'model.layers.31.mlp.upgate_proj.codebook_id', 'model.layers.2.self_attn.o_proj.Qidxs', 'model.layers.4.mlp.down_proj.codebook_id', 'model.layers.28.mlp.upgate_proj.Qidxs', 'model.layers.9.mlp.down_proj.Qidxs', 'model.layers.15.self_attn.o_proj.Wscale', 'model.layers.11.self_attn.qkv_proj.SU', 'model.layers.0.mlp.down_proj.SU', 'model.layers.9.mlp.upgate_proj.SU', 'model.layers.19.mlp.down_proj.codebook_id', 'model.layers.6.mlp.upgate_proj.Qidxs', 'model.layers.29.mlp.down_proj.SU', 'model.layers.18.mlp.upgate_proj.Qidxs', 'model.layers.12.self_attn.qkv_proj.SV', 'model.layers.30.mlp.down_proj.SU', 'model.layers.12.mlp.down_proj.SV', 'model.layers.16.mlp.upgate_proj.SV', 'model.layers.3.mlp.upgate_proj.SU', 'model.layers.16.self_attn.o_proj.SU', 'model.layers.11.mlp.upgate_proj.codebook_id', 'model.layers.4.self_attn.qkv_proj.fuse_scales', 'model.layers.29.mlp.down_proj.Wscale', 'model.layers.31.self_attn.qkv_proj.SV', 'model.layers.4.self_attn.qkv_proj.Wscale', 'model.layers.16.mlp.upgate_proj.fuse_scales', 'model.layers.18.mlp.upgate_proj.codebook_id', 'model.layers.24.mlp.upgate_proj.SV', 'model.layers.23.self_attn.o_proj.SV', 'model.layers.7.self_attn.qkv_proj.fuse_scales', 'model.layers.23.mlp.upgate_proj.Wscale', 'model.layers.19.mlp.down_proj.Wscale', 'model.layers.15.self_attn.qkv_proj.codebook_id', 'model.layers.2.mlp.down_proj.SU', 'model.layers.6.mlp.upgate_proj.Wscale', 'model.layers.0.self_attn.qkv_proj.SU', 'model.layers.2.self_attn.o_proj.Wscale', 'model.layers.24.mlp.upgate_proj.Wscale', 'model.layers.25.self_attn.qkv_proj.Qidxs', 'model.layers.1.mlp.upgate_proj.Qidxs', 'model.layers.10.mlp.upgate_proj.SV', 'model.layers.29.mlp.down_proj.Qidxs', 'model.layers.12.mlp.upgate_proj.Wscale', 'model.layers.15.mlp.upgate_proj.SV', 'model.layers.12.mlp.upgate_proj.SV', 'model.layers.30.mlp.upgate_proj.codebook_id', 'model.layers.0.mlp.upgate_proj.Qidxs', 'model.layers.20.self_attn.o_proj.Qidxs', 'model.layers.19.self_attn.qkv_proj.Qidxs', 'model.layers.29.self_attn.o_proj.SU', 'model.layers.10.self_attn.o_proj.SV', 'model.layers.17.mlp.upgate_proj.Wscale', 'model.layers.13.mlp.down_proj.codebook_id', 'model.layers.11.self_attn.o_proj.Qidxs', 'model.layers.12.self_attn.qkv_proj.codebook_id', 'model.layers.11.mlp.down_proj.Qidxs', 'model.layers.28.self_attn.qkv_proj.codebook_id', 'model.layers.29.mlp.upgate_proj.SU', 'model.layers.24.mlp.down_proj.Qidxs', 'model.layers.23.mlp.down_proj.SV', 'model.layers.20.mlp.down_proj.SV', 'model.layers.20.self_attn.o_proj.codebook_id', 'model.layers.27.self_attn.o_proj.SV', 'model.layers.8.self_attn.o_proj.Qidxs', 'model.layers.0.self_attn.o_proj.SU', 'model.layers.4.self_attn.o_proj.Qidxs', 'model.layers.15.self_attn.qkv_proj.Wscale', 'model.layers.21.self_attn.qkv_proj.Qidxs', 'model.layers.17.mlp.upgate_proj.fuse_scales', 'model.layers.19.self_attn.o_proj.SU', 'model.layers.31.mlp.down_proj.codebook_id', 'model.layers.24.self_attn.qkv_proj.Qidxs', 'model.layers.20.mlp.down_proj.Qidxs', 'model.layers.5.self_attn.o_proj.SU', 'model.layers.11.self_attn.o_proj.SV', 'model.layers.13.mlp.down_proj.Qidxs', 'model.layers.0.mlp.upgate_proj.SV', 'model.layers.19.mlp.upgate_proj.SV', 'model.layers.22.self_attn.qkv_proj.Wscale', 'model.layers.30.self_attn.o_proj.Qidxs', 'model.layers.30.mlp.upgate_proj.fuse_scales', 'model.layers.25.mlp.upgate_proj.SV', 'model.layers.8.self_attn.qkv_proj.SU', 'model.layers.17.self_attn.qkv_proj.SU', 'model.layers.19.mlp.upgate_proj.Qidxs', 'model.layers.20.mlp.down_proj.Wscale', 'model.layers.16.self_attn.o_proj.Qidxs', 'model.layers.26.self_attn.o_proj.SU', 'model.layers.11.mlp.down_proj.SU', 'model.layers.1.mlp.down_proj.SV', 'model.layers.12.self_attn.qkv_proj.Qidxs', 'model.layers.23.self_attn.qkv_proj.codebook_id', 'model.layers.26.mlp.down_proj.Qidxs', 'model.layers.3.self_attn.o_proj.Qidxs', 'model.layers.11.mlp.upgate_proj.SV', 'model.layers.14.mlp.upgate_proj.SU', 'model.layers.14.mlp.upgate_proj.codebook_id', 'model.layers.14.self_attn.o_proj.SU', 'model.layers.30.self_attn.o_proj.SV', 'model.layers.24.mlp.upgate_proj.Qidxs', 'model.layers.0.mlp.upgate_proj.fuse_scales', 'model.layers.7.mlp.upgate_proj.SV', 'model.layers.6.self_attn.o_proj.SV', 'model.layers.27.self_attn.o_proj.Wscale', 'model.layers.0.mlp.down_proj.Wscale', 'model.layers.28.mlp.down_proj.codebook_id', 'model.layers.29.self_attn.qkv_proj.Wscale', 'model.layers.22.mlp.down_proj.Qidxs', 'model.layers.15.mlp.down_proj.Qidxs', 'model.layers.10.self_attn.qkv_proj.SU', 'model.layers.25.self_attn.o_proj.Qidxs', 'model.layers.29.mlp.down_proj.codebook_id', 'model.layers.26.mlp.down_proj.Wscale', 'model.layers.22.self_attn.qkv_proj.SU', 'model.layers.17.self_attn.qkv_proj.Wscale', 'model.layers.9.self_attn.qkv_proj.Qidxs', 'model.layers.21.self_attn.o_proj.SV', 'model.layers.26.mlp.upgate_proj.codebook_id', 'model.layers.31.mlp.upgate_proj.SU', 'model.layers.10.self_attn.o_proj.SU', 'model.layers.17.self_attn.o_proj.SV', 'model.layers.10.mlp.down_proj.SU', 'model.layers.13.self_attn.qkv_proj.fuse_scales', 'model.layers.21.self_attn.o_proj.Qidxs', 'model.layers.5.mlp.down_proj.codebook_id', 'model.layers.25.mlp.down_proj.Wscale', 'model.layers.19.mlp.down_proj.SV', 'model.layers.14.self_attn.qkv_proj.Wscale', 'model.layers.25.self_attn.o_proj.Wscale', 'model.layers.3.self_attn.qkv_proj.fuse_scales', 'model.layers.3.self_attn.o_proj.SU', 'model.layers.9.mlp.upgate_proj.fuse_scales', 'model.layers.13.self_attn.qkv_proj.Qidxs', 'model.layers.2.mlp.upgate_proj.SU', 'model.layers.14.mlp.upgate_proj.fuse_scales', 'model.layers.28.mlp.upgate_proj.codebook_id', 'model.layers.6.mlp.down_proj.SU', 'model.layers.9.self_attn.o_proj.SV', 'model.layers.1.mlp.upgate_proj.Wscale', 'model.layers.26.self_attn.qkv_proj.fuse_scales', 'model.layers.16.self_attn.qkv_proj.Wscale', 'model.layers.24.mlp.upgate_proj.fuse_scales', 'model.layers.10.self_attn.o_proj.Wscale', 'model.layers.30.self_attn.qkv_proj.SU', 'model.layers.17.mlp.down_proj.Wscale', 'model.layers.28.mlp.down_proj.SU', 'model.layers.22.self_attn.o_proj.Wscale', 'model.layers.29.self_attn.o_proj.SV', 'model.layers.2.mlp.upgate_proj.Wscale', 'model.layers.5.self_attn.qkv_proj.Qidxs', 'model.layers.9.mlp.upgate_proj.Wscale', 'model.layers.31.mlp.down_proj.SU', 'model.layers.27.self_attn.qkv_proj.SU', 'model.layers.9.self_attn.o_proj.SU', 'model.layers.29.self_attn.o_proj.Wscale', 'model.layers.14.self_attn.o_proj.Qidxs', 'model.layers.30.mlp.down_proj.Qidxs', 'model.layers.2.mlp.down_proj.Qidxs', 'model.layers.21.mlp.down_proj.SU', 'model.layers.19.mlp.upgate_proj.Wscale', 'model.layers.11.self_attn.o_proj.codebook_id', 'model.layers.28.self_attn.qkv_proj.Wscale', 'model.layers.24.self_attn.qkv_proj.SV', 'model.layers.4.mlp.upgate_proj.Qidxs', 'model.layers.1.mlp.down_proj.SU', 'model.layers.7.mlp.down_proj.Wscale', 'model.layers.12.mlp.down_proj.Wscale', 'model.layers.26.self_attn.o_proj.SV', 'model.layers.5.mlp.upgate_proj.fuse_scales', 'model.layers.0.mlp.down_proj.Qidxs', 'model.layers.8.self_attn.o_proj.codebook_id', 'model.layers.20.self_attn.o_proj.Wscale', 'model.layers.18.mlp.down_proj.SV', 'model.layers.6.self_attn.o_proj.codebook_id', 'model.layers.5.mlp.down_proj.SV', 'model.layers.17.mlp.down_proj.SU', 'model.layers.29.self_attn.qkv_proj.SV', 'model.layers.5.self_attn.o_proj.SV', 'model.layers.3.mlp.upgate_proj.codebook_id', 'model.layers.6.mlp.upgate_proj.fuse_scales', 'model.layers.30.self_attn.qkv_proj.Qidxs', 'model.layers.9.self_attn.qkv_proj.SV', 'model.layers.30.mlp.down_proj.codebook_id', 'model.layers.13.self_attn.o_proj.codebook_id', 'model.layers.21.mlp.upgate_proj.Wscale', 'model.layers.4.self_attn.o_proj.SV', 'model.layers.5.self_attn.o_proj.Wscale', 'model.layers.15.self_attn.o_proj.SV', 'model.layers.10.mlp.upgate_proj.fuse_scales', 'model.layers.29.self_attn.qkv_proj.SU', 'model.layers.4.self_attn.o_proj.SU', 'model.layers.31.self_attn.o_proj.Qidxs', 'model.layers.4.self_attn.qkv_proj.SU', 'model.layers.11.mlp.upgate_proj.SU', 'model.layers.20.mlp.down_proj.SU', 'model.layers.30.self_attn.qkv_proj.SV', 'model.layers.30.mlp.upgate_proj.Wscale', 'model.layers.10.mlp.upgate_proj.Qidxs', 'model.layers.21.self_attn.qkv_proj.SU', 'model.layers.8.self_attn.qkv_proj.SV', 'model.layers.12.mlp.upgate_proj.codebook_id', 'model.layers.19.self_attn.o_proj.Qidxs', 'model.layers.31.self_attn.qkv_proj.Qidxs', 'model.layers.23.mlp.down_proj.codebook_id', 'model.layers.5.self_attn.qkv_proj.SV', 'model.layers.0.mlp.down_proj.codebook_id', 'model.layers.10.self_attn.o_proj.Qidxs', 'model.layers.26.mlp.upgate_proj.SV', 'model.layers.19.self_attn.qkv_proj.SV', 'model.layers.9.self_attn.qkv_proj.fuse_scales', 'model.layers.26.self_attn.qkv_proj.SU', 'model.layers.20.mlp.upgate_proj.SV', 'model.layers.0.mlp.upgate_proj.Wscale', 'model.layers.5.self_attn.o_proj.Qidxs', 'model.layers.4.self_attn.qkv_proj.Qidxs', 'model.layers.15.mlp.down_proj.SV', 'model.layers.16.mlp.upgate_proj.SU', 'model.layers.7.self_attn.qkv_proj.Qidxs', 'model.layers.14.mlp.upgate_proj.Wscale', 'model.layers.9.self_attn.o_proj.Qidxs', 'model.layers.18.self_attn.qkv_proj.SV', 'model.layers.6.mlp.down_proj.codebook_id', 'model.layers.3.mlp.upgate_proj.Wscale', 'model.layers.23.self_attn.qkv_proj.Qidxs', 'model.layers.0.self_attn.qkv_proj.fuse_scales', 'model.layers.14.self_attn.qkv_proj.Qidxs', 'model.layers.16.mlp.down_proj.Wscale', 'model.layers.21.mlp.down_proj.Qidxs', 'model.layers.14.mlp.down_proj.Qidxs', 'model.layers.6.self_attn.qkv_proj.SV', 'model.layers.16.self_attn.qkv_proj.fuse_scales', 'model.layers.12.mlp.upgate_proj.fuse_scales', 'model.layers.15.self_attn.qkv_proj.fuse_scales', 'model.layers.2.mlp.down_proj.codebook_id', 'model.layers.28.mlp.upgate_proj.SU', 'model.layers.27.mlp.upgate_proj.codebook_id', 'model.layers.20.mlp.down_proj.codebook_id', 'model.layers.27.self_attn.o_proj.Qidxs', 'model.layers.19.mlp.upgate_proj.fuse_scales', 'model.layers.2.self_attn.o_proj.SU', 'model.layers.21.self_attn.o_proj.Wscale', 'model.layers.12.mlp.down_proj.SU', 'model.layers.8.mlp.upgate_proj.codebook_id', 'model.layers.3.mlp.upgate_proj.Qidxs', 'model.layers.1.self_attn.qkv_proj.SV', 'model.layers.26.self_attn.qkv_proj.Wscale', 'model.layers.23.self_attn.qkv_proj.SU', 'model.layers.15.self_attn.o_proj.codebook_id', 'model.layers.24.mlp.down_proj.SV', 'model.layers.23.self_attn.qkv_proj.Wscale', 'model.layers.5.self_attn.qkv_proj.codebook_id', 'model.layers.21.self_attn.o_proj.codebook_id', 'model.layers.29.mlp.upgate_proj.SV', 'model.layers.24.self_attn.qkv_proj.fuse_scales', 'model.layers.16.self_attn.qkv_proj.SU', 'model.layers.1.self_attn.qkv_proj.codebook_id', 'model.layers.13.self_attn.o_proj.SU', 'model.layers.16.mlp.down_proj.Qidxs', 'model.layers.21.mlp.down_proj.codebook_id', 'model.layers.25.mlp.upgate_proj.Qidxs', 'model.layers.3.self_attn.qkv_proj.SV', 'model.layers.1.mlp.upgate_proj.codebook_id', 'model.layers.18.self_attn.qkv_proj.codebook_id', 'model.layers.26.mlp.upgate_proj.Wscale', 'model.layers.9.self_attn.qkv_proj.Wscale', 'model.layers.6.self_attn.o_proj.Qidxs', 'model.layers.22.self_attn.qkv_proj.Qidxs', 'model.layers.23.mlp.down_proj.Qidxs', 'model.layers.19.mlp.upgate_proj.codebook_id', 'model.layers.26.self_attn.o_proj.Wscale', 'model.layers.17.mlp.upgate_proj.codebook_id', 'model.layers.11.mlp.down_proj.SV', 'model.layers.6.mlp.down_proj.Qidxs', 'model.layers.30.mlp.upgate_proj.SU', 'model.layers.19.self_attn.qkv_proj.Wscale', 'model.layers.17.self_attn.o_proj.Qidxs', 'model.layers.7.self_attn.o_proj.Qidxs', 'model.layers.22.self_attn.qkv_proj.fuse_scales', 'model.layers.6.self_attn.qkv_proj.fuse_scales', 'model.layers.18.mlp.down_proj.Wscale', 'model.layers.4.self_attn.o_proj.codebook_id', 'model.layers.17.mlp.down_proj.Qidxs', 'model.layers.4.mlp.upgate_proj.codebook_id', 'model.layers.27.self_attn.o_proj.codebook_id', 'model.layers.30.mlp.upgate_proj.SV', 'model.layers.0.self_attn.qkv_proj.codebook_id', 'model.layers.15.mlp.upgate_proj.SU', 'model.layers.16.mlp.upgate_proj.Wscale', 'model.layers.13.self_attn.o_proj.Qidxs', 'model.layers.16.mlp.down_proj.SV', 'model.layers.30.self_attn.qkv_proj.Wscale', 'model.layers.7.mlp.upgate_proj.SU', 'model.layers.3.self_attn.qkv_proj.Wscale', 'model.layers.17.self_attn.qkv_proj.codebook_id', 'model.layers.2.mlp.down_proj.Wscale', 'model.layers.31.self_attn.o_proj.SV', 'model.layers.8.self_attn.qkv_proj.codebook_id', 'model.layers.17.self_attn.qkv_proj.fuse_scales', 'model.layers.3.self_attn.qkv_proj.Qidxs', 'model.layers.0.mlp.upgate_proj.codebook_id', 'model.layers.8.mlp.upgate_proj.SV', 'model.layers.28.self_attn.qkv_proj.SU', 'model.layers.4.mlp.down_proj.SU', 'model.layers.27.mlp.upgate_proj.SU', 'model.layers.9.self_attn.o_proj.codebook_id', 'model.layers.10.self_attn.qkv_proj.SV', 'model.layers.15.mlp.down_proj.codebook_id', 'model.layers.12.self_attn.qkv_proj.SU', 'model.layers.18.self_attn.qkv_proj.Qidxs', 'model.layers.10.self_attn.qkv_proj.codebook_id', 'model.layers.14.self_attn.o_proj.SV', 'model.layers.31.self_attn.qkv_proj.Wscale', 'model.layers.16.self_attn.qkv_proj.codebook_id', 'model.layers.20.mlp.upgate_proj.Qidxs', 'model.layers.3.mlp.down_proj.codebook_id', 'model.layers.13.self_attn.qkv_proj.Wscale', 'model.layers.30.mlp.down_proj.Wscale', 'model.layers.1.mlp.upgate_proj.SV', 'model.layers.28.self_attn.qkv_proj.fuse_scales', 'model.layers.31.mlp.upgate_proj.fuse_scales', 'model.layers.28.mlp.down_proj.SV', 'model.layers.18.self_attn.o_proj.SV', 'model.layers.0.self_attn.o_proj.codebook_id', 'model.layers.22.mlp.down_proj.SU', 'model.layers.6.self_attn.qkv_proj.codebook_id', 'model.layers.28.self_attn.o_proj.codebook_id', 'model.layers.22.mlp.upgate_proj.Qidxs', 'model.layers.18.self_attn.qkv_proj.fuse_scales', 'model.layers.24.mlp.down_proj.Wscale', 'model.layers.23.self_attn.qkv_proj.fuse_scales', 'model.layers.26.mlp.down_proj.codebook_id', 'model.layers.21.self_attn.o_proj.SU', 'model.layers.28.self_attn.o_proj.SU', 'model.layers.10.self_attn.qkv_proj.Qidxs', 'model.layers.16.mlp.down_proj.codebook_id', 'model.layers.28.mlp.upgate_proj.Wscale', 'model.layers.30.self_attn.o_proj.codebook_id', 'model.layers.31.self_attn.qkv_proj.fuse_scales', 'model.layers.14.self_attn.qkv_proj.codebook_id', 'model.layers.2.self_attn.qkv_proj.fuse_scales', 'model.layers.29.self_attn.o_proj.Qidxs', 'model.layers.21.self_attn.qkv_proj.SV', 'model.layers.2.mlp.upgate_proj.SV', 'model.layers.24.self_attn.o_proj.Wscale', 'model.layers.21.mlp.down_proj.SV', 'model.layers.4.mlp.upgate_proj.fuse_scales', 'model.layers.1.mlp.down_proj.codebook_id', 'model.layers.28.mlp.down_proj.Qidxs', 'model.layers.12.mlp.upgate_proj.Qidxs', 'model.layers.29.mlp.upgate_proj.Qidxs', 'model.layers.2.self_attn.o_proj.SV', 'model.layers.29.mlp.upgate_proj.codebook_id', 'model.layers.10.mlp.upgate_proj.SU', 'model.layers.15.mlp.down_proj.Wscale', 'model.layers.30.self_attn.o_proj.Wscale', 'model.layers.5.self_attn.qkv_proj.fuse_scales', 'model.layers.15.mlp.upgate_proj.codebook_id', 'model.layers.28.mlp.down_proj.Wscale', 'model.layers.18.mlp.upgate_proj.SU', 'model.layers.18.mlp.upgate_proj.SV', 'model.layers.23.self_attn.o_proj.Qidxs', 'model.layers.27.mlp.upgate_proj.Wscale', 'model.layers.25.self_attn.o_proj.codebook_id', 'model.layers.19.self_attn.qkv_proj.SU', 'model.layers.1.mlp.down_proj.Qidxs', 'model.layers.1.mlp.upgate_proj.SU', 'model.layers.28.mlp.upgate_proj.SV', 'model.layers.9.mlp.down_proj.codebook_id', 'model.layers.24.mlp.upgate_proj.codebook_id', 'model.layers.0.self_attn.o_proj.SV', 'model.layers.9.mlp.down_proj.SV', 'model.layers.1.self_attn.o_proj.Qidxs', 'model.layers.22.self_attn.qkv_proj.codebook_id', 'model.layers.25.mlp.upgate_proj.codebook_id', 'model.layers.1.self_attn.o_proj.codebook_id', 'model.layers.27.mlp.down_proj.SV', 'model.layers.27.self_attn.qkv_proj.fuse_scales', 'model.layers.5.self_attn.qkv_proj.SU', 'model.layers.17.self_attn.qkv_proj.SV', 'model.layers.27.self_attn.qkv_proj.SV', 'model.layers.24.self_attn.qkv_proj.Wscale', 'model.layers.15.self_attn.qkv_proj.SU', 'model.layers.22.mlp.down_proj.Wscale', 'model.layers.7.self_attn.o_proj.SV', 'model.layers.12.mlp.down_proj.Qidxs', 'model.layers.13.self_attn.o_proj.SV', 'model.layers.5.mlp.down_proj.Qidxs', 'model.layers.21.mlp.upgate_proj.fuse_scales', 'model.layers.12.mlp.down_proj.codebook_id', 'model.layers.20.mlp.upgate_proj.Wscale', 'model.layers.27.mlp.upgate_proj.fuse_scales', 'model.layers.1.self_attn.qkv_proj.Wscale', 'model.layers.0.mlp.upgate_proj.SU', 'model.layers.30.self_attn.o_proj.SU', 'model.layers.18.self_attn.o_proj.SU', 'model.layers.7.mlp.upgate_proj.Qidxs', 'model.layers.23.self_attn.qkv_proj.SV', 'model.layers.17.self_attn.o_proj.SU', 'model.layers.7.mlp.down_proj.SV', 'model.layers.30.self_attn.qkv_proj.codebook_id', 'model.layers.10.self_attn.qkv_proj.fuse_scales', 'model.layers.13.mlp.upgate_proj.SV', 'model.layers.16.self_attn.qkv_proj.Qidxs', 'model.layers.22.mlp.down_proj.codebook_id', 'model.layers.23.mlp.upgate_proj.Qidxs', 'model.layers.23.mlp.upgate_proj.fuse_scales', 'model.layers.22.self_attn.qkv_proj.SV', 'model.layers.13.self_attn.qkv_proj.SV', 'model.layers.5.mlp.upgate_proj.Wscale', 'model.layers.0.mlp.down_proj.SV', 'model.layers.20.self_attn.qkv_proj.Qidxs', 'model.layers.27.mlp.down_proj.SU', 'model.layers.19.self_attn.o_proj.codebook_id', 'model.layers.14.self_attn.o_proj.Wscale', 'model.layers.4.mlp.upgate_proj.SU', 'model.layers.20.self_attn.qkv_proj.fuse_scales', 'model.layers.10.mlp.upgate_proj.codebook_id', 'model.layers.13.mlp.upgate_proj.fuse_scales', 'model.layers.22.self_attn.o_proj.Qidxs', 'model.layers.5.self_attn.qkv_proj.Wscale', 'model.layers.22.mlp.upgate_proj.SV', 'model.layers.24.self_attn.o_proj.codebook_id', 'model.layers.25.self_attn.qkv_proj.SU', 'model.layers.6.self_attn.qkv_proj.Wscale', 'model.layers.7.mlp.upgate_proj.fuse_scales', 'model.layers.25.self_attn.qkv_proj.SV', 'model.layers.25.self_attn.qkv_proj.Wscale', 'model.layers.19.self_attn.o_proj.SV', 'model.layers.12.mlp.upgate_proj.SU', 'model.layers.10.mlp.upgate_proj.Wscale', 'model.layers.4.mlp.upgate_proj.SV', 'model.layers.4.mlp.upgate_proj.Wscale', 'model.layers.11.self_attn.o_proj.SU', 'model.layers.27.mlp.upgate_proj.Qidxs', 'model.layers.4.self_attn.qkv_proj.codebook_id', 'model.layers.2.self_attn.qkv_proj.codebook_id', 'model.layers.7.self_attn.o_proj.SU', 'model.layers.25.mlp.upgate_proj.fuse_scales', 'model.layers.12.self_attn.o_proj.SV', 'model.layers.20.self_attn.qkv_proj.SU', 'model.layers.22.self_attn.o_proj.SV', 'model.layers.16.mlp.upgate_proj.Qidxs', 'model.layers.5.mlp.upgate_proj.SU', 'model.layers.9.mlp.upgate_proj.SV', 'model.layers.4.self_attn.qkv_proj.SV', 'model.layers.4.mlp.down_proj.Qidxs', 'model.layers.26.mlp.upgate_proj.SU', 'model.layers.19.self_attn.qkv_proj.codebook_id', 'model.layers.20.self_attn.o_proj.SU', 'model.layers.29.mlp.upgate_proj.Wscale', 'model.layers.3.mlp.down_proj.SV', 'model.layers.14.self_attn.qkv_proj.fuse_scales', 'model.layers.21.self_attn.qkv_proj.Wscale', 'model.layers.26.mlp.upgate_proj.fuse_scales', 'model.layers.15.mlp.upgate_proj.fuse_scales', 'model.layers.25.self_attn.qkv_proj.codebook_id', 'model.layers.3.self_attn.qkv_proj.SU', 'model.layers.7.mlp.upgate_proj.codebook_id', 'model.layers.8.mlp.down_proj.Wscale', 'model.layers.25.mlp.down_proj.SV', 'model.layers.25.mlp.down_proj.SU', 'model.layers.22.mlp.upgate_proj.SU', 'model.layers.8.mlp.down_proj.codebook_id', 'model.layers.11.self_attn.qkv_proj.SV', 'model.layers.30.mlp.upgate_proj.Qidxs', 'model.layers.23.self_attn.o_proj.codebook_id', 'model.layers.21.mlp.upgate_proj.Qidxs', 'model.layers.5.mlp.down_proj.SU', 'model.layers.8.self_attn.qkv_proj.Qidxs', 'model.layers.31.mlp.upgate_proj.SV', 'model.layers.8.mlp.down_proj.SV', 'model.layers.18.mlp.down_proj.SU', 'model.layers.24.self_attn.qkv_proj.SU', 'model.layers.14.mlp.upgate_proj.Qidxs', 'model.layers.8.mlp.upgate_proj.Qidxs', 'model.layers.3.mlp.upgate_proj.fuse_scales', 'model.layers.27.self_attn.o_proj.SU', 'model.layers.13.mlp.down_proj.SV', 'model.layers.4.self_attn.o_proj.Wscale', 'model.layers.14.mlp.down_proj.SU', 'model.layers.9.mlp.upgate_proj.Qidxs', 'model.layers.1.mlp.upgate_proj.fuse_scales', 'model.layers.24.self_attn.o_proj.SV', 'model.layers.30.mlp.down_proj.SV', 'model.layers.20.mlp.upgate_proj.fuse_scales', 'model.layers.2.mlp.down_proj.SV', 'model.layers.23.mlp.upgate_proj.SV', 'model.layers.13.mlp.upgate_proj.Wscale', 'model.layers.11.self_attn.qkv_proj.codebook_id', 'model.layers.3.self_attn.o_proj.codebook_id', 'model.layers.8.self_attn.qkv_proj.Wscale', 'model.layers.28.self_attn.o_proj.Wscale', 'model.layers.13.mlp.upgate_proj.Qidxs', 'model.layers.19.mlp.down_proj.SU', 'model.layers.21.mlp.upgate_proj.SV', 'model.layers.17.mlp.upgate_proj.Qidxs', 'model.layers.26.mlp.down_proj.SV', 'model.layers.24.mlp.upgate_proj.SU', 'model.layers.11.self_attn.o_proj.Wscale', 'model.layers.15.mlp.upgate_proj.Qidxs', 'model.layers.11.mlp.down_proj.Wscale', 'model.layers.0.self_attn.o_proj.Wscale', 'model.layers.13.self_attn.qkv_proj.codebook_id', 'model.layers.8.mlp.down_proj.SU', 'model.layers.9.self_attn.qkv_proj.codebook_id', 'model.layers.31.mlp.down_proj.SV', 'model.layers.15.self_attn.qkv_proj.Qidxs', 'model.layers.30.self_attn.qkv_proj.fuse_scales', 'model.layers.15.self_attn.o_proj.Qidxs', 'model.layers.29.mlp.down_proj.SV', 'model.layers.1.self_attn.qkv_proj.Qidxs', 'model.layers.26.self_attn.qkv_proj.Qidxs', 'model.layers.11.self_attn.qkv_proj.Wscale', 'model.layers.5.mlp.down_proj.Wscale', 'model.layers.11.mlp.upgate_proj.Qidxs', 'model.layers.31.mlp.down_proj.Wscale', 'model.layers.10.self_attn.o_proj.codebook_id', 'model.layers.2.mlp.upgate_proj.Qidxs', 'model.layers.1.self_attn.qkv_proj.fuse_scales', 'model.layers.21.mlp.upgate_proj.SU', 'model.layers.13.mlp.upgate_proj.SU', 'model.layers.2.self_attn.qkv_proj.Qidxs', 'model.layers.0.self_attn.o_proj.Qidxs', 'model.layers.18.self_attn.o_proj.Wscale', 'model.layers.6.mlp.down_proj.Wscale', 'model.layers.7.mlp.down_proj.codebook_id', 'model.layers.12.self_attn.o_proj.Wscale', 'model.layers.27.mlp.down_proj.Wscale', 'model.layers.11.mlp.upgate_proj.fuse_scales', 'model.layers.17.self_attn.o_proj.codebook_id', 'model.layers.17.mlp.down_proj.SV', 'model.layers.7.self_attn.qkv_proj.Wscale', 'model.layers.14.self_attn.qkv_proj.SU', 'model.layers.9.mlp.down_proj.Wscale', 'model.layers.27.mlp.upgate_proj.SV', 'model.layers.29.self_attn.qkv_proj.Qidxs', 'model.layers.25.mlp.down_proj.codebook_id', 'model.layers.18.mlp.upgate_proj.fuse_scales', 'model.layers.31.self_attn.o_proj.SU', 'model.layers.5.mlp.upgate_proj.Qidxs', 'model.layers.3.mlp.down_proj.Wscale', 'model.layers.25.self_attn.o_proj.SV', 'model.layers.8.self_attn.o_proj.Wscale', 'model.layers.23.mlp.down_proj.Wscale', 'model.layers.14.self_attn.o_proj.codebook_id', 'model.layers.2.self_attn.qkv_proj.Wscale', 'model.layers.6.self_attn.o_proj.SU', 'model.layers.11.mlp.upgate_proj.Wscale', 'model.layers.19.self_attn.o_proj.Wscale', 'model.layers.6.mlp.down_proj.SV', 'model.layers.19.self_attn.qkv_proj.fuse_scales', 'model.layers.22.mlp.upgate_proj.Wscale', 'model.layers.27.mlp.down_proj.Qidxs', 'model.layers.31.self_attn.qkv_proj.codebook_id', 'model.layers.16.self_attn.o_proj.SV', 'model.layers.9.mlp.upgate_proj.codebook_id', 'model.layers.5.self_attn.o_proj.codebook_id', 'model.layers.7.mlp.upgate_proj.Wscale', 'model.layers.29.self_attn.o_proj.codebook_id', 'model.layers.24.mlp.down_proj.SU', 'model.layers.12.self_attn.o_proj.Qidxs', 'model.layers.21.self_attn.qkv_proj.fuse_scales', 'model.layers.13.mlp.upgate_proj.codebook_id', 'model.layers.14.mlp.down_proj.codebook_id', 'model.layers.16.self_attn.o_proj.codebook_id', 'model.layers.17.self_attn.o_proj.Wscale', 'model.layers.3.self_attn.qkv_proj.codebook_id', 'model.layers.8.self_attn.o_proj.SV', 'model.layers.25.self_attn.qkv_proj.fuse_scales', 'model.layers.8.mlp.upgate_proj.fuse_scales', 'model.layers.27.self_attn.qkv_proj.Qidxs', 'model.layers.23.self_attn.o_proj.Wscale', 'model.layers.12.self_attn.o_proj.SU', 'model.layers.20.mlp.upgate_proj.SU', 'model.layers.2.mlp.upgate_proj.codebook_id', 'model.layers.18.self_attn.o_proj.codebook_id', 'model.layers.29.self_attn.qkv_proj.codebook_id', 'model.layers.8.mlp.down_proj.Qidxs', 'model.layers.22.mlp.down_proj.SV', 'model.layers.8.self_attn.o_proj.SU', 'model.layers.13.self_attn.qkv_proj.SU', 'model.layers.10.self_attn.qkv_proj.Wscale', 'model.layers.4.mlp.down_proj.Wscale', 'model.layers.14.mlp.down_proj.SV', 'model.layers.1.self_attn.o_proj.SU', 'model.layers.17.self_attn.qkv_proj.Qidxs', 'model.layers.28.self_attn.qkv_proj.Qidxs', 'model.layers.16.self_attn.qkv_proj.SV', 'model.layers.3.mlp.upgate_proj.SV', 'model.layers.13.mlp.down_proj.SU', 'model.layers.10.mlp.down_proj.Qidxs', 'model.layers.27.self_attn.qkv_proj.Wscale', 'model.layers.12.self_attn.qkv_proj.fuse_scales', 'model.layers.24.self_attn.qkv_proj.codebook_id', 'model.layers.31.mlp.down_proj.Qidxs', 'model.layers.17.mlp.down_proj.codebook_id', 'model.layers.11.self_attn.qkv_proj.Qidxs', 'model.layers.22.self_attn.o_proj.SU', 'model.layers.6.mlp.upgate_proj.codebook_id', 'model.layers.6.self_attn.o_proj.Wscale', 'model.layers.5.mlp.upgate_proj.codebook_id', 'model.layers.23.mlp.upgate_proj.codebook_id', 'model.layers.11.mlp.down_proj.codebook_id', 'model.layers.26.mlp.down_proj.SU', 'model.layers.20.self_attn.o_proj.SV', 'model.layers.6.mlp.upgate_proj.SU', 'model.layers.10.mlp.down_proj.codebook_id', 'model.layers.3.mlp.down_proj.SU', 'model.layers.7.self_attn.qkv_proj.codebook_id', 'model.layers.12.self_attn.o_proj.codebook_id', 'model.layers.10.mlp.down_proj.Wscale', 'model.layers.7.self_attn.qkv_proj.SV', 'model.layers.6.self_attn.qkv_proj.SU', 'model.layers.22.self_attn.o_proj.codebook_id', 'model.layers.22.mlp.upgate_proj.fuse_scales', 'model.layers.3.mlp.down_proj.Qidxs', 'model.layers.15.self_attn.o_proj.SU', 'model.layers.21.self_attn.qkv_proj.codebook_id', 'model.layers.1.self_attn.qkv_proj.SU', 'model.layers.31.mlp.upgate_proj.Wscale', 'model.layers.8.mlp.upgate_proj.Wscale', 'model.layers.11.self_attn.qkv_proj.fuse_scales', 'model.layers.16.mlp.down_proj.SU', 'model.layers.31.mlp.upgate_proj.Qidxs', 'model.layers.12.self_attn.qkv_proj.Wscale', 'model.layers.15.mlp.down_proj.SU', 'model.layers.20.self_attn.qkv_proj.codebook_id', 'model.layers.18.self_attn.o_proj.Qidxs', 'model.layers.29.mlp.upgate_proj.fuse_scales', 'model.layers.26.self_attn.qkv_proj.SV', 'model.layers.9.self_attn.o_proj.Wscale', 'model.layers.28.mlp.upgate_proj.fuse_scales', 'model.layers.5.mlp.upgate_proj.SV', 'model.layers.14.mlp.down_proj.Wscale', 'model.layers.16.mlp.upgate_proj.codebook_id', 'model.layers.17.mlp.upgate_proj.SV', 'model.layers.28.self_attn.qkv_proj.SV', 'model.layers.2.self_attn.qkv_proj.SU', 'model.layers.1.mlp.down_proj.Wscale', 'model.layers.7.self_attn.qkv_proj.SU', 'model.layers.26.self_attn.o_proj.codebook_id', 'model.layers.18.self_attn.qkv_proj.Wscale', 'model.layers.18.mlp.upgate_proj.Wscale', 'model.layers.22.mlp.upgate_proj.codebook_id', 'model.layers.28.self_attn.o_proj.SV', 'model.layers.7.mlp.down_proj.SU', 'model.layers.18.self_attn.qkv_proj.SU', 'model.layers.9.self_attn.qkv_proj.SU', 'model.layers.14.self_attn.qkv_proj.SV', 'model.layers.7.mlp.down_proj.Qidxs', 'model.layers.20.self_attn.qkv_proj.SV', 'model.layers.7.self_attn.o_proj.codebook_id', 'model.layers.13.self_attn.o_proj.Wscale', 'model.layers.7.self_attn.o_proj.Wscale', 'model.layers.4.mlp.down_proj.SV', 'model.layers.1.self_attn.o_proj.SV', 'model.layers.13.mlp.down_proj.Wscale', 'model.layers.25.mlp.upgate_proj.SU', 'model.layers.14.mlp.upgate_proj.SV', 'model.layers.0.self_attn.qkv_proj.SV', 'model.layers.25.mlp.upgate_proj.Wscale', 'model.layers.25.mlp.down_proj.Qidxs', 'model.layers.15.mlp.upgate_proj.Wscale', 'model.layers.1.self_attn.o_proj.Wscale', 'model.layers.26.mlp.upgate_proj.Qidxs', 'model.layers.31.self_attn.qkv_proj.SU', 'model.layers.6.mlp.upgate_proj.SV']
- This IS expected if you are initializing LlamaForCausalLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
- This IS NOT expected if you are initializing LlamaForCausalLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).
Some weights of LlamaForCausalLM were not initialized from the model checkpoint at relaxml/Llama-2-7b-E8P-2Bit and are newly initialized: ['model.layers.15.self_attn.q_proj.weight', 'model.layers.24.mlp.down_proj.weight', 'model.layers.20.self_attn.v_proj.weight', 'model.layers.16.mlp.up_proj.weight', 'model.layers.21.mlp.gate_proj.weight', 'model.layers.18.self_attn.q_proj.weight', 'model.layers.3.mlp.up_proj.weight', 'model.layers.22.self_attn.o_proj.weight', 'model.layers.17.mlp.up_proj.weight', 'model.layers.26.self_attn.o_proj.weight', 'model.layers.26.mlp.gate_proj.weight', 'model.layers.23.self_attn.v_proj.weight', 'model.layers.2.self_attn.q_proj.weight', 'model.layers.21.mlp.down_proj.weight', 'model.layers.3.self_attn.o_proj.weight', 'model.layers.4.mlp.up_proj.weight', 'model.layers.5.self_attn.k_proj.weight', 'model.layers.16.self_attn.q_proj.weight', 'model.layers.20.self_attn.o_proj.weight', 'model.layers.15.mlp.up_proj.weight', 'model.layers.2.mlp.gate_proj.weight', 'model.layers.15.self_attn.o_proj.weight', 'model.layers.10.self_attn.o_proj.weight', 'model.layers.18.self_attn.o_proj.weight', 'model.layers.29.self_attn.q_proj.weight', 'model.layers.12.self_attn.v_proj.weight', 'model.layers.16.self_attn.v_proj.weight', 'model.layers.17.self_attn.k_proj.weight', 'model.layers.20.mlp.gate_proj.weight', 'model.layers.1.self_attn.o_proj.weight', 'model.layers.17.mlp.gate_proj.weight', 'model.layers.5.mlp.down_proj.weight', 'model.layers.29.mlp.down_proj.weight', 'model.layers.10.self_attn.k_proj.weight', 'model.layers.23.mlp.gate_proj.weight', 'model.layers.23.mlp.down_proj.weight', 'model.layers.24.mlp.up_proj.weight', 'model.layers.27.mlp.up_proj.weight', 'model.layers.2.self_attn.o_proj.weight', 'model.layers.26.self_attn.k_proj.weight', 'model.layers.10.mlp.gate_proj.weight', 'model.layers.28.mlp.up_proj.weight', 'model.layers.5.self_attn.o_proj.weight', 'model.layers.8.self_attn.v_proj.weight', 'model.layers.8.self_attn.o_proj.weight', 'model.layers.11.self_attn.q_proj.weight', 'model.layers.4.mlp.down_proj.weight', 'model.layers.12.self_attn.o_proj.weight', 'model.layers.12.mlp.down_proj.weight', 'model.layers.9.mlp.gate_proj.weight', 'model.layers.1.self_attn.q_proj.weight', 'model.layers.31.mlp.up_proj.weight', 'model.layers.19.self_attn.o_proj.weight', 'model.layers.23.mlp.up_proj.weight', 'model.layers.27.self_attn.k_proj.weight', 'model.layers.6.mlp.up_proj.weight', 'model.layers.17.self_attn.o_proj.weight', 'model.layers.1.mlp.down_proj.weight', 'model.layers.14.self_attn.o_proj.weight', 'model.layers.30.mlp.gate_proj.weight', 'model.layers.0.mlp.down_proj.weight', 'model.layers.3.mlp.gate_proj.weight', 'model.layers.1.mlp.gate_proj.weight', 'model.layers.19.self_attn.v_proj.weight', 'model.layers.14.self_attn.k_proj.weight', 'model.layers.16.self_attn.o_proj.weight', 'model.layers.25.mlp.down_proj.weight', 'model.layers.30.self_attn.q_proj.weight', 'model.layers.31.self_attn.k_proj.weight', 'model.layers.29.self_attn.k_proj.weight', 'model.layers.19.self_attn.k_proj.weight', 'model.layers.28.self_attn.q_proj.weight', 'model.layers.3.self_attn.q_proj.weight', 'model.layers.9.self_attn.v_proj.weight', 'model.layers.27.self_attn.q_proj.weight', 'model.layers.1.self_attn.v_proj.weight', 'model.layers.24.self_attn.q_proj.weight', 'model.layers.18.mlp.up_proj.weight', 'model.layers.6.self_attn.k_proj.weight', 'model.layers.22.self_attn.v_proj.weight', 'model.layers.22.mlp.up_proj.weight', 'model.layers.16.mlp.gate_proj.weight', 'model.layers.26.self_attn.v_proj.weight', 'model.layers.26.mlp.down_proj.weight', 'model.layers.2.self_attn.v_proj.weight', 'model.layers.7.mlp.down_proj.weight', 'model.layers.7.mlp.up_proj.weight', 'model.layers.8.mlp.up_proj.weight', 'model.layers.21.self_attn.q_proj.weight', 'model.layers.0.self_attn.v_proj.weight', 'model.layers.13.self_attn.v_proj.weight', 'model.layers.18.mlp.down_proj.weight', 'model.layers.25.self_attn.o_proj.weight', 'model.layers.7.mlp.gate_proj.weight', 'model.layers.7.self_attn.k_proj.weight', 'model.layers.11.self_attn.v_proj.weight', 'model.layers.21.self_attn.o_proj.weight', 'model.layers.21.mlp.up_proj.weight', 'model.layers.14.self_attn.q_proj.weight', 'model.layers.24.self_attn.k_proj.weight', 'model.layers.4.mlp.gate_proj.weight', 'model.layers.1.mlp.up_proj.weight', 'model.layers.20.self_attn.q_proj.weight', 'model.layers.30.self_attn.o_proj.weight', 'model.layers.8.mlp.gate_proj.weight', 'model.layers.25.self_attn.k_proj.weight', 'model.layers.3.self_attn.v_proj.weight', 'model.layers.4.self_attn.o_proj.weight', 'model.layers.23.self_attn.o_proj.weight', 'model.layers.7.self_attn.v_proj.weight', 'model.layers.22.mlp.gate_proj.weight', 'model.layers.24.self_attn.o_proj.weight', 'model.layers.2.mlp.down_proj.weight', 'model.layers.3.mlp.down_proj.weight', 'model.layers.19.mlp.gate_proj.weight', 'model.layers.15.mlp.gate_proj.weight', 'model.layers.18.self_attn.v_proj.weight', 'model.layers.12.self_attn.k_proj.weight', 'model.layers.24.mlp.gate_proj.weight', 'model.layers.22.self_attn.q_proj.weight', 'model.layers.26.mlp.up_proj.weight', 'model.layers.30.mlp.up_proj.weight', 'model.layers.3.self_attn.k_proj.weight', 'model.layers.27.mlp.down_proj.weight', 'model.layers.21.self_attn.v_proj.weight', 'model.layers.0.self_attn.q_proj.weight', 'model.layers.6.self_attn.q_proj.weight', 'model.layers.12.mlp.gate_proj.weight', 'model.layers.31.self_attn.q_proj.weight', 'model.layers.31.self_attn.o_proj.weight', 'model.layers.30.mlp.down_proj.weight', 'model.layers.31.mlp.gate_proj.weight', 'model.layers.8.self_attn.q_proj.weight', 'model.layers.20.self_attn.k_proj.weight', 'model.layers.0.mlp.gate_proj.weight', 'model.layers.1.self_attn.k_proj.weight', 'model.layers.0.self_attn.o_proj.weight', 'model.layers.8.mlp.down_proj.weight', 'model.layers.15.mlp.down_proj.weight', 'model.layers.5.mlp.gate_proj.weight', 'model.layers.0.self_attn.k_proj.weight', 'model.layers.5.self_attn.q_proj.weight', 'model.layers.18.self_attn.k_proj.weight', 'model.layers.9.self_attn.o_proj.weight', 'model.layers.27.self_attn.v_proj.weight', 'model.layers.30.self_attn.k_proj.weight', 'model.layers.28.mlp.down_proj.weight', 'model.layers.25.self_attn.v_proj.weight', 'model.layers.9.mlp.up_proj.weight', 'model.layers.11.mlp.gate_proj.weight', 'model.layers.11.mlp.down_proj.weight', 'model.layers.15.self_attn.k_proj.weight', 'model.layers.10.mlp.up_proj.weight', 'model.layers.2.mlp.up_proj.weight', 'model.layers.23.self_attn.q_proj.weight', 'model.layers.29.mlp.up_proj.weight', 'model.layers.31.mlp.down_proj.weight', 'model.layers.10.self_attn.q_proj.weight', 'model.layers.29.self_attn.v_proj.weight', 'model.layers.9.self_attn.k_proj.weight', 'model.layers.6.mlp.gate_proj.weight', 'model.layers.12.mlp.up_proj.weight', 'model.layers.23.self_attn.k_proj.weight', 'model.layers.18.mlp.gate_proj.weight', 'model.layers.19.mlp.down_proj.weight', 'model.layers.10.mlp.down_proj.weight', 'model.layers.20.mlp.down_proj.weight', 'model.layers.4.self_attn.v_proj.weight', 'model.layers.2.self_attn.k_proj.weight', 'model.layers.19.self_attn.q_proj.weight', 'model.layers.28.self_attn.v_proj.weight', 'model.layers.4.self_attn.q_proj.weight', 'model.layers.20.mlp.up_proj.weight', 'model.layers.28.self_attn.o_proj.weight', 'model.layers.19.mlp.up_proj.weight', 'model.layers.31.self_attn.v_proj.weight', 'model.layers.14.mlp.up_proj.weight', 'model.layers.14.mlp.gate_proj.weight', 'model.layers.5.self_attn.v_proj.weight', 'model.layers.13.mlp.gate_proj.weight', 'model.layers.7.self_attn.o_proj.weight', 'model.layers.13.self_attn.k_proj.weight', 'model.layers.17.self_attn.v_proj.weight', 'model.layers.29.mlp.gate_proj.weight', 'model.layers.11.self_attn.k_proj.weight', 'model.layers.14.self_attn.v_proj.weight', 'model.layers.14.mlp.down_proj.weight', 'model.layers.17.mlp.down_proj.weight', 'model.layers.25.self_attn.q_proj.weight', 'model.layers.16.mlp.down_proj.weight', 'model.layers.6.mlp.down_proj.weight', 'model.layers.22.mlp.down_proj.weight', 'model.layers.25.mlp.up_proj.weight', 'model.layers.0.mlp.up_proj.weight', 'model.layers.12.self_attn.q_proj.weight', 'model.layers.26.self_attn.q_proj.weight', 'model.layers.17.self_attn.q_proj.weight', 'model.layers.13.self_attn.o_proj.weight', 'model.layers.7.self_attn.q_proj.weight', 'model.layers.10.self_attn.v_proj.weight', 'model.layers.9.self_attn.q_proj.weight', 'model.layers.5.mlp.up_proj.weight', 'model.layers.13.self_attn.q_proj.weight', 'model.layers.13.mlp.down_proj.weight', 'model.layers.27.self_attn.o_proj.weight', 'model.layers.11.mlp.up_proj.weight', 'model.layers.24.self_attn.v_proj.weight', 'model.layers.15.self_attn.v_proj.weight', 'model.layers.16.self_attn.k_proj.weight', 'model.layers.21.self_attn.k_proj.weight', 'model.layers.28.self_attn.k_proj.weight', 'model.layers.13.mlp.up_proj.weight', 'model.layers.8.self_attn.k_proj.weight', 'model.layers.25.mlp.gate_proj.weight', 'model.layers.6.self_attn.o_proj.weight', 'model.layers.30.self_attn.v_proj.weight', 'model.layers.28.mlp.gate_proj.weight', 'model.layers.29.self_attn.o_proj.weight', 'model.layers.22.self_attn.k_proj.weight', 'model.layers.6.self_attn.v_proj.weight', 'model.layers.11.self_attn.o_proj.weight', 'model.layers.9.mlp.down_proj.weight', 'model.layers.27.mlp.gate_proj.weight', 'model.layers.4.self_attn.k_proj.weight']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.

gcc --version
gcc (Ubuntu 13.2.0-4ubuntu3) 13.2.0
Copyright (C) 2023 Free Software Foundation, Inc.
This is free software; see the source for copying conditions.  There is NO
warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.

cd quiptools && python setup.py install && cd ../
running install
/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/setuptools/command/install.py:34: SetuptoolsDeprecationWarning: setup.py install is deprecated. Use build and pip and other standards-based tools.
  warnings.warn(
/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/setuptools/command/easy_install.py:144: EasyInstallDeprecationWarning: easy_install command is deprecated. Use build and pip and other standards-based tools.
  warnings.warn(
running bdist_egg
running egg_info
writing quiptools_cuda.egg-info/PKG-INFO
writing dependency_links to quiptools_cuda.egg-info/dependency_links.txt
writing top-level names to quiptools_cuda.egg-info/top_level.txt
reading manifest file 'quiptools_cuda.egg-info/SOURCES.txt'
writing manifest file 'quiptools_cuda.egg-info/SOURCES.txt'
installing library code to build/bdist.linux-x86_64/egg
running install_lib
running build_ext
/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/torch/utils/cpp_extension.py:414: UserWarning: The detected CUDA version (12.0) has a minor version mismatch with the version that was used to compile PyTorch (12.1). Most likely this shouldn't be a problem.
  warnings.warn(CUDA_MISMATCH_WARN.format(cuda_str_version, torch.version.cuda))
/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/torch/utils/cpp_extension.py:424: UserWarning: There are no g++ version bounds defined for CUDA version 12.0
  warnings.warn(f'There are no {compiler_name} version bounds defined for CUDA version {cuda_str_version}')
building 'quiptools_cuda' extension
Emitting ninja build file /code/quip-sharp/quiptools/build/temp.linux-x86_64-cpython-311/build.ninja...
Compiling objects...
Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N)
[1/2] /usr/bin/nvcc  -I/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/torch/include -I/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/torch/include/torch/csrc/api/include -I/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/torch/include/TH -I/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/torch/include/THC -I/home/mihai/.pyenv/versions/quip/include -I/home/mihai/.pyenv/versions/3.11.7/include/python3.11 -c -c /code/quip-sharp/quiptools/quiptools.cu -o /code/quip-sharp/quiptools/build/temp.linux-x86_64-cpython-311/quiptools.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_BFLOAT16_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr --compiler-options ''"'"'-fPIC'"'"'' -O2 -g -Xcompiler -rdynamic -lineinfo -allow-unsupported-compiler -DTORCH_API_INCLUDE_EXTENSION_H '-DPYBIND11_COMPILER_TYPE="_gcc"' '-DPYBIND11_STDLIB="_libstdcpp"' '-DPYBIND11_BUILD_ABI="_cxxabi1011"' -DTORCH_EXTENSION_NAME=quiptools_cuda -D_GLIBCXX_USE_CXX11_ABI=0 -gencode=arch=compute_86,code=compute_86 -gencode=arch=compute_86,code=sm_86 -std=c++17
FAILED: /code/quip-sharp/quiptools/build/temp.linux-x86_64-cpython-311/quiptools.o 
/usr/bin/nvcc  -I/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/torch/include -I/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/torch/include/torch/csrc/api/include -I/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/torch/include/TH -I/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/torch/include/THC -I/home/mihai/.pyenv/versions/quip/include -I/home/mihai/.pyenv/versions/3.11.7/include/python3.11 -c -c /code/quip-sharp/quiptools/quiptools.cu -o /code/quip-sharp/quiptools/build/temp.linux-x86_64-cpython-311/quiptools.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_BFLOAT16_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr --compiler-options ''"'"'-fPIC'"'"'' -O2 -g -Xcompiler -rdynamic -lineinfo -allow-unsupported-compiler -DTORCH_API_INCLUDE_EXTENSION_H '-DPYBIND11_COMPILER_TYPE="_gcc"' '-DPYBIND11_STDLIB="_libstdcpp"' '-DPYBIND11_BUILD_ABI="_cxxabi1011"' -DTORCH_EXTENSION_NAME=quiptools_cuda -D_GLIBCXX_USE_CXX11_ABI=0 -gencode=arch=compute_86,code=compute_86 -gencode=arch=compute_86,code=sm_86 -std=c++17
/code/quip-sharp/quiptools/quiptools.cu(34): warning #177-D: function "gpuAssert" was declared but never referenced

Remark: The warnings can be suppressed with "-diag-suppress <warning-number>"

/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/torch/include/pybind11/detail/../cast.h: In function ‘typename pybind11::detail::type_caster<typename pybind11::detail::intrinsic_type<T>::type>::cast_op_type<T> pybind11::detail::cast_op(make_caster<T>&)’:
/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/torch/include/pybind11/detail/../cast.h:45:120: error: expected template-name before ‘<’ token
   45 |     return caster.operator typename make_caster<T>::template cast_op_type<T>();
      |                                                                                                                        ^
/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/torch/include/pybind11/detail/../cast.h:45:120: error: expected identifier before ‘<’ token
/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/torch/include/pybind11/detail/../cast.h:45:123: error: expected primary-expression before ‘>’ token
   45 |     return caster.operator typename make_caster<T>::template cast_op_type<T>();
      |                                                                                                                           ^
/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/torch/include/pybind11/detail/../cast.h:45:126: error: expected primary-expression before ‘)’ token
   45 |     return caster.operator typename make_caster<T>::template cast_op_type<T>();
      |                                                                                                                              ^
[2/2] /usr/bin/nvcc  -I/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/torch/include -I/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/torch/include/torch/csrc/api/include -I/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/torch/include/TH -I/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/torch/include/THC -I/home/mihai/.pyenv/versions/quip/include -I/home/mihai/.pyenv/versions/3.11.7/include/python3.11 -c -c /code/quip-sharp/quiptools/quiptools_e8p_gemv.cu -o /code/quip-sharp/quiptools/build/temp.linux-x86_64-cpython-311/quiptools_e8p_gemv.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_BFLOAT16_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr --compiler-options ''"'"'-fPIC'"'"'' -O2 -g -Xcompiler -rdynamic -lineinfo -allow-unsupported-compiler -DTORCH_API_INCLUDE_EXTENSION_H '-DPYBIND11_COMPILER_TYPE="_gcc"' '-DPYBIND11_STDLIB="_libstdcpp"' '-DPYBIND11_BUILD_ABI="_cxxabi1011"' -DTORCH_EXTENSION_NAME=quiptools_cuda -D_GLIBCXX_USE_CXX11_ABI=0 -gencode=arch=compute_86,code=compute_86 -gencode=arch=compute_86,code=sm_86 -std=c++17
FAILED: /code/quip-sharp/quiptools/build/temp.linux-x86_64-cpython-311/quiptools_e8p_gemv.o 
/usr/bin/nvcc  -I/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/torch/include -I/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/torch/include/torch/csrc/api/include -I/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/torch/include/TH -I/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/torch/include/THC -I/home/mihai/.pyenv/versions/quip/include -I/home/mihai/.pyenv/versions/3.11.7/include/python3.11 -c -c /code/quip-sharp/quiptools/quiptools_e8p_gemv.cu -o /code/quip-sharp/quiptools/build/temp.linux-x86_64-cpython-311/quiptools_e8p_gemv.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_BFLOAT16_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr --compiler-options ''"'"'-fPIC'"'"'' -O2 -g -Xcompiler -rdynamic -lineinfo -allow-unsupported-compiler -DTORCH_API_INCLUDE_EXTENSION_H '-DPYBIND11_COMPILER_TYPE="_gcc"' '-DPYBIND11_STDLIB="_libstdcpp"' '-DPYBIND11_BUILD_ABI="_cxxabi1011"' -DTORCH_EXTENSION_NAME=quiptools_cuda -D_GLIBCXX_USE_CXX11_ABI=0 -gencode=arch=compute_86,code=compute_86 -gencode=arch=compute_86,code=sm_86 -std=c++17
/code/quip-sharp/quiptools/quiptools_e8p_gemv.cu(85): warning #177-D: variable "shared_weights" was declared but never referenced

Remark: The warnings can be suppressed with "-diag-suppress <warning-number>"

/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/torch/include/pybind11/detail/../cast.h: In function ‘typename pybind11::detail::type_caster<typename pybind11::detail::intrinsic_type<T>::type>::cast_op_type<T> pybind11::detail::cast_op(make_caster<T>&)’:
/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/torch/include/pybind11/detail/../cast.h:45:120: error: expected template-name before ‘<’ token
   45 |     return caster.operator typename make_caster<T>::template cast_op_type<T>();
      |                                                                                                                        ^
/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/torch/include/pybind11/detail/../cast.h:45:120: error: expected identifier before ‘<’ token
/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/torch/include/pybind11/detail/../cast.h:45:123: error: expected primary-expression before ‘>’ token
   45 |     return caster.operator typename make_caster<T>::template cast_op_type<T>();
      |                                                                                                                           ^
/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/torch/include/pybind11/detail/../cast.h:45:126: error: expected primary-expression before ‘)’ token
   45 |     return caster.operator typename make_caster<T>::template cast_op_type<T>();
      |                                                                                                                              ^
ninja: build stopped: subcommand failed.
Traceback (most recent call last):
  File "/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/torch/utils/cpp_extension.py", line 2100, in _run_ninja_build
    subprocess.run(
  File "/home/mihai/.pyenv/versions/3.11.7/lib/python3.11/subprocess.py", line 571, in run
    raise CalledProcessError(retcode, process.args,
subprocess.CalledProcessError: Command '['ninja', '-v']' returned non-zero exit status 1.

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/code/quip-sharp/quiptools/setup.py", line 4, in <module>
    setup(name='quiptools_cuda',
  File "/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/setuptools/__init__.py", line 87, in setup
    return distutils.core.setup(**attrs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/setuptools/_distutils/core.py", line 185, in setup
    return run_commands(dist)
           ^^^^^^^^^^^^^^^^^^
  File "/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/setuptools/_distutils/core.py", line 201, in run_commands
    dist.run_commands()
  File "/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/setuptools/_distutils/dist.py", line 968, in run_commands
    self.run_command(cmd)
  File "/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/setuptools/dist.py", line 1217, in run_command
    super().run_command(command)
  File "/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/setuptools/_distutils/dist.py", line 987, in run_command
    cmd_obj.run()
  File "/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/setuptools/command/install.py", line 74, in run
    self.do_egg_install()
  File "/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/setuptools/command/install.py", line 123, in do_egg_install
    self.run_command('bdist_egg')
  File "/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/setuptools/_distutils/cmd.py", line 319, in run_command
    self.distribution.run_command(command)
  File "/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/setuptools/dist.py", line 1217, in run_command
    super().run_command(command)
  File "/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/setuptools/_distutils/dist.py", line 987, in run_command
    cmd_obj.run()
  File "/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/setuptools/command/bdist_egg.py", line 165, in run
    cmd = self.call_command('install_lib', warn_dir=0)
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/setuptools/command/bdist_egg.py", line 151, in call_command
    self.run_command(cmdname)
  File "/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/setuptools/_distutils/cmd.py", line 319, in run_command
    self.distribution.run_command(command)
  File "/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/setuptools/dist.py", line 1217, in run_command
    super().run_command(command)
  File "/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/setuptools/_distutils/dist.py", line 987, in run_command
    cmd_obj.run()
  File "/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/setuptools/command/install_lib.py", line 11, in run
    self.build()
  File "/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/setuptools/_distutils/command/install_lib.py", line 112, in build
    self.run_command('build_ext')
  File "/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/setuptools/_distutils/cmd.py", line 319, in run_command
    self.distribution.run_command(command)
  File "/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/setuptools/dist.py", line 1217, in run_command
    super().run_command(command)
  File "/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/setuptools/_distutils/dist.py", line 987, in run_command
    cmd_obj.run()
  File "/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/setuptools/command/build_ext.py", line 84, in run
    _build_ext.run(self)
  File "/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/setuptools/_distutils/command/build_ext.py", line 346, in run
    self.build_extensions()
  File "/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/torch/utils/cpp_extension.py", line 873, in build_extensions
    build_ext.build_extensions(self)
  File "/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/setuptools/_distutils/command/build_ext.py", line 466, in build_extensions
    self._build_extensions_serial()
  File "/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/setuptools/_distutils/command/build_ext.py", line 492, in _build_extensions_serial
    self.build_extension(ext)
  File "/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/setuptools/command/build_ext.py", line 246, in build_extension
    _build_ext.build_extension(self, ext)
  File "/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/setuptools/_distutils/command/build_ext.py", line 547, in build_extension
    objects = self.compiler.compile(
              ^^^^^^^^^^^^^^^^^^^^^^
  File "/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/torch/utils/cpp_extension.py", line 686, in unix_wrap_ninja_compile
    _write_ninja_file_and_compile_objects(
  File "/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/torch/utils/cpp_extension.py", line 1774, in _write_ninja_file_and_compile_objects
    _run_ninja_build(
  File "/home/mihai/.pyenv/versions/quip/lib/python3.11/site-packages/torch/utils/cpp_extension.py", line 2116, in _run_ninja_build
    raise RuntimeError(message) from e
RuntimeError: Error compiling objects for extension

(train) root@WIN-IUEXHKJBZLU:/home/luhao/quip-sharp# CUDA_VISIBLE_DEVICES=0 python quantize_llama.py --base_model Yi-34B-Chat --codebook E8P12
Loading checkpoint shards: 100%|████████████████████████████████████████████████████████| 15/15 [00:01<00:00, 10.49it/s]
Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.
Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.
I1204 12:02:37.810840 3560 quantize_llama.py:289] loaded model
W1204 12:02:40.726335 3560 warnings.py:109] /root/anaconda3/envs/train/lib/python3.9/site-packages/datasets/table.py:1421: FutureWarning: promote has been superseded by mode='default'.
  table = cls._concat_blocks(blocks, axis=0)

8
18
26
33
38
51
63
64
I1204 12:02:50.217392 3560 quantize_llama.py:293] loaded dataset and devset
Traceback (most recent call last):
  File "/home/luhao/quip-sharp/quantize_llama.py", line 417, in <module>
    main(args)
  File "/home/luhao/quip-sharp/quantize_llama.py", line 342, in main
    model.model.layers[i](
  File "/root/anaconda3/envs/train/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
  File "/root/anaconda3/envs/train/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl
    return forward_call(*args, **kwargs)
  File "/root/anaconda3/envs/train/lib/python3.9/site-packages/transformers/models/llama/modeling_llama.py", line 635, in forward
    hidden_states, self_attn_weights, present_key_value = self.self_attn(
  File "/root/anaconda3/envs/train/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
  File "/root/anaconda3/envs/train/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl
    return forward_call(*args, **kwargs)
  File "/root/anaconda3/envs/train/lib/python3.9/site-packages/transformers/models/llama/modeling_llama.py", line 389, in forward
    attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype)
  File "/root/anaconda3/envs/train/lib/python3.9/site-packages/torch/nn/functional.py", line 1858, in softmax
    ret = input.softmax(dim, dtype=dtype)
RuntimeError: CUDA error: unknown error
CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1.
Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.