wenzhengzhang / seq2seqcoref Goto Github PK

bash run_scripts/train.sh \
    /home/wyb/Seq2seqCoref-main/data/output/ontonotes \
    /home/wyb/.cache/huggingface/hub/models--VincentNLP--seq2seq-coref-t0-3b-integer-free/snapshots/4e4516c3cceb1bd165039629b90f84b59bdd8b45 \
    /home/wyb/Seq2seqCoref-main/training_evaluation/model_save \
    /home/wyb/Seq2seqCoref-main/training_evaluation/predict_save \
    /home/wyb/Seq2seqCoref-main/training_evaluation/logging \
    action \
    integer \
    3e-5 \
    100 \
    4096\
    2 \
    800 \
    800 \
    100 \
    30000 \
    1

[2023-12-15 02:24:30,083] [INFO] [utils.py:827:see_memory_usage] Before initializing optimizer states
[2023-12-15 02:24:30,084] [INFO] [utils.py:828:see_memory_usage] MA 15.94 GB         Max_MA 21.25 GB         CA 27.13 GB         Max_CA 27 GB 
[2023-12-15 02:24:30,084] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory:  used = 22.04 GB, percent = 4.4%
Traceback (most recent call last):
  File "main_trainer.py", line 167, in <module>
    main()
  File "main_trainer.py", line 154, in main
    test_results = trainer.evaluate(
  File "/home/wyb/anaconda3/envs/seq2seq/lib/python3.8/site-packages/transformers/trainer_seq2seq.py", line 78, in evaluate
    return super().evaluate(eval_dataset, ignore_keys=ignore_keys, metric_key_prefix=metric_key_prefix)
  File "/home/wyb/anaconda3/envs/seq2seq/lib/python3.8/site-packages/transformers/trainer.py", line 2796, in evaluate
    output = eval_loop(
  File "/home/wyb/Seq2seqCoref-main/trainer.py", line 821, in evaluation_loop
    deepspeed_engine, _, _ = deepspeed_init(
  File "/home/wyb/anaconda3/envs/seq2seq/lib/python3.8/site-packages/transformers/deepspeed.py", line 344, in deepspeed_init
    deepspeed_engine, optimizer, _, lr_scheduler = deepspeed.initialize(**kwargs)
  File "/home/wyb/anaconda3/envs/seq2seq/lib/python3.8/site-packages/deepspeed/__init__.py", line 124, in initialize
    engine = DeepSpeedEngine(args=args,
  File "/home/wyb/anaconda3/envs/seq2seq/lib/python3.8/site-packages/deepspeed/runtime/engine.py", line 327, in __init__
    self._configure_optimizer(optimizer, model_parameters)
  File "/home/wyb/anaconda3/envs/seq2seq/lib/python3.8/site-packages/deepspeed/runtime/engine.py", line 1153, in _configure_optimizer
    self.optimizer = self._configure_zero_optimizer(basic_optimizer)
  File "/home/wyb/anaconda3/envs/seq2seq/lib/python3.8/site-packages/deepspeed/runtime/engine.py", line 1404, in _configure_zero_optimizer
    optimizer = DeepSpeedZeroOptimizer(
  File "/home/wyb/anaconda3/envs/seq2seq/lib/python3.8/site-packages/deepspeed/runtime/zero/stage_1_and_2.py", line 521, in __init__
    self.initialize_optimizer_states()
  File "/home/wyb/anaconda3/envs/seq2seq/lib/python3.8/site-packages/deepspeed/runtime/zero/stage_1_and_2.py", line 647, in initialize_optimizer_states
    self.optimizer.step()
  File "/home/wyb/anaconda3/envs/seq2seq/lib/python3.8/site-packages/torch/optim/optimizer.py", line 113, in wrapper
    return func(*args, **kwargs)
  File "/home/wyb/anaconda3/envs/seq2seq/lib/python3.8/site-packages/deepspeed/ops/adam/fused_adam.py", line 169, in step
    multi_tensor_applier(self.multi_tensor_adam,
  File "/home/wyb/anaconda3/envs/seq2seq/lib/python3.8/site-packages/deepspeed/ops/adam/multi_tensor_apply.py", line 14, in __call__
    return op(self.chunk_size, noop_flag_buffer, tensor_lists, *args)
RuntimeError: CUDA error: an illegal memory access was encountered
[2023-12-15 02:24:31,823] [INFO] [launch.py:318:sigkill_handler] Killing subprocess 88854
[2023-12-15 02:24:31,824] [ERROR] [launch.py:324:sigkill_handler] ['/home/wyb/anaconda3/envs/seq2seq/bin/python', '-u', 'main_trainer.py', '--local_rank=0', '--output_dir', '/home/wyb/Seq2seqCoref-main/training_evaluation/model_save', '--model_name_or_path', '/home/wyb/.cache/huggingface/hub/models--VincentNLP--seq2seq-coref-t0-3b-integer-free/snapshots/4e4516c3cceb1bd165039629b90f84b59bdd8b45', '--do_train', 'False', '--save_strategy', 'steps', '--load_best_model_at_end', 'True', '--metric_for_best_model', 'average_f1', '--evaluation_strategy', 'steps', '--logging_steps', '100', '--eval_steps', '800', '--data_dir', '/home/wyb/Seq2seqCoref-main/data/output/ontonotes', '--save_dir', '/home/wyb/Seq2seqCoref-main/training_evaluation/predict_save', '--per_device_train_batch_size', '1', '--per_device_eval_batch_size', '1', '--learning_rate', '3e-5', '--num_train_epochs', '100', '--logging_dir', '/home/wyb/Seq2seqCoref-main/training_evaluation/logging', '--remove_unused_columns', 'False', '--overwrite_output_dir', 'True', '--dataloader_num_workers', '0', '--predict_with_generate', 'True', '--warmup_ratio', '0.1', '--max_train_len', '2048', '--max_train_len_out', '4096', '--max_eval_len', '4096', '--max_eval_len_out', '4096', '--generation_num_beams', '4', '--generation_max_length', '4096', '--weight_decay', '0.01', '--save_predicts', 'True', '--do_predict', 'True', '--bf16', 'True', '--save_total_limit', '2', '--save_steps', '800', '--eval_delay', '30000', '--deepspeed', 'ds_configs/ds_stage2.json', '--gradient_checkpointing', 'True', '--seq2seq_type', 'action', '--mark_sentence', 'True', '--action_type', 'integer', '--align_mode', 'l', '--min_num_mentions', '2', '--add_mention_end', 'False'] exits with return code = -6