xCount, true> &, const void *) [with Layout = cutlass::layout::RowMajor, MatrixCount = 4]: block: [24,0,0], thread: [52,0,0] Assertion `0` failed.
/niti/pytorch/cutlass-extension/include/cutlass/arch/memory_sm75.h:193: void cutlass::arch::ldsm(cutlass::Array<unsigned int, MatrixCount, true> &, const void *) [with Layout = cutlass::layout::RowMajor, MatrixCount = 4]: block: [24,0,0], thread: [53,0,0] Assertion `0` failed.
/niti/pytorch/cutlass-extension/include/cutlass/arch/memory_sm75.h:193: void cutlass::arch::ldsm(cutlass::Array<unsigned int, MatrixCount, true> &, const void *) [with Layout = cutlass::layout::RowMajor, MatrixCount = 4]: block: [24,0,0], thread: [54,0,0] Assertion `0` failed.
/niti/pytorch/cutlass-extension/include/cutlass/arch/memory_sm75.h:193: void cutlass::arch::ldsm(cutlass::Array<unsigned int, MatrixCount, true> &, const void *) [with Layout = cutlass::layout::RowMajor, MatrixCount = 4]: block: [24,0,0], thread: [55,0,0] Assertion `0` failed.
/niti/pytorch/cutlass-extension/include/cutlass/arch/memory_sm75.h:193: void cutlass::arch::ldsm(cutlass::Array<unsigned int, MatrixCount, true> &, const void *) [with Layout = cutlass::layout::RowMajor, MatrixCount = 4]: block: [24,0,0], thread: [56,0,0] Assertion `0` failed.
/niti/pytorch/cutlass-extension/include/cutlass/arch/memory_sm75.h:193: void cutlass::arch::ldsm(cutlass::Array<unsigned int, MatrixCount, true> &, const void *) [with Layout = cutlass::layout::RowMajor, MatrixCount = 4]: block: [24,0,0], thread: [57,0,0] Assertion `0` failed.
/niti/pytorch/cutlass-extension/include/cutlass/arch/memory_sm75.h:193: void cutlass::arch::ldsm(cutlass::Array<unsigned int, MatrixCount, true> &, const void *) [with Layout = cutlass::layout::RowMajor, MatrixCount = 4]: block: [24,0,0], thread: [58,0,0] Assertion `0` failed.
/niti/pytorch/cutlass-extension/include/cutlass/arch/memory_sm75.h:193: void cutlass::arch::ldsm(cutlass::Array<unsigned int, MatrixCount, true> &, const void *) [with Layout = cutlass::layout::RowMajor, MatrixCount = 4]: block: [24,0,0], thread: [59,0,0] Assertion `0` failed.
/niti/pytorch/cutlass-extension/include/cutlass/arch/memory_sm75.h:193: void cutlass::arch::ldsm(cutlass::Array<unsigned int, MatrixCount, true> &, const void *) [with Layout = cutlass::layout::RowMajor, MatrixCount = 4]: block: [24,0,0], thread: [60,0,0] Assertion `0` failed.
/niti/pytorch/cutlass-extension/include/cutlass/arch/memory_sm75.h:193: void cutlass::arch::ldsm(cutlass::Array<unsigned int, MatrixCount, true> &, const void *) [with Layout = cutlass::layout::RowMajor, MatrixCount = 4]: block: [24,0,0], thread: [61,0,0] Assertion `0` failed.
/niti/pytorch/cutlass-extension/include/cutlass/arch/memory_sm75.h:193: void cutlass::arch::ldsm(cutlass::Array<unsigned int, MatrixCount, true> &, const void *) [with Layout = cutlass::layout::RowMajor, MatrixCount = 4]: block: [24,0,0], thread: [62,0,0] Assertion `0` failed.
/niti/pytorch/cutlass-extension/include/cutlass/arch/memory_sm75.h:193: void cutlass::arch::ldsm(cutlass::Array<unsigned int, MatrixCount, true> &, const void *) [with Layout = cutlass::layout::RowMajor, MatrixCount = 4]: block: [24,0,0], thread: [63,0,0] Assertion `0` failed.
THCudaCheck FAIL file=/opt/conda/conda-bld/pytorch_1603729096996/work/aten/src/THC/THCCachingHostAllocator.cpp line=278 error=710 : device-side assert triggered
Traceback (most recent call last):
File "main.py", line 191, in <module>
main()
File "main.py", line 79, in main
train_loss, train_prec1, train_prec5= forward(
File "main.py", line 152, in forward
output, output_exp = model(inputs)
File "/opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py", line 727, in _call_impl
result = self.forward(*input, **kwargs)
File "/niti/ti_net.py", line 21, in forward
self.out, self.out_exp = self.forward_layers(x)
File "/opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py", line 727, in _call_impl
result = self.forward(*input, **kwargs)
File "/opt/conda/lib/python3.8/site-packages/torch/nn/modules/container.py", line 117, in forward
input = module(input)
File "/opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py", line 727, in _call_impl
result = self.forward(*input, **kwargs)
File "/niti/ti_torch.py", line 395, in forward
act_out, exp_out = act_calc(temp, self.act_in_exp + self.weight_exp)
File "/niti/ti_torch.py", line 108, in act_calc
int32_bitwidth = RangeEstimate(int32_acc)
File "/niti/ti_torch.py", line 374, in RangeEstimate
if range_ ==0:
RuntimeError: CUDA error: device-side assert triggered
Could you help identify the problem? thanks a lot.
My GPU is V100, with CUDA11.2 , driver version 460.91.03, docker version 20.10.12.