Comments (14)
I'm not sure we've ever tested NVIDIA binaries on ARM before, so I don't have any intuition as to the problem. From the backtraces, it's clear something has gone terribly wrong. Let's try a few things to see if we can get some "better" crash results.
- Build Dyninst will full debugging (-DCMAKE_BUILD_TYPE=Debug).
- echo "int test(){ return 1; }" | gcc -fPIC -shared -o libtest.so -x c++ -
- Run
$ OMP_NUM_THREADS=1 disassemble/disassemble libtest.so
If that works, then we'll look at instrumentation.
from dyninst.
This is the output of disassemble:
"_fini" :
55c: "hint 0x0"
560: "stp X29, X30, [SP, 0xfffffffffffffff0]"
564: "add X29, SP, 0x0, lsl 0x0"
568: "ldp X29, X30, [SP], 0x10"
56c: "ret X30"
"call_weak_fn" :
470: "adrp X0, PC + 0x10000"
474: "ldr X0, [X0, 0xfe0]"
478: "cbz X0, PC + 0x8"
47c: "b PC + 0xffffffffffffffe4"
480: "ret X30"
"deregister_tm_clones" :
490: "adrp X0, PC + 0x11000"
494: "add X0, X0, 0x18, lsl 0x0"
498: "adrp X1, PC + 0x11000"
49c: "add X1, X1, 0x18, lsl 0x0"
4a0: "subs XZR, X1, X0, lsl 0x0"
4a4: "b.eq PC + 0x18"
4a8: "adrp X1, PC + 0x10000"
4ac: "ldr X1, [X1, 0xfd8]"
4b0: "cbz X1, PC + 0xc"
4b4: "mov X16, X1"
4b8: "br X16"
4bc: "ret X30"
"register_tm_clones" :
4c0: "adrp X0, PC + 0x11000"
4c4: "add X0, X0, 0x18, lsl 0x0"
4c8: "adrp X1, PC + 0x11000"
4cc: "add X1, X1, 0x18, lsl 0x0"
4d0: "sub X1, X1, X0, lsl 0x0"
4d4: "lsr X2, X1, 0x3f"
4d8: "add X1, X2, X1 ASR 0x3"
4dc: "sbfx X1, X1, 0x1, 0x3f"
4e0: "cbz X1, PC + 0x18"
4e4: "adrp X2, PC + 0x10000"
4e8: "ldr X2, [X2, 0xfd0]"
4ec: "cbz X2, PC + 0xc"
4f0: "mov X16, X2"
4f4: "br X16"
4f8: "ret X30"
"__do_global_dtors_aux" :
500: "stp X29, X30, [SP, 0xffffffffffffffe0]"
504: "add X29, SP, 0x0, lsl 0x0"
508: "str X19, [SP, 0x10]"
50c: "adrp X19, PC + 0x11000"
510: "ldrb W0, [X19, 0x18]"
514: "cbnz W0, PC + 0x28"
518: "adrp X0, PC + 0x10000"
51c: "ldr X0, [X0, 0xfc8]"
520: "cbz X0, PC + 0x10"
524: "adrp X0, PC + 0x11000"
528: "ldr X0, [X0, 0x10]"
52c: "bl PC + 0xffffffffffffff24"
530: "bl PC + 0xffffffffffffff60"
534: "mov W0, 0x1, lsl 0x0"
538: "strb W0, [X19, 0x18]"
53c: "ldr X19, [SP, 0x10]"
540: "ldp X29, X30, [SP], 0x20"
544: "ret X30"
"frame_dummy" :
550: "b PC + 0xffffffffffffff70"
"test" :
554: "mov W0, 0x1, lsl 0x0"
558: "ret X30"
"_init" :
410: "hint 0x0"
414: "stp X29, X30, [SP, 0xfffffffffffffff0]"
418: "add X29, SP, 0x0, lsl 0x0"
41c: "bl PC + 0x54"
420: "ldp X29, X30, [SP], 0x10"
424: "ret X30"
"targ450" :
450: "adrp X16, PC + 0x11000"
454: "ldr X17, [X16, 0x0]"
458: "add X16, X16, 0x0, lsl 0x0"
45c: "br X17"
"targ460" :
460: "adrp X16, PC + 0x11000"
464: "ldr X17, [X16, 0x8]"
468: "add X16, X16, 0x8, lsl 0x0"
46c: "br X17"
from dyninst.
It's interesting we found two functions without symbols. I'll leave that for another day.
Let's see if we can instrument a known function. In instrumenting_a_function.cpp, change the mode to open
, then rebuild and run the below.
$ cp instrumentAFunction/instrumenting_a_function .
$ export DYNINSTAPI_RT_LIB=...
$ ./instrumenting_a_function
$ ./InterestingProgram-rewritten
from dyninst.
Output:
Calling function InterestingProcedure
InterestingProcedure called 963542153 times
Here it has failed.
from dyninst.
Output: Calling function InterestingProcedure InterestingProcedure called 963542153 times
Here is has failed.
Did it crash after printing that information? The counter value is wrong, but I'm pretty sure that's a bug in the example code. I see the same thing on x86.
from dyninst.
It didn't crash.
from dyninst.
Patching with dyninst/examples#51 and compiling with debugging turned on for testcc, I get
Program received signal SIGSEGV, Segmentation fault.
__strlen_generic () at ../sysdeps/aarch64/multiarch/../strlen.S:98
98 ../sysdeps/aarch64/multiarch/../strlen.S: No such file or directory.
(gdb) bt
#0 __strlen_generic () at ../sysdeps/aarch64/multiarch/../strlen.S:98
dyninst/examples#1 0x0000fffff6c8fd5c in std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::operator=(char const*) ()
from /lib/aarch64-linux-gnu/libstdc++.so.6
dyninst/examples#2 0x0000fffff7e16084 in registerFunc (id=0, name=0x1 <error: Cannot access memory at address 0x1>, modName=0x500000 "_init")
at /home/tim/paradyn/examples/codeCoverage/Inst.C:84
dyninst/examples#3 0x0000000000500b64 in ?? ()
dyninst/examples#4 0x0000000000000001 in ?? ()
Backtrace stopped: previous frame identical to this frame (corrupt stack?)
@kupsch It looks like we're putting the string literals in a bad location during rewriting. This works fine on x86_64. I'm testing on ppc right now. Thoughts?
from dyninst.
either that or something is corrupting the pointer.
from dyninst.
Here's another example on the Grace Hopper system at OACISS. The step that removes linking to libdwarf is necessary to get Tau instrumentation working when using Dyninst. That's a separate issue.
$ module load e4s
$ wget http://tau.uoregon.edu/tau_releases/tau-2.33.1.tar.gz
$ tar xf tau-2.33.1.tar.gz; cd tau-2.33.1
# Remove linking to libdwarf
$ sed -i 's/-ldwarf//' utils/include/Makefile.skel include/Makefile.skel
$ ./configure -bfd=download -dyninst=/storage/packages/e4s/23.11/mvapich/spack/opt/spack/linux-rhel9-aarch64/gcc-12.2.1/dyninst-master-gr2ba27d5gp75gnctfotykb2iwvklvlq -boost=/storage/packages/e4s/23.11/mvapich/spack/opt/spack/linux-rhel9-aarch64/gcc-12.2.1/boost-1.79.0-hpeegtovmhpjk5r3qkkoygnx7hn4mg3p -tbb=/storage/packages/e4s/23.11/mvapich/spack/opt/spack/linux-rhel9-aarch64/gcc-12.2.1/intel-tbb-2020.3-tttx5qweuu6p2aote7zc6ssxv5bh7vsj
$ export LD_LIBRARY_PATH=/storage/packages/e4s/23.11/mvapich/spack/opt/spack/linux-rhel9-aarch64/gcc-12.2.1/dyninst-master-gr2ba27d5gp75gnctfotykb2iwvklvlq/lib:/usr/local/packages/SRC/tau-2.33.1/arm64_linux/lib:$LD_LIBRARY_PATH
$ export DYNINSTAPI_RT_LIB=/storage/packages/e4s/23.11/mvapich/spack/opt/spack/linux-rhel9-aarch64/gcc-12.2.1/dyninst-master-gr2ba27d5gp75gnctfotykb2iwvklvlq/lib/libdyninstAPI_RT.so
$ make install
$ cd examples/mm
$ gcc *.c -o mm -g
$ ../../utils/tau_run -T serial ./mm -o mm.i
$ ./mm> Loading libTAUsh.so ...
$ ./mm.i
WARNING: trace_register_func: id does not match invocations
WARNING: trace_register_func: id does not match invocations
WARNING: trace_register_func: id does not match invocations
WARNING: trace_register_func: id does not match invocations
WARNING: trace_register_func: id does not match invocations
WARNING: trace_register_func: id does not match invocations
terminate called after throwing an instance of 'std::length_error'
what(): vector::_M_default_append
Aborted (core dumped)
[thaines@hopper1 mm]$ gdb --args ./mm.i
Reading symbols from ./mm.i...
(gdb) r
Starting program: /storage/users/thaines/workspace/hopper/paradyn/tau-without-libdwarf/examples/mm/mm.i
warning: Unable to determine the number of hardware watchpoints available.
warning: Unable to determine the number of hardware breakpoints available.
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib64/libthread_db.so.1".
WARNING: trace_register_func: id does not match invocations
WARNING: trace_register_func: id does not match invocations
WARNING: trace_register_func: id does not match invocations
WARNING: trace_register_func: id does not match invocations
WARNING: trace_register_func: id does not match invocations
WARNING: trace_register_func: id does not match invocations
terminate called after throwing an instance of 'std::length_error'
what(): vector::_M_default_append
Program received signal SIGABRT, Aborted.
0x0000fffff7e94650 in __pthread_kill_implementation () from /lib64/libc.so.6
(gdb) bt
#0 0x0000fffff7e94650 in __pthread_kill_implementation () from /lib64/libc.so.6
dyninst/examples#1 0x0000fffff7e4f86c in raise () from /lib64/libc.so.6
dyninst/examples#2 0x0000fffff7e37030 in abort () from /lib64/libc.so.6
dyninst/examples#3 0x0000fffff66a491c in __gnu_cxx::__verbose_terminate_handler() () from /lib64/libstdc++.so.6
dyninst/examples#4 0x0000fffff66a22bc in __cxxabiv1::__terminate(void (*)()) () from /lib64/libstdc++.so.6
dyninst/examples#5 0x0000fffff66a2320 in std::terminate() () from /lib64/libstdc++.so.6
dyninst/examples#6 0x0000fffff66a2604 in __cxa_throw () from /lib64/libstdc++.so.6
dyninst/examples#7 0x0000fffff669d900 in std::__throw_length_error(char const*) () from /lib64/libstdc++.so.6
dyninst/examples#8 0x0000fffff7cf38bc in std::vector<void*, std::allocator<void*> >::_M_check_len(unsigned long, char const*) const ()
from /home/users/thaines/workspace/hopper/paradyn/tau-without-libdwarf/arm64_linux/lib/libTAUsh.so
dyninst/examples#9 0x0000fffff7cf2e84 in std::vector<void*, std::allocator<void*> >::_M_default_append(unsigned long) ()
from /home/users/thaines/workspace/hopper/paradyn/tau-without-libdwarf/arm64_linux/lib/libTAUsh.so
dyninst/examples#10 0x0000fffff7cf20f8 in std::vector<void*, std::allocator<void*> >::resize(unsigned long) ()
from /home/users/thaines/workspace/hopper/paradyn/tau-without-libdwarf/arm64_linux/lib/libTAUsh.so
dyninst/examples#11 0x0000fffff7cf1060 in trace_register_func () from /home/users/thaines/workspace/hopper/paradyn/tau-without-libdwarf/arm64_linux/lib/libTAUsh.so
dyninst/examples#12 0x0000000000501c7c in ?? ()
dyninst/examples#13 0x0000fffff7cd7d48 in Tau_global_incr_insideTAU () from /home/users/thaines/workspace/hopper/paradyn/tau-without-libdwarf/arm64_linux/lib/libTAUsh.so
dyninst/examples#14 0x0000fffff7cd7d48 in Tau_global_incr_insideTAU () from /home/users/thaines/workspace/hopper/paradyn/tau-without-libdwarf/arm64_linux/lib/libTAUsh.so
dyninst/examples#15 0x0000fffff7cd7dc8 in Tau_global_decr_insideTAU () from /home/users/thaines/workspace/hopper/paradyn/tau-without-libdwarf/arm64_linux/lib/libTAUsh.so
dyninst/examples#16 0x0000fffff7e0cd68 in guard variable for TheTauBinDynFI()::FuncTauDynFI ()
from /home/users/thaines/workspace/hopper/paradyn/tau-without-libdwarf/arm64_linux/lib/libTAUsh.so
Backtrace stopped: previous frame identical to this frame (corrupt stack?)
from dyninst.
The problem is that ARM rewriting isn't adhering to the parameter-passing ABI. We should always be passing the second parameter in x1, but each call uses the next-hightest register number. In fact, we eventually go beyond x7 (the highest permitted register for parameters) to x14.
501a18: d2800000 mov x0, #0x0 // #0
501a1c: f2a00a00 movk x0, #0x50, lsl dyninst/examples#16
501a20: d2800001 mov x1, #0x0 // #0
501a24: 10ff65a2 adr x2, 5006d8 <trace_register_func@Base>
01a34: d2800f00 mov x0, #0x78 // #120
501a38: f2a00a00 movk x0, #0x50, lsl dyninst/examples#16
501a3c: d2800022 mov x2, #0x1 // dyninst/examples#1
501a40: 10ff64c3 adr x3, 5006d8 <trace_register_func@Base>
501a58: d2801d00 mov x0, #0xe8 // #232
501a5c: f2a00a00 movk x0, #0x50, lsl dyninst/examples#16
501a60: d2800043 mov x3, #0x2 // dyninst/examples#2
501a64: 10ff63a4 adr x4, 5006d8 <trace_register_func@Base>
501a84: d2802b00 mov x0, #0x158 // #344
501a88: f2a00a00 movk x0, #0x50, lsl dyninst/examples#16
501a8c: d2800064 mov x4, #0x3 // dyninst/examples#3
501a90: 10ff6245 adr x5, 5006d8 <trace_register_func@Base>
...
501df4: d2807000 mov x0, #0x380 // #896
501df8: f2a00a00 movk x0, #0x50, lsl dyninst/examples#16
501dfc: d28001ae mov x14, #0xd // dyninst/examples#13
501e00: 10ff46cf adr x15, 5006d8 <trace_register_func@Base>
from dyninst.
Hello,
are there any news regarding this issue?
from dyninst.
We've narrowed it down the the instruction emitter for ARM. The problem is that there is only one person working on Dyninst now (I'm volunteering), and neither of us knows much about ARM. I tried taking a look, but didn't get very far. Could you give us an idea of what you are doing here that this bug is impeding your progress?
from dyninst.
I've transferred this issue here from dyninst/examples so it will be more visible and because it's really a Dyninst issue.
from dyninst.
See also
from dyninst.
Related Issues (20)
- aarch64 stackwalking needs work. HOT 1
- Segmentation fault on libc-2.27.so:i386
- Make Parser::getSyscallNumber architecture-agnostic and handle multiple syscall arguments
- Issues with Static Instruction Insertion using Dyninst HOT 4
- Difficulty Inserting Instructions Using Dyninst Toolkit: Seeking Unified Solution HOT 1
- GitHub Release for 13.0.0 HOT 3
- release cmake configury: unable to install under /usr/lib64/... HOT 7
- AMD GPU Instruction Decoder Failed to expand operand HOT 1
- Upgrading SelectiveTaint: Replacing dyninst9.3 with dyninst13.0 in Ubuntu 14.04 Environment HOT 13
- (Unintentional) dependency in C++ static constructor order; causes SIGFPE when source files are reordered HOT 1
- How to instrument a non-x86_64 binary with dyninst library built upon x86_64 ? HOT 2
- Substantial increase in build times with gcc-14. HOT 1
- Build failures with gcc-14 HOT 1
- strange runtime error: odd - couldn't find kept register 2 HOT 1
- Instruction Operation missing x86 FLAG read/write properties
- Just upgrde gcc version to 8.5.0, also do those soft link trick,but found issue"Dyninst requires gcc >= 6.0" HOT 1
- Segmentation fault when -O2 used to compile application. HOT 4
- Missing entries in DWARF line information lookup HOT 1
- Incorrect jump table detection HOT 2
Recommend Projects
-
React
A declarative, efficient, and flexible JavaScript library for building user interfaces.
-
Vue.js
🖖 Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.
-
Typescript
TypeScript is a superset of JavaScript that compiles to clean JavaScript output.
-
TensorFlow
An Open Source Machine Learning Framework for Everyone
-
Django
The Web framework for perfectionists with deadlines.
-
Laravel
A PHP framework for web artisans
-
D3
Bring data to life with SVG, Canvas and HTML. 📊📈🎉
-
Recommend Topics
-
javascript
JavaScript (JS) is a lightweight interpreted programming language with first-class functions.
-
web
Some thing interesting about web. New door for the world.
-
server
A server is a program made to process requests and deliver data to clients.
-
Machine learning
Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.
-
Visualization
Some thing interesting about visualization, use data art
-
Game
Some thing interesting about game, make everyone happy.
Recommend Org
-
Facebook
We are working to build community through open source technology. NB: members must have two-factor auth.
-
Microsoft
Open source projects and samples from Microsoft.
-
Google
Google ❤️ Open Source for everyone.
-
Alibaba
Alibaba Open Source for everyone
-
D3
Data-Driven Documents codes.
-
Tencent
China tencent open source team.
from dyninst.