<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clip

<a class="user-mention notranslate" data-hovercard-type="user" data-hovercard-url="/us

It appears this is related to this change in go1.14: <a href="https://golang.google.cn

"fatal error: checkptr: unsafe pointer arithmetic" with -race enabled about sqlite HOT 3 CLOSED

crawshaw commented on June 12, 2024

"fatal error: checkptr: unsafe pointer arithmetic" with -race enabled

from sqlite.

Comments (3)

AdamSLevy commented on June 12, 2024 1

@zombiezen Thanks for your suggestion and example on how to address this issue.

from sqlite.

AdamSLevy commented on June 12, 2024

It appears this is related to this change in go1.14: https://golang.google.cn/doc/go1.14#compiler

This release adds -d=checkptr as a compile-time option for adding instrumentation to check that Go code is following unsafe.Pointer safety rules dynamically. This option is enabled by default (except on Windows) with the -race or -msan flags, and can be disabled with -gcflags=all=-d=checkptr=0. Specifically, -d=checkptr checks the following:

When converting unsafe.Pointer to *T, the resulting pointer must be aligned appropriately for T.
If the result of pointer arithmetic points into a Go heap object, one of the unsafe.Pointer-typed operands must point into the same object.

I'm not familiar enough with the streaming code here to precisely understand what is going on.

sqlite/session.go

Lines 511 to 609 in 251d886

 func (cg Changegroup) Output(w io.Writer) (n int, err error) { 

 xOut := newStrm(w, nil) 

 res := C.sqlite3changegroup_output_strm(cg.ptr, (*[0]byte)(C.strm_w_tramp), xOut.cptr()) 

 n = xOut.n 

 xOut.free() 

 return n, reserr("Changegroup.Output", "", "", res) 

 } 

 type strm struct { 

 id int 

 w io.Writer // one of w or r is set 

 r io.Reader 

 n int // number of bytes read or written 

 } 

 var strms = struct { 

 mu sync.Mutex 

 m map[int]*strm 

 next int 

 }{ 

 m: make(map[int]*strm), 

 } 

 func newStrm(w io.Writer, r io.Reader) *strm { 

 x := &strm{w: w, r: r} 

 strms.mu.Lock() 

 strms.next++ 

 x.id = strms.next 

 strms.m[x.id] = x 

 strms.mu.Unlock() 

 return x 

 } 

 func (x strm) free() { 

 strms.mu.Lock() 

 delete(strms.m, x.id) 

 strms.mu.Unlock() 

 } 

 func (x *strm) cptr() unsafe.Pointer { return unsafe.Pointer(uintptr(x.id)) } 

 func getStrm(cptr unsafe.Pointer) *strm { 

 strms.mu.Lock() 

 x := strms.m[int(uintptr(cptr))] 

 strms.mu.Unlock() 

 return x 

 } 

 //export strm_w_tramp 

 func strm_w_tramp(pOut unsafe.Pointer, pData *C.char, n C.int) C.int { 

 //println("strm_w_tramp start") 

 x := getStrm(pOut) 

 b := (*[1 << 30]byte)(unsafe.Pointer(pData))[:n:n] 

 for len(b) > 0 { 

 nw, err := x.w.Write(b) 

 x.n += nw 

 b = b[nw:] 

 if err != nil { 

 if code := ErrCode(err); code != SQLITE_ERROR { 

 return C.int(code) 

 } 

 return C.SQLITE_IOERR 

 } 

 } 

 //println("strm_w_tramp OK, nw=", nw) 

 return C.SQLITE_OK 

 } 

 //export strm_r_tramp 

 func strm_r_tramp(pIn unsafe.Pointer, pData *C.char, pnData *C.int) C.int { 

 x := getStrm(pIn) 

 b := (*[1 << 30]byte)(unsafe.Pointer(pData))[:*pnData:*pnData] 

 var n int 

 var err error 

 for n == 0 && err == nil { 

 // Technically an io.Reader is allowed to return (0, nil) 

 // and it is not treated as the end of the stream. 

 // 

 // So we spin here until the io.Reader is gracious enough 

 // to get off its butt and actually do something. 

 n, err = x.r.Read(b) 

 } 

 x.n += n 

 //println("*pnData:", *pnData, "n:", n) 

 *pnData = C.int(n) 

 if err != nil && err != io.EOF { 

 if code := ErrCode(err); code != SQLITE_ERROR { 

 return C.int(code) 

 } 

 return C.SQLITE_IOERR 

 } 

 return C.SQLITE_OK 

 }

from sqlite.

AdamSLevy commented on June 12, 2024

Walked through the streaming code and now I understand how it works. It appears that forcing an int into a uintptr is causing this check to fail here:

sqlite/session.go

Line 552 in 251d886

func (x *strm) cptr() unsafe.Pointer { return unsafe.Pointer(uintptr(x.id)) }

On the gopher slack I got some good suggestions. The main issue to work around here is that we can't pass a go pointer across to SQLite that it hangs onto, which is what is happening in the streaming functions. One way around this is to write an adapter function in C that converts an int to a void*. This hides this messy conversion from Go entirely. I have some initial code for this that I haven't had much time to work on or test but I'll get to it.

Another way would be to create a C pointer with malloc and use that as the strm.m map key. I personally like the C adapter approach better.

from sqlite.

"fatal error: checkptr: unsafe pointer arithmetic" with -race enabled about sqlite HOT 3 CLOSED

Comments (3)

Related Issues (20)

Recommend Projects

React

Vue.js

Typescript

TensorFlow

Django

Laravel

D3

Recommend Topics

javascript

web

server

Machine learning

Visualization

Game

Recommend Org

Facebook

Microsoft

Google

Alibaba

D3

Tencent

	func (cg Changegroup) Output(w io.Writer) (n int, err error) {
	xOut := newStrm(w, nil)
	res := C.sqlite3changegroup_output_strm(cg.ptr, (*[0]byte)(C.strm_w_tramp), xOut.cptr())
	n = xOut.n
	xOut.free()
	return n, reserr("Changegroup.Output", "", "", res)
	}

	type strm struct {
	id int
	w io.Writer // one of w or r is set
	r io.Reader
	n int // number of bytes read or written
	}

	var strms = struct {
	mu sync.Mutex
	m map[int]*strm
	next int
	}{
	m: make(map[int]*strm),
	}

	func newStrm(w io.Writer, r io.Reader) *strm {
	x := &strm{w: w, r: r}

	strms.mu.Lock()
	strms.next++
	x.id = strms.next
	strms.m[x.id] = x
	strms.mu.Unlock()

	return x
	}

	func (x strm) free() {
	strms.mu.Lock()
	delete(strms.m, x.id)
	strms.mu.Unlock()
	}

	func (x *strm) cptr() unsafe.Pointer { return unsafe.Pointer(uintptr(x.id)) }

	func getStrm(cptr unsafe.Pointer) *strm {
	strms.mu.Lock()
	x := strms.m[int(uintptr(cptr))]
	strms.mu.Unlock()

	return x
	}

	//export strm_w_tramp
	func strm_w_tramp(pOut unsafe.Pointer, pData *C.char, n C.int) C.int {
	//println("strm_w_tramp start")
	x := getStrm(pOut)
	b := (*[1 << 30]byte)(unsafe.Pointer(pData))[:n:n]
	for len(b) > 0 {
	nw, err := x.w.Write(b)
	x.n += nw
	b = b[nw:]

	if err != nil {
	if code := ErrCode(err); code != SQLITE_ERROR {
	return C.int(code)
	}
	return C.SQLITE_IOERR
	}
	}
	//println("strm_w_tramp OK, nw=", nw)
	return C.SQLITE_OK
	}

	//export strm_r_tramp
	func strm_r_tramp(pIn unsafe.Pointer, pData C.char, pnData C.int) C.int {
	x := getStrm(pIn)
	b := ([1 << 30]byte)(unsafe.Pointer(pData))[:pnData:*pnData]

	var n int
	var err error
	for n == 0 && err == nil {
	// Technically an io.Reader is allowed to return (0, nil)
	// and it is not treated as the end of the stream.
	//
	// So we spin here until the io.Reader is gracious enough
	// to get off its butt and actually do something.
	n, err = x.r.Read(b)
	}

	x.n += n
	//println("pnData:", pnData, "n:", n)
	*pnData = C.int(n)
	if err != nil && err != io.EOF {
	if code := ErrCode(err); code != SQLITE_ERROR {
	return C.int(code)
	}
	return C.SQLITE_IOERR
	}
	return C.SQLITE_OK
	}