On a large scan the following was encountered when trying to write out to a file at the duration of the scan. This should probably be worked on with #40 as the solution to this likely aligns up nicely,
fatal error: runtime: out of memory
runtime stack:
runtime.throw(0x6db5a8, 0x16)
/usr/lib/go-1.10/src/runtime/panic.go:616 +0x81
runtime.sysMap(0xc75d9e0000, 0x197ad0000, 0x456300, 0x8738f8)
/usr/lib/go-1.10/src/runtime/mem_linux.go:216 +0x20a
runtime.(*mheap).sysAlloc(0x85a7e0, 0x197ad0000, 0x7f51add9e5a8)
/usr/lib/go-1.10/src/runtime/malloc.go:470 +0xd4
runtime.(*mheap).grow(0x85a7e0, 0xcbd67, 0x0)
/usr/lib/go-1.10/src/runtime/mheap.go:907 +0x60
runtime.(*mheap).allocSpanLocked(0x85a7e0, 0xcbd67, 0x873908, 0x7f51acef0d88)
/usr/lib/go-1.10/src/runtime/mheap.go:820 +0x301
runtime.(*mheap).alloc_m(0x85a7e0, 0xcbd67, 0x410100, 0xc41fb8c2ff)
/usr/lib/go-1.10/src/runtime/mheap.go:686 +0x118
runtime.(*mheap).alloc.func1()
/usr/lib/go-1.10/src/runtime/mheap.go:753 +0x4d
runtime.(*mheap).alloc(0x85a7e0, 0xcbd67, 0x7f51ac010100, 0x41566c)
/usr/lib/go-1.10/src/runtime/mheap.go:752 +0x8a
runtime.largeAlloc(0x197ace000, 0x450001, 0x7f51ade8ad90)
/usr/lib/go-1.10/src/runtime/malloc.go:826 +0x94
runtime.mallocgc.func1()
/usr/lib/go-1.10/src/runtime/malloc.go:721 +0x46
runtime.systemstack(0xc400000000)
/usr/lib/go-1.10/src/runtime/asm_amd64.s:409 +0x79
runtime.mstart()
/usr/lib/go-1.10/src/runtime/proc.go:1175
goroutine 1 [running]:
runtime.systemstack_switch()
/usr/lib/go-1.10/src/runtime/asm_amd64.s:363 fp=0xc423969b18 sp=0xc423969b10 pc=0x453820
runtime.mallocgc(0x197ace000, 0x670f00, 0xc4202c1c01, 0x350)
/usr/lib/go-1.10/src/runtime/malloc.go:720 +0x8a2 fp=0xc423969bb8 sp=0xc423969b18 pc=0x411c42
runtime.growslice(0x670f00, 0xc42df20000, 0x14623e00, 0x14623e00, 0x14623e35, 0x35, 0x40, 0x35)
/usr/lib/go-1.10/src/runtime/slice.go:179 +0x14a fp=0xc423969c20 sp=0xc423969bb8 pc=0x44027a
github.com/Ice3man543/subfinder/libsubfinder/engines/passive.PassiveDiscovery(0xc42012a000, 0x34, 0x0, 0x0)
/root/go/src/github.com/Ice3man543/subfinder/libsubfinder/engines/passive/passive.go:379 +0x797 fp=0xc423969e28 sp=0xc423969c20 pc=0x63f8d7
main.main()
/opt/Web_Enumeration/subfinder/main.go:144 +0xf1d fp=0xc423969f88 sp=0xc423969e28 pc=0x642b0d
runtime.main()
/usr/lib/go-1.10/src/runtime/proc.go:198 +0x212 fp=0xc423969fe0 sp=0xc423969f88 pc=0x42c362
runtime.goexit()
/usr/lib/go-1.10/src/runtime/asm_amd64.s:2361 +0x1 fp=0xc423969fe8 sp=0xc423969fe0 pc=0x456371
goroutine 21617 [IO wait]:
internal/poll.runtime_pollWait(0x7f51ac16b090, 0x72, 0xc423c30858)
/usr/lib/go-1.10/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc42793b218, 0x72, 0xffffffffffffff00, 0x70c480, 0x81e478)
/usr/lib/go-1.10/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc42793b218, 0xc429598000, 0x2000, 0x2000)
/usr/lib/go-1.10/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc42793b200, 0xc429598000, 0x2000, 0x2000, 0x0, 0x0, 0x0)
/usr/lib/go-1.10/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc42793b200, 0xc429598000, 0x2000, 0x2000, 0x4336e9, 0xc420071680, 0x200000003)
/usr/lib/go-1.10/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc42d320288, 0xc429598000, 0x2000, 0x2000, 0x0, 0x0, 0x0)
/usr/lib/go-1.10/src/net/net.go:176 +0x6a
crypto/tls.(*block).readFromUntil(0xc42db70f00, 0x7f51ade58178, 0xc42d320288, 0x5, 0xc42d320288, 0xc423c30a10)
/usr/lib/go-1.10/src/crypto/tls/conn.go:493 +0x96
crypto/tls.(*Conn).readRecord(0xc42315a700, 0x6eb417, 0xc42315a820, 0xc420071680)
/usr/lib/go-1.10/src/crypto/tls/conn.go:595 +0xe0
crypto/tls.(*Conn).Read(0xc42315a700, 0xc42140b000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/lib/go-1.10/src/crypto/tls/conn.go:1156 +0x100
bufio.(*Reader).Read(0xc429e121e0, 0xc42012c038, 0x9, 0x9, 0xc42315aa30, 0x58fb00, 0xc42315aa30)
/usr/lib/go-1.10/src/bufio/bufio.go:216 +0x238
io.ReadAtLeast(0x70b660, 0xc429e121e0, 0xc42012c038, 0x9, 0x9, 0x9, 0x5e1ba1, 0xc42315a700, 0xc42140a000)
/usr/lib/go-1.10/src/io/io.go:309 +0x86
io.ReadFull(0x70b660, 0xc429e121e0, 0xc42012c038, 0x9, 0x9, 0x60e134, 0x7f51ade0fd68, 0xc42315a700)
/usr/lib/go-1.10/src/io/io.go:327 +0x58
net/http.http2readFrameHeader(0xc42012c038, 0x9, 0x9, 0x70b660, 0xc429e121e0, 0x0, 0xc400000000, 0x429f04, 0x6eb470)
/usr/lib/go-1.10/src/net/http/h2_bundle.go:1517 +0x7b
net/http.(*http2Framer).ReadFrame(0xc42012c000, 0xc427b91560, 0x0, 0x0, 0x0)
/usr/lib/go-1.10/src/net/http/h2_bundle.go:1775 +0xa4
net/http.(*http2clientConnReadLoop).run(0xc423c30fb8, 0x6eaef8, 0xc4200307b8)
/usr/lib/go-1.10/src/net/http/h2_bundle.go:8028 +0x90
net/http.(*http2ClientConn).readLoop(0xc420c88000)
/usr/lib/go-1.10/src/net/http/h2_bundle.go:7956 +0x68
created by net/http.(*http2Transport).newClientConn
/usr/lib/go-1.10/src/net/http/h2_bundle.go:7143 +0x65c
goroutine 21633 [IO wait]:
internal/poll.runtime_pollWait(0x7f51ac16b4a0, 0x72, 0xc423bdf858)
/usr/lib/go-1.10/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc421336398, 0x72, 0xffffffffffffff00, 0x70c480, 0x81e478)
/usr/lib/go-1.10/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc421336398, 0xc420164000, 0x4000, 0x4000)
/usr/lib/go-1.10/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc421336380, 0xc420164000, 0x4000, 0x4000, 0x0, 0x0, 0x0)
/usr/lib/go-1.10/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc421336380, 0xc420164000, 0x4000, 0x4000, 0x10, 0x1, 0x100000000000000)
/usr/lib/go-1.10/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc42d320320, 0xc420164000, 0x4000, 0x4000, 0x0, 0x0, 0x0)
/usr/lib/go-1.10/src/net/net.go:176 +0x6a
crypto/tls.(*block).readFromUntil(0xc423fd7080, 0x7f51ade58178, 0xc42d320320, 0x5, 0xc42d320320, 0x2c0)
/usr/lib/go-1.10/src/crypto/tls/conn.go:493 +0x96
crypto/tls.(*Conn).readRecord(0xc420097880, 0x6eb417, 0xc4200979a0, 0xc420457380)
/usr/lib/go-1.10/src/crypto/tls/conn.go:595 +0xe0
crypto/tls.(*Conn).Read(0xc420097880, 0xc423159000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/lib/go-1.10/src/crypto/tls/conn.go:1156 +0x100
bufio.(*Reader).Read(0xc4201d8cc0, 0xc4203da658, 0x9, 0x9, 0xc423bdfc50, 0x45c763, 0xc423bdfc60)
/usr/lib/go-1.10/src/bufio/bufio.go:216 +0x238
io.ReadAtLeast(0x70b660, 0xc4201d8cc0, 0xc4203da658, 0x9, 0x9, 0x9, 0xc42054ac01, 0xc423bdfd00, 0x5ebe2b)
/usr/lib/go-1.10/src/io/io.go:309 +0x86
io.ReadFull(0x70b660, 0xc4201d8cc0, 0xc4203da658, 0x9, 0x9, 0xc420086030, 0xc423bdfd30, 0x5ebd3d)
/usr/lib/go-1.10/src/io/io.go:327 +0x58
net/http.http2readFrameHeader(0xc4203da658, 0x9, 0x9, 0x70b660, 0xc4201d8cc0, 0x0, 0x0, 0x0, 0x0)
/usr/lib/go-1.10/src/net/http/h2_bundle.go:1517 +0x7b
net/http.(*http2Framer).ReadFrame(0xc4203da620, 0xc42089e240, 0x0, 0x0, 0x0)
/usr/lib/go-1.10/src/net/http/h2_bundle.go:1775 +0xa4
net/http.(*http2clientConnReadLoop).run(0xc423bdffb8, 0x6eaef8, 0xc4200317b8)
/usr/lib/go-1.10/src/net/http/h2_bundle.go:8028 +0x90
net/http.(*http2ClientConn).readLoop(0xc4291ca540)
/usr/lib/go-1.10/src/net/http/h2_bundle.go:7956 +0x68
created by net/http.(*http2Transport).newClientConn
/usr/lib/go-1.10/src/net/http/h2_bundle.go:7143 +0x65c
goroutine 21494 [IO wait]:
internal/poll.runtime_pollWait(0x7f51addca820, 0x72, 0xc428f832a8)
/usr/lib/go-1.10/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc4295b9d18, 0x72, 0xffffffffffffff00, 0x70c480, 0x81e478)
/usr/lib/go-1.10/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc4295b9d18, 0xc4295a0900, 0x364b, 0x364b)
/usr/lib/go-1.10/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc4295b9d00, 0xc4295a09b5, 0x364b, 0x364b, 0x0, 0x0, 0x0)
/usr/lib/go-1.10/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc4295b9d00, 0xc4295a09b5, 0x364b, 0x364b, 0x52b, 0x0, 0x0)
/usr/lib/go-1.10/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc4213ba2a0, 0xc4295a09b5, 0x364b, 0x364b, 0x0, 0x0, 0x0)
/usr/lib/go-1.10/src/net/net.go:176 +0x6a
crypto/tls.(*block).readFromUntil(0xc420175650, 0x7f51ade58178, 0xc4213ba2a0, 0x2025, 0xc4213ba2a0, 0x0)
/usr/lib/go-1.10/src/crypto/tls/conn.go:493 +0x96
crypto/tls.(*Conn).readRecord(0xc4205c2e00, 0x6eb417, 0xc4205c2f20, 0x85)
/usr/lib/go-1.10/src/crypto/tls/conn.go:640 +0x1fa
crypto/tls.(*Conn).Read(0xc4205c2e00, 0xc429026000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/lib/go-1.10/src/crypto/tls/conn.go:1156 +0x100
net/http.(*persistConn).Read(0xc421188000, 0xc429026000, 0x1000, 0x1000, 0x0, 0xc42734c000, 0x8000)
/usr/lib/go-1.10/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc427600c60)
/usr/lib/go-1.10/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).ReadSlice(0xc427600c60, 0x70b60a, 0xc42be34a80, 0x70c820, 0xc4295c4000, 0xc42be34a80, 0xa)
/usr/lib/go-1.10/src/bufio/bufio.go:341 +0x2c
net/http/internal.readChunkLine(0xc427600c60, 0x1, 0x0, 0x0, 0x0, 0xc428f837c8)
/usr/lib/go-1.10/src/net/http/internal/chunked.go:122 +0x34
net/http/internal.(*chunkedReader).beginChunk(0xc42019f1a0)
/usr/lib/go-1.10/src/net/http/internal/chunked.go:48 +0x32
net/http/internal.(*chunkedReader).Read(0xc42019f1a0, 0xc42959e000, 0x1000, 0x1000, 0xc428f83880, 0x429609, 0x8)
/usr/lib/go-1.10/src/net/http/internal/chunked.go:93 +0x113
net/http.(*body).readLocked(0xc4201e9e40, 0xc42959e000, 0x1000, 0x1000, 0xc421581640, 0xc428f83a18, 0x5b4d4b)
/usr/lib/go-1.10/src/net/http/transfer.go:778 +0x61
net/http.(*body).Read(0xc4201e9e40, 0xc42959e000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/lib/go-1.10/src/net/http/transfer.go:770 +0xdd
net/http.(*bodyEOFSignal).Read(0xc4201e9e80, 0xc42959e000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/lib/go-1.10/src/net/http/transport.go:2187 +0xdc
bufio.(*Reader).fill(0xc42be34a80)
/usr/lib/go-1.10/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).ReadByte(0xc42be34a80, 0x3b, 0x0, 0x0)
/usr/lib/go-1.10/src/bufio/bufio.go:242 +0x39
compress/flate.(*decompressor).moreBits(0xc4295c4000, 0x0, 0x0)
/usr/lib/go-1.10/src/compress/flate/inflate.go:694 +0x37
compress/flate.(*decompressor).huffSym(0xc4295c4000, 0xc4295c4028, 0x13, 0x0, 0x0)
/usr/lib/go-1.10/src/compress/flate/inflate.go:716 +0x4f
compress/flate.(*decompressor).huffmanBlock(0xc4295c4000)
/usr/lib/go-1.10/src/compress/flate/inflate.go:494 +0xf3
compress/flate.(*decompressor).nextBlock(0xc4295c4000)
/usr/lib/go-1.10/src/compress/flate/inflate.go:327 +0x1a1
compress/flate.(*decompressor).Read(0xc4295c4000, 0xc4295c0600, 0x200, 0x200, 0x0, 0x0, 0x0)
/usr/lib/go-1.10/src/compress/flate/inflate.go:347 +0x72
compress/gzip.(*Reader).Read(0xc42525b600, 0xc4295c0600, 0x200, 0x200, 0x0, 0xc428f83c70, 0x429f04)
/usr/lib/go-1.10/src/compress/gzip/gunzip.go:251 +0x96
net/http.(*gzipReader).Read(0xc4203ee4c0, 0xc4295c0600, 0x200, 0x200, 0x4685c1, 0x6710c0, 0x200)
/usr/lib/go-1.10/src/net/http/transport.go:2250 +0xe7
net/http.(*cancelTimerBody).Read(0xc4203ee4e0, 0xc4295c0600, 0x200, 0x200, 0x200, 0x200, 0xc4295c0600)
/usr/lib/go-1.10/src/net/http/client.go:820 +0x5c
bytes.(*Buffer).ReadFrom(0xc420391c00, 0x7f51ade0e3b8, 0xc4203ee4e0, 0xc428f83dc0, 0x613671, 0xc4201f4450)
/usr/lib/go-1.10/src/bytes/buffer.go:205 +0xa0
io/ioutil.readAll(0x7f51ade0e3b8, 0xc4203ee4e0, 0x200, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/lib/go-1.10/src/io/ioutil/ioutil.go:36 +0xb5
io/ioutil.ReadAll(0x7f51ade0e3b8, 0xc4203ee4e0, 0xc4203ee4e0, 0x7f51ade0e3b8, 0xc4203ee4e0, 0x0, 0x11)
/usr/lib/go-1.10/src/io/ioutil/ioutil.go:45 +0x3e
github.com/Ice3man543/subfinder/libsubfinder/sources/bing.Query(0xc42012a000, 0xc420bf4060)
/root/go/src/github.com/Ice3man543/subfinder/libsubfinder/sources/bing/bing.go:53 +0x6d5
created by github.com/Ice3man543/subfinder/libsubfinder/engines/passive.Discover
/root/go/src/github.com/Ice3man543/subfinder/libsubfinder/engines/passive/passive.go:148 +0x728
goroutine 21772 [select]:
net/http.setRequestCancel.func3(0x0, 0xc4201f4540, 0xc422f2a0f0, 0xc423b81f8c, 0xc42be6b2c0)
/usr/lib/go-1.10/src/net/http/client.go:323 +0xfa
created by net/http.setRequestCancel
/usr/lib/go-1.10/src/net/http/client.go:322 +0x2ac
goroutine 21800 [select]:
net/http.(*persistConn).writeLoop(0xc421188000)
/usr/lib/go-1.10/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
/usr/lib/go-1.10/src/net/http/transport.go:1238 +0x97f
goroutine 21799 [select]:
net/http.(*persistConn).readLoop(0xc421188000)
/usr/lib/go-1.10/src/net/http/transport.go:1717 +0x743
created by net/http.(*Transport).dialConn
/usr/lib/go-1.10/src/net/http/transport.go:1237 +0x95a