Hi, I have an issue with Cherami when I try to setup a local cluster using multiple docker cherami-server instances.
I might have the wrong configs or some bad setup, but I would appreciate your help.
time="2017-03-24T10:31:26Z" level=info msg="New Frontend" deploymentName= frntID=039683e9 serviceName=cherami-frontendhost
time="2017-03-24T10:31:26Z" level=info msg="RuntimeMetricsReporter started"
time="2017-03-24T10:31:26Z" level=info msg="tChannel listening 172.17.0.5:4254"
time="2017-03-24T10:31:26Z" level=debug msg=RingHosts RingHosts=[172.17.0.5 172.17.0.6] hostIP=172.17.0.5
time="2017-03-24T10:31:26Z" level=info msg="service started" hostport="172.17.0.5:4254" service=cherami-outputhost
time="2017-03-24T10:31:26Z" level=info msg="Load reporter started." deploymentName= hostIP="172.17.0.5:4254" outhID=039683e9
time="2017-03-24T10:31:26Z" level=info msg="HostIDHeartbeater started" deploymentName= hostAddr="172.17.0.5:4254" hostID=039683e9-e15f-417d-beb6-764cbf206626 hostName=35e0d2b285ff outhID=039683e9
time="2017-03-24T10:31:26Z" level=info msg="Diagnostic http endpoint listening on 127.0.0.1:14254"
time="2017-03-24T10:31:26Z" level=info msg="WebSocket listening 172.17.0.5:6190"
time="2017-03-24T10:31:26Z" level=info msg="RingpopMonitor worker loop started"
time="2017-03-24T10:31:26Z" level=info msg="Config refresher started" deploymentName= outhID=039683e9
time="2017-03-24T10:31:26Z" level=debug msg="Ring membership updated, new membershipMap" newembershipMap=map[cherami-controllerhost:0xc4202f3360 cherami-frontendhost:0xc4202f33a0 cherami-replicator:0xc4202f33e0 cherami-inputhost:0xc4202f32a0 cherami-outputhost:0xc4202a4c00 cherami-storehost:0xc4202f3320]
time="2017-03-24T10:31:26Z" level=info msg="RuntimeMetricsReporter started"
time="2017-03-24T10:31:26Z" level=info msg="tChannel listening 172.17.0.5:4253"
time="2017-03-24T10:31:26Z" level=debug msg=RingHosts RingHosts=[172.17.0.5 172.17.0.6] hostIP=172.17.0.5
time="2017-03-24T10:31:26Z" level=info msg="RingpopMonitor worker loop started"
time="2017-03-24T10:31:26Z" level=info msg="service started" hostport="172.17.0.5:4253" service=cherami-storehost
time="2017-03-24T10:31:26Z" level=info msg="HostIDHeartbeater started" deploymentName= hostAddr="172.17.0.5:4253" hostID=11111111-1111-1111-1111-111111111111 hostName=35e0d2b285ff storID=11111111
time="2017-03-24T10:31:26Z" level=info msg="StorageMonitor: started" deploymentName= storID=11111111
time="2017-03-24T10:31:26Z" level=info msg="Load reporter started." deploymentName= storID=11111111
time="2017-03-24T10:31:26Z" level=info msg="extStatsReporter: started" deploymentName= storID=11111111
time="2017-03-24T10:31:26Z" level=info msg="StoreHost: started" deploymentName= options="Store=UNKNOWN BaseDir=/tmp/cherami-store" storID=11111111
time="2017-03-24T10:31:26Z" level=info msg="Diagnostic http endpoint listening on 127.0.0.1:14253"
time="2017-03-24T10:31:26Z" level=info msg="ReplicationJobRunner: started" deploymentName= storID=11111111
time="2017-03-24T10:31:26Z" level=info msg="extStatsReporter: reporterPump started" deploymentName= storID=11111111
time="2017-03-24T10:31:26Z" level=info msg="WebSocket listening 172.17.0.5:6191"
time="2017-03-24T10:31:26Z" level=info msg="extStatsReporter: schedulerPump started" deploymentName= report-interval=1m0s storID=11111111
time="2017-03-24T10:31:26Z" level=info msg="Update the uconfig value" inputhost.HostOverallConnLimit=10000
time="2017-03-24T10:31:26Z" level=info msg="RuntimeMetricsReporter started"
time="2017-03-24T10:31:26Z" level=info msg="tChannel listening 172.17.0.5:4240"
time="2017-03-24T10:31:26Z" level=debug msg=RingHosts RingHosts=[172.17.0.5 172.17.0.6] hostIP=172.17.0.5
time="2017-03-24T10:31:26Z" level=info msg="service started" hostport="172.17.0.5:4240" service=cherami-inputhost
time="2017-03-24T10:31:26Z" level=info msg="HostIDHeartbeater started" deploymentName= hostAddr="172.17.0.5:4240" hostID=ed1490d7-444e-4658-b486-406e8fc03982 hostName=35e0d2b285ff inhoID=ed1490d7
time="2017-03-24T10:31:26Z" level=info msg="Load reporter started." deploymentName= inhoID=ed1490d7
time="2017-03-24T10:31:26Z" level=info msg="Diagnostic http endpoint listening on 127.0.0.1:14240"
time="2017-03-24T10:31:26Z" level=info msg="WebSocket listening 172.17.0.5:6189"
time="2017-03-24T10:31:26Z" level=info msg="RingpopMonitor worker loop started"
time="2017-03-24T10:31:26Z" level=info msg="New Frontend" deploymentName= frntID=3002148f serviceName=cherami-frontendhost
time="2017-03-24T10:31:26Z" level=info msg="RuntimeMetricsReporter started"
time="2017-03-24T10:31:26Z" level=info msg="tChannel listening 172.17.0.5:4922"
time="2017-03-24T10:31:26Z" level=debug msg=RingHosts RingHosts=[172.17.0.5 172.17.0.6] hostIP=172.17.0.5
time="2017-03-24T10:31:26Z" level=info msg="service started" hostport="172.17.0.5:4922" service=cherami-frontendhost
time="2017-03-24T10:31:26Z" level=info msg="HostIDHeartbeater started" deploymentName= frntID=3002148f hostAddr="172.17.0.5:4922" hostID=3002148f-941d-43ef-9f7c-1c464f48c137 hostName=35e0d2b285ff
time="2017-03-24T10:31:26Z" level=info msg="Diagnostic http endpoint listening on 127.0.0.1:14922"
time="2017-03-24T10:31:26Z" level=info msg="RingpopMonitor worker loop started"
time="2017-03-24T10:31:26Z" level=debug msg="Ring membership updated, new membershipMap" newembershipMap=map[cherami-storehost:0xc4203914e0 cherami-controllerhost:0xc420370ca0 cherami-frontendhost:0xc420370ce0 cherami-replicator:0xc420370d20 cherami-inputhost:0xc420370be0 cherami-outputhost:0xc420370c20]
time="2017-03-24T10:31:26Z" level=info msg="replication run started" deploymentName= storID=11111111
time="2017-03-24T10:31:26Z" level=debug msg="Ring membership updated, new membershipMap" newembershipMap=map[cherami-frontendhost:0xc4203a5900 cherami-replicator:0xc4203a5940 cherami-inputhost:0xc4203d2160 cherami-outputhost:0xc4203a5840 cherami-storehost:0xc4203a5880 cherami-controllerhost:0xc4203a58c0]
time="2017-03-24T10:31:26Z" level=info msg="replication run finished" deploymentName= stats="total extents: 0, remote extents:0, opened for replication: 0, primary: 0, secondary: 0, failed: 0" storID=11111111
time="2017-03-24T10:31:26Z" level=debug msg="Ring membership updated, new membershipMap" newembershipMap=map[cherami-inputhost:0xc4203efbe0 cherami-outputhost:0xc4203efc20 cherami-storehost:0xc4203efc60 cherami-controllerhost:0xc4203efca0 cherami-frontendhost:0xc420406140 cherami-replicator:0xc4203efd20]
- time="2017-03-24T10:31:27Z" level=error msg="Cannot initialize topology for placement" ctrlID=531fb4cb deploymentName= err="open : no such file or directory"
time="2017-03-24T10:31:27Z" level=info msg="RuntimeMetricsReporter started"
time="2017-03-24T10:31:27Z" level=info msg="tChannel listening 172.17.0.5:5425"
time="2017-03-24T10:31:27Z" level=debug msg=RingHosts RingHosts=[172.17.0.5 172.17.0.6] hostIP=172.17.0.5
time="2017-03-24T10:31:27Z" level=info msg="service started" hostport="172.17.0.5:5425" service=cherami-controllerhost
time="2017-03-24T10:31:27Z" level=info msg="RingpopMonitor worker loop started"
time="2017-03-24T10:31:27Z" level=debug msg="Ring membership updated, new membershipMap" newembershipMap=map[cherami-frontendhost:0xc420452fe0 cherami-replicator:0xc420453020 cherami-inputhost:0xc420452ee0 cherami-outputhost:0xc420452f20 cherami-storehost:0xc420452f60 cherami-controllerhost:0xc420427520]
time="2017-03-24T10:31:27Z" level=info msg="Event pipeline started" ctrlID=531fb4cb deploymentName= module=EventPipeline
time="2017-03-24T10:31:27Z" level=info msg="Timeslot metrics aggregator started" ctrlID=531fb4cb deploymentName=
time="2017-03-24T10:31:27Z" level=info msg="Failure Detector Daemon started" ctrlID=531fb4cb deploymentName=
time="2017-03-24T10:31:27Z" level=debug msg="retMgrRunner: Starting" ctrlID=531fb4cb deploymentName=
time="2017-03-24T10:31:27Z" level=info msg="HostIDHeartbeater started" ctrlID=531fb4cb deploymentName= hostAddr="172.17.0.5:5425" hostID=4e4af61f-185a-4e7d-87dc-54f67d737234 hostName=35e0d2b285ff
time="2017-03-24T10:31:27Z" level=info msg="ExtentStateMonitor started" ctrlID=531fb4cb deploymentName= module=extentMon
time="2017-03-24T10:31:27Z" level=info msg="Diagnostic http endpoint listening on 127.0.0.1:15425"
time="2017-03-24T10:31:27Z" level=info msg="Config refresher started" ctrlID=531fb4cb deploymentName=
time="2017-03-24T10:31:27Z" level=debug msg="retMgrRunner: not primary; not running here" ctrlID=531fb4cb deploymentName=
- time="2017-03-24T10:31:27Z" level=error msg="RingPop GetHosts failed" ctrlID=531fb4cb deploymentName= err="Not enough hosts to serve the request" module=extentMon
time="2017-03-24T10:31:27Z" level=debug msg="Ring membership updated, new membershipMap" newembershipMap=map[cherami-controllerhost:0xc4202f3360 cherami-frontendhost:0xc4202f33a0 cherami-replicator:0xc4202f33e0 cherami-inputhost:0xc4202f32a0 cherami-outputhost:0xc420427d60 cherami-storehost:0xc4202f3320]
time="2017-03-24T10:31:27Z" level=debug msg="Ring membership updated, new membershipMap" newembershipMap=map[cherami-controllerhost:0xc4203a58c0 cherami-frontendhost:0xc4203a5900 cherami-replicator:0xc4203a5940 cherami-inputhost:0xc4203ee720 cherami-outputhost:0xc4203a5840 cherami-storehost:0xc4203a5880]
time="2017-03-24T10:31:28Z" level=debug msg="Ring membership updated, new membershipMap" newembershipMap=map[cherami-controllerhost:0xc4203d2960 cherami-frontendhost:0xc420452fe0 cherami-replicator:0xc420453020 cherami-inputhost:0xc420452ee0 cherami-outputhost:0xc420452f20 cherami-storehost:0xc420452f60]
time="2017-03-24T10:31:28Z" level=debug msg="retMgrRunner: Starting retention manager" ctrlID=531fb4cb deploymentName=
time="2017-03-24T10:31:28Z" level=info msg="RetentionMgr starting" ctrlID=531fb4cb deploymentName= dlqinterval=0s interval=10m0s module=retMgr workers=8
time="2017-03-24T10:31:28Z" level=debug msg="retentionWorker: started" ctrlID=531fb4cb deploymentName= module=retMgr worker=0
time="2017-03-24T10:31:28Z" level=debug msg="retentionWorker: started" ctrlID=531fb4cb deploymentName= module=retMgr worker=1
time="2017-03-24T10:31:28Z" level=debug msg="retentionWorker: started" ctrlID=531fb4cb deploymentName= module=retMgr worker=2
time="2017-03-24T10:31:28Z" level=debug msg="retentionWorker: started" ctrlID=531fb4cb deploymentName= module=retMgr worker=3
time="2017-03-24T10:31:28Z" level=debug msg="retentionWorker: started" ctrlID=531fb4cb deploymentName= module=retMgr worker=4
time="2017-03-24T10:31:28Z" level=debug msg="retentionWorker: started" ctrlID=531fb4cb deploymentName= module=retMgr worker=5
time="2017-03-24T10:31:28Z" level=debug msg="retentionWorker: started" ctrlID=531fb4cb deploymentName= module=retMgr worker=6
time="2017-03-24T10:31:28Z" level=debug msg="retentionWorker: started" ctrlID=531fb4cb deploymentName= module=retMgr worker=7
- time="2017-03-24T10:31:28Z" level=error msg="RingPop GetHosts failed" ctrlID=531fb4cb deploymentName= err="Not enough hosts to serve the request" module=extentMon
- time="2017-03-24T10:31:28Z" level=error msg="RingPop GetHosts failed" ctrlID=531fb4cb deploymentName= err="Not enough hosts to serve the request" module=extentMon
- time="2017-03-24T10:31:28Z" level=error msg="getControllerClient - Failed to find controller host" err="Not enough hosts to serve the request"
...
root@35e0d2b285ff:/go/src/github.com/uber/cherami-server# ./cherami-cli --env=prod --hostport=172.17.0.5:4922 create destination /test/cherami
tchannel error ErrCodeUnexpected: InternalServiceError({Message:Not enough hosts to serve the request})
--env=staging is now the default. Did you mean '--env=prod' ?
goroutine 1 [running]:
runtime/debug.Stack(0xc4201bbd20, 0x4638f5, 0xa6345d)
/usr/local/go/src/runtime/debug/stack.go:24 +0x79
runtime/debug.PrintStack()
/usr/local/go/src/runtime/debug/stack.go:16 +0x22
github.com/uber/cherami-server/tools/common.ExitIfError(0xd7ec40, 0xc42013dd70)
/go/src/github.com/uber/cherami-server/tools/common/lib.go:101 +0x1cb
github.com/uber/cherami-server/tools/common.CreateDestination(0xc420153680, 0xd88b00, 0xc42013b900, 0xd84c80, 0xc42013ff60)
/go/src/github.com/uber/cherami-server/tools/common/lib.go:257 +0xa22
github.com/uber/cherami-server/tools/cli.CreateDestination(0xc420153680, 0xd84c80, 0xc42013ff60)
/go/src/github.com/uber/cherami-server/tools/cli/lib.go:42 +0x76
main.main.func1(0xc420153680)
/go/src/github.com/uber/cherami-server/cmd/tools/cli/main.go:129 +0x3d
github.com/uber/cherami-server/vendor/github.com/codegangsta/cli.HandleAction(0x981ac0, 0xc42013ff80, 0xc420153680, 0xc420148a00, 0x0)
/go/src/github.com/uber/cherami-server/vendor/github.com/codegangsta/cli/app.go:487 +0x7c
github.com/uber/cherami-server/vendor/github.com/codegangsta/cli.Command.Run(0xa5e189, 0xb, 0x0, 0x0, 0xc42013ffe0, 0x2, 0x2, 0xa705e5, 0x23, 0x0, ...)
/go/src/github.com/uber/cherami-server/vendor/github.com/codegangsta/cli/command.go:193 +0xb96
github.com/uber/cherami-server/vendor/github.com/codegangsta/cli.(*App).RunAsSubcommand(0xc420164d00, 0xc420153400, 0x0, 0x0)
/go/src/github.com/uber/cherami-server/vendor/github.com/codegangsta/cli/app.go:374 +0xb1a
github.com/uber/cherami-server/vendor/github.com/codegangsta/cli.Command.startApp(0xa59d23, 0x6, 0x0, 0x0, 0xc42013ffc0, 0x2, 0x2, 0xa7153d, 0x24, 0x0, ...)
/go/src/github.com/uber/cherami-server/vendor/github.com/codegangsta/cli/command.go:280 +0x82c
github.com/uber/cherami-server/vendor/github.com/codegangsta/cli.Command.Run(0xa59d23, 0x6, 0x0, 0x0, 0xc42013ffc0, 0x2, 0x2, 0xa7153d, 0x24, 0x0, ...)
/go/src/github.com/uber/cherami-server/vendor/github.com/codegangsta/cli/command.go:79 +0x16a5
github.com/uber/cherami-server/vendor/github.com/codegangsta/cli.(*App).Run(0xc4201649c0, 0xc42000c180, 0x6, 0x6, 0x0, 0x0)
/go/src/github.com/uber/cherami-server/vendor/github.com/codegangsta/cli/app.go:250 +0x812
main.main()
/go/src/github.com/uber/cherami-server/cmd/tools/cli/main.go:407 +0x2aca
I know this is a handful and I might miss something obvious, but because there is not much documentation it's hard to get all the things right.