As you can see, there are two volumes are being in Pending state...
Aug 23 16:17:03 mslave1 flocker-dataset-agent[11611]: {"timestamp": 1471983423.412408, "task_uuid": "eb6e8448-07e1-4124-a008-5317bb75a243", "action_type": "eliot:remote_task", "action_status": "started", "task_level": [7, 2, 1]}
Aug 23 16:17:03 mslave1 flocker-dataset-agent[11611]: {"exception": "exceptions.KeyError", "task_level": [7, 2, 2], "action_type": "eliot:remote_task", "reason": "\"Attribute or key '83790f44-1afb-4e43-b03b-e8749be46f01' not found in UnicodeManifestationPMap({u'b5afba76-949c-44ac-876f-08330f6c5bee': Manifestation(primary=True, dataset=Dataset(deleted=False, dataset_id=u'b5afba76-949c-44ac-876f-08330f6c5bee', maximum_size=5368709120, metadata=pmap({u'name': u'apples', u'size': u'small'})))})\"", "timestamp": 1471983423.427221, "task_uuid": "eb6e8448-07e1-4124-a008-5317bb75a243", "action_status": "failed"}
Aug 23 16:17:03 mslave1 flocker-dataset-agent[11611]: {"task_uuid": "996124a0-0d10-4e77-8e1e-947b7834b84b", "error": true, "timestamp": 1471983423.436451, "message": "Unhandled Error\nTraceback (most recent call last):\n File \"/opt/flocker/lib/python2.7/site-packages/twisted/protocols/amp.py\", line 1021, in _commandReceived\n deferred = self.dispatchCommand(box)\n File \"/opt/flocker/lib/python2.7/site-packages/twisted/protocols/amp.py\", line 1079, in dispatchCommand\n return maybeDeferred(responder, box)\n File \"/opt/flocker/lib/python2.7/site-packages/twisted/internet/defer.py\", line 150, in maybeDeferred\n result = f(*args, **kw)\n File \"/opt/flocker/lib/python2.7/site-packages/twisted/protocols/amp.py\", line 1166, in doit\n return maybeDeferred(aCallable, **kw).addCallback(\n--- <exception caught here> ---\n File \"/opt/flocker/lib/python2.7/site-packages/twisted/internet/defer.py\", line 150, in maybeDeferred\n result = f(*args, **kw)\n File \"/opt/flocker/lib/python2.7/site-packages/flocker/control/_protocol.py\", line 1157, in cluster_updated_diff\n self._current_configuration\n File \"/opt/flocker/lib/python2.7/site-packages/flocker/control/_diffing.py\", line 432, in apply\n proxy = c.apply(proxy)\n File \"/opt/flocker/lib/python2.7/site-packages/flocker/control/_diffing.py\", line 100, in apply\n self.path, lambda o: o.set(self.key, self.value)\n File \"/opt/flocker/lib/python2.7/site-packages/flocker/control/_diffing.py\", line 390, in transform\n target = _get_or_add_proxy_child(target, segment)\n File \"/opt/flocker/lib/python2.7/site-packages/flocker/control/_diffing.py\", line 348, in _get_or_add_proxy_child\n segment, parent_proxy._original\nexceptions.KeyError: \"Attribute or key '83790f44-1afb-4e43-b03b-e8749be46f01' not found in UnicodeManifestationPMap({u'b5afba76-949c-44ac-876f-08330f6c5bee': Manifestation(primary=True, dataset=Dataset(deleted=False, dataset_id=u'b5afba76-949c-44ac-876f-08330f6c5bee', maximum_size=5368709120, metadata=pmap({u'name': u'apples', u'size': u'small'})))})\"\n", "message_type": "twisted:log", "task_level": [1]}
Aug 23 16:17:03 mslave1 flocker-dataset-agent[11611]: {"task_uuid": "503dfc52-f314-4ca4-ab44-1ddc1dde43ce", "error": false, "timestamp": 1471983423.452151, "message": "AgentAMP connection lost (HOST:IPv4Address(TCP, '192.168.120.161', 55783) PEER:IPv4Address(TCP, '192.168.120.156', 4524))", "message_type": "twisted:log", "task_level": [1]}
Aug 23 16:17:03 mslave1 flocker-dataset-agent[11611]: {"fsm_identifier": "<flocker.node._loop.ClusterStatus object at 0x42daad0>", "fsm_input": "<ClusterStatusInputs=DISCONNECTED_FROM_CONTROL_SERVICE>", "timestamp": 1471983423.453291, "fsm_rich_input": null, "action_status": "started", "task_uuid": "13348bd8-2880-49f7-bff9-6806e8f4d2a0", "action_type": "fsm:transition", "fsm_state": "<ClusterStatusStates=KNOWLEDGEABLE>", "task_level": [1]}
Aug 23 16:17:03 mslave1 flocker-dataset-agent[11611]: {"fsm_identifier": "<flocker.node._loop.ConvergenceLoop object at 0x42bc7d0>", "fsm_input": "<ConvergenceLoopInputs=STOP>", "timestamp": 1471983423.454234, "fsm_rich_input": null, "action_status": "started", "task_uuid": "13348bd8-2880-49f7-bff9-6806e8f4d2a0", "action_type": "fsm:transition", "fsm_state": "<ConvergenceLoopStates=SLEEPING>", "task_level": [2, 1]}
Aug 23 16:17:03 mslave1 flocker-dataset-agent[11611]: {"fsm_next_state": "<ConvergenceLoopStates=STOPPED>", "task_level": [2, 2], "action_type": "fsm:transition", "timestamp": 1471983423.454929, "fsm_output": ["<ConvergenceLoopOutputs=CLEAR_WAKEUP>"], "task_uuid": "13348bd8-2880-49f7-bff9-6806e8f4d2a0", "action_status": "succeeded"}
Aug 23 16:17:03 mslave1 flocker-dataset-agent[11611]: {"fsm_next_state": "<ClusterStatusStates=DISCONNECTED>", "task_level": [3], "action_type": "fsm:transition", "timestamp": 1471983423.455499, "fsm_output": ["<ClusterStatusOutputs=STOP>"], "task_uuid": "13348bd8-2880-49f7-bff9-6806e8f4d2a0", "action_status": "succeeded"}
Aug 23 16:17:03 mslave1 flocker-dataset-agent[11611]: {"task_uuid": "3f7459a3-07b1-4c8f-a1b9-c700badb5934", "error": false, "timestamp": 1471983423.457014, "message": "<twisted.internet.tcp.Connector instance at 0x413c680> will retry in 3 seconds", "message_type": "twisted:log", "task_level": [1]}
Aug 23 16:17:03 mslave1 flocker-dataset-agent[11611]: {"task_uuid": "a500c269-3b5c-4b40-b3b3-6cc6d164aa4c", "error": false, "timestamp": 1471983423.467202, "message": "Stopping factory <twisted.internet.protocol.ReconnectingClientFactory instance at 0x7fcb1403edd0>", "message_type": "twisted:log", "task_level": [1]}
Aug 23 16:17:06 mslave1 flocker-dataset-agent[11611]: {"task_uuid": "c890d693-81e1-4318-ae14-a27f2adeb8c2", "error": false, "timestamp": 1471983426.653772, "message": "Starting factory <twisted.internet.protocol.ReconnectingClientFactory instance at 0x7fcb1403edd0>", "message_type": "twisted:log", "task_level": [1]}
Aug 23 16:17:06 mslave1 flocker-dataset-agent[11611]: {"task_uuid": "2bb2d11d-de76-4a0f-a7dc-e22548dfa0c7", "error": false, "timestamp": 1471983426.666226, "message": "AgentAMP connection established (HOST:IPv4Address(TCP, '192.168.120.161', 35975) PEER:IPv4Address(TCP, '192.168.120.156', 4524))", "message_type": "twisted:log", "task_level": [1]}
Aug 23 16:17:06 mslave1 flocker-dataset-agent[11611]: {"fsm_identifier": "<flocker.node._loop.ClusterStatus object at 0x42daad0>", "fsm_input": "<ClusterStatusInputs=CONNECTED_TO_CONTROL_SERVICE>", "timestamp": 1471983426.667331, "fsm_rich_input": "<_ConnectedToControlService>", "action_status": "started", "task_uuid": "beeb75dd-fc61-4ad8-94ed-580501d17a16", "action_type": "fsm:transition", "fsm_state": "<ClusterStatusStates=DISCONNECTED>", "task_level": [1]}
Aug 23 16:17:06 mslave1 flocker-dataset-agent[11611]: {"fsm_next_state": "<ClusterStatusStates=IGNORANT>", "task_level": [2], "action_type": "fsm:transition", "timestamp": 1471983426.667744, "fsm_output": ["<ClusterStatusOutputs=STORE_CLIENT>"], "task_uuid": "beeb75dd-fc61-4ad8-94ed-580501d17a16", "action_status": "succeeded"}
Aug 23 16:17:07 mslave1 flocker-dataset-agent[11611]: {"timestamp": 1471983427.795813, "task_uuid": "36d3303d-9691-4374-bbda-78b055c9c791", "action_type": "eliot:remote_task", "action_status": "started", "task_level": [4, 2, 1]}
Aug 23 16:17:07 mslave1 flocker-dataset-agent[11611]: {"fsm_identifier": "<flocker.node._loop.ClusterStatus object at 0x42daad0>", "fsm_input": "<ClusterStatusInputs=STATUS_UPDATE>", "timestamp": 1471983427.823243, "fsm_rich_input": "<_StatusUpdate>", "action_status": "started", "task_uuid": "36d3303d-9691-4374-bbda-78b055c9c791", "action_type": "fsm:transition", "fsm_state": "<ClusterStatusStates=IGNORANT>", "task_level": [4, 2, 2, 1]}
Aug 23 16:17:07 mslave1 flocker-dataset-agent[11611]: {"fsm_identifier": "<flocker.node._loop.ConvergenceLoop object at 0x42bc7d0>", "fsm_input": "<ConvergenceLoopInputs=STATUS_UPDATE>", "timestamp": 1471983427.823805, "fsm_rich_input": "<_ClientStatusUpdate>", "action_status": "started", "task_uuid": "36d3303d-9691-4374-bbda-78b055c9c791", "action_type": "fsm:transition", "fsm_state": "<ConvergenceLoopStates=STOPPED>", "task_level": [4, 2, 2, 2, 1]}
Aug 23 16:17:07 mslave1 flocker-dataset-agent[11611]: {"timestamp": 1471983427.824237, "task_uuid": "36d3303d-9691-4374-bbda-78b055c9c791", "action_type": "flocker:agent:converge", "action_status": "started", "task_level": [4, 2, 2, 2, 2, 1]}
Aug 23 16:17:07 mslave1 flocker-dataset-agent[11611]: {"timestamp": 1471983427.824591, "task_uuid": "36d3303d-9691-4374-bbda-78b055c9c791", "action_type": "flocker:agent:discovery", "action_status": "started", "task_level": [4, 2, 2, 2, 2, 2, 1]}
Aug 23 16:17:07 mslave1 flocker-dataset-agent[11611]: {"count": 135, "function": "_discover_raw_state", "task_uuid": "36d3303d-9691-4374-bbda-78b055c9c791", "timestamp": 1471983427.824912, "message_type": "flocker:node:agents:blockdevice:list_volumes", "task_level": [4, 2, 2, 2, 2, 2, 2]}
Aug 23 16:17:07 mslave1 flocker-dataset-agent[11611]: {"Info": "Found Purity volume managed by flocker {u'source': None, u'serial': u'7E4D75D0FFC14E7C00011010', u'size': 5368709120, u'name': u'flocker-bf10eb7b-1119-4699-b5afba76-949c-44ac-876f-08330f6c5bee', u'created': u'2016-08-23T18:21:24Z'}", "timestamp": 1471983427.897228, "task_uuid": "36d3303d-9691-4374-bbda-78b055c9c791", "task_level": [4, 2, 2, 2, 2, 2, 3]}
Aug 23 16:17:08 mslave1 flocker-dataset-agent[11611]: {"Info": "Volume flocker-bf10eb7b-1119-4699-b5afba76-949c-44ac-876f-08330f6c5bee attached_to = mslave2", "timestamp": 1471983428.065972, "task_uuid": "36d3303d-9691-4374-bbda-78b055c9c791", "task_level": [4, 2, 2, 2, 2, 2, 4]}
Aug 23 16:17:08 mslave1 flocker-dataset-agent[11611]: {"timestamp": 1471983428.073424, "task_uuid": "36d3303d-9691-4374-bbda-78b055c9c791", "message_type": "agent:blockdevice:raw_state", "raw_state": "RawState(compute_instance_id=u'mslave1', devices_with_filesystems=FilepathPSet([]), devices=UuidToFilepathPMap({}), system_mounts=FilepathToFilepathPMap({FilePath('/dev/mapper/vg_gluster-brick3'): FilePath('/bricks/brick3'), FilePath('/dev/mapper/vg_gluster-brick2'): FilePath('/bricks/brick2'), FilePath('/dev/mapper/vg_gluster-brick5'): FilePath('/bricks/brick5'), FilePath('/dev/mapper/vgsys-root'): FilePath('/'), FilePath('/dev/mapper/vgsys-diskdump'): FilePath('/diskdump'), FilePath('/dev/mapper/vg_gluster-brick4'): FilePath('/bricks/brick4'), FilePath('/dev/mapper/vgcore-opt_bsa_bladelogic'): FilePath('/opt/bsa/bladelogic'), FilePath('/dev/sda1'): FilePath('/boot'), FilePath('/dev/mapper/vgsys-var_tmp'): FilePath('/var/tmp'), FilePath('/dev/mapper/vgcore-users'): FilePath('/users'), FilePath('/dev/mapper/vgsys-tmp'): FilePath('/tmp'), FilePath('/dev/mapper/vgsys-var_log'): FilePath('/var/log'), FilePath('/dev/mapper/vgsys-var'): FilePath('/var'), FilePath('/dev/mapper/vg_gluster-brick1'): FilePath('/bricks/brick1'), FilePath('/dev/mapper/vgcore-usr_bltemp'): FilePath('/usr/bltemp')}), volumes=BlockdevicevolumePVector([BlockDeviceVolume(blockdevice_id=u'flocker-bf10eb7b-1119-4699-b5afba76-949c-44ac-876f-08330f6c5bee', dataset_id=UUID('b5afba76-949c-44ac-876f-08330f6c5bee'), attached_to=u'mslave2', size=5368709120)]), _live_instances=None)", "task_level": [4, 2, 2, 2, 2, 2, 5]}
Aug 23 16:17:08 mslave1 flocker-dataset-agent[11611]: {"task_uuid": "36d3303d-9691-4374-bbda-78b055c9c791", "task_level": [4, 2, 2, 2, 2, 2, 6], "action_type": "flocker:agent:discovery", "timestamp": 1471983428.075773, "state": "BlockDeviceDeployerLocalState(node_uuid=UUID('06978551-c7ee-41f6-8412-42f119777d04'), datasets=UuidToDiscovereddatasetPMap({UUID('b5afba76-949c-44ac-876f-08330f6c5bee'): DiscoveredDataset(blockdevice_id=u'flocker-bf10eb7b-1119-4699-b5afba76-949c-44ac-876f-08330f6c5bee', state=<DatasetStates=ATTACHED_ELSEWHERE>, maximum_size=5368709120, dataset_id=UUID('b5afba76-949c-44ac-876f-08330f6c5bee'))}), hostname=u'192.168.120.161')", "action_status": "succeeded"}
Aug 23 16:17:08 mslave1 flocker-dataset-agent[11611]: {"local_changes": [{"paths": {"values": [], "$__class__$": "PMap"}, "uuid": {"hex": "06978551-c7ee-41f6-8412-42f119777d04", "$__class__$": "UUID"}, "$__class__$": "NodeState", "hostname": "192.168.120.161", "devices": {"values": [], "$__class__$": "PMap"}, "applications": null, "manifestations": {"values": [], "$__class__$": "PMap"}}, {"datasets": {"values": [], "$__class__$": "PMap"}, "$__class__$": "NonManifestDatasets"}], "connection": "<AgentAMP at 0x473d310>", "task_level": [4, 2, 2, 2, 2, 3, 1], "action_type": "flocker:agent:send_to_control_service", "timestamp": 1471983428.079364, "task_uuid": "36d3303d-9691-4374-bbda-78b055c9c791", "action_status": "started"}
Aug 23 16:17:08 mslave1 flocker-dataset-agent[11611]: {"timestamp": 1471983428.082172, "task_uuid": "36d3303d-9691-4374-bbda-78b055c9c791", "message_type": "flocker:agent:converge:actions", "task_level": [4, 2, 2, 2, 2, 4], "calculated_actions": "NoOp(sleep=datetime.timedelta(0, 60))"}
Aug 23 16:17:08 mslave1 flocker-dataset-agent[11611]: {"timestamp": 1471983428.083624, "task_uuid": "36d3303d-9691-4374-bbda-78b055c9c791", "action_type": "flocker:change:noop", "action_status": "started", "task_level": [4, 2, 2, 2, 2, 5, 1]}
Aug 23 16:17:08 mslave1 flocker-dataset-agent[11611]: {"timestamp": 1471983428.08429, "task_uuid": "36d3303d-9691-4374-bbda-78b055c9c791", "action_type": "flocker:change:noop", "action_status": "succeeded", "task_level": [4, 2, 2, 2, 2, 5, 2]}
Aug 23 16:17:08 mslave1 flocker-dataset-agent[11611]: {"fsm_next_state": "<ConvergenceLoopStates=CONVERGING>", "task_level": [4, 2, 2, 2, 3], "action_type": "fsm:transition", "timestamp": 1471983428.087224, "fsm_output": ["<ConvergenceLoopOutputs=STORE_INFO>", "<ConvergenceLoopOutputs=CONVERGE>"], "task_uuid": "36d3303d-9691-4374-bbda-78b055c9c791", "action_status": "succeeded"}
Aug 23 16:17:08 mslave1 flocker-dataset-agent[11611]: {"fsm_next_state": "<ClusterStatusStates=KNOWLEDGEABLE>", "task_level": [4, 2, 2, 3], "action_type": "fsm:transition", "timestamp": 1471983428.090916, "fsm_output": ["<ClusterStatusOutputs=UPDATE_STATUS>"], "task_uuid": "36d3303d-9691-4374-bbda-78b055c9c791", "action_status": "succeeded"}
Aug 23 16:17:08 mslave1 flocker-dataset-agent[11611]: {"timestamp": 1471983428.091679, "task_uuid": "36d3303d-9691-4374-bbda-78b055c9c791", "action_type": "eliot:remote_task", "action_status": "succeeded", "task_level": [4, 2, 3]}
Aug 23 16:17:08 mslave1 flocker-dataset-agent[11611]: {"timestamp": 1471983428.115874, "task_uuid": "36d3303d-9691-4374-bbda-78b055c9c791", "action_type": "flocker:agent:send_to_control_service", "action_status": "succeeded", "task_level": [4, 2, 2, 2, 2, 3, 3]}
Aug 23 16:17:08 mslave1 flocker-dataset-agent[11611]: {"delay": 52.606892414095064, "task_uuid": "36d3303d-9691-4374-bbda-78b055c9c791", "log_level": "INFO", "timestamp": 1471983428.116817, "message": "Delaying until next convergence loop.", "message_type": "flocker:node:_loop:CONVERGE:delay", "task_level": [4, 2, 2, 2, 2, 6]}
Aug 23 16:17:08 mslave1 flocker-dataset-agent[11611]: {"fsm_identifier": "<flocker.node._loop.ConvergenceLoop object at 0x42bc7d0>", "fsm_input": "<ConvergenceLoopInputs=SLEEP>", "timestamp": 1471983428.119845, "fsm_rich_input": "<_Sleep>", "action_status": "started", "task_uuid": "36d3303d-9691-4374-bbda-78b055c9c791", "action_type": "fsm:transition", "fsm_state": "<ConvergenceLoopStates=CONVERGING>", "task_level": [4, 2, 2, 2, 2, 7, 1]}
Aug 23 16:17:08 mslave1 flocker-dataset-agent[11611]: {"fsm_next_state": "<ConvergenceLoopStates=SLEEPING>", "task_level": [4, 2, 2, 2, 2, 7, 2], "action_type": "fsm:transition", "timestamp": 1471983428.120741, "fsm_output": ["<ConvergenceLoopOutputs=SCHEDULE_WAKEUP>"], "task_uuid": "36d3303d-9691-4374-bbda-78b055c9c791", "action_status": "succeeded"}
Aug 23 16:17:08 mslave1 flocker-dataset-agent[11611]: {"timestamp": 1471983428.122268, "task_uuid": "36d3303d-9691-4374-bbda-78b055c9c791", "action_type": "flocker:agent:converge", "action_status": "succeeded", "task_level": [4, 2, 2, 2, 2, 8]}
Aug 23 16:17:09 mslave1 flocker-dataset-agent[11611]: {"timestamp": 1471983429.145261, "task_uuid": "9f7d164a-de82-4249-8b99-e5e28cd592bc", "action_type": "eliot:remote_task", "action_status": "started", "task_level": [5, 2, 1]}
Aug 23 16:17:09 mslave1 flocker-dataset-agent[11611]: {"fsm_identifier": "<flocker.node._loop.ClusterStatus object at 0x42daad0>", "fsm_input": "<ClusterStatusInputs=STATUS_UPDATE>", "timestamp": 1471983429.149796, "fsm_rich_input": "<_StatusUpdate>", "action_status": "started", "task_uuid": "9f7d164a-de82-4249-8b99-e5e28cd592bc", "action_type": "fsm:transition", "fsm_state": "<ClusterStatusStates=KNOWLEDGEABLE>", "task_level": [5, 2, 2, 1]}
Aug 23 16:17:09 mslave1 flocker-dataset-agent[11611]: {"fsm_identifier": "<flocker.node._loop.ConvergenceLoop object at 0x42bc7d0>", "fsm_input": "<ConvergenceLoopInputs=STATUS_UPDATE>", "timestamp": 1471983429.159345, "fsm_rich_input": "<_ClientStatusUpdate>", "action_status": "started", "task_uuid": "9f7d164a-de82-4249-8b99-e5e28cd592bc", "action_type": "fsm:transition", "fsm_state": "<ConvergenceLoopStates=SLEEPING>", "task_level": [5, 2, 2, 2, 1]}
Aug 23 16:17:09 mslave1 flocker-dataset-agent[11611]: {"fsm_next_state": "<ConvergenceLoopStates=SLEEPING>", "task_level": [5, 2, 2, 2, 2], "action_type": "fsm:transition", "timestamp": 1471983429.160953, "fsm_output": ["<ConvergenceLoopOutputs=STORE_INFO>", "<ConvergenceLoopOutputs=UPDATE_MAYBE_WAKEUP>"], "task_uuid": "9f7d164a-de82-4249-8b99-e5e28cd592bc", "action_status": "succeeded"}
Aug 23 16:17:09 mslave1 flocker-dataset-agent[11611]: {"fsm_next_state": "<ClusterStatusStates=KNOWLEDGEABLE>", "task_level": [5, 2, 2, 3], "action_type": "fsm:transition", "timestamp": 1471983429.161595, "fsm_output": ["<ClusterStatusOutputs=UPDATE_STATUS>"], "task_uuid": "9f7d164a-de82-4249-8b99-e5e28cd592bc", "action_status": "succeeded"}
Aug 23 16:17:09 mslave1 flocker-dataset-agent[11611]: {"timestamp": 1471983429.162255, "task_uuid": "9f7d164a-de82-4249-8b99-e5e28cd592bc", "action_type": "eliot:remote_task", "action_status": "succeeded", "task_level": [5, 2, 3]}