tests.system.aiplatform.test_featurestore.TestFeaturestore: test_ingest_feature_values_from_df_using_feature_time_column_and_online_read_multiple_entities failed about python-aiplatform HOT 2 CLOSED

self = 
def next(self):
    """Get the next response from the stream.

    Returns:
        protobuf.Message: A single response from the stream.
    """
    try:
        if hasattr(self, "_stored_first_result"):
            result = self._stored_first_result
            del self._stored_first_result
            return result


      return six.next(self._wrapped)


.nox/system-3-8/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:107:

self = <_MultiThreadedRendezvous of RPC that terminated with:

status = StatusCode.RESOURCE_EXHAUSTED

details = "Unable to s...rage. Check the online serving storage CPU usage. Provisioning more online serving nodes may help.","grpc_status":8}"


def __next__(self):


  return self._next()


.nox/system-3-8/lib/python3.8/site-packages/grpc/_channel.py:426:

self = <_MultiThreadedRendezvous of RPC that terminated with:

status = StatusCode.RESOURCE_EXHAUSTED

details = "Unable to s...rage. Check the online serving storage CPU usage. Provisioning more online serving nodes may help.","grpc_status":8}"


def _next(self):
    with self._state.condition:
        if self._state.code is None:
            event_handler = _event_handler(self._state,
                                           self._response_deserializer)
            self._state.due.add(cygrpc.OperationType.receive_message)
            operating = self._call.operate(
                (cygrpc.ReceiveMessageOperation(_EMPTY_FLAGS),),
                event_handler)
            if not operating:
                self._state.due.remove(cygrpc.OperationType.receive_message)
        elif self._state.code is grpc.StatusCode.OK:
            raise StopIteration()
        else:
            raise self

        def _response_ready():
            return (self._state.response is not None or
                    (cygrpc.OperationType.receive_message
                     not in self._state.due and
                     self._state.code is not None))

        _common.wait(self._state.condition.wait, _response_ready)
        if self._state.response is not None:
            response = self._state.response
            self._state.response = None
            return response
        elif cygrpc.OperationType.receive_message not in self._state.due:
            if self._state.code is grpc.StatusCode.OK:
                raise StopIteration()
            elif self._state.code is not None:


              raise self


E                   grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:

E                   	status = StatusCode.RESOURCE_EXHAUSTED

E                   	details = "Unable to serve request due to exhausted resource for online serving storage. Check the online serving storage CPU usage. Provisioning more online serving nodes may help."

E                   	debug_error_string = "{"created":"@1683625762.290306437","description":"Error received from peer ipv4:74.125.20.95:443","file":"src/core/lib/surface/call.cc","file_line":966,"grpc_message":"Unable to serve request due to exhausted resource for online serving storage. Check the online serving storage CPU usage. Provisioning more online serving nodes may help.","grpc_status":8}"

E                   >
.nox/system-3-8/lib/python3.8/site-packages/grpc/_channel.py:826: _MultiThreadedRendezvous
The above exception was the direct cause of the following exception:
self = <tests.system.aiplatform.test_featurestore.TestFeaturestore object at 0x7f391f18cd90>

shared_state = {'bigquery_client': <google.cloud.bigquery.client.Client object at 0x7f391f1f3af0>, 'bigquery_dataset': Dataset(Datase...130d_4e03_aa36_5d6ca85bbcfe', 'bucket': <Bucket: temp_vertex_sdk_e2e_featurestore_test-329394db-44ac-42c9-9982-e>, ...}

caplog = <_pytest.logging.LogCaptureFixture object at 0x7f391c471520>
def test_ingest_feature_values_from_df_using_feature_time_column_and_online_read_multiple_entities(
    self, shared_state, caplog
):

    assert shared_state["movie_entity_type"]
    movie_entity_type = shared_state["movie_entity_type"]

    caplog.set_level(logging.INFO)

    aiplatform.init(
        project=e2e_base._PROJECT,
        location=e2e_base._LOCATION,
    )

    read_feature_ids = ["average_rating", "title", "genres"]


  movie_entity_views_df_before_ingest = movie_entity_type.read(


        entity_ids=["movie_01", "movie_02"],
        feature_ids=read_feature_ids,
    )

tests/system/aiplatform/test_featurestore.py:291:

google/cloud/aiplatform/featurestore/_entity_type.py:1479: in read

streaming_read_feature_values_responses = list(

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:110: in next

six.raise_from(exceptions.from_grpc_error(exc), exc)

value = None

from_value = <_MultiThreadedRendezvous of RPC that terminated with:

status = StatusCode.RESOURCE_EXHAUSTED

details = "Unable to s...rage. Check the online serving storage CPU usage. Provisioning more online serving nodes may help.","grpc_status":8}"



???

E   google.api_core.exceptions.ResourceExhausted: 429 Unable to serve request due to exhausted resource for online serving storage. Check the online serving storage CPU usage. Provisioning more online serving nodes may help.

:3: ResourceExhausted