tests.system.aiplatform.test_experiment_model.TestExperimentModel: test_deploy_model_with_gpu_container failed about python-aiplatform HOT 2 CLOSED

args = (name: "projects/580378083368/locations/us-central1/endpoints/8650970681481101312/operations/1970395411877199872"
,)
kwargs = {'metadata': [('x-goog-request-params', 'name=projects/580378083368/locations/us-central1/endpoints/8650970681481101312/operations/1970395411877199872'), ('x-goog-api-client', 'gl-python/3.8.13 grpc/1.47.0 gax/1.32.0')], 'timeout': 20.0}
@six.wraps(callable_)
def error_remapped_callable(*args, **kwargs):
    try:


      return callable_(*args, **kwargs)


.nox/system-3-8/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:67:

self = <grpc._channel._UnaryUnaryMultiCallable object at 0x7fa318b64640>

request = name: "projects/580378083368/locations/us-central1/endpoints/8650970681481101312/operations/1970395411877199872"
timeout = 20.0

metadata = [('x-goog-request-params', 'name=projects/580378083368/locations/us-central1/endpoints/8650970681481101312/operations/1970395411877199872'), ('x-goog-api-client', 'gl-python/3.8.13 grpc/1.47.0 gax/1.32.0')]

credentials = None, wait_for_ready = None, compression = None
def __call__(self,
             request,
             timeout=None,
             metadata=None,
             credentials=None,
             wait_for_ready=None,
             compression=None):
    state, call, = self._blocking(request, timeout, metadata, credentials,
                                  wait_for_ready, compression)


  return _end_unary_response_blocking(state, call, False, None)


.nox/system-3-8/lib/python3.8/site-packages/grpc/_channel.py:946:

state = <grpc._channel._RPCState object at 0x7fa3193d2580>

call = <grpc._cython.cygrpc.SegregatedCall object at 0x7fa2a4898680>

with_call = False, deadline = None
def _end_unary_response_blocking(state, call, with_call, deadline):
    if state.code is grpc.StatusCode.OK:
        if with_call:
            rendezvous = _MultiThreadedRendezvous(state, call, None, deadline)
            return state.response, rendezvous
        else:
            return state.response
    else:


      raise _InactiveRpcError(state)


E           grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:

E           	status = StatusCode.DEADLINE_EXCEEDED

E           	details = "Deadline Exceeded"

E           	debug_error_string = "{"created":"@1683042356.339949371","description":"Deadline Exceeded","file":"src/core/ext/filters/deadline/deadline_filter.cc","file_line":81,"grpc_status":4}"

E           >
.nox/system-3-8/lib/python3.8/site-packages/grpc/_channel.py:849: _InactiveRpcError
The above exception was the direct cause of the following exception:
self = <tests.system.aiplatform.test_experiment_model.TestExperimentModel object at 0x7fa2a5f10370>

shared_state = {'bucket': <Bucket: test-vertex-sdk-e2e-experiment-model-6e88677f-a38a-4d56-a495-56>, 'resources': [<google.cloud.aipl...ent-model-6e88677f-a38a-4d56-a495-56', 'storage_client': <google.cloud.storage.client.Client object at 0x7fa26efd40d0>}
def test_deploy_model_with_gpu_container(self, shared_state):
    aiplatform.init(
        project=e2e_base._PROJECT,
        location=e2e_base._LOCATION,
        staging_bucket=f"gs://{shared_state['staging_bucket_name']}",
    )

    # It takes long time to deploy a model. To reduce the system test run
    # time, we randomly choose one registered model to test deployment.
    registered_model = random.choice(self.registered_models_gpu)

    # Deploy the registered model
    # Since we are using gpu, we need to specify accelerator_type and count
    endpoint = registered_model.deploy(
        accelerator_type="NVIDIA_TESLA_T4", accelerator_count=1, sync=False
    )


  pred = endpoint.predict([[1, 2]])


tests/system/aiplatform/test_experiment_model.py:365:

google/cloud/aiplatform/models.py:1526: in predict

self.wait()

google/cloud/aiplatform/base.py:257: in wait

self._raise_future_exception()

google/cloud/aiplatform/base.py:223: in _raise_future_exception

raise self._exception

google/cloud/aiplatform/base.py:235: in _complete_future

future.result()  # raises

/usr/local/lib/python3.8/concurrent/futures/_base.py:437: in result

return self.__get_result()

/usr/local/lib/python3.8/concurrent/futures/_base.py:389: in __get_result

raise self._exception

/usr/local/lib/python3.8/concurrent/futures/thread.py:57: in run

result = self.fn(*self.args, **self.kwargs)

google/cloud/aiplatform/base.py:325: in wait_for_dependencies_and_invoke

result = method(*args, **kwargs)

google/cloud/aiplatform/models.py:3488: in _deploy

endpoint._deploy_call(

google/cloud/aiplatform/models.py:1244: in _deploy_call

operation_future.result(timeout=None)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:130: in result

self._blocking_poll(timeout=timeout, **kwargs)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:108: in blocking_poll

retry(self._done_or_raise)(**kwargs)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:286: in retry_wrapped_func

return retry_target(

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:189: in retry_target

return target()

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:86: in _done_or_raise

if not self.done(**kwargs):

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/operation.py:170: in done

self._refresh_and_update(retry)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/operation.py:158: in _refresh_and_update

self._operation = self._refresh(retry=retry)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/operations_v1/operations_client.py:142: in get_operation

return self._get_operation(

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/gapic_v1/method.py:145: in call

return wrapped_func(*args, **kwargs)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:286: in retry_wrapped_func

return retry_target(

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:189: in retry_target

return target()

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/timeout.py:214: in func_with_timeout

return func(*args, **kwargs)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:69: in error_remapped_callable

six.raise_from(exceptions.from_grpc_error(exc), exc)

value = None

from_value = <_InactiveRpcError of RPC that terminated with:

status = StatusCode.DEADLINE_EXCEEDED

details = "Deadline Exceeded"

...ption":"Deadline Exceeded","file":"src/core/ext/filters/deadline/deadline_filter.cc","file_line":81,"grpc_status":4}"



???

E   google.api_core.exceptions.DeadlineExceeded: 504 Deadline Exceeded

:3: DeadlineExceeded