Comments (2)
Test passed for commit 575ee0f (Build Status, Sponge)! Closing this issue.
from python-aiplatform.
Looks like this issue is flaky. 😟
I'm going to leave this open and stop commenting.
A human should fix and close this.
commit: 921bfc0
buildURL: Build Status, Sponge
status: failed
Test output
args = (name: "projects/580378083368/locations/us-central1/endpoints/8650970681481101312/operations/1970395411877199872" ,) kwargs = {'metadata': [('x-goog-request-params', 'name=projects/580378083368/locations/us-central1/endpoints/8650970681481101312/operations/1970395411877199872'), ('x-goog-api-client', 'gl-python/3.8.13 grpc/1.47.0 gax/1.32.0')], 'timeout': 20.0}@six.wraps(callable_) def error_remapped_callable(*args, **kwargs): try:
return callable_(*args, **kwargs)
.nox/system-3-8/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:67:
self = <grpc._channel._UnaryUnaryMultiCallable object at 0x7fa318b64640>
request = name: "projects/580378083368/locations/us-central1/endpoints/8650970681481101312/operations/1970395411877199872"timeout = 20.0
metadata = [('x-goog-request-params', 'name=projects/580378083368/locations/us-central1/endpoints/8650970681481101312/operations/1970395411877199872'), ('x-goog-api-client', 'gl-python/3.8.13 grpc/1.47.0 gax/1.32.0')]
credentials = None, wait_for_ready = None, compression = Nonedef __call__(self, request, timeout=None, metadata=None, credentials=None, wait_for_ready=None, compression=None): state, call, = self._blocking(request, timeout, metadata, credentials, wait_for_ready, compression)
return _end_unary_response_blocking(state, call, False, None)
.nox/system-3-8/lib/python3.8/site-packages/grpc/_channel.py:946:
state = <grpc._channel._RPCState object at 0x7fa3193d2580>
call = <grpc._cython.cygrpc.SegregatedCall object at 0x7fa2a4898680>
with_call = False, deadline = Nonedef _end_unary_response_blocking(state, call, with_call, deadline): if state.code is grpc.StatusCode.OK: if with_call: rendezvous = _MultiThreadedRendezvous(state, call, None, deadline) return state.response, rendezvous else: return state.response else:
raise _InactiveRpcError(state)
E grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:
E status = StatusCode.DEADLINE_EXCEEDED
E details = "Deadline Exceeded"
E debug_error_string = "{"created":"@1683042356.339949371","description":"Deadline Exceeded","file":"src/core/ext/filters/deadline/deadline_filter.cc","file_line":81,"grpc_status":4}"
E >.nox/system-3-8/lib/python3.8/site-packages/grpc/_channel.py:849: _InactiveRpcError
The above exception was the direct cause of the following exception:
self = <tests.system.aiplatform.test_experiment_model.TestExperimentModel object at 0x7fa2a5f10370>
shared_state = {'bucket': <Bucket: test-vertex-sdk-e2e-experiment-model-6e88677f-a38a-4d56-a495-56>, 'resources': [<google.cloud.aipl...ent-model-6e88677f-a38a-4d56-a495-56', 'storage_client': <google.cloud.storage.client.Client object at 0x7fa26efd40d0>}def test_deploy_model_with_gpu_container(self, shared_state): aiplatform.init( project=e2e_base._PROJECT, location=e2e_base._LOCATION, staging_bucket=f"gs://{shared_state['staging_bucket_name']}", ) # It takes long time to deploy a model. To reduce the system test run # time, we randomly choose one registered model to test deployment. registered_model = random.choice(self.registered_models_gpu) # Deploy the registered model # Since we are using gpu, we need to specify accelerator_type and count endpoint = registered_model.deploy( accelerator_type="NVIDIA_TESLA_T4", accelerator_count=1, sync=False )
pred = endpoint.predict([[1, 2]])
tests/system/aiplatform/test_experiment_model.py:365:
google/cloud/aiplatform/models.py:1526: in predict
self.wait()
google/cloud/aiplatform/base.py:257: in wait
self._raise_future_exception()
google/cloud/aiplatform/base.py:223: in _raise_future_exception
raise self._exception
google/cloud/aiplatform/base.py:235: in _complete_future
future.result() # raises
/usr/local/lib/python3.8/concurrent/futures/_base.py:437: in result
return self.__get_result()
/usr/local/lib/python3.8/concurrent/futures/_base.py:389: in __get_result
raise self._exception
/usr/local/lib/python3.8/concurrent/futures/thread.py:57: in run
result = self.fn(*self.args, **self.kwargs)
google/cloud/aiplatform/base.py:325: in wait_for_dependencies_and_invoke
result = method(*args, **kwargs)
google/cloud/aiplatform/models.py:3488: in _deploy
endpoint._deploy_call(
google/cloud/aiplatform/models.py:1244: in _deploy_call
operation_future.result(timeout=None)
.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:130: in result
self._blocking_poll(timeout=timeout, **kwargs)
.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:108: in blocking_poll
retry(self._done_or_raise)(**kwargs)
.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:286: in retry_wrapped_func
return retry_target(
.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:189: in retry_target
return target()
.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:86: in _done_or_raise
if not self.done(**kwargs):
.nox/system-3-8/lib/python3.8/site-packages/google/api_core/operation.py:170: in done
self._refresh_and_update(retry)
.nox/system-3-8/lib/python3.8/site-packages/google/api_core/operation.py:158: in _refresh_and_update
self._operation = self._refresh(retry=retry)
.nox/system-3-8/lib/python3.8/site-packages/google/api_core/operations_v1/operations_client.py:142: in get_operation
return self._get_operation(
.nox/system-3-8/lib/python3.8/site-packages/google/api_core/gapic_v1/method.py:145: in call
return wrapped_func(*args, **kwargs)
.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:286: in retry_wrapped_func
return retry_target(
.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:189: in retry_target
return target()
.nox/system-3-8/lib/python3.8/site-packages/google/api_core/timeout.py:214: in func_with_timeout
return func(*args, **kwargs)
.nox/system-3-8/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:69: in error_remapped_callable
six.raise_from(exceptions.from_grpc_error(exc), exc)
value = None
from_value = <_InactiveRpcError of RPC that terminated with:
status = StatusCode.DEADLINE_EXCEEDED
details = "Deadline Exceeded"
...ption":"Deadline Exceeded","file":"src/core/ext/filters/deadline/deadline_filter.cc","file_line":81,"grpc_status":4}"???
E google.api_core.exceptions.DeadlineExceeded: 504 Deadline Exceeded:3: DeadlineExceeded
from python-aiplatform.
Related Issues (20)
- tests.system.aiplatform.test_model_upload.TestModelUploadAndUpdate: test_upload_and_deploy_xgboost_model failed HOT 1
- tests.system.aiplatform.test_experiments.TestExperiments: test_get_time_series_data_frame_batch_read_success failed HOT 1
- tests.system.vertex_ray.test_cluster_management.TestClusterManagement: test_cluster_management[2.9] failed HOT 2
- tests.system.vertexai.test_reasoning_engines.TestReasoningEngines: test_langchain_template failed HOT 4
- Gemini Batch Prediction API throws "INTERNAL" error HOT 3
- Getting AttributeError when importing cloud_profiler from google cloud aiplatform HOT 2
- tests.system.vertex_ray.test_ray_data.TestRayData: test_ray_data[2.9] failed HOT 2
- 401 Deadline exception using Imagen Python API
- tests.system.aiplatform.test_language_models.TestLanguageModels: test_text_generation[grpc] failed HOT 1
- tests.system.aiplatform.test_language_models.TestLanguageModels: test_chat_model_async[grpc] failed HOT 1
- tests.unit.aiplatform.test_metadata.TestExperiments: test_get_experiment_df failed HOT 4
- tests.system.vertex_ray.test_job_submission_dashboard.TestJobSubmissionDashboard: test_job_submission_dashboard[2.9] failed HOT 8
- vertex-aiplatform 1.54 throwing runtime exception: Gapic client context issue . This can occur due to parallelization HOT 5
- tests.system.vertexai.test_batch_prediction.TestBatchPrediction: test_batch_prediction_with_gcs_input failed HOT 1
- Allow overwrite_table to be set in vertex_ray.data.write_bigquery() HOT 1
- Batch prediction for Gemini: Failed to import data
- Palm2 TextGeneration parameter logit_bias not working HOT 2
- Gemini Batch Prediction Request Job failed: code: 3 HOT 1
- tests.unit.aiplatform.test_model_evaluation.TestModelEvaluationJob: test_model_evaluation_job_submit[{"pipelineInfo": {"name": "evaluation-default-pipeline"}, "root": {"dag": {"tasks": {}}, "inputDefinitions": {"parameters": {"batch_predict_gcs_source_uris": {"type": "STRING"}, "dataflow_service_account": {"type": "STRING"}, "batch_predict_instances_format": {"type": "STRING"}, "batch_predict_machine_type": {"type": "STRING"}, "evaluation_class_labels": {"type": "STRING"}, "location": {"type": "STRING"}, "model_name": {"type": "STRING"}, "project": {"type": "STRING"}, "batch_predict_gcs_destination_output_uri": {"type": "STRING"}, "target_field_name": {"type": "STRING"}}}}, "schemaVersion": "2.0.0", "sdkVersion": "kfp-1.8.12", "components": {}}] failed HOT 4
- tests.unit.aiplatform.test_model_evaluation.TestModelEvaluationJob: test_model_evaluation_job_submit_with_experiment[{"pipelineInfo": {"name": "evaluation-default-pipeline"}, "root": {"dag": {"tasks": {}}, "inputDefinitions": {"parameters": {"batch_predict_gcs_source_uris": {"type": "STRING"}, "dataflow_service_account": {"type": "STRING"}, "batch_predict_instances_format": {"type": "STRING"}, "batch_predict_machine_type": {"type": "STRING"}, "evaluation_class_labels": {"type": "STRING"}, "location": {"type": "STRING"}, "model_name": {"type": "STRING"}, "project": {"type": "STRING"}, "batch_predict_gcs_destination_output_uri": {"type": "STRING"}, "target_field_name": {"type": "STRING"}}}}, "schemaVersion": "2.0.0", "sdkVersion": "kfp-1.8.12", "components": {}}] failed HOT 4
Recommend Projects
React
A declarative, efficient, and flexible JavaScript library for building user interfaces.
Vue.js
🖖 Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.
Typescript
TypeScript is a superset of JavaScript that compiles to clean JavaScript output.
TensorFlow
An Open Source Machine Learning Framework for Everyone
Django
The Web framework for perfectionists with deadlines.
Laravel
A PHP framework for web artisans
D3
Bring data to life with SVG, Canvas and HTML. 📊📈🎉
Recommend Topics
javascript
JavaScript (JS) is a lightweight interpreted programming language with first-class functions.
web
Some thing interesting about web. New door for the world.
server
A server is a program made to process requests and deliver data to clients.
Machine learning
Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.
Visualization
Some thing interesting about visualization, use data art
Game
Some thing interesting about game, make everyone happy.
Recommend Org
We are working to build community through open source technology. NB: members must have two-factor auth.
Microsoft
Open source projects and samples from Microsoft.
Google ❤️ Open Source for everyone.
Alibaba
Alibaba Open Source for everyone
D3
Data-Driven Documents codes.
Tencent
China tencent open source team.
from python-aiplatform.