tests.system.aiplatform.test_language_models.TestLanguageModels: test_batch_prediction_for_text_generation failed about python-aiplatform HOT 40 CLOSED

self = 
def test_batch_prediction_for_text_generation(self):
    source_uri = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/batch_prediction_prompts1.jsonl"
    destination_uri_prefix = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/predictions/text-bison@001_"

    aiplatform.init(project=e2e_base._PROJECT, location=e2e_base._LOCATION)

    model = TextGenerationModel.from_pretrained("text-bison@001")


  job = model.batch_predict(


        dataset=source_uri,
        destination_uri_prefix=destination_uri_prefix,
        model_parameters={"temperature": 0, "top_p": 1, "top_k": 5},
    )

tests/system/aiplatform/test_language_models.py:170:

vertexai/language_models/_language_models.py:422: in batch_predict

return super().batch_predict(

vertexai/language_models/_language_models.py:380: in batch_predict

job = aiplatform.BatchPredictionJob.create(

google/cloud/aiplatform/jobs.py:794: in create

return cls._create(

google/cloud/aiplatform/base.py:814: in wrapper

return method(*args, **kwargs)

google/cloud/aiplatform/jobs.py:875: in _create

batch_prediction_job._block_until_complete()

self = <google.cloud.aiplatform.jobs.BatchPredictionJob object at 0x7f68d0c15520>

resource name: projects/580378083368/locations/us-central1/batchPredictionJobs/7276704910538702848
def _block_until_complete(self):
    """Helper method to block and check on job until complete.

    Raises:
        RuntimeError: If job failed or cancelled.
    """

    log_wait = _LOG_WAIT_TIME

    previous_time = time.time()
    while self.state not in _JOB_COMPLETE_STATES:
        current_time = time.time()
        if current_time - previous_time >= log_wait:
            self._log_job_state()
            log_wait = min(log_wait * _WAIT_TIME_MULTIPLIER, _MAX_WAIT_TIME)
            previous_time = current_time
        time.sleep(_JOB_WAIT_TIME)

    self._log_job_state()

    # Error is only populated when the job state is
    # JOB_STATE_FAILED or JOB_STATE_CANCELLED.
    if self._gca_resource.state in _JOB_ERROR_STATES:


      raise RuntimeError("Job failed with:\n%s" % self._gca_resource.error)


E           RuntimeError: Job failed with:

E           code: 3

E           message: "Failed to run inference job. Syntax error: Unexpected keyword AS at [78:129]"
google/cloud/aiplatform/jobs.py:241: RuntimeError

self = 
def test_batch_prediction_for_text_generation(self):
    source_uri = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/batch_prediction_prompts1.jsonl"
    destination_uri_prefix = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/predictions/text-bison@001_"

    aiplatform.init(project=e2e_base._PROJECT, location=e2e_base._LOCATION)

    model = TextGenerationModel.from_pretrained("text-bison@001")


  job = model.batch_predict(


        dataset=source_uri,
        destination_uri_prefix=destination_uri_prefix,
        model_parameters={"temperature": 0, "top_p": 1, "top_k": 5},
    )

tests/system/aiplatform/test_language_models.py:170:

vertexai/language_models/_language_models.py:422: in batch_predict

return super().batch_predict(

vertexai/language_models/_language_models.py:380: in batch_predict

job = aiplatform.BatchPredictionJob.create(

google/cloud/aiplatform/jobs.py:794: in create

return cls._create(

google/cloud/aiplatform/base.py:814: in wrapper

return method(*args, **kwargs)

google/cloud/aiplatform/jobs.py:875: in _create

batch_prediction_job._block_until_complete()

self = <google.cloud.aiplatform.jobs.BatchPredictionJob object at 0x7f05efd39fd0>

resource name: projects/580378083368/locations/us-central1/batchPredictionJobs/6611051774990090240
def _block_until_complete(self):
    """Helper method to block and check on job until complete.

    Raises:
        RuntimeError: If job failed or cancelled.
    """

    log_wait = _LOG_WAIT_TIME

    previous_time = time.time()
    while self.state not in _JOB_COMPLETE_STATES:
        current_time = time.time()
        if current_time - previous_time >= log_wait:
            self._log_job_state()
            log_wait = min(log_wait * _WAIT_TIME_MULTIPLIER, _MAX_WAIT_TIME)
            previous_time = current_time
        time.sleep(_JOB_WAIT_TIME)

    self._log_job_state()

    # Error is only populated when the job state is
    # JOB_STATE_FAILED or JOB_STATE_CANCELLED.
    if self._gca_resource.state in _JOB_ERROR_STATES:


      raise RuntimeError("Job failed with:\n%s" % self._gca_resource.error)


E           RuntimeError: Job failed with:

E           code: 3

E           message: "Failed to run inference job. Syntax error: Unexpected keyword AS at [78:129]"
google/cloud/aiplatform/jobs.py:241: RuntimeError

self = 
def test_batch_prediction_for_text_generation(self):
    source_uri = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/batch_prediction_prompts1.jsonl"
    destination_uri_prefix = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/predictions/text-bison@001_"

    aiplatform.init(project=e2e_base._PROJECT, location=e2e_base._LOCATION)

    model = TextGenerationModel.from_pretrained("text-bison@001")


  job = model.batch_predict(


        dataset=source_uri,
        destination_uri_prefix=destination_uri_prefix,
        model_parameters={"temperature": 0, "top_p": 1, "top_k": 5},
    )

tests/system/aiplatform/test_language_models.py:170:

vertexai/language_models/_language_models.py:422: in batch_predict

return super().batch_predict(

vertexai/language_models/_language_models.py:380: in batch_predict

job = aiplatform.BatchPredictionJob.create(

google/cloud/aiplatform/jobs.py:794: in create

return cls._create(

google/cloud/aiplatform/base.py:814: in wrapper

return method(*args, **kwargs)

google/cloud/aiplatform/jobs.py:875: in _create

batch_prediction_job._block_until_complete()

self = <google.cloud.aiplatform.jobs.BatchPredictionJob object at 0x7f6f4c4bfbe0>

resource name: projects/580378083368/locations/us-central1/batchPredictionJobs/7581788600920965120
def _block_until_complete(self):
    """Helper method to block and check on job until complete.

    Raises:
        RuntimeError: If job failed or cancelled.
    """

    log_wait = _LOG_WAIT_TIME

    previous_time = time.time()
    while self.state not in _JOB_COMPLETE_STATES:
        current_time = time.time()
        if current_time - previous_time >= log_wait:
            self._log_job_state()
            log_wait = min(log_wait * _WAIT_TIME_MULTIPLIER, _MAX_WAIT_TIME)
            previous_time = current_time
        time.sleep(_JOB_WAIT_TIME)

    self._log_job_state()

    # Error is only populated when the job state is
    # JOB_STATE_FAILED or JOB_STATE_CANCELLED.
    if self._gca_resource.state in _JOB_ERROR_STATES:


      raise RuntimeError("Job failed with:\n%s" % self._gca_resource.error)


E           RuntimeError: Job failed with:

E           code: 3

E           message: "Failed to run inference job. Syntax error: Unexpected keyword AS at [78:129]"
google/cloud/aiplatform/jobs.py:241: RuntimeError

self = 
def test_batch_prediction_for_text_generation(self):
    source_uri = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/batch_prediction_prompts1.jsonl"
    destination_uri_prefix = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/predictions/text-bison@001_"

    aiplatform.init(project=e2e_base._PROJECT, location=e2e_base._LOCATION)

    model = TextGenerationModel.from_pretrained("text-bison@001")


  job = model.batch_predict(


        dataset=source_uri,
        destination_uri_prefix=destination_uri_prefix,
        model_parameters={"temperature": 0, "top_p": 1, "top_k": 5},
    )

tests/system/aiplatform/test_language_models.py:170:

vertexai/language_models/_language_models.py:429: in batch_predict

return super().batch_predict(

vertexai/language_models/_language_models.py:387: in batch_predict

job = aiplatform.BatchPredictionJob.create(

google/cloud/aiplatform/jobs.py:794: in create

return cls._create(

google/cloud/aiplatform/base.py:814: in wrapper

return method(*args, **kwargs)

google/cloud/aiplatform/jobs.py:875: in _create

batch_prediction_job._block_until_complete()

self = <google.cloud.aiplatform.jobs.BatchPredictionJob object at 0x7f2ef4bb1970>

resource name: projects/580378083368/locations/us-central1/batchPredictionJobs/1662915198463246336
def _block_until_complete(self):
    """Helper method to block and check on job until complete.

    Raises:
        RuntimeError: If job failed or cancelled.
    """

    log_wait = _LOG_WAIT_TIME

    previous_time = time.time()
    while self.state not in _JOB_COMPLETE_STATES:
        current_time = time.time()
        if current_time - previous_time >= log_wait:
            self._log_job_state()
            log_wait = min(log_wait * _WAIT_TIME_MULTIPLIER, _MAX_WAIT_TIME)
            previous_time = current_time
        time.sleep(_JOB_WAIT_TIME)

    self._log_job_state()

    # Error is only populated when the job state is
    # JOB_STATE_FAILED or JOB_STATE_CANCELLED.
    if self._gca_resource.state in _JOB_ERROR_STATES:


      raise RuntimeError("Job failed with:\n%s" % self._gca_resource.error)


E           RuntimeError: Job failed with:

E           code: 3

E           message: "Failed to run inference job. Syntax error: Unexpected keyword AS at [78:129]"
google/cloud/aiplatform/jobs.py:241: RuntimeError

self = 
def test_batch_prediction_for_text_generation(self):
    source_uri = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/batch_prediction_prompts1.jsonl"
    destination_uri_prefix = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/predictions/text-bison@001_"

    aiplatform.init(project=e2e_base._PROJECT, location=e2e_base._LOCATION)

    model = TextGenerationModel.from_pretrained("text-bison@001")


  job = model.batch_predict(


        dataset=source_uri,
        destination_uri_prefix=destination_uri_prefix,
        model_parameters={"temperature": 0, "top_p": 1, "top_k": 5},
    )

tests/system/aiplatform/test_language_models.py:170:

vertexai/language_models/_language_models.py:429: in batch_predict

return super().batch_predict(

vertexai/language_models/_language_models.py:387: in batch_predict

job = aiplatform.BatchPredictionJob.create(

google/cloud/aiplatform/jobs.py:794: in create

return cls._create(

google/cloud/aiplatform/base.py:809: in wrapper

return method(*args, **kwargs)

google/cloud/aiplatform/jobs.py:875: in _create

batch_prediction_job._block_until_complete()

self = <google.cloud.aiplatform.jobs.BatchPredictionJob object at 0x7fbcf1090940>

resource name: projects/580378083368/locations/us-central1/batchPredictionJobs/2317801916152676352
def _block_until_complete(self):
    """Helper method to block and check on job until complete.

    Raises:
        RuntimeError: If job failed or cancelled.
    """

    log_wait = _LOG_WAIT_TIME

    previous_time = time.time()
    while self.state not in _JOB_COMPLETE_STATES:
        current_time = time.time()
        if current_time - previous_time >= log_wait:
            self._log_job_state()
            log_wait = min(log_wait * _WAIT_TIME_MULTIPLIER, _MAX_WAIT_TIME)
            previous_time = current_time
        time.sleep(_JOB_WAIT_TIME)

    self._log_job_state()

    # Error is only populated when the job state is
    # JOB_STATE_FAILED or JOB_STATE_CANCELLED.
    if self._gca_resource.state in _JOB_ERROR_STATES:


      raise RuntimeError("Job failed with:\n%s" % self._gca_resource.error)


E           RuntimeError: Job failed with:

E           code: 3

E           message: "Failed to run inference job. Syntax error: Unexpected keyword AS at [78:129]"
google/cloud/aiplatform/jobs.py:241: RuntimeError

self = 
def test_batch_prediction_for_text_generation(self):
    source_uri = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/batch_prediction_prompts1.jsonl"
    destination_uri_prefix = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/predictions/text-bison@001_"

    aiplatform.init(project=e2e_base._PROJECT, location=e2e_base._LOCATION)

    model = TextGenerationModel.from_pretrained("text-bison@001")


  job = model.batch_predict(


        dataset=source_uri,
        destination_uri_prefix=destination_uri_prefix,
        model_parameters={"temperature": 0, "top_p": 1, "top_k": 5},
    )

tests/system/aiplatform/test_language_models.py:170:

vertexai/language_models/_language_models.py:429: in batch_predict

return super().batch_predict(

vertexai/language_models/_language_models.py:387: in batch_predict

job = aiplatform.BatchPredictionJob.create(

google/cloud/aiplatform/jobs.py:794: in create

return cls._create(

google/cloud/aiplatform/base.py:809: in wrapper

return method(*args, **kwargs)

google/cloud/aiplatform/jobs.py:875: in _create

batch_prediction_job._block_until_complete()

self = <google.cloud.aiplatform.jobs.BatchPredictionJob object at 0x7f100b4beb50>

resource name: projects/580378083368/locations/us-central1/batchPredictionJobs/6439780848731422720
def _block_until_complete(self):
    """Helper method to block and check on job until complete.

    Raises:
        RuntimeError: If job failed or cancelled.
    """

    log_wait = _LOG_WAIT_TIME

    previous_time = time.time()
    while self.state not in _JOB_COMPLETE_STATES:
        current_time = time.time()
        if current_time - previous_time >= log_wait:
            self._log_job_state()
            log_wait = min(log_wait * _WAIT_TIME_MULTIPLIER, _MAX_WAIT_TIME)
            previous_time = current_time
        time.sleep(_JOB_WAIT_TIME)

    self._log_job_state()

    # Error is only populated when the job state is
    # JOB_STATE_FAILED or JOB_STATE_CANCELLED.
    if self._gca_resource.state in _JOB_ERROR_STATES:


      raise RuntimeError("Job failed with:\n%s" % self._gca_resource.error)


E           RuntimeError: Job failed with:

E           code: 3

E           message: "Failed to run inference job. Syntax error: Unexpected keyword AS at [78:129]"
google/cloud/aiplatform/jobs.py:241: RuntimeError

self = 
def test_batch_prediction_for_text_generation(self):
    source_uri = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/batch_prediction_prompts1.jsonl"
    destination_uri_prefix = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/predictions/text-bison@001_"

    aiplatform.init(project=e2e_base._PROJECT, location=e2e_base._LOCATION)

    model = TextGenerationModel.from_pretrained("text-bison@001")


  job = model.batch_predict(


        dataset=source_uri,
        destination_uri_prefix=destination_uri_prefix,
        model_parameters={"temperature": 0, "top_p": 1, "top_k": 5},
    )

tests/system/aiplatform/test_language_models.py:170:

vertexai/language_models/_language_models.py:429: in batch_predict

return super().batch_predict(

vertexai/language_models/_language_models.py:387: in batch_predict

job = aiplatform.BatchPredictionJob.create(

google/cloud/aiplatform/jobs.py:794: in create

return cls._create(

google/cloud/aiplatform/base.py:809: in wrapper

return method(*args, **kwargs)

google/cloud/aiplatform/jobs.py:875: in _create

batch_prediction_job._block_until_complete()

self = <google.cloud.aiplatform.jobs.BatchPredictionJob object at 0x7f3002887b80>

resource name: projects/580378083368/locations/us-central1/batchPredictionJobs/770690099825344512
def _block_until_complete(self):
    """Helper method to block and check on job until complete.

    Raises:
        RuntimeError: If job failed or cancelled.
    """

    log_wait = _LOG_WAIT_TIME

    previous_time = time.time()
    while self.state not in _JOB_COMPLETE_STATES:
        current_time = time.time()
        if current_time - previous_time >= log_wait:
            self._log_job_state()
            log_wait = min(log_wait * _WAIT_TIME_MULTIPLIER, _MAX_WAIT_TIME)
            previous_time = current_time
        time.sleep(_JOB_WAIT_TIME)

    self._log_job_state()

    # Error is only populated when the job state is
    # JOB_STATE_FAILED or JOB_STATE_CANCELLED.
    if self._gca_resource.state in _JOB_ERROR_STATES:


      raise RuntimeError("Job failed with:\n%s" % self._gca_resource.error)


E           RuntimeError: Job failed with:

E           code: 3

E           message: "Failed to run inference job. Syntax error: Unexpected keyword AS at [78:128]"
google/cloud/aiplatform/jobs.py:241: RuntimeError

self = 
def test_batch_prediction_for_text_generation(self):
    source_uri = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/batch_prediction_prompts1.jsonl"
    destination_uri_prefix = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/predictions/text-bison@001_"

    aiplatform.init(project=e2e_base._PROJECT, location=e2e_base._LOCATION)

    model = TextGenerationModel.from_pretrained("text-bison@001")


  job = model.batch_predict(


        dataset=source_uri,
        destination_uri_prefix=destination_uri_prefix,
        model_parameters={"temperature": 0, "top_p": 1, "top_k": 5},
    )

tests/system/aiplatform/test_language_models.py:170:

vertexai/language_models/_language_models.py:434: in batch_predict

return super().batch_predict(

vertexai/language_models/_language_models.py:392: in batch_predict

job = aiplatform.BatchPredictionJob.create(

google/cloud/aiplatform/jobs.py:794: in create

return cls._create(

google/cloud/aiplatform/base.py:809: in wrapper

return method(*args, **kwargs)

google/cloud/aiplatform/jobs.py:875: in _create

batch_prediction_job._block_until_complete()

self = <google.cloud.aiplatform.jobs.BatchPredictionJob object at 0x7f930599b850>

resource name: projects/580378083368/locations/us-central1/batchPredictionJobs/4670490717779394560
def _block_until_complete(self):
    """Helper method to block and check on job until complete.

    Raises:
        RuntimeError: If job failed or cancelled.
    """

    log_wait = _LOG_WAIT_TIME

    previous_time = time.time()
    while self.state not in _JOB_COMPLETE_STATES:
        current_time = time.time()
        if current_time - previous_time >= log_wait:
            self._log_job_state()
            log_wait = min(log_wait * _WAIT_TIME_MULTIPLIER, _MAX_WAIT_TIME)
            previous_time = current_time
        time.sleep(_JOB_WAIT_TIME)

    self._log_job_state()

    # Error is only populated when the job state is
    # JOB_STATE_FAILED or JOB_STATE_CANCELLED.
    if self._gca_resource.state in _JOB_ERROR_STATES:


      raise RuntimeError("Job failed with:\n%s" % self._gca_resource.error)


E           RuntimeError: Job failed with:

E           code: 3

E           message: "Failed to run inference job. Syntax error: Unexpected keyword AS at [78:129]"
google/cloud/aiplatform/jobs.py:241: RuntimeError

self = 
def test_batch_prediction_for_text_generation(self):
    source_uri = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/batch_prediction_prompts1.jsonl"
    destination_uri_prefix = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/predictions/text-bison@001_"

    aiplatform.init(project=e2e_base._PROJECT, location=e2e_base._LOCATION)

    model = TextGenerationModel.from_pretrained("text-bison@001")


  job = model.batch_predict(


        dataset=source_uri,
        destination_uri_prefix=destination_uri_prefix,
        model_parameters={"temperature": 0, "top_p": 1, "top_k": 5},
    )

tests/system/aiplatform/test_language_models.py:170:

vertexai/language_models/_language_models.py:434: in batch_predict

return super().batch_predict(

vertexai/language_models/_language_models.py:392: in batch_predict

job = aiplatform.BatchPredictionJob.create(

google/cloud/aiplatform/jobs.py:794: in create

return cls._create(

google/cloud/aiplatform/base.py:809: in wrapper

return method(*args, **kwargs)

google/cloud/aiplatform/jobs.py:875: in _create

batch_prediction_job._block_until_complete()

self = <google.cloud.aiplatform.jobs.BatchPredictionJob object at 0x7fa2442e9d30>

resource name: projects/580378083368/locations/us-central1/batchPredictionJobs/6576832774110445568
def _block_until_complete(self):
    """Helper method to block and check on job until complete.

    Raises:
        RuntimeError: If job failed or cancelled.
    """

    log_wait = _LOG_WAIT_TIME

    previous_time = time.time()
    while self.state not in _JOB_COMPLETE_STATES:
        current_time = time.time()
        if current_time - previous_time >= log_wait:
            self._log_job_state()
            log_wait = min(log_wait * _WAIT_TIME_MULTIPLIER, _MAX_WAIT_TIME)
            previous_time = current_time
        time.sleep(_JOB_WAIT_TIME)

    self._log_job_state()

    # Error is only populated when the job state is
    # JOB_STATE_FAILED or JOB_STATE_CANCELLED.
    if self._gca_resource.state in _JOB_ERROR_STATES:


      raise RuntimeError("Job failed with:\n%s" % self._gca_resource.error)


E           RuntimeError: Job failed with:

E           code: 3

E           message: "Failed to run inference job. Syntax error: Unexpected keyword AS at [78:129]"
google/cloud/aiplatform/jobs.py:241: RuntimeError

self = 
def test_batch_prediction_for_text_generation(self):
    source_uri = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/batch_prediction_prompts1.jsonl"
    destination_uri_prefix = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/predictions/text-bison@001_"

    aiplatform.init(project=e2e_base._PROJECT, location=e2e_base._LOCATION)

    model = TextGenerationModel.from_pretrained("text-bison@001")


  job = model.batch_predict(


        dataset=source_uri,
        destination_uri_prefix=destination_uri_prefix,
        model_parameters={"temperature": 0, "top_p": 1, "top_k": 5},
    )

tests/system/aiplatform/test_language_models.py:170:

vertexai/language_models/_language_models.py:479: in batch_predict

return super().batch_predict(

vertexai/language_models/_language_models.py:437: in batch_predict

job = aiplatform.BatchPredictionJob.create(

google/cloud/aiplatform/jobs.py:794: in create

return cls._create(

google/cloud/aiplatform/base.py:809: in wrapper

return method(*args, **kwargs)

google/cloud/aiplatform/jobs.py:875: in _create

batch_prediction_job._block_until_complete()

self = <google.cloud.aiplatform.jobs.BatchPredictionJob object at 0x7f4fd3254c40>

resource name: projects/580378083368/locations/us-central1/batchPredictionJobs/7712267645789143040
def _block_until_complete(self):
    """Helper method to block and check on job until complete.

    Raises:
        RuntimeError: If job failed or cancelled.
    """

    log_wait = _LOG_WAIT_TIME

    previous_time = time.time()
    while self.state not in _JOB_COMPLETE_STATES:
        current_time = time.time()
        if current_time - previous_time >= log_wait:
            self._log_job_state()
            log_wait = min(log_wait * _WAIT_TIME_MULTIPLIER, _MAX_WAIT_TIME)
            previous_time = current_time
        time.sleep(_JOB_WAIT_TIME)

    self._log_job_state()

    # Error is only populated when the job state is
    # JOB_STATE_FAILED or JOB_STATE_CANCELLED.
    if self._gca_resource.state in _JOB_ERROR_STATES:


      raise RuntimeError("Job failed with:\n%s" % self._gca_resource.error)


E           RuntimeError: Job failed with:

E           code: 3

E           message: "Failed to run inference job. Syntax error: Unexpected keyword AS at [78:129]"
google/cloud/aiplatform/jobs.py:241: RuntimeError

self = 
def test_batch_prediction_for_text_generation(self):
    source_uri = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/batch_prediction_prompts1.jsonl"
    destination_uri_prefix = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/predictions/text-bison@001_"

    aiplatform.init(project=e2e_base._PROJECT, location=e2e_base._LOCATION)

    model = TextGenerationModel.from_pretrained("text-bison@001")


  job = model.batch_predict(


        dataset=source_uri,
        destination_uri_prefix=destination_uri_prefix,
        model_parameters={"temperature": 0, "top_p": 1, "top_k": 5},
    )

tests/system/aiplatform/test_language_models.py:170:

vertexai/language_models/_language_models.py:488: in batch_predict

return super().batch_predict(

vertexai/language_models/_language_models.py:446: in batch_predict

job = aiplatform.BatchPredictionJob.create(

google/cloud/aiplatform/jobs.py:794: in create

return cls._create(

google/cloud/aiplatform/base.py:809: in wrapper

return method(*args, **kwargs)

google/cloud/aiplatform/jobs.py:875: in _create

batch_prediction_job._block_until_complete()

self = <google.cloud.aiplatform.jobs.BatchPredictionJob object at 0x7f21177162e0>

resource name: projects/580378083368/locations/us-central1/batchPredictionJobs/555111053990559744
def _block_until_complete(self):
    """Helper method to block and check on job until complete.

    Raises:
        RuntimeError: If job failed or cancelled.
    """

    log_wait = _LOG_WAIT_TIME

    previous_time = time.time()
    while self.state not in _JOB_COMPLETE_STATES:
        current_time = time.time()
        if current_time - previous_time >= log_wait:
            self._log_job_state()
            log_wait = min(log_wait * _WAIT_TIME_MULTIPLIER, _MAX_WAIT_TIME)
            previous_time = current_time
        time.sleep(_JOB_WAIT_TIME)

    self._log_job_state()

    # Error is only populated when the job state is
    # JOB_STATE_FAILED or JOB_STATE_CANCELLED.
    if self._gca_resource.state in _JOB_ERROR_STATES:


      raise RuntimeError("Job failed with:\n%s" % self._gca_resource.error)


E           RuntimeError: Job failed with:

E           code: 3

E           message: "Failed to run inference job. Syntax error: Unexpected keyword AS at [78:128]"
google/cloud/aiplatform/jobs.py:241: RuntimeError

self = 
def test_batch_prediction_for_text_generation(self):
    source_uri = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/batch_prediction_prompts1.jsonl"
    destination_uri_prefix = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/predictions/text-bison@001_"

    aiplatform.init(project=e2e_base._PROJECT, location=e2e_base._LOCATION)

    model = TextGenerationModel.from_pretrained("text-bison@001")


  job = model.batch_predict(


        dataset=source_uri,
        destination_uri_prefix=destination_uri_prefix,
        model_parameters={"temperature": 0, "top_p": 1, "top_k": 5},
    )

tests/system/aiplatform/test_language_models.py:170:

vertexai/language_models/_language_models.py:491: in batch_predict

return super().batch_predict(

vertexai/language_models/_language_models.py:449: in batch_predict

job = aiplatform.BatchPredictionJob.create(

google/cloud/aiplatform/jobs.py:794: in create

return cls._create(

google/cloud/aiplatform/base.py:809: in wrapper

return method(*args, **kwargs)

google/cloud/aiplatform/jobs.py:875: in _create

batch_prediction_job._block_until_complete()

self = <google.cloud.aiplatform.jobs.BatchPredictionJob object at 0x7feaa6efddc0>

resource name: projects/580378083368/locations/us-central1/batchPredictionJobs/3646198879482806272
def _block_until_complete(self):
    """Helper method to block and check on job until complete.

    Raises:
        RuntimeError: If job failed or cancelled.
    """

    log_wait = _LOG_WAIT_TIME

    previous_time = time.time()
    while self.state not in _JOB_COMPLETE_STATES:
        current_time = time.time()
        if current_time - previous_time >= log_wait:
            self._log_job_state()
            log_wait = min(log_wait * _WAIT_TIME_MULTIPLIER, _MAX_WAIT_TIME)
            previous_time = current_time
        time.sleep(_JOB_WAIT_TIME)

    self._log_job_state()

    # Error is only populated when the job state is
    # JOB_STATE_FAILED or JOB_STATE_CANCELLED.
    if self._gca_resource.state in _JOB_ERROR_STATES:


      raise RuntimeError("Job failed with:\n%s" % self._gca_resource.error)


E           RuntimeError: Job failed with:

E           code: 3

E           message: "Failed to run inference job. Syntax error: Unexpected keyword AS at [78:129]"
google/cloud/aiplatform/jobs.py:241: RuntimeError

self = 
def test_batch_prediction_for_text_generation(self):
    source_uri = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/batch_prediction_prompts1.jsonl"
    destination_uri_prefix = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/predictions/text-bison@001_"

    aiplatform.init(project=e2e_base._PROJECT, location=e2e_base._LOCATION)

    model = TextGenerationModel.from_pretrained("text-bison@001")


  job = model.batch_predict(


        dataset=source_uri,
        destination_uri_prefix=destination_uri_prefix,
        model_parameters={"temperature": 0, "top_p": 1, "top_k": 5},
    )

tests/system/aiplatform/test_language_models.py:222:

vertexai/language_models/_language_models.py:563: in batch_predict

return super().batch_predict(

vertexai/language_models/_language_models.py:521: in batch_predict

job = aiplatform.BatchPredictionJob.create(

google/cloud/aiplatform/jobs.py:794: in create

return cls._create(

google/cloud/aiplatform/base.py:809: in wrapper

return method(*args, **kwargs)

google/cloud/aiplatform/jobs.py:875: in _create

batch_prediction_job._block_until_complete()

self = <google.cloud.aiplatform.jobs.BatchPredictionJob object at 0x7f98501dac10>

resource name: projects/580378083368/locations/us-central1/batchPredictionJobs/5001588853274836992
def _block_until_complete(self):
    """Helper method to block and check on job until complete.

    Raises:
        RuntimeError: If job failed or cancelled.
    """

    log_wait = _LOG_WAIT_TIME

    previous_time = time.time()
    while self.state not in _JOB_COMPLETE_STATES:
        current_time = time.time()
        if current_time - previous_time >= log_wait:
            self._log_job_state()
            log_wait = min(log_wait * _WAIT_TIME_MULTIPLIER, _MAX_WAIT_TIME)
            previous_time = current_time
        time.sleep(_JOB_WAIT_TIME)

    self._log_job_state()

    # Error is only populated when the job state is
    # JOB_STATE_FAILED or JOB_STATE_CANCELLED.
    if self._gca_resource.state in _JOB_ERROR_STATES:


      raise RuntimeError("Job failed with:\n%s" % self._gca_resource.error)


E           RuntimeError: Job failed with:

E           code: 3

E           message: "Failed to run inference job. Syntax error: Unexpected keyword AS at [78:129]"
google/cloud/aiplatform/jobs.py:241: RuntimeError

self = 
def test_batch_prediction_for_text_generation(self):
    source_uri = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/batch_prediction_prompts1.jsonl"
    destination_uri_prefix = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/predictions/text-bison@001_"

    aiplatform.init(project=e2e_base._PROJECT, location=e2e_base._LOCATION)

    model = TextGenerationModel.from_pretrained("text-bison@001")


  job = model.batch_predict(


        dataset=source_uri,
        destination_uri_prefix=destination_uri_prefix,
        model_parameters={"temperature": 0, "top_p": 1, "top_k": 5},
    )

tests/system/aiplatform/test_language_models.py:223:

vertexai/language_models/_language_models.py:570: in batch_predict

return super().batch_predict(

vertexai/language_models/_language_models.py:528: in batch_predict

job = aiplatform.BatchPredictionJob.create(

google/cloud/aiplatform/jobs.py:794: in create

return cls._create(

google/cloud/aiplatform/base.py:809: in wrapper

return method(*args, **kwargs)

google/cloud/aiplatform/jobs.py:875: in _create

batch_prediction_job._block_until_complete()

self = <google.cloud.aiplatform.jobs.BatchPredictionJob object at 0x7fb3c8354130>

resource name: projects/580378083368/locations/us-central1/batchPredictionJobs/5528580378421362688
def _block_until_complete(self):
    """Helper method to block and check on job until complete.

    Raises:
        RuntimeError: If job failed or cancelled.
    """

    log_wait = _LOG_WAIT_TIME

    previous_time = time.time()
    while self.state not in _JOB_COMPLETE_STATES:
        current_time = time.time()
        if current_time - previous_time >= log_wait:
            self._log_job_state()
            log_wait = min(log_wait * _WAIT_TIME_MULTIPLIER, _MAX_WAIT_TIME)
            previous_time = current_time
        time.sleep(_JOB_WAIT_TIME)

    self._log_job_state()

    # Error is only populated when the job state is
    # JOB_STATE_FAILED or JOB_STATE_CANCELLED.
    if self._gca_resource.state in _JOB_ERROR_STATES:


      raise RuntimeError("Job failed with:\n%s" % self._gca_resource.error)


E           RuntimeError: Job failed with:

E           code: 3

E           message: "Failed to run inference job. Syntax error: Unexpected keyword AS at [78:129]"
google/cloud/aiplatform/jobs.py:241: RuntimeError

self = 
def test_batch_prediction_for_text_generation(self):
    source_uri = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/batch_prediction_prompts1.jsonl"
    destination_uri_prefix = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/predictions/text-bison@001_"

    aiplatform.init(project=e2e_base._PROJECT, location=e2e_base._LOCATION)

    model = TextGenerationModel.from_pretrained("text-bison@001")


  job = model.batch_predict(


        dataset=source_uri,
        destination_uri_prefix=destination_uri_prefix,
        model_parameters={"temperature": 0, "top_p": 1, "top_k": 5},
    )

tests/system/aiplatform/test_language_models.py:229:

vertexai/language_models/_language_models.py:563: in batch_predict

return super().batch_predict(

vertexai/language_models/_language_models.py:521: in batch_predict

job = aiplatform.BatchPredictionJob.create(

google/cloud/aiplatform/jobs.py:794: in create

return cls._create(

google/cloud/aiplatform/base.py:809: in wrapper

return method(*args, **kwargs)

google/cloud/aiplatform/jobs.py:875: in _create

batch_prediction_job._block_until_complete()

self = <google.cloud.aiplatform.jobs.BatchPredictionJob object at 0x7f6c70b3cf70>

resource name: projects/580378083368/locations/us-central1/batchPredictionJobs/8341315840225837056
def _block_until_complete(self):
    """Helper method to block and check on job until complete.

    Raises:
        RuntimeError: If job failed or cancelled.
    """

    log_wait = _LOG_WAIT_TIME

    previous_time = time.time()
    while self.state not in _JOB_COMPLETE_STATES:
        current_time = time.time()
        if current_time - previous_time >= log_wait:
            self._log_job_state()
            log_wait = min(log_wait * _WAIT_TIME_MULTIPLIER, _MAX_WAIT_TIME)
            previous_time = current_time
        time.sleep(_JOB_WAIT_TIME)

    self._log_job_state()

    # Error is only populated when the job state is
    # JOB_STATE_FAILED or JOB_STATE_CANCELLED.
    if self._gca_resource.state in _JOB_ERROR_STATES:


      raise RuntimeError("Job failed with:\n%s" % self._gca_resource.error)


E           RuntimeError: Job failed with:

E           code: 3

E           message: "Failed to run inference job. Syntax error: Unexpected keyword AS at [78:129]"
google/cloud/aiplatform/jobs.py:241: RuntimeError

self = 
def test_batch_prediction_for_text_generation(self):
    source_uri = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/batch_prediction_prompts1.jsonl"
    destination_uri_prefix = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/predictions/text-bison@001_"

    aiplatform.init(project=e2e_base._PROJECT, location=e2e_base._LOCATION)

    model = TextGenerationModel.from_pretrained("text-bison@001")


  job = model.batch_predict(


        dataset=source_uri,
        destination_uri_prefix=destination_uri_prefix,
        model_parameters={"temperature": 0, "top_p": 1, "top_k": 5},
    )

tests/system/aiplatform/test_language_models.py:245:

vertexai/language_models/_language_models.py:736: in batch_predict

return super().batch_predict(

vertexai/language_models/_language_models.py:694: in batch_predict

job = aiplatform.BatchPredictionJob.create(

google/cloud/aiplatform/jobs.py:794: in create

return cls._create(

google/cloud/aiplatform/base.py:809: in wrapper

return method(*args, **kwargs)

google/cloud/aiplatform/jobs.py:875: in _create

batch_prediction_job._block_until_complete()

self = <google.cloud.aiplatform.jobs.BatchPredictionJob object at 0x7fa310d2e820>

resource name: projects/580378083368/locations/us-central1/batchPredictionJobs/9217688180214464512
def _block_until_complete(self):
    """Helper method to block and check on job until complete.

    Raises:
        RuntimeError: If job failed or cancelled.
    """

    log_wait = _LOG_WAIT_TIME

    previous_time = time.time()
    while self.state not in _JOB_COMPLETE_STATES:
        current_time = time.time()
        if current_time - previous_time >= log_wait:
            self._log_job_state()
            log_wait = min(log_wait * _WAIT_TIME_MULTIPLIER, _MAX_WAIT_TIME)
            previous_time = current_time
        time.sleep(_JOB_WAIT_TIME)

    self._log_job_state()

    # Error is only populated when the job state is
    # JOB_STATE_FAILED or JOB_STATE_CANCELLED.
    if self._gca_resource.state in _JOB_ERROR_STATES:


      raise RuntimeError("Job failed with:\n%s" % self._gca_resource.error)


E           RuntimeError: Job failed with:

E           code: 3

E           message: "Failed to run inference job. Syntax error: Unexpected keyword AS at [78:129]"
google/cloud/aiplatform/jobs.py:241: RuntimeError

self = 
def test_batch_prediction_for_text_generation(self):
    source_uri = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/batch_prediction_prompts1.jsonl"
    destination_uri_prefix = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/predictions/text-bison@001_"

    aiplatform.init(project=e2e_base._PROJECT, location=e2e_base._LOCATION)

    model = TextGenerationModel.from_pretrained("text-bison@001")


  job = model.batch_predict(


        dataset=source_uri,
        destination_uri_prefix=destination_uri_prefix,
        model_parameters={"temperature": 0, "top_p": 1, "top_k": 5},
    )

tests/system/aiplatform/test_language_models.py:259:

vertexai/language_models/_language_models.py:890: in batch_predict

return super().batch_predict(

vertexai/language_models/_language_models.py:848: in batch_predict

job = aiplatform.BatchPredictionJob.create(

google/cloud/aiplatform/jobs.py:794: in create

return cls._create(

google/cloud/aiplatform/base.py:809: in wrapper

return method(*args, **kwargs)

google/cloud/aiplatform/jobs.py:875: in _create

batch_prediction_job._block_until_complete()

self = <google.cloud.aiplatform.jobs.BatchPredictionJob object at 0x7fa13013c940>

resource name: projects/580378083368/locations/us-central1/batchPredictionJobs/5908422063399370752
def _block_until_complete(self):
    """Helper method to block and check on job until complete.

    Raises:
        RuntimeError: If job failed or cancelled.
    """

    log_wait = _LOG_WAIT_TIME

    previous_time = time.time()
    while self.state not in _JOB_COMPLETE_STATES:
        current_time = time.time()
        if current_time - previous_time >= log_wait:
            self._log_job_state()
            log_wait = min(log_wait * _WAIT_TIME_MULTIPLIER, _MAX_WAIT_TIME)
            previous_time = current_time
        time.sleep(_JOB_WAIT_TIME)

    self._log_job_state()

    # Error is only populated when the job state is
    # JOB_STATE_FAILED or JOB_STATE_CANCELLED.
    if self._gca_resource.state in _JOB_ERROR_STATES:


      raise RuntimeError("Job failed with:\n%s" % self._gca_resource.error)


E           RuntimeError: Job failed with:

E           code: 3

E           message: "Failed to run inference job. Syntax error: Unexpected keyword AS at [78:129]"
google/cloud/aiplatform/jobs.py:241: RuntimeError

self = 
def test_batch_prediction_for_text_generation(self):
    source_uri = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/batch_prediction_prompts1.jsonl"
    destination_uri_prefix = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/predictions/text-bison@001_"

    aiplatform.init(project=e2e_base._PROJECT, location=e2e_base._LOCATION)

    model = TextGenerationModel.from_pretrained("text-bison@001")


  job = model.batch_predict(


        dataset=source_uri,
        destination_uri_prefix=destination_uri_prefix,
        model_parameters={"temperature": 0, "top_p": 1, "top_k": 5},
    )

tests/system/aiplatform/test_language_models.py:259:

vertexai/language_models/_language_models.py:890: in batch_predict

return super().batch_predict(

vertexai/language_models/_language_models.py:848: in batch_predict

job = aiplatform.BatchPredictionJob.create(

google/cloud/aiplatform/jobs.py:794: in create

return cls._create(

google/cloud/aiplatform/base.py:809: in wrapper

return method(*args, **kwargs)

google/cloud/aiplatform/jobs.py:875: in _create

batch_prediction_job._block_until_complete()

self = <google.cloud.aiplatform.jobs.BatchPredictionJob object at 0x7f4a090a8dc0>

resource name: projects/580378083368/locations/us-central1/batchPredictionJobs/7917572854973988864
def _block_until_complete(self):
    """Helper method to block and check on job until complete.

    Raises:
        RuntimeError: If job failed or cancelled.
    """

    log_wait = _LOG_WAIT_TIME

    previous_time = time.time()
    while self.state not in _JOB_COMPLETE_STATES:
        current_time = time.time()
        if current_time - previous_time >= log_wait:
            self._log_job_state()
            log_wait = min(log_wait * _WAIT_TIME_MULTIPLIER, _MAX_WAIT_TIME)
            previous_time = current_time
        time.sleep(_JOB_WAIT_TIME)

    self._log_job_state()

    # Error is only populated when the job state is
    # JOB_STATE_FAILED or JOB_STATE_CANCELLED.
    if self._gca_resource.state in _JOB_ERROR_STATES:


      raise RuntimeError("Job failed with:\n%s" % self._gca_resource.error)


E           RuntimeError: Job failed with:

E           code: 3

E           message: "Failed to run inference job. Syntax error: Unexpected keyword AS at [78:129]"
google/cloud/aiplatform/jobs.py:241: RuntimeError

self = 
def test_batch_prediction_for_text_generation(self):
    source_uri = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/batch_prediction_prompts1.jsonl"
    destination_uri_prefix = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/predictions/text-bison@001_"

    aiplatform.init(project=e2e_base._PROJECT, location=e2e_base._LOCATION)

    model = TextGenerationModel.from_pretrained("text-bison@001")


  job = model.batch_predict(


        dataset=source_uri,
        destination_uri_prefix=destination_uri_prefix,
        model_parameters={"temperature": 0, "top_p": 1, "top_k": 5},
    )

tests/system/aiplatform/test_language_models.py:259:

vertexai/language_models/_language_models.py:890: in batch_predict

return super().batch_predict(

vertexai/language_models/_language_models.py:848: in batch_predict

job = aiplatform.BatchPredictionJob.create(

google/cloud/aiplatform/jobs.py:794: in create

return cls._create(

google/cloud/aiplatform/base.py:809: in wrapper

return method(*args, **kwargs)

google/cloud/aiplatform/jobs.py:875: in _create

batch_prediction_job._block_until_complete()

self = <google.cloud.aiplatform.jobs.BatchPredictionJob object at 0x7fcd2c8d00d0>

resource name: projects/580378083368/locations/us-central1/batchPredictionJobs/6996270071827922944
def _block_until_complete(self):
    """Helper method to block and check on job until complete.

    Raises:
        RuntimeError: If job failed or cancelled.
    """

    log_wait = _LOG_WAIT_TIME

    previous_time = time.time()
    while self.state not in _JOB_COMPLETE_STATES:
        current_time = time.time()
        if current_time - previous_time >= log_wait:
            self._log_job_state()
            log_wait = min(log_wait * _WAIT_TIME_MULTIPLIER, _MAX_WAIT_TIME)
            previous_time = current_time
        time.sleep(_JOB_WAIT_TIME)

    self._log_job_state()

    # Error is only populated when the job state is
    # JOB_STATE_FAILED or JOB_STATE_CANCELLED.
    if self._gca_resource.state in _JOB_ERROR_STATES:


      raise RuntimeError("Job failed with:\n%s" % self._gca_resource.error)


E           RuntimeError: Job failed with:

E           code: 3

E           message: "Failed to run inference job. Syntax error: Unexpected keyword AS at [78:129]"
google/cloud/aiplatform/jobs.py:241: RuntimeError

self = 
def test_batch_prediction_for_text_generation(self):
    source_uri = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/batch_prediction_prompts1.jsonl"
    destination_uri_prefix = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/predictions/text-bison@001_"

    aiplatform.init(project=e2e_base._PROJECT, location=e2e_base._LOCATION)

    model = TextGenerationModel.from_pretrained("text-bison@001")


  job = model.batch_predict(


        dataset=source_uri,
        destination_uri_prefix=destination_uri_prefix,
        model_parameters={"temperature": 0, "top_p": 1, "top_k": 5},
    )

tests/system/aiplatform/test_language_models.py:259:

vertexai/language_models/_language_models.py:890: in batch_predict

return super().batch_predict(

vertexai/language_models/_language_models.py:848: in batch_predict

job = aiplatform.BatchPredictionJob.create(

google/cloud/aiplatform/jobs.py:795: in create

return cls._create(

google/cloud/aiplatform/base.py:809: in wrapper

return method(*args, **kwargs)

google/cloud/aiplatform/jobs.py:876: in _create

batch_prediction_job._block_until_complete()

self = <google.cloud.aiplatform.jobs.BatchPredictionJob object at 0x7f234c5e2370>

resource name: projects/580378083368/locations/us-central1/batchPredictionJobs/1697401380668440576
def _block_until_complete(self):
    """Helper method to block and check on job until complete.

    Raises:
        RuntimeError: If job failed or cancelled.
    """

    log_wait = _LOG_WAIT_TIME

    previous_time = time.time()
    while self.state not in _JOB_COMPLETE_STATES:
        current_time = time.time()
        if current_time - previous_time >= log_wait:
            self._log_job_state()
            log_wait = min(log_wait * _WAIT_TIME_MULTIPLIER, _MAX_WAIT_TIME)
            previous_time = current_time
        time.sleep(_JOB_WAIT_TIME)

    self._log_job_state()

    # Error is only populated when the job state is
    # JOB_STATE_FAILED or JOB_STATE_CANCELLED.
    if self._gca_resource.state in _JOB_ERROR_STATES:


      raise RuntimeError("Job failed with:\n%s" % self._gca_resource.error)


E           RuntimeError: Job failed with:

E           code: 3

E           message: "Failed to run inference job. Syntax error: Unexpected keyword AS at [78:129]"
google/cloud/aiplatform/jobs.py:241: RuntimeError

self = 
def test_batch_prediction_for_text_generation(self):
    source_uri = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/batch_prediction_prompts1.jsonl"
    destination_uri_prefix = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/predictions/text-bison@001_"

    aiplatform.init(project=e2e_base._PROJECT, location=e2e_base._LOCATION)

    model = TextGenerationModel.from_pretrained("text-bison@001")


  job = model.batch_predict(


        dataset=source_uri,
        destination_uri_prefix=destination_uri_prefix,
        model_parameters={"temperature": 0, "top_p": 1, "top_k": 5},
    )

tests/system/aiplatform/test_language_models.py:259:

vertexai/language_models/_language_models.py:890: in batch_predict

return super().batch_predict(

vertexai/language_models/_language_models.py:848: in batch_predict

job = aiplatform.BatchPredictionJob.create(

google/cloud/aiplatform/jobs.py:795: in create

return cls._create(

google/cloud/aiplatform/base.py:809: in wrapper

return method(*args, **kwargs)

google/cloud/aiplatform/jobs.py:876: in _create

batch_prediction_job._block_until_complete()

self = <google.cloud.aiplatform.jobs.BatchPredictionJob object at 0x7f62783669d0>

resource name: projects/580378083368/locations/us-central1/batchPredictionJobs/8573219335198212096
def _block_until_complete(self):
    """Helper method to block and check on job until complete.

    Raises:
        RuntimeError: If job failed or cancelled.
    """

    log_wait = _LOG_WAIT_TIME

    previous_time = time.time()
    while self.state not in _JOB_COMPLETE_STATES:
        current_time = time.time()
        if current_time - previous_time >= log_wait:
            self._log_job_state()
            log_wait = min(log_wait * _WAIT_TIME_MULTIPLIER, _MAX_WAIT_TIME)
            previous_time = current_time
        time.sleep(_JOB_WAIT_TIME)

    self._log_job_state()

    # Error is only populated when the job state is
    # JOB_STATE_FAILED or JOB_STATE_CANCELLED.
    if self._gca_resource.state in _JOB_ERROR_STATES:


      raise RuntimeError("Job failed with:\n%s" % self._gca_resource.error)


E           RuntimeError: Job failed with:

E           code: 3

E           message: "Failed to run inference job. Syntax error: Unexpected keyword AS at [78:129]"
google/cloud/aiplatform/jobs.py:241: RuntimeError

self = 
def test_batch_prediction_for_text_generation(self):
    source_uri = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/batch_prediction_prompts1.jsonl"
    destination_uri_prefix = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/predictions/text-bison@001_"

    aiplatform.init(project=e2e_base._PROJECT, location=e2e_base._LOCATION)

    model = TextGenerationModel.from_pretrained("text-bison@001")


  job = model.batch_predict(


        dataset=source_uri,
        destination_uri_prefix=destination_uri_prefix,
        model_parameters={"temperature": 0, "top_p": 1, "top_k": 5},
    )

tests/system/aiplatform/test_language_models.py:259:

vertexai/language_models/_language_models.py:890: in batch_predict

return super().batch_predict(

vertexai/language_models/_language_models.py:848: in batch_predict

job = aiplatform.BatchPredictionJob.create(

google/cloud/aiplatform/jobs.py:794: in create

return cls._create(

google/cloud/aiplatform/base.py:809: in wrapper

return method(*args, **kwargs)

google/cloud/aiplatform/jobs.py:875: in _create

batch_prediction_job._block_until_complete()

self = <google.cloud.aiplatform.jobs.BatchPredictionJob object at 0x7f8a24399220>

resource name: projects/580378083368/locations/us-central1/batchPredictionJobs/7626944443962097664
def _block_until_complete(self):
    """Helper method to block and check on job until complete.

    Raises:
        RuntimeError: If job failed or cancelled.
    """

    log_wait = _LOG_WAIT_TIME

    previous_time = time.time()
    while self.state not in _JOB_COMPLETE_STATES:
        current_time = time.time()
        if current_time - previous_time >= log_wait:
            self._log_job_state()
            log_wait = min(log_wait * _WAIT_TIME_MULTIPLIER, _MAX_WAIT_TIME)
            previous_time = current_time
        time.sleep(_JOB_WAIT_TIME)

    self._log_job_state()

    # Error is only populated when the job state is
    # JOB_STATE_FAILED or JOB_STATE_CANCELLED.
    if self._gca_resource.state in _JOB_ERROR_STATES:


      raise RuntimeError("Job failed with:\n%s" % self._gca_resource.error)


E           RuntimeError: Job failed with:

E           code: 3

E           message: "Failed to run inference job. Syntax error: Unexpected keyword AS at [78:129]"
google/cloud/aiplatform/jobs.py:241: RuntimeError

self = 
def test_batch_prediction_for_text_generation(self):
    source_uri = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/batch_prediction_prompts1.jsonl"
    destination_uri_prefix = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/predictions/text-bison@001_"

    aiplatform.init(project=e2e_base._PROJECT, location=e2e_base._LOCATION)

    model = TextGenerationModel.from_pretrained("text-bison@001")


  job = model.batch_predict(


        dataset=source_uri,
        destination_uri_prefix=destination_uri_prefix,
        model_parameters={"temperature": 0, "top_p": 1, "top_k": 5},
    )

tests/system/aiplatform/test_language_models.py:259:

vertexai/language_models/_language_models.py:890: in batch_predict

return super().batch_predict(

vertexai/language_models/_language_models.py:848: in batch_predict

job = aiplatform.BatchPredictionJob.create(

google/cloud/aiplatform/jobs.py:795: in create

return cls._create(

google/cloud/aiplatform/base.py:809: in wrapper

return method(*args, **kwargs)

google/cloud/aiplatform/jobs.py:876: in _create

batch_prediction_job._block_until_complete()

self = <google.cloud.aiplatform.jobs.BatchPredictionJob object at 0x7f6f6d3d3550>

resource name: projects/580378083368/locations/us-central1/batchPredictionJobs/7404544027988590592
def _block_until_complete(self):
    """Helper method to block and check on job until complete.

    Raises:
        RuntimeError: If job failed or cancelled.
    """

    log_wait = _LOG_WAIT_TIME

    previous_time = time.time()
    while self.state not in _JOB_COMPLETE_STATES:
        current_time = time.time()
        if current_time - previous_time >= log_wait:
            self._log_job_state()
            log_wait = min(log_wait * _WAIT_TIME_MULTIPLIER, _MAX_WAIT_TIME)
            previous_time = current_time
        time.sleep(_JOB_WAIT_TIME)

    self._log_job_state()

    # Error is only populated when the job state is
    # JOB_STATE_FAILED or JOB_STATE_CANCELLED.
    if self._gca_resource.state in _JOB_ERROR_STATES:


      raise RuntimeError("Job failed with:\n%s" % self._gca_resource.error)


E           RuntimeError: Job failed with:

E           code: 3

E           message: "Failed to run inference job. Syntax error: Unexpected keyword AS at [78:129]"
google/cloud/aiplatform/jobs.py:241: RuntimeError

self = 
def test_batch_prediction_for_text_generation(self):
    source_uri = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/batch_prediction_prompts1.jsonl"
    destination_uri_prefix = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/predictions/text-bison@001_"

    aiplatform.init(project=e2e_base._PROJECT, location=e2e_base._LOCATION)

    model = TextGenerationModel.from_pretrained("text-bison@001")


  job = model.batch_predict(


        dataset=source_uri,
        destination_uri_prefix=destination_uri_prefix,
        model_parameters={"temperature": 0, "top_p": 1, "top_k": 5},
    )

tests/system/aiplatform/test_language_models.py:259:

vertexai/language_models/_language_models.py:890: in batch_predict

return super().batch_predict(

vertexai/language_models/_language_models.py:848: in batch_predict

job = aiplatform.BatchPredictionJob.create(

google/cloud/aiplatform/jobs.py:795: in create

return cls._create(

google/cloud/aiplatform/base.py:809: in wrapper

return method(*args, **kwargs)

google/cloud/aiplatform/jobs.py:876: in _create

batch_prediction_job._block_until_complete()

self = <google.cloud.aiplatform.jobs.BatchPredictionJob object at 0x7fa55cb32010>

resource name: projects/580378083368/locations/us-central1/batchPredictionJobs/6356045341694885888
def _block_until_complete(self):
    """Helper method to block and check on job until complete.

    Raises:
        RuntimeError: If job failed or cancelled.
    """

    log_wait = _LOG_WAIT_TIME

    previous_time = time.time()
    while self.state not in _JOB_COMPLETE_STATES:
        current_time = time.time()
        if current_time - previous_time >= log_wait:
            self._log_job_state()
            log_wait = min(log_wait * _WAIT_TIME_MULTIPLIER, _MAX_WAIT_TIME)
            previous_time = current_time
        time.sleep(_JOB_WAIT_TIME)

    self._log_job_state()

    # Error is only populated when the job state is
    # JOB_STATE_FAILED or JOB_STATE_CANCELLED.
    if self._gca_resource.state in _JOB_ERROR_STATES:


      raise RuntimeError("Job failed with:\n%s" % self._gca_resource.error)


E           RuntimeError: Job failed with:

E           code: 3

E           message: "Failed to run inference job. Syntax error: Unexpected keyword AS at [78:129]"
google/cloud/aiplatform/jobs.py:241: RuntimeError

self = 
def test_batch_prediction_for_text_generation(self):
    source_uri = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/batch_prediction_prompts1.jsonl"
    destination_uri_prefix = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/predictions/text-bison@001_"

    aiplatform.init(project=e2e_base._PROJECT, location=e2e_base._LOCATION)

    model = TextGenerationModel.from_pretrained("text-bison@001")


  job = model.batch_predict(


        dataset=source_uri,
        destination_uri_prefix=destination_uri_prefix,
        model_parameters={"temperature": 0, "top_p": 1, "top_k": 5},
    )

tests/system/aiplatform/test_language_models.py:259:

vertexai/language_models/_language_models.py:885: in batch_predict

return super().batch_predict(

vertexai/language_models/_language_models.py:843: in batch_predict

job = aiplatform.BatchPredictionJob.create(

google/cloud/aiplatform/jobs.py:795: in create

return cls._create(

google/cloud/aiplatform/base.py:809: in wrapper

return method(*args, **kwargs)

google/cloud/aiplatform/jobs.py:876: in _create

batch_prediction_job._block_until_complete()

self = <google.cloud.aiplatform.jobs.BatchPredictionJob object at 0x7f4e78183ad0>

resource name: projects/580378083368/locations/us-central1/batchPredictionJobs/6633535688266481664
def _block_until_complete(self):
    """Helper method to block and check on job until complete.

    Raises:
        RuntimeError: If job failed or cancelled.
    """

    log_wait = _LOG_WAIT_TIME

    previous_time = time.time()
    while self.state not in _JOB_COMPLETE_STATES:
        current_time = time.time()
        if current_time - previous_time >= log_wait:
            self._log_job_state()
            log_wait = min(log_wait * _WAIT_TIME_MULTIPLIER, _MAX_WAIT_TIME)
            previous_time = current_time
        time.sleep(_JOB_WAIT_TIME)

    self._log_job_state()

    # Error is only populated when the job state is
    # JOB_STATE_FAILED or JOB_STATE_CANCELLED.
    if self._gca_resource.state in _JOB_ERROR_STATES:


      raise RuntimeError("Job failed with:\n%s" % self._gca_resource.error)


E           RuntimeError: Job failed with:

E           code: 3

E           message: "Failed to run inference job. Syntax error: Unexpected keyword AS at [78:129]"
google/cloud/aiplatform/jobs.py:241: RuntimeError

self = 
def test_batch_prediction_for_text_generation(self):
    source_uri = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/batch_prediction_prompts1.jsonl"
    destination_uri_prefix = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/predictions/text-bison@001_"

    aiplatform.init(project=e2e_base._PROJECT, location=e2e_base._LOCATION)

    model = TextGenerationModel.from_pretrained("text-bison@001")


  job = model.batch_predict(


        dataset=source_uri,
        destination_uri_prefix=destination_uri_prefix,
        model_parameters={"temperature": 0, "top_p": 1, "top_k": 5},
    )

tests/system/aiplatform/test_language_models.py:259:

vertexai/language_models/_language_models.py:905: in batch_predict

return super().batch_predict(

vertexai/language_models/_language_models.py:863: in batch_predict

job = aiplatform.BatchPredictionJob.create(

google/cloud/aiplatform/jobs.py:795: in create

return cls._create(

google/cloud/aiplatform/base.py:809: in wrapper

return method(*args, **kwargs)

google/cloud/aiplatform/jobs.py:876: in _create

batch_prediction_job._block_until_complete()

self = <google.cloud.aiplatform.jobs.BatchPredictionJob object at 0x7fb2cdd1bb10>

resource name: projects/580378083368/locations/us-central1/batchPredictionJobs/6254775922730205184
def _block_until_complete(self):
    """Helper method to block and check on job until complete.

    Raises:
        RuntimeError: If job failed or cancelled.
    """

    log_wait = _LOG_WAIT_TIME

    previous_time = time.time()
    while self.state not in _JOB_COMPLETE_STATES:
        current_time = time.time()
        if current_time - previous_time >= log_wait:
            self._log_job_state()
            log_wait = min(log_wait * _WAIT_TIME_MULTIPLIER, _MAX_WAIT_TIME)
            previous_time = current_time
        time.sleep(_JOB_WAIT_TIME)

    self._log_job_state()

    # Error is only populated when the job state is
    # JOB_STATE_FAILED or JOB_STATE_CANCELLED.
    if self._gca_resource.state in _JOB_ERROR_STATES:


      raise RuntimeError("Job failed with:\n%s" % self._gca_resource.error)


E           RuntimeError: Job failed with:

E           code: 3

E           message: "Failed to run inference job. Syntax error: Unexpected keyword AS at [78:129]"
google/cloud/aiplatform/jobs.py:241: RuntimeError

self = 
def test_batch_prediction_for_text_generation(self):
    source_uri = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/batch_prediction_prompts1.jsonl"
    destination_uri_prefix = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/predictions/text-bison@001_"

    aiplatform.init(project=e2e_base._PROJECT, location=e2e_base._LOCATION)

    model = TextGenerationModel.from_pretrained("text-bison@001")


  job = model.batch_predict(


        dataset=source_uri,
        destination_uri_prefix=destination_uri_prefix,
        model_parameters={"temperature": 0, "top_p": 1, "top_k": 5},
    )

tests/system/aiplatform/test_language_models.py:259:

vertexai/language_models/_language_models.py:905: in batch_predict

return super().batch_predict(

vertexai/language_models/_language_models.py:863: in batch_predict

job = aiplatform.BatchPredictionJob.create(

google/cloud/aiplatform/jobs.py:795: in create

return cls._create(

google/cloud/aiplatform/base.py:819: in wrapper

return method(*args, **kwargs)

google/cloud/aiplatform/jobs.py:876: in _create

batch_prediction_job._block_until_complete()

self = <google.cloud.aiplatform.jobs.BatchPredictionJob object at 0x7f304e65bb50>

resource name: projects/580378083368/locations/us-central1/batchPredictionJobs/8517535668321124352
def _block_until_complete(self):
    """Helper method to block and check on job until complete.

    Raises:
        RuntimeError: If job failed or cancelled.
    """

    log_wait = _LOG_WAIT_TIME

    previous_time = time.time()
    while self.state not in _JOB_COMPLETE_STATES:
        current_time = time.time()
        if current_time - previous_time >= log_wait:
            self._log_job_state()
            log_wait = min(log_wait * _WAIT_TIME_MULTIPLIER, _MAX_WAIT_TIME)
            previous_time = current_time
        time.sleep(_JOB_WAIT_TIME)

    self._log_job_state()

    # Error is only populated when the job state is
    # JOB_STATE_FAILED or JOB_STATE_CANCELLED.
    if self._gca_resource.state in _JOB_ERROR_STATES:


      raise RuntimeError("Job failed with:\n%s" % self._gca_resource.error)


E           RuntimeError: Job failed with:

E           code: 3

E           message: "Failed to run inference job. Syntax error: Unexpected keyword AS at [78:129]"
google/cloud/aiplatform/jobs.py:241: RuntimeError

self = 
def test_batch_prediction_for_text_generation(self):
    source_uri = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/batch_prediction_prompts1.jsonl"
    destination_uri_prefix = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/predictions/text-bison@001_"

    aiplatform.init(project=e2e_base._PROJECT, location=e2e_base._LOCATION)

    model = TextGenerationModel.from_pretrained("text-bison@001")


  job = model.batch_predict(


        dataset=source_uri,
        destination_uri_prefix=destination_uri_prefix,
        model_parameters={"temperature": 0, "top_p": 1, "top_k": 5},
    )

tests/system/aiplatform/test_language_models.py:259:

vertexai/language_models/_language_models.py:907: in batch_predict

return super().batch_predict(

vertexai/language_models/_language_models.py:865: in batch_predict

job = aiplatform.BatchPredictionJob.create(

google/cloud/aiplatform/jobs.py:795: in create

return cls._create(

google/cloud/aiplatform/base.py:819: in wrapper

return method(*args, **kwargs)

google/cloud/aiplatform/jobs.py:876: in _create

batch_prediction_job._block_until_complete()

self = <google.cloud.aiplatform.jobs.BatchPredictionJob object at 0x7f5039b73510>

resource name: projects/580378083368/locations/us-central1/batchPredictionJobs/3753325396888649728
def _block_until_complete(self):
    """Helper method to block and check on job until complete.

    Raises:
        RuntimeError: If job failed or cancelled.
    """

    log_wait = _LOG_WAIT_TIME

    previous_time = time.time()
    while self.state not in _JOB_COMPLETE_STATES:
        current_time = time.time()
        if current_time - previous_time >= log_wait:
            self._log_job_state()
            log_wait = min(log_wait * _WAIT_TIME_MULTIPLIER, _MAX_WAIT_TIME)
            previous_time = current_time
        time.sleep(_JOB_WAIT_TIME)

    self._log_job_state()

    # Error is only populated when the job state is
    # JOB_STATE_FAILED or JOB_STATE_CANCELLED.
    if self._gca_resource.state in _JOB_ERROR_STATES:


      raise RuntimeError("Job failed with:\n%s" % self._gca_resource.error)


E           RuntimeError: Job failed with:

E           code: 3

E           message: "Failed to run inference job. Syntax error: Unexpected keyword AS at [78:129]"
google/cloud/aiplatform/jobs.py:241: RuntimeError

self = 
def test_batch_prediction_for_text_generation(self):
    source_uri = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/batch_prediction_prompts1.jsonl"
    destination_uri_prefix = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/predictions/text-bison@001_"

    aiplatform.init(project=e2e_base._PROJECT, location=e2e_base._LOCATION)

    model = TextGenerationModel.from_pretrained("text-bison@001")


  job = model.batch_predict(


        dataset=source_uri,
        destination_uri_prefix=destination_uri_prefix,
        model_parameters={"temperature": 0, "top_p": 1, "top_k": 5},
    )

tests/system/aiplatform/test_language_models.py:259:

vertexai/language_models/_language_models.py:907: in batch_predict

return super().batch_predict(

vertexai/language_models/_language_models.py:865: in batch_predict

job = aiplatform.BatchPredictionJob.create(

google/cloud/aiplatform/jobs.py:795: in create

return cls._create(

google/cloud/aiplatform/base.py:819: in wrapper

return method(*args, **kwargs)

google/cloud/aiplatform/jobs.py:876: in _create

batch_prediction_job._block_until_complete()

self = <google.cloud.aiplatform.jobs.BatchPredictionJob object at 0x7fb26088f8d0>

resource name: projects/580378083368/locations/us-central1/batchPredictionJobs/8703298157833879552
def _block_until_complete(self):
    """Helper method to block and check on job until complete.

    Raises:
        RuntimeError: If job failed or cancelled.
    """

    log_wait = _LOG_WAIT_TIME

    previous_time = time.time()
    while self.state not in _JOB_COMPLETE_STATES:
        current_time = time.time()
        if current_time - previous_time >= log_wait:
            self._log_job_state()
            log_wait = min(log_wait * _WAIT_TIME_MULTIPLIER, _MAX_WAIT_TIME)
            previous_time = current_time
        time.sleep(_JOB_WAIT_TIME)

    self._log_job_state()

    # Error is only populated when the job state is
    # JOB_STATE_FAILED or JOB_STATE_CANCELLED.
    if self._gca_resource.state in _JOB_ERROR_STATES:


      raise RuntimeError("Job failed with:\n%s" % self._gca_resource.error)


E           RuntimeError: Job failed with:

E           code: 3

E           message: "Failed to run inference job. Syntax error: Unexpected keyword AS at [78:129]"
google/cloud/aiplatform/jobs.py:241: RuntimeError

self = 
def test_batch_prediction_for_text_generation(self):
    source_uri = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/batch_prediction_prompts1.jsonl"
    destination_uri_prefix = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/predictions/text-bison@001_"

    aiplatform.init(project=e2e_base._PROJECT, location=e2e_base._LOCATION)

    model = TextGenerationModel.from_pretrained("text-bison@001")


  job = model.batch_predict(


        dataset=source_uri,
        destination_uri_prefix=destination_uri_prefix,
        model_parameters={"temperature": 0, "top_p": 1, "top_k": 5},
    )

tests/system/aiplatform/test_language_models.py:259:

vertexai/language_models/_language_models.py:907: in batch_predict

return super().batch_predict(

vertexai/language_models/_language_models.py:865: in batch_predict

job = aiplatform.BatchPredictionJob.create(

google/cloud/aiplatform/jobs.py:795: in create

return cls._create(

google/cloud/aiplatform/base.py:819: in wrapper

return method(*args, **kwargs)

google/cloud/aiplatform/jobs.py:876: in _create

batch_prediction_job._block_until_complete()

self = <google.cloud.aiplatform.jobs.BatchPredictionJob object at 0x7f5d04d19350>

resource name: projects/580378083368/locations/us-central1/batchPredictionJobs/3908618220172476416
def _block_until_complete(self):
    """Helper method to block and check on job until complete.

    Raises:
        RuntimeError: If job failed or cancelled.
    """

    log_wait = _LOG_WAIT_TIME

    previous_time = time.time()
    while self.state not in _JOB_COMPLETE_STATES:
        current_time = time.time()
        if current_time - previous_time >= log_wait:
            self._log_job_state()
            log_wait = min(log_wait * _WAIT_TIME_MULTIPLIER, _MAX_WAIT_TIME)
            previous_time = current_time
        time.sleep(_JOB_WAIT_TIME)

    self._log_job_state()

    # Error is only populated when the job state is
    # JOB_STATE_FAILED or JOB_STATE_CANCELLED.
    if self._gca_resource.state in _JOB_ERROR_STATES:


      raise RuntimeError("Job failed with:\n%s" % self._gca_resource.error)


E           RuntimeError: Job failed with:

E           code: 3

E           message: "Failed to run inference job. Syntax error: Unexpected keyword AS at [78:129]"
google/cloud/aiplatform/jobs.py:241: RuntimeError

self = 
def test_batch_prediction_for_text_generation(self):
    source_uri = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/batch_prediction_prompts1.jsonl"
    destination_uri_prefix = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/predictions/text-bison@001_"

    aiplatform.init(project=e2e_base._PROJECT, location=e2e_base._LOCATION)

    model = TextGenerationModel.from_pretrained("text-bison@001")


  job = model.batch_predict(


        dataset=source_uri,
        destination_uri_prefix=destination_uri_prefix,
        model_parameters={"temperature": 0, "top_p": 1, "top_k": 5},
    )

tests/system/aiplatform/test_language_models.py:259:

vertexai/language_models/_language_models.py:911: in batch_predict

return super().batch_predict(

vertexai/language_models/_language_models.py:869: in batch_predict

job = aiplatform.BatchPredictionJob.create(

google/cloud/aiplatform/jobs.py:795: in create

return cls._create(

google/cloud/aiplatform/base.py:819: in wrapper

return method(*args, **kwargs)

google/cloud/aiplatform/jobs.py:876: in _create

batch_prediction_job._block_until_complete()

self = <google.cloud.aiplatform.jobs.BatchPredictionJob object at 0x7faa1e4db290>

resource name: projects/580378083368/locations/us-central1/batchPredictionJobs/4995872492322029568
def _block_until_complete(self):
    """Helper method to block and check on job until complete.

    Raises:
        RuntimeError: If job failed or cancelled.
    """

    log_wait = _LOG_WAIT_TIME

    previous_time = time.time()
    while self.state not in _JOB_COMPLETE_STATES:
        current_time = time.time()
        if current_time - previous_time >= log_wait:
            self._log_job_state()
            log_wait = min(log_wait * _WAIT_TIME_MULTIPLIER, _MAX_WAIT_TIME)
            previous_time = current_time
        time.sleep(_JOB_WAIT_TIME)

    self._log_job_state()

    # Error is only populated when the job state is
    # JOB_STATE_FAILED or JOB_STATE_CANCELLED.
    if self._gca_resource.state in _JOB_ERROR_STATES:


      raise RuntimeError("Job failed with:\n%s" % self._gca_resource.error)


E           RuntimeError: Job failed with:

E           code: 3

E           message: "Failed to run inference job. Syntax error: Unexpected keyword AS at [78:129]"
google/cloud/aiplatform/jobs.py:241: RuntimeError

self = 
def test_batch_prediction_for_text_generation(self):
    source_uri = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/batch_prediction_prompts1.jsonl"
    destination_uri_prefix = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/predictions/text-bison@001_"

    aiplatform.init(project=e2e_base._PROJECT, location=e2e_base._LOCATION)

    model = TextGenerationModel.from_pretrained("text-bison@001")


  job = model.batch_predict(


        dataset=source_uri,
        destination_uri_prefix=destination_uri_prefix,
        model_parameters={"temperature": 0, "top_p": 1, "top_k": 5},
    )

tests/system/aiplatform/test_language_models.py:259:

vertexai/language_models/_language_models.py:916: in batch_predict

return super().batch_predict(

vertexai/language_models/_language_models.py:874: in batch_predict

job = aiplatform.BatchPredictionJob.create(

google/cloud/aiplatform/jobs.py:795: in create

return cls._create(

google/cloud/aiplatform/base.py:819: in wrapper

return method(*args, **kwargs)

google/cloud/aiplatform/jobs.py:876: in _create

batch_prediction_job._block_until_complete()

self = <google.cloud.aiplatform.jobs.BatchPredictionJob object at 0x7f8a4d9c1990>

resource name: projects/580378083368/locations/us-central1/batchPredictionJobs/1022131916423299072
def _block_until_complete(self):
    """Helper method to block and check on job until complete.

    Raises:
        RuntimeError: If job failed or cancelled.
    """

    log_wait = _LOG_WAIT_TIME

    previous_time = time.time()
    while self.state not in _JOB_COMPLETE_STATES:
        current_time = time.time()
        if current_time - previous_time >= log_wait:
            self._log_job_state()
            log_wait = min(log_wait * _WAIT_TIME_MULTIPLIER, _MAX_WAIT_TIME)
            previous_time = current_time
        time.sleep(_JOB_WAIT_TIME)

    self._log_job_state()

    # Error is only populated when the job state is
    # JOB_STATE_FAILED or JOB_STATE_CANCELLED.
    if self._gca_resource.state in _JOB_ERROR_STATES:


      raise RuntimeError("Job failed with:\n%s" % self._gca_resource.error)


E           RuntimeError: Job failed with:

E           code: 3

E           message: "Failed to run inference job. Syntax error: Unexpected keyword AS at [78:129]"
google/cloud/aiplatform/jobs.py:241: RuntimeError

self = 
def test_batch_prediction_for_text_generation(self):
    source_uri = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/batch_prediction_prompts1.jsonl"
    destination_uri_prefix = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/predictions/text-bison@001_"

    aiplatform.init(project=e2e_base._PROJECT, location=e2e_base._LOCATION)

    model = TextGenerationModel.from_pretrained("text-bison@001")


  job = model.batch_predict(


        dataset=source_uri,
        destination_uri_prefix=destination_uri_prefix,
        model_parameters={"temperature": 0, "top_p": 1, "top_k": 5},
    )

tests/system/aiplatform/test_language_models.py:259:

vertexai/language_models/_language_models.py:916: in batch_predict

return super().batch_predict(

vertexai/language_models/_language_models.py:874: in batch_predict

job = aiplatform.BatchPredictionJob.create(

google/cloud/aiplatform/jobs.py:795: in create

return cls._create(

google/cloud/aiplatform/base.py:819: in wrapper

return method(*args, **kwargs)

google/cloud/aiplatform/jobs.py:876: in _create

batch_prediction_job._block_until_complete()

self = <google.cloud.aiplatform.jobs.BatchPredictionJob object at 0x7f318e94a510>

resource name: projects/580378083368/locations/us-central1/batchPredictionJobs/1053199716977737728
def _block_until_complete(self):
    """Helper method to block and check on job until complete.

    Raises:
        RuntimeError: If job failed or cancelled.
    """

    log_wait = _LOG_WAIT_TIME

    previous_time = time.time()
    while self.state not in _JOB_COMPLETE_STATES:
        current_time = time.time()
        if current_time - previous_time >= log_wait:
            self._log_job_state()
            log_wait = min(log_wait * _WAIT_TIME_MULTIPLIER, _MAX_WAIT_TIME)
            previous_time = current_time
        time.sleep(_JOB_WAIT_TIME)

    self._log_job_state()

    # Error is only populated when the job state is
    # JOB_STATE_FAILED or JOB_STATE_CANCELLED.
    if self._gca_resource.state in _JOB_ERROR_STATES:


      raise RuntimeError("Job failed with:\n%s" % self._gca_resource.error)


E           RuntimeError: Job failed with:

E           code: 3

E           message: "Failed to run inference job. Syntax error: Unexpected keyword AS at [78:129]"
google/cloud/aiplatform/jobs.py:241: RuntimeError

self = 
def test_batch_prediction_for_text_generation(self):
    source_uri = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/batch_prediction_prompts1.jsonl"
    destination_uri_prefix = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/predictions/text-bison@001_"

    aiplatform.init(project=e2e_base._PROJECT, location=e2e_base._LOCATION)

    model = TextGenerationModel.from_pretrained("text-bison@001")


  job = model.batch_predict(


        dataset=source_uri,
        destination_uri_prefix=destination_uri_prefix,
        model_parameters={"temperature": 0, "top_p": 1, "top_k": 5},
    )

tests/system/aiplatform/test_language_models.py:259:

vertexai/language_models/_language_models.py:916: in batch_predict

return super().batch_predict(

vertexai/language_models/_language_models.py:874: in batch_predict

job = aiplatform.BatchPredictionJob.create(

google/cloud/aiplatform/jobs.py:795: in create

return cls._create(

google/cloud/aiplatform/base.py:819: in wrapper

return method(*args, **kwargs)

google/cloud/aiplatform/jobs.py:876: in _create

batch_prediction_job._block_until_complete()

self = <google.cloud.aiplatform.jobs.BatchPredictionJob object at 0x7f0d846d4c50>

resource name: projects/580378083368/locations/us-central1/batchPredictionJobs/6373947590018334720
def _block_until_complete(self):
    """Helper method to block and check on job until complete.

    Raises:
        RuntimeError: If job failed or cancelled.
    """

    log_wait = _LOG_WAIT_TIME

    previous_time = time.time()
    while self.state not in _JOB_COMPLETE_STATES:
        current_time = time.time()
        if current_time - previous_time >= log_wait:
            self._log_job_state()
            log_wait = min(log_wait * _WAIT_TIME_MULTIPLIER, _MAX_WAIT_TIME)
            previous_time = current_time
        time.sleep(_JOB_WAIT_TIME)

    self._log_job_state()

    # Error is only populated when the job state is
    # JOB_STATE_FAILED or JOB_STATE_CANCELLED.
    if self._gca_resource.state in _JOB_ERROR_STATES:


      raise RuntimeError("Job failed with:\n%s" % self._gca_resource.error)


E           RuntimeError: Job failed with:

E           code: 3

E           message: "Failed to run inference job. Syntax error: Unexpected keyword AS at [78:129]"
google/cloud/aiplatform/jobs.py:241: RuntimeError

self = 
def test_batch_prediction_for_text_generation(self):
    source_uri = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/batch_prediction_prompts1.jsonl"
    destination_uri_prefix = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/predictions/text-bison@001_"

    aiplatform.init(project=e2e_base._PROJECT, location=e2e_base._LOCATION)

    model = TextGenerationModel.from_pretrained("text-bison@001")


  job = model.batch_predict(


        dataset=source_uri,
        destination_uri_prefix=destination_uri_prefix,
        model_parameters={"temperature": 0, "top_p": 1, "top_k": 5},
    )

tests/system/aiplatform/test_language_models.py:259:

vertexai/language_models/_language_models.py:916: in batch_predict

return super().batch_predict(

vertexai/language_models/_language_models.py:874: in batch_predict

job = aiplatform.BatchPredictionJob.create(

google/cloud/aiplatform/jobs.py:795: in create

return cls._create(

google/cloud/aiplatform/base.py:819: in wrapper

return method(*args, **kwargs)

google/cloud/aiplatform/jobs.py:876: in _create

batch_prediction_job._block_until_complete()

self = <google.cloud.aiplatform.jobs.BatchPredictionJob object at 0x7f4ee956f590>

resource name: projects/580378083368/locations/us-central1/batchPredictionJobs/255279730654183424
def _block_until_complete(self):
    """Helper method to block and check on job until complete.

    Raises:
        RuntimeError: If job failed or cancelled.
    """

    log_wait = _LOG_WAIT_TIME

    previous_time = time.time()
    while self.state not in _JOB_COMPLETE_STATES:
        current_time = time.time()
        if current_time - previous_time >= log_wait:
            self._log_job_state()
            log_wait = min(log_wait * _WAIT_TIME_MULTIPLIER, _MAX_WAIT_TIME)
            previous_time = current_time
        time.sleep(_JOB_WAIT_TIME)

    self._log_job_state()

    # Error is only populated when the job state is
    # JOB_STATE_FAILED or JOB_STATE_CANCELLED.
    if self._gca_resource.state in _JOB_ERROR_STATES:


      raise RuntimeError("Job failed with:\n%s" % self._gca_resource.error)


E           RuntimeError: Job failed with:

E           code: 3

E           message: "Failed to run inference job. Syntax error: Unexpected keyword AS at [78:128]"
google/cloud/aiplatform/jobs.py:241: RuntimeError

self = 
def test_batch_prediction_for_text_generation(self):
    source_uri = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/batch_prediction_prompts1.jsonl"
    destination_uri_prefix = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/predictions/text-bison@001_"

    aiplatform.init(project=e2e_base._PROJECT, location=e2e_base._LOCATION)

    model = TextGenerationModel.from_pretrained("text-bison@001")


  job = model.batch_predict(


        dataset=source_uri,
        destination_uri_prefix=destination_uri_prefix,
        model_parameters={"temperature": 0, "top_p": 1, "top_k": 5},
    )

tests/system/aiplatform/test_language_models.py:259:

vertexai/language_models/_language_models.py:916: in batch_predict

return super().batch_predict(

vertexai/language_models/_language_models.py:874: in batch_predict

job = aiplatform.BatchPredictionJob.create(

google/cloud/aiplatform/jobs.py:795: in create

return cls._create(

google/cloud/aiplatform/base.py:819: in wrapper

return method(*args, **kwargs)

google/cloud/aiplatform/jobs.py:876: in _create

batch_prediction_job._block_until_complete()

self = <google.cloud.aiplatform.jobs.BatchPredictionJob object at 0x7fab2419f8d0>

resource name: projects/580378083368/locations/us-central1/batchPredictionJobs/393044139568005120
def _block_until_complete(self):
    """Helper method to block and check on job until complete.

    Raises:
        RuntimeError: If job failed or cancelled.
    """

    log_wait = _LOG_WAIT_TIME

    previous_time = time.time()
    while self.state not in _JOB_COMPLETE_STATES:
        current_time = time.time()
        if current_time - previous_time >= log_wait:
            self._log_job_state()
            log_wait = min(log_wait * _WAIT_TIME_MULTIPLIER, _MAX_WAIT_TIME)
            previous_time = current_time
        time.sleep(_JOB_WAIT_TIME)

    self._log_job_state()

    # Error is only populated when the job state is
    # JOB_STATE_FAILED or JOB_STATE_CANCELLED.
    if self._gca_resource.state in _JOB_ERROR_STATES:


      raise RuntimeError("Job failed with:\n%s" % self._gca_resource.error)


E           RuntimeError: Job failed with:

E           code: 3

E           message: "Failed to run inference job. Syntax error: Unexpected keyword AS at [78:128]"
google/cloud/aiplatform/jobs.py:241: RuntimeError

self = 
def test_batch_prediction_for_text_generation(self):
    source_uri = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/batch_prediction_prompts1.jsonl"
    destination_uri_prefix = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/predictions/text-bison@001_"

    aiplatform.init(project=e2e_base._PROJECT, location=e2e_base._LOCATION)

    model = TextGenerationModel.from_pretrained("text-bison@001")


  job = model.batch_predict(


        dataset=source_uri,
        destination_uri_prefix=destination_uri_prefix,
        model_parameters={"temperature": 0, "top_p": 1, "top_k": 5},
    )

tests/system/aiplatform/test_language_models.py:259:

vertexai/language_models/_language_models.py:916: in batch_predict

return super().batch_predict(

vertexai/language_models/_language_models.py:874: in batch_predict

job = aiplatform.BatchPredictionJob.create(

google/cloud/aiplatform/jobs.py:795: in create

return cls._create(

google/cloud/aiplatform/base.py:819: in wrapper

return method(*args, **kwargs)

google/cloud/aiplatform/jobs.py:876: in _create

batch_prediction_job._block_until_complete()

self = <google.cloud.aiplatform.jobs.BatchPredictionJob object at 0x7f259b0c02d0>

resource name: projects/580378083368/locations/us-central1/batchPredictionJobs/6178376082396282880
def _block_until_complete(self):
    """Helper method to block and check on job until complete.

    Raises:
        RuntimeError: If job failed or cancelled.
    """

    log_wait = _LOG_WAIT_TIME

    previous_time = time.time()
    while self.state not in _JOB_COMPLETE_STATES:
        current_time = time.time()
        if current_time - previous_time >= log_wait:
            self._log_job_state()
            log_wait = min(log_wait * _WAIT_TIME_MULTIPLIER, _MAX_WAIT_TIME)
            previous_time = current_time
        time.sleep(_JOB_WAIT_TIME)

    self._log_job_state()

    # Error is only populated when the job state is
    # JOB_STATE_FAILED or JOB_STATE_CANCELLED.
    if self._gca_resource.state in _JOB_ERROR_STATES:


      raise RuntimeError("Job failed with:\n%s" % self._gca_resource.error)


E           RuntimeError: Job failed with:

E           code: 3

E           message: "Failed to run inference job. Syntax error: Unexpected keyword AS at [78:129]"
google/cloud/aiplatform/jobs.py:241: RuntimeError

self = 
def test_batch_prediction_for_text_generation(self):
    source_uri = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/batch_prediction_prompts1.jsonl"
    destination_uri_prefix = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/predictions/text-bison@001_"

    aiplatform.init(project=e2e_base._PROJECT, location=e2e_base._LOCATION)

    model = TextGenerationModel.from_pretrained("text-bison@001")


  job = model.batch_predict(


        dataset=source_uri,
        destination_uri_prefix=destination_uri_prefix,
        model_parameters={"temperature": 0, "top_p": 1, "top_k": 5},
    )

tests/system/aiplatform/test_language_models.py:259:

vertexai/language_models/_language_models.py:916: in batch_predict

return super().batch_predict(

vertexai/language_models/_language_models.py:874: in batch_predict

job = aiplatform.BatchPredictionJob.create(

google/cloud/aiplatform/jobs.py:795: in create

return cls._create(

google/cloud/aiplatform/base.py:819: in wrapper

return method(*args, **kwargs)

google/cloud/aiplatform/jobs.py:876: in _create

batch_prediction_job._block_until_complete()

self = <google.cloud.aiplatform.jobs.BatchPredictionJob object at 0x7f0065617250>

resource name: projects/580378083368/locations/us-central1/batchPredictionJobs/6793891487718309888
def _block_until_complete(self):
    """Helper method to block and check on job until complete.

    Raises:
        RuntimeError: If job failed or cancelled.
    """

    log_wait = _LOG_WAIT_TIME

    previous_time = time.time()
    while self.state not in _JOB_COMPLETE_STATES:
        current_time = time.time()
        if current_time - previous_time >= log_wait:
            self._log_job_state()
            log_wait = min(log_wait * _WAIT_TIME_MULTIPLIER, _MAX_WAIT_TIME)
            previous_time = current_time
        time.sleep(_JOB_WAIT_TIME)

    self._log_job_state()

    # Error is only populated when the job state is
    # JOB_STATE_FAILED or JOB_STATE_CANCELLED.
    if self._gca_resource.state in _JOB_ERROR_STATES:


      raise RuntimeError("Job failed with:\n%s" % self._gca_resource.error)


E           RuntimeError: Job failed with:

E           code: 3

E           message: "Failed to run inference job. Syntax error: Unexpected keyword AS at [78:129]"
google/cloud/aiplatform/jobs.py:241: RuntimeError

self = 
def test_batch_prediction_for_text_generation(self):
    source_uri = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/batch_prediction_prompts1.jsonl"
    destination_uri_prefix = "gs://ucaip-samples-us-central1/model/llm/batch_prediction/predictions/text-bison@001_"

    aiplatform.init(project=e2e_base._PROJECT, location=e2e_base._LOCATION)

    model = TextGenerationModel.from_pretrained("text-bison@001")


  job = model.batch_predict(


        dataset=source_uri,
        destination_uri_prefix=destination_uri_prefix,
        model_parameters={"temperature": 0, "top_p": 1, "top_k": 5},
    )

tests/system/aiplatform/test_language_models.py:259:

vertexai/language_models/_language_models.py:884: in batch_predict

return super().batch_predict(

vertexai/language_models/_language_models.py:842: in batch_predict

job = aiplatform.BatchPredictionJob.create(

google/cloud/aiplatform/jobs.py:795: in create

return cls._create(

google/cloud/aiplatform/base.py:819: in wrapper

return method(*args, **kwargs)

google/cloud/aiplatform/jobs.py:876: in _create

batch_prediction_job._block_until_complete()

self = <google.cloud.aiplatform.jobs.BatchPredictionJob object at 0x7f9ce06cf810>

resource name: projects/580378083368/locations/us-central1/batchPredictionJobs/6359100609630568448
def _block_until_complete(self):
    """Helper method to block and check on job until complete.

    Raises:
        RuntimeError: If job failed or cancelled.
    """

    log_wait = _LOG_WAIT_TIME

    previous_time = time.time()
    while self.state not in _JOB_COMPLETE_STATES:
        current_time = time.time()
        if current_time - previous_time >= log_wait:
            self._log_job_state()
            log_wait = min(log_wait * _WAIT_TIME_MULTIPLIER, _MAX_WAIT_TIME)
            previous_time = current_time
        time.sleep(_JOB_WAIT_TIME)

    self._log_job_state()

    # Error is only populated when the job state is
    # JOB_STATE_FAILED or JOB_STATE_CANCELLED.
    if self._gca_resource.state in _JOB_ERROR_STATES:


      raise RuntimeError("Job failed with:\n%s" % self._gca_resource.error)


E           RuntimeError: Job failed with:

E           code: 3

E           message: "Failed to run inference job. Syntax error: Unexpected keyword AS at [78:129]"
google/cloud/aiplatform/jobs.py:241: RuntimeError