jobs are failing on awx.main.commands.run_callback_receiver Database Error about awx HOT 3 OPEN

edited the post with formating.

@yaron-Kalatian can you provide the result_traceback from /api/v2/jobs/715793

from awx.

HTTP 200 OK
Allow: GET, DELETE, HEAD, OPTIONS
Content-Type: application/json
Vary: Accept
X-API-Node: awx-web-666c695b9c-8jgsv
X-API-Product-Name: AWX
X-API-Product-Version: 23.5.1
X-API-Time: 0.046s

{
"id": 715793,
"type": "job",
"url": "/api/v2/jobs/715793/",
"related": {
"created_by": "/api/v2/users/52/",
"labels": "/api/v2/jobs/715793/labels/",
"inventory": "/api/v2/inventories/13/",
"project": "/api/v2/projects/6/",
"organization": "/api/v2/organizations/1/",
"credentials": "/api/v2/jobs/715793/credentials/",
"unified_job_template": "/api/v2/job_templates/309/",
"stdout": "/api/v2/jobs/715793/stdout/",
"execution_environment": "/api/v2/execution_environments/1/",
"job_events": "/api/v2/jobs/715793/job_events/",
"job_host_summaries": "/api/v2/jobs/715793/job_host_summaries/",
"activity_stream": "/api/v2/jobs/715793/activity_stream/",
"notifications": "/api/v2/jobs/715793/notifications/",
"create_schedule": "/api/v2/jobs/715793/create_schedule/",
"job_template": "/api/v2/job_templates/309/",
"cancel": "/api/v2/jobs/715793/cancel/",
"relaunch": "/api/v2/jobs/715793/relaunch/"
},
"summary_fields": {
"organization": {
"id": 1,
"name": "Default",
"description": ""
},
"inventory": {
"id": 13,
"name": "ansible-general-inventory",
"description": "",
"has_active_failures": true,
"total_hosts": 196,
"hosts_with_active_failures": 33,
"total_groups": 46,
"has_inventory_sources": true,
"total_inventory_sources": 1,
"inventory_sources_with_failures": 0,
"organization_id": 3,
"kind": ""
},
"execution_environment": {
"id": 1,
"name": "AWX EE (latest)",
"description": "",
"image": "quay.io/ansible/awx-ee:latest"
},
"project": {
"id": 6,
"name": "ansible-infra",
"description": "",
"status": "successful",
"scm_type": "git",
"allow_override": false
},
"job_template": {
"id": 309,
"name": "OCI clean files by regex",
"description": "OCI clean files by regex"
},
"unified_job_template": {
"id": 309,
"name": "OCI clean files by regex",
"description": "OCI clean files by regex",
"unified_job_type": "job"
},
"instance_group": {
"id": 3,
"name": "default",
"is_container_group": true
},
"created_by": {
"id": 52,
"username": "automation",
"first_name": "Automation",
"last_name": ""
},
"user_capabilities": {
"delete": true,
"start": true
},
"labels": {
"count": 0,
"results": []
},
"credentials": [
{
"id": 19,
"name": "OCI-OPC",
"description": "",
"kind": "ssh",
"cloud": false
}
]
},
"created": "2024-05-26T06:20:45.151086Z",
"modified": "2024-05-26T06:20:46.164093Z",
"name": "OCI clean files by regex",
"description": "OCI clean files by regex",
"job_type": "run",
"inventory": 13,
"project": 6,
"playbook": "playbooks/clean_files_regex.yaml",
"scm_branch": "",
"forks": 0,
"limit": "oci-ash-sdp300",
"verbosity": 0,
"extra_vars": "{"become": true, "file_regex": ".part*", "file_age": 70, "directory_path": "/etl/flink///"}",
"job_tags": "",
"force_handlers": false,
"skip_tags": "",
"start_at_task": "",
"timeout": 0,
"use_fact_cache": true,
"organization": 1,
"unified_job_template": 309,
"launch_type": "manual",
"status": "error",
"execution_environment": 1,
"failed": true,
"started": "2024-05-26T06:20:46.186101Z",
"finished": "2024-05-26T06:20:53.456029Z",
"canceled_on": null,
"elapsed": 7.27,
"job_args": "["ssh-agent", "sh", "-c", "trap 'rm -f /runner/artifacts/715793/ssh_key_data' EXIT && ssh-add /runner/artifacts/715793/ssh_key_data && rm -f /runner/artifacts/715793/ssh_key_data && ansible-playbook -u opc --become -l oci-ash-sdp300 -i /runner/inventory/hosts -e @/runner/env/extravars playbooks/clean_files_regex.yaml"]",
"job_cwd": "/runner/project",
"job_env": {
"PWD": "/runner",
"HOME": "/var/lib/awx",
"PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"TERM": "xterm",
"SHLVL": "0",
"JOB_ID": "715793",
"AWX_HOST": "https://towerhost/",
"HOSTNAME": "automation-job-715793-cgrpb",
"LC_CTYPE": "C.UTF-8",
"INVENTORY_ID": "13",
"MAX_EVENT_RES": "700000",
"KUBERNETES_PORT": "tcp://10.32.0.1:443",
"AWX_SERVICE_PORT": "tcp://10.47.211.133:80",
"PROJECT_REVISION": "d349a2868641669a3f133bb42e39d461df00f9be",
"ANSIBLE_ROLES_PATH": "/runner/requirements_roles:roles:/.ansible/roles:/usr/share/ansible/roles:/etc/ansible/roles",
"RUNNER_OMIT_EVENTS": "False",
"ANSIBLE_FORCE_COLOR": "True",
"ANSIBLE_CACHE_PLUGIN": "jsonfile",
"AWX_PRIVATE_DATA_DIR": "/tmp/awx_715793_3bvr2nvk",
"AWX_ISOLATED_DATA_DIR": "/runner/artifacts/715793",
"ANSIBLE_STDOUT_CALLBACK": "awx_display",
"AWX_SERVICE_PORT_80_TCP": "tcp://10.47.211.133:80",
"KUBERNETES_PORT_443_TCP": "tcp://10.32.0.1:443",
"KUBERNETES_SERVICE_HOST": "10.32.0.1",
"KUBERNETES_SERVICE_PORT": "443",
"ANSIBLE_CALLBACK_PLUGINS": "/usr/local/lib/python3.9/site-packages/ansible_runner/display_callback/callback",
"ANSIBLE_COLLECTIONS_PATH": "/runner/requirements_collections:/.ansible/collections:/usr/share/ansible/collections",
"AWX_SERVICE_SERVICE_HOST": "10.47.211.133",
"AWX_SERVICE_SERVICE_PORT": "80",
"ANSIBLE_COLLECTIONS_PATHS": "/runner/requirements_collections:~/.ansible/collections:/usr/share/ansible/collections",
"ANSIBLE_HOST_KEY_CHECKING": "False",
"KUBERNETES_PORT_12250_TCP": "tcp://10.32.0.1:12250",
"PIP_BREAK_SYSTEM_PACKAGES": "1",
"RUNNER_ONLY_FAILED_EVENTS": "False",
"ANSIBLE_RETRY_FILES_ENABLED": "False",
"ANSIBLE_SSH_CONTROL_PATH_DIR": "/runner/cp",
"AWX_SERVICE_PORT_80_TCP_ADDR": "10.47.211.133",
"AWX_SERVICE_PORT_80_TCP_PORT": "80",
"KUBERNETES_PORT_443_TCP_ADDR": "10.32.0.1",
"KUBERNETES_PORT_443_TCP_PORT": "443",
"AWX_SERVICE_PORT_80_TCP_PROTO": "tcp",
"AWX_SERVICE_SERVICE_PORT_HTTP": "80",
"KUBERNETES_PORT_443_TCP_PROTO": "tcp",
"KUBERNETES_SERVICE_PORT_HTTPS": "443",
"KUBERNETES_PORT_12250_TCP_ADDR": "10.32.0.1",
"KUBERNETES_PORT_12250_TCP_PORT": "12250",
"ANSIBLE_CACHE_PLUGIN_CONNECTION": "/runner/artifacts/715793/fact_cache",
"KUBERNETES_PORT_12250_TCP_PROTO": "tcp",
"KUBERNETES_SERVICE_PORT_PROXYMUX": "12250",
"ANSIBLE_INVENTORY_UNPARSED_FAILED": "True",
"ANSIBLE_PARAMIKO_RECORD_HOST_KEYS": "False",
"AWX_OPERATOR_CONTROLLER_MANAGER_METRICS_SERVICE_PORT": "tcp://10.47.89.227:8443",
"AWX_OPERATOR_CONTROLLER_MANAGER_METRICS_SERVICE_SERVICE_HOST": "10.47.89.227",
"AWX_OPERATOR_CONTROLLER_MANAGER_METRICS_SERVICE_SERVICE_PORT": "8443",
"AWX_OPERATOR_CONTROLLER_MANAGER_METRICS_SERVICE_PORT_8443_TCP": "tcp://10.47.89.227:8443",
"AWX_OPERATOR_CONTROLLER_MANAGER_METRICS_SERVICE_PORT_8443_TCP_ADDR": "10.47.89.227",
"AWX_OPERATOR_CONTROLLER_MANAGER_METRICS_SERVICE_PORT_8443_TCP_PORT": "8443",
"AWX_OPERATOR_CONTROLLER_MANAGER_METRICS_SERVICE_SERVICE_PORT_HTTPS": "8443",
"AWX_OPERATOR_CONTROLLER_MANAGER_METRICS_SERVICE_PORT_8443_TCP_PROTO": "tcp"
},
"job_explanation": "Failed to JSON parse a line from worker stream. Error: Expecting ':' delimiter: line 1 column 16416 (char 16415) Line with invalid JSON data: b'{"uuid": "b2f5f01f-afd8-4704-877f-d1431cfe3039", "counter": 14, "stdout": "\\u001b[0;32mok: [oci-ash-prd-flink303.stapp.me]\\u001b[0m", "start_line": 8, "end_line": 9, "runner_ident": "715793", "event": "runner_on_ok", "job_id": 715793, "pid": 22, "created": "2024-05-26T06:20:53.360735+00:00", "parent_uuid": "f63d76a5-c700-6531-e90a-0000000000ca", "event_data": {"playbook": "playbooks/clean_files_regex.yaml", "playbook_uuid": "e0abda99-21fd-4455-b029-367ea9890353", "play": "clean files by regex", "play_uuid": "f63d76a5-c700-6531-e90a-0000000000c4", "play_pattern": "all", "task": "Gathering Facts", "task_uuid": "f63d76a5-c700-6531-e90a-0000000000ca", "task_action": "gather_facts", "resolved_action": "ansible.builtin.gather_facts", "task_args": "", "task_path": "/runner/project/playbooks/clean_files_regex.yaml:1", "host": "oci-ash-prd-flink303.stapp.me", "remote_addr": "oci-ash-prd-flink303.stapp.me", "res": {"ansible_facts": {"ansible_system_capabilities_enforced": "True", "ansible_system'",
"execution_node": "",
"controller_node": "awx-task-67c5d44dbf-thgs6",
"result_traceback": "Receptor detail:\nPod Running",
"event_processing_finished": true,
"launched_by": {
"id": 52,
"name": "automation",
"type": "user",
"url": "/api/v2/users/52/"
},
"work_unit_id": "pu6fZMps",
"job_template": 309,
"passwords_needed_to_start": [],
"allow_simultaneous": true,
"artifacts": {},
"scm_revision": "d349a2868641669a3f133bb42e39d461df00f9be",
"instance_group": 3,
"diff_mode": false,
"job_slice_number": 0,
"job_slice_count": 1,
"webhook_service": "",
"webhook_credential": null,
"webhook_guid": "",
"host_status_counts": null,
"playbook_counts": {
"play_count": 1,
"task_count": 1
},
"custom_virtualenv": null

from awx.

@yaron-Kalatian are you able to reproduce the error on the latest version of AWX?

from awx.