Try 6 out of 6
Exception:
Pod drop-older-than-hzcqugv returned a failure.
remote_pod: {'api_version': 'v1',
'kind': 'Pod',
'metadata': {'annotations': {'cni.projectcalico.org/containerID': '260afb034de4f062dc47d9b25113d81a0f97b8bc7ebcb8c90bc17c5023324952',
'cni.projectcalico.org/podIP': '',
'cni.projectcalico.org/podIPs': '',
'container.seccomp.security.alpha.kubernetes.io/base': 'runtime/default'},
'creation_timestamp': datetime.datetime(2026, 1, 31, 0, 28, 36, tzinfo=tzlocal()),
'deletion_grace_period_seconds': None,
'deletion_timestamp': None,
'finalizers': None,
'generate_name': None,
'generation': None,
'labels': {'airflow_kpo_in_cluster': 'True',
'airflow_version': '2.10.5',
'app': 'airflow',
'component': 'task-pod',
'dag_id': 'drop_old_data_daily',
'kubernetes_pod_operator': 'True',
'release': 'production',
'routed_via': 'production',
'run_id': 'scheduled__2026-01-30T0000000000-675e53b6f',
'task_id': 'drop_glent_prep_partitions',
'try_number': '6'},
'managed_fields': [{'api_version': 'v1',
'fields_type': 'FieldsV1',
'fields_v1': {'f:metadata': {'f:labels': {'.': {},
'f:airflow_kpo_in_cluster': {},
'f:airflow_version': {},
'f:app': {},
'f:component': {},
'f:dag_id': {},
'f:kubernetes_pod_operator': {},
'f:release': {},
'f:routed_via': {},
'f:run_id': {},
'f:task_id': {},
'f:try_number': {}}},
'f:spec': {'f:affinity': {},
'f:containers': {'k:{"name":"base"}': {'.': {},
'f:args': {},
'f:command': {},
'f:env': {'.': {},
'k:{"name":"AWS_REQUEST_CHECKSUM_CALCULATION"}': {'.': {},
'f:name': {},
'f:value': {}},
'k:{"name":"AWS_RESPONSE_CHECKSUM_VALIDATION"}': {'.': {},
'f:name': {},
'f:value': {}},
'k:{"name":"HADOOP_CONF_DIR"}': {'.': {},
'f:name': {},
'f:value': {}},
'k:{"name":"HIVE_CONF_DIR"}': {'.': {},
'f:name': {},
'f:value': {}},
'k:{"name":"KRB5CCNAME"}': {'.': {},
'f:name': {},
'f:value': {}},
'k:{"name":"KRB5_CONFIG"}': {'.': {},
'f:name': {},
'f:value': {}},
'k:{"name":"REQUESTS_CA_BUNDLE"}': {'.': {},
'f:name': {},
'f:value': {}}},
'f:image': {},
'f:imagePullPolicy': {},
'f:name': {},
'f:resources': {'.': {},
'f:limits': {'.': {},
'f:cpu': {},
'f:memory': {}},
'f:requests': {'.': {},
'f:cpu': {},
'f:memory': {}}},
'f:securityContext': {'.': {},
'f:allowPrivilegeEscalation': {},
'f:capabilities': {'.': {},
'f:drop': {}},
'f:runAsNonRoot': {},
'f:seccompProfile': {'.': {},
'f:type': {}}},
'f:terminationMessagePath': {},
'f:terminationMessagePolicy': {},
'f:volumeMounts': {'.': {},
'k:{"mountPath":"/etc/hadoop/conf"}': {'.': {},
'f:mountPath': {},
'f:name': {}},
'k:{"mountPath":"/etc/krb5.conf"}': {'.': {},
'f:mountPath': {},
'f:name': {},
'f:subPath': {}},
'k:{"mountPath":"/tmp/airflow_krb5_ccache"}': {'.': {},
'f:mountPath': {},
'f:name': {}}}}},
'f:dnsPolicy': {},
'f:enableServiceLinks': {},
'f:priorityClassName': {},
'f:restartPolicy': {},
'f:schedulerName': {},
'f:securityContext': {},
'f:terminationGracePeriodSeconds': {},
'f:volumes': {'.': {},
'k:{"name":"airflow-hadoop-configuration"}': {'.': {},
'f:configMap': {'.': {},
'f:defaultMode': {},
'f:name': {}},
'f:name': {}},
'k:{"name":"airflow-kerberos-client-config"}': {'.': {},
'f:configMap': {'.': {},
'f:defaultMode': {},
'f:name': {}},
'f:name': {}},
'k:{"name":"airflow-kerberos-token"}': {'.': {},
'f:name': {},
'f:persistentVolumeClaim': {'.': {},
'f:claimName': {}}}}}},
'manager': 'OpenAPI-Generator',
'operation': 'Update',
'subresource': None,
'time': datetime.datetime(2026, 1, 31, 0, 28, 36, tzinfo=tzlocal())},
{'api_version': 'v1',
'fields_type': 'FieldsV1',
'fields_v1': {'f:metadata': {'f:annotations': {'f:cni.projectcalico.org/containerID': {},
'f:cni.projectcalico.org/podIP': {},
'f:cni.projectcalico.org/podIPs': {}}}},
'manager': 'Go-http-client',
'operation': 'Update',
'subresource': 'status',
'time': datetime.datetime(2026, 1, 31, 0, 28, 37, tzinfo=tzlocal())},
{'api_version': 'v1',
'fields_type': 'FieldsV1',
'fields_v1': {'f:status': {'f:conditions': {'k:{"type":"ContainersReady"}': {'.': {},
'f:lastProbeTime': {},
'f:lastTransitionTime': {},
'f:reason': {},
'f:status': {},
'f:type': {}},
'k:{"type":"Initialized"}': {'.': {},
'f:lastProbeTime': {},
'f:lastTransitionTime': {},
'f:status': {},
'f:type': {}},
'k:{"type":"Ready"}': {'.': {},
'f:lastProbeTime': {},
'f:lastTransitionTime': {},
'f:reason': {},
'f:status': {},
'f:type': {}}},
'f:containerStatuses': {},
'f:hostIP': {},
'f:phase': {},
'f:podIP': {},
'f:podIPs': {'.': {},
'k:{"ip":"10.67.31.105"}': {'.': {},
'f:ip': {}},
'k:{"ip":"2620:0:861:302:98b9:b305:4ad3:6e9"}': {'.': {},
'f:ip': {}}},
'f:startTime': {}}},
'manager': 'kubelet',
'operation': 'Update',
'subresource': 'status',
'time': datetime.datetime(2026, 1, 31, 0, 28, 53, tzinfo=tzlocal())}],
'name': 'drop-older-than-hzcqugv',
'namespace': 'airflow-search',
'owner_references': None,
'resource_version': '1127754475',
'self_link': None,
'uid': '50af7739-5399-4cb0-bcc7-6a88af628ca4'},
'spec': {'active_deadline_seconds': None,
'affinity': {'node_affinity': None,
'pod_affinity': None,
'pod_anti_affinity': None},
'automount_service_account_token': None,
'containers': [{'args': ['--verbose',
'--database=glent',
'--tables=^(.*)$',
'--older-than=29',
'--allowed-interval=3',
'--execute=b31e910eb2946f5d7d464008a182b3e9'],
'command': ['/opt/refinery/bin/refinery-drop-older-than'],
'env': [{'name': 'REQUESTS_CA_BUNDLE',
'value': '/etc/ssl/certs/ca-certificates.crt',
'value_from': None},
{'name': 'AWS_REQUEST_CHECKSUM_CALCULATION',
'value': 'WHEN_REQUIRED',
'value_from': None},
{'name': 'AWS_RESPONSE_CHECKSUM_VALIDATION',
'value': 'WHEN_REQUIRED',
'value_from': None},
{'name': 'HADOOP_CONF_DIR',
'value': '/etc/hadoop/conf',
'value_from': None},
{'name': 'HIVE_CONF_DIR',
'value': '/etc/hadoop/conf',
'value_from': None},
{'name': 'KRB5CCNAME',
'value': '/tmp/airflow_krb5_ccache/krb5cc',
'value_from': None},
{'name': 'KRB5_CONFIG',
'value': '/etc/krb5.conf',
'value_from': None}],
'env_from': None,
'image': 'docker-registry.discovery.wmnet/repos/data-engineering/refinery:2025-01-16-124629-dc160ba98355aab28add89f314345f6b5b633ac0',
'image_pull_policy': 'IfNotPresent',
'lifecycle': None,
'liveness_probe': None,
'name': 'base',
'ports': None,
'readiness_probe': None,
'resize_policy': None,
'resources': {'claims': None,
'limits': {'cpu': '2', 'memory': '3Gi'},
'requests': {'cpu': '1',
'memory': '1500Mi'}},
'restart_policy': None,
'restart_policy_rules': None,
'security_context': {'allow_privilege_escalation': False,
'app_armor_profile': None,
'capabilities': {'add': None,
'drop': ['ALL']},
'privileged': None,
'proc_mount': None,
'read_only_root_filesystem': None,
'run_as_group': None,
'run_as_non_root': True,
'run_as_user': None,
'se_linux_options': None,
'seccomp_profile': {'localhost_profile': None,
'type': 'RuntimeDefault'},
'windows_options': None},
'startup_probe': None,
'stdin': None,
'stdin_once': None,
'termination_message_path': '/dev/termination-log',
'termination_message_policy': 'File',
'tty': None,
'volume_devices': None,
'volume_mounts': [{'mount_path': '/etc/hadoop/conf',
'mount_propagation': None,
'name': 'airflow-hadoop-configuration',
'read_only': None,
'recursive_read_only': None,
'sub_path': None,
'sub_path_expr': None},
{'mount_path': '/etc/krb5.conf',
'mount_propagation': None,
'name': 'airflow-kerberos-client-config',
'read_only': None,
'recursive_read_only': None,
'sub_path': 'krb5.conf',
'sub_path_expr': None},
{'mount_path': '/tmp/airflow_krb5_ccache',
'mount_propagation': None,
'name': 'airflow-kerberos-token',
'read_only': None,
'recursive_read_only': None,
'sub_path': None,
'sub_path_expr': None},
{'mount_path': '/var/run/secrets/kubernetes.io/serviceaccount',
'mount_propagation': None,
'name': 'kube-api-access-6t886',
'read_only': True,
'recursive_read_only': None,
'sub_path': None,
'sub_path_expr': None}],
'working_dir': None}],
'dns_config': None,
'dns_policy': 'ClusterFirst',
'enable_service_links': True,
'ephemeral_containers': None,
'host_aliases': None,
'host_ipc': None,
'host_network': None,
'host_pid': None,
'host_users': None,
'hostname': None,
'hostname_override': None,
'image_pull_secrets': None,
'init_containers': None,
'node_name': 'dse-k8s-worker1016.eqiad.wmnet',
'node_selector': None,
'os': None,
'overhead': None,
'preemption_policy': 'PreemptLowerPriority',
'priority': -100,
'priority_class_name': 'low-priority-pod',
'readiness_gates': None,
'resource_claims': None,
'resources': None,
'restart_policy': 'Never',
'runtime_class_name': None,
'scheduler_name': 'default-scheduler',
'scheduling_gates': None,
'security_context': {'app_armor_profile': None,
'fs_group': None,
'fs_group_change_policy': None,
'run_as_group': None,
'run_as_non_root': None,
'run_as_user': None,
'se_linux_change_policy': None,
'se_linux_options': None,
'seccomp_profile': None,
'supplemental_groups': None,
'supplemental_groups_policy': None,
'sysctls': None,
'windows_options': None},
'service_account': 'default',
'service_account_name': 'default',
'set_hostname_as_fqdn': None,
'share_process_namespace': None,
'subdomain': None,
'termination_grace_period_seconds': 30,
'tolerations': [{'effect': 'NoExecute',
'key': 'node.kubernetes.io/not-ready',
'operator': 'Exists',
'toleration_seconds': 300,
'value': None},
{'effect': 'NoExecute',
'key': 'node.kubernetes.io/unreachable',
'operator': 'Exists',
'toleration_seconds': 300,
'value': None}],
'topology_spread_constraints': None,
'volumes': [{'aws_elastic_block_store': None,
'azure_disk': None,
'azure_file': None,
'cephfs': None,
'cinder': None,
'config_map': {'default_mode': 420,
'items': None,
'name': 'airflow-hadoop-configuration',
'optional': None},
'csi': None,
'downward_api': None,
'empty_dir': None,
'ephemeral': None,
'fc': None,
'flex_volume': None,
'flocker': None,
'gce_persistent_disk': None,
'git_repo': None,
'glusterfs': None,
'host_path': None,
'image': None,
'iscsi': None,
'name': 'airflow-hadoop-configuration',
'nfs': None,
'persistent_volume_claim': None,
'photon_persistent_disk': None,
'portworx_volume': None,
'projected': None,
'quobyte': None,
'rbd': None,
'scale_io': None,
'secret': None,
'storageos': None,
'vsphere_volume': None},
{'aws_elastic_block_store': None,
'azure_disk': None,
'azure_file': None,
'cephfs': None,
'cinder': None,
'config_map': {'default_mode': 420,
'items': None,
'name': 'airflow-kerberos-client-config',
'optional': None},
'csi': None,
'downward_api': None,
'empty_dir': None,
'ephemeral': None,
'fc': None,
'flex_volume': None,
'flocker': None,
'gce_persistent_disk': None,
'git_repo': None,
'glusterfs': None,
'host_path': None,
'image': None,
'iscsi': None,
'name': 'airflow-kerberos-client-config',
'nfs': None,
'persistent_volume_claim': None,
'photon_persistent_disk': None,
'portworx_volume': None,
'projected': None,
'quobyte': None,
'rbd': None,
'scale_io': None,
'secret': None,
'storageos': None,
'vsphere_volume': None},
{'aws_elastic_block_store': None,
'azure_disk': None,
'azure_file': None,
'cephfs': None,
'cinder': None,
'config_map': None,
'csi': None,
'downward_api': None,
'empty_dir': None,
'ephemeral': None,
'fc': None,
'flex_volume': None,
'flocker': None,
'gce_persistent_disk': None,
'git_repo': None,
'glusterfs': None,
'host_path': None,
'image': None,
'iscsi': None,
'name': 'airflow-kerberos-token',
'nfs': None,
'persistent_volume_claim': {'claim_name': 'airflow-kerberos-token-pvc',
'read_only': None},
'photon_persistent_disk': None,
'portworx_volume': None,
'projected': None,
'quobyte': None,
'rbd': None,
'scale_io': None,
'secret': None,
'storageos': None,
'vsphere_volume': None},
{'aws_elastic_block_store': None,
'azure_disk': None,
'azure_file': None,
'cephfs': None,
'cinder': None,
'config_map': None,
'csi': None,
'downward_api': None,
'empty_dir': None,
'ephemeral': None,
'fc': None,
'flex_volume': None,
'flocker': None,
'gce_persistent_disk': None,
'git_repo': None,
'glusterfs': None,
'host_path': None,
'image': None,
'iscsi': None,
'name': 'kube-api-access-6t886',
'nfs': None,
'persistent_volume_claim': None,
'photon_persistent_disk': None,
'portworx_volume': None,
'projected': {'default_mode': 420,
'sources': [{'cluster_trust_bundle': None,
'config_map': None,
'downward_api': None,
'pod_certificate': None,
'secret': None,
'service_account_token': {'audience': None,
'expiration_seconds': 3607,
'path': 'token'}},
{'cluster_trust_bundle': None,
'config_map': {'items': [{'key': 'ca.crt',
'mode': None,
'path': 'ca.crt'}],
'name': 'kube-root-ca.crt',
'optional': None},
'downward_api': None,
'pod_certificate': None,
'secret': None,
'service_account_token': None},
{'cluster_trust_bundle': None,
'config_map': None,
'downward_api': {'items': [{'field_ref': {'api_version': 'v1',
'field_path': 'metadata.namespace'},
'mode': None,
'path': 'namespace',
'resource_field_ref': None}]},
'pod_certificate': None,
'secret': None,
'service_account_token': None}]},
'quobyte': None,
'rbd': None,
'scale_io': None,
'secret': None,
'storageos': None,
'vsphere_volume': None}]},
'status': {'conditions': [{'last_probe_time': None,
'last_transition_time': datetime.datetime(2026, 1, 31, 0, 28, 36, tzinfo=tzlocal()),
'message': None,
'observed_generation': None,
'reason': None,
'status': 'True',
'type': 'Initialized'},
{'last_probe_time': None,
'last_transition_time': datetime.datetime(2026, 1, 31, 0, 28, 53, tzinfo=tzlocal()),
'message': None,
'observed_generation': None,
'reason': 'PodFailed',
'status': 'False',
'type': 'Ready'},
{'last_probe_time': None,
'last_transition_time': datetime.datetime(2026, 1, 31, 0, 28, 53, tzinfo=tzlocal()),
'message': None,
'observed_generation': None,
'reason': 'PodFailed',
'status': 'False',
'type': 'ContainersReady'},
{'last_probe_time': None,
'last_transition_time': datetime.datetime(2026, 1, 31, 0, 28, 36, tzinfo=tzlocal()),
'message': None,
'observed_generation': None,
'reason': None,
'status': 'True',
'type': 'PodScheduled'}],
'container_statuses': [{'allocated_resources': None,
'allocated_resources_status': None,
'container_id': 'containerd://192abe370c8636e798682f8bd38c737d9fe0564b15efa7859a25b1ede3c1af55',
'image': 'docker-registry.discovery.wmnet/repos/data-engineering/refinery:2025-01-16-124629-dc160ba98355aab28add89f314345f6b5b633ac0',
'image_id': 'docker-registry.discovery.wmnet/repos/data-engineering/refinery@sha256:bb5e546380e49523dd7129e76b90990dd649ed72dd77a927e2bf82cfd8fdab26',
'last_state': {'running': None,
'terminated': None,
'waiting': None},
'name': 'base',
'ready': False,
'resources': None,
'restart_count': 0,
'started': False,
'state': {'running': None,
'terminated': {'container_id': 'containerd://192abe370c8636e798682f8bd38c737d9fe0564b15efa7859a25b1ede3c1af55',
'exit_code': 1,
'finished_at': datetime.datetime(2026, 1, 31, 0, 28, 53, tzinfo=tzlocal()),
'message': None,
'reason': 'Error',
'signal': None,
'started_at': datetime.datetime(2026, 1, 31, 0, 28, 37, tzinfo=tzlocal())},
'waiting': None},
'stop_signal': None,
'user': None,
'volume_mounts': None}],
'ephemeral_container_statuses': None,
'extended_resource_claim_status': None,
'host_i_ps': None,
'host_ip': '10.64.0.157',
'init_container_statuses': None,
'message': None,
'nominated_node_name': None,
'observed_generation': None,
'phase': 'Failed',
'pod_i_ps': [{'ip': '10.67.31.105'},
{'ip': '2620:0:861:302:98b9:b305:4ad3:6e9'}],
'pod_ip': '10.67.31.105',
'qos_class': 'Burstable',
'reason': None,
'resize': None,
'resource_claim_statuses': None,
'start_time': datetime.datetime(2026, 1, 31, 0, 28, 36, tzinfo=tzlocal())}}
Log: Link
Host: drop-old-data-daily-drop-glent-prep-partitions-hb279pit
Mark success: Link