Is your feature request related to a problem or challenge? What d

Plan: I think we should change coerc

Use non-comparison coercion for `Coalesce` or even avoid implicit casting for `Coalesce` about arrow-datafusion HOT 4 CLOSED

jayzhan211 commented on June 4, 2024 1

Use non-comparison coercion for `Coalesce` or even avoid implicit casting for `Coalesce`

from arrow-datafusion.

Comments (4)

alamb commented on June 4, 2024 1

BTW this is a really nicely written ticket

from arrow-datafusion.

alamb commented on June 4, 2024

If possible, I think we should follow the model of existing implementations here (rather than invent DataFusion specific semants)

Specifically if we are going to change the semantics of coalsce I think we should follow either postgres or spark's behavior -- I haven't done the research to know how close/far what DataFusion does compared to those systems

from arrow-datafusion.

jayzhan211 commented on June 4, 2024

~~Alright, so I think we should remove casting logic for coalesce, which follows what Postgres and DuckDB do. Returns error if there are mixed types for different columns~~.

It seems Postgres has casting for some of the types too.

from arrow-datafusion.

jayzhan211 commented on June 4, 2024

Plan:

I think we should change coerced_from to can_cast_types here, pre-computed reasonable valid types with TypeSignature in get_valid_types.

datafusion/datafusion/expr/src/type_coercion/functions.rs

Lines 67 to 71 in ed14682

for valid_types in valid_types {

if let Some(types) = maybe_data_types(&valid_types, current_types) {

return Ok(types);

}

}

datafusion/datafusion/expr/src/type_coercion/functions.rs

Lines 82 to 261 in ed14682

 fn get_valid_types( 

 signature: &TypeSignature, 

 current_types: &[DataType], 

 ) -> Result<Vec<Vec<DataType>>> { 

 fn array_element_and_optional_index( 

 current_types: &[DataType], 

 ) -> Result<Vec<Vec<DataType>>> { 

 // make sure there's 2 or 3 arguments 

 if !(current_types.len() == 2 || current_types.len() == 3) { 

 return Ok(vec![vec![]]); 

 } 

 let first_two_types = &current_types[0..2]; 

 let mut valid_types = array_append_or_prepend_valid_types(first_two_types, true)?; 

 // Early return if there are only 2 arguments 

 if current_types.len() == 2 { 

 return Ok(valid_types); 

 } 

 let valid_types_with_index = valid_types 

 .iter() 

 .map(|t| { 

 let mut t = t.clone(); 

 t.push(DataType::Int64); 

 t 

 }) 

 .collect::<Vec<_>>(); 

 valid_types.extend(valid_types_with_index); 

 Ok(valid_types) 

 } 

 fn array_append_or_prepend_valid_types( 

 current_types: &[DataType], 

 is_append: bool, 

 ) -> Result<Vec<Vec<DataType>>> { 

 if current_types.len() != 2 { 

 return Ok(vec![vec![]]); 

 } 

 let (array_type, elem_type) = if is_append { 

 (&current_types[0], &current_types[1]) 

 } else { 

 (&current_types[1], &current_types[0]) 

 }; 

 // We follow Postgres on `array_append(Null, T)`, which is not valid. 

 if array_type.eq(&DataType::Null) { 

 return Ok(vec![vec![]]); 

 } 

 // We need to find the coerced base type, mainly for cases like: 

 // `array_append(List(null), i64)` -> `List(i64)` 

 let array_base_type = datafusion_common::utils::base_type(array_type); 

 let elem_base_type = datafusion_common::utils::base_type(elem_type); 

 let new_base_type = comparison_coercion(&array_base_type, &elem_base_type); 

 let new_base_type = new_base_type.ok_or_else(|| { 

 internal_datafusion_err!( 

 "Coercion from {array_base_type:?} to {elem_base_type:?} not supported." 

 ) 

 })?; 

 let new_array_type = datafusion_common::utils::coerced_type_with_base_type_only( 

 array_type, 

 &new_base_type, 

 ); 

 match new_array_type { 

 DataType::List(ref field) 

 | DataType::LargeList(ref field) 

 | DataType::FixedSizeList(ref field, _) => { 

 let new_elem_type = field.data_type(); 

 if is_append { 

 Ok(vec![vec![new_array_type.clone(), new_elem_type.clone()]]) 

 } else { 

 Ok(vec![vec![new_elem_type.to_owned(), new_array_type.clone()]]) 

 } 

 } 

 _ => Ok(vec![vec![]]), 

 } 

 } 

 fn array(array_type: &DataType) -> Option<DataType> { 

 match array_type { 

 DataType::List(_) 

 | DataType::LargeList(_) 

 | DataType::FixedSizeList(_, _) => { 

 let array_type = coerced_fixed_size_list_to_list(array_type); 

 Some(array_type) 

 } 

 _ => None, 

 } 

 } 

 let valid_types = match signature { 

 TypeSignature::Variadic(valid_types) => valid_types 

 .iter() 

 .map(|valid_type| current_types.iter().map(|_| valid_type.clone()).collect()) 

 .collect(), 

 TypeSignature::Uniform(number, valid_types) => valid_types 

 .iter() 

 .map(|valid_type| (0..*number).map(|_| valid_type.clone()).collect()) 

 .collect(), 

 TypeSignature::VariadicEqual => { 

 let new_type = current_types.iter().skip(1).try_fold( 

 current_types.first().unwrap().clone(), 

 |acc, x| { 

 // The coerced types found by `comparison_coercion` are not guaranteed to be 

 // coercible for the arguments. `comparison_coercion` returns more loose 

 // types that can be coerced to both `acc` and `x` for comparison purpose. 

 // See `maybe_data_types` for the actual coercion. 

 let coerced_type = comparison_coercion(&acc, x); 

 if let Some(coerced_type) = coerced_type { 

 Ok(coerced_type) 

 } else { 

 internal_err!("Coercion from {acc:?} to {x:?} failed.") 

 } 

 }, 

 ); 

 match new_type { 

 Ok(new_type) => vec![vec![new_type; current_types.len()]], 

 Err(e) => return Err(e), 

 } 

 } 

 TypeSignature::VariadicAny => { 

 vec![current_types.to_vec()] 

 } 

 TypeSignature::Exact(valid_types) => vec![valid_types.clone()], 

 TypeSignature::ArraySignature(ref function_signature) => match function_signature 

 { 

 ArrayFunctionSignature::ArrayAndElement => { 

 array_append_or_prepend_valid_types(current_types, true)? 

 } 

 ArrayFunctionSignature::ElementAndArray => { 

 array_append_or_prepend_valid_types(current_types, false)? 

 } 

 ArrayFunctionSignature::ArrayAndIndex => { 

 if current_types.len() != 2 { 

 return Ok(vec![vec![]]); 

 } 

 array(&current_types[0]).map_or_else( 

 || vec![vec![]], 

 |array_type| vec![vec![array_type, DataType::Int64]], 

 ) 

 } 

 ArrayFunctionSignature::ArrayAndElementAndOptionalIndex => { 

 array_element_and_optional_index(current_types)? 

 } 

 ArrayFunctionSignature::Array => { 

 if current_types.len() != 1 { 

 return Ok(vec![vec![]]); 

 } 

 array(&current_types[0]) 

 .map_or_else(|| vec![vec![]], |array_type| vec![vec![array_type]]) 

 } 

 }, 

 TypeSignature::Any(number) => { 

 if current_types.len() != *number { 

 return plan_err!( 

 "The function expected {} arguments but received {}", 

 number, 

 current_types.len() 

 ); 

 } 

 vec![(0..*number).map(|i| current_types[i].clone()).collect()] 

 } 

 TypeSignature::OneOf(types) => types 

 .iter() 

 .filter_map(|t| get_valid_types(t, current_types).ok()) 

 .flatten() 

 .collect::<Vec<_>>(), 

 }; 

 Ok(valid_types) 

 }

Non-comparison coercion: type resolution coercion for Coalesce, MakeArray, and so on.

https://www.postgresql.org/docs/current/typeconv-union-case.html

Deprecate/Remove coerced_from at the end!

from arrow-datafusion.

Use non-comparison coercion for `Coalesce` or even avoid implicit casting for `Coalesce` about arrow-datafusion HOT 4 CLOSED

Comments (4)

Related Issues (20)

Recommend Projects

React

Vue.js

Typescript

TensorFlow

Django

Laravel

D3

Recommend Topics

javascript

web

server

Machine learning

Visualization

Game

Recommend Org

Facebook

Microsoft

Google

Alibaba

D3

Tencent

	for valid_types in valid_types {
	if let Some(types) = maybe_data_types(&valid_types, current_types) {
	return Ok(types);
	}
	}

	fn get_valid_types(
	signature: &TypeSignature,
	current_types: &[DataType],
	) -> Result<Vec<Vec<DataType>>> {
	fn array_element_and_optional_index(
	current_types: &[DataType],
	) -> Result<Vec<Vec<DataType>>> {
	// make sure there's 2 or 3 arguments
	if !(current_types.len() == 2 \|\| current_types.len() == 3) {
	return Ok(vec![vec![]]);
	}

	let first_two_types = &current_types[0..2];
	let mut valid_types = array_append_or_prepend_valid_types(first_two_types, true)?;

	// Early return if there are only 2 arguments
	if current_types.len() == 2 {
	return Ok(valid_types);
	}

	let valid_types_with_index = valid_types
	.iter()
	.map(\|t\| {
	let mut t = t.clone();
	t.push(DataType::Int64);
	t
	})
	.collect::<Vec<_>>();

	valid_types.extend(valid_types_with_index);

	Ok(valid_types)
	}

	fn array_append_or_prepend_valid_types(
	current_types: &[DataType],
	is_append: bool,
	) -> Result<Vec<Vec<DataType>>> {
	if current_types.len() != 2 {
	return Ok(vec![vec![]]);
	}

	let (array_type, elem_type) = if is_append {
	(&current_types[0], &current_types[1])
	} else {
	(&current_types[1], &current_types[0])
	};

	// We follow Postgres on `array_append(Null, T)`, which is not valid.
	if array_type.eq(&DataType::Null) {
	return Ok(vec![vec![]]);
	}

	// We need to find the coerced base type, mainly for cases like:
	// `array_append(List(null), i64)` -> `List(i64)`
	let array_base_type = datafusion_common::utils::base_type(array_type);
	let elem_base_type = datafusion_common::utils::base_type(elem_type);
	let new_base_type = comparison_coercion(&array_base_type, &elem_base_type);

	let new_base_type = new_base_type.ok_or_else(\|\| {
	internal_datafusion_err!(
	"Coercion from {array_base_type:?} to {elem_base_type:?} not supported."
	)
	})?;

	let new_array_type = datafusion_common::utils::coerced_type_with_base_type_only(
	array_type,
	&new_base_type,
	);

	match new_array_type {
	DataType::List(ref field)
	\| DataType::LargeList(ref field)
	\| DataType::FixedSizeList(ref field, _) => {
	let new_elem_type = field.data_type();
	if is_append {
	Ok(vec![vec![new_array_type.clone(), new_elem_type.clone()]])
	} else {
	Ok(vec![vec![new_elem_type.to_owned(), new_array_type.clone()]])
	}
	}
	_ => Ok(vec![vec![]]),
	}
	}
	fn array(array_type: &DataType) -> Option<DataType> {
	match array_type {
	DataType::List(_)
	\| DataType::LargeList(_)
	\| DataType::FixedSizeList(_, _) => {
	let array_type = coerced_fixed_size_list_to_list(array_type);
	Some(array_type)
	}
	_ => None,
	}
	}

	let valid_types = match signature {
	TypeSignature::Variadic(valid_types) => valid_types
	.iter()
	.map(\|valid_type\| current_types.iter().map(\|_\| valid_type.clone()).collect())
	.collect(),
	TypeSignature::Uniform(number, valid_types) => valid_types
	.iter()
	.map(\|valid_type\| (0..*number).map(\|_\| valid_type.clone()).collect())
	.collect(),
	TypeSignature::VariadicEqual => {
	let new_type = current_types.iter().skip(1).try_fold(
	current_types.first().unwrap().clone(),
	\|acc, x\| {
	// The coerced types found by `comparison_coercion` are not guaranteed to be
	// coercible for the arguments. `comparison_coercion` returns more loose
	// types that can be coerced to both `acc` and `x` for comparison purpose.
	// See `maybe_data_types` for the actual coercion.
	let coerced_type = comparison_coercion(&acc, x);
	if let Some(coerced_type) = coerced_type {
	Ok(coerced_type)
	} else {
	internal_err!("Coercion from {acc:?} to {x:?} failed.")
	}
	},
	);

	match new_type {
	Ok(new_type) => vec![vec![new_type; current_types.len()]],
	Err(e) => return Err(e),
	}
	}
	TypeSignature::VariadicAny => {
	vec![current_types.to_vec()]
	}

	TypeSignature::Exact(valid_types) => vec![valid_types.clone()],
	TypeSignature::ArraySignature(ref function_signature) => match function_signature
	{
	ArrayFunctionSignature::ArrayAndElement => {
	array_append_or_prepend_valid_types(current_types, true)?
	}
	ArrayFunctionSignature::ElementAndArray => {
	array_append_or_prepend_valid_types(current_types, false)?
	}
	ArrayFunctionSignature::ArrayAndIndex => {
	if current_types.len() != 2 {
	return Ok(vec![vec![]]);
	}
	array(&current_types[0]).map_or_else(
	\|\| vec![vec![]],
	\|array_type\| vec![vec![array_type, DataType::Int64]],
	)
	}
	ArrayFunctionSignature::ArrayAndElementAndOptionalIndex => {
	array_element_and_optional_index(current_types)?
	}
	ArrayFunctionSignature::Array => {
	if current_types.len() != 1 {
	return Ok(vec![vec![]]);
	}

	array(&current_types[0])
	.map_or_else(\|\| vec![vec![]], \|array_type\| vec![vec![array_type]])
	}
	},
	TypeSignature::Any(number) => {
	if current_types.len() != *number {
	return plan_err!(
	"The function expected {} arguments but received {}",
	number,
	current_types.len()
	);
	}
	vec![(0..*number).map(\|i\| current_types[i].clone()).collect()]
	}
	TypeSignature::OneOf(types) => types
	.iter()
	.filter_map(\|t\| get_valid_types(t, current_types).ok())
	.flatten()
	.collect::<Vec<_>>(),
	};

	Ok(valid_types)
	}