Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,10 @@ case class CharType private[sql] (length: Int, collation: Option[Int])

override def defaultSize: Int = length
override def typeName: String =
if (isUTF8BinaryCollation) s"char($length)"
if (collation.isEmpty) s"char($length)"
else s"char($length) collate $collationName"
override def toString: String =
if (isUTF8BinaryCollation) s"CharType($length)"
if (collation.isEmpty) s"CharType($length)"
else s"CharType($length, $collationName)"
private[spark] override def asNullable: CharType = this

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,12 +78,10 @@ class StringType private[sql] (
* `string` due to backwards compatibility.
*/
override def typeName: String =
if (isUTF8BinaryCollation) "string"
else s"string collate $collationName"
s"string collate $collationName"

override def toString: String =
if (isUTF8BinaryCollation) "StringType"
else s"StringType($collationName)"
s"StringType($collationName)"

private[sql] def collationName: String =
CollationFactory.fetchCollation(collationId).collationName
Expand Down Expand Up @@ -119,6 +117,10 @@ case object StringType
val collationId = CollationFactory.collationNameToId(collation)
new StringType(collationId)
}

override def typeName: String = "string"

override def toString: String = "StringType"
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,10 +38,10 @@ case class VarcharType private[sql] (length: Int, collation: Option[Int])

override def defaultSize: Int = length
override def typeName: String =
if (isUTF8BinaryCollation) s"varchar($length)"
if (collation.isEmpty) s"varchar($length)"
else s"varchar($length) collate $collationName"
override def toString: String =
if (isUTF8BinaryCollation) s"VarcharType($length)"
if (collation.isEmpty) s"VarcharType($length)"
else s"VarcharType($length, $collationName)"
private[spark] override def asNullable: VarcharType = this

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package org.apache.spark.sql.catalyst.types

import org.apache.spark.sql.catalyst.analysis.Resolver
import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, Cast, Literal}
import org.apache.spark.sql.catalyst.util.CollationFactory
import org.apache.spark.sql.catalyst.util.TypeUtils.toSQLId
import org.apache.spark.sql.errors.QueryCompilationErrors
import org.apache.spark.sql.internal.SQLConf.StoreAssignmentPolicy
Expand Down Expand Up @@ -327,5 +328,23 @@ object DataTypeUtils {
case _ => false
}
}

/**
* Recursively replaces all STRING, CHAR and VARCHAR types that do not have an explicit collation
* with the same type but with explicit `UTF8_BINARY` collation.
*
* Used for cases like `SHOW CREATE TABLE`, where we want to show the exact collation of the
* columns, because the default collation of the table may change the type of the column.
*/
def replaceNonCollatedTypesWithExplicitUTF8Binary(dataType: DataType): DataType = {
dataType.transformRecursively {
case charType: CharType if isDefaultStringCharOrVarcharType(charType) =>
CharType(charType.length, CollationFactory.UTF8_BINARY_COLLATION_ID)
case varcharType: VarcharType if isDefaultStringCharOrVarcharType(varcharType) =>
VarcharType(varcharType.length, CollationFactory.UTF8_BINARY_COLLATION_ID)
case stringType: StringType if isDefaultStringCharOrVarcharType(stringType) =>
StringType(CollationFactory.UTF8_BINARY_COLLATION_ID)
}
}
}

Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ class PlanGenerationTestSuite extends ConnectFunSuite with Logging {
}

private def test(name: String)(f: => Dataset[_]): Unit = super.test(name) {
val actual = trimJvmOriginFields(f.plan.getRoot)
val actual = normalizeProtoForComparison(f.plan.getRoot)
val goldenFile = queryFilePath.resolve(name.replace(' ', '_') + ".proto.bin")
Try(readRelation(goldenFile)) match {
case Success(expected) if expected == actual =>
Expand Down Expand Up @@ -195,7 +195,12 @@ class PlanGenerationTestSuite extends ConnectFunSuite with Logging {
}
}

private def trimJvmOriginFields[T <: protobuf.Message](message: T): T = {
/**
* Normalize proto messages for stable comparison:
* - Trim JVM origin fields (lines, stack traces, anonymous function names)
* - Populate default StringType collation when missing (UTF8_BINARY)
*/
private def normalizeProtoForComparison[T <: protobuf.Message](message: T): T = {
def trim(builder: proto.JvmOrigin.Builder): Unit = {
builder
.clearLine()
Expand All @@ -216,6 +221,17 @@ class PlanGenerationTestSuite extends ConnectFunSuite with Logging {
val builder = message.toBuilder

builder match {
// For comparison only, we add UTF8_BINARY when StringType collation is missing
// to ensure deterministic plan equality across environments.
case dt: proto.DataType.Builder if dt.getKindCase == proto.DataType.KindCase.STRING =>
val sb = dt.getStringBuilder
if (sb.getCollation.isEmpty) {
val defaultCollationName =
CollationFactory
.fetchCollation(CollationFactory.UTF8_BINARY_COLLATION_ID)
.collationName
sb.setCollation(defaultCollationName)
}
case exp: proto.Relation.Builder
if exp.hasCommon && exp.getCommon.hasOrigin && exp.getCommon.getOrigin.hasJvmOrigin =>
trim(exp.getCommonBuilder.getOriginBuilder.getJvmOriginBuilder)
Expand All @@ -227,10 +243,10 @@ class PlanGenerationTestSuite extends ConnectFunSuite with Logging {

builder.getAllFields.asScala.foreach {
case (desc, msg: protobuf.Message) =>
builder.setField(desc, trimJvmOriginFields(msg))
builder.setField(desc, normalizeProtoForComparison(msg))
case (desc, list: java.util.List[_]) =>
val newList = list.asScala.map {
case msg: protobuf.Message => trimJvmOriginFields(msg)
case msg: protobuf.Message => normalizeProtoForComparison(msg)
case other => other // Primitive types
}
builder.setField(desc, newList.asJava)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,12 +68,12 @@ class ColumnNodeToProtoConverterSuite extends ConnectFunSuite {
expr(_.getLiteralBuilder.setString("foo").build()))
val dataType = new StructType()
.add("_1", DoubleType)
.add("_2", StringType)
.add("_2", StringType("UTF8_LCASE"))
.add("_3", DoubleType)
.add("_4", StringType)
.add("_4", StringType("UTF8_LCASE"))
val stringTypeWithCollation = proto.DataType
.newBuilder()
.setString(proto.DataType.String.newBuilder().setCollation("UTF8_BINARY"))
.setString(proto.DataType.String.newBuilder().setCollation("UTF8_LCASE"))
.build()
testConversion(
Literal((12.0, "north", 60.0, "west"), Option(dataType)),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ object DataTypeProtoConverter {
}

private def toCatalystStringType(t: proto.DataType.String): StringType =
StringType(if (t.getCollation.nonEmpty) t.getCollation else "UTF8_BINARY")
if (t.getCollation.nonEmpty) StringType(t.getCollation) else StringType

private def toCatalystYearMonthIntervalType(t: proto.DataType.YearMonthInterval) = {
(t.hasStartField, t.hasEndField) match {
Expand Down Expand Up @@ -214,13 +214,15 @@ object DataTypeProtoConverter {

// StringType must be matched after CharType and VarcharType
case s: StringType =>
val stringBuilder = proto.DataType.String.newBuilder()
// Send collation only for explicit collations (including explicit UTF8_BINARY).
// Default STRING (case object) has no explicit collation and should omit it.
if (!s.eq(StringType)) {
stringBuilder.setCollation(CollationFactory.fetchCollation(s.collationId).collationName)
}
proto.DataType
.newBuilder()
.setString(
proto.DataType.String
.newBuilder()
.setCollation(CollationFactory.fetchCollation(s.collationId).collationName)
.build())
.setString(stringBuilder.build())
Comment on lines +217 to +225
Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Without this change, a user who uses JDBC, for example, and doesn't care about collations would suddenly get COLLATE UTF8_BINARY, as in this test case:

.build()

case DateType => ProtoDataTypes.DateType
Expand Down
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
Project [id#0L, id#0L, 1 AS 1#0, null AS NULL#0, true AS true#0, 68 AS 68#0, 9872 AS 9872#0, -8726532 AS -8726532#0, 7834609328726532 AS 7834609328726532#0L, 2.718281828459045 AS 2.718281828459045#0, -0.8 AS -0.8#0, 89.97620 AS 89.97620#0, 89889.7667231 AS 89889.7667231#0, connect! AS connect!#0, T AS T#0, ABCDEFGHIJ AS ABCDEFGHIJ#0, 0x78797A7B7C7D7E7F808182838485868788898A8B8C8D8E AS X'78797A7B7C7D7E7F808182838485868788898A8B8C8D8E'#0, 0x0806 AS X'0806'#0, [8,6] AS ARRAY(8, 6)#0, null AS NULL#0, 2020-10-10 AS DATE '2020-10-10'#0, 8.997620 AS 8.997620#0, 2023-02-23 04:31:59.808 AS TIMESTAMP '2023-02-23 04:31:59.808'#0, 1969-12-31 16:00:12.345 AS TIMESTAMP '1969-12-31 16:00:12.345'#0, 2023-02-23 20:36:00 AS TIMESTAMP_NTZ '2023-02-23 20:36:00'#0, 2023-02-23 AS DATE '2023-02-23'#0, INTERVAL '0 00:03:20' DAY TO SECOND AS INTERVAL '0 00:03:20' DAY TO SECOND#0, INTERVAL '0-0' YEAR TO MONTH AS INTERVAL '0-0' YEAR TO MONTH#0, 23:59:59.999999999 AS TIME '23:59:59.999999999'#0, 2 months 20 days 0.0001 seconds AS INTERVAL '2 months 20 days 0.0001 seconds'#0, [18545,1677155519808000,12345000,1677184560000000,19411,200000000,0,86399999999999,2 months 20 days 0.0001 seconds] AS NAMED_STRUCT('_1', DATE '2020-10-10', '_2', TIMESTAMP '2023-02-23 04:31:59.808', '_3', TIMESTAMP '1969-12-31 16:00:12.345', '_4', TIMESTAMP_NTZ '2023-02-23 20:36:00', '_5', DATE '2023-02-23', '_6', INTERVAL '0 00:03:20' DAY TO SECOND, '_7', INTERVAL '0-0' YEAR TO MONTH, '_8', TIME '23:59:59.999999999', '_9', INTERVAL '2 months 20 days 0.0001 seconds')#0, 1 AS 1#0, [1,2,3] AS ARRAY(1, 2, 3)#0, [null,null] AS ARRAY(CAST(NULL AS INT), CAST(NULL AS INT))#0, [null,null,[1,a],[2,null]] AS ARRAY(NULL, NULL, NAMED_STRUCT('_1', 1, '_2', 'a'), NAMED_STRUCT('_1', 2, '_2', CAST(NULL AS STRING)))#0, [null,null,[1,a]] AS ARRAY(NULL, NULL, NAMED_STRUCT('_1', 1, '_2', 'a'))#0, [1,2,3] AS ARRAY(1, 2, 3)#0, map(keys: [a,b], values: [1,2]) AS MAP('a', 1, 'b', 2)#0, map(keys: [a,b], values: [null,null]) AS MAP('a', CAST(NULL AS INT), 'b', CAST(NULL AS INT))#0, [a,2,1.0] AS NAMED_STRUCT('_1', 'a', '_2', 2, '_3', 1.0D)#0, null AS NULL#0, [1] AS ARRAY(1)#0, map(keys: [1], values: [null]) AS MAP(1, CAST(NULL AS INT))#0, map(keys: [1], values: [null]) AS MAP(1, CAST(NULL AS INT))#0, map(keys: [1], values: [null]) AS MAP(1, CAST(NULL AS INT))#0, [[1,2,3],[4,5,6],[7,8,9]] AS ARRAY(ARRAY(1, 2, 3), ARRAY(4, 5, 6), ARRAY(7, 8, 9))#0, [[1,2,[3,4]],[5,6,[]]] AS ARRAY(NAMED_STRUCT('_1', 1, '_2', '2', '_3', ARRAY('3', '4')), NAMED_STRUCT('_1', 5, '_2', '6', '_3', ARRAY()))#0, [[1,2],[3,4],[5,6]] AS ARRAY(NAMED_STRUCT('a', 1, 'b', '2'), NAMED_STRUCT('a', 3, 'b', '4'), NAMED_STRUCT('a', 5, 'b', '6'))#0, [keys: [a,b], values: [1,2],keys: [a,b], values: [3,4],keys: [a,b], values: [5,6]] AS ARRAY(MAP('a', 1, 'b', 2), MAP('a', 3, 'b', 4), MAP('a', 5, 'b', 6))#0, [keys: [a,b], values: [[1,2],[3,4]],keys: [a,b], values: [[5,6],[7,8]],keys: [a,b], values: [[],[]]] AS ARRAY(MAP('a', ARRAY('1', '2'), 'b', ARRAY('3', '4')), MAP('a', ARRAY('5', '6'), 'b', ARRAY('7', '8')), MAP('a', ARRAY(), 'b', ARRAY()))#0, map(keys: [1,2], values: [keys: [a,b], values: [1,2],keys: [a,b], values: [3,4]]) AS MAP(1, MAP('a', 1, 'b', 2), 2, MAP('a', 3, 'b', 4))#0, [[1,2,3],keys: [a,b], values: [1,2],[a,keys: [1,2], values: [a,b]]] AS NAMED_STRUCT('_1', ARRAY(1, 2, 3), '_2', MAP('a', 1, 'b', 2), '_3', NAMED_STRUCT('_1', 'a', '_2', MAP(1, 'a', 2, 'b')))#0]
Project [id#0L, id#0L, 1 AS 1#0, null AS NULL#0, true AS true#0, 68 AS 68#0, 9872 AS 9872#0, -8726532 AS -8726532#0, 7834609328726532 AS 7834609328726532#0L, 2.718281828459045 AS 2.718281828459045#0, -0.8 AS -0.8#0, 89.97620 AS 89.97620#0, 89889.7667231 AS 89889.7667231#0, connect! AS connect!#0, T AS T#0, ABCDEFGHIJ AS ABCDEFGHIJ#0, 0x78797A7B7C7D7E7F808182838485868788898A8B8C8D8E AS X'78797A7B7C7D7E7F808182838485868788898A8B8C8D8E'#0, 0x0806 AS X'0806'#0, [8,6] AS ARRAY(8, 6)#0, null AS NULL#0, 2020-10-10 AS DATE '2020-10-10'#0, 8.997620 AS 8.997620#0, 2023-02-23 04:31:59.808 AS TIMESTAMP '2023-02-23 04:31:59.808'#0, 1969-12-31 16:00:12.345 AS TIMESTAMP '1969-12-31 16:00:12.345'#0, 2023-02-23 20:36:00 AS TIMESTAMP_NTZ '2023-02-23 20:36:00'#0, 2023-02-23 AS DATE '2023-02-23'#0, INTERVAL '0 00:03:20' DAY TO SECOND AS INTERVAL '0 00:03:20' DAY TO SECOND#0, INTERVAL '0-0' YEAR TO MONTH AS INTERVAL '0-0' YEAR TO MONTH#0, 23:59:59.999999999 AS TIME '23:59:59.999999999'#0, 2 months 20 days 0.0001 seconds AS INTERVAL '2 months 20 days 0.0001 seconds'#0, [18545,1677155519808000,12345000,1677184560000000,19411,200000000,0,86399999999999,2 months 20 days 0.0001 seconds] AS NAMED_STRUCT('_1', DATE '2020-10-10', '_2', TIMESTAMP '2023-02-23 04:31:59.808', '_3', TIMESTAMP '1969-12-31 16:00:12.345', '_4', TIMESTAMP_NTZ '2023-02-23 20:36:00', '_5', DATE '2023-02-23', '_6', INTERVAL '0 00:03:20' DAY TO SECOND, '_7', INTERVAL '0-0' YEAR TO MONTH, '_8', TIME '23:59:59.999999999', '_9', INTERVAL '2 months 20 days 0.0001 seconds')#0, 1 AS 1#0, [1,2,3] AS ARRAY(1, 2, 3)#0, [null,null] AS ARRAY(CAST(NULL AS INT), CAST(NULL AS INT))#0, [null,null,[1,a],[2,null]] AS ARRAY(NULL, NULL, NAMED_STRUCT('_1', 1, '_2', 'a'), NAMED_STRUCT('_1', 2, '_2', CAST(NULL AS STRING COLLATE UTF8_BINARY)))#0, [null,null,[1,a]] AS ARRAY(NULL, NULL, NAMED_STRUCT('_1', 1, '_2', 'a'))#0, [1,2,3] AS ARRAY(1, 2, 3)#0, map(keys: [a,b], values: [1,2]) AS MAP('a', 1, 'b', 2)#0, map(keys: [a,b], values: [null,null]) AS MAP('a', CAST(NULL AS INT), 'b', CAST(NULL AS INT))#0, [a,2,1.0] AS NAMED_STRUCT('_1', 'a', '_2', 2, '_3', 1.0D)#0, null AS NULL#0, [1] AS ARRAY(1)#0, map(keys: [1], values: [null]) AS MAP(1, CAST(NULL AS INT))#0, map(keys: [1], values: [null]) AS MAP(1, CAST(NULL AS INT))#0, map(keys: [1], values: [null]) AS MAP(1, CAST(NULL AS INT))#0, [[1,2,3],[4,5,6],[7,8,9]] AS ARRAY(ARRAY(1, 2, 3), ARRAY(4, 5, 6), ARRAY(7, 8, 9))#0, [[1,2,[3,4]],[5,6,[]]] AS ARRAY(NAMED_STRUCT('_1', 1, '_2', '2', '_3', ARRAY('3', '4')), NAMED_STRUCT('_1', 5, '_2', '6', '_3', ARRAY()))#0, [[1,2],[3,4],[5,6]] AS ARRAY(NAMED_STRUCT('a', 1, 'b', '2'), NAMED_STRUCT('a', 3, 'b', '4'), NAMED_STRUCT('a', 5, 'b', '6'))#0, [keys: [a,b], values: [1,2],keys: [a,b], values: [3,4],keys: [a,b], values: [5,6]] AS ARRAY(MAP('a', 1, 'b', 2), MAP('a', 3, 'b', 4), MAP('a', 5, 'b', 6))#0, [keys: [a,b], values: [[1,2],[3,4]],keys: [a,b], values: [[5,6],[7,8]],keys: [a,b], values: [[],[]]] AS ARRAY(MAP('a', ARRAY('1', '2'), 'b', ARRAY('3', '4')), MAP('a', ARRAY('5', '6'), 'b', ARRAY('7', '8')), MAP('a', ARRAY(), 'b', ARRAY()))#0, map(keys: [1,2], values: [keys: [a,b], values: [1,2],keys: [a,b], values: [3,4]]) AS MAP(1, MAP('a', 1, 'b', 2), 2, MAP('a', 3, 'b', 4))#0, [[1,2,3],keys: [a,b], values: [1,2],[a,keys: [1,2], values: [a,b]]] AS NAMED_STRUCT('_1', ARRAY(1, 2, 3), '_2', MAP('a', 1, 'b', 2), '_3', NAMED_STRUCT('_1', 'a', '_2', MAP(1, 'a', 2, 'b')))#0]
+- LocalRelation <empty>, [id#0L, a#0, b#0]
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,12 @@ import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.catalyst.plans.DescribeCommandSchema
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.types.DataTypeUtils
import org.apache.spark.sql.catalyst.util.{escapeSingleQuotedString, quoteIfNeeded, CaseInsensitiveMap, CharVarcharUtils, DateTimeUtils, ResolveDefaultColumns}
import org.apache.spark.sql.catalyst.util.ResolveDefaultColumns.CURRENT_DEFAULT_COLUMN_METADATA_KEY
import org.apache.spark.sql.classic.ClassicConversions.castToImpl
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.TableIdentifierHelper
import org.apache.spark.sql.connector.catalog.TableCatalog
import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryExecutionErrors}
import org.apache.spark.sql.execution.CommandExecutionMode
import org.apache.spark.sql.execution.datasources.DataSource
Expand Down Expand Up @@ -1078,6 +1080,11 @@ trait ShowCreateTableCommandBase extends SQLConfHelper {
.foreach(builder.append)
}

protected def showTableCollation(metadata: CatalogTable, builder: StringBuilder): Unit = {
metadata.collation.map(collation => "DEFAULT COLLATION " + collation + "\n")
.foreach(builder.append)
}

protected def showTableProperties(metadata: CatalogTable, builder: StringBuilder): Unit = {
if (metadata.properties.nonEmpty) {
val props =
Expand All @@ -1099,6 +1106,7 @@ trait ShowCreateTableCommandBase extends SQLConfHelper {
protected def showCreateView(metadata: CatalogTable, builder: StringBuilder): Unit = {
showViewDataColumns(metadata, builder)
showTableComment(metadata, builder)
showTableCollation(metadata, builder)
showViewProperties(metadata, builder)
showViewSchemaBinding(metadata, builder)
showViewText(metadata, builder)
Expand All @@ -1121,6 +1129,7 @@ trait ShowCreateTableCommandBase extends SQLConfHelper {
private def showViewProperties(metadata: CatalogTable, builder: StringBuilder): Unit = {
val viewProps = metadata.properties
.filter { case (k, _) => !k.startsWith(CatalogTable.VIEW_PREFIX) }
.filter { case (k, _) => !k.startsWith(TableCatalog.PROP_COLLATION) }
if (viewProps.nonEmpty) {
val props = viewProps.toSeq.sortBy(_._1).map { case (key, value) =>
s"'${escapeSingleQuotedString(key)}' = '${escapeSingleQuotedString(value)}'"
Expand Down Expand Up @@ -1235,7 +1244,10 @@ case class ShowCreateTableCommand(

private def showDataSourceTableDataColumns(
metadata: CatalogTable, builder: StringBuilder): Unit = {
val columns = metadata.schema.fields.map(_.toDDL)
val rawSchema = CharVarcharUtils.getRawSchema(metadata.schema, conf)
val schemaWithExplicitCollations = DataTypeUtils.replaceNonCollatedTypesWithExplicitUTF8Binary(
rawSchema).asInstanceOf[StructType]
val columns = schemaWithExplicitCollations.fields.map(_.toDDL)
builder ++= concatByMultiLines(columns)
}

Expand Down Expand Up @@ -1280,6 +1292,7 @@ case class ShowCreateTableCommand(
showDataSourceTableOptions(metadata, builder)
showDataSourceTableNonDataColumns(metadata, builder)
showTableComment(metadata, builder)
showTableCollation(metadata, builder)
showTableLocation(metadata, builder)
showTableProperties(metadata, builder)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,12 @@ import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.analysis.ResolvedTable
import org.apache.spark.sql.catalyst.catalog.BucketSpec
import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.catalyst.types.DataTypeUtils
import org.apache.spark.sql.catalyst.util.{escapeSingleQuotedString, CharVarcharUtils}
import org.apache.spark.sql.connector.catalog.{CatalogV2Util, Table, TableCatalog}
import org.apache.spark.sql.connector.expressions.BucketTransform
import org.apache.spark.sql.execution.LeafExecNode
import org.apache.spark.sql.types.StructType
import org.apache.spark.unsafe.types.UTF8String

/**
Expand Down Expand Up @@ -64,7 +66,10 @@ case class ShowCreateTableExec(

private def showTableDataColumns(table: Table, builder: StringBuilder): Unit = {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
val columns = CharVarcharUtils.getRawSchema(table.columns.asSchema, conf).fields.map(_.toDDL)
val rawSchema = CharVarcharUtils.getRawSchema(table.columns.asSchema, conf)
val schemaWithExplicitCollations = DataTypeUtils.replaceNonCollatedTypesWithExplicitUTF8Binary(
rawSchema).asInstanceOf[StructType]
val columns = schemaWithExplicitCollations.fields.map(_.toDDL)
val constraints = table.constraints().map(_.toDDL)
builder ++= concatByMultiLines(columns ++ constraints)
}
Expand Down Expand Up @@ -159,8 +164,7 @@ case class ShowCreateTableExec(

private def showTableCollation(table: Table, builder: StringBuilder): Unit = {
Option(table.properties.get(TableCatalog.PROP_COLLATION))
.map("COLLATION '" + escapeSingleQuotedString(_) + "'\n")
.foreach(builder.append)
.map("DEFAULT COLLATION " + _ + "\n").foreach(builder.append)
}

private def concatByMultiLines(iter: Iterable[String]): String = {
Expand Down
Loading