[SPARK-56031][SQL] Make Natural Join column matching respect case sensitivity conf

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala

@@ -3613,7 +3613,9 @@ class Analyzer(
       case j @ Join(left, right, NaturalJoin(joinType), condition, hint)
           if j.resolvedExceptNatural =>
         // find common column names from both sides
-        val joinNames = left.output.map(_.name).intersect(right.output.map(_.name))
+        val joinNames = left.output.map(_.name).distinct.filter { leftName =>
+          right.output.map(_.name).exists(resolver(leftName, _))
+        }
         val project = commonNaturalJoinProcessing(
           left, right, joinType, joinNames, condition, hint)
         j.getTagValue(LogicalPlan.PLAN_ID_TAG)

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveNaturalJoinSuite.scala

@@ -154,4 +154,21 @@ class ResolveNaturalJoinSuite extends AnalysisTest {
       checkAnalysis(usingPlan, expected, caseSensitive = false)
     }
   }
+
+  test("natural join with a case insensitive analyzer") {
+    val aUpper = $"A".string
+    val r1Upper = LocalRelation(b, aUpper)
+    val naturalPlan = r1Upper.join(r2, NaturalJoin(Inner), None)
+    val expected = r1Upper.join(r2, Inner, Some(EqualTo(aUpper, a))).select(aUpper, b, c)
+    checkAnalysis(naturalPlan, expected, caseSensitive = false)
+  }
+
+  test("natural join with a case sensitive analyzer") {
+    val aUpper = $"A".string
+    val r1Upper = LocalRelation(b, aUpper)
+    // "A" and "a" should not match when case sensitive, resulting in a cross join
+    val naturalPlan = r1Upper.join(r2, NaturalJoin(Inner), None)
+    val expected = r1Upper.join(r2, Inner, None).select(b, aUpper, c, a)
+    checkAnalysis(naturalPlan, expected, caseSensitive = true)
+  }
 }

sql/core/src/test/resources/sql-tests/analyzer-results/natural-join.sql.out

@@ -678,3 +678,17 @@ Project [k#x, v1#x, k#x, v2#x, k#x, v3#x, k#x, v4#x]
                +- Project [k#x, v4#x]
                   +- SubqueryAlias nt4
                      +- LocalRelation [k#x, v4#x]
+
+
+-- !query
+SELECT * FROM (SELECT 1 AS ID) t1 NATURAL JOIN (SELECT 1 AS id) t2
+-- !query analysis
+Project [ID#x]
++- Project [ID#x]
+   +- Join Inner, (ID#x = id#x)
+      :- SubqueryAlias t1
+      :  +- Project [1 AS ID#x]
+      :     +- OneRowRelation
+      +- SubqueryAlias t2
+         +- Project [1 AS id#x]
+            +- OneRowRelation

sql/core/src/test/resources/sql-tests/inputs/natural-join.sql

@@ -75,3 +75,5 @@ SELECT * FROM nt1 natural join nt2 join nt3 on nt1.k = nt3.k;
 SELECT * FROM nt1 natural join nt2 join nt3 on nt2.k = nt3.k;
 
 SELECT nt1.*, nt2.*, nt3.*, nt4.* FROM nt1 natural join nt2 natural join nt3 natural join nt4;
+
+SELECT * FROM (SELECT 1 AS ID) t1 NATURAL JOIN (SELECT 1 AS id) t2;

sql/core/src/test/resources/sql-tests/results/natural-join.sql.out

@@ -340,3 +340,11 @@ one	1	one	5	one	4	one	9
 one	1	one	5	one	6	one	7
 one	1	one	5	one	6	one	9
 two	2	two	22	two	5	two	8
+
+
+-- !query
+SELECT * FROM (SELECT 1 AS ID) t1 NATURAL JOIN (SELECT 1 AS id) t2
+-- !query schema
+struct<ID:int>
+-- !query output
+1