From 57018b7cf4bed20a9d383645f1774a1d5622a00a Mon Sep 17 00:00:00 2001
From: Shehab <11789402+shehabgamin@users.noreply.github.com>
Date: Tue, 11 Mar 2025 14:21:10 -0700
Subject: [PATCH 01/25] feat: Add datafusion-spark crate

---
 Cargo.lock                                    |  26 ++
 Cargo.toml                                    |   1 +
 NOTICE.txt                                    |   4 +-
 datafusion/functions/src/utils.rs             |   2 +-
 datafusion/spark/Cargo.toml                   |  60 ++++
 datafusion/spark/LICENSE.txt                  |   1 +
 datafusion/spark/NOTICE.txt                   |   1 +
 datafusion/spark/README.md                    |  22 ++
 .../spark/src/function/aggregate/mod.rs       |  16 ++
 datafusion/spark/src/function/array/mod.rs    |  16 ++
 datafusion/spark/src/function/bitwise/mod.rs  |  16 ++
 .../spark/src/function/collection/mod.rs      |  16 ++
 .../spark/src/function/conditional/mod.rs     |  16 ++
 .../spark/src/function/conversion/mod.rs      |  16 ++
 datafusion/spark/src/function/csv/mod.rs      |  16 ++
 datafusion/spark/src/function/datetime/mod.rs |  16 ++
 datafusion/spark/src/function/error_utils.rs  |  71 +++++
 .../spark/src/function/generator/mod.rs       |  16 ++
 datafusion/spark/src/function/hash/mod.rs     |  16 ++
 datafusion/spark/src/function/json/mod.rs     |  16 ++
 datafusion/spark/src/function/lambda/mod.rs   |  16 ++
 datafusion/spark/src/function/map/mod.rs      |  16 ++
 datafusion/spark/src/function/math/expm1.rs   | 116 ++++++++
 datafusion/spark/src/function/math/mod.rs     |  18 ++
 datafusion/spark/src/function/misc/mod.rs     |  16 ++
 datafusion/spark/src/function/mod.rs          |  41 +++
 .../spark/src/function/predicate/mod.rs       |  16 ++
 datafusion/spark/src/function/string/ascii.rs | 258 ++++++++++++++++++
 datafusion/spark/src/function/string/mod.rs   |  18 ++
 datafusion/spark/src/function/struct/mod.rs   |  16 ++
 datafusion/spark/src/function/table/mod.rs    |  16 ++
 datafusion/spark/src/function/url/mod.rs      |  16 ++
 datafusion/spark/src/function/utils.rs        | 108 ++++++++
 datafusion/spark/src/function/window/mod.rs   |  16 ++
 datafusion/spark/src/function/xml/mod.rs      |  16 ++
 datafusion/spark/src/lib.rs                   |  48 ++++
 36 files changed, 1112 insertions(+), 3 deletions(-)
 create mode 100644 datafusion/spark/Cargo.toml
 create mode 120000 datafusion/spark/LICENSE.txt
 create mode 120000 datafusion/spark/NOTICE.txt
 create mode 100644 datafusion/spark/README.md
 create mode 100644 datafusion/spark/src/function/aggregate/mod.rs
 create mode 100644 datafusion/spark/src/function/array/mod.rs
 create mode 100644 datafusion/spark/src/function/bitwise/mod.rs
 create mode 100644 datafusion/spark/src/function/collection/mod.rs
 create mode 100644 datafusion/spark/src/function/conditional/mod.rs
 create mode 100644 datafusion/spark/src/function/conversion/mod.rs
 create mode 100644 datafusion/spark/src/function/csv/mod.rs
 create mode 100644 datafusion/spark/src/function/datetime/mod.rs
 create mode 100644 datafusion/spark/src/function/error_utils.rs
 create mode 100644 datafusion/spark/src/function/generator/mod.rs
 create mode 100644 datafusion/spark/src/function/hash/mod.rs
 create mode 100644 datafusion/spark/src/function/json/mod.rs
 create mode 100644 datafusion/spark/src/function/lambda/mod.rs
 create mode 100644 datafusion/spark/src/function/map/mod.rs
 create mode 100644 datafusion/spark/src/function/math/expm1.rs
 create mode 100644 datafusion/spark/src/function/math/mod.rs
 create mode 100644 datafusion/spark/src/function/misc/mod.rs
 create mode 100644 datafusion/spark/src/function/mod.rs
 create mode 100644 datafusion/spark/src/function/predicate/mod.rs
 create mode 100644 datafusion/spark/src/function/string/ascii.rs
 create mode 100644 datafusion/spark/src/function/string/mod.rs
 create mode 100644 datafusion/spark/src/function/struct/mod.rs
 create mode 100644 datafusion/spark/src/function/table/mod.rs
 create mode 100644 datafusion/spark/src/function/url/mod.rs
 create mode 100644 datafusion/spark/src/function/utils.rs
 create mode 100644 datafusion/spark/src/function/window/mod.rs
 create mode 100644 datafusion/spark/src/function/xml/mod.rs
 create mode 100644 datafusion/spark/src/lib.rs

diff --git a/Cargo.lock b/Cargo.lock
index 8105a36a8f3e..cfd4a795b6d3 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2444,6 +2444,32 @@ dependencies = [
  "serde_json",
 ]
 
+[[package]]
+name = "datafusion-spark"
+version = "46.0.0"
+dependencies = [
+ "arrow",
+ "arrow-ord",
+ "criterion",
+ "datafusion-common",
+ "datafusion-doc",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-expr-common",
+ "datafusion-functions",
+ "datafusion-functions-aggregate",
+ "datafusion-functions-aggregate-common",
+ "datafusion-functions-nested",
+ "datafusion-functions-table",
+ "datafusion-functions-window",
+ "datafusion-functions-window-common",
+ "datafusion-macros",
+ "datafusion-physical-expr-common",
+ "itertools 0.14.0",
+ "log",
+ "rand 0.8.5",
+]
+
 [[package]]
 name = "datafusion-sql"
 version = "46.0.0"
diff --git a/Cargo.toml b/Cargo.toml
index 871377f8dfc0..2fc392a8916d 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -47,6 +47,7 @@ members = [
     "datafusion/proto/gen",
     "datafusion/proto-common",
     "datafusion/proto-common/gen",
+    "datafusion/spark",
     "datafusion/sql",
     "datafusion/sqllogictest",
     "datafusion/substrait",
diff --git a/NOTICE.txt b/NOTICE.txt
index 21be1a20d554..7f3c80d606c0 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -1,5 +1,5 @@
 Apache DataFusion
-Copyright 2019-2024 The Apache Software Foundation
+Copyright 2019-2025 The Apache Software Foundation
 
 This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
\ No newline at end of file
+The Apache Software Foundation (http://www.apache.org/).
diff --git a/datafusion/functions/src/utils.rs b/datafusion/functions/src/utils.rs
index 47f3121ba2ce..33ea9fe5d28d 100644
--- a/datafusion/functions/src/utils.rs
+++ b/datafusion/functions/src/utils.rs
@@ -75,7 +75,7 @@ get_optimal_return_type!(utf8_to_int_type, DataType::Int64, DataType::Int32);
 /// Creates a scalar function implementation for the given function.
 /// * `inner` - the function to be executed
 /// * `hints` - hints to be used when expanding scalars to arrays
-pub(super) fn make_scalar_function<F>(
+pub fn make_scalar_function<F>(
     inner: F,
     hints: Vec<Hint>,
 ) -> impl Fn(&[ColumnarValue]) -> Result<ColumnarValue>
diff --git a/datafusion/spark/Cargo.toml b/datafusion/spark/Cargo.toml
new file mode 100644
index 000000000000..b29dc5f6c903
--- /dev/null
+++ b/datafusion/spark/Cargo.toml
@@ -0,0 +1,60 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+name = "datafusion-spark"
+description = "DataFusion expressions that emulate Apache Spark's behavior"
+version = { workspace = true }
+homepage = { workspace = true }
+repository = { workspace = true }
+authors = { workspace = true }
+readme = { workspace = true }
+license = { workspace = true }
+edition = { workspace = true }
+
+[package.metadata.docs.rs]
+all-features = true
+
+[lints]
+workspace = true
+
+[lib]
+name = "datafusion_spark"
+
+[dependencies]
+arrow = { workspace = true }
+arrow-ord = { workspace = true }
+datafusion-common = { workspace = true }
+datafusion-doc = { workspace = true }
+datafusion-execution = { workspace = true }
+datafusion-expr = { workspace = true }
+datafusion-expr-common = { workspace = true }
+datafusion-functions = { workspace = true }
+datafusion-functions-aggregate = { workspace = true }
+datafusion-functions-aggregate-common = { workspace = true }
+datafusion-functions-nested = { workspace = true }
+datafusion-functions-table = { workspace = true }
+datafusion-functions-window = { workspace = true }
+datafusion-functions-window-common = { workspace = true }
+datafusion-macros = { workspace = true }
+datafusion-physical-expr-common = { workspace = true }
+itertools = { workspace = true, features = ["use_std"] }
+log = { workspace = true }
+
+[dev-dependencies]
+criterion = { workspace = true, features = ["async_tokio"] }
+rand = { workspace = true }
diff --git a/datafusion/spark/LICENSE.txt b/datafusion/spark/LICENSE.txt
new file mode 120000
index 000000000000..1ef648f64b34
--- /dev/null
+++ b/datafusion/spark/LICENSE.txt
@@ -0,0 +1 @@
+../../LICENSE.txt
\ No newline at end of file
diff --git a/datafusion/spark/NOTICE.txt b/datafusion/spark/NOTICE.txt
new file mode 120000
index 000000000000..fb051c92b10b
--- /dev/null
+++ b/datafusion/spark/NOTICE.txt
@@ -0,0 +1 @@
+../../NOTICE.txt
\ No newline at end of file
diff --git a/datafusion/spark/README.md b/datafusion/spark/README.md
new file mode 100644
index 000000000000..afd94d2c0690
--- /dev/null
+++ b/datafusion/spark/README.md
@@ -0,0 +1,22 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# datafusion-spark: Spark-compatible Expressions
+
+This crate provides Apache Spark-compatible expressions for use with DataFusion.
diff --git a/datafusion/spark/src/function/aggregate/mod.rs b/datafusion/spark/src/function/aggregate/mod.rs
new file mode 100644
index 000000000000..b248758bc120
--- /dev/null
+++ b/datafusion/spark/src/function/aggregate/mod.rs
@@ -0,0 +1,16 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
diff --git a/datafusion/spark/src/function/array/mod.rs b/datafusion/spark/src/function/array/mod.rs
new file mode 100644
index 000000000000..b248758bc120
--- /dev/null
+++ b/datafusion/spark/src/function/array/mod.rs
@@ -0,0 +1,16 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
diff --git a/datafusion/spark/src/function/bitwise/mod.rs b/datafusion/spark/src/function/bitwise/mod.rs
new file mode 100644
index 000000000000..b248758bc120
--- /dev/null
+++ b/datafusion/spark/src/function/bitwise/mod.rs
@@ -0,0 +1,16 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
diff --git a/datafusion/spark/src/function/collection/mod.rs b/datafusion/spark/src/function/collection/mod.rs
new file mode 100644
index 000000000000..b248758bc120
--- /dev/null
+++ b/datafusion/spark/src/function/collection/mod.rs
@@ -0,0 +1,16 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
diff --git a/datafusion/spark/src/function/conditional/mod.rs b/datafusion/spark/src/function/conditional/mod.rs
new file mode 100644
index 000000000000..b248758bc120
--- /dev/null
+++ b/datafusion/spark/src/function/conditional/mod.rs
@@ -0,0 +1,16 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
diff --git a/datafusion/spark/src/function/conversion/mod.rs b/datafusion/spark/src/function/conversion/mod.rs
new file mode 100644
index 000000000000..b248758bc120
--- /dev/null
+++ b/datafusion/spark/src/function/conversion/mod.rs
@@ -0,0 +1,16 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
diff --git a/datafusion/spark/src/function/csv/mod.rs b/datafusion/spark/src/function/csv/mod.rs
new file mode 100644
index 000000000000..b248758bc120
--- /dev/null
+++ b/datafusion/spark/src/function/csv/mod.rs
@@ -0,0 +1,16 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
diff --git a/datafusion/spark/src/function/datetime/mod.rs b/datafusion/spark/src/function/datetime/mod.rs
new file mode 100644
index 000000000000..b248758bc120
--- /dev/null
+++ b/datafusion/spark/src/function/datetime/mod.rs
@@ -0,0 +1,16 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
diff --git a/datafusion/spark/src/function/error_utils.rs b/datafusion/spark/src/function/error_utils.rs
new file mode 100644
index 000000000000..b972d64ed3e9
--- /dev/null
+++ b/datafusion/spark/src/function/error_utils.rs
@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// TODO: https://github.com/apache/spark/tree/master/common/utils/src/main/resources/error
+
+use arrow::datatypes::DataType;
+use datafusion_common::{exec_datafusion_err, internal_datafusion_err, DataFusionError};
+
+pub fn invalid_arg_count_exec_err(
+    function_name: &str,
+    required_range: (i32, i32),
+    provided: usize,
+) -> DataFusionError {
+    let (min_required, max_required) = required_range;
+    let required = if min_required == max_required {
+        format!(
+            "{min_required} argument{}",
+            if min_required == 1 { "" } else { "s" }
+        )
+    } else {
+        format!("{min_required} to {max_required} arguments")
+    };
+    exec_datafusion_err!(
+        "Spark `{function_name}` function requires {required}, got {provided}"
+    )
+}
+
+pub fn unsupported_data_type_exec_err(
+    function_name: &str,
+    required: &str,
+    provided: &DataType,
+) -> DataFusionError {
+    exec_datafusion_err!("Unsupported Data Type: Spark `{function_name}` function expects {required}, got {provided}")
+}
+
+pub fn unsupported_data_types_exec_err(
+    function_name: &str,
+    required: &str,
+    provided: &[DataType],
+) -> DataFusionError {
+    exec_datafusion_err!(
+        "Unsupported Data Type: Spark `{function_name}` function expects {required}, got {}",
+        provided
+            .iter()
+            .map(|dt| format!("{dt}"))
+            .collect::<Vec<_>>()
+            .join(", ")
+    )
+}
+
+pub fn generic_exec_err(function_name: &str, message: &str) -> DataFusionError {
+    exec_datafusion_err!("Spark `{function_name}` function: {message}")
+}
+
+pub fn generic_internal_err(function_name: &str, message: &str) -> DataFusionError {
+    internal_datafusion_err!("Spark `{function_name}` function: {message}")
+}
diff --git a/datafusion/spark/src/function/generator/mod.rs b/datafusion/spark/src/function/generator/mod.rs
new file mode 100644
index 000000000000..b248758bc120
--- /dev/null
+++ b/datafusion/spark/src/function/generator/mod.rs
@@ -0,0 +1,16 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
diff --git a/datafusion/spark/src/function/hash/mod.rs b/datafusion/spark/src/function/hash/mod.rs
new file mode 100644
index 000000000000..b248758bc120
--- /dev/null
+++ b/datafusion/spark/src/function/hash/mod.rs
@@ -0,0 +1,16 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
diff --git a/datafusion/spark/src/function/json/mod.rs b/datafusion/spark/src/function/json/mod.rs
new file mode 100644
index 000000000000..b248758bc120
--- /dev/null
+++ b/datafusion/spark/src/function/json/mod.rs
@@ -0,0 +1,16 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
diff --git a/datafusion/spark/src/function/lambda/mod.rs b/datafusion/spark/src/function/lambda/mod.rs
new file mode 100644
index 000000000000..b248758bc120
--- /dev/null
+++ b/datafusion/spark/src/function/lambda/mod.rs
@@ -0,0 +1,16 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
diff --git a/datafusion/spark/src/function/map/mod.rs b/datafusion/spark/src/function/map/mod.rs
new file mode 100644
index 000000000000..b248758bc120
--- /dev/null
+++ b/datafusion/spark/src/function/map/mod.rs
@@ -0,0 +1,16 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
diff --git a/datafusion/spark/src/function/math/expm1.rs b/datafusion/spark/src/function/math/expm1.rs
new file mode 100644
index 000000000000..985c7bf20496
--- /dev/null
+++ b/datafusion/spark/src/function/math/expm1.rs
@@ -0,0 +1,116 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use arrow::array::{ArrayRef, AsArray};
+use arrow::datatypes::{DataType, Float64Type};
+use datafusion_common::{Result, ScalarValue};
+use datafusion_expr::{
+    ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,
+};
+use std::any::Any;
+use std::sync::Arc;
+
+use crate::function::error_utils::{
+    invalid_arg_count_exec_err, unsupported_data_type_exec_err,
+};
+
+#[derive(Debug)]
+pub struct SparkExpm1 {
+    signature: Signature,
+    aliases: Vec<String>,
+}
+
+impl Default for SparkExpm1 {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl SparkExpm1 {
+    pub fn new() -> Self {
+        Self {
+            signature: Signature::user_defined(Volatility::Immutable),
+            aliases: vec!["spark_expm1".to_string()],
+        }
+    }
+}
+
+impl ScalarUDFImpl for SparkExpm1 {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn name(&self) -> &str {
+        "expm1"
+    }
+
+    fn signature(&self) -> &Signature {
+        &self.signature
+    }
+
+    fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
+        Ok(DataType::Float64)
+    }
+
+    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
+        if args.args.len() != 1 {
+            return Err(invalid_arg_count_exec_err("expm1", (1, 1), args.args.len()));
+        }
+        match &args.args[0] {
+            ColumnarValue::Scalar(ScalarValue::Float64(value)) => Ok(
+                ColumnarValue::Scalar(ScalarValue::Float64(value.map(|x| x.exp_m1()))),
+            ),
+            ColumnarValue::Array(array) => match array.data_type() {
+                DataType::Float64 => Ok(ColumnarValue::Array(Arc::new(
+                    array
+                        .as_primitive::<Float64Type>()
+                        .unary::<_, Float64Type>(|x| x.exp_m1()),
+                )
+                    as ArrayRef)),
+                other => Err(unsupported_data_type_exec_err(
+                    "expm1",
+                    format!("{}", DataType::Float64).as_str(),
+                    other,
+                )),
+            },
+            other => Err(unsupported_data_type_exec_err(
+                "expm1",
+                format!("{}", DataType::Float64).as_str(),
+                &other.data_type(),
+            )),
+        }
+    }
+
+    fn aliases(&self) -> &[String] {
+        &self.aliases
+    }
+
+    fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+        if arg_types.len() != 1 {
+            return Err(invalid_arg_count_exec_err("expm1", (1, 1), arg_types.len()));
+        }
+        if arg_types[0].is_numeric() {
+            Ok(vec![DataType::Float64])
+        } else {
+            Err(unsupported_data_type_exec_err(
+                "expm1",
+                "Numeric Type",
+                &arg_types[0],
+            ))
+        }
+    }
+}
diff --git a/datafusion/spark/src/function/math/mod.rs b/datafusion/spark/src/function/math/mod.rs
new file mode 100644
index 000000000000..13c865a3a5b3
--- /dev/null
+++ b/datafusion/spark/src/function/math/mod.rs
@@ -0,0 +1,18 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+pub mod expm1;
diff --git a/datafusion/spark/src/function/misc/mod.rs b/datafusion/spark/src/function/misc/mod.rs
new file mode 100644
index 000000000000..b248758bc120
--- /dev/null
+++ b/datafusion/spark/src/function/misc/mod.rs
@@ -0,0 +1,16 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
diff --git a/datafusion/spark/src/function/mod.rs b/datafusion/spark/src/function/mod.rs
new file mode 100644
index 000000000000..dfdd94a040a9
--- /dev/null
+++ b/datafusion/spark/src/function/mod.rs
@@ -0,0 +1,41 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+pub mod aggregate;
+pub mod array;
+pub mod bitwise;
+pub mod collection;
+pub mod conditional;
+pub mod conversion;
+pub mod csv;
+pub mod datetime;
+pub mod error_utils;
+pub mod generator;
+pub mod hash;
+pub mod json;
+pub mod lambda;
+pub mod map;
+pub mod math;
+pub mod misc;
+pub mod predicate;
+pub mod string;
+pub mod r#struct;
+pub mod table;
+pub mod url;
+pub mod utils;
+pub mod window;
+pub mod xml;
diff --git a/datafusion/spark/src/function/predicate/mod.rs b/datafusion/spark/src/function/predicate/mod.rs
new file mode 100644
index 000000000000..b248758bc120
--- /dev/null
+++ b/datafusion/spark/src/function/predicate/mod.rs
@@ -0,0 +1,16 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
diff --git a/datafusion/spark/src/function/string/ascii.rs b/datafusion/spark/src/function/string/ascii.rs
new file mode 100644
index 000000000000..4eb46fac0c54
--- /dev/null
+++ b/datafusion/spark/src/function/string/ascii.rs
@@ -0,0 +1,258 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use arrow::array::{ArrayAccessor, ArrayIter, ArrayRef, AsArray, Int32Array};
+use arrow::datatypes::DataType;
+use arrow::error::ArrowError;
+use datafusion_common::{internal_err, plan_err, Result};
+use datafusion_expr::{ColumnarValue, Documentation};
+use datafusion_expr::{ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility};
+use datafusion_functions::utils::make_scalar_function;
+use datafusion_macros::user_doc;
+use std::any::Any;
+use std::sync::Arc;
+
+#[user_doc(
+    doc_section(label = "String Functions"),
+    description = "Returns the ASCII code point of the first character of string.",
+    syntax_example = "ascii(str)",
+    sql_example = r#"```sql
+> select ascii('abc');
++--------------------+
+| ascii(abc)         |
++--------------------+
+| 97                 |
++--------------------+
+> select ascii('🚀');
++-------------------+
+| ascii(🚀)         |
++-------------------+
+| 128640            |
++-------------------+
+> select ascii(2);
++----------------+
+| ascii(2)       |
++----------------+
+| 50             |
++----------------+
+> select ascii(X'44617461467573696F6E');
++--------------------------------------+
+| ascii(X'44617461467573696F6E')       |
++--------------------------------------+
+| 68                                   |
++--------------------------------------+
+```"#,
+    standard_argument(name = "str", prefix = "String")
+)]
+#[derive(Debug)]
+pub struct SparkAscii {
+    signature: Signature,
+    aliases: Vec<String>,
+}
+
+impl Default for SparkAscii {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl SparkAscii {
+    pub fn new() -> Self {
+        Self {
+            signature: Signature::user_defined(Volatility::Immutable),
+            aliases: vec!["spark_ascii".to_string()],
+        }
+    }
+}
+
+impl ScalarUDFImpl for SparkAscii {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn name(&self) -> &str {
+        "ascii"
+    }
+
+    fn signature(&self) -> &Signature {
+        &self.signature
+    }
+
+    fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
+        Ok(DataType::Int32)
+    }
+
+    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
+        make_scalar_function(ascii, vec![])(&args.args)
+    }
+
+    fn aliases(&self) -> &[String] {
+        &self.aliases
+    }
+
+    fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+        if arg_types.len() != 1 {
+            return plan_err!(
+                "The {} function requires 1 argument, but got {}.",
+                self.name(),
+                arg_types.len()
+            );
+        }
+        Ok(vec![DataType::Utf8])
+    }
+
+    fn documentation(&self) -> Option<&Documentation> {
+        self.doc()
+    }
+}
+
+fn calculate_ascii<'a, V>(array: V) -> Result<ArrayRef, ArrowError>
+where
+    V: ArrayAccessor<Item = &'a str>,
+{
+    let iter = ArrayIter::new(array);
+    let result = iter
+        .map(|string| {
+            string.map(|s| {
+                let mut chars = s.chars();
+                chars.next().map_or(0, |v| v as i32)
+            })
+        })
+        .collect::<Int32Array>();
+
+    Ok(Arc::new(result) as ArrayRef)
+}
+
+/// Returns the numeric code of the first character of the argument.
+pub fn ascii(args: &[ArrayRef]) -> Result<ArrayRef> {
+    match args[0].data_type() {
+        DataType::Utf8 => {
+            let string_array = args[0].as_string::<i32>();
+            Ok(calculate_ascii(string_array)?)
+        }
+        DataType::LargeUtf8 => {
+            let string_array = args[0].as_string::<i64>();
+            Ok(calculate_ascii(string_array)?)
+        }
+        DataType::Utf8View => {
+            let string_array = args[0].as_string_view();
+            Ok(calculate_ascii(string_array)?)
+        }
+        _ => internal_err!("Unsupported data type"),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::function::string::ascii::SparkAscii;
+    use crate::function::utils::test::test_scalar_function;
+    use arrow::array::{Array, Int32Array};
+    use arrow::datatypes::DataType::Int32;
+    use datafusion_common::{Result, ScalarValue};
+    use datafusion_expr::{ColumnarValue, ScalarUDFImpl};
+
+    macro_rules! test_ascii_string_invoke {
+        ($INPUT:expr, $EXPECTED:expr) => {
+            test_scalar_function!(
+                SparkAscii::new(),
+                vec![ColumnarValue::Scalar(ScalarValue::Utf8($INPUT))],
+                $EXPECTED,
+                i32,
+                Int32,
+                Int32Array
+            );
+
+            test_scalar_function!(
+                SparkAscii::new(),
+                vec![ColumnarValue::Scalar(ScalarValue::LargeUtf8($INPUT))],
+                $EXPECTED,
+                i32,
+                Int32,
+                Int32Array
+            );
+
+            test_scalar_function!(
+                SparkAscii::new(),
+                vec![ColumnarValue::Scalar(ScalarValue::Utf8View($INPUT))],
+                $EXPECTED,
+                i32,
+                Int32,
+                Int32Array
+            );
+        };
+    }
+
+    macro_rules! test_ascii_int_invoke {
+        ($INPUT:expr, $EXPECTED:expr) => {
+            test_scalar_function!(
+                SparkAscii::new(),
+                vec![ColumnarValue::Scalar(ScalarValue::Int32($INPUT))],
+                $EXPECTED,
+                i32,
+                Int32,
+                Int32Array
+            );
+        };
+    }
+
+    macro_rules! test_ascii_binary_invoke {
+        ($INPUT:expr, $EXPECTED:expr) => {
+            test_scalar_function!(
+                SparkAscii::new(),
+                vec![ColumnarValue::Scalar(ScalarValue::Binary($INPUT))],
+                $EXPECTED,
+                i32,
+                Int32,
+                Int32Array
+            );
+
+            test_scalar_function!(
+                SparkAscii::new(),
+                vec![ColumnarValue::Scalar(ScalarValue::BinaryView($INPUT))],
+                $EXPECTED,
+                i32,
+                Int32,
+                Int32Array
+            );
+
+            test_scalar_function!(
+                SparkAscii::new(),
+                vec![ColumnarValue::Scalar(ScalarValue::LargeBinary($INPUT))],
+                $EXPECTED,
+                i32,
+                Int32,
+                Int32Array
+            );
+        };
+    }
+
+    #[test]
+    fn test_ascii_invoke() -> Result<()> {
+        test_ascii_string_invoke!(Some(String::from("x")), Ok(Some(120)));
+        test_ascii_string_invoke!(Some(String::from("a")), Ok(Some(97)));
+        test_ascii_string_invoke!(Some(String::from("")), Ok(Some(0)));
+        test_ascii_string_invoke!(None, Ok(None));
+
+        test_ascii_int_invoke!(Some(2), Ok(Some(50)));
+        test_ascii_int_invoke!(None, Ok(None));
+
+        test_ascii_binary_invoke!(Some(b"DataFusion".to_vec()), Ok(Some(50)));
+        test_ascii_binary_invoke!(None, Ok(None));
+
+        Ok(())
+    }
+}
diff --git a/datafusion/spark/src/function/string/mod.rs b/datafusion/spark/src/function/string/mod.rs
new file mode 100644
index 000000000000..229be18e4161
--- /dev/null
+++ b/datafusion/spark/src/function/string/mod.rs
@@ -0,0 +1,18 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+pub mod ascii;
diff --git a/datafusion/spark/src/function/struct/mod.rs b/datafusion/spark/src/function/struct/mod.rs
new file mode 100644
index 000000000000..b248758bc120
--- /dev/null
+++ b/datafusion/spark/src/function/struct/mod.rs
@@ -0,0 +1,16 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
diff --git a/datafusion/spark/src/function/table/mod.rs b/datafusion/spark/src/function/table/mod.rs
new file mode 100644
index 000000000000..b248758bc120
--- /dev/null
+++ b/datafusion/spark/src/function/table/mod.rs
@@ -0,0 +1,16 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
diff --git a/datafusion/spark/src/function/url/mod.rs b/datafusion/spark/src/function/url/mod.rs
new file mode 100644
index 000000000000..b248758bc120
--- /dev/null
+++ b/datafusion/spark/src/function/url/mod.rs
@@ -0,0 +1,16 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
diff --git a/datafusion/spark/src/function/utils.rs b/datafusion/spark/src/function/utils.rs
new file mode 100644
index 000000000000..97651e87ac1e
--- /dev/null
+++ b/datafusion/spark/src/function/utils.rs
@@ -0,0 +1,108 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use arrow::array::ArrayRef;
+use arrow::datatypes::DataType;
+
+use datafusion_common::{Result, ScalarValue};
+use datafusion_expr::function::Hint;
+use datafusion_expr::ColumnarValue;
+
+#[cfg(test)]
+pub mod test {
+    /// $FUNC ScalarUDFImpl to test
+    /// $ARGS arguments (vec) to pass to function
+    /// $EXPECTED a Result<ColumnarValue>
+    /// $EXPECTED_TYPE is the expected value type
+    /// $EXPECTED_DATA_TYPE is the expected result type
+    /// $ARRAY_TYPE is the column type after function applied
+    macro_rules! test_scalar_function {
+        ($FUNC:expr, $ARGS:expr, $EXPECTED:expr, $EXPECTED_TYPE:ty, $EXPECTED_DATA_TYPE:expr, $ARRAY_TYPE:ident) => {
+            let expected: Result<Option<$EXPECTED_TYPE>> = $EXPECTED;
+            let func = $FUNC;
+
+            let type_array = $ARGS.iter().map(|arg| arg.data_type()).collect::<Vec<_>>();
+            let cardinality = $ARGS
+                .iter()
+                .fold(Option::<usize>::None, |acc, arg| match arg {
+                    ColumnarValue::Scalar(_) => acc,
+                    ColumnarValue::Array(a) => Some(a.len()),
+                })
+                .unwrap_or(1);
+
+            let scalar_arguments = $ARGS.iter().map(|arg| match arg {
+                ColumnarValue::Scalar(scalar) => Some(scalar.clone()),
+                ColumnarValue::Array(_) => None,
+            }).collect::<Vec<_>>();
+            let scalar_arguments_refs = scalar_arguments.iter().map(|arg| arg.as_ref()).collect::<Vec<_>>();
+
+            let nullables = $ARGS.iter().map(|arg| match arg {
+                ColumnarValue::Scalar(scalar) => scalar.is_null(),
+                ColumnarValue::Array(a) => a.null_count() > 0,
+            }).collect::<Vec<_>>();
+
+            let return_info = func.return_type_from_args(datafusion_expr::ReturnTypeArgs {
+                arg_types: &type_array,
+                scalar_arguments: &scalar_arguments_refs,
+                nullables: &nullables
+            });
+
+            match expected {
+                Ok(expected) => {
+                    assert_eq!(return_info.is_ok(), true);
+                    let (return_type, _nullable) = return_info.unwrap().into_parts();
+                    assert_eq!(return_type, $EXPECTED_DATA_TYPE);
+
+                    let result = func.invoke_with_args(datafusion_expr::ScalarFunctionArgs{args: $ARGS, number_rows: cardinality, return_type: &return_type});
+                    assert_eq!(result.is_ok(), true, "function returned an error: {}", result.unwrap_err());
+
+                    let result = result.unwrap().to_array(cardinality).expect("Failed to convert to array");
+                    let result = result.as_any().downcast_ref::<$ARRAY_TYPE>().expect("Failed to convert to type");
+                    assert_eq!(result.data_type(), &$EXPECTED_DATA_TYPE);
+
+                    // value is correct
+                    match expected {
+                        Some(v) => assert_eq!(result.value(0), v),
+                        None => assert!(result.is_null(0)),
+                    };
+                }
+                Err(expected_error) => {
+                    if return_info.is_err() {
+                        match return_info {
+                            Ok(_) => assert!(false, "expected error"),
+                            Err(error) => { datafusion_common::assert_contains!(expected_error.strip_backtrace(), error.strip_backtrace()); }
+                        }
+                    }
+                    else {
+                        let (return_type, _nullable) = return_info.unwrap().into_parts();
+
+                        // invoke is expected error - cannot use .expect_err() due to Debug not being implemented
+                        match func.invoke_with_args(datafusion_expr::ScalarFunctionArgs{args: $ARGS, number_rows: cardinality, return_type: &return_type}) {
+                            Ok(_) => assert!(false, "expected error"),
+                            Err(error) => {
+                                assert!(expected_error.strip_backtrace().starts_with(&error.strip_backtrace()));
+                            }
+                        }
+                    }
+                }
+            };
+        };
+    }
+
+    use super::*;
+    pub(crate) use test_scalar_function;
+}
diff --git a/datafusion/spark/src/function/window/mod.rs b/datafusion/spark/src/function/window/mod.rs
new file mode 100644
index 000000000000..b248758bc120
--- /dev/null
+++ b/datafusion/spark/src/function/window/mod.rs
@@ -0,0 +1,16 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
diff --git a/datafusion/spark/src/function/xml/mod.rs b/datafusion/spark/src/function/xml/mod.rs
new file mode 100644
index 000000000000..b248758bc120
--- /dev/null
+++ b/datafusion/spark/src/function/xml/mod.rs
@@ -0,0 +1,16 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
diff --git a/datafusion/spark/src/lib.rs b/datafusion/spark/src/lib.rs
new file mode 100644
index 000000000000..650259399315
--- /dev/null
+++ b/datafusion/spark/src/lib.rs
@@ -0,0 +1,48 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#![doc(
+    html_logo_url = "https://raw.githubusercontent.com/apache/datafusion/19fe44cf2f30cbdd63d4a4f52c74055163c6cc38/docs/logos/standalone_logo/logo_original.svg",
+    html_favicon_url = "https://raw.githubusercontent.com/apache/datafusion/19fe44cf2f30cbdd63d4a4f52c74055163c6cc38/docs/logos/standalone_logo/logo_original.svg"
+)]
+#![cfg_attr(docsrs, feature(doc_auto_cfg))]
+// Make cheap clones clear: https://github.com/apache/datafusion/issues/11143
+#![deny(clippy::clone_on_ref_ptr)]
+
+//! Spark Expression packages for [DataFusion].
+//!
+//! This crate contains a collection of various Spark expression packages for DataFusion,
+//! implemented using the extension API. Users may wish to control which functions
+//! are available to control the binary size of their application.
+//!
+//! Each package is implemented as a separate
+//! module, activated by a feature flag.
+//!
+//! [DataFusion]: https://crates.io/crates/datafusion
+//!
+//! # Available Packages
+//! See the list of [modules](#modules) in this crate for available packages.
+//!
+//! # Using A Package
+//! You can register all functions in all packages using the [`register_all`] function.
+//!
+//! Each package also exports an `expr_fn` submodule to help create [`Expr`]s that invoke
+//! functions using a fluent style. For example:
+//!
+//![`Expr`]: datafusion_expr::Expr
+
+pub mod function;

From 762ef03432b262476e82be935ff11dee1922dadb Mon Sep 17 00:00:00 2001
From: Shehab <11789402+shehabgamin@users.noreply.github.com>
Date: Wed, 12 Mar 2025 17:21:12 -0700
Subject: [PATCH 02/25] spark crate setup

---
 Cargo.lock                                    |   1 +
 datafusion/functions-aggregate/src/macros.rs  |   3 +
 datafusion/functions/src/macros.rs            |   2 +
 datafusion/spark/Cargo.toml                   |   1 +
 .../spark/src/function/aggregate/mod.rs       |   9 ++
 datafusion/spark/src/function/array/mod.rs    |   9 ++
 datafusion/spark/src/function/bitwise/mod.rs  |   9 ++
 .../spark/src/function/collection/mod.rs      |   9 ++
 .../spark/src/function/conditional/mod.rs     |   9 ++
 .../spark/src/function/conversion/mod.rs      |   9 ++
 datafusion/spark/src/function/csv/mod.rs      |   9 ++
 datafusion/spark/src/function/datetime/mod.rs |   9 ++
 .../spark/src/function/generator/mod.rs       |   9 ++
 datafusion/spark/src/function/hash/mod.rs     |   9 ++
 datafusion/spark/src/function/json/mod.rs     |   9 ++
 datafusion/spark/src/function/lambda/mod.rs   |   9 ++
 datafusion/spark/src/function/map/mod.rs      |   9 ++
 datafusion/spark/src/function/math/expm1.rs   |  30 ++++++
 datafusion/spark/src/function/math/mod.rs     |  16 +++
 datafusion/spark/src/function/misc/mod.rs     |   9 ++
 .../spark/src/function/predicate/mod.rs       |   9 ++
 datafusion/spark/src/function/string/ascii.rs |  50 ---------
 datafusion/spark/src/function/string/mod.rs   |  20 ++++
 datafusion/spark/src/function/struct/mod.rs   |   9 ++
 datafusion/spark/src/function/table/mod.rs    |   9 ++
 datafusion/spark/src/function/url/mod.rs      |   9 ++
 datafusion/spark/src/function/utils.rs        |  22 ++--
 datafusion/spark/src/function/window/mod.rs   |   9 ++
 datafusion/spark/src/function/xml/mod.rs      |   9 ++
 datafusion/spark/src/lib.rs                   | 100 ++++++++++++++++++
 30 files changed, 360 insertions(+), 65 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index cfd4a795b6d3..42522763d6a2 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2451,6 +2451,7 @@ dependencies = [
  "arrow",
  "arrow-ord",
  "criterion",
+ "datafusion-catalog",
  "datafusion-common",
  "datafusion-doc",
  "datafusion-execution",
diff --git a/datafusion/functions-aggregate/src/macros.rs b/datafusion/functions-aggregate/src/macros.rs
index b464dde6ccab..18f27c3c4ae3 100644
--- a/datafusion/functions-aggregate/src/macros.rs
+++ b/datafusion/functions-aggregate/src/macros.rs
@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#[macro_export]
 macro_rules! make_udaf_expr {
     ($EXPR_FN:ident, $($arg:ident)*, $DOC:expr, $AGGREGATE_UDF_FN:ident) => {
         // "fluent expr_fn" style function
@@ -34,6 +35,7 @@ macro_rules! make_udaf_expr {
     };
 }
 
+#[macro_export]
 macro_rules! make_udaf_expr_and_func {
     ($UDAF:ty, $EXPR_FN:ident, $($arg:ident)*, $DOC:expr, $AGGREGATE_UDF_FN:ident) => {
         make_udaf_expr!($EXPR_FN, $($arg)*, $DOC, $AGGREGATE_UDF_FN);
@@ -59,6 +61,7 @@ macro_rules! make_udaf_expr_and_func {
     };
 }
 
+#[macro_export]
 macro_rules! create_func {
     ($UDAF:ty, $AGGREGATE_UDF_FN:ident) => {
         create_func!($UDAF, $AGGREGATE_UDF_FN, <$UDAF>::default());
diff --git a/datafusion/functions/src/macros.rs b/datafusion/functions/src/macros.rs
index d2849c3abba0..30ebf8654ea0 100644
--- a/datafusion/functions/src/macros.rs
+++ b/datafusion/functions/src/macros.rs
@@ -40,6 +40,7 @@
 /// Exported functions accept:
 /// - `Vec<Expr>` argument (single argument followed by a comma)
 /// - Variable number of `Expr` arguments (zero or more arguments, must be without commas)
+#[macro_export]
 macro_rules! export_functions {
     ($(($FUNC:ident, $DOC:expr, $($arg:tt)*)),*) => {
         $(
@@ -69,6 +70,7 @@ macro_rules! export_functions {
 /// named `$NAME` which returns that singleton.
 ///
 /// This is used to ensure creating the list of `ScalarUDF` only happens once.
+#[macro_export]
 macro_rules! make_udf_function {
     ($UDF:ty, $NAME:ident) => {
         #[doc = concat!("Return a [`ScalarUDF`](datafusion_expr::ScalarUDF) implementation of ", stringify!($NAME))]
diff --git a/datafusion/spark/Cargo.toml b/datafusion/spark/Cargo.toml
index b29dc5f6c903..a40032928164 100644
--- a/datafusion/spark/Cargo.toml
+++ b/datafusion/spark/Cargo.toml
@@ -38,6 +38,7 @@ name = "datafusion_spark"
 [dependencies]
 arrow = { workspace = true }
 arrow-ord = { workspace = true }
+datafusion-catalog = { workspace = true }
 datafusion-common = { workspace = true }
 datafusion-doc = { workspace = true }
 datafusion-execution = { workspace = true }
diff --git a/datafusion/spark/src/function/aggregate/mod.rs b/datafusion/spark/src/function/aggregate/mod.rs
index b248758bc120..0856e2872d4f 100644
--- a/datafusion/spark/src/function/aggregate/mod.rs
+++ b/datafusion/spark/src/function/aggregate/mod.rs
@@ -14,3 +14,12 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
+
+use datafusion_expr::AggregateUDF;
+use std::sync::Arc;
+
+pub mod expr_fn {}
+
+pub fn functions() -> Vec<Arc<AggregateUDF>> {
+    vec![]
+}
diff --git a/datafusion/spark/src/function/array/mod.rs b/datafusion/spark/src/function/array/mod.rs
index b248758bc120..a87df9a2c87a 100644
--- a/datafusion/spark/src/function/array/mod.rs
+++ b/datafusion/spark/src/function/array/mod.rs
@@ -14,3 +14,12 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
+
+use datafusion_expr::ScalarUDF;
+use std::sync::Arc;
+
+pub mod expr_fn {}
+
+pub fn functions() -> Vec<Arc<ScalarUDF>> {
+    vec![]
+}
diff --git a/datafusion/spark/src/function/bitwise/mod.rs b/datafusion/spark/src/function/bitwise/mod.rs
index b248758bc120..a87df9a2c87a 100644
--- a/datafusion/spark/src/function/bitwise/mod.rs
+++ b/datafusion/spark/src/function/bitwise/mod.rs
@@ -14,3 +14,12 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
+
+use datafusion_expr::ScalarUDF;
+use std::sync::Arc;
+
+pub mod expr_fn {}
+
+pub fn functions() -> Vec<Arc<ScalarUDF>> {
+    vec![]
+}
diff --git a/datafusion/spark/src/function/collection/mod.rs b/datafusion/spark/src/function/collection/mod.rs
index b248758bc120..a87df9a2c87a 100644
--- a/datafusion/spark/src/function/collection/mod.rs
+++ b/datafusion/spark/src/function/collection/mod.rs
@@ -14,3 +14,12 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
+
+use datafusion_expr::ScalarUDF;
+use std::sync::Arc;
+
+pub mod expr_fn {}
+
+pub fn functions() -> Vec<Arc<ScalarUDF>> {
+    vec![]
+}
diff --git a/datafusion/spark/src/function/conditional/mod.rs b/datafusion/spark/src/function/conditional/mod.rs
index b248758bc120..a87df9a2c87a 100644
--- a/datafusion/spark/src/function/conditional/mod.rs
+++ b/datafusion/spark/src/function/conditional/mod.rs
@@ -14,3 +14,12 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
+
+use datafusion_expr::ScalarUDF;
+use std::sync::Arc;
+
+pub mod expr_fn {}
+
+pub fn functions() -> Vec<Arc<ScalarUDF>> {
+    vec![]
+}
diff --git a/datafusion/spark/src/function/conversion/mod.rs b/datafusion/spark/src/function/conversion/mod.rs
index b248758bc120..a87df9a2c87a 100644
--- a/datafusion/spark/src/function/conversion/mod.rs
+++ b/datafusion/spark/src/function/conversion/mod.rs
@@ -14,3 +14,12 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
+
+use datafusion_expr::ScalarUDF;
+use std::sync::Arc;
+
+pub mod expr_fn {}
+
+pub fn functions() -> Vec<Arc<ScalarUDF>> {
+    vec![]
+}
diff --git a/datafusion/spark/src/function/csv/mod.rs b/datafusion/spark/src/function/csv/mod.rs
index b248758bc120..a87df9a2c87a 100644
--- a/datafusion/spark/src/function/csv/mod.rs
+++ b/datafusion/spark/src/function/csv/mod.rs
@@ -14,3 +14,12 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
+
+use datafusion_expr::ScalarUDF;
+use std::sync::Arc;
+
+pub mod expr_fn {}
+
+pub fn functions() -> Vec<Arc<ScalarUDF>> {
+    vec![]
+}
diff --git a/datafusion/spark/src/function/datetime/mod.rs b/datafusion/spark/src/function/datetime/mod.rs
index b248758bc120..a87df9a2c87a 100644
--- a/datafusion/spark/src/function/datetime/mod.rs
+++ b/datafusion/spark/src/function/datetime/mod.rs
@@ -14,3 +14,12 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
+
+use datafusion_expr::ScalarUDF;
+use std::sync::Arc;
+
+pub mod expr_fn {}
+
+pub fn functions() -> Vec<Arc<ScalarUDF>> {
+    vec![]
+}
diff --git a/datafusion/spark/src/function/generator/mod.rs b/datafusion/spark/src/function/generator/mod.rs
index b248758bc120..a87df9a2c87a 100644
--- a/datafusion/spark/src/function/generator/mod.rs
+++ b/datafusion/spark/src/function/generator/mod.rs
@@ -14,3 +14,12 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
+
+use datafusion_expr::ScalarUDF;
+use std::sync::Arc;
+
+pub mod expr_fn {}
+
+pub fn functions() -> Vec<Arc<ScalarUDF>> {
+    vec![]
+}
diff --git a/datafusion/spark/src/function/hash/mod.rs b/datafusion/spark/src/function/hash/mod.rs
index b248758bc120..a87df9a2c87a 100644
--- a/datafusion/spark/src/function/hash/mod.rs
+++ b/datafusion/spark/src/function/hash/mod.rs
@@ -14,3 +14,12 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
+
+use datafusion_expr::ScalarUDF;
+use std::sync::Arc;
+
+pub mod expr_fn {}
+
+pub fn functions() -> Vec<Arc<ScalarUDF>> {
+    vec![]
+}
diff --git a/datafusion/spark/src/function/json/mod.rs b/datafusion/spark/src/function/json/mod.rs
index b248758bc120..a87df9a2c87a 100644
--- a/datafusion/spark/src/function/json/mod.rs
+++ b/datafusion/spark/src/function/json/mod.rs
@@ -14,3 +14,12 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
+
+use datafusion_expr::ScalarUDF;
+use std::sync::Arc;
+
+pub mod expr_fn {}
+
+pub fn functions() -> Vec<Arc<ScalarUDF>> {
+    vec![]
+}
diff --git a/datafusion/spark/src/function/lambda/mod.rs b/datafusion/spark/src/function/lambda/mod.rs
index b248758bc120..a87df9a2c87a 100644
--- a/datafusion/spark/src/function/lambda/mod.rs
+++ b/datafusion/spark/src/function/lambda/mod.rs
@@ -14,3 +14,12 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
+
+use datafusion_expr::ScalarUDF;
+use std::sync::Arc;
+
+pub mod expr_fn {}
+
+pub fn functions() -> Vec<Arc<ScalarUDF>> {
+    vec![]
+}
diff --git a/datafusion/spark/src/function/map/mod.rs b/datafusion/spark/src/function/map/mod.rs
index b248758bc120..a87df9a2c87a 100644
--- a/datafusion/spark/src/function/map/mod.rs
+++ b/datafusion/spark/src/function/map/mod.rs
@@ -14,3 +14,12 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
+
+use datafusion_expr::ScalarUDF;
+use std::sync::Arc;
+
+pub mod expr_fn {}
+
+pub fn functions() -> Vec<Arc<ScalarUDF>> {
+    vec![]
+}
diff --git a/datafusion/spark/src/function/math/expm1.rs b/datafusion/spark/src/function/math/expm1.rs
index 985c7bf20496..b54b8ec80fa9 100644
--- a/datafusion/spark/src/function/math/expm1.rs
+++ b/datafusion/spark/src/function/math/expm1.rs
@@ -114,3 +114,33 @@ impl ScalarUDFImpl for SparkExpm1 {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use crate::function::math::expm1::SparkExpm1;
+    use crate::function::utils::test::test_scalar_function;
+    use arrow::array::{Array, Float64Array};
+    use arrow::datatypes::DataType::Float64;
+    use datafusion_common::{Result, ScalarValue};
+    use datafusion_expr::{ColumnarValue, ScalarUDFImpl};
+
+    macro_rules! test_expm1_float64_invoke {
+        ($INPUT:expr, $EXPECTED:expr) => {
+            test_scalar_function!(
+                SparkExpm1::new(),
+                vec![ColumnarValue::Scalar(ScalarValue::Float64($INPUT))],
+                $EXPECTED,
+                f64,
+                Float64,
+                Float64Array
+            );
+        };
+    }
+
+    #[test]
+    fn test_expm1_invoke() -> Result<()> {
+        test_expm1_float64_invoke!(Some(0f64), Ok(Some(0.0f64)));
+        test_expm1_float64_invoke!(Some(1f64), Ok(Some(1.7182818284590453f64)));
+        Ok(())
+    }
+}
diff --git a/datafusion/spark/src/function/math/mod.rs b/datafusion/spark/src/function/math/mod.rs
index 13c865a3a5b3..31a40b244698 100644
--- a/datafusion/spark/src/function/math/mod.rs
+++ b/datafusion/spark/src/function/math/mod.rs
@@ -16,3 +16,19 @@
 // under the License.
 
 pub mod expm1;
+
+use datafusion_expr::ScalarUDF;
+use datafusion_functions::make_udf_function;
+use std::sync::Arc;
+
+make_udf_function!(expm1::SparkExpm1, spark_expm1);
+
+pub mod expr_fn {
+    use datafusion_functions::export_functions;
+
+    export_functions!((spark_expm1, "Returns exp(expr) - 1", arg1));
+}
+
+pub fn functions() -> Vec<Arc<ScalarUDF>> {
+    vec![spark_expm1()]
+}
diff --git a/datafusion/spark/src/function/misc/mod.rs b/datafusion/spark/src/function/misc/mod.rs
index b248758bc120..a87df9a2c87a 100644
--- a/datafusion/spark/src/function/misc/mod.rs
+++ b/datafusion/spark/src/function/misc/mod.rs
@@ -14,3 +14,12 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
+
+use datafusion_expr::ScalarUDF;
+use std::sync::Arc;
+
+pub mod expr_fn {}
+
+pub fn functions() -> Vec<Arc<ScalarUDF>> {
+    vec![]
+}
diff --git a/datafusion/spark/src/function/predicate/mod.rs b/datafusion/spark/src/function/predicate/mod.rs
index b248758bc120..a87df9a2c87a 100644
--- a/datafusion/spark/src/function/predicate/mod.rs
+++ b/datafusion/spark/src/function/predicate/mod.rs
@@ -14,3 +14,12 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
+
+use datafusion_expr::ScalarUDF;
+use std::sync::Arc;
+
+pub mod expr_fn {}
+
+pub fn functions() -> Vec<Arc<ScalarUDF>> {
+    vec![]
+}
diff --git a/datafusion/spark/src/function/string/ascii.rs b/datafusion/spark/src/function/string/ascii.rs
index 4eb46fac0c54..9f98a03646d9 100644
--- a/datafusion/spark/src/function/string/ascii.rs
+++ b/datafusion/spark/src/function/string/ascii.rs
@@ -196,50 +196,6 @@ mod tests {
         };
     }
 
-    macro_rules! test_ascii_int_invoke {
-        ($INPUT:expr, $EXPECTED:expr) => {
-            test_scalar_function!(
-                SparkAscii::new(),
-                vec![ColumnarValue::Scalar(ScalarValue::Int32($INPUT))],
-                $EXPECTED,
-                i32,
-                Int32,
-                Int32Array
-            );
-        };
-    }
-
-    macro_rules! test_ascii_binary_invoke {
-        ($INPUT:expr, $EXPECTED:expr) => {
-            test_scalar_function!(
-                SparkAscii::new(),
-                vec![ColumnarValue::Scalar(ScalarValue::Binary($INPUT))],
-                $EXPECTED,
-                i32,
-                Int32,
-                Int32Array
-            );
-
-            test_scalar_function!(
-                SparkAscii::new(),
-                vec![ColumnarValue::Scalar(ScalarValue::BinaryView($INPUT))],
-                $EXPECTED,
-                i32,
-                Int32,
-                Int32Array
-            );
-
-            test_scalar_function!(
-                SparkAscii::new(),
-                vec![ColumnarValue::Scalar(ScalarValue::LargeBinary($INPUT))],
-                $EXPECTED,
-                i32,
-                Int32,
-                Int32Array
-            );
-        };
-    }
-
     #[test]
     fn test_ascii_invoke() -> Result<()> {
         test_ascii_string_invoke!(Some(String::from("x")), Ok(Some(120)));
@@ -247,12 +203,6 @@ mod tests {
         test_ascii_string_invoke!(Some(String::from("")), Ok(Some(0)));
         test_ascii_string_invoke!(None, Ok(None));
 
-        test_ascii_int_invoke!(Some(2), Ok(Some(50)));
-        test_ascii_int_invoke!(None, Ok(None));
-
-        test_ascii_binary_invoke!(Some(b"DataFusion".to_vec()), Ok(Some(50)));
-        test_ascii_binary_invoke!(None, Ok(None));
-
         Ok(())
     }
 }
diff --git a/datafusion/spark/src/function/string/mod.rs b/datafusion/spark/src/function/string/mod.rs
index 229be18e4161..7e8e88fe8d1e 100644
--- a/datafusion/spark/src/function/string/mod.rs
+++ b/datafusion/spark/src/function/string/mod.rs
@@ -16,3 +16,23 @@
 // under the License.
 
 pub mod ascii;
+
+use datafusion_expr::ScalarUDF;
+use datafusion_functions::make_udf_function;
+use std::sync::Arc;
+
+make_udf_function!(ascii::SparkAscii, spark_ascii);
+
+pub mod expr_fn {
+    use datafusion_functions::export_functions;
+
+    export_functions!((
+        spark_ascii,
+        "Returns the ASCII code point of the first character of string.",
+        arg1
+    ));
+}
+
+pub fn functions() -> Vec<Arc<ScalarUDF>> {
+    vec![spark_ascii()]
+}
diff --git a/datafusion/spark/src/function/struct/mod.rs b/datafusion/spark/src/function/struct/mod.rs
index b248758bc120..a87df9a2c87a 100644
--- a/datafusion/spark/src/function/struct/mod.rs
+++ b/datafusion/spark/src/function/struct/mod.rs
@@ -14,3 +14,12 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
+
+use datafusion_expr::ScalarUDF;
+use std::sync::Arc;
+
+pub mod expr_fn {}
+
+pub fn functions() -> Vec<Arc<ScalarUDF>> {
+    vec![]
+}
diff --git a/datafusion/spark/src/function/table/mod.rs b/datafusion/spark/src/function/table/mod.rs
index b248758bc120..e23e96d44f6d 100644
--- a/datafusion/spark/src/function/table/mod.rs
+++ b/datafusion/spark/src/function/table/mod.rs
@@ -14,3 +14,12 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
+
+use datafusion_catalog::TableFunctionImpl;
+use std::sync::Arc;
+
+pub mod expr_fn {}
+
+pub fn functions() -> Vec<Arc<dyn TableFunctionImpl>> {
+    vec![]
+}
diff --git a/datafusion/spark/src/function/url/mod.rs b/datafusion/spark/src/function/url/mod.rs
index b248758bc120..a87df9a2c87a 100644
--- a/datafusion/spark/src/function/url/mod.rs
+++ b/datafusion/spark/src/function/url/mod.rs
@@ -14,3 +14,12 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
+
+use datafusion_expr::ScalarUDF;
+use std::sync::Arc;
+
+pub mod expr_fn {}
+
+pub fn functions() -> Vec<Arc<ScalarUDF>> {
+    vec![]
+}
diff --git a/datafusion/spark/src/function/utils.rs b/datafusion/spark/src/function/utils.rs
index 97651e87ac1e..b05e4acd20ae 100644
--- a/datafusion/spark/src/function/utils.rs
+++ b/datafusion/spark/src/function/utils.rs
@@ -15,13 +15,6 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use arrow::array::ArrayRef;
-use arrow::datatypes::DataType;
-
-use datafusion_common::{Result, ScalarValue};
-use datafusion_expr::function::Hint;
-use datafusion_expr::ColumnarValue;
-
 #[cfg(test)]
 pub mod test {
     /// $FUNC ScalarUDFImpl to test
@@ -32,27 +25,27 @@ pub mod test {
     /// $ARRAY_TYPE is the column type after function applied
     macro_rules! test_scalar_function {
         ($FUNC:expr, $ARGS:expr, $EXPECTED:expr, $EXPECTED_TYPE:ty, $EXPECTED_DATA_TYPE:expr, $ARRAY_TYPE:ident) => {
-            let expected: Result<Option<$EXPECTED_TYPE>> = $EXPECTED;
+            let expected: datafusion_common::Result<Option<$EXPECTED_TYPE>> = $EXPECTED;
             let func = $FUNC;
 
             let type_array = $ARGS.iter().map(|arg| arg.data_type()).collect::<Vec<_>>();
             let cardinality = $ARGS
                 .iter()
                 .fold(Option::<usize>::None, |acc, arg| match arg {
-                    ColumnarValue::Scalar(_) => acc,
-                    ColumnarValue::Array(a) => Some(a.len()),
+                    datafusion_expr::ColumnarValue::Scalar(_) => acc,
+                    datafusion_expr::ColumnarValue::Array(a) => Some(a.len()),
                 })
                 .unwrap_or(1);
 
             let scalar_arguments = $ARGS.iter().map(|arg| match arg {
-                ColumnarValue::Scalar(scalar) => Some(scalar.clone()),
-                ColumnarValue::Array(_) => None,
+                datafusion_expr::ColumnarValue::Scalar(scalar) => Some(scalar.clone()),
+                datafusion_expr::ColumnarValue::Array(_) => None,
             }).collect::<Vec<_>>();
             let scalar_arguments_refs = scalar_arguments.iter().map(|arg| arg.as_ref()).collect::<Vec<_>>();
 
             let nullables = $ARGS.iter().map(|arg| match arg {
-                ColumnarValue::Scalar(scalar) => scalar.is_null(),
-                ColumnarValue::Array(a) => a.null_count() > 0,
+                datafusion_expr::ColumnarValue::Scalar(scalar) => scalar.is_null(),
+                datafusion_expr::ColumnarValue::Array(a) => a.null_count() > 0,
             }).collect::<Vec<_>>();
 
             let return_info = func.return_type_from_args(datafusion_expr::ReturnTypeArgs {
@@ -103,6 +96,5 @@ pub mod test {
         };
     }
 
-    use super::*;
     pub(crate) use test_scalar_function;
 }
diff --git a/datafusion/spark/src/function/window/mod.rs b/datafusion/spark/src/function/window/mod.rs
index b248758bc120..97ab4a9e3542 100644
--- a/datafusion/spark/src/function/window/mod.rs
+++ b/datafusion/spark/src/function/window/mod.rs
@@ -14,3 +14,12 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
+
+use datafusion_expr::WindowUDF;
+use std::sync::Arc;
+
+pub mod expr_fn {}
+
+pub fn functions() -> Vec<Arc<WindowUDF>> {
+    vec![]
+}
diff --git a/datafusion/spark/src/function/xml/mod.rs b/datafusion/spark/src/function/xml/mod.rs
index b248758bc120..a87df9a2c87a 100644
--- a/datafusion/spark/src/function/xml/mod.rs
+++ b/datafusion/spark/src/function/xml/mod.rs
@@ -14,3 +14,12 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
+
+use datafusion_expr::ScalarUDF;
+use std::sync::Arc;
+
+pub mod expr_fn {}
+
+pub fn functions() -> Vec<Arc<ScalarUDF>> {
+    vec![]
+}
diff --git a/datafusion/spark/src/lib.rs b/datafusion/spark/src/lib.rs
index 650259399315..0cf627e72f23 100644
--- a/datafusion/spark/src/lib.rs
+++ b/datafusion/spark/src/lib.rs
@@ -46,3 +46,103 @@
 //![`Expr`]: datafusion_expr::Expr
 
 pub mod function;
+
+use datafusion_common::Result;
+use datafusion_execution::FunctionRegistry;
+use datafusion_expr::{AggregateUDF, ScalarUDF, WindowUDF};
+use log::debug;
+use std::sync::Arc;
+
+/// Fluent-style API for creating `Expr`s
+#[allow(unused)]
+pub mod expr_fn {
+    pub use super::function::aggregate::expr_fn::*;
+    pub use super::function::array::expr_fn::*;
+    pub use super::function::bitwise::expr_fn::*;
+    pub use super::function::collection::expr_fn::*;
+    pub use super::function::conditional::expr_fn::*;
+    pub use super::function::conversion::expr_fn::*;
+    pub use super::function::csv::expr_fn::*;
+    pub use super::function::datetime::expr_fn::*;
+    pub use super::function::generator::expr_fn::*;
+    pub use super::function::hash::expr_fn::*;
+    pub use super::function::json::expr_fn::*;
+    pub use super::function::lambda::expr_fn::*;
+    pub use super::function::map::expr_fn::*;
+    pub use super::function::math::expr_fn::*;
+    pub use super::function::misc::expr_fn::*;
+    pub use super::function::predicate::expr_fn::*;
+    pub use super::function::r#struct::expr_fn::*;
+    pub use super::function::string::expr_fn::*;
+    pub use super::function::table::expr_fn::*;
+    pub use super::function::url::expr_fn::*;
+    pub use super::function::window::expr_fn::*;
+    pub use super::function::xml::expr_fn::*;
+}
+
+/// Returns all default scalar functions
+pub fn all_default_scalar_functions() -> Vec<Arc<ScalarUDF>> {
+    function::array::functions()
+        .into_iter()
+        .chain(function::bitwise::functions())
+        .chain(function::collection::functions())
+        .chain(function::conditional::functions())
+        .chain(function::conversion::functions())
+        .chain(function::csv::functions())
+        .chain(function::datetime::functions())
+        .chain(function::generator::functions())
+        .chain(function::hash::functions())
+        .chain(function::json::functions())
+        .chain(function::lambda::functions())
+        .chain(function::map::functions())
+        .chain(function::math::functions())
+        .chain(function::misc::functions())
+        .chain(function::predicate::functions())
+        .chain(function::string::functions())
+        .chain(function::r#struct::functions())
+        .chain(function::url::functions())
+        .chain(function::xml::functions())
+        .collect::<Vec<_>>()
+}
+
+/// Returns all default aggregate functions
+pub fn all_default_aggregate_functions() -> Vec<Arc<AggregateUDF>> {
+    function::aggregate::functions()
+}
+
+/// Returns all default window functions
+pub fn all_default_window_functions() -> Vec<Arc<WindowUDF>> {
+    function::window::functions()
+}
+
+/// Registers all enabled packages with a [`FunctionRegistry`]
+pub fn register_all(registry: &mut dyn FunctionRegistry) -> Result<()> {
+    let scalar_functions: Vec<Arc<ScalarUDF>> = all_default_scalar_functions();
+    scalar_functions.into_iter().try_for_each(|udf| {
+        let existing_udf = registry.register_udf(udf)?;
+        if let Some(existing_udf) = existing_udf {
+            debug!("Overwrite existing UDF: {}", existing_udf.name());
+        }
+        Ok(()) as Result<()>
+    })?;
+
+    let aggregate_functions: Vec<Arc<AggregateUDF>> = all_default_aggregate_functions();
+    aggregate_functions.into_iter().try_for_each(|udf| {
+        let existing_udaf = registry.register_udaf(udf)?;
+        if let Some(existing_udaf) = existing_udaf {
+            debug!("Overwrite existing UDAF: {}", existing_udaf.name());
+        }
+        Ok(()) as Result<()>
+    })?;
+
+    let window_functions: Vec<Arc<WindowUDF>> = all_default_window_functions();
+    window_functions.into_iter().try_for_each(|udf| {
+        let existing_udwf = registry.register_udwf(udf)?;
+        if let Some(existing_udwf) = existing_udwf {
+            debug!("Overwrite existing UDWF: {}", existing_udwf.name());
+        }
+        Ok(()) as Result<()>
+    })?;
+
+    Ok(())
+}

From 6e1aa06f9c00fe60b67736f9473812120ef4befd Mon Sep 17 00:00:00 2001
From: Shehab <11789402+shehabgamin@users.noreply.github.com>
Date: Wed, 12 Mar 2025 17:30:36 -0700
Subject: [PATCH 03/25] clean up 2 example functions

---
 datafusion/spark/src/function/math/expm1.rs   | 31 ++++++++++++++++---
 datafusion/spark/src/function/math/mod.rs     |  2 +-
 datafusion/spark/src/function/string/ascii.rs |  2 +-
 3 files changed, 29 insertions(+), 6 deletions(-)

diff --git a/datafusion/spark/src/function/math/expm1.rs b/datafusion/spark/src/function/math/expm1.rs
index b54b8ec80fa9..fd96d76f1190 100644
--- a/datafusion/spark/src/function/math/expm1.rs
+++ b/datafusion/spark/src/function/math/expm1.rs
@@ -15,19 +15,42 @@
 // specific language governing permissions and limitations
 // under the License.
 
+use crate::function::error_utils::{
+    invalid_arg_count_exec_err, unsupported_data_type_exec_err,
+};
 use arrow::array::{ArrayRef, AsArray};
 use arrow::datatypes::{DataType, Float64Type};
 use datafusion_common::{Result, ScalarValue};
 use datafusion_expr::{
     ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,
 };
+use datafusion_macros::user_doc;
 use std::any::Any;
 use std::sync::Arc;
 
-use crate::function::error_utils::{
-    invalid_arg_count_exec_err, unsupported_data_type_exec_err,
-};
-
+#[user_doc(
+    doc_section(label = "Spark Math Functions"),
+    description = "Returns exp(expr) - 1 as a Float64.",
+    syntax_example = "expm1(expr)",
+    sql_example = r#"```sql
+> select expm1(0);
++--------------------+
+| expm1(0)           |
++--------------------+
+| 0.0                |
++--------------------+
+> select expm1(1);
++----------------+
+| expm1(1)       |
++----------------+
+| 50             |
++----------------+
+```"#,
+    argument(
+        name = "expr",
+        description = "An expression that evaluates to a numeric."
+    )
+)]
 #[derive(Debug)]
 pub struct SparkExpm1 {
     signature: Signature,
diff --git a/datafusion/spark/src/function/math/mod.rs b/datafusion/spark/src/function/math/mod.rs
index 31a40b244698..ee48bafb8e0c 100644
--- a/datafusion/spark/src/function/math/mod.rs
+++ b/datafusion/spark/src/function/math/mod.rs
@@ -26,7 +26,7 @@ make_udf_function!(expm1::SparkExpm1, spark_expm1);
 pub mod expr_fn {
     use datafusion_functions::export_functions;
 
-    export_functions!((spark_expm1, "Returns exp(expr) - 1", arg1));
+    export_functions!((spark_expm1, "Returns exp(expr) - 1 as a Float64.", arg1));
 }
 
 pub fn functions() -> Vec<Arc<ScalarUDF>> {
diff --git a/datafusion/spark/src/function/string/ascii.rs b/datafusion/spark/src/function/string/ascii.rs
index 9f98a03646d9..482292b0dfe5 100644
--- a/datafusion/spark/src/function/string/ascii.rs
+++ b/datafusion/spark/src/function/string/ascii.rs
@@ -27,7 +27,7 @@ use std::any::Any;
 use std::sync::Arc;
 
 #[user_doc(
-    doc_section(label = "String Functions"),
+    doc_section(label = "Spark String Functions"),
     description = "Returns the ASCII code point of the first character of string.",
     syntax_example = "ascii(str)",
     sql_example = r#"```sql

From eba0a514ba7f18525321f578dc5c8090f9052fae Mon Sep 17 00:00:00 2001
From: Shehab <11789402+shehabgamin@users.noreply.github.com>
Date: Wed, 12 Mar 2025 19:03:45 -0700
Subject: [PATCH 04/25] cleanup crate

---
 Cargo.lock                                    |  1 +
 Cargo.toml                                    |  1 +
 datafusion/core/Cargo.toml                    |  2 +
 .../src/execution/session_state_defaults.rs   | 43 ++++++++++++--
 datafusion/spark/README.md                    | 11 ++++
 datafusion/spark/src/function/table/mod.rs    |  4 +-
 datafusion/spark/src/lib.rs                   |  6 ++
 datafusion/sqllogictest/Cargo.toml            |  2 +-
 .../sqllogictest/test_files/spark/README.md   | 57 +++++++++++++++++++
 .../test_files/spark/math/expm1.slt           | 32 +++++++++++
 .../test_files/spark/string/ascii.slt         | 43 ++++++++++++++
 parquet-testing                               |  2 +-
 12 files changed, 196 insertions(+), 8 deletions(-)
 create mode 100644 datafusion/sqllogictest/test_files/spark/README.md
 create mode 100644 datafusion/sqllogictest/test_files/spark/math/expm1.slt
 create mode 100644 datafusion/sqllogictest/test_files/spark/string/ascii.slt

diff --git a/Cargo.lock b/Cargo.lock
index 42522763d6a2..84eb45324f88 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1775,6 +1775,7 @@ dependencies = [
  "datafusion-physical-expr-common",
  "datafusion-physical-optimizer",
  "datafusion-physical-plan",
+ "datafusion-spark",
  "datafusion-sql",
  "doc-comment",
  "env_logger",
diff --git a/Cargo.toml b/Cargo.toml
index 2fc392a8916d..3492b3655dd4 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -138,6 +138,7 @@ datafusion-physical-plan = { path = "datafusion/physical-plan", version = "46.0.
 datafusion-proto = { path = "datafusion/proto", version = "46.0.0" }
 datafusion-proto-common = { path = "datafusion/proto-common", version = "46.0.0" }
 datafusion-sql = { path = "datafusion/sql", version = "46.0.0" }
+datafusion-spark = { path = "datafusion/spark", version = "46.0.0" }
 doc-comment = "0.3"
 env_logger = "0.11"
 futures = "0.3"
diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml
index fd1fd4164da0..d6bfa37c7746 100644
--- a/datafusion/core/Cargo.toml
+++ b/datafusion/core/Cargo.toml
@@ -85,6 +85,7 @@ serde = [
     # statements in `arrow-schema` crate
     "arrow-schema/serde",
 ]
+spark = ["datafusion-spark"]
 string_expressions = ["datafusion-functions/string_expressions"]
 unicode_expressions = [
     "datafusion-sql/unicode_expressions",
@@ -124,6 +125,7 @@ datafusion-physical-expr-common = { workspace = true }
 datafusion-physical-optimizer = { workspace = true }
 datafusion-physical-plan = { workspace = true }
 datafusion-sql = { workspace = true }
+datafusion-spark = { workspace = true, optional = true }
 flate2 = { version = "1.1.0", optional = true }
 futures = { workspace = true }
 itertools = { workspace = true }
diff --git a/datafusion/core/src/execution/session_state_defaults.rs b/datafusion/core/src/execution/session_state_defaults.rs
index b48ef90f2bd5..2c2aa2a227df 100644
--- a/datafusion/core/src/execution/session_state_defaults.rs
+++ b/datafusion/core/src/execution/session_state_defaults.rs
@@ -104,28 +104,55 @@ impl SessionStateDefaults {
 
     /// returns the list of default [`ScalarUDF']'s
     pub fn default_scalar_functions() -> Vec<Arc<ScalarUDF>> {
-        #[cfg_attr(not(feature = "nested_expressions"), allow(unused_mut))]
+        #[cfg_attr(
+            not(any(feature = "nested_expressions", feature = "spark")),
+            allow(unused_mut)
+        )]
         let mut functions: Vec<Arc<ScalarUDF>> = functions::all_default_functions();
 
         #[cfg(feature = "nested_expressions")]
         functions.append(&mut functions_nested::all_default_nested_functions());
 
+        #[cfg(feature = "spark")]
+        functions.append(&mut datafusion_spark::all_default_scalar_functions());
+
         functions
     }
 
     /// returns the list of default [`AggregateUDF']'s
     pub fn default_aggregate_functions() -> Vec<Arc<AggregateUDF>> {
-        functions_aggregate::all_default_aggregate_functions()
+        #[cfg_attr(not(feature = "spark"), allow(unused_mut))]
+        let mut functions: Vec<Arc<AggregateUDF>> =
+            functions_aggregate::all_default_aggregate_functions();
+
+        #[cfg(feature = "spark")]
+        functions.append(&mut datafusion_spark::all_default_aggregate_functions());
+
+        functions
     }
 
     /// returns the list of default [`WindowUDF']'s
     pub fn default_window_functions() -> Vec<Arc<WindowUDF>> {
-        functions_window::all_default_window_functions()
+        #[cfg_attr(not(feature = "spark"), allow(unused_mut))]
+        let mut functions: Vec<Arc<WindowUDF>> =
+            functions_window::all_default_window_functions();
+
+        #[cfg(feature = "spark")]
+        functions.append(&mut datafusion_spark::all_default_window_functions());
+
+        functions
     }
 
     /// returns the list of default [`TableFunction`]s
     pub fn default_table_functions() -> Vec<Arc<TableFunction>> {
-        functions_table::all_default_table_functions()
+        #[cfg_attr(not(feature = "spark"), allow(unused_mut))]
+        let mut functions: Vec<Arc<TableFunction>> =
+            functions_table::all_default_table_functions();
+
+        #[cfg(feature = "spark")]
+        functions.append(&mut datafusion_spark::all_default_table_functions());
+
+        functions
     }
 
     /// returns the list of default [`FileFormatFactory']'s
@@ -148,6 +175,7 @@ impl SessionStateDefaults {
         Self::register_scalar_functions(state);
         Self::register_array_functions(state);
         Self::register_aggregate_functions(state);
+        Self::register_spark_functions(state);
     }
 
     /// registers all the builtin scalar functions
@@ -170,6 +198,13 @@ impl SessionStateDefaults {
             .expect("can not register aggregate functions");
     }
 
+    /// registers all the builtin Spark functions
+    #[cfg_attr(not(feature = "spark"), allow(unused_variables))]
+    pub fn register_spark_functions(state: &mut SessionState) {
+        #[cfg(feature = "spark")]
+        datafusion_spark::register_all(state).expect("can not register Spark functions");
+    }
+
     /// registers the default schema
     pub fn register_default_schema(
         config: &SessionConfig,
diff --git a/datafusion/spark/README.md b/datafusion/spark/README.md
index afd94d2c0690..9d453be5eb0e 100644
--- a/datafusion/spark/README.md
+++ b/datafusion/spark/README.md
@@ -20,3 +20,14 @@ under the License.
 # datafusion-spark: Spark-compatible Expressions
 
 This crate provides Apache Spark-compatible expressions for use with DataFusion.
+
+## Testing Guide
+
+When testing functions by directly invoking them (e.g., `test_scalar_function!()`), input coercion (from the `signature`
+or `coerce_types`) is not applied.
+
+Therefore, direct invocation tests should only be used to verify that the function is correctly implemented.
+
+Please be sure to add additional tests beyond direct invocation.
+For more detailed testing guidelines, refer to
+the [Spark SQLLogicTest README](../sqllogictest/test_files/spark/README.md).
diff --git a/datafusion/spark/src/function/table/mod.rs b/datafusion/spark/src/function/table/mod.rs
index e23e96d44f6d..aba7b7ceb78e 100644
--- a/datafusion/spark/src/function/table/mod.rs
+++ b/datafusion/spark/src/function/table/mod.rs
@@ -15,11 +15,11 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use datafusion_catalog::TableFunctionImpl;
+use datafusion_catalog::TableFunction;
 use std::sync::Arc;
 
 pub mod expr_fn {}
 
-pub fn functions() -> Vec<Arc<dyn TableFunctionImpl>> {
+pub fn functions() -> Vec<Arc<TableFunction>> {
     vec![]
 }
diff --git a/datafusion/spark/src/lib.rs b/datafusion/spark/src/lib.rs
index 0cf627e72f23..de50fd1551e2 100644
--- a/datafusion/spark/src/lib.rs
+++ b/datafusion/spark/src/lib.rs
@@ -47,6 +47,7 @@
 
 pub mod function;
 
+use datafusion_catalog::TableFunction;
 use datafusion_common::Result;
 use datafusion_execution::FunctionRegistry;
 use datafusion_expr::{AggregateUDF, ScalarUDF, WindowUDF};
@@ -115,6 +116,11 @@ pub fn all_default_window_functions() -> Vec<Arc<WindowUDF>> {
     function::window::functions()
 }
 
+/// Returns all default table functions
+pub fn all_default_table_functions() -> Vec<Arc<TableFunction>> {
+    function::table::functions()
+}
+
 /// Registers all enabled packages with a [`FunctionRegistry`]
 pub fn register_all(registry: &mut dyn FunctionRegistry) -> Result<()> {
     let scalar_functions: Vec<Arc<ScalarUDF>> = all_default_scalar_functions();
diff --git a/datafusion/sqllogictest/Cargo.toml b/datafusion/sqllogictest/Cargo.toml
index 5742f583acf7..8b7ce4671ba4 100644
--- a/datafusion/sqllogictest/Cargo.toml
+++ b/datafusion/sqllogictest/Cargo.toml
@@ -43,7 +43,7 @@ bigdecimal = { workspace = true }
 bytes = { workspace = true, optional = true }
 chrono = { workspace = true, optional = true }
 clap = { version = "4.5.31", features = ["derive", "env"] }
-datafusion = { workspace = true, default-features = true, features = ["avro"] }
+datafusion = { workspace = true, default-features = true, features = ["avro", "spark"] }
 futures = { workspace = true }
 half = { workspace = true, default-features = true }
 indicatif = "0.17"
diff --git a/datafusion/sqllogictest/test_files/spark/README.md b/datafusion/sqllogictest/test_files/spark/README.md
new file mode 100644
index 000000000000..0599b1a825ab
--- /dev/null
+++ b/datafusion/sqllogictest/test_files/spark/README.md
@@ -0,0 +1,57 @@
+<!---
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+# Spark Test Files
+
+This directory contains test files for the `spark` test suite.
+
+## Testing Guide
+
+When testing Spark functions:
+
+- Functions must be tested on both `Scalar` and `Array` inputs
+- Test cases should only contain `SELECT` statements with the function being tested
+- Add explicit casts to input values to ensure the correct data type is used (e.g., `0::INT`)
+    - Explicit casting is necessary because DataFusion and Spark do not infer data types in the same way
+
+### Finding Test Cases
+
+To verify and compare function behavior at a minimum, you can refer to the following documentation sources:
+
+1. Databricks SQL Function Reference:
+   https://docs.databricks.com/aws/en/sql/language-manual/functions/NAME
+2. Apache Spark SQL Function Reference:
+   https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.NAME.html
+3. PySpark SQL Function Reference:
+   https://spark.apache.org/docs/latest/api/sql/#NAME
+
+**Note:** Replace `NAME` in each URL with the actual function name (e.g., for the `ASCII` function, use `ascii` instead
+of `NAME`).
+
+### Scalar Example:
+
+```sql
+SELECT spark_expm1(0::INT);
+```
+
+### Array Example:
+
+```sql
+SELECT spark_expm1(a) FROM (VALUES (0::INT), (1::INT)) AS t(a);
+```
diff --git a/datafusion/sqllogictest/test_files/spark/math/expm1.slt b/datafusion/sqllogictest/test_files/spark/math/expm1.slt
new file mode 100644
index 000000000000..9455b7461f78
--- /dev/null
+++ b/datafusion/sqllogictest/test_files/spark/math/expm1.slt
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+
+#   http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+query R
+SELECT spark_expm1(0::INT);
+----
+0
+
+query R
+SELECT spark_expm1(1::INT);
+----
+1.718281828459
+
+query R
+SELECT spark_expm1(a) FROM (VALUES (0::INT), (1::INT)) AS t(a);
+----
+0
+1.718281828459
diff --git a/datafusion/sqllogictest/test_files/spark/string/ascii.slt b/datafusion/sqllogictest/test_files/spark/string/ascii.slt
new file mode 100644
index 000000000000..de434a6d15b6
--- /dev/null
+++ b/datafusion/sqllogictest/test_files/spark/string/ascii.slt
@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+
+#   http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+query I
+SELECT spark_ascii('234');
+----
+50
+
+query I
+SELECT spark_ascii('');
+----
+0
+
+query I
+SELECT spark_ascii('222');
+----
+50
+
+query I
+SELECT spark_ascii(2::INT);
+----
+50
+
+query I
+SELECT spark_ascii(a) FROM (VALUES ('Spark'), ('PySpark'), ('Pandas API')) AS t(a);
+----
+83
+80
+80
diff --git a/parquet-testing b/parquet-testing
index f4d7ed772a62..39b91cf85306 160000
--- a/parquet-testing
+++ b/parquet-testing
@@ -1 +1 @@
-Subproject commit f4d7ed772a62a95111db50fbcad2460833e8c882
+Subproject commit 39b91cf853062d92f0d20581d37b20dabe70a6a0

From 9ca4a52ac7a48530cea6827a157be6efc8f562dc Mon Sep 17 00:00:00 2001
From: Shehab <11789402+shehabgamin@users.noreply.github.com>
Date: Wed, 12 Mar 2025 19:14:48 -0700
Subject: [PATCH 05/25] Spark crate setup

---
 Cargo.toml                 | 2 +-
 datafusion/core/Cargo.toml | 2 +-
 datafusion/spark/README.md | 5 +++++
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 3492b3655dd4..dd7d46166cb9 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -137,8 +137,8 @@ datafusion-physical-optimizer = { path = "datafusion/physical-optimizer", versio
 datafusion-physical-plan = { path = "datafusion/physical-plan", version = "46.0.0" }
 datafusion-proto = { path = "datafusion/proto", version = "46.0.0" }
 datafusion-proto-common = { path = "datafusion/proto-common", version = "46.0.0" }
-datafusion-sql = { path = "datafusion/sql", version = "46.0.0" }
 datafusion-spark = { path = "datafusion/spark", version = "46.0.0" }
+datafusion-sql = { path = "datafusion/sql", version = "46.0.0" }
 doc-comment = "0.3"
 env_logger = "0.11"
 futures = "0.3"
diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml
index d6bfa37c7746..ab7795a93b15 100644
--- a/datafusion/core/Cargo.toml
+++ b/datafusion/core/Cargo.toml
@@ -124,8 +124,8 @@ datafusion-physical-expr = { workspace = true }
 datafusion-physical-expr-common = { workspace = true }
 datafusion-physical-optimizer = { workspace = true }
 datafusion-physical-plan = { workspace = true }
-datafusion-sql = { workspace = true }
 datafusion-spark = { workspace = true, optional = true }
+datafusion-sql = { workspace = true }
 flate2 = { version = "1.1.0", optional = true }
 futures = { workspace = true }
 itertools = { workspace = true }
diff --git a/datafusion/spark/README.md b/datafusion/spark/README.md
index 9d453be5eb0e..fe550123bf57 100644
--- a/datafusion/spark/README.md
+++ b/datafusion/spark/README.md
@@ -31,3 +31,8 @@ Therefore, direct invocation tests should only be used to verify that the functi
 Please be sure to add additional tests beyond direct invocation.
 For more detailed testing guidelines, refer to
 the [Spark SQLLogicTest README](../sqllogictest/test_files/spark/README.md).
+
+## Implementation References
+
+When implementing Spark-compatible functions, you can check if there are existing implementations in
+the [Sail](https://github.com/lakehq/sail) or [Comet](https://github.com/apache/datafusion-comet) projects first.

From 074fc70f118d95405d23bf7e026fbcd2792180d8 Mon Sep 17 00:00:00 2001
From: Shehab <11789402+shehabgamin@users.noreply.github.com>
Date: Wed, 12 Mar 2025 19:21:01 -0700
Subject: [PATCH 06/25] fix lint issue

---
 datafusion/sqllogictest/test_files/spark/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datafusion/sqllogictest/test_files/spark/README.md b/datafusion/sqllogictest/test_files/spark/README.md
index 0599b1a825ab..5e78cc721ece 100644
--- a/datafusion/sqllogictest/test_files/spark/README.md
+++ b/datafusion/sqllogictest/test_files/spark/README.md
@@ -28,7 +28,7 @@ When testing Spark functions:
 - Functions must be tested on both `Scalar` and `Array` inputs
 - Test cases should only contain `SELECT` statements with the function being tested
 - Add explicit casts to input values to ensure the correct data type is used (e.g., `0::INT`)
-    - Explicit casting is necessary because DataFusion and Spark do not infer data types in the same way
+  - Explicit casting is necessary because DataFusion and Spark do not infer data types in the same way
 
 ### Finding Test Cases
 

From 4e283100b7bac332c5e739f5262c3b64a46e69e4 Mon Sep 17 00:00:00 2001
From: Shehab <11789402+shehabgamin@users.noreply.github.com>
Date: Wed, 12 Mar 2025 19:27:53 -0700
Subject: [PATCH 07/25] cargo cleanup

---
 Cargo.lock                  | 6 ------
 datafusion/spark/Cargo.toml | 8 --------
 2 files changed, 14 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 84eb45324f88..d4c538aa9115 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2450,14 +2450,11 @@ name = "datafusion-spark"
 version = "46.0.0"
 dependencies = [
  "arrow",
- "arrow-ord",
- "criterion",
  "datafusion-catalog",
  "datafusion-common",
  "datafusion-doc",
  "datafusion-execution",
  "datafusion-expr",
- "datafusion-expr-common",
  "datafusion-functions",
  "datafusion-functions-aggregate",
  "datafusion-functions-aggregate-common",
@@ -2466,10 +2463,7 @@ dependencies = [
  "datafusion-functions-window",
  "datafusion-functions-window-common",
  "datafusion-macros",
- "datafusion-physical-expr-common",
- "itertools 0.14.0",
  "log",
- "rand 0.8.5",
 ]
 
 [[package]]
diff --git a/datafusion/spark/Cargo.toml b/datafusion/spark/Cargo.toml
index a40032928164..bea80cc6aa09 100644
--- a/datafusion/spark/Cargo.toml
+++ b/datafusion/spark/Cargo.toml
@@ -37,13 +37,11 @@ name = "datafusion_spark"
 
 [dependencies]
 arrow = { workspace = true }
-arrow-ord = { workspace = true }
 datafusion-catalog = { workspace = true }
 datafusion-common = { workspace = true }
 datafusion-doc = { workspace = true }
 datafusion-execution = { workspace = true }
 datafusion-expr = { workspace = true }
-datafusion-expr-common = { workspace = true }
 datafusion-functions = { workspace = true }
 datafusion-functions-aggregate = { workspace = true }
 datafusion-functions-aggregate-common = { workspace = true }
@@ -52,10 +50,4 @@ datafusion-functions-table = { workspace = true }
 datafusion-functions-window = { workspace = true }
 datafusion-functions-window-common = { workspace = true }
 datafusion-macros = { workspace = true }
-datafusion-physical-expr-common = { workspace = true }
-itertools = { workspace = true, features = ["use_std"] }
 log = { workspace = true }
-
-[dev-dependencies]
-criterion = { workspace = true, features = ["async_tokio"] }
-rand = { workspace = true }

From 7f448f3aa2bbcfbb482b1d1753f62d4a9c61984f Mon Sep 17 00:00:00 2001
From: Shehab <11789402+shehabgamin@users.noreply.github.com>
Date: Wed, 12 Mar 2025 19:42:25 -0700
Subject: [PATCH 08/25] fix collision in sqllogic

---
 datafusion/spark/src/function/math/expm1.rs   | 4 ++--
 datafusion/spark/src/function/string/ascii.rs | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/datafusion/spark/src/function/math/expm1.rs b/datafusion/spark/src/function/math/expm1.rs
index fd96d76f1190..2bfb2e70a33c 100644
--- a/datafusion/spark/src/function/math/expm1.rs
+++ b/datafusion/spark/src/function/math/expm1.rs
@@ -67,7 +67,7 @@ impl SparkExpm1 {
     pub fn new() -> Self {
         Self {
             signature: Signature::user_defined(Volatility::Immutable),
-            aliases: vec!["spark_expm1".to_string()],
+            aliases: vec![],
         }
     }
 }
@@ -78,7 +78,7 @@ impl ScalarUDFImpl for SparkExpm1 {
     }
 
     fn name(&self) -> &str {
-        "expm1"
+        "spark_expm1"
     }
 
     fn signature(&self) -> &Signature {
diff --git a/datafusion/spark/src/function/string/ascii.rs b/datafusion/spark/src/function/string/ascii.rs
index 482292b0dfe5..3a9c8dd1769d 100644
--- a/datafusion/spark/src/function/string/ascii.rs
+++ b/datafusion/spark/src/function/string/ascii.rs
@@ -74,7 +74,7 @@ impl SparkAscii {
     pub fn new() -> Self {
         Self {
             signature: Signature::user_defined(Volatility::Immutable),
-            aliases: vec!["spark_ascii".to_string()],
+            aliases: vec![],
         }
     }
 }
@@ -85,7 +85,7 @@ impl ScalarUDFImpl for SparkAscii {
     }
 
     fn name(&self) -> &str {
-        "ascii"
+        "spark_ascii"
     }
 
     fn signature(&self) -> &Signature {

From c29a9e421c5d51360183db2e3ebb37bedb28b575 Mon Sep 17 00:00:00 2001
From: Shehab <11789402+shehabgamin@users.noreply.github.com>
Date: Wed, 12 Mar 2025 22:34:19 -0700
Subject: [PATCH 09/25] remove redundant test

---
 datafusion/spark/src/function/math/expm1.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/datafusion/spark/src/function/math/expm1.rs b/datafusion/spark/src/function/math/expm1.rs
index 2bfb2e70a33c..d648ae1f2e7b 100644
--- a/datafusion/spark/src/function/math/expm1.rs
+++ b/datafusion/spark/src/function/math/expm1.rs
@@ -163,7 +163,6 @@ mod tests {
     #[test]
     fn test_expm1_invoke() -> Result<()> {
         test_expm1_float64_invoke!(Some(0f64), Ok(Some(0.0f64)));
-        test_expm1_float64_invoke!(Some(1f64), Ok(Some(1.7182818284590453f64)));
         Ok(())
     }
 }

From b2e9135cceb946ea4582b0c5088755424d8ed60e Mon Sep 17 00:00:00 2001
From: Shehab <11789402+shehabgamin@users.noreply.github.com>
Date: Thu, 13 Mar 2025 22:47:26 -0700
Subject: [PATCH 10/25] test float precision when casting to string

---
 datafusion/sqllogictest/test_files/spark/math/expm1.slt | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/datafusion/sqllogictest/test_files/spark/math/expm1.slt b/datafusion/sqllogictest/test_files/spark/math/expm1.slt
index 9455b7461f78..6b7ffd7fa3cb 100644
--- a/datafusion/sqllogictest/test_files/spark/math/expm1.slt
+++ b/datafusion/sqllogictest/test_files/spark/math/expm1.slt
@@ -30,3 +30,8 @@ SELECT spark_expm1(a) FROM (VALUES (0::INT), (1::INT)) AS t(a);
 ----
 0
 1.718281828459
+
+query T
+SELECT spark_expm1(1::INT)::STRING;
+----
+1.7182818284590453

From 4fedbf4e3d4a7798c9c1bcac786ece35f898f333 Mon Sep 17 00:00:00 2001
From: Shehab <11789402+shehabgamin@users.noreply.github.com>
Date: Thu, 13 Mar 2025 22:48:27 -0700
Subject: [PATCH 11/25] reorder

---
 .../sqllogictest/test_files/spark/math/expm1.slt       | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/datafusion/sqllogictest/test_files/spark/math/expm1.slt b/datafusion/sqllogictest/test_files/spark/math/expm1.slt
index 6b7ffd7fa3cb..f39487975552 100644
--- a/datafusion/sqllogictest/test_files/spark/math/expm1.slt
+++ b/datafusion/sqllogictest/test_files/spark/math/expm1.slt
@@ -25,13 +25,13 @@ SELECT spark_expm1(1::INT);
 ----
 1.718281828459
 
+query T
+SELECT spark_expm1(1::INT)::STRING;
+----
+1.7182818284590453
+
 query R
 SELECT spark_expm1(a) FROM (VALUES (0::INT), (1::INT)) AS t(a);
 ----
 0
 1.718281828459
-
-query T
-SELECT spark_expm1(1::INT)::STRING;
-----
-1.7182818284590453

From 94829647efda1324d7afe508a2867b6d3f9c646b Mon Sep 17 00:00:00 2001
From: Shehab <11789402+shehabgamin@users.noreply.github.com>
Date: Thu, 13 Mar 2025 23:14:00 -0700
Subject: [PATCH 12/25] undo

---
 datafusion/sqllogictest/test_files/spark/math/expm1.slt | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/datafusion/sqllogictest/test_files/spark/math/expm1.slt b/datafusion/sqllogictest/test_files/spark/math/expm1.slt
index f39487975552..9455b7461f78 100644
--- a/datafusion/sqllogictest/test_files/spark/math/expm1.slt
+++ b/datafusion/sqllogictest/test_files/spark/math/expm1.slt
@@ -25,11 +25,6 @@ SELECT spark_expm1(1::INT);
 ----
 1.718281828459
 
-query T
-SELECT spark_expm1(1::INT)::STRING;
-----
-1.7182818284590453
-
 query R
 SELECT spark_expm1(a) FROM (VALUES (0::INT), (1::INT)) AS t(a);
 ----

From 4964b0fb235ee02c974d258baedf5ad96020a245 Mon Sep 17 00:00:00 2001
From: Shehab <11789402+shehabgamin@users.noreply.github.com>
Date: Tue, 22 Apr 2025 20:54:13 -0700
Subject: [PATCH 13/25] save

---
 Cargo.lock                         | 22 ----------------------
 Cargo.toml                         |  2 --
 datafusion/core/Cargo.toml         |  2 --
 datafusion/sqllogictest/Cargo.toml |  2 +-
 parquet-testing                    |  1 -
 5 files changed, 1 insertion(+), 28 deletions(-)
 delete mode 160000 parquet-testing

diff --git a/Cargo.lock b/Cargo.lock
index a50c0ca4f420..e772b2466c34 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1774,7 +1774,6 @@ dependencies = [
  "datafusion-physical-expr-common",
  "datafusion-physical-optimizer",
  "datafusion-physical-plan",
- "datafusion-spark",
  "datafusion-sql",
  "doc-comment",
  "env_logger",
@@ -2444,27 +2443,6 @@ dependencies = [
  "serde_json",
 ]
 
-[[package]]
-name = "datafusion-spark"
-version = "46.0.0"
-dependencies = [
- "arrow",
- "datafusion-catalog",
- "datafusion-common",
- "datafusion-doc",
- "datafusion-execution",
- "datafusion-expr",
- "datafusion-functions",
- "datafusion-functions-aggregate",
- "datafusion-functions-aggregate-common",
- "datafusion-functions-nested",
- "datafusion-functions-table",
- "datafusion-functions-window",
- "datafusion-functions-window-common",
- "datafusion-macros",
- "log",
-]
-
 [[package]]
 name = "datafusion-sql"
 version = "46.0.0"
diff --git a/Cargo.toml b/Cargo.toml
index dd7d46166cb9..871377f8dfc0 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -47,7 +47,6 @@ members = [
     "datafusion/proto/gen",
     "datafusion/proto-common",
     "datafusion/proto-common/gen",
-    "datafusion/spark",
     "datafusion/sql",
     "datafusion/sqllogictest",
     "datafusion/substrait",
@@ -137,7 +136,6 @@ datafusion-physical-optimizer = { path = "datafusion/physical-optimizer", versio
 datafusion-physical-plan = { path = "datafusion/physical-plan", version = "46.0.0" }
 datafusion-proto = { path = "datafusion/proto", version = "46.0.0" }
 datafusion-proto-common = { path = "datafusion/proto-common", version = "46.0.0" }
-datafusion-spark = { path = "datafusion/spark", version = "46.0.0" }
 datafusion-sql = { path = "datafusion/sql", version = "46.0.0" }
 doc-comment = "0.3"
 env_logger = "0.11"
diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml
index ab7795a93b15..fd1fd4164da0 100644
--- a/datafusion/core/Cargo.toml
+++ b/datafusion/core/Cargo.toml
@@ -85,7 +85,6 @@ serde = [
     # statements in `arrow-schema` crate
     "arrow-schema/serde",
 ]
-spark = ["datafusion-spark"]
 string_expressions = ["datafusion-functions/string_expressions"]
 unicode_expressions = [
     "datafusion-sql/unicode_expressions",
@@ -124,7 +123,6 @@ datafusion-physical-expr = { workspace = true }
 datafusion-physical-expr-common = { workspace = true }
 datafusion-physical-optimizer = { workspace = true }
 datafusion-physical-plan = { workspace = true }
-datafusion-spark = { workspace = true, optional = true }
 datafusion-sql = { workspace = true }
 flate2 = { version = "1.1.0", optional = true }
 futures = { workspace = true }
diff --git a/datafusion/sqllogictest/Cargo.toml b/datafusion/sqllogictest/Cargo.toml
index 95dade7618f2..8550663b439f 100644
--- a/datafusion/sqllogictest/Cargo.toml
+++ b/datafusion/sqllogictest/Cargo.toml
@@ -43,7 +43,7 @@ bigdecimal = { workspace = true }
 bytes = { workspace = true, optional = true }
 chrono = { workspace = true, optional = true }
 clap = { version = "4.5.32", features = ["derive", "env"] }
-datafusion = { workspace = true, default-features = true, features = ["avro", "spark"] }
+datafusion = { workspace = true, default-features = true, features = ["avro"] }
 futures = { workspace = true }
 half = { workspace = true, default-features = true }
 indicatif = "0.17"
diff --git a/parquet-testing b/parquet-testing
deleted file mode 160000
index 39b91cf85306..000000000000
--- a/parquet-testing
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 39b91cf853062d92f0d20581d37b20dabe70a6a0

From 82a55d974f17744ea0d1e548c857eb748f185858 Mon Sep 17 00:00:00 2001
From: Shehab <11789402+shehabgamin@users.noreply.github.com>
Date: Tue, 22 Apr 2025 20:56:24 -0700
Subject: [PATCH 14/25] save

---
 datafusion-testing | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datafusion-testing b/datafusion-testing
index 3462eaa78745..e9f9e22ccf09 160000
--- a/datafusion-testing
+++ b/datafusion-testing
@@ -1 +1 @@
-Subproject commit 3462eaa787459957e38df267a4a21f5bea605807
+Subproject commit e9f9e22ccf09145a7368f80fd6a871f11e2b4481

From 4bac6cec477c0e05e3dcea24441a4bd580ee55d2 Mon Sep 17 00:00:00 2001
From: Shehab <11789402+shehabgamin@users.noreply.github.com>
Date: Tue, 22 Apr 2025 21:00:15 -0700
Subject: [PATCH 15/25] save

---
 parquet-testing | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/parquet-testing b/parquet-testing
index 6e851ddd768d..e845e41789ec 160000
--- a/parquet-testing
+++ b/parquet-testing
@@ -1 +1 @@
-Subproject commit 6e851ddd768d6af741c7b15dc594874399fc3cff
+Subproject commit e845e41789ec8d3aa6317e6464cdbbca987bf91d

From 2d7b44eca89b6625dfa4abe917e4077a8a270737 Mon Sep 17 00:00:00 2001
From: Shehab <11789402+shehabgamin@users.noreply.github.com>
Date: Tue, 22 Apr 2025 23:26:31 -0700
Subject: [PATCH 16/25] add spark crate

---
 Cargo.toml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Cargo.toml b/Cargo.toml
index 5a735666f8e7..e51081b9bf15 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -48,6 +48,7 @@ members = [
     "datafusion/proto-common",
     "datafusion/proto-common/gen",
     "datafusion/session",
+    "datafusion/spark",
     "datafusion/sql",
     "datafusion/sqllogictest",
     "datafusion/substrait",
@@ -138,6 +139,7 @@ datafusion-physical-plan = { path = "datafusion/physical-plan", version = "47.0.
 datafusion-proto = { path = "datafusion/proto", version = "47.0.0" }
 datafusion-proto-common = { path = "datafusion/proto-common", version = "47.0.0" }
 datafusion-session = { path = "datafusion/session", version = "47.0.0" }
+datafusion-spark = { path = "datafusion/spark", version = "47.0.0" }
 datafusion-sql = { path = "datafusion/sql", version = "47.0.0" }
 doc-comment = "0.3"
 env_logger = "0.11"

From ac2b90d62ee89dc187d1398fdd68dd442aa68601 Mon Sep 17 00:00:00 2001
From: Shehab <11789402+shehabgamin@users.noreply.github.com>
Date: Thu, 24 Apr 2025 21:35:35 -0700
Subject: [PATCH 17/25] remove spark from core

---
 .../src/execution/session_state_defaults.rs   | 43 ++-----------------
 1 file changed, 4 insertions(+), 39 deletions(-)

diff --git a/datafusion/core/src/execution/session_state_defaults.rs b/datafusion/core/src/execution/session_state_defaults.rs
index f86b4eb7d3ba..a241738bd3a4 100644
--- a/datafusion/core/src/execution/session_state_defaults.rs
+++ b/datafusion/core/src/execution/session_state_defaults.rs
@@ -104,55 +104,28 @@ impl SessionStateDefaults {
 
     /// returns the list of default [`ScalarUDF']'s
     pub fn default_scalar_functions() -> Vec<Arc<ScalarUDF>> {
-        #[cfg_attr(
-            not(any(feature = "nested_expressions", feature = "spark")),
-            allow(unused_mut)
-        )]
+        #[cfg_attr(not(feature = "nested_expressions"), allow(unused_mut))]
         let mut functions: Vec<Arc<ScalarUDF>> = functions::all_default_functions();
 
         #[cfg(feature = "nested_expressions")]
         functions.append(&mut functions_nested::all_default_nested_functions());
 
-        #[cfg(feature = "spark")]
-        functions.append(&mut datafusion_spark::all_default_scalar_functions());
-
         functions
     }
 
     /// returns the list of default [`AggregateUDF']'s
     pub fn default_aggregate_functions() -> Vec<Arc<AggregateUDF>> {
-        #[cfg_attr(not(feature = "spark"), allow(unused_mut))]
-        let mut functions: Vec<Arc<AggregateUDF>> =
-            functions_aggregate::all_default_aggregate_functions();
-
-        #[cfg(feature = "spark")]
-        functions.append(&mut datafusion_spark::all_default_aggregate_functions());
-
-        functions
+        functions_aggregate::all_default_aggregate_functions()
     }
 
     /// returns the list of default [`WindowUDF']'s
     pub fn default_window_functions() -> Vec<Arc<WindowUDF>> {
-        #[cfg_attr(not(feature = "spark"), allow(unused_mut))]
-        let mut functions: Vec<Arc<WindowUDF>> =
-            functions_window::all_default_window_functions();
-
-        #[cfg(feature = "spark")]
-        functions.append(&mut datafusion_spark::all_default_window_functions());
-
-        functions
+        functions_window::all_default_window_functions()
     }
 
     /// returns the list of default [`TableFunction`]s
     pub fn default_table_functions() -> Vec<Arc<TableFunction>> {
-        #[cfg_attr(not(feature = "spark"), allow(unused_mut))]
-        let mut functions: Vec<Arc<TableFunction>> =
-            functions_table::all_default_table_functions();
-
-        #[cfg(feature = "spark")]
-        functions.append(&mut datafusion_spark::all_default_table_functions());
-
-        functions
+        functions_table::all_default_table_functions()
     }
 
     /// returns the list of default [`FileFormatFactory']'s
@@ -175,7 +148,6 @@ impl SessionStateDefaults {
         Self::register_scalar_functions(state);
         Self::register_array_functions(state);
         Self::register_aggregate_functions(state);
-        Self::register_spark_functions(state);
     }
 
     /// registers all the builtin scalar functions
@@ -198,13 +170,6 @@ impl SessionStateDefaults {
             .expect("can not register aggregate functions");
     }
 
-    /// registers all the builtin Spark functions
-    #[cfg_attr(not(feature = "spark"), allow(unused_variables))]
-    pub fn register_spark_functions(state: &mut SessionState) {
-        #[cfg(feature = "spark")]
-        datafusion_spark::register_all(state).expect("can not register Spark functions");
-    }
-
     /// registers the default schema
     pub fn register_default_schema(
         config: &SessionConfig,

From 6070e0e8c90cbf574b908d9b52bb88cc768b93c2 Mon Sep 17 00:00:00 2001
From: Shehab <11789402+shehabgamin@users.noreply.github.com>
Date: Thu, 24 Apr 2025 22:05:37 -0700
Subject: [PATCH 18/25] add comment to import tests

---
 Cargo.lock                 | 21 +++++++++++++++++++++
 datafusion/spark/README.md |  2 ++
 2 files changed, 23 insertions(+)

diff --git a/Cargo.lock b/Cargo.lock
index 8c501a873133..d9c0c0e4e6ea 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2558,6 +2558,27 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "datafusion-spark"
+version = "47.0.0"
+dependencies = [
+ "arrow",
+ "datafusion-catalog",
+ "datafusion-common",
+ "datafusion-doc",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-functions",
+ "datafusion-functions-aggregate",
+ "datafusion-functions-aggregate-common",
+ "datafusion-functions-nested",
+ "datafusion-functions-table",
+ "datafusion-functions-window",
+ "datafusion-functions-window-common",
+ "datafusion-macros",
+ "log",
+]
+
 [[package]]
 name = "datafusion-sql"
 version = "47.0.0"
diff --git a/datafusion/spark/README.md b/datafusion/spark/README.md
index fe550123bf57..c92ada0ab477 100644
--- a/datafusion/spark/README.md
+++ b/datafusion/spark/README.md
@@ -36,3 +36,5 @@ the [Spark SQLLogicTest README](../sqllogictest/test_files/spark/README.md).
 
 When implementing Spark-compatible functions, you can check if there are existing implementations in
 the [Sail](https://github.com/lakehq/sail) or [Comet](https://github.com/apache/datafusion-comet) projects first.
+If you do port functionality from these sources, make sure to port over the corresponding tests too, to ensure
+correctness and compatibility.

From 4608b2aedea30a46609c695c9d025980ebbac531 Mon Sep 17 00:00:00 2001
From: Shehab <11789402+shehabgamin@users.noreply.github.com>
Date: Thu, 24 Apr 2025 22:15:43 -0700
Subject: [PATCH 19/25] Fix: reset submodule to main pointer and clean state

---
 parquet-testing | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/parquet-testing b/parquet-testing
index e845e41789ec..6e851ddd768d 160000
--- a/parquet-testing
+++ b/parquet-testing
@@ -1 +1 @@
-Subproject commit e845e41789ec8d3aa6317e6464cdbbca987bf91d
+Subproject commit 6e851ddd768d6af741c7b15dc594874399fc3cff

From dd48fa083dc8a3e3fb7e52a763504b63d82b8253 Mon Sep 17 00:00:00 2001
From: Shehab <11789402+shehabgamin@users.noreply.github.com>
Date: Fri, 25 Apr 2025 02:10:04 -0700
Subject: [PATCH 20/25] Save

---
 Cargo.lock                                        |  1 +
 datafusion/spark/src/function/math/expm1.rs       |  2 +-
 datafusion/spark/src/function/math/mod.rs         |  6 +++---
 datafusion/spark/src/function/string/ascii.rs     |  2 +-
 datafusion/spark/src/function/string/mod.rs       |  6 +++---
 datafusion/sqllogictest/Cargo.toml                |  1 +
 datafusion/sqllogictest/src/test_context.rs       | 15 +++++++++++----
 .../sqllogictest/test_files/spark/README.md       |  4 ++--
 .../sqllogictest/test_files/spark/math/expm1.slt  |  6 +++---
 .../test_files/spark/string/ascii.slt             | 10 +++++-----
 10 files changed, 31 insertions(+), 22 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 359d4b101fcc..9eea4ba4bb2f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2614,6 +2614,7 @@ dependencies = [
  "chrono",
  "clap 4.5.36",
  "datafusion",
+ "datafusion-spark",
  "env_logger",
  "futures",
  "half",
diff --git a/datafusion/spark/src/function/math/expm1.rs b/datafusion/spark/src/function/math/expm1.rs
index d648ae1f2e7b..438333827b1e 100644
--- a/datafusion/spark/src/function/math/expm1.rs
+++ b/datafusion/spark/src/function/math/expm1.rs
@@ -78,7 +78,7 @@ impl ScalarUDFImpl for SparkExpm1 {
     }
 
     fn name(&self) -> &str {
-        "spark_expm1"
+        "expm1"
     }
 
     fn signature(&self) -> &Signature {
diff --git a/datafusion/spark/src/function/math/mod.rs b/datafusion/spark/src/function/math/mod.rs
index ee48bafb8e0c..c5b007f40d7f 100644
--- a/datafusion/spark/src/function/math/mod.rs
+++ b/datafusion/spark/src/function/math/mod.rs
@@ -21,14 +21,14 @@ use datafusion_expr::ScalarUDF;
 use datafusion_functions::make_udf_function;
 use std::sync::Arc;
 
-make_udf_function!(expm1::SparkExpm1, spark_expm1);
+make_udf_function!(expm1::SparkExpm1, expm1);
 
 pub mod expr_fn {
     use datafusion_functions::export_functions;
 
-    export_functions!((spark_expm1, "Returns exp(expr) - 1 as a Float64.", arg1));
+    export_functions!((expm1, "Returns exp(expr) - 1 as a Float64.", arg1));
 }
 
 pub fn functions() -> Vec<Arc<ScalarUDF>> {
-    vec![spark_expm1()]
+    vec![expm1()]
 }
diff --git a/datafusion/spark/src/function/string/ascii.rs b/datafusion/spark/src/function/string/ascii.rs
index 3a9c8dd1769d..d2b53df82d7f 100644
--- a/datafusion/spark/src/function/string/ascii.rs
+++ b/datafusion/spark/src/function/string/ascii.rs
@@ -85,7 +85,7 @@ impl ScalarUDFImpl for SparkAscii {
     }
 
     fn name(&self) -> &str {
-        "spark_ascii"
+        "ascii"
     }
 
     fn signature(&self) -> &Signature {
diff --git a/datafusion/spark/src/function/string/mod.rs b/datafusion/spark/src/function/string/mod.rs
index 7e8e88fe8d1e..c01b6c45c008 100644
--- a/datafusion/spark/src/function/string/mod.rs
+++ b/datafusion/spark/src/function/string/mod.rs
@@ -21,18 +21,18 @@ use datafusion_expr::ScalarUDF;
 use datafusion_functions::make_udf_function;
 use std::sync::Arc;
 
-make_udf_function!(ascii::SparkAscii, spark_ascii);
+make_udf_function!(ascii::SparkAscii, ascii);
 
 pub mod expr_fn {
     use datafusion_functions::export_functions;
 
     export_functions!((
-        spark_ascii,
+        ascii,
         "Returns the ASCII code point of the first character of string.",
         arg1
     ));
 }
 
 pub fn functions() -> Vec<Arc<ScalarUDF>> {
-    vec![spark_ascii()]
+    vec![ascii()]
 }
diff --git a/datafusion/sqllogictest/Cargo.toml b/datafusion/sqllogictest/Cargo.toml
index b2e02d7d5dd8..ef31fcad0f48 100644
--- a/datafusion/sqllogictest/Cargo.toml
+++ b/datafusion/sqllogictest/Cargo.toml
@@ -44,6 +44,7 @@ bytes = { workspace = true, optional = true }
 chrono = { workspace = true, optional = true }
 clap = { version = "4.5.36", features = ["derive", "env"] }
 datafusion = { workspace = true, default-features = true, features = ["avro"] }
+datafusion-spark = { workspace = true, default-features = true }
 futures = { workspace = true }
 half = { workspace = true, default-features = true }
 indicatif = "0.17"
diff --git a/datafusion/sqllogictest/src/test_context.rs b/datafusion/sqllogictest/src/test_context.rs
index ce819f186454..2d245c745778 100644
--- a/datafusion/sqllogictest/src/test_context.rs
+++ b/datafusion/sqllogictest/src/test_context.rs
@@ -122,6 +122,13 @@ impl TestContext {
                 info!("Using default SessionContext");
             }
         };
+
+        if relative_path.starts_with("spark/") {
+            info!("Registering Spark functions");
+            datafusion_spark::register_all(&mut test_ctx.session_ctx().state())
+                .expect("Can not register Spark functions");
+        }
+
         Some(test_ctx)
     }
 
@@ -223,14 +230,14 @@ pub async fn register_temp_table(ctx: &SessionContext) {
             self
         }
 
-        fn table_type(&self) -> TableType {
-            self.0
-        }
-
         fn schema(&self) -> SchemaRef {
             unimplemented!()
         }
 
+        fn table_type(&self) -> TableType {
+            self.0
+        }
+
         async fn scan(
             &self,
             _state: &dyn Session,
diff --git a/datafusion/sqllogictest/test_files/spark/README.md b/datafusion/sqllogictest/test_files/spark/README.md
index 5e78cc721ece..0a7bb92371b5 100644
--- a/datafusion/sqllogictest/test_files/spark/README.md
+++ b/datafusion/sqllogictest/test_files/spark/README.md
@@ -47,11 +47,11 @@ of `NAME`).
 ### Scalar Example:
 
 ```sql
-SELECT spark_expm1(0::INT);
+SELECT expm1(0::INT);
 ```
 
 ### Array Example:
 
 ```sql
-SELECT spark_expm1(a) FROM (VALUES (0::INT), (1::INT)) AS t(a);
+SELECT expm1(a) FROM (VALUES (0::INT), (1::INT)) AS t(a);
 ```
diff --git a/datafusion/sqllogictest/test_files/spark/math/expm1.slt b/datafusion/sqllogictest/test_files/spark/math/expm1.slt
index 9455b7461f78..5e5c0c4354b5 100644
--- a/datafusion/sqllogictest/test_files/spark/math/expm1.slt
+++ b/datafusion/sqllogictest/test_files/spark/math/expm1.slt
@@ -16,17 +16,17 @@
 # under the License.
 
 query R
-SELECT spark_expm1(0::INT);
+SELECT expm1(0::INT);
 ----
 0
 
 query R
-SELECT spark_expm1(1::INT);
+SELECT expm1(1::INT);
 ----
 1.718281828459
 
 query R
-SELECT spark_expm1(a) FROM (VALUES (0::INT), (1::INT)) AS t(a);
+SELECT expm1(a) FROM (VALUES (0::INT), (1::INT)) AS t(a);
 ----
 0
 1.718281828459
diff --git a/datafusion/sqllogictest/test_files/spark/string/ascii.slt b/datafusion/sqllogictest/test_files/spark/string/ascii.slt
index de434a6d15b6..a90a70724550 100644
--- a/datafusion/sqllogictest/test_files/spark/string/ascii.slt
+++ b/datafusion/sqllogictest/test_files/spark/string/ascii.slt
@@ -16,27 +16,27 @@
 # under the License.
 
 query I
-SELECT spark_ascii('234');
+SELECT ascii('234');
 ----
 50
 
 query I
-SELECT spark_ascii('');
+SELECT ascii('');
 ----
 0
 
 query I
-SELECT spark_ascii('222');
+SELECT ascii('222');
 ----
 50
 
 query I
-SELECT spark_ascii(2::INT);
+SELECT ascii(2::INT);
 ----
 50
 
 query I
-SELECT spark_ascii(a) FROM (VALUES ('Spark'), ('PySpark'), ('Pandas API')) AS t(a);
+SELECT ascii(a) FROM (VALUES ('Spark'), ('PySpark'), ('Pandas API')) AS t(a);
 ----
 83
 80

From 51b882fb8dba1d5ad25bf33b228312046d32df2d Mon Sep 17 00:00:00 2001
From: Shehab <11789402+shehabgamin@users.noreply.github.com>
Date: Sat, 26 Apr 2025 00:10:18 -0700
Subject: [PATCH 21/25] fix registration

---
 datafusion/sqllogictest/src/test_context.rs | 22 ++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/datafusion/sqllogictest/src/test_context.rs b/datafusion/sqllogictest/src/test_context.rs
index 2d245c745778..0b5b3438a17a 100644
--- a/datafusion/sqllogictest/src/test_context.rs
+++ b/datafusion/sqllogictest/src/test_context.rs
@@ -42,6 +42,8 @@ use datafusion::{
 
 use async_trait::async_trait;
 use datafusion::common::cast::as_float64_array;
+use datafusion::execution::runtime_env::RuntimeEnv;
+use datafusion::execution::SessionStateBuilder;
 use log::info;
 use tempfile::TempDir;
 
@@ -70,8 +72,20 @@ impl TestContext {
         let config = SessionConfig::new()
             // hardcode target partitions so plans are deterministic
             .with_target_partitions(4);
+        let runtime = Arc::new(RuntimeEnv::default());
+        let mut state = SessionStateBuilder::new()
+            .with_config(config)
+            .with_runtime_env(runtime)
+            .with_default_features()
+            .build();
 
-        let mut test_ctx = TestContext::new(SessionContext::new_with_config(config));
+        if relative_path.starts_with("spark/") {
+            info!("Registering Spark functions");
+            datafusion_spark::register_all(&mut state)
+                .expect("Can not register Spark functions");
+        }
+
+        let mut test_ctx = TestContext::new(SessionContext::new_with_state(state));
 
         let file_name = relative_path.file_name().unwrap().to_str().unwrap();
         match file_name {
@@ -123,12 +137,6 @@ impl TestContext {
             }
         };
 
-        if relative_path.starts_with("spark/") {
-            info!("Registering Spark functions");
-            datafusion_spark::register_all(&mut test_ctx.session_ctx().state())
-                .expect("Can not register Spark functions");
-        }
-
         Some(test_ctx)
     }
 

From 0514ba65d9b6fd90f72671121e5963bf810b49d4 Mon Sep 17 00:00:00 2001
From: Shehab <11789402+shehabgamin@users.noreply.github.com>
Date: Sat, 26 Apr 2025 01:57:58 -0700
Subject: [PATCH 22/25] modify float64 precision for spark

---
 datafusion/spark/src/function/string/ascii.rs |  2 +
 .../sqllogictest/src/engines/conversion.rs    | 76 +++++++++++++------
 .../engines/datafusion_engine/normalize.rs    | 42 +++++-----
 .../src/engines/datafusion_engine/runner.rs   | 11 ++-
 datafusion/sqllogictest/src/test_context.rs   |  3 +-
 datafusion/sqllogictest/src/util.rs           |  4 +
 .../test_files/spark/math/expm1.slt           |  4 +-
 .../test_files/spark/string/ascii.slt         |  5 ++
 8 files changed, 97 insertions(+), 50 deletions(-)

diff --git a/datafusion/spark/src/function/string/ascii.rs b/datafusion/spark/src/function/string/ascii.rs
index d2b53df82d7f..22c5aa14706e 100644
--- a/datafusion/spark/src/function/string/ascii.rs
+++ b/datafusion/spark/src/function/string/ascii.rs
@@ -201,6 +201,8 @@ mod tests {
         test_ascii_string_invoke!(Some(String::from("x")), Ok(Some(120)));
         test_ascii_string_invoke!(Some(String::from("a")), Ok(Some(97)));
         test_ascii_string_invoke!(Some(String::from("")), Ok(Some(0)));
+        test_ascii_string_invoke!(Some(String::from("\n")), Ok(Some(10)));
+        test_ascii_string_invoke!(Some(String::from("\t")), Ok(Some(9)));
         test_ascii_string_invoke!(None, Ok(None));
 
         Ok(())
diff --git a/datafusion/sqllogictest/src/engines/conversion.rs b/datafusion/sqllogictest/src/engines/conversion.rs
index 516ec69e0b07..92ab64059bbd 100644
--- a/datafusion/sqllogictest/src/engines/conversion.rs
+++ b/datafusion/sqllogictest/src/engines/conversion.rs
@@ -49,7 +49,7 @@ pub(crate) fn f16_to_str(value: f16) -> String {
     } else if value == f16::NEG_INFINITY {
         "-Infinity".to_string()
     } else {
-        big_decimal_to_str(BigDecimal::from_str(&value.to_string()).unwrap())
+        big_decimal_to_str(BigDecimal::from_str(&value.to_string()).unwrap(), None)
     }
 }
 
@@ -63,7 +63,7 @@ pub(crate) fn f32_to_str(value: f32) -> String {
     } else if value == f32::NEG_INFINITY {
         "-Infinity".to_string()
     } else {
-        big_decimal_to_str(BigDecimal::from_str(&value.to_string()).unwrap())
+        big_decimal_to_str(BigDecimal::from_str(&value.to_string()).unwrap(), None)
     }
 }
 
@@ -77,7 +77,21 @@ pub(crate) fn f64_to_str(value: f64) -> String {
     } else if value == f64::NEG_INFINITY {
         "-Infinity".to_string()
     } else {
-        big_decimal_to_str(BigDecimal::from_str(&value.to_string()).unwrap())
+        big_decimal_to_str(BigDecimal::from_str(&value.to_string()).unwrap(), None)
+    }
+}
+
+pub(crate) fn spark_f64_to_str(value: f64) -> String {
+    if value.is_nan() {
+        // The sign of NaN can be different depending on platform.
+        // So the string representation of NaN ignores the sign.
+        "NaN".to_string()
+    } else if value == f64::INFINITY {
+        "Infinity".to_string()
+    } else if value == f64::NEG_INFINITY {
+        "-Infinity".to_string()
+    } else {
+        big_decimal_to_str(BigDecimal::from_str(&value.to_string()).unwrap(), Some(15))
     }
 }
 
@@ -86,6 +100,7 @@ pub(crate) fn decimal_128_to_str(value: i128, scale: i8) -> String {
     big_decimal_to_str(
         BigDecimal::from_str(&Decimal128Type::format_decimal(value, precision, scale))
             .unwrap(),
+        None,
     )
 }
 
@@ -94,17 +109,21 @@ pub(crate) fn decimal_256_to_str(value: i256, scale: i8) -> String {
     big_decimal_to_str(
         BigDecimal::from_str(&Decimal256Type::format_decimal(value, precision, scale))
             .unwrap(),
+        None,
     )
 }
 
 #[cfg(feature = "postgres")]
 pub(crate) fn decimal_to_str(value: Decimal) -> String {
-    big_decimal_to_str(BigDecimal::from_str(&value.to_string()).unwrap())
+    big_decimal_to_str(BigDecimal::from_str(&value.to_string()).unwrap(), None)
 }
 
-pub(crate) fn big_decimal_to_str(value: BigDecimal) -> String {
+/// Converts a `BigDecimal` to its plain string representation, optionally rounding to a specified number of decimal places.
+///
+/// If `round_digits` is `None`, the value is rounded to 12 decimal places by default.
+pub(crate) fn big_decimal_to_str(value: BigDecimal, round_digits: Option<i64>) -> String {
     // Round the value to limit the number of decimal places
-    let value = value.round(12).normalized();
+    let value = value.round(round_digits.unwrap_or(12)).normalized();
     // Format the value to a string
     value.to_plain_string()
 }
@@ -115,12 +134,12 @@ mod tests {
     use bigdecimal::{num_bigint::BigInt, BigDecimal};
 
     macro_rules! assert_decimal_str_eq {
-        ($integer:expr, $scale:expr, $expected:expr) => {
+        ($integer:expr, $scale:expr, $round_digits:expr, $expected:expr) => {
             assert_eq!(
-                big_decimal_to_str(BigDecimal::from_bigint(
-                    BigInt::from($integer),
-                    $scale
-                )),
+                big_decimal_to_str(
+                    BigDecimal::from_bigint(BigInt::from($integer), $scale),
+                    $round_digits
+                ),
                 $expected
             );
         };
@@ -128,44 +147,51 @@ mod tests {
 
     #[test]
     fn test_big_decimal_to_str() {
-        assert_decimal_str_eq!(110, 3, "0.11");
-        assert_decimal_str_eq!(11, 3, "0.011");
-        assert_decimal_str_eq!(11, 2, "0.11");
-        assert_decimal_str_eq!(11, 1, "1.1");
-        assert_decimal_str_eq!(11, 0, "11");
-        assert_decimal_str_eq!(11, -1, "110");
-        assert_decimal_str_eq!(0, 0, "0");
+        assert_decimal_str_eq!(110, 3, None, "0.11");
+        assert_decimal_str_eq!(11, 3, None, "0.011");
+        assert_decimal_str_eq!(11, 2, None, "0.11");
+        assert_decimal_str_eq!(11, 1, None, "1.1");
+        assert_decimal_str_eq!(11, 0, None, "11");
+        assert_decimal_str_eq!(11, -1, None, "110");
+        assert_decimal_str_eq!(0, 0, None, "0");
         assert_decimal_str_eq!(
             12345678901234567890123456789012345678_i128,
             0,
+            None,
             "12345678901234567890123456789012345678"
         );
         assert_decimal_str_eq!(
             12345678901234567890123456789012345678_i128,
             38,
+            None,
             "0.123456789012"
         );
 
         // Negative cases
-        assert_decimal_str_eq!(-110, 3, "-0.11");
-        assert_decimal_str_eq!(-11, 3, "-0.011");
-        assert_decimal_str_eq!(-11, 2, "-0.11");
-        assert_decimal_str_eq!(-11, 1, "-1.1");
-        assert_decimal_str_eq!(-11, 0, "-11");
-        assert_decimal_str_eq!(-11, -1, "-110");
+        assert_decimal_str_eq!(-110, 3, None, "-0.11");
+        assert_decimal_str_eq!(-11, 3, None, "-0.011");
+        assert_decimal_str_eq!(-11, 2, None, "-0.11");
+        assert_decimal_str_eq!(-11, 1, None, "-1.1");
+        assert_decimal_str_eq!(-11, 0, None, "-11");
+        assert_decimal_str_eq!(-11, -1, None, "-110");
         assert_decimal_str_eq!(
             -12345678901234567890123456789012345678_i128,
             0,
+            None,
             "-12345678901234567890123456789012345678"
         );
         assert_decimal_str_eq!(
             -12345678901234567890123456789012345678_i128,
             38,
+            None,
             "-0.123456789012"
         );
 
         // Round to 12 decimal places
         // 1.0000000000011 -> 1.000000000001
-        assert_decimal_str_eq!(10_i128.pow(13) + 11, 13, "1.000000000001");
+        assert_decimal_str_eq!(10_i128.pow(13) + 11, 13, None, "1.000000000001");
+        assert_decimal_str_eq!(10_i128.pow(13) + 11, 13, Some(12), "1.000000000001");
+
+        assert_decimal_str_eq!(10_i128.pow(13) + 11, 13, Some(13), "1.0000000000011");
     }
 }
diff --git a/datafusion/sqllogictest/src/engines/datafusion_engine/normalize.rs b/datafusion/sqllogictest/src/engines/datafusion_engine/normalize.rs
index eeb34186ea20..85bbd421827e 100644
--- a/datafusion/sqllogictest/src/engines/datafusion_engine/normalize.rs
+++ b/datafusion/sqllogictest/src/engines/datafusion_engine/normalize.rs
@@ -28,7 +28,10 @@ use std::path::PathBuf;
 use std::sync::LazyLock;
 
 /// Converts `batches` to a result as expected by sqllogictest.
-pub fn convert_batches(batches: Vec<RecordBatch>) -> Result<Vec<Vec<String>>> {
+pub fn convert_batches(
+    batches: Vec<RecordBatch>,
+    is_spark_path: bool,
+) -> Result<Vec<Vec<String>>> {
     if batches.is_empty() {
         Ok(vec![])
     } else {
@@ -46,7 +49,16 @@ pub fn convert_batches(batches: Vec<RecordBatch>) -> Result<Vec<Vec<String>>> {
                 )));
             }
 
-            let new_rows = convert_batch(batch)?
+            // Convert a single batch to a `Vec<Vec<String>>` for comparison, flatten expanded rows, and normalize each.
+            let new_rows = (0..batch.num_rows())
+                .map(|row| {
+                    batch
+                        .columns()
+                        .iter()
+                        .map(|col| cell_to_string(col, row, is_spark_path))
+                        .collect::<Result<Vec<String>>>()
+                })
+                .collect::<Result<Vec<Vec<String>>>>()?
                 .into_iter()
                 .flat_map(expand_row)
                 .map(normalize_paths);
@@ -162,19 +174,6 @@ static WORKSPACE_ROOT: LazyLock<object_store::path::Path> = LazyLock::new(|| {
     object_store::path::Path::parse(sanitized_workplace_root).unwrap()
 });
 
-/// Convert a single batch to a `Vec<Vec<String>>` for comparison
-fn convert_batch(batch: RecordBatch) -> Result<Vec<Vec<String>>> {
-    (0..batch.num_rows())
-        .map(|row| {
-            batch
-                .columns()
-                .iter()
-                .map(|col| cell_to_string(col, row))
-                .collect::<Result<Vec<String>>>()
-        })
-        .collect()
-}
-
 macro_rules! get_row_value {
     ($array_type:ty, $column: ident, $row: ident) => {{
         let array = $column.as_any().downcast_ref::<$array_type>().unwrap();
@@ -193,7 +192,7 @@ macro_rules! get_row_value {
 ///
 /// Floating numbers are rounded to have a consistent representation with the Postgres runner.
 ///
-pub fn cell_to_string(col: &ArrayRef, row: usize) -> Result<String> {
+pub fn cell_to_string(col: &ArrayRef, row: usize, is_spark_path: bool) -> Result<String> {
     if !col.is_valid(row) {
         // represent any null value with the string "NULL"
         Ok(NULL_STR.to_string())
@@ -210,7 +209,12 @@ pub fn cell_to_string(col: &ArrayRef, row: usize) -> Result<String> {
                 Ok(f32_to_str(get_row_value!(array::Float32Array, col, row)))
             }
             DataType::Float64 => {
-                Ok(f64_to_str(get_row_value!(array::Float64Array, col, row)))
+                let result = get_row_value!(array::Float64Array, col, row);
+                if is_spark_path {
+                    Ok(spark_f64_to_str(result))
+                } else {
+                    Ok(f64_to_str(result))
+                }
             }
             DataType::Decimal128(_, scale) => {
                 let value = get_row_value!(array::Decimal128Array, col, row);
@@ -236,12 +240,12 @@ pub fn cell_to_string(col: &ArrayRef, row: usize) -> Result<String> {
             DataType::Dictionary(_, _) => {
                 let dict = col.as_any_dictionary();
                 let key = dict.normalized_keys()[row];
-                Ok(cell_to_string(dict.values(), key)?)
+                Ok(cell_to_string(dict.values(), key, is_spark_path)?)
             }
             _ => {
                 let f =
                     ArrayFormatter::try_new(col.as_ref(), &DEFAULT_CLI_FORMAT_OPTIONS);
-                Ok(f.unwrap().value(row).to_string())
+                Ok(f?.value(row).to_string())
             }
         }
         .map_err(DFSqlLogicTestError::Arrow)
diff --git a/datafusion/sqllogictest/src/engines/datafusion_engine/runner.rs b/datafusion/sqllogictest/src/engines/datafusion_engine/runner.rs
index a3a29eda2ee9..a01ac7e2f985 100644
--- a/datafusion/sqllogictest/src/engines/datafusion_engine/runner.rs
+++ b/datafusion/sqllogictest/src/engines/datafusion_engine/runner.rs
@@ -31,6 +31,7 @@ use sqllogictest::DBOutput;
 use tokio::time::Instant;
 
 use crate::engines::output::{DFColumnType, DFOutput};
+use crate::is_spark_path;
 
 pub struct DataFusion {
     ctx: SessionContext,
@@ -79,7 +80,7 @@ impl sqllogictest::AsyncDB for DataFusion {
         }
 
         let start = Instant::now();
-        let result = run_query(&self.ctx, sql).await;
+        let result = run_query(&self.ctx, is_spark_path(&self.relative_path), sql).await;
         let duration = start.elapsed();
 
         if duration.gt(&Duration::from_millis(500)) {
@@ -115,7 +116,11 @@ impl sqllogictest::AsyncDB for DataFusion {
     async fn shutdown(&mut self) {}
 }
 
-async fn run_query(ctx: &SessionContext, sql: impl Into<String>) -> Result<DFOutput> {
+async fn run_query(
+    ctx: &SessionContext,
+    is_spark_path: bool,
+    sql: impl Into<String>,
+) -> Result<DFOutput> {
     let df = ctx.sql(sql.into().as_str()).await?;
     let task_ctx = Arc::new(df.task_ctx());
     let plan = df.create_physical_plan().await?;
@@ -123,7 +128,7 @@ async fn run_query(ctx: &SessionContext, sql: impl Into<String>) -> Result<DFOut
     let stream = execute_stream(plan, task_ctx)?;
     let types = normalize::convert_schema_to_types(stream.schema().fields());
     let results: Vec<RecordBatch> = collect(stream).await?;
-    let rows = normalize::convert_batches(results)?;
+    let rows = normalize::convert_batches(results, is_spark_path)?;
 
     if rows.is_empty() && types.is_empty() {
         Ok(DBOutput::StatementComplete(0))
diff --git a/datafusion/sqllogictest/src/test_context.rs b/datafusion/sqllogictest/src/test_context.rs
index 0b5b3438a17a..999f8fe08d4e 100644
--- a/datafusion/sqllogictest/src/test_context.rs
+++ b/datafusion/sqllogictest/src/test_context.rs
@@ -40,6 +40,7 @@ use datafusion::{
     prelude::{CsvReadOptions, SessionContext},
 };
 
+use crate::is_spark_path;
 use async_trait::async_trait;
 use datafusion::common::cast::as_float64_array;
 use datafusion::execution::runtime_env::RuntimeEnv;
@@ -79,7 +80,7 @@ impl TestContext {
             .with_default_features()
             .build();
 
-        if relative_path.starts_with("spark/") {
+        if is_spark_path(relative_path) {
             info!("Registering Spark functions");
             datafusion_spark::register_all(&mut state)
                 .expect("Can not register Spark functions");
diff --git a/datafusion/sqllogictest/src/util.rs b/datafusion/sqllogictest/src/util.rs
index 5ae640cc98a9..695fe463fa67 100644
--- a/datafusion/sqllogictest/src/util.rs
+++ b/datafusion/sqllogictest/src/util.rs
@@ -106,3 +106,7 @@ pub fn df_value_validator(
 
     normalized_actual == normalized_expected
 }
+
+pub fn is_spark_path(relative_path: &Path) -> bool {
+    relative_path.starts_with("spark/")
+}
diff --git a/datafusion/sqllogictest/test_files/spark/math/expm1.slt b/datafusion/sqllogictest/test_files/spark/math/expm1.slt
index 5e5c0c4354b5..96d4abb0414b 100644
--- a/datafusion/sqllogictest/test_files/spark/math/expm1.slt
+++ b/datafusion/sqllogictest/test_files/spark/math/expm1.slt
@@ -23,10 +23,10 @@ SELECT expm1(0::INT);
 query R
 SELECT expm1(1::INT);
 ----
-1.718281828459
+1.718281828459045
 
 query R
 SELECT expm1(a) FROM (VALUES (0::INT), (1::INT)) AS t(a);
 ----
 0
-1.718281828459
+1.718281828459045
diff --git a/datafusion/sqllogictest/test_files/spark/string/ascii.slt b/datafusion/sqllogictest/test_files/spark/string/ascii.slt
index a90a70724550..623154ffaa7b 100644
--- a/datafusion/sqllogictest/test_files/spark/string/ascii.slt
+++ b/datafusion/sqllogictest/test_files/spark/string/ascii.slt
@@ -30,6 +30,11 @@ SELECT ascii('222');
 ----
 50
 
+query I
+SELECT ascii('😀');
+----
+128512
+
 query I
 SELECT ascii(2::INT);
 ----

From 2601c269d422ce06832a95165d18476b0b6dbea4 Mon Sep 17 00:00:00 2001
From: Shehab Amin <11789402+shehabgamin@users.noreply.github.com>
Date: Tue, 29 Apr 2025 05:34:27 +0000
Subject: [PATCH 23/25] Update datafusion/spark/src/lib.rs

Co-authored-by: Andrew Lamb <andrew@nerdnetworks.org>
---
 datafusion/spark/src/lib.rs | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/datafusion/spark/src/lib.rs b/datafusion/spark/src/lib.rs
index de50fd1551e2..9d5f04f7a623 100644
--- a/datafusion/spark/src/lib.rs
+++ b/datafusion/spark/src/lib.rs
@@ -26,11 +26,7 @@
 //! Spark Expression packages for [DataFusion].
 //!
 //! This crate contains a collection of various Spark expression packages for DataFusion,
-//! implemented using the extension API. Users may wish to control which functions
-//! are available to control the binary size of their application.
-//!
-//! Each package is implemented as a separate
-//! module, activated by a feature flag.
+//! implemented using the extension API. 
 //!
 //! [DataFusion]: https://crates.io/crates/datafusion
 //!

From e7f6d5fc772f5d9285d3ee44c90a01466736f156 Mon Sep 17 00:00:00 2001
From: Shehab <11789402+shehabgamin@users.noreply.github.com>
Date: Mon, 28 Apr 2025 22:39:35 -0700
Subject: [PATCH 24/25] clean up code

---
 datafusion/spark/src/function/math/expm1.rs   | 24 +-------------
 datafusion/spark/src/function/string/ascii.rs | 33 +------------------
 datafusion/spark/src/lib.rs                   |  1 -
 3 files changed, 2 insertions(+), 56 deletions(-)

diff --git a/datafusion/spark/src/function/math/expm1.rs b/datafusion/spark/src/function/math/expm1.rs
index 438333827b1e..3110d15bf9f5 100644
--- a/datafusion/spark/src/function/math/expm1.rs
+++ b/datafusion/spark/src/function/math/expm1.rs
@@ -28,29 +28,7 @@ use datafusion_macros::user_doc;
 use std::any::Any;
 use std::sync::Arc;
 
-#[user_doc(
-    doc_section(label = "Spark Math Functions"),
-    description = "Returns exp(expr) - 1 as a Float64.",
-    syntax_example = "expm1(expr)",
-    sql_example = r#"```sql
-> select expm1(0);
-+--------------------+
-| expm1(0)           |
-+--------------------+
-| 0.0                |
-+--------------------+
-> select expm1(1);
-+----------------+
-| expm1(1)       |
-+----------------+
-| 50             |
-+----------------+
-```"#,
-    argument(
-        name = "expr",
-        description = "An expression that evaluates to a numeric."
-    )
-)]
+/// https://spark.apache.org/docs/latest/api/sql/index.html#expm1
 #[derive(Debug)]
 pub struct SparkExpm1 {
     signature: Signature,
diff --git a/datafusion/spark/src/function/string/ascii.rs b/datafusion/spark/src/function/string/ascii.rs
index 22c5aa14706e..bedeed3abe84 100644
--- a/datafusion/spark/src/function/string/ascii.rs
+++ b/datafusion/spark/src/function/string/ascii.rs
@@ -26,38 +26,7 @@ use datafusion_macros::user_doc;
 use std::any::Any;
 use std::sync::Arc;
 
-#[user_doc(
-    doc_section(label = "Spark String Functions"),
-    description = "Returns the ASCII code point of the first character of string.",
-    syntax_example = "ascii(str)",
-    sql_example = r#"```sql
-> select ascii('abc');
-+--------------------+
-| ascii(abc)         |
-+--------------------+
-| 97                 |
-+--------------------+
-> select ascii('🚀');
-+-------------------+
-| ascii(🚀)         |
-+-------------------+
-| 128640            |
-+-------------------+
-> select ascii(2);
-+----------------+
-| ascii(2)       |
-+----------------+
-| 50             |
-+----------------+
-> select ascii(X'44617461467573696F6E');
-+--------------------------------------+
-| ascii(X'44617461467573696F6E')       |
-+--------------------------------------+
-| 68                                   |
-+--------------------------------------+
-```"#,
-    standard_argument(name = "str", prefix = "String")
-)]
+/// https://spark.apache.org/docs/latest/api/sql/index.html#ascii
 #[derive(Debug)]
 pub struct SparkAscii {
     signature: Signature,
diff --git a/datafusion/spark/src/lib.rs b/datafusion/spark/src/lib.rs
index de50fd1551e2..64860704cba9 100644
--- a/datafusion/spark/src/lib.rs
+++ b/datafusion/spark/src/lib.rs
@@ -55,7 +55,6 @@ use log::debug;
 use std::sync::Arc;
 
 /// Fluent-style API for creating `Expr`s
-#[allow(unused)]
 pub mod expr_fn {
     pub use super::function::aggregate::expr_fn::*;
     pub use super::function::array::expr_fn::*;

From 2b02e3fc1f62567bbcb863ca1aef4109f835f5ce Mon Sep 17 00:00:00 2001
From: Shehab <11789402+shehabgamin@users.noreply.github.com>
Date: Mon, 28 Apr 2025 22:46:53 -0700
Subject: [PATCH 25/25] code cleanup

---
 Cargo.lock                                    | 7 -------
 datafusion/spark/Cargo.toml                   | 7 -------
 datafusion/spark/src/function/math/expm1.rs   | 3 +--
 datafusion/spark/src/function/string/ascii.rs | 9 ++-------
 datafusion/spark/src/lib.rs                   | 3 ++-
 5 files changed, 5 insertions(+), 24 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 9eea4ba4bb2f..6e9d2e45de75 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2565,16 +2565,9 @@ dependencies = [
  "arrow",
  "datafusion-catalog",
  "datafusion-common",
- "datafusion-doc",
  "datafusion-execution",
  "datafusion-expr",
  "datafusion-functions",
- "datafusion-functions-aggregate",
- "datafusion-functions-aggregate-common",
- "datafusion-functions-nested",
- "datafusion-functions-table",
- "datafusion-functions-window",
- "datafusion-functions-window-common",
  "datafusion-macros",
  "log",
 ]
diff --git a/datafusion/spark/Cargo.toml b/datafusion/spark/Cargo.toml
index bea80cc6aa09..1ded8c40aa4b 100644
--- a/datafusion/spark/Cargo.toml
+++ b/datafusion/spark/Cargo.toml
@@ -39,15 +39,8 @@ name = "datafusion_spark"
 arrow = { workspace = true }
 datafusion-catalog = { workspace = true }
 datafusion-common = { workspace = true }
-datafusion-doc = { workspace = true }
 datafusion-execution = { workspace = true }
 datafusion-expr = { workspace = true }
 datafusion-functions = { workspace = true }
-datafusion-functions-aggregate = { workspace = true }
-datafusion-functions-aggregate-common = { workspace = true }
-datafusion-functions-nested = { workspace = true }
-datafusion-functions-table = { workspace = true }
-datafusion-functions-window = { workspace = true }
-datafusion-functions-window-common = { workspace = true }
 datafusion-macros = { workspace = true }
 log = { workspace = true }
diff --git a/datafusion/spark/src/function/math/expm1.rs b/datafusion/spark/src/function/math/expm1.rs
index 3110d15bf9f5..3a3a0c3835d3 100644
--- a/datafusion/spark/src/function/math/expm1.rs
+++ b/datafusion/spark/src/function/math/expm1.rs
@@ -24,11 +24,10 @@ use datafusion_common::{Result, ScalarValue};
 use datafusion_expr::{
     ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,
 };
-use datafusion_macros::user_doc;
 use std::any::Any;
 use std::sync::Arc;
 
-/// https://spark.apache.org/docs/latest/api/sql/index.html#expm1
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#expm1>
 #[derive(Debug)]
 pub struct SparkExpm1 {
     signature: Signature,
diff --git a/datafusion/spark/src/function/string/ascii.rs b/datafusion/spark/src/function/string/ascii.rs
index bedeed3abe84..c05aa214ccc0 100644
--- a/datafusion/spark/src/function/string/ascii.rs
+++ b/datafusion/spark/src/function/string/ascii.rs
@@ -19,14 +19,13 @@ use arrow::array::{ArrayAccessor, ArrayIter, ArrayRef, AsArray, Int32Array};
 use arrow::datatypes::DataType;
 use arrow::error::ArrowError;
 use datafusion_common::{internal_err, plan_err, Result};
-use datafusion_expr::{ColumnarValue, Documentation};
+use datafusion_expr::ColumnarValue;
 use datafusion_expr::{ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility};
 use datafusion_functions::utils::make_scalar_function;
-use datafusion_macros::user_doc;
 use std::any::Any;
 use std::sync::Arc;
 
-/// https://spark.apache.org/docs/latest/api/sql/index.html#ascii
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#ascii>
 #[derive(Debug)]
 pub struct SparkAscii {
     signature: Signature,
@@ -83,10 +82,6 @@ impl ScalarUDFImpl for SparkAscii {
         }
         Ok(vec![DataType::Utf8])
     }
-
-    fn documentation(&self) -> Option<&Documentation> {
-        self.doc()
-    }
 }
 
 fn calculate_ascii<'a, V>(array: V) -> Result<ArrayRef, ArrowError>
diff --git a/datafusion/spark/src/lib.rs b/datafusion/spark/src/lib.rs
index 9f3585bc1d28..1fe5b6ecac8f 100644
--- a/datafusion/spark/src/lib.rs
+++ b/datafusion/spark/src/lib.rs
@@ -26,7 +26,7 @@
 //! Spark Expression packages for [DataFusion].
 //!
 //! This crate contains a collection of various Spark expression packages for DataFusion,
-//! implemented using the extension API. 
+//! implemented using the extension API.
 //!
 //! [DataFusion]: https://crates.io/crates/datafusion
 //!
@@ -51,6 +51,7 @@ use log::debug;
 use std::sync::Arc;
 
 /// Fluent-style API for creating `Expr`s
+#[allow(unused)]
 pub mod expr_fn {
     pub use super::function::aggregate::expr_fn::*;
     pub use super::function::array::expr_fn::*;
