Run the examples in the new format (#18946)

## Which issue does this PR close?

<!--
We generally require a GitHub issue to be filed for all bug fixes and
enhancements and this helps us generate change logs for our releases.
You can link an issue to this PR using the GitHub syntax. For example
`Closes #123` indicates that this PR will close issue #123.
-->

- Closes ##18512.

## Rationale for this change

<!--
Why are you proposing this change? If this is already explained clearly
in the issue then this section is not needed.
Explaining clearly why changes are proposed helps reviewers understand
your changes and offer better suggestions for fixes.
-->

## What changes are included in this PR?

<!--
There is no need to duplicate the description in the issue here but it
is sometimes worth providing a summary of the individual changes in this
PR.
-->

## Are these changes tested?

<!--
We typically require tests for all PRs in order to:
1. Prevent the code from being accidentally broken by subsequent changes
2. Serve as another way to document the expected behavior of the code

If tests are not included in your PR, please explain why (for example,
are they covered by existing tests)?
-->

## Are there any user-facing changes?

<!--
If there are user-facing changes then we may require documentation to be
updated before approving the PR.
-->

<!--
If there are any breaking changes to public APIs, please add the `api
change` label.
-->

---------

Co-authored-by: Sergey Zhukov <szhukov@aligntech.com>
Co-authored-by: Andrew Lamb <andrew@nerdnetworks.org>

Author: 3 people

ci/scripts/rust_example.sh

@@ -25,12 +25,26 @@ export CARGO_PROFILE_CI_STRIP=true
 cd datafusion-examples/examples/
 cargo build --profile ci --examples
 
-files=$(ls .)
-for filename in $files
-do
-  example_name=`basename $filename ".rs"`
-  # Skip tests that rely on external storage and flight
-  if [ ! -d $filename ]; then
-    cargo run --profile ci --example $example_name
-  fi
+SKIP_LIST=("external_dependency" "flight" "ffi")
+
+skip_example() {
+    local name="$1"
+    for skip in "${SKIP_LIST[@]}"; do
+        if [ "$name" = "$skip" ]; then
+            return 0
+        fi
+    done
+    return 1
+}
+
+for dir in */; do
+    example_name=$(basename "$dir")
+
+    if skip_example "$example_name"; then
+        echo "Skipping $example_name"
+        continue
+    fi
+
+    echo "Running example group: $example_name"
+    cargo run --profile ci --example "$example_name" -- all
 done

datafusion-examples/examples/builtin_functions/main.rs

@@ -21,10 +21,11 @@
 //!
 //! ## Usage
 //! ```bash
-//! cargo run --example builtin_functions -- [date_time|function_factory|regexp]
+//! cargo run --example builtin_functions -- [all|date_time|function_factory|regexp]
 //! ```
 //!
 //! Each subcommand runs a corresponding example:
+//! - `all` — run all examples included in this module
 //! - `date_time` — examples of date-time related functions and queries
 //! - `function_factory` — register `CREATE FUNCTION` handler to implement SQL macros
 //! - `regexp` — examples of using regular expression functions
@@ -38,6 +39,7 @@ use std::str::FromStr;
 use datafusion::error::{DataFusionError, Result};
 
 enum ExampleKind {
+    All,
     DateTime,
     FunctionFactory,
     Regexp,
@@ -46,6 +48,7 @@ enum ExampleKind {
 impl AsRef<str> for ExampleKind {
     fn as_ref(&self) -> &str {
         match self {
+            Self::All => "all",
             Self::DateTime => "date_time",
             Self::FunctionFactory => "function_factory",
             Self::Regexp => "regexp",
@@ -58,6 +61,7 @@ impl FromStr for ExampleKind {
 
     fn from_str(s: &str) -> Result<Self> {
         match s {
+            "all" => Ok(Self::All),
             "date_time" => Ok(Self::DateTime),
             "function_factory" => Ok(Self::FunctionFactory),
             "regexp" => Ok(Self::Regexp),
@@ -67,12 +71,38 @@ impl FromStr for ExampleKind {
 }
 
 impl ExampleKind {
-    const ALL: [Self; 3] = [Self::DateTime, Self::FunctionFactory, Self::Regexp];
+    const ALL_VARIANTS: [Self; 4] = [
+        Self::All,
+        Self::DateTime,
+        Self::FunctionFactory,
+        Self::Regexp,
+    ];
+
+    const RUNNABLE_VARIANTS: [Self; 3] =
+        [Self::DateTime, Self::FunctionFactory, Self::Regexp];
 
     const EXAMPLE_NAME: &str = "builtin_functions";
 
     fn variants() -> Vec<&'static str> {
-        Self::ALL.iter().map(|x| x.as_ref()).collect()
+        Self::ALL_VARIANTS
+            .iter()
+            .map(|example| example.as_ref())
+            .collect()
+    }
+
+    async fn run(&self) -> Result<()> {
+        match self {
+            ExampleKind::All => {
+                for example in ExampleKind::RUNNABLE_VARIANTS {
+                    println!("Running example: {}", example.as_ref());
+                    Box::pin(example.run()).await?;
+                }
+            }
+            ExampleKind::DateTime => date_time::date_time().await?,
+            ExampleKind::FunctionFactory => function_factory::function_factory().await?,
+            ExampleKind::Regexp => regexp::regexp().await?,
+        }
+        Ok(())
     }
 }
 
@@ -89,11 +119,6 @@ async fn main() -> Result<()> {
         DataFusionError::Execution("Missing argument".to_string())
     })?;
 
-    match arg.parse::<ExampleKind>()? {
-        ExampleKind::DateTime => date_time::date_time().await?,
-        ExampleKind::FunctionFactory => function_factory::function_factory().await?,
-        ExampleKind::Regexp => regexp::regexp().await?,
-    }
-
-    Ok(())
+    let example = arg.parse::<ExampleKind>()?;
+    example.run().await
 }

datafusion-examples/examples/builtin_functions/regexp.rs

@@ -18,9 +18,12 @@
 
 //! See `main.rs` for how to run it.
 
+use std::{fs::File, io::Write};
+
 use datafusion::common::{assert_batches_eq, assert_contains};
 use datafusion::error::Result;
 use datafusion::prelude::*;
+use tempfile::tempdir;
 
 /// This example demonstrates how to use the regexp_* functions
 ///
@@ -32,12 +35,30 @@ use datafusion::prelude::*;
 /// https://docs.rs/regex/latest/regex/#grouping-and-flags
 pub async fn regexp() -> Result<()> {
     let ctx = SessionContext::new();
-    ctx.register_csv(
-        "examples",
-        "datafusion/physical-expr/tests/data/regex.csv",
-        CsvReadOptions::new(),
-    )
-    .await?;
+    // content from file 'datafusion/physical-expr/tests/data/regex.csv'
+    let csv_data = r#"values,patterns,replacement,flags
+abc,^(a),bb\1bb,i
+ABC,^(A).*,B,i
+aBc,(b|d),e,i
+AbC,(B|D),e,
+aBC,^(b|c),d,
+4000,\b4([1-9]\d\d|\d[1-9]\d|\d\d[1-9])\b,xyz,
+4010,\b4([1-9]\d\d|\d[1-9]\d|\d\d[1-9])\b,xyz,
+Düsseldorf,[\p{Letter}-]+,München,
+Москва,[\p{L}-]+,Moscow,
+Köln,[a-zA-Z]ö[a-zA-Z]{2},Koln,
+اليوم,^\p{Arabic}+$,Today,"#;
+    let dir = tempdir()?;
+    let file_path = dir.path().join("regex.csv");
+    {
+        let mut file = File::create(&file_path)?;
+        // write CSV data
+        file.write_all(csv_data.as_bytes())?;
+    } // scope closes the file
+    let file_path = file_path.to_str().unwrap();
+
+    ctx.register_csv("examples", file_path, CsvReadOptions::new())
+        .await?;
 
     //
     //

datafusion-examples/examples/custom_data_source/custom_file_casts.rs

@@ -32,7 +32,7 @@ use datafusion::datasource::listing::{
 use datafusion::execution::context::SessionContext;
 use datafusion::execution::object_store::ObjectStoreUrl;
 use datafusion::parquet::arrow::ArrowWriter;
-use datafusion::physical_expr::expressions::CastExpr;
+use datafusion::physical_expr::expressions::{CastColumnExpr, CastExpr};
 use datafusion::physical_expr::PhysicalExpr;
 use datafusion::prelude::SessionConfig;
 use datafusion_physical_expr_adapter::{
@@ -192,6 +192,19 @@ impl PhysicalExprAdapter for CustomCastsPhysicalExprAdapter {
                     );
                 }
             }
+            if let Some(cast) = expr.as_any().downcast_ref::<CastColumnExpr>() {
+                let input_data_type =
+                    cast.expr().data_type(&self.physical_file_schema)?;
+                let output_data_type = cast.data_type(&self.physical_file_schema)?;
+                if !CastExpr::check_bigger_cast(
+                    cast.target_field().data_type(),
+                    &input_data_type,
+                ) {
+                    return not_impl_err!(
+                        "Unsupported CAST from {input_data_type} to {output_data_type}"
+                    );
+                }
+            }
             Ok(Transformed::no(expr))
         })
         .data()

datafusion-examples/examples/custom_data_source/main.rs

@@ -21,10 +21,11 @@
 //!
 //! ## Usage
 //! ```bash
-//! cargo run --example custom_data_source -- [csv_json_opener|csv_sql_streaming|custom_datasource|custom_file_casts|custom_file_format|default_column_values|file_stream_provider]
+//! cargo run --example custom_data_source -- [all|csv_json_opener|csv_sql_streaming|custom_datasource|custom_file_casts|custom_file_format|default_column_values|file_stream_provider]
 //! ```
 //!
 //! Each subcommand runs a corresponding example:
+//! - `all` — run all examples included in this module
 //! - `csv_json_opener` — use low level FileOpener APIs to read CSV/JSON into Arrow RecordBatches
 //! - `csv_sql_streaming` — build and run a streaming query plan from a SQL statement against a local CSV file
 //! - `custom_datasource` — run queries against a custom datasource (TableProvider)
@@ -46,25 +47,27 @@ use std::str::FromStr;
 use datafusion::error::{DataFusionError, Result};
 
 enum ExampleKind {
+    All,
     CsvJsonOpener,
     CsvSqlStreaming,
     CustomDatasource,
     CustomFileCasts,
     CustomFileFormat,
     DefaultColumnValues,
-    FileFtreamProvider,
+    FileStreamProvider,
 }
 
 impl AsRef<str> for ExampleKind {
     fn as_ref(&self) -> &str {
         match self {
+            Self::All => "all",
             Self::CsvJsonOpener => "csv_json_opener",
             Self::CsvSqlStreaming => "csv_sql_streaming",
             Self::CustomDatasource => "custom_datasource",
             Self::CustomFileCasts => "custom_file_casts",
             Self::CustomFileFormat => "custom_file_format",
             Self::DefaultColumnValues => "default_column_values",
-            Self::FileFtreamProvider => "file_stream_provider",
+            Self::FileStreamProvider => "file_stream_provider",
         }
     }
 }
@@ -74,33 +77,79 @@ impl FromStr for ExampleKind {
 
     fn from_str(s: &str) -> Result<Self> {
         match s {
+            "all" => Ok(Self::All),
             "csv_json_opener" => Ok(Self::CsvJsonOpener),
             "csv_sql_streaming" => Ok(Self::CsvSqlStreaming),
             "custom_datasource" => Ok(Self::CustomDatasource),
             "custom_file_casts" => Ok(Self::CustomFileCasts),
             "custom_file_format" => Ok(Self::CustomFileFormat),
             "default_column_values" => Ok(Self::DefaultColumnValues),
-            "file_stream_provider" => Ok(Self::FileFtreamProvider),
+            "file_stream_provider" => Ok(Self::FileStreamProvider),
             _ => Err(DataFusionError::Execution(format!("Unknown example: {s}"))),
         }
     }
 }
 
 impl ExampleKind {
-    const ALL: [Self; 7] = [
+    const ALL_VARIANTS: [Self; 8] = [
+        Self::All,
         Self::CsvJsonOpener,
         Self::CsvSqlStreaming,
         Self::CustomDatasource,
         Self::CustomFileCasts,
         Self::CustomFileFormat,
         Self::DefaultColumnValues,
-        Self::FileFtreamProvider,
+        Self::FileStreamProvider,
+    ];
+
+    const RUNNABLE_VARIANTS: [Self; 7] = [
+        Self::CsvJsonOpener,
+        Self::CsvSqlStreaming,
+        Self::CustomDatasource,
+        Self::CustomFileCasts,
+        Self::CustomFileFormat,
+        Self::DefaultColumnValues,
+        Self::FileStreamProvider,
     ];
 
     const EXAMPLE_NAME: &str = "custom_data_source";
 
     fn variants() -> Vec<&'static str> {
-        Self::ALL.iter().map(|x| x.as_ref()).collect()
+        Self::ALL_VARIANTS
+            .iter()
+            .map(|example| example.as_ref())
+            .collect()
+    }
+
+    async fn run(&self) -> Result<()> {
+        match self {
+            ExampleKind::All => {
+                for example in ExampleKind::RUNNABLE_VARIANTS {
+                    println!("Running example: {}", example.as_ref());
+                    Box::pin(example.run()).await?;
+                }
+            }
+            ExampleKind::CsvJsonOpener => csv_json_opener::csv_json_opener().await?,
+            ExampleKind::CsvSqlStreaming => {
+                csv_sql_streaming::csv_sql_streaming().await?
+            }
+            ExampleKind::CustomDatasource => {
+                custom_datasource::custom_datasource().await?
+            }
+            ExampleKind::CustomFileCasts => {
+                custom_file_casts::custom_file_casts().await?
+            }
+            ExampleKind::CustomFileFormat => {
+                custom_file_format::custom_file_format().await?
+            }
+            ExampleKind::DefaultColumnValues => {
+                default_column_values::default_column_values().await?
+            }
+            ExampleKind::FileStreamProvider => {
+                file_stream_provider::file_stream_provider().await?
+            }
+        }
+        Ok(())
     }
 }
 
@@ -117,19 +166,6 @@ async fn main() -> Result<()> {
         DataFusionError::Execution("Missing argument".to_string())
     })?;
 
-    match arg.parse::<ExampleKind>()? {
-        ExampleKind::CsvJsonOpener => csv_json_opener::csv_json_opener().await?,
-        ExampleKind::CsvSqlStreaming => csv_sql_streaming::csv_sql_streaming().await?,
-        ExampleKind::CustomDatasource => custom_datasource::custom_datasource().await?,
-        ExampleKind::CustomFileCasts => custom_file_casts::custom_file_casts().await?,
-        ExampleKind::CustomFileFormat => custom_file_format::custom_file_format().await?,
-        ExampleKind::DefaultColumnValues => {
-            default_column_values::default_column_values().await?
-        }
-        ExampleKind::FileFtreamProvider => {
-            file_stream_provider::file_stream_provider().await?
-        }
-    }
-
-    Ok(())
+    let example = arg.parse::<ExampleKind>()?;
+    example.run().await
 }