From e6cb54d323ccc5d33d461751e6389b5ec2a45bd8 Mon Sep 17 00:00:00 2001 From: Brendan Collins Date: Thu, 18 Dec 2025 07:09:01 -0800 Subject: [PATCH 1/4] added downloaded examples directory to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index d2576a70..0a630cf7 100644 --- a/.gitignore +++ b/.gitignore @@ -95,3 +95,4 @@ dmypy.json # airspeed velocity .asv/ +xrspatial-examples/ From 03e60fcb10de95dc353990ac8663870fff319a59 Mon Sep 17 00:00:00 2001 From: Brendan Collins Date: Thu, 18 Dec 2025 11:23:57 -0800 Subject: [PATCH 2/4] Dask in now a true optional dep. --- xrspatial/aspect.py | 11 ++++-- xrspatial/classify.py | 8 ++++- xrspatial/curvature.py | 8 ++++- xrspatial/datasets/__init__.py | 10 +++++- xrspatial/focal.py | 12 ++++++- xrspatial/hillshade.py | 11 ++++-- xrspatial/multispectral.py | 8 ++++- xrspatial/perlin.py | 8 ++++- xrspatial/proximity.py | 8 +++-- xrspatial/slope.py | 13 ++++--- xrspatial/terrain.py | 7 +++- xrspatial/tests/general_checks.py | 33 +++++++++++++----- xrspatial/tests/test_aspect.py | 7 +++- xrspatial/tests/test_classify.py | 10 ++++++ xrspatial/tests/test_curvature.py | 9 +++-- xrspatial/tests/test_datasets.py | 6 +++- xrspatial/tests/test_focal.py | 37 ++++++++++++++------ xrspatial/tests/test_hillshade.py | 8 +++-- xrspatial/tests/test_multispectral.py | 10 +++++- xrspatial/tests/test_perlin.py | 14 ++++++-- xrspatial/tests/test_proximity.py | 8 +++-- xrspatial/tests/test_slope.py | 4 ++- xrspatial/tests/test_terrain.py | 18 ++++++++-- xrspatial/tests/test_utils.py | 2 ++ xrspatial/tests/test_zonal.py | 50 +++++++++++++++++++++++---- xrspatial/utils.py | 32 +++++++++++++---- xrspatial/zonal.py | 30 ++++++++++++---- 27 files changed, 312 insertions(+), 70 deletions(-) diff --git a/xrspatial/aspect.py b/xrspatial/aspect.py index aa1f46d5..33ed3f37 100644 --- a/xrspatial/aspect.py +++ b/xrspatial/aspect.py @@ -1,13 +1,20 @@ +from __future__ import annotations + from functools import partial from math import atan2 from typing import Optional -import dask.array as da +try: + import dask.array as da +except ImportError: + da = None + + import numpy as np import xarray as xr from numba import cuda -from xrspatial.utils import ArrayTypeFunctionMapping, cuda_args, ngjit, not_implemented_func +from xrspatial.utils import ArrayTypeFunctionMapping, cuda_args, ngjit # 3rd-party try: diff --git a/xrspatial/classify.py b/xrspatial/classify.py index 9ec6b561..05fe83c5 100644 --- a/xrspatial/classify.py +++ b/xrspatial/classify.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import warnings from functools import partial from typing import List, Optional @@ -12,7 +14,11 @@ class cupy(object): ndarray = False -import dask.array as da +try: + import dask.array as da +except ImportError: + da = None + import numba as nb import numpy as np diff --git a/xrspatial/curvature.py b/xrspatial/curvature.py index 3a1292a4..fb8708a1 100644 --- a/xrspatial/curvature.py +++ b/xrspatial/curvature.py @@ -1,3 +1,5 @@ +from __future__ import annotations + # std lib from functools import partial from typing import Optional, Union @@ -9,7 +11,11 @@ class cupy(object): ndarray = False -import dask.array as da +try: + import dask.array as da +except ImportError: + da = None + import numpy as np import xarray as xr from numba import cuda diff --git a/xrspatial/datasets/__init__.py b/xrspatial/datasets/__init__.py index 9f061dd5..8bfea5b2 100644 --- a/xrspatial/datasets/__init__.py +++ b/xrspatial/datasets/__init__.py @@ -1,6 +1,10 @@ import os -import dask.array as da +try: + import dask.array as da +except ImportError: + da = None + import datashader as ds import noise import numpy as np @@ -76,6 +80,10 @@ def make_terrain( terrain : xarray.DataArray 2D array of generated terrain values. """ + + if da is None: + raise Exception("make terrain requires dask.Array (pip install dask)") + def _func(arr, block_id=None): block_ystart = block_id[0] * arr.shape[0] block_xstart = block_id[1] * arr.shape[1] diff --git a/xrspatial/focal.py b/xrspatial/focal.py index 7e29b300..79a595cf 100644 --- a/xrspatial/focal.py +++ b/xrspatial/focal.py @@ -1,16 +1,26 @@ +from __future__ import annotations + + import copy from functools import partial from math import isnan import math -import dask.array as da import numba as nb import numpy as np import pandas as pd import xarray as xr + from numba import cuda, prange from xarray import DataArray + +try: + import dask.array as da +except ImportError: + da = None + + try: import cupy except ImportError: diff --git a/xrspatial/hillshade.py b/xrspatial/hillshade.py index fa9a85dc..137be95a 100644 --- a/xrspatial/hillshade.py +++ b/xrspatial/hillshade.py @@ -2,8 +2,13 @@ from functools import partial from typing import Optional -import dask.array as da import numpy as np + +try: + import dask.array as da +except ImportError: + da = None + import xarray as xr from numba import cuda @@ -178,12 +183,12 @@ def hillshade(agg: xr.DataArray, out = _run_cupy(agg.data, azimuth, angle_altitude) # dask + cupy case - elif (has_cuda_and_cupy() and isinstance(agg.data, da.Array) and + elif (has_cuda_and_cupy() and da is not None and isinstance(agg.data, da.Array) and is_cupy_backed(agg)): raise NotImplementedError("Dask/CuPy hillshade not implemented") # dask + numpy case - elif isinstance(agg.data, da.Array): + elif da is not None and isinstance(agg.data, da.Array): out = _run_dask_numpy(agg.data, azimuth, angle_altitude) else: diff --git a/xrspatial/multispectral.py b/xrspatial/multispectral.py index 857e2ca9..c0852866 100644 --- a/xrspatial/multispectral.py +++ b/xrspatial/multispectral.py @@ -1,7 +1,8 @@ +from __future__ import annotations + import warnings from math import sqrt -import dask.array as da import numba as nb import numpy as np import xarray as xr @@ -18,6 +19,11 @@ class cupy(object): ndarray = False +try: + import dask.array as da +except ImportError: + da = None + @ngjit def _arvi_cpu(nir_data, red_data, blue_data): diff --git a/xrspatial/perlin.py b/xrspatial/perlin.py index dd9dd8f9..8e5bf18e 100644 --- a/xrspatial/perlin.py +++ b/xrspatial/perlin.py @@ -1,3 +1,5 @@ +from __future__ import annotations + # std lib from functools import partial @@ -11,7 +13,11 @@ class cupy(object): ndarray = False -import dask.array as da +try: + import dask.array as da +except ImportError: + da = None + import numba as nb from numba import cuda, jit diff --git a/xrspatial/proximity.py b/xrspatial/proximity.py index 5fc4e7d7..9a2f639c 100644 --- a/xrspatial/proximity.py +++ b/xrspatial/proximity.py @@ -1,6 +1,10 @@ from math import sqrt -import dask.array as da +try: + import dask.array as da +except ImportError: + da = None + import numpy as np import xarray as xr from numba import prange @@ -619,7 +623,7 @@ def _process_dask(raster, xs, ys): # numpy case result = _process_numpy(raster.data, xs, ys) - elif isinstance(raster.data, da.Array): + elif da is not None and isinstance(raster.data, da.Array): # dask + numpy case xs = da.from_array(xs, chunks=(raster.chunks)) ys = da.from_array(ys, chunks=(raster.chunks)) diff --git a/xrspatial/slope.py b/xrspatial/slope.py index 3abef6fe..042dfd27 100644 --- a/xrspatial/slope.py +++ b/xrspatial/slope.py @@ -1,3 +1,5 @@ +from __future__ import annotations + # std lib from functools import partial from math import atan @@ -10,14 +12,17 @@ class cupy(object): ndarray = False -import dask.array as da +try: + import dask.array as da +except ImportError: + da = None + import numpy as np import xarray as xr from numba import cuda # local modules -from xrspatial.utils import (ArrayTypeFunctionMapping, cuda_args, get_dataarray_resolution, ngjit, - not_implemented_func) +from xrspatial.utils import (ArrayTypeFunctionMapping, cuda_args, get_dataarray_resolution, ngjit) @ngjit @@ -64,6 +69,7 @@ def _run_dask_numpy(data: da.Array, meta=np.array(())) return out + def _run_dask_cupy(data: da.Array, cellsize_x: Union[int, float], cellsize_y: Union[int, float]) -> da.Array: @@ -79,7 +85,6 @@ def _run_dask_cupy(data: da.Array, return out - @cuda.jit(device=True) def _gpu(arr, cellsize_x, cellsize_y): a = arr[2, 0] diff --git a/xrspatial/terrain.py b/xrspatial/terrain.py index 3140653c..2d87a391 100644 --- a/xrspatial/terrain.py +++ b/xrspatial/terrain.py @@ -1,3 +1,5 @@ +from __future__ import annotations + # std lib from functools import partial from typing import List, Optional, Tuple, Union @@ -14,7 +16,10 @@ class cupy(object): ndarray = False -import dask.array as da +try: + import dask.array as da +except ImportError: + da = None # local modules from xrspatial.utils import (ArrayTypeFunctionMapping, cuda_args, get_dataarray_resolution, diff --git a/xrspatial/tests/general_checks.py b/xrspatial/tests/general_checks.py index f5f9bfba..75011fd6 100644 --- a/xrspatial/tests/general_checks.py +++ b/xrspatial/tests/general_checks.py @@ -1,15 +1,32 @@ -import dask.array as da +from __future__ import annotations +try: + import dask.array as da +except ImportError: + da = None + import numpy as np import pytest import xarray as xr -from xrspatial.utils import ArrayTypeFunctionMapping, has_cuda_and_cupy +from xrspatial.utils import ArrayTypeFunctionMapping +from xrspatial.utils import has_cuda_and_cupy +from xrspatial.utils import has_dask_array +from xrspatial.utils import has_dask_dataframe # Use this as a decorator to skip tests if do not have both CUDA and CuPy available. cuda_and_cupy_available = pytest.mark.skipif( not has_cuda_and_cupy(), reason="Requires CUDA and CuPy") +# Use this as a decorator to skip tests if do not have dask array +dask_array_available = pytest.mark.skipif( + not has_dask_array(), reason="Requires dask.Array") + +# Use this as a decorator to skip tests if do not have dask array +dask_dataframe_available = pytest.mark.skipif( + not has_dask_dataframe(), reason="Requires dask.DataFrame") + + def create_test_raster( data, backend='numpy', @@ -36,7 +53,7 @@ def create_test_raster( import cupy raster.data = cupy.asarray(raster.data) - if 'dask' in backend: + if 'dask' in backend and has_dask_array(): raster.data = da.from_array(raster.data, chunks=chunks) return raster @@ -52,7 +69,7 @@ def general_output_checks(input_agg: xr.DataArray, # type of output is the same as of input assert isinstance(output_agg.data, type(input_agg.data)) - if isinstance(input_agg.data, da.Array): + if has_dask_array() and isinstance(input_agg.data, da.Array): # dask case assert isinstance( output_agg.data.compute(), type(input_agg.data.compute())) @@ -123,13 +140,13 @@ def assert_numpy_equals_cupy(numpy_agg, cupy_agg, func, nan_edges=True, atol=0, numpy_result.data, cupy_result.data.get(), equal_nan=True, atol=atol, rtol=rtol) -def assert_numpy_equals_dask_cupy(numpy_agg, dask_cupy_agg, func, nan_edges=True, atol=0, rtol=1e-7): +def assert_numpy_equals_dask_cupy(numpy_agg, dask_cupy_agg, func, + nan_edges=True, atol=0, rtol=1e-7): numpy_result = func(numpy_agg) if nan_edges: assert_nan_edges_effect(numpy_result) dask_cupy_result = func(dask_cupy_agg) general_output_checks(dask_cupy_agg, dask_cupy_result) - np.testing.assert_allclose( - numpy_result.data, dask_cupy_result.data.compute().get(), equal_nan=True, atol=atol, rtol=rtol - ) + np.testing.assert_allclose(numpy_result.data, dask_cupy_result.data.compute().get(), + equal_nan=True, atol=atol, rtol=rtol) diff --git a/xrspatial/tests/test_aspect.py b/xrspatial/tests/test_aspect.py index 39fb1cfb..61c04bb7 100644 --- a/xrspatial/tests/test_aspect.py +++ b/xrspatial/tests/test_aspect.py @@ -5,7 +5,9 @@ from xrspatial.tests.general_checks import (assert_nan_edges_effect, assert_numpy_equals_cupy, assert_numpy_equals_dask_cupy, - assert_numpy_equals_dask_numpy, create_test_raster, + assert_numpy_equals_dask_numpy, + dask_array_available, + create_test_raster, cuda_and_cupy_available, general_output_checks) @@ -45,6 +47,7 @@ def test_numpy_equals_qgis(elevation_raster, qgis_aspect): assert_nan_edges_effect(xrspatial_aspect) +@dask_array_available def test_numpy_equals_dask_qgis_data(elevation_raster): # compare using the data run through QGIS numpy_agg = input_data(elevation_raster, 'numpy') @@ -52,6 +55,7 @@ def test_numpy_equals_dask_qgis_data(elevation_raster): assert_numpy_equals_dask_numpy(numpy_agg, dask_agg, aspect) +@dask_array_available @pytest.mark.parametrize("size", [(2, 4), (10, 15)]) @pytest.mark.parametrize( "dtype", [np.int32, np.int64, np.uint32, np.uint64, np.float32, np.float64]) @@ -79,6 +83,7 @@ def test_numpy_equals_cupy_random_data(random_data): assert_numpy_equals_cupy(numpy_agg, cupy_agg, aspect, atol=1e-6, rtol=1e-6) +@dask_array_available @cuda_and_cupy_available @pytest.mark.parametrize("size", [(2, 4), (10, 15)]) @pytest.mark.parametrize( diff --git a/xrspatial/tests/test_classify.py b/xrspatial/tests/test_classify.py index 0f3d6200..08f66296 100644 --- a/xrspatial/tests/test_classify.py +++ b/xrspatial/tests/test_classify.py @@ -6,6 +6,11 @@ from xrspatial.tests.general_checks import (create_test_raster, cuda_and_cupy_available, general_output_checks) +try: + import dask.array as da +except ImportError: + da = None + def input_data(backend='numpy'): elevation = np.array([ @@ -37,6 +42,7 @@ def test_binary_numpy(result_binary): general_output_checks(numpy_agg, numpy_result, expected_result) +@pytest.mark.skipif(da is not None, reason="dask is not installed") def test_binary_dask_numpy(result_binary): values, expected_result = result_binary dask_agg = input_data(backend='dask') @@ -89,6 +95,7 @@ def test_reclassify_numpy(result_reclassify): general_output_checks(numpy_agg, numpy_result, expected_result, verify_dtype=True) +@pytest.mark.skipif(da is not None, reason="dask is not installed") def test_reclassify_dask_numpy(result_reclassify): bins, new_values, expected_result = result_reclassify dask_agg = input_data(backend='dask') @@ -105,6 +112,7 @@ def test_reclassify_cupy(result_reclassify): @cuda_and_cupy_available +@pytest.mark.skipif(da is not None, reason="dask is not installed") def test_reclassify_dask_cupy(result_reclassify): bins, new_values, expected_result = result_reclassify dask_cupy_agg = input_data(backend='dask+cupy') @@ -140,6 +148,7 @@ def test_quantile_numpy(result_quantile): general_output_checks(numpy_agg, numpy_quantile, expected_result, verify_dtype=True) +@pytest.mark.skipif(da is not None, reason="dask is not installed") def test_quantile_dask_numpy(result_quantile): # Note that dask's percentile algorithm is # approximate, while numpy's is exact. @@ -258,6 +267,7 @@ def test_equal_interval_numpy(result_equal_interval): general_output_checks(numpy_agg, numpy_result, expected_result, verify_dtype=True) +@pytest.mark.skipif(da is not None, reason="dask is not installed") def test_equal_interval_dask_numpy(result_equal_interval): k, expected_result = result_equal_interval dask_agg = input_data('dask+numpy') diff --git a/xrspatial/tests/test_curvature.py b/xrspatial/tests/test_curvature.py index d10c18ca..3f7049be 100644 --- a/xrspatial/tests/test_curvature.py +++ b/xrspatial/tests/test_curvature.py @@ -4,8 +4,11 @@ from xrspatial import curvature from xrspatial.tests.general_checks import (assert_numpy_equals_cupy, assert_numpy_equals_dask_cupy, - assert_numpy_equals_dask_numpy, create_test_raster, - cuda_and_cupy_available, general_output_checks) + assert_numpy_equals_dask_numpy, + create_test_raster, + cuda_and_cupy_available, + dask_array_available, + general_output_checks) @pytest.fixture @@ -90,6 +93,7 @@ def test_numpy_equals_cupy_random_data(random_data): assert_numpy_equals_cupy(numpy_agg, cupy_agg, curvature) +@dask_array_available @pytest.mark.parametrize("size", [(2, 4), (10, 15)]) @pytest.mark.parametrize( "dtype", [np.int32, np.int64, np.uint32, np.uint64, np.float32, np.float64]) @@ -99,6 +103,7 @@ def test_numpy_equals_dask_random_data(random_data): assert_numpy_equals_dask_numpy(numpy_agg, dask_agg, curvature) +@dask_array_available @cuda_and_cupy_available @pytest.mark.parametrize("size", [(2, 4), (10, 15)]) @pytest.mark.parametrize( diff --git a/xrspatial/tests/test_datasets.py b/xrspatial/tests/test_datasets.py index 22d4490f..1de64f25 100644 --- a/xrspatial/tests/test_datasets.py +++ b/xrspatial/tests/test_datasets.py @@ -1,10 +1,14 @@ -import dask.array as da import xarray as xr from xrspatial.datasets import make_terrain +from xrspatial.tests.general_checks import dask_array_available + +@dask_array_available def test_make_terrain(): + import dask.array as da + terrain = make_terrain() assert terrain is not None assert isinstance(terrain, xr.DataArray) diff --git a/xrspatial/tests/test_focal.py b/xrspatial/tests/test_focal.py index 1ab72287..6dcdf989 100644 --- a/xrspatial/tests/test_focal.py +++ b/xrspatial/tests/test_focal.py @@ -1,4 +1,8 @@ -import dask.array as da +try: + import dask.array as da +except ImportError: + da = None + import numpy as np import pytest import xarray as xr @@ -7,7 +11,9 @@ from xrspatial.convolution import (annulus_kernel, calc_cellsize, circle_kernel, convolution_2d, convolve_2d, custom_kernel) from xrspatial.focal import apply, focal_stats, hotspots -from xrspatial.tests.general_checks import (create_test_raster, cuda_and_cupy_available, +from xrspatial.tests.general_checks import (create_test_raster, + cuda_and_cupy_available, + dask_array_available, general_output_checks) from xrspatial.utils import ngjit @@ -47,6 +53,14 @@ def test_mean_transfer_function_cpu(): numpy_mean = mean(numpy_agg) general_output_checks(numpy_agg, numpy_mean) + +@dask_array_available +def test_mean_transfer_function_dask_cpu(): + # numpy case + numpy_agg = xr.DataArray(data_random) + numpy_mean = mean(numpy_agg) + general_output_checks(numpy_agg, numpy_mean) + # dask + numpy case dask_numpy_agg = xr.DataArray(da.from_array(data_random, chunks=(3, 3))) dask_numpy_mean = mean(dask_numpy_agg) @@ -199,6 +213,7 @@ def test_convolution_numpy( ) +@dask_array_available def test_convolution_dask_numpy( convolve_2d_data, convolution_custom_kernel, @@ -261,14 +276,16 @@ def test_2d_convolution_gpu( ) # dask + cupy case not implemented - dask_cupy_agg = xr.DataArray( - da.from_array(cupy.asarray(convolve_2d_data), chunks=(3, 3)) - ) - result_kernel_annulus = convolve_2d(dask_cupy_agg.data, kernel_annulus_2_2_2_1) - assert isinstance(result_kernel_annulus, da.Array) - np.testing.assert_allclose( - result_kernel_annulus.compute().get(), convolution_kernel_annulus_2_2_1, equal_nan=True - ) + # TODO: break this into its own test. + if da is not None: + dask_cupy_agg = xr.DataArray( + da.from_array(cupy.asarray(convolve_2d_data), chunks=(3, 3)) + ) + result_kernel_annulus = convolve_2d(dask_cupy_agg.data, kernel_annulus_2_2_2_1) + assert isinstance(result_kernel_annulus, da.Array) + np.testing.assert_allclose( + result_kernel_annulus.compute().get(), convolution_kernel_annulus_2_2_1, equal_nan=True + ) def test_calc_cellsize_unit_input_attrs(convolve_2d_data): diff --git a/xrspatial/tests/test_hillshade.py b/xrspatial/tests/test_hillshade.py index 0d4cf07f..dc30e794 100644 --- a/xrspatial/tests/test_hillshade.py +++ b/xrspatial/tests/test_hillshade.py @@ -5,8 +5,11 @@ from xrspatial import hillshade from xrspatial.tests.general_checks import (assert_numpy_equals_cupy, - assert_numpy_equals_dask_numpy, create_test_raster, - cuda_and_cupy_available, general_output_checks) + assert_numpy_equals_dask_numpy, + create_test_raster, + cuda_and_cupy_available, + dask_array_available, + general_output_checks) from ..gpu_rtx import has_rtx @@ -36,6 +39,7 @@ def test_hillshade(data_gaussian): assert da_gaussian_shade[60, 60] > 0 +@dask_array_available @pytest.mark.parametrize("size", [(2, 4), (10, 15)]) @pytest.mark.parametrize( "dtype", [np.int32, np.int64, np.float32, np.float64]) diff --git a/xrspatial/tests/test_multispectral.py b/xrspatial/tests/test_multispectral.py index 72657c97..8248b558 100644 --- a/xrspatial/tests/test_multispectral.py +++ b/xrspatial/tests/test_multispectral.py @@ -5,6 +5,7 @@ from xrspatial.multispectral import (arvi, ebbi, evi, gci, nbr, nbr2, ndmi, ndvi, savi, sipi, true_color) from xrspatial.tests.general_checks import (create_test_raster, cuda_and_cupy_available, + dask_array_available, general_output_checks) @@ -358,12 +359,19 @@ def test_ndvi_data_contains_valid_values(): assert da_ndvi[15, 10] == da_ndvi[10, 15] == 0.5 -@pytest.mark.parametrize("backend", ["numpy", "dask+numpy"]) +@pytest.mark.parametrize("backend", ["numpy"]) def test_ndvi_cpu_against_qgis(nir_data, red_data, qgis_ndvi): result = ndvi(nir_data, red_data) general_output_checks(nir_data, result, qgis_ndvi, verify_dtype=True) +@dask_array_available +@pytest.mark.parametrize("backend", ["dask+numpy"]) +def test_ndvi_dask_cpu_against_qgis(nir_data, red_data, qgis_ndvi): + result = ndvi(nir_data, red_data) + general_output_checks(nir_data, result, qgis_ndvi, verify_dtype=True) + + @pytest.mark.parametrize("dtype", ["uint8", "uint16"]) def test_ndvi_uint_dtype(data_uint_dtype_normalized_ratio): nir_data, red_data, result_ndvi = data_uint_dtype_normalized_ratio diff --git a/xrspatial/tests/test_perlin.py b/xrspatial/tests/test_perlin.py index 70bce320..d5236a3a 100644 --- a/xrspatial/tests/test_perlin.py +++ b/xrspatial/tests/test_perlin.py @@ -1,9 +1,11 @@ -import dask.array as da import numpy as np import xarray as xr from xrspatial import perlin -from xrspatial.tests.general_checks import cuda_and_cupy_available, general_output_checks +from xrspatial.tests.general_checks import cuda_and_cupy_available +from xrspatial.tests.general_checks import dask_array_available +from xrspatial.tests.general_checks import general_output_checks + from xrspatial.utils import has_cuda_and_cupy @@ -29,6 +31,14 @@ def test_perlin_cpu(): perlin_numpy = perlin(data_numpy) general_output_checks(data_numpy, perlin_numpy) + +@dask_array_available +def test_perlin_dask_cpu(): + # vanilla numpy version + data_numpy = create_test_arr() + perlin_numpy = perlin(data_numpy) + general_output_checks(data_numpy, perlin_numpy) + # dask data_dask = create_test_arr(backend='dask') perlin_dask = perlin(data_dask) diff --git a/xrspatial/tests/test_proximity.py b/xrspatial/tests/test_proximity.py index 3d732ca0..837e297b 100644 --- a/xrspatial/tests/test_proximity.py +++ b/xrspatial/tests/test_proximity.py @@ -1,4 +1,8 @@ -import dask.array as da +try: + import dask.array as da +except ImportError: + da = None + import numpy as np import pytest import xarray as xr @@ -32,7 +36,7 @@ def test_raster(backend): raster = xr.DataArray(data, dims=['lat', 'lon']) raster['lon'] = _lon raster['lat'] = _lat - if 'dask' in backend: + if 'dask' in backend and da is not None: raster.data = da.from_array(data, chunks=(4, 3)) return raster diff --git a/xrspatial/tests/test_slope.py b/xrspatial/tests/test_slope.py index df043f4f..aafd1c0b 100644 --- a/xrspatial/tests/test_slope.py +++ b/xrspatial/tests/test_slope.py @@ -5,7 +5,8 @@ from xrspatial.tests.general_checks import (assert_nan_edges_effect, assert_numpy_equals_cupy, assert_numpy_equals_dask_cupy, assert_numpy_equals_dask_numpy, create_test_raster, - cuda_and_cupy_available, general_output_checks) + cuda_and_cupy_available, + dask_array_available, general_output_checks) def input_data(data, backend): @@ -51,6 +52,7 @@ def test_numpy_equals_qgis(elevation_raster, qgis_slope): assert_nan_edges_effect(xrspatial_slope_numpy) +@dask_array_available def test_numpy_equals_dask_qgis_data(elevation_raster): # compare using the data run through QGIS numpy_agg = input_data(elevation_raster, 'numpy') diff --git a/xrspatial/tests/test_terrain.py b/xrspatial/tests/test_terrain.py index 00296fd1..c39694b5 100644 --- a/xrspatial/tests/test_terrain.py +++ b/xrspatial/tests/test_terrain.py @@ -1,9 +1,14 @@ -import dask.array as da +try: + import dask.array as da +except ImportError: + da = None + import numpy as np import xarray as xr from xrspatial import generate_terrain from xrspatial.tests.general_checks import cuda_and_cupy_available +from xrspatial.tests.general_checks import dask_array_available from xrspatial.utils import has_cuda_and_cupy @@ -17,7 +22,8 @@ def create_test_arr(backend='numpy'): import cupy raster.data = cupy.asarray(raster.data) - if 'dask' in backend: + # TODO: restructure dask test cases to use skips if da is None + if 'dask' in backend and da is not None: raster.data = da.from_array(raster.data, chunks=(10, 10)) return raster @@ -27,8 +33,14 @@ def test_terrain_cpu(): # vanilla numpy version data_numpy = create_test_arr() terrain_numpy = generate_terrain(data_numpy) + assert isinstance(terrain_numpy, xr.DataArray) + - # dask +@dask_array_available +def test_terrain_dask_cpu(): + # vanilla numpy version + data_numpy = create_test_arr() + terrain_numpy = generate_terrain(data_numpy) data_dask = create_test_arr(backend='dask') terrain_dask = generate_terrain(data_dask) assert isinstance(terrain_dask.data, da.Array) diff --git a/xrspatial/tests/test_utils.py b/xrspatial/tests/test_utils.py index be7cc685..1a3fcd41 100644 --- a/xrspatial/tests/test_utils.py +++ b/xrspatial/tests/test_utils.py @@ -1,7 +1,9 @@ from xrspatial.datasets import make_terrain from xrspatial.utils import canvas_like +from xrspatial.tests.general_checks import dask_array_available +@dask_array_available def test_canvas_like(): # aspect ratio is 1:1 terrain_shape = (1000, 1000) diff --git a/xrspatial/tests/test_zonal.py b/xrspatial/tests/test_zonal.py index a827427d..47eb4faa 100644 --- a/xrspatial/tests/test_zonal.py +++ b/xrspatial/tests/test_zonal.py @@ -1,7 +1,15 @@ import copy -import dask.array as da -import dask.dataframe as dd +try: + import dask.array as da +except ImportError: + da = None + +try: + import dask.dataframe as dd +except ImportError: + dd = None + import numpy as np import pandas as pd import pytest @@ -14,7 +22,8 @@ from xrspatial.zonal import regions from .general_checks import ( - assert_input_data_unmodified, create_test_raster, general_output_checks, has_cuda_and_cupy + assert_input_data_unmodified, create_test_raster, general_output_checks, has_cuda_and_cupy, + dask_array_available, has_dask_array, has_dask_dataframe ) @@ -41,7 +50,7 @@ def data_values_2d(backend): @pytest.fixture def data_values_3d(backend): data = np.ones(4*3*8).reshape(3, 8, 4) - if 'dask' in backend: + if has_dask_array() and 'dask' in backend: data = da.from_array(data, chunks=(3, 4, 2)) agg = xr.DataArray(data, dims=['lat', 'lon', 'race']) @@ -336,7 +345,7 @@ def qgis_zonal_stats(): def check_results( backend, df_result, expected_results_dict, rtol=1e-05, atol=1e-07, equal_nan=True ): - if 'dask' in backend: + if has_dask_dataframe() and 'dask' in backend: # dask case, compute result assert isinstance(df_result, dd.DataFrame) df_result = df_result.compute() @@ -357,6 +366,9 @@ def test_default_stats(backend, data_zones, data_values_2d, result_default_stats if backend == 'cupy' and not has_cuda_and_cupy(): pytest.skip("Requires CUDA and CuPy") + if 'dask' in backend and not dask_array_available(): + pytest.skip("Requires Dask") + # copy input data to verify they're unchanged after running the function copied_data_zones = copy.deepcopy(data_zones) copied_data_values_2d = copy.deepcopy(data_values_2d) @@ -389,11 +401,15 @@ def test_default_stats_dataarray( assert_input_data_unmodified(data_zones, copied_data_zones) assert_input_data_unmodified(data_values_2d, copied_data_values_2d) + @pytest.mark.parametrize("backend", ['numpy', 'dask+numpy', 'cupy']) def test_zone_ids_stats(backend, data_zones, data_values_2d, result_zone_ids_stats): if backend == 'cupy' and not has_cuda_and_cupy(): pytest.skip("Requires CUDA and CuPy") + if 'dask' in backend and not dask_array_available(): + pytest.skip("Requires Dask") + # copy input data to verify they're unchanged after running the function copied_data_zones = copy.deepcopy(data_zones) copied_data_values_2d = copy.deepcopy(data_values_2d) @@ -431,6 +447,9 @@ def test_custom_stats(backend, data_zones, data_values_2d, result_custom_stats): if backend == 'cupy' and not has_cuda_and_cupy(): pytest.skip("Requires CUDA and CuPy") + if 'dask' in backend and not dask_array_available(): + pytest.skip("Requires Dask") + # copy input data to verify they're unchanged after running the function copied_data_zones = copy.deepcopy(data_zones) copied_data_values_2d = copy.deepcopy(data_values_2d) @@ -488,11 +507,14 @@ def test_zonal_stats_inputs_unmodified(backend, data_zones, data_values_2d, resu if backend == 'cupy' and not has_cuda_and_cupy(): pytest.skip("Requires CUDA and CuPy") + if 'dask' in backend and not dask_array_available(): + pytest.skip("Requires Dask") + # copy input data to verify they're unchanged after running the function copied_data_zones = copy.deepcopy(data_zones) copied_data_values_2d = copy.deepcopy(data_values_2d) - df_result = stats(zones=data_zones, values=data_values_2d) + _ = stats(zones=data_zones, values=data_values_2d) assert_input_data_unmodified(data_zones, copied_data_zones) assert_input_data_unmodified(data_values_2d, copied_data_values_2d) @@ -501,6 +523,10 @@ def test_zonal_stats_inputs_unmodified(backend, data_zones, data_values_2d, resu @pytest.mark.parametrize("backend", ['numpy', 'dask+numpy']) def test_count_crosstab_2d(backend, data_zones, data_values_2d, result_count_crosstab_2d): # copy input data to verify they're unchanged after running the function + + if 'dask' in backend and not dask_array_available(): + pytest.skip("Requires Dask") + copied_data_zones = copy.deepcopy(data_zones) copied_data_values_2d = copy.deepcopy(data_values_2d) @@ -516,6 +542,10 @@ def test_count_crosstab_2d(backend, data_zones, data_values_2d, result_count_cro @pytest.mark.parametrize("backend", ['numpy', 'dask+numpy']) def test_percentage_crosstab_2d(backend, data_zones, data_values_2d, result_percentage_crosstab_2d): # copy input data to verify they're unchanged after running the function + + if 'dask' in backend and not dask_array_available(): + pytest.skip("Requires Dask") + copied_data_zones = copy.deepcopy(data_zones) copied_data_values_2d = copy.deepcopy(data_values_2d) @@ -531,6 +561,10 @@ def test_percentage_crosstab_2d(backend, data_zones, data_values_2d, result_perc @pytest.mark.parametrize("backend", ['numpy', 'dask+numpy']) def test_crosstab_3d_count(backend, data_zones, data_values_3d, result_crosstab_3d): + + if 'dask' in backend and not dask_array_available(): + pytest.skip("Requires Dask") + # copy input data to verify they're unchanged after running the function copied_data_zones = copy.deepcopy(data_zones) copied_data_values_3d = copy.deepcopy(data_values_3d) @@ -566,6 +600,10 @@ def test_nodata_values_crosstab_3d( data_values_3d, result_nodata_values_crosstab_3d ): + + if 'dask' in backend and not dask_array_available(): + pytest.skip("Requires Dask") + # copy input data to verify they're unchanged after running the function copied_data_zones = copy.deepcopy(data_zones) copied_data_values_3d = copy.deepcopy(data_values_3d) diff --git a/xrspatial/utils.py b/xrspatial/utils.py index e7e2c4f9..9bff209f 100644 --- a/xrspatial/utils.py +++ b/xrspatial/utils.py @@ -1,6 +1,7 @@ +from __future__ import annotations + from math import ceil -import dask.array as da import datashader as ds import datashader.transfer_functions as tf import numpy as np @@ -13,6 +14,19 @@ except ImportError: cupy = None + +try: + import dask.array as da +except ImportError: + da = None + + +try: + import dask.dataframe as dd +except ImportError: + dd = None + + ngjit = jit(nopython=True, nogil=True) @@ -28,6 +42,14 @@ def is_cupy_array(arr): return _has_cupy() and isinstance(arr, cupy.ndarray) +def has_dask_array(): + return da is not None + + +def has_dask_dataframe(): + return dd is not None + + def _has_cuda(): """Check for supported CUDA device. If none found, return False""" local_cuda = False @@ -113,7 +135,7 @@ def __call__(self, arr): return self.dask_cupy_func # dask + numpy case - elif isinstance(arr.data, da.Array): + elif has_dask_array() and isinstance(arr.data, da.Array): return self.dask_func else: @@ -136,7 +158,7 @@ def validate_arrays(*arrays): raise ValueError("input arrays must have same type") # ensure dask chunksizes of all arrays are the same - if isinstance(first_array.data, da.Array): + if has_dask_array() and isinstance(first_array.data, da.Array): for i in range(1, len(arrays)): if first_array.chunks != arrays[i].chunks: arrays[i].data = arrays[i].data.rechunk(first_array.chunks) @@ -154,9 +176,7 @@ def get_xy_range(raster, xdim=None, ydim=None): If not provided, assume xdim is `raster.dims[-1]` ydim: str, default = None Name of the y coordinate dimension in input `raster` - If not provided, assume ydim is `raster.dims[-2]` - - Returns + If not provided, assume ydim is `raturns ---------- xrange, yrange Tuple of tuples: (x, y-range). diff --git a/xrspatial/zonal.py b/xrspatial/zonal.py index 0f2d7a4e..fc290f95 100644 --- a/xrspatial/zonal.py +++ b/xrspatial/zonal.py @@ -1,15 +1,30 @@ +from __future__ import annotations + # standard library import copy from math import sqrt from typing import Callable, Dict, List, Optional, Union + # 3rd-party -import dask.array as da -import dask.dataframe as dd +try: + import dask.array as da +except ImportError: + da = None + +try: + import dask.dataframe as dd +except ImportError: + dd = None + +try: + from dask import delayed +except ImportError: + delayed = lambda x: None # noqa + import numpy as np import pandas as pd import xarray as xr -from dask import delayed from xarray import DataArray try: @@ -20,6 +35,7 @@ class cupy(object): # local modules from xrspatial.utils import ArrayTypeFunctionMapping, ngjit, not_implemented_func, validate_arrays +from xrspatial.utils import has_dask_array TOTAL_COUNT = '_total_count' @@ -377,7 +393,7 @@ def _stats_cupy( raise ValueError(stats) result = stats_func(zone_values) - assert(len(result.shape) == 0) + assert (len(result.shape) == 0) stats_dict[stats].append(cupy.float_(result)) @@ -547,7 +563,7 @@ def stats( raise ValueError("`values` must be an array of integers or floats.") # validate stats_funcs - if isinstance(values.data, da.Array) and not isinstance(stats_funcs, list): + if has_dask_array() and isinstance(values.data, da.Array) and not isinstance(stats_funcs, list): raise ValueError( "Got dask-backed DataArray as `values` aggregate. " "`stats_funcs` must be a subset of default supported stats " @@ -1014,7 +1030,7 @@ def crosstab( raise ValueError( f"`agg` method for 3D numpy backed data array must be one of following {agg_3d_numpy}" # noqa ) - if isinstance(values.data, da.Array) and agg not in agg_3d_dask: + if has_dask_array() and isinstance(values.data, da.Array) and agg not in agg_3d_dask: raise ValueError( f"`agg` method for 3D dask backed data array must be one of following {agg_3d_dask}" ) @@ -1036,7 +1052,7 @@ def crosstab( if zones.shape != values.shape[1:]: raise ValueError("Incompatible shapes") - if isinstance(values.data, da.Array): + if has_dask_array() and isinstance(values.data, da.Array): # dask case, rechunk if necessary zones_chunks = zones.chunks expected_values_chunks = { From 9ae244d2d4869256526ef19d518143d00f598424 Mon Sep 17 00:00:00 2001 From: Brendan Collins Date: Thu, 18 Dec 2025 11:38:03 -0800 Subject: [PATCH 3/4] fixed some tests for cupy available but no dask case --- xrspatial/tests/test_focal.py | 19 +++++++++++++------ xrspatial/tests/test_perlin.py | 1 + xrspatial/tests/test_slope.py | 1 + 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/xrspatial/tests/test_focal.py b/xrspatial/tests/test_focal.py index 6dcdf989..1c05d083 100644 --- a/xrspatial/tests/test_focal.py +++ b/xrspatial/tests/test_focal.py @@ -89,6 +89,18 @@ def test_mean_transfer_function_gpu_equals_cpu(): np.testing.assert_allclose( numpy_mean.data, cupy_mean.data.get(), equal_nan=True) + +@dask_array_available +@cuda_and_cupy_available +def test_mean_transfer_dask_gpu_raise_not_implemented(): + + import cupy + + # cupy case + cupy_agg = xr.DataArray(cupy.asarray(data_random)) + cupy_mean = mean(cupy_agg) + general_output_checks(cupy_agg, cupy_mean) + # dask + cupy case not implemented dask_cupy_agg = xr.DataArray( da.from_array(cupy.asarray(data_random), chunks=(3, 3)) @@ -466,6 +478,7 @@ def test_hotspots_numpy(data_hotspots): assert numpy_hotspots.attrs['unit'] == '%' +@dask_array_available def test_hotspots_dask_numpy(data_hotspots): data, kernel, expected_result = data_hotspots dask_numpy_agg = create_test_raster(data, backend='dask') @@ -495,9 +508,3 @@ def test_hotspot_gpu(data_hotspots): cupy_hotspots[coord].data, cupy_agg[coord].data, equal_nan=True ) assert cupy_hotspots.attrs['unit'] == '%' - - # dask + cupy case not implemented - dask_cupy_agg = create_test_raster(data, backend='dask+cupy') - with pytest.raises(NotImplementedError) as e_info: - hotspots(dask_cupy_agg, kernel) - assert e_info diff --git a/xrspatial/tests/test_perlin.py b/xrspatial/tests/test_perlin.py index d5236a3a..953f673f 100644 --- a/xrspatial/tests/test_perlin.py +++ b/xrspatial/tests/test_perlin.py @@ -20,6 +20,7 @@ def create_test_arr(backend='numpy'): raster.data = cupy.asarray(raster.data) if 'dask' in backend: + import dask.array as da raster.data = da.from_array(raster.data, chunks=(10, 10)) return raster diff --git a/xrspatial/tests/test_slope.py b/xrspatial/tests/test_slope.py index aafd1c0b..091e770c 100644 --- a/xrspatial/tests/test_slope.py +++ b/xrspatial/tests/test_slope.py @@ -68,6 +68,7 @@ def test_numpy_equals_cupy_qgis_data(elevation_raster): assert_numpy_equals_cupy(numpy_agg, cupy_agg, slope) +@dask_array_available @cuda_and_cupy_available @pytest.mark.parametrize("size", [(2, 4), (10, 15)]) @pytest.mark.parametrize( From a764e8897501a0440a214324e217874ec2dcbcbc Mon Sep 17 00:00:00 2001 From: Brendan Collins Date: Thu, 18 Dec 2025 11:53:17 -0800 Subject: [PATCH 4/4] skipping dask tests if dask unavailable --- xrspatial/tests/test_classify.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/xrspatial/tests/test_classify.py b/xrspatial/tests/test_classify.py index 08f66296..5368239a 100644 --- a/xrspatial/tests/test_classify.py +++ b/xrspatial/tests/test_classify.py @@ -3,7 +3,9 @@ import xarray as xr from xrspatial import binary, equal_interval, natural_breaks, quantile, reclassify -from xrspatial.tests.general_checks import (create_test_raster, cuda_and_cupy_available, +from xrspatial.tests.general_checks import (create_test_raster, + cuda_and_cupy_available, + dask_array_available, general_output_checks) try: @@ -42,7 +44,7 @@ def test_binary_numpy(result_binary): general_output_checks(numpy_agg, numpy_result, expected_result) -@pytest.mark.skipif(da is not None, reason="dask is not installed") +@dask_array_available def test_binary_dask_numpy(result_binary): values, expected_result = result_binary dask_agg = input_data(backend='dask') @@ -58,6 +60,7 @@ def test_binary_cupy(result_binary): general_output_checks(cupy_agg, cupy_result, expected_result) +@dask_array_available @cuda_and_cupy_available def test_binary_dask_cupy(result_binary): values, expected_result = result_binary @@ -95,7 +98,7 @@ def test_reclassify_numpy(result_reclassify): general_output_checks(numpy_agg, numpy_result, expected_result, verify_dtype=True) -@pytest.mark.skipif(da is not None, reason="dask is not installed") +@dask_array_available def test_reclassify_dask_numpy(result_reclassify): bins, new_values, expected_result = result_reclassify dask_agg = input_data(backend='dask') @@ -111,8 +114,8 @@ def test_reclassify_cupy(result_reclassify): general_output_checks(cupy_agg, cupy_result, expected_result, verify_dtype=True) +@dask_array_available @cuda_and_cupy_available -@pytest.mark.skipif(da is not None, reason="dask is not installed") def test_reclassify_dask_cupy(result_reclassify): bins, new_values, expected_result = result_reclassify dask_cupy_agg = input_data(backend='dask+cupy') @@ -148,7 +151,7 @@ def test_quantile_numpy(result_quantile): general_output_checks(numpy_agg, numpy_quantile, expected_result, verify_dtype=True) -@pytest.mark.skipif(da is not None, reason="dask is not installed") +@dask_array_available def test_quantile_dask_numpy(result_quantile): # Note that dask's percentile algorithm is # approximate, while numpy's is exact. @@ -267,7 +270,7 @@ def test_equal_interval_numpy(result_equal_interval): general_output_checks(numpy_agg, numpy_result, expected_result, verify_dtype=True) -@pytest.mark.skipif(da is not None, reason="dask is not installed") +@dask_array_available def test_equal_interval_dask_numpy(result_equal_interval): k, expected_result = result_equal_interval dask_agg = input_data('dask+numpy')