diff --git a/ingestion-data/staging/dataset-config/oco3-co2-sam-cogs.json b/ingestion-data/staging/dataset-config/oco3-co2-sam-cogs.json new file mode 100644 index 00000000..111f0e8a --- /dev/null +++ b/ingestion-data/staging/dataset-config/oco3-co2-sam-cogs.json @@ -0,0 +1,26 @@ +{ + "collection": "oco3-co2-sam-cogs", + "title": "Orbiting Carbon Observatory 3 (OCO-3) Snapshot Area Mapping (SAM) Cloud-Optimized GeoTIFFs (COGs)", + "description": "NASA-JPL's OCO-3 is an instrument onboard the International Space Station that measures carbon dioxide in the Earth's atmosphere.", + "type": "cog", + "spatial_extent": { + "xmin": -180, + "ymin": -90, + "xmax": 180, + "ymax": 90 + }, + "temporal_extent": {}, + "license": "MIT", + "is_periodic": true, + "time_density": "day", + "stac_version": "1.0.0", + "discovery_items": [ + { + "prefix": "oco3-co2-sam-cogs_v2025.04.16/", + "bucket": "veda-data-store-staging", + "filename_regex": "^.+(2024|2025).+\\.tif$", + "discovery": "s3", + "datetime_range": "day" + } + ] +} diff --git a/transformation-scripts/oco-sam-cogs/test-and-move-files.ipynb b/transformation-scripts/oco-sam-cogs/test-and-move-files.ipynb new file mode 100644 index 00000000..3873a9df --- /dev/null +++ b/transformation-scripts/oco-sam-cogs/test-and-move-files.ipynb @@ -0,0 +1,846 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "55f6ccf7-4c0f-46a0-8483-3c8fe7a4f789", + "metadata": {}, + "source": [ + "# Moving OCO-3 and OCO-2 COGs\n", + "\n", + "This notebook moves COGs from the [OCO3-data-transformer](https://github.com/EarthDigitalTwin/OCO3-data-transformer) project into the VEDA staging bucket so they can be published to the VEDA staging catalog and previewed in the VEDA UI.\n", + "\n", + "## Step 1: Setup\n", + "\n", + "### Import necessary libraries." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "cfdf3f25-24a8-42c8-b364-0f695061197e", + "metadata": {}, + "outputs": [], + "source": [ + "import boto3\n", + "import concurrent.futures\n", + "from concurrent.futures import as_completed\n", + "from osgeo import gdal\n", + "\n", + "gdal.DontUseExceptions()" + ] + }, + { + "cell_type": "markdown", + "id": "9f953e76-3455-4170-82b9-1a854eda9996", + "metadata": {}, + "source": [ + "### Initiate the S3 Client" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "4750555a-152f-430a-9d31-0e51aa0c529f", + "metadata": {}, + "outputs": [], + "source": [ + "s3_client = boto3.client(\"s3\")" + ] + }, + { + "cell_type": "markdown", + "id": "e14f9fc6-579d-4a36-b400-f0208058e567", + "metadata": {}, + "source": [ + "### Set constants\n", + "\n", + "The constants below are not expected to change." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "4b4fc2c9-91d1-4f39-804f-f0ef9ae9449e", + "metadata": {}, + "outputs": [], + "source": [ + "source_bucket = \"sdap-dev-zarr\"\n", + "source_dir = \"OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog\"\n", + "# Set the number of worker processes\n", + "NUM_WORKERS = 4\n", + "destination_bucket = \"veda-data-store-staging\"\n", + "collection_prefix_collection_id_map = {\n", + " \"oco3-co2\": \"oco3-co2-sam-l3-cogs\",\n", + " \"oco2-co2\": \"oco2-co2-target-cogs\",\n", + " \"oco3-sif\": \"oco3-sif-sam-cogs\",\n", + "}" + ] + }, + { + "cell_type": "markdown", + "id": "abb49f0e-4ef1-4fef-a2d1-7f20f9efdc11", + "metadata": {}, + "source": [ + "The constants below are expected to change depending on what collection we are working on (oco3 co2, oco2 co2 or oco3 sif) and how many files we want to upload (set to lower than the total for testing)." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "e136c21d-2ef5-46a1-a9a4-44b6e38fb749", + "metadata": {}, + "outputs": [], + "source": [ + "collection = \"oco3-sif\"\n", + "prefix = f\"{source_dir}/{collection}\"\n", + "object_limit = 100" + ] + }, + { + "cell_type": "markdown", + "id": "975fe6a9-a5df-4ce7-82f7-346ce9314e06", + "metadata": {}, + "source": [ + "## Step 2: List all objects for this collection in the source bucket + directory" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "c1ffe44c-2a38-42de-844f-e71315bb9e81", + "metadata": {}, + "outputs": [], + "source": [ + "all_objects = []\n", + "continuation_token = None\n", + "\n", + "while all_objects == [] or continuation_token is not None:\n", + " args = dict(Bucket=source_bucket, Prefix=prefix)\n", + " if continuation_token is not None:\n", + " args[\"ContinuationToken\"] = continuation_token\n", + " response = s3_client.list_objects_v2(**args)\n", + " [all_objects.append(obj[\"Key\"]) for obj in response.get(\"Contents\", [])]\n", + " continuation_token = response.get(\"NextContinuationToken\", None)" + ] + }, + { + "cell_type": "markdown", + "id": "c6d0f167-b8e8-4109-9727-314542f4a927", + "metadata": {}, + "source": [ + "### Sanity checks" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "95e1f08b-1651-4b3d-90f5-cafea5d7eded", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "2832" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(all_objects)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "93227a49-b4f2-4539-813f-25dca1b241b9", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2019-10-10T190148Z_filtered_Daily_SIF_757nm.tif'" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "all_objects[0]" + ] + }, + { + "cell_type": "markdown", + "id": "d6f54b53-fe2f-414d-82a8-71d464ee603a", + "metadata": {}, + "source": [ + "## Step 3: Define process and copy operations\n", + "\n", + "The `copy_object` function copies the original file to the destination bucket and directory.\n", + "\n", + "The `process_file` function checks to make sure there are valid pixels in the file. We do this check before moving a file to the staging location because having invalid files in the `veda-data-store-staging` bucket will cause publication for all files in that bucket to fail (at time of writing). So we skip files that fail this check." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "11029195-d25b-44ca-997e-0fe7ad90cbd2", + "metadata": {}, + "outputs": [], + "source": [ + "def copy_object(object_key: str):\n", + " copy_source = {\"Bucket\": source_bucket, \"Key\": object_key}\n", + " destination_file_name = object_key.split(\"/\")[-1]\n", + " collection_prefix = destination_file_name.split(\"_\")[0]\n", + " destination_dir = collection_prefix_collection_id_map[collection_prefix]\n", + " destination_object_key = f\"{destination_dir}/{destination_file_name}\"\n", + " s3_client.copy_object(\n", + " CopySource=copy_source, Bucket=destination_bucket, Key=destination_object_key\n", + " )\n", + " print(\n", + " f\"Copied: s3://{source_bucket}/{object_key} to s3://{destination_bucket}/{destination_object_key}\"\n", + " )\n", + "\n", + "\n", + "def process_file(object_key: str):\n", + " \"\"\"Function to process a single file.\"\"\"\n", + " dataset = gdal.Open(f\"/vsis3/{source_bucket}/{object_key}\", gdal.GA_ReadOnly)\n", + "\n", + " if dataset is None:\n", + " return file_path, \"Failed to open\"\n", + "\n", + " band = dataset.GetRasterBand(1)\n", + " _ = band.GetStatistics(True, True)\n", + "\n", + " # Check for errors\n", + " err_msg = gdal.GetLastErrorMsg()\n", + "\n", + " if \"Failed to compute statistics, no valid pixels found in sampling\" in err_msg:\n", + " return dict(\n", + " object_key=object_key,\n", + " error_message=f\"GDAL Error: {err_msg}\",\n", + " status=\"error\",\n", + " )\n", + " else:\n", + " copy_object(object_key=object_key)\n", + " return dict(object_key=object_key, error_message=None, status=\"success\")" + ] + }, + { + "cell_type": "markdown", + "id": "7b55efa7-f244-439d-9514-736e4d5e3104", + "metadata": {}, + "source": [ + "## Step 4: Test on one file" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "c6915325-3e07-47e4-8ec7-3f4b79789e89", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2019-10-10T190148Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_cal001_2019-10-10T190148Z_filtered_Daily_SIF_757nm.tif\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2019-10-10T190148Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n" + ] + }, + { + "data": { + "text/plain": [ + "{'object_key': 'OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2019-10-10T190148Z_filtered_Daily_SIF_757nm.tif',\n", + " 'error_message': None,\n", + " 'status': 'success'}" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "first_object_key = all_objects[0]\n", + "process_file(first_object_key)" + ] + }, + { + "cell_type": "markdown", + "id": "b33272a6-a7c5-4ab2-a8b1-e60f9ab300c5", + "metadata": {}, + "source": [ + "### Optional: Check the file was moved" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "e6932be8-6dee-4631-8e0d-a31404b56c42", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-03-27 22:53:12 125814 oco3-sif_cal001_2019-10-10T190148Z_filtered_Daily_SIF_757nm.tif\n" + ] + } + ], + "source": [ + "!aws s3 ls s3://veda-data-store-staging/oco3-sif-sam-cogs/ #| wc -l" + ] + }, + { + "cell_type": "markdown", + "id": "4532766a-5c6f-444c-b581-975a3ac6e1b4", + "metadata": {}, + "source": [ + "## Step 5: Run process + copy for all files." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "5dd258aa-37d8-40d2-9767-b874abf0189a", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2019-10-10T190148Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_cal001_2019-10-10T190148Z_filtered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2020-02-02T211516Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_cal001_2020-02-02T211516Z_unfiltered_Daily_SIF_757nm.tif\n", + "processed file 0\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2019-10-10T190148Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_cal001_2019-10-10T190148Z_unfiltered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2020-02-10T180738Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_cal001_2020-02-10T180738Z_filtered_Daily_SIF_757nm.tif\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2020-02-02T211516Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2019-10-10T190148Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2019-10-10T190148Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2020-02-10T180738Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2020-02-10T180738Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2020-02-10T180738Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_cal001_2020-02-10T180738Z_unfiltered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2020-02-29T173738Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_cal001_2020-02-29T173738Z_unfiltered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2020-04-05T202057Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_cal001_2020-04-05T202057Z_filtered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2020-02-29T173750Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_cal001_2020-02-29T173750Z_filtered_Daily_SIF_757nm.tif\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2020-02-29T173738Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2020-04-05T202057Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2020-02-29T173750Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2020-04-05T202057Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2020-04-05T202057Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_cal001_2020-04-05T202057Z_unfiltered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2020-04-13T171716Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_cal001_2020-04-13T171716Z_filtered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2020-04-13T171716Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_cal001_2020-04-13T171716Z_unfiltered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2020-04-17T154526Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_cal001_2020-04-17T154526Z_unfiltered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2020-04-17T154551Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_cal001_2020-04-17T154551Z_filtered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2020-04-17T221622Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_cal001_2020-04-17T221622Z_filtered_Daily_SIF_757nm.tif\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2020-04-13T171716Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2020-04-13T171716Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2020-04-17T154526Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2020-04-17T154551Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2020-04-17T221622Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2020-04-21T204250Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_cal001_2020-04-21T204250Z_filtered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2020-04-17T221611Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_cal001_2020-04-17T221611Z_unfiltered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2020-04-21T204250Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_cal001_2020-04-21T204250Z_unfiltered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2020-04-29T173429Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_cal001_2020-04-29T173429Z_filtered_Daily_SIF_757nm.tif\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2020-04-21T204250Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2020-04-17T221611Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2020-04-21T204250Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2020-04-29T173429Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2020-04-29T173429Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_cal001_2020-04-29T173429Z_unfiltered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2020-05-03T160011Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_cal001_2020-05-03T160011Z_filtered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2020-05-03T160011Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_cal001_2020-05-03T160011Z_unfiltered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon100_2019-10-09T103320Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon100_2019-10-09T103320Z_filtered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon100_2019-10-09T103320Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon100_2019-10-09T103320Z_unfiltered_Daily_SIF_757nm.tif\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2020-04-29T173429Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2020-05-03T160011Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_cal001_2020-05-03T160011Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon100_2019-10-09T103320Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon100_2019-10-09T103320Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon100_2019-10-13T085540Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon100_2019-10-13T085540Z_filtered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon100_2020-02-09T093759Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon100_2020-02-09T093759Z_filtered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon100_2019-10-13T085540Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon100_2019-10-13T085540Z_unfiltered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon100_2020-02-09T093759Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon100_2020-02-09T093759Z_unfiltered_Daily_SIF_757nm.tif\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon100_2019-10-13T085540Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon100_2020-02-09T093759Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon100_2019-10-13T085540Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon100_2020-02-09T093759Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon100_2020-02-22T103920Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon100_2020-02-22T103920Z_filtered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon100_2020-02-22T103920Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon100_2020-02-22T103920Z_unfiltered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon100_2020-03-30T141003Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon100_2020-03-30T141003Z_filtered_Daily_SIF_757nm.tif\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon100_2020-02-22T103920Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon100_2020-02-22T103920Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon100_2020-03-30T141003Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon100_2020-03-30T140959Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon100_2020-03-30T140959Z_unfiltered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon100_2020-04-08T101838Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon100_2020-04-08T101838Z_filtered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon100_2020-04-08T101838Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon100_2020-04-08T101838Z_unfiltered_Daily_SIF_757nm.tif\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon100_2020-03-30T140959Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon100_2020-04-08T101838Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon100_2020-04-08T101838Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon100_2020-04-12T084648Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon100_2020-04-12T084648Z_unfiltered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon100_2020-04-12T084648Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon100_2020-04-12T084648Z_filtered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon100_2020-04-16T071458Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon100_2020-04-16T071458Z_unfiltered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon100_2020-04-16T071458Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon100_2020-04-16T071458Z_filtered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon100_2020-04-17T125746Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon100_2020-04-17T125746Z_filtered_Daily_SIF_757nm.tif\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon100_2020-04-12T084648Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon100_2020-04-12T084648Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon100_2020-04-16T071458Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon100_2020-04-16T071458Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon100_2020-04-17T125746Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon100_2020-04-17T125746Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon100_2020-04-17T125746Z_unfiltered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon100_2020-04-29T081617Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon100_2020-04-29T081617Z_filtered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon100_2020-04-29T081617Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon100_2020-04-29T081617Z_unfiltered_Daily_SIF_757nm.tif\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon100_2020-04-17T125746Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon100_2020-04-29T081617Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon100_2020-04-29T081617Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon100_2020-05-03T064158Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon100_2020-05-03T064158Z_unfiltered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon100_2020-05-03T064159Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon100_2020-05-03T064159Z_filtered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon101_2019-09-24T124619Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon101_2019-09-24T124619Z_unfiltered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon101_2019-09-24T124619Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon101_2019-09-24T124619Z_filtered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon101_2019-10-06T075433Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon101_2019-10-06T075433Z_filtered_Daily_SIF_757nm.tif\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon100_2020-05-03T064158Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon100_2020-05-03T064159Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon101_2019-09-24T124619Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon101_2019-09-24T124619Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon101_2019-10-06T075433Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon101_2019-10-06T075433Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon101_2019-10-06T075433Z_unfiltered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon101_2019-12-26T142803Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon101_2019-12-26T142803Z_unfiltered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon101_2019-12-26T142804Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon101_2019-12-26T142804Z_filtered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon101_2020-01-03T111737Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon101_2020-01-03T111737Z_unfiltered_Daily_SIF_757nm.tif\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon101_2019-10-06T075433Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon101_2019-12-26T142803Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon101_2019-12-26T142804Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon101_2020-01-03T111737Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon101_2020-01-11T080736Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon101_2020-01-11T080736Z_filtered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon101_2020-01-03T111737Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon101_2020-01-03T111737Z_filtered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon101_2020-01-11T080736Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon101_2020-01-11T080736Z_unfiltered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon101_2020-01-16T153359Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon101_2020-01-16T153359Z_filtered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon101_2020-03-19T143249Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon101_2020-03-19T143249Z_unfiltered_Daily_SIF_757nm.tif\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon101_2020-01-11T080736Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon101_2020-01-03T111737Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon101_2020-01-11T080736Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon101_2020-01-16T153359Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon101_2020-03-19T143249Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon101_2020-03-19T143249Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon101_2020-03-19T143249Z_filtered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon101_2020-01-16T153359Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon101_2020-01-16T153359Z_unfiltered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon101_2020-03-23T130008Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon101_2020-03-23T130008Z_filtered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon101_2020-03-23T130008Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon101_2020-03-23T130008Z_unfiltered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon101_2020-03-27T112729Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon101_2020-03-27T112729Z_filtered_Daily_SIF_757nm.tif\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon101_2020-03-19T143249Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon101_2020-01-16T153359Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon101_2020-03-23T130008Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon101_2020-03-23T130008Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon101_2020-03-27T112729Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon101_2020-03-27T112729Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon101_2020-03-27T112729Z_unfiltered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon101_2020-04-26T140436Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon101_2020-04-26T140436Z_unfiltered_Daily_SIF_757nm.tif\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon101_2020-03-27T112729Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon101_2020-04-26T140436Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon101_2020-04-04T082237Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon101_2020-04-04T082237Z_unfiltered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon101_2020-04-26T140436Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon101_2020-04-26T140436Z_filtered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon101_2020-04-04T082237Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon101_2020-04-04T082237Z_filtered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2019-09-13T070208Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon102_2019-09-13T070208Z_unfiltered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2019-09-13T070210Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon102_2019-09-13T070210Z_filtered_Daily_SIF_757nm.tif\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon101_2020-04-04T082237Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon101_2020-04-26T140436Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon101_2020-04-04T082237Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2019-09-13T070208Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2019-09-13T070210Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2019-09-22T124836Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon102_2019-09-22T124836Z_filtered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2019-09-22T124836Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon102_2019-09-22T124836Z_unfiltered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2019-12-24T142959Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon102_2019-12-24T142959Z_filtered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2019-12-24T142959Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon102_2019-12-24T142959Z_unfiltered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2020-01-05T094337Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon102_2020-01-05T094337Z_filtered_Daily_SIF_757nm.tif\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2019-09-22T124836Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2019-09-22T124836Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2019-12-24T142959Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2020-01-05T094337Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2019-12-24T142959Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2020-01-05T094337Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon102_2020-01-05T094337Z_unfiltered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2020-01-13T063326Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon102_2020-01-13T063326Z_filtered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2020-01-09T080826Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon102_2020-01-09T080826Z_unfiltered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2020-01-09T080826Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon102_2020-01-09T080826Z_filtered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2020-01-30T091800Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon102_2020-01-30T091800Z_filtered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2020-01-13T063326Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon102_2020-01-13T063326Z_unfiltered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2020-01-30T091757Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon102_2020-01-30T091757Z_unfiltered_Daily_SIF_757nm.tif\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2020-01-05T094337Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2020-01-13T063326Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2020-01-09T080826Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2020-01-09T080826Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2020-01-30T091800Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2020-01-13T063326Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2020-01-30T091757Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2020-03-21T130017Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon102_2020-03-21T130017Z_unfiltered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2020-03-21T130019Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon102_2020-03-21T130019Z_filtered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2020-03-25T112737Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon102_2020-03-25T112737Z_filtered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2020-03-25T112737Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon102_2020-03-25T112737Z_unfiltered_Daily_SIF_757nm.tif\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2020-03-21T130017Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2020-03-21T130019Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2020-03-25T112737Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2020-03-25T112737Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2020-04-02T082219Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon102_2020-04-02T082219Z_filtered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2020-04-02T082218Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon102_2020-04-02T082218Z_unfiltered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2020-04-06T065027Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon102_2020-04-06T065027Z_filtered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2020-04-06T065027Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon102_2020-04-06T065027Z_unfiltered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2020-05-02T105646Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon102_2020-05-02T105646Z_filtered_Daily_SIF_757nm.tif\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2020-04-02T082219Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2020-04-02T082218Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2020-04-06T065027Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2020-04-06T065027Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2020-05-02T105646Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2020-05-02T105646Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon102_2020-05-02T105646Z_unfiltered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2020-05-06T092226Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon102_2020-05-06T092226Z_filtered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2020-05-10T074806Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon102_2020-05-10T074806Z_filtered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2020-05-06T092226Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon102_2020-05-06T092226Z_unfiltered_Daily_SIF_757nm.tif\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2020-05-02T105646Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2020-05-06T092226Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2020-05-10T074806Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2020-05-06T092226Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon103_2019-09-24T125417Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon103_2019-09-24T125417Z_filtered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2020-05-10T074806Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon102_2020-05-10T074806Z_unfiltered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon103_2019-09-24T125417Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon103_2019-09-24T125417Z_unfiltered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon103_2019-12-24T124927Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon103_2019-12-24T124927Z_unfiltered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon103_2019-12-24T124929Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon103_2019-12-24T124929Z_filtered_Daily_SIF_757nm.tif\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon103_2019-09-24T125417Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon102_2020-05-10T074806Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon103_2019-09-24T125417Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon103_2019-12-24T124927Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon103_2019-12-24T124929Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon103_2020-01-05T080256Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon103_2020-01-05T080256Z_filtered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon103_2020-01-05T080256Z_unfiltered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon103_2020-01-05T080256Z_unfiltered_Daily_SIF_757nm.tif\n", + "Copied: s3://sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon103_2020-01-09T062756Z_filtered_Daily_SIF_757nm.tif to s3://veda-data-store-staging/oco3-sif-sam-cogs/oco3-sif_coccon103_2020-01-09T062756Z_filtered_Daily_SIF_757nm.tif\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon103_2020-01-05T080256Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon103_2020-01-05T080256Z_unfiltered_Daily_SIF_757nm.tif.aux.xml.\n", + "Warning 1: Unable to save auxiliary information in /vsis3/sdap-dev-zarr/OCO3/outputs/veda/demo-2024.10.28-target/SIMULTEST_TFP_cog/oco3-sif_coccon103_2020-01-09T062756Z_filtered_Daily_SIF_757nm.tif.aux.xml.\n" + ] + } + ], + "source": [ + "def main():\n", + " failures = []\n", + " with concurrent.futures.ThreadPoolExecutor(max_workers=NUM_WORKERS) as executor:\n", + " futures = {\n", + " executor.submit(process_file, object_key): object_key\n", + " for object_key in all_objects[0:object_limit]\n", + " }\n", + "\n", + " for idx, future in enumerate(as_completed(futures)):\n", + " if idx % 100 == 0:\n", + " print(f\"processed file {idx}\")\n", + " result = future.result()\n", + " if result[\"status\"] == \"error\":\n", + " object_key = result[\"object_key\"]\n", + " failures.append((object_key, result))\n", + " # print(f\"{object_key}: {result['error_message']}\")\n", + "\n", + " # Save results to a log file\n", + " with open(\"failed_files.log\", \"w\") as f:\n", + " for object_key, result in failures:\n", + " f.write(f\"{object_key}\\n\")\n", + "\n", + "\n", + "main()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}