From e5ff0e4d7c863105d0c80aad7076111f80bb41a7 Mon Sep 17 00:00:00 2001 From: ctuguinay Date: Thu, 25 Jun 2026 09:27:59 +0800 Subject: [PATCH 01/10] small file upload and raw2sv update --- README.md | 4 ++-- src/echodataflow/flows/flows_acoustics.py | 8 ++++++-- src/echodataflow/flows/flows_helper.py | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index fef6dac..f1649a1 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ Echodataflow streamlines echosounder data processing by combining [Prefect](http prefect worker start --pool "local" ``` -3. Download the recipes from the [echodataflow-recipes repository](https://github.com/echostack-org/echodataflow-recipes) by clonining it to your computer: +3. In a new terminal, download the recipes from the [echodataflow-recipes repository](https://github.com/echostack-org/echodataflow-recipes) by clonining it to your computer: ``` cd REPO_DIRECTORY # switch to where you want the recipes repo to sit git clone https://github.com/echostack-org/echodataflow-recipes.git @@ -106,7 +106,7 @@ to `.service` `ExecStart` usage, with no wrapper shell script required. 1. Copy and customize the templates for your user: ```shell mkdir -p ~/.config/echodataflow ~/Library/LaunchAgents ~/.local/var/log/echodataflow - cp src/echodataflow/services/services.env.example ~/.config/echodataflow/services.env + cp src/echodataflow/services/services.env.example_local ~/.config/echodataflow/services.env cp src/echodataflow/services/deploy_prefect_server.launchd.plist ~/Library/LaunchAgents/org.echodataflow.prefect-server.plist cp src/echodataflow/services/deploy_prefect_worker.launchd.plist ~/Library/LaunchAgents/org.echodataflow.prefect-worker.plist ``` diff --git a/src/echodataflow/flows/flows_acoustics.py b/src/echodataflow/flows/flows_acoustics.py index b2ad040..f28ebfe 100644 --- a/src/echodataflow/flows/flows_acoustics.py +++ b/src/echodataflow/flows/flows_acoustics.py @@ -122,7 +122,8 @@ def flow_raw2Sv( waveform_mode: str = "CW", depth_offset: float = 9.5, sonar_model: str = "EK80", - nmea_sentence: str = "GGA", + datagram_type: str|None = None, + nmea_sentence: str|None = None, filename_pattern: str = "*.raw", path_main: str = "", path_raw: str = "", @@ -224,6 +225,7 @@ async def cancel_run(): waveform_mode=waveform_mode, depth_offset=depth_offset, sonar_model=sonar_model, + datagram_type=datagram_type, nmea_sentence=nmea_sentence, ) @@ -302,7 +304,8 @@ def task_raw2Sv( waveform_mode: str = "CW", depth_offset: float = 9.5, # in meters sonar_model: str = "EK80", - nmea_sentence: str = "GGA", + datagram_type: str|None = None, + nmea_sentence: str|None = None, path_Sv_zarr: str = "PATH_TO_STORE_SV_ZARR", ): """ @@ -328,6 +331,7 @@ def task_raw2Sv( ds_Sv = ep.consolidate.add_location( ds=ds_Sv, echodata=ed, + datagram_type=datagram_type, nmea_sentence=nmea_sentence, ) diff --git a/src/echodataflow/flows/flows_helper.py b/src/echodataflow/flows/flows_helper.py index f896f3a..cc8be10 100644 --- a/src/echodataflow/flows/flows_helper.py +++ b/src/echodataflow/flows/flows_helper.py @@ -57,7 +57,7 @@ def flow_file_upload( if max_age == -1: command = f"rclone copy -v --no-traverse {src_dir} {dest_dir} --exclude-from {str(exclude_path)}" else: - command = f"rclone copy -v --max-age 2h --no-traverse {src_dir} {dest_dir} --exclude-from {str(exclude_path)}" + command = f"rclone copy -v --max-age {max_age}h --no-traverse {src_dir} {dest_dir} --exclude-from {str(exclude_path)}" print("command:", command) with ShellOperation(commands=[command], working_dir=src_dir) as file_upload_operation: From 344f43188f4dcb2239b4c6ff8ecb8a0efeb28579 Mon Sep 17 00:00:00 2001 From: ctuguinay Date: Thu, 2 Jul 2026 19:12:02 -0700 Subject: [PATCH 02/10] debug prediction datetime issue and add functionality for writing to EVR --- pyproject.toml | 1 - src/echodataflow/flows/flows_acoustics.py | 182 +-- src/echodataflow/flows/flows_helper.py | 5 +- src/echodataflow/utils/utils.py | 48 - test.ipynb | 1301 +++++++++++++++++++++ 5 files changed, 1396 insertions(+), 141 deletions(-) create mode 100644 test.ipynb diff --git a/pyproject.toml b/pyproject.toml index 7d3fb23..d849229 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,6 @@ dependencies = [ "PyYAML", "s3fs", "shapely", - "torch", "xarray", ] dynamic = ["version"] diff --git a/src/echodataflow/flows/flows_acoustics.py b/src/echodataflow/flows/flows_acoustics.py index f28ebfe..0b873f5 100644 --- a/src/echodataflow/flows/flows_acoustics.py +++ b/src/echodataflow/flows/flows_acoustics.py @@ -13,6 +13,7 @@ import numpy as np import echopype as ep +import echoregions as er import boto3 from botocore import UNSIGNED @@ -27,13 +28,14 @@ from echodataflow.flows.flows_helper import deployment_already_running from echodataflow.utils.utils import ( - # get_MVBS_tensor, - # get_hake_model, round_up_mins, get_slice_start_end_times, extract_datetime_from_filename, ) +from segmentation_inference.model import binary_hake_model +from segmentation_inference.utils import get_MVBS_tensor + import torch @@ -54,58 +56,9 @@ async def set_concurrency_limit(): ep.utils.log.verbose() -def _load_binary_hake_model_class(): - from src.model.BinaryHakeModel import BinaryHakeModel - - return BinaryHakeModel - - -def get_MVBS_tensor(ds_in, freq_wanted=[120000, 38000, 18000]): - # Find the right channel sequence - ch_wanted = [int((np.abs(ds_in["frequency_nominal"]-freq)).argmin()) for freq in freq_wanted] - - # Crucial for model prediction: - # - order of dimension (channel, depth, ping_time) - # - depth slice up to 590 m - mvbs_tensor = torch.tensor( - ( - ds_in["Sv"] - .transpose("channel", "depth", "ping_time") - .isel(channel=ch_wanted).sel(depth=slice(None, 590)).values - ), - dtype=torch.float32 - ) - - # Clip to Sv range - mvbs_tensor_clip = torch.clip( - mvbs_tensor.clone().detach().to(torch.float16), - min=-70, - max=-36, - ) - - # Replace NaN values with min Sv - mvbs_tensor_clip[torch.isnan(mvbs_tensor_clip)] = -70 - - # Normalize and conver to float - mvbs_tensor_clip_normalized = ( - (mvbs_tensor_clip - (-70.0)) / (-36.0 - (-70.0)) * 255.0 - ) - - return mvbs_tensor_clip_normalized.unsqueeze(0).float() - - # Load binary hake models with weights -def get_hake_model(model_path: str) -> BinaryHakeModel: - placeholder_score_dir = Path( - str(files("echodataflow.assets") / "placeholder_score_tensor_dir") - ) - binary_hake_model = _load_binary_hake_model_class() - model = binary_hake_model( - "placeholder_experiment_name", - placeholder_score_dir, - "placeholder_tensor_log_dir", - 0, - ).eval() +def get_hake_model(model_path: str) -> binary_hake_model: + model = binary_hake_model().eval() model.load_state_dict(torch.load(model_path, map_location=torch.device('cpu'))["state_dict"]) return model @@ -269,7 +222,6 @@ async def cancel_run(): results, columns=["raw_filename", "Sv_filename", "first_ping_time", "last_ping_time"] ) - print(df_new) # Concatenate with existing df_Sv and save df_Sv = pd.concat([df_Sv, df_new], ignore_index=True) @@ -347,7 +299,7 @@ def task_raw2Sv( return ( out_path.name, pd.to_datetime(ds_Sv["ping_time"][0].values), - pd.to_datetime(ds_Sv["ping_time"][-1].values) + pd.to_datetime(ds_Sv["ping_time"][-1].values), ) @@ -565,7 +517,6 @@ def task_create_MVBS( ) # Save to zarr: 1 chunk along each dimension - print("MVBS filename:", MVBS_filename) logger.info(f"Saving MVBS to {MVBS_filename}") ds_MVBS.chunk({"channel": -1, "ping_time": -1, "depth": -1}).to_zarr( store=Path(path_MVBS_zarrr) / MVBS_filename, # existing file will be overwritten @@ -576,7 +527,7 @@ def task_create_MVBS( return ( pd.to_datetime(ds_MVBS["ping_time"][0].values), - pd.to_datetime(ds_MVBS["ping_time"][-1].values) + pd.to_datetime(ds_MVBS["ping_time"][-1].values), ) @@ -587,6 +538,8 @@ async def flow_predict_hake( num_slices: int = 3, temperature: float = 0.5, softmax_threshold: float = 0.5, + max_depth: float = 590.0, + path_weight: str = "", path_main: str = "", file_MVBS_csv: str = "", file_prediction_csv: str = "" @@ -604,14 +557,15 @@ async def flow_predict_hake( The number of slices to create. temperature : float Temperature parameter for softmax scaling in prediction. + softmax_threshold : float + Threshold to determine hake presence. + max_depth : float + Max depth to predict hake. """ logger = get_run_logger() - # Load binary hake models with weights - model_epoch = 85 - model_folder = "/Users/feresa/code_git/echodataflow/temp_model/model_160_epochs/model_weights" - model_path = f"{model_folder}/binary_hake_model_1.0m_bottom_offset_1.0m_depth_2017_2019_epoch_{model_epoch:03d}.ckpt" - model = get_hake_model(model_path) + # Load binary hake model with weights + model = get_hake_model(path_weight) # Set end_time to current time - time_offset_seconds end_time = round_up_mins( @@ -637,16 +591,20 @@ async def flow_predict_hake( file_prediction_csv = Path(path_main) / file_prediction_csv path_MVBS_zarr = Path(path_main) / "MVBS" path_prediction_zarr = Path(path_main) / "prediction" + path_prediction_evr = Path(path_main) / "EVR" path_NASC_zarr = Path(path_main) / "NASC" if not path_MVBS_zarr.exists(): raise ValueError("MVBS zarr store does not exist, check create_MVBS flow!") if not path_prediction_zarr.exists(): path_prediction_zarr.mkdir(parents=True, exist_ok=True) + if not path_prediction_evr.exists(): + path_prediction_evr.mkdir(parents=True, exist_ok=True) if not path_NASC_zarr.exists(): path_NASC_zarr.mkdir(parents=True, exist_ok=True) # convert back to string to pass into task path_MVBS_zarr = str(path_MVBS_zarr) path_prediction_zarr = str(path_prediction_zarr) + path_prediction_evr = str(path_prediction_evr) path_NASC_zarr = str(path_NASC_zarr) # Load Sv and MVBS info dataframes @@ -673,7 +631,15 @@ async def flow_predict_hake( if not file_prediction_csv.exists(): df_prediction = pd.DataFrame( - columns=["prediction_filename_postfix", "score_filename", "softmax_filename", "first_ping_time", "last_ping_time"] + columns=[ + "prediction_filename_postfix", + "score_filename", + "softmax_filename", + "prediction_filename", + "evr_filename", + "first_ping_time", + "last_ping_time" + ] ) df_prediction.to_csv(file_prediction_csv) else: @@ -721,10 +687,12 @@ async def flow_predict_hake( ( # used for task_compute_NASC_direct ds_MVBS_combine, - da_score_softmax, + da_predict_hake, # used for book keeping score_filename, softmax_filename, + prediction_filename, + evr_filename, first_ping_time, last_ping_time ) = task_predict_hake.with_options( @@ -737,8 +705,11 @@ async def flow_predict_hake( end_time=end_time[snum], model=model, temperature=temperature, + softmax_threshold=softmax_threshold, + max_depth=max_depth, path_MVBS_zarr=path_MVBS_zarr, path_prediction_zarr=path_prediction_zarr, + path_prediction_evr=path_prediction_evr, ) # Compute NASC directly from the prediction @@ -748,8 +719,7 @@ async def flow_predict_hake( )( NASC_filename=f"NASC_{predict_filename_postfix}.zarr", ds_MVBS_combine=ds_MVBS_combine, - da_score_softmax=da_score_softmax, - softmax_threshold=softmax_threshold, + da_predict_hake=da_predict_hake, path_NASC_zarr=path_NASC_zarr, # use the same path as predictions ) @@ -761,7 +731,15 @@ async def flow_predict_hake( else: logger.info(f"Adding new prediction file {predict_filename_postfix} to tracking dataframe") idx_to_add = len(df_prediction) - df_prediction.loc[idx_to_add] = [predict_filename_postfix, score_filename, softmax_filename, first_ping_time, last_ping_time] + df_prediction.loc[idx_to_add] = [ + predict_filename_postfix, + score_filename, + softmax_filename, + prediction_filename, + evr_filename, + first_ping_time, + last_ping_time, + ] except Exception as e: errors.append(e) logger.error(f"Error during prediction for slice {snum+1}: {e}") @@ -786,10 +764,13 @@ def task_predict_hake( MVBS_filenames: list[str], start_time: pd.Timestamp, end_time: pd.Timestamp, - model: BinaryHakeModel, + model: binary_hake_model, temperature: int = 0.5, + softmax_threshold: float = 0.5, + max_depth: float = 590.0, path_MVBS_zarr: str = "PATH_TO_MVBS_ZARR", path_prediction_zarr: str = "PATH_TO_PREDICTION_ZARR", + path_prediction_evr: str = "PATH_TO_PREDICTION_EVR", ): """ Predict on a single MVBS file. @@ -811,7 +792,12 @@ def task_predict_hake( The trained model to use for prediction. temperature : float Temperature parameter for softmax scaling in prediction. + softmax_threshold : float + Threshold to determine hake presence. + max_depth : float + Max depth to predict hake. """ + logger = get_run_logger() # Remove timezone info for slicing start_time = start_time.replace(tzinfo=None) end_time = end_time.replace(tzinfo=None) @@ -828,23 +814,24 @@ def task_predict_hake( ).sel( # slice start/end, end exclusive ping_time=slice(start_time, end_time-pd.to_timedelta("10milliseconds")), - depth=slice(None, 590) # slice to what the model expects + depth=slice(None, max_depth) # slice to what the model expects ) # Prepare input tensor: slice depth and ensure order of coordinates input_tensor = get_MVBS_tensor(ds_MVBS_combine) # Predict using the model - score_tensor = model(input_tensor).detach().squeeze(0) - score_tensor_softmax = torch.nn.functional.softmax(score_tensor / temperature, dim=0) + output_dict = model.forward(input_tensor, softmax_temperature=temperature) + score_tensor = output_dict["interpolated_output"].detach() + score_tensor_softmax = output_dict["softmax_output"].detach() # Assemble output DataArrays da_score = xr.DataArray( score_tensor, coords={ "scatterer_class": ["background", "hake"], - "depth": ds_MVBS_combine["depth"].values, - "ping_time": ds_MVBS_combine["ping_time"].values, + "depth": ds_MVBS_combine["depth"], + "ping_time": ds_MVBS_combine["ping_time"], }, name="score", ) @@ -852,15 +839,23 @@ def task_predict_hake( score_tensor_softmax, coords={ "scatterer_class": ["background", "hake"], - "depth": ds_MVBS_combine["depth"].values, - "ping_time": ds_MVBS_combine["ping_time"].values, + "depth": ds_MVBS_combine["depth"], + "ping_time": ds_MVBS_combine["ping_time"], }, name="softmax_score", ) + da_predict_hake = ( + da_score_softmax + .sel(scatterer_class="hake") # only need hake class + .transpose("ping_time", "depth") # TODO: remove once update echopype to 0.10.2 + .drop_vars("scatterer_class") + ) > softmax_threshold + da_predict_hake.name = "hake_prediction" # Save to zarr score_filename = f"score_{predict_filename_postfix}.zarr" softmax_filename = f"softmax_{predict_filename_postfix}.zarr" + prediction_filename = f"prediction_{predict_filename_postfix}.zarr" da_score.chunk({"scatterer_class": -1, "ping_time": -1, "depth": -1}).to_zarr( store=Path(path_prediction_zarr) / score_filename, mode="w", @@ -871,14 +866,32 @@ def task_predict_hake( mode="w", consolidated=True, ) + da_predict_hake.chunk({"ping_time": -1, "depth": -1}).to_zarr( + store=Path(path_prediction_zarr) / prediction_filename, + mode="w", + consolidated=True, + ) + + # Save to evr + evr_filename = f"prediction_{predict_filename_postfix}.evr" + er.write_evr( + Path(path_prediction_evr) / evr_filename, + da_predict_hake, + region_classification="hake", + ) return ( ds_MVBS_combine, - da_score_softmax, + da_predict_hake, score_filename, softmax_filename, - pd.to_datetime(ds_MVBS_combine["ping_time"][0].values), - pd.to_datetime(ds_MVBS_combine["ping_time"][-1].values) + prediction_filename, + evr_filename, + # Need to enforce UTC as df_prediction for which these values will be + # put into is already read in as UTC (if it already exists before this + # flow). + pd.to_datetime(ds_MVBS_combine["ping_time"][0].values, utc=True), + pd.to_datetime(ds_MVBS_combine["ping_time"][-1].values, utc=True), ) @@ -886,24 +899,15 @@ def task_predict_hake( def task_compute_NASC( NASC_filename: str, ds_MVBS_combine: xr.Dataset, - da_score_softmax: xr.DataArray, - softmax_threshold: float = 0.5, + da_predict_hake: xr.DataArray, path_NASC_zarr: str = "PATH_TO_SAVE_NASC_ZARR", ): logger = get_run_logger() - # Prepare softmax score dimensions for apply_mask - da_score_softmax = ( - da_score_softmax - .sel(scatterer_class="hake") # only need hake class - .transpose("ping_time", "depth") # TODO: remove once update echopype to 0.10.2 - .drop_vars("scatterer_class") - ) - # Apply mask based on threshold ds_MVBS_combine_masked = ep.mask.apply_mask( source_ds=ds_MVBS_combine, - mask=da_score_softmax > softmax_threshold, + mask=da_predict_hake, var_name="Sv", fill_value=np.nan, ) diff --git a/src/echodataflow/flows/flows_helper.py b/src/echodataflow/flows/flows_helper.py index cc8be10..a49f114 100644 --- a/src/echodataflow/flows/flows_helper.py +++ b/src/echodataflow/flows/flows_helper.py @@ -55,10 +55,9 @@ def flow_file_upload( # Potentially long running so using a context manager if max_age == -1: - command = f"rclone copy -v --no-traverse {src_dir} {dest_dir} --exclude-from {str(exclude_path)}" + command = f"rclone copy -v --no-traverse --s3-no-check-bucket {src_dir} {dest_dir} --exclude-from {str(exclude_path)}" else: - command = f"rclone copy -v --max-age {max_age}h --no-traverse {src_dir} {dest_dir} --exclude-from {str(exclude_path)}" - print("command:", command) + command = f"rclone copy -v --max-age {max_age}h --no-traverse --s3-no-check-bucket {src_dir} {dest_dir} --exclude-from {str(exclude_path)}" with ShellOperation(commands=[command], working_dir=src_dir) as file_upload_operation: # Trigger runs the process in the background diff --git a/src/echodataflow/utils/utils.py b/src/echodataflow/utils/utils.py index 4109c01..d381bc2 100644 --- a/src/echodataflow/utils/utils.py +++ b/src/echodataflow/utils/utils.py @@ -3,54 +3,6 @@ import datetime import numpy as np import pandas as pd -# import torch -# from src.model.BinaryHakeModel import BinaryHakeModel - - -# def get_MVBS_tensor(ds_in, freq_wanted=[120000, 38000, 18000]): -# # Find the right channel sequence -# ch_wanted = [int((np.abs(ds_in["frequency_nominal"]-freq)).argmin()) for freq in freq_wanted] - -# # Crucial for model prediction: -# # - order of dimension (channel, depth, ping_time) -# # - depth slice up to 590 m -# mvbs_tensor = torch.tensor( -# ( -# ds_in["Sv"] -# .transpose("channel", "depth", "ping_time") -# .isel(channel=ch_wanted).sel(depth=slice(None, 590)).values -# ), -# dtype=torch.float32 -# ) - -# # Clip to Sv range -# mvbs_tensor_clip = torch.clip( -# mvbs_tensor.clone().detach().to(torch.float16), -# min=-70, -# max=-36, -# ) - -# # Replace NaN values with min Sv -# mvbs_tensor_clip[torch.isnan(mvbs_tensor_clip)] = -70 - -# # Normalize and conver to float -# mvbs_tensor_clip_normalized = ( -# (mvbs_tensor_clip - (-70.0)) / (-36.0 - (-70.0)) * 255.0 -# ) - -# return mvbs_tensor_clip_normalized.unsqueeze(0).float() - - -# # Load binary hake models with weights -# def get_hake_model(model_path: str) -> BinaryHakeModel: -# placeholder_score_dir = Path( -# str(files("echodataflow.assets") / "placeholder_score_tensor_dir") -# ) -# model = BinaryHakeModel("placeholder_experiment_name", -# placeholder_score_dir, -# "placeholder_tensor_log_dir", 0).eval() -# model.load_state_dict(torch.load(model_path, map_location=torch.device('cpu'))["state_dict"]) -# return model def round_up_mins(dt: datetime.datetime, slice_mins: int) -> datetime.datetime: diff --git a/test.ipynb b/test.ipynb new file mode 100644 index 0000000..ac8dd07 --- /dev/null +++ b/test.ipynb @@ -0,0 +1,1301 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "71656e3f", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "c4f45b0f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
raw_filenameSv_filenamefirst_ping_timelast_ping_time
0Hake-D20230804-T164148.rawHake-D20230804-T164148_Sv.zarr2023-08-04 16:41:48.3079492023-08-04 16:46:09.154161
1Hake-D20230804-T164610.rawHake-D20230804-T164610_Sv.zarr2023-08-04 16:46:10.3499792023-08-04 16:50:26.749968
2Hake-D20230804-T165027.rawHake-D20230804-T165027_Sv.zarr2023-08-04 16:50:27.9617292023-08-04 16:54:51.134266
3Hake-D20230804-T165452.rawHake-D20230804-T165452_Sv.zarr2023-08-04 16:54:52.3550052023-08-04 16:59:15.220356
4Hake-D20230804-T165916.rawHake-D20230804-T165916_Sv.zarr2023-08-04 16:59:16.4141652023-08-04 17:03:38.062754
5Hake-D20230804-T170339.rawHake-D20230804-T170339_Sv.zarr2023-08-04 17:03:39.2795012023-08-04 17:07:57.800514
6Hake-D20230804-T170758.rawHake-D20230804-T170758_Sv.zarr2023-08-04 17:07:58.9913162023-08-04 17:12:19.872564
7Hake-D20230804-T171221.rawHake-D20230804-T171221_Sv.zarr2023-08-04 17:12:21.0563832023-08-04 17:16:40.626137
8Hake-D20230804-T171641.rawHake-D20230804-T171641_Sv.zarr2023-08-04 17:16:41.8119812023-08-04 17:21:01.645643
9Hake-D20230804-T172102.rawHake-D20230804-T172102_Sv.zarr2023-08-04 17:21:02.8623872023-08-04 17:25:22.924578
10Hake-D20230804-T172524.rawHake-D20230804-T172524_Sv.zarr2023-08-04 17:25:24.1443142023-08-04 17:29:48.684889
11Hake-D20230804-T172949.rawHake-D20230804-T172949_Sv.zarr2023-08-04 17:29:49.9021302023-08-04 17:34:44.950945
12Hake-D20230804-T173446.rawHake-D20230804-T173446_Sv.zarr2023-08-04 17:34:46.4928232023-08-04 17:40:18.000994
13Hake-D20230804-T174019.rawHake-D20230804-T174019_Sv.zarr2023-08-04 17:40:19.5727922023-08-04 17:45:52.363268
14Hake-D20230804-T174553.rawHake-D20230804-T174553_Sv.zarr2023-08-04 17:45:53.9151252023-08-04 17:51:31.728239
15Hake-D20230804-T175133.rawHake-D20230804-T175133_Sv.zarr2023-08-04 17:51:33.2711322023-08-04 17:58:09.737223
16Hake-D20230804-T175811.rawHake-D20230804-T175811_Sv.zarr2023-08-04 17:58:11.7438582023-08-04 18:05:53.087282
17Hake-D20230804-T180555.rawHake-D20230804-T180555_Sv.zarr2023-08-04 18:05:55.2255672023-08-04 18:13:08.444262
18Hake-D20230804-T181310.rawHake-D20230804-T181310_Sv.zarr2023-08-04 18:13:10.6493722023-08-04 18:21:01.011724
19Hake-D20230804-T182103.rawHake-D20230804-T182103_Sv.zarr2023-08-04 18:21:03.2158372023-08-04 18:29:02.296200
\n", + "
" + ], + "text/plain": [ + " raw_filename Sv_filename \\\n", + "0 Hake-D20230804-T164148.raw Hake-D20230804-T164148_Sv.zarr \n", + "1 Hake-D20230804-T164610.raw Hake-D20230804-T164610_Sv.zarr \n", + "2 Hake-D20230804-T165027.raw Hake-D20230804-T165027_Sv.zarr \n", + "3 Hake-D20230804-T165452.raw Hake-D20230804-T165452_Sv.zarr \n", + "4 Hake-D20230804-T165916.raw Hake-D20230804-T165916_Sv.zarr \n", + "5 Hake-D20230804-T170339.raw Hake-D20230804-T170339_Sv.zarr \n", + "6 Hake-D20230804-T170758.raw Hake-D20230804-T170758_Sv.zarr \n", + "7 Hake-D20230804-T171221.raw Hake-D20230804-T171221_Sv.zarr \n", + "8 Hake-D20230804-T171641.raw Hake-D20230804-T171641_Sv.zarr \n", + "9 Hake-D20230804-T172102.raw Hake-D20230804-T172102_Sv.zarr \n", + "10 Hake-D20230804-T172524.raw Hake-D20230804-T172524_Sv.zarr \n", + "11 Hake-D20230804-T172949.raw Hake-D20230804-T172949_Sv.zarr \n", + "12 Hake-D20230804-T173446.raw Hake-D20230804-T173446_Sv.zarr \n", + "13 Hake-D20230804-T174019.raw Hake-D20230804-T174019_Sv.zarr \n", + "14 Hake-D20230804-T174553.raw Hake-D20230804-T174553_Sv.zarr \n", + "15 Hake-D20230804-T175133.raw Hake-D20230804-T175133_Sv.zarr \n", + "16 Hake-D20230804-T175811.raw Hake-D20230804-T175811_Sv.zarr \n", + "17 Hake-D20230804-T180555.raw Hake-D20230804-T180555_Sv.zarr \n", + "18 Hake-D20230804-T181310.raw Hake-D20230804-T181310_Sv.zarr \n", + "19 Hake-D20230804-T182103.raw Hake-D20230804-T182103_Sv.zarr \n", + "\n", + " first_ping_time last_ping_time \n", + "0 2023-08-04 16:41:48.307949 2023-08-04 16:46:09.154161 \n", + "1 2023-08-04 16:46:10.349979 2023-08-04 16:50:26.749968 \n", + "2 2023-08-04 16:50:27.961729 2023-08-04 16:54:51.134266 \n", + "3 2023-08-04 16:54:52.355005 2023-08-04 16:59:15.220356 \n", + "4 2023-08-04 16:59:16.414165 2023-08-04 17:03:38.062754 \n", + "5 2023-08-04 17:03:39.279501 2023-08-04 17:07:57.800514 \n", + "6 2023-08-04 17:07:58.991316 2023-08-04 17:12:19.872564 \n", + "7 2023-08-04 17:12:21.056383 2023-08-04 17:16:40.626137 \n", + "8 2023-08-04 17:16:41.811981 2023-08-04 17:21:01.645643 \n", + "9 2023-08-04 17:21:02.862387 2023-08-04 17:25:22.924578 \n", + "10 2023-08-04 17:25:24.144314 2023-08-04 17:29:48.684889 \n", + "11 2023-08-04 17:29:49.902130 2023-08-04 17:34:44.950945 \n", + "12 2023-08-04 17:34:46.492823 2023-08-04 17:40:18.000994 \n", + "13 2023-08-04 17:40:19.572792 2023-08-04 17:45:52.363268 \n", + "14 2023-08-04 17:45:53.915125 2023-08-04 17:51:31.728239 \n", + "15 2023-08-04 17:51:33.271132 2023-08-04 17:58:09.737223 \n", + "16 2023-08-04 17:58:11.743858 2023-08-04 18:05:53.087282 \n", + "17 2023-08-04 18:05:55.225567 2023-08-04 18:13:08.444262 \n", + "18 2023-08-04 18:13:10.649372 2023-08-04 18:21:01.011724 \n", + "19 2023-08-04 18:21:03.215837 2023-08-04 18:29:02.296200 " + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_Sv = pd.read_csv(\n", + " \"/home/ctuguinay/echodataflow_test/output/Sv_files.csv\",\n", + " index_col=0,\n", + " date_format=\"ISO8601\",\n", + " parse_dates=[\"first_ping_time\", \"last_ping_time\"]\n", + ")\n", + "df_Sv" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "df482d7d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
MVBS_filenamefirst_ping_timelast_ping_time
0MVBS_20230804T164000.zarr2023-08-04 16:41:452023-08-04 16:59:55
1MVBS_20230804T170000.zarr2023-08-04 17:00:002023-08-04 17:19:55
2MVBS_20230804T172000.zarr2023-08-04 17:20:002023-08-04 17:39:55
3MVBS_20230804T174000.zarr2023-08-04 17:40:002023-08-04 17:59:55
4MVBS_20230804T180000.zarr2023-08-04 18:00:002023-08-04 18:19:55
5MVBS_20230804T182000.zarr2023-08-04 18:20:002023-08-04 18:29:00
\n", + "
" + ], + "text/plain": [ + " MVBS_filename first_ping_time last_ping_time\n", + "0 MVBS_20230804T164000.zarr 2023-08-04 16:41:45 2023-08-04 16:59:55\n", + "1 MVBS_20230804T170000.zarr 2023-08-04 17:00:00 2023-08-04 17:19:55\n", + "2 MVBS_20230804T172000.zarr 2023-08-04 17:20:00 2023-08-04 17:39:55\n", + "3 MVBS_20230804T174000.zarr 2023-08-04 17:40:00 2023-08-04 17:59:55\n", + "4 MVBS_20230804T180000.zarr 2023-08-04 18:00:00 2023-08-04 18:19:55\n", + "5 MVBS_20230804T182000.zarr 2023-08-04 18:20:00 2023-08-04 18:29:00" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_MVBS = pd.read_csv(\n", + " \"/home/ctuguinay/echodataflow_test/output/MVBS_files.csv\",\n", + " index_col=0,\n", + " date_format=\"ISO8601\",\n", + " parse_dates=[\"first_ping_time\", \"last_ping_time\"]\n", + ")\n", + "df_MVBS" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "43c553a0", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
prediction_filename_postfixscore_filenamesoftmax_filenamefirst_ping_timelast_ping_time
020230804T164000score_20230804T164000.zarrsoftmax_20230804T164000.zarr2023-08-04 16:41:45+00:002023-08-04 17:19:55+00:00
120230804T172000score_20230804T172000.zarrsoftmax_20230804T172000.zarr2023-08-04 17:20:00+00:002023-08-04 17:39:55+00:00
220230804T180000score_20230804T180000.zarrsoftmax_20230804T180000.zarr2023-08-04 18:00:002023-08-04 18:19:55
\n", + "
" + ], + "text/plain": [ + " prediction_filename_postfix score_filename \\\n", + "0 20230804T164000 score_20230804T164000.zarr \n", + "1 20230804T172000 score_20230804T172000.zarr \n", + "2 20230804T180000 score_20230804T180000.zarr \n", + "\n", + " softmax_filename first_ping_time \\\n", + "0 softmax_20230804T164000.zarr 2023-08-04 16:41:45+00:00 \n", + "1 softmax_20230804T172000.zarr 2023-08-04 17:20:00+00:00 \n", + "2 softmax_20230804T180000.zarr 2023-08-04 18:00:00 \n", + "\n", + " last_ping_time \n", + "0 2023-08-04 17:19:55+00:00 \n", + "1 2023-08-04 17:39:55+00:00 \n", + "2 2023-08-04 18:19:55 " + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_prediction = pd.read_csv(\n", + " \"/home/ctuguinay/echodataflow_test/output/prediction_files.csv\",\n", + " index_col=0,\n", + " date_format=\"ISO8601\",\n", + " parse_dates=[\"first_ping_time\", \"last_ping_time\"]\n", + ")\n", + "df_prediction" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "9dee303d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0 2023-08-04 16:59:55\n", + "1 2023-08-04 17:19:55\n", + "2 2023-08-04 17:39:55\n", + "3 2023-08-04 17:59:55\n", + "4 2023-08-04 18:19:55\n", + "5 2023-08-04 18:29:00\n", + "Name: last_ping_time, dtype: datetime64[us]" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_MVBS[\"last_ping_time\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "97668be4", + "metadata": {}, + "outputs": [], + "source": [ + "import xarray as xr" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "996b55c2", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.DataArray 'ping_time' (ping_time: 109)> Size: 872B\n",
+       "array(['2023-08-04T18:20:00.000000000', '2023-08-04T18:20:05.000000000',\n",
+       "       '2023-08-04T18:20:10.000000000', '2023-08-04T18:20:15.000000000',\n",
+       "       '2023-08-04T18:20:20.000000000', '2023-08-04T18:20:25.000000000',\n",
+       "       '2023-08-04T18:20:30.000000000', '2023-08-04T18:20:35.000000000',\n",
+       "       '2023-08-04T18:20:40.000000000', '2023-08-04T18:20:45.000000000',\n",
+       "       '2023-08-04T18:20:50.000000000', '2023-08-04T18:20:55.000000000',\n",
+       "       '2023-08-04T18:21:00.000000000', '2023-08-04T18:21:05.000000000',\n",
+       "       '2023-08-04T18:21:10.000000000', '2023-08-04T18:21:15.000000000',\n",
+       "       '2023-08-04T18:21:20.000000000', '2023-08-04T18:21:25.000000000',\n",
+       "       '2023-08-04T18:21:30.000000000', '2023-08-04T18:21:35.000000000',\n",
+       "       '2023-08-04T18:21:40.000000000', '2023-08-04T18:21:45.000000000',\n",
+       "       '2023-08-04T18:21:50.000000000', '2023-08-04T18:21:55.000000000',\n",
+       "       '2023-08-04T18:22:00.000000000', '2023-08-04T18:22:05.000000000',\n",
+       "       '2023-08-04T18:22:10.000000000', '2023-08-04T18:22:15.000000000',\n",
+       "       '2023-08-04T18:22:20.000000000', '2023-08-04T18:22:25.000000000',\n",
+       "       '2023-08-04T18:22:30.000000000', '2023-08-04T18:22:35.000000000',\n",
+       "       '2023-08-04T18:22:40.000000000', '2023-08-04T18:22:45.000000000',\n",
+       "       '2023-08-04T18:22:50.000000000', '2023-08-04T18:22:55.000000000',\n",
+       "       '2023-08-04T18:23:00.000000000', '2023-08-04T18:23:05.000000000',\n",
+       "       '2023-08-04T18:23:10.000000000', '2023-08-04T18:23:15.000000000',\n",
+       "       '2023-08-04T18:23:20.000000000', '2023-08-04T18:23:25.000000000',\n",
+       "       '2023-08-04T18:23:30.000000000', '2023-08-04T18:23:35.000000000',\n",
+       "       '2023-08-04T18:23:40.000000000', '2023-08-04T18:23:45.000000000',\n",
+       "       '2023-08-04T18:23:50.000000000', '2023-08-04T18:23:55.000000000',\n",
+       "       '2023-08-04T18:24:00.000000000', '2023-08-04T18:24:05.000000000',\n",
+       "       '2023-08-04T18:24:10.000000000', '2023-08-04T18:24:15.000000000',\n",
+       "       '2023-08-04T18:24:20.000000000', '2023-08-04T18:24:25.000000000',\n",
+       "       '2023-08-04T18:24:30.000000000', '2023-08-04T18:24:35.000000000',\n",
+       "       '2023-08-04T18:24:40.000000000', '2023-08-04T18:24:45.000000000',\n",
+       "       '2023-08-04T18:24:50.000000000', '2023-08-04T18:24:55.000000000',\n",
+       "       '2023-08-04T18:25:00.000000000', '2023-08-04T18:25:05.000000000',\n",
+       "       '2023-08-04T18:25:10.000000000', '2023-08-04T18:25:15.000000000',\n",
+       "       '2023-08-04T18:25:20.000000000', '2023-08-04T18:25:25.000000000',\n",
+       "       '2023-08-04T18:25:30.000000000', '2023-08-04T18:25:35.000000000',\n",
+       "       '2023-08-04T18:25:40.000000000', '2023-08-04T18:25:45.000000000',\n",
+       "       '2023-08-04T18:25:50.000000000', '2023-08-04T18:25:55.000000000',\n",
+       "       '2023-08-04T18:26:00.000000000', '2023-08-04T18:26:05.000000000',\n",
+       "       '2023-08-04T18:26:10.000000000', '2023-08-04T18:26:15.000000000',\n",
+       "       '2023-08-04T18:26:20.000000000', '2023-08-04T18:26:25.000000000',\n",
+       "       '2023-08-04T18:26:30.000000000', '2023-08-04T18:26:35.000000000',\n",
+       "       '2023-08-04T18:26:40.000000000', '2023-08-04T18:26:45.000000000',\n",
+       "       '2023-08-04T18:26:50.000000000', '2023-08-04T18:26:55.000000000',\n",
+       "       '2023-08-04T18:27:00.000000000', '2023-08-04T18:27:05.000000000',\n",
+       "       '2023-08-04T18:27:10.000000000', '2023-08-04T18:27:15.000000000',\n",
+       "       '2023-08-04T18:27:20.000000000', '2023-08-04T18:27:25.000000000',\n",
+       "       '2023-08-04T18:27:30.000000000', '2023-08-04T18:27:35.000000000',\n",
+       "       '2023-08-04T18:27:40.000000000', '2023-08-04T18:27:45.000000000',\n",
+       "       '2023-08-04T18:27:50.000000000', '2023-08-04T18:27:55.000000000',\n",
+       "       '2023-08-04T18:28:00.000000000', '2023-08-04T18:28:05.000000000',\n",
+       "       '2023-08-04T18:28:10.000000000', '2023-08-04T18:28:15.000000000',\n",
+       "       '2023-08-04T18:28:20.000000000', '2023-08-04T18:28:25.000000000',\n",
+       "       '2023-08-04T18:28:30.000000000', '2023-08-04T18:28:35.000000000',\n",
+       "       '2023-08-04T18:28:40.000000000', '2023-08-04T18:28:45.000000000',\n",
+       "       '2023-08-04T18:28:50.000000000', '2023-08-04T18:28:55.000000000',\n",
+       "       '2023-08-04T18:29:00.000000000'], dtype='datetime64[ns]')\n",
+       "Coordinates:\n",
+       "  * ping_time  (ping_time) datetime64[ns] 872B 2023-08-04T18:20:00 ... 2023-0...\n",
+       "Attributes:\n",
+       "    long_name:      Ping time\n",
+       "    standard_name:  time\n",
+       "    axis:           T
" + ], + "text/plain": [ + " Size: 872B\n", + "array(['2023-08-04T18:20:00.000000000', '2023-08-04T18:20:05.000000000',\n", + " '2023-08-04T18:20:10.000000000', '2023-08-04T18:20:15.000000000',\n", + " '2023-08-04T18:20:20.000000000', '2023-08-04T18:20:25.000000000',\n", + " '2023-08-04T18:20:30.000000000', '2023-08-04T18:20:35.000000000',\n", + " '2023-08-04T18:20:40.000000000', '2023-08-04T18:20:45.000000000',\n", + " '2023-08-04T18:20:50.000000000', '2023-08-04T18:20:55.000000000',\n", + " '2023-08-04T18:21:00.000000000', '2023-08-04T18:21:05.000000000',\n", + " '2023-08-04T18:21:10.000000000', '2023-08-04T18:21:15.000000000',\n", + " '2023-08-04T18:21:20.000000000', '2023-08-04T18:21:25.000000000',\n", + " '2023-08-04T18:21:30.000000000', '2023-08-04T18:21:35.000000000',\n", + " '2023-08-04T18:21:40.000000000', '2023-08-04T18:21:45.000000000',\n", + " '2023-08-04T18:21:50.000000000', '2023-08-04T18:21:55.000000000',\n", + " '2023-08-04T18:22:00.000000000', '2023-08-04T18:22:05.000000000',\n", + " '2023-08-04T18:22:10.000000000', '2023-08-04T18:22:15.000000000',\n", + " '2023-08-04T18:22:20.000000000', '2023-08-04T18:22:25.000000000',\n", + " '2023-08-04T18:22:30.000000000', '2023-08-04T18:22:35.000000000',\n", + " '2023-08-04T18:22:40.000000000', '2023-08-04T18:22:45.000000000',\n", + " '2023-08-04T18:22:50.000000000', '2023-08-04T18:22:55.000000000',\n", + " '2023-08-04T18:23:00.000000000', '2023-08-04T18:23:05.000000000',\n", + " '2023-08-04T18:23:10.000000000', '2023-08-04T18:23:15.000000000',\n", + " '2023-08-04T18:23:20.000000000', '2023-08-04T18:23:25.000000000',\n", + " '2023-08-04T18:23:30.000000000', '2023-08-04T18:23:35.000000000',\n", + " '2023-08-04T18:23:40.000000000', '2023-08-04T18:23:45.000000000',\n", + " '2023-08-04T18:23:50.000000000', '2023-08-04T18:23:55.000000000',\n", + " '2023-08-04T18:24:00.000000000', '2023-08-04T18:24:05.000000000',\n", + " '2023-08-04T18:24:10.000000000', '2023-08-04T18:24:15.000000000',\n", + " '2023-08-04T18:24:20.000000000', '2023-08-04T18:24:25.000000000',\n", + " '2023-08-04T18:24:30.000000000', '2023-08-04T18:24:35.000000000',\n", + " '2023-08-04T18:24:40.000000000', '2023-08-04T18:24:45.000000000',\n", + " '2023-08-04T18:24:50.000000000', '2023-08-04T18:24:55.000000000',\n", + " '2023-08-04T18:25:00.000000000', '2023-08-04T18:25:05.000000000',\n", + " '2023-08-04T18:25:10.000000000', '2023-08-04T18:25:15.000000000',\n", + " '2023-08-04T18:25:20.000000000', '2023-08-04T18:25:25.000000000',\n", + " '2023-08-04T18:25:30.000000000', '2023-08-04T18:25:35.000000000',\n", + " '2023-08-04T18:25:40.000000000', '2023-08-04T18:25:45.000000000',\n", + " '2023-08-04T18:25:50.000000000', '2023-08-04T18:25:55.000000000',\n", + " '2023-08-04T18:26:00.000000000', '2023-08-04T18:26:05.000000000',\n", + " '2023-08-04T18:26:10.000000000', '2023-08-04T18:26:15.000000000',\n", + " '2023-08-04T18:26:20.000000000', '2023-08-04T18:26:25.000000000',\n", + " '2023-08-04T18:26:30.000000000', '2023-08-04T18:26:35.000000000',\n", + " '2023-08-04T18:26:40.000000000', '2023-08-04T18:26:45.000000000',\n", + " '2023-08-04T18:26:50.000000000', '2023-08-04T18:26:55.000000000',\n", + " '2023-08-04T18:27:00.000000000', '2023-08-04T18:27:05.000000000',\n", + " '2023-08-04T18:27:10.000000000', '2023-08-04T18:27:15.000000000',\n", + " '2023-08-04T18:27:20.000000000', '2023-08-04T18:27:25.000000000',\n", + " '2023-08-04T18:27:30.000000000', '2023-08-04T18:27:35.000000000',\n", + " '2023-08-04T18:27:40.000000000', '2023-08-04T18:27:45.000000000',\n", + " '2023-08-04T18:27:50.000000000', '2023-08-04T18:27:55.000000000',\n", + " '2023-08-04T18:28:00.000000000', '2023-08-04T18:28:05.000000000',\n", + " '2023-08-04T18:28:10.000000000', '2023-08-04T18:28:15.000000000',\n", + " '2023-08-04T18:28:20.000000000', '2023-08-04T18:28:25.000000000',\n", + " '2023-08-04T18:28:30.000000000', '2023-08-04T18:28:35.000000000',\n", + " '2023-08-04T18:28:40.000000000', '2023-08-04T18:28:45.000000000',\n", + " '2023-08-04T18:28:50.000000000', '2023-08-04T18:28:55.000000000',\n", + " '2023-08-04T18:29:00.000000000'], dtype='datetime64[ns]')\n", + "Coordinates:\n", + " * ping_time (ping_time) datetime64[ns] 872B 2023-08-04T18:20:00 ... 2023-0...\n", + "Attributes:\n", + " long_name: Ping time\n", + " standard_name: time\n", + " axis: T" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds_MVBS = xr.open_zarr(\"/home/ctuguinay/echodataflow_test/output_20260625/MVBS/MVBS_20230804T182000.zarr\",)\n", + "ds_MVBS[\"ping_time\"]" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "echodataflow", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 7a5b9dc14ef4dcb2c83cc1b467e0d579429a0a29 Mon Sep 17 00:00:00 2001 From: ctuguinay Date: Thu, 2 Jul 2026 19:12:55 -0700 Subject: [PATCH 03/10] remove test ipynb --- test.ipynb | 1301 ---------------------------------------------------- 1 file changed, 1301 deletions(-) delete mode 100644 test.ipynb diff --git a/test.ipynb b/test.ipynb deleted file mode 100644 index ac8dd07..0000000 --- a/test.ipynb +++ /dev/null @@ -1,1301 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "id": "71656e3f", - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "c4f45b0f", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
raw_filenameSv_filenamefirst_ping_timelast_ping_time
0Hake-D20230804-T164148.rawHake-D20230804-T164148_Sv.zarr2023-08-04 16:41:48.3079492023-08-04 16:46:09.154161
1Hake-D20230804-T164610.rawHake-D20230804-T164610_Sv.zarr2023-08-04 16:46:10.3499792023-08-04 16:50:26.749968
2Hake-D20230804-T165027.rawHake-D20230804-T165027_Sv.zarr2023-08-04 16:50:27.9617292023-08-04 16:54:51.134266
3Hake-D20230804-T165452.rawHake-D20230804-T165452_Sv.zarr2023-08-04 16:54:52.3550052023-08-04 16:59:15.220356
4Hake-D20230804-T165916.rawHake-D20230804-T165916_Sv.zarr2023-08-04 16:59:16.4141652023-08-04 17:03:38.062754
5Hake-D20230804-T170339.rawHake-D20230804-T170339_Sv.zarr2023-08-04 17:03:39.2795012023-08-04 17:07:57.800514
6Hake-D20230804-T170758.rawHake-D20230804-T170758_Sv.zarr2023-08-04 17:07:58.9913162023-08-04 17:12:19.872564
7Hake-D20230804-T171221.rawHake-D20230804-T171221_Sv.zarr2023-08-04 17:12:21.0563832023-08-04 17:16:40.626137
8Hake-D20230804-T171641.rawHake-D20230804-T171641_Sv.zarr2023-08-04 17:16:41.8119812023-08-04 17:21:01.645643
9Hake-D20230804-T172102.rawHake-D20230804-T172102_Sv.zarr2023-08-04 17:21:02.8623872023-08-04 17:25:22.924578
10Hake-D20230804-T172524.rawHake-D20230804-T172524_Sv.zarr2023-08-04 17:25:24.1443142023-08-04 17:29:48.684889
11Hake-D20230804-T172949.rawHake-D20230804-T172949_Sv.zarr2023-08-04 17:29:49.9021302023-08-04 17:34:44.950945
12Hake-D20230804-T173446.rawHake-D20230804-T173446_Sv.zarr2023-08-04 17:34:46.4928232023-08-04 17:40:18.000994
13Hake-D20230804-T174019.rawHake-D20230804-T174019_Sv.zarr2023-08-04 17:40:19.5727922023-08-04 17:45:52.363268
14Hake-D20230804-T174553.rawHake-D20230804-T174553_Sv.zarr2023-08-04 17:45:53.9151252023-08-04 17:51:31.728239
15Hake-D20230804-T175133.rawHake-D20230804-T175133_Sv.zarr2023-08-04 17:51:33.2711322023-08-04 17:58:09.737223
16Hake-D20230804-T175811.rawHake-D20230804-T175811_Sv.zarr2023-08-04 17:58:11.7438582023-08-04 18:05:53.087282
17Hake-D20230804-T180555.rawHake-D20230804-T180555_Sv.zarr2023-08-04 18:05:55.2255672023-08-04 18:13:08.444262
18Hake-D20230804-T181310.rawHake-D20230804-T181310_Sv.zarr2023-08-04 18:13:10.6493722023-08-04 18:21:01.011724
19Hake-D20230804-T182103.rawHake-D20230804-T182103_Sv.zarr2023-08-04 18:21:03.2158372023-08-04 18:29:02.296200
\n", - "
" - ], - "text/plain": [ - " raw_filename Sv_filename \\\n", - "0 Hake-D20230804-T164148.raw Hake-D20230804-T164148_Sv.zarr \n", - "1 Hake-D20230804-T164610.raw Hake-D20230804-T164610_Sv.zarr \n", - "2 Hake-D20230804-T165027.raw Hake-D20230804-T165027_Sv.zarr \n", - "3 Hake-D20230804-T165452.raw Hake-D20230804-T165452_Sv.zarr \n", - "4 Hake-D20230804-T165916.raw Hake-D20230804-T165916_Sv.zarr \n", - "5 Hake-D20230804-T170339.raw Hake-D20230804-T170339_Sv.zarr \n", - "6 Hake-D20230804-T170758.raw Hake-D20230804-T170758_Sv.zarr \n", - "7 Hake-D20230804-T171221.raw Hake-D20230804-T171221_Sv.zarr \n", - "8 Hake-D20230804-T171641.raw Hake-D20230804-T171641_Sv.zarr \n", - "9 Hake-D20230804-T172102.raw Hake-D20230804-T172102_Sv.zarr \n", - "10 Hake-D20230804-T172524.raw Hake-D20230804-T172524_Sv.zarr \n", - "11 Hake-D20230804-T172949.raw Hake-D20230804-T172949_Sv.zarr \n", - "12 Hake-D20230804-T173446.raw Hake-D20230804-T173446_Sv.zarr \n", - "13 Hake-D20230804-T174019.raw Hake-D20230804-T174019_Sv.zarr \n", - "14 Hake-D20230804-T174553.raw Hake-D20230804-T174553_Sv.zarr \n", - "15 Hake-D20230804-T175133.raw Hake-D20230804-T175133_Sv.zarr \n", - "16 Hake-D20230804-T175811.raw Hake-D20230804-T175811_Sv.zarr \n", - "17 Hake-D20230804-T180555.raw Hake-D20230804-T180555_Sv.zarr \n", - "18 Hake-D20230804-T181310.raw Hake-D20230804-T181310_Sv.zarr \n", - "19 Hake-D20230804-T182103.raw Hake-D20230804-T182103_Sv.zarr \n", - "\n", - " first_ping_time last_ping_time \n", - "0 2023-08-04 16:41:48.307949 2023-08-04 16:46:09.154161 \n", - "1 2023-08-04 16:46:10.349979 2023-08-04 16:50:26.749968 \n", - "2 2023-08-04 16:50:27.961729 2023-08-04 16:54:51.134266 \n", - "3 2023-08-04 16:54:52.355005 2023-08-04 16:59:15.220356 \n", - "4 2023-08-04 16:59:16.414165 2023-08-04 17:03:38.062754 \n", - "5 2023-08-04 17:03:39.279501 2023-08-04 17:07:57.800514 \n", - "6 2023-08-04 17:07:58.991316 2023-08-04 17:12:19.872564 \n", - "7 2023-08-04 17:12:21.056383 2023-08-04 17:16:40.626137 \n", - "8 2023-08-04 17:16:41.811981 2023-08-04 17:21:01.645643 \n", - "9 2023-08-04 17:21:02.862387 2023-08-04 17:25:22.924578 \n", - "10 2023-08-04 17:25:24.144314 2023-08-04 17:29:48.684889 \n", - "11 2023-08-04 17:29:49.902130 2023-08-04 17:34:44.950945 \n", - "12 2023-08-04 17:34:46.492823 2023-08-04 17:40:18.000994 \n", - "13 2023-08-04 17:40:19.572792 2023-08-04 17:45:52.363268 \n", - "14 2023-08-04 17:45:53.915125 2023-08-04 17:51:31.728239 \n", - "15 2023-08-04 17:51:33.271132 2023-08-04 17:58:09.737223 \n", - "16 2023-08-04 17:58:11.743858 2023-08-04 18:05:53.087282 \n", - "17 2023-08-04 18:05:55.225567 2023-08-04 18:13:08.444262 \n", - "18 2023-08-04 18:13:10.649372 2023-08-04 18:21:01.011724 \n", - "19 2023-08-04 18:21:03.215837 2023-08-04 18:29:02.296200 " - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df_Sv = pd.read_csv(\n", - " \"/home/ctuguinay/echodataflow_test/output/Sv_files.csv\",\n", - " index_col=0,\n", - " date_format=\"ISO8601\",\n", - " parse_dates=[\"first_ping_time\", \"last_ping_time\"]\n", - ")\n", - "df_Sv" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "df482d7d", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
MVBS_filenamefirst_ping_timelast_ping_time
0MVBS_20230804T164000.zarr2023-08-04 16:41:452023-08-04 16:59:55
1MVBS_20230804T170000.zarr2023-08-04 17:00:002023-08-04 17:19:55
2MVBS_20230804T172000.zarr2023-08-04 17:20:002023-08-04 17:39:55
3MVBS_20230804T174000.zarr2023-08-04 17:40:002023-08-04 17:59:55
4MVBS_20230804T180000.zarr2023-08-04 18:00:002023-08-04 18:19:55
5MVBS_20230804T182000.zarr2023-08-04 18:20:002023-08-04 18:29:00
\n", - "
" - ], - "text/plain": [ - " MVBS_filename first_ping_time last_ping_time\n", - "0 MVBS_20230804T164000.zarr 2023-08-04 16:41:45 2023-08-04 16:59:55\n", - "1 MVBS_20230804T170000.zarr 2023-08-04 17:00:00 2023-08-04 17:19:55\n", - "2 MVBS_20230804T172000.zarr 2023-08-04 17:20:00 2023-08-04 17:39:55\n", - "3 MVBS_20230804T174000.zarr 2023-08-04 17:40:00 2023-08-04 17:59:55\n", - "4 MVBS_20230804T180000.zarr 2023-08-04 18:00:00 2023-08-04 18:19:55\n", - "5 MVBS_20230804T182000.zarr 2023-08-04 18:20:00 2023-08-04 18:29:00" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df_MVBS = pd.read_csv(\n", - " \"/home/ctuguinay/echodataflow_test/output/MVBS_files.csv\",\n", - " index_col=0,\n", - " date_format=\"ISO8601\",\n", - " parse_dates=[\"first_ping_time\", \"last_ping_time\"]\n", - ")\n", - "df_MVBS" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "43c553a0", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
prediction_filename_postfixscore_filenamesoftmax_filenamefirst_ping_timelast_ping_time
020230804T164000score_20230804T164000.zarrsoftmax_20230804T164000.zarr2023-08-04 16:41:45+00:002023-08-04 17:19:55+00:00
120230804T172000score_20230804T172000.zarrsoftmax_20230804T172000.zarr2023-08-04 17:20:00+00:002023-08-04 17:39:55+00:00
220230804T180000score_20230804T180000.zarrsoftmax_20230804T180000.zarr2023-08-04 18:00:002023-08-04 18:19:55
\n", - "
" - ], - "text/plain": [ - " prediction_filename_postfix score_filename \\\n", - "0 20230804T164000 score_20230804T164000.zarr \n", - "1 20230804T172000 score_20230804T172000.zarr \n", - "2 20230804T180000 score_20230804T180000.zarr \n", - "\n", - " softmax_filename first_ping_time \\\n", - "0 softmax_20230804T164000.zarr 2023-08-04 16:41:45+00:00 \n", - "1 softmax_20230804T172000.zarr 2023-08-04 17:20:00+00:00 \n", - "2 softmax_20230804T180000.zarr 2023-08-04 18:00:00 \n", - "\n", - " last_ping_time \n", - "0 2023-08-04 17:19:55+00:00 \n", - "1 2023-08-04 17:39:55+00:00 \n", - "2 2023-08-04 18:19:55 " - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df_prediction = pd.read_csv(\n", - " \"/home/ctuguinay/echodataflow_test/output/prediction_files.csv\",\n", - " index_col=0,\n", - " date_format=\"ISO8601\",\n", - " parse_dates=[\"first_ping_time\", \"last_ping_time\"]\n", - ")\n", - "df_prediction" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "9dee303d", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0 2023-08-04 16:59:55\n", - "1 2023-08-04 17:19:55\n", - "2 2023-08-04 17:39:55\n", - "3 2023-08-04 17:59:55\n", - "4 2023-08-04 18:19:55\n", - "5 2023-08-04 18:29:00\n", - "Name: last_ping_time, dtype: datetime64[us]" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df_MVBS[\"last_ping_time\"]" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "97668be4", - "metadata": {}, - "outputs": [], - "source": [ - "import xarray as xr" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "996b55c2", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
<xarray.DataArray 'ping_time' (ping_time: 109)> Size: 872B\n",
-       "array(['2023-08-04T18:20:00.000000000', '2023-08-04T18:20:05.000000000',\n",
-       "       '2023-08-04T18:20:10.000000000', '2023-08-04T18:20:15.000000000',\n",
-       "       '2023-08-04T18:20:20.000000000', '2023-08-04T18:20:25.000000000',\n",
-       "       '2023-08-04T18:20:30.000000000', '2023-08-04T18:20:35.000000000',\n",
-       "       '2023-08-04T18:20:40.000000000', '2023-08-04T18:20:45.000000000',\n",
-       "       '2023-08-04T18:20:50.000000000', '2023-08-04T18:20:55.000000000',\n",
-       "       '2023-08-04T18:21:00.000000000', '2023-08-04T18:21:05.000000000',\n",
-       "       '2023-08-04T18:21:10.000000000', '2023-08-04T18:21:15.000000000',\n",
-       "       '2023-08-04T18:21:20.000000000', '2023-08-04T18:21:25.000000000',\n",
-       "       '2023-08-04T18:21:30.000000000', '2023-08-04T18:21:35.000000000',\n",
-       "       '2023-08-04T18:21:40.000000000', '2023-08-04T18:21:45.000000000',\n",
-       "       '2023-08-04T18:21:50.000000000', '2023-08-04T18:21:55.000000000',\n",
-       "       '2023-08-04T18:22:00.000000000', '2023-08-04T18:22:05.000000000',\n",
-       "       '2023-08-04T18:22:10.000000000', '2023-08-04T18:22:15.000000000',\n",
-       "       '2023-08-04T18:22:20.000000000', '2023-08-04T18:22:25.000000000',\n",
-       "       '2023-08-04T18:22:30.000000000', '2023-08-04T18:22:35.000000000',\n",
-       "       '2023-08-04T18:22:40.000000000', '2023-08-04T18:22:45.000000000',\n",
-       "       '2023-08-04T18:22:50.000000000', '2023-08-04T18:22:55.000000000',\n",
-       "       '2023-08-04T18:23:00.000000000', '2023-08-04T18:23:05.000000000',\n",
-       "       '2023-08-04T18:23:10.000000000', '2023-08-04T18:23:15.000000000',\n",
-       "       '2023-08-04T18:23:20.000000000', '2023-08-04T18:23:25.000000000',\n",
-       "       '2023-08-04T18:23:30.000000000', '2023-08-04T18:23:35.000000000',\n",
-       "       '2023-08-04T18:23:40.000000000', '2023-08-04T18:23:45.000000000',\n",
-       "       '2023-08-04T18:23:50.000000000', '2023-08-04T18:23:55.000000000',\n",
-       "       '2023-08-04T18:24:00.000000000', '2023-08-04T18:24:05.000000000',\n",
-       "       '2023-08-04T18:24:10.000000000', '2023-08-04T18:24:15.000000000',\n",
-       "       '2023-08-04T18:24:20.000000000', '2023-08-04T18:24:25.000000000',\n",
-       "       '2023-08-04T18:24:30.000000000', '2023-08-04T18:24:35.000000000',\n",
-       "       '2023-08-04T18:24:40.000000000', '2023-08-04T18:24:45.000000000',\n",
-       "       '2023-08-04T18:24:50.000000000', '2023-08-04T18:24:55.000000000',\n",
-       "       '2023-08-04T18:25:00.000000000', '2023-08-04T18:25:05.000000000',\n",
-       "       '2023-08-04T18:25:10.000000000', '2023-08-04T18:25:15.000000000',\n",
-       "       '2023-08-04T18:25:20.000000000', '2023-08-04T18:25:25.000000000',\n",
-       "       '2023-08-04T18:25:30.000000000', '2023-08-04T18:25:35.000000000',\n",
-       "       '2023-08-04T18:25:40.000000000', '2023-08-04T18:25:45.000000000',\n",
-       "       '2023-08-04T18:25:50.000000000', '2023-08-04T18:25:55.000000000',\n",
-       "       '2023-08-04T18:26:00.000000000', '2023-08-04T18:26:05.000000000',\n",
-       "       '2023-08-04T18:26:10.000000000', '2023-08-04T18:26:15.000000000',\n",
-       "       '2023-08-04T18:26:20.000000000', '2023-08-04T18:26:25.000000000',\n",
-       "       '2023-08-04T18:26:30.000000000', '2023-08-04T18:26:35.000000000',\n",
-       "       '2023-08-04T18:26:40.000000000', '2023-08-04T18:26:45.000000000',\n",
-       "       '2023-08-04T18:26:50.000000000', '2023-08-04T18:26:55.000000000',\n",
-       "       '2023-08-04T18:27:00.000000000', '2023-08-04T18:27:05.000000000',\n",
-       "       '2023-08-04T18:27:10.000000000', '2023-08-04T18:27:15.000000000',\n",
-       "       '2023-08-04T18:27:20.000000000', '2023-08-04T18:27:25.000000000',\n",
-       "       '2023-08-04T18:27:30.000000000', '2023-08-04T18:27:35.000000000',\n",
-       "       '2023-08-04T18:27:40.000000000', '2023-08-04T18:27:45.000000000',\n",
-       "       '2023-08-04T18:27:50.000000000', '2023-08-04T18:27:55.000000000',\n",
-       "       '2023-08-04T18:28:00.000000000', '2023-08-04T18:28:05.000000000',\n",
-       "       '2023-08-04T18:28:10.000000000', '2023-08-04T18:28:15.000000000',\n",
-       "       '2023-08-04T18:28:20.000000000', '2023-08-04T18:28:25.000000000',\n",
-       "       '2023-08-04T18:28:30.000000000', '2023-08-04T18:28:35.000000000',\n",
-       "       '2023-08-04T18:28:40.000000000', '2023-08-04T18:28:45.000000000',\n",
-       "       '2023-08-04T18:28:50.000000000', '2023-08-04T18:28:55.000000000',\n",
-       "       '2023-08-04T18:29:00.000000000'], dtype='datetime64[ns]')\n",
-       "Coordinates:\n",
-       "  * ping_time  (ping_time) datetime64[ns] 872B 2023-08-04T18:20:00 ... 2023-0...\n",
-       "Attributes:\n",
-       "    long_name:      Ping time\n",
-       "    standard_name:  time\n",
-       "    axis:           T
" - ], - "text/plain": [ - " Size: 872B\n", - "array(['2023-08-04T18:20:00.000000000', '2023-08-04T18:20:05.000000000',\n", - " '2023-08-04T18:20:10.000000000', '2023-08-04T18:20:15.000000000',\n", - " '2023-08-04T18:20:20.000000000', '2023-08-04T18:20:25.000000000',\n", - " '2023-08-04T18:20:30.000000000', '2023-08-04T18:20:35.000000000',\n", - " '2023-08-04T18:20:40.000000000', '2023-08-04T18:20:45.000000000',\n", - " '2023-08-04T18:20:50.000000000', '2023-08-04T18:20:55.000000000',\n", - " '2023-08-04T18:21:00.000000000', '2023-08-04T18:21:05.000000000',\n", - " '2023-08-04T18:21:10.000000000', '2023-08-04T18:21:15.000000000',\n", - " '2023-08-04T18:21:20.000000000', '2023-08-04T18:21:25.000000000',\n", - " '2023-08-04T18:21:30.000000000', '2023-08-04T18:21:35.000000000',\n", - " '2023-08-04T18:21:40.000000000', '2023-08-04T18:21:45.000000000',\n", - " '2023-08-04T18:21:50.000000000', '2023-08-04T18:21:55.000000000',\n", - " '2023-08-04T18:22:00.000000000', '2023-08-04T18:22:05.000000000',\n", - " '2023-08-04T18:22:10.000000000', '2023-08-04T18:22:15.000000000',\n", - " '2023-08-04T18:22:20.000000000', '2023-08-04T18:22:25.000000000',\n", - " '2023-08-04T18:22:30.000000000', '2023-08-04T18:22:35.000000000',\n", - " '2023-08-04T18:22:40.000000000', '2023-08-04T18:22:45.000000000',\n", - " '2023-08-04T18:22:50.000000000', '2023-08-04T18:22:55.000000000',\n", - " '2023-08-04T18:23:00.000000000', '2023-08-04T18:23:05.000000000',\n", - " '2023-08-04T18:23:10.000000000', '2023-08-04T18:23:15.000000000',\n", - " '2023-08-04T18:23:20.000000000', '2023-08-04T18:23:25.000000000',\n", - " '2023-08-04T18:23:30.000000000', '2023-08-04T18:23:35.000000000',\n", - " '2023-08-04T18:23:40.000000000', '2023-08-04T18:23:45.000000000',\n", - " '2023-08-04T18:23:50.000000000', '2023-08-04T18:23:55.000000000',\n", - " '2023-08-04T18:24:00.000000000', '2023-08-04T18:24:05.000000000',\n", - " '2023-08-04T18:24:10.000000000', '2023-08-04T18:24:15.000000000',\n", - " '2023-08-04T18:24:20.000000000', '2023-08-04T18:24:25.000000000',\n", - " '2023-08-04T18:24:30.000000000', '2023-08-04T18:24:35.000000000',\n", - " '2023-08-04T18:24:40.000000000', '2023-08-04T18:24:45.000000000',\n", - " '2023-08-04T18:24:50.000000000', '2023-08-04T18:24:55.000000000',\n", - " '2023-08-04T18:25:00.000000000', '2023-08-04T18:25:05.000000000',\n", - " '2023-08-04T18:25:10.000000000', '2023-08-04T18:25:15.000000000',\n", - " '2023-08-04T18:25:20.000000000', '2023-08-04T18:25:25.000000000',\n", - " '2023-08-04T18:25:30.000000000', '2023-08-04T18:25:35.000000000',\n", - " '2023-08-04T18:25:40.000000000', '2023-08-04T18:25:45.000000000',\n", - " '2023-08-04T18:25:50.000000000', '2023-08-04T18:25:55.000000000',\n", - " '2023-08-04T18:26:00.000000000', '2023-08-04T18:26:05.000000000',\n", - " '2023-08-04T18:26:10.000000000', '2023-08-04T18:26:15.000000000',\n", - " '2023-08-04T18:26:20.000000000', '2023-08-04T18:26:25.000000000',\n", - " '2023-08-04T18:26:30.000000000', '2023-08-04T18:26:35.000000000',\n", - " '2023-08-04T18:26:40.000000000', '2023-08-04T18:26:45.000000000',\n", - " '2023-08-04T18:26:50.000000000', '2023-08-04T18:26:55.000000000',\n", - " '2023-08-04T18:27:00.000000000', '2023-08-04T18:27:05.000000000',\n", - " '2023-08-04T18:27:10.000000000', '2023-08-04T18:27:15.000000000',\n", - " '2023-08-04T18:27:20.000000000', '2023-08-04T18:27:25.000000000',\n", - " '2023-08-04T18:27:30.000000000', '2023-08-04T18:27:35.000000000',\n", - " '2023-08-04T18:27:40.000000000', '2023-08-04T18:27:45.000000000',\n", - " '2023-08-04T18:27:50.000000000', '2023-08-04T18:27:55.000000000',\n", - " '2023-08-04T18:28:00.000000000', '2023-08-04T18:28:05.000000000',\n", - " '2023-08-04T18:28:10.000000000', '2023-08-04T18:28:15.000000000',\n", - " '2023-08-04T18:28:20.000000000', '2023-08-04T18:28:25.000000000',\n", - " '2023-08-04T18:28:30.000000000', '2023-08-04T18:28:35.000000000',\n", - " '2023-08-04T18:28:40.000000000', '2023-08-04T18:28:45.000000000',\n", - " '2023-08-04T18:28:50.000000000', '2023-08-04T18:28:55.000000000',\n", - " '2023-08-04T18:29:00.000000000'], dtype='datetime64[ns]')\n", - "Coordinates:\n", - " * ping_time (ping_time) datetime64[ns] 872B 2023-08-04T18:20:00 ... 2023-0...\n", - "Attributes:\n", - " long_name: Ping time\n", - " standard_name: time\n", - " axis: T" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds_MVBS = xr.open_zarr(\"/home/ctuguinay/echodataflow_test/output_20260625/MVBS/MVBS_20230804T182000.zarr\",)\n", - "ds_MVBS[\"ping_time\"]" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "echodataflow", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.13" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} From 0c9358118a0765649f14c5e877089f647b402ead Mon Sep 17 00:00:00 2001 From: Wu-Jung Lee Date: Thu, 9 Jul 2026 11:39:55 -0700 Subject: [PATCH 04/10] drop s3 no check bucket --- src/echodataflow/flows/flows_helper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/echodataflow/flows/flows_helper.py b/src/echodataflow/flows/flows_helper.py index a49f114..a973be0 100644 --- a/src/echodataflow/flows/flows_helper.py +++ b/src/echodataflow/flows/flows_helper.py @@ -55,9 +55,9 @@ def flow_file_upload( # Potentially long running so using a context manager if max_age == -1: - command = f"rclone copy -v --no-traverse --s3-no-check-bucket {src_dir} {dest_dir} --exclude-from {str(exclude_path)}" + command = f"rclone copy -v --no-traverse {src_dir} {dest_dir} --exclude-from {str(exclude_path)}" else: - command = f"rclone copy -v --max-age {max_age}h --no-traverse --s3-no-check-bucket {src_dir} {dest_dir} --exclude-from {str(exclude_path)}" + command = f"rclone copy -v --max-age {max_age}h --no-traverse {src_dir} {dest_dir} --exclude-from {str(exclude_path)}" with ShellOperation(commands=[command], working_dir=src_dir) as file_upload_operation: # Trigger runs the process in the background From 19b0b1dff37f18de3dc64ada39b59b294d88d040 Mon Sep 17 00:00:00 2001 From: Wu-Jung Lee Date: Thu, 9 Jul 2026 11:41:13 -0700 Subject: [PATCH 05/10] add mount to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 9ee6a7f..29f9491 100644 --- a/.gitignore +++ b/.gitignore @@ -144,3 +144,4 @@ _archive prefect.toml temp_model src/echodataflow/services/prototype_misc/ +mount/ From 1214b71f2fb3831f6a534a9ac2eb6b233582bd60 Mon Sep 17 00:00:00 2001 From: Wu-Jung Lee Date: Thu, 9 Jul 2026 11:44:51 -0700 Subject: [PATCH 06/10] update readme to include note about background mac services --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index f1649a1..e1325ce 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,8 @@ Echodataflow streamlines echosounder data processing by combining [Prefect](http ## Running the edge pipeline +Note: Starting the server and running work pool is unnecessary if local Mac Prefect background services are running. + 1. Start the local Prefect server: ```shell prefect server start From 196242c11dac1d0a4711806163a674d580ad5895 Mon Sep 17 00:00:00 2001 From: Wu-Jung Lee Date: Fri, 10 Jul 2026 16:17:49 -0700 Subject: [PATCH 07/10] add segmentation inference instructions --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e1325ce..4171ba4 100644 --- a/README.md +++ b/README.md @@ -35,8 +35,14 @@ Echodataflow streamlines echosounder data processing by combining [Prefect](http git clone git+https://github.com/echostack-org/echodataflow.git # clone the repo pip install -e ".[test,lint,docs]" # install in editable mode with dev tools ``` - +3. Pip install segmentation inference package + ```bash + cd .. + git clone git+https://github.com/uw-echospace/segmentation_inference.git # clone the repo + cd segmentation inference + pip install -e . + ``` ## Running the edge pipeline From 7240a7d9b0d03b6ea90ba4dbdee87d1602bd36a2 Mon Sep 17 00:00:00 2001 From: Wu-Jung Lee Date: Fri, 10 Jul 2026 16:18:51 -0700 Subject: [PATCH 08/10] remove mount from gitignore --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 29f9491..9ee6a7f 100644 --- a/.gitignore +++ b/.gitignore @@ -144,4 +144,3 @@ _archive prefect.toml temp_model src/echodataflow/services/prototype_misc/ -mount/ From 0efcdadcb01d7a50e8b0737d88bd5a3241cb3a75 Mon Sep 17 00:00:00 2001 From: Wu-Jung Lee Date: Sat, 18 Jul 2026 10:49:46 -0700 Subject: [PATCH 09/10] add automount smb plist --- README.md | 11 ++- .../services/auto_mount.launchd.plist | 71 +++++++++++++++++++ .../services/services.env.example_local | 8 +++ 3 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 src/echodataflow/services/auto_mount.launchd.plist diff --git a/README.md b/README.md index 4171ba4..d1ddf03 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ Note: Starting the server and running work pool is unnecessary if local Mac Pref 6. Start up system services that hosts the 2 sets of visualization -## Running Local Prefect Services on macOS (launchd) +## Running Local Prefect and auto mounting services on macOS (launchd) To run a local Prefect server and worker as background services on macOS, you can use launchd with the provided plist templates: @@ -111,12 +111,15 @@ use launchd with the provided plist templates: These templates intentionally use direct one-line `ProgramArguments` commands, similar to `.service` `ExecStart` usage, with no wrapper shell script required. +Included is a template and subsequent commands for auto mounting an SMB volume. These can be omitted if the volume is stable. + 1. Copy and customize the templates for your user: ```shell mkdir -p ~/.config/echodataflow ~/Library/LaunchAgents ~/.local/var/log/echodataflow cp src/echodataflow/services/services.env.example_local ~/.config/echodataflow/services.env cp src/echodataflow/services/deploy_prefect_server.launchd.plist ~/Library/LaunchAgents/org.echodataflow.prefect-server.plist cp src/echodataflow/services/deploy_prefect_worker.launchd.plist ~/Library/LaunchAgents/org.echodataflow.prefect-worker.plist + cp src/echodataflow/services/auto_mount.launchd.plist ~/Library/LaunchAgents/org.echodataflow.auto-mount.plist ``` 2. Edit `~/.config/echodataflow/services.env` as needed: @@ -125,13 +128,16 @@ to `.service` `ExecStart` usage, with no wrapper shell script required. - Adjust `MAMBA_BIN` - Adjust `PREFECT_POOL` - Adjust `PREFECT_API_URL` + - Adjust SMB parameters as needed 3. Load and start services: ```shell launchctl bootstrap gui/$(id -u) ~/Library/LaunchAgents/org.echodataflow.prefect-server.plist launchctl bootstrap gui/$(id -u) ~/Library/LaunchAgents/org.echodataflow.prefect-worker.plist + launchctl bootstrap gui/$(id -u) ~/Library/LaunchAgents/org.echodataflow.auto-mount.plist launchctl kickstart -k gui/$(id -u)/org.echodataflow.prefect-server launchctl kickstart -k gui/$(id -u)/org.echodataflow.prefect-worker + launchctl kickstart -k gui/$(id -u)/org.echodataflow.auto-mount ``` 4. Check status and logs: @@ -139,15 +145,18 @@ to `.service` `ExecStart` usage, with no wrapper shell script required. # make sure "state = running" and "runs" not increasing launchctl print gui/$(id -u)/org.echodataflow.prefect-server launchctl print gui/$(id -u)/org.echodataflow.prefect-worker + launchctl print gui/$(id -u)/org.echodataflow.auto-mount # -f to follow logs in real time tail -n 100 ~/.local/var/log/echodataflow/prefect-server.err.log tail -n 100 ~/.local/var/log/echodataflow/prefect-worker.err.log + tail -n 100 ~/.local/var/log/echodataflow/auto-mount.err.log ``` 5. To stop and unload services: ```shell launchctl bootout gui/$(id -u) ~/Library/LaunchAgents/org.echodataflow.prefect-server.plist launchctl bootout gui/$(id -u) ~/Library/LaunchAgents/org.echodataflow.prefect-worker.plist + launchctl bootout gui/$(id -u) ~/Library/LaunchAgents/org.echodataflow.auto-mount.plist ``` 6. SQLite health checks (local Prefect server): diff --git a/src/echodataflow/services/auto_mount.launchd.plist b/src/echodataflow/services/auto_mount.launchd.plist new file mode 100644 index 0000000..b37b7c9 --- /dev/null +++ b/src/echodataflow/services/auto_mount.launchd.plist @@ -0,0 +1,71 @@ + + + + + Label + org.echodataflow.auto-mount + + RunAtLoad + + + KeepAlive + + + ThrottleInterval + 30 + + EnvironmentVariables + + PATH + /opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin + + + ProgramArguments + + /bin/zsh + -lc + +ENV_FILE="${ECHODATAFLOW_SERVICES_ENV_FILE:-$HOME/.config/echodataflow/services.env}" +[[ -f "$ENV_FILE" ]] || { echo "Missing $ENV_FILE"; exit 1; } + +set -a +source "$ENV_FILE" +set +a + +: "${SMB_USER:?SMB_USER is not set in services.env}" +: "${SMB_PASSWORD:?SMB_PASSWORD is not set in services.env}" +: "${SMB_HOST:?SMB_HOST is not set in services.env}" +: "${SMB_SHARE:?SMB_SHARE is not set in services.env}" +: "${SMB_MOUNT_POINT:?SMB_MOUNT_POINT is not set in services.env}" + +LOG_DIR="${ECHODATAFLOW_LOG_DIR:-$HOME/.local/log/echodataflow}" +mkdir -p "$LOG_DIR" + +exec >>"$LOG_DIR/auto-mount.out.log" 2>&1 "$LOG_DIR/auto-mount.err.log" +set -x + +echo "$(date): auto mount service started" + +mkdir -p "$SMB_MOUNT_POINT" + +while true; do + if ! /sbin/mount | grep -q "$SMB_MOUNT_POINT"; then + echo "$(date): mounting $SMB_MOUNT_POINT" + + /sbin/mount_smbfs \ + "//${SMB_USER}:${SMB_PASSWORD}@${SMB_HOST}/${SMB_SHARE}" \ + "$SMB_MOUNT_POINT" + + if [[ $? -eq 0 ]]; then + echo "$(date): mount successful" + else + echo "$(date): mount failed" + fi + fi + + sleep 300 +done + + + + \ No newline at end of file diff --git a/src/echodataflow/services/services.env.example_local b/src/echodataflow/services/services.env.example_local index e29bc10..7af8834 100644 --- a/src/echodataflow/services/services.env.example_local +++ b/src/echodataflow/services/services.env.example_local @@ -29,3 +29,11 @@ PREFECT_DB_PATH=$ECHODATAFLOW_HOME/.prefect/prefect.db # - false: fail fast if DB quick_check is not ok # - true: archive old DB files and start with a new DB PREFECT_DB_AUTORESET_ON_FAILURE=true + +# SMB connection parameters. Only needed for SMB connections that +# disconnect. +SMB_USER=TODO +SMB_PASSWORD=TODO +SMB_HOST=TODO +SMB_SHARE=data_import +SMB_MOUNT_POINT=/Volumes/data_import \ No newline at end of file From 281430fe198ec5eccb67eed68beaca8acb499753 Mon Sep 17 00:00:00 2001 From: Wu-Jung Lee Date: Sat, 18 Jul 2026 10:50:42 -0700 Subject: [PATCH 10/10] set smb params as default todo --- src/echodataflow/services/services.env.example_local | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/echodataflow/services/services.env.example_local b/src/echodataflow/services/services.env.example_local index 7af8834..decdd3f 100644 --- a/src/echodataflow/services/services.env.example_local +++ b/src/echodataflow/services/services.env.example_local @@ -35,5 +35,5 @@ PREFECT_DB_AUTORESET_ON_FAILURE=true SMB_USER=TODO SMB_PASSWORD=TODO SMB_HOST=TODO -SMB_SHARE=data_import -SMB_MOUNT_POINT=/Volumes/data_import \ No newline at end of file +SMB_SHARE=TODO +SMB_MOUNT_POINT=TODO \ No newline at end of file