From b906e8d3314f3db396fdc143d883a4eb1e7b8b7d Mon Sep 17 00:00:00 2001 From: Sebastian Rowan <79868066+sebastianrowan@users.noreply.github.com> Date: Mon, 16 Mar 2026 11:18:04 -0400 Subject: [PATCH 1/9] add csv multihazard hazardprovider --- actions/compute-payload.go | 11 ++ go.mod | 5 +- go.sum | 6 +- hazardproviders/csv_multihazard_provider.go | 106 ++++++++++++++++++++ notes.md | 9 ++ 5 files changed, 131 insertions(+), 6 deletions(-) create mode 100644 hazardproviders/csv_multihazard_provider.go create mode 100644 notes.md diff --git a/actions/compute-payload.go b/actions/compute-payload.go index e6f10c5..9f19ab5 100644 --- a/actions/compute-payload.go +++ b/actions/compute-payload.go @@ -54,12 +54,14 @@ const ( computeEventActionName string = "compute-event" computeFrequencyActionName string = "compute-frequency" computeCoastalEventActionName string = "compute-coastal-event" + computeCoastalLifecycleActionName string = "compute-coastal-lifecycle" ) func init() { cc.ActionRegistry.RegisterAction(computeEventActionName, &ComputeEventAction{}) cc.ActionRegistry.RegisterAction(computeFrequencyActionName, &ComputeFrequencyAction{}) cc.ActionRegistry.RegisterAction(computeCoastalEventActionName, &ComputeCoastalEventAction{}) + cc.ActionRegistry.RegisterAction(computeCoastalLifecycleActionName, &ComputeCoastalLifecycleAction{}) } type ComputeEventAction struct { @@ -71,6 +73,9 @@ type ComputeFrequencyAction struct { type ComputeCoastalEventAction struct { cc.ActionRunnerBase } +type ComputeCoastalLifecycleAction struct { + cc.ActionRunnerBase +} func (ar *ComputeEventAction) Run() error { a := ar.Action @@ -505,3 +510,9 @@ func ComputeEAD(damages []float64, freq []float64) float64 { } return eadT } + + +func (ar *ComputeCoastalLifecycle) Run() error { + + +} \ No newline at end of file diff --git a/go.mod b/go.mod index 8eeec48..a580d81 100644 --- a/go.mod +++ b/go.mod @@ -4,12 +4,14 @@ go 1.24 require ( github.com/HydrologicEngineeringCenter/go-statistics v0.0.0-20240126145250-a17483ae0981 - github.com/USACE/go-consequences v0.0.0-20250731200338-7eee6afc377f + github.com/USACE/go-consequences v0.0.0-20260303155249-41dfc39dd84d github.com/dewberry/gdal v0.3.4 github.com/google/uuid v1.6.0 // indirect github.com/usace-cloud-compute/cc-go-sdk v0.0.0-20251124210849-b455e063a7ea ) +require github.com/usace-cloud-compute/filesapi v0.0.0-20251107191432-8084e0da4b5c + require ( github.com/aws/aws-sdk-go-v2 v1.39.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.1 // indirect @@ -40,7 +42,6 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/shopspring/decimal v1.4.0 // indirect github.com/spf13/cast v1.10.0 // indirect - github.com/usace-cloud-compute/filesapi v0.0.0-20251107191432-8084e0da4b5c // indirect github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect gonum.org/v1/gonum v0.16.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index b25f0ef..5b3da85 100644 --- a/go.sum +++ b/go.sum @@ -3,8 +3,8 @@ gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zum github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/HydrologicEngineeringCenter/go-statistics v0.0.0-20240126145250-a17483ae0981 h1:JFLYdHvF3mPN56AkXeaWm41bQ9eJFuSceCtXXtMWTNI= github.com/HydrologicEngineeringCenter/go-statistics v0.0.0-20240126145250-a17483ae0981/go.mod h1:ABek1cDzTXjGA4uwrB91/NriEROj8AnUDaX7xU+1L2E= -github.com/USACE/go-consequences v0.0.0-20250731200338-7eee6afc377f h1:i4a5qJ/6ghF6VxJKCadp+Z6JHUzLgHwuoAXr7WEi2sQ= -github.com/USACE/go-consequences v0.0.0-20250731200338-7eee6afc377f/go.mod h1:+HMvu5PLsam+TS7GCpeiraeCWjwD04/hcgz85kuQTM4= +github.com/USACE/go-consequences v0.0.0-20260303155249-41dfc39dd84d h1:Fi60yreROXSA9kWDBdW8rvAJiDgNV4jj2VMZuDTc1eA= +github.com/USACE/go-consequences v0.0.0-20260303155249-41dfc39dd84d/go.mod h1:+HMvu5PLsam+TS7GCpeiraeCWjwD04/hcgz85kuQTM4= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/aws/aws-sdk-go-v2 v1.39.0 h1:xm5WV/2L4emMRmMjHFykqiA4M/ra0DJVSWUkDyBjbg4= github.com/aws/aws-sdk-go-v2 v1.39.0/go.mod h1:sDioUELIUO9Znk23YVmIk86/9DOpkbyyVb1i/gUNFXY= @@ -105,8 +105,6 @@ github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qq github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/usace-cloud-compute/cc-go-sdk v0.0.0-20251118163833-620acc79f197 h1:oM3e8JDPNyYjK3/aZHQkNDeWiE0D2/SPtFAr6haz3Hc= -github.com/usace-cloud-compute/cc-go-sdk v0.0.0-20251118163833-620acc79f197/go.mod h1:HZsGTrNUUdjfqqJJaLpSBEKoIDmu/vnye2KK/AwchDo= github.com/usace-cloud-compute/cc-go-sdk v0.0.0-20251124210849-b455e063a7ea h1:tlM91GCRJ1gqc4dcFvfI6h8MI2l5FNR3Ehn+Rh4oDJw= github.com/usace-cloud-compute/cc-go-sdk v0.0.0-20251124210849-b455e063a7ea/go.mod h1:HZsGTrNUUdjfqqJJaLpSBEKoIDmu/vnye2KK/AwchDo= github.com/usace-cloud-compute/filesapi v0.0.0-20251107191432-8084e0da4b5c h1:1Srw2WpT9tcEIUY853oeojXJjPH8iTYFGLXPaIrUe+I= diff --git a/hazardproviders/csv_multihazard_provider.go b/hazardproviders/csv_multihazard_provider.go new file mode 100644 index 0000000..ca6ca50 --- /dev/null +++ b/hazardproviders/csv_multihazard_provider.go @@ -0,0 +1,106 @@ +package hazardproviders + +import ( + "errors" + "fmt" + + "github.com/USACE/go-consequences/geography" + gc "github.com/USACE/go-consequences/hazardproviders" +) + +type csvArrivalDepthDurationMultiHazardProvider struct { + f *os.File + arrivals []time.Time + depths []float64 + durations []float64 + process HazardFunction + bbox geography.BBox +} + +func InitCSV(fp string, b geography.BBox) (csvArrivalDepthDurationMultiHazardProvider, error) { + fmt.Println("Connecting to: " + fp) + file, err := os.Open(fp) + if err != nil { + panic(err) + } + + reader := csv.NewReader(file) + + rows, err := reader.ReadAll() + if err != nil { + panic(err) + } + arrivals := make([]time.Time, len(rows)-1) + depths := make([]float64, len(rows)-1) + durations := make([]float64, len(rows)-1) + + for i, row := range rows[1:] { + year, err := strconv.Atoi(row[2]) + if err != nil { + panic(err) + } + month, err := strconv.Atoi(strings.TrimSpace(row[3])) + if err != nil { + panic(err) + } + day, err := strconv.Atoi(strings.TrimSpace(row[4])) + if err != nil { + panic(err) + } + at := time.Date(year, time.Month(month), int(day), 0, 0, 0, 0, time.UTC) + depth, err := strconv.ParseFloat(row[5], 64) + if err != nil { + panic(err) + } + dur, err := strconv.ParseFloat(row[6], 64) + if err != nil { + panic(err) + } + + arrivals[i] = at + depths[i] = depth + durations[i] = dur + } + + return csvArrivalDepthDurationMultiHazardProvider{ + f: file, + arrivals: arrivals, + depths: depths, + durations: durations, + process: gc.ArrivalDepthAndDurationHazardFunction(), + bbox: b, + }, nil + +} + +func (c csvArrivalDepthDurationMultiHazardProvider) Close() { + c.f.Close() +} + +func (c csvArrivalDepthDurationMultiHazardProvider) Hazard(l geography.Location) (hazards.HazardEvent, error) { + var hm hazards.ArrivalDepthandDurationEventMulti + + for i, d := range c.depths { + hd := hazards.HazardData{ + Depth: d, + Velocity: 0, + ArrivalTime: c.arrivals[i], + Erosion: 0, + Duration: c.durations[i], + WaveHeight: 0, + Salinity: false, + Qualitative: "", + } + var h hazards.HazardEvent + h, err := c.process(hd, h) + if err != nil { + panic(err) + } + hm.Append(h.(hazards.ArrivalDepthandDurationEvent)) + } + return &hm, nil +} + +func (c csvArrivalDepthDurationMultiHazardProvider) HazardBoundary() (geography.BBox, error) { + return c.bbox, nil +} \ No newline at end of file diff --git a/notes.md b/notes.md new file mode 100644 index 0000000..e0d2631 --- /dev/null +++ b/notes.md @@ -0,0 +1,9 @@ +# Create new action: compute_coastal_lifecycle + +1. update `go.mod` to use go-consequences version with lifecycle compute + +2. add action name "compute_coastal_lifecycle" to compute manifest const + +3. implement csv hazard provider + +4. implement results writer for lifecycle results \ No newline at end of file From c910a3966104ee185c70eeac904e16097105bc98 Mon Sep 17 00:00:00 2001 From: Sebastian Rowan <79868066+sebastianrowan@users.noreply.github.com> Date: Wed, 18 Mar 2026 16:29:49 -0400 Subject: [PATCH 2/9] add ComputeCoastalLifecycleAction Run() func --- actions/compute-payload.go | 108 +++++++++++++++++++++++++++++++++++++ notes.md | 9 ---- 2 files changed, 108 insertions(+), 9 deletions(-) delete mode 100644 notes.md diff --git a/actions/compute-payload.go b/actions/compute-payload.go index 9f19ab5..b00f388 100644 --- a/actions/compute-payload.go +++ b/actions/compute-payload.go @@ -35,6 +35,8 @@ const ( depthgridDatasourceName string = "depth-grid" //plugin datasource name required velocitygridDatasourceName string = "velocity-grid" //plugin datasource name required durationgridDatasourceName string = "duration-grid" //plugin datasource name required + multiHazardDatasourceName string = "multiHazardcsv" // + studyAreaBboxKey string = "bbox" outputDatasourceName string = "Damages" //plugin output datasource name required localData string = "/app/data" pluginName string = "consequences" @@ -386,6 +388,112 @@ func (ar *ComputeFrequencyAction) Run() error { ComputeMultiFrequency(hps, frequencies, abstractSP, rw) return nil } + +func (ar *ComputeCoastalLifecycleAction) Run() error { + a := ar.Action + + // get all relevant parameters + tablename := a.Attributes.GetStringOrFail(tablenameKey) + hazardsCSVPathString := a.Attributes.GetStringOrFail(multiHazardDatasourceName) + studyAreaBbox := a.Attributes.GetStringOrFail(studyAreaBboxKey) //TODO: this will need to be cast as a geography.BBox{} + inventoryPath := a.Attributes.GetStringOrFail(inventoryPathKey) //expected this is local - needs to agree with the payload input datasource name + inventoryDriver := a.Attributes.GetStringOrFail(inventoryDriverKey) + + outputDriver := a.Attributes.GetStringOrFail(outputDriverKey) + outputFileName := a.Attributes.GetStringOrFail(outputFileNameKey) //expected this is local - needs to agree with the payload output datasource name + //useKnowledgeUncertainty, err := strconv.ParseBool(a.Parameters.GetStringOrFail(useKnowledgeUncertaintyKey)) + damageFunctionPath := a.Attributes.GetStringOrFail(damageFunctionPathKey) //expected this is local - needs to agree with the payload input datasource name + + b := geography>BBox{ + BBox: studyAreaBbox + } + + hp, err := InitCSV(hazardsCSVPathString, b) + if err != nil { + panic(err) + } + defer hp.Close() + + var abstractSP consequences.StreamProvider + var sr string + if inventoryDriver == "MILLIMAN" { + sp, err := structureproviders.InitMillimanStructureProviderwithOcctypePath(inventoryPath, damageFunctionPath) + sp.SetDeterministic(true) + if err != nil { + return err + } + fmt.Sprintln(sp.FilePath) + sr = sp.SpatialReference() + abstractSP = sp + } else { + sp, err := structureprovider.InitStructureProviderwithOcctypePath(inventoryPath, tablename, inventoryDriver, damageFunctionPath) + sp.SetDeterministic(true) + if err != nil { + return err + } + fmt.Sprintln(sp.FilePath) + sr = sp.SpatialReference() + abstractSP = sp + } + + //initalize a results writer + var rw consequences.ResultsWriter + if outputDriver == "PostgreSQL" { + pgUser := os.Getenv(pgUserKey) + pgPass := os.Getenv(pgPasswordKey) + pgDB := os.Getenv(pgDbnameKey) + pgHost := os.Getenv(pgHostKey) + pgPort := os.Getenv(pgPortKey) + pgSchema := os.Getenv(pgSchemaKey) + + outConnStr := fmt.Sprintf( + "PG:dbname=%s user=%s password=%s host=%s port=%s schemas=%s", + pgDB, pgUser, pgPass, pgHost, pgPort, pgSchema, + ) + + rw, err = lrw.InitSpatialResultsWriter_PSQL(outConnStr, outputLayerName, outputDriver, pgDB) + if err != nil { + log.Fatalf("Failed to initialize spatial psql result writer: %s\n", err) + } + } else { + outfp := fmt.Sprintf("%s/%s", localData, outputFileName) + + rw, err = resultswriters.InitSpatialResultsWriter_WKT_Projected(outfp, outputLayerName, outputDriver, sr) + if err != nil { + log.Fatalf("Failed to initialize spatial result writer: %s\n", err) + } + } + defer rw.Close() + + //compute results + //get boundingbox + fmt.Println("Getting bbox") + bbox, err := hp.HazardBoundary() + if err != nil { + log.Panicf("Unable to get the raster bounding box: %s", err) + } + fmt.Println(bbox.ToString()) + abstractSP.ByBbox(bbox, func(f consequences.Receptor) { + //ProvideHazard works off of a geography.Location + d, err2 := hp.Hazard(geography.Location{X: f.Location().X, Y: f.Location().Y}) + //compute damages based on hazard being able to provide depth + if err2 == nil { + r, err3 := f.Compute(d) + r.Headers = append(r.Headers, "multihazard") + bytes, err := json.Marshal(d) + s := "" + if err == nil { + s = string(bytes) + } + r.Result = append(r.Result, s) + if err3 == nil { + rw.Write(r) + } + } + }) + return nil + +} func ComputeMultiFrequency(hps []hazardproviders.HazardProvider, freqs []float64, sp consequences.StreamProvider, w consequences.ResultsWriter) { fmt.Printf("Computing %v frequencies\n", len(freqs)) //ASSUMPTION hazard providers and frequencies are in the same order diff --git a/notes.md b/notes.md deleted file mode 100644 index e0d2631..0000000 --- a/notes.md +++ /dev/null @@ -1,9 +0,0 @@ -# Create new action: compute_coastal_lifecycle - -1. update `go.mod` to use go-consequences version with lifecycle compute - -2. add action name "compute_coastal_lifecycle" to compute manifest const - -3. implement csv hazard provider - -4. implement results writer for lifecycle results \ No newline at end of file From 445d9d6d1f7b78ec687074471c60b14868774e06 Mon Sep 17 00:00:00 2001 From: Sebastian Rowan <79868066+sebastianrowan@users.noreply.github.com> Date: Fri, 20 Mar 2026 11:29:42 -0400 Subject: [PATCH 3/9] Add new hazardprovider that takes events, depth_timeseries, and reach boundary files from stormsim --- actions/compute-payload.go | 69 +++--- hazardproviders/csv_multihazard_provider.go | 13 +- .../stormSim_lifecycle_hazardprovider.go | 209 ++++++++++++++++++ 3 files changed, 249 insertions(+), 42 deletions(-) create mode 100644 hazardproviders/stormSim_lifecycle_hazardprovider.go diff --git a/actions/compute-payload.go b/actions/compute-payload.go index b00f388..20092e3 100644 --- a/actions/compute-payload.go +++ b/actions/compute-payload.go @@ -18,6 +18,7 @@ import ( "github.com/USACE/go-consequences/structures" "github.com/usace-cloud-compute/cc-go-sdk" lrw "github.com/usace-cloud-compute/consequences-runner/crresultswriters" + lhp "github.com/usace-cloud-compute/consequences-runner/hazardproviders" "github.com/usace-cloud-compute/consequences-runner/structureproviders" ) @@ -32,30 +33,31 @@ const ( outputLayerName string = "damages" structureInventoryPathKey string = "Inventory" //plugin datasource name required //seedsDatasourceName string = "seeds.json" //plugin datasource name required - depthgridDatasourceName string = "depth-grid" //plugin datasource name required - velocitygridDatasourceName string = "velocity-grid" //plugin datasource name required - durationgridDatasourceName string = "duration-grid" //plugin datasource name required - multiHazardDatasourceName string = "multiHazardcsv" // - studyAreaBboxKey string = "bbox" - outputDatasourceName string = "Damages" //plugin output datasource name required - localData string = "/app/data" - pluginName string = "consequences" - DepthGridPathsKey string = "depth-grids" // expected to contain the fully qualified vsis3 path set comma separated or the local path if the resource is included as an inputdatasource - VelocityGridPathsKey string = "velocity-grids" // expected to contain the fully qualified vsis3 path set comma separated or the local path if the resource is included as an inputdatasource - FrequenciesKey string = "frequencies" //expected to be comma separated string - inventoryPathKey string = "Inventory" //expected this is local - needs to agree with the payload input datasource name - damageFunctionPathKey string = "damage-functions" //expected this is local - needs to agree with the payload input datasource name - projectIdKey string = "project-id" - runIdKey string = "run-id" - pgUserKey string = "PG_USER" - pgPasswordKey string = "PG_PASSWORD" - pgDbnameKey string = "PG_DBNAME" - pgHostKey string = "PG_HOST" - pgPortKey string = "PG_PORT" - pgSchemaKey string = "PG_SCHEMA" - computeEventActionName string = "compute-event" - computeFrequencyActionName string = "compute-frequency" - computeCoastalEventActionName string = "compute-coastal-event" + depthgridDatasourceName string = "depth-grid" //plugin datasource name required + velocitygridDatasourceName string = "velocity-grid" //plugin datasource name required + durationgridDatasourceName string = "duration-grid" //plugin datasource name required + stormSimEventsPath string = "ss-events" // + stormSimResponsesPath string = "ss-responses" + stormSimReachesPath string = "ss-reaches" + outputDatasourceName string = "Damages" //plugin output datasource name required + localData string = "/app/data" + pluginName string = "consequences" + DepthGridPathsKey string = "depth-grids" // expected to contain the fully qualified vsis3 path set comma separated or the local path if the resource is included as an inputdatasource + VelocityGridPathsKey string = "velocity-grids" // expected to contain the fully qualified vsis3 path set comma separated or the local path if the resource is included as an inputdatasource + FrequenciesKey string = "frequencies" //expected to be comma separated string + inventoryPathKey string = "Inventory" //expected this is local - needs to agree with the payload input datasource name + damageFunctionPathKey string = "damage-functions" //expected this is local - needs to agree with the payload input datasource name + projectIdKey string = "project-id" + runIdKey string = "run-id" + pgUserKey string = "PG_USER" + pgPasswordKey string = "PG_PASSWORD" + pgDbnameKey string = "PG_DBNAME" + pgHostKey string = "PG_HOST" + pgPortKey string = "PG_PORT" + pgSchemaKey string = "PG_SCHEMA" + computeEventActionName string = "compute-event" + computeFrequencyActionName string = "compute-frequency" + computeCoastalEventActionName string = "compute-coastal-event" computeCoastalLifecycleActionName string = "compute-coastal-lifecycle" ) @@ -391,11 +393,12 @@ func (ar *ComputeFrequencyAction) Run() error { func (ar *ComputeCoastalLifecycleAction) Run() error { a := ar.Action - // get all relevant parameters tablename := a.Attributes.GetStringOrFail(tablenameKey) - hazardsCSVPathString := a.Attributes.GetStringOrFail(multiHazardDatasourceName) - studyAreaBbox := a.Attributes.GetStringOrFail(studyAreaBboxKey) //TODO: this will need to be cast as a geography.BBox{} + stormSimEventsPathString := a.Attributes.GetStringOrFail(stormSimEventsPath) + stormSimResponsesPathString := a.Attributes.GetStringOrFail(stormSimResponsesPath) + stormSimReachesPathString := a.Attributes.GetStringOrFail(stormSimReachesPath) + inventoryPath := a.Attributes.GetStringOrFail(inventoryPathKey) //expected this is local - needs to agree with the payload input datasource name inventoryDriver := a.Attributes.GetStringOrFail(inventoryDriverKey) @@ -404,11 +407,7 @@ func (ar *ComputeCoastalLifecycleAction) Run() error { //useKnowledgeUncertainty, err := strconv.ParseBool(a.Parameters.GetStringOrFail(useKnowledgeUncertaintyKey)) damageFunctionPath := a.Attributes.GetStringOrFail(damageFunctionPathKey) //expected this is local - needs to agree with the payload input datasource name - b := geography>BBox{ - BBox: studyAreaBbox - } - - hp, err := InitCSV(hazardsCSVPathString, b) + hp, err := lhp.InitStormSim(stormSimEventsPathString, stormSimResponsesPathString, stormSimReachesPathString) if err != nil { panic(err) } @@ -618,9 +617,3 @@ func ComputeEAD(damages []float64, freq []float64) float64 { } return eadT } - - -func (ar *ComputeCoastalLifecycle) Run() error { - - -} \ No newline at end of file diff --git a/hazardproviders/csv_multihazard_provider.go b/hazardproviders/csv_multihazard_provider.go index ca6ca50..d325183 100644 --- a/hazardproviders/csv_multihazard_provider.go +++ b/hazardproviders/csv_multihazard_provider.go @@ -1,11 +1,16 @@ package hazardproviders import ( - "errors" + "encoding/csv" "fmt" + "os" + "strconv" + "strings" + "time" "github.com/USACE/go-consequences/geography" gc "github.com/USACE/go-consequences/hazardproviders" + "github.com/USACE/go-consequences/hazards" ) type csvArrivalDepthDurationMultiHazardProvider struct { @@ -13,12 +18,12 @@ type csvArrivalDepthDurationMultiHazardProvider struct { arrivals []time.Time depths []float64 durations []float64 - process HazardFunction + process gc.HazardFunction bbox geography.BBox } func InitCSV(fp string, b geography.BBox) (csvArrivalDepthDurationMultiHazardProvider, error) { - fmt.Println("Connecting to: " + fp) + fmt.Println("Connecting to: " + fp) // should we confirm file is csv? file, err := os.Open(fp) if err != nil { panic(err) @@ -103,4 +108,4 @@ func (c csvArrivalDepthDurationMultiHazardProvider) Hazard(l geography.Location) func (c csvArrivalDepthDurationMultiHazardProvider) HazardBoundary() (geography.BBox, error) { return c.bbox, nil -} \ No newline at end of file +} diff --git a/hazardproviders/stormSim_lifecycle_hazardprovider.go b/hazardproviders/stormSim_lifecycle_hazardprovider.go new file mode 100644 index 0000000..96869bf --- /dev/null +++ b/hazardproviders/stormSim_lifecycle_hazardprovider.go @@ -0,0 +1,209 @@ +package hazardproviders + +import ( + "encoding/csv" + "os" + "strconv" + "time" + + "github.com/USACE/go-consequences/geography" + gc "github.com/USACE/go-consequences/hazardproviders" + "github.com/USACE/go-consequences/hazards" +) + +type stormsimLifecycleMultiHazardProvider struct { + eventsFile *os.File + stormResponsesFile *os.File // TODO: better name + reachesFile *os.File + // specify drivers for each? + arrivals []time.Time + depths []float64 + durations []float64 + process gc.HazardFunction + bbox geography.BBox +} + +func InitStormSim(eventsFP string, responsesFP string, reachesFP string) (stormsimLifecycleMultiHazardProvider, error) { + + // parse eventsFile + // not needed. all info in eventsFile is also in stormResponsesFile + + //TODO: check file type. For now assume csv + eventsFile, err := os.Open(eventsFP) + if err != nil { + panic(err) + } + eventReader := csv.NewReader(eventsFile) + rows, err := eventReader.ReadAll() + if err != nil { + panic(err) + } + storm_ids := []string{} // will use the length of this slice to pre-allocate slice sizes for parsed storms + for _, row := range rows[1:] { + lifecycle, err := strconv.Atoi(row[1]) + if err != nil { + panic(err) + } + storm_id := row[8] + if lifecycle == 0 { // allow user to specify lifecycle? + // compute for multiHazard currently doesn't support multiple lifecycles + storm_ids = append(storm_ids, storm_id) + } + } + + // parse stormResponsesFile + stormResponsesFile, err := os.Open(responsesFP) + if err != nil { + panic(err) + } + arrivals, depths, durations := parseStormsFile(stormResponsesFile, len(storm_ids)) + + // get bbox from reachesFile + //TODO: read geometry from gpkg + // using csv for now + reachesFile, err := os.Open(reachesFP) + if err != nil { + panic(err) + } + reachesReader := csv.NewReader(reachesFile) + rows2, err := reachesReader.ReadAll() + if err != nil { + panic(err) + } + coords := rows2[1] + xmin, err := strconv.ParseFloat(coords[1], 64) + if err != nil { + panic(err) + } + ymin, err := strconv.ParseFloat(coords[2], 64) + if err != nil { + panic(err) + } + xmax, err := strconv.ParseFloat(coords[3], 64) + if err != nil { + panic(err) + } + ymax, err := strconv.ParseFloat(coords[4], 64) + if err != nil { + panic(err) + } + b := geography.BBox{ + Bbox: []float64{xmin, ymin, xmax, ymax}, + } + + return stormsimLifecycleMultiHazardProvider{ + eventsFile: eventsFile, + stormResponsesFile: stormResponsesFile, + reachesFile: reachesFile, + arrivals: arrivals, + depths: depths, + durations: durations, + process: gc.ArrivalDepthAndDurationHazardFunction(), + bbox: b, + }, nil +} + +func parseStormsFile(file *os.File, n int) ([]time.Time, []float64, []float64) { + + reader := csv.NewReader(file) + rows, err := reader.ReadAll() + if err != nil { + panic(err) + } + + arrivals := make([]time.Time, n) + depths := make([]float64, n) + durations := make([]float64, n) + + var curStormIndex int = 0 + var curStormID string + var curStormLifecycle int + var curStormStart time.Time + var curStormEnd time.Time + // var curStormPeakTime time.Time // not sure if this is worth tracking + var curStormPeakStage float64 + for i, row := range rows[1:] { + date_i, err := time.Parse("2006-01-02 15:04:05", row[1]) + if err != nil { + panic(err) + } + storm_id := row[2] + lifecycle_i, err := strconv.Atoi(row[3]) + if err != nil { + panic(err) + } + stage_i, err := strconv.ParseFloat(row[4], 64) + if err != nil { + panic(err) + } + + if (storm_id == curStormID) && (lifecycle_i == curStormLifecycle) { + curStormEnd = date_i + if stage_i > curStormPeakStage { + curStormPeakStage = stage_i + // curStormPeakTime = date_i + } + } else { + // we've reached a new storm in the series + // save results from previous storm + if i > 0 { // can't save previous storm on row 0 + if curStormLifecycle == 0 { // not handling multiple lifecycles currently + arrivals[curStormIndex] = curStormStart + depths[curStormIndex] = curStormPeakStage + duration := curStormEnd.Sub(curStormStart) + durations[curStormIndex] = duration.Hours() * 24.0 + curStormIndex++ + } + } + curStormID = storm_id + curStormLifecycle = lifecycle_i + curStormStart = date_i + curStormEnd = date_i + curStormPeakStage = stage_i + // curStormPeakTime = date_i + } + } + // need to add final storm details to slices + if curStormLifecycle == 0 { + arrivals[curStormIndex] = curStormStart + depths[curStormIndex] = curStormPeakStage + duration := curStormEnd.Sub(curStormStart) + durations[curStormIndex] = duration.Hours() * 24.0 + + } + return arrivals, depths, durations +} + +func (c stormsimLifecycleMultiHazardProvider) Close() { + c.eventsFile.Close() + c.stormResponsesFile.Close() + c.reachesFile.Close() +} + +func (c stormsimLifecycleMultiHazardProvider) Hazard(l geography.Location) (hazards.HazardEvent, error) { + var hm hazards.ArrivalDepthandDurationEventMulti + + for i, d := range c.depths { + hd := hazards.HazardData{ + Depth: d, + Velocity: 0, + ArrivalTime: c.arrivals[i], + Erosion: 0, + Duration: c.durations[i], + WaveHeight: 0, + Salinity: false, + Qualitative: "", + } + var h hazards.HazardEvent + h, err := c.process(hd, h) + if err != nil { + panic(err) + } + hm.Append(h.(hazards.ArrivalDepthandDurationEvent)) + } + return &hm, nil +} + +func (c stormsimLifecycleMultiHazardProvider) HazardBoundary() (geography.BBox, error) { + return c.bbox, nil +} From 866ac4e4ea22f8779acc37433dc6a1c619dcee7d Mon Sep 17 00:00:00 2001 From: Sebastian Rowan <79868066+sebastianrowan@users.noreply.github.com> Date: Wed, 1 Apr 2026 14:34:30 -0400 Subject: [PATCH 4/9] use gdal to process responses file next step: use gdal to handle events and reaches files --- .env-coastal-lifecycle | 3 + .vscode/launch.json | 12 ++ actions/compute-payload.go | 9 +- .../stormSim_lifecycle_hazardprovider.go | 164 +++++++++++------- utils_test.go | 66 +++++++ 5 files changed, 190 insertions(+), 64 deletions(-) create mode 100644 .env-coastal-lifecycle diff --git a/.env-coastal-lifecycle b/.env-coastal-lifecycle new file mode 100644 index 0000000..9b9a033 --- /dev/null +++ b/.env-coastal-lifecycle @@ -0,0 +1,3 @@ +CC_STORE_TYPE=FS +FSB_ROOT_PATH="/workspaces/consequences-runner/data/" +CC_PAYLOAD_ID=coastal \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index 00a4c25..ac1afdf 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -4,6 +4,7 @@ // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ + { "name": "Launch Package", "type": "go", @@ -43,6 +44,17 @@ "Test_Compute" ], "envFile": "/workspaces/.env-milliman" + },{ + "name": "Test Coastal Lifecycle", + "type": "go", + "request": "launch", + "mode": "test", + "program": "${workspaceFolder}", + "args": [ + "-test.run", + "Test_ComputeCoastalEventAction" + ], + "envFile": "/workspaces/consequences-runner/.env-coastal-lifecycle" },{ "name": "Download", "type": "go", diff --git a/actions/compute-payload.go b/actions/compute-payload.go index 20092e3..5409fc7 100644 --- a/actions/compute-payload.go +++ b/actions/compute-payload.go @@ -37,8 +37,11 @@ const ( velocitygridDatasourceName string = "velocity-grid" //plugin datasource name required durationgridDatasourceName string = "duration-grid" //plugin datasource name required stormSimEventsPath string = "ss-events" // + stormSimEventsDriver string = "ss-events-driver" stormSimResponsesPath string = "ss-responses" + stormSimResponsesDriver string = "ss-responses-driver" stormSimReachesPath string = "ss-reaches" + stormSimReachesDriver string = "ss-reaches-driver" outputDatasourceName string = "Damages" //plugin output datasource name required localData string = "/app/data" pluginName string = "consequences" @@ -399,6 +402,10 @@ func (ar *ComputeCoastalLifecycleAction) Run() error { stormSimResponsesPathString := a.Attributes.GetStringOrFail(stormSimResponsesPath) stormSimReachesPathString := a.Attributes.GetStringOrFail(stormSimReachesPath) + ssEventsDriverString := a.Attributes.GetStringOrFail(stormSimEventsDriver) + ssResponsesDriverString := a.Attributes.GetStringOrFail(stormSimResponsesDriver) + ssReachesDriverString := a.Attributes.GetStringOrFail(stormSimReachesDriver) + inventoryPath := a.Attributes.GetStringOrFail(inventoryPathKey) //expected this is local - needs to agree with the payload input datasource name inventoryDriver := a.Attributes.GetStringOrFail(inventoryDriverKey) @@ -407,7 +414,7 @@ func (ar *ComputeCoastalLifecycleAction) Run() error { //useKnowledgeUncertainty, err := strconv.ParseBool(a.Parameters.GetStringOrFail(useKnowledgeUncertaintyKey)) damageFunctionPath := a.Attributes.GetStringOrFail(damageFunctionPathKey) //expected this is local - needs to agree with the payload input datasource name - hp, err := lhp.InitStormSim(stormSimEventsPathString, stormSimResponsesPathString, stormSimReachesPathString) + hp, err := lhp.InitStormSim(stormSimEventsPathString, ssEventsDriverString, stormSimResponsesPathString, ssResponsesDriverString, stormSimReachesPathString, ssReachesDriverString) if err != nil { panic(err) } diff --git a/hazardproviders/stormSim_lifecycle_hazardprovider.go b/hazardproviders/stormSim_lifecycle_hazardprovider.go index 96869bf..c587cd7 100644 --- a/hazardproviders/stormSim_lifecycle_hazardprovider.go +++ b/hazardproviders/stormSim_lifecycle_hazardprovider.go @@ -2,6 +2,7 @@ package hazardproviders import ( "encoding/csv" + "errors" "os" "strconv" "time" @@ -9,12 +10,10 @@ import ( "github.com/USACE/go-consequences/geography" gc "github.com/USACE/go-consequences/hazardproviders" "github.com/USACE/go-consequences/hazards" + "github.com/dewberry/gdal" ) type stormsimLifecycleMultiHazardProvider struct { - eventsFile *os.File - stormResponsesFile *os.File // TODO: better name - reachesFile *os.File // specify drivers for each? arrivals []time.Time depths []float64 @@ -23,7 +22,7 @@ type stormsimLifecycleMultiHazardProvider struct { bbox geography.BBox } -func InitStormSim(eventsFP string, responsesFP string, reachesFP string) (stormsimLifecycleMultiHazardProvider, error) { +func InitStormSim(eventsFP string, eventsDriver string, responsesFP string, responsesDriver string, reachesFP string, reachesDriver string) (stormsimLifecycleMultiHazardProvider, error) { // parse eventsFile // not needed. all info in eventsFile is also in stormResponsesFile @@ -52,11 +51,7 @@ func InitStormSim(eventsFP string, responsesFP string, reachesFP string) (storms } // parse stormResponsesFile - stormResponsesFile, err := os.Open(responsesFP) - if err != nil { - panic(err) - } - arrivals, depths, durations := parseStormsFile(stormResponsesFile, len(storm_ids)) + arrivals, depths, durations, err := ParseResponsesFile(responsesFP, "", eventsDriver, len(storm_ids)) // get bbox from reachesFile //TODO: read geometry from gpkg @@ -92,75 +87,120 @@ func InitStormSim(eventsFP string, responsesFP string, reachesFP string) (storms } return stormsimLifecycleMultiHazardProvider{ - eventsFile: eventsFile, - stormResponsesFile: stormResponsesFile, - reachesFile: reachesFile, - arrivals: arrivals, - depths: depths, - durations: durations, - process: gc.ArrivalDepthAndDurationHazardFunction(), - bbox: b, + arrivals: arrivals, + depths: depths, + durations: durations, + process: gc.ArrivalDepthAndDurationHazardFunction(), + bbox: b, }, nil } -func parseStormsFile(file *os.File, n int) ([]time.Time, []float64, []float64) { +// type gdalDataSet struct { +// FilePath string +// LayerName string +// schemaIDX []int +// ds *gdal.DataSource +// } - reader := csv.NewReader(file) - rows, err := reader.ReadAll() - if err != nil { - panic(err) - } +func eventsSchema() []string { + s := []string{"location_id", "lifecycle", "year_offset", "year", "month", "day", "hour", "timestamp", "storm_id"} + return s +} + +func responsesSchema() []string { + s := []string{"location_id", "date", "storm_id", "lifecycle", "stage"} + return s +} +func reachesSchema() []string { + s := []string{"Name", "NameAbbreviated", "Note", "NOAAStation", "NOAAStationName", "SavePoint", "geom"} + return s +} + +func ParseResponsesFile(filepath string, layername string, driver string, n int) ([]time.Time, []float64, []float64, error) { arrivals := make([]time.Time, n) depths := make([]float64, n) durations := make([]float64, n) + //TODO: combine arrivals, depths, durations into an ADD struct to reduce number of return values + + d := gdal.OGRDriverByName(driver) + // for i := 0; i < gdal.OGRDriverCount(); i++ { // can use this to ensure given driver is available + // fmt.Println(gdal.OGRDriverByIndex(i).Name()) + // } + + ds, dsok := d.Open(filepath, int(gdal.ReadOnly)) + if !dsok { + return arrivals, depths, durations, errors.New("error opening responses file of type " + driver) + } + layer := ds.LayerByIndex(0) + def := layer.Definition() + + s := responsesSchema() + sIDX := make([]int, len(s)) + + for i, f := range s { + idx := def.FieldIndex(f) + if idx < 0 { + return arrivals, depths, durations, errors.New("gdal dataset at path " + filepath + " Expected field named " + f + " none was found") + } + sIDX[i] = idx + } + + // fmt.Println(layer.Name()) + // fmt.Println(layer.Definition().FieldCount()) + + // for j := 0; j < layer.Definition().FieldCount(); j++ { + // fmt.Println(layer.Definition().FieldDefinition(j).Name()) + // } + + fc, _ := layer.FeatureCount(true) + // fmt.Printf("The layer has %v features\n", fc) var curStormIndex int = 0 var curStormID string var curStormLifecycle int var curStormStart time.Time var curStormEnd time.Time - // var curStormPeakTime time.Time // not sure if this is worth tracking var curStormPeakStage float64 - for i, row := range rows[1:] { - date_i, err := time.Parse("2006-01-02 15:04:05", row[1]) - if err != nil { - panic(err) - } - storm_id := row[2] - lifecycle_i, err := strconv.Atoi(row[3]) - if err != nil { - panic(err) - } - stage_i, err := strconv.ParseFloat(row[4], 64) - if err != nil { - panic(err) - } - if (storm_id == curStormID) && (lifecycle_i == curStormLifecycle) { - curStormEnd = date_i - if stage_i > curStormPeakStage { - curStormPeakStage = stage_i - // curStormPeakTime = date_i + for i := range fc { + feat := layer.NextFeature() + if feat != nil { + // "location_id", "date", "storm_id", "lifecycle", "stage" + d := feat.FieldAsString(sIDX[1]) // also returns a bool. Does true mean ok? + date_i, err := time.Parse("2006-01-02 15:04:05", d) + if err != nil { + panic(err) } - } else { - // we've reached a new storm in the series - // save results from previous storm - if i > 0 { // can't save previous storm on row 0 - if curStormLifecycle == 0 { // not handling multiple lifecycles currently - arrivals[curStormIndex] = curStormStart - depths[curStormIndex] = curStormPeakStage - duration := curStormEnd.Sub(curStormStart) - durations[curStormIndex] = duration.Hours() * 24.0 - curStormIndex++ + storm_id := feat.FieldAsString(sIDX[2]) + lifecycle_i := feat.FieldAsInteger(sIDX[3]) + stage_i := feat.FieldAsFloat64(sIDX[4]) // how to handle possible NA values here? + + if (storm_id == curStormID) && (lifecycle_i == curStormLifecycle) { + curStormEnd = date_i + if stage_i > curStormPeakStage { + curStormPeakStage = stage_i + // curStormPeakTime = date_i + } + } else { + // we've reached a new storm in the series + // save results from previous storm + if i > 0 { // can't save previous storm on row 0 + if curStormLifecycle == 0 { // not handling multiple lifecycles currently + arrivals[curStormIndex] = curStormStart + depths[curStormIndex] = curStormPeakStage + duration := curStormEnd.Sub(curStormStart) + durations[curStormIndex] = duration.Hours() / 24.0 + curStormIndex++ + } } + curStormID = storm_id + curStormLifecycle = lifecycle_i + curStormStart = date_i + curStormEnd = date_i + curStormPeakStage = stage_i + // curStormPeakTime = date_i } - curStormID = storm_id - curStormLifecycle = lifecycle_i - curStormStart = date_i - curStormEnd = date_i - curStormPeakStage = stage_i - // curStormPeakTime = date_i } } // need to add final storm details to slices @@ -171,13 +211,11 @@ func parseStormsFile(file *os.File, n int) ([]time.Time, []float64, []float64) { durations[curStormIndex] = duration.Hours() * 24.0 } - return arrivals, depths, durations + return arrivals, depths, durations, nil } func (c stormsimLifecycleMultiHazardProvider) Close() { - c.eventsFile.Close() - c.stormResponsesFile.Close() - c.reachesFile.Close() + // what goes here after switching to using gdal? } func (c stormsimLifecycleMultiHazardProvider) Hazard(l geography.Location) (hazards.HazardEvent, error) { diff --git a/utils_test.go b/utils_test.go index e6dfb42..42ba583 100644 --- a/utils_test.go +++ b/utils_test.go @@ -7,6 +7,9 @@ import ( "os/exec" "strings" "testing" + "time" + + "github.com/usace-cloud-compute/consequences-runner/hazardproviders" ) /* @@ -137,3 +140,66 @@ func Test_Format(t *testing.T) { } } + +func Test_ParseStormsFile(t *testing.T) { + responsesFP := "/workspaces/consequences-runner/data/coastal/stage_EventDate_LC_responses.csv" + + arrivals_expected := []time.Time{ + time.Date(2033, time.Month(8), 22, 14, 7, 0, 0, time.UTC), + time.Date(2034, time.Month(8), 19, 19, 31, 39, 0, time.UTC), + time.Date(2034, time.Month(8), 28, 8, 07, 19, 0, time.UTC), + time.Date(2034, time.Month(9), 16, 8, 3, 47, 0, time.UTC), + time.Date(2034, time.Month(10), 12, 7, 19, 13, 0, time.UTC), + time.Date(2035, time.Month(7), 17, 4, 16, 10, 0, time.UTC), + time.Date(2035, time.Month(10), 20, 9, 19, 48, 0, time.UTC), + time.Date(2036, time.Month(9), 17, 6, 47, 54, 0, time.UTC), + time.Date(2037, time.Month(10), 01, 18, 45, 0, 0, time.UTC), + time.Date(2037, time.Month(10), 24, 6, 15, 12, 0, time.UTC), + time.Date(2040, time.Month(6), 13, 14, 13, 24, 0, time.UTC), + time.Date(2040, time.Month(8), 19, 23, 56, 21, 0, time.UTC), + time.Date(2040, time.Month(8), 30, 6, 53, 14, 0, time.UTC), + time.Date(2040, time.Month(9), 26, 7, 11, 16, 0, time.UTC), + time.Date(2041, time.Month(10), 5, 0, 7, 32, 0, time.UTC), + time.Date(2041, time.Month(11), 15, 6, 8, 48, 0, time.UTC), + time.Date(2042, time.Month(9), 6, 23, 29, 45, 0, time.UTC), + } + depths_expected := []float64{ + 13.0, 1.19023548877452, 0.00189844601579208, 1.0171892210224, + 1.1684668225599, 2.67564711780564, 1.11368483591891, 5.95024604558782e-06, + 0.00610899031783363, 0.0051434691700571, 1.29154912885772, 1.10496133027909, + 0.00478200367925463, 1.09444238480238, 1.01478090469665, 13.0, 0.012841852964191, + } + durations_expected := []float64{ + 2.0, 2.0, 2.0, 2.0, 4.0, 2.0, + 4.0, 2.0, 2.0, 4.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 1.0, 2.0, + } + arrivals, depths, durations, err := hazardproviders.ParseResponsesFile(responsesFP, "using-layer_id=0", "CSV", len(arrivals_expected)) + if err != nil { + panic(err) + } + + for i, arrival := range arrivals { + if arrival != arrivals_expected[i] { + t.Errorf("Fail: Expected %v, got %v", arrivals_expected[i], arrival) + } else { + fmt.Println("Arrival Correct") + } + if depths[i] != depths_expected[i] { + t.Errorf("Fail: Expected %v, got %v", depths_expected[i], depths[i]) + } else { + fmt.Println("Depth correct") + } + if durations[i] != durations_expected[i] { + t.Errorf("Fail: Expected %v, got %v", durations_expected[i], durations[i]) + } else { + fmt.Println("Duration Correct") + } + } + +} + +func Test_ComputeCoastalEventAction(t *testing.T) { + + main() +} From 9e07b8d56a2ba07d241db5091ee4b645b62f3b44 Mon Sep 17 00:00:00 2001 From: Sebastian Rowan <79868066+sebastianrowan@users.noreply.github.com> Date: Wed, 1 Apr 2026 14:52:22 -0400 Subject: [PATCH 5/9] move tests --- .../stormSim_lifecycle_hazardprovider.go | 5 +- .../stormSim_lifecycle_hazardprovider_test.go | 65 +++++++++++++++++++ utils_test.go | 61 ----------------- 3 files changed, 67 insertions(+), 64 deletions(-) create mode 100644 hazardproviders/stormSim_lifecycle_hazardprovider_test.go diff --git a/hazardproviders/stormSim_lifecycle_hazardprovider.go b/hazardproviders/stormSim_lifecycle_hazardprovider.go index c587cd7..6eb366a 100644 --- a/hazardproviders/stormSim_lifecycle_hazardprovider.go +++ b/hazardproviders/stormSim_lifecycle_hazardprovider.go @@ -50,8 +50,7 @@ func InitStormSim(eventsFP string, eventsDriver string, responsesFP string, resp } } - // parse stormResponsesFile - arrivals, depths, durations, err := ParseResponsesFile(responsesFP, "", eventsDriver, len(storm_ids)) + arrivals, depths, durations, err := parseResponsesFile(responsesFP, "", eventsDriver, len(storm_ids)) // get bbox from reachesFile //TODO: read geometry from gpkg @@ -117,7 +116,7 @@ func reachesSchema() []string { return s } -func ParseResponsesFile(filepath string, layername string, driver string, n int) ([]time.Time, []float64, []float64, error) { +func parseResponsesFile(filepath string, layername string, driver string, n int) ([]time.Time, []float64, []float64, error) { arrivals := make([]time.Time, n) depths := make([]float64, n) durations := make([]float64, n) diff --git a/hazardproviders/stormSim_lifecycle_hazardprovider_test.go b/hazardproviders/stormSim_lifecycle_hazardprovider_test.go new file mode 100644 index 0000000..ec19884 --- /dev/null +++ b/hazardproviders/stormSim_lifecycle_hazardprovider_test.go @@ -0,0 +1,65 @@ +package hazardproviders + +import ( + "fmt" + "testing" + "time" +) + +func Test_ParseStormsFile(t *testing.T) { + responsesFP := "/workspaces/consequences-runner/data/coastal/stage_EventDate_LC_responses.csv" + + arrivals_expected := []time.Time{ + time.Date(2033, time.Month(8), 22, 14, 7, 0, 0, time.UTC), + time.Date(2034, time.Month(8), 19, 19, 31, 39, 0, time.UTC), + time.Date(2034, time.Month(8), 28, 8, 07, 19, 0, time.UTC), + time.Date(2034, time.Month(9), 16, 8, 3, 47, 0, time.UTC), + time.Date(2034, time.Month(10), 12, 7, 19, 13, 0, time.UTC), + time.Date(2035, time.Month(7), 17, 4, 16, 10, 0, time.UTC), + time.Date(2035, time.Month(10), 20, 9, 19, 48, 0, time.UTC), + time.Date(2036, time.Month(9), 17, 6, 47, 54, 0, time.UTC), + time.Date(2037, time.Month(10), 01, 18, 45, 0, 0, time.UTC), + time.Date(2037, time.Month(10), 24, 6, 15, 12, 0, time.UTC), + time.Date(2040, time.Month(6), 13, 14, 13, 24, 0, time.UTC), + time.Date(2040, time.Month(8), 19, 23, 56, 21, 0, time.UTC), + time.Date(2040, time.Month(8), 30, 6, 53, 14, 0, time.UTC), + time.Date(2040, time.Month(9), 26, 7, 11, 16, 0, time.UTC), + time.Date(2041, time.Month(10), 5, 0, 7, 32, 0, time.UTC), + time.Date(2041, time.Month(11), 15, 6, 8, 48, 0, time.UTC), + time.Date(2042, time.Month(9), 6, 23, 29, 45, 0, time.UTC), + } + depths_expected := []float64{ + 13.0, 1.19023548877452, 0.00189844601579208, 1.0171892210224, + 1.1684668225599, 2.67564711780564, 1.11368483591891, 5.95024604558782e-06, + 0.00610899031783363, 0.0051434691700571, 1.29154912885772, 1.10496133027909, + 0.00478200367925463, 1.09444238480238, 1.01478090469665, 13.0, 0.012841852964191, + } + durations_expected := []float64{ + 2.0, 2.0, 2.0, 2.0, 4.0, 2.0, + 4.0, 2.0, 2.0, 4.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 1.0, 2.0, + } + arrivals, depths, durations, err := parseResponsesFile(responsesFP, "using-layer_id=0", "CSV", len(arrivals_expected)) + if err != nil { + panic(err) + } + + for i, arrival := range arrivals { + if arrival != arrivals_expected[i] { + t.Errorf("Fail: Expected %v, got %v", arrivals_expected[i], arrival) + } else { + fmt.Println("Arrival Correct") + } + if depths[i] != depths_expected[i] { + t.Errorf("Fail: Expected %v, got %v", depths_expected[i], depths[i]) + } else { + fmt.Println("Depth correct") + } + if durations[i] != durations_expected[i] { + t.Errorf("Fail: Expected %v, got %v", durations_expected[i], durations[i]) + } else { + fmt.Println("Duration Correct") + } + } + +} diff --git a/utils_test.go b/utils_test.go index 42ba583..fd9110d 100644 --- a/utils_test.go +++ b/utils_test.go @@ -7,9 +7,6 @@ import ( "os/exec" "strings" "testing" - "time" - - "github.com/usace-cloud-compute/consequences-runner/hazardproviders" ) /* @@ -141,64 +138,6 @@ func Test_Format(t *testing.T) { } -func Test_ParseStormsFile(t *testing.T) { - responsesFP := "/workspaces/consequences-runner/data/coastal/stage_EventDate_LC_responses.csv" - - arrivals_expected := []time.Time{ - time.Date(2033, time.Month(8), 22, 14, 7, 0, 0, time.UTC), - time.Date(2034, time.Month(8), 19, 19, 31, 39, 0, time.UTC), - time.Date(2034, time.Month(8), 28, 8, 07, 19, 0, time.UTC), - time.Date(2034, time.Month(9), 16, 8, 3, 47, 0, time.UTC), - time.Date(2034, time.Month(10), 12, 7, 19, 13, 0, time.UTC), - time.Date(2035, time.Month(7), 17, 4, 16, 10, 0, time.UTC), - time.Date(2035, time.Month(10), 20, 9, 19, 48, 0, time.UTC), - time.Date(2036, time.Month(9), 17, 6, 47, 54, 0, time.UTC), - time.Date(2037, time.Month(10), 01, 18, 45, 0, 0, time.UTC), - time.Date(2037, time.Month(10), 24, 6, 15, 12, 0, time.UTC), - time.Date(2040, time.Month(6), 13, 14, 13, 24, 0, time.UTC), - time.Date(2040, time.Month(8), 19, 23, 56, 21, 0, time.UTC), - time.Date(2040, time.Month(8), 30, 6, 53, 14, 0, time.UTC), - time.Date(2040, time.Month(9), 26, 7, 11, 16, 0, time.UTC), - time.Date(2041, time.Month(10), 5, 0, 7, 32, 0, time.UTC), - time.Date(2041, time.Month(11), 15, 6, 8, 48, 0, time.UTC), - time.Date(2042, time.Month(9), 6, 23, 29, 45, 0, time.UTC), - } - depths_expected := []float64{ - 13.0, 1.19023548877452, 0.00189844601579208, 1.0171892210224, - 1.1684668225599, 2.67564711780564, 1.11368483591891, 5.95024604558782e-06, - 0.00610899031783363, 0.0051434691700571, 1.29154912885772, 1.10496133027909, - 0.00478200367925463, 1.09444238480238, 1.01478090469665, 13.0, 0.012841852964191, - } - durations_expected := []float64{ - 2.0, 2.0, 2.0, 2.0, 4.0, 2.0, - 4.0, 2.0, 2.0, 4.0, 2.0, 2.0, - 2.0, 2.0, 2.0, 1.0, 2.0, - } - arrivals, depths, durations, err := hazardproviders.ParseResponsesFile(responsesFP, "using-layer_id=0", "CSV", len(arrivals_expected)) - if err != nil { - panic(err) - } - - for i, arrival := range arrivals { - if arrival != arrivals_expected[i] { - t.Errorf("Fail: Expected %v, got %v", arrivals_expected[i], arrival) - } else { - fmt.Println("Arrival Correct") - } - if depths[i] != depths_expected[i] { - t.Errorf("Fail: Expected %v, got %v", depths_expected[i], depths[i]) - } else { - fmt.Println("Depth correct") - } - if durations[i] != durations_expected[i] { - t.Errorf("Fail: Expected %v, got %v", durations_expected[i], durations[i]) - } else { - fmt.Println("Duration Correct") - } - } - -} - func Test_ComputeCoastalEventAction(t *testing.T) { main() From c194046c58614d1d69e719270a465fb8f004d769 Mon Sep 17 00:00:00 2001 From: Sebastian Rowan <79868066+sebastianrowan@users.noreply.github.com> Date: Wed, 8 Apr 2026 12:50:29 -0400 Subject: [PATCH 6/9] update file parsers --- actions/compute-payload.go | 29 +- .../stormSim_lifecycle_hazardprovider.go | 282 +++++++++++------- .../stormSim_lifecycle_hazardprovider_test.go | 86 ++++-- 3 files changed, 262 insertions(+), 135 deletions(-) diff --git a/actions/compute-payload.go b/actions/compute-payload.go index 5409fc7..d28dc4b 100644 --- a/actions/compute-payload.go +++ b/actions/compute-payload.go @@ -38,10 +38,14 @@ const ( durationgridDatasourceName string = "duration-grid" //plugin datasource name required stormSimEventsPath string = "ss-events" // stormSimEventsDriver string = "ss-events-driver" + stormSimEventsLayer string = "ss-events-layer" stormSimResponsesPath string = "ss-responses" stormSimResponsesDriver string = "ss-responses-driver" + stormSimResponsesLayer string = "ss-responses-layer" stormSimReachesPath string = "ss-reaches" stormSimReachesDriver string = "ss-reaches-driver" + stormSimReachesLayer string = "ss-reaches-layer" + stormSimLifecycle string = "ss-lifecycle" outputDatasourceName string = "Damages" //plugin output datasource name required localData string = "/app/data" pluginName string = "consequences" @@ -406,6 +410,16 @@ func (ar *ComputeCoastalLifecycleAction) Run() error { ssResponsesDriverString := a.Attributes.GetStringOrFail(stormSimResponsesDriver) ssReachesDriverString := a.Attributes.GetStringOrFail(stormSimReachesDriver) + ssEventsLayerString := a.Attributes.GetStringOrDefault(stormSimEventsLayer, "") + ssResponsesLayerString := a.Attributes.GetStringOrDefault(stormSimResponsesLayer, "") + ssReachesLayerString := a.Attributes.GetStringOrDefault(stormSimReachesLayer, "") + + ssLC := a.Attributes.GetStringOrFail(stormSimLifecycle) + ssLifecycle, err := strconv.Atoi(ssLC) + if err != nil { + panic(err) + } + inventoryPath := a.Attributes.GetStringOrFail(inventoryPathKey) //expected this is local - needs to agree with the payload input datasource name inventoryDriver := a.Attributes.GetStringOrFail(inventoryDriverKey) @@ -414,7 +428,20 @@ func (ar *ComputeCoastalLifecycleAction) Run() error { //useKnowledgeUncertainty, err := strconv.ParseBool(a.Parameters.GetStringOrFail(useKnowledgeUncertaintyKey)) damageFunctionPath := a.Attributes.GetStringOrFail(damageFunctionPathKey) //expected this is local - needs to agree with the payload input datasource name - hp, err := lhp.InitStormSim(stormSimEventsPathString, ssEventsDriverString, stormSimResponsesPathString, ssResponsesDriverString, stormSimReachesPathString, ssReachesDriverString) + ssi := lhp.StormSimInfo{ + EventsFP: stormSimEventsPathString, + EventsDriver: ssEventsDriverString, + EventsLayername: ssEventsLayerString, + ResponsesFP: stormSimResponsesPathString, + ResponsesDriver: ssResponsesDriverString, + ResponsesLayername: ssResponsesLayerString, + ReachesFP: stormSimReachesPathString, + ReachesDriver: ssReachesDriverString, + ReachesLayername: ssReachesLayerString, + Lifecycle: ssLifecycle, + } + + hp, err := lhp.InitStormSim(ssi) if err != nil { panic(err) } diff --git a/hazardproviders/stormSim_lifecycle_hazardprovider.go b/hazardproviders/stormSim_lifecycle_hazardprovider.go index 6eb366a..2f337df 100644 --- a/hazardproviders/stormSim_lifecycle_hazardprovider.go +++ b/hazardproviders/stormSim_lifecycle_hazardprovider.go @@ -1,10 +1,7 @@ package hazardproviders import ( - "encoding/csv" "errors" - "os" - "strconv" "time" "github.com/USACE/go-consequences/geography" @@ -20,94 +17,56 @@ type stormsimLifecycleMultiHazardProvider struct { durations []float64 process gc.HazardFunction bbox geography.BBox + geom gdal.Geometry } -func InitStormSim(eventsFP string, eventsDriver string, responsesFP string, responsesDriver string, reachesFP string, reachesDriver string) (stormsimLifecycleMultiHazardProvider, error) { +// create struct to hold the arguments and - // parse eventsFile - // not needed. all info in eventsFile is also in stormResponsesFile +type StormSimInfo struct { + EventsFP string + EventsDriver string + EventsLayername string + ResponsesFP string + ResponsesDriver string + ResponsesLayername string + ReachesFP string + ReachesDriver string + ReachesLayername string + Lifecycle int +} - //TODO: check file type. For now assume csv - eventsFile, err := os.Open(eventsFP) - if err != nil { - panic(err) - } - eventReader := csv.NewReader(eventsFile) - rows, err := eventReader.ReadAll() - if err != nil { - panic(err) - } - storm_ids := []string{} // will use the length of this slice to pre-allocate slice sizes for parsed storms - for _, row := range rows[1:] { - lifecycle, err := strconv.Atoi(row[1]) - if err != nil { - panic(err) - } - storm_id := row[8] - if lifecycle == 0 { // allow user to specify lifecycle? - // compute for multiHazard currently doesn't support multiple lifecycles - storm_ids = append(storm_ids, storm_id) - } - } +func InitStormSim(ssi StormSimInfo) (stormsimLifecycleMultiHazardProvider, error) { - arrivals, depths, durations, err := parseResponsesFile(responsesFP, "", eventsDriver, len(storm_ids)) + reaches, err := parseReachesFile(ssi.ReachesFP, ssi.ReachesLayername, ssi.ReachesDriver) - // get bbox from reachesFile - //TODO: read geometry from gpkg - // using csv for now - reachesFile, err := os.Open(reachesFP) - if err != nil { - panic(err) - } - reachesReader := csv.NewReader(reachesFile) - rows2, err := reachesReader.ReadAll() - if err != nil { - panic(err) - } - coords := rows2[1] - xmin, err := strconv.ParseFloat(coords[1], 64) - if err != nil { - panic(err) - } - ymin, err := strconv.ParseFloat(coords[2], 64) + events, err := parseEventsFile(ssi.EventsFP, ssi.EventsLayername, ssi.EventsDriver) if err != nil { panic(err) } - xmax, err := strconv.ParseFloat(coords[3], 64) - if err != nil { - panic(err) - } - ymax, err := strconv.ParseFloat(coords[4], 64) - if err != nil { - panic(err) - } - b := geography.BBox{ - Bbox: []float64{xmin, ymin, xmax, ymax}, - } + + //TODO: how to handle case where there are multiple reaches? + reach := reaches[0] + reachevents := events[reach.reachID] + + add, err := parseResponsesFile(ssi.ResponsesFP, ssi.ResponsesLayername, ssi.ResponsesDriver, len(reachevents), ssi.Lifecycle) return stormsimLifecycleMultiHazardProvider{ - arrivals: arrivals, - depths: depths, - durations: durations, + arrivals: add.arrivals, + depths: add.depths, + durations: add.durations, process: gc.ArrivalDepthAndDurationHazardFunction(), - bbox: b, + bbox: reach.Bbox, + geom: reach.Geom, }, nil } -// type gdalDataSet struct { -// FilePath string -// LayerName string -// schemaIDX []int -// ds *gdal.DataSource -// } - func eventsSchema() []string { s := []string{"location_id", "lifecycle", "year_offset", "year", "month", "day", "hour", "timestamp", "storm_id"} return s } func responsesSchema() []string { - s := []string{"location_id", "date", "storm_id", "lifecycle", "stage"} + s := []string{"location_id", "date", "stormevent_id", "storm_id", "lifecycle", "stage"} return s } @@ -116,11 +75,129 @@ func reachesSchema() []string { return s } -func parseResponsesFile(filepath string, layername string, driver string, n int) ([]time.Time, []float64, []float64, error) { +type Reach struct { + reachID string + reachName string + Bbox geography.BBox + Geom gdal.Geometry +} + +func parseReachesFile(filepath string, layername string, driver string) ([]Reach, error) { + var ret = []Reach{} + + d := gdal.OGRDriverByName(driver) + ds, dsok := d.Open(filepath, int(gdal.ReadOnly)) + if !dsok { + return ret, errors.New("error opening reaches file of type " + driver) + } + + var layer gdal.Layer + if layername != "" { + layer = ds.LayerByName(layername) + } else { + layer = ds.LayerByIndex(0) + } + def := layer.Definition() + s := reachesSchema() + sIDX := make([]int, len(s)) + + for i, f := range s { + idx := def.FieldIndex(f) + if idx < 0 { + return ret, errors.New("gdal dataset at path " + filepath + " Expected field named " + f + " none was found") + } + sIDX[i] = idx + } + + fc, _ := layer.FeatureCount(true) + ret = make([]Reach, fc) + + for i := range fc { + feat := layer.NextFeature() + if feat != nil { + //NOTE: "Name", "NameAbbreviated", "Note", "NOAAStation", "NOAAStationName", "SavePoint", "geom" + reach_id := feat.FieldAsString(sIDX[1]) + reach_name := feat.FieldAsString(sIDX[0]) + geom := feat.Geometry() + env := geom.Envelope() + xmin := env.MinX() + xmax := env.MaxX() + ymin := env.MinY() + ymax := env.MaxY() + bb := geography.BBox{Bbox: []float64{xmin, ymin, xmax, ymax}} + ret[i] = Reach{reachID: reach_id, reachName: reach_name, Geom: geom, Bbox: bb} + } + } + + return ret, nil +} + +func parseEventsFile(filepath string, layername string, driver string) (map[string]map[int][]string, error) { + ret := make(map[string]map[int][]string) // map[location_id]map[lifecycle][]storm_ids + + d := gdal.OGRDriverByName(driver) + ds, dsok := d.Open(filepath, int(gdal.ReadOnly)) + if !dsok { + return ret, errors.New("error opening events file of type " + driver) + } + + var layer gdal.Layer + if layername != "" { + layer = ds.LayerByName(layername) + } else { + layer = ds.LayerByIndex(0) + } + def := layer.Definition() + s := eventsSchema() + sIDX := make([]int, len(s)) + + for i, f := range s { + idx := def.FieldIndex(f) + if idx < 0 { + return ret, errors.New("gdal dataset at path " + filepath + " Expected field named " + f + " none was found") + } + sIDX[i] = idx + } + + fc, _ := layer.FeatureCount(true) + + for range fc { + feat := layer.NextFeature() + if feat != nil { + //NOTE: "location_id", "lifecycle", "year_offset", "year", "month", "day", "hour", "timestamp", "storm_id" + loc_id := feat.FieldAsString(sIDX[0]) + lifecycle := feat.FieldAsInteger(sIDX[1]) + storm_id := feat.FieldAsString(sIDX[8]) + + if ret[loc_id] == nil { + ret[loc_id] = make(map[int][]string) + } + if ret[loc_id][lifecycle] == nil { + ret[loc_id][lifecycle] = []string{storm_id} + } else { + ret[loc_id][lifecycle] = append(ret[loc_id][lifecycle], storm_id) + } + } + } + + return ret, nil +} + +type ADDInfo struct { + arrivals []time.Time + depths []float64 + durations []float64 +} + +func parseResponsesFile(filepath string, layername string, driver string, n int, lifecycle int) (ADDInfo, error) { arrivals := make([]time.Time, n) depths := make([]float64, n) durations := make([]float64, n) - //TODO: combine arrivals, depths, durations into an ADD struct to reduce number of return values + ret := ADDInfo{ + arrivals: arrivals, + depths: depths, + durations: durations, + } d := gdal.OGRDriverByName(driver) // for i := 0; i < gdal.OGRDriverCount(); i++ { // can use this to ensure given driver is available @@ -129,10 +206,15 @@ func parseResponsesFile(filepath string, layername string, driver string, n int) ds, dsok := d.Open(filepath, int(gdal.ReadOnly)) if !dsok { - return arrivals, depths, durations, errors.New("error opening responses file of type " + driver) + return ret, errors.New("error opening responses file of type " + driver) } - layer := ds.LayerByIndex(0) + var layer gdal.Layer + if layername != "" { + layer = ds.LayerByName(layername) + } else { + layer = ds.LayerByIndex(0) + } def := layer.Definition() s := responsesSchema() @@ -141,23 +223,16 @@ func parseResponsesFile(filepath string, layername string, driver string, n int) for i, f := range s { idx := def.FieldIndex(f) if idx < 0 { - return arrivals, depths, durations, errors.New("gdal dataset at path " + filepath + " Expected field named " + f + " none was found") + return ret, errors.New("gdal dataset at path " + filepath + " Expected field named " + f + " none was found") } sIDX[i] = idx } - // fmt.Println(layer.Name()) - // fmt.Println(layer.Definition().FieldCount()) - - // for j := 0; j < layer.Definition().FieldCount(); j++ { - // fmt.Println(layer.Definition().FieldDefinition(j).Name()) - // } - fc, _ := layer.FeatureCount(true) // fmt.Printf("The layer has %v features\n", fc) var curStormIndex int = 0 var curStormID string - var curStormLifecycle int + var curStormLifecycle int // var curStormStart time.Time var curStormEnd time.Time var curStormPeakStage float64 @@ -165,17 +240,17 @@ func parseResponsesFile(filepath string, layername string, driver string, n int) for i := range fc { feat := layer.NextFeature() if feat != nil { - // "location_id", "date", "storm_id", "lifecycle", "stage" - d := feat.FieldAsString(sIDX[1]) // also returns a bool. Does true mean ok? + d := feat.FieldAsString(sIDX[1]) date_i, err := time.Parse("2006-01-02 15:04:05", d) if err != nil { panic(err) } storm_id := feat.FieldAsString(sIDX[2]) - lifecycle_i := feat.FieldAsInteger(sIDX[3]) - stage_i := feat.FieldAsFloat64(sIDX[4]) // how to handle possible NA values here? + lifecycle_i := feat.FieldAsInteger(sIDX[4]) + //TODO: handle NA values for stage + stage_i := feat.FieldAsFloat64(sIDX[5]) - if (storm_id == curStormID) && (lifecycle_i == curStormLifecycle) { + if storm_id == curStormID { curStormEnd = date_i if stage_i > curStormPeakStage { curStormPeakStage = stage_i @@ -184,15 +259,16 @@ func parseResponsesFile(filepath string, layername string, driver string, n int) } else { // we've reached a new storm in the series // save results from previous storm - if i > 0 { // can't save previous storm on row 0 - if curStormLifecycle == 0 { // not handling multiple lifecycles currently - arrivals[curStormIndex] = curStormStart - depths[curStormIndex] = curStormPeakStage + if i > 0 { // can't save previous storm if we're on row 0 + if curStormLifecycle == lifecycle { // not handling multiple lifecycles currently + ret.arrivals[curStormIndex] = curStormStart + ret.depths[curStormIndex] = curStormPeakStage duration := curStormEnd.Sub(curStormStart) - durations[curStormIndex] = duration.Hours() / 24.0 + ret.durations[curStormIndex] = duration.Hours() / 24.0 curStormIndex++ } } + // reset all tracking variables to current row curStormID = storm_id curStormLifecycle = lifecycle_i curStormStart = date_i @@ -202,15 +278,11 @@ func parseResponsesFile(filepath string, layername string, driver string, n int) } } } - // need to add final storm details to slices - if curStormLifecycle == 0 { - arrivals[curStormIndex] = curStormStart - depths[curStormIndex] = curStormPeakStage - duration := curStormEnd.Sub(curStormStart) - durations[curStormIndex] = duration.Hours() * 24.0 - - } - return arrivals, depths, durations, nil + ret.arrivals[curStormIndex] = curStormStart + ret.depths[curStormIndex] = curStormPeakStage + duration := curStormEnd.Sub(curStormStart) + ret.durations[curStormIndex] = duration.Hours() / 24.0 + return ret, nil } func (c stormsimLifecycleMultiHazardProvider) Close() { @@ -238,6 +310,12 @@ func (c stormsimLifecycleMultiHazardProvider) Hazard(l geography.Location) (haza } hm.Append(h.(hazards.ArrivalDepthandDurationEvent)) } + test_geom := gdal.Create(gdal.GT_Point) + test_geom.AddPoint(l.X, l.Y, 0) + + if !c.geom.Contains(test_geom) { + return &hm, errors.New("Provided hazard location is outside the reach boundary") + } return &hm, nil } diff --git a/hazardproviders/stormSim_lifecycle_hazardprovider_test.go b/hazardproviders/stormSim_lifecycle_hazardprovider_test.go index ec19884..90b8367 100644 --- a/hazardproviders/stormSim_lifecycle_hazardprovider_test.go +++ b/hazardproviders/stormSim_lifecycle_hazardprovider_test.go @@ -6,57 +6,79 @@ import ( "time" ) -func Test_ParseStormsFile(t *testing.T) { - responsesFP := "/workspaces/consequences-runner/data/coastal/stage_EventDate_LC_responses.csv" +func Test_ParseReachesFile(t *testing.T) {} + +func Test_ParseEventsFile(t *testing.T) { + eventsFP := "/workspaces/consequences-runner/data/coastal/EventDate_LC.csv" + + events, err := parseEventsFile(eventsFP, "", "CSV") + if err != nil { + panic(err) + } + + expected := []string{"190", "485", "833", "990", "575", "664", "176", "609", "984", "294", "610", "562", "112", "268", "161", "859", "335"} + got := events["DE001"][0] + + for i, val := range got { + e := expected[i] + if val != e { + t.Errorf("Fail: Expected %v, got %v", e, val) + } + } +} +func Test_ParseResponsesFile(t *testing.T) { + responsesFP := "/workspaces/consequences-runner/data/coastal/stage_responses_loc_DE001_lc_0.csv" arrivals_expected := []time.Time{ - time.Date(2033, time.Month(8), 22, 14, 7, 0, 0, time.UTC), - time.Date(2034, time.Month(8), 19, 19, 31, 39, 0, time.UTC), - time.Date(2034, time.Month(8), 28, 8, 07, 19, 0, time.UTC), - time.Date(2034, time.Month(9), 16, 8, 3, 47, 0, time.UTC), - time.Date(2034, time.Month(10), 12, 7, 19, 13, 0, time.UTC), - time.Date(2035, time.Month(7), 17, 4, 16, 10, 0, time.UTC), - time.Date(2035, time.Month(10), 20, 9, 19, 48, 0, time.UTC), - time.Date(2036, time.Month(9), 17, 6, 47, 54, 0, time.UTC), - time.Date(2037, time.Month(10), 01, 18, 45, 0, 0, time.UTC), - time.Date(2037, time.Month(10), 24, 6, 15, 12, 0, time.UTC), - time.Date(2040, time.Month(6), 13, 14, 13, 24, 0, time.UTC), - time.Date(2040, time.Month(8), 19, 23, 56, 21, 0, time.UTC), - time.Date(2040, time.Month(8), 30, 6, 53, 14, 0, time.UTC), - time.Date(2040, time.Month(9), 26, 7, 11, 16, 0, time.UTC), - time.Date(2041, time.Month(10), 5, 0, 7, 32, 0, time.UTC), - time.Date(2041, time.Month(11), 15, 6, 8, 48, 0, time.UTC), - time.Date(2042, time.Month(9), 6, 23, 29, 45, 0, time.UTC), + time.Date(2033, time.Month(8), 26, 5, 50, 29, 0, time.UTC), + time.Date(2033, time.Month(9), 10, 1, 13, 39, 0, time.UTC), + time.Date(2033, time.Month(10), 4, 7, 47, 32, 0, time.UTC), + time.Date(2033, time.Month(10), 14, 22, 47, 59, 0, time.UTC), + time.Date(2033, time.Month(11), 8, 11, 10, 48, 0, time.UTC), + time.Date(2034, time.Month(9), 19, 14, 33, 50, 0, time.UTC), + time.Date(2035, time.Month(8), 20, 22, 31, 42, 0, time.UTC), + time.Date(2036, time.Month(6), 19, 20, 38, 29, 0, time.UTC), + time.Date(2036, time.Month(9), 29, 4, 32, 9, 0, time.UTC), + time.Date(2037, time.Month(11), 4, 5, 20, 28, 0, time.UTC), + time.Date(2038, time.Month(7), 9, 2, 11, 13, 0, time.UTC), + time.Date(2039, time.Month(9), 18, 8, 33, 4, 0, time.UTC), + time.Date(2040, time.Month(9), 8, 14, 17, 10, 0, time.UTC), + time.Date(2041, time.Month(9), 3, 15, 12, 6, 0, time.UTC), + time.Date(2041, time.Month(10), 12, 5, 26, 57, 0, time.UTC), + time.Date(2042, time.Month(9), 20, 21, 50, 47, 0, time.UTC), + time.Date(2042, time.Month(11), 19, 2, 48, 24, 0, time.UTC), } depths_expected := []float64{ - 13.0, 1.19023548877452, 0.00189844601579208, 1.0171892210224, - 1.1684668225599, 2.67564711780564, 1.11368483591891, 5.95024604558782e-06, - 0.00610899031783363, 0.0051434691700571, 1.29154912885772, 1.10496133027909, - 0.00478200367925463, 1.09444238480238, 1.01478090469665, 13.0, 0.012841852964191, + 0.979270524147806, 1.54121902357747, 3.34393311836401e-07, + 2.54157606740306e-06, 13, 3.10613618627195, 0.0118805618230561, + 0.000230443433672538, 0.000439569934077712, 0.0123576597179041, + 7.99622779970534e-05, 13, 6.78775218723548e-05, 1.02305022821015, + 2.68301460998456e-05, 0.00243471711551489, 1.90291529387485e-06, } durations_expected := []float64{ - 2.0, 2.0, 2.0, 2.0, 4.0, 2.0, - 4.0, 2.0, 2.0, 4.0, 2.0, 2.0, - 2.0, 2.0, 2.0, 1.0, 2.0, + 2.0, 2.0, 2.0, 1.0, + 2.0, 2.0, 2.0, 1.0, + 2.0, 1.0, 2.0, 1.0, + 2.0, 4.0, 4.0, 1.0, 4.0, } - arrivals, depths, durations, err := parseResponsesFile(responsesFP, "using-layer_id=0", "CSV", len(arrivals_expected)) + add, err := parseResponsesFile(responsesFP, "", "CSV", len(arrivals_expected), 0) if err != nil { panic(err) } - for i, arrival := range arrivals { + for i, arrival := range add.arrivals { if arrival != arrivals_expected[i] { t.Errorf("Fail: Expected %v, got %v", arrivals_expected[i], arrival) } else { fmt.Println("Arrival Correct") } - if depths[i] != depths_expected[i] { - t.Errorf("Fail: Expected %v, got %v", depths_expected[i], depths[i]) + if add.depths[i] != depths_expected[i] { + t.Errorf("Fail: Expected %v, got %v", depths_expected[i], add.depths[i]) } else { fmt.Println("Depth correct") } - if durations[i] != durations_expected[i] { - t.Errorf("Fail: Expected %v, got %v", durations_expected[i], durations[i]) + if add.durations[i] != durations_expected[i] { + t.Errorf("Fail: Expected %v, got %v", durations_expected[i], add.durations[i]) } else { fmt.Println("Duration Correct") } From b8b62ffae6d5e1d60688a7b24912c41adc1a0b2c Mon Sep 17 00:00:00 2001 From: Sebastian Rowan <79868066+sebastianrowan@users.noreply.github.com> Date: Wed, 8 Apr 2026 13:52:46 -0400 Subject: [PATCH 7/9] fix error. only get events for specified lifecycle --- hazardproviders/stormSim_lifecycle_hazardprovider.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/hazardproviders/stormSim_lifecycle_hazardprovider.go b/hazardproviders/stormSim_lifecycle_hazardprovider.go index 2f337df..76fbdf3 100644 --- a/hazardproviders/stormSim_lifecycle_hazardprovider.go +++ b/hazardproviders/stormSim_lifecycle_hazardprovider.go @@ -38,7 +38,9 @@ type StormSimInfo struct { func InitStormSim(ssi StormSimInfo) (stormsimLifecycleMultiHazardProvider, error) { reaches, err := parseReachesFile(ssi.ReachesFP, ssi.ReachesLayername, ssi.ReachesDriver) - + if err != nil { + panic(err) + } events, err := parseEventsFile(ssi.EventsFP, ssi.EventsLayername, ssi.EventsDriver) if err != nil { panic(err) @@ -46,7 +48,7 @@ func InitStormSim(ssi StormSimInfo) (stormsimLifecycleMultiHazardProvider, error //TODO: how to handle case where there are multiple reaches? reach := reaches[0] - reachevents := events[reach.reachID] + reachevents := events[reach.reachID][ssi.Lifecycle] add, err := parseResponsesFile(ssi.ResponsesFP, ssi.ResponsesLayername, ssi.ResponsesDriver, len(reachevents), ssi.Lifecycle) @@ -71,7 +73,7 @@ func responsesSchema() []string { } func reachesSchema() []string { - s := []string{"Name", "NameAbbreviated", "Note", "NOAAStation", "NOAAStationName", "SavePoint", "geom"} + s := []string{"Name", "NameAbbreviated", "Note", "NOAAStation", "NOAAStationName", "SavePoint"} return s } @@ -103,6 +105,7 @@ func parseReachesFile(filepath string, layername string, driver string) ([]Reach for i, f := range s { idx := def.FieldIndex(f) + if idx < 0 { return ret, errors.New("gdal dataset at path " + filepath + " Expected field named " + f + " none was found") } From a7bff189f928797f365a27c25f1b062f0a730bfa Mon Sep 17 00:00:00 2001 From: Sebastian Rowan <79868066+sebastianrowan@users.noreply.github.com> Date: Thu, 16 Apr 2026 12:16:25 -0400 Subject: [PATCH 8/9] Update docker file to use gdal:ubuntu. Fix date parsing in parseResponsesFile() Default attempt to parse field as a DateTime. Fallback to parsing as a string for csv. Will still fail if csv is not using 'YYYY-MM-DD HH:MM:SS' format --- Dockerfile | 9 +++++---- .../stormSim_lifecycle_hazardprovider.go | 17 +++++++++++++---- .../stormSim_lifecycle_hazardprovider_test.go | 1 + 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index bcd6c86..d988cb9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,12 @@ -FROM ghcr.io/usace-cloud-compute/cc-tiledb-base:latest as dev +FROM ghcr.io/osgeo/gdal:ubuntu-full-3.8.3 as dev -ARG TILEDB_LIB=/usr/local/lib/tiledb +# ARG TILEDB_LIB=/usr/local/lib/tiledb ARG GO_VERSION=1.24.5 ARG TARGETARCH ENV PATH=/go/bin:$PATH -ENV CGO_LDFLAGS="-L${TILEDB_LIB}/lib" -ENV CGO_CFLAGS="-I${TILEDB_LIB}/include" +# ENV CGO_LDFLAGS="-L${TILEDB_LIB}/lib" +# ENV CGO_CFLAGS="-I${TILEDB_LIB}/include" ENV GOROOT=/go ENV GOPATH=/src/go @@ -15,6 +15,7 @@ RUN echo "Building for arch: ${TARGETARCH}" &&\ tar -xvzf /go${GO_VERSION}.linux-${TARGETARCH}.tar.gz -C / RUN apt update +RUN apt -y install git RUN apt -y install gdal-data RUN apt -y install gdal-bin RUN apt -y install libgdal-dev diff --git a/hazardproviders/stormSim_lifecycle_hazardprovider.go b/hazardproviders/stormSim_lifecycle_hazardprovider.go index 76fbdf3..06250ef 100644 --- a/hazardproviders/stormSim_lifecycle_hazardprovider.go +++ b/hazardproviders/stormSim_lifecycle_hazardprovider.go @@ -2,6 +2,7 @@ package hazardproviders import ( "errors" + "fmt" "time" "github.com/USACE/go-consequences/geography" @@ -243,11 +244,19 @@ func parseResponsesFile(filepath string, layername string, driver string, n int, for i := range fc { feat := layer.NextFeature() if feat != nil { - d := feat.FieldAsString(sIDX[1]) - date_i, err := time.Parse("2006-01-02 15:04:05", d) - if err != nil { - panic(err) + var date_i time.Time + date_i, dtok := feat.FieldAsDateTime(sIDX[1]) + if !dtok { + // GDAL doesn't seem to recognize DateTime strings in csv files as DateTimes + fmt.Println("Failed to parse date field as DateTime. Attempting to parse from string with format 'YYYY-MM-DD HH:MM:SS'") + ds := feat.FieldAsString(sIDX[1]) + var err error + date_i, err = time.Parse("2006-01-02 15:04:05", ds) + if err != nil { + panic(err) + } } + storm_id := feat.FieldAsString(sIDX[2]) lifecycle_i := feat.FieldAsInteger(sIDX[4]) //TODO: handle NA values for stage diff --git a/hazardproviders/stormSim_lifecycle_hazardprovider_test.go b/hazardproviders/stormSim_lifecycle_hazardprovider_test.go index 90b8367..58fda15 100644 --- a/hazardproviders/stormSim_lifecycle_hazardprovider_test.go +++ b/hazardproviders/stormSim_lifecycle_hazardprovider_test.go @@ -26,6 +26,7 @@ func Test_ParseEventsFile(t *testing.T) { } } } + func Test_ParseResponsesFile(t *testing.T) { responsesFP := "/workspaces/consequences-runner/data/coastal/stage_responses_loc_DE001_lc_0.csv" From 26c9fab27a108983fac336f3fbddc0da472691e2 Mon Sep 17 00:00:00 2001 From: Sebastian Rowan <79868066+sebastianrowan@users.noreply.github.com> Date: Thu, 16 Apr 2026 16:53:02 -0400 Subject: [PATCH 9/9] Add lifecycle resultsWriter --- .vscode/launch.json | 2 +- actions/compute-payload.go | 39 +++-------- crresultswriters/lifecycle-results-writer.go | 69 +++++++++++++++++++ .../stormSim_lifecycle_hazardprovider.go | 2 + utils_test.go | 5 -- 5 files changed, 83 insertions(+), 34 deletions(-) create mode 100644 crresultswriters/lifecycle-results-writer.go diff --git a/.vscode/launch.json b/.vscode/launch.json index ac1afdf..e9ed2a4 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -52,7 +52,7 @@ "program": "${workspaceFolder}", "args": [ "-test.run", - "Test_ComputeCoastalEventAction" + "Test_Main" ], "envFile": "/workspaces/consequences-runner/.env-coastal-lifecycle" },{ diff --git a/actions/compute-payload.go b/actions/compute-payload.go index d28dc4b..c6ff7a3 100644 --- a/actions/compute-payload.go +++ b/actions/compute-payload.go @@ -17,6 +17,7 @@ import ( "github.com/USACE/go-consequences/structureprovider" "github.com/USACE/go-consequences/structures" "github.com/usace-cloud-compute/cc-go-sdk" + "github.com/usace-cloud-compute/consequences-runner/crresultswriters" lrw "github.com/usace-cloud-compute/consequences-runner/crresultswriters" lhp "github.com/usace-cloud-compute/consequences-runner/hazardproviders" "github.com/usace-cloud-compute/consequences-runner/structureproviders" @@ -448,7 +449,7 @@ func (ar *ComputeCoastalLifecycleAction) Run() error { defer hp.Close() var abstractSP consequences.StreamProvider - var sr string + // var sr string if inventoryDriver == "MILLIMAN" { sp, err := structureproviders.InitMillimanStructureProviderwithOcctypePath(inventoryPath, damageFunctionPath) sp.SetDeterministic(true) @@ -456,7 +457,7 @@ func (ar *ComputeCoastalLifecycleAction) Run() error { return err } fmt.Sprintln(sp.FilePath) - sr = sp.SpatialReference() + // sr = sp.SpatialReference() abstractSP = sp } else { sp, err := structureprovider.InitStructureProviderwithOcctypePath(inventoryPath, tablename, inventoryDriver, damageFunctionPath) @@ -465,36 +466,17 @@ func (ar *ComputeCoastalLifecycleAction) Run() error { return err } fmt.Sprintln(sp.FilePath) - sr = sp.SpatialReference() + // sr = sp.SpatialReference() abstractSP = sp } //initalize a results writer - var rw consequences.ResultsWriter - if outputDriver == "PostgreSQL" { - pgUser := os.Getenv(pgUserKey) - pgPass := os.Getenv(pgPasswordKey) - pgDB := os.Getenv(pgDbnameKey) - pgHost := os.Getenv(pgHostKey) - pgPort := os.Getenv(pgPortKey) - pgSchema := os.Getenv(pgSchemaKey) - - outConnStr := fmt.Sprintf( - "PG:dbname=%s user=%s password=%s host=%s port=%s schemas=%s", - pgDB, pgUser, pgPass, pgHost, pgPort, pgSchema, - ) - - rw, err = lrw.InitSpatialResultsWriter_PSQL(outConnStr, outputLayerName, outputDriver, pgDB) - if err != nil { - log.Fatalf("Failed to initialize spatial psql result writer: %s\n", err) - } - } else { - outfp := fmt.Sprintf("%s/%s", localData, outputFileName) - - rw, err = resultswriters.InitSpatialResultsWriter_WKT_Projected(outfp, outputLayerName, outputDriver, sr) - if err != nil { - log.Fatalf("Failed to initialize spatial result writer: %s\n", err) - } + //TODO: add more key-value pairs to payload for summary and events results writer details + summaryResultsFile := fmt.Sprintf("summary_%s", outputFileName) + eventsResultsFile := fmt.Sprintf("events_%s", outputFileName) + rw, err := crresultswriters.InitLifecycleResultsWriter(summaryResultsFile, "summary_results", outputDriver, eventsResultsFile, "event_results", outputDriver) + if err != nil { + panic(err) } defer rw.Close() @@ -509,6 +491,7 @@ func (ar *ComputeCoastalLifecycleAction) Run() error { abstractSP.ByBbox(bbox, func(f consequences.Receptor) { //ProvideHazard works off of a geography.Location d, err2 := hp.Hazard(geography.Location{X: f.Location().X, Y: f.Location().Y}) + //compute damages based on hazard being able to provide depth if err2 == nil { r, err3 := f.Compute(d) diff --git a/crresultswriters/lifecycle-results-writer.go b/crresultswriters/lifecycle-results-writer.go new file mode 100644 index 0000000..55245de --- /dev/null +++ b/crresultswriters/lifecycle-results-writer.go @@ -0,0 +1,69 @@ +package crresultswriters + +import ( + "github.com/USACE/go-consequences/consequences" + "github.com/USACE/go-consequences/resultswriters" +) + +type lifecycleResultsWriter struct { + SummaryWriter consequences.ResultsWriter + EventsWriter consequences.ResultsWriter +} + +func InitLifecycleResultsWriter( + summaryFilepath string, + summaryLayername string, + summaryDriver string, + eventsFilepath string, + eventsLayername string, + eventsDriver string, +) (*lifecycleResultsWriter, error) { + srw, err := resultswriters.InitSpatialResultsWriter(summaryFilepath, summaryLayername, summaryDriver) + if err != nil { + panic(err) + } + erw, err := resultswriters.InitSpatialResultsWriter(eventsFilepath, eventsLayername, eventsDriver) + if err != nil { + panic(err) + } + return &lifecycleResultsWriter{SummaryWriter: srw, EventsWriter: erw}, nil +} + +func (lrw *lifecycleResultsWriter) Write(r consequences.Result) { + // Lifecycle compute includes a column which contains results for each event in the lifecycle + // 1. remove the event results column and parse them into a series of discrete Results + // 2. pass each event result to lrw.EventsWriter.Write() + // 3. pass the remaining columns to lrw.SummaryWriter.Write() + fd_id, err := r.Fetch("fd_id") + if err != nil { + panic(err) + } + e, err := r.Fetch("hazard results") + if err != nil { + panic(err) + } + eventResults := e.(consequences.Result) + for _, ei := range eventResults.Result { + er := ei.(consequences.Result) + hi := append([]string{"fd_id"}, er.Headers...) + ri := append([]interface{}{fd_id}, er.Result...) + lrw.EventsWriter.Write(consequences.Result{Headers: hi, Result: ri}) + } + + summaryHeaders := make([]string, 0) + summaryResults := make([]interface{}, 0) + + for i, h := range r.Headers { + if h != "hazard results" { + summaryHeaders = append(summaryHeaders, h) + summaryResults = append(summaryResults, r.Result[i]) + } + } + lrw.SummaryWriter.Write(consequences.Result{Headers: summaryHeaders, Result: summaryResults}) +} + +func (lrw *lifecycleResultsWriter) Close() { + // Will this fail if both writers are using the same geopackage? + lrw.SummaryWriter.Close() + lrw.EventsWriter.Close() +} diff --git a/hazardproviders/stormSim_lifecycle_hazardprovider.go b/hazardproviders/stormSim_lifecycle_hazardprovider.go index 06250ef..5735a0b 100644 --- a/hazardproviders/stormSim_lifecycle_hazardprovider.go +++ b/hazardproviders/stormSim_lifecycle_hazardprovider.go @@ -188,12 +188,14 @@ func parseEventsFile(filepath string, layername string, driver string) (map[stri } type ADDInfo struct { + storm_ids []string arrivals []time.Time depths []float64 durations []float64 } func parseResponsesFile(filepath string, layername string, driver string, n int, lifecycle int) (ADDInfo, error) { + arrivals := make([]time.Time, n) depths := make([]float64, n) durations := make([]float64, n) diff --git a/utils_test.go b/utils_test.go index fd9110d..e6dfb42 100644 --- a/utils_test.go +++ b/utils_test.go @@ -137,8 +137,3 @@ func Test_Format(t *testing.T) { } } - -func Test_ComputeCoastalEventAction(t *testing.T) { - - main() -}