From 53cb451bae4701b66f2be1ab9acc97e8fd296ef5 Mon Sep 17 00:00:00 2001 From: Tim H <6026716+tho@users.noreply.github.com> Date: Wed, 19 Mar 2025 06:25:07 -0700 Subject: [PATCH 1/4] Change JQ to process newline-delimited JSON https://jsonlines.org Example use case: Analyzes logs to count and display frequency of different log levels from newline-delimited JSON input. ``` cat log.json | ./goscript.sh -c 'script.Stdin().JQ(".level").Freq().Stdout()' 3 "INFO" 2 "WARN" 1 "ERROR" ``` --- script.go | 41 ++++++++++++++++----------- script_test.go | 76 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+), 16 deletions(-) diff --git a/script.go b/script.go index 1fd40bc..700d3df 100644 --- a/script.go +++ b/script.go @@ -712,9 +712,10 @@ func (p *Pipe) Join() *Pipe { }) } -// JQ executes query on the pipe's contents (presumed to be JSON), producing -// the result. An invalid query will set the appropriate error on the pipe. -// +// JQ executes query on the pipe's contents (presumed to be newline-delimited +// JSON), applying the query to each input value and producing the results. An +// invalid query or value will set the appropriate error on the pipe. + // The exact dialect of JQ supported is that provided by // [github.com/itchyny/gojq], whose documentation explains the differences // between it and standard JQ. @@ -724,26 +725,34 @@ func (p *Pipe) JQ(query string) *Pipe { if err != nil { return err } - var input interface{} - err = json.NewDecoder(r).Decode(&input) + c, err := gojq.Compile(q) if err != nil { return err } - iter := q.Run(input) - for { - v, ok := iter.Next() - if !ok { - return nil - } - if err, ok := v.(error); ok { - return err - } - result, err := gojq.Marshal(v) + dec := json.NewDecoder(r) + for dec.More() { + var input interface{} + err := dec.Decode(&input) if err != nil { return err } - fmt.Fprintln(w, string(result)) + iter := c.Run(input) + for { + v, ok := iter.Next() + if !ok { + break + } + if err, ok := v.(error); ok { + return err + } + result, err := gojq.Marshal(v) + if err != nil { + return err + } + fmt.Fprintln(w, string(result)) + } } + return nil }) } diff --git a/script_test.go b/script_test.go index a3f9124..193d12a 100644 --- a/script_test.go +++ b/script_test.go @@ -804,6 +804,59 @@ func TestJQHandlesGithubJSONWithRealWorldExampleQuery(t *testing.T) { } } +func TestJQWithNewlineDelimitedInputAndFieldQueryProducesSelectedFields(t *testing.T) { + t.Parallel() + input := `{"timestamp": 1649264191, "iss_position": {"longitude": "52.8439", "latitude": "10.8107"}, "message": "success"}` + "\n" + input += input + want := `{"latitude":"10.8107","longitude":"52.8439"}` + "\n" + want += want + got, err := script.Echo(input).JQ(".iss_position").String() + if err != nil { + t.Fatal(err) + } + if want != got { + t.Error(want, got) + t.Error(cmp.Diff(want, got)) + } +} + +func TestJQWithNewlineDelimitedInputAndArrayInputAndElementQueryProducesSelectedElements(t *testing.T) { + t.Parallel() + input := `[1, 2, 3]` + "\n" + `[4, 5, 6]` + want := "1\n4\n" + got, err := script.Echo(input).JQ(".[0]").String() + if err != nil { + t.Fatal(err) + } + if want != got { + t.Error(want, got) + t.Error(cmp.Diff(want, got)) + } +} + +func TestJQWithNewlineDelimitedMixedAndPrettyPrintedInputValues(t *testing.T) { + t.Parallel() + input := ` +{ + "key1": "val1", + "key2": "val2" +} +[ + 0, + 1 +] +` + want := `{"key1":"val1","key2":"val2"}` + "\n" + "[0,1]" + "\n" + got, err := script.Echo(input).JQ(".").String() + if err != nil { + t.Fatal(err) + } + if want != got { + t.Error(want, got) + t.Error(cmp.Diff(want, got)) + } +} + func TestJQErrorsWithInvalidQuery(t *testing.T) { t.Parallel() input := `[1, 2, 3]` @@ -813,6 +866,29 @@ func TestJQErrorsWithInvalidQuery(t *testing.T) { } } +func TestJQErrorsWithInvalidInput(t *testing.T) { + t.Parallel() + input := "invalid JSON value" + _, err := script.Echo(input).JQ(".").String() + if err == nil { + t.Error("want error from invalid JSON input, got nil") + } +} + +func TestJQWithNewlineDelimitedInputErrorsAfterFirstInvalidInput(t *testing.T) { + t.Parallel() + input := `[0]` + "\n" + `[1` + "\n" + `[2]` // missing `]` in second line + want := "0\n" + got, err := script.Echo(input).JQ(".[0]").String() + if err == nil { + t.Fatal("want error from invalid JSON, got nil") + } + if want != got { + t.Error(want, got) + t.Error(cmp.Diff(want, got)) + } +} + func TestLastDropsAllButLastNLinesOfInput(t *testing.T) { t.Parallel() input := "a\nb\nc\n" From 22c403ad7011c22eac6d401347c89746336dfd95 Mon Sep 17 00:00:00 2001 From: John Arundel Date: Fri, 21 Mar 2025 12:07:03 +0000 Subject: [PATCH 2/4] tweaks --- script.go | 31 +++++++++++++++++-------------- script_test.go | 46 +++++++++------------------------------------- 2 files changed, 26 insertions(+), 51 deletions(-) diff --git a/script.go b/script.go index 700d3df..d7d1bc3 100644 --- a/script.go +++ b/script.go @@ -712,31 +712,34 @@ func (p *Pipe) Join() *Pipe { }) } -// JQ executes query on the pipe's contents (presumed to be newline-delimited -// JSON), applying the query to each input value and producing the results. An -// invalid query or value will set the appropriate error on the pipe. - +// JQ executes query on the pipe's contents (presumed to be valid JSON or +// [JSONLines] data), applying the query to each newline-delimited input value +// and producing results until the first error is encountered. An invalid query +// or value will set the appropriate error on the pipe. +// // The exact dialect of JQ supported is that provided by // [github.com/itchyny/gojq], whose documentation explains the differences // between it and standard JQ. +// +// [JSONLines]: https://jsonlines.org/ func (p *Pipe) JQ(query string) *Pipe { + parsedQuery, err := gojq.Parse(query) + if err != nil { + return p.WithError(err) + } + code, err := gojq.Compile(parsedQuery) + if err != nil { + return p.WithError(err) + } return p.Filter(func(r io.Reader, w io.Writer) error { - q, err := gojq.Parse(query) - if err != nil { - return err - } - c, err := gojq.Compile(q) - if err != nil { - return err - } dec := json.NewDecoder(r) for dec.More() { - var input interface{} + var input any err := dec.Decode(&input) if err != nil { return err } - iter := c.Run(input) + iter := code.Run(input) for { v, ok := iter.Next() if !ok { diff --git a/script_test.go b/script_test.go index 193d12a..04435eb 100644 --- a/script_test.go +++ b/script_test.go @@ -804,23 +804,20 @@ func TestJQHandlesGithubJSONWithRealWorldExampleQuery(t *testing.T) { } } -func TestJQWithNewlineDelimitedInputAndFieldQueryProducesSelectedFields(t *testing.T) { +func TestJQCorrectlyQueriesMultilineInputFields(t *testing.T) { t.Parallel() - input := `{"timestamp": 1649264191, "iss_position": {"longitude": "52.8439", "latitude": "10.8107"}, "message": "success"}` + "\n" - input += input - want := `{"latitude":"10.8107","longitude":"52.8439"}` + "\n" - want += want - got, err := script.Echo(input).JQ(".iss_position").String() + input := `{"a":1}` + "\n" + `{"a":2}` + want := "1\n2\n" + got, err := script.Echo(input).JQ(".a").String() if err != nil { t.Fatal(err) } if want != got { - t.Error(want, got) t.Error(cmp.Diff(want, got)) } } -func TestJQWithNewlineDelimitedInputAndArrayInputAndElementQueryProducesSelectedElements(t *testing.T) { +func TestJQCorrectlyQueriesMultilineInputArrays(t *testing.T) { t.Parallel() input := `[1, 2, 3]` + "\n" + `[4, 5, 6]` want := "1\n4\n" @@ -829,30 +826,6 @@ func TestJQWithNewlineDelimitedInputAndArrayInputAndElementQueryProducesSelected t.Fatal(err) } if want != got { - t.Error(want, got) - t.Error(cmp.Diff(want, got)) - } -} - -func TestJQWithNewlineDelimitedMixedAndPrettyPrintedInputValues(t *testing.T) { - t.Parallel() - input := ` -{ - "key1": "val1", - "key2": "val2" -} -[ - 0, - 1 -] -` - want := `{"key1":"val1","key2":"val2"}` + "\n" + "[0,1]" + "\n" - got, err := script.Echo(input).JQ(".").String() - if err != nil { - t.Fatal(err) - } - if want != got { - t.Error(want, got) t.Error(cmp.Diff(want, got)) } } @@ -875,16 +848,15 @@ func TestJQErrorsWithInvalidInput(t *testing.T) { } } -func TestJQWithNewlineDelimitedInputErrorsAfterFirstInvalidInput(t *testing.T) { +func TestJQProducesValidResultsUntilFirstError(t *testing.T) { t.Parallel() - input := `[0]` + "\n" + `[1` + "\n" + `[2]` // missing `]` in second line - want := "0\n" + input := "[1]\ninvalid JSON value" + want := "1\n" got, err := script.Echo(input).JQ(".[0]").String() if err == nil { - t.Fatal("want error from invalid JSON, got nil") + t.Fatal("want error from invalid JSON input, got nil") } if want != got { - t.Error(want, got) t.Error(cmp.Diff(want, got)) } } From 6304e3e58f7fd9d5eebf497898361100234afdbc Mon Sep 17 00:00:00 2001 From: John Arundel Date: Fri, 21 Mar 2025 12:11:58 +0000 Subject: [PATCH 3/4] remove extra failure outputs --- script_test.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/script_test.go b/script_test.go index 04435eb..9aa93ff 100644 --- a/script_test.go +++ b/script_test.go @@ -743,7 +743,6 @@ func TestJQWithDotQueryPrettyPrintsInput(t *testing.T) { t.Fatal(err) } if want != got { - t.Error(want, got) t.Error(cmp.Diff(want, got)) } } @@ -757,7 +756,6 @@ func TestJQWithFieldQueryProducesSelectedField(t *testing.T) { t.Fatal(err) } if want != got { - t.Error(want, got) t.Error(cmp.Diff(want, got)) } } @@ -771,7 +769,6 @@ func TestJQWithArrayQueryProducesRequiredArray(t *testing.T) { t.Fatal(err) } if want != got { - t.Error(want, got) t.Error(cmp.Diff(want, got)) } } @@ -785,7 +782,6 @@ func TestJQWithArrayInputAndElementQueryProducesSelectedElement(t *testing.T) { t.Fatal(err) } if want != got { - t.Error(want, got) t.Error(cmp.Diff(want, got)) } } @@ -799,7 +795,6 @@ func TestJQHandlesGithubJSONWithRealWorldExampleQuery(t *testing.T) { t.Fatal(err) } if want != got { - t.Error(want, got) t.Error(cmp.Diff(want, got)) } } From 14b46056564400a58c6ec94f87699e8aaf06f7e8 Mon Sep 17 00:00:00 2001 From: John Arundel Date: Fri, 21 Mar 2025 15:59:16 +0000 Subject: [PATCH 4/4] add extra ignored test input --- script_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script_test.go b/script_test.go index 9aa93ff..12f4d90 100644 --- a/script_test.go +++ b/script_test.go @@ -845,11 +845,11 @@ func TestJQErrorsWithInvalidInput(t *testing.T) { func TestJQProducesValidResultsUntilFirstError(t *testing.T) { t.Parallel() - input := "[1]\ninvalid JSON value" + input := "[1]\ninvalid JSON value\n[2]" want := "1\n" got, err := script.Echo(input).JQ(".[0]").String() if err == nil { - t.Fatal("want error from invalid JSON input, got nil") + t.Error("want error from invalid JSON input, got nil") } if want != got { t.Error(cmp.Diff(want, got))