From 348e77b31fef2a2fdead67776edbf73f4d656e08 Mon Sep 17 00:00:00 2001 From: Tim H <6026716+tho@users.noreply.github.com> Date: Sat, 22 Mar 2025 04:59:54 -0700 Subject: [PATCH 1/2] Change JQ to process newline-delimited JSON (#227) * Change JQ to process newline-delimited JSON https://jsonlines.org Example use case: Analyzes logs to count and display frequency of different log levels from newline-delimited JSON input. ``` cat log.json | ./goscript.sh -c 'script.Stdin().JQ(".level").Freq().Stdout()' 3 "INFO" 2 "WARN" 1 "ERROR" ``` * tweaks * remove extra failure outputs * add extra ignored test input --------- Co-authored-by: John Arundel --- script.go | 56 ++++++++++++++++++++++++++++++-------------------- script_test.go | 53 ++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 82 insertions(+), 27 deletions(-) diff --git a/script.go b/script.go index 1fd40bc6..d7d1bc34 100644 --- a/script.go +++ b/script.go @@ -712,38 +712,50 @@ func (p *Pipe) Join() *Pipe { }) } -// JQ executes query on the pipe's contents (presumed to be JSON), producing -// the result. An invalid query will set the appropriate error on the pipe. +// JQ executes query on the pipe's contents (presumed to be valid JSON or +// [JSONLines] data), applying the query to each newline-delimited input value +// and producing results until the first error is encountered. An invalid query +// or value will set the appropriate error on the pipe. // // The exact dialect of JQ supported is that provided by // [github.com/itchyny/gojq], whose documentation explains the differences // between it and standard JQ. +// +// [JSONLines]: https://jsonlines.org/ func (p *Pipe) JQ(query string) *Pipe { + parsedQuery, err := gojq.Parse(query) + if err != nil { + return p.WithError(err) + } + code, err := gojq.Compile(parsedQuery) + if err != nil { + return p.WithError(err) + } return p.Filter(func(r io.Reader, w io.Writer) error { - q, err := gojq.Parse(query) - if err != nil { - return err - } - var input interface{} - err = json.NewDecoder(r).Decode(&input) - if err != nil { - return err - } - iter := q.Run(input) - for { - v, ok := iter.Next() - if !ok { - return nil - } - if err, ok := v.(error); ok { - return err - } - result, err := gojq.Marshal(v) + dec := json.NewDecoder(r) + for dec.More() { + var input any + err := dec.Decode(&input) if err != nil { return err } - fmt.Fprintln(w, string(result)) + iter := code.Run(input) + for { + v, ok := iter.Next() + if !ok { + break + } + if err, ok := v.(error); ok { + return err + } + result, err := gojq.Marshal(v) + if err != nil { + return err + } + fmt.Fprintln(w, string(result)) + } } + return nil }) } diff --git a/script_test.go b/script_test.go index a3f91247..12f4d90a 100644 --- a/script_test.go +++ b/script_test.go @@ -743,7 +743,6 @@ func TestJQWithDotQueryPrettyPrintsInput(t *testing.T) { t.Fatal(err) } if want != got { - t.Error(want, got) t.Error(cmp.Diff(want, got)) } } @@ -757,7 +756,6 @@ func TestJQWithFieldQueryProducesSelectedField(t *testing.T) { t.Fatal(err) } if want != got { - t.Error(want, got) t.Error(cmp.Diff(want, got)) } } @@ -771,7 +769,6 @@ func TestJQWithArrayQueryProducesRequiredArray(t *testing.T) { t.Fatal(err) } if want != got { - t.Error(want, got) t.Error(cmp.Diff(want, got)) } } @@ -785,7 +782,6 @@ func TestJQWithArrayInputAndElementQueryProducesSelectedElement(t *testing.T) { t.Fatal(err) } if want != got { - t.Error(want, got) t.Error(cmp.Diff(want, got)) } } @@ -799,7 +795,32 @@ func TestJQHandlesGithubJSONWithRealWorldExampleQuery(t *testing.T) { t.Fatal(err) } if want != got { - t.Error(want, got) + t.Error(cmp.Diff(want, got)) + } +} + +func TestJQCorrectlyQueriesMultilineInputFields(t *testing.T) { + t.Parallel() + input := `{"a":1}` + "\n" + `{"a":2}` + want := "1\n2\n" + got, err := script.Echo(input).JQ(".a").String() + if err != nil { + t.Fatal(err) + } + if want != got { + t.Error(cmp.Diff(want, got)) + } +} + +func TestJQCorrectlyQueriesMultilineInputArrays(t *testing.T) { + t.Parallel() + input := `[1, 2, 3]` + "\n" + `[4, 5, 6]` + want := "1\n4\n" + got, err := script.Echo(input).JQ(".[0]").String() + if err != nil { + t.Fatal(err) + } + if want != got { t.Error(cmp.Diff(want, got)) } } @@ -813,6 +834,28 @@ func TestJQErrorsWithInvalidQuery(t *testing.T) { } } +func TestJQErrorsWithInvalidInput(t *testing.T) { + t.Parallel() + input := "invalid JSON value" + _, err := script.Echo(input).JQ(".").String() + if err == nil { + t.Error("want error from invalid JSON input, got nil") + } +} + +func TestJQProducesValidResultsUntilFirstError(t *testing.T) { + t.Parallel() + input := "[1]\ninvalid JSON value\n[2]" + want := "1\n" + got, err := script.Echo(input).JQ(".[0]").String() + if err == nil { + t.Error("want error from invalid JSON input, got nil") + } + if want != got { + t.Error(cmp.Diff(want, got)) + } +} + func TestLastDropsAllButLastNLinesOfInput(t *testing.T) { t.Parallel() input := "a\nb\nc\n" From 6006cd08d4338a997e1801708f4678dcf3fba6be Mon Sep 17 00:00:00 2001 From: John Arundel Date: Sat, 22 Mar 2025 12:06:14 +0000 Subject: [PATCH 2/2] update changelog (#228) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 0510040d..461168e9 100644 --- a/README.md +++ b/README.md @@ -383,6 +383,7 @@ Sinks are methods that return some data from a pipe, ending the pipeline and ext | Version | New | | ----------- | ------- | +| 0.24.1 | [`JQ`](https://pkg.go.dev/github.com/bitfield/script#Pipe.JQ) accepts JSONLines data | | 0.24.0 | [`Hash`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Hash) | | | [`HashSums`](https://pkg.go.dev/github.com/bitfield/script#Pipe.HashSums) | | 0.23.0 | [`WithEnv`](https://pkg.go.dev/github.com/bitfield/script#Pipe.WithEnv) |