Sitelet https://github.com/GoogleCloudPlatform/testgrid/commit/9e39b10579bd02c31ce6956ecd5e3458643aa0f8
Skip to content

Commit 9e39b10

Browse files
google-oss-prow[bot]google-oss-prow[bot]
authored
Merge pull request #1274 from michelle192837/query
Add ability to query a single job from Prow in ResultStore.
2 parents dcf821b + ba42709 commit 9e39b10

7 files changed

Lines changed: 680 additions & 357 deletions

File tree

‎pb/config/config.pb.go‎

Lines changed: 350 additions & 339 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pb/config/config.proto‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -364,6 +364,9 @@ message GCSConfig {
364364
message ResultStoreConfig {
365365
// Google Cloud Platform project ID where ResultStore results are stored.
366366
string project = 1;
367+
// A simple query to filter for particular results.
368+
// Currently, only allows a query in the form of `target:"<target>"`.
369+
string query = 2;
367370
}
368371

369372
// Options for where to gather linked issues from.

‎pkg/updater/resultstore/BUILD.bazel‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ go_library(
44
name = "go_default_library",
55
srcs = [
66
"client.go",
7+
"query.go",
78
"resultstore.go",
89
],
910
importpath = "github.com/GoogleCloudPlatform/testgrid/pkg/updater/resultstore",
@@ -27,7 +28,10 @@ go_library(
2728

2829
go_test(
2930
name = "go_default_test",
30-
srcs = ["resultstore_test.go"],
31+
srcs = [
32+
"query_test.go",
33+
"resultstore_test.go",
34+
],
3135
embed = [":go_default_library"],
3236
deps = [
3337
"//pb/config:go_default_library",

‎pkg/updater/resultstore/query.go‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
/*
2+
Copyright 2024 The TestGrid Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package resultstore
18+
19+
import (
20+
"fmt"
21+
"regexp"
22+
"strings"
23+
)
24+
25+
func translateAtom(simpleAtom string) (string, error) {
26+
if simpleAtom == "" {
27+
return "", nil
28+
}
29+
// For now, we expect an atom with the exact form `target:"<target>"`
30+
// Split the `key:value` atom.
31+
parts := strings.SplitN(simpleAtom, ":", 2)
32+
if len(parts) != 2 {
33+
return "", fmt.Errorf("unrecognized atom %q", simpleAtom)
34+
}
35+
key := strings.TrimSpace(parts[0])
36+
val := strings.Trim(strings.TrimSpace(parts[1]), `"`)
37+
38+
switch {
39+
case key == "target":
40+
return fmt.Sprintf(`id.target_id="%s"`, val), nil
41+
default:
42+
return "", fmt.Errorf("unrecognized atom key %q", key)
43+
}
44+
}
45+
46+
var (
47+
queryRe = regexp.MustCompile(`^target:".*"$`)
48+
)
49+
50+
func translateQuery(simpleQuery string) (string, error) {
51+
if simpleQuery == "" {
52+
return "", nil
53+
}
54+
// For now, we expect a query with a single atom, with the exact form `target:"<target>"`
55+
if !queryRe.MatchString(simpleQuery) {
56+
return "", fmt.Errorf("invalid query %q: must match %q", simpleQuery, queryRe.String())
57+
}
58+
query, err := translateAtom(simpleQuery)
59+
if err != nil {
60+
return "", fmt.Errorf("invalid query %q: %v", simpleQuery, err)
61+
}
62+
return query, nil
63+
}

‎pkg/updater/resultstore/query_test.go‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
/*
2+
Copyright 2024 The TestGrid Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package resultstore
18+
19+
import (
20+
"testing"
21+
)
22+
23+
func TestTranslateAtom(t *testing.T) {
24+
cases := []struct {
25+
name string
26+
atom string
27+
want string
28+
wantError bool
29+
}{
30+
{
31+
name: "empty",
32+
atom: "",
33+
want: "",
34+
},
35+
{
36+
name: "basic",
37+
atom: `target:"//my-target"`,
38+
want: `id.target_id="//my-target"`,
39+
},
40+
{
41+
name: "case-sensitive key",
42+
atom: `TARGET:"//MY-TARGET"`,
43+
wantError: true,
44+
},
45+
{
46+
name: "multiple colons",
47+
atom: `target:"//path/to:my-target"`,
48+
want: `id.target_id="//path/to:my-target"`,
49+
},
50+
{
51+
name: "unquoted",
52+
atom: `target://my-target`,
53+
want: `id.target_id="//my-target"`,
54+
},
55+
{
56+
name: "partial quotes",
57+
atom: `target://my-target"`,
58+
want: `id.target_id="//my-target"`,
59+
},
60+
{
61+
name: "not enough parts",
62+
atom: "target",
63+
wantError: true,
64+
},
65+
{
66+
name: "unknown atom",
67+
atom: "label:foo",
68+
wantError: true,
69+
},
70+
}
71+
72+
for _, tc := range cases {
73+
t.Run(tc.name, func(t *testing.T) {
74+
got, err := translateAtom(tc.atom)
75+
if tc.want != got {
76+
t.Errorf("translateAtom(%q) differed; got %q, want %q", tc.atom, got, tc.want)
77+
}
78+
if err == nil && tc.wantError {
79+
t.Errorf("translateAtom(%q) did not error as expected", tc.atom)
80+
} else if err != nil && !tc.wantError {
81+
t.Errorf("translateAtom(%q) errored unexpectedly: %v", tc.atom, err)
82+
}
83+
})
84+
}
85+
}
86+
87+
func TestTranslateQuery(t *testing.T) {
88+
cases := []struct {
89+
name string
90+
query string
91+
want string
92+
wantError bool
93+
}{
94+
{
95+
name: "empty",
96+
query: "",
97+
want: "",
98+
},
99+
{
100+
name: "basic",
101+
query: `target:"//my-target"`,
102+
want: `id.target_id="//my-target"`,
103+
},
104+
{
105+
name: "case-sensitive key",
106+
query: `TARGET:"//MY-TARGET"`,
107+
wantError: true,
108+
},
109+
{
110+
name: "multiple colons",
111+
query: `target:"//path/to:my-target"`,
112+
want: `id.target_id="//path/to:my-target"`,
113+
},
114+
{
115+
name: "unquoted",
116+
query: `target://my-target`,
117+
wantError: true,
118+
},
119+
{
120+
name: "partial quotes",
121+
query: `target://my-target"`,
122+
wantError: true,
123+
},
124+
{
125+
name: "invalid query",
126+
query: `label:foo`,
127+
wantError: true,
128+
},
129+
{
130+
name: "partial match",
131+
query: `some_target:foo`,
132+
wantError: true,
133+
},
134+
}
135+
136+
for _, tc := range cases {
137+
t.Run(tc.name, func(t *testing.T) {
138+
got, err := translateQuery(tc.query)
139+
if tc.want != got {
140+
t.Errorf("translateQuery(%q) differed; got %q, want %q", tc.query, got, tc.want)
141+
}
142+
if tc.wantError && err == nil {
143+
t.Errorf("translateQuery(%q) did not error as expected", tc.query)
144+
} else if !tc.wantError && err != nil {
145+
t.Errorf("translateQuery(%q) errored unexpectedly: %v", tc.query, err)
146+
}
147+
})
148+
}
149+
}

‎pkg/updater/resultstore/resultstore.go‎

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,7 @@ func extractGroupID(tg *configpb.TestGroup, inv *invocation) string {
156156
func ColumnReader(client *DownloadClient, reprocess time.Duration) updater.ColumnReader {
157157
return func(ctx context.Context, log logrus.FieldLogger, tg *configpb.TestGroup, oldCols []updater.InflatedColumn, defaultStop time.Time, receivers chan<- updater.InflatedColumn) error {
158158
stop := updateStop(log, tg, time.Now(), oldCols, defaultStop, reprocess)
159-
ids, err := search(ctx, log, client, tg.GetResultSource().GetResultstoreConfig().GetProject(), stop)
159+
ids, err := search(ctx, log, client, tg.GetResultSource().GetResultstoreConfig(), stop)
160160
if err != nil {
161161
return fmt.Errorf("error searching invocations: %v", err)
162162
}
@@ -889,21 +889,35 @@ func queryAfter(query string, when time.Time) string {
889889
return fmt.Sprintf("%s timing.start_time>=\"%s\"", query, when.UTC().Format(time.RFC3339))
890890
}
891891

892-
// TODO: Replace these hardcoded values with adjustable ones.
893892
const (
894-
queryProw = "invocation_attributes.labels:\"prow\""
893+
prowLabel = `invocation_attributes.labels:"prow"`
895894
)
896895

897-
func search(ctx context.Context, log logrus.FieldLogger, client *DownloadClient, projectID string, stop time.Time) ([]string, error) {
896+
func queryProw(baseQuery string, stop time.Time) (string, error) {
897+
// TODO: ResultStore use is assumed to be Prow-only at the moment. Make this more flexible in future.
898+
if baseQuery == "" {
899+
return queryAfter(prowLabel, stop), nil
900+
}
901+
query, err := translateQuery(baseQuery)
902+
if err != nil {
903+
return "", err
904+
}
905+
return queryAfter(fmt.Sprintf("%s %s", query, prowLabel), stop), nil
906+
}
907+
908+
func search(ctx context.Context, log logrus.FieldLogger, client *DownloadClient, rsConfig *configpb.ResultStoreConfig, stop time.Time) ([]string, error) {
898909
if client == nil {
899910
return nil, fmt.Errorf("no ResultStore client provided")
900911
}
901-
query := queryAfter(queryProw, stop)
912+
query, err := queryProw(rsConfig.GetQuery(), stop)
913+
if err != nil {
914+
return nil, fmt.Errorf("queryProw() failed to create query: %v", err)
915+
}
902916
log.WithField("query", query).Debug("Searching ResultStore.")
903917
// Quit if search goes over 5 minutes.
904918
ctx, cancel := context.WithTimeout(ctx, 5*time.Minute)
905919
defer cancel()
906-
ids, err := client.Search(ctx, log, query, projectID)
920+
ids, err := client.Search(ctx, log, query, rsConfig.GetProject())
907921
log.WithField("ids", len(ids)).WithError(err).Debug("Searched ResultStore.")
908922
return ids, err
909923
}

0 commit comments

Comments
 (0)