Commit 35339a2f authored by Stefan Hengl's avatar Stefan Hengl Committed by GitHub

search: add support for streaming search (#485)

This adds the flag `-stream` to the `search` sub-command. Searches
submitted with `-stream` are routed to the endpoint `search/stream`
instead of the GraphQL API.

In addition, users can set `-display <int>` to set the display limit.

We will add support for JSON in a follow-up PR.
parent ecd2663b
......@@ -60,6 +60,9 @@ func parseTemplate(text string) (*template.Template, error) {
"addFloat": func(x, y float64) float64 {
return x + y
},
"addInt32": func(x, y int32) int32 {
return x + y
},
"debug": func(v interface{}) string {
data, _ := marshalIndent(v)
fmt.Println(string(data))
......@@ -91,6 +94,13 @@ func parseTemplate(text string) (*template.Template, error) {
"buildVersionHasNewSearchInterface": searchTemplateFuncs["buildVersionHasNewSearchInterface"],
"renderResult": searchTemplateFuncs["renderResult"],
// Register stream-search specific template functions.
"streamSearchSequentialLineNumber": streamSearchTemplateFuncs["streamSearchSequentialLineNumber"],
"streamSearchHighlightMatch": streamSearchTemplateFuncs["streamSearchHighlightMatch"],
"streamSearchHighlightCommit": streamSearchTemplateFuncs["streamSearchHighlightCommit"],
"streamSearchRenderCommitLabel": streamSearchTemplateFuncs["streamSearchRenderCommitLabel"],
"matchOrMatches": streamSearchTemplateFuncs["matchOrMatches"],
// Alert rendering
"searchAlertRender": func(alert searchResultsAlert) string {
if content, err := alert.Render(); err != nil {
......
......@@ -17,6 +17,7 @@ import (
isatty "github.com/mattn/go-isatty"
"github.com/sourcegraph/src-cli/internal/api"
"github.com/sourcegraph/src-cli/internal/streaming"
"jaytaylor.com/html2text"
)
......@@ -50,10 +51,12 @@ Other tips:
flagSet := flag.NewFlagSet("search", flag.ExitOnError)
var (
jsonFlag = flagSet.Bool("json", false, "Whether or not to output results as JSON")
jsonFlag = flagSet.Bool("json", false, "Whether or not to output results as JSON.")
explainJSONFlag = flagSet.Bool("explain-json", false, "Explain the JSON output schema and exit.")
apiFlags = api.NewFlags(flagSet)
lessFlag = flagSet.Bool("less", true, "Pipe output to 'less -R' (only if stdout is terminal, and not json flag)")
lessFlag = flagSet.Bool("less", true, "Pipe output to 'less -R' (only if stdout is terminal, and not json flag).")
streamFlag = flagSet.Bool("stream", false, "Consume results as stream. Streaming search only supports a subset of flags and parameters: trace, insecure-skip-verify, display.")
display = flagSet.Int("display", -1, "Limit the number of results that are displayed. Only supported together with stream flag. Statistics continue to report all results.")
)
handler := func(args []string) error {
......@@ -61,6 +64,15 @@ Other tips:
return err
}
if *streamFlag {
opts := streaming.Opts{
Display: *display,
Trace: apiFlags.Trace(),
}
client := cfg.apiClient(apiFlags, flagSet.Output())
return streamSearch(flagSet.Arg(0), opts, client, os.Stdout)
}
if *explainJSONFlag {
fmt.Println(searchJSONExplanation)
return nil
......
......@@ -19,15 +19,18 @@ func init() {
}
}
// ProposedQuery is a suggested query to run when we emit an alert.
type ProposedQuery struct {
Description string
Query string
}
// searchResultsAlert is a type that can be used to unmarshal values returned by
// the searchResultsAlertFragment GraphQL fragment below.
type searchResultsAlert struct {
Title string
Description string
ProposedQueries []struct {
Description string
Query string
}
ProposedQueries []ProposedQuery
}
// Render renders an alert to a string ready to be output to a console,
......
......@@ -13,10 +13,7 @@ func TestRender(t *testing.T) {
full := &searchResultsAlert{
Title: "foo",
Description: "bar",
ProposedQueries: []struct {
Description string
Query string
}{
ProposedQueries: []ProposedQuery{
{
Description: "quux",
Query: "xyz:abc",
......
This diff is collapsed.
package main
import (
"flag"
"io/ioutil"
"net"
"net/http"
"net/http/httptest"
"os"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/sourcegraph/src-cli/internal/api"
"github.com/sourcegraph/src-cli/internal/streaming"
)
var event []streaming.EventMatch
func mockStreamHandler(w http.ResponseWriter, _ *http.Request) {
writer, _ := streaming.NewWriter(w)
writer.Event("matches", event)
writer.Event("done", nil)
}
func testServer(t *testing.T, handler http.Handler) *httptest.Server {
t.Helper()
// We need a stable port, because src-cli output contains references to the host.
// Here we exchange the standard listener with our own.
l, err := net.Listen("tcp", "127.0.0.1:55128")
if err != nil {
t.Fatal(err)
}
s := httptest.NewUnstartedServer(handler)
s.Listener.Close()
s.Listener = l
s.Start()
return s
}
func TestSearchStream(t *testing.T) {
s := testServer(t, http.HandlerFunc(mockStreamHandler))
defer s.Close()
cfg = &config{
Endpoint: s.URL,
}
defer func() { cfg = nil }()
event = []streaming.EventMatch{
&streaming.EventFileMatch{
Type: streaming.FileMatchType,
Path: "path/to/file",
Repository: "org/repo",
Branches: nil,
Version: "",
LineMatches: []streaming.EventLineMatch{
{
Line: "foo bar",
LineNumber: 4,
OffsetAndLengths: [][2]int32{{4, 3}},
},
},
},
&streaming.EventRepoMatch{
Type: streaming.RepoMatchType,
Repository: "sourcegraph/sourcegraph",
Branches: []string{},
},
&streaming.EventSymbolMatch{
Type: streaming.SymbolMatchType,
Path: "path/to/file",
Repository: "org/repo",
Branches: []string{},
Version: "",
Symbols: []streaming.Symbol{
{
URL: "github.com/sourcegraph/sourcegraph/-/blob/cmd/frontend/graphqlbackend/search_results.go#L1591:26-1591:35",
Name: "doResults",
ContainerName: "",
Kind: "FUNCTION",
},
{
URL: "github.com/sourcegraph/sourcegraph/-/blob/cmd/frontend/graphqlbackend/search_results.go#L1591:26-1591:35",
Name: "Results",
ContainerName: "SearchResultsResolver",
Kind: "FIELD",
},
},
},
&streaming.EventCommitMatch{
Type: streaming.CommitMatchType,
Icon: "",
Label: "[sourcegraph/sourcegraph-atom](/github.com/sourcegraph/sourcegraph-atom) › [Stephen Gutekanst](/github.com/sourcegraph/sourcegraph-atom/-/commit/5b098d7fed963d88e23057ed99d73d3c7a33ad89): [all: release v1.0.5](/github.com/sourcegraph/sourcegraph-atom/-/commit/5b098d7fed963d88e23057ed99d73d3c7a33ad89)^",
URL: "",
Detail: "",
Content: "```COMMIT_EDITMSG\nfoo bar\n```",
Ranges: [][3]int32{
{1, 3, 3},
},
},
&streaming.EventCommitMatch{
Type: streaming.CommitMatchType,
Icon: "",
Label: "[sourcegraph/sourcegraph-atom](/github.com/sourcegraph/sourcegraph-atom) › [Stephen Gutekanst](/github.com/sourcegraph/sourcegraph-atom/-/commit/5b098d7fed963d88e23057ed99d73d3c7a33ad89): [all: release v1.0.5](/github.com/sourcegraph/sourcegraph-atom/-/commit/5b098d7fed963d88e23057ed99d73d3c7a33ad89)^",
URL: "",
Detail: "",
Content: "```diff\nsrc/data.ts src/data.ts\n@@ -0,0 +11,4 @@\n+ return of<Data>({\n+ title: 'Acme Corp open-source code search',\n+ summary: 'Instant code search across all Acme Corp open-source code.',\n+ githubOrgs: ['sourcegraph'],\n```",
Ranges: [][3]int32{
{4, 44, 6},
},
},
}
// Capture output.
r, w, err := os.Pipe()
if err != nil {
t.Fatal(err)
}
flagSet := flag.NewFlagSet("test", flag.ExitOnError)
flags := api.NewFlags(flagSet)
client := cfg.apiClient(flags, flagSet.Output())
err = streamSearch("", streaming.Opts{}, client, w)
if err != nil {
t.Fatal(err)
}
err = w.Close()
if err != nil {
t.Fatal(err)
}
got, err := ioutil.ReadAll(r)
if err != nil {
t.Fatal(err)
}
want, err := ioutil.ReadFile("./testdata/streaming_search_want.txt")
if err != nil {
t.Fatal(err)
}
if d := cmp.Diff(want, got); d != "" {
t.Fatalf("(-want +got): %s", d)
}
}
org/repo › path/to/file (1 match)
--------------------------------------------------------------------------------
  5 | foo bar
sourcegraph/sourcegraph (http://127.0.0.1:55128/sourcegraph/sourcegraph) (1 match)
org/repo › path/to/file (2 matches)
--------------------------------------------------------------------------------
doResults(FUNCTION) (http://127.0.0.1:55128/github.com/sourcegraph/sourcegraph/-/blob/cmd/frontend/graphqlbackend/search_results.go#L1591:26-1591:35)
Results(FIELD, SearchResultsResolver) (http://127.0.0.1:55128/github.com/sourcegraph/sourcegraph/-/blob/cmd/frontend/graphqlbackend/search_results.go#L1591:26-1591:35)

(http://127.0.0.1:55128)
sourcegraph/sourcegraph-atom > Stephen Gutekanst : all: release v1.0.5 (1 match)
--------------------------------------------------------------------------------
 oo bar
(http://127.0.0.1:55128)
sourcegraph/sourcegraph-atom > Stephen Gutekanst : all: release v1.0.5 (1 match)
--------------------------------------------------------------------------------
 src/data.ts src/data.ts
@@ -0,0 +11,4 @@
+ return of<Data>({
+ title: 'Acme Corp open-source code search',
+ summary: 'Instant code search across all Acme Corp open-source code.',
+ githubOrgs: ['sourcegraph'],
......@@ -11,6 +11,13 @@ type Flags struct {
insecureSkipVerify *bool
}
func (f *Flags) Trace() bool {
if f.trace == nil {
return false
}
return *(f.trace)
}
// NewFlags instantiates a new Flags structure and attaches flags to the given
// flag set.
func NewFlags(flagSet *flag.FlagSet) *Flags {
......
package streaming
import (
"context"
"fmt"
"net/url"
"os"
"strconv"
"github.com/sourcegraph/src-cli/internal/api"
)
// Opts contains the search options supported by Search.
type Opts struct {
Display int
Trace bool
}
// Search calls the streaming search endpoint and uses decoder to decode the
// response body.
func Search(query string, opts Opts, client api.Client, decoder Decoder) error {
// Create request.
req, err := client.NewHTTPRequest(context.Background(), "GET", "search/stream?q="+url.QueryEscape(query), nil)
if err != nil {
return err
}
req.Header.Set("Accept", "text/event-stream")
if opts.Display >= 0 {
q := req.URL.Query()
q.Add("display", strconv.Itoa(opts.Display))
req.URL.RawQuery = q.Encode()
}
// Send request.
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("error sending request: %w", err)
}
defer resp.Body.Close()
// Process response.
err = decoder.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("error during decoding: %w", err)
}
// Output trace.
if opts.Trace {
_, err = fmt.Fprintf(os.Stderr, fmt.Sprintf("\nx-trace: %s\n", resp.Header.Get("x-trace")))
if err != nil {
return err
}
}
return nil
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment