feat: Introduce more flags to filter HTTP records (#220)

* feat: introduce path-regex and path-prefix to sub cmd http

Signed-off-by: spencercjh <spencercjh@gmail.com>

* style: reformat with goimports

Signed-off-by: spencercjh <spencercjh@gmail.com>

* fix: save FilterByRequest's result as HttpFilter's field

Signed-off-by: spencercjh <spencercjh@gmail.com>

* docs: update docs about HttpFilter

Signed-off-by: spencercjh <spencercjh@gmail.com>

---------

Signed-off-by: spencercjh <spencercjh@gmail.com>

Makefile

@@ -101,6 +101,7 @@ format: format-go
 
 .PHONY: format-go
 format-go:
+	goimports -w .
 	gofmt -s -w .
 
 .PHONY: dlv

agent/protocol/http.go

@@ -8,8 +8,11 @@ import (
 	"kyanos/bpf"
 	"kyanos/common"
 	"net/http"
+	"regexp"
 	"slices"
 	"strings"
+
+	"k8s.io/utils/ptr"
 )
 
 var _ ProtocolStreamParser = &HTTPStreamParser{}
@@ -233,9 +236,12 @@ func (resp *ParsedHttpResponse) IsReq() bool {
 var _ ProtocolFilter = HttpFilter{}
 
 type HttpFilter struct {
-	TargetPath     string
+	TargetPath       string
-	TargetHostName string
+	TargetPathReg    *regexp.Regexp
-	TargetMethods  []string
+	TargetPathPrefix string
+	TargetHostName   string
+	TargetMethods    []string
+	needFilter       *bool
 }
 
 func (filter HttpFilter) FilterByProtocol(protocol bpf.AgentTrafficProtocolT) bool {
@@ -243,28 +249,57 @@ func (filter HttpFilter) FilterByProtocol(protocol bpf.AgentTrafficProtocolT) bo
 }
 
 func (filter HttpFilter) FilterByRequest() bool {
-	return filter.TargetPath != "" || len(filter.TargetMethods) > 0 || filter.TargetHostName != ""
+	if filter.needFilter != nil {
+		return *filter.needFilter
+	}
+	filter.needFilter = ptr.To(len(filter.TargetPath) > 0 ||
+		filter.TargetPathReg != nil ||
+		len(filter.TargetPathPrefix) > 0 ||
+		len(filter.TargetMethods) > 0 ||
+		len(filter.TargetHostName) > 0)
+	return *filter.needFilter
 }
 
 func (filter HttpFilter) FilterByResponse() bool {
 	return false
 }
 
-func (filter HttpFilter) Filter(parsedReq ParsedMessage, parsedResp ParsedMessage) bool {
+// Filter filters HTTP requests based on various criteria such as path, path prefix, path regex, method, and host name.
+// It returns true if the request matches all the specified criteria, otherwise it returns false.
+//
+// The filtering logic is as follows:
+// - If TargetPath is specified, the request path must exactly match TargetPath.
+// - If TargetPathPrefix is specified, the request path must start with TargetPathPrefix.
+// - If TargetPathReg is specified, the request path must match the regular expression TargetPathReg.
+// - If TargetMethods is specified, the request method must be one of the methods in TargetMethods.
+// - If TargetHostName is specified, the request host must exactly match TargetHostName.
+func (filter HttpFilter) Filter(parsedReq ParsedMessage, _ ParsedMessage) bool {
 	req, ok := parsedReq.(*ParsedHttpRequest)
 	if !ok {
 		common.ProtocolParserLog.Warnf("[HttpFilter] cast to http.Request failed: %v\n", req)
 		return false
 	}
+	common.ProtocolParserLog.Debugf("[HttpFilter] filtering request: %v\n", req)
 
-	if filter.TargetPath != "" && filter.TargetPath != req.Path {
+	if len(filter.TargetPath) > 0 && filter.TargetPath != req.Path {
 		return false
 	}
+
+	if len(filter.TargetPathPrefix) > 0 && !strings.HasPrefix(req.Path, filter.TargetPathPrefix) {
+		return false
+	}
+
+	if filter.TargetPathReg != nil && !filter.TargetPathReg.MatchString(req.Path) {
+		return false
+	}
+
 	if len(filter.TargetMethods) > 0 && !slices.Contains(filter.TargetMethods, req.Method) {
 		return false
 	}
+
 	if filter.TargetHostName != "" && filter.TargetHostName != req.Host {
 		return false
 	}
+
 	return true
 }

agent/protocol/http_test.go

@@ -9,6 +9,7 @@ import (
 	"kyanos/common"
 	"net/http"
 	"reflect"
+	"regexp"
 	"strings"
 	"testing"
 
@@ -162,3 +163,174 @@ func TestParseResponse(t *testing.T) {
 	assert.Equal(t, uint64(10), message.TimestampNs())
 	assert.Equal(t, uint64(20), message.Seq())
 }
+
+func TestHttpFilter_Filter(t *testing.T) {
+	type fields struct {
+		TargetPath       string
+		TargetPathReg    *regexp.Regexp
+		TargetPathPrefix string
+		TargetHostName   string
+		TargetMethods    []string
+	}
+	type args struct {
+		parsedReq protocol.ParsedMessage
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		args   args
+		want   bool
+	}{
+		{
+			name: "not_http_req",
+			args: args{
+				parsedReq: &protocol.RedisMessage{},
+			},
+			want: false,
+		},
+		{
+			name: "filter_by_path",
+			fields: fields{
+				TargetPath: "/foo/bar",
+			},
+			args: args{
+				parsedReq: &protocol.ParsedHttpRequest{
+					Path: "/foo/bar",
+				},
+			},
+			want: true,
+		},
+		{
+			name: "not_filter_by_path",
+			fields: fields{
+				TargetPath: "/foo/bar",
+			},
+			args: args{
+				parsedReq: &protocol.ParsedHttpRequest{
+					Path: "/foo/bar/baz",
+				},
+			},
+			want: false,
+		},
+		{
+			name: "filter_by_path_prefix",
+			fields: fields{
+				TargetPathPrefix: "/foo/bar",
+			},
+			args: args{
+				parsedReq: &protocol.ParsedHttpRequest{
+					Path: "/foo/bar/baz",
+				},
+			},
+			want: true,
+		},
+		{
+			name: "not_filter_by_path_prefix",
+			fields: fields{
+				TargetPathPrefix: "/foo/bar",
+			},
+			args: args{
+				parsedReq: &protocol.ParsedHttpRequest{
+					Path: "/test",
+				},
+			},
+			want: false,
+		},
+		{
+			name: "filter_by_regex",
+			fields: fields{
+				TargetPathReg: regexp.MustCompile("/foo/bar/\\d+/baz"),
+			},
+			args: args{
+				parsedReq: &protocol.ParsedHttpRequest{
+					Path: "/foo/bar/100/baz",
+				},
+			},
+			want: true,
+		},
+		{
+			name: "not_filter_by_regex",
+			fields: fields{
+				TargetPathReg: regexp.MustCompile("/foo/bar/\\w+"),
+			},
+			args: args{
+				parsedReq: &protocol.ParsedHttpRequest{
+					Path: "/test",
+				},
+			},
+			want: false,
+		},
+		{
+			name:   "no_filter",
+			fields: fields{},
+			args: args{
+				parsedReq: &protocol.ParsedHttpRequest{
+					Path:   "/test",
+					Host:   "test.com",
+					Method: "POST",
+				},
+			},
+			want: true,
+		},
+		{
+			name: "filter_by_method",
+			fields: fields{
+				TargetMethods: []string{"GET"},
+			},
+			args: args{
+				parsedReq: &protocol.ParsedHttpRequest{
+					Method: "GET",
+				},
+			},
+			want: true,
+		},
+		{
+			name: "not_filter_by_method",
+			fields: fields{
+				TargetMethods: []string{"GET"},
+			},
+			args: args{
+				parsedReq: &protocol.ParsedHttpRequest{
+					Method: "POST",
+				},
+			},
+			want: false,
+		},
+		{
+			name: "filter_by_host",
+			fields: fields{
+				TargetHostName: "foo.bar",
+			},
+			args: args{
+				parsedReq: &protocol.ParsedHttpRequest{
+					Host: "foo.bar",
+				},
+			},
+			want: true,
+		},
+		{
+			name: "not_filter_by_host",
+			fields: fields{
+				TargetHostName: "foo.bar",
+			},
+			args: args{
+				parsedReq: &protocol.ParsedHttpRequest{
+					Host: "foo.baz",
+				},
+			},
+			want: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			filter := protocol.HttpFilter{
+				TargetPath:       tt.fields.TargetPath,
+				TargetPathReg:    tt.fields.TargetPathReg,
+				TargetPathPrefix: tt.fields.TargetPathPrefix,
+				TargetHostName:   tt.fields.TargetHostName,
+				TargetMethods:    tt.fields.TargetMethods,
+			}
+			assert.Equalf(t, tt.want, filter.Filter(tt.args.parsedReq, nil), "Filter(%v, %v)", tt.args.parsedReq, nil)
+		})
+	}
+}

bpf/loader/utils.go

@@ -4,10 +4,11 @@ import (
 	"archive/tar"
 	"bytes"
 	"fmt"
-	"github.com/smira/go-xz"
 	"io"
 	"os"
 	"strings"
+
+	"github.com/smira/go-xz"
 )
 
 func saveDataToFile(data []byte, targetPath string) error {

cmd/http.go

@@ -2,13 +2,15 @@ package cmd
 
 import (
 	"kyanos/agent/protocol"
+	"regexp"
 
 	"github.com/spf13/cobra"
 )
 
-var httpCmd *cobra.Command = &cobra.Command{
+var httpCmd = &cobra.Command{
-	Use:   "http [--method METHODS|--path PATH|--host HOSTNAME]",
+	Use:   "http [--method METHODS|--path PATH|--path-regex REGEX|--path-prefix PREFIX|--host HOSTNAME]",
 	Short: "watch HTTP message",
+	Long:  `Filter HTTP messages based on method, path (strict, regex, prefix), or host. Filter flags are combined with AND(&&).`,
 	Run: func(cmd *cobra.Command, args []string) {
 		methods, err := cmd.Flags().GetStringSlice("method")
 		if err != nil {
@@ -22,10 +24,27 @@ var httpCmd *cobra.Command = &cobra.Command{
 		if err != nil {
 			logger.Fatalf("invalid host: %v\n", err)
 		}
+		var (
+			pathReg *regexp.Regexp
+		)
+		if pathRegStr, err := cmd.Flags().GetString("path-regex"); err != nil {
+			logger.Fatalf("invalid path-regex: %v\n", err)
+		} else if len(pathRegStr) > 0 {
+			if pathReg, err = regexp.Compile(pathRegStr); err != nil {
+				logger.Fatalf("invalid path-regex: %v\n", err)
+			}
+		}
+		pathPrefix, err := cmd.Flags().GetString("path-prefix")
+		if err != nil {
+			logger.Fatalf("invalid path-prefix: %v\n", err)
+		}
+
 		options.MessageFilter = protocol.HttpFilter{
-			TargetPath:     path,
+			TargetPath:       path,
-			TargetMethods:  methods,
+			TargetPathReg:    pathReg,
-			TargetHostName: host,
+			TargetPathPrefix: pathPrefix,
+			TargetHostName:   host,
+			TargetMethods:    methods,
 		}
 		options.LatencyFilter = initLatencyFilter(cmd)
 		options.SizeFilter = initSizeFilter(cmd)
@@ -37,6 +56,9 @@ func init() {
 	httpCmd.Flags().StringSlice("method", []string{}, "Specify the HTTP method to monitor(GET, POST), seperate by ','")
 	httpCmd.Flags().String("host", "", "Specify the HTTP host to monitor, like: 'ubuntu.com'")
 	httpCmd.Flags().String("path", "", "Specify the HTTP path to monitor, like: '/foo/bar'")
+	httpCmd.Flags().String("path-regex", "", "Specify the regex for HTTP path to monitor, like: '\\/foo\\/bar\\/\\d+'")
+	httpCmd.Flags().String("path-prefix", "", "Specify the prefix of HTTP path to monitor, like: '/foo'")
+
 	httpCmd.Flags().SortFlags = false
 	httpCmd.PersistentFlags().SortFlags = false
 	copy := *httpCmd

docs/cn/watch.md

@@ -100,12 +100,13 @@ kyanos 支持根据 IP 端口等三/四层信息过滤，可以指定以下选
 
 #### HTTP协议过滤
 
-| 过滤条件   | 命令行flag  | 示例                                               |
+| 过滤条件     | 命令行flag       | 示例                                                             |
-| :----- | :------- | :----------------------------------------------- |
+|:---------|:--------------|:---------------------------------------------------------------|
-| 请求Path | `path`   | `--path /foo/bar ` 只观察请求path为/foo/bar            |
+| 请求Path   | `path`        | `--path /foo/bar ` 只观察path为/foo/bar的请求                         |
-| 请求Host | `host`  | `--host www.baidu.com ` 只观察请求Host为www\.baidu.com |
+| 请求Path前缀 | `path-prefix` | `--path-prefix /foo/bar ` 只观察path前缀为/foo/bar的请求                |
-| 请求方法   | `method` | `--method GET` 只观察请求为GET                         |
+| 请求Path正则 | `path-regex`  | `--path-regex "\/foo\/bar\/.*" ` 只观察path匹配 `\/foo\/bar\/.*`的请求 |
-
+| 请求Host   | `host`        | `--host www.baidu.com ` 只观察Host为www.baidu.com的请求               |
+| 请求方法     | `method`      | `--method GET` 只观察方法为GET                                       |
 
 #### Redis协议过滤
 

docs/watch.md

@@ -105,11 +105,13 @@ Here are the options available for filtering by each protocol:
 
 #### HTTP Protocol Filtering
 
-| Filter Condition | Command Line Flag | Example                                                |
+| Filter Condition    | Command Line Flag | Example                                                                                                    |
-|------------------|-------------------|--------------------------------------------------------|
+|---------------------|-------------------|------------------------------------------------------------------------------------------------------------|
-| Request Path     | `path`            | `--path /foo/bar` <br> Only observe requests with the path `/foo/bar`. |
+| Request Path        | `path`            | `--path /foo/bar` <br> Only observe requests with the path `/foo/bar`.                                     |
-| Request Host     | `host`            | `--host www.baidu.com` <br> Only observe requests with the host `www.baidu.com`. |
+| Request Path Prefix | `path-prefix`     | `--path-prefix /foo/bar` <br> Only observe requests with paths started with `/foo/bar`.                    |
-| Request Method    | `method`          | `--method GET` <br> Only observe requests with the method `GET`. |
+| Request Path Regex  | `path-regex`      | `--path-regex "\/foo\/bar\/.*"` <br> Only observe requests with paths matching the regex `\/foo\/bar\/.*`. |
+| Request Host        | `host`            | `--host www.baidu.com` <br> Only observe requests with the host `www.baidu.com`.                           |
+| Request Method      | `method`          | `--method GET` <br> Only observe requests with the method `GET`.                                           |
 
 #### Redis Protocol Filtering