Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package fortio for openSUSE:Factory checked in at 2023-07-04 15:22:36 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/fortio (Old) and /work/SRC/openSUSE:Factory/.fortio.new.23466 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "fortio" Tue Jul 4 15:22:36 2023 rev:42 rq:1096659 version:1.56.0 Changes: -------- --- /work/SRC/openSUSE:Factory/fortio/fortio.changes 2023-06-27 23:16:58.951503140 +0200 +++ /work/SRC/openSUSE:Factory/.fortio.new.23466/fortio.changes 2023-07-04 15:23:32.230622901 +0200 @@ -1,0 +2,10 @@ +Tue Jul 04 04:35:40 UTC 2023 - ka...@b1-systems.de + +- Update to version 1.56.0: + * do abort early even with stdclient on bad urls/hosts. allow + most same query args on fetchers. (#785) + * add -X to override method. treat content-type in header as + ContentType on options (#783) + * Bump docker/setup-buildx-action from 2.7.0 to 2.8.0 (#786) + +------------------------------------------------------------------- Old: ---- fortio-1.55.2.obscpio New: ---- fortio-1.56.0.obscpio ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ fortio.spec ++++++ --- /var/tmp/diff_new_pack.DtFrHt/_old 2023-07-04 15:23:33.062627900 +0200 +++ /var/tmp/diff_new_pack.DtFrHt/_new 2023-07-04 15:23:33.066627925 +0200 @@ -19,7 +19,7 @@ %define __arch_install_post export NO_BRP_STRIP_DEBUG=true Name: fortio -Version: 1.55.2 +Version: 1.56.0 Release: 0 Summary: Load testing library, command line tool, advanced echo server and web UI License: Apache-2.0 ++++++ _service ++++++ --- /var/tmp/diff_new_pack.DtFrHt/_old 2023-07-04 15:23:33.098628117 +0200 +++ /var/tmp/diff_new_pack.DtFrHt/_new 2023-07-04 15:23:33.102628141 +0200 @@ -3,7 +3,7 @@ <param name="url">https://github.com/fortio/fortio</param> <param name="scm">git</param> <param name="exclude">.git</param> - <param name="revision">v1.55.2</param> + <param name="revision">v1.56.0</param> <param name="versionformat">@PARENT_TAG@</param> <param name="changesgenerate">enable</param> <param name="versionrewrite-pattern">v(.*)</param> ++++++ _servicedata ++++++ --- /var/tmp/diff_new_pack.DtFrHt/_old 2023-07-04 15:23:33.122628262 +0200 +++ /var/tmp/diff_new_pack.DtFrHt/_new 2023-07-04 15:23:33.126628285 +0200 @@ -1,6 +1,6 @@ <servicedata> <service name="tar_scm"> <param name="url">https://github.com/fortio/fortio</param> - <param name="changesrevision">c39b74f7f44b365dab0e7c753ab21691bb7c1861</param></service></servicedata> + <param name="changesrevision">8890d97fd5aac4ccf571cc9a40331d3ab2bd145b</param></service></servicedata> (No newline at EOF) ++++++ fortio-1.55.2.obscpio -> fortio-1.56.0.obscpio ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fortio-1.55.2/.github/workflows/main.yml new/fortio-1.56.0/.github/workflows/main.yml --- old/fortio-1.55.2/.github/workflows/main.yml 2023-06-27 03:23:18.000000000 +0200 +++ new/fortio-1.56.0/.github/workflows/main.yml 2023-07-04 03:07:15.000000000 +0200 @@ -29,7 +29,7 @@ - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@ecf95283f03858871ff00b787d79c419715afc34 # pin@v2 + uses: docker/setup-buildx-action@16c0bc4a6e6ada2cfd8afd41d22d95379cf7c32a # pin@v2 - name: Available platforms run: echo ${{ steps.buildx.outputs.platforms }} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fortio-1.55.2/.github/workflows/manual-build.yml new/fortio-1.56.0/.github/workflows/manual-build.yml --- old/fortio-1.55.2/.github/workflows/manual-build.yml 2023-06-27 03:23:18.000000000 +0200 +++ new/fortio-1.56.0/.github/workflows/manual-build.yml 2023-07-04 03:07:15.000000000 +0200 @@ -30,7 +30,7 @@ - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@ecf95283f03858871ff00b787d79c419715afc34 # pin@v2 + uses: docker/setup-buildx-action@16c0bc4a6e6ada2cfd8afd41d22d95379cf7c32a # pin@v2 - name: Build id: build diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fortio-1.55.2/Makefile new/fortio-1.56.0/Makefile --- old/fortio-1.55.2/Makefile 2023-06-27 03:23:18.000000000 +0200 +++ new/fortio-1.56.0/Makefile 2023-07-04 03:07:15.000000000 +0200 @@ -120,7 +120,7 @@ $(SED) --in-place=.bak -E -e 's!$(DOCKER_PREFIX).build:v[^ ]+!$(BUILD_IMAGE)!g' $(FILES_WITH_IMAGE) docker-default-platform: - @docker buildx --builder default inspect | tail -1 | sed -e "s/Platforms: //" -e "s/,//g" | awk '{print $$1}' + @docker buildx --builder default inspect | awk '/Platforms:/ {print $$2}' | sed -e 's/,//g' docker-version: @echo "### Docker is `which docker`" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fortio-1.55.2/README.md new/fortio-1.56.0/README.md --- old/fortio-1.55.2/README.md 2023-06-27 03:23:18.000000000 +0200 +++ new/fortio-1.56.0/README.md 2023-07-04 03:07:15.000000000 +0200 @@ -1,4 +1,4 @@ -<!-- 1.55.2 --> +<!-- 1.56.0 --> # Fortio [![Awesome Go](https://fortio.org/mentioned-badge.svg)](https://github.com/avelino/awesome-go#networking) @@ -60,13 +60,13 @@ The [releases](https://github.com/fortio/fortio/releases) page has binaries for many OS/architecture combinations (see assets): ```shell -curl -L https://github.com/fortio/fortio/releases/download/v1.55.2/fortio-linux_amd64-1.55.2.tgz \ +curl -L https://github.com/fortio/fortio/releases/download/v1.56.0/fortio-linux_amd64-1.56.0.tgz \ | sudo tar -C / -xvzpf - # or the debian package -wget https://github.com/fortio/fortio/releases/download/v1.55.2/fortio_1.55.2_amd64.deb -dpkg -i fortio_1.55.2_amd64.deb +wget https://github.com/fortio/fortio/releases/download/v1.56.0/fortio_1.56.0_amd64.deb +dpkg -i fortio_1.56.0_amd64.deb # or the rpm -rpm -i https://github.com/fortio/fortio/releases/download/v1.55.2/fortio-1.55.2-1.x86_64.rpm +rpm -i https://github.com/fortio/fortio/releases/download/v1.56.0/fortio-1.56.0-1.x86_64.rpm # and more, see assets in release page ``` @@ -76,7 +76,7 @@ brew install fortio ``` -On Windows, download https://github.com/fortio/fortio/releases/download/v1.55.2/fortio_win_1.55.2.zip and extract `fortio.exe` to any location, then using the Windows Command Prompt: +On Windows, download https://github.com/fortio/fortio/releases/download/v1.56.0/fortio_win_1.56.0.zip and extract `fortio.exe` to any location, then using the Windows Command Prompt: ``` fortio.exe server ``` @@ -127,7 +127,7 @@ <!-- use release/updateFlags.sh to update this section --> <pre> <!-- USAGE_START --> -ΦοÏÏίο 1.55.2 usage: +ΦοÏÏίο 1.56.0 usage: fortio command [flags] target where command is one of: load (load testing), server (starts ui, rest api, http-echo, redirect, proxies, tcp-echo, udp-echo and grpc ping servers), @@ -150,6 +150,8 @@ -P value Tcp proxies to run, e.g -P "localport1 dest_host1:dest_port1" -P "[::1]:0 www.google.com:443" ... + -X string + HTTP method to use instead of GET/POST depending on payload/content-type -a Automatically save JSON result with filename based on labels & timestamp -abort-on code Http code that if encountered aborts the run. e.g. 503 or -1 for socket errors. @@ -248,6 +250,9 @@ URL and hostname -log-errors Log http non 2xx/418 error codes as they occur (default true) + -logger-file-line + Filename and line numbers emitted in JSON logs, use -logger-file-line=false to +disable (default true) -logger-json Log in JSON format, use -logger-json=false to disable (default true) -logger-timestamp diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fortio-1.55.2/Webtest.sh new/fortio-1.56.0/Webtest.sh --- old/fortio-1.55.2/Webtest.sh 2023-06-27 03:23:18.000000000 +0200 +++ new/fortio-1.56.0/Webtest.sh 2023-07-04 03:07:15.000000000 +0200 @@ -14,7 +14,7 @@ # limitations under the License. set -x # Check we can build the image -NATIVE_PLATFORM=$(docker buildx --builder default inspect | tail -1 | sed -e "s/Platforms: //" -e "s/,//g" | awk '{print $1}') +NATIVE_PLATFORM=$(docker buildx --builder default inspect | awk '/Platforms:/ {print $2}' | sed -e 's/,//g') echo "Building for $NATIVE_PLATFORM" make docker-internal TAG=webtest BUILDX_PLATFORMS="$NATIVE_PLATFORM" MODE=dev || exit 1 FORTIO_UI_PREFIX=/newprefix/ # test the non default prefix (not /fortio/) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fortio-1.55.2/bincommon/commonflags.go new/fortio-1.56.0/bincommon/commonflags.go --- old/fortio-1.55.2/bincommon/commonflags.go 2023-06-27 03:23:18.000000000 +0200 +++ new/fortio-1.56.0/bincommon/commonflags.go 2023-07-04 03:07:15.000000000 +0200 @@ -106,6 +106,7 @@ // NoReResolveFlag is false if we want to resolve the DNS name for each new connection. NoReResolveFlag = flag.Bool("no-reresolve", false, "Keep the initial DNS resolution and "+ "don't re-resolve when making new connections (because of error or reuse limit reached)") + MethodFlag = flag.String("X", "", "HTTP method to use instead of GET/POST depending on payload/content-type") ) // SharedMain is the common part of main from fortio_main and fcurl. @@ -207,7 +208,10 @@ httpOpts.Insecure = TLSInsecure() httpOpts.Resolve = *resolve httpOpts.UserCredentials = *userCredentialsFlag - httpOpts.ContentType = *contentTypeFlag + if len(*contentTypeFlag) > 0 { + // only set content-type from flag if flag isn't empty as it can come also from -H content-type:... + httpOpts.ContentType = *contentTypeFlag + } if *PayloadStreamFlag { httpOpts.PayloadReader = os.Stdin } else { @@ -230,5 +234,6 @@ httpOpts.LogErrors = *LogErrorsFlag httpOpts.SequentialWarmup = *warmupFlag httpOpts.NoResolveEachConn = *NoReResolveFlag + httpOpts.MethodOverride = *MethodFlag return &httpOpts } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fortio-1.55.2/cli_test.go new/fortio-1.56.0/cli_test.go --- old/fortio-1.55.2/cli_test.go 1970-01-01 01:00:00.000000000 +0100 +++ new/fortio-1.56.0/cli_test.go 2023-07-04 03:07:15.000000000 +0200 @@ -0,0 +1,19 @@ +package main + +import ( + "os" + "testing" + + "fortio.org/testscript" +) + +func TestMain(m *testing.M) { + // Runs the cli_test.txtar (https://github.com/fortio/testscript#testscript) tests. + os.Exit(testscript.RunMain(m, map[string]func() int{ + "fortio": Main, + })) +} + +func TestDNSPing(t *testing.T) { + testscript.Run(t, testscript.Params{Dir: "./"}) +} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fortio-1.55.2/cli_test.txtar new/fortio-1.56.0/cli_test.txtar --- old/fortio-1.55.2/cli_test.txtar 1970-01-01 01:00:00.000000000 +0100 +++ new/fortio-1.56.0/cli_test.txtar 2023-07-04 03:07:15.000000000 +0200 @@ -0,0 +1,25 @@ +# testscript framework tests for fortio's main binary / command line +# Eventually we can convert most of Webtest.sh to here (except parts specifically about testing the Docker image) + +# Basic usage test +!fortio +!stdout . +stderr 'Missing command argument' + +# (short) version +fortio version +stdout '^dev$' +!stderr . + +# (long) version +fortio buildinfo +stdout '^dev go' +stdout 'path fortio.org/fortio' +!stderr . + +# Content-Type override through headers +fortio curl -H 'content-TYPE: foo/bar' -H 'xyz: bar blah' https://debug.fortio.org/test-path +stderr 'HTTP/1.1 200 OK' +stdout 'Xyz: bar blah' +stdout 'Content-Type: foo/bar' +stdout 'POST /test-path HTTP/1.1' diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fortio-1.55.2/fcurl/fcurl.go new/fortio-1.56.0/fcurl/fcurl.go --- old/fortio-1.55.2/fcurl/fcurl.go 2023-06-27 03:23:18.000000000 +0200 +++ new/fortio-1.56.0/fcurl/fcurl.go 2023-07-04 03:07:15.000000000 +0200 @@ -17,16 +17,25 @@ // Do not add any external dependencies we want to keep fortio minimal. import ( + "os" + "fortio.org/cli" "fortio.org/fortio/bincommon" + "fortio.org/log" ) -func main() { +func Main() int { cli.ProgramName = "ΦοÏÏίο fortio-curl" cli.ArgsHelp = "url" cli.MinArgs = 1 bincommon.SharedMain() cli.Main() o := bincommon.SharedHTTPOptions() + log.Debugf("Running curl with %+v", o) bincommon.FetchURL(o) + return 0 +} + +func main() { + os.Exit(Main()) } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fortio-1.55.2/fhttp/http_client.go new/fortio-1.56.0/fhttp/http_client.go --- old/fortio-1.55.2/fhttp/http_client.go 2023-06-27 03:23:18.000000000 +0200 +++ new/fortio-1.56.0/fhttp/http_client.go 2023-07-04 03:07:15.000000000 +0200 @@ -201,6 +201,7 @@ UserCredentials string // user credentials for authorization ContentType string // indicates request body type, implies POST instead of GET Payload []byte // body for http request, implies POST if not empty. + MethodOverride string // optional http method override. Otherwise GET or POST when a payload or ContentType is set. LogErrors bool // whether to log non 2xx code as they occur or not ID int `json:"-"` // thread/connect id to use for logging (thread id when used as a runner) UniqueID int64 `json:"-"` // Run identifier when used through a runner, copied from RunnerOptions.RunID @@ -287,6 +288,9 @@ // Method returns the method of the http req. func (h *HTTPOptions) Method() string { + if len(h.MethodOverride) > 0 { + return h.MethodOverride + } if len(h.Payload) > 0 || h.ContentType != "" { return fnet.POST } @@ -308,18 +312,23 @@ // No TrimSpace for the value, so we can set empty "" vs just whitespace " " which // will get trimmed later but treated differently: not emitted vs emitted empty for User-Agent. value := s[1] + // 2 headers need trimmed to not have extra spaces: + trimmedValue := strings.TrimSpace(value) switch strings.ToLower(key) { case "host": - log.LogVf("Will be setting special Host header to %s", value) - h.hostOverride = strings.TrimSpace(value) // This one needs to be trimmed + log.LogVf("Will be setting special Host header to %s", trimmedValue) + h.hostOverride = trimmedValue // This one needs to be trimmed case "user-agent": if value == "" { log.Infof("Deleting default User-Agent: header.") h.extraHeaders.Del(key) } else { - log.Infof("User-Agent being Set to %q", value) + log.Infof("User-Agent being set to %q", value) h.extraHeaders.Set(key, value) } + case "content-type": + log.LogVf("Content-Type being set to %q", trimmedValue) + h.ContentType = trimmedValue default: log.LogVf("Setting regular extra header %s: %s", key, value) h.extraHeaders.Add(key, value) @@ -362,6 +371,7 @@ // newHttpRequest makes a new http GET request for url with User-Agent. func newHTTPRequest(o *HTTPOptions) (*http.Request, error) { method := o.Method() + log.Debugf("newHTTPRequest %s %s", method, o.URL) var body io.Reader if o.PayloadReader != nil { body = o.PayloadReader @@ -370,6 +380,20 @@ } //nolint:noctx // we pass context later in Run()/Fetch() req, err := http.NewRequest(method, o.URL, body) + if err == nil { //nolint:nestif + // Additional validation for the URL so we abort early on fatal errors even for the std client. + // fixes #784 + if req.URL == nil || req.URL.Host == "" { + err = fmt.Errorf("invalid url '%s'", o.URL) + } else { + host := req.URL.Hostname() + if o.Resolve != "" { + host = o.Resolve + } + log.Debugf("Std client extra validation - host part of url (or resolve option) is %q", host) + _, err = fnet.ResolveAll(context.Background(), host, "ip") + } + } if err != nil { log.S(log.Error, "Unable to make request", log.Attr("method", method), log.Attr("url", o.URL), log.Attr("err", err), @@ -753,6 +777,7 @@ // the beginning and then reused many times. func NewFastClient(o *HTTPOptions) (Fetcher, error) { //nolint:funlen method := o.Method() + log.Debugf("NewFastClient %s %s", method, o.URL) payloadLen := len(o.Payload) o.Init(o.URL) proto := "1.1" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fortio-1.55.2/fhttp/http_forwarder.go new/fortio-1.56.0/fhttp/http_forwarder.go --- old/fortio-1.55.2/fhttp/http_forwarder.go 2023-06-27 03:23:18.000000000 +0200 +++ new/fortio-1.56.0/fhttp/http_forwarder.go 2023-07-04 03:07:15.000000000 +0200 @@ -94,18 +94,37 @@ // MakeSimpleRequest makes a new request for url but copies trace headers from input request r. // or all the headers if copyAllHeaders is true. -func MakeSimpleRequest(url string, r *http.Request, copyAllHeaders bool) *http.Request { - req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, url, nil) +func MakeSimpleRequest(url string, r *http.Request, copyAllHeaders bool) (*http.Request, *HTTPOptions) { + opts := CommonHTTPOptionsFromForm(r) + var body io.Reader + if len(opts.Payload) > 0 { + body = bytes.NewReader(opts.Payload) + } + req, err := http.NewRequestWithContext(r.Context(), opts.Method(), url, body) if err != nil { log.Warnf("new request error for %q: %v", url, err) - return nil + return nil, opts } // Copy only trace headers or all of them: CopyHeaders(req, r, copyAllHeaders) - if !copyAllHeaders { + if copyAllHeaders { + // Add the headers from the form/query args "H" arguments: (only in trusted/copy all headers mode) + for k, v := range opts.extraHeaders { + for _, vv := range v { + req.Header.Add(k, vv) + } + log.Debugf("Header %q is now %v", k, req.Header[k]) + } + if opts.ContentType != "" { + req.Header.Set("Content-Type", opts.ContentType) + log.Debugf("Setting Content-Type to %q", opts.ContentType) + } + // force correct content length: + req.Header.Set("Content-Length", strconv.Itoa(len(opts.Payload))) + } else { req.Header.Set(jrpc.UserAgentHeader, jrpc.UserAgent) } - return req + return req, opts } // TeeHandler common part between TeeSerialHandler and TeeParallelHandler. @@ -132,7 +151,7 @@ if t.MirrorOrigin { req = makeMirrorRequest(t.Destination, r, data) } else { - req = MakeSimpleRequest(t.Destination, r, false) + req, _ = MakeSimpleRequest(t.Destination, r, false) } if req == nil { // error already logged diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fortio-1.55.2/fhttp/http_forwarder_test.go new/fortio-1.56.0/fhttp/http_forwarder_test.go --- old/fortio-1.55.2/fhttp/http_forwarder_test.go 2023-06-27 03:23:18.000000000 +0200 +++ new/fortio-1.56.0/fhttp/http_forwarder_test.go 2023-07-04 03:07:15.000000000 +0200 @@ -115,5 +115,3 @@ } } } - -// -- end of benchmark tests / end of this file diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fortio-1.55.2/fhttp/http_server.go new/fortio-1.56.0/fhttp/http_server.go --- old/fortio-1.55.2/fhttp/http_server.go 2023-06-27 03:23:18.000000000 +0200 +++ new/fortio-1.56.0/fhttp/http_server.go 2023-07-04 03:07:15.000000000 +0200 @@ -18,6 +18,7 @@ import ( "bytes" + "crypto/tls" "fmt" "io" "net" @@ -513,12 +514,16 @@ if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") { url = "http://" + url } - req := MakeSimpleRequest(url, r, Fetch2CopiesAllHeader.Get()) + req, opts := MakeSimpleRequest(url, r, Fetch2CopiesAllHeader.Get()) if req == nil { http.Error(w, "parsing url failed, invalid url", http.StatusBadRequest) return } OnBehalfOfRequest(req, r) + tr := proxyClient.Transport.(*http.Transport) + if tr.TLSClientConfig == nil || tr.TLSClientConfig.InsecureSkipVerify != opts.Insecure { + tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: opts.Insecure} //nolint:gosec // as requested by the options. + } resp, err := proxyClient.Do(req) if err != nil { msg := fmt.Sprintf("Error for %q: %v", url, err) @@ -559,8 +564,11 @@ defer conn.Close() // Stripped prefix gets replaced by ./ - sometimes... url := strings.TrimPrefix(r.URL.String(), "./") - opts := NewHTTPOptions("http://" + url) - opts.HTTPReqTimeOut = 5 * time.Minute + opts := CommonHTTPOptionsFromForm(r) + if opts.HTTPReqTimeOut == 0 { + opts.HTTPReqTimeOut = 1 * time.Minute + } + opts.Init(url) OnBehalfOf(opts, r) //nolint:contextcheck // TODO: yes we should plug an aborter in the http options that's based on this request's context. client, _ := NewClient(opts) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fortio-1.55.2/fhttp/http_test.go new/fortio-1.56.0/fhttp/http_test.go --- old/fortio-1.55.2/fhttp/http_test.go 2023-06-27 03:23:18.000000000 +0200 +++ new/fortio-1.56.0/fhttp/http_test.go 2023-07-04 03:07:15.000000000 +0200 @@ -720,15 +720,20 @@ opts := NewHTTPOptions("not a valid url") cli, err := NewStdClient(opts) if cli != nil || err == nil { - t.Errorf("config1: got a client %v despite bogus url %s", cli, opts.URL) + t.Errorf("config1: got a client %v despite bogus url %q", cli, opts.URL) cli.Close() } - opts.URL = "http://doesnotexist.fortio.org" - cli, _ = NewStdClient(opts) - code, _, _ := cli.Fetch(context.Background()) - if code != -1 { - t.Errorf("config2: client can send request despite bogus url %s", opts.URL) + cli, err = NewStdClient(opts) + if cli != nil || err == nil { + t.Errorf("config2: got a client %v despite bogus host in url %q", cli, opts.URL) + cli.Close() + } + opts.URL = "" + cli, err = NewStdClient(opts) + if cli != nil || err == nil { + t.Errorf("config3: got a client %v despite empty url %q", cli, opts.URL) + cli.Close() } } @@ -863,16 +868,32 @@ nil, "GET", }, + { + "", + []byte{}, + "GET", + }, + { + "Foo", // once we set a content-type we get a POST instead of GET (alternative to -X) + nil, + "POST", + }, } for _, test := range tests { hOptions := HTTPOptions{} hOptions.URL = "www.google.com" - hOptions.ContentType = test.contentType + err := hOptions.AddAndValidateExtraHeader("conTENT-tYPE: " + test.contentType) + if err != nil { + t.Errorf("Got error %v adding header", err) + } + if hOptions.ContentType != test.contentType { + t.Errorf("Got %q, expected %q as a content type from header addition", hOptions.ContentType, test.contentType) + } hOptions.Payload = test.payload client, _ := NewStdClient(&hOptions) contentType := client.req.Header.Get("Content-Type") if contentType != test.contentType { - t.Errorf("Got %s, expected %s as a content type", contentType, test.contentType) + t.Errorf("Got %q, expected %q as a content type", contentType, test.contentType) } method := client.req.Method if method != test.expectedMethod { @@ -894,6 +915,47 @@ } } +func TestMethodOverride(t *testing.T) { + _, addr := ServeTCP("0", "/debug") + o := HTTPOptions{URL: fmt.Sprintf("http://localhost:%d/debug", addr.Port), MethodOverride: "FOO"} + for _, fastClient := range []bool{true, false} { + o.DisableFastClient = !fastClient + cli, _ := NewClient(&o) + ctx := context.Background() + code, data, _ := cli.Fetch(ctx) + if code != 200 { + t.Errorf("Non ok code %d for debug override method %q", code, o.MethodOverride) + } + expected := []byte("FOO /debug HTTP/1.1") + if !bytes.Contains(data, expected) { + t.Errorf("Method not echoed back in fastClient %v client %s (expecting %s)", fastClient, DebugSummary(data, 512), expected) + } + } +} + +func TestPostFromContentType(t *testing.T) { + _, addr := ServeTCP("0", "/debug") + o := HTTPOptions{URL: fmt.Sprintf("http://localhost:%d/debug", addr.Port)} + o.AddAndValidateExtraHeader("content-type: foo/bar; xyz") + for _, fastClient := range []bool{true, false} { + o.DisableFastClient = !fastClient + cli, _ := NewClient(&o) + ctx := context.Background() + code, data, _ := cli.Fetch(ctx) + if code != 200 { + t.Errorf("Non ok code %d for debug override method %q", code, o.MethodOverride) + } + expected := []byte("POST /debug HTTP/1.1") + if !bytes.Contains(data, expected) { + t.Errorf("POST not found in fastClient %v client %s (expecting %s)", fastClient, DebugSummary(data, 512), expected) + } + expected = []byte("Content-Type: foo/bar; xyz") + if !bytes.Contains(data, expected) { + t.Errorf("Content-type not found in fastClient %v client %s (expecting %s)", fastClient, DebugSummary(data, 512), expected) + } + } +} + func TestPayloadForFastClient(t *testing.T) { tests := []struct { contentType string @@ -1264,7 +1326,7 @@ } } -func TestFetch2Header(t *testing.T) { +func TestFetch2IncomingHeader(t *testing.T) { mux, addr := ServeTCP("0", "") mux.HandleFunc("/fetch2/", FetcherHandler2) url := fmt.Sprintf("localhost:%d/fetch2/?url=http://localhost:%d/echo%%3fheader%%3dFoo:Bar", addr.Port, addr.Port) @@ -1277,6 +1339,34 @@ } } +func TestFetch2OutgoingHeaders(t *testing.T) { + mux, addr := ServeTCP("0", "/debug") + mux.HandleFunc("/fetch2/", FetcherHandler2) + // added &H= and &H=incomplete to cover these 2 error cases + url := fmt.Sprintf("localhost:%d/fetch2/?url=http://localhost:%d/debug"+ + "&H=foo:Bar&payload=a-test&H=&H=IncompleteHeader&H=Content-TYPE:foo/bar42", + addr.Port, addr.Port) + code, data := Fetch(&HTTPOptions{URL: url, Payload: []byte("a-longer-different-payload")}) + if code != http.StatusOK { + t.Errorf("Got %d %s instead of ok for %s", code, DebugSummary(data, 256), url) + } + if !bytes.Contains(data, []byte("Foo: Bar")) { + t.Errorf("Result %s doesn't contain expected Foo: Bar header", DebugSummary(data, 1024)) + } + if !bytes.Contains(data, []byte("POST /debug HTTP/1.1")) { + t.Errorf("Passing payload should mean POST - got %s", DebugSummary(data, 1024)) + } + if !bytes.Contains(data, []byte("body:\n\na-test\n")) { + t.Errorf("Passing payload be echoed - got %s", DebugSummary(data, 1024)) + } + if !bytes.Contains(data, []byte("Content-Length: 6\n")) { + t.Errorf("Payload length should be the right one - got %s", DebugSummary(data, 1024)) + } + if !bytes.Contains(data, []byte("Content-Type: foo/bar42\n")) { + t.Errorf("Payload length should be the right one - got %s", DebugSummary(data, 1024)) + } +} + func TestFetch2Errors(t *testing.T) { mux, addr := ServeTCP("0", "") mux.HandleFunc("/fetch2/", FetcherHandler2) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fortio-1.55.2/fhttp/http_utils.go new/fortio-1.56.0/fhttp/http_utils.go --- old/fortio-1.55.2/fhttp/http_utils.go 2023-06-27 03:23:18.000000000 +0200 +++ new/fortio-1.56.0/fhttp/http_utils.go 2023-07-04 03:07:15.000000000 +0200 @@ -629,3 +629,40 @@ return r.reader.Read(p) } + +// CommonHTTPOptionsFromForm is used essentially in ui/uihandler.go but we want to reuse some options +// for fetching URLs too. Also ideally would get refactored to work for rapi/. +func CommonHTTPOptionsFromForm(r *http.Request) *HTTPOptions { + url := r.FormValue("url") + payload := r.FormValue("payload") + methodOverride := r.FormValue("X") + logErrors := (r.FormValue("log-errors") == "on") + h2 := (r.FormValue("h2") == "on") + httpsInsecure := (r.FormValue("https-insecure") == "on") + resolve := r.FormValue("resolve") + timeoutStr := strings.TrimSpace(r.FormValue("timeout")) + timeout, _ := time.ParseDuration(timeoutStr) // will be 0 if empty, which is handled by runner and opts + httpopts := &HTTPOptions{} + // to be normalized in init 0 replaced by default value only in http runner, not here as this could be a tcp or udp runner + httpopts.URL = url // fixes #651 - ie don't normalize here + httpopts.HTTPReqTimeOut = timeout + httpopts.Insecure = httpsInsecure + httpopts.Resolve = resolve + httpopts.H2 = h2 + httpopts.LogErrors = logErrors + httpopts.MethodOverride = methodOverride + if len(payload) > 0 { + httpopts.Payload = []byte(payload) + } + for _, header := range r.Form["H"] { + if len(header) == 0 { + continue + } + log.LogVf("adding header %v", header) + err := httpopts.AddAndValidateExtraHeader(header) + if err != nil { + log.Errf("Error adding custom headers: %v", err) + } + } + return httpopts +} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fortio-1.55.2/fhttp/httprunner_test.go new/fortio-1.56.0/fhttp/httprunner_test.go --- old/fortio-1.55.2/fhttp/httprunner_test.go 2023-06-27 03:23:18.000000000 +0200 +++ new/fortio-1.56.0/fhttp/httprunner_test.go 2023-07-04 03:07:15.000000000 +0200 @@ -79,7 +79,15 @@ } o1 = rawOpts o1.URL = "http://www.doesnotexist.badtld/" - c, _ := NewStdClient(&o1) + c, err := NewStdClient(&o1) + if err == nil || c != nil { + t.Errorf("Std Client bad host should error early") + } + o1.URL = "http://debug.fortio.org" // should resolve fine + c, err = NewStdClient(&o1) + if err != nil { + t.Errorf("Unexpected error once url is fixed: %v", err) + } c.ChangeURL(rawOpts.URL) if r, _, _ := c.Fetch(context.Background()); r != http.StatusOK { t.Errorf("Std Client with raw option should still work with warning in logs") diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fortio-1.55.2/go.mod new/fortio-1.56.0/go.mod --- old/fortio-1.55.2/go.mod 2023-06-27 03:23:18.000000000 +0200 +++ new/fortio-1.56.0/go.mod 2023-07-04 03:07:15.000000000 +0200 @@ -7,7 +7,8 @@ fortio.org/cli v1.1.0 fortio.org/dflag v1.5.2 fortio.org/log v1.5.0 - fortio.org/scli v1.6.0 + fortio.org/scli v1.7.0 + fortio.org/testscript v0.3.1 fortio.org/version v1.0.2 github.com/golang/protobuf v1.5.3 github.com/google/uuid v1.3.0 @@ -31,6 +32,7 @@ golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 // indirect golang.org/x/sys v0.9.0 // indirect golang.org/x/text v0.10.0 // indirect + golang.org/x/tools v0.8.0 // indirect google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect google.golang.org/protobuf v1.31.0 // indirect ) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fortio-1.55.2/go.sum new/fortio-1.56.0/go.sum --- old/fortio-1.55.2/go.sum 2023-06-27 03:23:18.000000000 +0200 +++ new/fortio-1.56.0/go.sum 2023-07-04 03:07:15.000000000 +0200 @@ -6,10 +6,12 @@ fortio.org/dflag v1.5.2/go.mod h1:ppb/A8u+KKg+qUUYZNYuvRnXuVb8IsdHb/XGzsmjkN8= fortio.org/log v1.5.0 h1:0f/O7QPXQoDSnRjy8t0IyxTlQOYQsDOe0EO4Wnw8yCA= fortio.org/log v1.5.0/go.mod h1:u/8/2lyczXq52aT5Nw6reD+3cR6m/EbS2jBiIYhgiTU= -fortio.org/scli v1.6.0 h1:orn3xqUVLtgkD9LgYtAovVZtfzOzN0qCuItRTd5Z+d4= -fortio.org/scli v1.6.0/go.mod h1:HV+Ucg3jV0n+i+mQt/MWlyLXINHmPwrQhney5j5oYW0= +fortio.org/scli v1.7.0 h1:NC3z2k+2NY5zTB+XoQGX2MKJulMq61gltK8OrkR3U9U= +fortio.org/scli v1.7.0/go.mod h1:HV+Ucg3jV0n+i+mQt/MWlyLXINHmPwrQhney5j5oYW0= fortio.org/sets v1.0.3 h1:HzewdGjH69YmyW06yzplL35lGr+X4OcqQt0qS6jbaO4= fortio.org/sets v1.0.3/go.mod h1:QZVj0r6KP/ZD9ebySW9SgxVNy/NjghUfyHW9NN+WU+4= +fortio.org/testscript v0.3.1 h1:MmRO64AsmzaU1KlYMzAbotJIMKRGxD1XXssJnBRiMGQ= +fortio.org/testscript v0.3.1/go.mod h1:7OJ+U4avooRNqc7p/VHKJadYgj9fA6+N0SbGU8FVWGs= fortio.org/version v1.0.2 h1:8NwxdX58aoeKx7T5xAPO0xlUu1Hpk42nRz5s6e6eKZ0= fortio.org/version v1.0.2/go.mod h1:2JQp9Ax+tm6QKiGuzR5nJY63kFeANcgrZ0osoQFDVm0= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= @@ -30,6 +32,8 @@ golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y= +golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fortio-1.55.2/main.go new/fortio-1.56.0/main.go --- old/fortio-1.55.2/main.go 2023-06-27 03:23:18.000000000 +0200 +++ new/fortio-1.56.0/main.go 2023-07-04 03:07:15.000000000 +0200 @@ -22,3 +22,9 @@ func main() { cli.FortioMain(nil /* no hook needed */) } + +// Same as real above but for testscript/txtar tests. +func Main() int { + cli.FortioMain(nil /* no hook needed */) + return 0 +} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fortio-1.55.2/rapi/restHandler.go new/fortio-1.56.0/rapi/restHandler.go --- old/fortio-1.55.2/rapi/restHandler.go 2023-06-27 03:23:18.000000000 +0200 +++ new/fortio-1.56.0/rapi/restHandler.go 2023-07-04 03:07:15.000000000 +0200 @@ -224,6 +224,7 @@ log.Infof("Starting API run %s load request from %v for %s", runner, r.RemoteAddr, url) async := (FormValue(r, jd, "async") == "on") payload := FormValue(r, jd, "payload") + methodOverride := FormValue(r, jd, "X") labels := FormValue(r, jd, "labels") resolution, _ := strconv.ParseFloat(FormValue(r, jd, "r"), 64) percList, _ := stats.ParsePercentiles(FormValue(r, jd, "p")) @@ -287,6 +288,7 @@ httpopts.Resolve = resolve httpopts.H2 = h2 httpopts.LogErrors = logErrors + httpopts.MethodOverride = methodOverride // Set the connection reuse range. err = bincommon.ConnectionReuseRange. WithValidator(bincommon.ConnectionReuseRangeValidator(httpopts)). diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fortio-1.55.2/release/bumpRelease.sh new/fortio-1.56.0/release/bumpRelease.sh --- old/fortio-1.55.2/release/bumpRelease.sh 2023-06-27 03:23:18.000000000 +0200 +++ new/fortio-1.56.0/release/bumpRelease.sh 2023-07-04 03:07:15.000000000 +0200 @@ -16,8 +16,12 @@ exit 1 fi if [ "$CURRENT" = "$RELEASE" ]; then - echo "Release $RELEASE is already in $FILENAME" - exit 0 + if [ "$2" = "-f" ]; then + echo "Forcing regen of same $CURRENT" + else + echo "Current release $CURRENT is the same as the requested, nothing to do, add -f to force" + exit 0 + fi fi echo "Changing $FILENAME from $CURRENT to release $RELEASE and updating usage section" ./release/updateFlags.sh | sed -e "s/ΦοÏÏίο dev/ΦοÏÏίο $RELEASE/" > /tmp/fortio_flags.txt diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fortio-1.55.2/ui/templates/main.html new/fortio-1.56.0/ui/templates/main.html --- old/fortio-1.55.2/ui/templates/main.html 2023-06-27 03:23:18.000000000 +0200 +++ new/fortio-1.56.0/ui/templates/main.html 2023-07-04 03:07:15.000000000 +0200 @@ -54,6 +54,7 @@ No Catch-Up (qps is a ceiling): <input type="checkbox" name="nocatchup" checked /><br /> Percentiles: <input type="text" name="p" size="20" value="50, 75, 90, 99, 99.9" /> <br /> Histogram Resolution: <input type="text" name="r" size="8" value="0.0001" /> <br /> + Method override (empty for GET/POST based on payload): <input type="text" name="X" size="8" value="" /> <br /> Extra Headers:<br /> <input type="text" name="H" size=40 value="" /> <br /> <button type="button" onclick="addCustomHeader()">+</button> diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fortio-1.55.2/ui/uihandler.go new/fortio-1.56.0/ui/uihandler.go --- old/fortio-1.55.2/ui/uihandler.go 2023-06-27 03:23:18.000000000 +0200 +++ new/fortio-1.56.0/ui/uihandler.go 2023-07-04 03:07:15.000000000 +0200 @@ -99,14 +99,15 @@ // Handler is the main UI handler creating the web forms and processing them. // TODO: refactor common option/args/flag parsing between restHandle.go and this. // -//nolint:funlen, gocognit, gocyclo, nestif, maintidx // should be refactored indeed (TODO) +//nolint:funlen, gocognit, nestif // should be refactored indeed (TODO) func Handler(w http.ResponseWriter, r *http.Request) { log.LogRequest(r, "UI") mode := menu JSONOnly := false - url := r.FormValue("url") runid := int64(0) runner := r.FormValue("runner") + httpopts := fhttp.CommonHTTPOptionsFromForm(r) + url := httpopts.URL if r.FormValue("load") == "Start" { mode = run if r.FormValue("json") == "on" { @@ -121,7 +122,6 @@ mode = stop } // Those only exist/make sense on run mode but go variable declaration... - payload := r.FormValue("payload") labels := r.FormValue("labels") resolution, _ := strconv.ParseFloat(r.FormValue("r"), 64) percList, _ := stats.ParsePercentiles(r.FormValue("p")) @@ -135,13 +135,7 @@ uniform := (r.FormValue("uniform") == "on") nocatchup := (r.FormValue("nocatchup") == "on") stdClient := (r.FormValue("stdclient") == "on") - logErrors := (r.FormValue("log-errors") == "on") - h2 := (r.FormValue("h2") == "on") sequentialWarmup := (r.FormValue("sequential-warmup") == "on") - httpsInsecure := (r.FormValue("https-insecure") == "on") - resolve := r.FormValue("resolve") - timeoutStr := strings.TrimSpace(r.FormValue("timeout")) - timeout, _ := time.ParseDuration(timeoutStr) // will be 0 if empty, which is handled by runner and opts var dur time.Duration if durStr == "on" || ((len(r.Form["t"]) > 1) && r.Form["t"][1] == "on") { dur = -1 @@ -165,9 +159,6 @@ out = fhttp.NewHTMLEscapeWriter(w) } n, _ := strconv.ParseInt(r.FormValue("n"), 10, 64) - if strings.TrimSpace(url) == "" { - url = "http://url.needed" // just because url validation doesn't like empty urls - } ro := periodic.RunnerOptions{ QPS: qps, Duration: dur, @@ -187,16 +178,8 @@ log.Infof("New run id %d", runid) ro.RunID = runid } - httpopts := &fhttp.HTTPOptions{} - // to be normalized in init 0 replaced by default value only in http runner, not here as this could be a tcp or udp runner - httpopts.URL = url // fixes #651 - httpopts.HTTPReqTimeOut = timeout httpopts.DisableFastClient = stdClient httpopts.SequentialWarmup = sequentialWarmup - httpopts.Insecure = httpsInsecure - httpopts.Resolve = resolve - httpopts.H2 = h2 - httpopts.LogErrors = logErrors // Set the connection reuse range. err := bincommon.ConnectionReuseRange. WithValidator(bincommon.ConnectionReuseRangeValidator(httpopts)). @@ -204,10 +187,6 @@ if err != nil { log.Errf("Fail to validate connection reuse range flag, err: %v", err) } - - if len(payload) > 0 { - httpopts.Payload = []byte(payload) - } if !JSONOnly { // Normal html mode if mainTemplate == nil { @@ -259,16 +238,6 @@ rapi.StopByRunID(runid, false) case run: // mode == run case: - for _, header := range r.Form["H"] { - if len(header) == 0 { - continue - } - log.LogVf("adding header %v", header) - err := httpopts.AddAndValidateExtraHeader(header) - if err != nil { - log.Errf("Error adding custom headers: %v", err) - } - } fhttp.OnBehalfOf(httpopts, r) runWriter := w if !JSONOnly { @@ -491,7 +460,7 @@ // If we had hundreds of thousands of entry we should stream, parallelize (connection pool) // and not do multiple passes over the same data, but for small tsv this is fine. // use std client to avoid chunked raw we can get with fast client: - client, _ := fhttp.NewStdClient(o) + client, _ := fhttp.NewStdClient(o) //nolint:contextcheck if client == nil { _, _ = w.Write([]byte("invalid url!<script>setPB(1,1)</script></body></html>\n")) // too late to write headers for real case but we do it anyway for the Sync() startup case ++++++ fortio.obsinfo ++++++ --- /var/tmp/diff_new_pack.DtFrHt/_old 2023-07-04 15:23:33.310629391 +0200 +++ /var/tmp/diff_new_pack.DtFrHt/_new 2023-07-04 15:23:33.310629391 +0200 @@ -1,5 +1,5 @@ name: fortio -version: 1.55.2 -mtime: 1687828998 -commit: c39b74f7f44b365dab0e7c753ab21691bb7c1861 +version: 1.56.0 +mtime: 1688432835 +commit: 8890d97fd5aac4ccf571cc9a40331d3ab2bd145b ++++++ vendor.tar.gz ++++++ ++++ 4453 lines of diff (skipped)