net/http Clients and Servers

package main

import (
	"context"
	"errors"
	"fmt"
	"net"
	"net/http"
	"net/http/httptest"
	"net/url"
	"time"
)

func main() {
	// A server that never answers within the test.
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		select {
		case <-r.Context().Done():
		case <-time.After(5 * time.Second):
		}
	}))
	defer srv.Close()

	client := &http.Client{Timeout: 150 * time.Millisecond}
	start := time.Now()
	_, err := client.Get(srv.URL + "/slow")

	var uerr *url.Error
	if errors.As(err, &uerr) {
		fmt.Println("op:", uerr.Op)
		fmt.Println("err:", uerr.Err)
		fmt.Println("timeout:", uerr.Timeout())
	}
	var nerr net.Error
	fmt.Println("net.Error:", errors.As(err, &nerr) && nerr.Timeout())
	fmt.Println("deadline exceeded:", errors.Is(err, context.DeadlineExceeded))
	fmt.Println("returned quickly:", time.Since(start) < 2*time.Second)
}

op: Get
err: context deadline exceeded (Client.Timeout exceeded while awaiting headers)
timeout: true
net.Error: true
deadline exceeded: true
returned quickly: true

package main

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/http/httptest"
	"time"

	"example.com/shelf/catalog"
)

// StatusError reports an unexpected HTTP status.
type StatusError struct {
	Code int
	Body string
}

func (e *StatusError) Error() string {
	return fmt.Sprintf("unexpected status %d: %s", e.Code, e.Body)
}

// getBook fetches one book from the shelf API.
func getBook(ctx context.Context, c *http.Client, base string, isbn catalog.ISBN) (catalog.Book, error) {
	var b catalog.Book
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, base+"/books/"+string(isbn), nil)
	if err != nil {
		return b, err
	}
	req.Header.Set("Accept", "application/json")

	resp, err := c.Do(req)
	if err != nil {
		return b, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		msg, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
		return b, &StatusError{Code: resp.StatusCode, Body: string(msg)}
	}
	// Never trust the size of a remote body.
	err = json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&b)
	return b, err
}

func main() {
	mux := http.NewServeMux()
	mux.HandleFunc("GET /books/{isbn}", func(w http.ResponseWriter, r *http.Request) {
		if r.PathValue("isbn") != "9780441172719" {
			http.Error(w, "no such book", http.StatusNotFound)
			return
		}
		w.Header().Set("Content-Type", "application/json")
		_ = json.NewEncoder(w).Encode(catalog.Book{
			ISBN:    "9780441172719",
			Title:   "Dune",
			Authors: []catalog.Author{{Name: "Frank Herbert"}},
		})
	})
	srv := httptest.NewServer(mux)
	defer srv.Close()

	client := &http.Client{Timeout: 5 * time.Second}
	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()

	b, err := getBook(ctx, client, srv.URL, "9780441172719")
	fmt.Printf("%+v %v\n", b, err)

	_, err = getBook(ctx, client, srv.URL, "9780000000002")
	var se *StatusError
	fmt.Println(errors.As(err, &se), err)
}

{ISBN:9780441172719 Title:Dune Authors:[{Name:Frank Herbert}]} <nil>
true unexpected status 404: no such book

package main

import (
	"fmt"
	"io"
	"net"
	"net/http"
	"net/http/httptest"
	"net/http/httptrace"
	"time"
)

// newClient returns a client for a service we call often.
func newClient() *http.Client {
	tr := http.DefaultTransport.(*http.Transport).Clone()
	tr.DialContext = (&net.Dialer{
		Timeout:   3 * time.Second, // TCP connect
		KeepAlive: 30 * time.Second,
	}).DialContext
	tr.TLSHandshakeTimeout = 3 * time.Second
	tr.ResponseHeaderTimeout = 5 * time.Second
	tr.IdleConnTimeout = 90 * time.Second
	tr.MaxIdleConns = 100
	tr.MaxIdleConnsPerHost = 10 // the default of 2 is too low for a busy client
	return &http.Client{Transport: tr, Timeout: 10 * time.Second}
}

func main() {
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprint(w, "ok")
	}))
	defer srv.Close()

	c := newClient()
	for i := 1; i <= 2; i++ {
		var reused bool
		trace := &httptrace.ClientTrace{
			GotConn: func(info httptrace.GotConnInfo) { reused = info.Reused },
		}
		req, _ := http.NewRequest(http.MethodGet, srv.URL, nil)
		req = req.WithContext(httptrace.WithClientTrace(req.Context(), trace))
		resp, err := c.Do(req)
		if err != nil {
			fmt.Println(err)
			return
		}
		_, _ = io.Copy(io.Discard, resp.Body)
		resp.Body.Close()
		fmt.Printf("request %d: connection reused: %t\n", i, reused)
	}
	tr := c.Transport.(*http.Transport)
	fmt.Println("MaxIdleConnsPerHost:", tr.MaxIdleConnsPerHost)
	fmt.Println("default MaxIdleConnsPerHost:", http.DefaultMaxIdleConnsPerHost)
}

request 1: connection reused: false
request 2: connection reused: true
MaxIdleConnsPerHost: 10
default MaxIdleConnsPerHost: 2

package main

import (
	"errors"
	"fmt"
	"net/http"
	"net/http/httptest"
	"net/url"
)

func main() {
	mux := http.NewServeMux()
	mux.HandleFunc("GET /old", func(w http.ResponseWriter, r *http.Request) {
		http.Redirect(w, r, "/new", http.StatusMovedPermanently)
	})
	mux.HandleFunc("GET /new", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, "you are here")
	})
	mux.HandleFunc("GET /loop", func(w http.ResponseWriter, r *http.Request) {
		http.Redirect(w, r, "/loop", http.StatusFound)
	})
	srv := httptest.NewServer(mux)
	defer srv.Close()

	// 1. The default client follows up to 10 redirects.
	resp, err := http.Get(srv.URL + "/old")
	if err != nil {
		fmt.Println(err)
		return
	}
	fmt.Println("followed:", resp.StatusCode, resp.Request.URL.Path)
	resp.Body.Close()

	// 2. Do not follow: inspect the redirect itself.
	noFollow := &http.Client{
		CheckRedirect: func(req *http.Request, via []*http.Request) error {
			return http.ErrUseLastResponse
		},
	}
	resp, err = noFollow.Get(srv.URL + "/old")
	if err != nil {
		fmt.Println(err)
		return
	}
	fmt.Println("not followed:", resp.StatusCode, resp.Header.Get("Location"))
	resp.Body.Close()

	// 3. A loop stops after 10 redirects.
	_, err = http.Get(srv.URL + "/loop")
	var uerr *url.Error
	fmt.Println("loop:", errors.As(err, &uerr), uerr.Err)
}

followed: 200 /new
not followed: 301 /new
loop: true stopped after 10 redirects

package main

import (
	"context"
	"fmt"
	"io"
	"net/http"
	"net/http/httptest"
	"strconv"
	"sync/atomic"
	"time"
)

// getWithRetry retries an idempotent GET on 429, 502, 503 and 504.
// It honours Retry-After (seconds) and stops as soon as ctx is done.
func getWithRetry(ctx context.Context, c *http.Client, url string, attempts int) (*http.Response, error) {
	backoff := 10 * time.Millisecond
	for attempt := 1; ; attempt++ {
		req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
		if err != nil {
			return nil, err
		}
		resp, err := c.Do(req)
		if err == nil && !retryable(resp.StatusCode) {
			return resp, nil // the caller closes the body
		}
		wait := backoff
		if err == nil {
			if s, perr := strconv.Atoi(resp.Header.Get("Retry-After")); perr == nil {
				wait = min(time.Duration(s)*time.Second, maxWait)
			}
			io.Copy(io.Discard, io.LimitReader(resp.Body, 4<<10))
			resp.Body.Close()
		}
		if attempt == attempts {
			if err == nil {
				err = fmt.Errorf("giving up after %d attempts: status %d", attempt, resp.StatusCode)
			}
			return nil, err
		}
		select {
		case <-time.After(wait):
		case <-ctx.Done():
			return nil, context.Cause(ctx)
		}
		backoff *= 2
	}
}

// maxWait caps what a server can ask for; use a few seconds in production.
const maxWait = 30 * time.Millisecond

func retryable(code int) bool {
	switch code {
	case http.StatusTooManyRequests, http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout:
		return true
	}
	return false
}

func main() {
	var calls atomic.Int32
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if calls.Add(1) < 3 {
			w.Header().Set("Retry-After", "1")
			http.Error(w, "busy", http.StatusServiceUnavailable)
			return
		}
		fmt.Fprintln(w, "finally")
	}))
	defer srv.Close()

	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
	defer cancel()
	resp, err := getWithRetry(ctx, srv.Client(), srv.URL, 5)
	if err != nil {
		fmt.Println(err)
		return
	}
	defer resp.Body.Close()
	b, _ := io.ReadAll(resp.Body)
	fmt.Printf("%d after %d calls: %s", resp.StatusCode, calls.Load(), b)
}

200 after 3 calls: finally

package main

import (
	"fmt"
	"io"
	"net"
	"net/http"
)

func main() {
	mux := http.NewServeMux()
	mux.HandleFunc("GET /", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, "served over %s\n", r.Proto)
	})

	// One cleartext port speaking both HTTP/1.1 and HTTP/2 (prior knowledge).
	srv := &http.Server{Handler: mux, Protocols: new(http.Protocols)}
	srv.Protocols.SetHTTP1(true)
	srv.Protocols.SetUnencryptedHTTP2(true)
	ln, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		panic(err)
	}
	go srv.Serve(ln)
	defer srv.Close()

	for _, h2 := range []bool{false, true} {
		tr := &http.Transport{Protocols: new(http.Protocols)}
		if h2 {
			tr.Protocols.SetUnencryptedHTTP2(true) // HTTP/2 only, no HTTP/1
		} else {
			tr.Protocols.SetHTTP1(true)
		}
		resp, err := (&http.Client{Transport: tr}).Get("http://" + ln.Addr().String())
		if err != nil {
			fmt.Println(err)
			continue
		}
		b, _ := io.ReadAll(resp.Body)
		resp.Body.Close()
		fmt.Printf("client proto %s: %s", resp.Proto, b)
		tr.CloseIdleConnections()
	}
}

client proto HTTP/1.1: served over HTTP/1.1
client proto HTTP/2.0: served over HTTP/2.0

type Handler interface {
    ServeHTTP(ResponseWriter, *Request)
}

package main

import (
	"bytes"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/http/httptest"
)

// hello is a Handler with state: a struct with a ServeHTTP method.
type hello struct{ greeting string }

func (h hello) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	fmt.Fprintf(w, "%s, %s\n", h.greeting, r.URL.Query().Get("name"))
}

// late shows the order rule: headers, then WriteHeader, then the body.
func late(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintln(w, "partial body") // implicit WriteHeader(200)
	w.Header().Set("X-Too-Late", "1")
	w.WriteHeader(http.StatusInternalServerError) // ignored, logged
}

func get(c *http.Client, url string) {
	resp, err := c.Get(url)
	if err != nil {
		fmt.Println(err)
		return
	}
	defer resp.Body.Close()
	body, _ := io.ReadAll(resp.Body)
	fmt.Printf("%d %q too-late=%q ctype=%q\n", resp.StatusCode, body,
		resp.Header.Get("X-Too-Late"), resp.Header.Get("Content-Type"))
}

func main() {
	mux := http.NewServeMux()
	mux.Handle("GET /hello", hello{greeting: "Hello"})
	mux.HandleFunc("GET /late", late)
	mux.HandleFunc("GET /sniff", func(w http.ResponseWriter, r *http.Request) {
		io.WriteString(w, "<html><body>hi</body></html>") // no Content-Type set
	})

	var logs bytes.Buffer
	srv := httptest.NewUnstartedServer(mux)
	srv.Config.ErrorLog = log.New(&logs, "", 0)
	srv.Start()
	defer srv.Close()

	get(srv.Client(), srv.URL+"/hello?name=Ada")
	get(srv.Client(), srv.URL+"/late")
	get(srv.Client(), srv.URL+"/sniff")
	fmt.Print("server log: ", logs.String())
}

200 "Hello, Ada\n" too-late="" ctype="text/plain; charset=utf-8"
200 "partial body\n" too-late="" ctype="text/plain; charset=utf-8"
200 "<html><body>hi</body></html>" too-late="" ctype="text/html; charset=utf-8"
server log: http: superfluous response.WriteHeader call from main.late (main.go:24)

[METHOD ][HOST]/[PATH]

package main

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"strings"
)

func main() {
	mux := http.NewServeMux()
	handle := func(pattern string) {
		mux.HandleFunc(pattern, func(w http.ResponseWriter, r *http.Request) {
			fmt.Fprintf(w, "matched %q", r.Pattern)
			for _, name := range []string{"isbn", "rest"} {
				if v := r.PathValue(name); v != "" {
					fmt.Fprintf(w, " %s=%q", name, v)
				}
			}
		})
	}
	handle("/{$}")
	handle("GET /books")
	handle("POST /books")
	handle("GET /books/{isbn}")
	handle("GET /books/{isbn}/loans")
	handle("GET /books/new")
	handle("/files/{rest...}")
	handle("GET /api/")
	handle("books.shelf.test/books")

	try := func(method, target string) {
		req := httptest.NewRequest(method, target, nil)
		rec := httptest.NewRecorder()
		mux.ServeHTTP(rec, req) // the mux is an http.Handler: no network needed
		line, _, _ := strings.Cut(rec.Body.String(), "\n")
		if allow := rec.Header().Get("Allow"); allow != "" {
			line += " (Allow: " + allow + ")"
		}
		fmt.Printf("%-6s %-34s %d %s\n", method, target, rec.Code, line)
	}
	try("GET", "/")
	try("GET", "/nothing")
	try("GET", "/books")
	try("POST", "/books")
	try("DELETE", "/books")
	try("GET", "/books/9780441172719")
	try("GET", "/books/new")
	try("GET", "/books/9780441172719/loans")
	try("GET", "/books/9780441172719/loans/extra")
	try("GET", "/files/a/b/c.txt")
	try("GET", "/files/")
	try("GET", "/api/anything/at/all")
	try("HEAD", "/books")
	try("GET", "http://books.shelf.test/books")
}

GET    /                                  200 matched "/{$}"
GET    /nothing                           404 404 page not found
GET    /books                             200 matched "GET /books"
POST   /books                             200 matched "POST /books"
DELETE /books                             405 Method Not Allowed (Allow: GET, HEAD, POST)
GET    /books/9780441172719               200 matched "GET /books/{isbn}" isbn="9780441172719"
GET    /books/new                         200 matched "GET /books/new"
GET    /books/9780441172719/loans         200 matched "GET /books/{isbn}/loans" isbn="9780441172719"
GET    /books/9780441172719/loans/extra   404 404 page not found
GET    /files/a/b/c.txt                   200 matched "/files/{rest...}" rest="a/b/c.txt"
GET    /files/                            200 matched "/files/{rest...}"
GET    /api/anything/at/all               200 matched "GET /api/"
HEAD   /books                             200 matched "GET /books"
GET    http://books.shelf.test/books      200 matched "books.shelf.test/books"

package main

import (
	"fmt"
	"net/http"
	"regexp"
)

func try(patterns ...string) {
	defer func() {
		if r := recover(); r != nil {
			msg := fmt.Sprint(r)
			msg = regexp.MustCompile(`\(registered at [^)]*\)`).ReplaceAllString(msg, "(registered at main.go)")
			fmt.Println("panic:", msg)
		}
	}()
	mux := http.NewServeMux()
	for _, p := range patterns {
		mux.HandleFunc(p, func(http.ResponseWriter, *http.Request) {})
	}
	fmt.Println("ok:", patterns)
}

func main() {
	try("GET /books/{isbn}", "GET /books/new") // the second is more specific
	try("/books/{id}", "/books/{isbn}")        // same set of requests
	try("GET /books/{id}", "/books/new")       // overlap, neither is more specific
	try("GET /books", "GET /books")            // registered twice
	try("GET /books/{id}/{id}")                // repeated wildcard name
	try("GET /files/{rest...}/more")           // ... only as the last segment
	try("GET")                                 // no path
	try("GET /books/{$}/more")                 // {$} only at the end
}

ok: [GET /books/{isbn} GET /books/new]
panic: pattern "/books/{isbn}" (registered at main.go) conflicts with pattern "/books/{id}" (registered at main.go):
/books/{isbn} matches the same requests as /books/{id}
panic: pattern "/books/new" (registered at main.go) conflicts with pattern "GET /books/{id}" (registered at main.go):
/books/new matches more methods than GET /books/{id}, but has a more specific path pattern
panic: pattern "GET /books" (registered at main.go) conflicts with pattern "GET /books" (registered at main.go):
GET /books matches the same requests as GET /books
panic: parsing "GET /books/{id}/{id}": at offset 16: duplicate wildcard name "id"
panic: parsing "GET /files/{rest...}/more": at offset 11: {...} wildcard not at end
panic: parsing "GET": at offset 0: host/path missing /
panic: parsing "GET /books/{$}/more": at offset 11: {$} not at end

package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/http/httptest"
	"slices"
	"strings"

	"example.com/shelf/catalog"
)

// bookJSON is the wire format; catalog.Book stays free of JSON concerns.
type bookJSON struct {
	ISBN    string   `json:"isbn"`
	Title   string   `json:"title"`
	Authors []string `json:"authors"`
}

func toJSON(b catalog.Book) bookJSON {
	out := bookJSON{ISBN: b.ISBN.String(), Title: b.Title, Authors: []string{}}
	for _, a := range b.Authors {
		out.Authors = append(out.Authors, a.Name)
	}
	return out
}

type api struct {
	store catalog.Store
}

// routes returns the whole service as one http.Handler.
func (a *api) routes() http.Handler {
	mux := http.NewServeMux()
	mux.HandleFunc("GET /books", a.listBooks)
	mux.HandleFunc("POST /books", a.createBook)
	mux.HandleFunc("GET /books/{isbn}", a.getBook)
	return mux
}

func writeJSON(w http.ResponseWriter, status int, v any) {
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(status)
	_ = json.NewEncoder(w).Encode(v)
}

func writeError(w http.ResponseWriter, status int, msg string) {
	writeJSON(w, status, map[string]string{"error": msg})
}

// decodeBody reads exactly one JSON value of at most 4 KiB.
func decodeBody(w http.ResponseWriter, r *http.Request, dst any) error {
	r.Body = http.MaxBytesReader(w, r.Body, 4<<10)
	dec := json.NewDecoder(r.Body)
	dec.DisallowUnknownFields()
	if err := dec.Decode(dst); err != nil {
		return err
	}
	if _, err := dec.Token(); err != io.EOF {
		return errors.New("body must contain a single JSON value")
	}
	return nil
}

func (a *api) listBooks(w http.ResponseWriter, r *http.Request) {
	books := a.store.All()
	slices.SortFunc(books, func(x, y catalog.Book) int { return strings.Compare(x.Title, y.Title) })
	out := make([]bookJSON, 0, len(books))
	for _, b := range books {
		out = append(out, toJSON(b))
	}
	writeJSON(w, http.StatusOK, out)
}

func (a *api) getBook(w http.ResponseWriter, r *http.Request) {
	isbn, err := catalog.ParseISBN(r.PathValue("isbn"))
	if err != nil {
		writeError(w, http.StatusBadRequest, err.Error())
		return
	}
	b, ok := a.store.Get(isbn)
	if !ok {
		writeError(w, http.StatusNotFound, "no such book")
		return
	}
	writeJSON(w, http.StatusOK, toJSON(b))
}

func (a *api) createBook(w http.ResponseWriter, r *http.Request) {
	var in bookJSON
	if err := decodeBody(w, r, &in); err != nil {
		var tooBig *http.MaxBytesError
		if errors.As(err, &tooBig) {
			writeError(w, http.StatusRequestEntityTooLarge, fmt.Sprintf("body larger than %d bytes", tooBig.Limit))
			return
		}
		writeError(w, http.StatusBadRequest, err.Error())
		return
	}
	isbn, err := catalog.ParseISBN(in.ISBN)
	if err != nil {
		writeError(w, http.StatusUnprocessableEntity, err.Error())
		return
	}
	book := catalog.Book{ISBN: isbn, Title: in.Title}
	for _, name := range in.Authors {
		book.Authors = append(book.Authors, catalog.Author{Name: name})
	}
	a.store.Put(book)
	w.Header().Set("Location", "/books/"+isbn.String())
	writeJSON(w, http.StatusCreated, toJSON(book))
}

func main() {
	srv := httptest.NewServer((&api{store: catalog.NewMemStore()}).routes())
	defer srv.Close()
	c := srv.Client()

	do := func(method, path, body string) {
		req, _ := http.NewRequest(method, srv.URL+path, strings.NewReader(body))
		if body != "" {
			req.Header.Set("Content-Type", "application/json")
		}
		resp, err := c.Do(req)
		if err != nil {
			fmt.Println(err)
			return
		}
		defer resp.Body.Close()
		b, _ := io.ReadAll(resp.Body)
		loc := ""
		if l := resp.Header.Get("Location"); l != "" {
			loc = " Location=" + l
		}
		short := body
		if len(short) > 40 {
			short = short[:40] + "..."
		}
		fmt.Printf("%s %s %s\n  -> %d%s %s", method, path, short, resp.StatusCode, loc, b)
	}

	do("POST", "/books", `{"isbn":"978-0-441-17271-9","title":"Dune","authors":["Frank Herbert"]}`)
	do("GET", "/books/9780441172719", "")
	do("GET", "/books", "")
	do("GET", "/books/9780000000002", "")
	do("GET", "/books/123", "")
	do("POST", "/books", `{"isbn":"978-0-441-17271-9","titel":"typo"}`)
	do("POST", "/books", `{"isbn":"978-0-441-17271-9"} {"again":true}`)
	do("POST", "/books", `{"isbn":"1234567890123","title":"Bad checksum"}`)
	do("POST", "/books", `{"isbn":"978-0-441-17271-9","title":"`+strings.Repeat("x", 5000)+`"}`)
	do("DELETE", "/books/9780441172719", "")
}

POST /books {"isbn":"978-0-441-17271-9","title":"Dun...
  -> 201 Location=/books/9780441172719 {"isbn":"9780441172719","title":"Dune","authors":["Frank Herbert"]}
GET /books/9780441172719
  -> 200 {"isbn":"9780441172719","title":"Dune","authors":["Frank Herbert"]}
GET /books
  -> 200 [{"isbn":"9780441172719","title":"Dune","authors":["Frank Herbert"]}]
GET /books/9780000000002
  -> 404 {"error":"no such book"}
GET /books/123
  -> 400 {"error":"catalog: invalid ISBN: \"123\""}
POST /books {"isbn":"978-0-441-17271-9","titel":"typ...
  -> 400 {"error":"json: unknown field \"titel\""}
POST /books {"isbn":"978-0-441-17271-9"} {"again":tr...
  -> 400 {"error":"body must contain a single JSON value"}
POST /books {"isbn":"1234567890123","title":"Bad che...
  -> 422 {"error":"catalog: invalid ISBN: \"1234567890123\""}
POST /books {"isbn":"978-0-441-17271-9","title":"xxx...
  -> 413 {"error":"body larger than 4096 bytes"}
DELETE /books/9780441172719
  -> 405 Method Not Allowed

package main

import (
	"fmt"
	"io"
	"net/http"
	"net/http/cookiejar"
	"net/http/httptest"
	"net/url"
	"strings"
)

func main() {
	mux := http.NewServeMux()

	// Forms: ParseForm reads the query string and a urlencoded body.
	mux.HandleFunc("POST /loans", func(w http.ResponseWriter, r *http.Request) {
		r.Body = http.MaxBytesReader(w, r.Body, 1<<10)
		if err := r.ParseForm(); err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		fmt.Fprintf(w, "isbn=%s member=%s via-query=%s\n",
			r.PostFormValue("isbn"), r.PostForm.Get("member"), r.URL.Query().Get("src"))
	})

	// Cookies: the server sets one, the client jar returns it.
	mux.HandleFunc("POST /login", func(w http.ResponseWriter, r *http.Request) {
		http.SetCookie(w, &http.Cookie{
			Name:     "session",
			Value:    "demo-session-id", // a real one is long, random and stored server-side
			Path:     "/",
			HttpOnly: true,                 // not readable by JavaScript
			Secure:   false,                // true in production (HTTPS only)
			SameSite: http.SameSiteLaxMode, // sent on same-site and top-level navigations
			MaxAge:   3600,
		})
		fmt.Fprintln(w, "logged in")
	})
	mux.HandleFunc("GET /me", func(w http.ResponseWriter, r *http.Request) {
		c, err := r.Cookie("session")
		if err != nil { // http.ErrNoCookie
			http.Error(w, err.Error(), http.StatusUnauthorized)
			return
		}
		fmt.Fprintf(w, "hello, session %s\n", c.Value)
	})

	srv := httptest.NewServer(mux)
	defer srv.Close()

	show := func(resp *http.Response, err error) {
		if err != nil {
			fmt.Println(err)
			return
		}
		defer resp.Body.Close()
		b, _ := io.ReadAll(resp.Body)
		fmt.Printf("%d %s", resp.StatusCode, b)
	}

	show(http.PostForm(srv.URL+"/loans?src=web", url.Values{"isbn": {"9780441172719"}, "member": {"ada"}}))
	show(http.Post(srv.URL+"/loans", "application/json", strings.NewReader(`{"isbn":"1"}`)))

	jar, _ := cookiejar.New(nil)
	client := &http.Client{Jar: jar}
	show(client.Get(srv.URL + "/me"))
	show(client.Post(srv.URL+"/login", "", nil))
	show(client.Get(srv.URL + "/me"))
	u, _ := url.Parse(srv.URL)
	for _, c := range jar.Cookies(u) {
		fmt.Println("jar holds:", c.Name)
	}
}

200 isbn=9780441172719 member=ada via-query=web
200 isbn= member= via-query=
401 http: named cookie not present
200 logged in
200 hello, session demo-session-id
jar holds: session

package main

import (
	"context"
	"fmt"
	"io"
	"log/slog"
	"net/http"
	"net/http/httptest"
	"os"
	"strconv"
	"sync/atomic"
)

// Middleware wraps a handler with behaviour that runs before and/or after it.
type Middleware func(http.Handler) http.Handler

// Chain applies middleware so that the first one listed is the outermost.
func Chain(h http.Handler, mws ...Middleware) http.Handler {
	for i := len(mws) - 1; i >= 0; i-- {
		h = mws[i](h)
	}
	return h
}

type ctxKey struct{}

// RequestID stores a request ID in the context and echoes it in the response.
func RequestID() Middleware {
	var n atomic.Int64
	return func(next http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			id := "req-" + strconv.FormatInt(n.Add(1), 10)
			w.Header().Set("X-Request-ID", id)
			next.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), ctxKey{}, id)))
		})
	}
}

// statusRecorder remembers what the handler wrote.
type statusRecorder struct {
	http.ResponseWriter
	status int
	bytes  int
}

func (s *statusRecorder) WriteHeader(code int) {
	s.status = code
	s.ResponseWriter.WriteHeader(code)
}

func (s *statusRecorder) Write(p []byte) (int, error) {
	if s.status == 0 {
		s.status = http.StatusOK
	}
	n, err := s.ResponseWriter.Write(p)
	s.bytes += n
	return n, err
}

// Unwrap lets http.ResponseController reach Flusher, deadlines and Hijacker.
func (s *statusRecorder) Unwrap() http.ResponseWriter { return s.ResponseWriter }

// Logging logs one structured line per request, after the handler returns.
func Logging(log *slog.Logger) Middleware {
	return func(next http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			rec := &statusRecorder{ResponseWriter: w}
			next.ServeHTTP(rec, r)
			id, _ := r.Context().Value(ctxKey{}).(string)
			log.LogAttrs(r.Context(), slog.LevelInfo, "request",
				slog.String("id", id),
				slog.String("method", r.Method),
				slog.String("pattern", r.Pattern),
				slog.Int("status", rec.status),
				slog.Int("bytes", rec.bytes))
		})
	}
}

// Recover turns a panic in a handler into a 500 instead of a dropped connection.
func Recover(log *slog.Logger) Middleware {
	return func(next http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			defer func() {
				if v := recover(); v != nil {
					if v == http.ErrAbortHandler {
						panic(v) // deliberate abort: let net/http handle it
					}
					log.Error("panic in handler", "panic", v)
					http.Error(w, "internal server error", http.StatusInternalServerError)
				}
			}()
			next.ServeHTTP(w, r)
		})
	}
}

func main() {
	log := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{
		ReplaceAttr: func(groups []string, a slog.Attr) slog.Attr {
			if a.Key == slog.TimeKey && len(groups) == 0 {
				return slog.Attr{} // drop the time to keep the output stable
			}
			return a
		},
	}))

	mux := http.NewServeMux()
	mux.HandleFunc("GET /ping", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, "pong")
	})
	mux.HandleFunc("GET /boom", func(w http.ResponseWriter, r *http.Request) {
		panic("catalogue is on fire")
	})

	// Logging is inside RequestID (it can read the ID) and outside Recover
	// (it sees the 500 that Recover writes).
	h := Chain(mux, RequestID(), Logging(log), Recover(log))

	srv := httptest.NewServer(h)
	defer srv.Close()
	for _, path := range []string{"/ping", "/boom", "/missing"} {
		resp, err := http.Get(srv.URL + path)
		if err != nil {
			fmt.Println(err)
			return
		}
		body, _ := io.ReadAll(resp.Body)
		resp.Body.Close()
		fmt.Printf("%s -> %d %s id=%s\n", path, resp.StatusCode, string(body[:len(body)-1]), resp.Header.Get("X-Request-ID"))
	}
}

level=INFO msg=request id=req-1 method=GET pattern="GET /ping" status=200 bytes=5
/ping -> 200 pong id=req-1
level=ERROR msg="panic in handler" panic="catalogue is on fire"
level=INFO msg=request id=req-2 method=GET pattern="GET /boom" status=500 bytes=22
/boom -> 500 internal server error id=req-2
level=INFO msg=request id=req-3 method=GET pattern="" status=404 bytes=19
/missing -> 404 404 page not found id=req-3

package main

import (
	"fmt"
	"net/http"
	"net/http/httptest"
)

func main() {
	mux := http.NewServeMux()
	mux.HandleFunc("POST /loans", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, "loan created")
	})
	mux.HandleFunc("GET /loans", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, "loan list")
	})
	mux.HandleFunc("POST /webhooks/payments", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, "webhook accepted")
	})

	cop := http.NewCrossOriginProtection()
	if err := cop.AddTrustedOrigin("https://admin.shelf.example"); err != nil {
		panic(err)
	}
	// Server-to-server callbacks are not browser requests; authenticate them
	// by signature instead (see the OAuth pages for token-based options).
	cop.AddInsecureBypassPattern("POST /webhooks/payments")
	h := cop.Handler(mux)

	try := func(label, method, target string, hdr map[string]string) {
		req := httptest.NewRequest(method, target, nil)
		for k, v := range hdr {
			req.Header.Set(k, v)
		}
		rec := httptest.NewRecorder()
		h.ServeHTTP(rec, req)
		fmt.Printf("%-34s %d %s", label, rec.Code, rec.Body.String())
	}
	try("curl, no browser headers", "POST", "/loans", nil)
	try("same-origin form post", "POST", "/loans", map[string]string{"Sec-Fetch-Site": "same-origin"})
	try("cross-site form post", "POST", "/loans", map[string]string{"Sec-Fetch-Site": "cross-site"})
	try("cross-site GET (safe method)", "GET", "/loans", map[string]string{"Sec-Fetch-Site": "cross-site"})
	try("old browser, other Origin", "POST", "/loans", map[string]string{"Origin": "https://evil.example"})
	try("old browser, same Origin", "POST", "/loans", map[string]string{"Origin": "http://example.com"})
	try("trusted admin origin", "POST", "/loans", map[string]string{"Origin": "https://admin.shelf.example", "Sec-Fetch-Site": "cross-site"})
	try("bypass pattern, cross-site", "POST", "/webhooks/payments", map[string]string{"Sec-Fetch-Site": "cross-site"})

	err := cop.Check(func() *http.Request {
		r := httptest.NewRequest("POST", "/loans", nil)
		r.Header.Set("Sec-Fetch-Site", "cross-site")
		return r
	}())
	fmt.Println("Check:", err)
}

curl, no browser headers           200 loan created
same-origin form post              200 loan created
cross-site form post               403 cross-origin request detected from Sec-Fetch-Site header
cross-site GET (safe method)       200 loan list
old browser, other Origin          403 cross-origin request detected, and/or browser is out of date: Sec-Fetch-Site is missing, and Origin does not match Host
old browser, same Origin           200 loan created
trusted admin origin               200 loan created
bypass pattern, cross-site         200 webhook accepted
Check: cross-origin request detected from Sec-Fetch-Site header

package main

import (
	"bufio"
	"fmt"
	"io"
	"net"
	"net/http"
	"strings"
	"time"
)

func main() {
	mux := http.NewServeMux()
	mux.HandleFunc("GET /fast", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, "fast")
	})
	mux.HandleFunc("GET /slow", func(w http.ResponseWriter, r *http.Request) {
		time.Sleep(600 * time.Millisecond) // longer than WriteTimeout
		fmt.Fprintln(w, "slow but finished")
	})
	mux.HandleFunc("GET /export", func(w http.ResponseWriter, r *http.Request) {
		// This one endpoint is allowed more time than the server default.
		rc := http.NewResponseController(w)
		if err := rc.SetWriteDeadline(time.Now().Add(5 * time.Second)); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		time.Sleep(600 * time.Millisecond)
		fmt.Fprintln(w, "export finished")
	})

	srv := &http.Server{
		Handler:           mux,
		ReadHeaderTimeout: 200 * time.Millisecond,
		WriteTimeout:      300 * time.Millisecond,
		IdleTimeout:       300 * time.Millisecond,
	}
	ln, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		panic(err)
	}
	go srv.Serve(ln)
	defer srv.Close()
	addr := ln.Addr().String()

	// 1. ReadHeaderTimeout: a client that sends half a request line and stalls.
	conn, _ := net.Dial("tcp", addr)
	fmt.Fprint(conn, "GET /fast HTTP/1.1\r\nHost: x\r\nX-Slow: ")
	conn.SetReadDeadline(time.Now().Add(3 * time.Second))
	reply, err := io.ReadAll(conn)
	conn.Close()
	fmt.Printf("stalled header: %d bytes received, connection closed by server: %t\n", len(reply), err == nil)

	// 2. WriteTimeout: the handler outlives the write deadline.
	for _, path := range []string{"/fast", "/slow", "/export"} {
		resp, err := http.Get("http://" + addr + path)
		if err != nil {
			fmt.Printf("GET %-7s error: %v\n", path, strings.TrimPrefix(err.Error(), `Get "http://`+addr+path+`": `))
			continue
		}
		body, _ := io.ReadAll(resp.Body)
		resp.Body.Close()
		fmt.Printf("GET %-7s %d %q\n", path, resp.StatusCode, body)
	}

	// 3. IdleTimeout: an idle keep-alive connection is closed by the server.
	conn, _ = net.Dial("tcp", addr)
	fmt.Fprint(conn, "GET /fast HTTP/1.1\r\nHost: x\r\n\r\n")
	br := bufio.NewReader(conn)
	resp, _ := http.ReadResponse(br, nil)
	io.Copy(io.Discard, resp.Body)
	resp.Body.Close()
	conn.SetReadDeadline(time.Now().Add(3 * time.Second))
	_, err = br.ReadByte()
	fmt.Printf("idle connection: first response %d, then %v\n", resp.StatusCode, err)
	conn.Close()
}

stalled header: 0 bytes received, connection closed by server: true
GET /fast   200 "fast\n"
GET /slow   error: EOF
GET /export 200 "export finished\n"
idle connection: first response 200, then EOF

package main

import (
	"bufio"
	"fmt"
	"io"
	"net/http"
	"net/http/httptest"
	"strings"
	"time"
)

func main() {
	mux := http.NewServeMux()

	// Server-Sent Events: flush after every event so the client sees it now.
	mux.HandleFunc("GET /events", func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "text/event-stream")
		w.Header().Set("Cache-Control", "no-cache")
		rc := http.NewResponseController(w)
		for i := 1; i <= 3; i++ {
			select {
			case <-r.Context().Done(): // client went away
				return
			case <-time.After(20 * time.Millisecond):
			}
			fmt.Fprintf(w, "event: loan\ndata: book %d lent\n\n", i)
			if err := rc.Flush(); err != nil {
				return // the writer cannot flush (wrapped without Unwrap)
			}
		}
	})

	// TimeoutHandler: the whole handler gets 100 ms, then the client sees a 503.
	slow := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		select {
		case <-r.Context().Done(): // TimeoutHandler cancels the context
		case <-time.After(2 * time.Second):
		}
	})
	mux.Handle("GET /report", http.TimeoutHandler(slow, 100*time.Millisecond, "report took too long\n"))

	// MaxBytesHandler: a limit for the whole subtree, applied before any handler.
	upload := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		n, err := io.Copy(io.Discard, r.Body)
		if err != nil {
			http.Error(w, err.Error(), http.StatusRequestEntityTooLarge)
			return
		}
		fmt.Fprintf(w, "read %d bytes\n", n)
	})
	mux.Handle("POST /import", http.MaxBytesHandler(upload, 16))

	srv := httptest.NewServer(mux)
	defer srv.Close()

	resp, _ := http.Get(srv.URL + "/events")
	sc := bufio.NewScanner(resp.Body)
	for sc.Scan() {
		if sc.Text() != "" {
			fmt.Println("sse:", sc.Text())
		}
	}
	resp.Body.Close()

	resp, _ = http.Get(srv.URL + "/report")
	b, _ := io.ReadAll(resp.Body)
	resp.Body.Close()
	fmt.Printf("report: %d %q\n", resp.StatusCode, b)

	for _, size := range []int{10, 100} {
		resp, _ = http.Post(srv.URL+"/import", "text/plain", strings.NewReader(strings.Repeat("x", size)))
		b, _ = io.ReadAll(resp.Body)
		resp.Body.Close()
		fmt.Printf("import %3d bytes: %d %s", size, resp.StatusCode, b)
	}
}

sse: event: loan
sse: data: book 1 lent
sse: event: loan
sse: data: book 2 lent
sse: event: loan
sse: data: book 3 lent
report: 503 "report took too long\n"
import  10 bytes: 200 read 10 bytes
import 100 bytes: 413 http: request body too large

package main

import (
	"context"
	"errors"
	"fmt"
	"io"
	"log/slog"
	"net"
	"net/http"
	"os"
	"os/signal"
	"syscall"
	"time"
)

// run serves h on ln until ctx is cancelled, then drains in-flight requests.
func run(ctx context.Context, ln net.Listener, h http.Handler) error {
	srv := &http.Server{
		Handler:           h,
		ReadHeaderTimeout: 5 * time.Second,
		IdleTimeout:       60 * time.Second,
	}

	errc := make(chan error, 1)
	go func() {
		// Serve returns ErrServerClosed as soon as Shutdown is called.
		if err := srv.Serve(ln); !errors.Is(err, http.ErrServerClosed) {
			errc <- err
			return
		}
		errc <- nil
	}()

	select {
	case err := <-errc:
		return err // the listener failed
	case <-ctx.Done():
	}

	slog.Info("shutting down")
	shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	defer cancel()
	if err := srv.Shutdown(shutdownCtx); err != nil {
		srv.Close() // deadline passed: drop whatever is left
		return fmt.Errorf("graceful shutdown failed: %w", err)
	}
	return <-errc
}

func main() {
	slog.SetDefault(slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{
		ReplaceAttr: func(groups []string, a slog.Attr) slog.Attr {
			if a.Key == slog.TimeKey && len(groups) == 0 {
				return slog.Attr{}
			}
			return a
		},
	})))

	// SIGINT and SIGTERM cancel ctx; stop() restores default signal handling.
	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
	defer stop()

	started := make(chan struct{})
	mux := http.NewServeMux()
	mux.HandleFunc("GET /slow", func(w http.ResponseWriter, r *http.Request) {
		close(started)
		time.Sleep(300 * time.Millisecond)
		fmt.Fprintln(w, "finished the in-flight request")
	})

	ln, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		slog.Error("listen", "err", err)
		os.Exit(1)
	}

	done := make(chan error, 1)
	go func() { done <- run(ctx, ln, mux) }()

	// Demo client: start a request, then "press Ctrl+C" while it is in flight.
	got := make(chan string, 1)
	go func() {
		resp, err := http.Get("http://" + ln.Addr().String() + "/slow")
		if err != nil {
			got <- "client error: " + err.Error()
			return
		}
		defer resp.Body.Close()
		b, _ := io.ReadAll(resp.Body)
		got <- fmt.Sprintf("client got: %d %s", resp.StatusCode, b)
	}()
	<-started
	_ = syscall.Kill(os.Getpid(), syscall.SIGINT)

	fmt.Println("run returned:", <-done)
	fmt.Print(<-got)
	_, err = http.Get("http://" + ln.Addr().String() + "/slow")
	fmt.Println("after shutdown, new connections fail:", err != nil)
}

level=INFO msg="shutting down"
run returned: <nil>
client got: 200 finished the in-flight request
after shutdown, new connections fail: true

package main

import (
	"embed"
	"fmt"
	"io"
	"io/fs"
	"net/http"
	"net/http/httptest"
	"path"
	"strings"
)

//go:embed all:assets
var assets embed.FS

func main() {
	// Serve the files at /assets/ without the "assets" directory prefix of
	// the embedded tree being doubled.
	mux := http.NewServeMux()
	mux.Handle("GET /assets/", http.FileServerFS(assets))

	// The same tree, but only some files and no directory listings.
	sub, err := fs.Sub(assets, "assets")
	if err != nil {
		panic(err)
	}
	mux.Handle("GET /static/", http.StripPrefix("/static/", http.FileServerFS(noDirs{sub})))

	// One file with ServeFileFS (never put a request path into the name).
	mux.HandleFunc("GET /{$}", func(w http.ResponseWriter, r *http.Request) {
		http.ServeFileFS(w, r, assets, "assets/index.html")
	})

	srv := httptest.NewServer(mux)
	defer srv.Close()
	for _, p := range []string{"/", "/assets/shelf.css", "/assets/covers/", "/static/covers/", "/static/covers/dune.txt", "/assets/.notes.txt", "/static/.notes.txt", "/static/"} {
		resp, err := http.Get(srv.URL + p)
		if err != nil {
			fmt.Println(err)
			continue
		}
		b, _ := io.ReadAll(resp.Body)
		resp.Body.Close()
		first, _, _ := strings.Cut(strings.TrimSpace(string(b)), "\n")
		if strings.Contains(string(b), "<pre>") {
			first = "(directory listing)"
		}
		fmt.Printf("%-20s %d %-24s %s\n", p, resp.StatusCode, resp.Header.Get("Content-Type"), first)
	}
}

// noDirs hides directory listings and dot files behind a 404.
type noDirs struct{ fs fs.FS }

func (n noDirs) Open(name string) (fs.File, error) {
	if base := name[strings.LastIndex(name, "/")+1:]; base != "." && strings.HasPrefix(base, ".") {
		return nil, fs.ErrNotExist
	}
	f, err := n.fs.Open(name)
	if err != nil {
		return nil, err
	}
	if st, err := f.Stat(); err == nil && st.IsDir() {
		// Allow a directory only if it holds an index.html.
		idx, err := n.fs.Open(path.Join(name, "index.html"))
		if err != nil {
			f.Close()
			return nil, fs.ErrNotExist
		}
		idx.Close()
	}
	return f, nil
}

/                    200 text/html; charset=utf-8 <!doctype html>
/assets/shelf.css    200 text/css; charset=utf-8  body { font-family: sans-serif; }
/assets/covers/      200 text/html; charset=utf-8 (directory listing)
/static/covers/      404 text/plain; charset=utf-8 404 page not found
/static/covers/dune.txt 200 text/plain; charset=utf-8 cover art placeholder
/assets/.notes.txt   200 text/plain; charset=utf-8 internal notes
/static/.notes.txt   404 text/plain; charset=utf-8 404 page not found
/static/             200 text/html; charset=utf-8 <!doctype html>

package main

import (
	"fmt"
	"io"
	"net/http"
	"net/http/httptest"
	"net/http/httputil"
	"net/url"
)

func main() {
	backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, "backend saw %s %s, X-Forwarded-For set: %t, X-Secret present: %t\n",
			r.Method, r.URL.Path, r.Header.Get("X-Forwarded-For") != "", r.Header.Get("X-Secret") != "")
	}))
	defer backend.Close()
	target, _ := url.Parse(backend.URL)

	// Rewrite replaces the deprecated Director hook (Go 1.20 added it,
	// Go 1.26 deprecated Director).
	proxy := &httputil.ReverseProxy{
		Rewrite: func(r *httputil.ProxyRequest) {
			r.SetURL(target)             // route to the backend, joining the paths
			r.SetXForwarded()            // add X-Forwarded-For, -Host and -Proto
			r.Out.Header.Del("X-Secret") // never forward internal headers
		},
	}
	front := httptest.NewServer(proxy)
	defer front.Close()

	req, _ := http.NewRequest(http.MethodGet, front.URL+"/books", nil)
	req.Header.Set("X-Secret", "internal-only")
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		fmt.Println(err)
		return
	}
	defer resp.Body.Close()
	b, _ := io.ReadAll(resp.Body)
	fmt.Print(string(b))
}

backend saw GET /books, X-Forwarded-For set: true, X-Secret present: false