I/O, Files and the Filesystem

package main

import (
	"fmt"
	"io"
	"strings"
	"testing/iotest"
	"unicode"
)

// upperReader upper-cases everything read through it.
type upperReader struct {
	r io.Reader
}

func (u upperReader) Read(p []byte) (int, error) {
	n, err := u.r.Read(p)
	for i := 0; i < n; i++ {
		p[i] = byte(unicode.ToUpper(rune(p[i])))
	}
	return n, err // report n > 0 and err together, exactly as the inner reader did
}

func dump(label string, r io.Reader) {
	fmt.Println(label)
	buf := make([]byte, 8)
	for {
		n, err := r.Read(buf)
		fmt.Printf("  n=%d err=%v data=%q\n", n, err, buf[:n])
		if err != nil {
			return
		}
	}
}

func main() {
	title := "Go in Practice"
	dump("plain strings.Reader:", strings.NewReader(title))
	dump("upperReader:", upperReader{strings.NewReader(title)})
	dump("data and EOF in the same call:", upperReader{iotest.DataErrReader(strings.NewReader(title))})
	dump("one byte at a time:", iotest.OneByteReader(strings.NewReader("ISBN")))
}

plain strings.Reader:
  n=8 err=<nil> data="Go in Pr"
  n=6 err=<nil> data="actice"
  n=0 err=EOF data=""
upperReader:
  n=8 err=<nil> data="GO IN PR"
  n=6 err=<nil> data="ACTICE"
  n=0 err=EOF data=""
data and EOF in the same call:
  n=8 err=<nil> data="GO IN PR"
  n=6 err=EOF data="ACTICE"
one byte at a time:
  n=1 err=<nil> data="I"
  n=1 err=<nil> data="S"
  n=1 err=<nil> data="B"
  n=1 err=<nil> data="N"
  n=0 err=EOF data=""

package main

import (
	"fmt"
	"io"
	"os"
)

func main() {
	f, _ := os.CreateTemp("", "seek-*")
	defer os.Remove(f.Name())
	defer f.Close()

	f.WriteString("0123456789")
	pos, _ := f.Seek(0, io.SeekCurrent)
	fmt.Println("position after writing:", pos)

	// ReadAt ignores and does not move the file offset; it is safe to call concurrently.
	buf := make([]byte, 4)
	n, err := f.ReadAt(buf, 3)
	fmt.Printf("ReadAt(3): %q n=%d err=%v\n", buf[:n], n, err)
	n, err = f.ReadAt(buf, 8)
	fmt.Printf("ReadAt(8): %q n=%d err=%v\n", buf[:n], n, err)

	// Seek moves the offset used by Read and Write.
	f.Seek(-3, io.SeekEnd)
	rest, _ := io.ReadAll(f)
	fmt.Printf("last three bytes: %q\n", rest)

	// WriteAt writes at an offset without touching the file offset.
	f.WriteAt([]byte("AB"), 1)
	f.Seek(0, io.SeekStart)
	all, _ := io.ReadAll(f)
	fmt.Printf("after WriteAt: %s\n", all)

	// Truncate shrinks (or extends with zero bytes) to the given size.
	f.Truncate(4)
	st, _ := f.Stat()
	fmt.Println("size after Truncate(4):", st.Size())

	// Writing past the end leaves a hole filled with zero bytes.
	f.WriteAt([]byte("Z"), 6)
	f.Seek(0, io.SeekStart)
	all, _ = io.ReadAll(f)
	fmt.Printf("%q\n", all)
}

position after writing: 10
ReadAt(3): "3456" n=4 err=<nil>
ReadAt(8): "89" n=2 err=EOF
last three bytes: "789"
after WriteAt: 0AB3456789
size after Truncate(4): 4
"0AB3\x00\x00Z"

package main

import (
	"fmt"
	"io"
	"strings"
)

// loud reports which interface io.Copy picked.
type loud struct{ *strings.Reader }

func (l loud) WriteTo(w io.Writer) (int64, error) {
	fmt.Println("  io.Copy used WriterTo")
	return l.Reader.WriteTo(w)
}

type sink struct{ n int }

func (s *sink) Write(p []byte) (int, error) { s.n += len(p); return len(p), nil }

type sinkFrom struct{ sink }

func (s *sinkFrom) ReadFrom(r io.Reader) (int64, error) {
	fmt.Println("  io.Copy used ReaderFrom")
	n, err := io.Copy(&s.sink, struct{ io.Reader }{r}) // hide the interfaces to avoid recursion
	return n, err
}

func main() {
	fmt.Println("source has WriterTo:")
	io.Copy(&sink{}, loud{strings.NewReader("abc")})

	fmt.Println("both ends: the source's WriteTo wins over the destination's ReadFrom:")
	io.Copy(&sinkFrom{}, loud{strings.NewReader("abc")})

	fmt.Println("destination has ReadFrom, source hidden:")
	io.Copy(&sinkFrom{}, struct{ io.Reader }{strings.NewReader("abc")})

	fmt.Println("both hidden behind plain wrappers (uses a 32 KiB buffer loop):")
	n, _ := io.Copy(&sink{}, struct{ io.Reader }{strings.NewReader("abc")})
	fmt.Println("  copied", n)

	fmt.Println("CopyBuffer with a small custom buffer:")
	buf := make([]byte, 2)
	n, err := io.CopyBuffer(&sink{}, struct{ io.Reader }{strings.NewReader("abcdef")}, buf)
	fmt.Println("  copied", n, err)
}

source has WriterTo:
  io.Copy used WriterTo
both ends: the source's WriteTo wins over the destination's ReadFrom:
  io.Copy used WriterTo
destination has ReadFrom, source hidden:
  io.Copy used ReaderFrom
both hidden behind plain wrappers (uses a 32 KiB buffer loop):
  copied 3
CopyBuffer with a small custom buffer:
  copied 6 <nil>

package main

import (
	"errors"
	"fmt"
	"io"
)

func main() {
	// The writer goroutine produces; the reader consumes. There is no buffer:
	// every Write blocks until the reads have consumed all of its bytes.
	pr, pw := io.Pipe()
	go func() {
		for _, title := range []string{"Go in Action", "Concurrency in Go", "Learning Go"} {
			fmt.Fprintln(pw, title)
		}
		pw.Close() // the reader now sees io.EOF
	}()
	b, err := io.ReadAll(pr)
	fmt.Printf("err=%v\n%s", err, b)

	// CloseWithError delivers a custom error to the reader.
	pr2, pw2 := io.Pipe()
	boom := errors.New("export aborted")
	go func() {
		pw2.Write([]byte("partial"))
		pw2.CloseWithError(boom)
	}()
	b, err = io.ReadAll(pr2)
	fmt.Printf("got %q, err=%v, is boom: %t\n", b, err, errors.Is(err, boom))

	// If the reader goes away, the writer gets io.ErrClosedPipe (or the reader's error).
	pr3, pw3 := io.Pipe()
	pr3.Close()
	_, err = pw3.Write([]byte("nobody listens"))
	fmt.Println("write after reader closed:", err, errors.Is(err, io.ErrClosedPipe))
}

err=<nil>
Go in Action
Concurrency in Go
Learning Go
got "partial", err=export aborted, is boom: true
write after reader closed: io: read/write on closed pipe true

package main

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"strings"
)

func main() {
	in := "978-0-13-419044-0|The Go Programming Language|Donovan\nlast line without newline"
	r := bufio.NewReaderSize(strings.NewReader(in), 16) // 16 is the minimum buffer size

	// Peek looks ahead without consuming.
	head, err := r.Peek(5)
	fmt.Printf("Peek: %q %v\n", head, err)

	// ReadString consumes up to and including the delimiter.
	isbn, err := r.ReadString('|')
	fmt.Printf("ReadString: %q %v\n", isbn, err)

	// ReadRune decodes UTF-8; UnreadRune undoes exactly one.
	ru, size, _ := r.ReadRune()
	fmt.Printf("ReadRune: %q size=%d\n", ru, size)
	r.UnreadRune()

	// ReadSlice returns a view into the buffer and fails when the delimiter is further away than the buffer.
	_, err = r.ReadSlice('\n')
	fmt.Println("ReadSlice with a 16-byte buffer:", err, errors.Is(err, bufio.ErrBufferFull))

	// ReadLine reports isPrefix when the line is longer than the buffer.
	r = bufio.NewReaderSize(strings.NewReader("abcdefghijklmnopqrstuvwxyz\nshort\n"), 16)
	for {
		line, isPrefix, err := r.ReadLine()
		if err != nil {
			fmt.Println("ReadLine:", err)
			break
		}
		fmt.Printf("ReadLine: %q prefix=%t\n", line, isPrefix)
	}

	// ReadString has no such limit and returns the data together with io.EOF.
	r = bufio.NewReader(strings.NewReader(in))
	for {
		line, err := r.ReadString('\n')
		fmt.Printf("line %q err=%v\n", line, err)
		if err == io.EOF {
			break
		}
	}
	fmt.Println("buffered now:", r.Buffered())
}

Peek: "978-0" <nil>
ReadString: "978-0-13-419044-0|" <nil>
ReadRune: 'T' size=1
ReadSlice with a 16-byte buffer: bufio: buffer full true
ReadLine: "abcdefghijklmnop" prefix=true
ReadLine: "qrstuvwxyz" prefix=false
ReadLine: "short" prefix=false
ReadLine: EOF
line "978-0-13-419044-0|The Go Programming Language|Donovan\n" err=<nil>
line "last line without newline" err=EOF
buffered now: 0

package main

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
)

type failingWriter struct{ left int }

var errDiskFull = errors.New("disk full")

func (f *failingWriter) Write(p []byte) (int, error) {
	if len(p) > f.left {
		n := f.left
		f.left = 0
		return n, errDiskFull
	}
	f.left -= len(p)
	return len(p), nil
}

func main() {
	var dst bytes.Buffer
	w := bufio.NewWriterSize(&dst, 32)

	w.WriteString("ISBN,Title\n")
	fmt.Printf("after write: buffered=%d available=%d destination=%q\n", w.Buffered(), w.Available(), dst.String())

	w.Flush()
	fmt.Printf("after Flush: buffered=%d destination=%q\n", w.Buffered(), dst.String())

	// Writes larger than the free space flush to the destination on their own.
	w.WriteString("978-0-13-419044-0,The Go Programming Language\n")
	fmt.Printf("big write: buffered=%d destination bytes=%d\n", w.Buffered(), dst.Len())

	// Forgetting Flush loses the tail.
	w.WriteString("tail")
	fmt.Printf("never flushed: %d bytes still in the buffer\n", w.Buffered())

	// Errors are sticky: after the first failure every call returns it, and Flush reports it.
	fw := &failingWriter{left: 10}
	bw := bufio.NewWriterSize(fw, 16)
	for i := range 4 {
		_, err := bw.WriteString("0123456789")
		fmt.Printf("write %d: err=%v\n", i, err)
	}
	fmt.Println("Flush:", bw.Flush())
}

after write: buffered=11 available=21 destination=""
after Flush: buffered=0 destination="ISBN,Title\n"
big write: buffered=0 destination bytes=57
never flushed: 4 bytes still in the buffer
write 0: err=<nil>
write 1: err=disk full
write 2: err=disk full
write 3: err=disk full
Flush: disk full

package main

import (
	"bufio"
	"fmt"
	"strings"
)

func main() {
	text := "The Go Programming Language\nLearning Go\r\n\nConcurrency in Go"

	// ScanLines (the default) strips \n and an optional \r; empty lines are tokens.
	sc := bufio.NewScanner(strings.NewReader(text))
	for sc.Scan() {
		fmt.Printf("line %q\n", sc.Text())
	}
	if err := sc.Err(); err != nil {
		fmt.Println("error:", err)
	}

	// ScanWords splits on Unicode white space.
	sc = bufio.NewScanner(strings.NewReader(text))
	sc.Split(bufio.ScanWords)
	words := 0
	for sc.Scan() {
		words++
	}
	fmt.Println("words:", words)

	// ScanRunes yields one UTF-8 sequence per token; ScanBytes one byte.
	sc = bufio.NewScanner(strings.NewReader("Ünï"))
	sc.Split(bufio.ScanRunes)
	for sc.Scan() {
		fmt.Printf("%q ", sc.Text())
	}
	fmt.Println()

	// Bytes returns a slice that the next Scan overwrites; copy it if you keep it.
	sc = bufio.NewScanner(strings.NewReader("ab\ncd\n"))
	var kept [][]byte
	var cloned [][]byte
	for sc.Scan() {
		kept = append(kept, sc.Bytes())
		cloned = append(cloned, append([]byte(nil), sc.Bytes()...))
	}
	fmt.Printf("kept without copying: %q, cloned: %q\n", kept, cloned)
}

line "The Go Programming Language"
line "Learning Go"
line ""
line "Concurrency in Go"
words: 9
"Ü" "n" "ï"
kept without copying: ["ab" "cd"], cloned: ["ab" "cd"]

package main

import (
	"errors"
	"fmt"
	"io/fs"
	"os"
	"strings"
)

func main() {
	dir, _ := os.MkdirTemp("", "shelf-*")
	defer os.RemoveAll(dir)
	show := func(what string, err error) {
		fmt.Printf("%-26s %s\n", what, strings.ReplaceAll(fmt.Sprint(err), dir, "$DIR"))
	}

	// Reading a directory as if it were a file.
	_, err := os.ReadFile(dir)
	show("ReadFile on a directory:", err)

	// Opening a directory for writing.
	_, err = os.OpenFile(dir, os.O_WRONLY, 0)
	show("OpenFile(O_WRONLY) dir:", err)

	// Exclusive creation.
	p := dir + "/a.txt"
	os.WriteFile(p, nil, 0o644)
	_, err = os.OpenFile(p, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o644)
	show("O_EXCL on existing file:", err)
	fmt.Println("  errors.Is(fs.ErrExist):", errors.Is(err, fs.ErrExist))

	// Permission errors (skipped when running as root).
	os.Chmod(p, 0o000)
	_, err = os.Open(p)
	show("Open without permission:", err)
	fmt.Println("  errors.Is(fs.ErrPermission):", errors.Is(err, fs.ErrPermission), "as root:", os.Geteuid() == 0)
	os.Chmod(p, 0o644)

	// Removing a non-empty directory.
	os.Mkdir(dir+"/d", 0o755)
	os.WriteFile(dir+"/d/x", nil, 0o644)
	show("Remove non-empty dir:", os.Remove(dir+"/d"))

	// An empty file is not an error: ReadFile returns an empty slice.
	b, err := os.ReadFile(p)
	fmt.Printf("%-26s len=%d err=%v\n", "ReadFile on empty file:", len(b), err)

	// A *os.File that is nil.
	var f *os.File
	_, err = f.Write([]byte("x"))
	show("Write on nil *os.File:", err)
	fmt.Println("  errors.Is(os.ErrInvalid):", errors.Is(err, os.ErrInvalid))
}

ReadFile on a directory:   read $DIR: is a directory
OpenFile(O_WRONLY) dir:    open $DIR: is a directory
O_EXCL on existing file:   open $DIR/a.txt: file exists
  errors.Is(fs.ErrExist): true
Open without permission:   open $DIR/a.txt: permission denied
  errors.Is(fs.ErrPermission): true as root: false
Remove non-empty dir:      remove $DIR/d: directory not empty
ReadFile on empty file:    len=0 err=<nil>
Write on nil *os.File:     invalid argument
  errors.Is(os.ErrInvalid): true

package main

import (
	"errors"
	"fmt"
	"os"
	"time"
)

func main() {
	// Deadlines work on pipes and sockets (pollable descriptors)...
	r, w, _ := os.Pipe()
	defer r.Close()
	defer w.Close()

	r.SetReadDeadline(time.Now().Add(20 * time.Millisecond))
	buf := make([]byte, 8)
	_, err := r.Read(buf) // nobody writes, so the deadline fires
	fmt.Println("pipe read:", err, errors.Is(err, os.ErrDeadlineExceeded))

	r.SetReadDeadline(time.Time{}) // clear the deadline
	w.WriteString("ready")
	n, err := r.Read(buf)
	fmt.Printf("pipe read after write: %q %v\n", buf[:n], err)

	// ...but not on regular files.
	f, _ := os.CreateTemp("", "deadline-*")
	defer os.Remove(f.Name())
	defer f.Close()
	err = f.SetDeadline(time.Now().Add(time.Second))
	fmt.Println("regular file:", err, errors.Is(err, os.ErrNoDeadline))
}

pipe read: read |0: i/o timeout true
pipe read after write: "ready" <nil>
regular file: file type does not support deadline true

package main

import (
	"context"
	"errors"
	"fmt"
	"io"
	"strings"
)

// ctxReader stops reading when ctx is done. It can only check between Read calls:
// a Read that is already blocked inside the kernel is not interrupted.
type ctxReader struct {
	ctx context.Context
	r   io.Reader
}

func (c ctxReader) Read(p []byte) (int, error) {
	if err := c.ctx.Err(); err != nil {
		return 0, err
	}
	return c.r.Read(p)
}

// cancelAfter cancels the context once n bytes have been read.
type cancelAfter struct {
	r      io.Reader
	n      int
	cancel context.CancelFunc
}

func (c *cancelAfter) Read(p []byte) (int, error) {
	k, err := c.r.Read(p)
	c.n -= k
	if c.n <= 0 {
		c.cancel()
	}
	return k, err
}

func main() {
	ctx, cancel := context.WithCancel(context.Background())
	src := &cancelAfter{r: strings.NewReader(strings.Repeat("x", 100)), n: 10, cancel: cancel}

	buf := make([]byte, 4)
	r := ctxReader{ctx, src}
	total := 0
	for {
		n, err := r.Read(buf)
		total += n
		if err != nil {
			fmt.Println("stopped after", total, "bytes:", err, errors.Is(err, context.Canceled))
			return
		}
	}
}

stopped after 12 bytes: context canceled true

package main

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strings"
)

// writeLines writes lines to path. The deferred Close error is joined into the result,
// because for a file opened for writing Close is where a failed flush surfaces.
func writeLines(path string, lines []string) (err error) {
	f, err := os.Create(path)
	if err != nil {
		return err
	}
	defer func() {
		err = errors.Join(err, f.Close())
	}()
	_, err = f.WriteString(strings.Join(lines, "\n") + "\n")
	return err
}

func main() {
	dir, _ := os.MkdirTemp("", "shelf-*")
	defer os.RemoveAll(dir)

	p := filepath.Join(dir, "titles.txt")
	fmt.Println("first call:", writeLines(p, []string{"Learning Go", "Concurrency in Go"}))
	b, _ := os.ReadFile(p)
	fmt.Printf("%q\n", b)

	// A path in a directory that does not exist fails at Create.
	err := writeLines(filepath.Join(dir, "nope", "titles.txt"), nil)
	fmt.Println("missing directory:", strings.ReplaceAll(err.Error(), dir, "$DIR"))

	// Close twice: the second Close reports os.ErrClosed, which is why a deferred
	// Close on a read-only file can be left alone but a write path must not ignore it.
	f, _ := os.Open(p)
	fmt.Println("Close:", f.Close())
	err = f.Close()
	fmt.Println("second Close:", strings.ReplaceAll(err.Error(), dir, "$DIR"), errors.Is(err, os.ErrClosed))
	_, err = f.Write([]byte("x"))
	fmt.Println("Write after Close:", strings.ReplaceAll(err.Error(), dir, "$DIR"))
}

first call: <nil>
"Learning Go\nConcurrency in Go\n"
missing directory: open $DIR/nope/titles.txt: no such file or directory
Close: <nil>
second Close: close $DIR/titles.txt: file already closed true
Write after Close: write $DIR/titles.txt: file already closed

package main

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"
)

// writeFileAtomic replaces path so that readers see either the old or the new content, never a mixture.
func writeFileAtomic(path string, data []byte, perm os.FileMode) (err error) {
	dir := filepath.Dir(path)
	tmp, err := os.CreateTemp(dir, filepath.Base(path)+".*.tmp") // same directory: same filesystem
	if err != nil {
		return err
	}
	defer func() {
		if err != nil {
			tmp.Close()
			os.Remove(tmp.Name())
		}
	}()
	if _, err = tmp.Write(data); err != nil {
		return err
	}
	if err = tmp.Chmod(perm); err != nil {
		return err
	}
	if err = tmp.Sync(); err != nil { // flush to stable storage before the rename
		return err
	}
	if err = tmp.Close(); err != nil {
		return err
	}
	return os.Rename(tmp.Name(), path) // atomic on POSIX file systems
}

func main() {
	dir, _ := os.MkdirTemp("", "shelf-*")
	defer os.RemoveAll(dir)
	p := filepath.Join(dir, "state.json")

	fmt.Println(writeFileAtomic(p, []byte(`{"version":1}`), 0o600))
	fmt.Println(writeFileAtomic(p, []byte(`{"version":2}`), 0o600))

	b, _ := os.ReadFile(p)
	st, _ := os.Stat(p)
	fmt.Printf("%s %v\n", b, st.Mode())

	entries, _ := os.ReadDir(dir)
	fmt.Println("files left in the directory:", len(entries))

	// A failure leaves the old content and no temporary file behind.
	err := writeFileAtomic(filepath.Join(dir, "missing", "state.json"), nil, 0o600)
	fmt.Println("unwritable target:", errors.Is(err, os.ErrNotExist))
}

<nil>
<nil>
{"version":2} -rw-------
files left in the directory: 1
unwritable target: true

package main

import (
	"fmt"
	"os"
	"strings"
)

func main() {
	os.Setenv("SHELF_DB", "postgres://localhost/shelf")
	os.Unsetenv("SHELF_DEBUG")

	// Getenv cannot tell unset from empty; LookupEnv can.
	fmt.Printf("Getenv(SHELF_DEBUG) = %q\n", os.Getenv("SHELF_DEBUG"))
	v, ok := os.LookupEnv("SHELF_DEBUG")
	fmt.Printf("LookupEnv(SHELF_DEBUG) = %q, %t\n", v, ok)
	os.Setenv("SHELF_DEBUG", "")
	v, ok = os.LookupEnv("SHELF_DEBUG")
	fmt.Printf("after Setenv to empty: %q, %t\n", v, ok)

	// ExpandEnv and Expand substitute $VAR and ${VAR}.
	fmt.Println(os.ExpandEnv("db=${SHELF_DB} user=$SHELF_USER."))
	fmt.Println(os.Expand("$ISBN is ${TITLE}", func(name string) string {
		return map[string]string{"ISBN": "978-0-13-419044-0", "TITLE": "The Go Programming Language"}[name]
	}))

	// Environ returns "KEY=value" strings.
	n := 0
	for _, kv := range os.Environ() {
		if strings.HasPrefix(kv, "SHELF_") {
			n++
		}
	}
	fmt.Println("SHELF_ variables:", n)

	// Directories that depend on the platform and the environment.
	home, err := os.UserHomeDir()
	fmt.Println("UserHomeDir ok:", home != "" && err == nil)
	cfg, err := os.UserConfigDir()
	fmt.Println("UserConfigDir ok:", cfg != "" && err == nil)
	fmt.Println("TempDir ok:", os.TempDir() != "")
	wd, _ := os.Getwd()
	exe, _ := os.Executable()
	fmt.Println("Getwd and Executable ok:", wd != "" && exe != "")

	// Arguments: Args[0] is the program name.
	fmt.Println("len(os.Args):", len(os.Args))

	// Standard streams are *os.File values.
	fmt.Fprintln(os.Stderr, "this line goes to standard error")
	fi, _ := os.Stdin.Stat()
	fmt.Println("stdin is a terminal:", fi.Mode()&os.ModeCharDevice != 0)
}

Getenv(SHELF_DEBUG) = ""
LookupEnv(SHELF_DEBUG) = "", false
after Setenv to empty: "", true
db=postgres://localhost/shelf user=.
978-0-13-419044-0 is The Go Programming Language
SHELF_ variables: 2
UserHomeDir ok: true
UserConfigDir ok: true
TempDir ok: true
Getwd and Executable ok: true
len(os.Args): 1
this line goes to standard error
stdin is a terminal: false

package main

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"strings"
)

// run is the testable core: it takes the streams as parameters instead of using os.Stdin and os.Stdout.
func run(in io.Reader, out, errOut io.Writer, args []string) int {
	w := bufio.NewWriter(out)
	defer w.Flush()

	sc := bufio.NewScanner(in)
	n := 0
	for sc.Scan() {
		line := strings.TrimSpace(sc.Text())
		if line == "" {
			continue
		}
		n++
		fmt.Fprintf(w, "%d: %s\n", n, strings.ToUpper(line))
	}
	if err := sc.Err(); err != nil {
		fmt.Fprintln(errOut, "shelf: reading input:", err)
		return 1
	}
	if n == 0 {
		fmt.Fprintln(errOut, "shelf: no input")
		return 2
	}
	return 0
}

func main() {
	// Typical wiring: the process streams go in here, and the exit code comes out.
	code := run(strings.NewReader("learning go\n\nconcurrency in go\n"), os.Stdout, os.Stderr, os.Args[1:])
	fmt.Println("exit code would be", code)
	code = run(strings.NewReader("\n\n"), os.Stdout, os.Stdout, nil)
	fmt.Println("exit code would be", code)
	// In a real main: os.Exit(run(os.Stdin, os.Stdout, os.Stderr, os.Args[1:])) -- after deferred work is done.
}

1: LEARNING GO
2: CONCURRENCY IN GO
exit code would be 0
shelf: no input
exit code would be 2

// upper.go
// Package upper provides a reader that upper-cases ASCII text.
package upper

import "io"

type reader struct{ r io.Reader }

// NewReader returns a reader that upper-cases the ASCII letters read from r.
func NewReader(r io.Reader) io.Reader { return reader{r} }

func (u reader) Read(p []byte) (int, error) {
	n, err := u.r.Read(p)
	for i := range p[:n] {
		if c := p[i]; 'a' <= c && c <= 'z' {
			p[i] = c - 'a' + 'A'
		}
	}
	return n, err
}

// upper_test.go
package upper_test

import (
	"errors"
	"io"
	"strings"
	"testing"
	"testing/iotest"

	"example.com/shelf/ex/upper"
)

func TestReaderContract(t *testing.T) {
	// TestReader reads the stream in many ways (Read, ReadAt, Seek...) and checks the content and the contract.
	if err := iotest.TestReader(upper.NewReader(strings.NewReader("LEARNING GO")), []byte("LEARNING GO")); err != nil {
		t.Fatal(err)
	}
}

func TestUpper(t *testing.T) {
	const in = "the go programming language"
	for name, wrap := range map[string]func(io.Reader) io.Reader{
		"plain":      func(r io.Reader) io.Reader { return r },
		"one byte":   iotest.OneByteReader,
		"data+EOF":   iotest.DataErrReader,
		"half reads": iotest.HalfReader,
	} {
		t.Run(name, func(t *testing.T) {
			got, err := io.ReadAll(upper.NewReader(wrap(strings.NewReader(in))))
			if err != nil || string(got) != strings.ToUpper(in) {
				t.Fatalf("got %q, %v", got, err)
			}
		})
	}
}

func TestReadError(t *testing.T) {
	boom := errors.New("boom")
	_, err := io.ReadAll(upper.NewReader(iotest.ErrReader(boom)))
	if !errors.Is(err, boom) {
		t.Fatalf("err = %v, want %v", err, boom)
	}
}

=== RUN   TestReaderContract
--- PASS: TestReaderContract (0.00s)
=== RUN   TestUpper
=== RUN   TestUpper/plain
=== RUN   TestUpper/one_byte
=== RUN   TestUpper/data+EOF
=== RUN   TestUpper/half_reads
--- PASS: TestUpper (0.00s)
    --- PASS: TestUpper/plain (0.00s)
    --- PASS: TestUpper/one_byte (0.00s)
    --- PASS: TestUpper/data+EOF (0.00s)
    --- PASS: TestUpper/half_reads (0.00s)
=== RUN   TestReadError
--- PASS: TestReadError (0.00s)
PASS
ok  	example.com/shelf/ex/upper	0.181s

package main

import (
	"fmt"
	"io/fs"
	"os"
	"path/filepath"
)

func main() {
	dir, _ := os.MkdirTemp("", "shelf-*")
	defer os.RemoveAll(dir)
	os.Chdir(dir)
	for _, p := range []string{"a/one.json", "a/two.txt", "b/three.json", ".git/config", "b/deep/four.json"} {
		os.MkdirAll(filepath.Dir(p), 0o755)
		os.WriteFile(p, nil, 0o644)
	}
	os.Symlink("a", "link-to-a")

	// WalkDir visits in lexical order, calls fn for the root first, and does not follow symlinks.
	err := filepath.WalkDir(".", func(p string, d fs.DirEntry, err error) error {
		if err != nil {
			return err // for example a directory that cannot be read
		}
		if d.IsDir() && d.Name() == ".git" {
			return filepath.SkipDir // do not descend
		}
		kind := "file"
		switch {
		case d.IsDir():
			kind = "dir"
		case d.Type()&fs.ModeSymlink != 0:
			kind = "symlink"
		}
		fmt.Printf("%-18s %s\n", p, kind)
		return nil
	})
	fmt.Println("WalkDir:", err)

	// SkipAll (Go 1.20) stops the whole walk successfully.
	n := 0
	err = filepath.WalkDir(".", func(p string, d fs.DirEntry, err error) error {
		if filepath.Ext(p) == ".json" {
			n++
			if n == 2 {
				return filepath.SkipAll
			}
		}
		return nil
	})
	fmt.Println("stopped after", n, "JSON files, err =", err)

	// Glob returns the matches of one pattern level at a time (no **).
	m, _ := filepath.Glob("*/*.json")
	fmt.Println(m)

	// EvalSymlinks resolves links and cleans the result.
	resolved, err := filepath.EvalSymlinks("link-to-a/one.json")
	fmt.Println(resolved, err)
}

.                  dir
a                  dir
a/one.json         file
a/two.txt          file
b                  dir
b/deep             dir
b/deep/four.json   file
b/three.json       file
link-to-a          symlink
WalkDir: <nil>
stopped after 2 JSON files, err = <nil>
[a/one.json b/three.json link-to-a/one.json]
a/one.json <nil>

package main

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
)

// naiveJoin is the classic hand-rolled check: join, then test the prefix.
func naiveJoin(base, name string) (string, error) {
	p := filepath.Join(base, name)
	if !strings.HasPrefix(p, filepath.Clean(base)+string(filepath.Separator)) {
		return "", fmt.Errorf("%q escapes %s", name, filepath.Base(base))
	}
	return p, nil
}

func main() {
	for _, name := range []string{"loans.json", "2026/loans.json", "a/../b", "../x", "a/../../x", "/etc/passwd", "", ".", "..", "a/./b"} {
		fmt.Printf("IsLocal(%q) = %t\n", name, filepath.IsLocal(name))
	}

	// Localize converts a slash path (such as an fs.FS or URL path) to an OS path or fails.
	for _, name := range []string{"2026/loans.json", "../x", `a\b`} {
		p, err := filepath.Localize(name)
		fmt.Printf("Localize(%q) = %q, %v\n", name, p, err)
	}

	// IsLocal and the prefix check are lexical: neither sees symbolic links.
	base, _ := os.MkdirTemp("", "shelf-*")
	defer os.RemoveAll(base)
	uploads := filepath.Join(base, "uploads")
	os.Mkdir(uploads, 0o755)
	os.WriteFile(filepath.Join(base, "secret.txt"), []byte("outside"), 0o644)
	os.Symlink("..", filepath.Join(uploads, "up"))

	name := "up/secret.txt"
	fmt.Println("IsLocal:", filepath.IsLocal(name))
	p, err := naiveJoin(uploads, name)
	fmt.Println("naive check passes:", err == nil)
	b, _ := os.ReadFile(p)
	fmt.Printf("os.ReadFile read %q\n", b)

	_, err = os.OpenInRoot(uploads, name)
	fmt.Println("os.OpenInRoot:", strings.ReplaceAll(err.Error(), base, "$BASE"))
}

IsLocal("loans.json") = true
IsLocal("2026/loans.json") = true
IsLocal("a/../b") = true
IsLocal("../x") = false
IsLocal("a/../../x") = false
IsLocal("/etc/passwd") = false
IsLocal("") = false
IsLocal(".") = true
IsLocal("..") = false
IsLocal("a/./b") = true
Localize("2026/loans.json") = "2026/loans.json", <nil>
Localize("../x") = "", invalid path
Localize("a\\b") = "a\\b", <nil>
IsLocal: true
naive check passes: true
os.ReadFile read "outside"
os.OpenInRoot: openat up/secret.txt: path escapes from parent

package main

import (
	"fmt"
	"io/fs"
	"os"
	"path"
	"testing/fstest"
	"time"
)

// countTitles accepts any fs.FS: the OS, an embed.FS, a zip archive or a test double.
func countTitles(fsys fs.FS, dir string) (int, error) {
	n := 0
	err := fs.WalkDir(fsys, dir, func(p string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		if !d.IsDir() && path.Ext(p) == ".txt" {
			b, err := fs.ReadFile(fsys, p)
			if err != nil {
				return err
			}
			if len(b) > 0 {
				n++
			}
		}
		return nil
	})
	return n, err
}

func main() {
	mem := fstest.MapFS{
		"shelf/go.txt":       {Data: []byte("The Go Programming Language"), ModTime: time.Date(2026, 10, 18, 9, 30, 0, 0, time.UTC)},
		"shelf/rust.txt":     {Data: []byte("Programming Rust")},
		"shelf/empty.txt":    {Data: nil},
		"shelf/sub/more.txt": {Data: []byte("Learning Go")},
		"shelf/readme.md":    {Data: []byte("# shelf")},
	}
	fmt.Println(countTitles(mem, "shelf"))

	// The same function runs against a real directory.
	dir, _ := os.MkdirTemp("", "shelf-*")
	defer os.RemoveAll(dir)
	os.WriteFile(dir+"/a.txt", []byte("x"), 0o644)
	fmt.Println(countTitles(os.DirFS(dir), "."))

	// fs names are slash-separated, unrooted and clean: ValidPath says whether a name qualifies.
	for _, name := range []string{"a/b", "/a", "a/../b", "./a", "a//b", "."} {
		fmt.Printf("ValidPath(%q) = %t\n", name, fs.ValidPath(name))
	}
	_, err := mem.Open("/shelf/go.txt")
	fmt.Println(err)

	// Stat, ReadDir, Glob, Sub work on any FS.
	info, _ := fs.Stat(mem, "shelf/go.txt")
	fmt.Println(fs.FormatFileInfo(info))
	entries, _ := fs.ReadDir(mem, "shelf")
	for _, e := range entries {
		fmt.Print(fs.FormatDirEntry(e), " ")
	}
	fmt.Println()
	matches, _ := fs.Glob(mem, "shelf/*.txt")
	fmt.Println(matches)
	sub, _ := fs.Sub(mem, "shelf/sub")
	b, _ := fs.ReadFile(sub, "more.txt")
	fmt.Println(string(b))
}

3 <nil>
1 <nil>
ValidPath("a/b") = true
ValidPath("/a") = false
ValidPath("a/../b") = false
ValidPath("./a") = false
ValidPath("a//b") = false
ValidPath(".") = true
open /shelf/go.txt: file does not exist
---------- 27 2026-10-18 09:30:00 go.txt
- empty.txt - go.txt - readme.md - rust.txt d sub/
[shelf/empty.txt shelf/go.txt shelf/rust.txt]
Learning Go

package main

import (
	"errors"
	"fmt"
	"io"
	"io/fs"
	"strings"
	"testing/fstest"
)

func hidden(name string) bool { return strings.HasSuffix(name, ".tmp") }

// naiveFS hides ".tmp" files in Open but forgets that directories list their children too.
type naiveFS struct{ inner fs.FS }

func (v naiveFS) Open(name string) (fs.File, error) {
	if hidden(name) {
		return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist}
	}
	return v.inner.Open(name)
}

// visibleFS also filters directory listings, both through Open and through ReadDir.
type visibleFS struct{ inner fs.FS }

func (v visibleFS) Open(name string) (fs.File, error) {
	if hidden(name) {
		return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist}
	}
	f, err := v.inner.Open(name)
	if err != nil {
		return nil, err
	}
	rd, ok := f.(fs.ReadDirFile)
	if !ok {
		return f, nil
	}
	all, err := rd.ReadDir(-1)
	if err != nil {
		f.Close()
		return nil, err
	}
	return &dirFile{File: f, entries: filter(all)}, nil
}

func (v visibleFS) ReadDir(name string) ([]fs.DirEntry, error) {
	entries, err := fs.ReadDir(v.inner, name)
	return filter(entries), err
}

func filter(entries []fs.DirEntry) []fs.DirEntry {
	var out []fs.DirEntry
	for _, e := range entries {
		if !hidden(e.Name()) {
			out = append(out, e)
		}
	}
	return out
}

// dirFile serves a precomputed, filtered listing through the fs.ReadDirFile contract.
type dirFile struct {
	fs.File
	entries []fs.DirEntry
	off     int
}

func (d *dirFile) ReadDir(n int) ([]fs.DirEntry, error) {
	rest := d.entries[d.off:]
	if n <= 0 {
		d.off = len(d.entries)
		return rest, nil
	}
	if len(rest) == 0 {
		return nil, io.EOF
	}
	n = min(n, len(rest))
	d.off += n
	return rest[:n], nil
}

func main() {
	mem := fstest.MapFS{
		"books.json":      {Data: []byte("[]")},
		"books.json.tmp":  {Data: []byte("half written")},
		"2026/loans.json": {Data: []byte("[]")},
	}

	err := fstest.TestFS(naiveFS{mem}, "books.json", "2026/loans.json")
	lines := strings.Split(err.Error(), "\n")
	fmt.Println("naive:", lines[0])
	fmt.Println("  ", lines[1])

	v := visibleFS{mem}
	entries, _ := fs.ReadDir(v, ".")
	for _, e := range entries {
		fmt.Println(e.Name())
	}
	_, err = fs.ReadFile(v, "books.json.tmp")
	fmt.Println(err, errors.Is(err, fs.ErrNotExist))
	fmt.Println("visible: TestFS:", fstest.TestFS(v, "books.json", "2026/loans.json"))
}

naive: TestFS found errors:
   books.json.tmp: Open: open books.json.tmp: file does not exist
2026
books.json
open books.json.tmp: file does not exist true
visible: TestFS: <nil>

package main

import (
	"errors"
	"fmt"
	"io/fs"
	"os"
	"path/filepath"
	"testing/fstest"
)

func main() {
	src := fstest.MapFS{
		"templates/book.html": {Data: []byte("<h1>{{.Title}}</h1>"), Mode: 0o644},
		"bin/export.sh":       {Data: []byte("#!/bin/sh\n"), Mode: 0o755},
		"README":              {Data: []byte("shelf\n")},
	}
	base, _ := os.MkdirTemp("", "shelf-*")
	defer os.RemoveAll(base)
	dst := filepath.Join(base, "site") // CopyFS creates dst when it is missing

	fmt.Println("first copy:", os.CopyFS(dst, src))

	filepath.WalkDir(dst, func(p string, d fs.DirEntry, err error) error {
		rel, _ := filepath.Rel(dst, p)
		info, _ := d.Info()
		fmt.Printf("  %-22s %v\n", rel, info.Mode().Perm())
		return nil
	})

	// It never overwrites: an existing file makes the whole call fail with fs.ErrExist.
	err := os.CopyFS(dst, src)
	fmt.Println("second copy is ErrExist:", errors.Is(err, fs.ErrExist))
}

first copy: <nil>
  .                      -rwxr-xr-x
  README                 -rw-r--r--
  bin                    -rwxr-xr-x
  bin/export.sh          -rwxr-xr-x
  templates              -rwxr-xr-x
  templates/book.html    -rw-r--r--
second copy is ErrExist: true