Strings, Runes and UTF-8

package main

import "fmt"

func main() {
	title := "Café 日本"

	fmt.Println("len (bytes):", len(title))
	for i := range len(title) {
		fmt.Printf("%#02x ", title[i])
	}
	fmt.Println()
	fmt.Printf("title[3] = %#x, title[4] = %#x\n", title[3], title[4])

	// Slicing counts bytes, so it can cut a character in half.
	fmt.Printf("%q\n", title[:4])
	fmt.Printf("%q\n", title[:5])
	fmt.Printf("%q\n", title[6:])
}

len (bytes): 12
0x43 0x61 0x66 0xc3 0xa9 0x20 0xe6 0x97 0xa5 0xe6 0x9c 0xac
title[3] = 0xc3, title[4] = 0xa9
"Caf\xc3"
"Café"
"日本"

./main.go:7:2: cannot assign to s[0] (neither addressable nor a map index expression)

./main.go:7:8: invalid operation: cannot take address of title[0] (value of type byte)

package main

import "fmt"

func main() {
	title := "Café 日本"

	// range decodes UTF-8: i is the byte offset, r the rune (code point).
	for i, r := range title {
		fmt.Printf("byte %2d: %c  %U  %d\n", i, r, r, r)
	}

	// Converting to []rune yields one element per code point.
	runes := []rune(title)
	fmt.Println(len(title), len(runes), string(runes[5:]))

	// Invalid UTF-8: range yields U+FFFD and advances one byte.
	bad := "a\xffb\xc3"
	for i, r := range bad {
		fmt.Printf("%d: %U %q\n", i, r, r)
	}
}

byte  0: C  U+0043  67
byte  1: a  U+0061  97
byte  2: f  U+0066  102
byte  3: é  U+00E9  233
byte  5:    U+0020  32
byte  6: 日  U+65E5  26085
byte  9: 本  U+672C  26412
12 7 日本
0: U+0061 'a'
1: U+FFFD '�'
2: U+0062 'b'
3: U+FFFD '�'

package main

import "fmt"

func main() {
	// Rune literals are untyped rune constants (default type rune = int32).
	letters := []rune{'a', '\n', '\x41', '\u00e9', '\U0001F4DA', '\'', '\101', '日'}
	for _, r := range letters {
		fmt.Printf("%-12q %-10U %3d %T\n", r, r, r, r)
	}

	// Interpreted strings process escapes; raw strings do not.
	interpreted := "C:\\shelf\tbooks\n"
	raw := `C:\shelf\tbooks\n`
	fmt.Printf("%q\n%q\n", interpreted, raw)

	// \x writes a single byte, \u and \U write the UTF-8 encoding of a code point.
	fmt.Printf("% x\n", "\xe6\x97\xa5")
	fmt.Printf("% x\n", "\u65e5")
	fmt.Println("\xe6\x97\xa5" == "\u65e5", "日" == "\u65e5")

	query := `
SELECT title
  FROM books
 WHERE isbn = ?`
	fmt.Println(query)
}

'a'          U+0061      97 int32
'\n'         U+000A      10 int32
'A'          U+0041      65 int32
'é'          U+00E9     233 int32
'📚'          U+1F4DA    128218 int32
'\''         U+0027      39 int32
'A'          U+0041      65 int32
'日'          U+65E5     26085 int32
"C:\\shelf\tbooks\n"
"C:\\shelf\\tbooks\\n"
e6 97 a5
e6 97 a5
true true

SELECT title
  FROM books
 WHERE isbn = ?

package main

import "fmt"

func main() {
	fmt.Println('\ud800', "\U00110000", '\400')
}

./main.go:6:21: escape is invalid Unicode code point U+D800
./main.go:6:35: escape is invalid Unicode code point U+110000
./main.go:6:43: octal escape value 256 > 255

./main.go:7:17: invalid operation: "x" + 'y' (mismatched types untyped string and untyped rune)

package main

import (
	"fmt"
	"testing"
)

var (
	sinkS string
	sinkB []byte
	sinkR []rune
	sinkN int
)

func main() {
	title := "Café 日本"

	b := []byte(title)         // copy of the bytes
	r := []rune(title)         // decoded code points
	s1 := string(b)            // copy back
	s2 := string(r)            // encode as UTF-8
	s3 := string(rune(0x65e5)) // a single code point
	fmt.Println(len(b), len(r), s1 == title, s2 == title, s3)

	// Mutating the copy does not touch the original string.
	b[0] = 'c'
	fmt.Println(string(b), title)

	// Out-of-range or surrogate code points become U+FFFD.
	fmt.Printf("%q %q\n", string(rune(0xD800)), string(rune(-1)))

	long := make([]byte, 4096)
	for i := range long {
		long[i] = 'x'
	}
	ls := string(long)
	fmt.Println("allocs string([]byte):", testing.AllocsPerRun(100, func() { sinkS = string(long) }))
	fmt.Println("allocs []byte(string):", testing.AllocsPerRun(100, func() { sinkB = []byte(ls) }))
	fmt.Println("allocs []rune(string):", testing.AllocsPerRun(100, func() { sinkR = []rune(ls) }))
	m := map[string]int{"xx": 1}
	key := []byte("xx")
	fmt.Println("allocs m[string(key)]:", testing.AllocsPerRun(100, func() { sinkN = m[string(key)] }))
}

12 7 true true 日
café 日本 Café 日本
"�" "�"
allocs string([]byte): 1
allocs []byte(string): 1
allocs []rune(string): 1
allocs m[string(key)]: 0

package main

import (
	"fmt"
	"unicode/utf8"
)

func main() {
	title := "Café 日本"

	fmt.Println(utf8.RuneCountInString(title), len(title), utf8.ValidString(title))
	fmt.Println(utf8.ValidString("a\xffb"), utf8.ValidRune(0xD800), utf8.ValidRune('日'))

	r, size := utf8.DecodeRuneInString(title[3:])
	fmt.Printf("%c %d\n", r, size)
	r, size = utf8.DecodeLastRuneInString(title)
	fmt.Printf("%c %d\n", r, size)
	r, size = utf8.DecodeRuneInString("\xff")
	fmt.Printf("%U %d %t\n", r, size, r == utf8.RuneError)
	r, size = utf8.DecodeRuneInString("")
	fmt.Printf("%U %d\n", r, size)

	fmt.Println(utf8.RuneLen('a'), utf8.RuneLen('é'), utf8.RuneLen('日'), utf8.RuneLen('📚'), utf8.RuneLen(0xD800), utf8.UTFMax)

	buf := make([]byte, 0, 16)
	buf = utf8.AppendRune(buf, 'é')
	buf = utf8.AppendRune(buf, '日')
	fmt.Printf("% x\n", buf)

	var enc [utf8.UTFMax]byte
	n := utf8.EncodeRune(enc[:], '📚')
	fmt.Printf("% x (%d)\n", enc[:n], n)

	fmt.Println(utf8.FullRuneInString("\xe6\x97"), utf8.FullRuneInString("\xe6\x97\xa5"), utf8.RuneStart(0xa5))
}

7 12 true
false false true
é 2
本 3
U+FFFD 1 true
U+FFFD 0
1 2 3 4 -1 4
c3 a9 e6 97 a5
f0 9f 93 9a (4)
false true false

package main

import (
	"fmt"
	"unicode"
	"unicode/utf8"
)

// truncate shortens s to at most n runes without splitting a character.
func truncate(s string, n int) string {
	if utf8.RuneCountInString(s) <= n {
		return s
	}
	i := 0
	for count := 0; count < n; count++ {
		_, size := utf8.DecodeRuneInString(s[i:])
		i += size
	}
	return s[:i]
}

func main() {
	title := "Crónica de una muerte anunciada"
	fmt.Println(truncate(title, 10), "|", truncate("日本語の本", 3))

	for _, r := range "aÉ9٣ 日_\t" {
		fmt.Printf("%-6q letter=%-5t upper=%-5t digit=%-5t number=%-5t space=%-5t han=%t\n",
			r, unicode.IsLetter(r), unicode.IsUpper(r), unicode.IsDigit(r),
			unicode.IsNumber(r), unicode.IsSpace(r), unicode.Is(unicode.Han, r))
	}
	fmt.Println(string(unicode.ToUpper('é')), string(unicode.ToTitle('dž')), string(unicode.SimpleFold('K')), unicode.In('λ', unicode.Greek, unicode.Latin))
	fmt.Println(unicode.MaxRune, unicode.ReplacementChar == utf8.RuneError, unicode.Version)
}

Crónica de | 日本語
'a'    letter=true  upper=false digit=false number=false space=false han=false
'É'    letter=true  upper=true  digit=false number=false space=false han=false
'9'    letter=false upper=false digit=true  number=true  space=false han=false
'٣'    letter=false upper=false digit=true  number=true  space=false han=false
' '    letter=false upper=false digit=false number=false space=true  han=false
'日'    letter=true  upper=false digit=false number=false space=false han=true
'_'    letter=false upper=false digit=false number=false space=false han=false
'\t'   letter=false upper=false digit=false number=false space=true  han=false
É Dž k true
1114111 true 17.0.0

package main

import (
	"fmt"
	"strings"
	"unicode"
)

func main() {
	text := "Dune; Emma ; Ulysses\nLolita"

	for part := range strings.SplitSeq(text, ";") {
		fmt.Printf("%q\n", strings.TrimSpace(part))
	}
	for line := range strings.Lines(text) {
		fmt.Printf("%q\n", line)
	}
	for f := range strings.FieldsSeq(text) {
		fmt.Print(f, "|")
	}
	fmt.Println()

	before, after, found := strings.Cut("isbn:9780441172719", ":")
	fmt.Println(before, after, found)
	fmt.Println(strings.EqualFold("Straße", "STRASSE"), strings.EqualFold("Éa", "éA"))
	fmt.Println(strings.Map(func(r rune) rune {
		if unicode.IsSpace(r) {
			return '-'
		}
		return unicode.ToLower(r)
	}, "The Great Gatsby"))
	fmt.Printf("%q\n", strings.ToValidUTF8("a\xff\xfeb", "?"))
	fmt.Println(strings.ToUpper("straße"))
	fmt.Println(strings.Compare("a", "b"), "a" < "b", "Z" < "a")
}

"Dune"
"Emma"
"Ulysses\nLolita"
"Dune; Emma ; Ulysses\n"
"Lolita"
Dune;|Emma|;|Ulysses|Lolita|
isbn 9780441172719 true
false true
the-great-gatsby
"a?b"
STRAßE
-1 true true

package main

import (
	"fmt"
	"strings"
)

func main() {
	// Cut splits at the first separator, CutLast (Go 1.27) at the last one.
	name := "dune.first.edition.json"

	first, rest, _ := strings.Cut(name, ".")
	fmt.Println(first, "|", rest)

	base, ext, found := strings.CutLast(name, ".")
	fmt.Println(base, "|", ext, "|", found)

	base, ext, found = strings.CutLast("README", ".")
	fmt.Printf("%q %q %t\n", base, ext, found)
}

dune | first.edition.json
dune.first.edition | json | true
"README" "" false

package main

import (
	"bytes"
	"fmt"
	"strings"
	"testing"
)

var sink string

func concat(parts []string) string {
	s := ""
	for _, p := range parts {
		s += p + ", "
	}
	return s
}

func build(parts []string) string {
	var sb strings.Builder
	sb.Grow(len(parts) * 9)
	for _, p := range parts {
		sb.WriteString(p)
		sb.WriteString(", ")
	}
	return sb.String()
}

func main() {
	parts := make([]string, 200)
	for i := range parts {
		parts[i] = fmt.Sprintf("book%03d", i)
	}
	fmt.Println(concat(parts) == build(parts))
	fmt.Println("allocs +=     :", testing.AllocsPerRun(50, func() { sink = concat(parts) }))
	fmt.Println("allocs Builder:", testing.AllocsPerRun(50, func() { sink = build(parts) }))

	var sb strings.Builder
	fmt.Fprintf(&sb, "%s (%d)", "Dune", 1965)
	sb.WriteByte(' ')
	sb.WriteRune('📚')
	fmt.Println(sb.String(), sb.Len(), sb.Cap() >= sb.Len())

	var buf bytes.Buffer
	buf.WriteString("isbn=")
	buf.Write([]byte("9780441172719"))
	fmt.Println(buf.String(), buf.Len())
	line, err := buf.ReadString('=')
	fmt.Println(line, err, buf.String())

	// Join is the usual answer for a slice of strings.
	fmt.Println(strings.Join(parts[:3], "; "))
}

true
allocs +=     : 200
allocs Builder: 1
Dune (1965) 📚 16 true
isbn=9780441172719 18
isbn= <nil> 9780441172719
book000; book001; book002

panic: strings: illegal use of non-zero Builder copied by value

package main

import (
	"fmt"
	"unicode/utf8"
)

func main() {
	composed := "caf\u00e9"
	decomposed := "cafe\u0301"

	fmt.Println(composed, decomposed, composed == decomposed)
	fmt.Println(len(composed), utf8.RuneCountInString(composed))
	fmt.Println(len(decomposed), utf8.RuneCountInString(decomposed))

	// A flag is two regional-indicator code points: one visible symbol, 8 bytes.
	flag := "\U0001F1EA\U0001F1F8"
	fmt.Println(flag, len(flag), utf8.RuneCountInString(flag))
}

café café false
5 4
6 5
🇪🇸 8 2