Syntax and Lexical Structure

# source() and parse() read the whole text before anything is evaluated, so a
# syntax error on the last line stops the first line from running.
script <- c(
  "cat(\"step one ran\\n\")",
  "step_two <- 2",
  "oops oops"
)
tryCatch(
  source(textConnection(script)),
  error = \(e) writeLines(conditionMessage(e))
)
#> textConnection(script):3:6: unexpected symbol
#> 2: step_two <- 2
#> 3: oops oops
#>         ^
exists("step_two")
#> [1] FALSE

# Parsing the lines one at a time, as the console does, runs the valid ones.
for (line in script) {
  tryCatch(
    eval(parse(text = line)),
    error = \(e) writeLines("syntax error on this line")
  )
}
#> step one ran
#> syntax error on this line
exists("step_two")
#> [1] TRUE

# Run a piece of text through the parser and evaluator; report the outcome.
outcome <- function(text) {
  tryCatch(
    {
      eval(parse(text = text))
      "no error"
    },
    error = \(e) conditionMessage(e)
  )
}

# The reserved words are recognised by the parser, so they cannot be names.
outcome("if <- 1")
#> [1] "<text>:1:4: unexpected assignment\n1: if <-\n       ^"
outcome("function <- 1")
#> [1] "<text>:1:10: unexpected assignment\n1: function <-\n             ^"
outcome("repeat <- 1")
#> [1] "<text>:1:8: unexpected assignment\n1: repeat <-\n           ^"

# TRUE, FALSE, NULL, Inf and NaN parse as constants, so the
# parser accepts the assignment but the evaluator refuses it.
outcome("TRUE <- 0")
#> [1] "invalid (do_set) left-hand side to assignment"
outcome("NULL <- 1")
#> [1] "invalid (do_set) left-hand side to assignment"
outcome("Inf <- 1")
#> [1] "invalid (do_set) left-hand side to assignment"

# Backticks turn a reserved word into an ordinary, non-syntactic name.
`if` <- "just a variable" # nolint: object_name_linter.
`if`
#> [1] "just a variable"
rm(`if`)

# As a list name or an argument tag, a reserved word needs the same quoting.
outcome("list(if = 1)")
#> [1] "<text>:1:9: unexpected '='\n1: list(if =\n            ^"
outcome("list(`if` = 1)")
#> [1] "no error"
shelf_item <- list(`if` = "interlibrary", `in` = "shelf 4")
outcome("shelf_item$if")
#> [1] "<text>:1:12: unexpected 'if'\n1: shelf_item$if\n               ^"
shelf_item$`if`
#> [1] "interlibrary"
shelf_item$"in"
#> [1] "shelf 4"

# Words that merely look special are ordinary variables and can be shadowed:
# "c" is a function, but assigning to it does not stop c() from working.
c <- 3 # nolint: object_name_linter.
c(c, c)
#> [1] 3 3
rm(c)

# Types of the constants.
vapply(
  list(TRUE, 1L, 1, 1i, "a", NULL, NA, NA_integer_, NA_real_, NA_character_),
  typeof, character(1L)
)
#>  [1] "logical"   "integer"   "double"    "complex"   "character" "NULL"
#>  [7] "logical"   "integer"   "double"    "character"
typeof(Inf)
#> [1] "double"
typeof(NaN)
#> [1] "double"

# Decimal constants are doubles, even when they look like integers.
vapply(list(1, 10, 0.1, .2, 1e-7, 1.2e+7, 5.), typeof, character(1L))
#> [1] "double" "double" "double" "double" "double" "double" "double"
class(2)
#> [1] "numeric"
2 == 2L
#> [1] TRUE
identical(2, 2L)
#> [1] FALSE

# The suffix L makes an integer when the value allows it.
vapply(list(1L, 0x10L, 1000000L, 1e6L, 1e3L), typeof, character(1L))
#> [1] "integer" "integer" "integer" "integer" "integer"
identical(0xFL, 15L)
#> [1] TRUE

# Hexadecimal constants, with an optional binary exponent (p or P).
0x1F
#> [1] 31
0xA0
#> [1] 160
0x1.8p1
#> [1] 3
0x123p4
#> [1] 4656
0x1.1 # accepted since R 4.4.3, as 0x1.1p0
#> [1] 1.0625

# Too large for an integer, or with a fractional part: a warning and a double.
catch_warning <- function(expr) {
  tryCatch(expr, warning = \(w) paste("warning:", conditionMessage(w)))
}
catch_warning(eval(parse(text = "1.5L")))
#> [1] "warning: integer literal 1.5L contains decimal; using numeric value"
catch_warning(eval(parse(text = "1.L")))
#> [1] "warning: integer literal 1.L contains unnecessary decimal point"
catch_warning(eval(parse(text = "3000000000L")))
#> [1] "warning: non-integer value 3000000000L qualified with L; using numeric value"
catch_warning(eval(parse(text = "1e-3L")))
#> [1] "warning: non-integer value 1e-3L qualified with L; using numeric value"
.Machine$integer.max
#> [1] 2147483647
catch_warning(.Machine$integer.max + 1L)
#> [1] "warning: NAs produced by integer overflow"

# Overflow becomes Inf, underflow becomes 0.
1e308 * 10
#> [1] Inf
1e400
#> [1] Inf
1e-400
#> [1] 0

# Complex constants: only the imaginary part is a constant.
typeof(2i)
#> [1] "complex"
3 + 2i
#> [1] 3+2i
class(quote(3 + 2i))
#> [1] "call"
sqrt(-1 + 0i)
#> [1] 0+1i

# A sign is a unary operator, not part of the constant.
-2^2
#> [1] -4
(-2)^2
#> [1] 4
class(quote(-1))
#> [1] "call"
class(quote(1))
#> [1] "numeric"

# There is no digit grouping, no binary or octal prefix and no "d" exponent.
parse_message <- function(text) {
  tryCatch(
    {
      parse(text = text)
      "parsed"
    },
    error = \(e) conditionMessage(e)
  )
}
for (text in c("1_000", "0b101", "1d3", "12iL", "1e")) {
  writeLines(parse_message(text))
}
#> <text>:1:2: unexpected input
#> 1: 1_
#>      ^
#> <text>:1:2: unexpected symbol
#> 1: 0b101
#>      ^
#> <text>:1:2: unexpected symbol
#> 1: 1d3
#>      ^
#> <text>:1:4: unexpected symbol
#> 1: 12iL
#>        ^
#> <text>:1:1: unexpected input
#> 1: 1e
#>     ^

# Single and double quotes are interchangeable; printing uses double quotes.
'single quotes work too' # nolint: quotes_linter.
#> [1] "single quotes work too"
identical('"It\'s alive!", he screamed.', "\"It's alive!\", he screamed.")
#> [1] TRUE

# print() shows the string as R source; writeLines() shows its characters.
path <- "C:\\shelf\\books.csv"
print(path)
#> [1] "C:\\shelf\\books.csv"
writeLines(path)
#> C:\shelf\books.csv
nchar(path)
#> [1] 18

# Escape sequences.
line <- "title\tauthor\nThe Long Table\tIris Fenwick"
print(line)
#> [1] "title\tauthor\nThe Long Table\tIris Fenwick"
writeLines(line)
#> title	author
#> The Long Table	Iris Fenwick

# Octal, hexadecimal and Unicode escapes can spell the same text.
hello_octal <- "\110\145\154\154\157"
hello_hex <- "\x48\x65\x6c\x6c\x6f"
hello_unicode <- "\u48\u65\u6c\u6c\u6f"
c(hello_octal, hello_hex, hello_unicode)
#> [1] "Hello" "Hello" "Hello"
identical(hello_octal, hello_hex) && identical(hello_hex, hello_unicode)
#> [1] TRUE
"\u00e9\u0126\U0001F4DA"
#> [1] "éĦ📚"
"\u{e9}\U{1F4DA}"
#> [1] "é📚"
nchar("\U0001F4DA")
#> [1] 1
nchar("\U0001F4DA", type = "bytes")
#> [1] 4

# A character constant may span lines: the newline is part of the string.
two_lines <- "first line
second line"
two_lines
#> [1] "first line\nsecond line"
nchar(two_lines)
#> [1] 22

# Bad escapes and embedded NULs are errors when the code is parsed.
parse_message <- function(text) {
  tryCatch(
    {
      parse(text = text)
      "parsed"
    },
    error = \(e) conditionMessage(e)
  )
}
parse_message('"\\q"')
#> [1] "'\\q' is an unrecognized escape in character string (<input>:1:3)"
parse_message('"\\x48\\u65"')
#> [1] "mixing Unicode and octal/hex escapes in a string is not allowed (<input>:1:10)"
parse_message('"foo\\0bar"')
#> [1] "nul character not allowed (<input>:1:6)"
parse_message('"\\U110000"')
#> [1] "invalid \\Uxxxxxxxx value 110000 (<input>:1:9)"
parse_message("'unterminated")
#> [1] "<text>:1:1: unexpected INCOMPLETE_STRING\n1: 'unterminated\n    ^"

# Strings do not interpolate: use paste0(), sprintf() or format().
member <- "Ana Ruiz"
"Hello, member"
#> [1] "Hello, member"
paste0("Hello, ", member)
#> [1] "Hello, Ana Ruiz"
sprintf("Hello, %s", member)
#> [1] "Hello, Ana Ruiz"

# TRUE and FALSE are reserved words (constants); T and F are ordinary variables
# defined in the base environment.
T # nolint: T_and_F_symbol_linter.
#> [1] TRUE
F # nolint: T_and_F_symbol_linter.
#> [1] FALSE
find("T")
#> [1] "package:base"
find("TRUE")
#> character(0)
tryCatch(
  eval(parse(text = "TRUE <- 0")),
  error = \(e) conditionMessage(e)
)
#> [1] "invalid (do_set) left-hand side to assignment"

# Because T is a variable, any code can change what it means.
local({
  T <- 0 # nolint: T_and_F_symbol_linter, object_name_linter.
  c(spelled_out = mean(c(1, NA, 3), na.rm = TRUE),
    shortcut = mean(c(1, NA, 3), na.rm = T)) # nolint: T_and_F_symbol_linter.
})
#> spelled_out    shortcut
#>           2          NA

# NULL is the empty object; NA marks a missing value; they are different things.
length(NULL)
#> [1] 0
length(NA)
#> [1] 1
is.null(NA)
#> [1] FALSE
is.na(NULL)
#> logical(0)
c(1, NULL, 3)
#> [1] 1 3
c(1, NA, 3)
#> [1]  1 NA  3

# The NA family: one constant per atomic type.
vapply(
  list(NA, NA_integer_, NA_real_, NA_complex_, NA_character_),
  typeof, character(1L)
)
#> [1] "logical"   "integer"   "double"    "complex"   "character"
c(NA, "text")
#> [1] NA     "text"
c(NA, 1L)
#> [1] NA  1
identical(NA_real_, NA_integer_)
#> [1] FALSE

# Inf and NaN are numeric constants (type double).
1 / 0
#> [1] Inf
-1 / 0
#> [1] -Inf
0 / 0
#> [1] NaN
is.nan(c(NaN, NA, 1))
#> [1]  TRUE FALSE FALSE
is.na(c(NaN, NA, 1))
#> [1]  TRUE  TRUE FALSE
is.finite(c(Inf, NaN, NA, 1))
#> [1] FALSE FALSE FALSE  TRUE
Inf - Inf
#> [1] NaN
NA > 1
#> [1] NA
NA & FALSE
#> [1] FALSE
NA || TRUE
#> [1] TRUE

# parse() reports a syntax error as an ordinary error condition, so a text
# can be checked without running it. The message has the form
# <source>:<line>:<column>: unexpected <token>, then the offending line.
check <- function(text) {
  tryCatch(
    {
      parse(text = text)
      writeLines("parsed")
    },
    error = \(e) writeLines(conditionMessage(e))
  )
}

check("books <- read.csv(\"books.csv\") loans")  # unexpected symbol
#> <text>:1:32: unexpected symbol
#> 1: books <- read.csv("books.csv") loans
#>                                    ^
check("x <- c(1, 2))")                           # unexpected ')'
#> <text>:1:13: unexpected ')'
#> 1: x <- c(1, 2))
#>                 ^
check("total <- 5 5")                            # unexpected numeric constant
#> <text>:1:12: unexpected numeric constant
#> 1: total <- 5 5
#>                ^
check("title <- \"Dune\" \"Emma\"")              # unexpected string constant
#> <text>:1:17: unexpected string constant
#> 1: title <- "Dune" "Emma"
#>                     ^
check("x <- 1 \\ 2")                             # unexpected '\\'
#> <text>:1:8: unexpected '\\'
#> 1: x <- 1 \
#>            ^
check("price <- “10”")                 # unexpected input
#> <text>:1:10: unexpected input
#> 1: price <- “
#>              ^
check("rate <- 5 %% % 2")                        # unexpected input
#> <text>:1:14: unexpected input
#> 1: rate <- 5 %% % 2
#>                  ^
check("if (TRUE) {\n  1\n}}")                    # unexpected '}'
#> <text>:3:2: unexpected '}'
#> 2:   1
#> 3: }}
#>     ^
check("f <- function(x) {\n  x + 1\n")           # unexpected end of input
#> <text>:4:0: unexpected end of input
#> 2:   x + 1
#> 3:
#>   ^
check("x <- 'unterminated")                      # INCOMPLETE_STRING
#> <text>:1:6: unexpected INCOMPLETE_STRING
#> 1: x <- 'unterminated
#>          ^
check("if <- 1")                                 # unexpected assignment
#> <text>:1:4: unexpected assignment
#> 1: if <-
#>        ^
check("a = = 2")                                 # unexpected '='
#> <text>:1:5: unexpected '='
#> 1: a = =
#>         ^

# The top-level "else" rule: a newline before else ends the if at the top
# level of a script, but not inside braces (see Control Flow).
check("if (TRUE) 1\nelse 2")
#> <text>:2:1: unexpected 'else'
#> 1: if (TRUE) 1
#> 2: else
#>    ^
check("{\n  if (TRUE) 1\n  else 2\n}")
#> parsed

# Since R 4.3.0 some syntax errors, notably those of the native pipe, are
# classed conditions with line and column fields; "unexpected ..." errors are
# still plain simpleError conditions.
class(tryCatch(parse(text = "a b"), error = identity))
#> [1] "simpleError" "error"       "condition"
pipe_err <- tryCatch(parse(text = "a |> b"), error = identity)
class(pipe_err)
#> [1] "RHSnotFnCall" "parseError"   "error"        "condition"
conditionMessage(pipe_err)
#> [1] "The pipe operator requires a function call as RHS (<input>:1:6)"
c(line = pipe_err$lineno, column = pipe_err$colno)
#>   line column
#>      1      6

# Text that parses but cannot be evaluated is not a syntax error.
check("2 + 2 <- 5")
#> parsed
tryCatch(
  eval(parse(text = "2 + 2 <- 5")),
  error = \(e) writeLines(conditionMessage(e))
)
#> target of assignment expands to non-language object

# An incomplete expression is how the console knows to show the "+" prompt:
# parse() reports it as "unexpected end of input".
check("sum(1,")
#> <text>:2:0: unexpected end of input
#> 1: sum(1,
#>    ^

# parse() turns text into an expression vector: one element per expression.
exprs <- parse(text = "total <- price * 2L\nprice")
class(exprs)
#> [1] "expression"
length(exprs)
#> [1] 2
exprs[[1L]]
#> total <- price * 2L
class(exprs[[1L]]) # a call
#> [1] "<-"
class(exprs[[2L]]) # a symbol
#> [1] "name"

# quote() builds the same objects from code that is already parsed.
call_obj <- quote(total <- price * 2L)
identical(call_obj, exprs[[1L]])
#> [1] TRUE
as.list(call_obj)
#> [[1]]
#> `<-`
#>
#> [[2]]
#> total
#>
#> [[3]]
#> price * 2L
#>
as.list(call_obj[[3L]])
#> [[1]]
#> `*`
#>
#> [[2]]
#> price
#>
#> [[3]]
#> [1] 2
#>

# Every part of a call is addressable: function first, then arguments.
call_obj[[1L]]
#> `<-`
call_obj[[3L]][[1L]]
#> `*`
call_obj[[3L]][[3L]]
#> [1] 2
typeof(call_obj[[3L]][[3L]])
#> [1] "integer"

# deparse() goes the other way, from an object back to text.
deparse(call_obj)
#> [1] "total <- price * 2L"
deparse(c(1, 2, 3))
#> [1] "c(1, 2, 3)"
deparse(1:5)
#> [1] "1:5"
deparse(quote(if (x > 1) "big" else "small"))
#> [1] "if (x > 1) \"big\" else \"small\""
cat(deparse(function(x, y = 2) x + y), sep = "\n")
#> function (x, y = 2)
#> x + y

# Parsing the deparsed text of an expression gives back the same expression.
identical(parse(text = deparse(call_obj))[[1L]], call_obj)
#> [1] TRUE
identical(eval(parse(text = deparse(1:5))), 1:5)
#> [1] TRUE
deparse(1 / 3)
#> [1] "0.333333333333333"
identical(eval(parse(text = deparse(1 / 3))), 1 / 3)
#> [1] FALSE
identical(eval(parse(text = deparse(1 / 3, control = "digits17"))), 1 / 3)
#> [1] TRUE

# str2lang() and str2expression() parse a single expression from a string.
str2lang("loan_days * 2")
#> loan_days * 2
class(str2lang("loan_days"))
#> [1] "name"
str2expression("loan_days * 2")
#> expression(loan_days * 2)

# With keep.source = TRUE the parser keeps the original text, comments included.
getOption("keep.source") # FALSE under Rscript, TRUE in an interactive session
#> [1] FALSE
src_text <- "function(x) {\n  x + 1 # add one\n}"
kept <- eval(parse(text = src_text, keep.source = TRUE))
kept
#> function(x) {
#>   x + 1 # add one
#> }
dropped <- eval(parse(text = src_text, keep.source = FALSE))
dropped
#> function (x)
#> {
#>     x + 1
#> }

books <- read.csv("books.csv")

# Operators, indexing and control flow are function calls with backquoted names.
`+`(1, 2)
#> [1] 3
`-`(10)
#> [1] -10
`[`(books$year, 2L)
#> [1] 1813
`[[`(books, "title")[1L:2L]
#> [1] "The Dispossessed"    "Pride and Prejudice"
`$`(books, "pages")[1L:3L]
#> [1] 387 432 214
`if`(nrow(books) > 10L, "long shelf", "short shelf")
#> [1] "long shelf"
`(`(1 + 2) * 3
#> [1] 9
`{`(1, 2, 3)
#> [1] 3
x <- `<-`(y, 5) # nolint: object_name_linter.
c(x, y)
#> [1] 5 5

# Calling a function by its name as a string works the same way.
"sum"(1, 2, 3)
#> [1] 6
(function(n) n^2)(5)
#> [1] 25
`%in%`(2, 1:3)
#> [1] TRUE

# The parsed form of an expression is a call: the function is the first element.
call_plus <- quote(1 + 2)
class(call_plus)
#> [1] "call"
call_plus[[1L]]
#> `+`
as.list(call_plus)
#> [[1]]
#> `+`
#>
#> [[2]]
#> [1] 1
#>
#> [[3]]
#> [1] 2
#>
eval(call_plus)
#> [1] 3
identical(call_plus, as.call(list(as.name("+"), 1, 2)))
#> [1] TRUE

# Precedence is just the shape of the tree.
show_tree <- function(x, indent = "") {
  if (is.call(x)) {
    cat(indent, "call: ", deparse(x[[1L]]), "\n", sep = "")
    for (arg in as.list(x)[-1L]) {
      show_tree(arg, paste0(indent, "  "))
    }
  } else {
    cat(indent, deparse(x), "\n", sep = "")
  }
}
show_tree(quote(1 + 2 * 3))
#> call: +
#>   1
#>   call: *
#>     2
#>     3
show_tree(quote((1 + 2) * 3))
#> call: *
#>   call: (
#>     call: +
#>       1
#>       2
#>   3
show_tree(quote(-2^2))
#> call: -
#>   call: ^
#>     2
#>     2

# Control flow and function definitions are calls too.
as.list(quote(if (a) b else c))
#> [[1]]
#> `if`
#>
#> [[2]]
#> a
#>
#> [[3]]
#> b
#>
#> [[4]]
#> c
#>
as.list(quote(for (i in 1:3) print(i)))[[1L]]
#> `for`
as.list(quote(while (TRUE) break))[[1L]]
#> `while`
quote(function(x, y = 2) x + y)[[1L]]
#> `function`
quote(\(x) x + 1)
#> function(x) x + 1
as.list(quote(x[1]))
#> [[1]]
#> `[`
#>
#> [[2]]
#> x
#>
#> [[3]]
#> [1] 1
#>
as.list(quote(x$name))
#> [[1]]
#> `$`
#>
#> [[2]]
#> x
#>
#> [[3]]
#> name
#>
as.list(quote(x[[2]]))
#> [[1]]
#> `[[`
#>
#> [[2]]
#> x
#>
#> [[3]]
#> [1] 2
#>

# A for loop is a call that returns NULL (invisibly); if returns its branch.
result <- for (i in 1L:3L) i
is.null(result)
#> [1] TRUE
value <- if (nrow(books) > 100L) "big" else "small"
value
#> [1] "small"

books <- read.csv("books.csv")
source("scenario.R")

# The columns of both tables are already syntactic names.
all(names(books) == make.names(names(books)))
#> [1] TRUE
all(names(loans) == make.names(names(loans)))
#> [1] TRUE
names(loans)
#> [1] "loan_id"     "isbn"        "member"      "loan_date"   "return_date"

# A filter typed by a user is only text: parse it first, run it afterwards.
filter_titles <- function(expr_text, data) {
  expr <- tryCatch(str2lang(expr_text), error = \(e) NULL)
  if (is.null(expr)) {
    return(paste("not a valid R expression:", expr_text))
  }
  keep <- eval(expr, data, baseenv())
  data$title[which(keep)]
}
filter_titles("pages > 400", books)
#> [1] "Pride and Prejudice" "Notes on Gravity"
filter_titles("genre == 'Poetry'", books)
#> [1] "Garden of Small Fires" "Paper Moons"
filter_titles("pages >> 400", books)
#> [1] "not a valid R expression: pages >> 400"
filter_titles("genre ==", books)
#> [1] "not a valid R expression: genre =="

# loan_days is an integer constant (the literal 21L) from the scenario file.
loan_days
#> [1] 21
class(loan_days)
#> [1] "integer"